File size: 1,177 Bytes
9880769 3f0a7d5 9880769 d2b8f1b 3f0a7d5 d2b8f1b 8cd8309 70e3867 3f0a7d5 8cd8309 9880769 c28f293 9880769 50d6cdc 9880769 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
import transformers
import os
import torch
import requests
MAX_LEN = 150 #256
TRAIN_BATCH_SIZE = 8
VALID_BATCH_SIZE = 4
EPOCHS = 5
# Folder to contain all the datasets
# from huggingface_hub import hf_hub_download
# print("hi")
# hf_hub_download(repo_id="thak123/bert-emoji-latvian-twitter-classifier", filename="model.bin",local_dir="./")
# from huggingface_hub import snapshot_download
# snapshot_download(repo_id="thak123/bert-emoji-latvian-twitter-classifier", allow_patterns="*.bin")
# import requests
URL = "https://huggingface.co/thak123/bert-emoji-latvian-twitter-classifier/resolve/main/model.bin"
response = requests.get(URL)
open("model.bin", "wb").write(response.content)
DATASET_LOCATION = "" #
MODEL_PATH = "model.bin"
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# 7 EPOCH Version
BERT_PATH = "FFZG-cleopatra/bert-emoji-latvian-twitter"
# TODO check if lower casing is required
# BertTokenizer
TOKENIZER = transformers.BertTokenizer.from_pretrained(
BERT_PATH,
do_lower_case=True
)
####################################################################################################################################
|