bananabot commited on
Commit
b4b033d
1 Parent(s): 193ebc9

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -0
app.py CHANGED
@@ -6,10 +6,14 @@ from gradio.mix import Parallel, Series
6
  #import torch.nn.functional as F
7
  from aitextgen import aitextgen
8
 
 
 
9
  from datasets import load_dataset
10
  dataset = load_dataset("bananabot/engMollywoodSummaries")
11
  tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neo-1.3B")
12
 
 
 
13
  def tokenize_function(examples):
14
  return tokenizer(examples["text"], padding="max_length", truncation=True)
15
  tokenized_datasets = dataset.map(tokenize_function, batched=True)
 
6
  #import torch.nn.functional as F
7
  from aitextgen import aitextgen
8
 
9
+
10
+
11
  from datasets import load_dataset
12
  dataset = load_dataset("bananabot/engMollywoodSummaries")
13
  tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neo-1.3B")
14
 
15
+ tokenizer.pad_token = tokenizer.eos_token
16
+
17
  def tokenize_function(examples):
18
  return tokenizer(examples["text"], padding="max_length", truncation=True)
19
  tokenized_datasets = dataset.map(tokenize_function, batched=True)