Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -6,10 +6,14 @@ from gradio.mix import Parallel, Series
|
|
6 |
#import torch.nn.functional as F
|
7 |
from aitextgen import aitextgen
|
8 |
|
|
|
|
|
9 |
from datasets import load_dataset
|
10 |
dataset = load_dataset("bananabot/engMollywoodSummaries")
|
11 |
tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neo-1.3B")
|
12 |
|
|
|
|
|
13 |
def tokenize_function(examples):
|
14 |
return tokenizer(examples["text"], padding="max_length", truncation=True)
|
15 |
tokenized_datasets = dataset.map(tokenize_function, batched=True)
|
|
|
6 |
#import torch.nn.functional as F
|
7 |
from aitextgen import aitextgen
|
8 |
|
9 |
+
|
10 |
+
|
11 |
from datasets import load_dataset
|
12 |
dataset = load_dataset("bananabot/engMollywoodSummaries")
|
13 |
tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neo-1.3B")
|
14 |
|
15 |
+
tokenizer.pad_token = tokenizer.eos_token
|
16 |
+
|
17 |
def tokenize_function(examples):
|
18 |
return tokenizer(examples["text"], padding="max_length", truncation=True)
|
19 |
tokenized_datasets = dataset.map(tokenize_function, batched=True)
|