Duplicate from RWKV/rwkv-5-world-3b

Co-authored-by: Kalei Neely <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,173 @@
+### Run Huggingface RWKV5 World Model
+
+
+#### CPU
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+def generate_prompt(instruction, input=""):
+ instruction = instruction.strip().replace('\r\n','\n').replace('\n\n','\n')
+ input = input.strip().replace('\r\n','\n').replace('\n\n','\n')
+ if input:
+ return f"""Instruction: {instruction}
+
+Input: {input}
+
+Response:"""
+ else:
+ return f"""User: hi
+
+Assistant: Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.
+
+User: {instruction}
+
+Assistant:"""
+
+
+model = AutoModelForCausalLM.from_pretrained("RWKV/rwkv-5-world-3b", trust_remote_code=True).to(torch.float32)
+tokenizer = AutoTokenizer.from_pretrained("RWKV/rwkv-5-world-3b", trust_remote_code=True)
+
+text = "请介绍北京的旅游景点"
+prompt = generate_prompt(text)
+
+inputs = tokenizer(prompt, return_tensors="pt")
+output = model.generate(inputs["input_ids"], max_new_tokens=333, do_sample=True, temperature=1.0, top_p=0.3, top_k=0, )
+print(tokenizer.decode(output[0].tolist(), skip_special_tokens=True))
+```
+
+output:
+
+```shell
+User: hi
+
+Assistant: Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.
+
+User: 请介绍北京的旅游景点
+
+Assistant: 北京是中国的首都，拥有众多的旅游景点，以下是其中一些著名的景点：
+1. 故宫：位于北京市中心，是明清两代的皇宫，内有大量的文物和艺术品。
+2. 天安门广场：是中国最著名的广场之一，是中国人民政治协商会议的旧址，也是中国人民政治协商会议的中心。
+3. 颐和园：是中国古代皇家园林之一，有着悠久的历史和丰富的文化内涵。
+4. 长城：是中国古代的一道长城，全长约万里，是中国最著名的旅游景点之一。
+5. 北京大学：是中国著名的高等教育机构之一，有着悠久的历史和丰富的文化内涵。
+6. 北京动物园：是中国最大的动物园之一，有着丰富的动物资源和丰富的文化内涵。
+7. 故宫博物院：是中国最著名的博物馆之一，收藏了大量的文物和艺术品，是中国最重要的文化遗产之一。
+8. 天坛：是中国古代皇家
+```
+
+#### GPU
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+def generate_prompt(instruction, input=""):
+ instruction = instruction.strip().replace('\r\n','\n').replace('\n\n','\n')
+ input = input.strip().replace('\r\n','\n').replace('\n\n','\n')
+ if input:
+ return f"""Instruction: {instruction}
+
+Input: {input}
+
+Response:"""
+ else:
+ return f"""User: hi
+
+Assistant: Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.
+
+User: {instruction}
+
+Assistant:"""
+
+
+model = AutoModelForCausalLM.from_pretrained("RWKV/rwkv-5-world-3b", trust_remote_code=True, torch_dtype=torch.float16).to(0)
+tokenizer = AutoTokenizer.from_pretrained("RWKV/rwkv-5-world-3b", trust_remote_code=True)
+
+text = "介绍一下大熊猫"
+prompt = generate_prompt(text)
+
+inputs = tokenizer(prompt, return_tensors="pt").to(0)
+output = model.generate(inputs["input_ids"], max_new_tokens=128, do_sample=True, temperature=1.0, top_p=0.3, top_k=0, )
+print(tokenizer.decode(output[0].tolist(), skip_special_tokens=True))
+```
+
+output:
+
+```shell
+User: hi
+
+Assistant: Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.
+
+User: 介绍一下大熊猫
+
+Assistant: 大熊猫是一种中国特有的哺乳动物，也是中国的国宝之一。它们的外貌特征是圆形的黑白相间的身体，有着黑色的毛发和白色的耳朵。大熊猫的食物主要是竹子，它们会在竹林中寻找竹子，并且会将竹子放在竹笼中进行储存。大熊猫的寿命约为20至30年，但由于栖息地的丧失和人类活动的
+```
+
+#### Batch Inference
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+def generate_prompt(instruction, input=""):
+ instruction = instruction.strip().replace('\r\n', '\n').replace('\n\n', '\n')
+ input = input.strip().replace('\r\n', '\n').replace('\n\n', '\n')
+ if input:
+ return f"""Instruction: {instruction}
+
+Input: {input}
+
+Response:"""
+ else:
+ return f"""User: hi
+
+Assistant: Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.
+
+User: {instruction}
+
+Assistant:"""
+
+model = AutoModelForCausalLM.from_pretrained("RWKV/rwkv-5-world-3b", trust_remote_code=True).to(torch.float32)
+tokenizer = AutoTokenizer.from_pretrained("RWKV/rwkv-5-world-3b", trust_remote_code=True, padding_side='left')
+
+texts = ["请介绍北京的旅游景点", "介绍一下大熊猫", "乌兰察布"]
+prompts = [generate_prompt(text) for text in texts]
+
+inputs = tokenizer(prompts, return_tensors="pt", padding=True)
+outputs = model.generate(inputs["input_ids"], max_new_tokens=128, do_sample=True, temperature=1.0, top_p=0.3, top_k=0, )
+
+for output in outputs:
+ print(tokenizer.decode(output.tolist(), skip_special_tokens=True))
+
+```
+
+output:
+
+```shell
+User: hi
+
+Assistant: Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.
+
+User: 请介绍北京的旅游景点
+
+Assistant: 北京是中国的首都，拥有丰富的旅游资源和历史文化遗产。以下是一些北京的旅游景点：
+1. 故宫：位于北京市中心，是明清两代的皇宫，是中国最大的古代宫殿建筑群之一。
+2. 天安门广场：位于北京市中心，是中国最著名的城市广场之一，也是中国最大的城市广场。
+3. 颐和
+User: hi
+
+Assistant: Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.
+
+User: 介绍一下大熊猫
+
+Assistant: 大熊猫是一种生活在中国中部地区的哺乳动物，也是中国的国宝之一。它们的外貌特征是圆形的黑白相间的身体，有着黑色的毛发和圆圆的眼睛。大熊猫是一种濒危物种，目前只有在野外的几个保护区才能看到它们的身影。大熊猫的食物主要是竹子，它们会在竹子上寻找食物，并且可以通
+User: hi
+
+Assistant: Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.
+
+User: 乌兰察布
+
+Assistant: 乌兰察布是中国新疆维吾尔自治区的一个县级市，位于新疆维吾尔自治区中部，是新疆的第二大城市。乌兰察布市是新疆的第一大城市，也是新疆的重要城市之一。乌兰察布市是新疆的经济中心，也是新疆的重要交通枢纽之一。乌兰察布市的人口约为2.5万人，其中汉族占绝大多数。乌
+```

config.json

@@ -0,0 +1,24 @@
+{
+ "architectures": [
+ "Rwkv5ForCausalLM"
+ ],
+ "auto_map": {
+ "AutoConfig": "configuration_rwkv5.Rwkv5Config",
+ "AutoModelForCausalLM": "modeling_rwkv5.Rwkv5ForCausalLM"
+ },
+ "attention_hidden_size": 2560,
+ "bos_token_id": 0,
+ "eos_token_id": 0,
+ "head_size": 64,
+ "hidden_size": 2560,
+ "intermediate_size": null,
+ "layer_norm_epsilon": 1e-05,
+ "model_type": "rwkv5",
+ "num_attention_heads": 64,
+ "num_hidden_layers": 32,
+ "rescale_every": 6,
+ "tie_word_embeddings": false,
+ "transformers_version": "4.34.0",
+ "use_cache": true,
+ "vocab_size": 65536
+}

configuration_rwkv5.py

@@ -0,0 +1,118 @@
+# coding=utf-8
+# Copyright 2023 The OpenAI Team Authors and HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" RWKV configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+RWKV5_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
+
+
+class Rwkv5Config(PretrainedConfig):
+ """
+ This is the configuration class to store the configuration of a [`Rwkv5Model`]. It is used to instantiate a RWKV5
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the RWVK-4
+ [RWKV/rwkv-5-world-1b5](https://huggingface.co/RWKV/rwkv-5-world-1b5) architecture.
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 65536):
+ Vocabulary size of the RWKV5 model. Defines the number of different tokens that can be represented by the
+ `inputs_ids` passed when calling [`Rwkv5Model`].
+ hidden_size (`int`, *optional*, defaults to 768):
+ Dimensionality of the embeddings and hidden states.
+ num_hidden_layers (`int`, *optional*, defaults to 24):
+ Number of hidden layers in the model.
+ attention_hidden_size (`int`, *optional*):
+ Dimensionality of the attention hidden states. Will default to `hidden_size` if unset.
+ num_attention_heads (`int`, *optional*, defaults to 64):
+ The attention heads to use in rwkv5 self_attention module.
+ head_size (`int`, *optional*, defaults to 64): head_size of rwkv5 self_attention module.
+ intermediate_size (`int`, *optional*):
+ Dimensionality of the inner feed-forward layers. Will default to 4 times `hidden_size` if unset.
+ layer_norm_epsilon (`float`, *optional*, defaults to 1e-05):
+ The epsilon to use in the layer normalization layers.
+ bos_token_id (`int`, *optional*, defaults to 0):
+ The id of the beginning of sentence token in the vocabulary. Defaults to 0.
+ eos_token_id (`int`, *optional*, defaults to 0):
+ The id of the end of sentence token in the vocabulary. Defaults to 0.
+ rescale_every (`int`, *optional*, defaults to 6):
+ At inference, the hidden states (and weights of the correponding output layers) are divided by 2 every
+ `rescale_every` layer. If set to 0 or a negative number, no rescale is done.
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+ Whether or not to tie the word embeddings with the input token embeddings.
+ use_cache (`bool`, *optional*, defaults to `True`):
+ Whether or not the model should return the last state.
+
+
+ Example:
+
+ ```python
+ >>> from transformers import Rwkv5Config, Rwkv5Model
+
+ >>> # Initializing a Rwkv5 configuration
+ >>> configuration = Rwkv5Config()
+
+ >>> # Initializing a model (with random weights) from the configuration
+ >>> model = Rwkv5Model(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "rwkv5"
+
+ def __init__(
+ self,
+ vocab_size=65536,
+ hidden_size=768,
+ num_hidden_layers=24,
+ attention_hidden_size=None,
+ head_size=64,
+ head_size_divisor=8,
+ intermediate_size=None,
+ layer_norm_epsilon=1e-5,
+ bos_token_id=0,
+ eos_token_id=0,
+ rescale_every=6,
+ tie_word_embeddings=False,
+ use_cache=True,
+ **kwargs,
+ ):
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ self.num_hidden_layers = num_hidden_layers
+ self.attention_hidden_size = attention_hidden_size if attention_hidden_size is not None else hidden_size
+ self.head_size = head_size
+ self.head_size_divisor = head_size_divisor
+ self.intermediate_size = None
+ self.layer_norm_epsilon = layer_norm_epsilon
+ self.rescale_every = rescale_every
+ self.use_cache = use_cache
+
+ self.bos_token_id = bos_token_id
+ self.eos_token_id = eos_token_id
+
+ super().__init__(
+ tie_word_embeddings=tie_word_embeddings, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs
+ )

generation_config.json

@@ -0,0 +1,12 @@
+{
+ "chat_format": "chatml",
+ "eos_token_id": 0,
+ "pad_token_id": 0,
+ "max_window_size": 4096,
+ "max_new_tokens": 4096,
+ "do_sample": true,
+ "top_k": 0,
+ "top_p": 0.1,
+ "repetition_penalty": 1.0,
+ "transformers_version": "4.31.1"
+}

modeling_rwkv5.py

@@ -0,0 +1,845 @@
+# coding=utf-8
+# Copyright 2024 The RWKV team and HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch RWKV5 World model."""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import CrossEntropyLoss
+
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import (
+ ModelOutput,
+ add_code_sample_docstrings,
+ add_start_docstrings,
+ add_start_docstrings_to_model_forward,
+ is_bitsandbytes_available,
+ is_ninja_available,
+ is_torch_cuda_available,
+ logging,
+)
+
+from .configuration_rwkv5 import Rwkv5Config
+
+
+logger = logging.get_logger(__name__)
+
+_CHECKPOINT_FOR_DOC = "RWKV/rwkv-5-world-1b5"
+_CONFIG_FOR_DOC = "Rwkv5Config"
+
+rwkv5_cuda_kernel = None
+
+
+# Copied from https://github.com/huggingface/transformers/blob/18cbaf13dcaca7145f5652aefb9b19734c56c3cd/src/transformers/models/rwkv/modeling_rwkv.py#L65
+def load_wkv5_cuda_kernel(head_size):
+ from torch.utils.cpp_extension import load as load_kernel
+
+ global rwkv5_cuda_kernel
+
+ kernel_folder = Path(__file__).parent.resolve()
+ cuda_kernel_files = [kernel_folder / f for f in ["wkv5_op.cpp", "wkv5_cuda.cu"]]
+
+ # Only load the kernel if it's not been loaded yet or if we changed the context length
+ if rwkv5_cuda_kernel is not None and rwkv5_cuda_kernel.head_size == head_size:
+ return
+
+ logger.info(f"Loading CUDA kernel for RWKV5 at head size of {head_size}.")
+
+ flags = [
+ "-res-usage",
+ "--maxrregcount 60",
+ "--use_fast_math",
+ "-O3",
+ "-Xptxas -O3",
+ "--extra-device-vectorization",
+ f"-D_N_={head_size}",
+ ]
+ rwkv5_cuda_kernel = load_kernel(
+ name=f"wkv_{head_size}",
+ sources=cuda_kernel_files,
+ verbose=(logging.get_verbosity() == logging.DEBUG),
+ extra_cuda_cflags=flags,
+ )
+ rwkv5_cuda_kernel.head_size = head_size
+
+
+class Rwkv5LinearAttention(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, receptance, key, value, time_decay, time_first, state):
+ with torch.no_grad():
+ assert receptance.dtype == torch.bfloat16
+ assert key.dtype == torch.bfloat16
+ assert value.dtype == torch.bfloat16
+ assert time_decay.dtype == torch.bfloat16
+ assert time_first.dtype == torch.bfloat16
+ assert state.dtype == torch.float32
+ batch, seq_length, hidden_size = key.shape
+ num_heads = time_decay.shape[0]
+ ctx.batch = batch
+ ctx.seq_length = seq_length
+ ctx.hidden_size = hidden_size
+ ctx.num_heads = num_heads
+ e_time_decay = (-torch.exp(time_decay.float())).contiguous()
+ ee_time_decay = (torch.exp(e_time_decay)).contiguous()
+ assert ee_time_decay.dtype == torch.float32
+ ctx.save_for_backward(receptance, key, value, ee_time_decay, e_time_decay, time_first)
+ out = torch.empty(
+ (batch, seq_length, hidden_size),
+ device=receptance.device,
+ dtype=torch.bfloat16,
+ memory_format=torch.contiguous_format,
+ )
+ state = state.clone()
+ rwkv5_cuda_kernel.forward_bf16(
+ batch,
+ seq_length,
+ hidden_size,
+ num_heads,
+ state,
+ receptance,
+ key,
+ value,
+ ee_time_decay,
+ time_first,
+ out,
+ )
+ return out, state
+
+ @staticmethod
+ def backward(ctx, gout):
+ with torch.no_grad():
+ assert gout.dtype == torch.bfloat16
+ batch = ctx.batch
+ seq_length = ctx.seq_length
+ hidden_size = ctx.hidden_size
+ num_heads = ctx.num_heads
+ receptance, key, value, ee_time_decay, e_time_decay, time_first = ctx.saved_tensors
+
+ global_shape = (batch, seq_length, hidden_size)
+
+ # TODO dtype should not be forced here IMO
+ greceptance = torch.empty(
+ global_shape,
+ device=gout.device,
+ requires_grad=False,
+ dtype=torch.bfloat16,
+ memory_format=torch.contiguous_format,
+ )
+ g_key = torch.empty(
+ global_shape,
+ device=gout.device,
+ requires_grad=False,
+ dtype=torch.bfloat16,
+ memory_format=torch.contiguous_format,
+ )
+ g_value = torch.empty(
+ global_shape,
+ device=gout.device,
+ requires_grad=False,
+ dtype=torch.bfloat16,
+ memory_format=torch.contiguous_format,
+ )
+ g_time_decay = torch.empty(
+ (batch, hidden_size),
+ device=gout.device,
+ requires_grad=False,
+ dtype=torch.bfloat16,
+ memory_format=torch.contiguous_format,
+ )
+ g_time_first = torch.empty(
+ (batch, hidden_size),
+ device=gout.device,
+ requires_grad=False,
+ dtype=torch.bfloat16,
+ memory_format=torch.contiguous_format,
+ )
+ rwkv5_cuda_kernel.backward_bf16(
+ batch,
+ seq_length,
+ hidden_size,
+ num_heads,
+ receptance,
+ key,
+ value,
+ ee_time_decay,
+ e_time_decay,
+ time_first,
+ gout,
+ greceptance,
+ g_key,
+ g_value,
+ g_time_decay,
+ g_time_first,
+ )
+ head_size = hidden_size // num_heads
+ g_time_decay = torch.sum(g_time_decay, 0).view(num_heads, head_size)
+ g_time_first = torch.sum(g_time_first, 0).view(num_heads, head_size)
+ return (None, None, None, None, greceptance, g_key, g_value, g_time_decay, g_time_first)
+
+
+def rwkv5_linear_attention_cpu(receptance, key, value, time_decay, time_first, state):
+ input_dtype = receptance.dtype
+ # For CPU fallback. Will be slower and probably take more memory than the custom CUDA kernel if not executed
+ # within a torch.no_grad.
+ batch, seq_length, hidden_size = receptance.shape
+ num_heads, head_size = time_first.shape
+ key = key.float().view(batch, seq_length, num_heads, head_size).transpose(1, 2).transpose(-2, -1)
+ value = value.float().view(batch, seq_length, num_heads, head_size).transpose(1, 2)
+ receptance = receptance.float().view(batch, seq_length, num_heads, head_size).transpose(1, 2)
+ time_decay = torch.exp(-torch.exp(time_decay.float())).reshape(-1, 1, 1).reshape(num_heads, -1, 1)
+ time_first = time_first.float().reshape(-1, 1, 1).reshape(num_heads, -1, 1)
+ out = torch.zeros_like(key).reshape(batch, seq_length, num_heads, head_size)
+
+ for current_index in range(seq_length):
+ current_receptance = receptance[:, :, current_index:current_index+1, :]
+ current_key = key[:, :, :, current_index:current_index+1]
+ current_value = value[:, :, current_index:current_index+1, :]
+ attention_output = current_key @ current_value
+ out[:, current_index] = (current_receptance @ (time_first * attention_output + state)).squeeze(2)
+ with torch.no_grad():
+ state = attention_output + time_decay * state
+
+ return out, state
+
+# copied from RWKV but with receptance
+def RWKV5_linear_attention(training, receptance, key, value, time_decay, time_first, state):
+ no_cuda = any(t.device.type != "cuda" for t in [time_decay, time_first, key, value])
+ # Launching the CUDA kernel for just one token will actually be slower (there is no for loop in the CPU version
+ # in this case).
+ one_token = key.size(1) == 1
+ if not training or rwkv5_cuda_kernel is None or no_cuda or one_token:
+ return rwkv5_linear_attention_cpu(
+ receptance, key, value, time_decay, time_first, state
+ )
+ else:
+ return Rwkv5LinearAttention.apply(receptance, key, value, time_decay, time_first, state)
+
+
+class Rwkv5SelfAttention(nn.Module):
+ def __init__(self, config, layer_id=0):
+ super().__init__()
+ self.config = config
+ kernel_loaded = rwkv5_cuda_kernel is not None and rwkv5_cuda_kernel.head_size == config.head_size
+ if is_ninja_available() and is_torch_cuda_available() and not kernel_loaded:
+ try:
+ load_wkv5_cuda_kernel(config.head_size)
+ except Exception:
+ logger.info("Could not load the custom CUDA kernel for RWKV5 attention.")
+ self.layer_id = layer_id
+ hidden_size = config.hidden_size
+ attention_hidden_size = config.attention_hidden_size
+ self.attention_hidden_size = attention_hidden_size
+ head_size = config.head_size
+ num_heads = attention_hidden_size // head_size
+
+ self.time_decay = nn.Parameter(torch.empty(num_heads, head_size))
+ self.time_faaaa = nn.Parameter(torch.empty(num_heads, head_size))
+ self.time_mix_gate = nn.Parameter(torch.empty(1, 1, hidden_size))
+
+ self.time_mix_key = nn.Parameter(torch.empty(1, 1, hidden_size))
+ self.time_mix_value = nn.Parameter(torch.empty(1, 1, hidden_size))
+ self.time_mix_receptance = nn.Parameter(torch.empty(1, 1, hidden_size))
+
+ self.time_shift = nn.ZeroPad2d((0, 0, 1, -1))
+ self.key = nn.Linear(hidden_size, attention_hidden_size, bias=False)
+ self.value = nn.Linear(hidden_size, attention_hidden_size, bias=False)
+ self.receptance = nn.Linear(hidden_size, attention_hidden_size, bias=False)
+ self.gate = nn.Linear(hidden_size, attention_hidden_size, bias=False)
+ self.output = nn.Linear(attention_hidden_size, hidden_size, bias=False)
+ self.ln_x = nn.GroupNorm(num_heads, hidden_size)
+
+ def extract_key_value(self, hidden, state=None):
+ # Mix hidden with the previous timestep to produce key, value, receptance
+ if hidden.size(1) == 1 and state is not None:
+ shifted = state[0][:, :, self.layer_id]
+ else:
+ shifted = self.time_shift(hidden)
+ if state is not None:
+ shifted[:, 0] = state[0][:, :, self.layer_id]
+ if len(shifted.size()) == 2:
+ shifted = shifted.unsqueeze(1)
+ 
+ key = hidden * self.time_mix_key + shifted * (1 - self.time_mix_key)
+ value = hidden * self.time_mix_value + shifted * (1 - self.time_mix_value)
+ receptance = hidden * self.time_mix_receptance + shifted * (1 - self.time_mix_receptance)
+ gate = hidden * self.time_mix_gate + shifted * (1 - self.time_mix_gate)
+
+ key = self.key(key)
+ value = self.value(value)
+ receptance = self.receptance(receptance)
+ gate = F.silu(self.gate(gate))
+
+ if state is not None:
+ state[0][:, :, self.layer_id] = hidden[:, -1]
+
+ return receptance, key, value, gate, state
+
+ def forward(self, hidden, state=None, use_cache=False, seq_mode=True):
+ receptance, key, value, gate, state = self.extract_key_value(hidden, state=state)
+
+ B,T,C = receptance.shape
+ H, S = self.time_faaaa.shape
+
+ layer_state = state[1][:, :, :, :, self.layer_id] if state is not None else None
+ out, layer_state = RWKV5_linear_attention(
+ self.training, receptance, key, value, self.time_decay, self.time_faaaa, layer_state
+ )
+
+ if layer_state is not None:
+ state[1][:, :, :, :, self.layer_id] = layer_state
+
+ out = out.reshape(B * T, H * S)
+ out = F.group_norm(out / self.config.head_size_divisor, num_groups=H, weight=self.ln_x.weight.to(out.dtype), bias=self.ln_x.bias.to(out.dtype), eps=self.ln_x.eps).reshape(B, T, H * S)
+ out = out.to(dtype=hidden.dtype) * gate
+ out = self.output(out)
+ return out, state
+
+# Copied from rwkv exceot for the intermediate size
+class Rwkv5FeedForward(nn.Module):
+ def __init__(self, config, layer_id=0):
+ super().__init__()
+ self.config = config
+ self.layer_id = layer_id
+ hidden_size = config.hidden_size
+ intermediate_size = (
+ config.intermediate_size
+ if config.intermediate_size is not None
+ else int((config.hidden_size * 3.5) // 32 * 32)
+ )
+
+ self.time_shift = nn.ZeroPad2d((0, 0, 1, -1))
+ self.time_mix_key = nn.Parameter(torch.empty(1, 1, hidden_size))
+ self.time_mix_receptance = nn.Parameter(torch.empty(1, 1, hidden_size))
+
+ self.key = nn.Linear(hidden_size, intermediate_size, bias=False)
+ self.receptance = nn.Linear(hidden_size, hidden_size, bias=False)
+ self.value = nn.Linear(intermediate_size, hidden_size, bias=False)
+
+ def forward(self, hidden, state=None):
+ if hidden.size(1) == 1 and state is not None:
+ shifted = state[2][:, :, self.layer_id]
+ else:
+ shifted = self.time_shift(hidden)
+ if state is not None:
+ shifted[:, 0] = state[2][:, :, self.layer_id]
+ if len(shifted.size()) == 2:
+ shifted = shifted.unsqueeze(1)
+ key = hidden * self.time_mix_key + shifted * (1 - self.time_mix_key)
+ receptance = hidden * self.time_mix_receptance + shifted * (1 - self.time_mix_receptance)
+
+ key = torch.square(torch.relu(self.key(key)))
+ value = self.value(key)
+ receptance = torch.sigmoid(self.receptance(receptance))
+
+ if state is not None:
+ state[2][:, :, self.layer_id] = hidden[:, -1]
+
+ return receptance * value, state
+
+
+# Copied from transformers.models.rwkv.modeling_rwkv.RwkvBlock with Rwkv->Rwkv5
+class Rwkv5Block(nn.Module):
+ def __init__(self, config, layer_id):
+ super().__init__()
+ self.config = config
+ self.layer_id = layer_id
+
+ if layer_id == 0:
+ self.pre_ln = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_epsilon)
+
+ self.ln1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_epsilon)
+ self.ln2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_epsilon)
+
+ self.attention = Rwkv5SelfAttention(config, layer_id)
+ self.feed_forward = Rwkv5FeedForward(config, layer_id)
+
+ def forward(self, hidden, state=None, use_cache=False, output_attentions=False, seq_mode=True):
+ if self.layer_id == 0:
+ hidden = self.pre_ln(hidden)
+ attention, state = self.attention(self.ln1(hidden), state=state, use_cache=use_cache, seq_mode=seq_mode)
+ hidden = hidden + attention
+
+ feed_forward, state = self.feed_forward(self.ln2(hidden), state=state)
+ hidden = hidden + feed_forward
+
+ outputs = (hidden, state)
+ if output_attentions:
+ outputs += (attention,)
+ else:
+ outputs += (None,)
+
+ return outputs
+
+
+# Copied from transformers.models.rwkv.modeling_rwkv.RwkvPreTrainedModel with Rwkv->Rwkv5
+class Rwkv5PreTrainedModel(PreTrainedModel):
+ """
+ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+ models.
+ """
+
+ config_class = Rwkv5Config
+ base_model_prefix = "rwkv5"
+ _no_split_modules = ["Rwkv5Block"]
+ _keep_in_fp32_modules = ["time_decay", "time_first"]
+ supports_gradient_checkpointing = True
+
+ def _init_weights(self, module):
+ """Initialize the weights."""
+ if isinstance(module, Rwkv5SelfAttention):
+ layer_id = module.layer_id
+ num_hidden_layers = module.config.num_hidden_layers
+ hidden_size = module.config.hidden_size
+ attention_hidden_size = module.attention_hidden_size
+ head_size = module.config.head_size
+ num_heads = attention_hidden_size // head_size
+
+ ratio_0_to_1 = layer_id / (num_hidden_layers - 1) # 0 to 1
+ ratio_1_to_almost0 = 1.0 - (layer_id / num_hidden_layers) # 1 to ~0
+
+ time_weight = torch.tensor(
+ [i / hidden_size for i in range(hidden_size)],
+ dtype=module.time_mix_key.dtype,
+ device=module.time_mix_key.device,
+ )
+ time_weight = time_weight[None, None, :]
+
+ decay_speed = [
+ -6.0 + 5.0 * (h / (attention_hidden_size - 1)) ** (0.7 + 1.3 * ratio_0_to_1)
+ for h in range(attention_hidden_size)
+ ]
+ decay_speed = torch.tensor(decay_speed, dtype=module.time_decay.dtype, device=module.time_decay.device)
+ tmp = torch.tensor(
+ [
+ (1.0 - (i / (attention_hidden_size - 1.0))) * ratio_0_to_1 + 0.1 * ((i + 1) % 3 - 1)
+ for i in range(attention_hidden_size)
+ ],
+ dtype=module.time_faaaa.dtype,
+ device=module.time_faaaa.device,
+ )
+
+ with torch.no_grad():
+ module.time_decay.data = decay_speed.reshape(num_heads, head_size)
+ module.time_faaaa.data = tmp.reshape(num_heads, head_size)
+ module.time_mix_key.data = torch.pow(time_weight, ratio_1_to_almost0)
+
+ module.time_mix_value.data = torch.pow(time_weight, ratio_1_to_almost0) + 0.3 * ratio_0_to_1
+ module.time_mix_receptance.data = torch.pow(time_weight, 0.5 * ratio_1_to_almost0)
+ module.time_mix_gate.data = torch.pow(time_weight, 0.5 * ratio_1_to_almost0)
+
+ elif isinstance(module, Rwkv5FeedForward):
+ layer_id = module.layer_id
+ num_hidden_layers = module.config.num_hidden_layers
+ hidden_size = module.config.hidden_size
+
+ ratio_1_to_almost0 = 1.0 - (layer_id / num_hidden_layers) # 1 to ~0
+
+ time_weight = torch.tensor(
+ [i / hidden_size for i in range(hidden_size)],
+ dtype=module.time_mix_key.dtype,
+ device=module.time_mix_key.device,
+ )
+ time_weight = time_weight[None, None, :]
+
+ with torch.no_grad():
+ module.time_mix_key.data = torch.pow(time_weight, ratio_1_to_almost0)
+ module.time_mix_receptance.data = torch.pow(time_weight, ratio_1_to_almost0)
+
+
+# Copied from transformers.models.rwkv.modeling_rwkv.RwkvOutput with Rwkv->Rwkv5
+@dataclass
+class Rwkv5Output(ModelOutput):
+ """
+ Class for the RWKV5 model outputs.
+
+ Args:
+ last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+ Sequence of hidden-states at the output of the last layer of the model.
+ state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`):
+ The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
+ avoid providing the old `input_ids`.
+ hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+ one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of
+ the model at the output of each layer plus the optional initial embedding outputs.
+ attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
+ """
+
+ last_hidden_state: torch.FloatTensor = None
+ state: Optional[List[torch.FloatTensor]] = None
+ hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+ attentions: Optional[Tuple[torch.FloatTensor]] = None
+
+
+# Copied from transformers.models.rwkv.modeling_rwkv.RwkvCausalLMOutput with Rwkv->Rwkv5
+@dataclass
+class Rwkv5CausalLMOutput(ModelOutput):
+ """
+ Base class for causal language model (or autoregressive) outputs.
+
+ Args:
+ loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
+ Language modeling loss (for next-token prediction).
+ logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
+ Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+ state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`):
+ The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
+ avoid providing the old `input_ids`.
+ hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+ one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of
+ the model at the output of each layer plus the optional initial embedding outputs.
+ attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
+ """
+
+ loss: Optional[torch.FloatTensor] = None
+ logits: torch.FloatTensor = None
+ state: Optional[List[torch.FloatTensor]] = None
+ hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+ attentions: Optional[Tuple[torch.FloatTensor]] = None
+
+
+RWKV5_START_DOCSTRING = r"""
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
+ subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
+ general usage and behavior.
+
+ Parameters:
+ config ([`Rwkv5Config`]): Model configuration class with all the parameters of the model.
+ Initializing with a config file does not load the weights associated with the model, only the
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+RWKV5_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
+ `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
+ `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input
+ sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their
+ past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See
+ [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input
+ IDs?](../glossary#input-ids)
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
+ state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*):
+ If passed along, the model uses the previous state in all the blocks (which will give the output for the
+ `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
+ use_cache (`bool`, *optional*):
+ If set to `True`, the last state is returned and can be used to quickly generate the next logits.
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+@add_start_docstrings(
+ "The bare RWKV5 Model transformer outputting raw hidden-states without any specific head on top.",
+ RWKV5_START_DOCSTRING,
+)
+class Rwkv5Model(Rwkv5PreTrainedModel):
+ def __init__(self, config):
+ super().__init__(config)
+
+ self.embeddings = nn.Embedding(config.vocab_size, config.hidden_size)
+ self.blocks = nn.ModuleList([Rwkv5Block(config, layer_id=idx) for idx in range(config.num_hidden_layers)])
+ self.ln_out = nn.LayerNorm(config.hidden_size)
+
+ self.layers_are_rescaled = False
+ self.gradient_checkpointing = False
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.embeddings
+
+ def set_input_embeddings(self, new_embeddings):
+ self.embeddings = new_embeddings
+
+ @add_start_docstrings_to_model_forward(RWKV5_INPUTS_DOCSTRING)
+ @add_code_sample_docstrings(
+ checkpoint=_CHECKPOINT_FOR_DOC,
+ output_type=Rwkv5Output,
+ config_class=_CONFIG_FOR_DOC,
+ )
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ attention_mask: Optional[torch.LongTensor] = None, # noqa
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ state: Optional[List[torch.FloatTensor]] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, Rwkv5Output]:
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ # FIXME - training is supportable with the CUDA code
+ # rwkv5 only support inference in huggingface.
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ if self.training == self.layers_are_rescaled and (
+ self.embeddings.weight.dtype == torch.float16 or self.embeddings.weight.dtype == torch.bfloat16
+ ):
+ self._rescale_layers()
+
+ if input_ids is not None and inputs_embeds is not None:
+ raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+ elif input_ids is None and inputs_embeds is None:
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+ if inputs_embeds is None:
+ inputs_embeds = self.embeddings(input_ids)
+
+ if state is None:
+ state = []
+ head_size = self.config.head_size
+ num_heads = self.config.attention_hidden_size // head_size
+ state_attn_x = torch.zeros(
+ (inputs_embeds.size(0), self.config.hidden_size, self.config.num_hidden_layers),
+ dtype=inputs_embeds.dtype,
+ requires_grad=False,
+ device=inputs_embeds.device,
+ ).contiguous()
+ state_attn_kv = torch.zeros(
+ (
+ inputs_embeds.size(0),
+ num_heads,
+ head_size,
+ head_size,
+ self.config.num_hidden_layers,
+ ),
+ dtype=torch.float32,
+ requires_grad=False,
+ device=inputs_embeds.device,
+ ).contiguous()
+ state_ffn_x = torch.zeros(
+ (inputs_embeds.size(0), self.config.hidden_size, self.config.num_hidden_layers),
+ dtype=inputs_embeds.dtype,
+ requires_grad=False,
+ device=inputs_embeds.device,
+ ).contiguous()
+ state.append(state_attn_x)
+ state.append(state_attn_kv)
+ state.append(state_ffn_x)
+
+ seq_mode = inputs_embeds.shape[1] > 1
+ hidden_states = inputs_embeds
+
+ all_self_attentions = () if output_attentions else None
+ all_hidden_states = () if output_hidden_states else None
+ for idx, block in enumerate(self.blocks):
+ hidden_states, state, attentions = block(
+ hidden_states, state=state, use_cache=use_cache, output_attentions=output_attentions, seq_mode=seq_mode
+ )
+ if (
+ self.layers_are_rescaled
+ and self.config.rescale_every > 0
+ and (idx + 1) % self.config.rescale_every == 0
+ ):
+ hidden_states = hidden_states / 2
+
+ if output_hidden_states:
+ all_hidden_states = all_hidden_states + (hidden_states,)
+
+ if output_attentions:
+ all_self_attentions = all_self_attentions + (attentions,)
+
+ hidden_states = self.ln_out(hidden_states)
+
+ if output_hidden_states:
+ all_hidden_states = all_hidden_states + (hidden_states,)
+
+ if not return_dict:
+ return (hidden_states, state, all_hidden_states, all_self_attentions)
+
+ return Rwkv5Output(
+ last_hidden_state=hidden_states,
+ state=state,
+ hidden_states=all_hidden_states, # None
+ attentions=all_self_attentions, # None
+ )
+
+ def _rescale_layers(self):
+ # Layers should be rescaled for inference only.
+ if self.layers_are_rescaled == (not self.training):
+ return
+ if self.config.rescale_every > 0:
+ with torch.no_grad():
+ for block_id, block in enumerate(self.blocks):
+ if self.training:
+ block.attention.output.weight.mul_(2 ** int(block_id // self.config.rescale_every))
+ block.feed_forward.value.weight.mul_(2 ** int(block_id // self.config.rescale_every))
+ else:
+ # Deal with quantization statistics
+ if hasattr(block.attention.output.weight, "SCB"):
+ block.attention.output.weight.SCB.div_(2 ** int(block_id // self.config.rescale_every))
+ block.feed_forward.value.weight.SCB.div_(2 ** int(block_id // self.config.rescale_every))
+ elif hasattr(block.attention.output.weight, "quant_state"):
+ self._bnb_4bit_dequantize_and_rescale(block.attention.output, block_id)
+ self._bnb_4bit_dequantize_and_rescale(block.feed_forward.value, block_id)
+ else:
+ block.attention.output.weight.div_(2 ** int(block_id // self.config.rescale_every))
+ block.feed_forward.value.weight.div_(2 ** int(block_id // self.config.rescale_every))
+
+ self.layers_are_rescaled = not self.training
+
+ def _bnb_4bit_dequantize_and_rescale(self, target_layer, block_id):
+ r"""
+ Perform the dequantization and rescaling of the weights of a given layer. After that operation the layer will
+ be quantized again.
+ """
+ if not is_bitsandbytes_available():
+ raise ImportError("Please install bitsandbytes to use this method.")
+ import bitsandbytes as bnb
+
+ dequant_weights = bnb.functional.dequantize_4bit(target_layer.weight.data, target_layer.weight.quant_state)
+
+ dequant_weights.div_(2 ** int(block_id // self.config.rescale_every))
+
+ # re-quantize the model:
+ # we need to put it first on CPU then back to the device
+ # this will create an overhead :/
+ # We set requires_grad=False as we cannot compute gradients on top of 4bit parameters anyway and to avoid
+ # bugs with bnb
+ quant_weight = bnb.nn.Params4bit(dequant_weights.to("cpu"), requires_grad=False).to(dequant_weights.device)
+ setattr(target_layer, "weight", quant_weight)
+
+
+# copied from HuggingFace https://github.com/huggingface/transformers/blob/main/src/transformers/models/rwkv/modeling_rwkv.py
+@add_start_docstrings(
+ """
+ The RWKV5 Model transformer with a language modeling head on top (linear layer with weights tied to the input
+ embeddings).
+ """,
+ RWKV5_START_DOCSTRING,
+)
+# Copied from transformers.models.rwkv.modeling_rwkv.RwkvForCausalLM with Rwkv->Rwkv5
+class Rwkv5ForCausalLM(Rwkv5PreTrainedModel):
+ _tied_weights_keys = ["head.weight"]
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.rwkv = Rwkv5Model(config)
+ self.head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_output_embeddings(self):
+ return self.head
+
+ def set_output_embeddings(self, new_embeddings):
+ self.head = new_embeddings
+
+ def prepare_inputs_for_generation(self, input_ids, state=None, inputs_embeds=None, **kwargs):
+ # only last token for inputs_ids if the state is passed along.
+ if state is not None:
+ input_ids = input_ids[:, -1].unsqueeze(-1)
+
+ # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+ if inputs_embeds is not None and state is None:
+ model_inputs = {"inputs_embeds": inputs_embeds}
+ else:
+ model_inputs = {"input_ids": input_ids}
+
+ model_inputs["state"] = state
+ return model_inputs
+
+ @add_start_docstrings_to_model_forward(RWKV5_INPUTS_DOCSTRING)
+ @add_code_sample_docstrings(
+ checkpoint=_CHECKPOINT_FOR_DOC,
+ output_type=Rwkv5CausalLMOutput,
+ config_class=_CONFIG_FOR_DOC,
+ )
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ attention_mask: Optional[torch.LongTensor] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ state: Optional[List[torch.FloatTensor]] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, Rwkv5CausalLMOutput]:
+ r"""
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
+ `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+ are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+ """
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ outputs = self.rwkv(
+ input_ids,
+ inputs_embeds=inputs_embeds,
+ state=state,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+ hidden_states = outputs[0]
+
+ logits = self.head(hidden_states)
+
+ loss = None
+ if labels is not None:
+ # move labels to correct device to enable model parallelism
+ labels = labels.to(logits.device)
+ # Shift so that tokens < n predict n
+ shift_logits = logits[..., :-1, :].contiguous()
+ shift_labels = labels[..., 1:].contiguous()
+ # Flatten the tokens
+ loss_fct = CrossEntropyLoss()
+ loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+
+ if not return_dict:
+ output = (logits,) + outputs[1:]
+ return ((loss,) + output) if loss is not None else output
+
+ return Rwkv5CausalLMOutput(
+ loss=loss,
+ logits=logits,
+ state=outputs.state,
+ hidden_states=outputs.hidden_states,
+ attentions=outputs.attentions,
+ )

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:270b507b47e08b5f71442380a08668567124a77ea0b1d2e273f85751bc795aa3
+size 6126083713

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+ "bos_token": "<s>",
+ "eos_token": "<s>",
+ "pad_token": "<s>",
+ "unk_token": "<s>"
+}

tokenization_rwkv5.py

@@ -0,0 +1,230 @@
+# coding=utf-8
+# Copyright 2024 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for RWKV5."""
+
+import os
+from typing import TYPE_CHECKING, List, Optional, Tuple
+import re
+
+from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
+from transformers.utils import logging
+
+
+if TYPE_CHECKING:
+ pass
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {
+ "vocab_file": "vocab.txt",
+}
+PRETRAINED_VOCAB_FILES_MAP = {
+ "vocab_file": {
+ "ArthurZ/rwkv-5-utf": "https://huggingface.co/ArthurZ/rwkv-5-utf/blob/main/vocab.txt",
+ },
+}
+
+
+
+def whitespace_tokenize(text):
+ """Runs basic whitespace cleaning and splitting on a piece of text.
+ The separators are kept
+ """
+ text = text.strip()
+ if not text:
+ return []
+ tokens = re.split(b"(?= )", text)
+ return tokens
+
+
+class WordpieceTokenizer(object):
+ """Runs WordPiece tokenization."""
+
+ def __init__(self, vocab, unk_token, max_input_chars_per_word=100):
+ self.vocab = vocab
+ self.unk_token = unk_token
+ self.max_input_chars_per_word = max_input_chars_per_word
+
+ def tokenize(self, text):
+ """
+ Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform
+ tokenization using the given vocabulary.
+
+ For example, `input = "unaffable"` wil return as output `["un", "##aff", "##able"]`.
+
+ Args:
+ text: A single token or whitespace separated tokens. This should have
+ already been passed through *BasicTokenizer*.
+
+ Returns:
+ A list of wordpiece tokens.
+ """
+
+ output_tokens = []
+ for token in whitespace_tokenize(text):
+ chars = list(token)
+ if len(chars) > self.max_input_chars_per_word:
+ output_tokens.append(self.unk_token)
+ continue
+
+ is_bad = False
+ start = 0
+ sub_tokens = []
+ while start < len(chars):
+ end = len(chars)
+ cur_substr = None
+ while start < end:
+ substr = bytes(chars[start:end])
+ if substr in self.vocab:
+ cur_substr = substr
+ break
+ end -= 1
+ if cur_substr is None:
+ is_bad = True
+ break
+ sub_tokens.append(cur_substr.decode())
+ start = end
+
+ if is_bad:
+ output_tokens.append(self.unk_token)
+ else:
+ output_tokens.extend(sub_tokens)
+ return output_tokens
+
+
+class Rwkv5Tokenizer(PreTrainedTokenizer):
+ vocab_files_names = VOCAB_FILES_NAMES
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+ max_model_input_sizes = {"ArthurZ/rwkv-5-utf": 2048}
+
+ model_input_names = ["input_ids", "attention_mask"]
+
+ def __init__(self, vocab_file, bos_token="<s>", eos_token="<s>", unk_token="<s>", pad_token="<s>",**kwargs):
+ if not os.path.isfile(vocab_file):
+ raise ValueError(
+ f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"
+ " model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
+ )
+
+ with open(vocab_file, "r") as reader:
+ tokens = reader.readlines()
+ vocab = {}
+ for index, token in enumerate(tokens):
+ token = eval(token.rstrip("\n"))
+ vocab[token] = index
+
+ self.add_bos_token = True
+ self.encoder = vocab
+ self.decoder = {v: k for k, v in vocab.items()}
+ self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.encoder, unk_token=str(unk_token))
+ self._added_tokens_decoder = {0: AddedToken(str(bos_token))}
+ super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, pad_token=pad_token, **kwargs)
+
+ @property
+ def vocab_size(self):
+ return len(self.encoder)
+
+ def get_vocab(self):
+ vocab = {str(self.convert_ids_to_tokens(i)): i for i in range(self.vocab_size)}
+ vocab.update(self.added_tokens_encoder)
+ return vocab
+
+ def _tokenize(self, text, split_special_tokens=False):
+ return self.wordpiece_tokenizer.tokenize(text.encode("utf-8"))
+
+ def _convert_token_to_id(self, token):
+ """Converts a token (byte) to an id using the vocab."""
+ if not isinstance(token, bytes):
+ token = token.encode("utf-8", errors="replace")
+ return self.encoder.get(token, self.unk_token_id)
+
+ def _convert_id_to_token(self, index):
+ """Converts an index (integer) in a token (byte) using the vocab."""
+ token = self.decoder.get(index, self.unk_token)
+ if isinstance(token, (bytes)):
+ token = token.decode("utf-8", errors="replace")
+ return token
+
+ def convert_tokens_to_string(self, tokens):
+ """Converts a sequence of tokens (bytes) in a single string. Additional tokens are encoded to bytes"""
+ out_string = b"".join([k.encode(errors="replace") if isinstance(k, str) else k for k in tokens]).decode(
+ "utf-8"
+ )
+ return out_string
+
+ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+ index = 0
+ if os.path.isdir(save_directory):
+ vocab_file = os.path.join(
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+ )
+ else:
+ vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory
+ with open(vocab_file, "w") as writer:
+ for token, token_index in sorted(self.encoder.items(), key=lambda kv: kv[1]):
+ if index != token_index:
+ logger.warning(
+ f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
+ " Please check that the vocabulary is not corrupted!"
+ )
+ index = token_index
+ writer.write(str(token) + "\n")
+ index += 1
+ return (vocab_file,)
+
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+ if self.add_bos_token:
+ bos_token_ids = [self.bos_token_id]
+ else:
+ bos_token_ids = []
+
+ output = bos_token_ids + token_ids_0
+
+ if token_ids_1 is None:
+ return output
+
+ return output + bos_token_ids + token_ids_1
+
+ def get_special_tokens_mask(
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+ ) -> List[int]:
+ """
+ Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
+ special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.
+
+ Args:
+ token_ids_0 (`List[int]`):
+ List of IDs.
+ token_ids_1 (`List[int]`, *optional*):
+ Optional second list of IDs for sequence pairs.
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+ Whether or not the token list is already formatted with special tokens for the model.
+
+ Returns:
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+ """
+ if already_has_special_tokens:
+ return super().get_special_tokens_mask(
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+ )
+
+ if not self.add_bos_token:
+ return super().get_special_tokens_mask(
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=False
+ )
+
+ if token_ids_1 is None:
+ return [1] + ([0] * len(token_ids_0))
+ return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1))

tokenization_rwkv_world.py

@@ -0,0 +1,549 @@
+# coding=utf-8
+# Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for RWKV5."""
+
+import json
+import os
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
+
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.tokenization_utils_base import (
+ BatchEncoding,
+ EncodedInput,
+ TextInput,
+ TruncationStrategy,
+)
+from transformers.utils import PaddingStrategy, TensorType, logging, to_py_obj
+
+
+if TYPE_CHECKING:
+ from transformers.pipelines.conversational import Conversation
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {
+ "vocab_file": "rwkv_vocab_v20230424.txt",
+}
+PRETRAINED_VOCAB_FILES_MAP = {
+ "vocab_file": {
+ "RWKV/rwkv-5-world-169m": "https://huggingface.co/RWKV/rwkv-5-world-169m/blob/main/rwkv_vocab_v20230424.txt",
+ },
+}
+
+
+class TRIE:
+ __slots__ = tuple("ch,to,values,front".split(","))
+ to: list
+ values: set
+
+ def __init__(self, front=None, ch=None):
+ self.ch = ch
+ self.to = [None for ch in range(256)]
+ self.values = set()
+ self.front = front
+
+ def __repr__(self):
+ fr = self
+ ret = []
+ while fr is not None:
+ if fr.ch is not None:
+ ret.append(fr.ch)
+ fr = fr.front
+ return "<TRIE %s %s>" % (ret[::-1], self.values)
+
+ def add(self, key: bytes, idx: int = 0, val=None):
+ if idx == len(key):
+ if val is None:
+ val = key
+ self.values.add(val)
+ return self
+ ch = key[idx]
+ if self.to[ch] is None:
+ self.to[ch] = TRIE(front=self, ch=ch)
+ return self.to[ch].add(key, idx=idx + 1, val=val)
+
+ def find_longest(self, key: bytes, idx: int = 0):
+ u: TRIE = self
+ ch: int = key[idx]
+
+ while u.to[ch] is not None:
+ u = u.to[ch]
+ idx += 1
+ if u.values:
+ ret = idx, u, u.values
+ if idx == len(key):
+ break
+ ch = key[idx]
+ return ret
+
+
+class RWKVWorldTokenizer(PreTrainedTokenizer):
+ vocab_files_names = VOCAB_FILES_NAMES
+ model_input_names = ["input_ids", "attention_mask"]
+
+ def __init__(self, vocab_file, errors="replace", pad_token="0", **kwargs):
+ self.add_bos_token = False
+ self.encoder = {}
+ sorted = [] # must be already sorted
+ with open(vocab_file, "r", encoding="utf-8") as f:
+ lines = f.readlines()
+ for l in lines:
+ idx = int(l[: l.index(" ")])
+ x = eval(l[l.index(" ") : l.rindex(" ")])
+ x = x.encode("utf-8") if isinstance(x, str) else x
+ assert isinstance(x, bytes)
+ assert len(x) == int(l[l.rindex(" ") :])
+ sorted += [x]
+ self.encoder[idx] = x
+
+ self.decoder = {}
+ for k, v in self.encoder.items():
+ self.decoder[v] = int(k)
+
+ self.trie = TRIE()
+ for t, i in self.decoder.items():
+ _ = self.trie.add(t, val=(t, i))
+ self.errors = errors # how to handle errors in decoding
+ self.cache = {}
+ self.first_max_length = 0
+ super().__init__(
+ errors=errors,
+ **kwargs,
+ )
+
+ @property
+ def eos_token_id(self) -> Optional[int]:
+ return 0
+
+ @property
+ def eot_token_id(self) -> Optional[int]:
+ return 0
+
+ @property
+ def pad_token_id(self) -> Optional[int]:
+ return 0
+
+ @property
+ def vocab_size(self):
+ return len(self.encoder)
+
+ def get_vocab(self):
+ return dict(self.encoder, **self.added_tokens_encoder)
+
+ def add_tokens(self, new_tokens, special_tokens: bool = False):
+ for token in new_tokens:
+ token_id = self.convert_tokens_to_ids(token)
+ self.added_tokens_decoder[token_id] = token
+
+ def convert_ids_to_tokens(self, ids, skip_special_tokens=False):
+ if isinstance(ids, int):
+ ids = [ids]
+ tokens = []
+ for id_ in ids:
+ if id_ in self.added_tokens_decoder:
+ tokens.append(self.added_tokens_decoder[id_])
+ else:
+ tokens.append(self._convert_id_to_token(id_))
+ return tokens
+
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+ if self.add_bos_token:
+ bos_token_ids = [self.bos_token_id]
+ else:
+ bos_token_ids = []
+
+ output = bos_token_ids + token_ids_0
+
+ if token_ids_1 is None:
+ return output
+
+ return output + bos_token_ids + token_ids_1
+
+ def get_special_tokens_mask(
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+ ) -> List[int]:
+ """
+ Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
+ special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.
+
+ Args:
+ token_ids_0 (`List[int]`):
+ List of IDs.
+ token_ids_1 (`List[int]`, *optional*):
+ Optional second list of IDs for sequence pairs.
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+ Whether or not the token list is already formatted with special tokens for the model.
+
+ Returns:
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+ """
+ if already_has_special_tokens:
+ return super().get_special_tokens_mask(
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+ )
+
+ if not self.add_bos_token:
+ return super().get_special_tokens_mask(
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=False
+ )
+
+ if token_ids_1 is None:
+ return [1] + ([0] * len(token_ids_0))
+ return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1))
+
+ def encodeBytes(self, src: bytes):
+ idx: int = 0
+ tokens = []
+ while idx < len(src):
+ _idx: int = idx
+ idx, _, values = self.trie.find_longest(src, idx)
+ assert idx != _idx
+ _, token = next(iter(values))
+ tokens.append(token)
+ return tokens
+
+ def decodeBytes(self, tokens):
+ return b"".join(map(lambda i: self.encoder[i], tokens)) # noqa
+
+ def _tokenize(self, text, **kwargs):
+ """Tokenize a string."""
+ return self.encodeBytes(text.encode("utf-8"))
+
+ def _decode_tokens(self, tokens):
+ try:
+ return self.decodeBytes(tokens).decode("utf-8")
+ except Exception:
+ return "\ufffd" # bad utf-8
+
+ def _decode(
+ self,
+ token_ids: Union[int, List[int]],
+ skip_special_tokens: bool = False,
+ **kwargs,
+ ) -> str:
+ def remove_zeros_from_first_segment(token_ids, first_max_length):
+ first_segment = token_ids[:first_max_length]
+ first_segment_cleaned = [token for token in first_segment if token != 0]
+ return first_segment_cleaned + token_ids[first_max_length:]
+
+ # Convert inputs to python lists
+ token_ids = to_py_obj(token_ids)
+ token_ids = remove_zeros_from_first_segment(token_ids, self.first_max_length)
+ if isinstance(token_ids, int):
+ if token_ids in self.all_special_ids and skip_special_tokens:
+ return ""
+ return self.encoder.get(token_ids, self.unk_token)
+ elif isinstance(token_ids, list):
+ self.first_max_length
+ out_str = ""
+ out_last = 0
+ out_tokens = []
+ for i, token in enumerate(token_ids):
+ if token == 0:
+ break
+ out_tokens += [token]
+ tmp = self._decode_tokens(out_tokens[out_last:])
+ if "\ufffd" not in tmp:
+ out_str += tmp
+ out_last = i + 1
+ return out_str
+ else:
+ return token_ids
+
+ def _convert_token_to_id(self, token):
+ """Converts a token (str) in an id using the vocab."""
+ return self.encoder.get(token, self.encoder.get(self.unk_token))
+
+ def _convert_id_to_token(self, index):
+ """Converts an index (integer) in a token (str) using the vocab."""
+ return self.decoder.get(index)
+
+ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+ if not os.path.exists(save_directory):
+ os.mkdir(save_directory)
+ if not os.path.isdir(save_directory):
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+ return
+ vocab_file = os.path.join(
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+ )
+
+ with open(vocab_file, "w", encoding="utf-8") as f:
+ for idx, x in self.encoder.items():
+ if isinstance(x, str):
+ x = x.decode("utf-8")
+ line = f"{idx} {repr(x)} {len(x)}\n"
+ f.write(line)
+
+ return (vocab_file,)
+
+ def prepare_for_tokenization(self, text, **kwargs):
+ return (text, kwargs)
+
+ def _get_padding_truncation_strategies(
+ self, padding=False, truncation=None, max_length=None, pad_to_multiple_of=None, verbose=True, **kwargs
+ ):
+ return PaddingStrategy.LONGEST, TruncationStrategy.DO_NOT_TRUNCATE, -1, kwargs
+
+ def _encode_plus(
+ self,
+ text: Union[TextInput, EncodedInput],
+ add_special_tokens: bool = True,
+ padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
+ truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
+ max_length: Optional[int] = None,
+ stride: int = 0,
+ pad_to_multiple_of: Optional[int] = None,
+ return_tensors: Optional[Union[str, TensorType]] = None,
+ return_token_type_ids: Optional[bool] = None,
+ return_attention_mask: Optional[bool] = None,
+ return_overflowing_tokens: bool = False,
+ return_special_tokens_mask: bool = False,
+ return_offsets_mapping: bool = False,
+ return_length: bool = False,
+ verbose: bool = True,
+ **kwargs,
+ ) -> BatchEncoding:
+ def get_input_ids(text, max_length=None, pad_token_id=0):
+ def pad_sequence(seq, max_len, pad_tok):
+ return [pad_tok] * (max_len - len(seq)) + seq
+
+ if isinstance(text, str):
+ tokens = self._tokenize(text)
+ if max_length is not None:
+ tokens = pad_sequence(tokens, max_length, pad_token_id)
+ return tokens
+
+ elif isinstance(text, list) and len(text) > 0 and isinstance(text[0], str):
+ tokenized_texts = [self._tokenize(t) for t in text]
+ if max_length is None:
+ max_length = max(len(t) for t in tokenized_texts)
+ return [pad_sequence(t, max_length, pad_token_id) for t in tokenized_texts]
+
+ elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):
+ if max_length is not None and len(text) < max_length:
+ return pad_sequence(text, max_length, pad_token_id)
+ return text
+
+ else:
+ raise ValueError(
+ "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
+ )
+
+ if return_offsets_mapping:
+ raise NotImplementedError(
+ "return_offset_mapping is not available when using Python tokenizers. "
+ "To use this feature, change your tokenizer to one deriving from "
+ "transformers.PreTrainedTokenizerFast. "
+ "More information on available tokenizers at "
+ "https://github.com/huggingface/transformers/pull/2674"
+ )
+
+ first_ids = get_input_ids(text)
+
+ return self.prepare_for_model(
+ first_ids,
+ pair_ids=None,
+ add_special_tokens=add_special_tokens,
+ padding=padding_strategy.value,
+ truncation=truncation_strategy.value,
+ max_length=max_length,
+ stride=stride,
+ pad_to_multiple_of=pad_to_multiple_of,
+ return_tensors=return_tensors,
+ prepend_batch_axis=True,
+ return_attention_mask=return_attention_mask,
+ return_token_type_ids=return_token_type_ids,
+ return_overflowing_tokens=return_overflowing_tokens,
+ return_special_tokens_mask=return_special_tokens_mask,
+ return_length=return_length,
+ verbose=verbose,
+ )
+
+ def _batch_encode_plus(
+ self,
+ batch_text_or_text_pairs: Union[
+ List[TextInput],
+ List[EncodedInput],
+ ],
+ add_special_tokens: bool = True,
+ padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
+ truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
+ max_length: Optional[int] = None,
+ stride: int = 0,
+ pad_to_multiple_of: Optional[int] = None,
+ return_tensors: Optional[Union[str, TensorType]] = None,
+ return_token_type_ids: Optional[bool] = None,
+ return_attention_mask: Optional[bool] = None,
+ return_overflowing_tokens: bool = False,
+ return_special_tokens_mask: bool = False,
+ return_offsets_mapping: bool = False,
+ return_length: bool = False,
+ verbose: bool = True,
+ **kwargs,
+ ) -> BatchEncoding:
+ def get_input_ids(text, max_length=None, pad_token_id=0):
+ def pad_sequence(seq, max_len, pad_tok):
+ return [pad_tok] * (max_len - len(seq)) + seq
+
+ if isinstance(text, str):
+ tokens = self._tokenize(text)
+ if max_length is not None:
+ tokens = pad_sequence(tokens, max_length, pad_token_id)
+ return tokens
+
+ elif isinstance(text, list) and len(text) > 0 and isinstance(text[0], str):
+ tokenized_texts = [self._tokenize(t) for t in text]
+ if max_length is None:
+ max_length = max(len(t) for t in tokenized_texts)
+ return [pad_sequence(t, max_length, pad_token_id) for t in tokenized_texts]
+
+ elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):
+ if max_length is not None and len(text) < max_length:
+ return pad_sequence(text, max_length, pad_token_id)
+ return text
+
+ else:
+ raise ValueError(
+ "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
+ )
+
+ if return_offsets_mapping:
+ raise NotImplementedError(
+ "return_offset_mapping is not available when using Python tokenizers. "
+ "To use this feature, change your tokenizer to one deriving from "
+ "transformers.PreTrainedTokenizerFast."
+ )
+
+ first_max_length = 0
+ second_max_length = 0
+ for ids_or_pair_ids in batch_text_or_text_pairs:
+ if not isinstance(ids_or_pair_ids, (list, tuple)):
+ ids, pair_ids = ids_or_pair_ids, None
+ else:
+ ids, pair_ids = ids_or_pair_ids
+ first_ids = get_input_ids(ids)
+ second_ids = get_input_ids(pair_ids) if pair_ids is not None else None
+ first_max_length = max(first_max_length, len(first_ids))
+ if second_ids is not None:
+ second_max_length = max(second_max_length, len(second_ids))
+
+ self.first_max_length = first_max_length
+ input_ids = []
+ for ids_or_pair_ids in batch_text_or_text_pairs:
+ if not isinstance(ids_or_pair_ids, (list, tuple)):
+ ids, pair_ids = ids_or_pair_ids, None
+ else:
+ ids, pair_ids = ids_or_pair_ids
+
+ first_ids = get_input_ids(ids, max_length=first_max_length)
+ second_ids = get_input_ids(pair_ids, max_length=second_max_length) if pair_ids is not None else None
+ input_ids.append((first_ids, second_ids))
+
+ batch_outputs = self._batch_prepare_for_model(
+ input_ids,
+ add_special_tokens=add_special_tokens,
+ padding_strategy=padding_strategy,
+ truncation_strategy=truncation_strategy,
+ max_length=max_length,
+ stride=stride,
+ pad_to_multiple_of=pad_to_multiple_of,
+ return_attention_mask=return_attention_mask,
+ return_token_type_ids=return_token_type_ids,
+ return_overflowing_tokens=return_overflowing_tokens,
+ return_special_tokens_mask=return_special_tokens_mask,
+ return_length=return_length,
+ return_tensors=return_tensors,
+ verbose=verbose,
+ )
+
+ return BatchEncoding(batch_outputs)
+
+ def decode(
+ self,
+ token_ids: Union[int, List[int]],
+ skip_special_tokens: bool = False,
+ clean_up_tokenization_spaces: bool = None,
+ **kwargs,
+ ) -> str:
+ """
+ Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
+ tokens and clean up tokenization spaces.
+
+ Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.
+
+ Args:
+ token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
+ List of tokenized input ids. Can be obtained using the `__call__` method.
+ skip_special_tokens (`bool`, *optional*, defaults to `False`):
+ Whether or not to remove special tokens in the decoding.
+ clean_up_tokenization_spaces (`bool`, *optional*):
+ Whether or not to clean up the tokenization spaces. If `None`, will default to
+ `self.clean_up_tokenization_spaces`.
+ kwargs (additional keyword arguments, *optional*):
+ Will be passed to the underlying model specific decode method.
+
+ Returns:
+ `str`: The decoded sentence.
+ """
+ # Convert inputs to python lists
+ return self._decode(
+ token_ids=token_ids,
+ skip_special_tokens=skip_special_tokens,
+ clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+ **kwargs,
+ )
+
+ def batch_decode(
+ self,
+ sequences: Union[List[int], List[List[int]]],
+ skip_special_tokens: bool = False,
+ clean_up_tokenization_spaces: bool = None,
+ **kwargs,
+ ) -> List[str]:
+ """
+ Convert a list of lists of token ids into a list of strings by calling decode.
+
+ Args:
+ sequences (`Union[List[int], List[List[int]], np.ndarray, torch.Tensor, tf.Tensor]`):
+ List of tokenized input ids. Can be obtained using the `__call__` method.
+ skip_special_tokens (`bool`, *optional*, defaults to `False`):
+ Whether or not to remove special tokens in the decoding.
+ clean_up_tokenization_spaces (`bool`, *optional*):
+ Whether or not to clean up the tokenization spaces. If `None`, will default to
+ `self.clean_up_tokenization_spaces`.
+ kwargs (additional keyword arguments, *optional*):
+ Will be passed to the underlying model specific decode method.
+
+ Returns:
+ `List[str]`: The list of decoded sentences.
+ """
+ return [
+ self.decode(
+ seq,
+ skip_special_tokens=skip_special_tokens,
+ clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+ **kwargs,
+ )
+ for seq in sequences
+ ]
+
+ def _build_conversation_input_ids(self, conversation: "Conversation") -> List[int]:
+ input_ids = []
+ for is_user, text in conversation.iter_texts():
+ input_ids.extend(self.encode(text, add_special_tokens=False) + [self.eos_token_id])
+ if len(input_ids) > self.model_max_length:
+ input_ids = input_ids[-self.model_max_length :]
+ return input_ids

tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+ "auto_map": {
+ "AutoTokenizer": [
+ "tokenization_rwkv5.Rwkv5Tokenizer",
+ null
+ ]
+ },
+ "added_tokens_decoder": {
+ "0": {
+ "content": "<s>",
+ "lstrip": false,
+ "normalized": true,
+ "rstrip": false,
+ "single_word": false,
+ "special": false
+ }
+ },
+ "bos_token": "<s>",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<s>",
+ "model_max_length": 1000000000000000019884624838656,
+ "unk_token": "<s>"
+}