Spaces:
Runtime error
Runtime error
File size: 2,298 Bytes
6831a54 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 |
import gguf
import torch
quants_mapping = {
gguf.GGMLQuantizationType.Q2_K: gguf.Q2_K,
gguf.GGMLQuantizationType.Q3_K: gguf.Q3_K,
gguf.GGMLQuantizationType.Q4_0: gguf.Q4_0,
gguf.GGMLQuantizationType.Q4_K: gguf.Q4_K,
gguf.GGMLQuantizationType.Q4_1: gguf.Q4_1,
gguf.GGMLQuantizationType.Q5_0: gguf.Q5_0,
gguf.GGMLQuantizationType.Q5_1: gguf.Q5_1,
gguf.GGMLQuantizationType.Q5_K: gguf.Q5_K,
gguf.GGMLQuantizationType.Q6_K: gguf.Q6_K,
gguf.GGMLQuantizationType.Q8_0: gguf.Q8_0,
}
class ParameterGGUF(torch.nn.Parameter):
def __init__(self, tensor=None, requires_grad=False, no_init=False):
super().__init__()
self.is_gguf = True
if no_init:
return
self.gguf_type = tensor.tensor_type
self.gguf_real_shape = torch.Size(reversed(list(tensor.shape)))
self.gguf_cls = quants_mapping.get(self.gguf_type, None)
@property
def shape(self):
return self.gguf_real_shape
def __new__(cls, tensor=None, requires_grad=False, no_init=False):
return super().__new__(cls, torch.tensor(tensor.data), requires_grad=requires_grad)
def to(self, *args, **kwargs):
new = ParameterGGUF(self.data.to(*args, **kwargs), no_init=True)
new.gguf_type = self.gguf_type
new.gguf_real_shape = self.gguf_real_shape
new.gguf_cls = self.gguf_cls
return new
def pin_memory(self, device=None):
new = ParameterGGUF(torch.Tensor.pin_memory(self, device=device), no_init=True)
new.gguf_type = self.gguf_type
new.gguf_real_shape = self.gguf_real_shape
new.gguf_cls = self.gguf_cls
return new
@classmethod
def make(cls, data, gguf_type, gguf_cls, gguf_real_shape):
new = ParameterGGUF(data, no_init=True)
new.gguf_type = gguf_type
new.gguf_real_shape = gguf_real_shape
new.gguf_cls = gguf_cls
return new
def dequantize_tensor(tensor):
if tensor is None:
return None
if not hasattr(tensor, 'gguf_cls'):
return tensor
data = torch.tensor(tensor.data)
gguf_cls = tensor.gguf_cls
gguf_real_shape = tensor.gguf_real_shape
if gguf_cls is None:
return data
return gguf_cls.dequantize_pytorch(data, gguf_real_shape)
|