Spaces:
Runtime error
Runtime error
Commit
•
9ee08ed
0
Parent(s):
Duplicate from safetensors/convert
Browse filesCo-authored-by: Julien Chaumond <[email protected]>
- .gitattributes +33 -0
- .gitignore +1 -0
- .vscode/settings.json +4 -0
- README.md +17 -0
- app.py +94 -0
- convert.py +285 -0
- requirements.txt +4 -0
.gitattributes
ADDED
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
6 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
7 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
8 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
9 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
10 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
11 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
12 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
13 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
14 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
15 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
16 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
17 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
18 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
19 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
20 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
21 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
22 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
23 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
24 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
25 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
26 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
27 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
28 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
29 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
30 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
31 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
32 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
33 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
.env/
|
.vscode/settings.json
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"editor.formatOnSave": true,
|
3 |
+
"python.formatting.provider": "black"
|
4 |
+
}
|
README.md
ADDED
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
title: Convert to Safetensors
|
3 |
+
emoji: 🐶
|
4 |
+
colorFrom: yellow
|
5 |
+
colorTo: red
|
6 |
+
sdk: gradio
|
7 |
+
sdk_version: 3.8.1
|
8 |
+
app_file: app.py
|
9 |
+
pinned: true
|
10 |
+
license: apache-2.0
|
11 |
+
models: []
|
12 |
+
datasets:
|
13 |
+
- safetensors/conversions
|
14 |
+
duplicated_from: safetensors/convert
|
15 |
+
---
|
16 |
+
|
17 |
+
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
app.py
ADDED
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import csv
|
2 |
+
from datetime import datetime
|
3 |
+
import os
|
4 |
+
from typing import Optional
|
5 |
+
import gradio as gr
|
6 |
+
|
7 |
+
from convert import convert
|
8 |
+
from huggingface_hub import HfApi, Repository
|
9 |
+
|
10 |
+
|
11 |
+
DATASET_REPO_URL = "https://huggingface.co/datasets/safetensors/conversions"
|
12 |
+
DATA_FILENAME = "data.csv"
|
13 |
+
DATA_FILE = os.path.join("data", DATA_FILENAME)
|
14 |
+
|
15 |
+
HF_TOKEN = os.environ.get("HF_TOKEN")
|
16 |
+
|
17 |
+
repo: Optional[Repository] = None
|
18 |
+
if HF_TOKEN:
|
19 |
+
repo = Repository(local_dir="data", clone_from=DATASET_REPO_URL, token=HF_TOKEN)
|
20 |
+
|
21 |
+
|
22 |
+
def run(token: str, model_id: str) -> str:
|
23 |
+
if token == "" or model_id == "":
|
24 |
+
return """
|
25 |
+
### Invalid input 🐞
|
26 |
+
|
27 |
+
Please fill a token and model_id.
|
28 |
+
"""
|
29 |
+
try:
|
30 |
+
api = HfApi(token=token)
|
31 |
+
is_private = api.model_info(repo_id=model_id).private
|
32 |
+
print("is_private", is_private)
|
33 |
+
|
34 |
+
commit_info = convert(api=api, model_id=model_id)
|
35 |
+
print("[commit_info]", commit_info)
|
36 |
+
|
37 |
+
# save in a (public) dataset:
|
38 |
+
if repo is not None and not is_private:
|
39 |
+
repo.git_pull(rebase=True)
|
40 |
+
print("pulled")
|
41 |
+
with open(DATA_FILE, "a") as csvfile:
|
42 |
+
writer = csv.DictWriter(
|
43 |
+
csvfile, fieldnames=["model_id", "pr_url", "time"]
|
44 |
+
)
|
45 |
+
writer.writerow(
|
46 |
+
{
|
47 |
+
"model_id": model_id,
|
48 |
+
"pr_url": commit_info.pr_url,
|
49 |
+
"time": str(datetime.now()),
|
50 |
+
}
|
51 |
+
)
|
52 |
+
commit_url = repo.push_to_hub()
|
53 |
+
print("[dataset]", commit_url)
|
54 |
+
|
55 |
+
return f"""
|
56 |
+
### Success 🔥
|
57 |
+
|
58 |
+
Yay! This model was successfully converted and a PR was open using your token, here:
|
59 |
+
|
60 |
+
[{commit_info.pr_url}]({commit_info.pr_url})
|
61 |
+
"""
|
62 |
+
except Exception as e:
|
63 |
+
return f"""
|
64 |
+
### Error 😢😢😢
|
65 |
+
|
66 |
+
{e}
|
67 |
+
"""
|
68 |
+
|
69 |
+
|
70 |
+
DESCRIPTION = """
|
71 |
+
The steps are the following:
|
72 |
+
|
73 |
+
- Paste a read-access token from hf.co/settings/tokens. Read access is enough given that we will open a PR against the source repo.
|
74 |
+
- Input a model id from the Hub
|
75 |
+
- Click "Submit"
|
76 |
+
- That's it! You'll get feedback if it works or not, and if it worked, you'll get the URL of the opened PR 🔥
|
77 |
+
|
78 |
+
⚠️ For now only `pytorch_model.bin` files are supported but we'll extend in the future.
|
79 |
+
"""
|
80 |
+
|
81 |
+
demo = gr.Interface(
|
82 |
+
title="Convert any model to Safetensors and open a PR",
|
83 |
+
description=DESCRIPTION,
|
84 |
+
allow_flagging="never",
|
85 |
+
article="Check out the [Safetensors repo on GitHub](https://github.com/huggingface/safetensors)",
|
86 |
+
inputs=[
|
87 |
+
gr.Text(max_lines=1, label="your_hf_token"),
|
88 |
+
gr.Text(max_lines=1, label="model_id"),
|
89 |
+
],
|
90 |
+
outputs=[gr.Markdown(label="output")],
|
91 |
+
fn=run,
|
92 |
+
)
|
93 |
+
|
94 |
+
demo.launch()
|
convert.py
ADDED
@@ -0,0 +1,285 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import argparse
|
2 |
+
import json
|
3 |
+
import os
|
4 |
+
import shutil
|
5 |
+
from collections import defaultdict
|
6 |
+
from inspect import signature
|
7 |
+
from tempfile import TemporaryDirectory
|
8 |
+
from typing import Dict, List, Optional, Set
|
9 |
+
|
10 |
+
import torch
|
11 |
+
|
12 |
+
from huggingface_hub import CommitInfo, CommitOperationAdd, Discussion, HfApi, hf_hub_download
|
13 |
+
from huggingface_hub.file_download import repo_folder_name
|
14 |
+
from safetensors.torch import load_file, save_file
|
15 |
+
from transformers import AutoConfig
|
16 |
+
from transformers.pipelines.base import infer_framework_load_model
|
17 |
+
|
18 |
+
|
19 |
+
class AlreadyExists(Exception):
|
20 |
+
pass
|
21 |
+
|
22 |
+
|
23 |
+
def shared_pointers(tensors):
|
24 |
+
ptrs = defaultdict(list)
|
25 |
+
for k, v in tensors.items():
|
26 |
+
ptrs[v.data_ptr()].append(k)
|
27 |
+
failing = []
|
28 |
+
for ptr, names in ptrs.items():
|
29 |
+
if len(names) > 1:
|
30 |
+
failing.append(names)
|
31 |
+
return failing
|
32 |
+
|
33 |
+
|
34 |
+
def check_file_size(sf_filename: str, pt_filename: str):
|
35 |
+
sf_size = os.stat(sf_filename).st_size
|
36 |
+
pt_size = os.stat(pt_filename).st_size
|
37 |
+
|
38 |
+
if (sf_size - pt_size) / pt_size > 0.01:
|
39 |
+
raise RuntimeError(
|
40 |
+
f"""The file size different is more than 1%:
|
41 |
+
- {sf_filename}: {sf_size}
|
42 |
+
- {pt_filename}: {pt_size}
|
43 |
+
"""
|
44 |
+
)
|
45 |
+
|
46 |
+
|
47 |
+
def rename(pt_filename: str) -> str:
|
48 |
+
filename, ext = os.path.splitext(pt_filename)
|
49 |
+
local = f"{filename}.safetensors"
|
50 |
+
local = local.replace("pytorch_model", "model")
|
51 |
+
return local
|
52 |
+
|
53 |
+
|
54 |
+
def convert_multi(model_id: str, folder: str) -> List["CommitOperationAdd"]:
|
55 |
+
filename = hf_hub_download(repo_id=model_id, filename="pytorch_model.bin.index.json")
|
56 |
+
with open(filename, "r") as f:
|
57 |
+
data = json.load(f)
|
58 |
+
|
59 |
+
filenames = set(data["weight_map"].values())
|
60 |
+
local_filenames = []
|
61 |
+
for filename in filenames:
|
62 |
+
pt_filename = hf_hub_download(repo_id=model_id, filename=filename)
|
63 |
+
|
64 |
+
sf_filename = rename(pt_filename)
|
65 |
+
sf_filename = os.path.join(folder, sf_filename)
|
66 |
+
convert_file(pt_filename, sf_filename)
|
67 |
+
local_filenames.append(sf_filename)
|
68 |
+
|
69 |
+
index = os.path.join(folder, "model.safetensors.index.json")
|
70 |
+
with open(index, "w") as f:
|
71 |
+
newdata = {k: v for k, v in data.items()}
|
72 |
+
newmap = {k: rename(v) for k, v in data["weight_map"].items()}
|
73 |
+
newdata["weight_map"] = newmap
|
74 |
+
json.dump(newdata, f, indent=4)
|
75 |
+
local_filenames.append(index)
|
76 |
+
|
77 |
+
operations = [
|
78 |
+
CommitOperationAdd(path_in_repo=local.split("/")[-1], path_or_fileobj=local) for local in local_filenames
|
79 |
+
]
|
80 |
+
|
81 |
+
return operations
|
82 |
+
|
83 |
+
|
84 |
+
def convert_single(model_id: str, folder: str) -> List["CommitOperationAdd"]:
|
85 |
+
pt_filename = hf_hub_download(repo_id=model_id, filename="pytorch_model.bin")
|
86 |
+
|
87 |
+
sf_name = "model.safetensors"
|
88 |
+
sf_filename = os.path.join(folder, sf_name)
|
89 |
+
convert_file(pt_filename, sf_filename)
|
90 |
+
operations = [CommitOperationAdd(path_in_repo=sf_name, path_or_fileobj=sf_filename)]
|
91 |
+
return operations
|
92 |
+
|
93 |
+
|
94 |
+
def convert_file(
|
95 |
+
pt_filename: str,
|
96 |
+
sf_filename: str,
|
97 |
+
):
|
98 |
+
loaded = torch.load(pt_filename, map_location="cpu")
|
99 |
+
if "state_dict" in loaded:
|
100 |
+
loaded = loaded["state_dict"]
|
101 |
+
shared = shared_pointers(loaded)
|
102 |
+
for shared_weights in shared:
|
103 |
+
for name in shared_weights[1:]:
|
104 |
+
loaded.pop(name)
|
105 |
+
|
106 |
+
# For tensors to be contiguous
|
107 |
+
loaded = {k: v.contiguous() for k, v in loaded.items()}
|
108 |
+
|
109 |
+
dirname = os.path.dirname(sf_filename)
|
110 |
+
os.makedirs(dirname, exist_ok=True)
|
111 |
+
save_file(loaded, sf_filename, metadata={"format": "pt"})
|
112 |
+
check_file_size(sf_filename, pt_filename)
|
113 |
+
reloaded = load_file(sf_filename)
|
114 |
+
for k in loaded:
|
115 |
+
pt_tensor = loaded[k]
|
116 |
+
sf_tensor = reloaded[k]
|
117 |
+
if not torch.equal(pt_tensor, sf_tensor):
|
118 |
+
raise RuntimeError(f"The output tensors do not match for key {k}")
|
119 |
+
|
120 |
+
|
121 |
+
def create_diff(pt_infos: Dict[str, List[str]], sf_infos: Dict[str, List[str]]) -> str:
|
122 |
+
errors = []
|
123 |
+
for key in ["missing_keys", "mismatched_keys", "unexpected_keys"]:
|
124 |
+
pt_set = set(pt_infos[key])
|
125 |
+
sf_set = set(sf_infos[key])
|
126 |
+
|
127 |
+
pt_only = pt_set - sf_set
|
128 |
+
sf_only = sf_set - pt_set
|
129 |
+
|
130 |
+
if pt_only:
|
131 |
+
errors.append(f"{key} : PT warnings contain {pt_only} which are not present in SF warnings")
|
132 |
+
if sf_only:
|
133 |
+
errors.append(f"{key} : SF warnings contain {sf_only} which are not present in PT warnings")
|
134 |
+
return "\n".join(errors)
|
135 |
+
|
136 |
+
|
137 |
+
def check_final_model(model_id: str, folder: str):
|
138 |
+
config = hf_hub_download(repo_id=model_id, filename="config.json")
|
139 |
+
shutil.copy(config, os.path.join(folder, "config.json"))
|
140 |
+
config = AutoConfig.from_pretrained(folder)
|
141 |
+
|
142 |
+
_, (pt_model, pt_infos) = infer_framework_load_model(model_id, config, output_loading_info=True)
|
143 |
+
_, (sf_model, sf_infos) = infer_framework_load_model(folder, config, output_loading_info=True)
|
144 |
+
|
145 |
+
if pt_infos != sf_infos:
|
146 |
+
error_string = create_diff(pt_infos, sf_infos)
|
147 |
+
raise ValueError(f"Different infos when reloading the model: {error_string}")
|
148 |
+
|
149 |
+
pt_params = pt_model.state_dict()
|
150 |
+
sf_params = sf_model.state_dict()
|
151 |
+
|
152 |
+
pt_shared = shared_pointers(pt_params)
|
153 |
+
sf_shared = shared_pointers(sf_params)
|
154 |
+
if pt_shared != sf_shared:
|
155 |
+
raise RuntimeError("The reconstructed model is wrong, shared tensors are different {shared_pt} != {shared_tf}")
|
156 |
+
|
157 |
+
sig = signature(pt_model.forward)
|
158 |
+
input_ids = torch.arange(10).unsqueeze(0)
|
159 |
+
pixel_values = torch.randn(1, 3, 224, 224)
|
160 |
+
input_values = torch.arange(1000).float().unsqueeze(0)
|
161 |
+
kwargs = {}
|
162 |
+
if "input_ids" in sig.parameters:
|
163 |
+
kwargs["input_ids"] = input_ids
|
164 |
+
if "decoder_input_ids" in sig.parameters:
|
165 |
+
kwargs["decoder_input_ids"] = input_ids
|
166 |
+
if "pixel_values" in sig.parameters:
|
167 |
+
kwargs["pixel_values"] = pixel_values
|
168 |
+
if "input_values" in sig.parameters:
|
169 |
+
kwargs["input_values"] = input_values
|
170 |
+
if "bbox" in sig.parameters:
|
171 |
+
kwargs["bbox"] = torch.zeros((1, 10, 4)).long()
|
172 |
+
if "image" in sig.parameters:
|
173 |
+
kwargs["image"] = pixel_values
|
174 |
+
|
175 |
+
if torch.cuda.is_available():
|
176 |
+
pt_model = pt_model.cuda()
|
177 |
+
sf_model = sf_model.cuda()
|
178 |
+
kwargs = {k: v.cuda() for k, v in kwargs.items()}
|
179 |
+
|
180 |
+
pt_logits = pt_model(**kwargs)[0]
|
181 |
+
sf_logits = sf_model(**kwargs)[0]
|
182 |
+
|
183 |
+
torch.testing.assert_close(sf_logits, pt_logits)
|
184 |
+
print(f"Model {model_id} is ok !")
|
185 |
+
|
186 |
+
|
187 |
+
def previous_pr(api: "HfApi", model_id: str, pr_title: str) -> Optional["Discussion"]:
|
188 |
+
try:
|
189 |
+
discussions = api.get_repo_discussions(repo_id=model_id)
|
190 |
+
except Exception:
|
191 |
+
return None
|
192 |
+
for discussion in discussions:
|
193 |
+
if discussion.status == "open" and discussion.is_pull_request and discussion.title == pr_title:
|
194 |
+
return discussion
|
195 |
+
|
196 |
+
|
197 |
+
def convert_generic(model_id: str, folder: str, filenames: Set[str]) -> List["CommitOperationAdd"]:
|
198 |
+
operations = []
|
199 |
+
|
200 |
+
extensions = set([".bin", ".ckpt"])
|
201 |
+
for filename in filenames:
|
202 |
+
prefix, ext = os.path.splitext(filename)
|
203 |
+
if ext in extensions:
|
204 |
+
pt_filename = hf_hub_download(model_id, filename=filename)
|
205 |
+
_, raw_filename = os.path.split(filename)
|
206 |
+
if raw_filename == "pytorch_model.bin":
|
207 |
+
# XXX: This is a special case to handle `transformers` and the
|
208 |
+
# `transformers` part of the model which is actually loaded by `transformers`.
|
209 |
+
sf_in_repo = "model.safetensors"
|
210 |
+
else:
|
211 |
+
sf_in_repo = f"{prefix}.safetensors"
|
212 |
+
sf_filename = os.path.join(folder, sf_in_repo)
|
213 |
+
convert_file(pt_filename, sf_filename)
|
214 |
+
operations.append(CommitOperationAdd(path_in_repo=sf_in_repo, path_or_fileobj=sf_filename))
|
215 |
+
return operations
|
216 |
+
|
217 |
+
|
218 |
+
def convert(api: "HfApi", model_id: str, force: bool = False) -> Optional["CommitInfo"]:
|
219 |
+
pr_title = "Adding `safetensors` variant of this model"
|
220 |
+
info = api.model_info(model_id)
|
221 |
+
filenames = set(s.rfilename for s in info.siblings)
|
222 |
+
|
223 |
+
with TemporaryDirectory() as d:
|
224 |
+
folder = os.path.join(d, repo_folder_name(repo_id=model_id, repo_type="models"))
|
225 |
+
os.makedirs(folder)
|
226 |
+
new_pr = None
|
227 |
+
try:
|
228 |
+
operations = None
|
229 |
+
pr = previous_pr(api, model_id, pr_title)
|
230 |
+
|
231 |
+
library_name = getattr(info, "library_name", None)
|
232 |
+
if any(filename.endswith(".safetensors") for filename in filenames) and not force:
|
233 |
+
raise AlreadyExists(f"Model {model_id} is already converted, skipping..")
|
234 |
+
elif pr is not None and not force:
|
235 |
+
url = f"https://huggingface.co/{model_id}/discussions/{pr.num}"
|
236 |
+
new_pr = pr
|
237 |
+
raise AlreadyExists(f"Model {model_id} already has an open PR check out {url}")
|
238 |
+
elif library_name == "transformers":
|
239 |
+
if "pytorch_model.bin" in filenames:
|
240 |
+
operations = convert_single(model_id, folder)
|
241 |
+
elif "pytorch_model.bin.index.json" in filenames:
|
242 |
+
operations = convert_multi(model_id, folder)
|
243 |
+
else:
|
244 |
+
raise RuntimeError(f"Model {model_id} doesn't seem to be a valid pytorch model. Cannot convert")
|
245 |
+
check_final_model(model_id, folder)
|
246 |
+
else:
|
247 |
+
operations = convert_generic(model_id, folder, filenames)
|
248 |
+
|
249 |
+
if operations:
|
250 |
+
new_pr = api.create_commit(
|
251 |
+
repo_id=model_id,
|
252 |
+
operations=operations,
|
253 |
+
commit_message=pr_title,
|
254 |
+
create_pr=True,
|
255 |
+
)
|
256 |
+
print(f"Pr created at {new_pr.pr_url}")
|
257 |
+
else:
|
258 |
+
print("No files to convert")
|
259 |
+
finally:
|
260 |
+
shutil.rmtree(folder)
|
261 |
+
return new_pr
|
262 |
+
|
263 |
+
|
264 |
+
if __name__ == "__main__":
|
265 |
+
DESCRIPTION = """
|
266 |
+
Simple utility tool to convert automatically some weights on the hub to `safetensors` format.
|
267 |
+
It is PyTorch exclusive for now.
|
268 |
+
It works by downloading the weights (PT), converting them locally, and uploading them back
|
269 |
+
as a PR on the hub.
|
270 |
+
"""
|
271 |
+
parser = argparse.ArgumentParser(description=DESCRIPTION)
|
272 |
+
parser.add_argument(
|
273 |
+
"model_id",
|
274 |
+
type=str,
|
275 |
+
help="The name of the model on the hub to convert. E.g. `gpt2` or `facebook/wav2vec2-base-960h`",
|
276 |
+
)
|
277 |
+
parser.add_argument(
|
278 |
+
"--force",
|
279 |
+
action="store_true",
|
280 |
+
help="Create the PR even if it already exists of if the model was already converted.",
|
281 |
+
)
|
282 |
+
args = parser.parse_args()
|
283 |
+
model_id = args.model_id
|
284 |
+
api = HfApi()
|
285 |
+
convert(api, model_id, force=args.force)
|
requirements.txt
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
git+https://github.com/huggingface/huggingface_hub@main
|
2 |
+
safetensors
|
3 |
+
torch==1.12.0
|
4 |
+
transformers
|