Update README.md
Browse files
README.md
CHANGED
@@ -7,7 +7,7 @@ base_model:
|
|
7 |
---
|
8 |
|
9 |
#Usage
|
10 |
-
|
11 |
!pip install -U bitsandbytes
|
12 |
!pip install -U transformers
|
13 |
!pip install -U accelerate
|
@@ -87,4 +87,13 @@ for data in tqdm(datasets):
|
|
87 |
)[0]
|
88 |
output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
|
89 |
|
90 |
-
results.append({"task_id": data["task_id"], "input": input, "output": output})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
7 |
---
|
8 |
|
9 |
#Usage
|
10 |
+
```python
|
11 |
!pip install -U bitsandbytes
|
12 |
!pip install -U transformers
|
13 |
!pip install -U accelerate
|
|
|
87 |
)[0]
|
88 |
output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
|
89 |
|
90 |
+
results.append({"task_id": data["task_id"], "input": input, "output": output})
|
91 |
+
|
92 |
+
import re
|
93 |
+
jsonl_id = re.sub(".*/", "", adapter_id)
|
94 |
+
with open(f"./{jsonl_id}-outputs.jsonl", 'w', encoding='utf-8') as f:
|
95 |
+
for result in results:
|
96 |
+
json.dump(result, f, ensure_ascii=False) # ensure_ascii=False for handling non-ASCII characters
|
97 |
+
f.write('\n')
|
98 |
+
```python
|
99 |
+
|