Upload schema.py with huggingface_hub
Browse files
schema.py
CHANGED
@@ -34,9 +34,14 @@ class ToUnitxtGroup(StreamInstanceOperatorValidator):
|
|
34 |
|
35 |
def process(self, instance: Dict[str, Any], stream_name: str = None) -> Dict[str, Any]:
|
36 |
if self.remove_unnecessary_fields:
|
|
|
|
|
37 |
for key in instance.keys():
|
38 |
if key not in UNITXT_DATASET_SCHEMA:
|
39 |
-
|
|
|
|
|
|
|
40 |
|
41 |
instance["group"] = self.group
|
42 |
if self.metrics is not None:
|
|
|
34 |
|
35 |
def process(self, instance: Dict[str, Any], stream_name: str = None) -> Dict[str, Any]:
|
36 |
if self.remove_unnecessary_fields:
|
37 |
+
keys_to_delete = []
|
38 |
+
|
39 |
for key in instance.keys():
|
40 |
if key not in UNITXT_DATASET_SCHEMA:
|
41 |
+
keys_to_delete.append(key)
|
42 |
+
|
43 |
+
for key in keys_to_delete:
|
44 |
+
del instance[key]
|
45 |
|
46 |
instance["group"] = self.group
|
47 |
if self.metrics is not None:
|