Upload folder using huggingface_hub

metrics.py

@@ -1166,7 +1166,9 @@ class HuggingfaceBulkMetric(BulkInstanceMetric):
 
     def prepare(self):
         super().prepare()
-        self.metric = evaluate.load(self.hf_metric_name)
+        self.metric = evaluate.load(
+            self.hf_metric_name, experiment_id=str(uuid.uuid4())
+        )
 
     def compute(
         self,
@@ -1213,7 +1215,7 @@ class F1(GlobalMetric):
 
     def prepare(self):
         super().prepare()
-        self._metric = evaluate.load(self.metric)
+        self._metric = evaluate.load(self.metric, experiment_id=str(uuid.uuid4()))
 
     def get_str_id(self, str):
         if str not in self.str_to_id:
@@ -1337,7 +1339,9 @@ class F1MultiLabel(GlobalMetric):
 
     def prepare(self):
         super().prepare()
-        self._metric = evaluate.load(self.metric, "multilabel")
+        self._metric = evaluate.load(
+            self.metric, "multilabel", experiment_id=str(uuid.uuid4())
+        )
 
     def add_str_to_id(self, str):
         if str not in self.str_to_id:

task.py

@@ -1,3 +1,4 @@
+from functools import lru_cache
 from typing import Any, Dict, List, Optional, Union
 
 from .artifact import fetch_artifact
@@ -75,11 +76,16 @@ class FormTask(Tasker, StreamInstanceOperator):
                 augmentable_input in self.inputs
             ), f"augmentable_input {augmentable_input} is not part of {self.inputs}"
 
+    @staticmethod
+    @lru_cache(maxsize=None)
+    def get_metric_prediction_type(metric_id: str):
+        metric = fetch_artifact(metric_id)[0]
+        return metric.get_prediction_type()
+
     def check_metrics_type(self) -> None:
         prediction_type = parse_type_string(self.prediction_type)
-        for metric_name in self.metrics:
-            metric = fetch_artifact(metric_name)[0]
-            metric_prediction_type = metric.get_prediction_type()
+        for metric_id in self.metrics:
+            metric_prediction_type = FormTask.get_metric_prediction_type(metric_id)
 
             if (
                 prediction_type == metric_prediction_type
@@ -93,7 +99,7 @@ class FormTask(Tasker, StreamInstanceOperator):
                 continue
 
             raise ValueError(
-                f"The task's prediction type ({prediction_type}) and '{metric_name}' "
+                f"The task's prediction type ({prediction_type}) and '{metric_id}' "
                 f"metric's prediction type ({metric_prediction_type}) are different."
             )
 

version.py

@@ -1 +1 @@
-version = "1.8.0"
+version = "1.8.1"