# Copyright 2023 The HuggingFace Evaluate Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Jaccard similarity metric.""" import evaluate import datasets from sklearn.metrics import jaccard_score import numpy as np _CITATION = """\ @article{jaccard1912distribution, title={The distribution of the flora in the alpine zone}, author={Jaccard, Paul}, journal={New phytologist}, volume={11}, number={2}, pages={37--50}, year={1912}, publisher={Wiley Online Library} } """ _DESCRIPTION = """\ Jaccard similarity is a statistic used for gauging the similarity and diversity of sample sets. The Jaccard coefficient measures similarity between finite sample sets, and is defined as the size of the intersection divided by the size of the union of the sample sets. This implementation uses scikit-learn's jaccard_score function. """ _KWARGS_DESCRIPTION = """ Calculates the Jaccard similarity between predictions and references using scikit-learn. Args: predictions: 1d array-like, or label indicator array / sparse matrix Predicted labels, as returned by a classifier. references: 1d array-like, or label indicator array / sparse matrix Ground truth (correct) labels. labels: array-like of shape (n_classes,), default=None The set of labels to include when average != 'binary', and their order if average is None. pos_label: int, float, bool or str, default=1 The class to report if average='binary' and the data is binary. average: {'micro', 'macro', 'samples', 'weighted', 'binary'} or None, default='binary' This parameter is required for multiclass/multilabel targets. sample_weight: array-like of shape (n_samples,), default=None Sample weights. zero_division: "warn", {0.0, 1.0}, default="warn" Sets the value to return when there is a zero division. Returns: jaccard_similarity: float or ndarray of shape (n_unique_labels,) Jaccard similarity score. Examples: >>> jaccard_metric = evaluate.load("jaccard_similarity") >>> predictions = [0, 2, 1, 3] >>> references = [0, 1, 2, 3] >>> results = jaccard_metric.compute(predictions=predictions, references=references, average='macro') >>> print(results) {'jaccard_similarity': 0.5} """ @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) class JaccardSimilarity(evaluate.Metric): def _info(self): return evaluate.MetricInfo( module_type="metric", description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "predictions": datasets.Sequence(datasets.Value("int32")), "references": datasets.Sequence(datasets.Value("int32")), }), reference_urls=[ "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.jaccard_score.html", "https://en.wikipedia.org/wiki/Jaccard_index" ], ) def _compute(self, predictions, references, labels=None, pos_label=1, average='binary', sample_weight=None, zero_division='warn'): predictions = np.array(predictions) references = np.array(references) # Handle different input shapes if predictions.ndim == 1 and references.ndim == 1: # Binary or multiclass case pass elif predictions.ndim == 2 and references.ndim == 2: # Multilabel case if average == 'binary': average = 'micro' # 'binary' doesn't make sense for multilabel else: raise ValueError("Predictions and references should have the same shape") return { "jaccard_similarity": jaccard_score( references, predictions, labels=labels, pos_label=pos_label, average=average, sample_weight=sample_weight, zero_division=zero_division ) }