|
from typing import Dict |
|
|
|
from .llm_as_judge_constants import ( |
|
EVALUATORS_METADATA, |
|
MODEL_RENAMINGS, |
|
EvaluatorMetadata, |
|
EvaluatorNameEnum, |
|
ModelProviderEnum, |
|
) |
|
|
|
|
|
def get_parsed_context(context: Dict[str, str]): |
|
return ( |
|
"\n".join([f"{key}: {value}" for key, value in context.items()]) |
|
if len(context) > 1 |
|
or not (len(context) == 1 and next(iter(context.keys())).lower() == "context") |
|
else context[next(iter(context.keys()))] |
|
) |
|
|
|
|
|
def get_evaluator_metadata( |
|
name: EvaluatorNameEnum |
|
) -> EvaluatorMetadata: |
|
evaluator_search = [ |
|
e for e in EVALUATORS_METADATA if e.name == name |
|
] |
|
if len(evaluator_search) == 0: |
|
|
|
raise ValueError(f"An evaluator with id {name} does not exist.") |
|
if len(evaluator_search) > 1: |
|
|
|
raise ValueError(f"An evaluator with id {name} matched several models.") |
|
return evaluator_search[0] |
|
|
|
|
|
def rename_model_if_required(model_name: str, provider: ModelProviderEnum) -> str: |
|
if provider in MODEL_RENAMINGS and model_name in MODEL_RENAMINGS[provider]: |
|
return MODEL_RENAMINGS[provider][model_name] |
|
return model_name |
|
|
|
|
|
def rank_indexes(numbers): |
|
|
|
indices = list(range(len(numbers))) |
|
|
|
|
|
sorted_indices = sorted(indices, key=lambda x: -numbers[x]) |
|
|
|
|
|
rankings = [0] * len(numbers) |
|
|
|
|
|
current_rank = 0 |
|
for i in range(len(sorted_indices)): |
|
if i > 0 and numbers[sorted_indices[i]] != numbers[sorted_indices[i - 1]]: |
|
current_rank = i |
|
rankings[sorted_indices[i]] = current_rank |
|
|
|
return rankings |
|
|