|
import os |
|
|
|
from vbench2_beta_i2v.utils import init_submodules, save_json, load_json |
|
from vbench import VBench |
|
import importlib |
|
|
|
|
|
class VBenchI2V(VBench): |
|
def build_full_dimension_list(self, ): |
|
return ["subject_consistency", "background_consistency", "aesthetic_quality", "imaging_quality", "object_class", "multiple_objects", "color", "spatial_relationship", "scene", "temporal_style", 'overall_consistency', "human_action", "temporal_flickering", "motion_smoothness", "dynamic_degree", "appearance_style", "i2v_subject", "i2v_background", "camera_motion"] |
|
|
|
def evaluate(self, videos_path, name, dimension_list=None, local=False, read_frame=False, custom_prompt=False, resolution="1-1"): |
|
results_dict = {} |
|
if dimension_list is None: |
|
dimension_list = self.build_full_dimension_list() |
|
submodules_dict = init_submodules(dimension_list, local=local, read_frame=read_frame, resolution=resolution) |
|
|
|
cur_full_info_path = self.build_full_info_json(videos_path, name, dimension_list, custom_prompt=custom_prompt) |
|
|
|
for dimension in dimension_list: |
|
try: |
|
dimension_module = importlib.import_module(f'vbench2_beta_i2v.{dimension}') |
|
evaluate_func = getattr(dimension_module, f'compute_{dimension}') |
|
except Exception as e: |
|
raise NotImplementedError(f'UnImplemented dimension {dimension}!, {e}') |
|
submodules_list = submodules_dict[dimension] |
|
print(f'cur_full_info_path: {cur_full_info_path}') |
|
results = evaluate_func(cur_full_info_path, self.device, submodules_list) |
|
results_dict[dimension] = results |
|
output_name = os.path.join(self.output_path, name+'_eval_results.json') |
|
save_json(results_dict, output_name) |
|
print(f'Evaluation results saved to {output_name}') |
|
|
