add mmcm

MuseV/MMCM/.gitignore

@@ -0,0 +1,139 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don’t work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+*.swp
+.*.swp
+dataset/files
+experiments
+log
+csvs
+
+.idea
+.vscode
+__pycache__/
+*.code-workspace
+.DS_Store
+third_party/
+.polaris_cache/
+*.lock

MuseV/MMCM/Dockerfile

@@ -0,0 +1,83 @@
+# FROM mirrors.tencent.com/todacc/venus-std-base-cuda11.8:0.1.0
+FROM mirrors.tencent.com/todacc/venus-std-ext-cuda11.8-pytorch2.0-tf2.12-py3.10:0.7.0
+
+#MAINTAINER 维护者信息
+LABEL MAINTAINER="anchorxia"
+LABEL Email="[email protected]"
+LABEL Description="gpu development image, from mirrors.tencent.com/todacc/venus-std-ext-cuda11.8-pytorch2.0-tf2.12-py3.10:0.7.0"
+
+USER root
+# 安装必须软件
+# RUN GENERIC_REPO_URL="http://mirrors.tencent.com/repository/generic/venus_repo/image_res" \
+# && cd /data/ \
+# && wget -q $GENERIC_REPO_URL/gcc/gcc-11.2.0.zip \
+# && unzip -q gcc-11.2.0.zip \
+# && cd gcc-releases-gcc-11.2.0 \
+# && ./contrib/download_prerequisites \
+# && ./configure --enable-bootstrap --enable-languages=c,c++ --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib \
+# && make --silent -j10 \
+# && make --silent install \
+# && gcc -v \
+# && rm -rf /data/gcc-releases-gcc-11.2.0 /data/gcc-11.2.0.zip 
+
+# RUN yum update -y \
+# && yum install -y epel-release \
+# && yum install -y ffmpeg \
+# && yum install -y Xvfb \
+# && yum install -y centos-release-scl devtoolset-11
+RUN yum install -y wget zsh git curl tmux cmake htop iotop git-lfs zip \
+ && yum install -y autojump autojump-zsh portaudio portaudio-devel \
+ && yum clean all
+
+USER mqq
+RUN source ~/.bashrc \
+ && GENERIC_REPO_URL="http://mirrors.tencent.com/repository/generic/venus_repo/image_res" \
+ && conda deactivate \
+ # && conda remove -y -n env-2.7.18 --all \
+ # && conda remove -y -n env-3.6.8 --all \
+ # && conda remove -y -n env-3.7.7 --all \
+ # && conda remove -y -n env-3.8.8 --all \
+ # && conda remove -y -n env-3.9.2 --all \
+ # && conda remove -y -n env-novelai --all \
+ && conda create -n projectv python=3.10.6 -y \
+ && conda activate projectv \
+ && pip install venus-sdk -q -i https://mirrors.tencent.com/repository/pypi/tencent_pypi/simple \
+ --extra-index-url https://mirrors.tencent.com/pypi/simple/ \
+ && pip install tensorflow==2.12.0 tensorboard==2.12.0 \
+ && pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 -f https://mirror.sjtu.edu.cn/pytorch-wheels/torch_stable.html -i https://mirrors.bfsu.edu.cn/pypi/web/simple -U \
+ # 安装xformers，支持不同型号gpu
+ && pip install ninja==1.11.1 \
+ # && git clone https://github.com/facebookresearch/xformers.git \
+ # && cd xformers \
+ # && git checkout v0.0.17rc482 \
+ # && git submodule update --init --recursive \
+ # && pip install numpy==1.23.4 pyre-extensions==0.0.23 \
+ # && FORCE_CUDA="1" MAX_JOBS=1 TORCH_CUDA_ARCH_LIST="6.1;7.0;7.5;8.0;8.6" pip install -e . \
+ # && cd .. \
+ # 安装一堆包
+ && pip install --no-cache-dir transformers bitsandbytes decord accelerate xformers omegaconf einops imageio==2.31.1 \
+ && pip install --no-cache-dir pandas h5py matplotlib modelcards pynvml black pytest moviepy torch-tb-profiler scikit-learn librosa ffmpeg easydict webp controlnet_aux mediapipe \
+ && pip install --no-cache-dir Cython easydict gdown infomap insightface ipython librosa onnx onnxruntime onnxsim opencv_python Pillow protobuf pytube PyYAML \
+ && pip install --no-cache-dir requests scipy six tqdm gradio albumentations opencv-contrib-python imageio-ffmpeg pytorch-lightning test-tube \
+ && pip install --no-cache-dir timm addict yapf prettytable safetensors basicsr fvcore pycocotools wandb gunicorn \
+ && pip install --no-cache-dir streamlit webdataset kornia open_clip_torch streamlit-drawable-canvas torchmetrics \
+ # 安装暗水印
+ && pip install --no-cache-dir invisible-watermark==0.1.5 gdown==4.5.3 ftfy==6.1.1 modelcards==0.1.6 \
+ # 安装openmm相关包
+ && pip install--no-cache-dir -U openmim \
+ && mim install mmengine \
+ && mim install "mmcv>=2.0.1" \
+ && mim install "mmdet>=3.1.0" \
+ && mim install "mmpose>=1.1.0" \
+ # jupyters
+ && pip install ipywidgets==8.0.3 \
+ && python -m ipykernel install --user --name projectv --display-name "python(projectv)" \
+ && pip install --no-cache-dir matplotlib==3.6.2 redis==4.5.1 pydantic[dotenv]==1.10.2 loguru==0.6.0 IProgress==0.4 \
+ && pip install --no-cache-dir cos-python-sdk-v5==1.9.22 coscmd==1.8.6.30 \
+ # 必须放在最后pip，避免和jupyter的不兼容
+ && pip install --no-cache-dir markupsafe==2.0.1 \
+ && wget -P /tmp $GENERIC_REPO_URL/cpu/clean-layer.sh \
+ && sh /tmp/clean-layer.sh
+
+ENV LD_LIBRARY_PATH=/usr/local/lib64:$LD_LIBRARY_PATH
+USER root

MuseV/MMCM/README.md

@@ -0,0 +1,2 @@
+# MMCM
+Process package for multi media, cross multi modal.

MuseV/MMCM/mmcm/__init__.py

@@ -0,0 +1,6 @@
+from .audio import *
+from .data import *
+from .music import *
+from .text import *
+from .vision import *
+from .t2p import *

MuseV/MMCM/mmcm/data/__init__.py

@@ -0,0 +1,9 @@
+from .general.items import Items, Item
+
+from .emb.emb import MediaMapEmb
+from .emb.h5py_emb import H5pyMediaMapEmb, H5pyMediaMapEmbProxy
+
+from .media_map.media_map import MediaMap, MetaInfo, MetaInfoList, MediaMapSeq
+from .media_map.media_map_process import get_sub_mediamap_by_clip_idx, get_sub_mediamap_by_stage, get_subseq_by_time
+from .clip.clip import Clip, ClipSeq
+from .clip.clipid import ClipIds, ClipIdsSeq, MatchedClipIds, MatchedClipIdsSeq

MuseV/MMCM/mmcm/data/clip.py

@@ -0,0 +1,324 @@
+from copy import deepcopy
+from typing import Iterable
+import logging
+
+import numpy as np
+
+from ..utils.util import convert_class_attr_to_dict
+
+logger = logging.getLogger(__name__) # pylint: disable=invalid-name
+
+
+class Clip(object, Item):
+ """媒体片段, 指转场点与转场点之间的部分"""
+
+ def __init__(
+ self,
+ time_start,
+ duration,
+ clipid=None,
+ media_type=None,
+ mediaid=None,
+ timepoint_type=None,
+ text=None,
+ stage=None,
+ path=None,
+ duration_num=None,
+ group_time_start=0,
+ group_clipid=None,
+ original_clipid=None,
+ emb=None,
+ multi_factor=None,
+ similar_clipseq=None,
+ rythm: float = None,
+ **kwargs
+ ):
+ """
+ Args:
+ time_start (float): 开始时间,秒为单位,对应该媒体文件的, 和media_map.json上的序号一一对应
+ duration (_type_): 片段持续时间
+ clipid (int, or [int]): 由media_map提供的片段序号, 和media_map.json上的序号一一对应
+ media_type (str, optional): music, video,text, Defaults to None.
+ mediaid (int): 多媒体id, 当clipid是列表时,表示该片段是个融合片段
+ timepoint_type(int, ): 开始点的转场类型. Defaults to None.
+ text(str, optional): 该片段的文本描述,音乐可以是歌词,视频可以是台词,甚至可以是弹幕. Defaults to None.
+ stage(str, optional): 该片段在整个媒体文件中的结构位置,如音乐的intro、chrous、vesa,视频的片头、片尾、开始、高潮、转场等. Defaults to None.
+ path (_type_, optional): 该媒体文件的路径,用于后续媒体读取、处理. Defaults to None.
+ duration_num (_type_, optional): 片段持续帧数, Defaults to None.
+ group_time_start (int, optional): 当多歌曲、多视频剪辑时,group_time_start 表示该片段所对应的子媒体前所有子媒体的片段时长总和。
+ 默认0, 表示只有1个媒体文件. Defaults to 0.
+ group_clipid (int, optional): # MediaInfo.sub_meta_info 中的实际序号.
+ original_clipid (None or [int], optional): 有些片段由其他片段合并,该字段用于片段来源,id是 media_map.json 中的实际序号. Defaults to None.
+ emb (np.array, optional): 片段 综合emb,. Defaults to None.
+ multi_factor (MultiFactorFeature), optional): 多维度特征. Defaults to None.
+ similar_clipseq ([Clip]], optional): 与该片段相似的片段，具体结构待定义. Defaults to None.
+ """
+ self.media_type = media_type
+ self.mediaid = mediaid
+ self.time_start = time_start
+ self.duration = duration
+ self.clipid = clipid
+ self.path = path
+ self.timepoint_type = timepoint_type
+ self.text = text
+ self.stage = stage
+ self.group_time_start = group_time_start
+ self.group_clipid = group_clipid
+ self.duration_num = duration_num
+ self.original_clipid = original_clipid if original_clipid is not None else []
+ self.emb = emb
+ self.multi_factor = multi_factor
+ self.similar_clipseq = similar_clipseq
+ self.rythm = rythm
+ # TODO: 目前谱面中会有一些不必要的中间结果，比较占内存，现在代码里删掉，待后续数据协议确定
+ kwargs = {k: v for k, v in kwargs.items()}
+ self.__dict__.update(kwargs)
+ self.preprocess()
+
+ def preprocess(self):
+ pass
+
+ def spread_parameters(self):
+ pass
+
+ @property
+ def time_end(
+ self,
+ ):
+ return self.time_start + self.duration
+
+ @property
+ def mvp_clip(self):
+ """读取实际的片段数据为moviepy格式
+
+ Raises:
+ NotImplementedError: _description_
+ """
+ raise NotImplementedError
+
+
+class ClipSeq(object):
+ """媒体片段序列"""
+
+ ClipClass = Clip
+
+ def __init__(self, clips) -> None:
+ """_summary_
+
+ Args:
+ clips ([Clip]]): 媒体片段序列
+ """
+ if not isinstance(clips, list):
+ clips = [clips]
+ if len(clips) == 0:
+ self.clips = []
+ elif isinstance(clips[0], dict):
+ self.clips = [self.ClipClass(**d) for d in clips]
+ else:
+ self.clips = clips
+
+ def set_clip_value(self, k, v):
+ """给序列中的每一个clip 赋值"""
+ for i in range(len(self.clips)):
+ self.clips[i].__setattr__(k, v)
+
+ def __len__(
+ self,
+ ):
+ return len(self.clips)
+
+ def merge(self, other, group_time_start_delta=None, groupid_delta=None):
+ """融合其他ClipSeq。media_info 融合时需要记录 clip 所在的 groupid 和 group_time_start，delta用于表示变化
+
+ Args:
+ other (ClipSeq): 待融合的ClipSeq
+ group_time_start_delta (float, optional): . Defaults to None.
+ groupid_delta (int, optional): _description_. Defaults to None.
+ """
+ if group_time_start_delta is not None or groupid_delta is not None:
+ for i, clip in enumerate(other):
+ if group_time_start_delta is not None:
+ clip.group_time_start += group_time_start_delta
+ if groupid_delta is not None:
+ clip.groupid += groupid_delta
+ self.clips.extend(other.clips)
+ for i in range(len(self.clips)):
+ self.clips[i].group_clipid = i
+
+ @property
+ def duration(
+ self,
+ ):
+ """Clip.duration的和
+
+ Returns:
+ float: 序列总时长
+ """
+ if len(self.clips) == 0:
+ return 0
+ else:
+ return sum([c.duration for c in self.clips])
+
+ def __getitem__(self, i) -> Clip:
+ """支持索引和切片操作，如果输入是整数则返回Clip，如果是切片，则返回ClipSeq
+
+ Args:
+ i (int or slice): 索引
+
+ Raises:
+ ValueError: 需要按照给的输入类型索引
+
+ Returns:
+ Clip or ClipSeq:
+ """
+ if "int" in str(type(i)):
+ i = int(i)
+ if isinstance(i, int):
+ clip = self.clips[i]
+ return clip
+ elif isinstance(i, Iterable):
+ clips = [self.__getitem__(x) for x in i]
+ clipseq = ClipSeq(clips)
+ return clipseq
+ elif isinstance(i, slice):
+ if i.step is None:
+ step = 1
+ else:
+ step = i.step
+ clips = [self.__getitem__(x) for x in range(i.start, i.stop, step)]
+ clipseq = ClipSeq(clips)
+ return clipseq
+ else:
+ raise ValueError(
+ "unsupported input, should be int or slice, but given {}, type={}".format(
+ i, type(i)
+ )
+ )
+
+ def insert(self, idx, obj):
+ self.clips.insert(idx, obj)
+
+ def append(self, obj):
+ self.clips.append(obj)
+
+ def extend(self, objs):
+ self.clips.extend(objs)
+
+ @property
+ def duration_seq_emb(
+ self,
+ ):
+ emb = np.array([c.duration for c in self.clips])
+ return emb
+
+ @property
+ def timestamp_seq_emb(self):
+ emb = np.array([c.time_start for c in self.clips])
+ return emb
+
+ @property
+ def rela_timestamp_seq_emb(self):
+ emb = self.timestamp_seq_emb / self.duration
+ return emb
+
+ def get_factor_seq_emb(self, factor, dim):
+ emb = []
+ for c in self.clips:
+ if factor not in c.multi_factor or c.multi_factor[factor] is None:
+ v = np.full(dim, np.inf)
+ else:
+ v = c.multi_factor[factor]
+ emb.append(v)
+ emb = np.stack(emb, axis=0)
+ return emb
+
+ def semantic_seq_emb(self, dim):
+ return self.get_factor_seq_emb(factor="semantics", dim=dim)
+
+ def emotion_seq_emb(self, dim):
+ return self.get_factor_seq_emb(factor="emotion", dim=dim)
+
+ def theme_seq_emb(self, dim):
+ return self.get_factor_seq_emb(factor="theme", dim=dim)
+
+ def to_dct(
+ self,
+ target_keys=None,
+ ignored_keys=None,
+ ):
+ if ignored_keys is None:
+ ignored_keys = ["kwargs", "audio_path", "lyric_path", "start", "end"]
+ clips = [
+ clip.to_dct(target_keys=target_keys, ignored_keys=ignored_keys)
+ for clip in self.clips
+ ]
+ return clips
+
+ @property
+ def mvp_clip(self):
+ """读取实际的片段数据为moviepy格式
+
+ Raises:
+ NotImplementedError: _description_
+ """
+ raise NotImplementedError
+
+
+class ClipIds(object):
+ def __init__(
+ self,
+ clipids: list or int,
+ ) -> None:
+ """ClipSeq 中的 Clip序号，主要用于多个 Clip 融合后的 Clip, 使用场景如
+ 1. 一个 MusicClip 可以匹配到多个 VideoClip，VideoClip 的索引便可以使用 ClipIds 定义。
+
+ Args:
+ clipids (list or int): ClipSeq 中的序号
+ """
+ self.clipids = clipids if isinstance(clipids, list) else [clipids]
+
+
+class ClipIdsSeq(object):
+ def __init__(self, clipids_seq: list) -> None:
+ """多个 ClipIds，使用场景可以是
+ 1. 将MediaClipSeq 进行重组，拆分重组成更粗粒度的ClipSeq；
+
+ Args:
+ clipids_seq (list): 组合后的 ClipIds 列表
+ """
+ self.clipids_seq = (
+ clipids_seq if isinstance(clipids_seq, ClipIds) else [clipids_seq]
+ )
+
+
+# TODO: metric后续可能是字典
+class MatchedClipIds(object):
+ def __init__(
+ self, id1: ClipIds, id2: ClipIds, metric: float = None, **kwargs
+ ) -> None:
+ """两种模态数据的片段匹配对，���用场景 可以是
+ 1. 音乐片段和视频片段 之间的匹配关系，
+
+ Args:
+ id1 (ClipIds): 第一种模态的片段
+ id2 (ClipIds): 第二种模态的片段
+ metric (float): 匹配度量距离
+ """
+ self.id1 = id1 if isinstance(id1, ClipIds) else ClipIds(id1)
+ self.id2 = id2 if isinstance(id2, ClipIds) else ClipIds(id2)
+ self.metric = metric
+ self.__dict__.update(**kwargs)
+
+
+class MatchedClipIdsSeq(object):
+ def __init__(self, seq: list, metric: float = None, **kwargs) -> None:
+ """两种模态数据的序列匹配对，使用场景可以是
+ 1. 音乐片段序列和视频片段序列 之间的匹配，每一个元素都是MatchedClipIds:
+
+ Args:
+ seq (list): 两种模态数据的序列匹配对列表
+ metric (float): 匹配度量距离
+ """
+ self.seq = seq
+ self.metric = metric
+ self.__dict__.update(**kwargs)

MuseV/MMCM/mmcm/data/clip/__init__.py

@@ -0,0 +1,5 @@
+from .clip import Clip, ClipSeq
+from .clipid import ClipIds, MatchedClipIds, ClipIdsSeq, MatchedClipIdsSeq
+from .clip_process import find_idx_by_time, find_idx_by_clip, get_subseq_by_time, get_subseq_by_idx, clip_is_top, clip_is_middle, clip_is_end, abadon_old_return_new, reset_clipseq_id, insert_endclip, insert_startclip, drop_start_end_by_time, complete_clipseq, complete_gap
+from .clip_stat import stat_clipseq_duration
+from .clip_filter import ClipFilter, ClipSeqFilter

MuseV/MMCM/mmcm/data/clip/clip.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+from copy import deepcopy
+
+from typing import Iterable, List, Tuple, Dict, Hashable, Any, Union
+
+import numpy as np
+
+from ...utils.util import convert_class_attr_to_dict
+
+
+from ..general.items import Items, Item
+from .clipid import MatchedClipIds
+
+
+import logging
+
+logger = logging.getLogger(__name__) # pylint: disable=invalid-name
+
+
+__all__ = ["Clip", "ClipSeq"]
+
+
+class Clip(Item):
+ """媒体片段, 指转场点与转场点之间的部分"""
+
+ def __init__(
+ self,
+ time_start: float,
+ duration: float,
+ clipid: int = None,
+ media_type: str = None,
+ mediaid: str = None,
+ timepoint_type: str = None,
+ text: str = None,
+ stage: str = None,
+ path: str = None,
+ duration_num: int = None,
+ similar_clipseq: MatchedClipIds = None,
+ dynamic: float = None,
+ **kwargs,
+ ):
+ """
+ Args:
+ time_start (float): 开始时间,秒为单位,对应该媒体文件的, 和media_map.json上的序号一一对应
+ duration (_type_): 片段持续时间
+ clipid (int, or [int]): 由media_map提供的片段序号, 和media_map.json上的序号一一对应
+ media_type (str, optional): music, video,text, Defaults to None.
+ mediaid (int): 多媒体id, 当clipid是列表时,表示该片段是个融合片段
+ timepoint_type(int, ): 开始点的转场类型. Defaults to None.
+ text(str, optional): 该片段的文本描述,音乐可以是歌词,视频可以是台词,甚至可以是弹幕. Defaults to None.
+ stage(str, optional): 该片段在整个媒体文件中的结构位置,如音乐的intro、chrous、vesa,视频的片头、片尾、开始、高潮、转场等. Defaults to None.
+ path (str, optional): 该媒体文件的路径,用于后续媒体读取、处理. Defaults to None.
+ duration_num (_type_, optional): 片段持续帧数, Defaults to None.
+ similar_clipseq ([Clip]], optional): 与该片段相似的片段，具体结构待定义. Defaults to None.
+ """
+ self.media_type = media_type
+ self.mediaid = mediaid
+ self.time_start = time_start
+ self.duration = duration
+ self.clipid = clipid
+ self.path = path
+ self.timepoint_type = timepoint_type
+ self.text = text
+ self.stage = stage
+ self.duration_num = duration_num
+ self.similar_clipseq = similar_clipseq
+ self.dynamic = dynamic
+ self.__dict__.update(**kwargs)
+
+ def preprocess(self):
+ pass
+
+ def spread_parameters(self):
+ pass
+
+ @property
+ def time_end(
+ self,
+ ) -> float:
+ return self.time_start + self.duration
+
+ def get_emb(self, key: str, idx: int) -> np.float:
+ return self.emb.get_value(key, idx)
+
+
+class ClipSeq(Items):
+ """媒体片段序列"""
+
+ def __init__(self, items: List[Clip] = None):
+ super().__init__(items)
+ self.clipseq = self.data
+
+ def preprocess(self):
+ pass
+
+ def set_clip_value(self, k: Hashable, v: Any) -> None:
+ """给序列中的每一个clip 赋值"""
+ for i in range(len(self.clipseq)):
+ self.clipseq[i].__setattr__(k, v)
+
+ def __len__(
+ self,
+ ) -> int:
+ return len(self.clipseq)
+
+ @property
+ def duration(
+ self,
+ ) -> float:
+ """Clip.duration的和
+
+ Returns:
+ float: 序列总时长
+ """
+ if len(self.clipseq) == 0:
+ return 0
+ else:
+ return sum([c.duration for c in self.clipseq])
+
+ def __getitem__(self, i: Union[int, Iterable]) -> Union[Clip, ClipSeq]:
+ """支持索引和切片操作，如果输入是整数则返回Clip，如果是切片，则返回ClipSeq
+
+ Args:
+ i (int or slice): 索引
+
+ Raises:
+ ValueError: 需要按照给的输入类型索引
+
+ Returns:
+ Clip or ClipSeq:
+ """
+ if "int" in str(type(i)):
+ i = int(i)
+ if isinstance(i, int):
+ clip = self.clipseq[i]
+ return clip
+ elif isinstance(i, Iterable):
+ clipseq = [self.__getitem__(x) for x in i]
+ clipseq = ClipSeq(clipseq)
+ return clipseq
+ elif isinstance(i, slice):
+ if i.step is None:
+ step = 1
+ else:
+ step = i.step
+ clipseq = [self.__getitem__(x) for x in range(i.start, i.stop, step)]
+ clipseq = ClipSeq(clipseq)
+ return clipseq
+ else:
+ raise ValueError(
+ "unsupported input, should be int or slice, but given {}, type={}".format(
+ i, type(i)
+ )
+ )
+
+ @property
+ def mvp_clip(self):
+ """读取实际的片段数据为moviepy格式
+
+ Raises:
+ NotImplementedError: _description_
+ """
+ raise NotImplementedError
+
+ @property
+ def duration_seq_emb(
+ self,
+ ) -> np.array:
+ emb = np.array([c.duration for c in self.clipseq])
+ return emb
+
+ @property
+ def timestamp_seq_emb(self) -> np.array:
+ emb = np.array([c.time_start for c in self.clipseq])
+ return emb
+
+ @property
+ def rela_timestamp_seq_emb(self) -> np.array:
+ duration_seq = [c.duration for c in self.clipseq]
+ emb = np.cumsum(duration_seq) / self.duration
+ return emb
+
+ def get_emb(self, key: str, idx: int) -> np.float:
+ clip_start_idx = self.clipseq[0].clipid
+ clip_end_idx = self.clipseq[-1].clipid
+ # TODO: 待修改为更通用的形式
+ if idx is None:
+ idx = range(clip_start_idx, clip_end_idx + 1)
+ elif isinstance(idx, int):
+ idx += clip_start_idx
+ elif isinstance(idx, Iterable):
+ idx = [x + clip_start_idx for x in idx]
+ else:
+ raise ValueError(
+ f"idx only support None, int, Iterable, but given {idx},type is {type(idx)}"
+ )
+ return self.emb.get_value(key, idx=idx)

MuseV/MMCM/mmcm/data/clip/clip_filter.py

@@ -0,0 +1,46 @@
+from typing import Callable, List, Union
+
+from .clip import ClipSeq
+
+from .clip_process import reset_clipseq_id
+
+
+class ClipFilter(object):
+ """clip滤波器，判断 Clip 是否符合标准
+
+ Args:
+ object (bool): 是否符合输入函数
+ """
+
+ def __init__(self, funcs: Union[Callable, List[Callable]], logic_func: Callable=all) -> None:
+ """多个 clip 判断函数，通过 逻辑与、或当综合结果。
+
+ Args:
+ funcs (list of func): 列表判断函数
+ logic_func (func, optional): all or any. Defaults to all.
+ """
+ self.funcs = funcs if isinstance(funcs, list) else [funcs]
+ self.logic_func = logic_func
+
+ def __call__(self, clip) -> bool:
+ flag = [func(clip) for func in self.funcs]
+ flag = self.logic_func(flag)
+ return flag
+
+
+
+# TODO
+class ClipSeqFilter(object):
+ def __init__(self, filter: Callable) -> None:
+ self.filter = filter
+
+ def __call__(self, clipseq: ClipSeq) -> ClipSeq:
+ new_clipseq = []
+ n_clipseq = len(clipseq)
+ for i in range(n_clipseq):
+ clip = clipseq[i]
+ if self.filter(clip):
+ new_clipseq.append(clip)
+ new_clipseq = reset_clipseq_id(new_clipseq)
+ # logger.debug("ClipSeqFilter: clipseq length before={}, after={}".format(n_clipseq, len(new_clipseq)))
+ return new_clipseq

MuseV/MMCM/mmcm/data/clip/clip_fusion.py

@@ -0,0 +1,64 @@
+from typing import List, Union, Callable
+
+from copy import deepcopy
+
+from .clip import ClipSeq
+from .clip_process import reset_clipseq_id
+import logging
+
+logger = logging.getLogger(__name__) # pylint: disable=invalid-name
+
+
+# TODO: 不同类型的clip需要不同的融合方式
+def fuse_clips(s1: ClipSeq, s2: ClipSeq) -> ClipSeq:
+ """合并2个clip
+
+ Args:
+ s1 (Clip):
+ s2 (Clip):
+
+ Returns:
+ Clip: 合并后Clip
+ """
+ if not isinstance(s2, list):
+ s2 = [s2]
+ s1 = deepcopy(s1)
+ for other_clip in s2:
+ s1.duration += other_clip.duration
+ if s1.stage is not None and other_clip.stage is not None:
+ # TODO：如何保留融合的clip信息
+ s1.stage = "{}_{}".format(s1.stage, other_clip.stage)
+ s1.origin_clipid.extend(other_clip.origin_clipid)
+ if s1.timepoint_type is not None and other_clip.timepoint_type is not None:
+ s1.timepoint_type = "{}_{}".format(
+ s1.timepoint_type, other_clip.timepoint_type
+ )
+ return s1
+
+
+# TODO: 不同的filter和fusion函数不适用同一种流程，待优化
+class ClipSeqFusion(object):
+ """_summary_
+
+ Args:
+ object (_type_): _description_
+ """
+
+ def __init__(self, filter: Callable, fuse_func: Callable = None) -> None:
+ self.filter = filter
+ self.fuse_func = fuse_func
+
+ def __call__(self, clipseq: ClipSeq) -> ClipSeq:
+ new_clipseq = []
+ n_clipseq = len(clipseq)
+ for i in range(n_clipseq):
+ clip = clipseq[i]
+ if self.filter(clip):
+ new_clipseq.append(clip)
+ new_clipseq = reset_clipseq_id(new_clipseq)
+ logger.debug(
+ "ClipSeqFilter: clipseq length before={}, after={}".format(
+ n_clipseq, len(new_clipseq)
+ )
+ )
+ return new_clipseq

MuseV/MMCM/mmcm/data/clip/clip_process.py

@@ -0,0 +1,366 @@
+from functools import partial
+from copy import deepcopy
+from typing import Iterable, List, Tuple, Union
+import bisect
+import logging
+
+import numpy as np
+
+
+from .clip import Clip, ClipSeq
+from .clipid import ClipIds, ClipIdsSeq, MatchedClipIds, MatchedClipIdsSeq
+
+logger = logging.getLogger(__name__) # pylint: disable=invalid-name
+
+__all__ = [
+ "find_idx_by_rela_time",
+ "find_idx_by_time",
+ "find_idx_by_clip",
+ "get_subseq_by_time",
+ "get_subseq_by_idx",
+ "clip_is_top",
+ "clip_is_middle",
+ "clip_is_end",
+ "abadon_old_return_new",
+ "reset_clipseq_id",
+ "insert_endclip",
+ "insert_startclip",
+ "drop_start_end_by_time",
+ "complete_clipseq",
+ "complete_gap",
+ "get_subseq_by_stages",
+ "find_time_by_stage",
+]
+
+
+def find_idx_by_rela_time(clipseq: ClipSeq, timepoint: float) -> int:
+ clipseq_duration = clipseq.duration
+ timepoint = clipseq_duration * timepoint
+ clipseq_times = [c.duration for c in clipseq]
+ clipseq_times.insert(0, 0)
+ clipseq_times = np.cumsum(clipseq_times)
+ idx = bisect.bisect_right(clipseq_times, timepoint)
+ idx = min(max(0, idx - 1), len(clipseq) - 1)
+ return idx
+
+
+def find_idx_by_time(clipseq: ClipSeq, timepoint: float) -> int:
+ """寻找指定时间timepoint 在 clipseq 中的片段位置
+
+ Args:
+ clipseq (ClipSeq): 待寻找的片段序列
+ timepoint (float): 指定时间位置
+
+ Returns:
+ _type_: _description_
+ """
+ clipseq_times = [c.time_start for c in clipseq]
+ idx = bisect.bisect_right(clipseq_times, timepoint)
+ idx = min(max(0, idx - 1), len(clipseq) - 1)
+ return idx
+
+
+def find_idx_by_clip(clipseq: ClipSeq, clip: Clip, eps: float = 1e-4) -> int:
+ """通过计算目标clip和clipseq中所有候选clip的交集占比来找最近clip
+
+ Args:
+ clipseq (ClipSeq): 候选clip序列
+ clip (Clip): 目标clip
+ eps (float, optional): 最小交集占比. Defaults to 1e-4.
+
+ Returns:
+ int: 目标clip在候选clip序列的位置，若无则为None
+ """
+ timepoints = np.array([[c.time_start, c.time_start + c.duration] for c in clipseq])
+ clip_time_start = clip.time_start
+ clip_duraiton = clip.duration
+ clip_time_end = clip_time_start + clip_duraiton
+ max_time_start = np.maximum(timepoints[:, 0], clip_time_start)
+ min_time_end = np.minimum(timepoints[:, 1], clip_time_end)
+ intersection = min_time_end - max_time_start
+ intersection_ratio = intersection / clip_duraiton
+ max_intersection_ratio = np.max(intersection_ratio)
+ idx = np.argmax(intersection_ratio) if max_intersection_ratio > eps else None
+ return idx
+
+
+def get_subseq_by_time(
+ clipseq: ClipSeq,
+ start: float = 0,
+ duration: float = None,
+ end: float = 1,
+ eps: float = 1e-2,
+) -> ClipSeq:
+ """根据时间对媒体整体做掐头去尾，保留中间部分。，也可以是大于1的数。
+ start和end如果是0-1的小数，则认为是是相对时间位置，实际位置会乘以duration；
+ start和end如果是大于1的数，则是绝对时间位置。
+
+ Args:
+ clipseq (ClipSeq): 待处理的序列
+ start (float,): 保留部分的开始，. Defaults to 0.
+ duration (float, optional): 媒体文件当前总时长
+ end (float, optional): 保留部分的结尾. Defaults to 1.
+
+ Returns:
+ ClipSeq: 处理后的序列
+ """
+ if (start == 0 or start is None) and (end is None or end == 1):
+ logger.warning("you should set start or end")
+ return clipseq
+ if duration is None:
+ duration = clipseq.duration
+ if start is None or start == 0:
+ clip_start_idx = 0
+ else:
+ if start < 1:
+ start = start * duration
+ clip_start_idx = find_idx_by_time(clipseq, start)
+ if end is None or end == 1 or np.abs(duration - end) < eps:
+ clip_end_idx = -1
+ else:
+ if end < 1:
+ end = end * duration
+ clip_end_idx = find_idx_by_time(clipseq, end)
+ if clip_end_idx != -1 and clip_start_idx >= clip_end_idx:
+ logger.error(
+ f"clip_end_idx({clip_end_idx}) should be > clip_start_idx({clip_start_idx})"
+ )
+ subseq = get_subseq_by_idx(clipseq, clip_start_idx, clip_end_idx)
+ return subseq
+
+
+def get_subseq_by_idx(clipseq: ClipSeq, start: int = None, end: int = None) -> ClipSeq:
+ """通过指定索引范围，切片子序列
+
+ Args:
+ clipseq (ClipSeq):
+ start (int, optional): 开始索引. Defaults to None.
+ end (int, optional): 结尾索引. Defaults to None.
+
+ Returns:
+ _type_: _description_
+ """
+ if start is None and end is None:
+ return clipseq
+ if start is None:
+ start = 0
+ if end is None:
+ end = len(clipseq)
+ return clipseq[start:end]
+
+
+def clip_is_top(clip: Clip, total: float, th: float = 0.1) -> bool:
+ """判断Clip是否属于开始部分
+
+ Args:
+ clip (Clip):
+ total (float): 所在ClipSeq总时长
+ th (float, optional): 开始范围的截止位置. Defaults to 0.05.
+
+ Returns:
+ Bool: 是不是头部Clip
+ """
+ clip_time = clip.time_start
+ if clip_time / total <= th:
+ return True
+ else:
+ return False
+
+
+def clip_is_end(clip: Clip, total: float, th: float = 0.9) -> bool:
+ """判断Clip是否属于结尾部分
+
+ Args:
+ clip (Clip):
+ total (float): 所在ClipSeq总时长
+ th (float, optional): 结尾范围的开始位置. Defaults to 0.9.
+
+ Returns:
+ Bool: 是不是尾部Clip
+ """
+ clip_time = clip.time_start + clip.duration
+ if clip_time / total >= th:
+ return True
+ else:
+ return False
+
+
+def clip_is_middle(
+ clip: Clip, total: float, start: float = 0.05, end: float = 0.9
+) -> bool:
+ """判断Clip是否属于中间部分
+
+ Args:
+ clip (Clip):
+ total (float): 所在ClipSeq总时长
+ start (float, optional): 中间范围的开始位置. Defaults to 0.05.
+ start (float, optional): 中间范围的截止位置. Defaults to 0.9.
+
+ Returns:
+ Bool: 是不是中间Clip
+ """
+ if start >= 0 and start < 1:
+ start = total * start
+ if end > 0 and end <= 1:
+ end = total * end
+ clip_time_start = clip.time_start
+ clip_time_end = clip.time_start + clip.duration
+ if (clip_time_start >= start) and (clip_time_end <= end):
+ return True
+ else:
+ return False
+
+
+def abadon_old_return_new(s1: Clip, s2: Clip) -> Clip:
+ """特殊的融合方式
+ Args:
+
+ s1 (Clip): 靠前的clip
+ s2 (Clip): 靠后的clip
+
+ Returns:
+ Clip: 融合后的Clip
+ """
+ return s2
+
+
+# TODO：待确认是否要更新clipid，不方便对比着json进行debug
+def reset_clipseq_id(clipseq: ClipSeq) -> ClipSeq:
+ for i in range(len(clipseq)):
+ if isinstance(clipseq[i], dict):
+ clipseq[i]["clipid"] = i
+ else:
+ clipseq[i].clipid = i
+ return clipseq
+
+
+def insert_startclip(clipseq: ClipSeq) -> ClipSeq:
+ """给ClipSeq插入一个开始片段。
+
+ Args:
+ clipseq (ClipSeq):
+ clip_class (Clip, optional): 插入的Clip类型. Defaults to Clip.
+
+ Returns:
+ ClipSeq: 插入头部Clip的新ClipSeq
+ """
+ if clipseq[0].time_start > 0:
+ start = clipseq.ClipClass(
+ time_start=0, duration=round(clipseq[0].time_start, 3), timepoint_type=0
+ )
+ clipseq.insert(0, start)
+ clipseq = reset_clipseq_id(clipseq)
+ return clipseq
+
+
+def insert_endclip(clipseq: ClipSeq, duration: float) -> ClipSeq:
+ """给ClipSeq插入一个尾部片段。
+
+ Args:
+ clipseq (ClipSeq):
+ duration(float, ): 序列的总时长
+ clip_class (Clip, optional): 插入的Clip类型. Defaults to Clip.
+
+ Returns:
+ ClipSeq: 插入尾部Clip的新ClipSeq
+ """
+ clipseq_endtime = clipseq[-1].time_start + clipseq[-1].duration
+ if duration - clipseq_endtime > 1:
+ end = clipseq.ClipClass(
+ time_start=round(clipseq_endtime, 3),
+ duration=round(duration - clipseq_endtime, 3),
+ timepoint_type=0,
+ )
+ clipseq.append(end)
+ clipseq = reset_clipseq_id(clipseq)
+ return clipseq
+
+
+def drop_start_end_by_time(
+ clipseq: ClipSeq, start: float, end: float, duration: float = None
+):
+ return get_subseq_by_time(clipseq=clipseq, start=start, end=end, duration=duration)
+
+
+def complete_clipseq(
+ clipseq: ClipSeq, duration: float = None, gap_th: float = 2
+) -> ClipSeq:
+ """绝大多数需要clipseq中的时间信息是连续、完备的，有时候是空的，需要补足的部分。
+ 如歌词时间戳生成的music_map缺头少尾、中间有空的部分。
+
+ Args:
+ clipseq (ClipSeq): 待补集的序列
+ duration (float, optional): 整个序列持续时间. Defaults to None.
+ gap_th (float, optional): 有时候中间空隙过短就会被融合到上一个片段中. Defaults to 2.
+
+ Returns:
+ ClipSeq: 补集后的序列，时间连续、完备。
+ """
+ if isinstance(clipseq, list):
+ clipseq = ClipSeq(clipseq)
+ return complete_clipseq(clipseq=clipseq, duration=duration, gap_th=gap_th)
+ clipseq = complete_gap(clipseq, th=gap_th)
+ clipseq = insert_startclip(clipseq)
+ if duration is not None:
+ clipseq = insert_endclip(clipseq, duration)
+ return clipseq
+
+
+def complete_gap(clipseq: ClipSeq, th: float = 2) -> ClipSeq:
+ """generate blank clip timepoint = 0，如果空白时间过短，则空白附到上一个歌词片段中。
+
+
+ Args:
+ clipseq (ClipSeq): 原始的歌词生成的MusicClipSeq
+ th (float, optional): 有时候中间空隙过短就会被融合到上一个片段中. Defaults to 2.
+
+ Returns:
+ ClipSeq: 补全后的
+ """
+ gap_clipseq = []
+ clipid = 0
+ for i in range(len(clipseq) - 1):
+ time_start = clipseq[i].time_start
+ duration = clipseq[i].duration
+ time_end = time_start + duration
+ next_time_start = clipseq[i + 1].time_start
+ time_diff = next_time_start - time_end
+ if time_diff >= th:
+ blank_clip = clipseq.ClipClass(
+ time_start=time_end,
+ duration=time_diff,
+ timepoint_type=0,
+ clipid=clipid,
+ )
+ gap_clipseq.append(blank_clip)
+ clipid += 1
+ else:
+ clipseq[i].duration = next_time_start - time_start
+ clipseq.extend(gap_clipseq)
+ clipseq.clips = sorted(clipseq.clips, key=lambda clip: clip.time_start)
+ reset_clipseq_id(clipseq)
+ return clipseq
+
+
+def find_time_by_stage(
+ clipseq: ClipSeq, stages: Union[str, List[str]] = None
+) -> Tuple[float, float]:
+ if isinstance(stages, list):
+ stages = [stages]
+ for clip in clipseq:
+ if clip.stage in stages:
+ return clip.time_start, clip.time_end
+ return None, None
+
+
+def get_subseq_by_stages(clipseq: ClipSeq, stages: Union[str, List[str]]) -> ClipSeq:
+ if isinstance(stages, List):
+ stages = [stages]
+ start, _ = find_time_by_stage(clipseq, stages[0])
+ _, end = find_time_by_stage(clipseq, stages[-1])
+ if start1 is None:
+ start1 = 0
+ if end2 is None:
+ end2 = clipseq.duration
+ subseq = get_subseq_by_time(clipseq=clipseq, start=start, end=end)
+ return subseq

MuseV/MMCM/mmcm/data/clip/clip_stat.py

@@ -0,0 +1,13 @@
+from typing import Tuple
+
+import numpy as np
+
+from .clip import ClipSeq
+
+
+def stat_clipseq_duration(
+ clipseq: ClipSeq,
+) -> Tuple[np.array, np.array]:
+ clip_duration = [clip.duration for clip in clipseq]
+ (hist, bin_edges) = np.histogram(clip_duration)
+ return hist, bin_edges

MuseV/MMCM/mmcm/data/clip/clipid.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from typing import Union, List
+
+__all__ = [
+ "ClipIds",
+ "ClipIdsSeq",
+ "MatchedClipIds",
+ "MatchedClipIdsSeq",
+]
+
+
+class ClipIds(object):
+ def __init__(
+ self,
+ clipids: Union[int, List[int]],
+ ) -> None:
+ """ClipSeq 中的 Clip序号，主要用于多个 Clip 融合后的 Clip, 使用场景如
+ 1. 一个 MusicClip 可以匹配到多个 VideoClip，VideoClip 的索引便可以使用 ClipIds 定义。
+
+ Args:
+ clipids (list or int): ClipSeq 中的序号
+ """
+ self.clipids = clipids if isinstance(clipids, list) else [clipids]
+
+
+class ClipIdsSeq(object):
+ def __init__(self, clipids_seq: List[ClipIds]) -> None:
+ """多个 ClipIds，使用场景可以是
+ 1. 将MediaClipSeq 进行重组，拆分重组成更粗粒度的ClipSeq；
+
+ Args:
+ clipids_seq (list): 组合后的 ClipIds 列表
+ """
+ self.clipids_seq = (
+ clipids_seq if isinstance(clipids_seq, ClipIds) else [clipids_seq]
+ )
+
+
+# TODO: metric后续可能是字典
+class MatchedClipIds(object):
+ def __init__(
+ self, id1: ClipIds, id2: ClipIds, metric: float = None, **kwargs
+ ) -> None:
+ """两种模态数据的片段匹配对，使用场景 可以是
+ 1. 音乐片段和视频片段 之间的匹配关系，
+
+ Args:
+ id1 (ClipIds): 第一种模态的片段
+ id2 (ClipIds): 第二种模态的片段
+ metric (float): 匹配度量距离
+ """
+ self.id1 = id1 if isinstance(id1, ClipIds) else ClipIds(id1)
+ self.id2 = id2 if isinstance(id2, ClipIds) else ClipIds(id2)
+ self.metric = metric
+ self.__dict__.update(**kwargs)
+
+
+class MatchedClipIdsSeq(object):
+ def __init__(self, seq: List[MatchedClipIds], metric: float = None, **kwargs) -> None:
+ """两种模态数据的序列匹配对，使用场景可以是
+ 1. 音乐片段序列和视频片段序列 之间的匹配，每一个元素都是MatchedClipIds:
+
+ Args:
+ seq (list): 两种模态数据的序列匹配对列表
+ metric (float): 匹配度量距离
+ """
+ self.seq = seq
+ self.metric = metric
+ self.__dict__.update(**kwargs)

MuseV/MMCM/mmcm/data/crawl/download.py

@@ -0,0 +1,72 @@
+
+from collections import namedtuple
+from typing import NamedTuple, Tuple, List
+import logging
+import os
+import numpy as np
+import subprocess
+
+import requests
+
+import wget 
+
+from .youtube import download_youtube
+from .flicker import download_flickr
+from .ffmpeg import ffmpeg_load
+
+logger = logging.getLogger(__name__)
+
+# DownloadStatus = namedtuple("DownloadStatus", ["status_code", "msg"])
+
+status_code = {0: "download: succ",
+ -1: "download: failed",
+ -2: "clip: failed",
+ -3: "directory not exists",
+ -4: "skip task",
+ - 404: "param error"}
+
+
+def download_with_request(url, path):
+ res = requests.get(url)
+ if res.status_code == '200' or res.status_code == 200:
+ with open(path, "wb") as f:
+ f.write(res.content)
+ else:
+ print('request failed')
+ return path
+
+def download_video(url, save_path:str=None, save_dir:str=None, basename:str=None, filename:str=None, format:str=None, data_type: str="wget", **kwargs) -> Tuple[int, str]:
+ if save_path is None:
+ if basename is None:
+ basename = f"{filename}.{format}"
+ save_path = os.path.join(save_dir, basename)
+ if save_dir is None:
+ save_dir = os.path.dirname(save_path)
+ if basename is None:
+ basename = os.path.basename(save_path)
+ if filename is None:
+ filename, format = os.path.splitext(basename)
+ os.makedirs(save_dir, exist_ok=True)
+
+ if os.path.exists(save_path):
+ return (-4, save_path)
+
+ try:
+ if data_type == "requests":
+ save_path = download_with_request(url=url, path=save_path)
+ elif data_type == "wget":
+ save_path = wget.download(url=url, out=save_path)
+ elif data_type == "youtube":
+ save_path = download_youtube(url, format=format, save_dir=save_dir, filename=basename)
+ elif data_type == "flickr":
+ save_path = download_flickr(url, save_path)
+ elif data_type == "ffmpeg":
+ code = ffmpeg_load(url=url, save_path=save_path)
+ else:
+ raise ValueError(f"data_type shoulbe one of [wget, youtube, flickr, ffmpeg], but given {data_type}")
+ except Exception as e:
+ logger.error("failed download file {} to {} failed!".format(url, save_path))
+ logger.exception(e)
+ return (-1, None)
+
+ return (0, save_path)

MuseV/MMCM/mmcm/data/crawl/error.py

@@ -0,0 +1,20 @@
+
+
+class SubprocessError(Exception):
+ """
+ Exception object that contains information about an error that occurred
+ when running a command line command with a subprocess.
+ """
+
+ def __init__(self, cmd, return_code, stdout, stderr, *args):
+ msg = 'Got non-zero exit code ({1}) from command "{0}": {2}'
+ if stderr.strip():
+ err_msg = stderr
+ else:
+ err_msg = stdout
+ msg = msg.format(cmd[0], return_code, err_msg)
+ self.cmd = cmd
+ self.cmd_return_code = return_code
+ self.cmd_stdout = stdout
+ self.cmd_stderr = stderr
+ super(SubprocessError, self).__init__(msg, *args)

MuseV/MMCM/mmcm/data/crawl/ffmpeg.py

@@ -0,0 +1,39 @@
+import subprocess
+
+from .error import SubprocessError
+
+
+class FfmpegInvalidURLError(Exception):
+ """
+ Exception raised when a 4XX or 5XX error is returned when making a request
+ """
+
+ def __init__(self, url, error, *args):
+ self.url = url
+ self.error = error
+ msg = 'Got error when making request to "{}": {}'.format(url, error)
+ super(FfmpegInvalidURLError, self).__init__(msg, *args)
+
+
+def ffmpeg_load(url: str, save_path: str) -> str:
+
+ def run(cmd):
+ proc = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = proc.communicate()
+ return_code = proc.returncode
+
+ if return_code != 0:
+ raise SubprocessError(
+ cmd, return_code, stdout.decode(), stderr.decode())
+ return return_code
+
+ command = ['ffmpeg', '-n', '-i', url, '-t', '10', '-f', 'mp4',
+ '-r', '30', '-vcodec', 'h264', save_path, '-loglevel', 'error']
+ code = run(command)
+ return code
+
+
+
+
+

MuseV/MMCM/mmcm/data/crawl/flicker.py

@@ -0,0 +1,22 @@
+import os
+
+from .ffmpeg import ffmpeg_load
+
+
+def extract_flickr_id(url):
+ return url.strip('/').split('/')[-4]
+
+
+def download_flickr(url: str, save_path: str) -> str:
+ code = -1
+ code = ffmpeg_load(url=url,
+ save_path=save_path)
+ if code == 0:
+ return (code, save_path)
+ # only retry when failed!
+ flickr_id = extract_flickr_id(url)
+ url = 'https://www.flickr.com/video_download.gne?id={}'.format(
+ flickr_id)
+ code = ffmpeg_load(url=url,
+ save_path=save_path)
+ return save_path

MuseV/MMCM/mmcm/data/crawl/youtube.py

@@ -0,0 +1,13 @@
+
+import os
+
+from pytube import YouTube
+
+
+def download_youtube(url, format, save_dir, filename):
+ youtube = YouTube(url)
+ streams = youtube.streams.filter(progressive=True,
+ file_extension=format)
+ save_path = streams.get_highest_resolution().download(output_path=save_dir,
+ filename=filename)
+ return save_path

MuseV/MMCM/mmcm/data/emb/__init__.py

@@ -0,0 +1,2 @@
+from .emb import *
+from .h5py_emb import H5pyMediaMapEmb, H5pyMediaMapEmbProxy

MuseV/MMCM/mmcm/data/emb/emb.py

@@ -0,0 +1,104 @@
+"""用于将 mediamap中的emb存储独立出去，仍处于开发中
+"""
+import logging
+
+import numpy as np
+
+
+logger = logging.getLogger(__name__) # pylint: disable=invalid-name
+
+__all__ = ["MediaMapEmb"]
+
+
+class MediaMapEmb(object):
+ def __init__(self, path: str) -> None:
+ """
+ OfflineEmb = {
+ "overall_algo": Emb, # 整个文件的Emb
+ # 整个文件的多维度 Emb
+ "theme": np.array, # 主题，
+ "emotion_algo": np.array, # 情绪，
+ "semantic_algo": np.array, # 语义
+
+ "clips_overall_algo": np.array, n_clip x clip_emb
+ "clips_emotion_algo": np.array, n_clip x clip_emb
+ "clips_semantic_algo": np.array, n_clip x clip_emb
+ "clips_theme_algo": np.array, n_clip x clip_emb
+
+ "scenes_overall_algo": np.array, n_scenes x scene_emb
+ "scenes_emotion_algo": np.array, n_scenes x scene_emb
+ "scenes_semantic_algo": np.array, n_scenes x scene_emb
+ "scenes_theme_algo": E np.arraymb, n_scenes x scene_emb
+ # 片段可以是转场切分、MusicStage等, clips目前属于转场切分片段
+ # 若后续需要新增段落分割，可以和clips同级新增 stage字段。
+
+ "frames_overall_algo": np.array, n_frames x frame_emb
+ "frames_emotion_algo": np.array, n_frames x frame_emb
+ "frames_semantic_algo": np.array, n_frames x frame_emb
+ "frames_theme_algo": np.array, n_frames x frame_emb
+ "frames_objs": {
+ "frame_id": { #
+ "overall_algo": np.array, n_objs x obj_emb
+ "emotion_algo": np.array, n_objs x obj_emb
+ "semantic_algo": np.array, n_objs x obj_emb
+ "theme_algo": np.array, n_objs x obj_emb
+ }
+ }
+ "roles_algo": {
+ "roleid": np.array, n x obj_emb
+ }
+ }
+
+
+ Args:
+ path (str): hdf5 存储路径
+ """
+ self.path = path
+
+ def get_value(self, key, idx=None):
+ raise NotImplementedError
+
+ def __getitem__(self, key):
+ return self.get_value(key)
+
+ def get_media(self, factor, algo):
+ return self.get_value(f"{factor}_{algo}")
+
+ def get_clips(self, factor, algo, idx=None):
+ return self.get_value(f"clips_{factor}_{algo}", idx=idx)
+
+ def get_frames(self, factor, algo, idx=None):
+ return self.get_value(f"frames_{factor}_{algo}", idx=idx)
+
+ def get_frame_objs(self, frame_idx, factor, algo, idx=None):
+ return self.get_value(["frames_objs", frame_idx, f"{factor}_{algo}"], idx=idx)
+
+ def set_value(self, key, value, idx=None):
+ raise NotImplementedError
+
+ def set_media(self, factor, value, algo):
+ self.set_value([f"{factor}_{algo}"], value)
+
+ def set_clips(self, factor, value, algo, idx=None):
+ self.set_value([f"clips_{factor}_{algo}"], value, idx=idx)
+
+ def set_frames(self, factor, value, algo, idx=None):
+ self.set_value([f"frames_{factor}_{algo}"], value)
+
+ def set_frame_objs(self, frame_idx, factor, value, algo, idx=None):
+ return self.set_value(
+ ["frames_objs", frame_idx, f"{factor}_{algo}"], value, idx=idx
+ )
+
+ def set_roles(self, algo, value, idx=None):
+ return self.set_value(f"roles_{algo}", value, idx=idx)
+
+ def get_roles(self, algo, idx=None):
+ return self.get_value(f"roles_{algo}", idx=idx)
+
+ def __setitem__(self, key, value):
+ self.set_value(self, key, value)
+
+
+class MediaMapEmbProxy(MediaMapEmb):
+ pass

MuseV/MMCM/mmcm/data/emb/h5py_emb.py

@@ -0,0 +1,119 @@
+from typing import Union, List
+import logging
+
+import h5py
+import numpy as np
+
+from .emb import MediaMapEmb
+
+logger = logging.getLogger(__name__) # pylint: disable=invalid-name
+
+__all__ = ["H5pyMediaMapEmb", "save_value_with_h5py"]
+
+
+def save_value_with_h5py(
+ path: str,
+ value: Union[np.ndarray, None],
+ key: str,
+ idx: Union[int, List[int]] = None,
+ dtype=None,
+ shape=None,
+ overwrite: bool = False,
+):
+ with h5py.File(path, "a") as f:
+ if dtype is None:
+ dtype = value.dtype
+ if shape is None:
+ shape = value.shape
+ del_key = False
+ if key in f:
+ if overwrite:
+ del_key = True
+ if f[key].dtype != h5py.special_dtype(vlen=str):
+ if f[key].shape != value.shape:
+ del_key = True
+ if del_key:
+ del f[key]
+ if key not in f:
+ f.create_dataset(key, shape=shape, dtype=dtype)
+ if idx is None:
+ f[key][...] = value
+ else:
+ f[key][idx] = value
+
+
+class H5pyMediaMapEmb(MediaMapEmb):
+ def __init__(self, path: str) -> None:
+ """
+ OfflineEmb = {
+ "overall_algo": Emb, # 整个文件的Emb
+ # 整个文件的多维度 Emb
+ "theme": np.array, # 主题，
+ "emotion_algo": np.array, # 情绪，
+ "semantic_algo": np.array, # 语义
+
+ "clips_overall_algo": np.array, n_clip x clip_emb
+ "clips_emotion_algo": np.array, n_clip x clip_emb
+ "clips_semantic_algo": np.array, n_clip x clip_emb
+ "clips_theme_algo": np.array, n_clip x clip_emb
+
+ "scenes_overall_algo": np.array, n_scenes x scene_emb
+ "scenes_emotion_algo": np.array, n_scenes x scene_emb
+ "scenes_semantic_algo": np.array, n_scenes x scene_emb
+ "scenes_theme_algo": E np.arraymb, n_scenes x scene_emb
+ # 片段可以是转场切分、MusicStage等, clips目前属于转场切分片段
+ # 若后续需要新增段落分割，可以和clips同级新增 stage字段。
+
+ "frames_overall_algo": np.array, n_frames x frame_emb
+ "frames_emotion_algo": np.array, n_frames x frame_emb
+ "frames_semantic_algo": np.array, n_frames x frame_emb
+ "frames_theme_algo": np.array, n_frames x frame_emb
+ "frames_objs_algo": {
+ "frame_id_algo": { #
+ "overall_algo": np.array, n_objs x obj_emb
+ "emotion_algo": np.array, n_objs x obj_emb
+ "semantic_algo": np.array, n_objs x obj_emb
+ "theme_algo": np.array, n_objs x obj_emb
+ }
+ }
+ "roles_algo": {
+ "roleid": np.array, n x obj_emb
+ }
+ }
+
+ Args:
+ path (str): hdf5 存储路径
+ """
+ super().__init__(path)
+ # 待优化支持 with open 的方式来读写
+ self.f = h5py.File(path, "a")
+
+ def _keys_index(self, key):
+ if not isinstance(key, list):
+ key = [key]
+ key = "/".join([str(x) for x in key if x is not None])
+ return key
+
+ def get_value(self, key, idx=None):
+ new_key = self._keys_index(key)
+ if idx is None:
+ data = np.array(self.f[new_key])
+ else:
+ data = np.array(self.f[new_key][idx])
+ return data
+
+ def set_value(self, key, value, idx=None):
+ new_key = self._keys_index(key)
+ if new_key not in self.f:
+ self.f.create_dataset(new_key, shape=value.shape, dtype=value.dtype)
+ if idx is None:
+ self.f[new_key][...] = value
+ else:
+ self.f[new_key][idx] = value
+
+ def close(self):
+ self.f.close()
+
+
+class H5pyMediaMapEmbProxy(H5pyMediaMapEmb):
+ pass

MuseV/MMCM/mmcm/data/extract_feature/base_extract_feature.py

@@ -0,0 +1,28 @@
+from typing import List, Union, Any
+
+import torch
+from torch import nn
+import numpy as np
+import h5py
+
+
+class BaseFeatureExtractor(nn.Module):
+ def __init__(self, device: str = "cpu", dtype=torch.float32, name: str = None):
+ super().__init__()
+ self.device = device
+ self.dtype = dtype
+ self.name = name
+
+ def extract(
+ self, data: Any, return_type: Union[str, str] = "numpy"
+ ) -> Union[np.ndarray, torch.tensor]:
+ raise NotADirectoryError
+
+ def __call__(self, *args: Any, **kwds: Any) -> Any:
+ return self.extract(*args, **kwds)
+
+ def save_with_h5py(self, f: Union[h5py.File, str], *args, **kwds):
+ raise NotImplementedError
+
+ def forward(self, *args: Any, **kwds: Any) -> Any:
+ return self.extract(*args, **kwds)

MuseV/MMCM/mmcm/data/general/__init__.py

@@ -0,0 +1 @@
+from .items import Items

MuseV/MMCM/mmcm/data/general/items.py

@@ -0,0 +1,69 @@
+from collections import UserList
+from collections.abc import Iterable
+from typing import Iterator, Any, List
+
+from ...utils.util import convert_class_attr_to_dict
+
+__all__ = ["Item", "Items"]
+
+
+class Item(object):
+ def __init__(self) -> None:
+ pass
+
+ def to_dct(self, target_keys: List[str] = None, ignored_keys: List[str] = None):
+ base_ignored_keys = [
+ "kwargs",
+ ]
+ if isinstance(ignored_keys, list):
+ base_ignored_keys.extend(ignored_keys)
+ elif isinstance(ignored_keys, str):
+ base_ignored_keys.append(ignored_keys)
+ else:
+ pass
+ return convert_class_attr_to_dict(
+ self, target_keys=target_keys, ignored_keys=base_ignored_keys
+ )
+
+ def preprocess(self):
+ pass
+
+
+class Items(UserList):
+ def __init__(
+ self,
+ data: Any = None,
+ ):
+ if data is None:
+ data = list()
+ if not isinstance(data, list):
+ data = [data]
+ super().__init__(data)
+
+ def __len__(self):
+ return len(self.data)
+
+ def __getitem__(self, i):
+ return self.data[i]
+
+ def __delitem__(self, i):
+ del self.data[i]
+
+ def __setitem__(self, i, v):
+ self.data[i] = v
+
+ def insert(self, i, v):
+ self.data.insert(i, v)
+
+ def __str__(self):
+ return str(self.data)
+
+ def to_dct(self, target_keys: List[str] = None, ignored_keys: List[str] = None):
+ items = [item.to_dct(target_keys, ignored_keys) for item in self.data]
+ return items
+
+ def __iter__(self) -> Iterator:
+ return iter(self.data)
+
+ def preprocess(self):
+ pass

MuseV/MMCM/mmcm/data/media_map/__init__.py

@@ -0,0 +1 @@
+from .media_map import MetaInfo, MediaMap, MetaInfoList

MuseV/MMCM/mmcm/data/media_map/media_map.py

@@ -0,0 +1,393 @@
+from __future__ import annotations
+import bisect
+
+import logging
+from copy import deepcopy
+
+from functools import partial
+from typing import Any, Callable, Iterable, List, Union, Tuple, Dict
+
+import numpy as np
+from ..clip.clip_process import get_subseq_by_time
+from ..clip.clip_stat import stat_clipseq_duration
+from ..clip import Clip, ClipSeq, ClipIds, MatchedClipIds, MatchedClipIdsSeq
+from .media_map_process import get_sub_mediamap_by_time
+from ..emb import MediaMapEmb, H5pyMediaMapEmb
+from ..general.items import Item, Items
+from ...utils.data_util import pick_subdct
+from ...utils.util import convert_class_attr_to_dict, load_dct_from_file
+
+logger = logging.getLogger(__name__) # pylint: disable=invalid-name
+
+
+__all__ = ["MetaInfo", "MetaInfoList", "MediaMap", "MediaMapSeq"]
+
+
+class MetaInfo(Item):
+ """歌曲、视频等媒体文件级别的元信息"""
+
+ def __init__(
+ self,
+ mediaid=None,
+ media_name=None,
+ media_duration=None,
+ signature=None,
+ media_path: str = None,
+ media_map_path: str = None,
+ start: float = None,
+ end: float = None,
+ ext=None,
+ **kwargs,
+ ):
+ super(MetaInfo).__init__()
+ self.mediaid = mediaid
+ self.media_name = media_name
+ self.media_duration = media_duration
+ self.signature = signature
+ self.media_path = media_path
+ self.media_map_path = media_map_path
+ self.start = start
+ self.end = end
+ self.ext = ext
+ self.__dict__.update(**kwargs)
+ self.preprocess()
+
+ def preprocess(self):
+ self.set_start_end()
+
+ def set_start_end(self):
+ if self.start is None:
+ self.start = 0
+ elif self.start >= 0 and self.start <= 1:
+ self.start = self.start * self.media_duration
+
+ if self.end is None:
+ self.end = self.media_duration
+ elif self.end >= 0 and self.end <= 1:
+ self.end = self.end * self.media_duration
+
+
+class MetaInfoList(Items):
+ """媒体元数据列表，主要用于多歌曲、多视频剪辑时存储原单一媒体文件的元信息"""
+
+ def __init__(self, items: Union[MetaInfo, List[MetaInfo]] = None):
+ """
+ Args:
+ meta_info_list (list, optional): MetaInfo 列表. Defaults to None.
+ """
+ if items is None:
+ items = []
+ else:
+ items = items if isinstance(items, list) else [items]
+ super().__init__(items)
+ self.meta_info_list = self.items
+ if len(self.items) > 1:
+ self.reset()
+
+ def __len__(self):
+ return len(self.meta_info_list)
+
+ def __getitem__(self, i) -> MetaInfo:
+ return self.meta_info_list[i]
+
+ @property
+ def groupnum(self) -> int:
+ return len(self.meta_info_list)
+
+
+class MediaMap(object):
+ """媒体信息基类，也可以理解为音乐谱面、视觉谱面、音游谱面基类。主要有 MetaInfo、MetaInfoList、ClipSeq 属性。
+ 不同的媒体信息的 属性 类会有不同，所以在类变量里做定义。如有变化，可以定义自己的属性类。
+ """
+
+ def __init__(
+ self,
+ meta_info: MetaInfo = None,
+ clipseq: ClipSeq = None,
+ stageseq: ClipSeq = None,
+ frameseq: ClipSeq = None,
+ emb: H5pyMediaMapEmb = None,
+ **kwargs,
+ ):
+ """用于存储media的相关信息，media_info是json或直接字典
+
+ Args:
+ meta_info (MetaInfo): 当sub_meta_info不为None时, meta_info由sub_meta_info整合而成
+ sub_meta_info (None or [MetaInfo]): 当多个MediaInfo拼在一起时,用于保留子MediaInfo的信息
+ clipseq (ClipSeq): # 按照clipidx排序;
+ stageseq (ClipSeq): # 比 clipseq 更高纬度的片段划分，例如clips是镜头分割，stages是scenes分割；clips是关键点分割，stages是结构分割；
+ frameseq (ClipSeq): # 比 clipseq 更低纬度的片段划分
+ kwargs (dict, optional): 所有相关信息都会作为 meta_info 的补充，赋值到 meta_info 中
+ """
+ self.meta_info = meta_info
+ self.clipseq = clipseq
+ self.frameseq = frameseq
+ self.stageseq = stageseq
+ self.emb = emb
+ self.meta_info.__dict__.update(**kwargs)
+ self.preprocess()
+
+ def preprocess(
+ self,
+ ):
+ if (self.meta_info.start != 0 and self.meta_info.start is not None) or (
+ self.meta_info.end is not None and self.meta_info.end == 1
+ ):
+ self.drop_head_and_tail()
+ self.meta_info.preprocess()
+ if self.clipseq is not None:
+ self.clipseq.preprocess()
+ if self.frameseq is not None:
+ self.frameseq.preprocess()
+ if self.stageseq is not None:
+ self.stageseq.preprocess()
+ self.clip_start_idx = self.clipseq[0].clipid
+ self.clip_end_idx = self.clipseq[-1].clipid
+
+ def drop_head_and_tail(self) -> MediaMap:
+ self.clipseq = get_subseq_by_time(
+ self.clipseq,
+ start=self.meta_info.start,
+ end=self.meta_info.end,
+ duration=self.meta_info.media_duration,
+ )
+ if self.stageseq is not None:
+ self.stageseq = get_subseq_by_time(
+ self.clipseq,
+ start=self.meta_info.start,
+ end=self.meta_info.end,
+ duration=self.meta_info.media_duration,
+ )
+
+ def set_clip_value(self, k, v):
+ """为clipseq中的每个clip赋值，
+
+ Args:
+ k (str): Clip中字段名
+ v (any): Clip中字段值
+ """
+ self.clipseq.set_clip_value(k, v)
+
+ def spread_metainfo_2_clip(
+ self, target_keys: List = None, ignored_keys: List = None
+ ) -> None:
+ """将metainfo中的信息赋值到clip中，便于clip后面做相关处理。
+
+ Args:
+ target_keys ([str]): 待赋值的目标字段
+ """
+ dst = pick_subdct(
+ self.meta_info.__dict__, target_keys=target_keys, ignored_keys=ignored_keys
+ )
+ for k, v in dst.items():
+ self.set_clip_value(k, v)
+
+ def spread_parameters(self, target_keys: list, ignored_keys) -> None:
+ """元数据广播，将 media_info 的元数据广播到 clip 中，以及调用 clip 自己的参数传播。"""
+ self.spread_metainfo_2_clip(target_keys=target_keys, ignored_keys=ignored_keys)
+ for clip in self.clipseq:
+ clip.spread_parameters()
+
+ def stat(
+ self,
+ ):
+ """统计 media_info 相关信息，便于了解，目前统计内容有
+ 1. 片段长度
+ """
+ self.stat_clipseq_duration()
+
+ def stat_clipseq_duration(
+ self,
+ ):
+ hist, bin_edges = stat_clipseq_duration(self.clipseq)
+ print(self.media_name, "bin_edges", bin_edges)
+ print(self.media_name, "hist", hist)
+
+ def to_dct(self, target_keys: list = None, ignored_keys: list = None):
+ raise NotImplementedError
+
+ @property
+ def duration(
+ self,
+ ):
+ return self.clipseq.duration
+
+ @property
+ def mediaid(
+ self,
+ ):
+ return self.meta_info.mediaid
+
+ @property
+ def media_name(
+ self,
+ ):
+ return self.meta_info.media_name
+
+ @property
+ def duration_seq_emb(self):
+ return self.clipseq.duration_seq_emb
+
+ @property
+ def timestamp_seq_emb(self):
+ return self.clipseq.timestamp_seq_emb
+
+ @property
+ def rela_timestamp_seq_emb(self):
+ return self.clipseq.rela_timestamp_seq_emb
+
+ def get_emb(self, key, idx=None):
+ # TODO: 待修改为更通用的形式
+ if idx is None:
+ idx = range(self.clip_start_idx, self.clip_end_idx + 1)
+ elif isinstance(idx, int):
+ idx += self.clip_start_idx
+ elif isinstance(idx, Iterable):
+ idx = [x + self.clip_start_idx for x in idx]
+ else:
+ raise ValueError(
+ f"idx only support None, int, Iterable, but given {idx},type is {type(idx)}"
+ )
+ return self.emb.get_value(key, idx=idx)
+
+ def get_meta_info_attr(self, key: str) -> Any:
+ return getattr(self.meta_info, key)
+
+ @classmethod
+ def from_json_path(
+ cls, path: Dict, emb_path: str, media_path: str = None, **kwargs
+ ) -> MediaMap:
+ media_map = load_dct_from_file(path)
+ emb = H5pyMediaMapEmb(emb_path)
+ return cls.from_data(media_map, emb=emb, media_path=media_path, **kwargs)
+
+
+class MediaMapSeq(Items):
+ def __init__(self, maps: List[MediaMap]) -> None:
+ super().__init__(maps)
+ self.maps = self.data
+ self.preprocess()
+ self.each_map_clipseq_num = [len(m.clipseq) for m in self.maps]
+ self.each_map_clipseq_num_cumsum = np.cumsum([0] + self.each_map_clipseq_num)
+
+ @property
+ def clipseq(self):
+ clipseq = []
+ for m in self.maps:
+ clipseq.extend(m.clipseq.data)
+ return type(self.maps[0].clipseq)(clipseq)
+
+ @property
+ def stagesseq(self):
+ stagesseq = []
+ for m in self.maps:
+ stagesseq.extend(m.stagesseq.data)
+ return type(self.maps[0].stagesseq)(stagesseq)
+
+ @property
+ def frameseq(self):
+ frameseq = []
+ for m in self.maps:
+ frameseq.extend(m.frameseq.data)
+ return type(self.maps[0].frameseq)(frameseq)
+
+ def preprocess(self):
+ for m in self.maps:
+ m.preprocess()
+
+ def _combine_str(
+ self,
+ attrs: List[str],
+ sep: str = "|",
+ single_maxlen: int = 10,
+ total_max_length: int = 60,
+ ) -> str:
+ return sep.join([str(attr)[:single_maxlen] for attr in attrs])[
+ :total_max_length
+ ]
+
+ def get_meta_info_attr(self, key: str, func: Callable) -> Any:
+ attrs = [m.get_meta_info_attr(key) for m in self.maps]
+ return func(attrs)
+
+ @property
+ def mediaid(self) -> str:
+ return self.get_meta_info_attr(key="mediaid", func=self._combine_str)
+
+ @property
+ def media_name(self) -> str:
+ return self.get_meta_info_attr(key="media_name", func=self._combine_str)
+
+ @property
+ def duration(self) -> float:
+ return sum([m.duration for m in self.maps])
+
+ @property
+ def media_duration(self) -> float:
+ return self.get_meta_info_attr(key="media_duration", func=sum)
+
+ @classmethod
+ def from_json_paths(
+ cls,
+ media_map_class: MediaMap,
+ media_paths: str,
+ media_map_paths: str,
+ emb_paths: str,
+ **kwargs,
+ ) -> MediaMapSeq:
+ map_seq = [
+ media_map_class.from_json_path(
+ path=media_map_paths[i],
+ emb_path=emb_paths[i],
+ media_path=media_paths[i],
+ **kwargs,
+ )
+ for i in range(len(media_map_paths))
+ ]
+ return cls(map_seq)
+
+ # TODO: implement mapseq stat func
+ def stat(self):
+ for m in self.maps:
+ m.stat()
+
+ def _combine_embs(self, embs):
+ return np.concatenate(embs, axis=0)
+
+ @property
+ def duration_seq_emb(self):
+ embs = [m.duration_seq_emb for m in self.maps]
+ return self._combine_embs(embs)
+
+ @property
+ def timestamp_seq_emb(self):
+ embs = [m.timestamp_seq_emb for m in self.maps]
+ return self._combine_embs(embs)
+
+ @property
+ def rela_timestamp_seq_emb(self):
+ embs = [m.rela_timestamp_seq_emb for m in self.maps]
+ return self._combine_embs(embs)
+
+ def clip_idx_2_map_idx(self, idx):
+ target_map_idx = bisect.bisect_right(self.each_map_clipseq_num_cumsum, idx)
+ target_map_idx = min(max(0, target_map_idx - 1), len(self.maps) - 1)
+ target_map_clip_idx = idx - self.each_map_clipseq_num_cumsum[target_map_idx]
+ return target_map_idx, target_map_clip_idx
+
+ def get_emb(self, key: str, idx: Union[None, int, List[int]] = None) -> np.array:
+ if idx is None:
+ embs = [m.get_emb(key, idx=idx) for m in self.maps]
+ else:
+ if not isinstance(idx, list):
+ idx = [idx]
+ embs = []
+ for c_idx in idx:
+ target_map_idx, target_map_clip_idx = self.clip_idx_2_map_idx(c_idx)
+ embs.append(
+ self.maps[target_map_idx].get_emb(key, int(target_map_clip_idx))
+ )
+ if len(embs) == 1:
+ return embs[0]
+ else:
+ return self._combine_embs(embs)

MuseV/MMCM/mmcm/data/media_map/media_map_process.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from typing import List, Union, TYPE_CHECKING
+from ..clip.clip_process import (
+ get_subseq_by_time,
+ find_time_by_stage, 
+
+)
+if TYPE_CHECKING:
+ from ..media_map.media_map import MediaMap
+ from ..clip import Clip, ClipSeq
+
+
+__all__ =[
+ "get_sub_mediamap_by_clip_idx",
+ "get_sub_mediamap_by_stage",
+ "get_sub_mediamap_by_time",
+]
+
+
+def get_sub_mediamap_by_time(media_map:MediaMap, start: int=0, end:int=1, eps=1e-2) -> MediaMap:
+ """获取子片段序列，同时更新media_map中的相关信息
+
+ Args:
+ media_map (MediaInfo): _description_
+ start (float): 开始时间
+ end (float): 结束时间
+
+ Returns:
+ _type_: _description_
+ """
+ if start < 1:
+ start = media_map.duration * start
+ if end is None:
+ end = media_map.meta_info.media_duration
+ elif end <= 1:
+ end = media_map.duration * end
+ media_map.meta_info.start = start
+ media_map.meta_info.end = end
+ media_map.clipseq = get_subseq_by_time(
+ media_map.clipseq,
+ start=start,
+ end=end,
+ )
+ if media_map.stageseq is not None:
+ media_map.stageseq = get_subseq_by_time(media_map.stageseq, start=start, end=end) 
+ return media_map
+
+
+def get_sub_mediamap_by_clip_idx(media_map: MediaMap, start: int=None, end: int=None) -> MediaMap:
+ """不仅获取子片段序列，还要更新media_map中的相关信息
+
+ Args:
+ media_map (_type_): _description_
+ """
+ if start is None:
+ start = 0
+ if end is None:
+ end = -1
+ start = media_map.clipseq[start].time_start
+ end = media_map.clipseq[end].time_end
+ media_map = get_sub_mediamap_by_time(media_map=media_map, start=start, end=end) 
+ return media_map
+
+
+def get_sub_mediamap_by_stage(media_map: MediaMap, stages: Union[str, List[str]]) -> MediaMap:
+ if isinstance(stages, List):
+ stages = [stages]
+ start, _ = find_time_by_stage(media_map.stageseq, stages[0])
+ _, end = find_time_by_stage(media_map.stageseq, stages[-1])
+ media_map = get_sub_mediamap_by_time(media_map=media_map, start=start, end=end)
+ return media_map

MuseV/MMCM/mmcm/music/__init__.py

@@ -0,0 +1,6 @@
+from .music_map.music_map import MusicMap, MusicMapSeq
+from .music_map.music_clip import MusicClip, MusicClipSeq
+from .music_map.meta_info import MusicMetaInfo
+from .music_map.load_music_map import load_music_map
+
+from .utils.path_util import get_audio_path_dct

MuseV/MMCM/mmcm/music/music_map/beat_map.py

@@ -0,0 +1,82 @@
+import numpy as np
+
+from librosa.core.audio import get_duration
+
+from ...data.clip.clip_process import insert_endclip, insert_startclip
+
+from .clip_process import filter_clipseq_target_point
+from .music_clip import MusicClip, MusicClipSeq
+
+
+def beatnet2TMEType(beat: np.array, duration: float) -> MusicClipSeq:
+ """conver beatnet beat to tme beat type
+
+ Args:
+ beat (np.array): Nx2,
+ 1st column is time,
+ 2rd is type,
+ 0, end point
+ 1, strong beat
+ 2,3,4 weak beat
+ -1 lyric
+ duration (float): audio time length
+ Returns:
+ MusicClipSeq:
+ """
+ n = len(beat)
+ beat = np.insert(beat, 0, 0, axis=0)
+ beat = np.insert(beat, n + 1, [duration, 0], axis=0)
+ clips = []
+ for i in range(n + 1):
+ beat_type = int(beat[i + 1, 1])
+ clip = MusicClip(
+ time_start=beat[i, 0], # 开始时间
+ duration=round(beat[i + 1, 0] - beat[i, 0], 3), # 片段持续时间
+ clipid=i, # 片段序号，
+ timepoint_type=beat_type,
+ )
+ clips.append(clip)
+ clipseq = MusicClipSeq(clips=clips)
+ return clipseq
+
+
+def generate_beatseq_with_beatnet(audio_path: str) -> np.array:
+ """使用beatnet生成beat序列
+
+ Args:
+ audio_path (str):
+ Returns:
+ np.array: beat序列 Nx2,
+ 1st column is time,
+ 2rd is type,
+ 0, end point
+ 1, strong beat
+ 2,3,4 weak beat
+ """
+ from BeatNet.BeatNet import BeatNet
+
+ estimator = BeatNet(1, mode="offline", inference_model="DBN", plot=[], thread=False)
+ output = estimator.process(audio_path=audio_path)
+ return output
+
+
+def generate_music_map_with_beatnet(
+ audio_path: str, target: list = [0, 1]
+) -> MusicClipSeq:
+ """使用beatnet生成beat MusicClipseq
+
+ Args:
+ audio_path (str):
+ target (list, optional): 只保留相应的拍点. Defaults to [0, 1].
+
+ Returns:
+ MusicClipSeq: 返回的beat序列
+ beat: np.array, 原始的beat检测结果
+ """
+ output = generate_beatseq_with_beatnet(audio_path)
+ duration = get_duration(filename=audio_path)
+ clipseq = beatnet2TMEType(output, duration)
+ clipseq = insert_startclip(clipseq)
+ clipseq = insert_endclip(clipseq, duration)
+ clipseq = filter_clipseq_target_point(clipseq, target=target)
+ return clipseq, output

MuseV/MMCM/mmcm/music/music_map/clip_process.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING, Dict, List
+
+import numpy as np
+
+from ...data.clip.clip_process import find_idx_by_time, reset_clipseq_id
+from ...data.clip.clip_fusion import fuse_clips
+from ...utils.util import merge_list_continuous_same_element
+
+if TYPE_CHECKING:
+ from .music_clip import MusicClip, MusicClipSeq
+ from .music_map import MusicMap, MusicMapSeq
+
+
+# TODO: 待和clip操作做整合
+def music_clip_is_short(clip: MusicClip, th: float = 3) -> bool:
+ """判断音乐片段是否过短
+
+ Args:
+ clip (MusicClip): 待判断的音乐片段
+ th (float, optional): 短篇的参数. Defaults to 3.
+
+ Returns:
+ bool: 是或不是 短片段
+ """
+ if clip.duration < th:
+ return False
+ else:
+ return True
+
+
+def music_clip_timepoint_is_target(clip: MusicClip, target: list = [-1, 1, 0]) -> bool:
+ """音乐片段的关键点类型是否是目标关键点
+ 关键点类型暂时参考：VideoMashup/videomashup/data_structure/music_data_structure.py
+ Args:
+ clip (MusicClip): 待判断的音乐片段
+ target (list, optional): 目标关键点类别. Defaults to [-1, 1, 0].
+
+ Returns:
+ bool: 是还是不是
+ """
+ timepoint = clip.timepoint_type
+ if isinstance(timepoint, int):
+ timepoint = {timepoint}
+ else:
+ timepoint = {int(x) for x in timepoint.split("_")}
+ if timepoint & set(target):
+ return True
+ else:
+ return False
+
+
+def filter_clipseq_target_point(
+ clipseq: MusicClipSeq, target: list = [-1, 1, 0]
+) -> MusicClipSeq:
+ """删除目标关键点之外的点，对相应的片段做融合
+
+ Args:
+ clipseq (MusicClipSeq): 待处理的音乐片段序列
+ target (list, optional): 保留的目标关键点. Defaults to [-1, 1, 0].
+
+ Returns:
+ MusicClipSeq: 处理后的音乐片段序列
+ """
+ n_clipseq = len(clipseq)
+ if n_clipseq == 1:
+ return clipseq
+ newclipseq = []
+ start_clip = clipseq[0]
+ if music_clip_timepoint_is_target(start_clip, target=target):
+ has_start_clip = True
+ else:
+ has_start_clip = False
+ i = 1
+ while i <= n_clipseq - 1:
+ clip = clipseq[i]
+ start_clip_is_target = music_clip_timepoint_is_target(start_clip, target=target)
+ next_clip_is_target = music_clip_timepoint_is_target(clip, target=target)
+ # logger.debug("filter_clipseq_target_point: i={},start={}, clip={}".format(i, start_clip["timepoint_type"], clip["timepoint_type"]))
+ # logger.debug("start_clip_is_target: {}, next_clip_is_target {}".format(start_clip_is_target, next_clip_is_target))
+ if not has_start_clip:
+ start_clip = clip
+ has_start_clip = next_clip_is_target
+ else:
+ if start_clip_is_target:
+ has_start_clip = True
+ if next_clip_is_target:
+ newclipseq.append(start_clip)
+ start_clip = clip
+ if i == n_clipseq - 1:
+ newclipseq.append(clip)
+ else:
+ start_clip = fuse_clips(start_clip, clip)
+ if i == n_clipseq - 1:
+ newclipseq.append(start_clip)
+ # logger.debug("filter_clipseq_target_point: fuse {}, {}".format(i, clip["timepoint_type"]))
+ else:
+ start_clip = clip
+ i += 1
+ newclipseq = reset_clipseq_id(newclipseq)
+ return newclipseq
+
+
+def merge_musicclip_into_clipseq(
+ clip: MusicClipSeq, clipseq: MusicClip, th: float = 1
+) -> MusicClipSeq:
+ """给clipseq插入一个新的音乐片段，会根据插入后片段是否过短来判断。
+
+ Args:
+ clip (MusicClipSeq): 要插入的音乐片段
+ clipseq (MusicClip): 待插入的音乐片段序列
+ th (float, optional): 插入后如果受影响的片段长度过短，则放弃插入. Defaults to 1.
+
+ Returns:
+ MusicClipSeq: _description_
+ """
+ n_clipseq = len(clipseq)
+ clip_time = clip.time_start
+ idx = find_idx_by_time(clipseq, clip_time)
+ last_clip_time_start = clipseq[idx].time_start
+ next_clip_time_start = clipseq[idx].time_start + clipseq[idx].duration
+ last_clip_time_delta = clip_time - last_clip_time_start
+ clip_duration = next_clip_time_start - clip_time
+ # TODO: 副歌片段改变th参数来提升音符密度，暂不使用，等待音游谱面
+ # TODO: 待抽离独立的业务逻辑为单独的函数
+ # 只针对副歌片段插入关键点
+ if clipseq[idx].text is None or (
+ clipseq[idx].text is not None
+ and clipseq[idx].stage is not None
+ and "C" in clipseq[idx].stage
+ ):
+ if (last_clip_time_delta > th) and (clip_duration > th):
+ clip.duration = clip_duration
+ clipseq[idx].duration = last_clip_time_delta
+ clipseq.insert(idx + 1, clip)
+ clipseq = reset_clipseq_id(clipseq)
+ return clipseq
+
+
+def merge_music_clipseq(clipseq1: MusicClipSeq, clipseq2: MusicClipSeq) -> MusicClipSeq:
+ """将片段序列clipseq2融合到音乐片段序列clipseq1中。融合过程也会判断新片段长度。
+
+ Args:
+ clipseq1 (MusicClipSeq): 要融合的目标音乐片段序列
+ clipseq2 (MusicClipSeq): 待融合的音乐片段序列
+
+ Returns:
+ MusicClipSeq: 融合后的音乐片段序列
+ """
+ while len(clipseq2) > 0:
+ clip = clipseq2[0]
+ clipseq1 = merge_musicclip_into_clipseq(clip, clipseq1)
+ del clipseq2[0]
+ return clipseq1
+
+
+def merge_lyricseq_beatseq(
+ lyric_clipseq: MusicClipSeq, beat_clipseq: MusicClipSeq
+) -> MusicClipSeq:
+ """将beat序列融合到歌词序列中
+
+ Args:
+ lyric_clipseq (MusicClipSeq): 歌词序列
+ beat_clipseq (MusicClipSeq): beat序列
+
+ Returns:
+ MusicClipSeq: 融合后的音乐片段序列
+ """
+ newclipseq = merge_music_clipseq(lyric_clipseq, beat_clipseq)
+ # for i, clip in enumerate(newclipseq):
+ # logger.debug("i={}, time_start={}, duration={}".format(i, clip.time_start, clip.duration))
+ return newclipseq
+
+
+def get_stageseq_from_clipseq(clipseq: MusicClipSeq) -> List[Dict]:
+ """对clip.stage做近邻融合，返回总时间
+
+ Returns:
+ List[Dict]: 根据音乐结构进行分割的片段序列
+ """
+ stages = [clip.stage for clip in clipseq]
+ merge_stages_idx = merge_list_continuous_same_element(stages)
+ merge_stages = []
+ for n, stages_idx in enumerate(merge_stages_idx):
+ dct = {
+ "clipid": n,
+ "time_start": clipseq[stages_idx["start"]].time_start,
+ "time_end": clipseq[stages_idx["end"]].time_end,
+ "stage": stages_idx["element"],
+ "original_clipid": list(
+ range(stages_idx["start"], stages_idx["end"] + 1)
+ ), # mss都是左闭、 右闭的方式
+ }
+ dct["duration"] = dct["time_end"] - dct["time_start"]
+ merge_stages.append(dct)
+ return merge_stages

MuseV/MMCM/mmcm/music/music_map/convert_type.py

@@ -0,0 +1,57 @@
+from ...data.clip.clip_process import (
+ insert_startclip,
+ insert_endclip,
+ reset_clipseq_id,
+)
+
+from .music_clip import MusicClip, MusicClipSeq
+
+
+def read_osu_hitobjs(path: str) -> list:
+ """读取osu的音游谱面
+
+ Args:
+ path (str): 谱面低质
+
+ Returns:
+ list: 只包含HitObjects的行字符串信息
+ """
+ lines = []
+ is_hit_info_start = False
+ with open(path, "r") as f:
+ for line in f:
+ if is_hit_info_start:
+ lines.append(line.strip())
+ if "[HitObjects]" in line:
+ is_hit_info_start = True
+ return lines
+
+
+def osu2itech(src: list, duration: float = None) -> MusicClipSeq:
+ """将osu的音游谱面转换为我们的目标格式
+
+ Args:
+ src (list): 音游谱面路径或者是读取的目标行字符串列表
+ duration (float, optional): 歌曲长度. Defaults to None.
+
+ Returns:
+ MusicClipSeq: 音乐片段序列
+ """
+ if isinstance(src, str):
+ src = read_osu_hitobjs(src)
+ timepoints = [float(line.split(",")[2]) for line in src]
+ clips = []
+ for i in range(len(timepoints) - 1):
+ clip = MusicClip(
+ time_start=round(timepoints[i] / 1000, 3),
+ timepoint_type=0,
+ duration=round((timepoints[i + 1] - timepoints[i]) / 1000, 3),
+ clipid=i,
+ )
+ clips.append(clip)
+ if len(clips) > 0:
+ clips = insert_startclip(clips)
+ if duration is not None:
+ clips = insert_endclip(clips, duration=duration)
+ clips = reset_clipseq_id(clips)
+ return MusicClipSeq(clips)

MuseV/MMCM/mmcm/music/music_map/load_music_map.py

@@ -0,0 +1,38 @@
+
+
+from typing import List
+
+from .music_map import MusicMap, MusicMapSeq
+
+
+def load_music_map(
+ music_map_paths,
+ music_paths,
+ emb_paths,
+ start: float=None,
+ end: None=None,
+ target_stages: List[str] = None,
+ **kwargs,
+):
+ """读取视频谱面，转化成MusicInfo。当 musicinfo_path_lst 为列表时，表示多歌曲
+
+ Args:
+ musicinfo_path_lst (str or [str]): 视频谱面路径文件列表
+ music_path_lst (str or [str]): 视频文件路径文件列表，须与musicinfo_path_lst等长度
+
+
+ Returns:
+ MusicInfo: 视频谱面信息
+ """
+ dct ={
+ "start": start,
+ "end": end,
+ "target_stages": target_stages,
+ }
+ if isinstance(music_map_paths, list):
+ music_map = MusicMapSeq.from_json_paths(media_map_class=MusicMapSeq, media_paths=music_paths, media_map_paths=music_map_paths, emb_paths=emb_paths, **dct, **kwargs)
+ if len(music_map) == 1:
+ music_map = music_map[0]
+ else:
+ music_map = MusicMap.from_json_path(path=music_map_paths, emb_path=emb_paths, media_path=music_paths, **dct, **kwargs)
+ return music_map

MuseV/MMCM/mmcm/music/music_map/lyric_map.py

@@ -0,0 +1,149 @@
+import numpy as np
+from sklearn.preprocessing import normalize, minmax_scale
+from scipy.signal import savgol_filter
+
+# TODO：待更新音乐谱面的类信息
+from ...data.clip.clip_process import (
+ complete_clipseq,
+ find_idx_by_clip,
+ insert_endclip,
+ insert_startclip,
+ reset_clipseq_id,
+)
+
+from .music_clip import Clip, ClipSeq
+from .music_clip import MusicClipSeq
+from .music_map import MusicMap
+
+
+def generate_lyric_map(
+ path: str, duration: float = None, gap_th: float = 2
+) -> MusicClipSeq:
+ """从歌词文件中生成音乐谱面
+
+ Args:
+ path (str): 歌词文件路径
+ duration (float, optional): 歌词对应音频的总时长. Defaults to None.
+ gap_th (float, optional): 歌词中间的空白部分是否融合到上一个片段中. Defaults to 3.
+
+ Returns:
+ MusicClipSeq: 以歌词文件生成的音乐谱面
+ """
+ from ..music_map.lyric_process import lyricfile2musicinfo
+
+ lyric_info = lyricfile2musicinfo(path)
+ lyric_info = MusicMap(lyric_info, duration=duration)
+ clipseq = lyric_info.clipseq
+ lyric_info.meta_info.duration = duration
+ # set part of nonlyric as clip whose timepoint is 0
+ for i in range(len(clipseq)):
+ clipseq[i].timepoint_type = -1
+ lyric_info.clipseq = complete_clipseq(
+ clipseq=clipseq, duration=duration, gap_th=gap_th
+ )
+ return lyric_info
+
+
+def insert_field_2_clipseq(clipseq: ClipSeq, reference: ClipSeq, field: str) -> ClipSeq:
+ """将reference中每个clip的字段信息根据赋给clipseq中最近的clip
+
+ Args:
+ clipseq (ClipSeq): 目标clip序列
+ reference (ClipSeq): 参考clip序列
+ field (str): 目标字段
+
+ Returns:
+ ClipSeq: 更新目标字段新值后的clip序列
+ """
+ for i, clip in enumerate(clipseq):
+ idx = find_idx_by_clip(reference, clip=clip)
+ if idx is not None:
+ if getattr(reference[idx], field) is not None:
+ clipseq[i].__dict__[field] = getattr(reference[idx], field)
+ return clipseq
+
+
+def insert_rythm_2_clipseq(clipseq, reference):
+ """参考MSS字段的结构信息设置rythm信息。目前策略非常简单，主歌(Vx)0.25，副歌(Cx)0.75，其他为None
+
+ Args:
+ clipseq (ClipSeq): 目标clip序列，设置rythm字段
+ reference (ClipSeq): 参考clip序列，参考stage字段
+
+ Returns:
+ ClipSeq: 更新rythm字段新值后的clip序列
+ """
+
+ def stage2rythm(stage):
+ if "V" in stage:
+ return 0.25
+ elif "C" in stage:
+ return 0.75
+ else:
+ return None
+
+ for i, clip in enumerate(clipseq):
+ idx = find_idx_by_clip(reference, clip=clip)
+ if idx is not None:
+ if reference[idx].rythm is not None:
+ clipseq[i].rythm = stage2rythm(reference[idx].stage)
+ return clipseq
+
+
+def insert_rythm_from_clip(clipseq: MusicClipSeq, beat: np.array) -> MusicClipSeq:
+ """给MusicClipSeq中的每个Clip新增节奏信息。目前使用
+ 1. 单位时间内的歌词数量特征, 使用 min-max 归一化到 0 - 1 之间
+ 2. 单位时间内的关键点数量，目前使用beatnet,使用 min-max 归一化到 0 - 1 之间
+ 3. 对1、2中的特征相加，并根据歌曲结构不同进行加权
+ Args:
+ clipseq (MusicClipSeq): 待处理的 MusicClipSeq
+ beat (np.array): beat检测结果，Nx2,，用于结算单位时间内的关键点数。
+ 1st column is time,
+ 2rd is type,
+ 0, end point
+ 1, strong beat
+ 2,3,4 weak beat
+
+ Returns:
+ MusicClipSeq: 新增 rythm 的 MusicClipSeq
+ """
+ mss_cofficient = {
+ "intro": 1.0,
+ "bridge": 1.0,
+ "end": 0.8,
+ "VA": 1.0,
+ "VB": 1.0,
+ "CA": 1.6,
+ "CB": 1.6,
+ }
+ # text_num_per_second
+ text_num_per_second_lst = [clip.tnps for clip in clipseq if clip.tnps != 0]
+ common_tnps = np.min(text_num_per_second_lst)
+ tnps = np.array([clip.tnps if clip.tnps != 0 else common_tnps for clip in clipseq])
+ tnps = minmax_scale(tnps)
+ # beat point _num_per_second
+ beat_pnps = np.zeros(len(clipseq))
+ for i, clip in enumerate(clipseq):
+ time_start = clip.time_start
+ time_end = clip.time_end
+ target_beat = beat[(beat[:, 0] >= time_start) & (beat[:, 0] < time_end)]
+ beat_pnps[i] = len(target_beat) / clip.duration
+ beat_pnps = minmax_scale(beat_pnps)
+
+ # cofficient
+ cofficients = np.array(
+ [
+ mss_cofficient[clip.stage]
+ if clip.stage in mss_cofficient and clip.stage is not None
+ else 1.0
+ for clip in clipseq
+ ]
+ )
+
+ rythm = cofficients * (tnps + beat_pnps)
+ rythm = minmax_scale(rythm)
+ rythm = savgol_filter(rythm, window_length=5, polyorder=3)
+ rythm = minmax_scale(rythm)
+ for i, clip in enumerate(clipseq):
+ clip.dynamic = rythm[i]
+ return clipseq

MuseV/MMCM/mmcm/music/music_map/lyric_process.py

@@ -0,0 +1,515 @@
+from genericpath import isfile
+import re
+import os
+
+from ...text.utils.read_text import read_xml2json
+
+
+# 一个正则表达式非常好用的网站
+# https://regex101.com/r/cW8jA6/2
+
+
+CHINESE_PATTERN = r"[\u4e00-\u9fff]+"
+NOT_CHINESE_PATTERN = r"[^\u4e00-\u9fa5]"
+ENGLISH_CHARACHTER_PATTERN = r"[a-zA-Z]+"
+WORD_PATTERN = r"\w+" # equal to [a-zA-Z0-9_].
+NOT_WORD_PATTERN = r"\W+"
+
+
+def has_target_string(lyric: str, pattern: str) -> bool:
+ """本句歌词是否有目标字符串
+
+ Args:
+ lyric (str):
+ pattern (str): 目标字符串的正则表达式式patteren
+
+ Returns:
+ bool: 有没有目标字符串
+ """
+ matched = re.findall(pattern, lyric)
+ flag = len(matched) > 0
+ return flag
+
+
+def has_chinese_char(lyric: str) -> bool:
+ """是否有中文字符
+
+ Args:
+ lyric (str):
+
+ Returns:
+ bool: 是否有中文字符
+ """
+ return has_target_string(lyric, CHINESE_PATTERN)
+
+
+def has_non_chinese_char(lyric: str) -> bool:
+ """是否有非中文字符，参考https://git.woa.com/innovative_tech/CopyrightGroup/LyricTools/blob/master/lyric_tools/dataProcess.py#L53
+
+ Args:
+ lyric (str):
+
+ Returns:
+ bool: 是否有中文字符
+ """
+ return has_target_string(lyric, NOT_CHINESE_PATTERN)
+
+
+def has_english_alphabet_char(lyric: str) -> bool:
+ """是否有英文字母表字符
+
+ Args:
+ lyric (str):
+
+ Returns:
+ bool:
+ """
+ return has_target_string(lyric, ENGLISH_CHARACHTER_PATTERN)
+
+
+def check_is_lyric_row(lyric: str) -> bool:
+ """该字符串是否是歌词
+
+ Args:
+ lyric (str): 待判断的字符串
+
+ Returns:
+ bool: 该字符串是否是歌词
+ """
+ is_not_lyric = [
+ re.search(r"\[ti[:：]?", lyric),
+ re.search(r"\[ar[:：]?", lyric),
+ re.search(r"\[al[:：]?", lyric),
+ re.search(r"\[by[:：]?", lyric),
+ re.search(r"\[offset[:：]?", lyric),
+ re.search(r"词[:：]?\(\d+,\d+\)[:：]?", lyric),
+ re.search(r"曲[:：]?\(\d+,\d+\)[:：]?", lyric),
+ re.search(r"作\(\d+,\d+\)词[:：]?", lyric),
+ re.search(r"作\(\d+,\d+\)曲[:：]?", lyric),
+ re.search(r"演\(\d+,\d+\)唱[:：]?", lyric),
+ re.search(r"编\(\d+,\d+\)曲[:：]?", lyric),
+ re.search(r"吉\(\d+,\d+\)他[:：]", lyric),
+ re.search(r"人\(\d+,\d+\)声\(\d+,\d+\)录\(\d+,\d+\)音\(\d+,\d+\)师[:：]?", lyric),
+ re.search(r"人\(\d+,\d+\)声\(\d+,\d+\)录\(\d+,\d+\)音\(\d+,\d+\)棚[:：]?", lyric),
+ re.search(r"Vocal\s+\(\d+,\d+\)edite[:：]?", lyric),
+ re.search(r"混\(\d+,\d+\)音\(\d+,\d+\)/\(\d+,\d+\)母\(\d+,\d+\)带[:：]?", lyric),
+ re.search(r"混\(\d+,\d+\)音", lyric),
+ re.search(r"和\(\d+,\d+\)声\(\d+,\d+\)编\(\d+,\d+\)写[:：]?", lyric),
+ re.search(
+ r"词\(\d+,\d+\)版\(\d+,\d+\)权\(\d+,\d+\)管\(\d+,\d+\)理\(\d+,\d+\)方[:：]?", lyric
+ ),
+ re.search(
+ r"曲\(\d+,\d+\)版\(\d+,\d+\)权\(\d+,\d+\)管\(\d+,\d+\)理\(\d+,\d+\)方[:：]?", lyric
+ ),
+ re.search(r"联\(\d+,\d+\)合\(\d+,\d+\)出\(\d+,\d+\)品[:：]?", lyric),
+ re.search(r"录\(\d+,\d+\)音\(\d+,\d+\)作\(\d+,\d+\)品", lyric),
+ re.search(
+ r"录\(\d+,\d+\)音\(\d+,\d+\)作\(\d+,\d+\)品\(\d+,\d+\)监\(\d+,\d+\)制[:：]?", lyric
+ ),
+ re.search(r"制\(\d+,\d+\)作\(\d+,\d+\)人[:：]?", lyric),
+ re.search(r"制\(\d+,\d+\)作\(\d+,\d+\)人[:：]?", lyric),
+ re.search(r"不\(\d+,\d+\)得\(\d+,\d+\)翻\(\d+,\d+\)唱", lyric),
+ re.search(r"未\(\d+,\d+\)经\(\d+,\d+\)许\(\d+,\d+\)可", lyric),
+ re.search(r"酷\(\d+,\d+\)狗\(\d+,\d+\)音\(\d+,\d+\)乐", lyric),
+ re.search(r"[:：]", lyric),
+ ]
+ is_not_lyric = [x is not None for x in is_not_lyric]
+ is_not_lyric = any(is_not_lyric)
+ is_lyric = not is_not_lyric
+ return is_lyric
+
+
+def lyric2clip(lyric: str) -> dict:
+ """convert a line of lyric into a clip
+ Clip定义可以参考 https://git.woa.com/innovative_tech/VideoMashup/blob/master/videomashup/media/clip.py
+ Args:
+ lyric (str): _description_
+
+ Returns:
+ dict: 转化成Clip 字典
+ """
+ time_str_groups = re.findall(r"\d+,\d+", lyric)
+ line_time_start = round(int(time_str_groups[0].split(",")[0]) / 1000, 3)
+ line_duration = round(int(time_str_groups[0].split(",")[-1]) / 1000, 3)
+ line_end_time = line_time_start + line_duration
+ last_word_time_start = round(int(time_str_groups[-1].split(",")[0]) / 1000, 3)
+ last_word_duration = round(int(time_str_groups[-1].split(",")[-1]) / 1000, 3)
+ last_word_end_time = last_word_time_start + last_word_duration
+ actual_duration = min(line_end_time, last_word_end_time) - line_time_start
+ lyric = re.sub(r"\[\d+,\d+\]", "", lyric)
+
+ # by yuuhong: 把每个字的起始时间点、结束时间点、具体的字拆分出来
+ words_with_timestamp = get_words_with_timestamp(lyric)
+
+ lyric = re.sub(r"\(\d+,\d+\)", "", lyric)
+ dct = {
+ "time_start": line_time_start,
+ "duration": actual_duration,
+ "text": lyric,
+ "original_text": lyric,
+ "timepoint_type": -1,
+ "clips": words_with_timestamp,
+ }
+ return dct
+
+
+# by yuuhong
+# 把一句QRC中的每个字拆分出来
+# lyric示例：漫(17316,178)步(17494,174)走(17668,193)在(17861,183) (18044,0)莎(18044,153)玛(18197,159)丽(18356,176)丹(18532,200)
+def get_words_with_timestamp(lyric):
+ words_with_timestamp = []
+ elements = lyric.split(")")
+ for element in elements:
+ sub_elements = element.split("(")
+ if len(sub_elements) != 2:
+ continue
+ text = sub_elements[0]
+ timestamp = sub_elements[1]
+ if re.match(r"\d+,\d+", timestamp):
+ # 有效时间戳
+ time_start_str = timestamp.split(",")[0]
+ time_start = round(int(time_start_str) / 1000, 3)
+ duration_str = timestamp.split(",")[1]
+ duration = round(int(duration_str) / 1000, 3)
+ clip = {"text": text, "time_start": time_start, "duration": duration}
+ words_with_timestamp.append(clip)
+ return words_with_timestamp
+
+
+def lyric2clips(lyric: str, th: float = 0.75) -> list:
+ """将一句歌词转换为至少1个的clip。拆分主要是针对中文空格拆分，如果拆分后片段过短，也会整句处理。
+ Args:
+ lyric (str): such as [173247,3275]去(173247,403)吗(173649,677) 配(174326,189)吗(174516,593) 这(175108,279)
+ th (float, optional): 后面如果拆分后片段过短，也会整句处理. Defaults to 1.0.
+
+ Returns:
+ list: 歌词Clip序列
+ """
+ # 目前只对中文的一句歌词按照空格拆分，如果是英文空格则整句处理
+ # 后面如果拆分后片段过短，也会整句处理
+ if has_english_alphabet_char(lyric):
+ return [lyric2clip(lyric)]
+ splited_lyric = lyric.split(" ")
+ if len(splited_lyric) == 1:
+ return [lyric2clip(splited_lyric[0])]
+ line_time_str, sub_lyric = re.split(r"]", splited_lyric[0])
+ line_time_groups = re.findall(r"\d+,\d+", line_time_str)
+ line_time_start = round(int(line_time_groups[0].split(",")[0]) / 1000, 3)
+ line_duration = round(int(line_time_groups[0].split(",")[-1]) / 1000, 3)
+ splited_lyric[0] = sub_lyric
+ # 歌词xml都是歌词仅跟着时间，如果有空格 空格也应该是在时间后面，但有时候空格却在字后面、在时间前，因此需要修正
+ # 错误的：[173247,3275]去(173247,403)吗 (173649,677)配(174326,189)吗 (174516,593)这(175108,279)
+ # 错误的：[46122,2082]以(46122,213)身(46335,260)淬(46595,209)炼(46804,268)天(47072,250)地(47322,370)造(47692,341)化 (48033,172)
+ # 修正成：[173247,3275]去(173247,403)吗(173649,677) 配(174326,189)吗(174516,593) 这(175108,279)
+ for i in range(len(splited_lyric)):
+ if splited_lyric[i] == "":
+ del splited_lyric[i]
+ break
+ if splited_lyric[i][-1] != ")":
+ next_lyric_time_start = re.search(
+ r"\(\d+,\d+\)", splited_lyric[i + 1]
+ ).group(0)
+ splited_lyric[i] += next_lyric_time_start
+ splited_lyric[i + 1] = re.sub(
+ next_lyric_time_start, "", splited_lyric[i + 1]
+ )
+ splited_lyric[i + 1] = re.sub("\(\)", "", splited_lyric[i + 1])
+ lyric_text = re.sub(r"\[\d+,\d+\]", "", lyric)
+ lyric_text = re.sub(r"\(\d+,\d+\)", "", lyric_text)
+ clips = []
+ has_short_clip = False
+ for sub_lyric in splited_lyric:
+ sub_lyric_groups = re.findall(r"\d+,\d+", sub_lyric)
+ sub_lyric_1st_word_time_start = round(
+ int(sub_lyric_groups[0].split(",")[0]) / 1000, 3
+ )
+ sub_lyric_last_word_time_start = round(
+ int(sub_lyric_groups[-1].split(",")[0]) / 1000, 3
+ )
+ sub_lyric_last_word_duration = round(
+ int(sub_lyric_groups[-1].split(",")[-1]) / 1000, 3
+ )
+ sub_lyric_last_word_time_end = (
+ sub_lyric_last_word_time_start + sub_lyric_last_word_duration
+ )
+ sub_lyric_duration = (
+ sub_lyric_last_word_time_end - sub_lyric_1st_word_time_start
+ )
+ if sub_lyric_duration <= th:
+ has_short_clip = True
+ break
+ sub_lyric_text = re.sub(r"\[\d+,\d+\]", "", sub_lyric)
+ sub_lyric_text = re.sub(r"\(\d+,\d+\)", "", sub_lyric_text)
+ # 使用原始lyric，而不是sub_lyric_text 主要是保留相关clip的歌词信息，便于语义连续
+ dct = {
+ "time_start": sub_lyric_1st_word_time_start,
+ "duration": sub_lyric_duration,
+ "text": sub_lyric_text,
+ "original_text": lyric_text,
+ "timepoint_type": -1,
+ }
+ clips.append(dct)
+ if has_short_clip:
+ clips = [lyric2clip(lyric)]
+ return clips
+
+
+def is_songname(lyric: str) -> bool:
+ """是否是歌名，歌名文本含有ti, 如[ti:霍元甲 (《霍元甲》电影主题曲)]
+
+ Args:
+ lyric (str):
+
+ Returns:
+ bool:
+ """
+ return has_target_string(lyric, r"\[ti[:：]?")
+
+
+def get_songname(lyric: str) -> str:
+ """获取文本中的歌名，输入必须类似[ti:霍元甲 (《霍元甲》电影主题曲)]
+
+ Args:
+ lyric (str): 含有歌名的QRC文本行
+
+ Returns:
+ str: 歌名
+ """
+ return lyric.split("(")[0][4:-1]
+
+
+def is_album(lyric: str) -> bool:
+ """是否含有专辑名，文本必须类似[al:霍元甲]
+
+ Args:
+ lyric (str): _description_
+
+ Returns:
+ bool: _description_
+ """
+
+ return has_target_string(lyric, r"\[al[:：]?")
+
+
+def get_album(lyric: str) -> str:
+ """提取专辑名，文本必须类似[al:霍元甲]
+
+
+ Args:
+ lyric (str): 含有专辑名的QRC文本行
+
+ Returns:
+ str: 专辑名
+ """
+ return lyric[4:-1]
+
+
+def is_singer(lyric: str) -> bool:
+ """是否有歌手名，目标文本类似 [ar:周杰伦]
+
+ Args:
+ lyric (str): _description_
+
+ Returns:
+ bool: _description_
+ """
+ return has_target_string(lyric, r"\[ar[:：]?")
+
+
+def get_singer(lyric: str) -> str:
+ """提取歌手信息，文本必须类似[ar:周杰伦]
+
+ Args:
+ lyric (str): 含有歌手名的QRC文本行
+
+ Returns:
+ str: 歌手名
+ """
+ return lyric[4:-1]
+
+
+def lyric2musicinfo(lyric: str) -> dict:
+ """convert lyric content from str into musicinfo, a dict
+ 参考https://git.woa.com/innovative_tech/VideoMashup/blob/master/videomashup/media/media_info.py#L19
+ {
+ "meta_info": {},
+ "sub_meta_info": [],
+ "clips": [
+ clip
+ ]
+ }
+
+ Args:
+ lyric (str): 来自QRC的歌词字符串
+
+ Returns:
+ musicinfo: 音乐谱面字典，https://git.woa.com/innovative_tech/VideoMashup/blob/master/videomashup/media/media_info.py#L19
+ """
+ lyrics = lyric["QrcInfos"]["LyricInfo"]["Lyric_1"]["@LyricContent"]
+ musicinfo = {
+ "meta_info": {
+ "mediaid": None,
+ "media_name": None,
+ "singer": None,
+ },
+ "sub_meata_info": {},
+ "clips": [],
+ }
+ # lyrics = [line.strip() for line in re.split(r"[\t\n\s+]", lyrics)]
+ lyrics = ["[" + line.strip() for line in re.split(r"\[", lyrics)]
+ next_is_title_row = False
+ lyric_clips = []
+ for line in lyrics:
+ if is_songname(line):
+ musicinfo["meta_info"]["media_name"] = get_songname(line)
+ continue
+ if is_singer(line):
+ musicinfo["meta_info"]["singer"] = get_singer(line)
+ continue
+ if is_album(line):
+ musicinfo["meta_info"]["album"] = get_album(line)
+ continue
+ is_lyric_row = check_is_lyric_row(line)
+ if next_is_title_row:
+ next_is_title_row = False
+ continue
+ # remove tille row
+ if not next_is_title_row and re.search(r"\[offset[:：]", line):
+ next_is_title_row = True
+ if is_lyric_row and re.match(r"\[\d+,\d+\]", line):
+ lyric_clip = lyric2clip(line)
+ lyric_clips.append(lyric_clip)
+ clips = lyric2clips(line)
+ musicinfo["clips"].extend(clips)
+ musicinfo["meta_info"]["lyric"] = lyric_clips
+ return musicinfo
+
+
+def lrc_timestr2time(time_str: str) -> float:
+ """提取lrc中的时间戳文本，类似[00:00.00]，转化成秒的浮点数
+
+ Args:
+ time_str (str):
+
+ Returns:
+ float: 时间浮点数
+ """
+ m, s, ms = (float(x) for x in re.split(r"[:.]", time_str))
+ return round((m * 60 + s + ms / 1000), 3)
+
+
+def get_lrc_line_time(text: str, time_pattern: str) -> str:
+ """提取lrc中的时间字符串, 类似 \"[00:00.00]本字幕由天琴实验室独家AI字幕技术生成\"
+
+ Args:
+ text (str): 输入文本
+ time_pattern (str): 时间字符串正则表达式
+
+ Returns:
+ str: 符合正则表达式的时间信息文本
+ """
+ time_str = re.search(time_pattern, text).group(0)
+ return lrc_timestr2time(time_str)
+
+
+def lrc_lyric2clip(lyric: str, time_pattern: str, duration: float) -> dict:
+ """将一行lrc文本字符串转化为Clip 字典
+
+ Args:
+ lyric (str): 类似 \"[00:00.00]本字幕由天琴实验室独家AI字幕技术生成\"
+ time_pattern (str): 时间字符串正则表达式，类似 r"\d+:\d+\.\d+"
+ duration (float): clip的时长信息，
+
+ Returns:
+ dict: 转化后Clip
+ Clip定义可以参考 https://git.woa.com/innovative_tech/VideoMashup/blob/master/videomashup/media/clip.py
+ """
+ time_str = get_lrc_line_time(lyric, time_pattern=time_pattern)
+ text = re.sub(time_pattern, "", lyric)
+ text = text[2:]
+ clip = {
+ "time_start": time_str,
+ "duration": duration,
+ "text": text,
+ "timepoint_type": -1,
+ }
+ return clip
+
+
+def lrc2musicinfo(lyric: str, time_pattern: str = "\d+:\d+\.\d+") -> dict:
+ """将lrc转化为音乐谱面
+
+ Args:
+ lyric (str): lrc文本路径
+ time_pattern (str, optional): lrc时间戳字符串正则表达式. Defaults to "\d+:\d+\.\d+".
+
+ Returns:
+ dict: 生成的音乐谱面字典，定义可参考 https://git.woa.com/innovative_tech/VideoMashup/blob/master/videomashup/music/music_info.py
+ """
+ if isinstance(lyric, str):
+ if os.path.isfile(lyric):
+ with open(lyric, "r") as f:
+ lyric = [line.strip() for line in f.readlines()]
+ return lrc2musicinfo(lyric)
+ else:
+ lyric = lyric.split("\n")
+ return lrc2musicinfo(lyric)
+ else:
+ musicinfo = {
+ "meta_info": {
+ "mediaid": None,
+ "media_name": None,
+ "singer": None,
+ },
+ "sub_meata_info": {},
+ "clips": [],
+ }
+ # lyrics = [line.strip() for line in re.split(r"[\t\n\s+]", lyrics)]
+ lyric_clips = []
+ rows = len(lyric)
+ for i, line in enumerate(lyric):
+ if is_songname(line):
+ musicinfo["meta_info"]["media_name"] = line[4:-1]
+ continue
+ if is_singer(line):
+ musicinfo["meta_info"]["singer"] = line[4:-1]
+ continue
+ if is_album(line):
+ musicinfo["meta_info"]["album"] = line[4:-1]
+ continue
+ if len(re.findall(time_pattern, line)) > 0:
+ if i < rows - 1:
+ time_start = get_lrc_line_time(line, time_pattern=time_pattern)
+ next_line_time_start = get_lrc_line_time(
+ lyric[i + 1], time_pattern=time_pattern
+ )
+ duration = next_line_time_start - time_start
+ else:
+ duration = 1
+ clip = lrc_lyric2clip(
+ line, duration=duration, time_pattern=time_pattern
+ )
+ musicinfo["clips"].append(clip)
+ musicinfo["meta_info"]["lyric"] = lyric_clips
+ return musicinfo
+
+
+def lyricfile2musicinfo(path: str) -> dict:
+ """将歌词文件转化为音乐谱面，歌词文件可以是QRC的xml文件、也可以是lrc对应的lrc文件
+ TODO： 待支持osu
+
+ Args:
+ path (str): 歌词文件路径
+
+ Returns:
+ dict: 音乐谱面字典，定义可参考 https://git.woa.com/innovative_tech/VideoMashup/blob/master/videomashup/music/music_info.py
+ """
+
+ filename, ext = os.path.basename(path).split(".")
+ if ext == "xml":
+ lyric = read_xml2json(path)
+ musicinfo = lyric2musicinfo(lyric)
+ elif ext == "lrc":
+ musicinfo = lrc2musicinfo(path)
+ musicinfo["meta_info"]["mediaid"] = filename
+ return musicinfo

MuseV/MMCM/mmcm/music/music_map/meta_info.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from ...data import MetaInfo
+
+
+class MusicMetaInfo(MetaInfo):
+ def __init__(self, mediaid=None, media_name=None, media_duration=None, signature=None, media_path: str = None, media_map_path: str = None,
+ singer=None,
+ lyric_path=None,
+ genre=None,
+ language=None,
+ start: float = None, end: float = None, ext=None, **kwargs):
+ super().__init__(mediaid, media_name, media_duration, signature, media_path, media_map_path, start, end, ext, **kwargs)
+ self.singer = singer
+ self.genre = genre
+ self.language = language
+ self.lyric_path = lyric_path
+
+ @classmethod
+ def from_data(cls, data) -> MusicMetaInfo:
+ return MusicMetaInfo(**data)

MuseV/MMCM/mmcm/music/music_map/mss_map.py

@@ -0,0 +1,185 @@
+import logging
+
+from .music_clip import MusicClip, MusicClipSeq
+from .music_map import MusicMap
+from ...data.clip.clip_process import find_idx_by_time
+
+logger = logging.getLogger(__name__) # pylint: disable=invalid-name
+
+
+def insert_mss_2_clipseq(
+ clipseq: MusicClipSeq, mss_clipseq: MusicClipSeq
+) -> MusicClipSeq:
+ """将mss中的结构字段信息赋予到目标clipseq中的最近clip
+
+ Args:
+ clipseq (ClipSeq): 目标clip序列
+ reference (ClipSeq): 参考clip序列
+ field (str): 目标字段
+
+ Returns:
+ ClipSeq: 更新目标字段新值后的clip序列
+ """
+ for i, clip in enumerate(clipseq):
+ idx = find_idx_by_time(mss_clipseq, clip.time_start)
+ if idx is not None:
+ clipseq[i].stage = mss_clipseq[idx].stage
+ else:
+ clipseq[i].stage = "unknow"
+ return clipseq
+
+
+def get_mss_musicinfo(songid: str) -> MusicMap:
+ """通过调用media_data中的接口 获取天琴实验室的歌曲结构信息
+
+ Args:
+ songid (str): 歌词id
+
+ Returns:
+ MusicMap: mss结构信息生成的音乐谱面
+ """
+ try:
+ from media_data.oi.tianqin_database import get_mss
+
+ mss = get_mss(songid=songid)
+ except Exception as e:
+ logger.warning("get mss failed, mss={}".format(songid))
+ logger.exception(e)
+ mss = None
+ mss_musicinfo = MusicMap(mss) if mss is not None else None
+ return mss_musicinfo
+
+
+def merge_mss(musicinfo: MusicMap, mss: MusicMap) -> MusicMap:
+ """融合mss音乐谱面到目标音乐谱面
+
+ Args:
+ musicinfo (MusicMap): 目标音乐谱面
+ mss (MusicMap): 待融合的mss音乐谱面
+
+ Returns:
+ MusicMap: 融合后的音乐谱面
+ """
+ musicinfo.meta_info.bpm = mss.meta_info.bpm
+ if len(mss.clipseq) > 0:
+ musicinfo.clipseq = insert_mss_2_clipseq(musicinfo.clipseq, mss.clipseq)
+ return musicinfo
+
+
+def generate_mss_from_lyric(lyrics: list, audio_duration: float, th=8) -> MusicClipSeq:
+ # "intro", "VA", "CA", "bridge", "VB", "CB", "end"]
+ mss = []
+ n_lyric = len(lyrics)
+ for lyric_idx, line_lyric_dct in enumerate(lyrics):
+ time_start = line_lyric_dct["time_start"]
+ duration = line_lyric_dct["duration"]
+ time_end = time_start + duration
+ # text = line_lyric_dct["text"]
+ if lyric_idx == 0:
+ sub_mss = {
+ "stage": "intro",
+ "time_start": 0,
+ "duration": time_start,
+ }
+ mss.append(sub_mss)
+ continue
+ if lyric_idx == n_lyric - 1:
+ sub_mss = {
+ "stage": "end",
+ "time_start": time_end,
+ "duration": audio_duration - time_end,
+ }
+ mss.append(sub_mss)
+ continue
+
+ if lyrics[lyric_idx + 1]["time_start"] - time_end >= th:
+ sub_mss = {
+ "stage": "bridge",
+ "time_start": time_end,
+ "duration": lyrics[lyric_idx + 1]["time_start"] - time_end,
+ }
+ mss.append(sub_mss)
+ mss_lyric = []
+ for sub_idx, sub_mss in enumerate(mss):
+ if sub_idx == len(mss) - 1:
+ continue
+ time_end = sub_mss["time_start"] + sub_mss["duration"]
+ next_time_start = mss[sub_idx + 1]["time_start"]
+ if next_time_start - time_end > 0.1:
+ mss_lyric.append(
+ {
+ "stage": "lyric",
+ "time_start": time_end,
+ "duration": next_time_start - time_end,
+ }
+ )
+ mss.extend(mss_lyric)
+ mss = sorted(mss, key=lambda x: x["time_start"])
+ mss = MusicClipSeq(mss)
+ return mss
+
+
+def refine_mss_info_from_tianqin(
+ mss_info: MusicMap, lyricseq: MusicClipSeq
+) -> MusicMap:
+ """优化天琴的歌曲结信息,
+ 优化前：天琴歌曲结构里面只有每句歌词和结构信息，时间前后不连续，对于整首歌去时间结构不完备。
+ 优化后：增加intro,bridge,end，将相近的结构信息合并，时间前后连续，时间完备
+
+ Args:
+ mss_info (MusicMap): 天琴歌曲结构
+ lyricseq (ClipSeq): 原始歌曲信息，用于计算Intro,bridge,end。其实也可以从mss_info中获取。
+
+ Returns:
+ MusicMap: 优化后的歌曲结构信息
+ """
+ lyric_mss_clipseq = generate_mss_from_lyric(
+ lyricseq, audio_duration=mss_info.meta_info.duration
+ )
+ new_mss_clipseq = []
+ # lyric_mss_dct = lyric_mss_clipseq.to_dct()
+ # mss_dct = mss_info.clipseq.to_dct()
+ for l_clip_idx, lyric_clip in enumerate(lyric_mss_clipseq):
+ if lyric_clip.stage != "lyric":
+ new_mss_clipseq.append(lyric_clip)
+ else:
+ new_clip_time_start = lyric_clip.time_start
+ last_stage = "ANewClipStart"
+ for clip_idx, clip in enumerate(mss_info.clipseq):
+ if clip.time_start < new_clip_time_start:
+ continue
+ if (
+ clip.time_start >= lyric_mss_clipseq[l_clip_idx + 1].time_start
+ or clip_idx == len(mss_info.clipseq) - 1
+ ):
+ if clip.time_start >= lyric_mss_clipseq[l_clip_idx + 1].time_start:
+ stage = last_stage
+ # 像偶阵雨这首歌最后一个歌词段落 只有一句歌词
+ if clip_idx == len(mss_info.clipseq) - 1:
+ stage = clip.stage
+ new_clip_time_end = lyric_mss_clipseq[l_clip_idx + 1].time_start
+ new_stage_clip = {
+ "time_start": new_clip_time_start,
+ "duration": new_clip_time_end - new_clip_time_start,
+ "stage": stage,
+ }
+ new_mss_clipseq.append(MusicClip(**new_stage_clip))
+ new_clip_time_start = new_clip_time_end
+ last_stage = clip.stage
+ break
+ if clip.stage != last_stage:
+ if last_stage == "ANewClipStart":
+ last_stage = clip.stage
+ continue
+ new_clip_time_end = mss_info.clipseq[clip_idx].time_start
+ new_stage_clip = {
+ "time_start": new_clip_time_start,
+ "duration": new_clip_time_end - new_clip_time_start,
+ "stage": last_stage,
+ }
+ new_mss_clipseq.append(MusicClip(**new_stage_clip))
+ new_clip_time_start = new_clip_time_end
+ last_stage = clip.stage
+ new_mss_clipseq = MusicClipSeq(sorted(new_mss_clipseq, key=lambda x: x.time_start))
+ mss_info.clipseq = new_mss_clipseq
+ return mss_info

MuseV/MMCM/mmcm/music/music_map/music_clip.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+from typing import Dict, List
+
+from ...data.clip import Clip, ClipSeq
+
+
+class MusicClip(Clip):
+ def __init__(self, time_start: float, duration: float, clipid: int = None, media_type: str = None, mediaid: str = None, timepoint_type: str = None, text: str = None, stage: str = None, path: str = None, duration_num: int = None, similar_clipseq: MatchedClipIds = None, dynamic: float = None, **kwargs):
+ super().__init__(time_start, duration, clipid, media_type, mediaid, timepoint_type, text, stage, path, duration_num, similar_clipseq, dynamic, **kwargs)
+
+ @property
+ def text_num(self):
+ return self._cal_text_num()
+
+ @property
+ def original_text_num(self):
+ return self._cal_text_num(text_mode=1)
+
+ def _cal_text_num(self, text_mode: int = 0) -> int:
+ """计算 文本 字的数量
+
+ Args:
+ text_mode (int, optional): 0选text， 其他选original_text. Defaults to 0.
+
+ Returns:
+ int: _description_
+ """
+ if text_mode == 0:
+ text = self.text
+ else:
+ text = self.original_text
+ if text is None:
+ n_text = 0
+ else:
+ text = text.strip().split(" ")
+ n_text = len(text)
+ return n_text
+
+ @property
+ def text_num_per_second(self):
+ """单位时间内的text数量"""
+ return self._cal_text_num_per_second(mode=0)
+
+ @property
+ def original_text_num_per_second(self):
+ """单位时间内的original_text数量"""
+ return self._cal_text_num_per_second(mode=1)
+
+ @property
+ def tnps(self):
+ """单位时间内的text数量"""
+ return self.text_num_per_second
+
+ @property
+ def original_tnps(self):
+ """单位时间内的original_text数量"""
+ return self.original_text_num_per_second
+
+ def _cal_text_num_per_second(self, mode=0):
+ """计算单位时间内的文本数量"""
+ text_num = self.text_num if mode == 0 else self.original_text_num
+ return text_num / self.duration
+
+ @classmethod
+ def from_data(cls, data: Dict):
+ return MusicClip(**data)
+
+
+class MusicClipSeq(ClipSeq):
+
+ def __init__(self, items: List[Clip] = None):
+ super().__init__(items)
+ self.clipseq = self.data
+
+ @classmethod
+ def from_data(cls, clipseq: List[Dict]) -> MusicClipSeq:
+ new_clipseq = []
+ for clip in clipseq:
+ video_clip = MusicClip.from_data(clip)
+ new_clipseq.append(video_clip)
+ video_clipseq = MusicClipSeq(new_clipseq)
+ return video_clipseq
+ 

MuseV/MMCM/mmcm/music/music_map/music_map.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+from typing import List, Dict
+
+from moviepy.editor import concatenate_audioclips, AudioClip, AudioFileClip
+
+from ...data import MediaMap, MediaMapEmb, MetaInfo, MediaMapSeq
+from ...data.clip.clip_process import find_time_by_stage
+from ...data.emb.h5py_emb import H5pyMediaMapEmb
+from ...utils.util import load_dct_from_file
+
+from .clip_process import get_stageseq_from_clipseq
+from .music_clip import MusicClip, MusicClipSeq
+from .meta_info import MusicMetaInfo
+
+
+class MusicMap(MediaMap):
+ def __init__(
+ self,
+ meta_info: MetaInfo,
+ clipseq: MusicClipSeq,
+ lyricseq: MusicClipSeq = None,
+ stageseq: MusicClipSeq = None,
+ frameseq: MusicClipSeq = None,
+ emb: MediaMapEmb = None,
+ **kwargs,
+ ):
+ self.lyricseq = lyricseq
+ super().__init__(meta_info, clipseq, stageseq, frameseq, emb, **kwargs)
+ if self.stageseq is None:
+ self.stageseq = MusicClipSeq.from_data(
+ get_stageseq_from_clipseq(self.clipseq)
+ )
+ self.stageseq.preprocess()
+
+ def preprocess(self):
+ if (
+ hasattr(self.meta_info, "target_stages")
+ and self.meta_info.target_stages is not None
+ ):
+ self.set_start_end_by_target_stages()
+ super().preprocess()
+ self.spread_metainfo_2_clip(
+ target_keys=[
+ "media_path",
+ "media_map_path",
+ "emb_path",
+ "media_duration",
+ "mediaid",
+ "media_name",
+ "emb",
+ ]
+ )
+
+ def set_start_end_by_target_stages(self):
+ target_stages = self.meta_info.target_stages
+ if not isinstance(target_stages, List):
+ target_stages = [target_stages]
+ start, _ = find_time_by_stage(self.stageseq, target_stages[0])
+ _, end = find_time_by_stage(self.stageseq, target_stages[-1])
+ self.meta_info.start = start
+ self.meta_info.end = end
+
+ @property
+ def audio_clip(self) -> AudioFileClip:
+ """读取实际ClipSeq中的音频
+
+ Returns:
+ AudioClip: Moviepy中的audio_clip
+ """
+ audio_clip = AudioFileClip(self.meta_info.media_path)
+ audio_clip = audio_clip.subclip(self.meta_info.start, self.meta_info.end)
+ return audio_clip
+
+ @classmethod
+ def from_json_path(
+ cls, path: Dict, emb_path: str, media_path: str = None, **kwargs
+ ) -> MusicMap:
+ media_map = load_dct_from_file(path)
+ emb = H5pyMediaMapEmb(emb_path)
+ return cls.from_data(media_map, emb=emb, media_path=media_path, **kwargs)
+
+ @classmethod
+ def from_data(
+ cls, data: Dict, emb: H5pyMediaMapEmb, media_path: str = None, **kwargs
+ ) -> MusicMap:
+ meta_info = MusicMetaInfo.from_data(data.get("meta_info", {}))
+ meta_info.media_path = media_path
+ clipseq = MusicClipSeq.from_data(data.get("clipseq", []))
+ stageseq = MusicClipSeq.from_data(data.get("stageseq", []))
+ lyricseq = MusicClipSeq.from_data(data.get("lyricseq", []))
+ target_keys = ["meta_info", "clipseq", "frameseq", "stageseq", "lyricseq"]
+ dct = {k: data[k] for k in data.keys() if k not in target_keys}
+ dct.update(**kwargs)
+ video_map = MusicMap(
+ meta_info=meta_info,
+ clipseq=clipseq,
+ stageseq=stageseq,
+ lyricseq=lyricseq,
+ emb=emb,
+ **dct,
+ )
+ return video_map
+
+ def to_dct(
+ self, target_keys: List[str] = None, ignored_keys: List[str] = None
+ ) -> Dict:
+ dct = {}
+ dct["meta_info"] = self.meta_info.to_dct(
+ target_keys=target_keys, ignored_keys=ignored_keys
+ )
+ dct["clipseq"] = self.clipseq.to_dct(
+ target_keys=target_keys, ignored_keys=ignored_keys
+ )
+ if self.frameseq is not None:
+ dct["frameseq"] = self.frameseq.to_dct(
+ target_keys=target_keys, ignored_keys=ignored_keys
+ )
+ else:
+ dct["frameseq"] = None
+ if self.stageseq is not None:
+ dct["stageseq"] = self.stageseq.to_dct(
+ target_keys=target_keys, ignored_keys=ignored_keys
+ )
+ else:
+ dct["stageseq"] = None
+ dct["lyricseq"] = self.lyricseq.to_dct(
+ target_keys=target_keys, ignored_keys=ignored_keys
+ )
+ return dct
+
+
+class MusicMapSeq(MediaMapSeq):
+ def __init__(self, maps: List[MusicMap]) -> None:
+ super().__init__(maps)
+
+ @property
+ def audio_clip(self) -> AudioFileClip:
+ audio_clip_lst = [m.audi_clip for m in self.maps]
+ audio_clip = concatenate_audioclips(audio_clip_lst)
+ return audio_clip

MuseV/MMCM/mmcm/music/music_map/music_map_demp.py

@@ -0,0 +1,58 @@
+from moviepy.editor import (
+ ColorClip,
+ concatenate_videoclips,
+ AudioFileClip,
+ CompositeVideoClip,
+)
+
+from ...vision.video_map.video_lyric import render_lyric2video
+from ...vision.video_map.video_writer import write_videoclip
+from .music_map import MusicMap
+
+
+def generate_music_map_videodemo(
+ music_map: MusicMap,
+ path: str,
+ audio_path: str,
+ render_lyric: bool = True,
+ width: int = 360,
+ height: int = 240,
+ fps: int = 25,
+ n_thread: int = 8,
+ colors: list = [[51, 161, 201], [46, 139, 87]],
+) -> None:
+ """输入音乐谱面，生成对应的转场视频Demo，视频内容只是简单的颜色切换
+
+ Args:
+ music_map (MusicInfo): 待可视化的音乐谱面
+ path (str): 可视化视频的存储路径
+ audio_path (str): 音乐谱面对应的音频路径
+ render_lyric (bool, optional): 是否渲染歌词，歌词在音乐谱面中. Defaults to True.
+ width (int, optional): 可视化视频的宽. Defaults to 360.
+ height (int, optional): 可视化视频的高. Defaults to 240.
+ fps (int, optional): 可视化视频的fps. Defaults to 25.
+ n_thread (int, optional): 可视化视频的写入线程数. Defaults to 8.
+ colors (list, optional): 可视化的视频颜色. Defaults to [[51, 161, 201], [46, 139, 87]].
+ """
+ audio_clip = AudioFileClip(audio_path)
+ video_clips = []
+ size = (width, height)
+ for i, clip in enumerate(music_map.clipseq):
+ clip = ColorClip(
+ size=size, color=colors[i % len(colors)], duration=clip.duration
+ )
+ video_clips.append(clip)
+ video_clips = concatenate_videoclips(video_clips, method="compose")
+ if render_lyric:
+ video_clips = render_lyric2video(
+ videoclip=video_clips,
+ lyric=music_map,
+ lyric_info_type="music_map",
+ )
+ video_clips = video_clips.set_audio(audio_clip)
+ write_videoclip(
+ video_clips,
+ path=path,
+ fps=fps,
+ n_thread=n_thread,
+ )

MuseV/MMCM/mmcm/music/utils/path_util.py

@@ -0,0 +1,9 @@
+import os
+from typing import Dict, Tuple
+
+from ...utils.path_util import get_dir_file_map
+
+
+def get_audio_path_dct(path, exts=["mp3", "flac", "wav"]) -> Dict[str, str]:
+ """遍历目标文件夹及子文件夹下所有音频文件，生成字典。"""
+ return get_dir_file_map(path, exts=exts)

MuseV/MMCM/mmcm/t2p/.gitignore

@@ -0,0 +1,158 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.vscode
+dataset/dataset_TM_train_cb1_temp.py
+train_gpt_cnn_temp.py
+train_gpt_cnn_mask.py
+start.sh
+start_eval.sh
+config.json
+output_GPT_Final
+output_vqfinal
+output_transformer
+glove
+checkpoints
+dataset/HumanML3D
+dataset/KIT-ML
+output
+matrix_multi.py
+body_models
+render_final_diffuse.py
+render_final_mdm.py
+pretrained
+MDM
+Motiondiffusion
+Visualize_temp.py
+new.sh
+T2M_render
+render_final_t2m.py
+
+pose

MuseV/MMCM/mmcm/t2p/GPT_eval_multi.py

@@ -0,0 +1,121 @@
+import os 
+import torch
+import numpy as np
+from torch.utils.tensorboard import SummaryWriter
+import json
+import clip
+
+import options.option_transformer as option_trans
+import models.vqvae as vqvae
+import utils.utils_model as utils_model
+import utils.eval_trans as eval_trans
+from dataset import dataset_TM_eval
+import models.t2m_trans as trans
+from options.get_eval_option import get_opt
+from models.evaluator_wrapper import EvaluatorModelWrapper
+import warnings
+warnings.filterwarnings('ignore')
+
+##### ---- Exp dirs ---- #####
+args = option_trans.get_args_parser()
+torch.manual_seed(args.seed)
+
+args.out_dir = os.path.join(args.out_dir, f'{args.exp_name}')
+os.makedirs(args.out_dir, exist_ok = True)
+
+##### ---- Logger ---- #####
+logger = utils_model.get_logger(args.out_dir)
+writer = SummaryWriter(args.out_dir)
+logger.info(json.dumps(vars(args), indent=4, sort_keys=True))
+
+from utils.word_vectorizer import WordVectorizer
+w_vectorizer = WordVectorizer('./glove', 'our_vab')
+val_loader = dataset_TM_eval.DATALoader(args.dataname, True, 32, w_vectorizer)
+
+dataset_opt_path = 'checkpoints/kit/Comp_v6_KLD005/opt.txt' if args.dataname == 'kit' else 'checkpoints/t2m/Comp_v6_KLD005/opt.txt'
+
+wrapper_opt = get_opt(dataset_opt_path, torch.device('cuda'))
+eval_wrapper = EvaluatorModelWrapper(wrapper_opt)
+
+##### ---- Network ---- #####
+
+## load clip model and datasets
+clip_model, clip_preprocess = clip.load("ViT-B/32", device=torch.device('cuda'), jit=False) # Must set jit=False for training
+clip.model.convert_weights(clip_model) # Actually this line is unnecessary since clip by default already on float16
+clip_model.eval()
+for p in clip_model.parameters():
+ p.requires_grad = False
+
+net = vqvae.HumanVQVAE(args, ## use args to define different parameters in different quantizers
+ args.nb_code,
+ args.code_dim,
+ args.output_emb_width,
+ args.down_t,
+ args.stride_t,
+ args.width,
+ args.depth,
+ args.dilation_growth_rate)
+
+
+trans_encoder = trans.Text2Motion_Transformer(num_vq=args.nb_code, 
+ embed_dim=args.embed_dim_gpt, 
+ clip_dim=args.clip_dim, 
+ block_size=args.block_size, 
+ num_layers=args.num_layers, 
+ n_head=args.n_head_gpt, 
+ drop_out_rate=args.drop_out_rate, 
+ fc_rate=args.ff_rate)
+
+
+print ('loading checkpoint from {}'.format(args.resume_pth))
+ckpt = torch.load(args.resume_pth, map_location='cpu')
+net.load_state_dict(ckpt['net'], strict=True)
+net.eval()
+net.cuda()
+
+if args.resume_trans is not None:
+ print ('loading transformer checkpoint from {}'.format(args.resume_trans))
+ ckpt = torch.load(args.resume_trans, map_location='cpu')
+ trans_encoder.load_state_dict(ckpt['trans'], strict=True)
+trans_encoder.train()
+trans_encoder.cuda()
+
+
+fid = []
+div = []
+top1 = []
+top2 = []
+top3 = []
+matching = []
+multi = []
+repeat_time = 20
+
+ 
+for i in range(repeat_time):
+ best_fid, best_iter, best_div, best_top1, best_top2, best_top3, best_matching, best_multi, writer, logger = eval_trans.evaluation_transformer_test(args.out_dir, val_loader, net, trans_encoder, logger, writer, 0, best_fid=1000, best_iter=0, best_div=100, best_top1=0, best_top2=0, best_top3=0, best_matching=100, best_multi=0, clip_model=clip_model, eval_wrapper=eval_wrapper, draw=False, savegif=False, save=False, savenpy=(i==0))
+ fid.append(best_fid)
+ div.append(best_div)
+ top1.append(best_top1)
+ top2.append(best_top2)
+ top3.append(best_top3)
+ matching.append(best_matching)
+ multi.append(best_multi)
+
+print('final result:')
+print('fid: ', sum(fid)/repeat_time)
+print('div: ', sum(div)/repeat_time)
+print('top1: ', sum(top1)/repeat_time)
+print('top2: ', sum(top2)/repeat_time)
+print('top3: ', sum(top3)/repeat_time)
+print('matching: ', sum(matching)/repeat_time)
+print('multi: ', sum(multi)/repeat_time)
+
+fid = np.array(fid)
+div = np.array(div)
+top1 = np.array(top1)
+top2 = np.array(top2)
+top3 = np.array(top3)
+matching = np.array(matching)
+multi = np.array(multi)
+msg_final = f"FID. {np.mean(fid):.3f}, conf. {np.std(fid)*1.96/np.sqrt(repeat_time):.3f}, Diversity. {np.mean(div):.3f}, conf. {np.std(div)*1.96/np.sqrt(repeat_time):.3f}, TOP1. {np.mean(top1):.3f}, conf. {np.std(top1)*1.96/np.sqrt(repeat_time):.3f}, TOP2. {np.mean(top2):.3f}, conf. {np.std(top2)*1.96/np.sqrt(repeat_time):.3f}, TOP3. {np.mean(top3):.3f}, conf. {np.std(top3)*1.96/np.sqrt(repeat_time):.3f}, Matching. {np.mean(matching):.3f}, conf. {np.std(matching)*1.96/np.sqrt(repeat_time):.3f}, Multi. {np.mean(multi):.3f}, conf. {np.std(multi)*1.96/np.sqrt(repeat_time):.3f}"
+logger.info(msg_final)

MuseV/MMCM/mmcm/t2p/LICENSE

@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2023 tencent
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.