OPENGIT
/
Retrieval-based-Voice-Conversion-WebUI
mirror of https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(web): check assets integrity on start

This commit is contained in:
源文雨 2024-04-21 00:16:10 +09:00
parent a30d54f61c
commit 4ed830f785
6 changed files with 206 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
.env
View File

@ -7,3 +7,43 @@ weight_uvr5_root = assets/uvr5_weights
index_root = logs
outside_index_root = assets/indices
rmvpe_root = assets/rmvpe
sha256_hubert_base_pt = f54b40fd2802423a5643779c4861af1e9ee9c1564dc9d32f54f20b5ffba7db96
sha256_rmvpe_pt = 6d62215f4306e3ca278246188607209f09af3dc77ed4232efdd069798c4ec193
sha256_rmvpe_onnx = 5370e71ac80af8b4b7c793d27efd51fd8bf962de3a7ede0766dac0befa3660fd
sha256_v1_D32k_pth = 2ab20645829460fdad0d3c44254f1ab53c32cae50c22a66c926ae5aa30abda6f
sha256_v1_D40k_pth = 547f66dbbcd9023b9051ed244d12ab043ba8a4e854b154cc28761ac7c002909b
sha256_v1_D48k_pth = 8cc013fa60ed9c3f902f5bd99f48c7e3b9352d763d4d3cd6bc241c37b0bfd9ad
sha256_v1_G32k_pth = 81817645cde7ed2e2d83f23ef883f33dda564924b497e84d792743912eca4c23
sha256_v1_G40k_pth = e428573bda1124b0ae0ae843fd8dcded6027d3993444790b3e9b0100938b2113
sha256_v1_G48k_pth = 3862a67ea6313e8ffefc05cee6bee656ef3e089442e9ecf4a6618d60721f3e95
sha256_v1_f0D32k_pth = 294db3087236e2c75260d6179056791c9231245daf5d0485545d9e54c4057c77
sha256_v1_f0D40k_pth = 7d4f5a441594b470d67579958b2fd4c6b992852ded28ff9e72eda67abcebe423
sha256_v1_f0D48k_pth = 1b84c8bf347ad1e539c842e8f2a4c36ecd9e7fb23c16041189e4877e9b07925c
sha256_v1_f0G32k_pth = 285f524bf48bb692c76ad7bd0bc654c12bd9e5edeb784dddf7f61a789a608574
sha256_v1_f0G40k_pth = 9115654aeef1995f7dd3c6fc4140bebbef0ca9760bed798105a2380a34299831
sha256_v1_f0G48k_pth = 78bc9cab27e34bcfc194f93029374d871d8b3e663ddedea32a9709e894cc8fe8
sha256_v2_D32k_pth = d8043378cc6619083d385f5a045de09b83fb3bf8de45c433ca863b71723ac3ca
sha256_v2_D40k_pth = 471378e894e7191f89a94eda8288c5947b16bbe0b10c3f1f17efdb7a1d998242
sha256_v2_D48k_pth = db01094a93c09868a278e03dafe8bb781bfcc1a5ba8df168c948bf9168c84d82
sha256_v2_G32k_pth = 869b26a47f75168d6126f64ac39e6de5247017a8658cfd68aca600f7323efb9f
sha256_v2_G40k_pth = a3843da7fde33db1dab176146c70d6c2df06eafe9457f4e3aa10024e9c6a4b69
sha256_v2_G48k_pth = 2e2b1581a436d07a76b10b9d38765f64aa02836dc65c7dee1ce4140c11ea158b
sha256_v2_f0D32k_pth = bd7134e7793674c85474d5145d2d982e3c5d8124fc7bb6c20f710ed65808fa8a
sha256_v2_f0D40k_pth = 6b6ab091e70801b28e3f41f335f2fc5f3f35c75b39ae2628d419644ec2b0fa09
sha256_v2_f0D48k_pth = 2269b73c7a4cf34da09aea99274dabf99b2ddb8a42cbfb065fb3c0aa9a2fc748
sha256_v2_f0G32k_pth = 2332611297b8d88c7436de8f17ef5f07a2119353e962cd93cda5806d59a1133d
sha256_v2_f0G40k_pth = 3b2c44035e782c4b14ddc0bede9e2f4a724d025cd073f736d4f43708453adfcb
sha256_v2_f0G48k_pth = b5d51f589cc3632d4eae36a315b4179397695042edc01d15312e1bddc2b764a4
sha256_uvr5_HP2-人声vocals+非人声instrumentals_pth = 39796caa5db18d7f9382d8ac997ac967bfd85f7761014bb807d2543cc844ef05
sha256_uvr5_HP2_all_vocals_pth = 39796caa5db18d7f9382d8ac997ac967bfd85f7761014bb807d2543cc844ef05
sha256_uvr5_HP3_all_vocals_pth = 45e6b65199e781b4a6542002699be9f19cd3d1cb7d1558bc2bfbcd84674dfe28
sha256_uvr5_HP5-主旋律人声vocals+其他instrumentals_pth = 5908891829634926119720241e8573d97cbeb8277110a7512bdb0bd7563258ee
sha256_uvr5_HP5_only_main_vocal_pth = 5908891829634926119720241e8573d97cbeb8277110a7512bdb0bd7563258ee
sha256_uvr5_VR-DeEchoAggressive_pth = 8c8fd1582f9aabc363e47af62ddb88df6cae7e064cae75bbf041a067a5e0aee2
sha256_uvr5_VR-DeEchoDeReverb_pth = 01376dd2a571bf3cb9cced680732726d2d732609d09216a610b0d110f133febe
sha256_uvr5_VR-DeEchoNormal_pth = 56aba59db3bcdd14a14464e62f3129698ecdea62eee0f003b9360923eb3ac79e
sha256_uvr5_vocals_onnx = 233bb5c6aaa365e568659a0a81211746fa881f8f47f82d9e864fce1f7692db80

5
.github/workflows/unitest.yml vendored
View File

@ -19,13 +19,14 @@ jobs:
run: |
sudo apt update
sudo apt -y install ffmpeg
sudo apt -y install -qq aria2
aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/hubert_base.pt -d ./ -o hubert_base.pt
wget https://github.com/RVC-Project/RVC-Models-Downloader/releases/download/v0.2.1/rvcmd_linux_amd64.deb
sudo apt -y install ./rvcmd_linux_amd64.deb
python -m pip install --upgrade pip
python -m pip install --upgrade setuptools
python -m pip install --upgrade wheel
pip install torch torchvision torchaudio
pip install -r requirements.txt
rvcmd -notrs -w 1 -notui assets/all
- name: Test step 1 & 2
run: |
mkdir -p logs/mi-test

5
configs/config.py
View File

@ -57,6 +57,7 @@ class Config:
self.noparallel,
self.noautoopen,
self.dml,
self.nocheck,
) = self.arg_parse()
self.instead = ""
self.preprocess_per = 3.7
@ -93,6 +94,9 @@ class Config:
action="store_true",
help="torch_dml",
)
parser.add_argument(
"--nocheck", action="store_true", help="Run without checking assets"
)
cmd_opts = parser.parse_args()
cmd_opts.port = cmd_opts.port if 0 <= cmd_opts.port <= 65535 else 7865
@ -104,6 +108,7 @@ class Config:
cmd_opts.noparallel,
cmd_opts.noautoopen,
cmd_opts.dml,
cmd_opts.nocheck,
)
# has_mps is only available in nightly pytorch (for now) and MasOS 12.3+.

6
infer-web.py
View File

@ -13,6 +13,7 @@ from infer.lib.train.process_ckpt import (
merge,
show_info,
)
from infer.lib.rvcmd import check_all_assets, download_all_assets
from i18n.i18n import I18nAuto
from configs.config import Config
from sklearn.cluster import MiniBatchKMeans
@ -53,6 +54,11 @@ torch.manual_seed(114514)
config = Config()
vc = VC(config)
if not config.nocheck and not check_all_assets():
download_all_assets(tmpdir=tmp)
if not check_all_assets():
logging.error("counld not satisfy all assets needed.")
exit(1)
if config.dml == True:

150
infer/lib/rvcmd.py Normal file
View File

@ -0,0 +1,150 @@
import os
from pathlib import Path
import hashlib
import requests
from io import BytesIO
import logging
logger = logging.getLogger(__name__)
def sha256(f) -> str:
sha256_hash = hashlib.sha256()
# Read and update hash in chunks of 4M
for byte_block in iter(lambda: f.read(4*1024*1024), b""):
sha256_hash.update(byte_block)
return sha256_hash.hexdigest()
def check_model(dir_name: Path, model_name: str, hash: str) -> bool:
target = dir_name / model_name
relname = str(target)
relname = relname[relname.rindex("assets/"):]
logger.debug(f"checking {relname}...")
if not os.path.exists(target):
logger.info(f"{target} not exist.")
return False
with open(target, "rb") as f:
digest = sha256(f)
if digest != hash:
logger.info(f"{target} sha256 hash mismatch.")
logger.info(f"expected: {hash}")
logger.info(f"real val: {digest}")
return False
return True
def check_all_assets() -> bool:
BASE_DIR = Path(__file__).resolve().parent.parent.parent
logger.info("checking hubret & rmvpe...")
if not check_model(BASE_DIR / "assets/hubert", "hubert_base.pt", os.environ["sha256_hubert_base_pt"]):
return False
if not check_model(BASE_DIR / "assets/rmvpe", "rmvpe.pt", os.environ["sha256_rmvpe_pt"]):
return False
if not check_model(BASE_DIR / "assets/rmvpe", "rmvpe.onnx", os.environ["sha256_rmvpe_onnx"]):
return False
rvc_models_dir = BASE_DIR / "assets/pretrained"
logger.info("checking pretrained models...")
model_names = [
"D32k.pth",
"D40k.pth",
"D48k.pth",
"G32k.pth",
"G40k.pth",
"G48k.pth",
"f0D32k.pth",
"f0D40k.pth",
"f0D48k.pth",
"f0G32k.pth",
"f0G40k.pth",
"f0G48k.pth",
]
for model in model_names:
menv = model.replace(".", "_")
if not check_model(rvc_models_dir, model, os.environ[f"sha256_v1_{menv}"]):
return False
rvc_models_dir = BASE_DIR / "assets/pretrained_v2"
logger.info("checking pretrained models v2...")
for model in model_names:
menv = model.replace(".", "_")
if not check_model(rvc_models_dir, model, os.environ[f"sha256_v2_{menv}"]):
return False
logger.info("checking uvr5_weights...")
rvc_models_dir = BASE_DIR / "assets/uvr5_weights"
model_names = [
"HP2-人声vocals+非人声instrumentals.pth",
"HP2_all_vocals.pth",
"HP3_all_vocals.pth",
"HP5-主旋律人声vocals+其他instrumentals.pth",
"HP5_only_main_vocal.pth",
"VR-DeEchoAggressive.pth",
"VR-DeEchoDeReverb.pth",
"VR-DeEchoNormal.pth",
]
for model in model_names:
menv = model.replace(".", "_")
if not check_model(rvc_models_dir, model, os.environ[f"sha256_uvr5_{menv}"]):
return False
if not check_model(
BASE_DIR / "assets/uvr5_weights/onnx_dereverb_By_FoxJoy",
"vocals.onnx",
os.environ[f"sha256_uvr5_vocals_onnx"],
): return False
logger.info("all assets are already latest.")
return True
def download_and_extract_tar_gz(url: str, folder: str):
import tarfile
logger.info(f"downloading {url}")
response = requests.get(url, stream=True)
with BytesIO() as out_file:
out_file.write(response.content)
out_file.seek(0)
logger.info(f"downloaded.")
with tarfile.open(fileobj=out_file, mode="r:gz") as tar:
tar.extractall(folder)
logger.info(f"extracted into {folder}")
def download_and_extract_zip(url: str, folder: str):
import zipfile
logger.info(f"downloading {url}")
response = requests.get(url)
with BytesIO() as out_file:
out_file.write(response.content)
out_file.seek(0)
logger.info(f"downloaded.")
with zipfile.ZipFile(out_file) as zip_ref:
zip_ref.extractall(folder)
logger.info(f"extracted into {folder}")
def download_all_assets(tmpdir: str, version="0.2.1"):
import subprocess
import platform
archs = {
"aarch64": "arm64", "armv8l": "arm64", "arm64": "arm64",
"x86": "386", "i386": "386", "i686": "386", "386": "386",
"x86_64": "amd64", "x64": "amd64", "amd64": "amd64",
}
system_type = platform.system().lower()
architecture = platform.machine().lower()
is_win = architecture == "windows"
architecture = archs.get(architecture, None)
if not architecture:
logger.error(f"architecture {architecture} is not supported")
exit(1)
BASE_URL = "https://github.com/RVC-Project/RVC-Models-Downloader/releases/download/"
suffix = "zip" if is_win else "tar.gz"
RVCMD_URL = BASE_URL+f"v{version}/rvcmd_{system_type}_{architecture}.{suffix}"
cmdfile = tmpdir + "/rvcmd"
if is_win:
download_and_extract_zip(RVCMD_URL, tmpdir)
cmdfile += ".exe"
else:
download_and_extract_tar_gz(RVCMD_URL, tmpdir)
os.chmod(cmdfile, 0o755)
subprocess.run([cmdfile, "-notui", "-w", "0", "assets/all"])

7
run.sh
View File

@ -4,9 +4,6 @@ if [ "$(uname)" = "Darwin" ]; then
# macOS specific env:
export PYTORCH_ENABLE_MPS_FALLBACK=1
export PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0
elif [ "$(uname)" != "Linux" ]; then
echo "Unsupported operating system."
exit 1
fi
if [ -d ".venv" ]; then
@ -22,8 +19,8 @@ else
if [ "$(uname)" = "Darwin" ] && command -v brew >/dev/null 2>&1; then
brew install python@3.8
elif [ "$(uname)" = "Linux" ] && command -v apt-get >/dev/null 2>&1; then
sudo apt-get update
sudo apt-get install python3.8
sudo apt update
sudo apt install -y python3.8
else
echo "Please install Python 3.8 manually."
exit 1