feat(web): check assets integrity on start

.env

@@ -7,3 +7,43 @@ weight_uvr5_root = assets/uvr5_weights
 index_root = logs
 outside_index_root = assets/indices
 rmvpe_root = assets/rmvpe
+
+sha256_hubert_base_pt = f54b40fd2802423a5643779c4861af1e9ee9c1564dc9d32f54f20b5ffba7db96
+sha256_rmvpe_pt      = 6d62215f4306e3ca278246188607209f09af3dc77ed4232efdd069798c4ec193
+sha256_rmvpe_onnx    = 5370e71ac80af8b4b7c793d27efd51fd8bf962de3a7ede0766dac0befa3660fd
+
+sha256_v1_D32k_pth   = 2ab20645829460fdad0d3c44254f1ab53c32cae50c22a66c926ae5aa30abda6f
+sha256_v1_D40k_pth   = 547f66dbbcd9023b9051ed244d12ab043ba8a4e854b154cc28761ac7c002909b
+sha256_v1_D48k_pth   = 8cc013fa60ed9c3f902f5bd99f48c7e3b9352d763d4d3cd6bc241c37b0bfd9ad
+sha256_v1_G32k_pth   = 81817645cde7ed2e2d83f23ef883f33dda564924b497e84d792743912eca4c23
+sha256_v1_G40k_pth   = e428573bda1124b0ae0ae843fd8dcded6027d3993444790b3e9b0100938b2113
+sha256_v1_G48k_pth   = 3862a67ea6313e8ffefc05cee6bee656ef3e089442e9ecf4a6618d60721f3e95
+sha256_v1_f0D32k_pth = 294db3087236e2c75260d6179056791c9231245daf5d0485545d9e54c4057c77
+sha256_v1_f0D40k_pth = 7d4f5a441594b470d67579958b2fd4c6b992852ded28ff9e72eda67abcebe423
+sha256_v1_f0D48k_pth = 1b84c8bf347ad1e539c842e8f2a4c36ecd9e7fb23c16041189e4877e9b07925c
+sha256_v1_f0G32k_pth = 285f524bf48bb692c76ad7bd0bc654c12bd9e5edeb784dddf7f61a789a608574
+sha256_v1_f0G40k_pth = 9115654aeef1995f7dd3c6fc4140bebbef0ca9760bed798105a2380a34299831
+sha256_v1_f0G48k_pth = 78bc9cab27e34bcfc194f93029374d871d8b3e663ddedea32a9709e894cc8fe8
+
+sha256_v2_D32k_pth   = d8043378cc6619083d385f5a045de09b83fb3bf8de45c433ca863b71723ac3ca
+sha256_v2_D40k_pth   = 471378e894e7191f89a94eda8288c5947b16bbe0b10c3f1f17efdb7a1d998242
+sha256_v2_D48k_pth   = db01094a93c09868a278e03dafe8bb781bfcc1a5ba8df168c948bf9168c84d82
+sha256_v2_G32k_pth   = 869b26a47f75168d6126f64ac39e6de5247017a8658cfd68aca600f7323efb9f
+sha256_v2_G40k_pth   = a3843da7fde33db1dab176146c70d6c2df06eafe9457f4e3aa10024e9c6a4b69
+sha256_v2_G48k_pth   = 2e2b1581a436d07a76b10b9d38765f64aa02836dc65c7dee1ce4140c11ea158b
+sha256_v2_f0D32k_pth = bd7134e7793674c85474d5145d2d982e3c5d8124fc7bb6c20f710ed65808fa8a
+sha256_v2_f0D40k_pth = 6b6ab091e70801b28e3f41f335f2fc5f3f35c75b39ae2628d419644ec2b0fa09
+sha256_v2_f0D48k_pth = 2269b73c7a4cf34da09aea99274dabf99b2ddb8a42cbfb065fb3c0aa9a2fc748
+sha256_v2_f0G32k_pth = 2332611297b8d88c7436de8f17ef5f07a2119353e962cd93cda5806d59a1133d
+sha256_v2_f0G40k_pth = 3b2c44035e782c4b14ddc0bede9e2f4a724d025cd073f736d4f43708453adfcb
+sha256_v2_f0G48k_pth = b5d51f589cc3632d4eae36a315b4179397695042edc01d15312e1bddc2b764a4
+
+sha256_uvr5_HP2-人声vocals+非人声instrumentals_pth = 39796caa5db18d7f9382d8ac997ac967bfd85f7761014bb807d2543cc844ef05
+sha256_uvr5_HP2_all_vocals_pth =   39796caa5db18d7f9382d8ac997ac967bfd85f7761014bb807d2543cc844ef05
+sha256_uvr5_HP3_all_vocals_pth =   45e6b65199e781b4a6542002699be9f19cd3d1cb7d1558bc2bfbcd84674dfe28
+sha256_uvr5_HP5-主旋律人声vocals+其他instrumentals_pth = 5908891829634926119720241e8573d97cbeb8277110a7512bdb0bd7563258ee   
+sha256_uvr5_HP5_only_main_vocal_pth = 5908891829634926119720241e8573d97cbeb8277110a7512bdb0bd7563258ee
+sha256_uvr5_VR-DeEchoAggressive_pth = 8c8fd1582f9aabc363e47af62ddb88df6cae7e064cae75bbf041a067a5e0aee2
+sha256_uvr5_VR-DeEchoDeReverb_pth = 01376dd2a571bf3cb9cced680732726d2d732609d09216a610b0d110f133febe
+sha256_uvr5_VR-DeEchoNormal_pth =   56aba59db3bcdd14a14464e62f3129698ecdea62eee0f003b9360923eb3ac79e
+sha256_uvr5_vocals_onnx = 233bb5c6aaa365e568659a0a81211746fa881f8f47f82d9e864fce1f7692db80

.github/workflows/unitest.yml

@@ -19,13 +19,14 @@ jobs:
       run: |
         sudo apt update
         sudo apt -y install ffmpeg
-        sudo apt -y install -qq aria2
+        wget https://github.com/RVC-Project/RVC-Models-Downloader/releases/download/v0.2.1/rvcmd_linux_amd64.deb
-        aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/hubert_base.pt -d ./ -o hubert_base.pt
+        sudo apt -y install ./rvcmd_linux_amd64.deb
         python -m pip install --upgrade pip
         python -m pip install --upgrade setuptools
         python -m pip install --upgrade wheel
         pip install torch torchvision torchaudio
         pip install -r requirements.txt
+        rvcmd -notrs -w 1 -notui assets/all
     - name: Test step 1 & 2
       run: |
         mkdir -p logs/mi-test

configs/config.py

@@ -57,6 +57,7 @@ class Config:
             self.noparallel,
             self.noautoopen,
             self.dml,
+            self.nocheck,
         ) = self.arg_parse()
         self.instead = ""
         self.preprocess_per = 3.7
@@ -93,6 +94,9 @@ class Config:
             action="store_true",
             help="torch_dml",
         )
+        parser.add_argument(
+            "--nocheck", action="store_true", help="Run without checking assets"
+        )
         cmd_opts = parser.parse_args()
 
         cmd_opts.port = cmd_opts.port if 0 <= cmd_opts.port <= 65535 else 7865
@@ -104,6 +108,7 @@ class Config:
             cmd_opts.noparallel,
             cmd_opts.noautoopen,
             cmd_opts.dml,
+            cmd_opts.nocheck,
         )
 
     # has_mps is only available in nightly pytorch (for now) and MasOS 12.3+.

infer-web.py

@@ -13,6 +13,7 @@ from infer.lib.train.process_ckpt import (
     merge,
     show_info,
 )
+from infer.lib.rvcmd import check_all_assets, download_all_assets
 from i18n.i18n import I18nAuto
 from configs.config import Config
 from sklearn.cluster import MiniBatchKMeans
@@ -53,6 +54,11 @@ torch.manual_seed(114514)
 config = Config()
 vc = VC(config)
 
+if not config.nocheck and not check_all_assets():
+    download_all_assets(tmpdir=tmp)
+    if not check_all_assets():
+        logging.error("counld not satisfy all assets needed.")
+        exit(1)
 
 if config.dml == True:
 

infer/lib/rvcmd.py

@@ -0,0 +1,150 @@
+import os
+from pathlib import Path
+import hashlib
+import requests
+from io import BytesIO
+import logging
+
+logger = logging.getLogger(__name__)
+
+def sha256(f) -> str:
+    sha256_hash = hashlib.sha256()
+    # Read and update hash in chunks of 4M
+    for byte_block in iter(lambda: f.read(4*1024*1024), b""):
+        sha256_hash.update(byte_block)
+    return sha256_hash.hexdigest()
+
+def check_model(dir_name: Path, model_name: str, hash: str) -> bool:
+    target = dir_name / model_name
+    relname = str(target)
+    relname = relname[relname.rindex("assets/"):]
+    logger.debug(f"checking {relname}...")
+    if not os.path.exists(target):
+        logger.info(f"{target} not exist.")
+        return False
+    with open(target, "rb") as f:
+        digest = sha256(f)
+        if digest != hash:
+            logger.info(f"{target} sha256 hash mismatch.")
+            logger.info(f"expected: {hash}")
+            logger.info(f"real val: {digest}")
+            return False
+    return True
+
+def check_all_assets() -> bool:
+    BASE_DIR = Path(__file__).resolve().parent.parent.parent
+
+    logger.info("checking hubret & rmvpe...")
+
+    if not check_model(BASE_DIR / "assets/hubert", "hubert_base.pt", os.environ["sha256_hubert_base_pt"]):
+        return False
+    if not check_model(BASE_DIR / "assets/rmvpe", "rmvpe.pt", os.environ["sha256_rmvpe_pt"]):
+        return False
+    if not check_model(BASE_DIR / "assets/rmvpe", "rmvpe.onnx", os.environ["sha256_rmvpe_onnx"]):
+        return False
+
+    rvc_models_dir = BASE_DIR / "assets/pretrained"
+    logger.info("checking pretrained models...")
+    model_names = [
+        "D32k.pth",
+        "D40k.pth",
+        "D48k.pth",
+        "G32k.pth",
+        "G40k.pth",
+        "G48k.pth",
+        "f0D32k.pth",
+        "f0D40k.pth",
+        "f0D48k.pth",
+        "f0G32k.pth",
+        "f0G40k.pth",
+        "f0G48k.pth",
+    ]
+    for model in model_names:
+        menv = model.replace(".", "_")
+        if not check_model(rvc_models_dir, model, os.environ[f"sha256_v1_{menv}"]):
+            return False
+
+    rvc_models_dir = BASE_DIR / "assets/pretrained_v2"
+    logger.info("checking pretrained models v2...")
+    for model in model_names:
+        menv = model.replace(".", "_")
+        if not check_model(rvc_models_dir, model, os.environ[f"sha256_v2_{menv}"]):
+            return False
+
+    logger.info("checking uvr5_weights...")
+    rvc_models_dir = BASE_DIR / "assets/uvr5_weights"
+    model_names = [
+        "HP2-人声vocals+非人声instrumentals.pth",
+        "HP2_all_vocals.pth",
+        "HP3_all_vocals.pth",
+        "HP5-主旋律人声vocals+其他instrumentals.pth",
+        "HP5_only_main_vocal.pth",
+        "VR-DeEchoAggressive.pth",
+        "VR-DeEchoDeReverb.pth",
+        "VR-DeEchoNormal.pth",
+    ]
+    for model in model_names:
+        menv = model.replace(".", "_")
+        if not check_model(rvc_models_dir, model, os.environ[f"sha256_uvr5_{menv}"]):
+            return False
+    if not check_model(
+        BASE_DIR / "assets/uvr5_weights/onnx_dereverb_By_FoxJoy",
+        "vocals.onnx",
+        os.environ[f"sha256_uvr5_vocals_onnx"],
+    ): return False
+
+    logger.info("all assets are already latest.")
+    return True
+
+def download_and_extract_tar_gz(url: str, folder: str):
+    import tarfile
+    logger.info(f"downloading {url}")
+    response = requests.get(url, stream=True)
+    with BytesIO() as out_file:
+        out_file.write(response.content)
+        out_file.seek(0)
+        logger.info(f"downloaded.")
+        with tarfile.open(fileobj=out_file, mode="r:gz") as tar:
+            tar.extractall(folder)
+        logger.info(f"extracted into {folder}")
+
+def download_and_extract_zip(url: str, folder: str):
+    import zipfile
+    logger.info(f"downloading {url}")
+    response = requests.get(url)
+    with BytesIO() as out_file:
+        out_file.write(response.content)
+        out_file.seek(0)
+        logger.info(f"downloaded.")
+        with zipfile.ZipFile(out_file) as zip_ref:
+            zip_ref.extractall(folder)
+        logger.info(f"extracted into {folder}")
+
+def download_all_assets(tmpdir: str, version="0.2.1"):
+    import subprocess
+    import platform
+
+    archs = {
+        "aarch64": "arm64", "armv8l": "arm64", "arm64": "arm64",
+        "x86": "386", "i386": "386", "i686": "386", "386": "386",
+        "x86_64": "amd64", "x64": "amd64", "amd64": "amd64",
+    }
+    system_type = platform.system().lower()
+    architecture = platform.machine().lower()
+    is_win = architecture == "windows"
+
+    architecture = archs.get(architecture, None)
+    if not architecture:
+        logger.error(f"architecture {architecture} is not supported")
+        exit(1)
+    BASE_URL = "https://github.com/RVC-Project/RVC-Models-Downloader/releases/download/"
+    suffix = "zip" if is_win else "tar.gz"
+    RVCMD_URL = BASE_URL+f"v{version}/rvcmd_{system_type}_{architecture}.{suffix}"
+    cmdfile = tmpdir + "/rvcmd"
+    if is_win:
+        download_and_extract_zip(RVCMD_URL, tmpdir)
+        cmdfile += ".exe"
+    else:
+        download_and_extract_tar_gz(RVCMD_URL, tmpdir)
+        os.chmod(cmdfile, 0o755)
+    subprocess.run([cmdfile, "-notui", "-w", "0", "assets/all"])

run.sh

@@ -4,9 +4,6 @@ if [ "$(uname)" = "Darwin" ]; then
   # macOS specific env:
   export PYTORCH_ENABLE_MPS_FALLBACK=1
   export PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0
-elif [ "$(uname)" != "Linux" ]; then
-  echo "Unsupported operating system."
-  exit 1
 fi
 
 if [ -d ".venv" ]; then
@@ -22,8 +19,8 @@ else
     if [ "$(uname)" = "Darwin" ] && command -v brew >/dev/null 2>&1; then
       brew install python@3.8
     elif [ "$(uname)" = "Linux" ] && command -v apt-get >/dev/null 2>&1; then
-      sudo apt-get update
+      sudo apt update
-      sudo apt-get install python3.8
+      sudo apt install -y python3.8
     else
       echo "Please install Python 3.8 manually."
       exit 1