add formant shift for realtime-gui

configs/config.json

@@ -1 +1 @@
-{"pth_path": "assets/weights/kikiV1.pth", "index_path": "logs/kikiV1.index", "sg_hostapi": "MME", "sg_wasapi_exclusive": false, "sg_input_device": "VoiceMeeter Output (VB-Audio Vo", "sg_output_device": "VoiceMeeter Input (VB-Audio Voi", "sr_type": "sr_device", "threhold": -60.0, "pitch": 12.0, "rms_mix_rate": 0.5, "index_rate": 0.0, "block_time": 0.15, "crossfade_length": 0.08, "extra_time": 2.0, "n_cpu": 4.0, "use_jit": false, "use_pv": false, "f0method": "fcpe"}
+{"pth_path": "assets/weights/kikiV1.pth", "index_path": "logs/kikiV1.index", "sg_hostapi": "MME", "sg_wasapi_exclusive": false, "sg_input_device": "VoiceMeeter Output (VB-Audio Vo", "sg_output_device": "VoiceMeeter Input (VB-Audio Voi", "sr_type": "sr_device", "threhold": -60.0, "pitch": 12.0, "formant": 0.0, "rms_mix_rate": 0.5, "index_rate": 0.0, "block_time": 0.15, "crossfade_length": 0.08, "extra_time": 2.0, "n_cpu": 4.0, "use_jit": false, "use_pv": false, "f0method": "fcpe"}

gui_v1.py

@@ -114,6 +114,7 @@ if __name__ == "__main__":
             self.pth_path: str = ""
             self.index_path: str = ""
             self.pitch: int = 0
+            self.formant: float = 0.0
             self.sr_type: str = "sr_model"
             self.block_time: float = 0.25  # s
             self.threhold: int = -60
@@ -212,6 +213,7 @@ if __name__ == "__main__":
                         "sr_type": "sr_model",
                         "threhold": -60,
                         "pitch": 0,
+                        "formant": 0.0,
                         "index_rate": 0,
                         "rms_mix_rate": 0,
                         "block_time": 0.25,
@@ -353,6 +355,17 @@ if __name__ == "__main__":
                                     default_value=data.get("pitch", 0),
                                     enable_events=True,
                                 ),
+                            ],
+                                                        [
+                                sg.Text(i18n("共振偏移")),
+                                sg.Slider(
+                                    range=(-5, 5),
+                                    key="formant",
+                                    resolution=0.01,
+                                    orientation="h",
+                                    default_value=data.get("formant", 0.0),
+                                    enable_events=True,
+                                ),
                             ],
                             [
                                 sg.Text(i18n("Index Rate")),
@@ -579,6 +592,7 @@ if __name__ == "__main__":
                             ],
                             "threhold": values["threhold"],
                             "pitch": values["pitch"],
+                            "formant": values["formant"],
                             "rms_mix_rate": values["rms_mix_rate"],
                             "index_rate": values["index_rate"],
                             # "device_latency": values["device_latency"],
@@ -621,6 +635,10 @@ if __name__ == "__main__":
                     self.gui_config.pitch = values["pitch"]
                     if hasattr(self, "rvc"):
                         self.rvc.change_key(values["pitch"])
+                elif event == "formant":
+                    self.gui_config.formant = values["formant"]
+                    if hasattr(self, "rvc"):
+                        self.rvc.change_formant(values["formant"])
                 elif event == "index_rate":
                     self.gui_config.index_rate = values["index_rate"]
                     if hasattr(self, "rvc"):
@@ -679,6 +697,7 @@ if __name__ == "__main__":
             ]
             self.gui_config.threhold = values["threhold"]
             self.gui_config.pitch = values["pitch"]
+            self.gui_config.formant = values["formant"]
             self.gui_config.block_time = values["block_time"]
             self.gui_config.crossfade_time = values["crossfade_length"]
             self.gui_config.extra_time = values["extra_time"]
@@ -703,6 +722,7 @@ if __name__ == "__main__":
             torch.cuda.empty_cache()
             self.rvc = rtrvc.RVC(
                 self.gui_config.pitch,
+                self.gui_config.formant,
                 self.gui_config.pth_path,
                 self.gui_config.index_path,
                 self.gui_config.index_rate,

infer/lib/infer_pack/models.py

@@ -10,7 +10,6 @@ from torch import nn
 from torch.nn import AvgPool1d, Conv1d, Conv2d, ConvTranspose1d
 from torch.nn import functional as F
 from torch.nn.utils import remove_weight_norm, spectral_norm, weight_norm
-
 from infer.lib.infer_pack import attentions, commons, modules
 from infer.lib.infer_pack.commons import get_padding, init_weights
 
@@ -250,7 +249,12 @@ class Generator(torch.nn.Module):
         if gin_channels != 0:
             self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
 
-    def forward(self, x: torch.Tensor, g: Optional[torch.Tensor] = None):
+    def forward(self, x: torch.Tensor, g: Optional[torch.Tensor] = None, n_res: Optional[torch.Tensor] = None):
+        if n_res is not None:
+            assert isinstance(n_res, torch.Tensor)
+            n = int(n_res.item())
+            if n != x.shape[-1]:
+                x = F.interpolate(x, size=n, mode='linear')
         x = self.conv_pre(x)
         if g is not None:
             x = x + self.cond(g)
@@ -529,9 +533,16 @@ class GeneratorNSF(torch.nn.Module):
 
         self.lrelu_slope = modules.LRELU_SLOPE
                 
-    def forward(self, x, f0, g: Optional[torch.Tensor] = None):
+    def forward(self, x, f0, g: Optional[torch.Tensor] = None, n_res: Optional[torch.Tensor] = None):
         har_source, noi_source, uv = self.m_source(f0, self.upp)
         har_source = har_source.transpose(1, 2)
+        if n_res is not None:
+            assert isinstance(n_res, torch.Tensor)
+            n = int(n_res.item())
+            if n * self.upp != har_source.shape[-1]:
+                har_source = F.interpolate(har_source, size=n*self.upp, mode='linear')
+            if n != x.shape[-1]:
+                x = F.interpolate(x, size=n, mode='linear')
         x = self.conv_pre(x)
         if g is not None:
             x = x + self.cond(g)
@@ -558,6 +569,7 @@ class GeneratorNSF(torch.nn.Module):
         x = F.leaky_relu(x)
         x = self.conv_post(x)
         x = torch.tanh(x)
+
         return x
 
     def remove_weight_norm(self):
@@ -748,6 +760,7 @@ class SynthesizerTrnMs256NSFsid(nn.Module):
         sid: torch.Tensor,
         skip_head: Optional[torch.Tensor] = None,
         return_length: Optional[torch.Tensor] = None,
+        return_length2: Optional[torch.Tensor] = None,
     ):
         g = self.emb_g(sid).unsqueeze(-1)
         if skip_head is not None and return_length is not None:
@@ -767,7 +780,7 @@ class SynthesizerTrnMs256NSFsid(nn.Module):
             m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)
             z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask
             z = self.flow(z_p, x_mask, g=g, reverse=True)
-        o = self.dec(z * x_mask, nsff0, g=g)
+        o = self.dec(z * x_mask, nsff0, g=g, n_res=return_length2)
         return o, x_mask, (z, z_p, m_p, logs_p)
 
 
@@ -963,6 +976,7 @@ class SynthesizerTrnMs256NSFsid_nono(nn.Module):
         sid: torch.Tensor,
         skip_head: Optional[torch.Tensor] = None,
         return_length: Optional[torch.Tensor] = None,
+        return_length2: Optional[torch.Tensor] = None,
     ):
         g = self.emb_g(sid).unsqueeze(-1)
         if skip_head is not None and return_length is not None:
@@ -981,7 +995,7 @@ class SynthesizerTrnMs256NSFsid_nono(nn.Module):
             m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths)
             z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask
             z = self.flow(z_p, x_mask, g=g, reverse=True)
-        o = self.dec(z * x_mask, g=g)
+        o = self.dec(z * x_mask, g=g, n_res=return_length2)
         return o, x_mask, (z, z_p, m_p, logs_p)
 
 

infer/lib/rtrvc.py

@@ -15,6 +15,7 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 import torchcrepe
+from torchaudio.transforms import Resample
 
 now_dir = os.getcwd()
 sys.path.append(now_dir)
@@ -40,6 +41,7 @@ class RVC:
     def __init__(
         self,
         key,
+        formant,
         pth_path,
         index_path,
         index_rate,
@@ -68,6 +70,7 @@ class RVC:
             # device="cpu"########强制cpu测试
             self.device = config.device
             self.f0_up_key = key
+            self.formant_shift = formant
             self.f0_min = 50
             self.f0_max = 1100
             self.f0_mel_min = 1127 * np.log(1 + self.f0_min / 700)
@@ -90,6 +93,8 @@ class RVC:
                 1024, device=self.device, dtype=torch.float32
             )
             
+            self.resample_kernel = {}
+            
             if last_rvc is None:
                 models, _, _ = fairseq.checkpoint_utils.load_model_ensemble_and_task(
                     ["assets/hubert/hubert_base.pt"],
@@ -187,6 +192,9 @@ class RVC:
     def change_key(self, new_key):
         self.f0_up_key = new_key
     
+    def change_formant(self, new_formant):
+        self.formant_shift = new_formant
+        
     def change_index_rate(self, new_index_rate):
         if new_index_rate != 0 and self.index_rate == 0:
             self.index = faiss.read_index(self.index_path)
@@ -198,7 +206,7 @@ class RVC:
         if not torch.is_tensor(f0):
             f0 = torch.from_numpy(f0)
         f0 = f0.float().to(self.device).squeeze()
-        f0_mel = 1127 * torch.log(1 + f0 / 700)
+        f0_mel = 1127 * torch.log(1 + f0 * pow(2, -self.formant_shift / 12) / 700)
         f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - self.f0_mel_min) * 254 / (
             self.f0_mel_max - self.f0_mel_min
         ) + 1
@@ -410,6 +418,8 @@ class RVC:
         p_len = torch.LongTensor([p_len]).to(self.device)
         sid = torch.LongTensor([0]).to(self.device)
         skip_head = torch.LongTensor([skip_head])
+        factor = pow(2, self.formant_shift / 12)
+        return_length2 = torch.LongTensor([int(np.ceil(return_length * factor))])
         return_length = torch.LongTensor([return_length])
         with torch.no_grad():
             if self.if_f0 == 1:
@@ -421,11 +431,22 @@ class RVC:
                     sid,
                     skip_head,
                     return_length,
+                    return_length2,
                 )
             else:
                 infered_audio, _, _ = self.net_g.infer(
-                    feats, p_len, sid, skip_head, return_length
+                    feats, p_len, sid, skip_head, return_length, return_length2
                 )
+        infered_audio = infered_audio.squeeze(1).float()
+        upp_res = int(np.floor(factor * self.tgt_sr // 100))
+        if upp_res != self.tgt_sr // 100:
+            if upp_res not in self.resample_kernel:
+                self.resample_kernel[upp_res] = Resample(
+                    orig_freq=upp_res, 
+                    new_freq=self.tgt_sr // 100, 
+                    dtype=torch.float32,
+                ).to(self.device)
+            infered_audio = self.resample_kernel[upp_res](infered_audio[: ,: return_length * upp_res])
         t5 = ttime()
         printt(
             "Spent time: fea = %.3fs, index = %.3fs, f0 = %.3fs, model = %.3fs",
@@ -434,4 +455,4 @@ class RVC:
             t4 - t3,
             t5 - t4,
         )
-        return infered_audio.squeeze().float()
+        return infered_audio.squeeze()