Update trainset_preprocess_pipeline_print.py
This commit is contained in:
RVC-Boss
GitHub
parent
4a2c9c062f
commit
2c4ec6db93
|
|
@ -1,135 +1,135 @@
|
||||||
import sys, os, multiprocessing
|
import sys, os, multiprocessing
|
||||||
from scipy import signal
|
from scipy import signal
|
||||||
|
|
||||||
now_dir = os.getcwd()
|
now_dir = os.getcwd()
|
||||||
sys.path.append(now_dir)
|
sys.path.append(now_dir)
|
||||||
|
|
||||||
inp_root = sys.argv[1]
|
inp_root = sys.argv[1]
|
||||||
sr = int(sys.argv[2])
|
sr = int(sys.argv[2])
|
||||||
n_p = int(sys.argv[3])
|
n_p = int(sys.argv[3])
|
||||||
exp_dir = sys.argv[4]
|
exp_dir = sys.argv[4]
|
||||||
noparallel = sys.argv[5] == "True"
|
noparallel = sys.argv[5] == "True"
|
||||||
import numpy as np, os, traceback
|
import numpy as np, os, traceback
|
||||||
from slicer2 import Slicer
|
from slicer2 import Slicer
|
||||||
import librosa, traceback
|
import librosa, traceback
|
||||||
from scipy.io import wavfile
|
from scipy.io import wavfile
|
||||||
import multiprocessing
|
import multiprocessing
|
||||||
from my_utils import load_audio
|
from my_utils import load_audio
|
||||||
|
|
||||||
mutex = multiprocessing.Lock()
|
mutex = multiprocessing.Lock()
|
||||||
f = open("%s/preprocess.log" % exp_dir, "a+")
|
f = open("%s/preprocess.log" % exp_dir, "a+")
|
||||||
|
|
||||||
|
|
||||||
def println(strr):
|
def println(strr):
|
||||||
mutex.acquire()
|
mutex.acquire()
|
||||||
print(strr)
|
print(strr)
|
||||||
f.write("%s\n" % strr)
|
f.write("%s\n" % strr)
|
||||||
f.flush()
|
f.flush()
|
||||||
mutex.release()
|
mutex.release()
|
||||||
|
|
||||||
|
|
||||||
class PreProcess:
|
class PreProcess:
|
||||||
def __init__(self, sr, exp_dir):
|
def __init__(self, sr, exp_dir):
|
||||||
self.slicer = Slicer(
|
self.slicer = Slicer(
|
||||||
sr=sr,
|
sr=sr,
|
||||||
threshold=-40,
|
threshold=-40,
|
||||||
min_length=800,
|
min_length=800,
|
||||||
min_interval=400,
|
min_interval=400,
|
||||||
hop_size=15,
|
hop_size=15,
|
||||||
max_sil_kept=150,
|
max_sil_kept=150,
|
||||||
)
|
)
|
||||||
self.sr = sr
|
self.sr = sr
|
||||||
self.bh, self.ah = signal.butter(N=5, Wn=48, btype="high", fs=self.sr)
|
self.bh, self.ah = signal.butter(N=5, Wn=48, btype="high", fs=self.sr)
|
||||||
self.per = 3.7
|
self.per = 3.7
|
||||||
self.overlap = 0.3
|
self.overlap = 0.3
|
||||||
self.tail = self.per + self.overlap
|
self.tail = self.per + self.overlap
|
||||||
self.max = 0.95
|
self.max = 0.95
|
||||||
self.alpha = 0.8
|
self.alpha = 0.8
|
||||||
self.exp_dir = exp_dir
|
self.exp_dir = exp_dir
|
||||||
self.gt_wavs_dir = "%s/0_gt_wavs" % exp_dir
|
self.gt_wavs_dir = "%s/0_gt_wavs" % exp_dir
|
||||||
self.wavs16k_dir = "%s/1_16k_wavs" % exp_dir
|
self.wavs16k_dir = "%s/1_16k_wavs" % exp_dir
|
||||||
os.makedirs(self.exp_dir, exist_ok=True)
|
os.makedirs(self.exp_dir, exist_ok=True)
|
||||||
os.makedirs(self.gt_wavs_dir, exist_ok=True)
|
os.makedirs(self.gt_wavs_dir, exist_ok=True)
|
||||||
os.makedirs(self.wavs16k_dir, exist_ok=True)
|
os.makedirs(self.wavs16k_dir, exist_ok=True)
|
||||||
|
|
||||||
def norm_write(self, tmp_audio, idx0, idx1):
|
def norm_write(self, tmp_audio, idx0, idx1):
|
||||||
tmp_audio = (tmp_audio / np.abs(tmp_audio).max() * (self.max * self.alpha)) + (
|
tmp_audio = (tmp_audio / np.abs(tmp_audio).max() * (self.max * self.alpha)) + (
|
||||||
1 - self.alpha
|
1 - self.alpha
|
||||||
) * tmp_audio
|
) * tmp_audio
|
||||||
wavfile.write(
|
wavfile.write(
|
||||||
"%s/%s_%s.wav" % (self.gt_wavs_dir, idx0, idx1),
|
"%s/%s_%s.wav" % (self.gt_wavs_dir, idx0, idx1),
|
||||||
self.sr,
|
self.sr,
|
||||||
tmp_audio.astype(np.float32),
|
tmp_audio.astype(np.float32),
|
||||||
)
|
)
|
||||||
tmp_audio = librosa.resample(
|
tmp_audio = librosa.resample(
|
||||||
tmp_audio, orig_sr=self.sr, target_sr=16000
|
tmp_audio, orig_sr=self.sr, target_sr=16000
|
||||||
) # , res_type="soxr_vhq"
|
) # , res_type="soxr_vhq"
|
||||||
wavfile.write(
|
wavfile.write(
|
||||||
"%s/%s_%s.wav" % (self.wavs16k_dir, idx0, idx1),
|
"%s/%s_%s.wav" % (self.wavs16k_dir, idx0, idx1),
|
||||||
16000,
|
16000,
|
||||||
tmp_audio.astype(np.float32),
|
tmp_audio.astype(np.float32),
|
||||||
)
|
)
|
||||||
|
|
||||||
def pipeline(self, path, idx0):
|
def pipeline(self, path, idx0):
|
||||||
try:
|
try:
|
||||||
audio = load_audio(path, self.sr)
|
audio = load_audio(path, self.sr)
|
||||||
# zero phased digital filter cause pre-ringing noise...
|
# zero phased digital filter cause pre-ringing noise...
|
||||||
# audio = signal.filtfilt(self.bh, self.ah, audio)
|
# audio = signal.filtfilt(self.bh, self.ah, audio)
|
||||||
audio = signal.lfilter(self.bh, self.ah, audio)
|
audio = signal.lfilter(self.bh, self.ah, audio)
|
||||||
|
|
||||||
idx1 = 0
|
idx1 = 0
|
||||||
for audio in self.slicer.slice(audio):
|
for audio in self.slicer.slice(audio):
|
||||||
i = 0
|
i = 0
|
||||||
while 1:
|
while 1:
|
||||||
start = int(self.sr * (self.per - self.overlap) * i)
|
start = int(self.sr * (self.per - self.overlap) * i)
|
||||||
i += 1
|
i += 1
|
||||||
if len(audio[start:]) > self.tail * self.sr:
|
if len(audio[start:]) > self.tail * self.sr:
|
||||||
tmp_audio = audio[start : start + int(self.per * self.sr)]
|
tmp_audio = audio[start : start + int(self.per * self.sr)]
|
||||||
self.norm_write(tmp_audio, idx0, idx1)
|
self.norm_write(tmp_audio, idx0, idx1)
|
||||||
idx1 += 1
|
idx1 += 1
|
||||||
else:
|
else:
|
||||||
tmp_audio = audio[start:]
|
tmp_audio = audio[start:]
|
||||||
idx1 += 1
|
idx1 += 1
|
||||||
break
|
break
|
||||||
self.norm_write(tmp_audio, idx0, idx1)
|
self.norm_write(tmp_audio, idx0, idx1)
|
||||||
println("%s->Suc." % path)
|
println("%s->Suc." % path)
|
||||||
except:
|
except:
|
||||||
println("%s->%s" % (path, traceback.format_exc()))
|
println("%s->%s" % (path, traceback.format_exc()))
|
||||||
|
|
||||||
def pipeline_mp(self, infos):
|
def pipeline_mp(self, infos):
|
||||||
for path, idx0 in infos:
|
for path, idx0 in infos:
|
||||||
self.pipeline(path, idx0)
|
self.pipeline(path, idx0)
|
||||||
|
|
||||||
def pipeline_mp_inp_dir(self, inp_root, n_p):
|
def pipeline_mp_inp_dir(self, inp_root, n_p):
|
||||||
try:
|
try:
|
||||||
infos = [
|
infos = [
|
||||||
("%s/%s" % (inp_root, name), idx)
|
("%s/%s" % (inp_root, name), idx)
|
||||||
for idx, name in enumerate(sorted(list(os.listdir(inp_root))))
|
for idx, name in enumerate(sorted(list(os.listdir(inp_root))))
|
||||||
]
|
]
|
||||||
if noparallel:
|
if noparallel:
|
||||||
for i in range(n_p):
|
for i in range(n_p):
|
||||||
self.pipeline_mp(infos[i::n_p])
|
self.pipeline_mp(infos[i::n_p])
|
||||||
else:
|
else:
|
||||||
ps = []
|
ps = []
|
||||||
for i in range(n_p):
|
for i in range(n_p):
|
||||||
p = multiprocessing.Process(
|
p = multiprocessing.Process(
|
||||||
target=self.pipeline_mp, args=(infos[i::n_p],)
|
target=self.pipeline_mp, args=(infos[i::n_p],)
|
||||||
)
|
)
|
||||||
p.start()
|
p.start()
|
||||||
ps.append(p)
|
ps.append(p)
|
||||||
for p in ps:
|
for p in ps:
|
||||||
p.join()
|
p.join()
|
||||||
except:
|
except:
|
||||||
println("Fail. %s" % traceback.format_exc())
|
println("Fail. %s" % traceback.format_exc())
|
||||||
|
|
||||||
|
|
||||||
def preprocess_trainset(inp_root, sr, n_p, exp_dir):
|
def preprocess_trainset(inp_root, sr, n_p, exp_dir):
|
||||||
pp = PreProcess(sr, exp_dir)
|
pp = PreProcess(sr, exp_dir)
|
||||||
println("start preprocess")
|
println("start preprocess")
|
||||||
println(sys.argv)
|
println(sys.argv)
|
||||||
pp.pipeline_mp_inp_dir(inp_root, n_p)
|
pp.pipeline_mp_inp_dir(inp_root, n_p)
|
||||||
println("end preprocess")
|
println("end preprocess")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
preprocess_trainset(inp_root, sr, n_p, exp_dir)
|
preprocess_trainset(inp_root, sr, n_p, exp_dir)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue