feat: fallback to system encoding when fail to read file with utf-8

This commit is contained in:
多玩幻灵qwq 2024-06-09 00:05:12 +08:00 committed by GitHub
parent 64b78bed3b
commit f0dd73a419
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -278,8 +278,13 @@ def load_wav_to_torch(full_path):
def load_filepaths_and_text(filename, split="|"): def load_filepaths_and_text(filename, split="|"):
with open(filename, encoding="utf-8") as f: try:
filepaths_and_text = [line.strip().split(split) for line in f] with open(filename, encoding="utf-8") as f:
filepaths_and_text = [line.strip().split(split) for line in f]
except UnicodeDecodeError:
with open(filename) as f:
filepaths_and_text = [line.strip().split(split) for line in f]
return filepaths_and_text return filepaths_and_text