Skip to content

SOTA Open Source TTS

Add compress tool

Lengyue38d57129f056
diff --git a/preparing_data/wenet_clean/compress_tar.py b/preparing_data/wenet_clean/compress_tar.py
new file mode 100644
index 0000000..6609be1
--- /dev/null
+++ b/preparing_data/wenet_clean/compress_tar.py
@@ -0,0 +1,80 @@
+import tarfile
+from pathlib import Path
+from tqdm import tqdm
+import io
+import random
+from multiprocessing import Process
+
+
+def chunked_tarring(rank, file_list, base_folder, output_folder, chunk_size=1024**3):
+ chunk_count = 1
+ total_size = 0
+ saved_count = 0
+
+ buffer = io.BytesIO()
+ tar = tarfile.open(fileobj=buffer, mode="w")
+
+ for audio_file in file_list:
+ txt_file = audio_file.with_suffix(".txt")
+ if not txt_file.exists():
+ continue
+
+ file_size = audio_file.stat().st_size + txt_file.stat().st_size
+ if total_size + file_size > chunk_size:
+ tar.close()
+
+ # write the buffer to disk
+ buffer.seek(0)
+ with open(output_folder / f"chunk-{rank:03d}-{chunk_count:04d}.tar", "wb") as f:
+ f.write(buffer.read())
+
+ chunk_count += 1
+ total_size = 0
+ buffer = io.BytesIO()
+ tar = tarfile.open(fileobj=buffer, mode="w")
+
+ tar.add(audio_file, arcname=audio_file.relative_to(base_folder))
+ tar.add(txt_file, arcname=txt_file.relative_to(base_folder))
+
+ total_size += file_size
+
+ if saved_count % 1000 == 0:
+ print(f"Rank {rank}: {saved_count}/{len(file_list)}")
+
+ saved_count += 1
+
+ tar.close()
+ buffer.seek(0)
+ with open(output_folder / f"chunk-{rank:03d}-{chunk_count:04d}.tar", "wb") as f:
+ f.write(buffer.read())
+
+ print(f"Rank {rank}: {saved_count}/{len(file_list)}")
+
+
+if __name__ == "__main__":
+ base_folder = Path("/mnt/nvme1/multi-modal-test/WenetSpeech/cleaned")
+ output_folder = Path("/mnt/nvme1/multi-modal-test/WenetSpeech/compressed")
+ output_folder.mkdir(exist_ok=True, parents=True)
+ num_workers = 50
+
+ file_list = list(tqdm(base_folder.rglob("*.flac")))
+ random.shuffle(file_list)
+ print(f"Total files: {len(file_list)}")
+
+ chunk_size = len(file_list) // num_workers
+ processes = []
+
+ for i in range(num_workers):
+ start = i * chunk_size
+ end = (i + 1) * chunk_size
+ if i == num_workers - 1:
+ end = len(file_list)
+
+ p = Process(target=chunked_tarring, args=(i, file_list[start:end], base_folder, output_folder))
+ p.start()
+ processes.append(p)
+
+ for p in processes:
+ p.join()
+
+ print("Done")