Skip to content

FunASR-powered video transcription, subtitle generation, and LLM-assisted clipping tool with a local Gradio UI.

Fix repeated text matching in audio and video clips (#177)

zhifu gaod30daccff432
diff --git a/funclip/videoclipper.py b/funclip/videoclipper.py
index 9a4cdb5..ccb6165 100644
--- a/funclip/videoclipper.py
+++ b/funclip/videoclipper.py
@@ -170,33 +170,41 @@ class VideoClipper():
sr, data = audio_input
data = data.astype(np.float64)
+ log_append = ""
if timestamp_list is None:
all_ts = []
+ warning_messages = []
if dest_spk is None or dest_spk == '' or 'sd_sentences' not in state:
- for _dest_text in dest_text.split('#'):
+ for text_index, _dest_text in enumerate(dest_text.split('#'), start=1):
+ offset_match = None
if '[' in _dest_text:
- match = re.search(r'\[(\d+),\s*(\d+)\]', _dest_text)
- if match:
- offset_b, offset_e = map(int, match.groups())
- log_append = ""
+ offset_match = re.search(r'\[(\d+),\s*(\d+)\]', _dest_text)
+ if offset_match:
+ offset_b, offset_e = map(int, offset_match.groups())
else:
offset_b, offset_e = 0, 0
- log_append = "(Bracket detected in dest_text but offset time matching failed)"
+ warning_messages.append(
+ "(Bracket detected in dest_text but offset time matching failed)"
+ )
_dest_text = _dest_text[:_dest_text.find('[')]
else:
- log_append = ""
offset_b, offset_e = 0, 0
_dest_text = pre_proc(_dest_text)
ts = proc(recog_res_raw, timestamp, _dest_text)
for _ts in ts: all_ts.append([_ts[0]+offset_b*16, _ts[1]+offset_e*16])
- if len(ts) > 1 and match:
- log_append += '(offsets detected but No.{} sub-sentence matched to {} periods in audio, \
- offsets are applied to all periods)'
+ if len(ts) > 1 and offset_match:
+ warning_messages.append(
+ "(offsets detected but No.{} sub-sentence matched to {} "
+ "periods in audio, offsets are applied to all periods)".format(
+ text_index, len(ts)
+ )
+ )
else:
for _dest_spk in dest_spk.split('#'):
ts = proc_spk(_dest_spk, state['sd_sentences'])
for _ts in ts: all_ts.append(_ts)
- log_append = ""
+ if warning_messages:
+ log_append = " " + " ".join(warning_messages)
else:
all_ts = timestamp_list
ts = all_ts
@@ -222,7 +230,7 @@ class VideoClipper():
if len(ts):
message = "{} periods found in the speech: ".format(len(ts)) + start_end_info + log_append
else:
- message = "No period found in the speech, return raw speech. You may check the recognition result and try other destination text."
+ message = "No period found in the speech, return raw speech. You may check the recognition result and try other destination text." + log_append
res_audio = data
return (sr, res_audio), message, clip_srt
@@ -277,33 +285,42 @@ class VideoClipper():
clip_video_file = state['clip_video_file']
video_filename = state['video_filename']
+ log_append = ""
if timestamp_list is None:
all_ts = []
+ warning_messages = []
if dest_spk is None or dest_spk == '' or 'sd_sentences' not in state:
- for _dest_text in dest_text.split('#'):
+ for text_index, _dest_text in enumerate(dest_text.split('#'), start=1):
+ offset_match = None
if '[' in _dest_text:
- match = re.search(r'\[(\d+),\s*(\d+)\]', _dest_text)
- if match:
- offset_b, offset_e = map(int, match.groups())
- log_append = ""
+ offset_match = re.search(r'\[(\d+),\s*(\d+)\]', _dest_text)
+ if offset_match:
+ offset_b, offset_e = map(int, offset_match.groups())
else:
offset_b, offset_e = 0, 0
- log_append = "(Bracket detected in dest_text but offset time matching failed)"
+ warning_messages.append(
+ "(Bracket detected in dest_text but offset time matching failed)"
+ )
_dest_text = _dest_text[:_dest_text.find('[')]
else:
offset_b, offset_e = 0, 0
- log_append = ""
# import pdb; pdb.set_trace()
_dest_text = pre_proc(_dest_text)
ts = proc(recog_res_raw, timestamp, _dest_text.lower())
for _ts in ts: all_ts.append([_ts[0]+offset_b*16, _ts[1]+offset_e*16])
- if len(ts) > 1 and match:
- log_append += '(offsets detected but No.{} sub-sentence matched to {} periods in audio, \
- offsets are applied to all periods)'
+ if len(ts) > 1 and offset_match:
+ warning_messages.append(
+ "(offsets detected but No.{} sub-sentence matched to {} "
+ "periods in audio, offsets are applied to all periods)".format(
+ text_index, len(ts)
+ )
+ )
else:
for _dest_spk in dest_spk.split('#'):
ts = proc_spk(_dest_spk, state['sd_sentences'])
for _ts in ts: all_ts.append(_ts)
+ if warning_messages:
+ log_append = " " + " ".join(warning_messages)
else: # AI clip pass timestamp as input directly
all_ts = [[i[0]*16.0, i[1]*16.0] for i in timestamp_list]
@@ -347,7 +364,7 @@ class VideoClipper():
# _video_clip.write_videofile("debug.mp4", audio_codec="aac")
concate_clip.append(copy.copy(_video_clip))
time_acc_ost += end - start
- message = "{} periods found in the audio: ".format(len(ts)) + start_end_info
+ message = "{} periods found in the audio: ".format(len(ts)) + start_end_info + log_append
logging.warning("Concating...")
if len(concate_clip) > 1:
video_clip = concatenate_videoclips(concate_clip)
@@ -366,7 +383,7 @@ class VideoClipper():
self.GLOBAL_COUNT += 1
else:
clip_video_file = video_filename
- message = "No period found in the audio, return raw speech. You may check the recognition result and try other destination text."
+ message = "No period found in the audio, return raw speech. You may check the recognition result and try other destination text." + log_append
srt_clip = ''
return clip_video_file, message, clip_srt
diff --git a/tests/test_duplicate_text_matching.py b/tests/test_duplicate_text_matching.py
new file mode 100644
index 0000000..5bf1428
--- /dev/null
+++ b/tests/test_duplicate_text_matching.py
@@ -0,0 +1,151 @@
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import numpy as np
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "funclip"))
+
+from videoclipper import VideoClipper # noqa: E402
+
+
+class DummyVideoClip:
+ def __init__(self):
+ self.write_calls = []
+
+ def write_videofile(self, *args, **kwargs):
+ self.write_calls.append((args, kwargs))
+
+
+class DummyVideo:
+ def __init__(self):
+ self.subclip_calls = []
+
+ def subclip(self, start, end):
+ self.subclip_calls.append((start, end))
+ return DummyVideoClip()
+
+
+class TestDuplicateTextMatching(unittest.TestCase):
+ def setUp(self):
+ self.raw_text = "重 复 其 他 重 复"
+ self.timestamps = [[i * 100, (i + 1) * 100] for i in range(6)]
+ self.offset_warning = (
+ "(offsets detected but No.1 sub-sentence matched to 2 periods in audio, "
+ "offsets are applied to all periods)"
+ )
+
+ @patch("videoclipper.generate_srt_clip", return_value=("", [], 0))
+ def test_audio_clip_keeps_all_repeated_matches_without_offsets(self, _generate_srt):
+ clipper = VideoClipper(None)
+ audio = np.arange(10000, dtype=np.float32)
+ state = {
+ "audio_input": (16000, audio),
+ "recog_res_raw": self.raw_text,
+ "timestamp": self.timestamps,
+ "sentences": [],
+ }
+
+ (sample_rate, clipped), message, _ = clipper.clip("重复", 0, 0, state)
+
+ self.assertEqual(sample_rate, 16000)
+ self.assertEqual(len(clipped), 6400)
+ self.assertIn("2 periods found", message)
+
+ @patch(
+ "videoclipper.generate_srt_clip",
+ return_value=("", [((0.0, 0.2), "重复")], 1),
+ )
+ def test_video_clip_keeps_all_repeated_matches_without_offsets(self, _generate_srt):
+ clipper = VideoClipper(None)
+ clipper.lang = "zh"
+ video = DummyVideo()
+ output_clip = DummyVideoClip()
+ state = {
+ "recog_res_raw": self.raw_text,
+ "timestamp": self.timestamps,
+ "sentences": [],
+ "video": video,
+ "clip_video_file": "/tmp/duplicate_clip.mp4",
+ "video_filename": "/tmp/duplicate.mp4",
+ }
+
+ with patch(
+ "videoclipper.concatenate_videoclips", return_value=output_clip
+ ) as concatenate:
+ _, message, _ = clipper.video_clip("重复", 0, 0, state)
+
+ self.assertEqual(len(video.subclip_calls), 2)
+ self.assertAlmostEqual(video.subclip_calls[0][0], 0.0)
+ self.assertAlmostEqual(video.subclip_calls[0][1], 0.2)
+ self.assertAlmostEqual(video.subclip_calls[1][0], 0.4)
+ self.assertAlmostEqual(video.subclip_calls[1][1], 0.6)
+ self.assertEqual(len(concatenate.call_args.args[0]), 2)
+ self.assertEqual(len(output_clip.write_calls), 1)
+ self.assertIn("2 periods found", message)
+
+ @patch("videoclipper.generate_srt_clip", return_value=("", [], 0))
+ def test_audio_clip_formats_offset_warning_for_repeated_matches(
+ self, _generate_srt
+ ):
+ clipper = VideoClipper(None)
+ state = {
+ "audio_input": (16000, np.arange(10000, dtype=np.float32)),
+ "recog_res_raw": self.raw_text,
+ "timestamp": self.timestamps,
+ "sentences": [],
+ }
+
+ _, message, _ = clipper.clip("重复[10, 20]", 0, 0, state)
+
+ self.assertIn(self.offset_warning, message)
+ self.assertNotIn("No.{}", message)
+
+ @patch("videoclipper.generate_srt_clip", return_value=("", [], 0))
+ def test_audio_clip_accepts_explicit_timestamps_without_text_matching(
+ self, _generate_srt
+ ):
+ clipper = VideoClipper(None)
+ state = {
+ "audio_input": (16000, np.arange(10000, dtype=np.float32)),
+ "recog_res_raw": self.raw_text,
+ "timestamp": self.timestamps,
+ "sentences": [],
+ }
+
+ (_, clipped), message, _ = clipper.clip(
+ "", 0, 0, state, timestamp_list=[[0, 1600]]
+ )
+
+ self.assertEqual(len(clipped), 1600)
+ self.assertIn("1 periods found", message)
+
+ @patch(
+ "videoclipper.generate_srt_clip",
+ return_value=("", [((0.0, 0.2), "重复")], 1),
+ )
+ def test_video_clip_includes_offset_warning_for_repeated_matches(
+ self, _generate_srt
+ ):
+ clipper = VideoClipper(None)
+ clipper.lang = "zh"
+ output_clip = DummyVideoClip()
+ state = {
+ "recog_res_raw": self.raw_text,
+ "timestamp": self.timestamps,
+ "sentences": [],
+ "video": DummyVideo(),
+ "clip_video_file": "/tmp/duplicate_clip.mp4",
+ "video_filename": "/tmp/duplicate.mp4",
+ }
+
+ with patch("videoclipper.concatenate_videoclips", return_value=output_clip):
+ _, message, _ = clipper.video_clip("重复[10, 20]", 0, 0, state)
+
+ self.assertIn(self.offset_warning, message)
+
+
+if __name__ == "__main__":
+ unittest.main()