Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

Dev/pxy (#61)

xyupenge933ed8727b3
diff --git a/tools/datasets/filter_panda10m.py b/tools/datasets/filter_panda10m.py
new file mode 100644
index 0000000..d4397c2
--- /dev/null
+++ b/tools/datasets/filter_panda10m.py
@@ -0,0 +1,289 @@
+# TODO: remove this file before releasing
+
+import argparse
+import os
+import pandas as pd
+import json
+import html
+from tqdm import tqdm
+import re
+
+tqdm.pandas()
+
+try:
+ from pandarallel import pandarallel
+
+ pandarallel.initialize(progress_bar=True)
+ pandas_has_parallel = True
+except ImportError:
+ pandas_has_parallel = False
+
+
+def apply(df, func, **kwargs):
+ if pandas_has_parallel:
+ return df.parallel_apply(func, **kwargs)
+ return df.progress_apply(func, **kwargs)
+
+
+def basic_clean(text):
+ import ftfy
+
+ text = ftfy.fix_text(text)
+ text = html.unescape(html.unescape(text))
+ return text.strip()
+
+
+BAD_PUNCT_REGEX = re.compile(
+ r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\|" + "\\" + "\/" + "\*" + r"]{1,}"
+) # noqa
+
+
+def clean_caption(caption):
+ import urllib.parse as ul
+
+ from bs4 import BeautifulSoup
+
+ caption = str(caption)
+ caption = ul.unquote_plus(caption)
+ caption = caption.strip().lower()
+ caption = re.sub("<person>", "person", caption)
+ # urls:
+ caption = re.sub(
+ r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa
+ "",
+ caption,
+ ) # regex for urls
+ caption = re.sub(
+ r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa
+ "",
+ caption,
+ ) # regex for urls
+ # html:
+ caption = BeautifulSoup(caption, features="html.parser").text
+
+ # @<nickname>
+ caption = re.sub(r"@[\w\d]+\b", "", caption)
+
+ # 31C0—31EF CJK Strokes
+ # 31F0—31FF Katakana Phonetic Extensions
+ # 3200—32FF Enclosed CJK Letters and Months
+ # 3300—33FF CJK Compatibility
+ # 3400—4DBF CJK Unified Ideographs Extension A
+ # 4DC0—4DFF Yijing Hexagram Symbols
+ # 4E00—9FFF CJK Unified Ideographs
+ caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)
+ caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)
+ caption = re.sub(r"[\u3200-\u32ff]+", "", caption)
+ caption = re.sub(r"[\u3300-\u33ff]+", "", caption)
+ caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)
+ caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)
+ caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)
+ #######################################################
+
+ # все виды тире / all types of dash --> "-"
+ caption = re.sub(
+ r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa
+ "-",
+ caption,
+ )
+
+ # кавычки к одному стандарту
+ caption = re.sub(r"[`´«»“”¨]", '"', caption)
+ caption = re.sub(r"[‘’]", "'", caption)
+
+ # &quot;
+ caption = re.sub(r"&quot;?", "", caption)
+ # &amp
+ caption = re.sub(r"&amp", "", caption)
+
+ # ip adresses:
+ caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)
+
+ # article ids:
+ caption = re.sub(r"\d:\d\d\s+$", "", caption)
+
+ # \n
+ caption = re.sub(r"\\n", " ", caption)
+
+ # "#123"
+ caption = re.sub(r"#\d{1,3}\b", "", caption)
+ # "#12345.."
+ caption = re.sub(r"#\d{5,}\b", "", caption)
+ # "123456.."
+ caption = re.sub(r"\b\d{6,}\b", "", caption)
+ # filenames:
+ caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)
+
+ #
+ caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""
+ caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""
+
+ caption = re.sub(BAD_PUNCT_REGEX, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT
+ caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "
+
+ # this-is-my-cute-cat / this_is_my_cute_cat
+ regex2 = re.compile(r"(?:\-|\_)")
+ if len(re.findall(regex2, caption)) > 3:
+ caption = re.sub(regex2, " ", caption)
+
+ caption = basic_clean(caption)
+
+ caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640
+ caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc
+ caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231
+
+ caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)
+ caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)
+ caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)
+ caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption)
+ caption = re.sub(r"\bpage\s+\d+\b", "", caption)
+
+ caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a...
+
+ caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)
+
+ caption = re.sub(r"\b\s+\:\s+", r": ", caption)
+ caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)
+ caption = re.sub(r"\s+", " ", caption)
+
+ caption.strip()
+
+ caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)
+ caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)
+ caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)
+ caption = re.sub(r"^\.\S+$", "", caption)
+
+ return caption.strip()
+
+
+def get_10m_set():
+ meta_path_10m = '/mnt/hdd/data/Panda-70M/raw/meta/train/panda70m_training_10m.csv'
+ meta_10m = pd.read_csv(meta_path_10m)
+
+ def process_single_caption(row):
+ text_list = eval(row['caption'])
+ clean_list = [clean_caption(x) for x in text_list]
+ return str(clean_list)
+
+ ret = apply(meta_10m, process_single_caption, axis=1)
+ # ret = meta_10m.progress_apply(process_single_caption, axis=1)
+ print('==> text processed.')
+
+ text_list = []
+ for x in ret:
+ text_list += eval(x)
+ # text_set = text_set.union(set(eval(x)))
+ text_set = set(text_list)
+ # meta_10m['caption_new'] = ret
+ # meta_10m.to_csv('/mnt/hdd/data/Panda-70M/raw/meta/train/panda70m_training_10m_new-cap.csv')
+
+ # video_id_set = set(meta_10m['videoID'])
+ # id2t = {}
+ # for idx, row in tqdm(meta_10m.iterrows(), total=len(meta_10m)):
+ # video_id = row['videoID']
+ # text_list = eval(row['caption'])
+ # id2t[video_id] = set(text_list)
+
+ print(f"==> Loaded meta_10m from '{meta_path_10m}'")
+ return text_set
+
+
+def filter_panda10m_text(meta_path, text_set):
+ def process_single_row(row):
+ # path = row['path']
+ t = row['text']
+ # fname = os.path.basename(path)
+ # video_id = fname[:fname.rindex('_')]
+ if t not in text_set:
+ return False
+ return True
+
+ meta = pd.read_csv(meta_path)
+ ret = apply(meta, process_single_row, axis=1)
+ # ret = meta.progress_apply(process_single_row, axis=1)
+
+ meta = meta[ret]
+ wo_ext, ext = os.path.splitext(meta_path)
+ out_path = f"{wo_ext}_filter-10m{ext}"
+ meta.to_csv(out_path, index=False)
+ print(f"New meta (shape={meta.shape}) saved to '{out_path}'.")
+
+
+def filter_panda10m_timestamp(meta_path):
+ meta_path_10m = '/mnt/hdd/data/Panda-70M/raw/meta/train/panda70m_training_10m.csv'
+ meta_10m = pd.read_csv(meta_path_10m)
+
+ id2t = {}
+ for idx, row in tqdm(meta_10m.iterrows(), total=len(meta_10m)):
+ video_id = row['videoID']
+ timestamp = eval(row['timestamp'])
+ timestamp = [str(tuple(x)) for x in timestamp]
+ id2t[video_id] = timestamp
+
+ # video_id_set_10m = set(meta_10m['videoID'])
+ print(f"==> Loaded meta_10m from '{meta_path_10m}'")
+
+ def process_single_row(row):
+ path = row['path']
+ t = row['timestamp']
+ fname = os.path.basename(path)
+ video_id = fname[:fname.rindex('_')]
+ if video_id not in id2t:
+ return False
+ if t not in id2t[video_id]:
+ return False
+ return True
+ # return video_id in video_id_set_10m
+
+ meta = pd.read_csv(meta_path)
+ ret = apply(meta, process_single_row, axis=1)
+
+ meta = meta[ret]
+ wo_ext, ext = os.path.splitext(meta_path)
+ out_path = f"{wo_ext}_filter-10m{ext}"
+ meta.to_csv(out_path, index=False)
+ print(f"New meta (shape={meta.shape}) saved to '{out_path}'.")
+
+
+def append_timestamp(meta_path):
+
+ def process_single_row(row):
+ path = row['path']
+ wo_ext, ext = os.path.splitext(path)
+ json_path = f'{wo_ext}.json'
+ try:
+ with open(json_path, 'r') as f:
+ data = json.load(f)
+ timestamp = data['clips'][2:-2]
+ a, b = timestamp.split(', ')
+ timestamp = f"('{a}', '{b}')"
+ except Exception as e:
+ timestamp = ''
+ return timestamp
+
+ meta = pd.read_csv(meta_path)
+ ret = apply(meta, process_single_row, axis=1)
+ meta['timestamp'] = ret
+
+ wo_ext, ext = os.path.splitext(meta_path)
+ out_path = f"{wo_ext}_timestamp{ext}"
+ meta.to_csv(out_path, index=False)
+ print(f"New meta (shape={meta.shape}) saved to '{out_path}'.")
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--meta_path', type=str, nargs='+')
+ parser.add_argument('--num_workers', default=5, type=int)
+
+ args = parser.parse_args()
+ return args
+
+
+if __name__ == '__main__':
+ args = parse_args()
+ # append_timestamp(args.meta_path)
+
+ text_set = get_10m_set()
+ for x in args.meta_path:
+ filter_panda10m_text(x, text_set)
diff --git a/tools/scene_cut/README.md b/tools/scene_cut/README.md
index 492704d..87aa201 100644
--- a/tools/scene_cut/README.md
+++ b/tools/scene_cut/README.md
@@ -1,35 +1,56 @@
-## Scene Detection and Video Splitting
+# Scene Detection and Video Splitting
+In many cases, raw videos contain several scenes and are too long for training. Thus, it is essential to split them into shorter
+clips based on scenes. Here, we provide code for scene detection and video splitting.
-### Formatting
-
-Input meta should be `{prefix}.csv` with column `'videoId'`
+## Formatting
+At this step, you should have a raw video dataset prepared. We need a meta file for the dataset. To create a meta file from a folder, run:
```bash
-python tools/scene_cut/process_meta.py --task append_format --meta_path /mnt/hdd/data/pexels_new/raw/meta/popular_6.csv --split popular_6
+python -m tools.datasets.convert video /path/to/video/folder --output /path/to/save/meta.csv
```
+This should output a `.csv` file with column `path`.
+
+If you already have a meta file for the videos and want to keep the information.
+**Make sure** the meta file has column `id`, which is the id for each video, and the video is named as `{id}.mp4`.
+The following command will add a new column `path` to the meta file.
-Output is `{prefix}_format.csv` (with column `path`) and `{prefix}_intact.csv` (with column `intact` and `path`)
+```bash
+python tools/scene_cut/process_meta.py --task append_path --meta_path /path/to/meta.csv --folder_path /path/to/video/folder
+```
+This should output
+- `{prefix}_path-filtered.csv` with column `path` (broken videos filtered)
+- `{prefix}_path_intact.csv` with column `path` and `intact` (`intact` indicating a video is intact or not)
-### Scene Detection
-Input meta should be `{prefix}_format.csv`
+## Scene Detection
+The next step is to detect scenes in a video.
+We use [`PySceneDetect`](https://github.com/Breakthrough/PySceneDetect) for this job.
+**Make sure** the input meta file has column `path`, which is the path of a video.
```bash
+python tools/scene_cut/scene_detect.py --meta_path /path/to/meta.csv
python tools/scene_cut/scene_detect.py --meta_path /mnt/hdd/data/pexels_new/raw/meta/popular_6_format.csv
```
+The output is `{prefix}_timestamp.csv` with column `timestamp`. Each cell in column `timestamp` is a list of tuples,
+with each tuple indicating the start and end timestamp of a scene
+(e.g., `[('00:00:01.234', '00:00:02.345'), ('00:00:03.456', '00:00:04.567')]`).
-Output is `{prefix}_format_timestamp.csv`
-
-### Video Splitting
+## Video Splitting
+After obtaining timestamps for scenes, we conduct video splitting (cutting).
+**Make sure** the meta file contains column `timestamp`.
-Input meta should be `{prefix}_timestamp.csv`
+TODO: output video size, min_duration, max_duration
```bash
+python tools/scene_cut/main_cut_pandarallel.py \
+ --meta_path /path/to/meta.csv \
+ --out_dir /path/to/output/dir
+
python tools/scene_cut/main_cut_pandarallel.py \
--meta_path /mnt/hdd/data/pexels_new/raw/meta/popular_6_format_timestamp.csv \
--out_dir /mnt/hdd/data/pexels_new/scene_cut/data/popular_6
```
-Output is `{out_dir}/{wo_ext}_scene-{sid}.mp4`
+This yields video clips saved in `/path/to/output/dir`. The video clips are named as `{video_id}_scene-{scene_id}.mp4`
TODO: meta for video clips
diff --git a/tools/scene_cut/process_meta.py b/tools/scene_cut/process_meta.py
index 7a664b0..9d982de 100644
--- a/tools/scene_cut/process_meta.py
+++ b/tools/scene_cut/process_meta.py
@@ -36,136 +36,48 @@ def has_downloaded_success(json_path):
return True
-def split_meta_csv(chunk_size=60000):
- """
- Split csv into multiple small csv in order
- """
- root = "./data/Panda-70M"
- # meta_name = 'meta/panda70m_training_full.csv'
- meta_name = "meta/panda70m_training_10m.csv"
- # meta_name = 'meta/training_10m/train_0.csv'
- meta_path = os.path.join(root, meta_name)
-
- df = pd.read_csv(meta_path)
- num_rows = len(df)
-
- # Split the DataFrame into smaller DataFrames
- for idx, i in enumerate(range(0, num_rows, chunk_size)):
- df_i = df.iloc[i : i + chunk_size]
- out_path = os.path.join(root, f"meta/train_{idx}.csv")
- df_i.to_csv(out_path, index=False)
-
- # If there are remaining rows
- if num_rows > chunk_size and num_rows % chunk_size != 0:
- df_last = df.iloc[-(num_rows % chunk_size) :]
- out_path = os.path.join(root, f"meta/train_{idx + 1}.csv")
- df_last.to_csv(out_path, index=False)
-
-
-def remove_index():
- df = pd.read_csv("your_file.csv", index_col=0)
- df.to_csv("your_file_without_index.csv", index=False)
-
-
-def append_format(meta_path, mode=".mp4"):
- """
- Append _format to csv file:
- - filter broken videos; only intact videos are kept
- - add column 'path'
-
- input csv should satisfy:
- - name should be: {split}.csv
- - contain column 'videoID'/'videoId'
- """
- # meta_path = os.path.join(root, f'raw/meta/{split}.csv')
- meta_dirname = os.path.dirname(meta_path)
- assert meta_dirname.endswith("raw/meta")
- root_raw = os.path.dirname(meta_dirname)
-
- meta_fname = os.path.basename(meta_path)
- split, ext = os.path.splitext(meta_fname)
-
- meta = pd.read_csv(meta_path)
-
- path_list = []
- new_meta = []
- for idx, row in tqdm(meta.iterrows(), total=len(meta)):
- # video_id = row['videoID'] # panda
- video_id = row["videoId"] # pexels_new
- video_path = os.path.join(root_raw, f"data/{split}/{video_id}.mp4")
- if mode == ".mp4":
- if not is_intact_video(video_path):
- continue
- elif mode == ".json":
- json_path = os.path.join(root_raw, f"data/{split}/{video_id}.json")
- if not has_downloaded_success(json_path):
- continue
- else:
- raise ValueError
-
- new_meta.append(row)
- path_list.append(video_path)
-
- new_meta = pd.DataFrame(new_meta)
- new_meta["path"] = path_list
-
- out_path = os.path.join(root_raw, f"meta/{split}_format.csv")
- new_meta.to_csv(out_path, index=False)
- print(f"New meta (shape={new_meta.shape}) saved to '{out_path}'")
-
-
-def append_format_pandarallel(meta_path, split, mode=".mp4"):
- """
- Append _format to csv file:
- - filter broken videos; only intact videos are kept
- - add column 'path'
-
- input csv should satisfy:
- - name should be: {split}.csv
- - contain column 'videoID'/'videoId'
- """
- # meta_path = os.path.join(root, f'raw/meta/{split}.csv')
- meta_dirname = os.path.dirname(meta_path)
- assert meta_dirname.endswith("raw/meta")
- root_raw = os.path.dirname(meta_dirname)
-
- meta_fname = os.path.basename(meta_path)
- wo_ext, ext = os.path.splitext(meta_fname)
-
- meta = pd.read_csv(meta_path)
-
+def append_format_pandarallel(meta_path, folder_path, mode=".json"):
def is_intact(row, mode=".json"):
- video_id = row["videoId"] # pexels_new
- video_path = os.path.join(root_raw, f"data/{split}/{video_id}.mp4")
+ video_id = row["id"]
+ # video_path = os.path.join(root_raw, f"data/{split}/{video_id}.mp4")
+ video_path = os.path.join(folder_path, f"{video_id}.mp4")
row["path"] = video_path
if mode == ".mp4":
if is_intact_video(video_path):
return True, video_path
return False, video_path
elif mode == ".json":
- json_path = os.path.join(root_raw, f"data/{split}/{video_id}.json")
+ # json_path = os.path.join(root_raw, f"data/{split}/{video_id}.json")
+ json_path = os.path.join(folder_path, f"{video_id}.json")
if has_downloaded_success(json_path):
return True, video_path
return False, video_path
+ elif mode is None:
+ return True, video_path
else:
raise ValueError
+ meta_dirpath = os.path.dirname(meta_path)
+ meta_fname = os.path.basename(meta_path)
+ wo_ext, ext = os.path.splitext(meta_fname)
+
pandarallel.initialize(progress_bar=True)
is_intact_partial = partial(is_intact, mode=mode)
- ret = meta.parallel_apply(is_intact_partial, axis=1)
+ meta = pd.read_csv(meta_path)
+ ret = meta.parallel_apply(is_intact_partial, axis=1)
intact, paths = list(zip(*ret))
meta["intact"] = intact
meta["path"] = paths
- out_path = os.path.join(root_raw, f"meta/{wo_ext}_intact.csv")
+ out_path = os.path.join(meta_dirpath, f"{wo_ext}_path_intact.csv")
meta.to_csv(out_path, index=False)
print(f"New meta (shape={meta.shape}) with intact info saved to '{out_path}'")
# meta_format = meta[meta['intact']]
meta_format = meta[np.array(intact)]
meta_format.drop("intact", axis=1, inplace=True)
- out_path = os.path.join(root_raw, f"meta/{wo_ext}_format.csv")
+ out_path = os.path.join(meta_dirpath, f"{wo_ext}_path-filtered.csv")
meta_format.to_csv(out_path, index=False)
print(f"New meta (shape={meta_format.shape}) with format info saved to '{out_path}'")
@@ -180,84 +92,30 @@ def create_subset(meta_path):
print(f"New meta (shape={meta_subset.shape}) saved to '{out_path}'")
-def append_cut(root="./data/Panda-70M"):
- """
- Append _cut to csv file
- input csv should satisfy:
- - name_should be {split}_intact.csv
- - contain column 'timestamp': list of timestamp
- """
- split = "test"
- meta_path = os.path.join(root, f"processed/meta/{split}_intact.csv")
-
- wo_ext, ext = os.path.splitext(meta_path)
- suffix = "cut"
- out_path = f"{wo_ext}_{suffix}{ext}"
-
- meta = pd.read_csv(meta_path)
-
- new_meta = []
- for idx, row in tqdm(meta.iterrows(), total=len(meta)):
- video_id = row["videoID"]
- timestamps = eval(row["timestamp"])
- captions = eval(row["caption"])
- scores = eval(row["matching_score"])
-
- num_clips = len(timestamps)
- for idx_c in range(num_clips):
- path_i = os.path.join(root, f"processed/{split}/{video_id}_scene-{idx_c}.mp4")
- # if not is_intact_video(path_i):
- # continue
-
- row_i = [f"{video_id}_scene-{idx_c}", path_i, timestamps[idx_c], captions[idx_c], scores[idx_c]]
-
- new_meta.append(row_i)
-
- columns = ["videoID", "path", "timestamp", "text", "match_official"]
- new_meta = pd.DataFrame(new_meta, columns=columns)
-
- new_meta.to_csv(out_path, index=False)
- print(f"New meta (shape={new_meta.shape}) saved to '{out_path}'")
-
-
-def debug_meta_topk():
- meta_path = "F:/Panda-70M/meta/test_intact_cut_flow.csv"
- meta = pd.read_csv(meta_path)
-
- score_column = "flow_score"
- topk = meta.nlargest(10, columns=score_column)
- topk_s = meta.nsmallest(200, columns=score_column)
-
- [(row["path"], row["caption"], row[score_column]) for idx, row in topk.iterrows()]
- [(row["path"], row["caption"], row[score_column]) for idx, row in topk_s.iterrows()]
-
-
def parse_args():
parser = argparse.ArgumentParser()
- parser.add_argument("--task", default="append_format")
- parser.add_argument("--meta_path", default="./data/pexels_new/raw/meta/popular_1.csv")
- parser.add_argument("--split", default="popular_5")
+ parser.add_argument("--task", default="append_path", required=True)
+ parser.add_argument("--meta_path", type=str, required=True)
+ parser.add_argument("--folder_path", type=str, required=True)
+ parser.add_argument("--mode", type=str, default=None)
parser.add_argument("--num_workers", default=5, type=int)
args = parser.parse_args()
return args
-if __name__ == "__main__":
- # split_meta_csv()
-
+def main():
args = parse_args()
meta_path = args.meta_path
task = args.task
- if task == "append_format":
- # append_format(meta_path=meta_path, mode='.mp4')
- append_format_pandarallel(meta_path=meta_path, split=args.split, mode=".json")
+ if task == "append_path":
+ append_format_pandarallel(meta_path=meta_path, folder_path=args.folder_path, mode=args.mode)
elif task == "create_subset":
create_subset(meta_path=meta_path)
else:
raise ValueError
- # append_cut(root=root)
- # append_score(root=root)
- # debug_meta_topk()
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/scoring/README.md b/tools/scoring/README.md
index 500d52b..4e18d71 100644
--- a/tools/scoring/README.md
+++ b/tools/scoring/README.md
@@ -51,7 +51,7 @@ wget https://s3.eu-central-1.amazonaws.com/avg-projects/unimatch/pretrained/gmfl
Then run:
```bash
-torchrun --standalone --nproc_per_node 8 tools/scoring/optical_flow/inference.py /path/to/meta.csv
+torchrun --standalone --nproc_per_node 8 tools/scoring/optical_flow/inference.py --meta_path /path/to/meta.csv
```
The output should be `/path/to/meta_flow.csv` with column `flow`.
@@ -64,7 +64,7 @@ For videos, we compute the matching score of the middle frame and the caption.
**Make sure** meta files contain the column `text`, which is the caption of the sample. Then run:
```bash
-torchrun --standalone --nproc_per_node 8 tools/scoring/matching/inference.py /path/to/meta.csv
+torchrun --standalone --nproc_per_node 8 tools/scoring/matching/inference.py --meta_path /path/to/meta.csv
```
The output should be `/path/to/meta_match.csv` with column `match`. Higher matching scores indicate better image-text/video-text alignment.
diff --git a/tools/scoring/matching/inference.py b/tools/scoring/matching/inference.py
index 74c74b7..b1751a8 100644
--- a/tools/scoring/matching/inference.py
+++ b/tools/scoring/matching/inference.py
@@ -61,7 +61,7 @@ def merge_scores(gathered_list: list, meta: pd.DataFrame):
def main():
colossalai.launch_from_torch({})
parser = argparse.ArgumentParser()
- parser.add_argument("meta_path", type=str, help="Path to the input CSV file")
+ parser.add_argument("--meta_path", type=str, help="Path to the input CSV file")
parser.add_argument("--bs", type=int, default=16, help="Batch size")
parser.add_argument("--num_workers", type=int, default=16, help="Number of workers")
args = parser.parse_args()
diff --git a/tools/scoring/ocr/__init__.py b/tools/scoring/ocr/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/scoring/ocr/dbnetpp.py b/tools/scoring/ocr/dbnetpp.py
new file mode 100644
index 0000000..e313fd4
--- /dev/null
+++ b/tools/scoring/ocr/dbnetpp.py
@@ -0,0 +1,64 @@
+model = dict(
+ type='DBNet',
+ backbone=dict(
+ type='CLIPResNet',
+ depth=50,
+ num_stages=4,
+ out_indices=(0, 1, 2, 3),
+ frozen_stages=-1,
+ norm_cfg=dict(type='BN', requires_grad=True),
+ norm_eval=False,
+ style='pytorch',
+ dcn=dict(type='DCNv2', deform_groups=1, fallback_on_stride=False),
+ # init_cfg=dict(
+ # type='Pretrained',
+ # checkpoint='https://download.openmmlab.com/mmocr/backbone/resnet50-oclip-7ba0c533.pth'),
+ stage_with_dcn=(False, True, True, True),
+ ),
+ neck=dict(
+ type='FPNC',
+ in_channels=[256, 512, 1024, 2048],
+ lateral_channels=256,
+ asf_cfg=dict(attention_type='ScaleChannelSpatial'),
+ ),
+ det_head=dict(
+ type='DBHead',
+ in_channels=256,
+ module_loss=dict(type='DBModuleLoss'),
+ postprocessor=dict(
+ type='DBPostprocessor', text_repr_type='quad',
+ epsilon_ratio=0.002,
+ ),
+ ),
+ data_preprocessor=dict(
+ type='TextDetDataPreprocessor',
+ mean=[123.675, 116.28, 103.53],
+ std=[58.395, 57.12, 57.375],
+ bgr_to_rgb=True,
+ pad_size_divisor=32,
+ ),
+ init_cfg=dict(
+ type='Pretrained',
+ checkpoint='https://download.openmmlab.com/mmocr/textdet/dbnetpp/'
+ 'dbnetpp_resnet50-oclip_fpnc_1200e_icdar2015/'
+ 'dbnetpp_resnet50-oclip_fpnc_1200e_icdar2015_20221101_124139-4ecb39ac.pth',
+ )
+)
+
+test_pipeline = [
+ # dict(type='LoadImageFromFile', color_type='color_ignore_orientation'),
+ dict(type='Resize', scale=(4068, 1024), keep_ratio=True),
+ dict(
+ type='PackTextDetInputs',
+ # meta_keys=('img_path', 'ori_shape', 'img_shape', 'scale_factor'),
+ meta_keys=('img_shape', 'scale_factor'),
+ )
+]
+
+# Visualization
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='TextDetLocalVisualizer',
+ name='visualizer',
+ vis_backends=vis_backends,
+)
diff --git a/tools/scoring/ocr/dbnetpp_debug.py b/tools/scoring/ocr/dbnetpp_debug.py
new file mode 100644
index 0000000..c13a433
--- /dev/null
+++ b/tools/scoring/ocr/dbnetpp_debug.py
@@ -0,0 +1,96 @@
+model = dict(
+ type='DBNet',
+ backbone=dict(
+ type='CLIPResNet',
+ depth=50,
+ num_stages=4,
+ out_indices=(0, 1, 2, 3),
+ frozen_stages=-1,
+ norm_cfg=dict(type='BN', requires_grad=True),
+ norm_eval=False,
+ style='pytorch',
+ dcn=dict(type='DCNv2', deform_groups=1, fallback_on_stride=False),
+ # init_cfg=dict(
+ # type='Pretrained',
+ # checkpoint='https://download.openmmlab.com/mmocr/backbone/resnet50-oclip-7ba0c533.pth'),
+ stage_with_dcn=(False, True, True, True),
+ ),
+ neck=dict(
+ type='FPNC',
+ in_channels=[256, 512, 1024, 2048],
+ lateral_channels=256,
+ asf_cfg=dict(attention_type='ScaleChannelSpatial'),
+ ),
+ det_head=dict(
+ type='DBHead',
+ in_channels=256,
+ module_loss=dict(type='DBModuleLoss'),
+ postprocessor=dict(
+ type='DBPostprocessor', text_repr_type='quad',
+ epsilon_ratio=0.002,
+ ),
+ ),
+ data_preprocessor=dict(
+ type='TextDetDataPreprocessor',
+ mean=[123.675, 116.28, 103.53],
+ std=[58.395, 57.12, 57.375],
+ bgr_to_rgb=True,
+ pad_size_divisor=32,
+ ),
+ init_cfg=dict(
+ type='Pretrained',
+ checkpoint='https://download.openmmlab.com/mmocr/textdet/dbnetpp/'
+ 'dbnetpp_resnet50-oclip_fpnc_1200e_icdar2015/'
+ 'dbnetpp_resnet50-oclip_fpnc_1200e_icdar2015_20221101_124139-4ecb39ac.pth',
+ )
+)
+
+train_pipeline = [
+ dict(type='LoadImageFromFile', color_type='color_ignore_orientation'),
+ dict(
+ type='LoadOCRAnnotations',
+ with_bbox=True,
+ with_polygon=True,
+ with_label=True,
+ ),
+ dict(
+ type='TorchVisionWrapper',
+ op='ColorJitter',
+ brightness=32.0 / 255,
+ saturation=0.5),
+ dict(
+ type='ImgAugWrapper',
+ args=[['Fliplr', 0.5],
+ dict(cls='Affine', rotate=[-10, 10]), ['Resize', [0.5, 3.0]]]),
+ dict(type='RandomCrop', min_side_ratio=0.1),
+ dict(type='Resize', scale=(640, 640), keep_ratio=True),
+ dict(type='Pad', size=(640, 640)),
+ dict(
+ type='PackTextDetInputs',
+ meta_keys=('img_path', 'ori_shape', 'img_shape'))
+]
+
+test_pipeline = [
+ dict(type='LoadImageFromFile', color_type='color_ignore_orientation'),
+ dict(type='Resize', scale=(4068, 1024), keep_ratio=True),
+ # dict(
+ # type='LoadOCRAnnotations',
+ # with_polygon=True,
+ # with_bbox=True,
+ # with_label=True,
+ # ),
+ dict(
+ type='PackTextDetInputs',
+ # meta_keys=('img_path', 'ori_shape', 'img_shape', 'scale_factor',
+ # 'instances'),
+ meta_keys=('img_path', 'ori_shape', 'img_shape', 'scale_factor'),
+ )
+]
+
+# Visualization
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='TextDetLocalVisualizer',
+ name='visualizer',
+ vis_backends=vis_backends,
+)
diff --git a/tools/scoring/ocr/debug.py b/tools/scoring/ocr/debug.py
new file mode 100644
index 0000000..6028888
--- /dev/null
+++ b/tools/scoring/ocr/debug.py
@@ -0,0 +1,108 @@
+import os
+import os.path as osp
+from PIL import Image
+import numpy as np
+import torch
+
+import mmcv
+import mmengine
+from mmengine import Config
+from mmengine.registry import DefaultScope
+from mmengine.dataset import Compose, default_collate
+from mmocr.registry import MODELS, VISUALIZERS
+
+
+def visualize(visualizer,
+ inputs,
+ preds,
+ # return_vis: bool = False,
+ show: bool = False,
+ wait_time: int = 0,
+ draw_pred: bool = True,
+ pred_score_thr: float = 0.3,
+ save_vis: bool = False,
+ img_out_dir: str = ''):
+ """Visualize predictions.
+
+ Args:
+ inputs (List[Union[str, np.ndarray]]): Inputs for the inferencer.
+ preds (List[Dict]): Predictions of the model.
+ return_vis (bool): Whether to return the visualization result.
+ Defaults to False.
+ show (bool): Whether to display the image in a popup window.
+ Defaults to False.
+ wait_time (float): The interval of show (s). Defaults to 0.
+ draw_pred (bool): Whether to draw predicted bounding boxes.
+ Defaults to True.
+ pred_score_thr (float): Minimum score of bboxes to draw.
+ Defaults to 0.3.
+ save_vis (bool): Whether to save the visualization result. Defaults
+ to False.
+ img_out_dir (str): Output directory of visualization results.
+ If left as empty, no file will be saved. Defaults to ''.
+
+ Returns:
+ List[np.ndarray] or None: Returns visualization results only if
+ applicable.
+ """
+ results = []
+
+ for single_input, pred in zip(inputs, preds):
+ if isinstance(single_input, str):
+ img_bytes = mmengine.fileio.get(single_input)
+ img = mmcv.imfrombytes(img_bytes, channel_order='rgb')
+ elif isinstance(single_input, np.ndarray):
+ img = single_input.copy()[:, :, ::-1] # to RGB
+ else:
+ raise ValueError('Unsupported input type: '
+ f'{type(single_input)}')
+ img_name = osp.splitext(osp.basename(pred.img_path))[0]
+
+ if save_vis and img_out_dir:
+ out_file = osp.splitext(img_name)[0]
+ out_file = f'{out_file}.jpg'
+ out_file = osp.join(img_out_dir, out_file)
+ else:
+ out_file = None
+
+ visualization = visualizer.add_datasample(
+ img_name,
+ img,
+ pred,
+ show=show,
+ wait_time=wait_time,
+ draw_gt=False,
+ draw_pred=draw_pred,
+ pred_score_thr=pred_score_thr,
+ out_file=out_file,
+ )
+ results.append(visualization)
+
+ return results
+
+
+cfg = Config.fromfile('./tools/scoring/ocr/dbnetpp_debug.py')
+
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+print(device)
+DefaultScope.get_instance('ocr', scope_name='mmocr')
+
+model = MODELS.build(cfg.model)
+model.init_weights()
+model.to(device)
+pipeline = Compose(cfg.test_pipeline)
+visualizer = VISUALIZERS.build(cfg.visualizer)
+
+inputs = {
+ 'img_path': './assets/images/ocr/demo_text_ocr.jpg',
+ # 'img_path': './assets/images/ocr/demo_text_det.jpg',
+}
+results = pipeline(inputs)
+results['index'] = 0
+# imgs = results['inputs'].unsqueeze(0)
+# pred = model.predict(imgs, results['data_samples'])
+data = default_collate([results]) # list[Dict] to Dict
+with torch.no_grad():
+ pred = model.test_step(data)
+vis_results = visualize(visualizer, [x.img_path for x in data['data_samples']], pred, show=True)
+x = 0
\ No newline at end of file
diff --git a/tools/scoring/ocr/inference.py b/tools/scoring/ocr/inference.py
new file mode 100644
index 0000000..9c54bfd
--- /dev/null
+++ b/tools/scoring/ocr/inference.py
@@ -0,0 +1,150 @@
+import argparse
+import os
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.distributed as dist
+import torch.nn.functional as F
+from torchvision.transforms import Resize, CenterCrop, Compose
+from torch.utils.data import DataLoader, DistributedSampler
+from torchvision.datasets.folder import pil_loader
+from tqdm import tqdm
+
+import colossalai
+from mmengine import Config
+from mmengine.registry import DefaultScope
+from mmengine.dataset import Compose, default_collate
+from mmocr.registry import MODELS
+from mmocr.datasets import PackTextDetInputs
+
+from tools.datasets.utils import extract_frames, is_video
+
+
+def merge_scores(gathered_list: list, meta: pd.DataFrame):
+ # reorder
+ indices_list = list(map(lambda x: x[0], gathered_list))
+ scores_list = list(map(lambda x: x[1], gathered_list))
+ flat_indices = []
+ for x in zip(*indices_list):
+ flat_indices.extend(x)
+ flat_scores = []
+ for x in zip(*scores_list):
+ flat_scores.extend(x)
+ flat_indices = np.array(flat_indices)
+ flat_scores = np.array(flat_scores)
+ # filter duplicates
+ unique_indices, unique_indices_idx = np.unique(flat_indices, return_index=True)
+ meta.loc[unique_indices, "ocr"] = flat_scores[unique_indices_idx]
+
+
+class VideoTextDataset(torch.utils.data.Dataset):
+ def __init__(self, meta_path, transform):
+ self.meta_path = meta_path
+ self.meta = pd.read_csv(meta_path)
+ self.transform = transform
+ self.transform = Compose([
+ Resize(1024),
+ CenterCrop(1024),
+ ])
+ self.formatting = PackTextDetInputs(meta_keys=['scale_factor'])
+
+ def __getitem__(self, index):
+ row = self.meta.iloc[index]
+ path = row["path"]
+
+ if is_video(path):
+ img = extract_frames(path, frame_inds=[10], backend="opencv")[0]
+ else:
+ img = pil_loader(path)
+
+ img = self.transform(img)
+ img_array = np.array(img)[:, :, ::-1].copy() # bgr
+ results = {
+ 'img': img_array,
+ 'scale_factor': 1.0,
+ # 'img_shape': img_array.shape[-2],
+ # 'ori_shape': img_array.shape[-2],
+ }
+ results = self.formatting(results)
+ results['index'] = index
+
+ return results
+
+ def __len__(self):
+ return len(self.meta)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--meta_path", type=str, help="Path to the input CSV file")
+ parser.add_argument("--bs", type=int, default=16, help="Batch size")
+ parser.add_argument("--num_workers", type=int, default=16, help="Number of workers")
+ args = parser.parse_args()
+
+ return args
+
+
+def main():
+ args = parse_args()
+ cfg = Config.fromfile('./tools/scoring/ocr/dbnetpp.py')
+
+ meta_path = args.meta_path
+
+ colossalai.launch_from_torch({})
+ device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+ DefaultScope.get_instance('ocr', scope_name='mmocr') # use mmocr Registry as default
+
+ # build model
+ model = MODELS.build(cfg.model)
+ model.init_weights()
+ model.to(device) # set data_preprocessor._device
+ print('==> Model built.')
+
+ # build dataset
+ transform = Compose(cfg.test_pipeline)
+ dataset = VideoTextDataset(meta_path=meta_path, transform=transform)
+ dataloader = DataLoader(
+ dataset,
+ batch_size=args.bs,
+ num_workers=args.num_workers,
+ sampler=DistributedSampler(
+ dataset,
+ num_replicas=dist.get_world_size(),
+ rank=dist.get_rank(),
+ shuffle=False,
+ drop_last=False,
+ ),
+ collate_fn=default_collate,
+ )
+ print('==> Dataloader built.')
+
+ # compute scores
+ dataset.meta["ocr"] = np.nan
+ indices_list = []
+ scores_list = []
+ model.eval()
+ for data in tqdm(dataloader, disable=dist.get_rank() != 0):
+ indices_i = data['index']
+ indices_list.extend(indices_i.tolist())
+ del data['index']
+
+ pred = model.test_step(data) # this line will cast data to device
+
+ num_texts_i = [(x.pred_instances.scores > 0.3).sum().item() for x in pred]
+ scores_list.extend(num_texts_i)
+
+ gathered_list = [None] * dist.get_world_size()
+ dist.all_gather_object(gathered_list, (indices_list, scores_list))
+
+ if dist.get_rank() == 0:
+ merge_scores(gathered_list, dataset.meta)
+
+ wo_ext, ext = os.path.splitext(meta_path)
+ out_path = f"{wo_ext}_ocr{ext}"
+ dataset.meta.to_csv(out_path, index=False)
+ print(f"New meta (shape={dataset.meta.shape}) with ocr results saved to '{out_path}'.")
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/scoring/optical_flow/inference.py b/tools/scoring/optical_flow/inference.py
index 238f406..487121f 100644
--- a/tools/scoring/optical_flow/inference.py
+++ b/tools/scoring/optical_flow/inference.py
@@ -63,7 +63,7 @@ def main():
torch.backends.cudnn.benchmark = False
colossalai.launch_from_torch({})
parser = argparse.ArgumentParser()
- parser.add_argument("meta_path", type=str, help="Path to the input CSV file")
+ parser.add_argument("--meta_path", type=str, help="Path to the input CSV file")
parser.add_argument("--bs", type=int, default=4, help="Batch size")
parser.add_argument("--num_workers", type=int, default=16, help="Number of workers")
args = parser.parse_args()