Open-Sora: Democratizing Efficient Video Production for All
Python
Dev/pxy (#61)
xyupenge933ed8727b3
diff --git a/tools/datasets/filter_panda10m.py b/tools/datasets/filter_panda10m.pynew file mode 100644index 0000000..d4397c2--- /dev/null+++ b/tools/datasets/filter_panda10m.py@@ -0,0 +1,289 @@+# TODO: remove this file before releasing++import argparse+import os+import pandas as pd+import json+import html+from tqdm import tqdm+import re++tqdm.pandas()++try:+ from pandarallel import pandarallel++ pandarallel.initialize(progress_bar=True)+ pandas_has_parallel = True+except ImportError:+ pandas_has_parallel = False+++def apply(df, func, **kwargs):+ if pandas_has_parallel:+ return df.parallel_apply(func, **kwargs)+ return df.progress_apply(func, **kwargs)+++def basic_clean(text):+ import ftfy++ text = ftfy.fix_text(text)+ text = html.unescape(html.unescape(text))+ return text.strip()+++BAD_PUNCT_REGEX = re.compile(+ r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\|" + "\\" + "\/" + "\*" + r"]{1,}"+) # noqa+++def clean_caption(caption):+ import urllib.parse as ul++ from bs4 import BeautifulSoup++ caption = str(caption)+ caption = ul.unquote_plus(caption)+ caption = caption.strip().lower()+ caption = re.sub("<person>", "person", caption)+ # urls:+ caption = re.sub(+ r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ caption = re.sub(+ r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ # html:+ caption = BeautifulSoup(caption, features="html.parser").text++ # @<nickname>+ caption = re.sub(r"@[\w\d]+\b", "", caption)++ # 31C0—31EF CJK Strokes+ # 31F0—31FF Katakana Phonetic Extensions+ # 3200—32FF Enclosed CJK Letters and Months+ # 3300—33FF CJK Compatibility+ # 3400—4DBF CJK Unified Ideographs Extension A+ # 4DC0—4DFF Yijing Hexagram Symbols+ # 4E00—9FFF CJK Unified Ideographs+ caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)+ caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)+ caption = re.sub(r"[\u3200-\u32ff]+", "", caption)+ caption = re.sub(r"[\u3300-\u33ff]+", "", caption)+ caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)+ caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)+ caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)+ #######################################################++ # все виды тире / all types of dash --> "-"+ caption = re.sub(+ r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa+ "-",+ caption,+ )++ # кавычки к одному стандарту+ caption = re.sub(r"[`´«»“”¨]", '"', caption)+ caption = re.sub(r"[‘’]", "'", caption)++ # "+ caption = re.sub(r""?", "", caption)+ # &+ caption = re.sub(r"&", "", caption)++ # ip adresses:+ caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)++ # article ids:+ caption = re.sub(r"\d:\d\d\s+$", "", caption)++ # \n+ caption = re.sub(r"\\n", " ", caption)++ # "#123"+ caption = re.sub(r"#\d{1,3}\b", "", caption)+ # "#12345.."+ caption = re.sub(r"#\d{5,}\b", "", caption)+ # "123456.."+ caption = re.sub(r"\b\d{6,}\b", "", caption)+ # filenames:+ caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)++ #+ caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""+ caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""++ caption = re.sub(BAD_PUNCT_REGEX, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT+ caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "++ # this-is-my-cute-cat / this_is_my_cute_cat+ regex2 = re.compile(r"(?:\-|\_)")+ if len(re.findall(regex2, caption)) > 3:+ caption = re.sub(regex2, " ", caption)++ caption = basic_clean(caption)++ caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640+ caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc+ caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231++ caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)+ caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)+ caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)+ caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption)+ caption = re.sub(r"\bpage\s+\d+\b", "", caption)++ caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a...++ caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)++ caption = re.sub(r"\b\s+\:\s+", r": ", caption)+ caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)+ caption = re.sub(r"\s+", " ", caption)++ caption.strip()++ caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)+ caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)+ caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)+ caption = re.sub(r"^\.\S+$", "", caption)++ return caption.strip()+++def get_10m_set():+ meta_path_10m = '/mnt/hdd/data/Panda-70M/raw/meta/train/panda70m_training_10m.csv'+ meta_10m = pd.read_csv(meta_path_10m)++ def process_single_caption(row):+ text_list = eval(row['caption'])+ clean_list = [clean_caption(x) for x in text_list]+ return str(clean_list)++ ret = apply(meta_10m, process_single_caption, axis=1)+ # ret = meta_10m.progress_apply(process_single_caption, axis=1)+ print('==> text processed.')++ text_list = []+ for x in ret:+ text_list += eval(x)+ # text_set = text_set.union(set(eval(x)))+ text_set = set(text_list)+ # meta_10m['caption_new'] = ret+ # meta_10m.to_csv('/mnt/hdd/data/Panda-70M/raw/meta/train/panda70m_training_10m_new-cap.csv')++ # video_id_set = set(meta_10m['videoID'])+ # id2t = {}+ # for idx, row in tqdm(meta_10m.iterrows(), total=len(meta_10m)):+ # video_id = row['videoID']+ # text_list = eval(row['caption'])+ # id2t[video_id] = set(text_list)++ print(f"==> Loaded meta_10m from '{meta_path_10m}'")+ return text_set+++def filter_panda10m_text(meta_path, text_set):+ def process_single_row(row):+ # path = row['path']+ t = row['text']+ # fname = os.path.basename(path)+ # video_id = fname[:fname.rindex('_')]+ if t not in text_set:+ return False+ return True++ meta = pd.read_csv(meta_path)+ ret = apply(meta, process_single_row, axis=1)+ # ret = meta.progress_apply(process_single_row, axis=1)++ meta = meta[ret]+ wo_ext, ext = os.path.splitext(meta_path)+ out_path = f"{wo_ext}_filter-10m{ext}"+ meta.to_csv(out_path, index=False)+ print(f"New meta (shape={meta.shape}) saved to '{out_path}'.")+++def filter_panda10m_timestamp(meta_path):+ meta_path_10m = '/mnt/hdd/data/Panda-70M/raw/meta/train/panda70m_training_10m.csv'+ meta_10m = pd.read_csv(meta_path_10m)++ id2t = {}+ for idx, row in tqdm(meta_10m.iterrows(), total=len(meta_10m)):+ video_id = row['videoID']+ timestamp = eval(row['timestamp'])+ timestamp = [str(tuple(x)) for x in timestamp]+ id2t[video_id] = timestamp++ # video_id_set_10m = set(meta_10m['videoID'])+ print(f"==> Loaded meta_10m from '{meta_path_10m}'")++ def process_single_row(row):+ path = row['path']+ t = row['timestamp']+ fname = os.path.basename(path)+ video_id = fname[:fname.rindex('_')]+ if video_id not in id2t:+ return False+ if t not in id2t[video_id]:+ return False+ return True+ # return video_id in video_id_set_10m++ meta = pd.read_csv(meta_path)+ ret = apply(meta, process_single_row, axis=1)++ meta = meta[ret]+ wo_ext, ext = os.path.splitext(meta_path)+ out_path = f"{wo_ext}_filter-10m{ext}"+ meta.to_csv(out_path, index=False)+ print(f"New meta (shape={meta.shape}) saved to '{out_path}'.")+++def append_timestamp(meta_path):++ def process_single_row(row):+ path = row['path']+ wo_ext, ext = os.path.splitext(path)+ json_path = f'{wo_ext}.json'+ try:+ with open(json_path, 'r') as f:+ data = json.load(f)+ timestamp = data['clips'][2:-2]+ a, b = timestamp.split(', ')+ timestamp = f"('{a}', '{b}')"+ except Exception as e:+ timestamp = ''+ return timestamp++ meta = pd.read_csv(meta_path)+ ret = apply(meta, process_single_row, axis=1)+ meta['timestamp'] = ret++ wo_ext, ext = os.path.splitext(meta_path)+ out_path = f"{wo_ext}_timestamp{ext}"+ meta.to_csv(out_path, index=False)+ print(f"New meta (shape={meta.shape}) saved to '{out_path}'.")+++def parse_args():+ parser = argparse.ArgumentParser()+ parser.add_argument('--meta_path', type=str, nargs='+')+ parser.add_argument('--num_workers', default=5, type=int)++ args = parser.parse_args()+ return args+++if __name__ == '__main__':+ args = parse_args()+ # append_timestamp(args.meta_path)++ text_set = get_10m_set()+ for x in args.meta_path:+ filter_panda10m_text(x, text_set)diff --git a/tools/scene_cut/README.md b/tools/scene_cut/README.mdindex 492704d..87aa201 100644--- a/tools/scene_cut/README.md+++ b/tools/scene_cut/README.md@@ -1,35 +1,56 @@-## Scene Detection and Video Splitting+# Scene Detection and Video Splitting+In many cases, raw videos contain several scenes and are too long for training. Thus, it is essential to split them into shorter+clips based on scenes. Here, we provide code for scene detection and video splitting.-### Formatting--Input meta should be `{prefix}.csv` with column `'videoId'`+## Formatting+At this step, you should have a raw video dataset prepared. We need a meta file for the dataset. To create a meta file from a folder, run:```bash-python tools/scene_cut/process_meta.py --task append_format --meta_path /mnt/hdd/data/pexels_new/raw/meta/popular_6.csv --split popular_6+python -m tools.datasets.convert video /path/to/video/folder --output /path/to/save/meta.csv```+This should output a `.csv` file with column `path`.++If you already have a meta file for the videos and want to keep the information.+**Make sure** the meta file has column `id`, which is the id for each video, and the video is named as `{id}.mp4`.+The following command will add a new column `path` to the meta file.-Output is `{prefix}_format.csv` (with column `path`) and `{prefix}_intact.csv` (with column `intact` and `path`)+```bash+python tools/scene_cut/process_meta.py --task append_path --meta_path /path/to/meta.csv --folder_path /path/to/video/folder+```+This should output+- `{prefix}_path-filtered.csv` with column `path` (broken videos filtered)+- `{prefix}_path_intact.csv` with column `path` and `intact` (`intact` indicating a video is intact or not)-### Scene Detection-Input meta should be `{prefix}_format.csv`+## Scene Detection+The next step is to detect scenes in a video.+We use [`PySceneDetect`](https://github.com/Breakthrough/PySceneDetect) for this job.+**Make sure** the input meta file has column `path`, which is the path of a video.```bash+python tools/scene_cut/scene_detect.py --meta_path /path/to/meta.csvpython tools/scene_cut/scene_detect.py --meta_path /mnt/hdd/data/pexels_new/raw/meta/popular_6_format.csv```+The output is `{prefix}_timestamp.csv` with column `timestamp`. Each cell in column `timestamp` is a list of tuples,+with each tuple indicating the start and end timestamp of a scene+(e.g., `[('00:00:01.234', '00:00:02.345'), ('00:00:03.456', '00:00:04.567')]`).-Output is `{prefix}_format_timestamp.csv`--### Video Splitting+## Video Splitting+After obtaining timestamps for scenes, we conduct video splitting (cutting).+**Make sure** the meta file contains column `timestamp`.-Input meta should be `{prefix}_timestamp.csv`+TODO: output video size, min_duration, max_duration```bash+python tools/scene_cut/main_cut_pandarallel.py \+ --meta_path /path/to/meta.csv \+ --out_dir /path/to/output/dir+python tools/scene_cut/main_cut_pandarallel.py \--meta_path /mnt/hdd/data/pexels_new/raw/meta/popular_6_format_timestamp.csv \--out_dir /mnt/hdd/data/pexels_new/scene_cut/data/popular_6```-Output is `{out_dir}/{wo_ext}_scene-{sid}.mp4`+This yields video clips saved in `/path/to/output/dir`. The video clips are named as `{video_id}_scene-{scene_id}.mp4`TODO: meta for video clipsdiff --git a/tools/scene_cut/process_meta.py b/tools/scene_cut/process_meta.pyindex 7a664b0..9d982de 100644--- a/tools/scene_cut/process_meta.py+++ b/tools/scene_cut/process_meta.py@@ -36,136 +36,48 @@ def has_downloaded_success(json_path):return True-def split_meta_csv(chunk_size=60000):- """- Split csv into multiple small csv in order- """- root = "./data/Panda-70M"- # meta_name = 'meta/panda70m_training_full.csv'- meta_name = "meta/panda70m_training_10m.csv"- # meta_name = 'meta/training_10m/train_0.csv'- meta_path = os.path.join(root, meta_name)-- df = pd.read_csv(meta_path)- num_rows = len(df)-- # Split the DataFrame into smaller DataFrames- for idx, i in enumerate(range(0, num_rows, chunk_size)):- df_i = df.iloc[i : i + chunk_size]- out_path = os.path.join(root, f"meta/train_{idx}.csv")- df_i.to_csv(out_path, index=False)-- # If there are remaining rows- if num_rows > chunk_size and num_rows % chunk_size != 0:- df_last = df.iloc[-(num_rows % chunk_size) :]- out_path = os.path.join(root, f"meta/train_{idx + 1}.csv")- df_last.to_csv(out_path, index=False)---def remove_index():- df = pd.read_csv("your_file.csv", index_col=0)- df.to_csv("your_file_without_index.csv", index=False)---def append_format(meta_path, mode=".mp4"):- """- Append _format to csv file:- - filter broken videos; only intact videos are kept- - add column 'path'-- input csv should satisfy:- - name should be: {split}.csv- - contain column 'videoID'/'videoId'- """- # meta_path = os.path.join(root, f'raw/meta/{split}.csv')- meta_dirname = os.path.dirname(meta_path)- assert meta_dirname.endswith("raw/meta")- root_raw = os.path.dirname(meta_dirname)-- meta_fname = os.path.basename(meta_path)- split, ext = os.path.splitext(meta_fname)-- meta = pd.read_csv(meta_path)-- path_list = []- new_meta = []- for idx, row in tqdm(meta.iterrows(), total=len(meta)):- # video_id = row['videoID'] # panda- video_id = row["videoId"] # pexels_new- video_path = os.path.join(root_raw, f"data/{split}/{video_id}.mp4")- if mode == ".mp4":- if not is_intact_video(video_path):- continue- elif mode == ".json":- json_path = os.path.join(root_raw, f"data/{split}/{video_id}.json")- if not has_downloaded_success(json_path):- continue- else:- raise ValueError-- new_meta.append(row)- path_list.append(video_path)-- new_meta = pd.DataFrame(new_meta)- new_meta["path"] = path_list-- out_path = os.path.join(root_raw, f"meta/{split}_format.csv")- new_meta.to_csv(out_path, index=False)- print(f"New meta (shape={new_meta.shape}) saved to '{out_path}'")---def append_format_pandarallel(meta_path, split, mode=".mp4"):- """- Append _format to csv file:- - filter broken videos; only intact videos are kept- - add column 'path'-- input csv should satisfy:- - name should be: {split}.csv- - contain column 'videoID'/'videoId'- """- # meta_path = os.path.join(root, f'raw/meta/{split}.csv')- meta_dirname = os.path.dirname(meta_path)- assert meta_dirname.endswith("raw/meta")- root_raw = os.path.dirname(meta_dirname)-- meta_fname = os.path.basename(meta_path)- wo_ext, ext = os.path.splitext(meta_fname)-- meta = pd.read_csv(meta_path)-+def append_format_pandarallel(meta_path, folder_path, mode=".json"):def is_intact(row, mode=".json"):- video_id = row["videoId"] # pexels_new- video_path = os.path.join(root_raw, f"data/{split}/{video_id}.mp4")+ video_id = row["id"]+ # video_path = os.path.join(root_raw, f"data/{split}/{video_id}.mp4")+ video_path = os.path.join(folder_path, f"{video_id}.mp4")row["path"] = video_pathif mode == ".mp4":if is_intact_video(video_path):return True, video_pathreturn False, video_pathelif mode == ".json":- json_path = os.path.join(root_raw, f"data/{split}/{video_id}.json")+ # json_path = os.path.join(root_raw, f"data/{split}/{video_id}.json")+ json_path = os.path.join(folder_path, f"{video_id}.json")if has_downloaded_success(json_path):return True, video_pathreturn False, video_path+ elif mode is None:+ return True, video_pathelse:raise ValueError+ meta_dirpath = os.path.dirname(meta_path)+ meta_fname = os.path.basename(meta_path)+ wo_ext, ext = os.path.splitext(meta_fname)+pandarallel.initialize(progress_bar=True)is_intact_partial = partial(is_intact, mode=mode)- ret = meta.parallel_apply(is_intact_partial, axis=1)+ meta = pd.read_csv(meta_path)+ ret = meta.parallel_apply(is_intact_partial, axis=1)intact, paths = list(zip(*ret))meta["intact"] = intactmeta["path"] = paths- out_path = os.path.join(root_raw, f"meta/{wo_ext}_intact.csv")+ out_path = os.path.join(meta_dirpath, f"{wo_ext}_path_intact.csv")meta.to_csv(out_path, index=False)print(f"New meta (shape={meta.shape}) with intact info saved to '{out_path}'")# meta_format = meta[meta['intact']]meta_format = meta[np.array(intact)]meta_format.drop("intact", axis=1, inplace=True)- out_path = os.path.join(root_raw, f"meta/{wo_ext}_format.csv")+ out_path = os.path.join(meta_dirpath, f"{wo_ext}_path-filtered.csv")meta_format.to_csv(out_path, index=False)print(f"New meta (shape={meta_format.shape}) with format info saved to '{out_path}'")@@ -180,84 +92,30 @@ def create_subset(meta_path):print(f"New meta (shape={meta_subset.shape}) saved to '{out_path}'")-def append_cut(root="./data/Panda-70M"):- """- Append _cut to csv file- input csv should satisfy:- - name_should be {split}_intact.csv- - contain column 'timestamp': list of timestamp- """- split = "test"- meta_path = os.path.join(root, f"processed/meta/{split}_intact.csv")-- wo_ext, ext = os.path.splitext(meta_path)- suffix = "cut"- out_path = f"{wo_ext}_{suffix}{ext}"-- meta = pd.read_csv(meta_path)-- new_meta = []- for idx, row in tqdm(meta.iterrows(), total=len(meta)):- video_id = row["videoID"]- timestamps = eval(row["timestamp"])- captions = eval(row["caption"])- scores = eval(row["matching_score"])-- num_clips = len(timestamps)- for idx_c in range(num_clips):- path_i = os.path.join(root, f"processed/{split}/{video_id}_scene-{idx_c}.mp4")- # if not is_intact_video(path_i):- # continue-- row_i = [f"{video_id}_scene-{idx_c}", path_i, timestamps[idx_c], captions[idx_c], scores[idx_c]]-- new_meta.append(row_i)-- columns = ["videoID", "path", "timestamp", "text", "match_official"]- new_meta = pd.DataFrame(new_meta, columns=columns)-- new_meta.to_csv(out_path, index=False)- print(f"New meta (shape={new_meta.shape}) saved to '{out_path}'")---def debug_meta_topk():- meta_path = "F:/Panda-70M/meta/test_intact_cut_flow.csv"- meta = pd.read_csv(meta_path)-- score_column = "flow_score"- topk = meta.nlargest(10, columns=score_column)- topk_s = meta.nsmallest(200, columns=score_column)-- [(row["path"], row["caption"], row[score_column]) for idx, row in topk.iterrows()]- [(row["path"], row["caption"], row[score_column]) for idx, row in topk_s.iterrows()]--def parse_args():parser = argparse.ArgumentParser()- parser.add_argument("--task", default="append_format")- parser.add_argument("--meta_path", default="./data/pexels_new/raw/meta/popular_1.csv")- parser.add_argument("--split", default="popular_5")+ parser.add_argument("--task", default="append_path", required=True)+ parser.add_argument("--meta_path", type=str, required=True)+ parser.add_argument("--folder_path", type=str, required=True)+ parser.add_argument("--mode", type=str, default=None)parser.add_argument("--num_workers", default=5, type=int)args = parser.parse_args()return args-if __name__ == "__main__":- # split_meta_csv()-+def main():args = parse_args()meta_path = args.meta_pathtask = args.task- if task == "append_format":- # append_format(meta_path=meta_path, mode='.mp4')- append_format_pandarallel(meta_path=meta_path, split=args.split, mode=".json")+ if task == "append_path":+ append_format_pandarallel(meta_path=meta_path, folder_path=args.folder_path, mode=args.mode)elif task == "create_subset":create_subset(meta_path=meta_path)else:raise ValueError- # append_cut(root=root)- # append_score(root=root)- # debug_meta_topk()++if __name__ == "__main__":+ main()diff --git a/tools/scoring/README.md b/tools/scoring/README.mdindex 500d52b..4e18d71 100644--- a/tools/scoring/README.md+++ b/tools/scoring/README.md@@ -51,7 +51,7 @@ wget https://s3.eu-central-1.amazonaws.com/avg-projects/unimatch/pretrained/gmflThen run:```bash-torchrun --standalone --nproc_per_node 8 tools/scoring/optical_flow/inference.py /path/to/meta.csv+torchrun --standalone --nproc_per_node 8 tools/scoring/optical_flow/inference.py --meta_path /path/to/meta.csv```The output should be `/path/to/meta_flow.csv` with column `flow`.@@ -64,7 +64,7 @@ For videos, we compute the matching score of the middle frame and the caption.**Make sure** meta files contain the column `text`, which is the caption of the sample. Then run:```bash-torchrun --standalone --nproc_per_node 8 tools/scoring/matching/inference.py /path/to/meta.csv+torchrun --standalone --nproc_per_node 8 tools/scoring/matching/inference.py --meta_path /path/to/meta.csv```The output should be `/path/to/meta_match.csv` with column `match`. Higher matching scores indicate better image-text/video-text alignment.diff --git a/tools/scoring/matching/inference.py b/tools/scoring/matching/inference.pyindex 74c74b7..b1751a8 100644--- a/tools/scoring/matching/inference.py+++ b/tools/scoring/matching/inference.py@@ -61,7 +61,7 @@ def merge_scores(gathered_list: list, meta: pd.DataFrame):def main():colossalai.launch_from_torch({})parser = argparse.ArgumentParser()- parser.add_argument("meta_path", type=str, help="Path to the input CSV file")+ parser.add_argument("--meta_path", type=str, help="Path to the input CSV file")parser.add_argument("--bs", type=int, default=16, help="Batch size")parser.add_argument("--num_workers", type=int, default=16, help="Number of workers")args = parser.parse_args()diff --git a/tools/scoring/ocr/__init__.py b/tools/scoring/ocr/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/tools/scoring/ocr/dbnetpp.py b/tools/scoring/ocr/dbnetpp.pynew file mode 100644index 0000000..e313fd4--- /dev/null+++ b/tools/scoring/ocr/dbnetpp.py@@ -0,0 +1,64 @@+model = dict(+ type='DBNet',+ backbone=dict(+ type='CLIPResNet',+ depth=50,+ num_stages=4,+ out_indices=(0, 1, 2, 3),+ frozen_stages=-1,+ norm_cfg=dict(type='BN', requires_grad=True),+ norm_eval=False,+ style='pytorch',+ dcn=dict(type='DCNv2', deform_groups=1, fallback_on_stride=False),+ # init_cfg=dict(+ # type='Pretrained',+ # checkpoint='https://download.openmmlab.com/mmocr/backbone/resnet50-oclip-7ba0c533.pth'),+ stage_with_dcn=(False, True, True, True),+ ),+ neck=dict(+ type='FPNC',+ in_channels=[256, 512, 1024, 2048],+ lateral_channels=256,+ asf_cfg=dict(attention_type='ScaleChannelSpatial'),+ ),+ det_head=dict(+ type='DBHead',+ in_channels=256,+ module_loss=dict(type='DBModuleLoss'),+ postprocessor=dict(+ type='DBPostprocessor', text_repr_type='quad',+ epsilon_ratio=0.002,+ ),+ ),+ data_preprocessor=dict(+ type='TextDetDataPreprocessor',+ mean=[123.675, 116.28, 103.53],+ std=[58.395, 57.12, 57.375],+ bgr_to_rgb=True,+ pad_size_divisor=32,+ ),+ init_cfg=dict(+ type='Pretrained',+ checkpoint='https://download.openmmlab.com/mmocr/textdet/dbnetpp/'+ 'dbnetpp_resnet50-oclip_fpnc_1200e_icdar2015/'+ 'dbnetpp_resnet50-oclip_fpnc_1200e_icdar2015_20221101_124139-4ecb39ac.pth',+ )+)++test_pipeline = [+ # dict(type='LoadImageFromFile', color_type='color_ignore_orientation'),+ dict(type='Resize', scale=(4068, 1024), keep_ratio=True),+ dict(+ type='PackTextDetInputs',+ # meta_keys=('img_path', 'ori_shape', 'img_shape', 'scale_factor'),+ meta_keys=('img_shape', 'scale_factor'),+ )+]++# Visualization+vis_backends = [dict(type='LocalVisBackend')]+visualizer = dict(+ type='TextDetLocalVisualizer',+ name='visualizer',+ vis_backends=vis_backends,+)diff --git a/tools/scoring/ocr/dbnetpp_debug.py b/tools/scoring/ocr/dbnetpp_debug.pynew file mode 100644index 0000000..c13a433--- /dev/null+++ b/tools/scoring/ocr/dbnetpp_debug.py@@ -0,0 +1,96 @@+model = dict(+ type='DBNet',+ backbone=dict(+ type='CLIPResNet',+ depth=50,+ num_stages=4,+ out_indices=(0, 1, 2, 3),+ frozen_stages=-1,+ norm_cfg=dict(type='BN', requires_grad=True),+ norm_eval=False,+ style='pytorch',+ dcn=dict(type='DCNv2', deform_groups=1, fallback_on_stride=False),+ # init_cfg=dict(+ # type='Pretrained',+ # checkpoint='https://download.openmmlab.com/mmocr/backbone/resnet50-oclip-7ba0c533.pth'),+ stage_with_dcn=(False, True, True, True),+ ),+ neck=dict(+ type='FPNC',+ in_channels=[256, 512, 1024, 2048],+ lateral_channels=256,+ asf_cfg=dict(attention_type='ScaleChannelSpatial'),+ ),+ det_head=dict(+ type='DBHead',+ in_channels=256,+ module_loss=dict(type='DBModuleLoss'),+ postprocessor=dict(+ type='DBPostprocessor', text_repr_type='quad',+ epsilon_ratio=0.002,+ ),+ ),+ data_preprocessor=dict(+ type='TextDetDataPreprocessor',+ mean=[123.675, 116.28, 103.53],+ std=[58.395, 57.12, 57.375],+ bgr_to_rgb=True,+ pad_size_divisor=32,+ ),+ init_cfg=dict(+ type='Pretrained',+ checkpoint='https://download.openmmlab.com/mmocr/textdet/dbnetpp/'+ 'dbnetpp_resnet50-oclip_fpnc_1200e_icdar2015/'+ 'dbnetpp_resnet50-oclip_fpnc_1200e_icdar2015_20221101_124139-4ecb39ac.pth',+ )+)++train_pipeline = [+ dict(type='LoadImageFromFile', color_type='color_ignore_orientation'),+ dict(+ type='LoadOCRAnnotations',+ with_bbox=True,+ with_polygon=True,+ with_label=True,+ ),+ dict(+ type='TorchVisionWrapper',+ op='ColorJitter',+ brightness=32.0 / 255,+ saturation=0.5),+ dict(+ type='ImgAugWrapper',+ args=[['Fliplr', 0.5],+ dict(cls='Affine', rotate=[-10, 10]), ['Resize', [0.5, 3.0]]]),+ dict(type='RandomCrop', min_side_ratio=0.1),+ dict(type='Resize', scale=(640, 640), keep_ratio=True),+ dict(type='Pad', size=(640, 640)),+ dict(+ type='PackTextDetInputs',+ meta_keys=('img_path', 'ori_shape', 'img_shape'))+]++test_pipeline = [+ dict(type='LoadImageFromFile', color_type='color_ignore_orientation'),+ dict(type='Resize', scale=(4068, 1024), keep_ratio=True),+ # dict(+ # type='LoadOCRAnnotations',+ # with_polygon=True,+ # with_bbox=True,+ # with_label=True,+ # ),+ dict(+ type='PackTextDetInputs',+ # meta_keys=('img_path', 'ori_shape', 'img_shape', 'scale_factor',+ # 'instances'),+ meta_keys=('img_path', 'ori_shape', 'img_shape', 'scale_factor'),+ )+]++# Visualization+vis_backends = [dict(type='LocalVisBackend')]+visualizer = dict(+ type='TextDetLocalVisualizer',+ name='visualizer',+ vis_backends=vis_backends,+)diff --git a/tools/scoring/ocr/debug.py b/tools/scoring/ocr/debug.pynew file mode 100644index 0000000..6028888--- /dev/null+++ b/tools/scoring/ocr/debug.py@@ -0,0 +1,108 @@+import os+import os.path as osp+from PIL import Image+import numpy as np+import torch++import mmcv+import mmengine+from mmengine import Config+from mmengine.registry import DefaultScope+from mmengine.dataset import Compose, default_collate+from mmocr.registry import MODELS, VISUALIZERS+++def visualize(visualizer,+ inputs,+ preds,+ # return_vis: bool = False,+ show: bool = False,+ wait_time: int = 0,+ draw_pred: bool = True,+ pred_score_thr: float = 0.3,+ save_vis: bool = False,+ img_out_dir: str = ''):+ """Visualize predictions.++ Args:+ inputs (List[Union[str, np.ndarray]]): Inputs for the inferencer.+ preds (List[Dict]): Predictions of the model.+ return_vis (bool): Whether to return the visualization result.+ Defaults to False.+ show (bool): Whether to display the image in a popup window.+ Defaults to False.+ wait_time (float): The interval of show (s). Defaults to 0.+ draw_pred (bool): Whether to draw predicted bounding boxes.+ Defaults to True.+ pred_score_thr (float): Minimum score of bboxes to draw.+ Defaults to 0.3.+ save_vis (bool): Whether to save the visualization result. Defaults+ to False.+ img_out_dir (str): Output directory of visualization results.+ If left as empty, no file will be saved. Defaults to ''.++ Returns:+ List[np.ndarray] or None: Returns visualization results only if+ applicable.+ """+ results = []++ for single_input, pred in zip(inputs, preds):+ if isinstance(single_input, str):+ img_bytes = mmengine.fileio.get(single_input)+ img = mmcv.imfrombytes(img_bytes, channel_order='rgb')+ elif isinstance(single_input, np.ndarray):+ img = single_input.copy()[:, :, ::-1] # to RGB+ else:+ raise ValueError('Unsupported input type: '+ f'{type(single_input)}')+ img_name = osp.splitext(osp.basename(pred.img_path))[0]++ if save_vis and img_out_dir:+ out_file = osp.splitext(img_name)[0]+ out_file = f'{out_file}.jpg'+ out_file = osp.join(img_out_dir, out_file)+ else:+ out_file = None++ visualization = visualizer.add_datasample(+ img_name,+ img,+ pred,+ show=show,+ wait_time=wait_time,+ draw_gt=False,+ draw_pred=draw_pred,+ pred_score_thr=pred_score_thr,+ out_file=out_file,+ )+ results.append(visualization)++ return results+++cfg = Config.fromfile('./tools/scoring/ocr/dbnetpp_debug.py')++device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")+print(device)+DefaultScope.get_instance('ocr', scope_name='mmocr')++model = MODELS.build(cfg.model)+model.init_weights()+model.to(device)+pipeline = Compose(cfg.test_pipeline)+visualizer = VISUALIZERS.build(cfg.visualizer)++inputs = {+ 'img_path': './assets/images/ocr/demo_text_ocr.jpg',+ # 'img_path': './assets/images/ocr/demo_text_det.jpg',+}+results = pipeline(inputs)+results['index'] = 0+# imgs = results['inputs'].unsqueeze(0)+# pred = model.predict(imgs, results['data_samples'])+data = default_collate([results]) # list[Dict] to Dict+with torch.no_grad():+ pred = model.test_step(data)+vis_results = visualize(visualizer, [x.img_path for x in data['data_samples']], pred, show=True)+x = 0\ No newline at end of filediff --git a/tools/scoring/ocr/inference.py b/tools/scoring/ocr/inference.pynew file mode 100644index 0000000..9c54bfd--- /dev/null+++ b/tools/scoring/ocr/inference.py@@ -0,0 +1,150 @@+import argparse+import os++import numpy as np+import pandas as pd+import torch+import torch.distributed as dist+import torch.nn.functional as F+from torchvision.transforms import Resize, CenterCrop, Compose+from torch.utils.data import DataLoader, DistributedSampler+from torchvision.datasets.folder import pil_loader+from tqdm import tqdm++import colossalai+from mmengine import Config+from mmengine.registry import DefaultScope+from mmengine.dataset import Compose, default_collate+from mmocr.registry import MODELS+from mmocr.datasets import PackTextDetInputs++from tools.datasets.utils import extract_frames, is_video+++def merge_scores(gathered_list: list, meta: pd.DataFrame):+ # reorder+ indices_list = list(map(lambda x: x[0], gathered_list))+ scores_list = list(map(lambda x: x[1], gathered_list))+ flat_indices = []+ for x in zip(*indices_list):+ flat_indices.extend(x)+ flat_scores = []+ for x in zip(*scores_list):+ flat_scores.extend(x)+ flat_indices = np.array(flat_indices)+ flat_scores = np.array(flat_scores)+ # filter duplicates+ unique_indices, unique_indices_idx = np.unique(flat_indices, return_index=True)+ meta.loc[unique_indices, "ocr"] = flat_scores[unique_indices_idx]+++class VideoTextDataset(torch.utils.data.Dataset):+ def __init__(self, meta_path, transform):+ self.meta_path = meta_path+ self.meta = pd.read_csv(meta_path)+ self.transform = transform+ self.transform = Compose([+ Resize(1024),+ CenterCrop(1024),+ ])+ self.formatting = PackTextDetInputs(meta_keys=['scale_factor'])++ def __getitem__(self, index):+ row = self.meta.iloc[index]+ path = row["path"]++ if is_video(path):+ img = extract_frames(path, frame_inds=[10], backend="opencv")[0]+ else:+ img = pil_loader(path)++ img = self.transform(img)+ img_array = np.array(img)[:, :, ::-1].copy() # bgr+ results = {+ 'img': img_array,+ 'scale_factor': 1.0,+ # 'img_shape': img_array.shape[-2],+ # 'ori_shape': img_array.shape[-2],+ }+ results = self.formatting(results)+ results['index'] = index++ return results++ def __len__(self):+ return len(self.meta)+++def parse_args():+ parser = argparse.ArgumentParser()+ parser.add_argument("--meta_path", type=str, help="Path to the input CSV file")+ parser.add_argument("--bs", type=int, default=16, help="Batch size")+ parser.add_argument("--num_workers", type=int, default=16, help="Number of workers")+ args = parser.parse_args()++ return args+++def main():+ args = parse_args()+ cfg = Config.fromfile('./tools/scoring/ocr/dbnetpp.py')++ meta_path = args.meta_path++ colossalai.launch_from_torch({})+ device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")+ DefaultScope.get_instance('ocr', scope_name='mmocr') # use mmocr Registry as default++ # build model+ model = MODELS.build(cfg.model)+ model.init_weights()+ model.to(device) # set data_preprocessor._device+ print('==> Model built.')++ # build dataset+ transform = Compose(cfg.test_pipeline)+ dataset = VideoTextDataset(meta_path=meta_path, transform=transform)+ dataloader = DataLoader(+ dataset,+ batch_size=args.bs,+ num_workers=args.num_workers,+ sampler=DistributedSampler(+ dataset,+ num_replicas=dist.get_world_size(),+ rank=dist.get_rank(),+ shuffle=False,+ drop_last=False,+ ),+ collate_fn=default_collate,+ )+ print('==> Dataloader built.')++ # compute scores+ dataset.meta["ocr"] = np.nan+ indices_list = []+ scores_list = []+ model.eval()+ for data in tqdm(dataloader, disable=dist.get_rank() != 0):+ indices_i = data['index']+ indices_list.extend(indices_i.tolist())+ del data['index']++ pred = model.test_step(data) # this line will cast data to device++ num_texts_i = [(x.pred_instances.scores > 0.3).sum().item() for x in pred]+ scores_list.extend(num_texts_i)++ gathered_list = [None] * dist.get_world_size()+ dist.all_gather_object(gathered_list, (indices_list, scores_list))++ if dist.get_rank() == 0:+ merge_scores(gathered_list, dataset.meta)++ wo_ext, ext = os.path.splitext(meta_path)+ out_path = f"{wo_ext}_ocr{ext}"+ dataset.meta.to_csv(out_path, index=False)+ print(f"New meta (shape={dataset.meta.shape}) with ocr results saved to '{out_path}'.")+++if __name__ == '__main__':+ main()diff --git a/tools/scoring/optical_flow/inference.py b/tools/scoring/optical_flow/inference.pyindex 238f406..487121f 100644--- a/tools/scoring/optical_flow/inference.py+++ b/tools/scoring/optical_flow/inference.py@@ -63,7 +63,7 @@ def main():torch.backends.cudnn.benchmark = Falsecolossalai.launch_from_torch({})parser = argparse.ArgumentParser()- parser.add_argument("meta_path", type=str, help="Path to the input CSV file")+ parser.add_argument("--meta_path", type=str, help="Path to the input CSV file")parser.add_argument("--bs", type=int, default=4, help="Batch size")parser.add_argument("--num_workers", type=int, default=16, help="Number of workers")args = parser.parse_args()