Peach: Local-first video metadata scraper and media manager.
fix(upgrade): 清掉旧版从文件名误认的番号
longmeidao2de9795dd2cb
diff --git a/docs/SOURCING.md b/docs/SOURCING.mdindex 6b18bc10e..bdeb35ca3 100644--- a/docs/SOURCING.md+++ b/docs/SOURCING.md@@ -645,6 +645,10 @@ Instagram 的独立用户登录会话未取得,自动适配器不进正式依是「无码流出」;明写 `破解`/`AI去码`/`-LTX`/`-UMR`/`-C-U` 是「无码破解」,`-C-U` 不另算中字。有码片商的番号只带`-U`/`-UC`/`-UN`/`uncensored` 或标签「无码」时也算破解:真流出的片很少,流传的去码版绝大多数是破解;FC2 这类本身可能无码的片仍是「无码」。JavDB 的「無碼流出」是作品级标签,只留在来源快照里,不据此判断本地文件。+- 旧版从文件名误认的番号由升级补修清一次(`upgrade_repairs.stale_filename_codes`):手机录像的时间戳+ (`VID_20220818_125735_816` 记成 `125735-816`)、个人投稿者被截断的「名字_日期」(`KUZU_250103` 记成 `KUZU-25010`)。+ 只清无主或文件名扫描写入、现行规则从文件名和文件夹都认不出的番号,且番号已不合现行形状,或番号没完整出现在名字里、+ 这一行也没有任何刮削资料。清掉的行与原值记在 `ledger_upgrade_repair` 的 `stale-filename-codes-1` 回执里,可整批撤回。- 推广域名会出现在番号的头、尾和方括号里(`www.98t.la@ABW-358-U`、`ABP-762-fuckbe.com`、`[xxx.cc]ABC-123`)。番号提取与目录判重共用 `strip_promo_markers` 这一层;各剥一半,叠了两层的 `[98t.tv][98t.tv]ABW-251` 就会在其中一处漏网。diff --git a/src/peach/upgrade_repairs.py b/src/peach/upgrade_repairs.pyindex cb1e2984f..7616e9918 100644--- a/src/peach/upgrade_repairs.py+++ b/src/peach/upgrade_repairs.py@@ -59,6 +59,43 @@ def _manual_name(connection, entity_id: int) -> bool:(entity_id,)).fetchone())+#: 旧版从文件名误认的番号只清一次;清掉的行与原值记进同一条回执,可以按它整批撤回。+STALE_CODES_KEY = 'stale-filename-codes-1'+++def stale_filename_codes(connection) -> list[dict]:+ """旧版从文件名误认、现行规则已经不认的番号。++ 实例是手机录像的时间戳(`VID_20220818_125735_816` 记成 `125735-816`)和被截断的「名字_日期」+ (个人投稿者的 `KUZU_250103` 记成 `KUZU-25010`)。这类行有了番号就被当成 JAV,日期式番号还会拿到+ 「无码」徽章。只看无主或由文件名扫描写入的番号,且文件名与所在文件夹按现行规则都认不出番号;+ 再满足下面一条才清:番号本身已不合现行形状;或番号没有完整出现在文件名与文件夹里,这一行也没有+ 任何目录资料(标题、片商、发行日)。带资料的那类是刮削按另一份身份改过的(`1pon-092415-001-fhd2`+ 实为 `092415_159`),不碰。+ """+ import re+ from pathlib import PureWindowsPath+ from .catalog_rules import compile_code_patterns, is_jav_code, normalise_code_key, release_code_from_filename+ from .profiles import read_settings+ patterns = compile_code_patterns(read_settings(connection).get('codePatterns'))+ rows = []+ for row in connection.execute(+ "SELECT id,name,path,code,field_owners,catalog_title,studio,release_date FROM asset "+ "WHERE medium='video' AND code IS NOT NULL AND code<>''"):+ owner = owner_of(row['field_owners'], 'code')+ if owner not in ('', SCAN_FILENAME):+ continue+ texts = (str(row['name'] or ''), PureWindowsPath(str(row['path'] or '')).parent.name)+ if any(release_code_from_filename(text, patterns=patterns) for text in texts):+ continue+ body = re.escape(normalise_code_key(row['code'])).replace(r'\-', '[-_ ]?')+ whole = any(re.search(rf'(?<![A-Za-z0-9]){body}(?![A-Za-z0-9])', text, re.I) for text in texts)+ described = row['catalog_title'] or row['studio'] or row['release_date']+ if not is_jav_code(row['code']) or (not whole and not described):+ rows.append({'id': row['id'], 'code': row['code']})+ return rows++def _catalog_plan(connection, sources: Path, *, catalog: bool = True) -> dict:text = metadata_text_repair.collect(connection, sources / 'library-metadata')if not catalog:@@ -231,6 +268,18 @@ class UpgradeRepairs:(REVISION, str(time.time()), json.dumps(receipt, ensure_ascii=False)))atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))+ def _clear_stale_codes(self) -> None:+ """清掉旧版从文件名误认的番号(`stale_filename_codes`),只做一次。"""+ with self.contract.database.write_transaction() as connection:+ if connection.execute('SELECT 1 FROM ledger_upgrade_repair WHERE id=?', (STALE_CODES_KEY,)).fetchone():+ return+ rows = stale_filename_codes(connection)+ if rows:+ write_owned_fields(connection, [row['id'] for row in rows], {'code': None}, SCAN_FILENAME)+ connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',+ (STALE_CODES_KEY, str(time.time()), json.dumps({'cleared': rows})))+ LOGGER.info('旧版误认的番号:清掉 %s 条', len(rows))+def _release_studio_misses(self) -> None:"""旧版断网时记下的「官网未取得」各放一次重试(`studio_followup.release_unbounded_misses`)。@@ -446,6 +495,7 @@ class UpgradeRepairs:self.scheduler.add_job(self.tick, 'interval', seconds=60, max_instances=1, coalesce=True)self.scheduler.add_job(self._settle_verification, max_instances=1)self.scheduler.add_job(self._release_studio_misses, max_instances=1)+ self.scheduler.add_job(self._clear_stale_codes, max_instances=1)self.scheduler.start()def stop(self) -> None:diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.pyindex 6f700e56e..6dfc9d007 100644--- a/tests/test_upgrade_repairs.py+++ b/tests/test_upgrade_repairs.py@@ -280,6 +280,39 @@ class UpgradeRepairTests(unittest.TestCase):"VALUES(?,'local',?,?,'video',?,?,?)",(key, f'{key}.mp4', 'SSIS-123.mp4', code, json.dumps({'code': owner}), duration))+ def test_codes_misread_from_old_filenames_are_cleared_once_with_a_receipt(self):+ rows = [+ # 手机录像的时间戳与被截断的「名字_日期」:现行规则都不认,账本里还记着旧值。+ (1, r'R:\media\某人\VID_20220818_125735_816.mp4', '125735-816', None, None),+ (2, r'A:\pen\KUZU_250103-U_iris3.mp4', 'KUZU-25010', None, None),+ # 现行规则认不出文件名,但番号完整写在文件名里:保留。+ (3, r'B:\x\(アリスJAPAN)(DVAJ-185)标题.mp4', 'DVAJ-185', None, None),+ # 番号不在文件名里,可这一行有刮削资料:是按另一份身份改过的,保留。+ (4, r'B:\一本道\092415-001\1pon-092415-001-fhd2_(new).mp4', '092415_159', '另一部的标题', None),+ # 用户自己填的番号不碰。+ (5, r'A:\pen\KUZU_142-U_iris3.mp4', 'KUZU-142', None, 'user:manual'),+ ]+ with self.database.write_transaction() as connection:+ for key, path, code, title, owner in rows:+ connection.execute(+ 'INSERT INTO asset(id,location,path,name,medium,code,catalog_title,field_owners) '+ "VALUES(?,'local',?,?,'video',?,?,?)",+ (key, path, path.rsplit('\\', 1)[1], code, title, json.dumps({'code': owner}) if owner else None))+ self.service._clear_stale_codes()+ with self.database.read_connection() as connection:+ codes = dict(connection.execute('SELECT id,code FROM asset ORDER BY id').fetchall())+ receipt = connection.execute(f"SELECT result_json FROM ledger_upgrade_repair "+ f"WHERE id='{upgrade_repairs.STALE_CODES_KEY}'").fetchone()+ self.assertEqual(codes, {1: None, 2: None, 3: 'DVAJ-185', 4: '092415_159', 5: 'KUZU-142'})+ self.assertEqual(json.loads(receipt[0]), {'cleared': [{'id': 1, 'code': '125735-816'},+ {'id': 2, 'code': 'KUZU-25010'}]})+ # 只做一次:之后再记上的旧值不再清。+ with self.database.write_transaction() as connection:+ connection.execute("UPDATE asset SET code='125735-816' WHERE id=1")+ self.service._clear_stale_codes()+ with self.database.read_connection() as connection:+ self.assertEqual(connection.execute('SELECT code FROM asset WHERE id=1').fetchone()[0], '125735-816')+def test_old_studio_misses_are_released_only_once(self):from peach.followups import Attempts, attempts_rootroot = attempts_root(self.contract.candidate_root)