Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(upgrade): 清掉旧版从文件名误认的番号

longmeidao2de9795dd2cb
diff --git a/docs/SOURCING.md b/docs/SOURCING.md
index 6b18bc10e..bdeb35ca3 100644
--- a/docs/SOURCING.md
+++ b/docs/SOURCING.md
@@ -645,6 +645,10 @@ Instagram 的独立用户登录会话未取得,自动适配器不进正式依
是「无码流出」;明写 `破解`/`AI去码`/`-LTX`/`-UMR`/`-C-U` 是「无码破解」,`-C-U` 不另算中字。有码片商的番号只带
`-U`/`-UC`/`-UN`/`uncensored` 或标签「无码」时也算破解:真流出的片很少,流传的去码版绝大多数是破解;FC2 这类
本身可能无码的片仍是「无码」。JavDB 的「無碼流出」是作品级标签,只留在来源快照里,不据此判断本地文件。
+- 旧版从文件名误认的番号由升级补修清一次(`upgrade_repairs.stale_filename_codes`):手机录像的时间戳
+ (`VID_20220818_125735_816` 记成 `125735-816`)、个人投稿者被截断的「名字_日期」(`KUZU_250103` 记成 `KUZU-25010`)。
+ 只清无主或文件名扫描写入、现行规则从文件名和文件夹都认不出的番号,且番号已不合现行形状,或番号没完整出现在名字里、
+ 这一行也没有任何刮削资料。清掉的行与原值记在 `ledger_upgrade_repair` 的 `stale-filename-codes-1` 回执里,可整批撤回。
- 推广域名会出现在番号的头、尾和方括号里(`www.98t.la@ABW-358-U`、`ABP-762-fuckbe.com`、`[xxx.cc]ABC-123`)。
番号提取与目录判重共用 `strip_promo_markers` 这一层;各剥一半,叠了两层的 `[98t.tv][98t.tv]ABW-251` 就会在
其中一处漏网。
diff --git a/src/peach/upgrade_repairs.py b/src/peach/upgrade_repairs.py
index cb1e2984f..7616e9918 100644
--- a/src/peach/upgrade_repairs.py
+++ b/src/peach/upgrade_repairs.py
@@ -59,6 +59,43 @@ def _manual_name(connection, entity_id: int) -> bool:
(entity_id,)).fetchone())
+#: 旧版从文件名误认的番号只清一次;清掉的行与原值记进同一条回执,可以按它整批撤回。
+STALE_CODES_KEY = 'stale-filename-codes-1'
+
+
+def stale_filename_codes(connection) -> list[dict]:
+ """旧版从文件名误认、现行规则已经不认的番号。
+
+ 实例是手机录像的时间戳(`VID_20220818_125735_816` 记成 `125735-816`)和被截断的「名字_日期」
+ (个人投稿者的 `KUZU_250103` 记成 `KUZU-25010`)。这类行有了番号就被当成 JAV,日期式番号还会拿到
+ 「无码」徽章。只看无主或由文件名扫描写入的番号,且文件名与所在文件夹按现行规则都认不出番号;
+ 再满足下面一条才清:番号本身已不合现行形状;或番号没有完整出现在文件名与文件夹里,这一行也没有
+ 任何目录资料(标题、片商、发行日)。带资料的那类是刮削按另一份身份改过的(`1pon-092415-001-fhd2`
+ 实为 `092415_159`),不碰。
+ """
+ import re
+ from pathlib import PureWindowsPath
+ from .catalog_rules import compile_code_patterns, is_jav_code, normalise_code_key, release_code_from_filename
+ from .profiles import read_settings
+ patterns = compile_code_patterns(read_settings(connection).get('codePatterns'))
+ rows = []
+ for row in connection.execute(
+ "SELECT id,name,path,code,field_owners,catalog_title,studio,release_date FROM asset "
+ "WHERE medium='video' AND code IS NOT NULL AND code<>''"):
+ owner = owner_of(row['field_owners'], 'code')
+ if owner not in ('', SCAN_FILENAME):
+ continue
+ texts = (str(row['name'] or ''), PureWindowsPath(str(row['path'] or '')).parent.name)
+ if any(release_code_from_filename(text, patterns=patterns) for text in texts):
+ continue
+ body = re.escape(normalise_code_key(row['code'])).replace(r'\-', '[-_ ]?')
+ whole = any(re.search(rf'(?<![A-Za-z0-9]){body}(?![A-Za-z0-9])', text, re.I) for text in texts)
+ described = row['catalog_title'] or row['studio'] or row['release_date']
+ if not is_jav_code(row['code']) or (not whole and not described):
+ rows.append({'id': row['id'], 'code': row['code']})
+ return rows
+
+
def _catalog_plan(connection, sources: Path, *, catalog: bool = True) -> dict:
text = metadata_text_repair.collect(connection, sources / 'library-metadata')
if not catalog:
@@ -231,6 +268,18 @@ class UpgradeRepairs:
(REVISION, str(time.time()), json.dumps(receipt, ensure_ascii=False)))
atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))
+ def _clear_stale_codes(self) -> None:
+ """清掉旧版从文件名误认的番号(`stale_filename_codes`),只做一次。"""
+ with self.contract.database.write_transaction() as connection:
+ if connection.execute('SELECT 1 FROM ledger_upgrade_repair WHERE id=?', (STALE_CODES_KEY,)).fetchone():
+ return
+ rows = stale_filename_codes(connection)
+ if rows:
+ write_owned_fields(connection, [row['id'] for row in rows], {'code': None}, SCAN_FILENAME)
+ connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',
+ (STALE_CODES_KEY, str(time.time()), json.dumps({'cleared': rows})))
+ LOGGER.info('旧版误认的番号:清掉 %s 条', len(rows))
+
def _release_studio_misses(self) -> None:
"""旧版断网时记下的「官网未取得」各放一次重试(`studio_followup.release_unbounded_misses`)。
@@ -446,6 +495,7 @@ class UpgradeRepairs:
self.scheduler.add_job(self.tick, 'interval', seconds=60, max_instances=1, coalesce=True)
self.scheduler.add_job(self._settle_verification, max_instances=1)
self.scheduler.add_job(self._release_studio_misses, max_instances=1)
+ self.scheduler.add_job(self._clear_stale_codes, max_instances=1)
self.scheduler.start()
def stop(self) -> None:
diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.py
index 6f700e56e..6dfc9d007 100644
--- a/tests/test_upgrade_repairs.py
+++ b/tests/test_upgrade_repairs.py
@@ -280,6 +280,39 @@ class UpgradeRepairTests(unittest.TestCase):
"VALUES(?,'local',?,?,'video',?,?,?)",
(key, f'{key}.mp4', 'SSIS-123.mp4', code, json.dumps({'code': owner}), duration))
+ def test_codes_misread_from_old_filenames_are_cleared_once_with_a_receipt(self):
+ rows = [
+ # 手机录像的时间戳与被截断的「名字_日期」:现行规则都不认,账本里还记着旧值。
+ (1, r'R:\media\某人\VID_20220818_125735_816.mp4', '125735-816', None, None),
+ (2, r'A:\pen\KUZU_250103-U_iris3.mp4', 'KUZU-25010', None, None),
+ # 现行规则认不出文件名,但番号完整写在文件名里:保留。
+ (3, r'B:\x\(アリスJAPAN)(DVAJ-185)标题.mp4', 'DVAJ-185', None, None),
+ # 番号不在文件名里,可这一行有刮削资料:是按另一份身份改过的,保留。
+ (4, r'B:\一本道\092415-001\1pon-092415-001-fhd2_(new).mp4', '092415_159', '另一部的标题', None),
+ # 用户自己填的番号不碰。
+ (5, r'A:\pen\KUZU_142-U_iris3.mp4', 'KUZU-142', None, 'user:manual'),
+ ]
+ with self.database.write_transaction() as connection:
+ for key, path, code, title, owner in rows:
+ connection.execute(
+ 'INSERT INTO asset(id,location,path,name,medium,code,catalog_title,field_owners) '
+ "VALUES(?,'local',?,?,'video',?,?,?)",
+ (key, path, path.rsplit('\\', 1)[1], code, title, json.dumps({'code': owner}) if owner else None))
+ self.service._clear_stale_codes()
+ with self.database.read_connection() as connection:
+ codes = dict(connection.execute('SELECT id,code FROM asset ORDER BY id').fetchall())
+ receipt = connection.execute(f"SELECT result_json FROM ledger_upgrade_repair "
+ f"WHERE id='{upgrade_repairs.STALE_CODES_KEY}'").fetchone()
+ self.assertEqual(codes, {1: None, 2: None, 3: 'DVAJ-185', 4: '092415_159', 5: 'KUZU-142'})
+ self.assertEqual(json.loads(receipt[0]), {'cleared': [{'id': 1, 'code': '125735-816'},
+ {'id': 2, 'code': 'KUZU-25010'}]})
+ # 只做一次:之后再记上的旧值不再清。
+ with self.database.write_transaction() as connection:
+ connection.execute("UPDATE asset SET code='125735-816' WHERE id=1")
+ self.service._clear_stale_codes()
+ with self.database.read_connection() as connection:
+ self.assertEqual(connection.execute('SELECT code FROM asset WHERE id=1').fetchone()[0], '125735-816')
+
def test_old_studio_misses_are_released_only_once(self):
from peach.followups import Attempts, attempts_root
root = attempts_root(self.contract.candidate_root)