Peach: Local-first video metadata scraper and media manager.
refactor(media): 分离来源扫描与资料资产查询
longmeidao9c70ec04b00b
diff --git a/src/peach/library_processing.py b/src/peach/library_processing.pyindex 51326dbed..0f16cf0bd 100644--- a/src/peach/library_processing.py+++ b/src/peach/library_processing.py@@ -11,6 +11,7 @@ import timeimport uuidimport xml.etree.ElementTree as ETfrom contextlib import closing+from dataclasses import dataclassfrom pathlib import Pathfrom urllib.parse import urlparse@@ -1589,6 +1590,65 @@ def _online_source_root(source, root, chosen_paths, *, retrying, issue):return None+@dataclass+class _SourceScan:+ online_roots: list[Path]+ scanned: int = 0+ probed: int = 0+++def _scan_sources(config, db_path, *, locations, asset_ids, stage, guard, update, issue):+ """发现本趟可访问的来源根,按阶段登记文件并探测新视频。"""+ result = _SourceScan([])+ chosen_paths = _chosen_asset_paths(db_path, asset_ids)+ scan = asset_ids is None and stage in (ALL_STAGES, SCAN_STAGE)+ mounts = {key: tuple(str(value) for value in values) for key, values in config.mounts.items()}+ ffprobe = _ffprobe_path(config) if scan else None+ for source, roots in locations.items():+ if stage == PREVIEWS_STAGE and source != 'local':+ continue+ for root in roots:+ guard.check()+ translated_root = _online_source_root(+ source, root, chosen_paths, retrying=asset_ids is not None, issue=issue)+ if translated_root is None:+ continue+ result.online_roots.append(translated_root)+ if scan:+ registered = scan_location(db_path, source, root, declared_roots=config.locations,+ mounts=mounts, report=lambda line: update(stage='扫描文件'))+ result.scanned += registered.files+ result.probed += probe_unmeasured(+ db_path, ffprobe, source, root,+ report=lambda done, total: update(stage='读取时长与分辨率', checked=done, total=total))+ reattach_in(db_path, registered.new_ids)+ update(scanned=result.scanned, probed=result.probed)+ return result+++def _collection_rows(db_path, *, locations, online_roots, asset_ids):+ """读取可访问视频与采集所需的演员、内容标签和已收录 genre。"""+ multi = ("(SELECT group_concat(entity.canonical_name) FROM asset_entity"+ " JOIN entity ON entity.id=asset_entity.entity_id"+ " WHERE asset_entity.asset_id=asset.id AND entity.kind='performer') AS performers,"+ " (SELECT group_concat(asset_tag.tag) FROM asset_tag"+ " WHERE asset_tag.asset_id=asset.id AND asset_tag.tag NOT LIKE '演员:%') AS tags")+ query = f"SELECT asset.*, {multi} FROM asset WHERE medium='video' AND disposal IS NULL"+ parameters = []+ if asset_ids is not None:+ query += ' AND id IN (' + ','.join('?' * len(asset_ids)) + ')'+ parameters = asset_ids+ query += ' ORDER BY id'+ with closing(sqlite3.connect(db_path, timeout=30)) as connection:+ connection.row_factory = sqlite3.Row+ # 路径归属只比字面,避免逐行 resolve() 访问挂载服务。+ rows = [dict(row) for row in connection.execute(query, parameters)+ if row['location'] in locations+ and any(translate_ledger_path(row['path']).is_relative_to(root) for root in online_roots)]+ genre_decisions = load_genre_decisions(connection)+ return rows, genre_decisions++def _processing_issue_log(config, path, job_id, isolated):"""定向任务保留独立问题清单;全库任务只留本趟的清单。"""log_path = path.with_suffix('.issues.jsonl') if isolated else issues_path(config, job_id)@@ -1757,7 +1817,6 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='cotargeted = retrying or selected_ids is not Nonechosen_ids = selected_ids if selected_ids is not None else list(dict.fromkeys(map(int, retry_ids or [])))scope_ids = chosen_ids if targeted else None- chosen_paths = _chosen_asset_paths(db_path, chosen_ids)with FileLock(str(state_path(config)) + '.lock', timeout=0):state = dict(job_id=job_id or uuid.uuid4().hex, status='running',stage='读取本地资料' if retrying else '扫描文件',@@ -1825,62 +1884,17 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='coguard = DiskGuard(config.data_root, 1)guard.check(force=True)locations = config.locations if location == 'configured' else {location: config.locations[location]}- mounts = {key: tuple(str(value) for value in values) for key, values in config.mounts.items()}- ffprobe = _ffprobe_path(config)- online_roots = []- for source, roots in locations.items():- if stage == PREVIEWS_STAGE and source != 'local':- continue- for root in roots:- guard.check()- translated_root = _online_source_root(- source, root, chosen_paths, retrying=targeted, issue=issue)- if translated_root is None:- continue- online_roots.append(translated_root)- if not targeted and stage in (ALL_STAGES, SCAN_STAGE):- result = scan_location(db_path, source, root, declared_roots=config.locations,- mounts=mounts, report=lambda line: update(stage='扫描文件'))- state['scanned'] += result.files- state['probed'] += probe_unmeasured(- db_path, ffprobe, source, root,- report=lambda done, total: update(stage='读取时长与分辨率', checked=done, total=total))- # 登记那一刻新文件还没有时长;探完再判一轮,多个候选按时长挑、无番号按时长配。- reattach_in(db_path, result.new_ids)+ sources = _scan_sources(config, db_path, locations=locations, asset_ids=scope_ids,+ stage=stage, guard=guard, update=update, issue=issue)+ state.update(scanned=sources.scanned, probed=sources.probed)if stage == SCAN_STAGE:# `checked`/`total` 在探时长时借给进度条用过;只扫描这一段没有采集,读数归零。update(status='failed' if state['issue_count'] else 'complete',stage='处理结束', completed_at=time.time(), checked=0, total=0,error=issue_summary(state['issue_count'], state['paused_count']))return state- # 演员和标签是另外两张表,`asset` 上没有这两列。不带上它们,采集就把每部片都- # 当成缺演员缺标签,逐个去问 r18,再把账本早就有的写法变成一道复核题。- # `演员:` 是出演者在 `asset_tag` 上的扁平投影(ADR-0005),不算内容标签:- # 算进来的话,一部片只要有演员就永远不缺标签,它的 genre 再也采不回来。- # 本机实测 110 部片正好卡在这上面,`asset_tag` 里只有 `演员:` 那几行。- multi = ("(SELECT group_concat(entity.canonical_name) FROM asset_entity"- " JOIN entity ON entity.id=asset_entity.entity_id"- " WHERE asset_entity.asset_id=asset.id AND entity.kind='performer') AS performers,"- " (SELECT group_concat(asset_tag.tag) FROM asset_tag"- " WHERE asset_tag.asset_id=asset.id AND asset_tag.tag NOT LIKE '演员:%') AS tags")- if targeted:- placeholders = ','.join('?' * len(chosen_ids))- query = (f"SELECT asset.*, {multi} FROM asset WHERE id IN ({placeholders}) AND medium='video' "- "AND disposal IS NULL ORDER BY id")- parameters = chosen_ids- else:- query = (f"SELECT asset.*, {multi} FROM asset "- "WHERE medium='video' AND disposal IS NULL ORDER BY id")- parameters = []- with closing(sqlite3.connect(db_path, timeout=30)) as connection:- connection.row_factory = sqlite3.Row- # 归属判断只比路径字面:声明根与账本路径同出一个口径,`resolve()` 在网盘- # 挂载上是每行一次往返,几万行就是几分钟还没开始干活。- rows = [dict(row) for row in connection.execute(query, parameters)- if row['location'] in locations- and any(translate_ledger_path(row['path']).is_relative_to(root) for root in online_roots)]- # 用户在复核页收录过的 genre 从这一批起就是已知词,不该再作为未收录回来问一遍。- genre_decisions = load_genre_decisions(connection)+ rows, genre_decisions = _collection_rows(+ db_path, locations=locations, online_roots=sources.online_roots, asset_ids=scope_ids)if stage in (ALL_STAGES, COLLECT_STAGE, PREVIEWS_STAGE):_generate_previews(config, rows, database, update=update, issue=issue, guard=guard)if stage == PREVIEWS_STAGE:diff --git a/tests/test_complexity_ratchet.py b/tests/test_complexity_ratchet.pyindex 661887527..625afd360 100644--- a/tests/test_complexity_ratchet.py+++ b/tests/test_complexity_ratchet.py@@ -72,9 +72,9 @@ ACCEPTED: dict[str, tuple[int, str]] = {41, "应用工厂:按启动顺序构造账本、各项后台服务与路由并接成一个 app,每个可注入的""依赖各带一个缺省;挂载探测按配置状态启动。拆开会把同一条构造顺序分散到几处"),"src/peach/library_processing.py:process_library": (- 65, "顺序流水线:扫描、探时长、读本地资料、联网采集、写候选、收尾共用一份进度状态"- "与问题记录;能独立命名的段已经拆出(`_RemoteSession`、`_merge_candidates`、"- "`_record_issue`),剩下的是编排本身"),+ 62, "顺序流水线:本地预览、读本地资料、联网采集、写候选、收尾共用一份进度状态"+ "与问题记录;来源扫描、资产查询与外部会话分别归 `_scan_sources`、"+ "`_collection_rows`、`_RemoteSession`,剩下的是任务编排与逐项目资料处理"),"src/peach/jav_cover_fetch.py:run": (50, "批处理的主循环:每个番号依次量本机尺寸、择优、比较、落盘、记日志,统计、日志行""和出错后换新的连接池在整轮里共用,逐段拆开只是把这几样来回传"),diff --git a/tests/test_followups.py b/tests/test_followups.pyindex 9fbf82ad9..ef368cd7e 100644--- a/tests/test_followups.py+++ b/tests/test_followups.py@@ -1378,10 +1378,11 @@ class ProcessLibraryTests(LedgerTestCase):def test_a_scrape_declares_dispatches_and_runs_its_avatar_followups(self):_media, config = self.sample()self.seed()- result = process_library(config, self.db, self.root / 'generated',- self.root / 'covers',- provider_factory=mock.Mock(return_value=self.provider()),- database=self.database)+ with mock.patch('peach.library_processing._generate_previews', return_value=0):+ result = process_library(config, self.db, self.root / 'generated',+ self.root / 'covers',+ provider_factory=mock.Mock(return_value=self.provider()),+ database=self.database)self.assertEqual(result['status'], 'complete')with self.database.read_connection() as connection:entity_id = connection.execute(