Peach: Local-first video metadata scraper and media manager.
fix(telemetry): 跳过空研究快照并核实每周投递状态
longmeidaoe7191e7da4a0
diff --git a/src/peach/library_sharing.py b/src/peach/library_sharing.pyindex b29651459..966967ce9 100644--- a/src/peach/library_sharing.py+++ b/src/peach/library_sharing.py@@ -30,6 +30,31 @@ def _path(config):return config.directory("state") / "library-sharing.json"+def _state_path(config):+ return config.directory("state") / "library-sharing-attempt.json"+++def _state_lock(config):+ from filelock import FileLock+ path = _state_path(config)+ path.parent.mkdir(parents=True, exist_ok=True)+ return FileLock(str(path) + ".lock", timeout=5)+++def _period():+ return datetime.now(timezone.utc).strftime("%G-W%V")+++def _used(value, period):+ return value.get("period") == period and (value.get("attempted") is True+ or ("attempted" not in value and value.get("status") in {"attempted", "failed"}))+++def useful_payload(payload):+ """公共编号或任一偏好分组提供可分析内容。"""+ return bool(payload["library"] or any(payload["preferences"].values()))++def consent(config):value = remote_reporting._read(_path(config))return value if value.get("enabled") is True and value.get("schema") == SCHEMA else {}@@ -41,7 +66,12 @@ def save_choice(config, enabled):raise ValueError("馆藏与口味上传选项必须是布尔值")value = {"enabled": enabled, "schema": SCHEMA, "revision": uuid.uuid4().hex,"selected_at": datetime.now(timezone.utc).isoformat()}- atomic_write_text(_path(config), json.dumps(value))+ with _state_lock(config):+ atomic_write_text(_path(config), json.dumps(value))+ state = remote_reporting._read(_state_path(config))+ if state.get("status") == "queued":+ status = "unknown" if state.get("attempted") else "blocked"+ atomic_write_text(_state_path(config), json.dumps({**state, "status": status, "reason": "consent"}))return snapshot(config)@@ -56,6 +86,8 @@ def target():def snapshot(config):attempt = remote_reporting._read(config.directory("state") / "library-sharing-attempt.json")+ if attempt.get("status") == "attempted":+ attempt = {**attempt, "status": "unknown", "attempted": True}return {"enabled": bool(consent(config)), "available": bool(target()),"schema": SCHEMA, "fields": FIELDS, "notice": NOTICE,"last_attempt": attempt if attempt.get("schema") == SCHEMA else {}}@@ -123,7 +155,7 @@ class Service:self._wake.set()return snapshot(self.config)- def _gate(self, revision, envelope):+ def _gate(self, revision, envelope, attempt_id=None):current = consent(self.config)if self._stop.is_set() or not current or current.get("revision") != revision:return False@@ -141,7 +173,7 @@ class Service:if len(raw) > MAX_RAW_BYTES:return Falsepayload = json.loads(raw)- if not valid_payload(payload):+ if not valid_payload(payload) or not useful_payload(payload):return Falseexcept (ValueError, KeyError, TypeError, OSError, EOFError):return False@@ -149,38 +181,97 @@ class Service:for key in tuple(event):if key not in {"event_id", "timestamp", "platform"}:del event[key]- event.update(message="Peach library and taste research", level="info",- tags={"purpose": "population-research", "schema": str(SCHEMA), "period": payload["period"]})+ metadata = remote_reporting.event_metadata("research")+ event.update(**metadata, message="Peach library and taste research", level="info")+ event["tags"].update(schema=str(SCHEMA), period=payload["period"])for key in tuple(envelope.headers):if key not in {"event_id", "sent_at"}:del envelope.headers[key]+ if attempt_id is None:+ return True+ with _state_lock(self.config):+ current = consent(self.config)+ state = remote_reporting._read(_state_path(self.config))+ if (self._stop.is_set() or not current or current.get("revision") != revision+ or state.get("attempt_id") != attempt_id or state.get("revision") != revision+ or state.get("period") != _period() or state.get("status") != "queued"+ or _used(state, _period())):+ return False+ atomic_write_text(_state_path(self.config), json.dumps({**state, "attempted": True}))return True+ def _delivery(self, identity, result):+ """仅更新当前投递的脱敏回执,不把 HTTP 接收解释为留存。"""+ if result.get("status") not in {"accepted", "failed", "unknown", "blocked"}:+ return+ with _state_lock(self.config):+ state = remote_reporting._read(_state_path(self.config))+ if (any(state.get(key) != value for key, value in identity.items())+ or consent(self.config).get("revision") != identity["revision"]+ or state.get("status") not in {"queued", "unknown"}):+ return+ if state.get("status") == "unknown" and result["status"] == "unknown":+ return+ status = result["status"]+ if status == "unknown" and not state.get("attempted"):+ status = "failed"+ value = {key: value for key, value in state.items() if key not in {"reason", "status_code"}}+ value["status"] = status+ if type(result.get("status_code")) is int and 100 <= result["status_code"] <= 599:+ value["status_code"] = result["status_code"]+ if result.get("reason") in {"network", "http_status", "rate_limit", "queue_full", "consent", "disabled", "unknown"}:+ value["reason"] = result["reason"]+ atomic_write_text(_state_path(self.config), json.dumps(value))+def attempt(self):current, destination = consent(self.config), target()if not current or not destination or self._stop.is_set():return- period = datetime.now(timezone.utc).strftime("%G-W%V")- state = self.config.directory("state") / "library-sharing-attempt.json"- from filelock import FileLock- state.parent.mkdir(parents=True, exist_ok=True)- with FileLock(str(state) + ".lock", timeout=.1):+ period = _period()+ state = _state_path(self.config)+ with _state_lock(self.config):previous = remote_reporting._read(state)- if previous.get("schema") == SCHEMA and previous.get("period") == period:+ if _used(previous, period):return- atomic_write_text(state, json.dumps({"schema": SCHEMA, "period": period, "status": "attempted"}))- payload, data = collect(self.contract)- data = compressed(data)- from sentry_sdk.attachments import Attachment- client = remote_reporting._new_client(destination, envelope_gate=lambda envelope: self._gate(current["revision"], envelope))+ identity = {"schema": SCHEMA, "period": period, "revision": current["revision"], "attempt_id": uuid.uuid4().hex}+ value = {**identity, "attempted": False}+ stage = "collection"+ try:+ payload, data = collect(self.contract)+ if useful_payload(payload):+ stage = "compression"+ data = compressed(data)+ value["status"] = "queued"+ else:+ value["status"] = "waiting-data"+ except Exception:+ value.update(status="failed", reason=stage)+ with _state_lock(self.config):+ if (consent(self.config).get("revision") != identity["revision"] or _period() != period+ or _used(remote_reporting._read(state), period)):+ return+ atomic_write_text(state, json.dumps(value))+ if value["status"] != "queued":+ return+ client = Nonetry:+ from sentry_sdk.attachments import Attachment+ client = remote_reporting._new_client(destination,+ envelope_gate=lambda envelope: self._gate(current["revision"], envelope, identity["attempt_id"]),+ delivery_callback=lambda result: self._delivery(identity, result))client.capture_event({"message": "Peach library and taste research", "level": "info",- "tags": {"purpose": "population-research", "schema": str(SCHEMA), "period": payload["period"]}},+ **remote_reporting.event_metadata("research")},hint={"attachments": [Attachment(bytes=data, filename=FILENAME,content_type="application/gzip")]})client.flush(timeout=5)+ except Exception:+ passfinally:- client.close(timeout=0)+ try:+ if client is not None:+ client.close(timeout=0)+ finally:+ self._delivery(identity, {"status": "unknown", "reason": "unknown"})def start(self):if self._thread and self._thread.is_alive():@@ -190,14 +281,9 @@ class Service:while not self._stop.is_set():try:self.attempt()- except Exception as error:+ except Exception:# 分享失败不把数据或错误原文转交诊断通道。- try:- state = self.config.directory("state") / "library-sharing-attempt.json"- value = remote_reporting._read(state)- atomic_write_text(state, json.dumps({**value, "status": "failed", "reason": type(error).__name__}))- except OSError:- pass+ passself._wake.wait(3600)self._wake.clear()self._thread = threading.Thread(target=run, name="PeachLibrarySharing", daemon=True)diff --git a/tests/test_library_sharing.py b/tests/test_library_sharing.pyindex e63bf9015..5948da56a 100644--- a/tests/test_library_sharing.py+++ b/tests/test_library_sharing.py@@ -3,6 +3,7 @@ import gzipimport jsonimport osimport tempfile+import threadingimport unittestfrom pathlib import Pathfrom types import SimpleNamespace@@ -37,6 +38,22 @@ class LibrarySharingTests(unittest.TestCase):return self.service.save({'enabled': True, 'schema': sharing.SCHEMA,'consent': True, 'preview_token': preview['token']})+ def add_work(self):+ with self.database.write_transaction() as connection:+ connection.execute("INSERT INTO asset(location,path,name,code) VALUES('local','work','title','EXM-001')")++ def attempt_state(self):+ return sharing.remote_reporting._read(sharing._state_path(self.config))++ def envelope(self, data=None):+ from sentry_sdk.envelope import Envelope+ from sentry_sdk.attachments import Attachment+ envelope = Envelope()+ envelope.add_event({'message': 'Peach research'})+ wire = sharing.compressed(sharing.collect(self.contract)[1]) if data is None else data+ envelope.add_item(Attachment(bytes=wire, filename=sharing.FILENAME).to_envelope_item())+ return envelope+def test_disabled_and_missing_endpoint_do_not_read_library_or_construct_sdk(self):with mock.patch.object(sharing, 'collect') as collect, \mock.patch('peach.remote_reporting._new_client') as client:@@ -192,6 +209,7 @@ class LibrarySharingTests(unittest.TestCase):def test_real_sdk_sends_only_research_event_and_reviewed_attachment_once_per_week(self):from sentry_sdk.transport import HttpTransport+ self.add_work()self.enable()sent = []with mock.patch.object(HttpTransport, '_send_envelope', side_effect=lambda envelope: sent.append(envelope)):@@ -204,7 +222,9 @@ class LibrarySharingTests(unittest.TestCase):self.assertNotIn('extra', event)self.assertNotIn('server_name', event)self.assertNotIn('user', event)- self.assertNotIn('release', event)+ self.assertEqual(event['release'], sharing.remote_reporting.event_metadata('research')['release'])+ self.assertEqual(event['environment'], sharing.remote_reporting.event_metadata('research')['environment'])+ self.assertEqual(event['tags']['runtime'], sharing.remote_reporting.event_metadata('research')['tags']['runtime'])self.assertEqual(event['tags']['purpose'], 'population-research')attachment = json.loads(gzip.decompress(envelope.items[1].payload.get_bytes()))self.assertEqual(attachment['schema'], sharing.SCHEMA)@@ -215,6 +235,7 @@ class LibrarySharingTests(unittest.TestCase):def test_disable_invalidates_queued_attachment(self):from sentry_sdk.envelope import Envelopefrom sentry_sdk.attachments import Attachment+ self.add_work()self.enable()previous = sharing.consent(self.config)['revision']envelope = Envelope()@@ -258,11 +279,239 @@ class LibrarySharingTests(unittest.TestCase):with mock.patch.object(sharing, 'MAX_RAW_BYTES', 1), self.assertRaises(ValueError):self.service.preview()self.assertFalse(sharing.snapshot(self.config)['enabled'])+ self.add_work()self.enable()with mock.patch.object(sharing, 'MAX_BYTES', 1), mock.patch('peach.remote_reporting._new_client') as client:- with self.assertRaises(ValueError):- self.service.attempt()+ self.service.attempt()+ client.assert_not_called()+ self.assertEqual(self.attempt_state()['status'], 'failed')+ self.assertEqual(self.attempt_state()['reason'], 'compression')+ self.assertFalse(self.attempt_state()['attempted'])++ def test_empty_and_uncoded_without_preferences_wait_for_data(self):+ self.enable()+ with mock.patch('peach.remote_reporting._new_client') as client:+ self.service.attempt()+ with self.database.write_transaction() as connection:+ connection.execute("INSERT INTO asset(location,path,name) VALUES('local','uncoded','title')")+ self.service.attempt()+ client.assert_not_called()+ self.assertEqual(self.attempt_state()['status'], 'waiting-data')+ self.assertFalse(self.attempt_state()['attempted'])+ self.assertFalse(self.service._gate(sharing.consent(self.config)['revision'], self.envelope()))++ def test_uncoded_preference_group_can_be_sent(self):+ with self.database.write_transaction() as connection:+ connection.execute("INSERT INTO asset(location,path,name) VALUES('local','uncoded','title')")+ connection.execute("INSERT INTO asset_tag(asset_id,tag,source) VALUES(1,'手动标签','web-user')")+ self.enable()+ with mock.patch('peach.remote_reporting._new_client') as client:+ self.service.attempt()+ client.assert_called_once()++ def test_collection_failure_does_not_spend_weekly_network_attempt(self):+ self.enable()+ with mock.patch.object(sharing, 'collect', side_effect=OSError('private path')):+ self.service.attempt()+ self.assertEqual(self.attempt_state()['status'], 'failed')+ self.assertEqual(self.attempt_state()['reason'], 'collection')+ self.assertFalse(self.attempt_state()['attempted'])+ self.assertNotIn('private path', json.dumps(self.attempt_state()))+ self.add_work()+ with mock.patch('peach.remote_reporting._new_client') as client:+ self.service.attempt()+ client.assert_called_once()++ def test_disable_during_collection_prevents_queueing(self):+ self.add_work()+ self.enable()+ collect = sharing.collect+ def disable_then_collect(contract):+ sharing.save_choice(self.config, False)+ return collect(contract)+ with mock.patch.object(sharing, 'collect', side_effect=disable_then_collect), \+ mock.patch('peach.remote_reporting._new_client') as client:+ self.service.attempt()client.assert_not_called()+ self.assertFalse(sharing.snapshot(self.config)['enabled'])++ def test_queue_failure_does_not_spend_network_attempt(self):+ self.add_work()+ self.enable()+ def make_client(*_args, envelope_gate, delivery_callback):+ def capture(*_args, **_kwargs):+ delivery_callback({'status': 'failed', 'reason': 'queue_full'})+ return SimpleNamespace(capture_event=capture, flush=lambda **_kw: None, close=lambda **_kw: None)+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client) as client:+ self.service.attempt()+ self.service.attempt()+ self.assertEqual(client.call_count, 2)+ self.assertEqual(self.attempt_state()['status'], 'failed')+ self.assertEqual(self.attempt_state()['reason'], 'queue_full')+ self.assertFalse(self.attempt_state()['attempted'])++ def test_delivery_receipts_record_only_status_and_safe_details(self):+ self.add_work()+ self.enable()+ for result in ({'status': 'accepted', 'status_code': 200},+ {'status': 'failed', 'status_code': 403, 'reason': 'http_status'},+ {'status': 'unknown', 'reason': 'network'}):+ sharing._state_path(self.config).unlink(missing_ok=True)+ def make_client(*_args, envelope_gate, delivery_callback):+ def capture(*_args, **_kwargs):+ self.assertTrue(envelope_gate(self.envelope()))+ delivery_callback({**result, 'url': 'private-url', 'detail': 'private-content'})+ return SimpleNamespace(capture_event=capture, flush=lambda **_kw: None, close=lambda **_kw: None)+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):+ self.service.attempt()+ self.enable()+ self.service.attempt()+ state = self.attempt_state()+ self.assertEqual(state['status'], result['status'])+ self.assertTrue(state['attempted'])+ if 'reason' in result:+ self.assertEqual(state['reason'], result['reason'])+ self.assertNotIn('private', json.dumps(state))++ def test_missing_receipt_is_unknown_and_does_not_retry_network(self):+ self.add_work()+ self.enable()+ callbacks = []+ def make_client(*_args, envelope_gate, delivery_callback):+ callbacks.append(delivery_callback)+ def capture(*_args, **_kwargs):+ self.assertTrue(envelope_gate(self.envelope()))+ self.assertEqual(self.attempt_state()['status'], 'queued')+ return SimpleNamespace(capture_event=capture, flush=lambda **_kw: None, close=lambda **_kw: None)+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client) as client:+ self.service.attempt()+ self.assertEqual(self.attempt_state()['status'], 'unknown')+ self.service.attempt()+ client.assert_called_once()+ callbacks[0]({'status': 'accepted', 'status_code': 200})+ self.assertEqual(self.attempt_state()['status'], 'accepted')+ self.assertNotIn('reason', self.attempt_state())++ def test_real_sdk_http_receipt_controls_local_research_status(self):+ from sentry_sdk.transport import HttpTransport+ self.add_work()+ self.enable()+ for status, headers, expected in ((200, {}, 'accepted'), (413, {}, 'failed'), (429, {}, 'failed'),+ (200, {'x-sentry-rate-limits': '60:error:organization'}, 'unknown'), (None, {}, 'unknown')):+ with self.subTest(status=status, headers=headers):+ sharing._state_path(self.config).unlink(missing_ok=True)+ response = SimpleNamespace(status=status, headers=headers, close=mock.Mock(), data=b'private')+ with mock.patch.dict(os.environ, {'PEACH_REPORTING_TEST_MODE': '0'}), \+ mock.patch.object(HttpTransport, '_request', return_value=response,+ side_effect=OSError('private proxy') if status is None else None) as request:+ self.service.attempt()+ self.service.attempt()+ request.assert_called_once()+ state = self.attempt_state()+ self.assertEqual(state['status'], expected)+ self.assertTrue(state['attempted'])+ if status is not None:+ self.assertEqual(state['status_code'], status)+ self.assertNotIn('private', json.dumps(state))++ def test_cross_week_receipt_cannot_overwrite_new_attempt(self):+ self.add_work()+ self.enable()+ callbacks = []+ def make_client(*_args, envelope_gate, delivery_callback):+ callbacks.append(delivery_callback)+ return mock.Mock()+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):+ self.service.attempt()+ with mock.patch.object(sharing, '_period', return_value='2099-W01'):+ self.service.attempt()+ state = self.attempt_state()+ callbacks[0]({'status': 'accepted', 'status_code': 200})+ self.assertEqual(self.attempt_state(), state)++ def test_legacy_attempt_is_unknown_and_conserves_weekly_budget(self):+ self.enable()+ sharing._state_path(self.config).write_text(json.dumps(+ {'schema': sharing.SCHEMA, 'period': sharing._period(), 'status': 'attempted'}))+ self.assertEqual(sharing.snapshot(self.config)['last_attempt']['status'], 'unknown')+ with mock.patch.object(sharing, 'collect') as collect:+ self.service.attempt()+ collect.assert_not_called()++ def test_concurrent_services_allow_only_one_network_reservation(self):+ self.add_work()+ self.enable()+ sent, errors = [], []+ def make_client(*_args, envelope_gate, delivery_callback):+ def capture(*_args, **_kwargs):+ if envelope_gate(self.envelope()):+ sent.append(True)+ delivery_callback({'status': 'accepted', 'status_code': 200})+ return SimpleNamespace(capture_event=capture, flush=lambda **_kw: None, close=lambda **_kw: None)+ def run():+ try:+ sharing.Service(self.config, self.contract).attempt()+ except Exception as error:+ errors.append(error)+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):+ threads = [threading.Thread(target=run) for _ in range(5)]+ for thread in threads:+ thread.start()+ for thread in threads:+ thread.join(timeout=10)+ self.assertFalse(any(thread.is_alive() for thread in threads))+ self.assertEqual(errors, [])+ self.assertEqual(len(sent), 1)++ def test_obsolete_delivery_cannot_overwrite_current_attempt(self):+ self.add_work()+ self.enable()+ callbacks = []+ def make_client(*_args, envelope_gate, delivery_callback):+ callbacks.append(delivery_callback)+ return mock.Mock()+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):+ self.service.attempt()+ first = self.attempt_state()+ self.service.attempt()+ second = self.attempt_state()+ self.assertNotEqual(first['attempt_id'], second['attempt_id'])+ callbacks[0]({'status': 'accepted', 'status_code': 200})+ self.assertEqual(self.attempt_state(), second)+ self.enable()+ callbacks[1]({'status': 'accepted', 'status_code': 200})+ self.assertEqual(self.attempt_state(), second)++ def test_queued_gate_rechecks_consent_and_version(self):+ self.add_work()+ self.enable()+ gates = []+ def make_client(*_args, envelope_gate, delivery_callback):+ gates.append(envelope_gate)+ self.service.save({'enabled': False})+ return mock.Mock()+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):+ self.service.attempt()+ self.assertFalse(gates[0](self.envelope()))+ self.assertEqual(self.attempt_state()['status'], 'blocked')+ self.assertFalse(self.attempt_state()['attempted'])++ def test_queued_gate_rejects_expired_schema_and_period(self):+ self.add_work()+ self.enable()+ def make_client(*_args, envelope_gate, delivery_callback):+ def capture(*_args, **_kwargs):+ envelope = self.envelope()+ with mock.patch.object(sharing, 'SCHEMA', sharing.SCHEMA + 1):+ self.assertFalse(envelope_gate(envelope))+ with mock.patch.object(sharing, '_period', return_value='2099-W01'):+ self.assertFalse(envelope_gate(envelope))+ delivery_callback({'status': 'blocked', 'reason': 'consent'})+ return SimpleNamespace(capture_event=capture, flush=lambda **_kw: None, close=lambda **_kw: None)+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):+ self.service.attempt()+ self.assertEqual(self.attempt_state()['status'], 'blocked')+ self.assertFalse(self.attempt_state()['attempted'])def test_preview_and_consent_api_are_local_authenticated_and_same_origin(self):import asyncio