Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(telemetry): 跳过空研究快照并核实每周投递状态

longmeidaoe7191e7da4a0
diff --git a/src/peach/library_sharing.py b/src/peach/library_sharing.py
index b29651459..966967ce9 100644
--- a/src/peach/library_sharing.py
+++ b/src/peach/library_sharing.py
@@ -30,6 +30,31 @@ def _path(config):
return config.directory("state") / "library-sharing.json"
+def _state_path(config):
+ return config.directory("state") / "library-sharing-attempt.json"
+
+
+def _state_lock(config):
+ from filelock import FileLock
+ path = _state_path(config)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ return FileLock(str(path) + ".lock", timeout=5)
+
+
+def _period():
+ return datetime.now(timezone.utc).strftime("%G-W%V")
+
+
+def _used(value, period):
+ return value.get("period") == period and (value.get("attempted") is True
+ or ("attempted" not in value and value.get("status") in {"attempted", "failed"}))
+
+
+def useful_payload(payload):
+ """公共编号或任一偏好分组提供可分析内容。"""
+ return bool(payload["library"] or any(payload["preferences"].values()))
+
+
def consent(config):
value = remote_reporting._read(_path(config))
return value if value.get("enabled") is True and value.get("schema") == SCHEMA else {}
@@ -41,7 +66,12 @@ def save_choice(config, enabled):
raise ValueError("馆藏与口味上传选项必须是布尔值")
value = {"enabled": enabled, "schema": SCHEMA, "revision": uuid.uuid4().hex,
"selected_at": datetime.now(timezone.utc).isoformat()}
- atomic_write_text(_path(config), json.dumps(value))
+ with _state_lock(config):
+ atomic_write_text(_path(config), json.dumps(value))
+ state = remote_reporting._read(_state_path(config))
+ if state.get("status") == "queued":
+ status = "unknown" if state.get("attempted") else "blocked"
+ atomic_write_text(_state_path(config), json.dumps({**state, "status": status, "reason": "consent"}))
return snapshot(config)
@@ -56,6 +86,8 @@ def target():
def snapshot(config):
attempt = remote_reporting._read(config.directory("state") / "library-sharing-attempt.json")
+ if attempt.get("status") == "attempted":
+ attempt = {**attempt, "status": "unknown", "attempted": True}
return {"enabled": bool(consent(config)), "available": bool(target()),
"schema": SCHEMA, "fields": FIELDS, "notice": NOTICE,
"last_attempt": attempt if attempt.get("schema") == SCHEMA else {}}
@@ -123,7 +155,7 @@ class Service:
self._wake.set()
return snapshot(self.config)
- def _gate(self, revision, envelope):
+ def _gate(self, revision, envelope, attempt_id=None):
current = consent(self.config)
if self._stop.is_set() or not current or current.get("revision") != revision:
return False
@@ -141,7 +173,7 @@ class Service:
if len(raw) > MAX_RAW_BYTES:
return False
payload = json.loads(raw)
- if not valid_payload(payload):
+ if not valid_payload(payload) or not useful_payload(payload):
return False
except (ValueError, KeyError, TypeError, OSError, EOFError):
return False
@@ -149,38 +181,97 @@ class Service:
for key in tuple(event):
if key not in {"event_id", "timestamp", "platform"}:
del event[key]
- event.update(message="Peach library and taste research", level="info",
- tags={"purpose": "population-research", "schema": str(SCHEMA), "period": payload["period"]})
+ metadata = remote_reporting.event_metadata("research")
+ event.update(**metadata, message="Peach library and taste research", level="info")
+ event["tags"].update(schema=str(SCHEMA), period=payload["period"])
for key in tuple(envelope.headers):
if key not in {"event_id", "sent_at"}:
del envelope.headers[key]
+ if attempt_id is None:
+ return True
+ with _state_lock(self.config):
+ current = consent(self.config)
+ state = remote_reporting._read(_state_path(self.config))
+ if (self._stop.is_set() or not current or current.get("revision") != revision
+ or state.get("attempt_id") != attempt_id or state.get("revision") != revision
+ or state.get("period") != _period() or state.get("status") != "queued"
+ or _used(state, _period())):
+ return False
+ atomic_write_text(_state_path(self.config), json.dumps({**state, "attempted": True}))
return True
+ def _delivery(self, identity, result):
+ """仅更新当前投递的脱敏回执,不把 HTTP 接收解释为留存。"""
+ if result.get("status") not in {"accepted", "failed", "unknown", "blocked"}:
+ return
+ with _state_lock(self.config):
+ state = remote_reporting._read(_state_path(self.config))
+ if (any(state.get(key) != value for key, value in identity.items())
+ or consent(self.config).get("revision") != identity["revision"]
+ or state.get("status") not in {"queued", "unknown"}):
+ return
+ if state.get("status") == "unknown" and result["status"] == "unknown":
+ return
+ status = result["status"]
+ if status == "unknown" and not state.get("attempted"):
+ status = "failed"
+ value = {key: value for key, value in state.items() if key not in {"reason", "status_code"}}
+ value["status"] = status
+ if type(result.get("status_code")) is int and 100 <= result["status_code"] <= 599:
+ value["status_code"] = result["status_code"]
+ if result.get("reason") in {"network", "http_status", "rate_limit", "queue_full", "consent", "disabled", "unknown"}:
+ value["reason"] = result["reason"]
+ atomic_write_text(_state_path(self.config), json.dumps(value))
+
def attempt(self):
current, destination = consent(self.config), target()
if not current or not destination or self._stop.is_set():
return
- period = datetime.now(timezone.utc).strftime("%G-W%V")
- state = self.config.directory("state") / "library-sharing-attempt.json"
- from filelock import FileLock
- state.parent.mkdir(parents=True, exist_ok=True)
- with FileLock(str(state) + ".lock", timeout=.1):
+ period = _period()
+ state = _state_path(self.config)
+ with _state_lock(self.config):
previous = remote_reporting._read(state)
- if previous.get("schema") == SCHEMA and previous.get("period") == period:
+ if _used(previous, period):
return
- atomic_write_text(state, json.dumps({"schema": SCHEMA, "period": period, "status": "attempted"}))
- payload, data = collect(self.contract)
- data = compressed(data)
- from sentry_sdk.attachments import Attachment
- client = remote_reporting._new_client(destination, envelope_gate=lambda envelope: self._gate(current["revision"], envelope))
+ identity = {"schema": SCHEMA, "period": period, "revision": current["revision"], "attempt_id": uuid.uuid4().hex}
+ value = {**identity, "attempted": False}
+ stage = "collection"
+ try:
+ payload, data = collect(self.contract)
+ if useful_payload(payload):
+ stage = "compression"
+ data = compressed(data)
+ value["status"] = "queued"
+ else:
+ value["status"] = "waiting-data"
+ except Exception:
+ value.update(status="failed", reason=stage)
+ with _state_lock(self.config):
+ if (consent(self.config).get("revision") != identity["revision"] or _period() != period
+ or _used(remote_reporting._read(state), period)):
+ return
+ atomic_write_text(state, json.dumps(value))
+ if value["status"] != "queued":
+ return
+ client = None
try:
+ from sentry_sdk.attachments import Attachment
+ client = remote_reporting._new_client(destination,
+ envelope_gate=lambda envelope: self._gate(current["revision"], envelope, identity["attempt_id"]),
+ delivery_callback=lambda result: self._delivery(identity, result))
client.capture_event({"message": "Peach library and taste research", "level": "info",
- "tags": {"purpose": "population-research", "schema": str(SCHEMA), "period": payload["period"]}},
+ **remote_reporting.event_metadata("research")},
hint={"attachments": [Attachment(bytes=data, filename=FILENAME,
content_type="application/gzip")]})
client.flush(timeout=5)
+ except Exception:
+ pass
finally:
- client.close(timeout=0)
+ try:
+ if client is not None:
+ client.close(timeout=0)
+ finally:
+ self._delivery(identity, {"status": "unknown", "reason": "unknown"})
def start(self):
if self._thread and self._thread.is_alive():
@@ -190,14 +281,9 @@ class Service:
while not self._stop.is_set():
try:
self.attempt()
- except Exception as error:
+ except Exception:
# 分享失败不把数据或错误原文转交诊断通道。
- try:
- state = self.config.directory("state") / "library-sharing-attempt.json"
- value = remote_reporting._read(state)
- atomic_write_text(state, json.dumps({**value, "status": "failed", "reason": type(error).__name__}))
- except OSError:
- pass
+ pass
self._wake.wait(3600)
self._wake.clear()
self._thread = threading.Thread(target=run, name="PeachLibrarySharing", daemon=True)
diff --git a/tests/test_library_sharing.py b/tests/test_library_sharing.py
index e63bf9015..5948da56a 100644
--- a/tests/test_library_sharing.py
+++ b/tests/test_library_sharing.py
@@ -3,6 +3,7 @@ import gzip
import json
import os
import tempfile
+import threading
import unittest
from pathlib import Path
from types import SimpleNamespace
@@ -37,6 +38,22 @@ class LibrarySharingTests(unittest.TestCase):
return self.service.save({'enabled': True, 'schema': sharing.SCHEMA,
'consent': True, 'preview_token': preview['token']})
+ def add_work(self):
+ with self.database.write_transaction() as connection:
+ connection.execute("INSERT INTO asset(location,path,name,code) VALUES('local','work','title','EXM-001')")
+
+ def attempt_state(self):
+ return sharing.remote_reporting._read(sharing._state_path(self.config))
+
+ def envelope(self, data=None):
+ from sentry_sdk.envelope import Envelope
+ from sentry_sdk.attachments import Attachment
+ envelope = Envelope()
+ envelope.add_event({'message': 'Peach research'})
+ wire = sharing.compressed(sharing.collect(self.contract)[1]) if data is None else data
+ envelope.add_item(Attachment(bytes=wire, filename=sharing.FILENAME).to_envelope_item())
+ return envelope
+
def test_disabled_and_missing_endpoint_do_not_read_library_or_construct_sdk(self):
with mock.patch.object(sharing, 'collect') as collect, \
mock.patch('peach.remote_reporting._new_client') as client:
@@ -192,6 +209,7 @@ class LibrarySharingTests(unittest.TestCase):
def test_real_sdk_sends_only_research_event_and_reviewed_attachment_once_per_week(self):
from sentry_sdk.transport import HttpTransport
+ self.add_work()
self.enable()
sent = []
with mock.patch.object(HttpTransport, '_send_envelope', side_effect=lambda envelope: sent.append(envelope)):
@@ -204,7 +222,9 @@ class LibrarySharingTests(unittest.TestCase):
self.assertNotIn('extra', event)
self.assertNotIn('server_name', event)
self.assertNotIn('user', event)
- self.assertNotIn('release', event)
+ self.assertEqual(event['release'], sharing.remote_reporting.event_metadata('research')['release'])
+ self.assertEqual(event['environment'], sharing.remote_reporting.event_metadata('research')['environment'])
+ self.assertEqual(event['tags']['runtime'], sharing.remote_reporting.event_metadata('research')['tags']['runtime'])
self.assertEqual(event['tags']['purpose'], 'population-research')
attachment = json.loads(gzip.decompress(envelope.items[1].payload.get_bytes()))
self.assertEqual(attachment['schema'], sharing.SCHEMA)
@@ -215,6 +235,7 @@ class LibrarySharingTests(unittest.TestCase):
def test_disable_invalidates_queued_attachment(self):
from sentry_sdk.envelope import Envelope
from sentry_sdk.attachments import Attachment
+ self.add_work()
self.enable()
previous = sharing.consent(self.config)['revision']
envelope = Envelope()
@@ -258,11 +279,239 @@ class LibrarySharingTests(unittest.TestCase):
with mock.patch.object(sharing, 'MAX_RAW_BYTES', 1), self.assertRaises(ValueError):
self.service.preview()
self.assertFalse(sharing.snapshot(self.config)['enabled'])
+ self.add_work()
self.enable()
with mock.patch.object(sharing, 'MAX_BYTES', 1), mock.patch('peach.remote_reporting._new_client') as client:
- with self.assertRaises(ValueError):
- self.service.attempt()
+ self.service.attempt()
+ client.assert_not_called()
+ self.assertEqual(self.attempt_state()['status'], 'failed')
+ self.assertEqual(self.attempt_state()['reason'], 'compression')
+ self.assertFalse(self.attempt_state()['attempted'])
+
+ def test_empty_and_uncoded_without_preferences_wait_for_data(self):
+ self.enable()
+ with mock.patch('peach.remote_reporting._new_client') as client:
+ self.service.attempt()
+ with self.database.write_transaction() as connection:
+ connection.execute("INSERT INTO asset(location,path,name) VALUES('local','uncoded','title')")
+ self.service.attempt()
+ client.assert_not_called()
+ self.assertEqual(self.attempt_state()['status'], 'waiting-data')
+ self.assertFalse(self.attempt_state()['attempted'])
+ self.assertFalse(self.service._gate(sharing.consent(self.config)['revision'], self.envelope()))
+
+ def test_uncoded_preference_group_can_be_sent(self):
+ with self.database.write_transaction() as connection:
+ connection.execute("INSERT INTO asset(location,path,name) VALUES('local','uncoded','title')")
+ connection.execute("INSERT INTO asset_tag(asset_id,tag,source) VALUES(1,'手动标签','web-user')")
+ self.enable()
+ with mock.patch('peach.remote_reporting._new_client') as client:
+ self.service.attempt()
+ client.assert_called_once()
+
+ def test_collection_failure_does_not_spend_weekly_network_attempt(self):
+ self.enable()
+ with mock.patch.object(sharing, 'collect', side_effect=OSError('private path')):
+ self.service.attempt()
+ self.assertEqual(self.attempt_state()['status'], 'failed')
+ self.assertEqual(self.attempt_state()['reason'], 'collection')
+ self.assertFalse(self.attempt_state()['attempted'])
+ self.assertNotIn('private path', json.dumps(self.attempt_state()))
+ self.add_work()
+ with mock.patch('peach.remote_reporting._new_client') as client:
+ self.service.attempt()
+ client.assert_called_once()
+
+ def test_disable_during_collection_prevents_queueing(self):
+ self.add_work()
+ self.enable()
+ collect = sharing.collect
+ def disable_then_collect(contract):
+ sharing.save_choice(self.config, False)
+ return collect(contract)
+ with mock.patch.object(sharing, 'collect', side_effect=disable_then_collect), \
+ mock.patch('peach.remote_reporting._new_client') as client:
+ self.service.attempt()
client.assert_not_called()
+ self.assertFalse(sharing.snapshot(self.config)['enabled'])
+
+ def test_queue_failure_does_not_spend_network_attempt(self):
+ self.add_work()
+ self.enable()
+ def make_client(*_args, envelope_gate, delivery_callback):
+ def capture(*_args, **_kwargs):
+ delivery_callback({'status': 'failed', 'reason': 'queue_full'})
+ return SimpleNamespace(capture_event=capture, flush=lambda **_kw: None, close=lambda **_kw: None)
+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client) as client:
+ self.service.attempt()
+ self.service.attempt()
+ self.assertEqual(client.call_count, 2)
+ self.assertEqual(self.attempt_state()['status'], 'failed')
+ self.assertEqual(self.attempt_state()['reason'], 'queue_full')
+ self.assertFalse(self.attempt_state()['attempted'])
+
+ def test_delivery_receipts_record_only_status_and_safe_details(self):
+ self.add_work()
+ self.enable()
+ for result in ({'status': 'accepted', 'status_code': 200},
+ {'status': 'failed', 'status_code': 403, 'reason': 'http_status'},
+ {'status': 'unknown', 'reason': 'network'}):
+ sharing._state_path(self.config).unlink(missing_ok=True)
+ def make_client(*_args, envelope_gate, delivery_callback):
+ def capture(*_args, **_kwargs):
+ self.assertTrue(envelope_gate(self.envelope()))
+ delivery_callback({**result, 'url': 'private-url', 'detail': 'private-content'})
+ return SimpleNamespace(capture_event=capture, flush=lambda **_kw: None, close=lambda **_kw: None)
+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):
+ self.service.attempt()
+ self.enable()
+ self.service.attempt()
+ state = self.attempt_state()
+ self.assertEqual(state['status'], result['status'])
+ self.assertTrue(state['attempted'])
+ if 'reason' in result:
+ self.assertEqual(state['reason'], result['reason'])
+ self.assertNotIn('private', json.dumps(state))
+
+ def test_missing_receipt_is_unknown_and_does_not_retry_network(self):
+ self.add_work()
+ self.enable()
+ callbacks = []
+ def make_client(*_args, envelope_gate, delivery_callback):
+ callbacks.append(delivery_callback)
+ def capture(*_args, **_kwargs):
+ self.assertTrue(envelope_gate(self.envelope()))
+ self.assertEqual(self.attempt_state()['status'], 'queued')
+ return SimpleNamespace(capture_event=capture, flush=lambda **_kw: None, close=lambda **_kw: None)
+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client) as client:
+ self.service.attempt()
+ self.assertEqual(self.attempt_state()['status'], 'unknown')
+ self.service.attempt()
+ client.assert_called_once()
+ callbacks[0]({'status': 'accepted', 'status_code': 200})
+ self.assertEqual(self.attempt_state()['status'], 'accepted')
+ self.assertNotIn('reason', self.attempt_state())
+
+ def test_real_sdk_http_receipt_controls_local_research_status(self):
+ from sentry_sdk.transport import HttpTransport
+ self.add_work()
+ self.enable()
+ for status, headers, expected in ((200, {}, 'accepted'), (413, {}, 'failed'), (429, {}, 'failed'),
+ (200, {'x-sentry-rate-limits': '60:error:organization'}, 'unknown'), (None, {}, 'unknown')):
+ with self.subTest(status=status, headers=headers):
+ sharing._state_path(self.config).unlink(missing_ok=True)
+ response = SimpleNamespace(status=status, headers=headers, close=mock.Mock(), data=b'private')
+ with mock.patch.dict(os.environ, {'PEACH_REPORTING_TEST_MODE': '0'}), \
+ mock.patch.object(HttpTransport, '_request', return_value=response,
+ side_effect=OSError('private proxy') if status is None else None) as request:
+ self.service.attempt()
+ self.service.attempt()
+ request.assert_called_once()
+ state = self.attempt_state()
+ self.assertEqual(state['status'], expected)
+ self.assertTrue(state['attempted'])
+ if status is not None:
+ self.assertEqual(state['status_code'], status)
+ self.assertNotIn('private', json.dumps(state))
+
+ def test_cross_week_receipt_cannot_overwrite_new_attempt(self):
+ self.add_work()
+ self.enable()
+ callbacks = []
+ def make_client(*_args, envelope_gate, delivery_callback):
+ callbacks.append(delivery_callback)
+ return mock.Mock()
+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):
+ self.service.attempt()
+ with mock.patch.object(sharing, '_period', return_value='2099-W01'):
+ self.service.attempt()
+ state = self.attempt_state()
+ callbacks[0]({'status': 'accepted', 'status_code': 200})
+ self.assertEqual(self.attempt_state(), state)
+
+ def test_legacy_attempt_is_unknown_and_conserves_weekly_budget(self):
+ self.enable()
+ sharing._state_path(self.config).write_text(json.dumps(
+ {'schema': sharing.SCHEMA, 'period': sharing._period(), 'status': 'attempted'}))
+ self.assertEqual(sharing.snapshot(self.config)['last_attempt']['status'], 'unknown')
+ with mock.patch.object(sharing, 'collect') as collect:
+ self.service.attempt()
+ collect.assert_not_called()
+
+ def test_concurrent_services_allow_only_one_network_reservation(self):
+ self.add_work()
+ self.enable()
+ sent, errors = [], []
+ def make_client(*_args, envelope_gate, delivery_callback):
+ def capture(*_args, **_kwargs):
+ if envelope_gate(self.envelope()):
+ sent.append(True)
+ delivery_callback({'status': 'accepted', 'status_code': 200})
+ return SimpleNamespace(capture_event=capture, flush=lambda **_kw: None, close=lambda **_kw: None)
+ def run():
+ try:
+ sharing.Service(self.config, self.contract).attempt()
+ except Exception as error:
+ errors.append(error)
+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):
+ threads = [threading.Thread(target=run) for _ in range(5)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(timeout=10)
+ self.assertFalse(any(thread.is_alive() for thread in threads))
+ self.assertEqual(errors, [])
+ self.assertEqual(len(sent), 1)
+
+ def test_obsolete_delivery_cannot_overwrite_current_attempt(self):
+ self.add_work()
+ self.enable()
+ callbacks = []
+ def make_client(*_args, envelope_gate, delivery_callback):
+ callbacks.append(delivery_callback)
+ return mock.Mock()
+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):
+ self.service.attempt()
+ first = self.attempt_state()
+ self.service.attempt()
+ second = self.attempt_state()
+ self.assertNotEqual(first['attempt_id'], second['attempt_id'])
+ callbacks[0]({'status': 'accepted', 'status_code': 200})
+ self.assertEqual(self.attempt_state(), second)
+ self.enable()
+ callbacks[1]({'status': 'accepted', 'status_code': 200})
+ self.assertEqual(self.attempt_state(), second)
+
+ def test_queued_gate_rechecks_consent_and_version(self):
+ self.add_work()
+ self.enable()
+ gates = []
+ def make_client(*_args, envelope_gate, delivery_callback):
+ gates.append(envelope_gate)
+ self.service.save({'enabled': False})
+ return mock.Mock()
+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):
+ self.service.attempt()
+ self.assertFalse(gates[0](self.envelope()))
+ self.assertEqual(self.attempt_state()['status'], 'blocked')
+ self.assertFalse(self.attempt_state()['attempted'])
+
+ def test_queued_gate_rejects_expired_schema_and_period(self):
+ self.add_work()
+ self.enable()
+ def make_client(*_args, envelope_gate, delivery_callback):
+ def capture(*_args, **_kwargs):
+ envelope = self.envelope()
+ with mock.patch.object(sharing, 'SCHEMA', sharing.SCHEMA + 1):
+ self.assertFalse(envelope_gate(envelope))
+ with mock.patch.object(sharing, '_period', return_value='2099-W01'):
+ self.assertFalse(envelope_gate(envelope))
+ delivery_callback({'status': 'blocked', 'reason': 'consent'})
+ return SimpleNamespace(capture_event=capture, flush=lambda **_kw: None, close=lambda **_kw: None)
+ with mock.patch('peach.remote_reporting._new_client', side_effect=make_client):
+ self.service.attempt()
+ self.assertEqual(self.attempt_state()['status'], 'blocked')
+ self.assertFalse(self.attempt_state()['attempted'])
def test_preview_and_consent_api_are_local_authenticated_and_same_origin(self):
import asyncio