Open-source SEO, GEO, and marketing skills for AI agents.
fix: keep OpenClaw follow-up tasks visible (#52)
Tong Chen02928b693a0b
diff --git a/openclaw/artifacts/schemas/queue-item.schema.json b/openclaw/artifacts/schemas/queue-item.schema.jsonindex 05b5f1de..be105ac9 100644--- a/openclaw/artifacts/schemas/queue-item.schema.json+++ b/openclaw/artifacts/schemas/queue-item.schema.json@@ -17,8 +17,10 @@"enum": ["pending","in_progress",+ "ready_for_attention","blocked",- "done"+ "done",+ "processed"]},"due_at": {diff --git a/openclaw/bin/followups_due.py b/openclaw/bin/followups_due.pyindex 071c4a63..22c4944e 100755--- a/openclaw/bin/followups_due.py+++ b/openclaw/bin/followups_due.py@@ -13,14 +13,17 @@ CURRENT_DIR = Path(__file__).resolve().parentif str(CURRENT_DIR) not in sys.path:sys.path.insert(0, str(CURRENT_DIR))-from runtime import bootstrap_workspace, load_json, now_iso, workspace_root+from runtime import bootstrap_workspace, load_json, now_iso, reconcile_schedule_from_queue, save_json, workspace_rootdef parse_iso(value: str | None) -> datetime:if not value:return datetime.max.replace(tzinfo=timezone.utc)normalized = value.replace("Z", "+00:00")- return datetime.fromisoformat(normalized)+ parsed = datetime.fromisoformat(normalized)+ if parsed.tzinfo is None:+ return parsed.replace(tzinfo=timezone.utc)+ return parseddef main(argv: list[str]) -> int:@@ -31,11 +34,16 @@ def main(argv: list[str]) -> int:args = parser.parse_args(argv[1:])root = bootstrap_workspace(workspace_root())- schedule = load_json(root / "schedule.json", {"upcoming": []})+ schedule_path = root / "schedule.json"+ schedule = load_json(schedule_path, {"upcoming": []})+ schedule, restored_from_queue = reconcile_schedule_from_queue(root, schedule)+ save_json(schedule_path, schedule)cutoff = parse_iso(args.as_of)items = []for item in schedule.get("upcoming", []):+ if item.get("status") in {"done", "processed"}:+ continueif args.site and item.get("site_id") != args.site:continuedue_at = parse_iso(item.get("due_at"))@@ -49,6 +57,7 @@ def main(argv: list[str]) -> int:"site_filter": args.site,"count": len(items),"items": items,+ "restored_from_queue": restored_from_queue,}print(json.dumps(result, indent=2))return 0diff --git a/openclaw/bin/run_scheduler.py b/openclaw/bin/run_scheduler.pyindex 76fb04c6..644f6ba5 100755--- a/openclaw/bin/run_scheduler.py+++ b/openclaw/bin/run_scheduler.py@@ -14,14 +14,17 @@ if str(CURRENT_DIR) not in sys.path:sys.path.insert(0, str(CURRENT_DIR))from persist_run import persist_payload-from runtime import bootstrap_workspace, load_json, now_iso, save_json, update_learned_patterns, workspace_root+from runtime import bootstrap_workspace, load_json, now_iso, reconcile_schedule_from_queue, save_json, update_learned_patterns, workspace_rootfrom score_feedback import score_itemdef parse_iso(value: str | None) -> datetime:if not value:return datetime.max.replace(tzinfo=timezone.utc)- return datetime.fromisoformat(value.replace('Z', '+00:00'))+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))+ if parsed.tzinfo is None:+ return parsed.replace(tzinfo=timezone.utc)+ return parseddef update_queue_file(root: Path, item: dict, **updates) -> str | None:@@ -38,12 +41,12 @@ def update_queue_file(root: Path, item: dict, **updates) -> str | None:return str(queue_path)-def build_feedback_payload(item: dict, root: Path) -> dict:+def build_feedback_payload(item: dict, root: Path, score: dict | None = None) -> dict:site_id = item["site_id"]latest = load_json(root / "sites" / site_id / "latest-state.json", {"summary": "No recent state summary.", "open_issues": []})summary = item.get("notes") or latest.get("summary") or "Scheduled follow-up review."action_id = item.get("item_id", "feedback_check")- score = score_item(item)+ score = score or score_item(item)outcome = score.get("outcome", "inconclusive")reason = score.get("reason", "no scoring reason available")score_summary = f"Feedback outcome: {outcome}. Reason: {reason}."@@ -126,6 +129,25 @@ def build_feedback_payload(item: dict, root: Path) -> dict:}+def mark_for_attention(root: Path, item: dict, reason: str) -> dict:+ item["status"] = "ready_for_attention"+ item["surfaced_at"] = now_iso()+ item["attention_reason"] = reason+ update_queue_file(+ root,+ item,+ status="ready_for_attention",+ surfaced_at=item["surfaced_at"],+ attention_reason=reason,+ )+ return {+ "item_id": item.get("item_id"),+ "site_id": item.get("site_id"),+ "type": item.get("type"),+ "reason": reason,+ }++def main(argv: list[str]) -> int:parser = argparse.ArgumentParser(description=__doc__)parser.add_argument("--site")@@ -136,6 +158,7 @@ def main(argv: list[str]) -> int:root = bootstrap_workspace(workspace_root())schedule_path = root / "schedule.json"schedule = load_json(schedule_path, {"schema_version": "1", "upcoming": []})+ schedule, restored_from_queue = reconcile_schedule_from_queue(root, schedule)upcoming = schedule.setdefault("upcoming", [])as_of_dt = parse_iso(args.as_of)@@ -152,9 +175,32 @@ def main(argv: list[str]) -> int:if item.get("type") == "feedback_check":if args.dry_run:- processed.append({"item_id": item.get("item_id"), "site_id": item.get("site_id"), "dry_run": True})+ score = score_item(item)+ if score.get("outcome") == "inconclusive":+ manual_attention.append(+ {+ "item_id": item.get("item_id"),+ "site_id": item.get("site_id"),+ "type": item.get("type"),+ "reason": score.get("reason", "feedback scoring was inconclusive"),+ "dry_run": True,+ }+ )+ else:+ processed.append(+ {+ "item_id": item.get("item_id"),+ "site_id": item.get("site_id"),+ "outcome": score.get("outcome"),+ "dry_run": True,+ }+ )+ continue+ score = score_item(item)+ if score.get("outcome") == "inconclusive":+ manual_attention.append(mark_for_attention(root, item, score.get("reason", "feedback scoring was inconclusive")))continue- payload = build_feedback_payload(item, root)+ payload = build_feedback_payload(item, root, score)result = persist_payload(item["site_id"], payload, root=root)feedback_outcome = payload.get("feedback", {}).get("outcome", "inconclusive")item["status"] = "processed"@@ -185,10 +231,7 @@ def main(argv: list[str]) -> int:)processed.append({"item_id": item.get("item_id"), "site_id": item.get("site_id"), "result_run_dir": result["run_dir"], "outcome": feedback_outcome})else:- item["status"] = "ready_for_attention"- item["surfaced_at"] = now_iso()- update_queue_file(root, item, status="ready_for_attention", surfaced_at=item["surfaced_at"])- manual_attention.append({"item_id": item.get("item_id"), "site_id": item.get("site_id"), "type": item.get("type")})+ manual_attention.append(mark_for_attention(root, item, f"unsupported schedule item type: {item.get('type')}"))if not args.dry_run:save_json(schedule_path, schedule)@@ -198,6 +241,7 @@ def main(argv: list[str]) -> int:"as_of": args.as_of,"processed": processed,"manual_attention": manual_attention,+ "restored_from_queue": restored_from_queue,"dry_run": args.dry_run,}print(json.dumps(result, indent=2))diff --git a/openclaw/bin/runtime.py b/openclaw/bin/runtime.pyindex 1bd9b33c..a893c5df 100755--- a/openclaw/bin/runtime.py+++ b/openclaw/bin/runtime.py@@ -9,6 +9,8 @@ from datetime import datetime, timezonefrom pathlib import Pathfrom typing import Any+TERMINAL_QUEUE_STATUSES = {"done", "processed"}+def workspace_root() -> Path:configured = os.environ.get("TOPRANK_OPENCLAW_HOME")@@ -144,7 +146,15 @@ def upsert_schedule_items(items: list[dict[str, Any]], root: Path | None = None)item_id = item.get("item_id")if not item_id:continue- existing = next((entry for entry in upcoming if entry.get("item_id") == item_id), None)+ site_id = item.get("site_id")+ existing = next(+ (+ entry+ for entry in upcoming+ if entry.get("item_id") == item_id and (site_id is None or entry.get("site_id") == site_id)+ ),+ None,+ )if existing is None:upcoming.append(item)written.append(item)@@ -156,6 +166,58 @@ def upsert_schedule_items(items: list[dict[str, Any]], root: Path | None = None)return written+def reconcile_schedule_from_queue(root: Path | None = None, schedule: dict[str, Any] | None = None) -> tuple[dict[str, Any], list[dict[str, Any]]]:+ """Restore scheduled work from per-site queue files.++ Queue files are the durable per-site task records. `schedule.json` is an+ index used by the runner, so a missing schedule entry should not make a+ pending queue item disappear from backend processing.+ """+ root = bootstrap_workspace(root)+ if schedule is None:+ schedule = load_json(root / "schedule.json", {"schema_version": "1", "upcoming": []})+ upcoming = schedule.setdefault("upcoming", [])+ restored: list[dict[str, Any]] = []++ existing_by_key = {+ (entry.get("site_id"), entry.get("item_id")): entry+ for entry in upcoming+ if entry.get("site_id") and entry.get("item_id")+ }++ sites_dir = root / "sites"+ if not sites_dir.exists():+ return schedule, restored++ for queue_path in sites_dir.glob("*/queue/*.json"):+ try:+ item = load_json(queue_path, {})+ except json.JSONDecodeError:+ continue+ if not isinstance(item, dict):+ continue+ if not item.get("due_at"):+ continue+ status = item.get("status", "pending")+ if status in TERMINAL_QUEUE_STATUSES:+ continue++ site_id = item.get("site_id") or queue_path.parents[1].name+ item_id = item.get("item_id") or queue_path.stem+ item["site_id"] = site_id+ item["item_id"] = item_id+ key = (site_id, item_id)+ existing = existing_by_key.get(key)+ if existing is None:+ upcoming.append(dict(item))+ existing_by_key[key] = upcoming[-1]+ restored.append(upcoming[-1])+ elif existing.get("status") not in TERMINAL_QUEUE_STATUSES:+ existing.update(item)++ return schedule, restored++def upsert_portfolio_site(site_id: str,*,diff --git a/openclaw/tests/test_scheduler.py b/openclaw/tests/test_scheduler.pynew file mode 100644index 00000000..f343cff1--- /dev/null+++ b/openclaw/tests/test_scheduler.py@@ -0,0 +1,166 @@+import contextlib+import io+import json+import os+import sys+import tempfile+import unittest+from pathlib import Path++BIN_DIR = Path(__file__).resolve().parents[1] / "bin"+if str(BIN_DIR) not in sys.path:+ sys.path.insert(0, str(BIN_DIR))++import run_scheduler+from runtime import bootstrap_workspace, load_json, reconcile_schedule_from_queue, save_json, upsert_schedule_items+++class OpenClawSchedulerTest(unittest.TestCase):+ def setUp(self) -> None:+ self.tempdir = tempfile.TemporaryDirectory()+ self.root = Path(self.tempdir.name)+ self.previous_home = os.environ.get("TOPRANK_OPENCLAW_HOME")+ os.environ["TOPRANK_OPENCLAW_HOME"] = str(self.root)+ bootstrap_workspace(self.root)++ def tearDown(self) -> None:+ if self.previous_home is None:+ os.environ.pop("TOPRANK_OPENCLAW_HOME", None)+ else:+ os.environ["TOPRANK_OPENCLAW_HOME"] = self.previous_home+ self.tempdir.cleanup()++ def write_queue_item(self, site_id: str, item: dict) -> Path:+ queue_path = self.root / "sites" / site_id / "queue" / f"{item['item_id']}.json"+ save_json(queue_path, {"site_id": site_id, **item})+ return queue_path++ def run_scheduler(self, *args: str) -> dict:+ output = io.StringIO()+ with contextlib.redirect_stdout(output):+ code = run_scheduler.main(["run_scheduler.py", *args])+ self.assertEqual(code, 0)+ return json.loads(output.getvalue())++ def test_upsert_schedule_items_uses_site_and_item_id(self) -> None:+ upsert_schedule_items(+ [+ {"site_id": "a.com", "item_id": "same", "type": "feedback_check", "status": "pending", "due_at": "2026-01-01T00:00:00Z"},+ {"site_id": "b.com", "item_id": "same", "type": "feedback_check", "status": "pending", "due_at": "2026-01-01T00:00:00Z"},+ ],+ self.root,+ )++ schedule = load_json(self.root / "schedule.json", {})+ self.assertEqual(len(schedule["upcoming"]), 2)+ self.assertEqual({item["site_id"] for item in schedule["upcoming"]}, {"a.com", "b.com"})++ def test_reconcile_restores_pending_queue_item_missing_from_schedule(self) -> None:+ self.write_queue_item(+ "example.com",+ {+ "item_id": "orphaned_followup",+ "type": "feedback_check",+ "status": "pending",+ "due_at": "2026-01-01T00:00:00Z",+ },+ )++ schedule, restored = reconcile_schedule_from_queue(self.root)++ self.assertEqual(len(restored), 1)+ self.assertEqual(schedule["upcoming"][0]["item_id"], "orphaned_followup")++ def test_scheduler_accepts_naive_due_timestamps(self) -> None:+ self.write_queue_item(+ "example.com",+ {+ "item_id": "naive_due_followup",+ "type": "feedback_check",+ "status": "pending",+ "due_at": "2026-01-01T00:00:00",+ "primary_metric": "organic_clicks_7d",+ "baseline_metrics": {"organic_clicks_7d": 10.0},+ },+ )++ result = self.run_scheduler("--dry-run", "--as-of", "2026-01-02T00:00:00Z")++ self.assertEqual(result["manual_attention"][0]["item_id"], "naive_due_followup")++ def test_inconclusive_feedback_is_not_marked_done(self) -> None:+ queue_path = self.write_queue_item(+ "example.com",+ {+ "item_id": "missing_metrics_followup",+ "type": "feedback_check",+ "status": "pending",+ "due_at": "2026-01-01T00:00:00Z",+ "primary_metric": "organic_clicks_7d",+ "baseline_metrics": {"organic_clicks_7d": 10.0},+ },+ )++ result = self.run_scheduler("--as-of", "2026-01-02T00:00:00Z")+ queue_item = load_json(queue_path, {})+ schedule = load_json(self.root / "schedule.json", {})++ self.assertEqual(result["processed"], [])+ self.assertEqual(result["manual_attention"][0]["item_id"], "missing_metrics_followup")+ self.assertIn("missing metric data", result["manual_attention"][0]["reason"])+ self.assertEqual(queue_item["status"], "ready_for_attention")+ self.assertEqual(schedule["upcoming"][0]["status"], "ready_for_attention")+ self.assertFalse((self.root / "sites" / "example.com" / "feedback" / "missing_metrics_followup.json").exists())++ def test_dry_run_reports_inconclusive_feedback_as_manual_attention(self) -> None:+ queue_path = self.write_queue_item(+ "example.com",+ {+ "item_id": "dry_run_missing_metrics",+ "type": "feedback_check",+ "status": "pending",+ "due_at": "2026-01-01T00:00:00Z",+ "primary_metric": "organic_clicks_7d",+ "baseline_metrics": {"organic_clicks_7d": 10.0},+ },+ )++ result = self.run_scheduler("--dry-run", "--as-of", "2026-01-02T00:00:00Z")+ queue_item = load_json(queue_path, {})++ self.assertEqual(result["processed"], [])+ self.assertEqual(result["manual_attention"][0]["item_id"], "dry_run_missing_metrics")+ self.assertTrue(result["manual_attention"][0]["dry_run"])+ self.assertEqual(queue_item["status"], "pending")++ def test_feedback_with_observed_metrics_is_processed(self) -> None:+ queue_path = self.write_queue_item(+ "example.com",+ {+ "item_id": "scored_followup",+ "type": "feedback_check",+ "status": "pending",+ "due_at": "2026-01-01T00:00:00Z",+ "notes": "Re-check organic clicks.",+ "action_type": "page_improvement",+ "primary_metric": "organic_clicks_7d",+ "primary_direction": "higher_better",+ "success_threshold_pct": 0.1,+ "baseline_metrics": {"organic_clicks_7d": 10.0},+ "observed_metrics": {"organic_clicks_7d": 12.0},+ "guardrail_metrics": [],+ },+ )++ result = self.run_scheduler("--as-of", "2026-01-02T00:00:00Z")+ queue_item = load_json(queue_path, {})+ feedback = load_json(self.root / "sites" / "example.com" / "feedback" / "scored_followup.json", {})++ self.assertEqual(result["manual_attention"], [])+ self.assertEqual(result["processed"][0]["outcome"], "win")+ self.assertEqual(queue_item["status"], "done")+ self.assertEqual(feedback["status"], "win")+++if __name__ == "__main__":+ unittest.main()