Skip to content

Open-source SEO, GEO, and marketing skills for AI agents.

fix: keep OpenClaw follow-up tasks visible (#52)

Tong Chen02928b693a0b
diff --git a/openclaw/artifacts/schemas/queue-item.schema.json b/openclaw/artifacts/schemas/queue-item.schema.json
index 05b5f1de..be105ac9 100644
--- a/openclaw/artifacts/schemas/queue-item.schema.json
+++ b/openclaw/artifacts/schemas/queue-item.schema.json
@@ -17,8 +17,10 @@
"enum": [
"pending",
"in_progress",
+ "ready_for_attention",
"blocked",
- "done"
+ "done",
+ "processed"
]
},
"due_at": {
diff --git a/openclaw/bin/followups_due.py b/openclaw/bin/followups_due.py
index 071c4a63..22c4944e 100755
--- a/openclaw/bin/followups_due.py
+++ b/openclaw/bin/followups_due.py
@@ -13,14 +13,17 @@ CURRENT_DIR = Path(__file__).resolve().parent
if str(CURRENT_DIR) not in sys.path:
sys.path.insert(0, str(CURRENT_DIR))
-from runtime import bootstrap_workspace, load_json, now_iso, workspace_root
+from runtime import bootstrap_workspace, load_json, now_iso, reconcile_schedule_from_queue, save_json, workspace_root
def parse_iso(value: str | None) -> datetime:
if not value:
return datetime.max.replace(tzinfo=timezone.utc)
normalized = value.replace("Z", "+00:00")
- return datetime.fromisoformat(normalized)
+ parsed = datetime.fromisoformat(normalized)
+ if parsed.tzinfo is None:
+ return parsed.replace(tzinfo=timezone.utc)
+ return parsed
def main(argv: list[str]) -> int:
@@ -31,11 +34,16 @@ def main(argv: list[str]) -> int:
args = parser.parse_args(argv[1:])
root = bootstrap_workspace(workspace_root())
- schedule = load_json(root / "schedule.json", {"upcoming": []})
+ schedule_path = root / "schedule.json"
+ schedule = load_json(schedule_path, {"upcoming": []})
+ schedule, restored_from_queue = reconcile_schedule_from_queue(root, schedule)
+ save_json(schedule_path, schedule)
cutoff = parse_iso(args.as_of)
items = []
for item in schedule.get("upcoming", []):
+ if item.get("status") in {"done", "processed"}:
+ continue
if args.site and item.get("site_id") != args.site:
continue
due_at = parse_iso(item.get("due_at"))
@@ -49,6 +57,7 @@ def main(argv: list[str]) -> int:
"site_filter": args.site,
"count": len(items),
"items": items,
+ "restored_from_queue": restored_from_queue,
}
print(json.dumps(result, indent=2))
return 0
diff --git a/openclaw/bin/run_scheduler.py b/openclaw/bin/run_scheduler.py
index 76fb04c6..644f6ba5 100755
--- a/openclaw/bin/run_scheduler.py
+++ b/openclaw/bin/run_scheduler.py
@@ -14,14 +14,17 @@ if str(CURRENT_DIR) not in sys.path:
sys.path.insert(0, str(CURRENT_DIR))
from persist_run import persist_payload
-from runtime import bootstrap_workspace, load_json, now_iso, save_json, update_learned_patterns, workspace_root
+from runtime import bootstrap_workspace, load_json, now_iso, reconcile_schedule_from_queue, save_json, update_learned_patterns, workspace_root
from score_feedback import score_item
def parse_iso(value: str | None) -> datetime:
if not value:
return datetime.max.replace(tzinfo=timezone.utc)
- return datetime.fromisoformat(value.replace('Z', '+00:00'))
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ if parsed.tzinfo is None:
+ return parsed.replace(tzinfo=timezone.utc)
+ return parsed
def update_queue_file(root: Path, item: dict, **updates) -> str | None:
@@ -38,12 +41,12 @@ def update_queue_file(root: Path, item: dict, **updates) -> str | None:
return str(queue_path)
-def build_feedback_payload(item: dict, root: Path) -> dict:
+def build_feedback_payload(item: dict, root: Path, score: dict | None = None) -> dict:
site_id = item["site_id"]
latest = load_json(root / "sites" / site_id / "latest-state.json", {"summary": "No recent state summary.", "open_issues": []})
summary = item.get("notes") or latest.get("summary") or "Scheduled follow-up review."
action_id = item.get("item_id", "feedback_check")
- score = score_item(item)
+ score = score or score_item(item)
outcome = score.get("outcome", "inconclusive")
reason = score.get("reason", "no scoring reason available")
score_summary = f"Feedback outcome: {outcome}. Reason: {reason}."
@@ -126,6 +129,25 @@ def build_feedback_payload(item: dict, root: Path) -> dict:
}
+def mark_for_attention(root: Path, item: dict, reason: str) -> dict:
+ item["status"] = "ready_for_attention"
+ item["surfaced_at"] = now_iso()
+ item["attention_reason"] = reason
+ update_queue_file(
+ root,
+ item,
+ status="ready_for_attention",
+ surfaced_at=item["surfaced_at"],
+ attention_reason=reason,
+ )
+ return {
+ "item_id": item.get("item_id"),
+ "site_id": item.get("site_id"),
+ "type": item.get("type"),
+ "reason": reason,
+ }
+
+
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--site")
@@ -136,6 +158,7 @@ def main(argv: list[str]) -> int:
root = bootstrap_workspace(workspace_root())
schedule_path = root / "schedule.json"
schedule = load_json(schedule_path, {"schema_version": "1", "upcoming": []})
+ schedule, restored_from_queue = reconcile_schedule_from_queue(root, schedule)
upcoming = schedule.setdefault("upcoming", [])
as_of_dt = parse_iso(args.as_of)
@@ -152,9 +175,32 @@ def main(argv: list[str]) -> int:
if item.get("type") == "feedback_check":
if args.dry_run:
- processed.append({"item_id": item.get("item_id"), "site_id": item.get("site_id"), "dry_run": True})
+ score = score_item(item)
+ if score.get("outcome") == "inconclusive":
+ manual_attention.append(
+ {
+ "item_id": item.get("item_id"),
+ "site_id": item.get("site_id"),
+ "type": item.get("type"),
+ "reason": score.get("reason", "feedback scoring was inconclusive"),
+ "dry_run": True,
+ }
+ )
+ else:
+ processed.append(
+ {
+ "item_id": item.get("item_id"),
+ "site_id": item.get("site_id"),
+ "outcome": score.get("outcome"),
+ "dry_run": True,
+ }
+ )
+ continue
+ score = score_item(item)
+ if score.get("outcome") == "inconclusive":
+ manual_attention.append(mark_for_attention(root, item, score.get("reason", "feedback scoring was inconclusive")))
continue
- payload = build_feedback_payload(item, root)
+ payload = build_feedback_payload(item, root, score)
result = persist_payload(item["site_id"], payload, root=root)
feedback_outcome = payload.get("feedback", {}).get("outcome", "inconclusive")
item["status"] = "processed"
@@ -185,10 +231,7 @@ def main(argv: list[str]) -> int:
)
processed.append({"item_id": item.get("item_id"), "site_id": item.get("site_id"), "result_run_dir": result["run_dir"], "outcome": feedback_outcome})
else:
- item["status"] = "ready_for_attention"
- item["surfaced_at"] = now_iso()
- update_queue_file(root, item, status="ready_for_attention", surfaced_at=item["surfaced_at"])
- manual_attention.append({"item_id": item.get("item_id"), "site_id": item.get("site_id"), "type": item.get("type")})
+ manual_attention.append(mark_for_attention(root, item, f"unsupported schedule item type: {item.get('type')}"))
if not args.dry_run:
save_json(schedule_path, schedule)
@@ -198,6 +241,7 @@ def main(argv: list[str]) -> int:
"as_of": args.as_of,
"processed": processed,
"manual_attention": manual_attention,
+ "restored_from_queue": restored_from_queue,
"dry_run": args.dry_run,
}
print(json.dumps(result, indent=2))
diff --git a/openclaw/bin/runtime.py b/openclaw/bin/runtime.py
index 1bd9b33c..a893c5df 100755
--- a/openclaw/bin/runtime.py
+++ b/openclaw/bin/runtime.py
@@ -9,6 +9,8 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Any
+TERMINAL_QUEUE_STATUSES = {"done", "processed"}
+
def workspace_root() -> Path:
configured = os.environ.get("TOPRANK_OPENCLAW_HOME")
@@ -144,7 +146,15 @@ def upsert_schedule_items(items: list[dict[str, Any]], root: Path | None = None)
item_id = item.get("item_id")
if not item_id:
continue
- existing = next((entry for entry in upcoming if entry.get("item_id") == item_id), None)
+ site_id = item.get("site_id")
+ existing = next(
+ (
+ entry
+ for entry in upcoming
+ if entry.get("item_id") == item_id and (site_id is None or entry.get("site_id") == site_id)
+ ),
+ None,
+ )
if existing is None:
upcoming.append(item)
written.append(item)
@@ -156,6 +166,58 @@ def upsert_schedule_items(items: list[dict[str, Any]], root: Path | None = None)
return written
+def reconcile_schedule_from_queue(root: Path | None = None, schedule: dict[str, Any] | None = None) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+ """Restore scheduled work from per-site queue files.
+
+ Queue files are the durable per-site task records. `schedule.json` is an
+ index used by the runner, so a missing schedule entry should not make a
+ pending queue item disappear from backend processing.
+ """
+ root = bootstrap_workspace(root)
+ if schedule is None:
+ schedule = load_json(root / "schedule.json", {"schema_version": "1", "upcoming": []})
+ upcoming = schedule.setdefault("upcoming", [])
+ restored: list[dict[str, Any]] = []
+
+ existing_by_key = {
+ (entry.get("site_id"), entry.get("item_id")): entry
+ for entry in upcoming
+ if entry.get("site_id") and entry.get("item_id")
+ }
+
+ sites_dir = root / "sites"
+ if not sites_dir.exists():
+ return schedule, restored
+
+ for queue_path in sites_dir.glob("*/queue/*.json"):
+ try:
+ item = load_json(queue_path, {})
+ except json.JSONDecodeError:
+ continue
+ if not isinstance(item, dict):
+ continue
+ if not item.get("due_at"):
+ continue
+ status = item.get("status", "pending")
+ if status in TERMINAL_QUEUE_STATUSES:
+ continue
+
+ site_id = item.get("site_id") or queue_path.parents[1].name
+ item_id = item.get("item_id") or queue_path.stem
+ item["site_id"] = site_id
+ item["item_id"] = item_id
+ key = (site_id, item_id)
+ existing = existing_by_key.get(key)
+ if existing is None:
+ upcoming.append(dict(item))
+ existing_by_key[key] = upcoming[-1]
+ restored.append(upcoming[-1])
+ elif existing.get("status") not in TERMINAL_QUEUE_STATUSES:
+ existing.update(item)
+
+ return schedule, restored
+
+
def upsert_portfolio_site(
site_id: str,
*,
diff --git a/openclaw/tests/test_scheduler.py b/openclaw/tests/test_scheduler.py
new file mode 100644
index 00000000..f343cff1
--- /dev/null
+++ b/openclaw/tests/test_scheduler.py
@@ -0,0 +1,166 @@
+import contextlib
+import io
+import json
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+BIN_DIR = Path(__file__).resolve().parents[1] / "bin"
+if str(BIN_DIR) not in sys.path:
+ sys.path.insert(0, str(BIN_DIR))
+
+import run_scheduler
+from runtime import bootstrap_workspace, load_json, reconcile_schedule_from_queue, save_json, upsert_schedule_items
+
+
+class OpenClawSchedulerTest(unittest.TestCase):
+ def setUp(self) -> None:
+ self.tempdir = tempfile.TemporaryDirectory()
+ self.root = Path(self.tempdir.name)
+ self.previous_home = os.environ.get("TOPRANK_OPENCLAW_HOME")
+ os.environ["TOPRANK_OPENCLAW_HOME"] = str(self.root)
+ bootstrap_workspace(self.root)
+
+ def tearDown(self) -> None:
+ if self.previous_home is None:
+ os.environ.pop("TOPRANK_OPENCLAW_HOME", None)
+ else:
+ os.environ["TOPRANK_OPENCLAW_HOME"] = self.previous_home
+ self.tempdir.cleanup()
+
+ def write_queue_item(self, site_id: str, item: dict) -> Path:
+ queue_path = self.root / "sites" / site_id / "queue" / f"{item['item_id']}.json"
+ save_json(queue_path, {"site_id": site_id, **item})
+ return queue_path
+
+ def run_scheduler(self, *args: str) -> dict:
+ output = io.StringIO()
+ with contextlib.redirect_stdout(output):
+ code = run_scheduler.main(["run_scheduler.py", *args])
+ self.assertEqual(code, 0)
+ return json.loads(output.getvalue())
+
+ def test_upsert_schedule_items_uses_site_and_item_id(self) -> None:
+ upsert_schedule_items(
+ [
+ {"site_id": "a.com", "item_id": "same", "type": "feedback_check", "status": "pending", "due_at": "2026-01-01T00:00:00Z"},
+ {"site_id": "b.com", "item_id": "same", "type": "feedback_check", "status": "pending", "due_at": "2026-01-01T00:00:00Z"},
+ ],
+ self.root,
+ )
+
+ schedule = load_json(self.root / "schedule.json", {})
+ self.assertEqual(len(schedule["upcoming"]), 2)
+ self.assertEqual({item["site_id"] for item in schedule["upcoming"]}, {"a.com", "b.com"})
+
+ def test_reconcile_restores_pending_queue_item_missing_from_schedule(self) -> None:
+ self.write_queue_item(
+ "example.com",
+ {
+ "item_id": "orphaned_followup",
+ "type": "feedback_check",
+ "status": "pending",
+ "due_at": "2026-01-01T00:00:00Z",
+ },
+ )
+
+ schedule, restored = reconcile_schedule_from_queue(self.root)
+
+ self.assertEqual(len(restored), 1)
+ self.assertEqual(schedule["upcoming"][0]["item_id"], "orphaned_followup")
+
+ def test_scheduler_accepts_naive_due_timestamps(self) -> None:
+ self.write_queue_item(
+ "example.com",
+ {
+ "item_id": "naive_due_followup",
+ "type": "feedback_check",
+ "status": "pending",
+ "due_at": "2026-01-01T00:00:00",
+ "primary_metric": "organic_clicks_7d",
+ "baseline_metrics": {"organic_clicks_7d": 10.0},
+ },
+ )
+
+ result = self.run_scheduler("--dry-run", "--as-of", "2026-01-02T00:00:00Z")
+
+ self.assertEqual(result["manual_attention"][0]["item_id"], "naive_due_followup")
+
+ def test_inconclusive_feedback_is_not_marked_done(self) -> None:
+ queue_path = self.write_queue_item(
+ "example.com",
+ {
+ "item_id": "missing_metrics_followup",
+ "type": "feedback_check",
+ "status": "pending",
+ "due_at": "2026-01-01T00:00:00Z",
+ "primary_metric": "organic_clicks_7d",
+ "baseline_metrics": {"organic_clicks_7d": 10.0},
+ },
+ )
+
+ result = self.run_scheduler("--as-of", "2026-01-02T00:00:00Z")
+ queue_item = load_json(queue_path, {})
+ schedule = load_json(self.root / "schedule.json", {})
+
+ self.assertEqual(result["processed"], [])
+ self.assertEqual(result["manual_attention"][0]["item_id"], "missing_metrics_followup")
+ self.assertIn("missing metric data", result["manual_attention"][0]["reason"])
+ self.assertEqual(queue_item["status"], "ready_for_attention")
+ self.assertEqual(schedule["upcoming"][0]["status"], "ready_for_attention")
+ self.assertFalse((self.root / "sites" / "example.com" / "feedback" / "missing_metrics_followup.json").exists())
+
+ def test_dry_run_reports_inconclusive_feedback_as_manual_attention(self) -> None:
+ queue_path = self.write_queue_item(
+ "example.com",
+ {
+ "item_id": "dry_run_missing_metrics",
+ "type": "feedback_check",
+ "status": "pending",
+ "due_at": "2026-01-01T00:00:00Z",
+ "primary_metric": "organic_clicks_7d",
+ "baseline_metrics": {"organic_clicks_7d": 10.0},
+ },
+ )
+
+ result = self.run_scheduler("--dry-run", "--as-of", "2026-01-02T00:00:00Z")
+ queue_item = load_json(queue_path, {})
+
+ self.assertEqual(result["processed"], [])
+ self.assertEqual(result["manual_attention"][0]["item_id"], "dry_run_missing_metrics")
+ self.assertTrue(result["manual_attention"][0]["dry_run"])
+ self.assertEqual(queue_item["status"], "pending")
+
+ def test_feedback_with_observed_metrics_is_processed(self) -> None:
+ queue_path = self.write_queue_item(
+ "example.com",
+ {
+ "item_id": "scored_followup",
+ "type": "feedback_check",
+ "status": "pending",
+ "due_at": "2026-01-01T00:00:00Z",
+ "notes": "Re-check organic clicks.",
+ "action_type": "page_improvement",
+ "primary_metric": "organic_clicks_7d",
+ "primary_direction": "higher_better",
+ "success_threshold_pct": 0.1,
+ "baseline_metrics": {"organic_clicks_7d": 10.0},
+ "observed_metrics": {"organic_clicks_7d": 12.0},
+ "guardrail_metrics": [],
+ },
+ )
+
+ result = self.run_scheduler("--as-of", "2026-01-02T00:00:00Z")
+ queue_item = load_json(queue_path, {})
+ feedback = load_json(self.root / "sites" / "example.com" / "feedback" / "scored_followup.json", {})
+
+ self.assertEqual(result["manual_attention"], [])
+ self.assertEqual(result["processed"][0]["outcome"], "win")
+ self.assertEqual(queue_item["status"], "done")
+ self.assertEqual(feedback["status"], "win")
+
+
+if __name__ == "__main__":
+ unittest.main()