AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(blocks): add Hacker News search, stories, item and user blocks (#15191)
Nicholas Tindle0b8bf5bbca04
diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/__init__.py b/autogpt_platform/backend/backend/blocks/hacker_news/__init__.pynew file mode 100644index 0000000000..e69de29bb2diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/_api.py b/autogpt_platform/backend/backend/blocks/hacker_news/_api.pynew file mode 100644index 0000000000..5544f443fa--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/_api.py@@ -0,0 +1,171 @@+"""Requests to the two public Hacker News APIs the blocks use.++HN Search (hn.algolia.com, run by Algolia) answers searches, and returns an+item with its whole comment thread in one call. HN's official API on Firebase+(hacker-news.firebaseio.com) has the live story lists, single items and user+profiles. Neither needs an account or a key.+"""++import asyncio+import re+from typing import Any+from urllib.parse import parse_qs, urlparse++from backend.util.request import Requests, Response++ALGOLIA_URL = "https://hn.algolia.com/api/v1"+FIREBASE_URL = "https://hacker-news.firebaseio.com/v0"+HN_URL = "https://news.ycombinator.com"++# The official API has one URL per item, so a list of stories takes one request+# per story. This many run at once.+MAX_CONCURRENT_REQUESTS = 10++_TRUSTED_ORIGINS = ["https://hn.algolia.com", "https://hacker-news.firebaseio.com"]+_DIGITS = re.compile(r"[0-9]+")+# HN usernames are letters, digits, - and _. New accounts get 2 to 15 of them,+# but older ones predate that rule, so the length check is deliberately loose:+# a name HN doesn't know fails as "not found" instead.+_USERNAME = re.compile(r"[A-Za-z0-9_-]{1,32}")+++class HackerNewsError(Exception):+ """A Hacker News API failure, with a message the user can act on."""+++async def search(endpoint: str, params: dict[str, str]) -> dict[str, Any]:+ """Run an HN Search query; `endpoint` is "search" or "search_by_date"."""+ return await get_json(f"{ALGOLIA_URL}/{endpoint}", params) or {}+++async def get_thread(item_id: int) -> dict[str, Any] | None:+ """An item and its whole comment tree from HN Search.++ None when HN Search doesn't have the item: there is no such item, it is+ dead, or it was posted in the last few minutes and isn't indexed yet.+ """+ return await get_json(f"{ALGOLIA_URL}/items/{item_id}")+++async def get_item(item_id: int) -> dict[str, Any] | None:+ """An item from HN's official API, or None when there is no such item."""+ return await get_json(f"{FIREBASE_URL}/item/{item_id}.json")+++async def get_items(item_ids: list[int]) -> list[dict[str, Any] | None]:+ """Items from HN's official API, in the order of `item_ids`."""+ slots = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)++ async def fetch(item_id: int) -> dict[str, Any] | None:+ async with slots:+ return await get_item(item_id)++ return list(await asyncio.gather(*(fetch(item_id) for item_id in item_ids)))+++async def get_replies(kids: list[int], limit: int) -> dict[int, dict[str, Any]]:+ """Up to `limit` comments below an item from HN's official API, by id.++ Fetched breadth first, so every comment's parent is fetched before it.+ Dead and deleted comments are fetched too, and count towards the limit.+ Each id is fetched once, even if the API lists it twice.+ """+ found: dict[int, dict[str, Any]] = {}+ queue = list(dict.fromkeys(kids))+ queued = set(queue)+ while queue and len(found) < limit:+ room = limit - len(found)+ batch, queue = queue[:room], queue[room:]+ for item in await get_items(batch):+ if item:+ found[item["id"]] = item+ new = [kid for kid in item.get("kids") or [] if kid not in queued]+ queued.update(new)+ queue.extend(new)+ return found+++async def get_story_ids(list_name: str) -> list[int]:+ """The ids on one of HN's story lists, e.g. "topstories", in HN's order."""+ return await get_json(f"{FIREBASE_URL}/{list_name}.json") or []+++async def get_user(username: str) -> dict[str, Any] | None:+ """A user's profile from HN's official API, or None when there is no such user."""+ return await get_json(f"{FIREBASE_URL}/user/{username}.json")+++async def get_json(url: str, params: dict[str, str] | None = None) -> Any:+ """GET a Hacker News API URL and return its JSON, or None for a 404.++ The official API answers `null` rather than 404 for a missing item or+ user, which also comes back as None. 429 and 5xx responses are retried+ twice before giving up.+ """+ response = await Requests(+ trusted_origins=_TRUSTED_ORIGINS, raise_for_status=False, retry_max_attempts=3+ ).get(url, params=params)+ if response.status == 404:+ return None+ if not response.ok:+ raise api_error(url, response)+ return response.json()+++def api_error(url: str, response: Response) -> HackerNewsError:+ """Turn a failed Hacker News API response into a message the user can act on."""+ if urlparse(url).hostname == "hn.algolia.com":+ api = "Hacker News search (hn.algolia.com)"+ else:+ api = "The Hacker News API"+ status = response.status+ if status == 429:+ return HackerNewsError(+ f"{api} is limiting how many requests it takes from this server "+ "(HTTP 429). Wait a few minutes, then try again."+ )+ if status >= 500:+ return HackerNewsError(+ f"{api} had a temporary problem (HTTP {status}). Try again in a minute."+ )+ body = response.json(fallback={}) if response.content else {}+ message = body.get("message") if isinstance(body, dict) else None+ detail = message or response.reason or "no details given"+ return HackerNewsError(f"{api} rejected the request (HTTP {status}): {detail}")+++def parse_item_id(text: str) -> int | None:+ """The id in '8863' or in a link such as news.ycombinator.com/item?id=8863."""+ text = text.strip()+ if _DIGITS.fullmatch(text):+ return int(text)+ value = _link_id(text, "/item")+ return int(value) if value and _DIGITS.fullmatch(value) else None+++def parse_username(text: str) -> str | None:+ """The username in 'pg' or in a link such as news.ycombinator.com/user?id=pg."""+ text = text.strip()+ if not _USERNAME.fullmatch(text):+ text = _link_id(text, "/user") or ""+ return text if _USERNAME.fullmatch(text) else None+++def item_url(item_id: int) -> str:+ return f"{HN_URL}/item?id={item_id}"+++def user_url(username: str) -> str:+ return f"{HN_URL}/user?id={username}"+++def _link_id(text: str, path: str) -> str | None:+ """The id in a Hacker News link to `path`, written with or without https://."""+ try:+ url = urlparse(text if "://" in text else f"https://{text}")+ host = url.hostname+ except ValueError: # e.g. an unclosed [ in the host+ return None+ if host != "news.ycombinator.com" or url.path != path:+ return None+ return next(iter(parse_qs(url.query).get("id", [])), None)diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/_api_test.py b/autogpt_platform/backend/backend/blocks/hacker_news/_api_test.pynew file mode 100644index 0000000000..5eb2fb197b--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/_api_test.py@@ -0,0 +1,268 @@+"""Unit tests for the Hacker News API helpers: requests, error messages,+concurrent item fetches, and reading ids and usernames from text."""++import asyncio+import json+from typing import Any+from unittest.mock import MagicMock++import pytest++from backend.blocks.hacker_news import _api+from backend.blocks.hacker_news._api import (+ ALGOLIA_URL,+ FIREBASE_URL,+ MAX_CONCURRENT_REQUESTS,+ HackerNewsError,+ parse_item_id,+ parse_username,+)+from backend.util.request import Response+++def _response(status: int, body: Any = None, reason: str = "Reason") -> Response:+ content = body if isinstance(body, bytes) else json.dumps(body).encode()+ raw = MagicMock(status=status, headers={}, reason=reason)+ return Response(response=raw, url="https://example.invalid", body=content)+++class _FakeRequests:+ """Stands in for the Requests class: records its options and each GET."""++ def __init__(self, *responses: Response):+ self.responses = list(responses)+ self.options: list[dict[str, Any]] = []+ self.calls: list[tuple[str, dict[str, Any]]] = []++ def __call__(self, **kwargs: Any) -> "_FakeRequests":+ self.options.append(kwargs)+ return self++ async def get(self, url: str, **kwargs: Any) -> Response:+ self.calls.append((url, kwargs))+ return self.responses.pop(0)+++def fake_requests(monkeypatch: pytest.MonkeyPatch):+ def install(*responses: Response) -> _FakeRequests:+ fake = _FakeRequests(*responses)+ monkeypatch.setattr(_api, "Requests", fake)+ return fake++ return install+++async def test_search_calls_hn_search_with_the_params(fake_requests):+ fake = fake_requests(_response(200, {"hits": [], "nbHits": 0}))+ params = {"query": "agpt.co", "tags": "story"}+ assert await _api.search("search_by_date", params) == {"hits": [], "nbHits": 0}+ assert fake.calls == [(f"{ALGOLIA_URL}/search_by_date", {"params": params})]+ # Only the two Hacker News hosts skip the SSRF IP checks, and 429/5xx+ # responses are retried twice, then reported rather than raised as-is.+ assert fake.options == [+ {+ "trusted_origins": [+ "https://hn.algolia.com",+ "https://hacker-news.firebaseio.com",+ ],+ "raise_for_status": False,+ "retry_max_attempts": 3,+ }+ ]+++async def test_item_user_and_list_urls(fake_requests):+ fake = fake_requests(+ _response(200, {"id": 8863}),+ _response(200, {"id": 8863, "children": []}),+ _response(200, {"id": "pg"}),+ _response(200, [3, 2, 1]),+ )+ assert await _api.get_item(8863) == {"id": 8863}+ assert await _api.get_thread(8863) == {"id": 8863, "children": []}+ assert await _api.get_user("pg") == {"id": "pg"}+ assert await _api.get_story_ids("jobstories") == [3, 2, 1]+ assert [url for url, _ in fake.calls] == [+ f"{FIREBASE_URL}/item/8863.json",+ f"{ALGOLIA_URL}/items/8863",+ f"{FIREBASE_URL}/user/pg.json",+ f"{FIREBASE_URL}/jobstories.json",+ ]+++async def test_missing_items_and_users_come_back_as_none(fake_requests):+ # HN Search answers 404; the official API answers 200 with `null`.+ fake_requests(+ _response(404, {"error": "Not Found", "status": 404}),+ _response(200, b"null"),+ _response(200, b"null"),+ _response(200, b"null"),+ )+ assert await _api.get_thread(1) is None+ assert await _api.get_item(1) is None+ assert await _api.get_user("nobody") is None+ assert await _api.get_story_ids("topstories") == []+++ "url, status, body, expected",+ [+ (+ f"{ALGOLIA_URL}/search",+ 429,+ None,+ "Hacker News search (hn.algolia.com) is limiting how many requests",+ ),+ (+ f"{FIREBASE_URL}/item/1.json",+ 503,+ b"<html>Unavailable</html>",+ "The Hacker News API had a temporary problem (HTTP 503)",+ ),+ (+ f"{ALGOLIA_URL}/search",+ 400,+ {"message": "invalid setting for restrictSearchableAttributes"},+ "rejected the request (HTTP 400): invalid setting for "+ "restrictSearchableAttributes",+ ),+ (f"{FIREBASE_URL}/item/1.json", 401, b"", "(HTTP 401): Permission denied"),+ ],+)+async def test_errors_become_messages_a_user_can_act_on(+ fake_requests, url: str, status: int, body: Any, expected: str+):+ fake_requests(_response(status, body, reason="Permission denied"))+ with pytest.raises(HackerNewsError) as error:+ await _api.get_json(url)+ assert expected in str(error.value)+++async def test_get_items_keeps_order_and_limits_concurrency(+ monkeypatch: pytest.MonkeyPatch,+):+ running = peak = 0++ async def fake_get_item(item_id: int) -> dict[str, Any] | None:+ nonlocal running, peak+ running += 1+ peak = max(peak, running)+ await asyncio.sleep(0.001 * (item_id % 4)) # finish out of order+ running -= 1+ return None if item_id == 7 else {"id": item_id}++ monkeypatch.setattr(_api, "get_item", fake_get_item)+ items = await _api.get_items(list(range(30)))+ assert [item["id"] if item else None for item in items] == [+ None if item_id == 7 else item_id for item_id in range(30)+ ]+ assert peak == MAX_CONCURRENT_REQUESTS+++ "limit, batches, found",+ [+ (10, [[1, 2], [11, 12, 21], [111]], {1, 2, 11, 21, 111}),+ # 12 answers null, so the last slot goes to 21, still a level up from 111.+ (4, [[1, 2], [11, 12], [21]], {1, 2, 11, 21}),+ (3, [[1, 2], [11]], {1, 2, 11}),+ ],+)+async def test_get_replies_fetches_breadth_first_up_to_the_limit(+ monkeypatch: pytest.MonkeyPatch,+ limit: int,+ batches: list[list[int]],+ found: set[int],+):+ kids = {1: [11, 12], 2: [21], 11: [111], 21: [], 111: []}+ requested: list[list[int]] = []++ async def fake_get_items(item_ids: list[int]) -> list[dict[str, Any] | None]:+ requested.append(item_ids)+ return [+ {"id": item_id, "kids": kids[item_id]} if item_id in kids else None+ for item_id in item_ids+ ]++ monkeypatch.setattr(_api, "get_items", fake_get_items)+ replies = await _api.get_replies([1, 2], limit=limit)+ assert requested == batches+ assert set(replies) == found+++async def test_get_replies_fetches_each_id_once_even_in_a_loop(+ monkeypatch: pytest.MonkeyPatch,+):+ # Malformed data: 1 is listed twice, and 2 and 3 list each other.+ kids = {1: [2], 2: [3], 3: [2, 1]}+ requested: list[int] = []++ async def fake_get_items(item_ids: list[int]) -> list[dict[str, Any] | None]:+ requested.extend(item_ids)+ return [{"id": item_id, "kids": kids[item_id]} for item_id in item_ids]++ monkeypatch.setattr(_api, "get_items", fake_get_items)+ replies = await _api.get_replies([1, 1], limit=100)+ assert sorted(requested) == [1, 2, 3]+ assert set(replies) == {1, 2, 3}+++async def test_get_replies_with_no_comments_makes_no_requests(+ monkeypatch: pytest.MonkeyPatch,+):+ fake = MagicMock()+ monkeypatch.setattr(_api, "get_items", fake)+ assert await _api.get_replies([], limit=200) == {}+ fake.assert_not_called()+++ "text, expected",+ [+ ("8863", 8863),+ (" 008863 ", 8863),+ ("https://news.ycombinator.com/item?id=8863", 8863),+ ("http://news.ycombinator.com/item?id=8863&p=2", 8863),+ ("news.ycombinator.com/item?id=8863", 8863),+ ("https://news.ycombinator.com/item?id=8863#8865", 8863),+ ("https://news.ycombinator.com/user?id=8863", None),+ ("https://example.com/item?id=8863", None),+ ("https://news.ycombinator.com/item?id=abc", None),+ ("https://news.ycombinator.com/item", None),+ ("https://[news.ycombinator.com/item?id=1", None),+ ("-5", None),+ ("8863", None), # full-width digits+ ("", None),+ ],+)+def test_parse_item_id(text: str, expected: int | None):+ assert parse_item_id(text) == expected+++ "text, expected",+ [+ ("pg", "pg"),+ (" dang ", "dang"),+ ("some_user-1", "some_user-1"),+ ("https://news.ycombinator.com/user?id=pg", "pg"),+ ("news.ycombinator.com/user?id=tptacek", "tptacek"),+ ("https://news.ycombinator.com/item?id=pg", None),+ ("https://example.com/user?id=pg", None),+ ("pg,author_dang", None),+ ("../item/1", None),+ ("a" * 33, None),+ ("", None),+ ],+)+def test_parse_username(text: str, expected: str | None):+ assert parse_username(text) == expecteddiff --git a/autogpt_platform/backend/backend/blocks/hacker_news/_html.py b/autogpt_platform/backend/backend/blocks/hacker_news/_html.pynew file mode 100644index 0000000000..048487d888--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/_html.py@@ -0,0 +1,109 @@+"""Plain text from the HTML Hacker News stores for comments, posts and profiles.++HN keeps what people type as a little HTML: <p> between paragraphs, <i> for+text typed between asterisks, <pre><code> for indented code, and an <a> for+every URL, whose visible text HN cuts short ("https://example.com/a/lo...")+while href keeps the whole URL. This turns it back into the text the person+typed: blank lines between paragraphs, *asterisks* for italics, code kept+exactly as written, and full URLs.+"""++import re+from html.parser import HTMLParser++_SPACES = re.compile(r"\s+")+_SCHEME = re.compile(r"^[a-z][a-z0-9+.-]*:(//)?", re.IGNORECASE)+++def html_to_text(html: str | None) -> str:+ """The plain text of HN's HTML, or an empty string when there is none."""+ if not html:+ return ""+ parser = _HackerNewsHTML()+ parser.feed(html)+ parser.close()+ return parser.text()+++class _HackerNewsHTML(HTMLParser):+ def __init__(self) -> None:+ # Entities such as ' and > arrive in handle_data decoded.+ super().__init__(convert_charrefs=True)+ self._blocks: list[str] = []+ self._paragraph: list[str] = []+ self._code: list[str] | None = None # set while inside <pre>+ self._link: list[str] | None = None # set while inside <a>+ self._href = ""++ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:+ if tag == "p":+ self._end_paragraph()+ elif tag == "pre":+ self._end_paragraph()+ self._code = []+ elif tag == "br":+ self._write("\n")+ elif tag in ("i", "em"):+ self._write("*")+ elif tag == "a":+ self._link = []+ self._href = dict(attrs).get("href") or ""++ def handle_endtag(self, tag: str) -> None:+ if tag == "p":+ self._end_paragraph()+ elif tag == "pre":+ self._end_code()+ elif tag in ("i", "em"):+ self._write("*")+ elif tag == "a" and self._link is not None:+ visible = "".join(self._link)+ self._link = None+ self._write(_link_text(visible, self._href))++ def handle_data(self, data: str) -> None:+ if self._link is not None:+ self._link.append(data)+ elif self._code is not None:+ self._code.append(data)+ else:+ # Line breaks in HN's HTML are only spaces; <br> adds a real one.+ self._paragraph.append(_SPACES.sub(" ", data))++ def text(self) -> str:+ self._end_code()+ self._end_paragraph()+ return "\n\n".join(self._blocks)++ def _write(self, text: str) -> None:+ if self._code is not None:+ self._code.append(text)+ else:+ self._paragraph.append(text)++ def _end_paragraph(self) -> None:+ lines = "".join(self._paragraph).split("\n")+ self._paragraph = []+ paragraph = "\n".join(" ".join(line.split()) for line in lines).strip()+ if paragraph:+ self._blocks.append(paragraph)++ def _end_code(self) -> None:+ if self._code is None:+ return+ # Keep the indentation: HN only shows text indented by 2+ spaces as code.+ code = "".join(self._code).strip("\n").rstrip()+ self._code = None+ if code:+ self._blocks.append(code)+++def _link_text(visible: str, href: str) -> str:+ """The full URL for one of HN's links, which show it shortened with '...'."""+ visible = visible.strip()+ shown = visible.removesuffix("...")+ if not href:+ return visible+ if not shown or _SCHEME.sub("", href).startswith(_SCHEME.sub("", shown)):+ return href+ return f"{visible} ({href})"diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/_html_test.py b/autogpt_platform/backend/backend/blocks/hacker_news/_html_test.pynew file mode 100644index 0000000000..b2fba2ee92--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/_html_test.py@@ -0,0 +1,85 @@+"""Unit tests for turning Hacker News's HTML into plain text."""++import pytest++from backend.blocks.hacker_news._html import html_to_text++# Shaped like comments from both HN APIs: entities, <p> between paragraphs, a+# line break that is only a space, indented code, and a shortened link.+COMMENT = (+ "I'd try this:<p><pre><code> def main():\n"+ " print("Hello, world!")\n</code></pre>\n"+ "It's what the docs[1] suggest, more or less, if you read\n"+ "them closely.<p>[1] "+ '<a href="https://docs.example.com/guides/getting-started'+ '/first-steps.html#hello" rel="nofollow">https://'+ "docs.example.com/guides/getting-started/f...</a>"+)+++def test_a_comment_reads_as_typed():+ assert html_to_text(COMMENT) == (+ "I'd try this:\n"+ "\n"+ " def main():\n"+ ' print("Hello, world!")\n'+ "\n"+ "It's what the docs[1] suggest, more or less, if you read them closely.\n"+ "\n"+ "[1] https://docs.example.com/guides/getting-started/first-steps.html#hello"+ )+++ "html, expected",+ [+ ("First.<p>Second.<p>Third.", "First.\n\nSecond.\n\nThird."),+ ("<p>Closed</p><p></p><p>paragraphs</p>", "Closed\n\nparagraphs"),+ (+ "It's 2 > 1 & "fine" /s",+ 'It\'s 2 > 1 & "fine" /s',+ ),+ ("Not <i>that</i> one", "Not *that* one"),+ ("one line\nin the HTML", "one line in the HTML"),+ ("line<br>break", "line\nbreak"),+ ("<i>typed tags</i> stay text", "<i>typed tags</i> stay text"),+ ("Plain text with no HTML", "Plain text with no HTML"),+ ],+)+def test_paragraphs_entities_and_italics(html: str, expected: str):+ assert html_to_text(html) == expected+++ "html, expected",+ [+ # HN shortens what it shows; the whole URL is in href.+ (+ '<a href="https://example.com/a/very/long/path?x=1">https://example.com/a/very/lo...</a>',+ "https://example.com/a/very/long/path?x=1",+ ),+ (+ '<a href="https://news.ycombinator.com/item?id=1">news.ycombinator.com/item?id=1</a>',+ "https://news.ycombinator.com/item?id=1",+ ),+ ('see <a href="https://x.co">our site</a>.', "see our site (https://x.co)."),+ ("<a>no href</a>", "no href"),+ ('<a href="https://x.co"></a>', "https://x.co"),+ ],+)+def test_links_keep_the_full_url(html: str, expected: str):+ assert html_to_text(html) == expected+++def test_code_keeps_its_indentation_and_blank_lines():+ html = "Try:<p><pre><code> def f():\n\n return 1\n</code></pre>It works."+ assert html_to_text(html) == "Try:\n\n def f():\n\n return 1\n\nIt works."+++def test_an_unclosed_code_block_is_kept():+ assert html_to_text("<pre><code> x = 1\n y = 2") == " x = 1\n y = 2"++[email protected]("html", [None, "", "<p>", "<p> </p>"])+def test_nothing_to_read_gives_an_empty_string(html: str | None):+ assert html_to_text(html) == ""diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/_models.py b/autogpt_platform/backend/backend/blocks/hacker_news/_models.pynew file mode 100644index 0000000000..0bbafc5e95--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/_models.py@@ -0,0 +1,211 @@+"""Output models for the Hacker News blocks, and how each API's JSON maps to them.++HN Search search hits, HN Search items (/items/<id>) and items from HN's+official API name the same things differently (`author` or `by`, `points` or+`score`, `created_at_i` or `time`), so each has its own mapping.+"""++from datetime import datetime, timezone+from typing import Any++from pydantic import BaseModel, Field++from ._api import item_url+from ._html import html_to_text++# HN Search lists the item type among a hit's _tags, next to tags such as+# "author_pg", "story_8863" and "show_hn".+_ITEM_TYPES = ("story", "comment", "poll", "pollopt", "job")+++class HackerNewsItem(BaseModel):+ """A Hacker News story, comment, poll or job."""++ id: int = Field(description="Hacker News item id")+ type: str = Field(description="story, comment, poll, pollopt or job")+ title: str | None = Field(+ default=None,+ description="Title. For a comment found by search, the title of its story.",+ )+ url: str | None = Field(+ default=None,+ description=(+ "The link a story points to. Empty for text posts such as Ask HN, "+ "and for comments."+ ),+ )+ text: str | None = Field(+ default=None,+ description="Plain text of a comment, or of a text post such as Ask HN",+ )+ author: str | None = Field(default=None, description="Username of the poster")+ points: int | None = Field(+ default=None, description="Points (upvotes). Only stories and polls have them."+ )+ num_comments: int | None = Field(+ default=None, description="Number of comments on a story or poll"+ )+ created_at: str | None = Field(+ default=None, description="When it was posted, in ISO 8601 (UTC)"+ )+ hn_url: str = Field(description="Link to the item on Hacker News")+ story_id: int | None = Field(+ default=None, description="Id of the story it belongs to, when known"+ )+ parent_id: int | None = Field(+ default=None, description="For a comment, the id of the item it replies to"+ )+++class HackerNewsStory(BaseModel):+ """A story on one of Hacker News's lists, with its place on the list."""++ id: int = Field(description="Hacker News item id")+ rank: int = Field(description="Position on the list, starting at 1")+ type: str = Field(description="story, poll or job")+ title: str | None = Field(default=None, description="Title")+ url: str | None = Field(+ default=None,+ description="The link the story points to. Empty for text posts such as Ask HN.",+ )+ text: str | None = Field(+ default=None, description="Plain text of a text post such as Ask HN"+ )+ author: str | None = Field(default=None, description="Username of the poster")+ points: int | None = Field(+ default=None, description="Points (upvotes). Jobs have none."+ )+ num_comments: int | None = Field(+ default=None, description="Number of comments. Jobs have none."+ )+ created_at: str | None = Field(+ default=None, description="When it was posted, in ISO 8601 (UTC)"+ )+ hn_url: str = Field(description="Link to the story on Hacker News")+++class HackerNewsComment(BaseModel):+ """A comment in a thread, with how deep it is nested."""++ id: int = Field(description="Hacker News item id of the comment")+ author: str = Field(description="Username of the commenter")+ text: str = Field(description="Plain text of the comment")+ created_at: str | None = Field(+ default=None, description="When it was posted, in ISO 8601 (UTC)"+ )+ parent_id: int | None = Field(+ default=None, description="Id of the story or comment it replies to"+ )+ depth: int = Field(description="Nesting level: 1 is a direct reply to the item")+ hn_url: str = Field(description="Link to the comment on Hacker News")+++def item_from_hit(hit: dict[str, Any]) -> HackerNewsItem:+ """An HN Search hit: a story, comment, poll or job."""+ tags = hit.get("_tags") or []+ item_type = next((tag for tag in tags if tag in _ITEM_TYPES), "story")+ is_comment = item_type == "comment"+ item_id = int(hit["objectID"])+ return HackerNewsItem(+ id=item_id,+ type=item_type,+ title=(hit.get("story_title") if is_comment else hit.get("title")) or None,+ url=None if is_comment else hit.get("url") or None,+ text=html_to_text(hit.get("comment_text") or hit.get("story_text")) or None,+ author=hit.get("author") or None,+ points=hit.get("points"),+ num_comments=hit.get("num_comments"),+ created_at=iso_time(hit.get("created_at_i")),+ hn_url=item_url(item_id),+ story_id=hit.get("story_id"),+ parent_id=hit.get("parent_id"),+ )+++def item_from_thread(+ node: dict[str, Any], comment_count: int, points: int | None+) -> HackerNewsItem:+ """The item at the top of an HN Search thread (the /items/<id> endpoint).++ That endpoint has no comment count, and its points trail the live site,+ so the caller supplies both.+ """+ item_id = node["id"]+ item_type = node.get("type") or "story"+ return HackerNewsItem(+ id=item_id,+ type=item_type,+ title=node.get("title") or None,+ url=node.get("url") or None,+ text=html_to_text(node.get("text")) or None,+ author=node.get("author") or None,+ points=points,+ num_comments=comment_count if item_type in ("story", "poll") else None,+ created_at=iso_time(node.get("created_at_i")),+ hn_url=item_url(item_id),+ story_id=node.get("story_id"),+ parent_id=node.get("parent_id"),+ )+++def item_from_firebase(item: dict[str, Any]) -> HackerNewsItem:+ """An item from HN's official API."""+ item_id = item["id"]+ item_type = item.get("type") or "story"+ return HackerNewsItem(+ id=item_id,+ type=item_type,+ title=item.get("title") or None,+ url=item.get("url") or None,+ text=html_to_text(item.get("text")) or None,+ author=item.get("by") or None,+ # The API gives every job 1 point, which HN never shows.+ points=None if item_type == "job" else item.get("score"),+ num_comments=item.get("descendants"),+ created_at=iso_time(item.get("time")),+ hn_url=item_url(item_id),+ story_id=item_id if item_type in ("story", "poll") else None,+ parent_id=item.get("parent"),+ )+++def story_from_firebase(item: dict[str, Any], rank: int) -> HackerNewsStory:+ """An item from one of HN's story lists, at `rank` on the list."""+ fields = item_from_firebase(item).model_dump(exclude={"story_id", "parent_id"})+ return HackerNewsStory(rank=rank, **fields)+++def comment_from_node(node: dict[str, Any], depth: int) -> HackerNewsComment:+ """A comment from an HN Search thread, `depth` levels below the item."""+ return HackerNewsComment(+ id=node["id"],+ author=node["author"],+ text=html_to_text(node["text"]),+ created_at=iso_time(node.get("created_at_i")),+ parent_id=node.get("parent_id"),+ depth=depth,+ hn_url=item_url(node["id"]),+ )+++def thread_node_from_firebase(item: dict[str, Any]) -> dict[str, Any]:+ """A comment from HN's official API, shaped like a comment in an HN Search+ thread. A dead or deleted comment gets no author or text, as HN Search+ shows it."""+ hidden = item.get("deleted") or item.get("dead")+ return {+ "id": item["id"],+ "author": None if hidden else item.get("by"),+ "text": None if hidden else item.get("text"),+ "created_at_i": item.get("time"),+ "parent_id": item.get("parent"),+ "kids": item.get("kids") or [],+ }+++def iso_time(timestamp: int | None) -> str | None:+ """Unix seconds as ISO 8601 in UTC, e.g. 2026-10-05T14:44:04Z."""+ if timestamp is None:+ return None+ moment = datetime.fromtimestamp(timestamp, tz=timezone.utc)+ return moment.strftime("%Y-%m-%dT%H:%M:%SZ")diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/_models_test.py b/autogpt_platform/backend/backend/blocks/hacker_news/_models_test.pynew file mode 100644index 0000000000..97433cfcb9--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/_models_test.py@@ -0,0 +1,278 @@+"""Unit tests for mapping each Hacker News API's JSON to the blocks' models."""++import pytest++from backend.blocks.hacker_news._models import (+ HackerNewsItem,+ HackerNewsStory,+ iso_time,+ item_from_firebase,+ item_from_hit,+ item_from_thread,+ story_from_firebase,+ thread_node_from_firebase,+)+++def test_story_hit():+ item = item_from_hit(+ {+ "_tags": ["story", "author_maker_jane", "story_49965697", "show_hn"],+ "objectID": "49965697",+ "title": "Show HN: Pocketlog, a tiny log viewer",+ "url": "https://github.com/example/pocketlog",+ "author": "maker_jane",+ "points": 1,+ "num_comments": 0,+ "created_at": "2026-10-05T14:56:05Z",+ "created_at_i": 1791212165,+ "story_id": 49965697,+ }+ )+ assert item == HackerNewsItem(+ id=49965697,+ type="story",+ title="Show HN: Pocketlog, a tiny log viewer",+ url="https://github.com/example/pocketlog",+ author="maker_jane",+ points=1,+ num_comments=0,+ created_at="2026-10-05T14:56:05Z",+ hn_url="https://news.ycombinator.com/item?id=49965697",+ story_id=49965697,+ )+++def test_ask_hn_hit_has_text_and_no_link():+ item = item_from_hit(+ {+ "_tags": ["story", "author_someone", "story_1", "ask_hn"],+ "objectID": "1",+ "title": "Ask HN: Anyone else?",+ "story_text": "I wonder.<p>Do you?",+ "created_at_i": 1693926441,+ }+ )+ assert (item.url, item.text) == (None, "I wonder.\n\nDo you?")+++def test_comment_hit_takes_its_story_title_but_not_its_link():+ item = item_from_hit(+ {+ "_tags": ["comment", "author_fan_club", "story_49917536"],+ "objectID": "49965549",+ "author": "fan_club",+ "comment_text": "Ours came with <i>no switch</i>.<p>I don't know why.",+ "points": None,+ "story_id": 49917536,+ "story_title": "Why ceiling fans got so complicated",+ "story_url": "https://example.com/ceiling-fans",+ "parent_id": 49945418,+ "created_at_i": 1791211444,+ }+ )+ assert item == HackerNewsItem(+ id=49965549,+ type="comment",+ title="Why ceiling fans got so complicated",+ url=None,+ text="Ours came with *no switch*.\n\nI don't know why.",+ author="fan_club",+ created_at="2026-10-05T14:44:04Z",+ hn_url="https://news.ycombinator.com/item?id=49965549",+ story_id=49917536,+ parent_id=49945418,+ )+++ "tags, expected",+ [+ (["poll", "author_x", "story_5"], "poll"),+ (["pollopt", "author_x"], "pollopt"),+ (["job", "author_x"], "job"),+ (["author_x", "comment", "story_5"], "comment"),+ ([], "story"),+ ],+)+def test_hit_type_comes_from_its_tags(tags: list[str], expected: str):+ assert item_from_hit({"_tags": tags, "objectID": "5"}).type == expected+++def test_hit_with_missing_and_empty_fields():+ item = item_from_hit({"objectID": "7", "title": "", "url": "", "author": ""})+ assert item == HackerNewsItem(+ id=7, type="story", hn_url="https://news.ycombinator.com/item?id=7"+ )+++def test_thread_item_takes_the_count_and_points_it_is_given():+ item = item_from_thread(+ {+ "id": 8863,+ "type": "story",+ "author": "dhouston",+ "title": "My YC app: Dropbox - Throw away your USB drive",+ "url": "http://www.getdropbox.com/u/2/screencast.html",+ "text": None,+ "points": 104,+ "parent_id": None,+ "story_id": 8863,+ "created_at": "2007-04-04T19:16:40.000Z",+ "created_at_i": 1175714200,+ "options": [],+ "children": [],+ },+ comment_count=71,+ points=110,+ )+ assert item == HackerNewsItem(+ id=8863,+ type="story",+ title="My YC app: Dropbox - Throw away your USB drive",+ url="http://www.getdropbox.com/u/2/screencast.html",+ author="dhouston",+ points=110,+ num_comments=71,+ created_at="2007-04-04T19:16:40Z",+ hn_url="https://news.ycombinator.com/item?id=8863",+ story_id=8863,+ )+++def test_thread_comment_has_no_comment_count():+ item = item_from_thread(+ {+ "id": 8865,+ "type": "comment",+ "author": "dhouston",+ "text": "oh, and a mac port is coming :)",+ "points": None,+ "title": None,+ "url": None,+ "parent_id": 8863,+ "story_id": 8863,+ "created_at_i": 1175714575,+ },+ comment_count=3,+ points=None,+ )+ assert (item.num_comments, item.points, item.title) == (None, None, None)+ assert (item.story_id, item.parent_id) == (8863, 8863)+ assert item.text == "oh, and a mac port is coming :)"+++def test_firebase_story():+ item = item_from_firebase(+ {+ "by": "ops_person",+ "descendants": 20,+ "id": 49964045,+ "kids": [49965311, 49964775],+ "score": 39,+ "text": "Our IP is on a blocklist.<p>It's the whole ASN.",+ "time": 1791207720,+ "title": "Ask HN: Is your site blocked too?",+ "type": "story",+ }+ )+ assert item == HackerNewsItem(+ id=49964045,+ type="story",+ title="Ask HN: Is your site blocked too?",+ text="Our IP is on a blocklist.\n\nIt's the whole ASN.",+ author="ops_person",+ points=39,+ num_comments=20,+ created_at="2026-10-05T13:42:00Z",+ hn_url="https://news.ycombinator.com/item?id=49964045",+ story_id=49964045,+ )+++def test_firebase_job_shows_no_points():+ item = item_from_firebase(+ {+ "by": "acme_hiring",+ "id": 49945904,+ "score": 1,+ "time": 1791046812,+ "title": "Acme (YC W24) Is Hiring",+ "type": "job",+ "url": "https://www.ycombinator.com/companies/acme/jobs/x",+ }+ )+ assert (item.type, item.points, item.num_comments, item.story_id) == (+ "job",+ None,+ None,+ None,+ )+++def test_firebase_deleted_comment():+ item = item_from_firebase(+ {+ "deleted": True,+ "id": 11116487,+ "parent": 11116274,+ "time": 1455701075,+ "type": "comment",+ }+ )+ assert (item.author, item.text, item.parent_id, item.story_id) == (+ None,+ None,+ 11116274,+ None,+ )+++def test_story_from_a_list_keeps_its_rank():+ story = story_from_firebase(+ {+ "by": "dhouston",+ "descendants": 71,+ "id": 8863,+ "score": 104,+ "time": 1175714200,+ "title": "My YC app: Dropbox - Throw away your USB drive",+ "type": "story",+ "url": "http://www.getdropbox.com/u/2/screencast.html",+ },+ rank=4,+ )+ assert story == HackerNewsStory(+ id=8863,+ rank=4,+ type="story",+ title="My YC app: Dropbox - Throw away your USB drive",+ url="http://www.getdropbox.com/u/2/screencast.html",+ author="dhouston",+ points=104,+ num_comments=71,+ created_at="2007-04-04T19:16:40Z",+ hn_url="https://news.ycombinator.com/item?id=8863",+ )+++ "item, author, text",+ [+ ({"id": 1, "by": "a", "text": "hi", "time": 5, "parent": 9}, "a", "hi"),+ ({"id": 1, "by": "a", "text": "", "dead": True, "parent": 9}, None, None),+ ({"id": 1, "deleted": True, "parent": 9, "kids": [2]}, None, None),+ ],+)+def test_firebase_comment_as_a_thread_node(item: dict, author: str, text: str):+ node = thread_node_from_firebase(item)+ assert (node["author"], node["text"], node["parent_id"]) == (author, text, 9)+ assert node["kids"] == item.get("kids", [])+++ "timestamp, expected",+ [(0, "1970-01-01T00:00:00Z"), (1791211444, "2026-10-05T14:44:04Z"), (None, None)],+)+def test_iso_time(timestamp: int | None, expected: str | None):+ assert iso_time(timestamp) == expecteddiff --git a/autogpt_platform/backend/backend/blocks/hacker_news/_thread.py b/autogpt_platform/backend/backend/blocks/hacker_news/_thread.pynew file mode 100644index 0000000000..99a0e22049--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/_thread.py@@ -0,0 +1,91 @@+"""Comment threads as flat lists, for the Hacker News Get Item block."""++from typing import Any, Callable++from ._models import (+ HackerNewsComment,+ HackerNewsItem,+ comment_from_node,+ item_from_firebase,+ item_from_thread,+ thread_node_from_firebase,+)++Node = dict[str, Any]+Read = tuple[HackerNewsItem, list[HackerNewsComment], int]+++def read_thread(thread: Node, live: Node | None, limit: int) -> Read:+ """The item, up to `limit` of its comments, and its comment count, from an+ HN Search thread and `live`, the same item from HN's official API.++ HN Search keeps every comment's replies oldest first. The official API+ lists the item's direct replies (`kids`) in HN's ranking, so the direct+ replies are put in that order; deeper replies stay oldest first. HN Search+ also trails the live site by a minute or two, so the points and comment+ count come from `live` when it has them. The comments still come from HN+ Search, so the newest can be missing from the list.+ """+ live = live or {}+ ranking = {kid: rank for rank, kid in enumerate(live.get("kids") or [])}+ replies = sorted(+ thread.get("children") or [],+ key=lambda node: ranking.get(node.get("id"), len(ranking)),+ )+ comments, found = flatten_thread(+ replies, lambda node: node.get("children") or [], limit+ )+ count = _comment_count(found, live)+ # The official API gives every job 1 point, which HN never shows.+ score = None if thread.get("type") == "job" else live.get("score")+ points = thread.get("points") if score is None else score+ return item_from_thread(thread, count, points), comments, count+++def read_live(live: Node, replies: dict[int, Node], limit: int) -> Read:+ """The same from HN's official API, for an item HN Search doesn't have.++ `replies` holds the comments fetched below the item, by id. The official+ API lists replies in HN's ranking at every level.+ """+ nodes = {reply_id: thread_node_from_firebase(r) for reply_id, r in replies.items()}++ def children(node: Node) -> list[Node]:+ return [nodes[kid] for kid in node["kids"] if kid in nodes]++ roots = [nodes[kid] for kid in live.get("kids") or [] if kid in nodes]+ comments, found = flatten_thread(roots, children, limit)+ return item_from_firebase(live), comments, _comment_count(found, live)+++def flatten_thread(+ roots: list[Node], children: Callable[[Node], list[Node]], limit: int+) -> tuple[list[HackerNewsComment], int]:+ """Comments depth first, each followed by its replies, and how many there are.++ A comment with no author or text (deleted or dead) is left out, but its+ replies are kept. Only the first `limit` comments are returned; the count+ covers them all. A comment that appears twice is only read once.+ """+ comments: list[HackerNewsComment] = []+ count = 0+ seen: set[Any] = set()+ stack = [(node, 1) for node in reversed(roots)]+ while stack:+ node, depth = stack.pop()+ if node.get("id") in seen:+ continue+ seen.add(node.get("id"))+ stack.extend((child, depth + 1) for child in reversed(children(node)))+ if not (node.get("author") and node.get("text")):+ continue+ count += 1+ if len(comments) < limit:+ comments.append(comment_from_node(node, depth))+ return comments, count+++def _comment_count(found: int, live: Node) -> int:+ """HN's own count (`descendants`) when it is higher than the comments+ found, which can miss the newest ones or the ones not fetched."""+ return max(found, live.get("descendants") or 0)diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/_thread_test.py b/autogpt_platform/backend/backend/blocks/hacker_news/_thread_test.pynew file mode 100644index 0000000000..7145b8c097--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/_thread_test.py@@ -0,0 +1,210 @@+"""Unit tests for flattening Hacker News comment threads into reading order."""++from typing import Any++from backend.blocks.hacker_news._thread import flatten_thread, read_live, read_thread+++def comment(+ comment_id: int, parent_id: int, *replies: dict[str, Any], deleted: bool = False+) -> dict[str, Any]:+ """A comment as HN Search returns it inside a thread."""+ return {+ "id": comment_id,+ "author": None if deleted else f"user{comment_id}",+ "text": None if deleted else f"Comment {comment_id}",+ "parent_id": parent_id,+ "created_at_i": 1790780400 + comment_id,+ "children": list(replies),+ }+++def story(*replies: dict[str, Any]) -> dict[str, Any]:+ return {"id": 1, "type": "story", "title": "A story", "children": list(replies)}+++def children(node: dict[str, Any]) -> list[dict[str, Any]]:+ return node.get("children") or []+++def ids_and_depths(comments) -> list[tuple[int, int]]:+ return [(c.id, c.depth) for c in comments]+++def test_flatten_puts_each_comment_before_its_replies():+ thread = [+ comment(10, 1, comment(11, 10, comment(111, 11)), comment(12, 10)),+ comment(20, 1),+ ]+ comments, count = flatten_thread(thread, children, limit=100)+ assert ids_and_depths(comments) == [(10, 1), (11, 2), (111, 3), (12, 2), (20, 1)]+ assert count == 5+ assert comments[2].parent_id == 11+ assert comments[2].text == "Comment 111"+ assert comments[2].hn_url == "https://news.ycombinator.com/item?id=111"+++def test_flatten_leaves_out_deleted_comments_but_keeps_their_replies():+ thread = [comment(10, 1, comment(11, 10), deleted=True), comment(20, 1)]+ comments, count = flatten_thread(thread, children, limit=100)+ assert ids_and_depths(comments) == [(11, 2), (20, 1)]+ assert comments[0].parent_id == 10+ assert count == 2+++def test_flatten_cuts_the_list_but_counts_every_comment():+ thread = [comment(10, 1, comment(11, 10)), comment(20, 1), comment(30, 1)]+ comments, count = flatten_thread(thread, children, limit=2)+ assert ids_and_depths(comments) == [(10, 1), (11, 2)]+ assert count == 4+ assert flatten_thread(thread, children, limit=0) == ([], 4)+++def test_flatten_handles_threads_deeper_than_the_recursion_limit():+ deepest = comment(5000, 4999)+ node = deepest+ for comment_id in range(4999, 9, -1):+ node = comment(comment_id, comment_id - 1, node)+ comments, count = flatten_thread([node], children, limit=10_000)+ assert count == 4991+ assert comments[-1].depth == 4991+++def test_thread_replies_follow_hn_ranking_and_deeper_ones_stay_oldest_first():+ thread = story(+ comment(10, 1, comment(11, 10), comment(12, 10)),+ comment(20, 1),+ comment(30, 1),+ comment(40, 1), # newer than the live item: not ranked yet+ )+ live = {"id": 1, "kids": [30, 10, 99, 20]}+ item, comments, count = read_thread(thread, live, limit=100)+ assert ids_and_depths(comments) == [+ (30, 1),+ (10, 1),+ (11, 2),+ (12, 2),+ (20, 1),+ (40, 1),+ ]+ assert count == 6+ assert (item.id, item.num_comments) == (1, 6)+++def test_thread_without_a_live_item_stays_oldest_first():+ thread = story(comment(10, 1), comment(20, 1))+ _, comments, _ = read_thread(thread, None, limit=100)+ assert ids_and_depths(comments) == [(10, 1), (20, 1)]+++def test_thread_with_no_comments():+ item, comments, count = read_thread(story(), {"id": 1}, limit=100)+ assert (comments, count, item.num_comments) == ([], 0, 0)+++def test_thread_points_and_count_come_from_the_live_item():+ # HN Search trails the live site: it has 2 of the story's 5 comments.+ thread = {**story(comment(10, 1), comment(20, 1)), "points": 10}+ live = {"id": 1, "type": "story", "score": 15, "descendants": 5, "kids": [20, 10]}+ item, comments, count = read_thread(thread, live, limit=100)+ assert (item.points, item.num_comments, count) == (15, 5, 5)+ assert ids_and_depths(comments) == [(20, 1), (10, 1)]+++def test_thread_count_is_never_below_the_comments_found():+ thread = story(comment(10, 1), comment(20, 1), comment(30, 1))+ live = {"id": 1, "type": "story", "score": 3, "descendants": 2}+ item, _, count = read_thread(thread, live, limit=100)+ assert (item.num_comments, count) == (3, 3)+++def test_thread_without_a_live_item_uses_hn_search_points_and_count():+ thread = {**story(comment(10, 1)), "points": 10}+ item, _, count = read_thread(thread, None, limit=100)+ assert (item.points, item.num_comments, count) == (10, 1, 1)+++def test_thread_job_shows_no_points_even_from_the_live_item():+ job = {"id": 2, "type": "job", "title": "Acme is hiring", "points": None}+ item, _, count = read_thread(job, {"id": 2, "type": "job", "score": 1}, limit=10)+ assert (item.points, item.num_comments, count) == (None, None, 0)+++def test_thread_for_a_comment_counts_its_replies():+ thread = {+ **comment(10, 1, comment(11, 10), comment(12, 10)),+ "type": "comment",+ "story_id": 1,+ }+ live = {"id": 10, "type": "comment", "by": "user10", "parent": 1, "kids": [12, 11]}+ item, comments, count = read_thread(thread, live, limit=100)+ assert (item.type, item.points, item.num_comments) == ("comment", None, None)+ assert (item.story_id, item.parent_id) == (1, 1)+ assert ids_and_depths(comments) == [(12, 1), (11, 1)]+ assert count == 2+++def firebase(item_id: int, parent: int, *kids: int, **extra: Any) -> dict[str, Any]:+ """A comment as HN's official API returns it."""+ return {+ "id": item_id,+ "by": f"user{item_id}",+ "text": f"Comment {item_id}",+ "time": 1790780400 + item_id,+ "parent": parent,+ "kids": list(kids),+ "type": "comment",+ **extra,+ }+++def test_live_comments_follow_hn_ranking_at_every_level():+ live = {"id": 1, "type": "story", "by": "op", "kids": [20, 10], "descendants": 4}+ replies = {+ 10: firebase(10, 1, 12, 11),+ 11: firebase(11, 10),+ 12: firebase(12, 10),+ 20: firebase(20, 1),+ }+ item, comments, count = read_live(live, replies, limit=100)+ assert ids_and_depths(comments) == [(20, 1), (10, 1), (12, 2), (11, 2)]+ assert comments[1].author == "user10"+ assert count == 4+ assert (item.id, item.author, item.num_comments) == (1, "op", 4)+++def test_live_comments_skip_dead_deleted_and_unfetched_ones():+ live = {"id": 1, "type": "story", "kids": [10, 20, 30, 40]}+ replies = {+ 10: firebase(10, 1, 11, dead=True),+ 11: firebase(11, 10),+ 20: {"id": 20, "deleted": True, "parent": 1, "type": "comment"},+ 30: firebase(30, 1),+ # 40 wasn't fetched (over the limit, or gone)+ }+ _, comments, count = read_live(live, replies, limit=100)+ assert ids_and_depths(comments) == [(11, 2), (30, 1)]+ assert count == 2+++def test_live_count_uses_hn_count_when_not_every_reply_was_fetched():+ live = {"id": 1, "type": "story", "kids": [10, 20], "descendants": 57}+ replies = {10: firebase(10, 1), 20: firebase(20, 1)}+ _, comments, count = read_live(live, replies, limit=1)+ assert ids_and_depths(comments) == [(10, 1)]+ assert count == 57+++def test_live_comments_that_list_each_other_are_read_once():+ # Malformed data: 10 and 11 reply to each other, and 10 is listed twice.+ live = {"id": 1, "type": "story", "kids": [10, 10]}+ replies = {10: firebase(10, 1, 11), 11: firebase(11, 10, 10)}+ _, comments, count = read_live(live, replies, limit=100)+ assert ids_and_depths(comments) == [(10, 1), (11, 2)]+ assert count == 2+++def test_live_item_without_replies():+ live = {"id": 5, "type": "comment", "by": "x", "text": "hi", "parent": 1}+ item, comments, count = read_live(live, {}, limit=100)+ assert (item.id, item.parent_id, comments, count) == (5, 1, [], 0)diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/hacker_news_effects_test.py b/autogpt_platform/backend/backend/blocks/hacker_news/hacker_news_effects_test.pynew file mode 100644index 0000000000..1d7bdf2ff2--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/hacker_news_effects_test.py@@ -0,0 +1,38 @@+import pytest++from backend.blocks._base import Block, BlockEffect+from backend.blocks.hacker_news.items import HackerNewsGetItemBlock+from backend.blocks.hacker_news.search import HackerNewsSearchBlock+from backend.blocks.hacker_news.stories import HackerNewsGetStoriesBlock+from backend.blocks.hacker_news.users import HackerNewsGetUserBlock+++ ("block", "effect"),+ [+ (HackerNewsSearchBlock, BlockEffect.READ),+ (HackerNewsGetStoriesBlock, BlockEffect.READ),+ (HackerNewsGetItemBlock, BlockEffect.READ),+ (HackerNewsGetUserBlock, BlockEffect.READ),+ ],+)+def test_block_declares_what_it_does(block: type[Block], effect: BlockEffect):+ # AutoPilot runs reads without asking, and treats an undeclared block as one+ # that changes something outside the platform.+ assert block().effect is effect+++ "block",+ [+ HackerNewsSearchBlock,+ HackerNewsGetStoriesBlock,+ HackerNewsGetItemBlock,+ HackerNewsGetUserBlock,+ ],+)+def test_block_stays_a_primitive(block: type[Block]):+ # A provider-less "service" vanishes from AutoPilot's search when the query+ # names another service ("send hacker news stories to slack"); a primitive+ # still comes back as a fallback there.+ assert block().capability_kind == "primitive"diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/items.py b/autogpt_platform/backend/backend/blocks/hacker_news/items.pynew file mode 100644index 0000000000..dd94ca2636--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/items.py@@ -0,0 +1,235 @@+import asyncio+from typing import Any++from backend.blocks._base import (+ Block,+ BlockCategory,+ BlockEffect,+ BlockOutput,+ BlockSchemaInput,+ BlockSchemaOutput,+)+from backend.data.model import SchemaField+from backend.util.exceptions import BlockExecutionError, BlockInputError++from ._api import HackerNewsError, get_item, get_replies, get_thread, parse_item_id+from ._models import HackerNewsComment, HackerNewsItem+from ._thread import read_live, read_thread++_TEST_THREAD: dict[str, Any] = {+ "id": 49900001,+ "type": "story",+ "author": "builder42",+ "title": "Show HN: Hacker News blocks for AutoGPT",+ "url": "https://agpt.co/blog/hacker-news-blocks",+ "text": None,+ "points": 42,+ "parent_id": None,+ "story_id": 49900001,+ "created_at_i": 1790780400,+ "children": [+ {+ "id": 49900107,+ "author": "hn_reader",+ "text": "Does it read <i>Ask HN</i> posts too?",+ "parent_id": 49900001,+ "created_at_i": 1790781600,+ "children": [+ {+ "id": 49900215,+ "author": "builder42",+ "text": "Yes, as plain text.",+ "parent_id": 49900107,+ "created_at_i": 1790783100,+ "children": [],+ }+ ],+ },+ {+ # Deleted: HN Search keeps it, without author or text, for its reply.+ "id": 49900150,+ "author": None,+ "text": None,+ "parent_id": 49900001,+ "created_at_i": 1790782000,+ "children": [+ {+ "id": 49900301,+ "author": "lurker",+ "text": "What did it say?",+ "parent_id": 49900150,+ "created_at_i": 1790787900,+ "children": [],+ }+ ],+ },+ ],+}+# The same story from HN's official API. It ranks the deleted comment first,+# and has 3 more points and 1 more comment than HN Search has indexed so far.+_TEST_LIVE_ITEM: dict[str, Any] = {+ "id": 49900001,+ "type": "story",+ "by": "builder42",+ "time": 1790780400,+ "score": 45,+ "descendants": 4,+ "kids": [49900150, 49900107],+}+++class HackerNewsGetItemBlock(Block):+ """A Hacker News story, comment, poll or job, with its comments."""++ class Input(BlockSchemaInput):+ item: str = SchemaField(+ description=(+ "The item's id, such as 8863, or its link, such as "+ "https://news.ycombinator.com/item?id=8863"+ ),+ placeholder="e.g. 8863",+ )+ include_comments: bool = SchemaField(+ description="Also return the item's comments", default=True+ )+ max_comments: int = SchemaField(+ description=(+ "Most comments to return, in reading order. comment_count still "+ "counts them all."+ ),+ default=200,+ ge=1,+ le=1000,+ advanced=True,+ )++ class Output(BlockSchemaOutput):+ item: HackerNewsItem = SchemaField(+ description="The story, comment, poll or job"+ )+ comments: list[HackerNewsComment] = SchemaField(+ description=(+ "Its comments in reading order: each comment followed by its "+ "replies, with direct replies in Hacker News's ranking"+ )+ )+ comment: HackerNewsComment = SchemaField(description="Each comment")+ comment_count: int = SchemaField(+ description=(+ "How many comments the item has, including any that max_comments "+ "left out. Hacker News's own count when it is higher."+ )+ )++ def __init__(self):+ item = HackerNewsItem(+ id=49900001,+ type="story",+ title="Show HN: Hacker News blocks for AutoGPT",+ url="https://agpt.co/blog/hacker-news-blocks",+ author="builder42",+ points=45,+ num_comments=4,+ created_at="2026-09-30T15:00:00Z",+ hn_url="https://news.ycombinator.com/item?id=49900001",+ story_id=49900001,+ )+ # The deleted comment is ranked first: it is left out, its reply kept.+ comments = [+ HackerNewsComment(+ id=49900301,+ author="lurker",+ text="What did it say?",+ created_at="2026-09-30T17:05:00Z",+ parent_id=49900150,+ depth=2,+ hn_url="https://news.ycombinator.com/item?id=49900301",+ ),+ HackerNewsComment(+ id=49900107,+ author="hn_reader",+ text="Does it read *Ask HN* posts too?",+ created_at="2026-09-30T15:20:00Z",+ parent_id=49900001,+ depth=1,+ hn_url="https://news.ycombinator.com/item?id=49900107",+ ),+ ]+ super().__init__(+ id="3150fc1d-f7bc-46be-bdbe-2650ff7b2d62",+ description=(+ "Get a Hacker News story, comment, poll or job by its id or link, with "+ "its comments as a list in reading order. Each comment is followed by "+ "its replies, and direct replies come in Hacker News's ranking."+ ),+ categories={BlockCategory.SEARCH, BlockCategory.SOCIAL},+ input_schema=HackerNewsGetItemBlock.Input,+ output_schema=HackerNewsGetItemBlock.Output,+ test_input={+ "item": "https://news.ycombinator.com/item?id=49900001",+ "max_comments": 2,+ },+ test_output=[+ ("item", item),+ ("comments", comments),+ ("comment", comments[0]),+ ("comment", comments[1]),+ ("comment_count", 4),+ ],+ test_mock={+ "_fetch": lambda *args, **kwargs: (_TEST_THREAD, _TEST_LIVE_ITEM, {})+ },+ effect=BlockEffect.READ,+ )++ async def run(self, input_data: Input, **kwargs) -> BlockOutput:+ item_id = parse_item_id(input_data.item)+ if item_id is None:+ raise BlockInputError(+ message=(+ f"'{input_data.item.strip()}' isn't a Hacker News item. Give its "+ "id, such as 8863, or its link, such as "+ "https://news.ycombinator.com/item?id=8863."+ ),+ block_name=self.name,+ block_id=self.id,+ )+ limit = input_data.max_comments if input_data.include_comments else 0+ try:+ thread, live, replies = await self._fetch(item_id, limit)+ except HackerNewsError as e:+ raise BlockExecutionError(+ message=str(e), block_name=self.name, block_id=self.id+ ) from e++ if thread is not None:+ item, comments, count = read_thread(thread, live, limit)+ elif live is not None:+ item, comments, count = read_live(live, replies, limit)+ else:+ raise BlockExecutionError(+ message=f"Hacker News has no item {item_id}. Check the id or link.",+ block_name=self.name,+ block_id=self.id,+ )+ yield "item", item+ if input_data.include_comments:+ yield "comments", comments+ for comment in comments:+ yield "comment", comment+ yield "comment_count", count++ @staticmethod+ async def _fetch(+ item_id: int, max_replies: int+ ) -> tuple[dict[str, Any] | None, dict[str, Any] | None, dict[int, dict[str, Any]]]:+ """The item's thread from HN Search and the item from HN's official API.++ HN Search doesn't have items from the last few minutes, or dead ones.+ For those, up to `max_replies` comments come from the official API.+ """+ thread, live = await asyncio.gather(get_thread(item_id), get_item(item_id))+ replies: dict[int, dict[str, Any]] = {}+ if thread is None and live is not None and max_replies:+ replies = await get_replies(live.get("kids") or [], max_replies)+ return thread, live, repliesdiff --git a/autogpt_platform/backend/backend/blocks/hacker_news/items_test.py b/autogpt_platform/backend/backend/blocks/hacker_news/items_test.pynew file mode 100644index 0000000000..5b0890df4d--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/items_test.py@@ -0,0 +1,233 @@+"""Unit tests for the Hacker News Get Item block.++The block's own test_input/test_mock case mocks the API calls away; these+cover the requests, the fallback to HN's official API, turning comments off,+and errors. Comment order is covered in _thread_test.py.+"""++from typing import Any++import pytest++from backend.blocks.hacker_news import items+from backend.blocks.hacker_news._api import HackerNewsError+from backend.blocks.hacker_news.items import HackerNewsGetItemBlock+from backend.util.exceptions import BlockExecutionError, BlockInputError++THREAD = {+ "id": 8863,+ "type": "story",+ "author": "dhouston",+ "title": "My YC app: Dropbox - Throw away your USB drive",+ "points": 104,+ "story_id": 8863,+ "created_at_i": 1175714200,+ "children": [+ {+ "id": 8865,+ "author": "dhouston",+ "text": "oh, and a mac port is coming :)",+ "parent_id": 8863,+ "created_at_i": 1175714575,+ "children": [],+ }+ ],+}+LIVE = {"id": 8863, "type": "story", "by": "dhouston", "kids": [8865], "score": 104}+++def block() -> HackerNewsGetItemBlock:+ return HackerNewsGetItemBlock()+++class _FakeApi:+ """Stands in for the HN API calls _fetch makes, and records them."""++ def __init__(self, thread: Any, live: Any, replies: dict[int, Any] | None = None):+ self.thread, self.live, self.replies = thread, live, replies or {}+ self.calls: list[tuple[str, Any]] = []++ async def get_thread(self, item_id: int):+ self.calls.append(("get_thread", item_id))+ return self.thread++ async def get_item(self, item_id: int):+ self.calls.append(("get_item", item_id))+ return self.live++ async def get_replies(self, kids: list[int], limit: int):+ self.calls.append(("get_replies", (kids, limit)))+ return self.replies++ def install(self, monkeypatch: pytest.MonkeyPatch) -> "_FakeApi":+ for name in ("get_thread", "get_item", "get_replies"):+ monkeypatch.setattr(items, name, getattr(self, name))+ return self+++async def test_fetch_reads_both_apis_and_no_replies_when_hn_search_has_the_item(+ monkeypatch: pytest.MonkeyPatch,+):+ fake = _FakeApi(THREAD, LIVE).install(monkeypatch)+ assert await HackerNewsGetItemBlock._fetch(8863, 200) == (THREAD, LIVE, {})+ assert sorted(fake.calls) == [("get_item", 8863), ("get_thread", 8863)]+++async def test_fetch_falls_back_to_the_official_api_for_replies(+ monkeypatch: pytest.MonkeyPatch,+):+ replies = {8865: {"id": 8865, "by": "dhouston", "text": "hi", "parent": 8863}}+ fake = _FakeApi(None, LIVE, replies).install(monkeypatch)+ assert await HackerNewsGetItemBlock._fetch(8863, 50) == (None, LIVE, replies)+ assert ("get_replies", ([8865], 50)) in fake.calls+++ "live, max_replies", [(LIVE, 0), (None, 200)], ids=["comments off", "no item"]+)+async def test_fetch_skips_replies_when_none_are_wanted_or_there_is_no_item(+ monkeypatch: pytest.MonkeyPatch, live: Any, max_replies: int+):+ fake = _FakeApi(None, live).install(monkeypatch)+ assert await HackerNewsGetItemBlock._fetch(8863, max_replies) == (None, live, {})+ assert all(name != "get_replies" for name, _ in fake.calls)+++async def test_item_with_comments(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetItemBlock+):+ limits = mock_fetch(monkeypatch, block, (THREAD, LIVE, {}))+ outputs = await collect(block, item="8863", max_comments=5)+ assert [name for name, _ in outputs] == [+ "item",+ "comments",+ "comment",+ "comment_count",+ ]+ assert outputs[0][1].title == "My YC app: Dropbox - Throw away your USB drive"+ assert outputs[2][1].text == "oh, and a mac port is coming :)"+ assert outputs[3][1] == 1+ assert limits == [(8863, 5)]+++async def test_item_points_and_count_are_live_but_comments_are_hn_search_s(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetItemBlock+):+ # Measured on a front-page story: live 205 points and 112 comments while+ # HN Search had 198 and 106.+ live = {**LIVE, "score": 205, "descendants": 112}+ mock_fetch(monkeypatch, block, (THREAD, live, {}))+ outputs = dict(await collect(block, item="8863"))+ assert (outputs["item"].points, outputs["item"].num_comments) == (205, 112)+ assert outputs["comment_count"] == 112+ assert [c.id for c in outputs["comments"]] == [8865]+++async def test_item_without_comments_still_counts_them(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetItemBlock+):+ limits = mock_fetch(monkeypatch, block, (THREAD, LIVE, {}))+ outputs = await collect(block, item="8863", include_comments=False)+ assert [name for name, _ in outputs] == ["item", "comment_count"]+ assert outputs[1][1] == 1+ assert limits == [(8863, 0)]+++async def test_item_hn_search_has_not_indexed_comes_from_the_official_api(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetItemBlock+):+ live = {+ "id": 49965665,+ "type": "story",+ "by": "newposter",+ "title": "Posted a minute ago",+ "url": "https://example.com/new",+ "score": 2,+ "descendants": 1,+ "time": 1791212400,+ "kids": [49965700],+ }+ reply = {+ "id": 49965700,+ "type": "comment",+ "by": "fast",+ "text": "First!",+ "parent": 49965665,+ "time": 1791212460,+ }+ mock_fetch(monkeypatch, block, (None, live, {49965700: reply}))+ outputs = dict(await collect(block, item="news.ycombinator.com/item?id=49965665"))+ assert outputs["item"].title == "Posted a minute ago"+ assert outputs["item"].num_comments == 1+ assert [(c.id, c.author, c.depth) for c in outputs["comments"]] == [+ (49965700, "fast", 1)+ ]+ assert outputs["comment_count"] == 1+++async def test_unknown_item(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetItemBlock+):+ mock_fetch(monkeypatch, block, (None, None, {}))+ with pytest.raises(BlockExecutionError, match="Hacker News has no item 999999999"):+ await collect(block, item="999999999")+++ "text",+ [+ "",+ "abc",+ "https://example.com/item?id=1",+ "https://news.ycombinator.com/user?id=pg",+ ],+)+async def test_input_that_is_not_an_item(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetItemBlock, text: str+):+ limits = mock_fetch(monkeypatch, block, (THREAD, LIVE, {}))+ with pytest.raises(BlockInputError, match="isn't a Hacker News item"):+ await collect(block, item=text)+ assert limits == []+++async def test_item_turns_api_errors_into_block_errors(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetItemBlock+):+ async def fake_fetch(item_id: int, max_replies: int):+ raise HackerNewsError("Hacker News search (hn.algolia.com) is limiting")++ monkeypatch.setattr(block, "_fetch", fake_fetch)+ with pytest.raises(BlockExecutionError, match="is limiting"):+ await collect(block, item="8863")+++def mock_fetch(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetItemBlock, result: tuple+) -> list[tuple[int, int]]:+ """Make the block's _fetch return `result`; returns the (id, limit) it gets."""+ calls: list[tuple[int, int]] = []++ async def fake_fetch(item_id: int, max_replies: int):+ calls.append((item_id, max_replies))+ return result++ monkeypatch.setattr(block, "_fetch", fake_fetch)+ return calls+++async def collect(+ block: HackerNewsGetItemBlock, **inputs: Any+) -> list[tuple[str, Any]]:+ return [output async for output in block.run(block.Input.model_validate(inputs))]diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/search.py b/autogpt_platform/backend/backend/blocks/hacker_news/search.pynew file mode 100644index 0000000000..2ac35d77b7--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/search.py@@ -0,0 +1,327 @@+import re+from datetime import datetime, timedelta, timezone+from enum import Enum+from typing import Any++from backend.blocks._base import (+ Block,+ BlockCategory,+ BlockEffect,+ BlockOutput,+ BlockSchemaInput,+ BlockSchemaOutput,+)+from backend.data.model import SchemaField+from backend.util.exceptions import BlockExecutionError, BlockInputError++from ._api import HackerNewsError, parse_username, search+from ._models import HackerNewsItem, item_from_hit+++class SearchFor(str, Enum):+ STORIES = "stories"+ COMMENTS = "comments"+ STORIES_AND_COMMENTS = "stories_and_comments"+ SHOW_HN = "show_hn"+ ASK_HN = "ask_hn"+ FRONT_PAGE = "front_page"+++class MatchIn(str, Enum):+ ANYWHERE = "anywhere"+ TITLE = "title"+ URL = "url"+++class SortBy(str, Enum):+ RELEVANCE = "relevance"+ NEWEST = "newest"+++# HN Search tags: a comma means AND, brackets mean OR.+_TAGS = {+ SearchFor.STORIES: "story",+ SearchFor.COMMENTS: "comment",+ SearchFor.STORIES_AND_COMMENTS: "(story,comment)",+ SearchFor.SHOW_HN: "show_hn",+ SearchFor.ASK_HN: "ask_hn",+ SearchFor.FRONT_PAGE: "front_page",+}+_AGE = re.compile(r"(\d{1,5})\s*(h|hours?|d|days?|w|weeks?)", re.IGNORECASE)+_HOURS_PER_UNIT = {"h": 1, "d": 24, "w": 24 * 7}++_TEST_STORY_HIT: dict[str, Any] = {+ "_tags": ["story", "author_builder42", "story_49900001", "show_hn"],+ "objectID": "49900001",+ "title": "Show HN: Hacker News blocks for AutoGPT",+ "url": "https://agpt.co/blog/hacker-news-blocks",+ "author": "builder42",+ "points": 42,+ "num_comments": 7,+ "created_at_i": 1790780400,+ "story_id": 49900001,+}+_TEST_COMMENT_HIT: dict[str, Any] = {+ "_tags": ["comment", "author_hn_reader", "story_49900001"],+ "objectID": "49900107",+ "comment_text": (+ "I've tried <i>AutoGPT</i> for this.<p>Docs: "+ '<a href="https://agpt.co/docs/integrations" '+ 'rel="nofollow">https://agpt.co/docs/...</a>'+ ),+ "author": "hn_reader",+ "points": None,+ "story_id": 49900001,+ "story_title": "Show HN: Hacker News blocks for AutoGPT",+ "story_url": "https://agpt.co/blog/hacker-news-blocks",+ "parent_id": 49900001,+ "created_at_i": 1790785800,+}+++class HackerNewsSearchBlock(Block):+ """Search Hacker News stories and comments through HN Search (Algolia)."""++ class Input(BlockSchemaInput):+ query: str = SchemaField(+ description=(+ "Words to search for. Put a phrase in double quotes to match it as "+ "a phrase, and put - before a word to leave out items that have it. "+ "Leave empty to match everything, e.g. for the top stories of the "+ "past week."+ ),+ default="",+ placeholder='e.g. "AutoGPT" or agpt.co',+ )+ search_for: SearchFor = SchemaField(+ description=(+ "What to search: stories, comments, stories_and_comments, show_hn or "+ "ask_hn posts, or front_page (only the stories on the front page now)"+ ),+ default=SearchFor.STORIES,+ )+ match_in: MatchIn = SchemaField(+ description=(+ "Where the query must match: anywhere (title, link, text and author), "+ "title, or url (the link a story points to, e.g. to find links to a "+ "domain). title and url only match stories."+ ),+ default=MatchIn.ANYWHERE,+ )+ exact_match: bool = SchemaField(+ description=(+ "Match the query words as written, without typo tolerance. On by "+ "default, because typo tolerance makes 'autogpt' also match "+ "'automotive' and 'automation'. Plurals still match. Turn it off to "+ "also catch misspellings."+ ),+ default=True,+ )+ sort_by: SortBy = SchemaField(+ description=(+ "relevance: best matches first, then most points, then most comments. "+ "newest: newest first."+ ),+ default=SortBy.RELEVANCE,+ )+ created_after: str = SchemaField(+ description=(+ "Only items posted at or after this time: an ISO 8601 date or time "+ "(2026-09-01, 2026-09-01T12:00:00Z; UTC unless it has an offset), or "+ "an age such as 24h, 7d or 2w"+ ),+ default="",+ placeholder="e.g. 7d or 2026-09-01",+ )+ created_before: str = SchemaField(+ description=(+ "Only items posted before this time, written the same way as "+ "created_after"+ ),+ default="",+ placeholder="e.g. 2026-10-01",+ )+ author: str = SchemaField(+ description="Only items posted by this Hacker News username (case-sensitive)",+ default="",+ placeholder="e.g. pg",+ advanced=True,+ )+ min_points: int = SchemaField(+ description=(+ "Only stories with at least this many points. Comments have no "+ "points, so any value above 0 leaves comments out."+ ),+ default=0,+ ge=0,+ advanced=True,+ )+ min_comments: int = SchemaField(+ description=(+ "Only stories with at least this many comments. Any value above 0 "+ "leaves comments out."+ ),+ default=0,+ ge=0,+ advanced=True,+ )+ max_results: int = SchemaField(+ description="Most results to return (up to 1,000)",+ default=20,+ ge=1,+ le=1000,+ )++ class Output(BlockSchemaOutput):+ results: list[HackerNewsItem] = SchemaField(+ description="Matching stories and comments, in the chosen order"+ )+ result: HackerNewsItem = SchemaField(+ description="Each matching story or comment"+ )+ total_matches: int = SchemaField(+ description=(+ "How many items match in all. Can be more than the results returned: "+ "HN Search returns at most 1,000."+ )+ )++ def __init__(self):+ story = item_from_hit(_TEST_STORY_HIT)+ comment = item_from_hit(_TEST_COMMENT_HIT)+ super().__init__(+ id="f86d4a13-e470-4a7c-9eba-2bb6a76c0552",+ description=(+ "Search Hacker News stories and comments by keyword, author, date, "+ "points or linked domain, sorted by relevance or newest first. Use it "+ "to find mentions of a product, links to a website, or the most "+ "upvoted stories of the week. Uses HN Search by Algolia, which needs "+ "no account. exact_match is on by default, so words that are only "+ "spelled alike are left out; turn it off to also catch misspellings."+ ),+ categories={BlockCategory.SEARCH, BlockCategory.SOCIAL},+ input_schema=HackerNewsSearchBlock.Input,+ output_schema=HackerNewsSearchBlock.Output,+ test_input={+ "query": "AutoGPT",+ "search_for": SearchFor.STORIES_AND_COMMENTS,+ "sort_by": SortBy.NEWEST,+ },+ test_output=[+ ("results", [comment, story]),+ ("result", comment),+ ("result", story),+ ("total_matches", 2),+ ],+ test_mock={+ "_search": lambda *args, **kwargs: {+ "hits": [_TEST_COMMENT_HIT, _TEST_STORY_HIT],+ "nbHits": 2,+ }+ },+ effect=BlockEffect.READ,+ )++ async def run(self, input_data: Input, **kwargs) -> BlockOutput:+ params = self.search_params(input_data, datetime.now(timezone.utc))+ endpoint = "search_by_date" if input_data.sort_by == SortBy.NEWEST else "search"+ try:+ response = await self._search(endpoint, params)+ except HackerNewsError as e:+ raise BlockExecutionError(+ message=str(e), block_name=self.name, block_id=self.id+ ) from e++ results = [item_from_hit(hit) for hit in response.get("hits") or []]+ yield "results", results+ for result in results:+ yield "result", result+ yield "total_matches", response.get("nbHits", len(results))++ @staticmethod+ async def _search(endpoint: str, params: dict[str, str]) -> dict[str, Any]:+ return await search(endpoint, params)++ def search_params(self, input_data: Input, now: datetime) -> dict[str, str]:+ """HN Search query parameters for the block's inputs."""+ if (+ input_data.search_for == SearchFor.COMMENTS+ and input_data.match_in != MatchIn.ANYWHERE+ ):+ raise self._input_error(+ "Comments have no title or link of their own, so they can only be "+ "matched anywhere. Set match_in to anywhere, or search stories."+ )+ tags = _TAGS[input_data.search_for]+ if input_data.author.strip():+ tags += f",author_{self._username(input_data.author)}"+ params = {+ "query": input_data.query.strip(),+ "tags": tags,+ "hitsPerPage": str(input_data.max_results),+ }+ if filters := self._numeric_filters(input_data, now):+ params["numericFilters"] = filters+ if input_data.match_in != MatchIn.ANYWHERE:+ params["restrictSearchableAttributes"] = input_data.match_in.value+ if input_data.exact_match:+ params["typoTolerance"] = "false"+ return params++ def _numeric_filters(self, input_data: Input, now: datetime) -> str:+ after = self._time(input_data.created_after, "created_after", now)+ before = self._time(input_data.created_before, "created_before", now)+ if after is not None and before is not None and after >= before:+ raise self._input_error(+ "created_after must be earlier than created_before."+ )+ filters: list[str] = []+ if after is not None:+ filters.append(f"created_at_i>={after}")+ if before is not None:+ filters.append(f"created_at_i<{before}")+ if input_data.min_points:+ filters.append(f"points>={input_data.min_points}")+ if input_data.min_comments:+ filters.append(f"num_comments>={input_data.min_comments}")+ return ",".join(filters)++ def _time(self, text: str, field: str, now: datetime) -> int | None:+ try:+ return parse_time(text, now)+ except ValueError as e:+ raise self._input_error(+ f"{field} '{text.strip()}' isn't a date or an age. Use an ISO 8601 "+ "date or time such as 2026-09-01 or 2026-09-01T12:00:00Z, or an age "+ "such as 24h, 7d or 2w."+ ) from e++ def _username(self, text: str) -> str:+ username = parse_username(text)+ if not username:+ raise self._input_error(+ f"'{text.strip()}' isn't a Hacker News username. Usernames have only "+ "letters, digits, - and _."+ )+ return username++ def _input_error(self, message: str) -> BlockInputError:+ return BlockInputError(message=message, block_name=self.name, block_id=self.id)+++def parse_time(text: str, now: datetime) -> int | None:+ """Unix seconds for an ISO 8601 date or time, or for an age such as 24h, 7d+ or 2w before `now`. None for empty text; ValueError for anything else.++ A date or time without a UTC offset is taken as UTC.+ """+ text = text.strip()+ if not text:+ return None+ if age := _AGE.fullmatch(text):+ hours = int(age[1]) * _HOURS_PER_UNIT[age[2][0].lower()]+ return int((now - timedelta(hours=hours)).timestamp())+ moment = datetime.fromisoformat(text)+ if moment.tzinfo is None:+ moment = moment.replace(tzinfo=timezone.utc)+ return int(moment.timestamp())diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/search_test.py b/autogpt_platform/backend/backend/blocks/hacker_news/search_test.pynew file mode 100644index 0000000000..482f4d0894--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/search_test.py@@ -0,0 +1,241 @@+"""Unit tests for the Hacker News Search block: query parameters, tag and+numeric filter strings, dates, input errors and outputs.++The block's own test_input/test_mock case mocks the API call away; these+cover what that mock skips.+"""++from datetime import datetime, timezone+from typing import Any++import pytest++from backend.blocks._base import BlockOutput+from backend.blocks.hacker_news._api import HackerNewsError+from backend.blocks.hacker_news.search import HackerNewsSearchBlock, parse_time+from backend.util.exceptions import BlockExecutionError, BlockInputError++NOW = datetime(2026, 10, 5, 12, 0, tzinfo=timezone.utc)+NOW_TS = int(NOW.timestamp())+HOUR = 3600+DAY = 24 * HOUR+++def params(**inputs: Any) -> dict[str, str]:+ block = HackerNewsSearchBlock()+ return block.search_params(block.Input.model_validate(inputs), NOW)+++def test_defaults_search_stories_without_typo_tolerance():+ assert params() == {+ "query": "",+ "tags": "story",+ "hitsPerPage": "20",+ "typoTolerance": "false",+ }+++def test_typo_tolerance_can_be_turned_back_on():+ assert "typoTolerance" not in params(query="autogtp", exact_match=False)+++ "search_for, author, tags",+ [+ ("stories", "", "story"),+ ("comments", "", "comment"),+ ("stories_and_comments", "", "(story,comment)"),+ ("show_hn", "", "show_hn"),+ ("ask_hn", "", "ask_hn"),+ ("front_page", "", "front_page"),+ ("stories", "pg", "story,author_pg"),+ ("stories_and_comments", " dang ", "(story,comment),author_dang"),+ (+ "comments",+ "https://news.ycombinator.com/user?id=tptacek",+ "comment,author_tptacek",+ ),+ ],+)+def test_tags(search_for: str, author: str, tags: str):+ assert params(search_for=search_for, author=author)["tags"] == tags+++def test_domain_search_matches_story_links_exactly():+ assert params(query=" agpt.co ", match_in="url", max_results=1000) == {+ "query": "agpt.co",+ "tags": "story",+ "hitsPerPage": "1000",+ "restrictSearchableAttributes": "url",+ "typoTolerance": "false",+ }+++def test_title_search_on_stories_and_comments_is_allowed():+ found = params(search_for="stories_and_comments", match_in="title")+ assert found["restrictSearchableAttributes"] == "title"++[email protected]("match_in", ["title", "url"])+def test_comments_cannot_be_matched_by_title_or_link(match_in: str):+ with pytest.raises(BlockInputError, match="Set match_in to anywhere"):+ params(search_for="comments", match_in=match_in)+++def test_numeric_filters_combine_with_commas():+ found = params(+ created_after="2025-09-01",+ created_before="2025-10-01T00:00:00Z",+ min_points=100,+ min_comments=10,+ )+ assert found["numericFilters"] == (+ "created_at_i>=1756684800,created_at_i<1759276800,"+ "points>=100,num_comments>=10"+ )+++ "created_after, created_at_or_after",+ [+ ("24h", NOW_TS - DAY),+ ("7d", NOW_TS - 7 * DAY),+ ("2w", NOW_TS - 14 * DAY),+ ("3 days", NOW_TS - 3 * DAY),+ ("1 Hour", NOW_TS - HOUR),+ ],+)+def test_relative_ages_count_back_from_now(+ created_after: str, created_at_or_after: int+):+ found = params(created_after=created_after)+ assert found["numericFilters"] == f"created_at_i>={created_at_or_after}"+++def test_zero_minimums_add_no_filter():+ assert "numericFilters" not in params(min_points=0, min_comments=0)+++ "field, value",+ [+ ("created_after", "yesterday"),+ ("created_after", "7x"),+ ("created_before", "2026-13-01"),+ ("created_before", "1/2/2026"),+ ],+)+def test_bad_dates_name_the_input(field: str, value: str):+ with pytest.raises(BlockInputError, match=f"{field} '{value}' isn't a date"):+ params(**{field: value})+++def test_dates_must_be_in_order():+ with pytest.raises(BlockInputError, match="earlier than created_before"):+ params(created_after="2026-09-02", created_before="2026-09-01")++[email protected]("author", ["pg,dang", "(pg)", "pg dang"])+def test_bad_authors_are_refused_before_they_reach_the_tags(author: str):+ with pytest.raises(BlockInputError, match="isn't a Hacker News username"):+ params(author=author)+++ "text, expected",+ [+ ("2026-09-01", 1788220800),+ ("2026-09-01T12:00:00Z", 1788264000),+ ("2026-09-01T12:00:00", 1788264000),+ ("2026-09-01T14:00:00+02:00", 1788264000),+ (" 48h ", NOW_TS - 2 * DAY),+ ("0d", NOW_TS),+ ("", None),+ (" ", None),+ ],+)+def test_parse_time(text: str, expected: int | None):+ assert parse_time(text, NOW) == expected++[email protected]("text", ["last week", "7", "d7", "-7d", "123456d"])+def test_parse_time_rejects_anything_else(text: str):+ with pytest.raises(ValueError):+ parse_time(text, NOW)+++STORY_HIT = {+ "_tags": ["story", "author_pg", "story_1"],+ "objectID": "1",+ "title": "Y Combinator",+ "url": "http://ycombinator.com",+ "author": "pg",+ "points": 57,+ "num_comments": 18,+ "created_at_i": 1160418111,+ "story_id": 1,+}+++ "sort_by, endpoint", [("relevance", "search"), ("newest", "search_by_date")]+)+async def test_run_searches_and_outputs_each_result(+ monkeypatch: pytest.MonkeyPatch, sort_by: str, endpoint: str+):+ block = HackerNewsSearchBlock()+ calls: list[tuple[str, dict[str, str]]] = []++ async def fake_search(endpoint: str, params: dict[str, str]) -> dict[str, Any]:+ calls.append((endpoint, params))+ return {"hits": [STORY_HIT], "nbHits": 4321}++ monkeypatch.setattr(block, "_search", fake_search)+ outputs = await collect(block, query="ycombinator", sort_by=sort_by)+ assert [name for name, _ in outputs] == ["results", "result", "total_matches"]+ assert outputs[0][1][0].title == "Y Combinator"+ assert outputs[1][1] == outputs[0][1][0]+ assert outputs[2][1] == 4321+ assert calls == [+ (+ endpoint,+ {+ "query": "ycombinator",+ "tags": "story",+ "hitsPerPage": "20",+ "typoTolerance": "false",+ },+ )+ ]+++async def test_run_with_no_matches(monkeypatch: pytest.MonkeyPatch):+ block = HackerNewsSearchBlock()++ async def fake_search(endpoint: str, params: dict[str, str]) -> dict[str, Any]:+ return {"hits": [], "nbHits": 0}++ monkeypatch.setattr(block, "_search", fake_search)+ assert await collect(block, query="zzzzqqqq") == [+ ("results", []),+ ("total_matches", 0),+ ]+++async def test_run_turns_api_errors_into_block_errors(monkeypatch: pytest.MonkeyPatch):+ block = HackerNewsSearchBlock()++ async def fake_search(endpoint: str, params: dict[str, str]) -> dict[str, Any]:+ raise HackerNewsError("Hacker News search (hn.algolia.com) had a problem")++ monkeypatch.setattr(block, "_search", fake_search)+ with pytest.raises(BlockExecutionError, match="had a problem"):+ await collect(block, query="x")+++async def collect(block: HackerNewsSearchBlock, **inputs: Any) -> list[tuple[str, Any]]:+ outputs: BlockOutput = block.run(block.Input.model_validate(inputs))+ return [output async for output in outputs]diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/stories.py b/autogpt_platform/backend/backend/blocks/hacker_news/stories.pynew file mode 100644index 0000000000..b213b2a7d0--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/stories.py@@ -0,0 +1,144 @@+from enum import Enum+from typing import Any++from backend.blocks._base import (+ Block,+ BlockCategory,+ BlockEffect,+ BlockOutput,+ BlockSchemaInput,+ BlockSchemaOutput,+)+from backend.data.model import SchemaField+from backend.util.exceptions import BlockExecutionError++from ._api import HackerNewsError, get_items, get_story_ids+from ._models import HackerNewsStory, story_from_firebase+++class StoryList(str, Enum):+ TOP = "top"+ NEW = "new"+ BEST = "best"+ ASK = "ask"+ SHOW = "show"+ JOBS = "jobs"+++_LIST_NAMES = {+ StoryList.TOP: "topstories",+ StoryList.NEW: "newstories",+ StoryList.BEST: "beststories",+ StoryList.ASK: "askstories",+ StoryList.SHOW: "showstories",+ StoryList.JOBS: "jobstories",+}++_TEST_STORY: dict[str, Any] = {+ "id": 49963366,+ "type": "story",+ "by": "builder42",+ "time": 1791201600,+ "title": "Show HN: Hacker News blocks for AutoGPT",+ "url": "https://agpt.co/blog/hacker-news-blocks",+ "score": 224,+ "descendants": 37,+ "kids": [49963489, 49963513],+}+_TEST_DEAD_STORY: dict[str, Any] = {"id": 49963400, "type": "story", "dead": True}+_TEST_JOB: dict[str, Any] = {+ "id": 49945904,+ "type": "job",+ "by": "acme_hiring",+ "time": 1791205800,+ "title": "Acme (YC W24) Is Hiring Engineers",+ "url": "https://www.ycombinator.com/companies/acme/jobs",+ "score": 1,+}+++class HackerNewsGetStoriesBlock(Block):+ """The stories on one of Hacker News's lists, such as the front page."""++ class Input(BlockSchemaInput):+ story_list: StoryList = SchemaField(+ description=(+ "Which list: top (the front page ranking), new, best (most upvoted "+ "lately), ask (Ask HN), show (Show HN) or jobs"+ ),+ default=StoryList.TOP,+ )+ max_results: int = SchemaField(+ description=(+ "Most stories to return, from the top of the list. Top, new and best "+ "hold up to 500 stories; ask, show and jobs up to 200."+ ),+ default=30,+ ge=1,+ le=500,+ )++ class Output(BlockSchemaOutput):+ stories: list[HackerNewsStory] = SchemaField(+ description="The stories, in the list's order"+ )+ story: HackerNewsStory = SchemaField(description="Each story")++ def __init__(self):+ story = story_from_firebase(_TEST_STORY, rank=1)+ job = story_from_firebase(_TEST_JOB, rank=3)+ super().__init__(+ id="dda11fa8-5d5f-4254-a4b6-1d23afffea3b",+ description=(+ "Get the top (front page), new, best, Ask HN, Show HN or job stories "+ "from Hacker News, in the order Hacker News ranks them. Each story "+ "comes with its rank, title, link, points, comment count and poster. "+ "Uses Hacker News's official API, which needs no account."+ ),+ categories={BlockCategory.SEARCH, BlockCategory.SOCIAL},+ input_schema=HackerNewsGetStoriesBlock.Input,+ output_schema=HackerNewsGetStoriesBlock.Output,+ test_input={"story_list": StoryList.TOP, "max_results": 3},+ test_output=[+ ("stories", [story, job]),+ ("story", story),+ ("story", job),+ ],+ test_mock={+ "_fetch_stories": lambda *args, **kwargs: [+ _TEST_STORY,+ _TEST_DEAD_STORY,+ _TEST_JOB,+ ]+ },+ effect=BlockEffect.READ,+ )++ async def run(self, input_data: Input, **kwargs) -> BlockOutput:+ try:+ items = await self._fetch_stories(+ input_data.story_list, input_data.max_results+ )+ except HackerNewsError as e:+ raise BlockExecutionError(+ message=str(e), block_name=self.name, block_id=self.id+ ) from e++ # Rank is the place on HN's list, so it skips the items left out here.+ stories = [+ story_from_firebase(item, rank)+ for rank, item in enumerate(items, start=1)+ if item and not item.get("deleted") and not item.get("dead")+ ]+ yield "stories", stories+ for story in stories:+ yield "story", story++ @staticmethod+ async def _fetch_stories(+ story_list: StoryList, limit: int+ ) -> list[dict[str, Any] | None]:+ """The first `limit` items on the list. The official API has one URL+ per item, so this is one request per story, 10 at a time."""+ story_ids = await get_story_ids(_LIST_NAMES[story_list])+ return await get_items(story_ids[:limit])diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/stories_test.py b/autogpt_platform/backend/backend/blocks/hacker_news/stories_test.pynew file mode 100644index 0000000000..b9f21b44b1--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/stories_test.py@@ -0,0 +1,121 @@+"""Unit tests for the Hacker News Get Stories block.++The block's own test_input/test_mock case mocks the API calls away; these+cover which list is read, how many items are fetched, the items left out, and+errors.+"""++from typing import Any++import pytest++from backend.blocks.hacker_news import stories+from backend.blocks.hacker_news._api import HackerNewsError+from backend.blocks.hacker_news.stories import HackerNewsGetStoriesBlock, StoryList+from backend.util.exceptions import BlockExecutionError+++def story(item_id: int, **extra: Any) -> dict[str, Any]:+ return {+ "id": item_id,+ "type": "story",+ "by": "pg",+ "title": f"Story {item_id}",+ "score": 10,+ "descendants": 2,+ "time": 1160418111,+ **extra,+ }+++ "story_list, list_name",+ [+ (StoryList.TOP, "topstories"),+ (StoryList.NEW, "newstories"),+ (StoryList.BEST, "beststories"),+ (StoryList.ASK, "askstories"),+ (StoryList.SHOW, "showstories"),+ (StoryList.JOBS, "jobstories"),+ ],+)+async def test_fetch_stories_reads_the_list_then_its_first_items(+ monkeypatch: pytest.MonkeyPatch, story_list: StoryList, list_name: str+):+ requested: list[Any] = []++ async def fake_story_ids(name: str) -> list[int]:+ requested.append(name)+ return [5, 4, 3, 2, 1]++ async def fake_items(item_ids: list[int]) -> list[dict[str, Any] | None]:+ requested.append(item_ids)+ return [story(item_id) for item_id in item_ids]++ monkeypatch.setattr(stories, "get_story_ids", fake_story_ids)+ monkeypatch.setattr(stories, "get_items", fake_items)+ items = await HackerNewsGetStoriesBlock._fetch_stories(story_list, 3)+ assert requested == [list_name, [5, 4, 3]]+ assert [item["id"] for item in items if item] == [5, 4, 3]+++async def test_stories_skip_missing_deleted_and_dead_items_but_keep_hn_rank(+ monkeypatch: pytest.MonkeyPatch,+):+ block = HackerNewsGetStoriesBlock()++ async def fake_fetch(story_list: StoryList, limit: int):+ assert (story_list, limit) == (StoryList.NEW, 6)+ return [+ story(10),+ None,+ story(12, deleted=True),+ story(13, dead=True),+ {"id": 14, "type": "job", "by": "acme", "title": "Hiring", "score": 1},+ story(15),+ ]++ monkeypatch.setattr(block, "_fetch_stories", fake_fetch)+ outputs = await collect(block, story_list="new", max_results=6)+ found = outputs[0][1]+ assert outputs[0][0] == "stories"+ assert [(s.id, s.rank, s.type) for s in found] == [+ (10, 1, "story"),+ (14, 5, "job"),+ (15, 6, "story"),+ ]+ assert found[1].points is None+ assert outputs[1:] == [("story", s) for s in found]+++async def test_an_empty_list(monkeypatch: pytest.MonkeyPatch):+ block = HackerNewsGetStoriesBlock()++ async def fake_fetch(story_list: StoryList, limit: int):+ return []++ monkeypatch.setattr(block, "_fetch_stories", fake_fetch)+ assert await collect(block, story_list="jobs") == [("stories", [])]+++async def test_stories_turn_api_errors_into_block_errors(+ monkeypatch: pytest.MonkeyPatch,+):+ block = HackerNewsGetStoriesBlock()++ async def fake_fetch(story_list: StoryList, limit: int):+ raise HackerNewsError("The Hacker News API had a temporary problem")++ monkeypatch.setattr(block, "_fetch_stories", fake_fetch)+ with pytest.raises(BlockExecutionError, match="temporary problem"):+ await collect(block)+++async def collect(+ block: HackerNewsGetStoriesBlock, **inputs: Any+) -> list[tuple[str, Any]]:+ return [output async for output in block.run(block.Input.model_validate(inputs))]diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/users.py b/autogpt_platform/backend/backend/blocks/hacker_news/users.pynew file mode 100644index 0000000000..a96b14e831--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/users.py@@ -0,0 +1,119 @@+from typing import Any++from backend.blocks._base import (+ Block,+ BlockCategory,+ BlockEffect,+ BlockOutput,+ BlockSchemaInput,+ BlockSchemaOutput,+)+from backend.data.model import SchemaField+from backend.util.exceptions import BlockExecutionError, BlockInputError++from ._api import HackerNewsError, get_user, parse_username, user_url+from ._html import html_to_text+from ._models import iso_time++_TEST_USER: dict[str, Any] = {+ "id": "builder42",+ "created": 1160418092,+ "karma": 1337,+ "about": (+ "Builds agents.<p>Blog: "+ '<a href="https://agpt.co">https://agpt.co</a>'+ ),+ "submitted": [49900215, 49900001, 49871234],+}+++class HackerNewsGetUserBlock(Block):+ """A Hacker News user's public profile."""++ class Input(BlockSchemaInput):+ username: str = SchemaField(+ description=(+ "The username, such as pg (case-sensitive), or a profile link such "+ "as https://news.ycombinator.com/user?id=pg"+ ),+ placeholder="e.g. pg",+ )++ class Output(BlockSchemaOutput):+ username: str = SchemaField(+ description="The username, spelled the way Hacker News has it"+ )+ karma: int = SchemaField(description="The user's karma")+ created_at: str = SchemaField(+ description="When the account was created, in ISO 8601 (UTC)"+ )+ about: str = SchemaField(+ description="The user's about text as plain text; empty if they wrote none"+ )+ submission_count: int = SchemaField(+ description="How many stories, comments and polls the user has posted"+ )+ profile_url: str = SchemaField(description="Link to the profile on Hacker News")++ def __init__(self):+ super().__init__(+ id="7c78fe61-29e0-45fe-9613-888ec40ee349",+ description=(+ "Get a Hacker News user's profile by username: karma, when the "+ "account was created, the about text and how many items they have "+ "posted. Uses Hacker News's official API, which needs no account."+ ),+ categories={BlockCategory.SEARCH, BlockCategory.SOCIAL},+ input_schema=HackerNewsGetUserBlock.Input,+ output_schema=HackerNewsGetUserBlock.Output,+ test_input={"username": "https://news.ycombinator.com/user?id=builder42"},+ test_output=[+ ("username", "builder42"),+ ("karma", 1337),+ ("created_at", "2006-10-09T18:21:32Z"),+ ("about", "Builds agents.\n\nBlog: https://agpt.co"),+ ("submission_count", 3),+ ("profile_url", "https://news.ycombinator.com/user?id=builder42"),+ ],+ test_mock={"_fetch_user": lambda *args, **kwargs: _TEST_USER},+ effect=BlockEffect.READ,+ )++ async def run(self, input_data: Input, **kwargs) -> BlockOutput:+ username = parse_username(input_data.username)+ if not username:+ raise BlockInputError(+ message=(+ f"'{input_data.username.strip()}' isn't a Hacker News username. "+ "Usernames have only letters, digits, - and _."+ ),+ block_name=self.name,+ block_id=self.id,+ )+ try:+ user = await self._fetch_user(username)+ except HackerNewsError as e:+ raise BlockExecutionError(+ message=str(e), block_name=self.name, block_id=self.id+ ) from e+ if user is None:+ raise BlockExecutionError(+ message=(+ f"Hacker News has no user named '{username}'. Usernames are "+ "case-sensitive, so check the capitals."+ ),+ block_name=self.name,+ block_id=self.id,+ )++ yield "username", user.get("id") or username+ yield "karma", user.get("karma") or 0+ if created_at := iso_time(user.get("created")):+ yield "created_at", created_at+ yield "about", html_to_text(user.get("about"))+ yield "submission_count", len(user.get("submitted") or [])+ yield "profile_url", user_url(user.get("id") or username)++ @staticmethod+ async def _fetch_user(username: str) -> dict[str, Any] | None:+ return await get_user(username)diff --git a/autogpt_platform/backend/backend/blocks/hacker_news/users_test.py b/autogpt_platform/backend/backend/blocks/hacker_news/users_test.pynew file mode 100644index 0000000000..af96c025c3--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/hacker_news/users_test.py@@ -0,0 +1,126 @@+"""Unit tests for the Hacker News Get User block.++The block's own test_input/test_mock case mocks the API call away; these+cover profiles with missing fields, unknown users and bad input.+"""++from typing import Any++import pytest++from backend.blocks.hacker_news import users+from backend.blocks.hacker_news._api import HackerNewsError+from backend.blocks.hacker_news.users import HackerNewsGetUserBlock+from backend.util.exceptions import BlockExecutionError, BlockInputError+++def block() -> HackerNewsGetUserBlock:+ return HackerNewsGetUserBlock()+++async def test_fetch_user_asks_for_the_username(monkeypatch: pytest.MonkeyPatch):+ asked: list[str] = []++ async def fake_get_user(username: str) -> dict[str, Any] | None:+ asked.append(username)+ return {"id": username}++ monkeypatch.setattr(users, "get_user", fake_get_user)+ assert await HackerNewsGetUserBlock._fetch_user("dang") == {"id": "dang"}+ assert asked == ["dang"]+++async def test_profile(monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetUserBlock):+ asked = mock_fetch(+ monkeypatch,+ block,+ {+ "id": "pg",+ "created": 1160418092,+ "karma": 157316,+ "about": "Bug fixer.",+ "submitted": [1, 2, 3, 4],+ },+ )+ assert await collect(block, username=" pg ") == [+ ("username", "pg"),+ ("karma", 157316),+ ("created_at", "2006-10-09T18:21:32Z"),+ ("about", "Bug fixer."),+ ("submission_count", 4),+ ("profile_url", "https://news.ycombinator.com/user?id=pg"),+ ]+ assert asked == ["pg"]+++async def test_profile_with_nothing_but_an_id(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetUserBlock+):+ mock_fetch(monkeypatch, block, {"id": "quiet"})+ assert dict(await collect(block, username="quiet")) == {+ "username": "quiet",+ "karma": 0,+ "about": "",+ "submission_count": 0,+ "profile_url": "https://news.ycombinator.com/user?id=quiet",+ }+++async def test_unknown_user(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetUserBlock+):+ mock_fetch(monkeypatch, block, None)+ with pytest.raises(BlockExecutionError, match="no user named 'PG'.*case-sensitive"):+ await collect(block, username="PG")+++ "text", ["", "pg dang", "../item/1", "https://example.com/user?id=pg"]+)+async def test_input_that_is_not_a_username(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetUserBlock, text: str+):+ asked = mock_fetch(monkeypatch, block, {"id": "x"})+ with pytest.raises(BlockInputError, match="isn't a Hacker News username"):+ await collect(block, username=text)+ assert asked == []+++async def test_api_errors_become_block_errors(+ monkeypatch: pytest.MonkeyPatch, block: HackerNewsGetUserBlock+):+ async def fake_fetch(username: str):+ raise HackerNewsError("The Hacker News API had a temporary problem")++ monkeypatch.setattr(block, "_fetch_user", fake_fetch)+ with pytest.raises(BlockExecutionError, match="temporary problem"):+ await collect(block, username="pg")+++def mock_fetch(+ monkeypatch: pytest.MonkeyPatch,+ block: HackerNewsGetUserBlock,+ user: dict[str, Any] | None,+) -> list[str]:+ """Make the block's _fetch_user return `user`; returns the usernames it gets."""+ asked: list[str] = []++ async def fake_fetch(username: str):+ asked.append(username)+ return user++ monkeypatch.setattr(block, "_fetch_user", fake_fetch)+ return asked+++async def collect(+ block: HackerNewsGetUserBlock, **inputs: Any+) -> list[tuple[str, Any]]:+ return [output async for output in block.run(block.Input.model_validate(inputs))]diff --git a/docs/integrations/README.md b/docs/integrations/README.mdindex 7c4df3e4ed..b00b20058b 100644--- a/docs/integrations/README.md+++ b/docs/integrations/README.md@@ -336,6 +336,10 @@ Below is a comprehensive list of all available blocks, categorized by their prim| [Get Person Detail](block-integrations/apollo/person.md#get-person-detail) | Get detailed person data with Apollo API, including email reveal || [Get Wikipedia Summary](block-integrations/search.md#get-wikipedia-summary) | This block fetches the summary of a given topic from Wikipedia || [Google Maps Search](block-integrations/search.md#google-maps-search) | This block searches for local businesses using Google Maps API |+| [Hacker News Get Item](block-integrations/hacker_news/items.md#hacker-news-get-item) | Get a Hacker News story, comment, poll or job by its id or link, with its comments as a list in reading order |+| [Hacker News Get Stories](block-integrations/hacker_news/stories.md#hacker-news-get-stories) | Get the top (front page), new, best, Ask HN, Show HN or job stories from Hacker News, in the order Hacker News ranks them |+| [Hacker News Get User](block-integrations/hacker_news/users.md#hacker-news-get-user) | Get a Hacker News user's profile by username: karma, when the account was created, the about text and how many items they have posted |+| [Hacker News Search](block-integrations/hacker_news/search.md#hacker-news-search) | Search Hacker News stories and comments by keyword, author, date, points or linked domain, sorted by relevance or newest first || [Search Filter Typeahead](block-integrations/dataforb2b/typeahead.md#search-filter-typeahead) | Resolve the exact filter value (company, industry, job title, skill, school, investor, location, category) for people and company searches with DataForB2B || [Search Organizations](block-integrations/apollo/organization.md#search-organizations) | Search for organizations in Apollo || [Search People](block-integrations/apollo/people.md#search-people) | Search for people in Apollo |diff --git a/docs/integrations/SUMMARY.md b/docs/integrations/SUMMARY.mdindex febb81c0fa..7054b95351 100644--- a/docs/integrations/SUMMARY.md+++ b/docs/integrations/SUMMARY.md@@ -108,6 +108,10 @@* [Google Docs](block-integrations/google/docs.md)* [Google Gmail](block-integrations/google/gmail.md)* [Google Sheets](block-integrations/google/sheets.md)+* [Hacker News Items](block-integrations/hacker_news/items.md)+* [Hacker News Search](block-integrations/hacker_news/search.md)+* [Hacker News Stories](block-integrations/hacker_news/stories.md)+* [Hacker News Users](block-integrations/hacker_news/users.md)* [HubSpot Company](block-integrations/hubspot/company.md)* [HubSpot Contact](block-integrations/hubspot/contact.md)* [HubSpot Engagement](block-integrations/hubspot/engagement.md)diff --git a/docs/integrations/block-integrations/hacker_news/items.md b/docs/integrations/block-integrations/hacker_news/items.mdnew file mode 100644index 0000000000..c007fe2a1d--- /dev/null+++ b/docs/integrations/block-integrations/hacker_news/items.md@@ -0,0 +1,45 @@+# Hacker News Items+<!-- MANUAL: file_description -->+Read one Hacker News story, comment, poll or job with its comment thread. The block gets the thread from HN Search (hn.algolia.com), and the live points, comment count and ranking of the replies from Hacker News's official API (hacker-news.firebaseio.com). Neither needs an account or API key.+<!-- END MANUAL -->++## Hacker News Get Item++### What it is+Get a Hacker News story, comment, poll or job by its id or link, with its comments as a list in reading order. Each comment is followed by its replies, and direct replies come in Hacker News's ranking.++### How it works+<!-- MANUAL: how_it_works -->+Takes an id such as `8863` or a link such as `https://news.ycombinator.com/item?id=8863`, and makes two requests at once: HN Search's `items` endpoint, which returns the item and its whole comment tree in one call, and the official API's `item` endpoint. HN Search keeps replies oldest first, not in the order Hacker News shows them, so the block puts the item's direct replies in the order the official API ranks them. Deeper replies stay oldest first. Each comment is followed by its replies, and `depth` is 1 for a direct reply. Deleted comments are left out but their replies are kept, with `parent_id` still pointing at the deleted comment. `max_comments` (up to 1,000) cuts the list, while `comment_count` counts every comment in the thread. With `include_comments` off you get only the item and its count.++HN Search adds new items and comments a minute or two after they are posted, and doesn't keep killed (dead) ones. So `points`, `num_comments` and `comment_count` come from the official API, which is live: the count is Hacker News's own, or the number of comments found if that is higher. The newest comments can be missing from the list for a minute or two while the count already includes them. When HN Search doesn't have the item at all, the block reads it from the official API and fetches up to `max_comments` comments one request each, 10 at a time, in Hacker News's order at every level. A comment has no `title` or `url`; `story_id` names its story. A poll comes back without its options. Text comes back as plain text, and times are in UTC.+<!-- END MANUAL -->++### Inputs++| Input | Description | Type | Required |+|-------|-------------|------|----------|+| item | The item's id, such as 8863, or its link, such as https://news.ycombinator.com/item?id=8863 | str | Yes |+| include_comments | Also return the item's comments | bool | No |+| max_comments | Most comments to return, in reading order. comment_count still counts them all. | int | No |++### Outputs++| Output | Description | Type |+|--------|-------------|------|+| error | Error message if the operation failed | str |+| item | The story, comment, poll or job | HackerNewsItem |+| comments | Its comments in reading order: each comment followed by its replies, with direct replies in Hacker News's ranking | List[HackerNewsComment] |+| comment | Each comment | HackerNewsComment |+| comment_count | How many comments the item has, including any that max_comments left out. Hacker News's own count when it is higher. | int |++### Possible use case+<!-- MANUAL: use_case -->+**Discussion Summary**: Read the comments on a launch post and have an AI block sum up what people liked and what they asked for.++**Reply Tracking**: Check a comment you posted for new replies.++**Story Details**: Feed ids from Hacker News Search or Hacker News Get Stories into this block to read each story's text and top comments.+<!-- END MANUAL -->++---diff --git a/docs/integrations/block-integrations/hacker_news/search.md b/docs/integrations/block-integrations/hacker_news/search.mdnew file mode 100644index 0000000000..9bc9539301--- /dev/null+++ b/docs/integrations/block-integrations/hacker_news/search.md@@ -0,0 +1,52 @@+# Hacker News Search+<!-- MANUAL: file_description -->+Search Hacker News stories and comments through HN Search (hn.algolia.com), the public search API that Algolia runs for Hacker News. It needs no account or API key. HN Search allows 10,000 requests an hour from one IP address, and every agent running on the same server shares that allowance.+<!-- END MANUAL -->++## Hacker News Search++### What it is+Search Hacker News stories and comments by keyword, author, date, points or linked domain, sorted by relevance or newest first. Use it to find mentions of a product, links to a website, or the most upvoted stories of the week. Uses HN Search by Algolia, which needs no account. exact_match is on by default, so words that are only spelled alike are left out; turn it off to also catch misspellings.++### How it works+<!-- MANUAL: how_it_works -->+Calls HN Search's `search` endpoint, or `search_by_date` when `sort_by` is newest, and gets up to 1,000 results in one request. Relevance puts the best matches first, then the stories with the most points, then the most comments. With an empty query everything matches equally, so relevance sorts by points. `search_for` becomes HN Search's `tags` filter (`story`, `comment`, `(story,comment)`, `show_hn`, `ask_hn` or `front_page`), plus `author_<username>` when `author` is set. The time inputs and the minimums become `numericFilters` on `created_at_i`, `points` and `num_comments`. Only stories have points and comment counts, so a minimum above 0 leaves comments out. `match_in` limits matching to a story's `title` or `url`. Comments have neither, so the block refuses that combination instead of returning nothing.++HN Search allows typos unless told not to. Then `autogpt` also matches `automotive` and `automation`, hundreds of thousands of comments instead of a few hundred, and sorted newest first those near misses take over. So `exact_match` is on by default and turns typo tolerance off. Plurals still match, and the last word still matches the start of longer words (`agent` finds `agents` and `agentic`). Turn `exact_match` off to also catch misspellings. Quotes around a phrase and a `-` before a word work too. `total_matches` is HN Search's count of every match, even past the 1,000 it returns. Text comes back as plain text: paragraphs become blank lines, italics become `*asterisks*`, code keeps its indentation, and links show their full URL. HN Search adds new stories and comments a minute or two after they are posted. Times are in UTC.+<!-- END MANUAL -->++### Inputs++| Input | Description | Type | Required |+|-------|-------------|------|----------|+| query | Words to search for. Put a phrase in double quotes to match it as a phrase, and put - before a word to leave out items that have it. Leave empty to match everything, e.g. for the top stories of the past week. | str | No |+| search_for | What to search: stories, comments, stories_and_comments, show_hn or ask_hn posts, or front_page (only the stories on the front page now) | "stories" \| "comments" \| "stories_and_comments" \| "show_hn" \| "ask_hn" \| "front_page" | No |+| match_in | Where the query must match: anywhere (title, link, text and author), title, or url (the link a story points to, e.g. to find links to a domain). title and url only match stories. | "anywhere" \| "title" \| "url" | No |+| exact_match | Match the query words as written, without typo tolerance. On by default, because typo tolerance makes 'autogpt' also match 'automotive' and 'automation'. Plurals still match. Turn it off to also catch misspellings. | bool | No |+| sort_by | relevance: best matches first, then most points, then most comments. newest: newest first. | "relevance" \| "newest" | No |+| created_after | Only items posted at or after this time: an ISO 8601 date or time (2026-09-01, 2026-09-01T12:00:00Z; UTC unless it has an offset), or an age such as 24h, 7d or 2w | str | No |+| created_before | Only items posted before this time, written the same way as created_after | str | No |+| author | Only items posted by this Hacker News username (case-sensitive) | str | No |+| min_points | Only stories with at least this many points. Comments have no points, so any value above 0 leaves comments out. | int | No |+| min_comments | Only stories with at least this many comments. Any value above 0 leaves comments out. | int | No |+| max_results | Most results to return (up to 1,000) | int | No |++### Outputs++| Output | Description | Type |+|--------|-------------|------|+| error | Error message if the operation failed | str |+| results | Matching stories and comments, in the chosen order | List[HackerNewsItem] |+| result | Each matching story or comment | HackerNewsItem |+| total_matches | How many items match in all. Can be more than the results returned: HN Search returns at most 1,000. | int |++### Possible use case+<!-- MANUAL: use_case -->+**Brand Monitoring**: Every morning, search stories and comments from the past day for your product's name, newest first, and post new mentions to Slack.++**Who Links to Us**: Set `match_in` to `url` and search for your domain to find every story that links to your site, with its points and comment count.++**Weekly Digest**: Leave the query empty, set `created_after` to `7d` and sort by relevance to get the most upvoted stories of the week.+<!-- END MANUAL -->++---diff --git a/docs/integrations/block-integrations/hacker_news/stories.md b/docs/integrations/block-integrations/hacker_news/stories.mdnew file mode 100644index 0000000000..3bc62390f9--- /dev/null+++ b/docs/integrations/block-integrations/hacker_news/stories.md@@ -0,0 +1,42 @@+# Hacker News Stories+<!-- MANUAL: file_description -->+Get Hacker News's live story lists: the front page, new, best, Ask HN, Show HN and jobs. The block calls Hacker News's official API (hacker-news.firebaseio.com), which needs no account or API key and publishes no rate limit.+<!-- END MANUAL -->++## Hacker News Get Stories++### What it is+Get the top (front page), new, best, Ask HN, Show HN or job stories from Hacker News, in the order Hacker News ranks them. Each story comes with its rank, title, link, points, comment count and poster. Uses Hacker News's official API, which needs no account.++### How it works+<!-- MANUAL: how_it_works -->+Reads the list's ids from the official API (`topstories`, `newstories`, `beststories`, `askstories`, `showstories` or `jobstories`), keeps the first `max_results` and fetches each story, 10 at a time. The official API returns one item per request, so 500 stories take 501 requests and a few seconds. Stories stay in the list's order, and `rank` is each story's place on the list. Stories deleted or killed since the list was made are left out, so a run can return a few fewer than `max_results`, and the ranks skip their places.++Top, new and best hold up to 500 stories; Ask HN, Show HN and jobs up to 200, and often far fewer. Jobs have no points or comments. The text of Ask HN and other text posts comes back as plain text, and times are in UTC.+<!-- END MANUAL -->++### Inputs++| Input | Description | Type | Required |+|-------|-------------|------|----------|+| story_list | Which list: top (the front page ranking), new, best (most upvoted lately), ask (Ask HN), show (Show HN) or jobs | "top" \| "new" \| "best" \| "ask" \| "show" \| "jobs" | No |+| max_results | Most stories to return, from the top of the list. Top, new and best hold up to 500 stories; ask, show and jobs up to 200. | int | No |++### Outputs++| Output | Description | Type |+|--------|-------------|------|+| error | Error message if the operation failed | str |+| stories | The stories, in the list's order | List[HackerNewsStory] |+| story | Each story | HackerNewsStory |++### Possible use case+<!-- MANUAL: use_case -->+**Front Page Digest**: Get the top 30 stories every morning and send a summary to Slack or email.++**Launch Watch**: Check the Show HN list for launches in your market and alert the team.++**Hiring Signals**: Read the jobs list to see which YC companies are hiring, and for what roles.+<!-- END MANUAL -->++---diff --git a/docs/integrations/block-integrations/hacker_news/users.md b/docs/integrations/block-integrations/hacker_news/users.mdnew file mode 100644index 0000000000..3a44bb913b--- /dev/null+++ b/docs/integrations/block-integrations/hacker_news/users.md@@ -0,0 +1,45 @@+# Hacker News Users+<!-- MANUAL: file_description -->+Read a Hacker News user's public profile from Hacker News's official API (hacker-news.firebaseio.com), which needs no account or API key.+<!-- END MANUAL -->++## Hacker News Get User++### What it is+Get a Hacker News user's profile by username: karma, when the account was created, the about text and how many items they have posted. Uses Hacker News's official API, which needs no account.++### How it works+<!-- MANUAL: how_it_works -->+Calls the official API's `user` endpoint. Usernames are case-sensitive (`pg` exists, `PG` doesn't), and a profile link such as `https://news.ycombinator.com/user?id=pg` works too. The API answers `null` for a user that doesn't exist, and the block turns that into an error.++`about` comes back as plain text, with paragraphs as blank lines and links as full URLs. `submission_count` counts the stories, comments and polls on the user's submission list. The profile holds only what Hacker News shows publicly.+<!-- END MANUAL -->++### Inputs++| Input | Description | Type | Required |+|-------|-------------|------|----------|+| username | The username, such as pg (case-sensitive), or a profile link such as https://news.ycombinator.com/user?id=pg | str | Yes |++### Outputs++| Output | Description | Type |+|--------|-------------|------|+| error | Error message if the operation failed | str |+| username | The username, spelled the way Hacker News has it | str |+| karma | The user's karma | int |+| created_at | When the account was created, in ISO 8601 (UTC) | str |+| about | The user's about text as plain text; empty if they wrote none | str |+| submission_count | How many stories, comments and polls the user has posted | int |+| profile_url | Link to the profile on Hacker News | str |++### Possible use case+<!-- MANUAL: use_case -->+**Lead Context**: Before replying to someone who mentioned your product, check their karma and what they say about themselves.++**Expert Finder**: Look up the authors of the best comments on a topic to see who they are.++**New Account Check**: Flag mentions that come from accounts created in the last few days.+<!-- END MANUAL -->++---