Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

20.3 KB
Raw Blame
"""
Bedrock Agent Runtime service parity tests — all 31 ops covered.

boto3 round-trip on every operation family. Verified shapes against botocore
bedrock-agent-runtime-2023-07-26. Streaming ops drive boto3's EventStream
parser to validate eventstream framing + event sequence.
"""

import json
import uuid as _uuid_mod

import botocore.exceptions
import pytest
from conftest import make_client


def _ar():
    return make_client("bedrock-agent-runtime")


# ---------------------------------------------------------------------------
# Retrieve / RetrieveAndGenerate / Rerank / GenerateQuery / OptimizePrompt
# ---------------------------------------------------------------------------


def _make_kb(name):
    return make_client("bedrock-agent").create_knowledge_base(
        name=name,
        roleArn="arn:aws:iam::000000000000:role/kb-role",
        knowledgeBaseConfiguration={
            "type": "VECTOR",
            "vectorKnowledgeBaseConfiguration": {
                "embeddingModelArn":
                    "arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0",
            },
        },
        storageConfiguration={
            "type": "OPENSEARCH_SERVERLESS",
            "opensearchServerlessConfiguration": {
                "collectionArn": "arn:aws:aoss:us-east-1:000000000000:collection/c",
                "vectorIndexName": "idx",
                "fieldMapping": {"vectorField": "v", "textField": "t",
                                 "metadataField": "m"},
            },
        },
    )["knowledgeBase"]


def test_bedrock_ar_retrieve_returns_results_envelope():
    kb = _make_kb("kb-ar-envelope")
    resp = _ar().retrieve(
        knowledgeBaseId=kb["knowledgeBaseId"],
        retrievalQuery={"text": "what is AWS?"},
    )
    assert "retrievalResults" in resp
    assert isinstance(resp["retrievalResults"], list)


def test_bedrock_ar_retrieve_unknown_kb_is_not_found():
    try:
        _ar().retrieve(knowledgeBaseId="KBNOPE1234",
                       retrievalQuery={"text": "anything"})
        raise AssertionError("expected ResourceNotFoundException")
    except botocore.exceptions.ClientError as e:
        assert e.response["Error"]["Code"] == "ResourceNotFoundException"


def test_bedrock_ar_retrieve_returns_ingested_documents():
    s3 = make_client("s3")
    s3.create_bucket(Bucket="kb-ar-docs")
    s3.put_object(Bucket="kb-ar-docs", Key="ports.txt",
                  Body=b"MiniStack serves every service on port 4566.")
    s3.put_object(Bucket="kb-ar-docs", Key="pets.txt",
                  Body=b"Cats sleep most of the day.")
    kb = _make_kb("kb-ar-retrieve")
    agent_api = make_client("bedrock-agent")
    ds = agent_api.create_data_source(
        knowledgeBaseId=kb["knowledgeBaseId"], name="ds-ar",
        dataSourceConfiguration={"type": "S3", "s3Configuration": {
            "bucketArn": "arn:aws:s3:::kb-ar-docs"}},
    )["dataSource"]
    agent_api.start_ingestion_job(
        knowledgeBaseId=kb["knowledgeBaseId"], dataSourceId=ds["dataSourceId"])

    resp = _ar().retrieve(
        knowledgeBaseId=kb["knowledgeBaseId"],
        retrievalQuery={"text": "which port does ministack serve on"},
    )
    results = resp["retrievalResults"]
    assert results
    top = results[0]
    assert "4566" in top["content"]["text"]
    assert top["location"]["s3Location"]["uri"] == "s3://kb-ar-docs/ports.txt"
    assert top["location"]["type"] == "S3"
    assert 0 < top["score"] <= 1


def test_bedrock_ar_retrieve_validation_missing_query():
    try:
        _ar().retrieve(knowledgeBaseId="kb-x", retrievalQuery={})
    except botocore.exceptions.ParamValidationError:
        return
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ValidationException"
    else:
        raise AssertionError("expected validation")


def test_bedrock_ar_retrieve_and_generate_returns_output_and_session():
    resp = _ar().retrieve_and_generate(
        input={"text": "what is ministack?"},
        retrieveAndGenerateConfiguration={
            "type": "KNOWLEDGE_BASE",
            "knowledgeBaseConfiguration": {
                "knowledgeBaseId": "kb-1",
                "modelArn": "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-3-haiku-20240307-v1:0",
            },
        },
    )
    assert "output" in resp
    assert "text" in resp["output"]
    assert "sessionId" in resp
    assert "citations" in resp


def test_bedrock_ar_retrieve_and_generate_stream():
    resp = _ar().retrieve_and_generate_stream(
        input={"text": "stream this"},
        retrieveAndGenerateConfiguration={
            "type": "KNOWLEDGE_BASE",
            "knowledgeBaseConfiguration": {
                "knowledgeBaseId": "kb-1",
                "modelArn": "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-3-haiku-20240307-v1:0",
            },
        },
    )
    events = list(resp["stream"])
    assert any("output" in e for e in events)


def test_bedrock_ar_rerank():
    resp = _ar().rerank(
        queries=[{"type": "TEXT", "textQuery": {"text": "what is ministack?"}}],
        sources=[
            {"type": "INLINE", "inlineDocumentSource": {"type": "TEXT", "textDocument": {"text": "ministack is an AWS emulator"}}},
            {"type": "INLINE", "inlineDocumentSource": {"type": "TEXT", "textDocument": {"text": "unrelated content"}}},
        ],
        rerankingConfiguration={
            "type": "BEDROCK_RERANKING_MODEL",
            "bedrockRerankingConfiguration": {
                "modelConfiguration": {
                    "modelArn": "arn:aws:bedrock:us-east-1::foundation-model/cohere.command-r-v1:0",
                },
            },
        },
    )
    assert "results" in resp
    assert len(resp["results"]) == 2


def test_bedrock_ar_generate_query():
    resp = _ar().generate_query(
        queryGenerationInput={"type": "TEXT", "text": "find all users"},
        transformationConfiguration={
            "mode": "TEXT_TO_SQL",
            "textToSqlConfiguration": {
                "type": "KNOWLEDGE_BASE",
                "knowledgeBaseConfiguration": {"knowledgeBaseArn":
                                                  "arn:aws:bedrock:us-east-1:000000000000:knowledge-base/kb"},
            },
        },
    )
    assert "queries" in resp


# ---------------------------------------------------------------------------
# Sessions + invocations
# ---------------------------------------------------------------------------


def test_bedrock_ar_session_lifecycle():
    create = _ar().create_session(
        sessionMetadata={"userId": "u-1"},
    )
    sid = create["sessionId"]
    assert ":session/" in create["sessionArn"]
    assert create["sessionStatus"] == "ACTIVE"

    g = _ar().get_session(sessionIdentifier=sid)
    assert g["sessionId"] == sid
    assert g["sessionMetadata"]["userId"] == "u-1"

    _ar().update_session(
        sessionIdentifier=sid,
        sessionMetadata={"userId": "u-2"},
    )
    g2 = _ar().get_session(sessionIdentifier=sid)
    assert g2["sessionMetadata"]["userId"] == "u-2"

    end = _ar().end_session(sessionIdentifier=sid)
    assert end["sessionStatus"] == "ENDED"

    _ar().delete_session(sessionIdentifier=sid)
    try:
        _ar().get_session(sessionIdentifier=sid)
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ResourceNotFoundException"


def test_bedrock_ar_list_sessions():
    _ar().create_session(sessionMetadata={"x": "1"})
    _ar().create_session(sessionMetadata={"x": "2"})
    resp = _ar().list_sessions()
    assert len(resp["sessionSummaries"]) >= 2


def test_bedrock_ar_invocation_steps_lifecycle():
    session = _ar().create_session()
    sid = session["sessionId"]
    inv = _ar().create_invocation(
        sessionIdentifier=sid, description="step test",
    )
    iid = inv["invocationId"]
    # Put step
    _ar().put_invocation_step(
        sessionIdentifier=sid,
        invocationIdentifier=iid,
        invocationStepTime="2026-06-05T00:00:00Z",
        payload={"contentBlocks": [{"text": "step payload"}]},
    )
    # List steps
    lst = _ar().list_invocation_steps(
        sessionIdentifier=sid, invocationIdentifier=iid,
    )
    assert len(lst["invocationStepSummaries"]) >= 1


# ---------------------------------------------------------------------------
# Flow runtime
# ---------------------------------------------------------------------------


def test_bedrock_ar_invoke_flow_streams_output_and_completion():
    resp = _ar().invoke_flow(
        flowIdentifier="FL-1",
        flowAliasIdentifier="FA-1",
        inputs=[{
            "content": {"document": "hello"},
            "nodeName": "InputNode",
            "nodeOutputName": "document",
        }],
    )
    events = list(resp["responseStream"])
    # Should include flowOutputEvent and flowCompletionEvent
    names = []
    for e in events:
        if isinstance(e, dict):
            names.extend(e.keys())
    assert "flowOutputEvent" in names
    assert "flowCompletionEvent" in names


def test_bedrock_ar_start_get_list_stop_flow_execution():
    create = _ar().start_flow_execution(
        flowIdentifier="FL-1",
        flowAliasIdentifier="FA-1",
        inputs=[{
            "content": {"document": "input"},
            "nodeName": "InputNode",
            "nodeOutputName": "document",
        }],
    )
    arn = create["executionArn"]
    exec_id = arn.rsplit("/", 1)[-1]
    g = _ar().get_flow_execution(
        flowIdentifier="FL-1", flowAliasIdentifier="FA-1",
        executionIdentifier=exec_id,
    )
    assert g["executionArn"] == arn
    lst = _ar().list_flow_executions(flowIdentifier="FL-1")
    assert any(e["executionArn"] == arn for e in lst["flowExecutionSummaries"])
    stop = _ar().stop_flow_execution(
        flowIdentifier="FL-1", flowAliasIdentifier="FA-1",
        executionIdentifier=exec_id,
    )
    assert stop["status"] == "Aborted"


def test_bedrock_ar_list_flow_execution_events():
    create = _ar().start_flow_execution(
        flowIdentifier="FL-1", flowAliasIdentifier="FA-1",
        inputs=[{"content": {"document": "i"}, "nodeName": "N", "nodeOutputName": "document"}],
    )
    exec_id = create["executionArn"].rsplit("/", 1)[-1]
    resp = _ar().list_flow_execution_events(
        flowIdentifier="FL-1", flowAliasIdentifier="FA-1",
        executionIdentifier=exec_id, eventType="Node",
    )
    assert "flowExecutionEvents" in resp


def test_bedrock_ar_get_execution_flow_snapshot():
    create = _ar().start_flow_execution(
        flowIdentifier="FL-1", flowAliasIdentifier="FA-1",
        inputs=[{"content": {"document": "i"}, "nodeName": "N", "nodeOutputName": "document"}],
    )
    exec_id = create["executionArn"].rsplit("/", 1)[-1]
    resp = _ar().get_execution_flow_snapshot(
        flowIdentifier="FL-1", flowAliasIdentifier="FA-1",
        executionIdentifier=exec_id,
    )
    assert resp["flowIdentifier"] == "FL-1"


# ---------------------------------------------------------------------------
# InvokeAgent (eventstream)
# ---------------------------------------------------------------------------


def test_bedrock_ar_invoke_agent_returns_chunk_eventstream():
    resp = _ar().invoke_agent(
        agentId="AG-1",
        agentAliasId="AL-1",
        sessionId="sess-1",
        inputText="hello agent",
    )
    events = list(resp["completion"])
    names = []
    for e in events:
        if isinstance(e, dict):
            names.extend(e.keys())
    assert "chunk" in names


def test_bedrock_ar_invoke_agent_emits_no_fabricated_trace():
    # No orchestration happens locally, so enableTrace must not produce an
    # ORCHESTRATE trace describing reasoning that never ran.
    resp = _ar().invoke_agent(
        agentId="AG-2",
        agentAliasId="AL-2",
        sessionId="sess-trace",
        inputText="what is the weather",
        enableTrace=True,
    )
    names = []
    for e in resp["completion"]:
        if isinstance(e, dict):
            names.extend(e.keys())
    assert "chunk" in names
    assert "trace" not in names


def test_bedrock_ar_get_and_delete_agent_memory():
    resp = _ar().get_agent_memory(
        agentId="AG-1", agentAliasId="AL-1",
        memoryId="mem-1", memoryType="SESSION_SUMMARY",
    )
    assert "memoryContents" in resp
    _ar().delete_agent_memory(agentId="AG-1", agentAliasId="AL-1")


# ---------------------------------------------------------------------------
# Tags
# ---------------------------------------------------------------------------


def test_bedrock_ar_tag_resource():
    session = _ar().create_session()
    arn = session["sessionArn"]
    _ar().tag_resource(resourceArn=arn, tags={"env": "prod"})
    lst = _ar().list_tags_for_resource(resourceArn=arn)
    assert lst["tags"]["env"] == "prod"
    _ar().untag_resource(resourceArn=arn, tagKeys=["env"])
    lst2 = _ar().list_tags_for_resource(resourceArn=arn)
    assert "env" not in lst2["tags"]


def test_bedrock_ar_tag_resource_rejects_malformed_arn():
    try:
        _ar().tag_resource(resourceArn="not-an-arn-but-long-enough", tags={"env": "prod"})
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ValidationException"
    else:
        raise AssertionError("expected ValidationException")


def test_bedrock_ar_tag_resource_rejects_malformed_session_arn():
    arn = "arn:aws:bedrock:us-east-1:000000000000:session"
    try:
        _ar().tag_resource(resourceArn=arn, tags={"env": "prod"})
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ValidationException"
    else:
        raise AssertionError("expected ValidationException")


def test_bedrock_ar_tag_resource_rejects_wrong_scope_arn():
    session = _ar().create_session()
    wrong_region = session["sessionArn"].replace(":us-east-1:", ":us-west-2:")
    try:
        _ar().tag_resource(resourceArn=wrong_region, tags={"env": "prod"})
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ResourceNotFoundException"
    else:
        raise AssertionError("expected ResourceNotFoundException")


def test_bedrock_ar_tag_resource_rejects_noncanonical_partition_arn():
    session = _ar().create_session()
    wrong_partition = session["sessionArn"].replace("arn:aws:", "arn:aws-cn:", 1)
    try:
        _ar().tag_resource(resourceArn=wrong_partition, tags={"env": "prod"})
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ResourceNotFoundException"
    else:
        raise AssertionError("expected ResourceNotFoundException")


def test_bedrock_ar_tag_resource_rejects_wrong_service_arn():
    session = _ar().create_session()
    wrong_service = session["sessionArn"].replace(":bedrock:", ":lambda:")
    try:
        _ar().tag_resource(resourceArn=wrong_service, tags={"env": "prod"})
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ValidationException"
    else:
        raise AssertionError("expected ValidationException")


def test_bedrock_ar_tag_resource_rejects_flow_execution_arn():
    execution = _ar().start_flow_execution(
        flowIdentifier="FLTAG",
        flowAliasIdentifier="FATAG",
        inputs=[{"content": {"document": "input"}, "nodeName": "InputNode", "nodeOutputName": "document"}],
    )
    try:
        _ar().tag_resource(resourceArn=execution["executionArn"], tags={"env": "prod"})
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ValidationException"
    else:
        raise AssertionError("expected ValidationException")


def test_bedrock_ar_list_tags_rejects_unknown_resource_arn():
    arn = "arn:aws:bedrock:us-east-1:000000000000:session/SESSIONMISSING"
    try:
        _ar().list_tags_for_resource(resourceArn=arn)
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ResourceNotFoundException"
    else:
        raise AssertionError("expected ResourceNotFoundException")


def _ingest_with_metadata(suffix):
    """Three documents sharing a query term: one without metadata, one with the
    typed sidecar form, one with a plain scalar."""
    bucket = f"kb-filter-{suffix}"
    s3 = make_client("s3")
    s3.create_bucket(Bucket=bucket)
    s3.put_object(Bucket=bucket, Key="kb/a.txt", Body=b"alpha shared keyword")
    s3.put_object(Bucket=bucket, Key="kb/b.txt", Body=b"beta shared keyword")
    s3.put_object(Bucket=bucket, Key="kb/b.txt.metadata.json", Body=json.dumps({
        "metadataAttributes": {
            "file_id": {"value": {"type": "STRING", "stringValue": "file-abc123"},
                        "includeForEmbedding": False},
            "rank": {"value": {"type": "NUMBER", "numberValue": 7}},
            "tags": {"value": {"type": "STRING_LIST", "stringListValue": ["x", "y"]}},
        },
    }).encode())
    s3.put_object(Bucket=bucket, Key="kb/c.txt", Body=b"gamma shared keyword")
    s3.put_object(Bucket=bucket, Key="kb/c.txt.metadata.json",
                  Body=json.dumps({"metadataAttributes": {"file_id": "plain-form"}}).encode())
    kb = _make_kb(f"kb-filter-{suffix}")["knowledgeBaseId"]
    agent_api = make_client("bedrock-agent")
    ds = agent_api.create_data_source(
        knowledgeBaseId=kb, name="ds",
        dataSourceConfiguration={"type": "S3", "s3Configuration": {
            "bucketArn": f"arn:aws:s3:::{bucket}", "inclusionPrefixes": ["kb/"]}},
    )["dataSource"]["dataSourceId"]
    agent_api.start_ingestion_job(knowledgeBaseId=kb, dataSourceId=ds)
    return kb


def _names(kb, metadata_filter=None):
    config = {"vectorSearchConfiguration": {"numberOfResults": 10}}
    if metadata_filter is not None:
        config["vectorSearchConfiguration"]["filter"] = metadata_filter
    response = _ar().retrieve(
        knowledgeBaseId=kb, retrievalQuery={"text": "shared keyword"},
        retrievalConfiguration=config,
    )
    return sorted(r["location"]["s3Location"]["uri"].rsplit("/", 1)[-1]
                  for r in response["retrievalResults"])


@pytest.mark.parametrize(("metadata_filter", "expected"), [
    (None, ["a.txt", "b.txt", "c.txt"]),
    ({"equals": {"key": "file_id", "value": "file-abc123"}}, ["b.txt"]),
    ({"equals": {"key": "file_id", "value": "plain-form"}}, ["c.txt"]),
    ({"notEquals": {"key": "file_id", "value": "file-abc123"}}, ["a.txt", "c.txt"]),
    ({"startsWith": {"key": "file_id", "value": "file-"}}, ["b.txt"]),
    ({"stringContains": {"key": "file_id", "value": "abc"}}, ["b.txt"]),
    ({"in": {"key": "file_id", "value": ["file-abc123", "plain-form"]}},
     ["b.txt", "c.txt"]),
    ({"notIn": {"key": "file_id", "value": ["file-abc123"]}}, ["a.txt", "c.txt"]),
    ({"greaterThan": {"key": "rank", "value": 5}}, ["b.txt"]),
    ({"greaterThanOrEquals": {"key": "rank", "value": 7}}, ["b.txt"]),
    ({"lessThan": {"key": "rank", "value": 7}}, []),
    ({"lessThanOrEquals": {"key": "rank", "value": 7}}, ["b.txt"]),
    ({"listContains": {"key": "tags", "value": "x"}}, ["b.txt"]),
    ({"andAll": [{"equals": {"key": "file_id", "value": "file-abc123"}},
                 {"greaterThan": {"key": "rank", "value": 5}}]}, ["b.txt"]),
    ({"orAll": [{"equals": {"key": "file_id", "value": "file-abc123"}},
                {"equals": {"key": "file_id", "value": "plain-form"}}]},
     ["b.txt", "c.txt"]),
    ({"equals": {"key": "missing", "value": "x"}}, []),
])
def test_bedrock_ar_retrieve_applies_the_metadata_filter(metadata_filter, expected):
    """Retrieve honours retrievalConfiguration.vectorSearchConfiguration.filter
    across the RetrievalFilter grammar: the eleven comparators plus andAll and
    orAll. Without it a shared knowledge base leaks every tenant's documents."""
    kb = _ingest_with_metadata(f"{_uuid_mod.uuid4().hex[:8]}")
    assert _names(kb, metadata_filter) == expected


def test_bedrock_ar_retrieve_returns_the_document_metadata():
    """RetrievalResultMetadata carries the attributes the sidecar defined; a
    document without a sidecar omits the key, since the map has min 1."""
    kb = _ingest_with_metadata(f"{_uuid_mod.uuid4().hex[:8]}")
    response = _ar().retrieve(
        knowledgeBaseId=kb, retrievalQuery={"text": "shared keyword"},
        retrievalConfiguration={"vectorSearchConfiguration": {"numberOfResults": 10}},
    )
    by_name = {r["location"]["s3Location"]["uri"].rsplit("/", 1)[-1]: r
               for r in response["retrievalResults"]}
    assert by_name["b.txt"]["metadata"] == {
        "file_id": "file-abc123", "rank": 7, "tags": ["x", "y"]}
    assert by_name["c.txt"]["metadata"] == {"file_id": "plain-form"}
    assert "metadata" not in by_name["a.txt"]