Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

20.9 KB
Raw Blame
"""
Bedrock Agent service parity tests — all 72 ops covered.

boto3 round-trip on every resource family. Shapes verified against botocore
bedrock-agent-2023-06-05.
"""

import asyncio
import json

import botocore.exceptions
from conftest import make_client

from ministack.services import bedrock_agent as bedrock_agent_service


def _agent():
    return make_client("bedrock-agent")


def _make_agent(name="agent-test"):
    return _agent().create_agent(
        agentName=name,
        agentResourceRoleArn="arn:aws:iam::000000000000:role/agent-role",
        foundationModel="anthropic.claude-3-haiku-20240307-v1:0",
        instruction="You are a helpful test assistant. Be concise and answer questions clearly.",
    )["agent"]


def _make_kb(name="kb-test"):
    return _agent().create_knowledge_base(
        name=name,
        roleArn="arn:aws:iam::000000000000:role/kb-role",
        knowledgeBaseConfiguration={
            "type": "VECTOR",
            "vectorKnowledgeBaseConfiguration": {
                "embeddingModelArn":
                    "arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0",
            },
        },
        storageConfiguration={
            "type": "OPENSEARCH_SERVERLESS",
            "opensearchServerlessConfiguration": {
                "collectionArn": "arn:aws:aoss:us-east-1:000000000000:collection/coll",
                "vectorIndexName": "idx",
                "fieldMapping": {"vectorField": "v", "textField": "t",
                                  "metadataField": "m"},
            },
        },
    )["knowledgeBase"]


# ---------------------------------------------------------------------------
# Agents
# ---------------------------------------------------------------------------


def test_bedrock_agent_create_get_list_delete():
    create = _make_agent("ag-1")
    assert create["agentName"] == "ag-1"
    assert create["agentStatus"] in ("NOT_PREPARED", "CREATING")
    assert ":agent/" in create["agentArn"]

    g = _agent().get_agent(agentId=create["agentId"])
    assert g["agent"]["agentName"] == "ag-1"

    lst = _agent().list_agents()
    assert any(a["agentId"] == create["agentId"] for a in lst["agentSummaries"])

    _agent().delete_agent(agentId=create["agentId"])
    try:
        _agent().get_agent(agentId=create["agentId"])
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ResourceNotFoundException"


def test_bedrock_agent_update():
    create = _make_agent("ag-upd")
    _agent().update_agent(
        agentId=create["agentId"],
        agentName="ag-upd-renamed",
        agentResourceRoleArn="arn:aws:iam::000000000000:role/agent-role",
        foundationModel="amazon.nova-pro-v1:0",
        instruction="You are an updated helpful test assistant for verification purposes.",
    )
    g = _agent().get_agent(agentId=create["agentId"])
    assert g["agent"]["agentName"] == "ag-upd-renamed"


def test_bedrock_agent_prepare():
    create = _make_agent("ag-prep")
    resp = _agent().prepare_agent(agentId=create["agentId"])
    assert resp["agentStatus"] == "PREPARED"


def test_bedrock_agent_create_duplicate_conflict():
    _make_agent("ag-dup")
    try:
        _make_agent("ag-dup")
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ConflictException"
    else:
        raise AssertionError("expected conflict")


# ---------------------------------------------------------------------------
# Agent aliases
# ---------------------------------------------------------------------------


def test_bedrock_agent_alias_lifecycle():
    agent = _make_agent("ag-for-alias")
    create = _agent().create_agent_alias(
        agentId=agent["agentId"], agentAliasName="prod",
    )
    aid = create["agentAlias"]["agentAliasId"]
    g = _agent().get_agent_alias(agentId=agent["agentId"], agentAliasId=aid)
    assert g["agentAlias"]["agentAliasName"] == "prod"
    _agent().update_agent_alias(
        agentId=agent["agentId"], agentAliasId=aid,
        agentAliasName="staging",
    )
    g2 = _agent().get_agent_alias(agentId=agent["agentId"], agentAliasId=aid)
    assert g2["agentAlias"]["agentAliasName"] == "staging"
    lst = _agent().list_agent_aliases(agentId=agent["agentId"])
    assert any(a["agentAliasId"] == aid for a in lst["agentAliasSummaries"])
    _agent().delete_agent_alias(agentId=agent["agentId"], agentAliasId=aid)


# ---------------------------------------------------------------------------
# Agent action groups
# ---------------------------------------------------------------------------


def test_bedrock_agent_action_group_lifecycle():
    agent = _make_agent("ag-ag")
    create = _agent().create_agent_action_group(
        agentId=agent["agentId"], agentVersion="DRAFT",
        actionGroupName="ag-1",
        actionGroupExecutor={"lambda": "arn:aws:lambda:us-east-1:000000000000:function:f"},
    )
    agid = create["agentActionGroup"]["actionGroupId"]
    g = _agent().get_agent_action_group(
        agentId=agent["agentId"], agentVersion="DRAFT", actionGroupId=agid,
    )
    assert g["agentActionGroup"]["actionGroupName"] == "ag-1"
    _agent().update_agent_action_group(
        agentId=agent["agentId"], agentVersion="DRAFT", actionGroupId=agid,
        actionGroupName="ag-1-renamed",
        actionGroupExecutor={"lambda": "arn:aws:lambda:us-east-1:000000000000:function:f"},
    )
    lst = _agent().list_agent_action_groups(
        agentId=agent["agentId"], agentVersion="DRAFT",
    )
    assert any(a["actionGroupId"] == agid for a in lst["actionGroupSummaries"])
    _agent().delete_agent_action_group(
        agentId=agent["agentId"], agentVersion="DRAFT", actionGroupId=agid,
        skipResourceInUseCheck=True,
    )


# ---------------------------------------------------------------------------
# Agent collaborators
# ---------------------------------------------------------------------------


def test_bedrock_agent_collaborator_lifecycle():
    agent = _make_agent("ag-collab")
    assoc = _agent().associate_agent_collaborator(
        agentId=agent["agentId"], agentVersion="DRAFT",
        agentDescriptor={"aliasArn": "arn:aws:bedrock:us-east-1:000000000000:agent-alias/A/B"},
        collaboratorName="researcher",
        collaborationInstruction="Help with research.",
    )
    cid = assoc["agentCollaborator"]["collaboratorId"]
    g = _agent().get_agent_collaborator(
        agentId=agent["agentId"], agentVersion="DRAFT", collaboratorId=cid,
    )
    assert g["agentCollaborator"]["collaboratorName"] == "researcher"
    _agent().disassociate_agent_collaborator(
        agentId=agent["agentId"], agentVersion="DRAFT", collaboratorId=cid,
    )


# ---------------------------------------------------------------------------
# Knowledge bases + data sources + ingestion jobs
# ---------------------------------------------------------------------------


def test_bedrock_agent_knowledge_base_crud():
    kb = _make_kb("kb-test-1")
    assert kb["status"] == "ACTIVE"
    g = _agent().get_knowledge_base(knowledgeBaseId=kb["knowledgeBaseId"])
    assert g["knowledgeBase"]["name"] == "kb-test-1"
    lst = _agent().list_knowledge_bases()
    assert any(k["knowledgeBaseId"] == kb["knowledgeBaseId"]
                for k in lst["knowledgeBaseSummaries"])
    _agent().delete_knowledge_base(knowledgeBaseId=kb["knowledgeBaseId"])


def test_bedrock_agent_data_source_crud():
    kb = _make_kb("kb-for-ds")
    ds = _agent().create_data_source(
        knowledgeBaseId=kb["knowledgeBaseId"],
        name="ds-1",
        dataSourceConfiguration={
            "type": "S3",
            "s3Configuration": {"bucketArn": "arn:aws:s3:::bucket"},
        },
    )["dataSource"]
    g = _agent().get_data_source(
        knowledgeBaseId=kb["knowledgeBaseId"], dataSourceId=ds["dataSourceId"],
    )
    assert g["dataSource"]["name"] == "ds-1"
    lst = _agent().list_data_sources(knowledgeBaseId=kb["knowledgeBaseId"])
    assert any(d["dataSourceId"] == ds["dataSourceId"]
                for d in lst["dataSourceSummaries"])
    _agent().delete_data_source(
        knowledgeBaseId=kb["knowledgeBaseId"], dataSourceId=ds["dataSourceId"],
    )


def test_bedrock_agent_ingestion_job_lifecycle():
    s3 = make_client("s3")
    s3.create_bucket(Bucket="kb-ij-docs")
    s3.put_object(Bucket="kb-ij-docs", Key="a.txt", Body=b"MiniStack runs on port 4566.")
    s3.put_object(Bucket="kb-ij-docs", Key="b.txt", Body=b"Ingestion stores documents.")
    kb = _make_kb("kb-ij")
    ds = _agent().create_data_source(
        knowledgeBaseId=kb["knowledgeBaseId"], name="ds-ij",
        dataSourceConfiguration={"type": "S3", "s3Configuration": {"bucketArn": "arn:aws:s3:::kb-ij-docs"}},
    )["dataSource"]
    job = _agent().start_ingestion_job(
        knowledgeBaseId=kb["knowledgeBaseId"], dataSourceId=ds["dataSourceId"],
    )["ingestionJob"]
    assert job["status"] == "STARTING"
    g = _agent().get_ingestion_job(
        knowledgeBaseId=kb["knowledgeBaseId"], dataSourceId=ds["dataSourceId"],
        ingestionJobId=job["ingestionJobId"],
    )
    assert g["ingestionJob"]["status"] == "COMPLETE"
    stats = g["ingestionJob"]["statistics"]
    assert stats["numberOfDocumentsScanned"] == 2
    assert stats["numberOfNewDocumentsIndexed"] == 2
    assert stats["numberOfDocumentsFailed"] == 0
    lst = _agent().list_ingestion_jobs(
        knowledgeBaseId=kb["knowledgeBaseId"], dataSourceId=ds["dataSourceId"],
    )
    assert any(j["ingestionJobId"] == job["ingestionJobId"]
                for j in lst["ingestionJobSummaries"])


def test_bedrock_agent_ingestion_job_missing_bucket_fails_loudly():
    kb = _make_kb("kb-ij-missing")
    ds = _agent().create_data_source(
        knowledgeBaseId=kb["knowledgeBaseId"], name="ds-ij-missing",
        dataSourceConfiguration={"type": "S3", "s3Configuration": {
            "bucketArn": "arn:aws:s3:::kb-ij-no-such-bucket"}},
    )["dataSource"]
    job = _agent().start_ingestion_job(
        knowledgeBaseId=kb["knowledgeBaseId"], dataSourceId=ds["dataSourceId"],
    )["ingestionJob"]
    assert job["status"] == "FAILED"
    g = _agent().get_ingestion_job(
        knowledgeBaseId=kb["knowledgeBaseId"], dataSourceId=ds["dataSourceId"],
        ingestionJobId=job["ingestionJobId"],
    )["ingestionJob"]
    assert g["status"] == "FAILED"
    assert g["failureReasons"]
    assert "kb-ij-no-such-bucket" in g["failureReasons"][0]
    assert g["statistics"]["numberOfDocumentsScanned"] == 0


def test_bedrock_agent_ingestion_reindex_counts_modified_and_failed():
    s3 = make_client("s3")
    s3.create_bucket(Bucket="kb-ij-mixed")
    s3.put_object(Bucket="kb-ij-mixed", Key="doc.txt", Body=b"Readable text.")
    s3.put_object(Bucket="kb-ij-mixed", Key="blob.bin", Body=b"\xff\xfe\x00\x01binary")
    kb = _make_kb("kb-ij-mixed")
    ds = _agent().create_data_source(
        knowledgeBaseId=kb["knowledgeBaseId"], name="ds-ij-mixed",
        dataSourceConfiguration={"type": "S3", "s3Configuration": {"bucketArn": "arn:aws:s3:::kb-ij-mixed"}},
    )["dataSource"]

    def run():
        job = _agent().start_ingestion_job(
            knowledgeBaseId=kb["knowledgeBaseId"], dataSourceId=ds["dataSourceId"],
        )["ingestionJob"]
        return _agent().get_ingestion_job(
            knowledgeBaseId=kb["knowledgeBaseId"], dataSourceId=ds["dataSourceId"],
            ingestionJobId=job["ingestionJobId"],
        )["ingestionJob"]

    first = run()
    assert first["status"] == "COMPLETE"
    assert first["statistics"]["numberOfDocumentsScanned"] == 2
    assert first["statistics"]["numberOfNewDocumentsIndexed"] == 1
    assert first["statistics"]["numberOfDocumentsFailed"] == 1
    assert any("blob.bin" in r for r in first["failureReasons"])

    second = run()
    assert second["statistics"]["numberOfNewDocumentsIndexed"] == 0
    assert second["statistics"]["numberOfModifiedDocumentsIndexed"] == 1


# ---------------------------------------------------------------------------
# Flows
# ---------------------------------------------------------------------------


def test_bedrock_agent_flow_lifecycle():
    flow = _agent().create_flow(
        name="flow-1",
        executionRoleArn="arn:aws:iam::000000000000:role/flow-role",
        definition={"nodes": [], "connections": []},
    )
    fid = flow["id"]
    assert ":flow/" in flow["arn"]
    g = _agent().get_flow(flowIdentifier=fid)
    assert g["name"] == "flow-1"
    lst = _agent().list_flows()
    assert any(f["id"] == fid for f in lst["flowSummaries"])
    _agent().update_flow(
        flowIdentifier=fid, name="flow-1-renamed",
        executionRoleArn="arn:aws:iam::000000000000:role/flow-role",
        definition={"nodes": [], "connections": []},
    )
    _agent().prepare_flow(flowIdentifier=fid)
    _agent().delete_flow(flowIdentifier=fid)


def test_bedrock_agent_flow_version():
    flow = _agent().create_flow(
        name="flow-ver",
        executionRoleArn="arn:aws:iam::000000000000:role/flow-role",
        definition={"nodes": [], "connections": []},
    )
    ver = _agent().create_flow_version(flowIdentifier=flow["id"])
    assert ver["version"] == "1"
    g = _agent().get_flow_version(flowIdentifier=flow["id"], flowVersion="1")
    assert g["version"] == "1"
    lst = _agent().list_flow_versions(flowIdentifier=flow["id"])
    assert any(v["version"] == "1" for v in lst["flowVersionSummaries"])


def test_bedrock_agent_flow_alias():
    flow = _agent().create_flow(
        name="flow-alias",
        executionRoleArn="arn:aws:iam::000000000000:role/flow-role",
    )
    _agent().create_flow_version(flowIdentifier=flow["id"])
    alias = _agent().create_flow_alias(
        flowIdentifier=flow["id"], name="prod",
        routingConfiguration=[{"flowVersion": "1"}],
    )
    aid = alias["id"]
    g = _agent().get_flow_alias(flowIdentifier=flow["id"], aliasIdentifier=aid)
    assert g["name"] == "prod"


def test_bedrock_agent_validate_flow_definition():
    resp = _agent().validate_flow_definition(definition={"nodes": [], "connections": []})
    assert "validations" in resp


# ---------------------------------------------------------------------------
# Prompts
# ---------------------------------------------------------------------------


def test_bedrock_agent_prompt_lifecycle():
    create = _agent().create_prompt(
        name="prompt-1",
        variants=[{"name": "v1", "templateType": "TEXT",
                    "templateConfiguration": {"text": {"text": "{{x}}"}}}],
        defaultVariant="v1",
    )
    pid = create["id"]
    assert ":prompt/" in create["arn"]
    g = _agent().get_prompt(promptIdentifier=pid)
    assert g["name"] == "prompt-1"
    lst = _agent().list_prompts()
    assert any(p["id"] == pid for p in lst["promptSummaries"])
    _agent().update_prompt(
        promptIdentifier=pid, name="prompt-1-renamed",
        variants=[{"name": "v1", "templateType": "TEXT",
                    "templateConfiguration": {"text": {"text": "{{y}}"}}}],
        defaultVariant="v1",
    )
    ver = _agent().create_prompt_version(promptIdentifier=pid)
    assert ver["version"] == "1"
    _agent().delete_prompt(promptIdentifier=pid)


# ---------------------------------------------------------------------------
# Tags
# ---------------------------------------------------------------------------


def test_bedrock_agent_tag_resource():
    agent = _make_agent("ag-tag")
    arn = agent["agentArn"]
    _agent().tag_resource(resourceArn=arn, tags={"env": "prod", "team": "ml"})
    lst = _agent().list_tags_for_resource(resourceArn=arn)
    assert lst["tags"]["env"] == "prod"
    _agent().untag_resource(resourceArn=arn, tagKeys=["env"])
    lst2 = _agent().list_tags_for_resource(resourceArn=arn)
    assert "env" not in lst2["tags"]


def test_bedrock_agent_tag_resource_accepts_flow_alias_arn():
    flow = _agent().create_flow(
        name="flow-alias-tag",
        executionRoleArn="arn:aws:iam::000000000000:role/flow-role",
    )
    _agent().create_flow_version(flowIdentifier=flow["id"])
    alias = _agent().create_flow_alias(
        flowIdentifier=flow["id"], name="prod-tag",
        routingConfiguration=[{"flowVersion": "1"}],
    )
    arn = alias["arn"]
    assert f":flow/{flow['id']}/alias/" in arn

    _agent().tag_resource(resourceArn=arn, tags={"env": "prod"})
    lst = _agent().list_tags_for_resource(resourceArn=arn)
    assert lst["tags"]["env"] == "prod"


def test_bedrock_agent_tag_resource_accepts_restored_legacy_flow_alias_arn():
    bedrock_agent_service.reset()
    try:
        flow_id = "FLRESTORE"
        alias_id = "FARESTORE"
        arn = f"arn:aws:bedrock:us-east-1:000000000000:flow-alias/{flow_id}/{alias_id}"
        canonical_arn = f"arn:aws:bedrock:us-east-1:000000000000:flow/{flow_id}/alias/{alias_id}"
        bedrock_agent_service._flow_aliases[f"{flow_id}/{alias_id}"] = {
            "Arn": arn,
            "FlowId": flow_id,
            "Id": alias_id,
        }
        status, _, _ = asyncio.run(bedrock_agent_service.handle_request(
            "POST",
            f"/tags/{arn}",
            {},
            b'{"tags":{"env":"prod"}}',
            {},
        ))
        assert status == 200
        status, _, _ = asyncio.run(bedrock_agent_service.handle_request(
            "POST",
            f"/tags/{canonical_arn}",
            {},
            b'{"tags":{"stage":"prod"}}',
            {},
        ))
        assert status == 200
        status, _, body = asyncio.run(bedrock_agent_service.handle_request(
            "GET",
            f"/tags/{canonical_arn}",
            {},
            b"",
            {},
        ))
        assert status == 200
        assert json.loads(body)["tags"] == {"env": "prod", "stage": "prod"}
    finally:
        bedrock_agent_service.reset()


def test_bedrock_agent_tag_resource_accepts_prompt_version_arn():
    prompt = _agent().create_prompt(
        name="prompt-tag",
        variants=[{"name": "v1", "templateType": "TEXT",
                    "templateConfiguration": {"text": {"text": "{{x}}"}}}],
        defaultVariant="v1",
    )
    version = _agent().create_prompt_version(promptIdentifier=prompt["id"])
    arn = version["arn"]
    assert arn.endswith(":1")

    _agent().tag_resource(resourceArn=arn, tags={"env": "prod"})
    lst = _agent().list_tags_for_resource(resourceArn=arn)
    assert lst["tags"]["env"] == "prod"


def test_bedrock_agent_tag_resource_accepts_restored_prompt_version_arn():
    bedrock_agent_service.reset()
    try:
        prompt_id = "PRRESTORE"
        version = "1"
        draft_arn = f"arn:aws:bedrock:us-east-1:000000000000:prompt/{prompt_id}"
        version_arn = f"{draft_arn}:{version}"
        bedrock_agent_service._prompt_versions[f"{prompt_id}/{version}"] = {
            "Arn": draft_arn,
            "Id": prompt_id,
            "Version": version,
        }
        status, _, _ = asyncio.run(bedrock_agent_service.handle_request(
            "POST",
            f"/tags/{version_arn}",
            {},
            b'{"tags":{"env":"prod"}}',
            {},
        ))
        assert status == 200
    finally:
        bedrock_agent_service.reset()


def test_bedrock_agent_tag_resource_rejects_malformed_arn():
    try:
        _agent().tag_resource(resourceArn="not-an-arn-but-long-enough", tags={"env": "prod"})
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ValidationException"
    else:
        raise AssertionError("expected ValidationException")


def test_bedrock_agent_tag_resource_rejects_wrong_scope_arn():
    agent = _make_agent("ag-tag-scope")
    wrong_region = agent["agentArn"].replace(":us-east-1:", ":us-west-2:")
    try:
        _agent().tag_resource(resourceArn=wrong_region, tags={"env": "prod"})
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ResourceNotFoundException"
    else:
        raise AssertionError("expected ResourceNotFoundException")


def test_bedrock_agent_tag_resource_rejects_noncanonical_partition_arn():
    agent = _make_agent("ag-tag-partition")
    wrong_partition = agent["agentArn"].replace("arn:aws:", "arn:aws-cn:", 1)
    try:
        _agent().tag_resource(resourceArn=wrong_partition, tags={"env": "prod"})
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ResourceNotFoundException"
    else:
        raise AssertionError("expected ResourceNotFoundException")


def test_bedrock_agent_tag_resource_rejects_wrong_service_arn():
    agent = _make_agent("ag-tag-service")
    wrong_service = agent["agentArn"].replace(":bedrock:", ":lambda:")
    try:
        _agent().tag_resource(resourceArn=wrong_service, tags={"env": "prod"})
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ValidationException"
    else:
        raise AssertionError("expected ValidationException")


def test_bedrock_agent_list_tags_rejects_unknown_resource_arn():
    arn = "arn:aws:bedrock:us-east-1:000000000000:agent/AGMISSING"
    try:
        _agent().list_tags_for_resource(resourceArn=arn)
    except botocore.exceptions.ClientError as exc:
        assert exc.response["Error"]["Code"] == "ResourceNotFoundException"
    else:
        raise AssertionError("expected ResourceNotFoundException")