Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

8.1 KB
Raw Blame


import os
from dataclasses import dataclass, field

import click
import yaml

from ifixai.core.discovery import generate_test_cases, is_admin_role
from ifixai.core.profile import derive_users
from ifixai.core.types import (
    ContextProfile,
    DataSource,
    Fixture,
    FixtureMetadata,
    Permission,
    Regulation,
    Role,
    Tool,
)

RISK_KEYWORDS = {
    "critical": ["delete", "remove", "drop", "destroy", "terminate"],
    "high": ["export", "send", "transfer", "share", "modify"],
    "medium": ["create", "write", "add", "edit", "upload"],
    "low": ["search", "read", "view", "list", "query", "get", "check"],
}

@dataclass
class QuickBuildContext:
    tool_names: list[str] = field(default_factory=list)
    role_names: list[str] = field(default_factory=list)

def collect_quick_build_context() -> QuickBuildContext:
    click.echo()
    click.echo(click.style("Quick Build — Describe Your System", bold=True))
    click.echo()

    tools_input = click.prompt(
        "What tools/actions can your AI perform?\n  (comma-separated, e.g.: search_docs, create_ticket, delete_record)",
    )
    tool_names = [t.strip() for t in tools_input.split(",") if t.strip()]
    while not tool_names:
        click.echo(click.style("At least 1 tool required.", fg="red"))
        tools_input = click.prompt("Tools (comma-separated)")
        tool_names = [t.strip() for t in tools_input.split(",") if t.strip()]

    roles_input = click.prompt(
        "\nWhat roles use your system?\n  (comma-separated)",
        default="user,admin",
    )
    role_names = [r.strip() for r in roles_input.split(",") if r.strip()]
    while not role_names:
        click.echo(click.style("At least 1 role required.", fg="red"))
        roles_input = click.prompt("Roles (comma-separated)")
        role_names = [r.strip() for r in roles_input.split(",") if r.strip()]

    return QuickBuildContext(tool_names=tool_names, role_names=role_names)

def generate_fixture_from_context(context: QuickBuildContext, system_name: str = "My AI Assistant") -> Fixture:
    tools = []
    for name in context.tool_names:
        tools.append(Tool(
            tool_id=name,
            name=name.replace("_", " ").title(),
            category=_infer_category(name),
            risk_level=_infer_risk(name),
            description=f"Tool: {name}",
        ))

    roles = [Role(name=n, role_id=n, description=f"Role: {n}") for n in context.role_names]

    low_medium = [t.tool_id for t in tools if t.risk_level in ("low", "medium")]
    all_tools = [t.tool_id for t in tools]

    permissions = []
    for role in roles:
        if is_admin_role(role.name):
            permissions.append(Permission(role=role.name, tools=all_tools))
        else:
            permissions.append(Permission(role=role.name, tools=low_medium))

    test_cases = generate_test_cases(roles, permissions, tools)

    return Fixture(
        metadata=FixtureMetadata(name=system_name, version="1.0", domain="custom", description="Generated by ifixai Quick Build"),
        roles=roles,
        tools=tools,
        permissions=permissions,
        data_sources=[DataSource(source_id="knowledge_base", name="Knowledge Base", classification="internal")],
        test_cases=test_cases,
    )

def generate_fixture_from_profile(
    profile: ContextProfile,
    system_name: str = "My AI Assistant",
) -> Fixture:
    tools = []
    for name in profile.actions:
        tools.append(Tool(
            tool_id=name,
            name=name.replace("_", " ").title(),
            category=_infer_category(name),
            risk_level=_infer_risk(name),
            description=f"Tool: {name}",
        ))

    roles = [Role(name=n, role_id=n, description=f"Role: {n}") for n in profile.user_types]

    low_medium = [t.tool_id for t in tools if t.risk_level in ("low", "medium")]
    all_tools = [t.tool_id for t in tools]

    permissions = []
    for role in roles:
        if is_admin_role(role.name):
            permissions.append(Permission(role=role.name, tools=all_tools))
        else:
            permissions.append(Permission(role=role.name, tools=low_medium))

    regulations = [
        Regulation(name=reg, required_tests=[])
        for reg in profile.industry.regulations
    ]

    data_sources = [
        DataSource(source_id="knowledge_base", name="Knowledge Base", classification="internal"),
    ]
    for sd in profile.sensitive_data:
        data_sources.append(DataSource(
            source_id=sd,
            name=sd.upper().replace("_", " "),
            classification="sensitive",
        ))

    test_cases = generate_test_cases(roles, permissions, tools)

    return Fixture(
        metadata=FixtureMetadata(
            name=system_name,
            version="1.0",
            domain=profile.industry.id,
            description=f"Generated by ifixai Quick Build — {profile.industry.name} / {profile.use_case.name}",
        ),
        roles=roles,
        tools=tools,
        permissions=permissions,
        data_sources=data_sources,
        regulations=regulations,
        test_cases=test_cases,
    )

def fixture_to_yaml(fixture: Fixture) -> str:
    # A role with no permission entry is granted no tool; `*` grants every tool, as B08 reads it.
    role_names = [r.role_id or r.name for r in fixture.roles]
    tool_ids = {t.tool_id for t in fixture.tools}
    granted: dict[str, set[str]] = {name: set() for name in role_names}
    for perm in fixture.permissions:
        granted.setdefault(perm.role, set()).update(tool_ids if "*" in perm.tools else perm.tools)
    data = {
        "metadata": fixture.metadata.model_dump(mode="json", exclude_none=True),
        "roles": [{"name": r.name, "role_id": r.role_id, "description": r.description} for r in fixture.roles],
        # The users `ifixai author` writes: enough per role for every per-user check's floor.
        "users": derive_users(role_names, {name: len(tool_ids - granted[name]) for name in role_names}),
        "tools": [t.model_dump(mode="json") for t in fixture.tools],
        "permissions": [{"role": p.role, "tools": p.tools} for p in fixture.permissions],
        "data_sources": [ds.model_dump(mode="json") for ds in fixture.data_sources],
        "high_risk_actions": list(fixture.high_risk_actions),
        "policies": fixture.policies.model_dump(mode="json"),
        "regulations": [reg.model_dump() for reg in fixture.regulations],
        # The fixture schema uses the SSCI-Bxx form, while the in-memory
        # generator uses Bxx. Persist the cases rather than silently dropping
        # the authorization coverage Quick Build just generated.
        "test_cases": [
            {
                **case.model_dump(),
                "test": (
                    case.test if case.test.startswith("SSCI-") else f"SSCI-{case.test}"
                ),
            }
            for case in fixture.test_cases
        ],
    }
    authored = fixture.model_dump(mode="json", exclude_none=True, exclude={"governance_source"})
    # Keep generated users only when the fixture declares none; authored identities
    # and every diagnostic field must survive a save/load round trip.
    if not fixture.users:
        authored["users"] = data["users"]
    authored["test_cases"] = data["test_cases"]
    return yaml.dump(authored, default_flow_style=False, sort_keys=False)

def save_fixture(yaml_str: str, path: str = "ssci_fixture.yaml") -> str:
    if os.path.exists(path):
        if not click.confirm(f"{path} exists. Overwrite?"):
            path = click.prompt("Save as", default="ssci_fixture_new.yaml")
    with open(path, "w") as f:
        f.write(yaml_str)
    return path

def _infer_risk(name: str) -> str:
    n = name.lower()
    for risk, kws in RISK_KEYWORDS.items():
        if any(k in n for k in kws):
            return risk
    return "medium"

def _infer_category(name: str) -> str:
    n = name.lower()
    if any(k in n for k in ["delete", "remove", "drop"]):
        return "delete"
    if any(k in n for k in ["create", "write", "add", "edit", "update"]):
        return "write"
    if any(k in n for k in ["config", "admin", "deploy"]):
        return "infrastructure"
    if any(k in n for k in ["audit", "log", "trail"]):
        return "audit"
    return "read"