YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: enforce restricted enterprise onboarding gate
louis288989a4aa69
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex f23f2e8f2..02aff84b8 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -380,3 +380,22 @@ an unrelated newer GLM provider import is replaced by an unused constant.This verifies connection configuration, not full route dispatch, cryptographicverification, website membership computation, real gateway spend enforcement,settlement, SDK requests, model outcomes or filesystem isolation.+++## Restricted enterprise onboarding++`app-restricted-enterprise-onboarding` renders the actual React entitlement gate+and real entitlement policy. Ten outcomes cover live sign-in, cached identity+and build-resolution transitions, alongside ordinary/paid/tokenless consumer,+managed-build and explicit no-auth paths. The parent fails two visibility+assertions and preserves eight outcomes; reference and current source pass ten.+Permitted paths also preserve absence of recorder-stop commands.++Run `bun test evals/coding-agent/calibrate-restricted-onboarding.test.js` with+existing frontend dependencies installed. Calibration rejects blanket onboarding+access/refusal, premature build classification, lost paid-personal exceptions,+missing token requirements and ignored restriction flags. Equivalent internal+names pass; missing source remains setup failure. The grader is installed only+after an evaluated trajectory. Settings, managed policy, analytics and native+commands are synthetic. Native auth, recorder durability, persisted enterprise+policy, live downloads and execution isolation are outside this UI evidence.diff --git a/evals/coding-agent/calibrate-restricted-onboarding.test.js b/evals/coding-agent/calibrate-restricted-onboarding.test.jsnew file mode 100644index 000000000..1a326686a--- /dev/null+++ b/evals/coding-agent/calibrate-restricted-onboarding.test.js@@ -0,0 +1,28 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..');+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-restricted-enterprise-onboarding');+const path=item.oracle_paths[0],appPath='apps/screenpipe-app-tauri';+const show=(ref,p)=>execFileSync('git',['show',`${ref}:${p}`],{cwd:repo,encoding:'utf8'});+const fixed=show(item.oracle_ref,path),broken=show(item.base_ref,path);+const root=mkdtempSync(join(tmpdir(),'restricted-onboarding-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));+execFileSync('tar',['-x','-C',root],{input:execFileSync('git',['archive',item.oracle_ref,appPath],{cwd:repo,maxBuffer:128*1024*1024})});+const app=join(root,appPath),runtime=join(repo,appPath,'node_modules');symlinkSync(runtime,join(app,'node_modules'),'dir');+writeFileSync(join(app,'components/restricted-onboarding.eval.test.tsx'),readFileSync(join(import.meta.dir,'graders/restricted-onboarding.fixture.tsx')));+function replace(s,old,next){expect(s.split(old)).toHaveLength(2);return s.replace(old,next);}+function grade(source){writeFileSync(join(root,path),source);return spawnSync(join(runtime,'.bin/vitest'),['run','--config','vitest.config.ts','components/restricted-onboarding.eval.test.tsx','--maxWorkers=1','--minWorkers=1'],{cwd:app,encoding:'utf8',timeout:60000,env:{...process.env,NODE_OPTIONS:'--localstorage-file=.eval-localstorage'}});}+function fail(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stdout).toContain('failed');expect(r.stderr).not.toContain('Failed to resolve import');expect(r.stderr).not.toContain('Unhandled Errors');}+test('parent exposes restricted onboarding in two sequences and preserves eight',()=>{const r=grade(broken);fail(r);expect(r.stdout).toContain('2 failed');expect(r.stdout).toContain('8 passed');});+test('reference passes all ten UI outcomes',()=>{const r=grade(fixed);expect(r.status).toBe(0);expect(r.stdout).toContain('10 passed');});+test('equivalent local decision naming passes',()=>expect(grade(fixed.replaceAll('shouldGateForEnterpriseApp','restrictedBuildRequired')).status).toBe(0));+test('blanket onboarding refusal is rejected',()=>fail(grade(replace(fixed,'? isManagedDeploymentResolved && shouldGateForEnterpriseApp','? true'))));+test('premature build classification is rejected',()=>fail(grade(replace(fixed,'? isManagedDeploymentResolved && shouldGateForEnterpriseApp','? shouldGateForEnterpriseApp'))));+test('ignoring explicit enterprise restriction loses paid restricted case',()=>fail(grade(replace(fixed,'enterpriseAccount.restrict_consumer_build_access === true','false'))));+test('requiring enterprise app despite paid unrestricted account is rejected',()=>fail(grade(replace(fixed,'(!hasConsumerSubscription ||\n enterpriseAccount.restrict_consumer_build_access === true)','true'))));+test('tokenless cached organization must not gate onboarding',()=>fail(grade(replace(fixed,'Boolean(user?.token) &&\n enterpriseAccount?.requires_enterprise_app','true &&\n enterpriseAccount?.requires_enterprise_app'))));+test('missing component is setup failure',()=>{rmSync(join(root,path));const r=spawnSync(join(runtime,'.bin/vitest'),['run','--config','vitest.config.ts','components/restricted-onboarding.eval.test.tsx','--maxWorkers=1','--minWorkers=1'],{cwd:app,encoding:'utf8',timeout:60000,env:{...process.env,NODE_OPTIONS:'--localstorage-file=.eval-localstorage'}});expect(r.status).toBe(1);expect(r.stderr).toContain('Failed to resolve import');expect(r.stdout).toContain('no tests');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 0ce4843bc..df69e67ac 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-20.4",+ "dataset_version": "2026-09-20.5","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1770,6 +1770,45 @@],"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/enterprise-allowance.test.js"}+ },+ {+ "id": "app-restricted-enterprise-onboarding",+ "name": "Gate restricted enterprise identity when onboarding resolves consumer build",+ "source": {+ "kind": "git_regression",+ "fix_commit": "46a170f92ea371feecc5a8db139d2fc1e3cac8ba"+ },+ "base_ref": "e594a3c7ab8bed3e90310ff05af4c89cff32a166",+ "oracle_ref": "46a170f92ea371feecc5a8db139d2fc1e3cac8ba",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/app-entitlement-gate.tsx"+ ],+ "tags": [+ "regression",+ "onboarding",+ "enterprise",+ "access",+ "identity"+ ],+ "gate": "advisory",+ "prompt": "A restricted enterprise member can continue consumer onboarding after sign-in because the onboarding route bypasses the enterprise-app gate. Once build detection confirms the consumer app, hide protected onboarding content and show the enterprise-app-required state for a signed-in member whose organization requires that app and restricts consumer access, even if the member also has a paid personal subscription. Observe both live sign-in and a cached identity while detection resolves. Preserve ordinary consumer onboarding, tokenless onboarding, paid personal access when the organization does not restrict it, and accounts whose organization does not require the enterprise app. Do not misclassify an unresolved or managed enterprise build; managed onboarding keeps ownership of its authentication flow. Preserve explicit not_required startup semantics, including cached restricted identities, and do not stop recording on these permitted onboarding paths. Test the real React gate with synthetic settings/managed-policy/native/provider ports. This case covers visible gate decisions and attempted command effects, not real recorder durability or native authentication. No helper or state variable name is required.",+ "agent_timeout_seconds": 1200,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 120,+ "fixtures": [+ {+ "local_path": "graders/restricted-onboarding.fixture.tsx",+ "destination_path": "apps/screenpipe-app-tauri/components/restricted-onboarding.eval.test.tsx"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage ./node_modules/.bin/vitest run --config vitest.config.ts components/restricted-onboarding.eval.test.tsx --maxWorkers=1 --minWorkers=1"+ }}]}diff --git a/evals/coding-agent/graders/restricted-onboarding.fixture.tsx b/evals/coding-agent/graders/restricted-onboarding.fixture.tsxnew file mode 100644index 000000000..23b31f39a--- /dev/null+++ b/evals/coding-agent/graders/restricted-onboarding.fixture.tsx@@ -0,0 +1,312 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import {+ act,+ cleanup,+ fireEvent,+ render,+ screen,+ waitFor,+} from "@testing-library/react";++// Mutable harness state + spies. The gate reads everything through useSettings+// and the tauri `commands` object, so we drive entitlement scenarios by swapping+// `mocks.state.user` and assert on the engine start/stop calls it makes.+const mocks = vi.hoisted(() => ({+ capture: vi.fn(),+ open: vi.fn().mockResolvedValue(undefined),+ stopScreenpipe: vi.fn().mockResolvedValue(undefined),+ spawnScreenpipe: vi.fn().mockResolvedValue(undefined),+ openLoginWindow: vi.fn().mockResolvedValue(undefined),+ setCloudToken: vi.fn().mockResolvedValue(undefined),+ getCloudToken: vi.fn().mockResolvedValue(null),+ piUpdateConfig: vi.fn().mockResolvedValue(undefined),+ platform: vi.fn(() => "windows"),+ arch: vi.fn(() => "x86_64"),+ windowLabel: "home",+ loadUser: vi.fn().mockResolvedValue(undefined),+ updateSettings: vi.fn().mockResolvedValue(undefined),+ state: { isSettingsLoaded: true, user: null as any },+ enterpriseResolutionError: false,+ enterprise: {+ isManagedDeployment: false,+ isManagedDeploymentResolved: true,+ authenticationState: "authenticated",+ authenticationError: null as string | null,+ isManagedAuthenticated: true,+ },+ selectAuthenticationMethod: vi.fn(),+ submitLicenseKey: vi.fn(async () => ({ ok: true })),+}));++vi.mock("@/lib/hooks/use-settings", () => ({+ useSettings: () => ({+ settings: { user: mocks.state.user },+ isSettingsLoaded: mocks.state.isSettingsLoaded,+ loadUser: mocks.loadUser,+ updateSettings: mocks.updateSettings,+ }),+}));++vi.mock("@/lib/utils/tauri", () => ({+ commands: {+ stopScreenpipe: mocks.stopScreenpipe,+ spawnScreenpipe: mocks.spawnScreenpipe,+ openLoginWindow: mocks.openLoginWindow,+ setCloudToken: mocks.setCloudToken,+ getCloudToken: mocks.getCloudToken,+ piUpdateConfig: mocks.piUpdateConfig,+ },+}));++vi.mock("@/lib/hooks/use-managed-policy", () => ({+ useManagedPolicy: () => ({+ isManagedDeployment: mocks.enterprise.isManagedDeployment,+ isManagedDeploymentResolved: mocks.enterprise.isManagedDeploymentResolved,+ managedDeploymentResolutionError: mocks.enterpriseResolutionError,+ authenticationState: mocks.enterprise.authenticationState,+ authenticationError: mocks.enterprise.authenticationError,+ isManagedAuthenticated: mocks.enterprise.isManagedAuthenticated,+ selectAuthenticationMethod: mocks.selectAuthenticationMethod,+ submitLicenseKey: mocks.submitLicenseKey,+ }),+}));++vi.mock("posthog-js", () => ({ default: { capture: mocks.capture } }));+vi.mock("@tauri-apps/plugin-shell", () => ({ open: mocks.open }));+vi.mock("@tauri-apps/plugin-os", () => ({+ platform: mocks.platform,+ arch: mocks.arch,+}));++// The resume effect only restarts the engine from the primary window, which it+// detects via getCurrentWindow().label. Stand in as the primary "home" window so+// the sequenced stop -> settle -> spawn actually runs under test.+vi.mock("@tauri-apps/api/window", () => ({+ getCurrentWindow: () => ({ label: mocks.windowLabel }),+}));++import { AppEntitlementGate } from "./app-entitlement-gate";++// Build timestamps relative to the real clock so freshness checks are stable+// without fake timers.+const minsAgo = (m: number) => new Date(Date.now() - m * 60_000).toISOString();+const daysAgo = (d: number) =>+ new Date(Date.now() - d * 86_400_000).toISOString();+const daysAhead = (d: number) =>+ new Date(Date.now() + d * 86_400_000).toISOString();++function baseUser(overrides: Record<string, any> = {}) {+ return {+ id: "free-user",+ token: "tok",+ email: "[email protected]",+ cloud_subscribed: false,+ app_entitled: false,+ subscription_plan: "none",+ entitlement: {+ active: false,+ plan: "none",+ source: "none",+ checked_at: minsAgo(1),+ features: { app: false, cloud: false },+ },+ ...overrides,+ };+}++const protectedApp = <div data-testid="protected-app">app</div>;++describe("Restricted enterprise onboarding outcomes", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ window.history.replaceState({}, "", "/");+ // Production-like env so the dev billing bypass stays off and the gate runs.+ vi.stubEnv("TAURI_ENV_DEBUG", "false");+ vi.stubEnv("NEXT_PUBLIC_SCREENPIPE_DEV_BILLING_BYPASS", "false");+ vi.stubEnv("NEXT_PUBLIC_SCREENPIPE_DEV_LOGIN_SKIP", "false");+ mocks.state = { isSettingsLoaded: true, user: null };+ mocks.enterpriseResolutionError = false;+ mocks.windowLabel = "home";+ mocks.enterprise = {+ isManagedDeployment: false,+ isManagedDeploymentResolved: true,+ authenticationState: "authenticated",+ authenticationError: null,+ isManagedAuthenticated: true,+ };+ });++ afterEach(() => {+ cleanup();+ window.history.replaceState({}, "", "/");+ vi.unstubAllEnvs();+ });++ it("starts the application without an account when authentication is not required", () => {+ render(+ <AppEntitlementGate authenticationStatus="not_required">+ {protectedApp}+ </AppEntitlementGate>,+ );++ expect(screen.getByTestId("protected-app")).toBeInTheDocument();+ expect(mocks.stopScreenpipe).not.toHaveBeenCalled();+ });++ it("leaves enterprise authentication on the onboarding login step", () => {+ window.history.replaceState({}, "", "/onboarding");+ mocks.enterprise = {+ isManagedDeployment: true,+ isManagedDeploymentResolved: true,+ authenticationState: "choice",+ authenticationError: null,+ isManagedAuthenticated: false,+ };++ render(<AppEntitlementGate>{protectedApp}</AppEntitlementGate>);++ expect(screen.getByTestId("protected-app")).toBeInTheDocument();+ expect(screen.queryByText(/enterprise access/i)).not.toBeInTheDocument();+ });++ it("keeps an ordinary signed-in consumer in onboarding", () => {+ window.history.replaceState({}, "", "/onboarding");+ mocks.state.user = baseUser();++ render(+ <AppEntitlementGate authenticationStatus="logged_out">+ {protectedApp}+ </AppEntitlementGate>,+ );++ expect(screen.getByTestId("protected-app")).toBeInTheDocument();+ expect(+ screen.queryByText(/enterprise app required/i),+ ).not.toBeInTheDocument();+ });++ it("gates a restricted enterprise member as soon as onboarding sign-in resolves", () => {+ window.history.replaceState({}, "", "/onboarding");+ mocks.windowLabel = "onboarding";++ const { rerender } = render(+ <AppEntitlementGate authenticationStatus="logged_out">+ {protectedApp}+ </AppEntitlementGate>,+ );++ expect(screen.getByTestId("protected-app")).toBeInTheDocument();++ mocks.state.user = baseUser({+ app_entitled: true,+ cloud_subscribed: true,+ subscription_plan: "pro",+ enterprise_account: {+ org_name: "Example Org",+ role: "member",+ requires_enterprise_app: true,+ restrict_consumer_build_access: true,+ },+ entitlement: {+ active: true,+ plan: "pro",+ source: "subscription",+ checked_at: minsAgo(1),+ features: { app: true, cloud: true },+ },+ });+ rerender(+ <AppEntitlementGate authenticationStatus="logged_out">+ {protectedApp}+ </AppEntitlementGate>,+ );++ expect(screen.getByText(/enterprise app required/i)).toBeInTheDocument();+ expect(screen.queryByTestId("protected-app")).not.toBeInTheDocument();+ });++ it("does not misclassify a cached restricted user before build detection resolves", () => {+ window.history.replaceState({}, "", "/onboarding");+ mocks.state.user = baseUser({+ enterprise_account: {+ org_name: "Example Org",+ role: "member",+ requires_enterprise_app: true,+ restrict_consumer_build_access: true,+ },+ });+ mocks.enterprise.isManagedDeploymentResolved = false;++ const { rerender } = render(+ <AppEntitlementGate authenticationStatus="logged_out">+ {protectedApp}+ </AppEntitlementGate>,+ );++ expect(screen.getByTestId("protected-app")).toBeInTheDocument();+ expect(+ screen.queryByText(/enterprise app required/i),+ ).not.toBeInTheDocument();++ mocks.enterprise = {+ isManagedDeployment: true,+ isManagedDeploymentResolved: true,+ authenticationState: "choice",+ authenticationError: null,+ isManagedAuthenticated: false,+ };+ rerender(+ <AppEntitlementGate authenticationStatus="logged_out">+ {protectedApp}+ </AppEntitlementGate>,+ );++ expect(screen.getByTestId("protected-app")).toBeInTheDocument();+ expect(+ screen.queryByText(/enterprise app required/i),+ ).not.toBeInTheDocument();+ });++ function restrictedUser(overrides: Record<string, any> = {}) {+ return baseUser({+ app_entitled: true, cloud_subscribed: true, subscription_plan: "pro",+ enterprise_account: { org_name: "Example Org", role: "member", requires_enterprise_app: true, restrict_consumer_build_access: true },+ entitlement: { active: true, plan: "pro", source: "subscription", checked_at: minsAgo(1), features: {app:true,cloud:true} },+ ...overrides,+ });+ }+ it("gates a cached restricted identity after detection confirms consumer build", () => {+ window.history.replaceState({}, "", "/onboarding");mocks.windowLabel="onboarding";+ mocks.state.user=restrictedUser();mocks.enterprise.isManagedDeploymentResolved=false;+ const {rerender}=render(<AppEntitlementGate authenticationStatus="logged_out">{protectedApp}</AppEntitlementGate>);+ expect(screen.getByTestId("protected-app")).toBeInTheDocument();expect(mocks.stopScreenpipe).not.toHaveBeenCalled();+ mocks.enterprise.isManagedDeploymentResolved=true;+ rerender(<AppEntitlementGate authenticationStatus="logged_out">{protectedApp}</AppEntitlementGate>);+ expect(screen.getByRole("heading",{name:/enterprise app required/i})).toBeInTheDocument();expect(screen.queryByTestId("protected-app")).not.toBeInTheDocument();+ });+ it("preserves paid consumer access when enterprise policy does not restrict it", () => {+ window.history.replaceState({}, "", "/onboarding");+ mocks.state.user=restrictedUser({enterprise_account:{org_name:"Example Org",role:"member",requires_enterprise_app:true,restrict_consumer_build_access:false}});+ render(<AppEntitlementGate authenticationStatus="logged_out">{protectedApp}</AppEntitlementGate>);+ expect(screen.getByTestId("protected-app")).toBeInTheDocument();expect(mocks.stopScreenpipe).not.toHaveBeenCalled();+ });+ it("preserves tokenless onboarding rather than gating a cached organization", () => {+ window.history.replaceState({}, "", "/onboarding");mocks.state.user=restrictedUser({token:null});+ render(<AppEntitlementGate authenticationStatus="logged_out">{protectedApp}</AppEntitlementGate>);+ expect(screen.getByTestId("protected-app")).toBeInTheDocument();expect(mocks.stopScreenpipe).not.toHaveBeenCalled();+ });+ it("does not infer a required enterprise build from restriction flag alone", () => {+ window.history.replaceState({}, "", "/onboarding");mocks.state.user=restrictedUser({enterprise_account:{org_name:"Example Org",role:"member",requires_enterprise_app:false,restrict_consumer_build_access:true}});+ render(<AppEntitlementGate authenticationStatus="logged_out">{protectedApp}</AppEntitlementGate>);+ expect(screen.getByTestId("protected-app")).toBeInTheDocument();expect(mocks.stopScreenpipe).not.toHaveBeenCalled();+ });+ it("keeps explicit no-auth startup semantics for a cached restricted user", () => {+ window.history.replaceState({}, "", "/onboarding");mocks.state.user=restrictedUser();+ render(<AppEntitlementGate authenticationStatus="not_required">{protectedApp}</AppEntitlementGate>);+ expect(screen.getByTestId("protected-app")).toBeInTheDocument();expect(mocks.stopScreenpipe).not.toHaveBeenCalled();+ });++});