Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
/**
* The engine's browser row source: tables are read as Soup database rows through the
* normalized cache, newest first; `people` come from the contacts query.
*/
import { readRecordsByKeys, selectRecords } from '@app/lib/graphql-cache';
import type {
DatabaseSqlFetchFailure,
RowSource,
} from '@core/database-sql/driver';
import type {
Bin,
Catalog,
Cell,
GqlQuery,
KeyHint,
Page,
Propf,
Row,
Table,
} from '@core/database-sql/generated/types';
import type { DatabaseSqlStepTrace } from '@core/database-sql/trace';
import type { CacheHost } from '@graphql-cache/host/types';
import { buildGraphqlEntitySoupInput } from '@queries/soup/graphql/entity-input';
import {
materializeReconciledSoup,
soupReconciliationBaseline,
} from '@queries/soup/graphql/reconciliation';
import {
type DatabaseRowFieldsFragment,
DatabaseRowFieldsFragmentDoc,
DatabaseRowsDocument,
type DatabaseRowsQuery,
type DatabaseRowsQueryVariables,
type GraphqlDatabaseRowExpr,
type GraphqlEntityFilterAst,
type GraphqlFilterPropertiesExpr,
GroupSoupDocument,
type GroupSoupQuery,
type GroupSoupQueryVariables,
type SoupInput,
type SoupPropertyValueFieldsFragment,
} from '@service-storage/graphql/generated/graphql';
import {
type AnyVariables,
type Client,
createRequest,
type DocumentInput,
getOperationName,
type RequestPolicy,
stringifyDocument,
} from '@urql/core';
import { err, errAsync, ok, okAsync, Result, ResultAsync } from 'neverthrow';
import { match, P } from 'ts-pattern';
import { NIL as NIL_UUID, v5 as uuidV5 } from 'uuid';
/** One person the viewer can see, as the `people` table lists them. */
export interface Person {
id: string;
name: string;
email: string;
}
interface GraphqlRowSourceCapabilities {
/** The app's GraphQL client; its exchanges decide cache or network. */
client: Client;
/** The statement's catalog, for the value kinds of grouped columns. */
catalog: Catalog;
requestPolicy: RequestPolicy;
/** Probe the cache without subscribing to a shared urql operation. */
cache?: Pick<CacheHost, 'readQuery'>;
/** Everyone the viewer can see. */
people: () => Promise<Person[]>;
/**
* Table membership as the local filter index knows it. A read from the
* network records each first page as evidence; a later read with
* `reconcile` set answers a table that fit in one page from the cache's
* index instead, so rows the cache learned about since show up.
*/
membership?: LocalMembership;
}
/** Local reconciliation for first pages, shared across runs of a statement. */
export interface LocalMembership {
host: Pick<CacheHost, 'entityFilter' | 'readRecordsByKeys'>;
/** The last network first page for each Soup input. */
baselines: Map<
string,
{ items: readonly DatabaseRowFieldsFragment[]; complete: boolean }
>;
reconcile: boolean;
}
/** Past this many join values a narrowed filter costs more than it saves. */
const MAX_KEY_HINT_VALUES = 100;
/**
* `people` rows need UUID ids: the RFC 4122 OID namespace names a person's
* row as the engine's own tests do (`Uuid::NAMESPACE_OID`).
*/
const PERSON_ROW_NAMESPACE = '6ba7b812-9dad-11d1-80b4-00c04fd430c8';
type DatabaseRowItem = Extract<
DatabaseRowFieldsFragment,
{ __typename: 'GraphqlSoupDatabaseRow' }
>;
const rowSelection = selectRecords(DatabaseRowFieldsFragmentDoc);
function fetchFailure(message: string): DatabaseSqlFetchFailure {
return { kind: 'fetch', message };
}
/** Anything the source could not read is a fetch failure, in its own words. */
function thrownFetchFailure(thrown: unknown): DatabaseSqlFetchFailure {
return fetchFailure(
thrown instanceof Error ? thrown.message : String(thrown)
);
}
/**
* One GraphQL read inside the step's trace; a GraphQL error or a missing
* answer is a fetch failure.
*/
function graphqlQuery<Data, Variables extends AnyVariables>(
client: Client,
document: DocumentInput<Data, Variables>,
variables: Variables,
requestPolicy: RequestPolicy,
name: string,
step: DatabaseSqlStepTrace,
cache?: Pick<CacheHost, 'readQuery'>
): ResultAsync<Data, DatabaseSqlFetchFailure> {
step.request(stringifyDocument(document), variables);
const network = () =>
ResultAsync.fromPromise(
client
.query(document, variables, {
requestPolicy:
cache && requestPolicy === 'cache-first'
? 'network-only'
: requestPolicy,
fetchOptions: { headers: step.headers() },
})
.toPromise(),
thrownFetchFailure
).andThen((result): Result<Data, DatabaseSqlFetchFailure> => {
if (!result)
return err(fetchFailure(`the ${name} query ended without data`));
if (result.error) return err(fetchFailure(result.error.message));
if (!result.data)
return err(fetchFailure(`the ${name} query returned no data`));
return ok(result.data);
});
// Cache probes must not register an urql operation: a simultaneous refresh
// with the same key would join that probe and never send a network request.
if (
cache &&
(requestPolicy === 'cache-only' || requestPolicy === 'cache-first')
) {
return ResultAsync.fromPromise(
cache.readQuery({
query: stringifyDocument(document),
operationName: getOperationName(
createRequest(document, variables).query
),
variables: variables || undefined,
}),
thrownFetchFailure
)
.orElse(() => okAsync({ kind: 'miss' as const }))
.andThen((result) =>
result.kind === 'hit'
? okAsync(result.data as Data)
: requestPolicy === 'cache-only'
? errAsync(fetchFailure(`the ${name} query returned no data`))
: network()
);
}
return network();
}
export function createGraphqlRowSource({
client,
catalog,
requestPolicy,
cache,
people,
membership,
}: GraphqlRowSourceCapabilities): RowSource {
return {
page: (query, _needs, cursor, limit, step) =>
match(query)
.returnType<ResultAsync<Page, DatabaseSqlFetchFailure>>()
.with({ type: 'soup' }, (soup) =>
soupInput(soup, cursor, limit).asyncAndThen((input) =>
soupPage(client, requestPolicy, input, membership, step, cache)
)
)
.with({ type: 'people' }, ({ ids }) => peoplePage(catalog, people, ids))
.with({ type: 'groupSoup' }, () =>
errAsync(fetchFailure('a grouped query is read as bins, not pages'))
)
.exhaustive(() =>
errAsync(fetchFailure('an unknown query has no pages'))
),
bins: (query, step) =>
match(query)
.returnType<ResultAsync<Bin[], DatabaseSqlFetchFailure>>()
.with({ type: 'groupSoup' }, (grouped) =>
groupBins(client, requestPolicy, catalog, grouped, step, cache)
)
.with({ type: P.union('soup', 'people') }, () =>
errAsync(fetchFailure('only a grouped query has bins'))
)
.exhaustive(() =>
errAsync(fetchFailure('only a grouped query has bins'))
),
};
}
/** Rows of one table and nothing else, with the pushed-down filter. */
function tableFilters(
table: string,
propf: Propf | null,
keyHint: KeyHint | null
): Result<GraphqlEntityFilterAst, DatabaseSqlFetchFailure> {
const base = buildGraphqlEntitySoupInput('DATABASE_ROW', NIL_UUID)?.initial
?.filters;
if (!base)
return err(fetchFailure('a database row Soup input is unavailable'));
let rows: GraphqlDatabaseRowExpr = { literal: { tableId: table } };
let properties = propf ? propertiesExpression(propf) : undefined;
const hint = keyHint ? narrowing(keyHint) : undefined;
if (hint?.kind === 'rows')
rows = { and: { left: rows, right: hint.expression } };
if (hint?.kind === 'properties') {
properties = properties
? { and: { left: properties, right: hint.expression } }
: hint.expression;
}
return ok({
...base,
databaseRowFilter: rows,
...(properties ? { propertiesFilter: properties } : {}),
});
}
function soupInput(
{ table, propf, keyHint }: Extract<GqlQuery, { type: 'soup' }>,
cursor: string | null,
limit: number
): Result<SoupInput, DatabaseSqlFetchFailure> {
if (cursor) {
return ok({
continuation: { cursor, expand: true, sortDirection: 'DESC' },
});
}
return tableFilters(table, propf, keyHint).map((filters) => ({
initial: {
limit,
expand: true,
sortMethod: 'CREATED_AT',
sortDirection: 'DESC',
filters,
},
}));
}
/** The engine's `propf` wire form as the GraphQL properties filter. */
function propertiesExpression(propf: Propf): GraphqlFilterPropertiesExpr {
return match(propf)
.returnType<GraphqlFilterPropertiesExpr>()
.with({ '&': P.nonNullable }, ({ '&': [left, right] }) => ({
and: {
left: propertiesExpression(left),
right: propertiesExpression(right),
},
}))
.with({ '|': P.nonNullable }, ({ '|': [left, right] }) => ({
or: {
left: propertiesExpression(left),
right: propertiesExpression(right),
},
}))
.with({ '!': P.nonNullable }, ({ '!': inner }) => ({
not: propertiesExpression(inner),
}))
.with({ l: P.nonNullable }, ({ l: { pd, v } }) => ({
literal: {
propertyDefinitionId: pd,
value: match(v)
.with({ so: P.string }, ({ so }) => ({ selectOption: so }))
.with({ er: P.string }, ({ er }) => ({ entityRef: er }))
.exhaustive(() => ({ entityRef: NIL_UUID })),
},
}))
.exhaustive(() => ({
literal: {
propertyDefinitionId: NIL_UUID,
value: { entityRef: NIL_UUID },
},
}));
}
/** A balanced OR keeps a long list inside the filter depth limit. */
function balancedOr<Expression>(
items: Expression[],
or: (left: Expression, right: Expression) => Expression
): Expression | undefined {
if (items.length < 2) return items[0];
const middle = Math.floor(items.length / 2);
const left = balancedOr(items.slice(0, middle), or);
const right = balancedOr(items.slice(middle), or);
return left && right ? or(left, right) : (left ?? right);
}
/**
* A filter fetching only the joined rows the join can match. The fold
* applies the join regardless, so a hint that cannot be expressed fetches
* the whole table instead.
*/
function narrowing(
hint: KeyHint
):
| { kind: 'rows'; expression: GraphqlDatabaseRowExpr }
| { kind: 'properties'; expression: GraphqlFilterPropertiesExpr }
| undefined {
type Member = { kind: 'entity' | 'option' | 'other'; id: string };
const members = hint.values.flatMap((value): Member[] =>
match(value)
.returnType<Member[]>()
.with({ type: 'entities' }, ({ value }) =>
value.map((id) => ({ kind: 'entity', id }))
)
.with({ type: 'row' }, ({ value }) => [{ kind: 'entity', id: value }])
.with({ type: 'options' }, ({ value }) =>
value.map((id) => ({ kind: 'option', id }))
)
.otherwise(() => [{ kind: 'other', id: '' }])
);
if (
members.length === 0 ||
members.length > MAX_KEY_HINT_VALUES ||
members.some((member) => member.kind === 'other')
)
return undefined;
const column = hint.column;
if (column === null) {
const expression = balancedOr<GraphqlDatabaseRowExpr>(
members.map(({ id }) => ({ literal: { id } })),
(left, right) => ({ or: { left, right } })
);
return expression && { kind: 'rows', expression };
}
const expression = balancedOr<GraphqlFilterPropertiesExpr>(
members.map(({ kind, id }) => ({
literal: {
propertyDefinitionId: column,
value: kind === 'option' ? { selectOption: id } : { entityRef: id },
},
})),
(left, right) => ({ or: { left, right } })
);
return expression && { kind: 'properties', expression };
}
function soupPage(
client: Client,
requestPolicy: RequestPolicy,
input: SoupInput,
membership: LocalMembership | undefined,
step: DatabaseSqlStepTrace,
cache?: Pick<CacheHost, 'readQuery'>
): ResultAsync<Page, DatabaseSqlFetchFailure> {
const evidence = input.initial ? JSON.stringify(input) : undefined;
const local: ResultAsync<Page | undefined, DatabaseSqlFetchFailure> =
evidence && membership?.reconcile
? reconciledPage(input, evidence, membership, step)
: okAsync(undefined);
return local.andThen((page) =>
page
? okAsync(page)
: graphqlQuery<DatabaseRowsQuery, DatabaseRowsQueryVariables>(
client,
DatabaseRowsDocument,
{ input },
requestPolicy,
'database rows',
step,
cache
).andThen((data) => {
const { items, nextCursor } = data.user.soup;
if (evidence && membership && isNetworkRead(requestPolicy)) {
membership.baselines.set(evidence, {
items,
complete: nextCursor === null,
});
}
return tableRows(items, step).map((rows) => ({
rows,
next: nextCursor,
}));
})
);
}
function isNetworkRead(requestPolicy: RequestPolicy): boolean {
return (
requestPolicy === 'network-only' || requestPolicy === 'cache-and-network'
);
}
/**
* The first page as the local filter index reconciles it with the last
* network page, when that page held the whole table. `undefined` leaves the
* read to the GraphQL client.
*/
function reconciledPage(
input: SoupInput,
evidence: string,
{ host, baselines }: LocalMembership,
step: DatabaseSqlStepTrace
): ResultAsync<Page | undefined, DatabaseSqlFetchFailure> {
const initial = input.initial;
const baseline = baselines.get(evidence);
if (!initial || !baseline?.complete) return okAsync(undefined);
const limit = initial.limit ?? 0;
const baselineKeys = soupReconciliationBaseline(baseline.items, 'CREATED_AT');
if (!baselineKeys) return okAsync(undefined);
return ResultAsync.fromPromise(
host.entityFilter({
filters: initial.filters ?? {},
sortMethod: 'CREATED_AT',
sortDirection: 'DESC',
limit,
baseline: baselineKeys,
}),
thrownFetchFailure
).andThen(
(result): ResultAsync<Page | undefined, DatabaseSqlFetchFailure> => {
if (result.kind !== 'reconciled' && result.kind !== 'complete')
return okAsync(undefined);
// A full page may continue past the limit; only the server's cursor knows.
if (result.keys.length >= limit) return okAsync(undefined);
return ResultAsync.fromPromise(
readRecordsByKeys<DatabaseRowFieldsFragment>(
host,
rowSelection,
result.keys
),
thrownFetchFailure
).andThen(({ records }) =>
tableRows(
materializeReconciledSoup(result.keys, records, baseline.items),
step
).map((rows) => ({ rows, next: null }))
);
}
);
}
function tableRows(
items: readonly DatabaseRowFieldsFragment[],
step: DatabaseSqlStepTrace
): Result<Row[], DatabaseSqlFetchFailure> {
return step.decode(items.length, () =>
Result.combine(
items.map((item) =>
item.__typename === 'GraphqlSoupDatabaseRow'
? ok({ id: item.id, position: item.position, cells: rowCells(item) })
: err(fetchFailure(`a table query returned a ${item.__typename}`))
)
)
);
}
/** A row's cells by property definition; an empty property is no cell. */
export function rowCells(item: DatabaseRowItem): Record<string, Cell> {
const cells: Record<string, Cell> = {};
for (const property of item.properties) {
const value = cell(property.value);
if (value) cells[property.propertyDefinitionId] = value;
}
return cells;
}
/** A property value as the engine reads it, matching the server's source. */
function cell(
value: SoupPropertyValueFieldsFragment | null | undefined
): Cell | undefined {
if (!value) return undefined;
return match(value)
.returnType<Cell | undefined>()
.with({ __typename: 'GraphqlBooleanPropertyValue' }, ({ boolValue }) => ({
type: 'bool',
value: boolValue,
}))
.with({ __typename: 'GraphqlNumberPropertyValue' }, ({ numberValue }) => ({
type: 'number',
value: numberValue,
}))
.with({ __typename: 'GraphqlStringPropertyValue' }, ({ stringValue }) => ({
type: 'text',
value: stringValue,
}))
.with({ __typename: 'GraphqlDatePropertyValue' }, ({ dateValue }) => ({
type: 'date',
value: dateValue,
}))
.with(
{ __typename: 'GraphqlDateRangePropertyValue' },
({ rangeStart, rangeEnd }) => ({
type: 'dateRange',
value: { start: rangeStart, end: rangeEnd },
})
)
.with(
{ __typename: 'GraphqlSelectOptionPropertyValue' },
({ optionIds }) => ({
type: 'options',
value: optionIds,
})
)
.with(
{ __typename: 'GraphqlEntityReferencePropertyValue' },
({ references }) => ({
type: 'entities',
value: references.map((reference) => reference.entityId),
})
)
.with({ __typename: 'GraphqlLinkPropertyValue' }, ({ urls }) => ({
type: 'text',
value: urls.join(' '),
}))
.exhaustive(() => undefined);
}
function groupBins(
client: Client,
requestPolicy: RequestPolicy,
catalog: Catalog,
{ table, propf, groupBy }: Extract<GqlQuery, { type: 'groupSoup' }>,
step: DatabaseSqlStepTrace,
cache?: Pick<CacheHost, 'readQuery'>
): ResultAsync<Bin[], DatabaseSqlFetchFailure> {
const kind = catalog.tables
.find((candidate) => candidate.id === table)
?.columns.find((column) => column.id === groupBy)?.kind;
const binKey = (
key: string
): Result<Cell | null, DatabaseSqlFetchFailure> => {
// Soup files rows with an empty cell under the empty key.
if (key === '') return ok(null);
return match(kind)
.returnType<Result<Cell, DatabaseSqlFetchFailure>>()
.with({ kind: 'select' }, () => ok({ type: 'options', value: [key] }))
.with({ kind: 'entity' }, () => ok({ type: 'entities', value: [key] }))
.otherwise(() =>
err(fetchFailure(`column ${groupBy} cannot be grouped by Soup`))
);
};
return tableFilters(table, propf, null)
.asyncAndThen((filters) =>
graphqlQuery<GroupSoupQuery, GroupSoupQueryVariables>(
client,
GroupSoupDocument,
{
input: {
initial: {
groupBy: {
field: 'PROPERTY',
propertyDefinitionId: groupBy,
entityType: 'DATABASE_ROW',
},
// The bins' totals answer the count; one item each is enough.
limit: 1,
sortMethod: 'CREATED_AT',
filters,
},
},
},
requestPolicy,
'grouped Soup',
step,
cache
)
)
.andThen((data) =>
Result.combine(
data.user.groupSoup.bins.map(({ key, totalCount }) =>
binKey(key).map((cell) => ({ key: cell, count: totalCount }))
)
)
);
}
function peoplePage(
catalog: Catalog,
people: () => Promise<Person[]>,
ids: string[] | null
): ResultAsync<Page, DatabaseSqlFetchFailure> {
const table = catalog.tables.find(
(candidate): candidate is Table => candidate.source === 'people'
);
if (!table) return errAsync(fetchFailure('this catalog has no people table'));
const column = (name: string): Result<string, DatabaseSqlFetchFailure> => {
const found = table.columns.find((candidate) => candidate.name === name);
return found
? ok(found.id)
: err(fetchFailure(`the people table has no ${name} column`));
};
const wanted = ids ? new Set(ids) : undefined;
return Result.combine([
column('id'),
column('name'),
column('email'),
]).asyncAndThen(([id, name, email]) =>
ResultAsync.fromPromise(people(), thrownFetchFailure).map(
(everyone): Page => ({
rows: everyone
.filter((person) => !wanted || wanted.has(person.id))
.map((person) => ({
id: uuidV5(person.id, PERSON_ROW_NAMESPACE),
cells: {
[id]: { type: 'entities', value: [person.id] },
[name]: { type: 'text', value: person.name },
[email]: { type: 'text', value: person.email },
},
})),
next: null,
})
)
);
}