Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: the fast page filters hide question pages and pinned-question keys like the full filters

garrytanb940908c2c47
diff --git a/src/core/edge-whitespace.ts b/src/core/edge-whitespace.ts
index 678cfef2fd..891bff8630 100644
--- a/src/core/edge-whitespace.ts
+++ b/src/core/edge-whitespace.ts
@@ -2,7 +2,7 @@
* The whitespace JavaScript's `String.prototype.trim` strips (WhiteSpace and LineTerminator), for SQL twins of
* TypeScript predicates. Postgres `btrim(x)` with no set strips spaces only, so a twin that trims there disagrees with
* its TypeScript side on a tab, newline, NBSP or other edge whitespace. Twins trim with edgeTrimSql instead;
- * test/edge-whitespace.test.ts holds the set equal to what `trim` strips.
+ * test/edge-whitespace-twins.test.ts holds the set equal to what `trim` strips.
*/
export const EDGE_WHITESPACE = ' \t\n\v\f\r\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff';
diff --git a/src/core/repair/private-labels.ts b/src/core/repair/private-labels.ts
index f95edac77a..6b13a76c38 100644
--- a/src/core/repair/private-labels.ts
+++ b/src/core/repair/private-labels.ts
@@ -48,6 +48,7 @@ import { importFromContent } from '../import-file.ts';
import { isSegmentedPage } from '../page-body/page-shape.ts';
import { clearApprovedSet, loadApprovedSet, previewChangedError, previewHash, saveApprovedSet } from '../persistence/preview-approval.ts';
import { PRIVACY_ALIASES, privacyAliasSql } from '../search/private-visibility.ts';
+import { asciiLower } from '../edge-whitespace.ts';
import { afterCursor, repairRequestId, type RepairHandler, type RepairItem, type RepairItemOutcome, type RepairPlan, type RepairPlanOptions, type RepairScope } from './core.ts';
export type ClearClass = 'alias' | 'alias-and-visibility' | 'git-publish';
@@ -77,7 +78,7 @@ type Decision = { keep: KeepClass } | { clear: ClearClass; removed: Labels } | n
/** KEEP or CLEAR for a page that is private by its label; null for a page no label makes private. */
export function classifyPrivateLabel(row: Pick<LabelRow, 'type' | 'frontmatter' | 'git'>): Decision {
const fm = row.frontmatter ?? {};
- const value = (key: string) => typeof fm[key] === 'string' ? (fm[key] as string).trim().toLowerCase() : null;
+ const value = (key: string) => typeof fm[key] === 'string' ? asciiLower((fm[key] as string).trim()) : null;
const aliased = Object.entries(PRIVACY_ALIASES).some(([key, values]) => values.includes(value(key) ?? ''));
if (!(fm.visibility === 'private' || (fm.visibility == null && aliased))) return null;
if (Object.hasOwn(fm, 'pinned_question') || row.type === 'question') return { keep: 'pinned_question' };
diff --git a/src/core/search/private-visibility.ts b/src/core/search/private-visibility.ts
index 1ad125597b..bce7757bb1 100644
--- a/src/core/search/private-visibility.ts
+++ b/src/core/search/private-visibility.ts
@@ -173,15 +173,16 @@ function originAllowedSql(p: string): string {
/**
* `privatePagesFilterFragment` with a cheap first test: a page with no
* `visibility` or `derived_from` key (nor, unless the operator turned
- * `visibility.honor_aliases` off, a privacy-alias key) whose type is not atom, concept or event is visible by every rule
+ * `visibility.honor_aliases` off, a privacy-alias key, nor `readable_by` or `pinned_question`) whose type is not atom, concept,
+ * event or question is visible by every rule
* above, so only the remaining pages run the full check. For scans over thousands of
* candidate pages (an entity's referrers).
*/
export function privatePagesFilterFragmentFast(pageAlias: string): string {
return `(${folderRulesAllowedSql(pageAlias, [])} AND ((${pageAlias}.frontmatter->'visibility' IS NULL AND ${pageAlias}.frontmatter->'derived_from' IS NULL${honorAliases()
? `\n AND ${pageAlias}.frontmatter->'access' IS NULL AND ${pageAlias}.frontmatter->'confidentiality' IS NULL` : ''}
- AND ${pageAlias}.frontmatter->'readable_by' IS NULL
- AND ${pageAlias}.type NOT IN ('atom', 'concept', 'event'))
+ AND ${pageAlias}.frontmatter->'readable_by' IS NULL AND ${pageAlias}.frontmatter->'pinned_question' IS NULL
+ AND ${pageAlias}.type NOT IN ('atom', 'concept', 'event', 'question'))
OR ${privatePagesFilterFragment(pageAlias)}))`;
}
@@ -393,7 +394,8 @@ export function privatePagesFilterFastForCaller(pageAlias: string): string {
function privacyPartFastSql(pageAlias: string): string {
return `((${pageAlias}.frontmatter->'visibility' IS NULL AND ${pageAlias}.frontmatter->'derived_from' IS NULL${honorAliases()
? `\n AND ${pageAlias}.frontmatter->'access' IS NULL AND ${pageAlias}.frontmatter->'confidentiality' IS NULL` : ''}
- AND ${pageAlias}.type NOT IN ('atom', 'concept', 'event'))
+ AND ${pageAlias}.frontmatter->'pinned_question' IS NULL
+ AND ${pageAlias}.type NOT IN ('atom', 'concept', 'event', 'question'))
OR ${privacyPartSql(pageAlias)})`;
}
diff --git a/src/core/trust/channel.ts b/src/core/trust/channel.ts
index fc2ff36979..3024da1418 100644
--- a/src/core/trust/channel.ts
+++ b/src/core/trust/channel.ts
@@ -24,7 +24,7 @@
import type { BrainEngine } from '../engine.ts';
import type { WriteRequest } from '../persistence/model.ts';
import { CONNECTOR_SOURCE_KINDS } from '../persistence/connector-identity.ts';
-import { edgeTrimSql } from '../edge-whitespace.ts';
+import { asciiLower, asciiLowerSql, edgeTrimSql } from '../edge-whitespace.ts';
import {
CONTENT_ORIGINS, OWNER_TIER_FLOOR, TRUST_TIER_RANK, compareTrust, contentOriginTier, effectiveWriteTrust, isTrustTier, minTrust, trustRankSql,
type ContentOrigin, type TrustTier, type WriteTrust,
@@ -137,7 +137,7 @@ export function frontmatterTrustCaps(frontmatter: Record<string, unknown> | null
}
const transcript = fm.transcript_import as { harness?: unknown } | undefined;
if (transcript && typeof transcript === 'object') {
- caps.push(OWN_SESSION_HARNESSES.includes(str(transcript.harness).toLowerCase()) ? 'agent_written' : 'external_untrusted');
+ caps.push(OWN_SESSION_HARNESSES.includes(asciiLower(str(transcript.harness))) ? 'agent_written' : 'external_untrusted');
}
if (str(fm.source_url) && (str(fm.clipped_at) || str(fm.clipper) || str(fm.captured_via) === 'clipper' || str(fm.type) === 'clipping')) {
caps.push('external_untrusted');
@@ -164,7 +164,7 @@ export function frontmatterTrustCapsSql(fm: string): string[] {
`CASE WHEN jsonb_typeof(${fm}->'trust_tier') = 'string' THEN NULLIF(${trustRankSql(`(${fm}->>'trust_tier')`)}, 0) END`,
...['source_kind', 'ingested_via', 'captured_via'].map(stamp),
`CASE WHEN jsonb_typeof(${fm}->'transcript_import') IN ('object', 'array') THEN
- CASE WHEN lower(${text('harness', `(${fm}->'transcript_import')`)}) IN (${list(OWN_SESSION_HARNESSES)}) THEN ${TRUST_TIER_RANK.agent_written} ELSE ${TRUST_TIER_RANK.external_untrusted} END END`,
+ CASE WHEN ${asciiLowerSql(text('harness', `(${fm}->'transcript_import')`))} IN (${list(OWN_SESSION_HARNESSES)}) THEN ${TRUST_TIER_RANK.agent_written} ELSE ${TRUST_TIER_RANK.external_untrusted} END END`,
`CASE WHEN ${text('source_url')} IS NOT NULL AND (${text('clipped_at')} IS NOT NULL OR ${text('clipper')} IS NOT NULL
OR ${text('captured_via')} = 'clipper' OR ${text('type')} = 'clipping') THEN ${TRUST_TIER_RANK.external_untrusted} END`,
`CASE WHEN ${text('provenance')} = 'auto-extracted' OR ${fm}->'dream_generated' = 'true'::jsonb OR ${text('dream_generated')} = 'true'
diff --git a/test/edge-whitespace-twins.test.ts b/test/edge-whitespace-twins.test.ts
index b20a75989a..c068fd99fa 100644
--- a/test/edge-whitespace-twins.test.ts
+++ b/test/edge-whitespace-twins.test.ts
@@ -22,7 +22,10 @@ import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { databaseOnlyPageSql, isDatabaseOnlyPage } from '../src/core/persistence/unbound-pages.ts';
import { memberIdentityFor, stdioMemberId, STDIO_MEMBER_KEY } from '../src/core/access/caller.ts';
import { addMember } from '../src/core/access/members.ts';
-import { PRIVACY_ALIASES, __resetPrivateVisibilityCacheForTests, isPrivacyPrivatePage, privacyAliasSql } from '../src/core/search/private-visibility.ts';
+import {
+ PRIVACY_ALIASES, __resetPrivateVisibilityCacheForTests, isPrivacyPrivatePage, privacyAliasSql,
+ privatePagesFilterFastForCaller, privatePagesFilterForCaller, privatePagesFilterFragment, privatePagesFilterFragmentFast,
+} from '../src/core/search/private-visibility.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { testBackends } from './helpers/test-backends.ts';
@@ -71,10 +74,10 @@ describe.each(testBackends())('%s', (backend) => {
__resetPrivateVisibilityCacheForTests();
const words = ['private', 'restricted', 'confidential', 'secret', 'public', 'internal', 'privates', ''];
const shapes = (w: string) => [w, w.toUpperCase(), `${w[0]?.toUpperCase() ?? ''}${w.slice(1)}`, ...EDGES.flatMap(c => [`${c}${w}`, `${w}${c}`, `${c}${w.toUpperCase()}${c}`])];
- const frontmatters = Object.keys(PRIVACY_ALIASES).flatMap(key => words.flatMap(w => shapes(w).map(v => ({ [key]: v }))));
+ const frontmatters: Array<Record<string, unknown>> = Object.keys(PRIVACY_ALIASES).flatMap(key => words.flatMap(w => shapes(w).map(v => ({ [key]: v }))));
frontmatters.push({ access: 'PR\u0130VATE' }, { access: 'pri\u200bvate' }, { confidentiality: 'SECRET\u212a' }, { access: ['private'] }, { access: 1 });
const rows = await engine().executeRaw<{ i: number; private: boolean }>(
- `SELECT i, ${privacyAliasSql('p')} AS private FROM jsonb_array_elements($1::jsonb) WITH ORDINALITY AS t(frontmatter, i), LATERAL (SELECT t.frontmatter) AS p`,
+ `SELECT i, ${privacyAliasSql('p')} AS private FROM jsonb_array_elements($1::text::jsonb) WITH ORDINALITY AS t(frontmatter, i), LATERAL (SELECT t.frontmatter) AS p`,
[JSON.stringify(frontmatters)]);
expect(rows).toHaveLength(frontmatters.length);
for (const row of rows) {
@@ -111,6 +114,24 @@ describe.each(testBackends())('%s', (backend) => {
expect(await searched(ctx(false))).toEqual(Object.keys(pages).sort());
}, 120_000);
+ test('the fast page filters hide exactly what the full filters hide', async () => {
+ __resetPrivateVisibilityCacheForTests();
+ const sourceId = `fast-${randomUUID().slice(0, 8)}`;
+ await engine().executeRaw('INSERT INTO sources(id,name) VALUES($1,$1)', [sourceId]);
+ const types = ['note', 'meeting', 'question', 'atom', 'concept', 'event'];
+ const frontmatters: Array<Record<string, unknown>> = [{}, { pinned_question: 'q' }, { visibility: 'private' }, { visibility: 'world' }, { access: 'private\n' },
+ { confidentiality: 'internal' }, { derived_from: 'notes/none-example' }, { readable_by: ['alice-example'] }, { synthesized_by: 'x' }];
+ let n = 0;
+ for (const type of types) for (const frontmatter of frontmatters) {
+ await engine().executeRaw(`INSERT INTO pages(slug,type,title,compiled_truth,source_id,frontmatter) VALUES($1,$2,'T','body',$3,$4::text::jsonb)`,
+ [`fast/p${n++}`, type, sourceId, JSON.stringify(frontmatter)]);
+ }
+ const visible = async (filter: string) => (await engine().executeRaw<{ slug: string }>(
+ `SELECT p.slug FROM pages p WHERE p.source_id = $1 AND ${filter} ORDER BY p.slug`, [sourceId])).map(r => r.slug);
+ expect(await visible(privatePagesFilterFragmentFast('p'))).toEqual(await visible(privatePagesFilterFragment('p')));
+ expect(await visible(privatePagesFilterFastForCaller('p'))).toEqual(await visible(privatePagesFilterForCaller('p')));
+ });
+
test('the stdio member binding trims like stdioMemberId', async () => {
await addMember(engine(), 'carol-example');
try {
diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txt
index 5204d12ad7..1be80ed8e5 100644
--- a/test/postgres-unit-arms.txt
+++ b/test/postgres-unit-arms.txt
@@ -78,6 +78,7 @@ test/doctor-image-embedding-coverage.test.ts
test/doctor-keyword-plan-6495.test.ts
test/doctor-routing-skill-pack-source.test.ts
test/doctor-sync-freshness-skill-pack-6076.test.ts
+test/edge-whitespace-twins.test.ts
test/embed-once-per-content-version.serial.test.ts
test/embed-stale-images-managed-6529.test.ts
test/embedding-completion-atomic.serial.test.ts