fix: statement_timeout scoped to search, upload-raw writes pointer, publish inlines marked.js

1. statement_timeout: 8s moved from global connection config to
   searchKeyword/searchVector only. Prevents DoS on search without
   killing embed --all or bulk imports that need longer than 8s.

2. upload-raw now writes the .redirect.yaml pointer file to disk
   (was creating the pointer object but never calling writeFileSync).

3. publish inlines marked.js from node_modules instead of loading
   from cdn.jsdelivr.net. Generated HTML is now truly self-contained
   with no external dependencies.

4. v0.9.1 migration doc updated with slug authority breaking change
   warning for brains that use frontmatter slug: overrides.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-04-12 13:23:55 -10:00
co-authored by Claude Opus 4.6
parent fa62e61994
commit 004ac6c66f
8 changed files with 86 additions and 52 deletions
+3
View File
@@ -10,6 +10,7 @@
"@electric-sql/pglite": "^0.4.4",
"@modelcontextprotocol/sdk": "^1.0.0",
"gray-matter": "^4.0.3",
"marked": "^18.0.0",
"openai": "^4.0.0",
"pgvector": "^0.2.0",
"postgres": "^3.4.0",
@@ -358,6 +359,8 @@
"kind-of": ["kind-of@6.0.3", "", {}, "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw=="],
"marked": ["marked@18.0.0", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-2e7Qiv/HJSXj8rDEpgTvGKsP8yYtI9xXHKDnrftrmnrJPaFNM7VRb2YCzWaX4BP1iCJ/XPduzDJZMFoqTCcIMA=="],
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
"media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="],
+1
View File
@@ -34,6 +34,7 @@
"@electric-sql/pglite": "^0.4.4",
"@modelcontextprotocol/sdk": "^1.0.0",
"gray-matter": "^4.0.3",
"marked": "^18.0.0",
"openai": "^4.0.0",
"pgvector": "^0.2.0",
"postgres": "^3.4.0"
+13
View File
@@ -20,6 +20,19 @@ The content hash now includes all page fields (title, type, frontmatter), not
just compiled_truth + timeline. This means every existing page's hash will differ
on next sync, triggering a full re-import. This is expected and correct.
### Slug authority (BREAKING for some brains)
The file path is now the source of truth for slugs. If a markdown file has a
frontmatter `slug:` field that disagrees with its path, the import is REJECTED
(skipped with an error). This prevents page-hijack attacks in shared brains.
**If your brain uses frontmatter `slug:` overrides intentionally**, those pages
will be skipped on next sync. Fix by either:
- Removing the `slug:` line from frontmatter (let the path determine the slug)
- Moving the file to match the declared slug
Run `gbrain sync` and check for "Skipped ... does not match path-derived slug"
errors. Fix any before proceeding.
### Search limit ceiling
Search results are now capped at 100. If your agent requests more, it gets a
warning and 100 results. Use the new `--offset` flag to paginate.
+4 -3
View File
@@ -240,9 +240,10 @@ async function uploadRaw(args: string[]) {
uploaded: new Date().toISOString(),
...(fileType ? { type: fileType } : {}),
});
// Write pointer next to the page that references it
pointerPath = `${pageSlug}/${filename}.redirect.yaml`;
console.error(`Pointer: ${pointerPath}`);
// Write pointer next to the original file
pointerPath = filePath + '.redirect.yaml';
writeFileSync(pointerPath, pointer);
console.error(`Pointer written: ${pointerPath}`);
}
// Record in DB
+7 -2
View File
@@ -14,7 +14,12 @@
import { readFileSync, writeFileSync, mkdirSync } from 'fs';
import { randomBytes, createCipheriv, pbkdf2Sync } from 'crypto';
import { dirname, basename } from 'path';
import { dirname, basename, join } from 'path';
import { createRequire } from 'module';
// Inline marked.js so published HTML is truly self-contained (no CDN dependency)
const require = createRequire(import.meta.url);
const MARKED_JS = readFileSync(join(dirname(require.resolve('marked')), 'marked.umd.js'), 'utf8');
// ── Content stripping ──────────────────────────────────────────────
@@ -314,7 +319,7 @@ export function generateHtml({ title, markdown, encrypted }: GenerateHtmlOptions
${passwordHtml}
<div id="content"></div>
${encryptedVars}
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"><\/script>
<script>${MARKED_JS}<\/script>
${contentScript}
</body>
</html>`;
-1
View File
@@ -39,7 +39,6 @@ export async function connect(config: EngineConfig): Promise<void> {
max: 10,
idle_timeout: 20,
connect_timeout: 10,
connection: { statement_timeout: '8s' },
types: {
// Register pgvector type
bigint: postgres.BigInt,
+55 -44
View File
@@ -188,34 +188,40 @@ export class PostgresEngine implements BrainEngine {
console.warn(`[gbrain] Warning: search limit clamped from ${opts.limit} to ${MAX_SEARCH_LIMIT}`);
}
// CTE: rank pages by FTS score, then pick the best chunk per page in SQL
const rows = await sql`
WITH ranked_pages AS (
SELECT p.id, p.slug, p.title, p.type,
ts_rank(p.search_vector, websearch_to_tsquery('english', ${query})) AS score
FROM pages p
WHERE p.search_vector @@ websearch_to_tsquery('english', ${query})
${type ? sql`AND p.type = ${type}` : sql``}
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
// Search-only timeout: prevents DoS via expensive queries without
// affecting long-running operations like embed --all or bulk import
await sql`SET statement_timeout = '8s'`;
try {
// CTE: rank pages by FTS score, then pick the best chunk per page in SQL
const rows = await sql`
WITH ranked_pages AS (
SELECT p.id, p.slug, p.title, p.type,
ts_rank(p.search_vector, websearch_to_tsquery('english', ${query})) AS score
FROM pages p
WHERE p.search_vector @@ websearch_to_tsquery('english', ${query})
${type ? sql`AND p.type = ${type}` : sql``}
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
ORDER BY score DESC
LIMIT ${limit}
OFFSET ${offset}
),
best_chunks AS (
SELECT DISTINCT ON (rp.slug)
rp.slug, rp.id as page_id, rp.title, rp.type, rp.score,
cc.chunk_text, cc.chunk_source
FROM ranked_pages rp
JOIN content_chunks cc ON cc.page_id = rp.id
ORDER BY rp.slug, cc.chunk_index
)
SELECT slug, page_id, title, type, chunk_text, chunk_source, score,
false AS stale
FROM best_chunks
ORDER BY score DESC
LIMIT ${limit}
OFFSET ${offset}
),
best_chunks AS (
SELECT DISTINCT ON (rp.slug)
rp.slug, rp.id as page_id, rp.title, rp.type, rp.score,
cc.chunk_text, cc.chunk_source
FROM ranked_pages rp
JOIN content_chunks cc ON cc.page_id = rp.id
ORDER BY rp.slug, cc.chunk_index
)
SELECT slug, page_id, title, type, chunk_text, chunk_source, score,
false AS stale
FROM best_chunks
ORDER BY score DESC
`;
return rows.map(rowToSearchResult);
`;
return rows.map(rowToSearchResult);
} finally {
await sql`SET statement_timeout = '0'`;
}
}
async searchVector(embedding: Float32Array, opts?: SearchOpts): Promise<SearchResult[]> {
@@ -231,23 +237,28 @@ export class PostgresEngine implements BrainEngine {
const vecStr = '[' + Array.from(embedding).join(',') + ']';
const rows = await sql`
SELECT
p.slug, p.id as page_id, p.title, p.type,
cc.chunk_text, cc.chunk_source,
1 - (cc.embedding <=> ${vecStr}::vector) AS score,
false AS stale
FROM content_chunks cc
JOIN pages p ON p.id = cc.page_id
WHERE cc.embedding IS NOT NULL
${type ? sql`AND p.type = ${type}` : sql``}
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
ORDER BY cc.embedding <=> ${vecStr}::vector
LIMIT ${limit}
OFFSET ${offset}
`;
return rows.map(rowToSearchResult);
// Search-only timeout (see searchKeyword for rationale)
await sql`SET statement_timeout = '8s'`;
try {
const rows = await sql`
SELECT
p.slug, p.id as page_id, p.title, p.type,
cc.chunk_text, cc.chunk_source,
1 - (cc.embedding <=> ${vecStr}::vector) AS score,
false AS stale
FROM content_chunks cc
JOIN pages p ON p.id = cc.page_id
WHERE cc.embedding IS NOT NULL
${type ? sql`AND p.type = ${type}` : sql``}
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
ORDER BY cc.embedding <=> ${vecStr}::vector
LIMIT ${limit}
OFFSET ${offset}
`;
return rows.map(rowToSearchResult);
} finally {
await sql`SET statement_timeout = '0'`;
}
}
// Chunks
+3 -2
View File
@@ -210,8 +210,9 @@ describe('generateHtml', () => {
expect(html).toContain('prefers-color-scheme: dark');
});
test('includes marked.js CDN', () => {
test('inlines marked.js (no CDN dependency)', () => {
const html = generateHtml({ title: 'T', markdown: 'x' });
expect(html).toContain('cdn.jsdelivr.net/npm/marked');
expect(html).not.toContain('cdn.jsdelivr.net');
expect(html).toContain('marked');
});
});