mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 21:19:18 +00:00
fix: statement_timeout scoped to search, upload-raw writes pointer, publish inlines marked.js
1. statement_timeout: 8s moved from global connection config to searchKeyword/searchVector only. Prevents DoS on search without killing embed --all or bulk imports that need longer than 8s. 2. upload-raw now writes the .redirect.yaml pointer file to disk (was creating the pointer object but never calling writeFileSync). 3. publish inlines marked.js from node_modules instead of loading from cdn.jsdelivr.net. Generated HTML is now truly self-contained with no external dependencies. 4. v0.9.1 migration doc updated with slug authority breaking change warning for brains that use frontmatter slug: overrides. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
fa62e61994
commit
004ac6c66f
@@ -10,6 +10,7 @@
|
||||
"@electric-sql/pglite": "^0.4.4",
|
||||
"@modelcontextprotocol/sdk": "^1.0.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"marked": "^18.0.0",
|
||||
"openai": "^4.0.0",
|
||||
"pgvector": "^0.2.0",
|
||||
"postgres": "^3.4.0",
|
||||
@@ -358,6 +359,8 @@
|
||||
|
||||
"kind-of": ["kind-of@6.0.3", "", {}, "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw=="],
|
||||
|
||||
"marked": ["marked@18.0.0", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-2e7Qiv/HJSXj8rDEpgTvGKsP8yYtI9xXHKDnrftrmnrJPaFNM7VRb2YCzWaX4BP1iCJ/XPduzDJZMFoqTCcIMA=="],
|
||||
|
||||
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
|
||||
|
||||
"media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="],
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
"@electric-sql/pglite": "^0.4.4",
|
||||
"@modelcontextprotocol/sdk": "^1.0.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"marked": "^18.0.0",
|
||||
"openai": "^4.0.0",
|
||||
"pgvector": "^0.2.0",
|
||||
"postgres": "^3.4.0"
|
||||
|
||||
@@ -20,6 +20,19 @@ The content hash now includes all page fields (title, type, frontmatter), not
|
||||
just compiled_truth + timeline. This means every existing page's hash will differ
|
||||
on next sync, triggering a full re-import. This is expected and correct.
|
||||
|
||||
### Slug authority (BREAKING for some brains)
|
||||
The file path is now the source of truth for slugs. If a markdown file has a
|
||||
frontmatter `slug:` field that disagrees with its path, the import is REJECTED
|
||||
(skipped with an error). This prevents page-hijack attacks in shared brains.
|
||||
|
||||
**If your brain uses frontmatter `slug:` overrides intentionally**, those pages
|
||||
will be skipped on next sync. Fix by either:
|
||||
- Removing the `slug:` line from frontmatter (let the path determine the slug)
|
||||
- Moving the file to match the declared slug
|
||||
|
||||
Run `gbrain sync` and check for "Skipped ... does not match path-derived slug"
|
||||
errors. Fix any before proceeding.
|
||||
|
||||
### Search limit ceiling
|
||||
Search results are now capped at 100. If your agent requests more, it gets a
|
||||
warning and 100 results. Use the new `--offset` flag to paginate.
|
||||
|
||||
@@ -240,9 +240,10 @@ async function uploadRaw(args: string[]) {
|
||||
uploaded: new Date().toISOString(),
|
||||
...(fileType ? { type: fileType } : {}),
|
||||
});
|
||||
// Write pointer next to the page that references it
|
||||
pointerPath = `${pageSlug}/${filename}.redirect.yaml`;
|
||||
console.error(`Pointer: ${pointerPath}`);
|
||||
// Write pointer next to the original file
|
||||
pointerPath = filePath + '.redirect.yaml';
|
||||
writeFileSync(pointerPath, pointer);
|
||||
console.error(`Pointer written: ${pointerPath}`);
|
||||
}
|
||||
|
||||
// Record in DB
|
||||
|
||||
@@ -14,7 +14,12 @@
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync } from 'fs';
|
||||
import { randomBytes, createCipheriv, pbkdf2Sync } from 'crypto';
|
||||
import { dirname, basename } from 'path';
|
||||
import { dirname, basename, join } from 'path';
|
||||
import { createRequire } from 'module';
|
||||
|
||||
// Inline marked.js so published HTML is truly self-contained (no CDN dependency)
|
||||
const require = createRequire(import.meta.url);
|
||||
const MARKED_JS = readFileSync(join(dirname(require.resolve('marked')), 'marked.umd.js'), 'utf8');
|
||||
|
||||
// ── Content stripping ──────────────────────────────────────────────
|
||||
|
||||
@@ -314,7 +319,7 @@ export function generateHtml({ title, markdown, encrypted }: GenerateHtmlOptions
|
||||
${passwordHtml}
|
||||
<div id="content"></div>
|
||||
${encryptedVars}
|
||||
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"><\/script>
|
||||
<script>${MARKED_JS}<\/script>
|
||||
${contentScript}
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
@@ -39,7 +39,6 @@ export async function connect(config: EngineConfig): Promise<void> {
|
||||
max: 10,
|
||||
idle_timeout: 20,
|
||||
connect_timeout: 10,
|
||||
connection: { statement_timeout: '8s' },
|
||||
types: {
|
||||
// Register pgvector type
|
||||
bigint: postgres.BigInt,
|
||||
|
||||
+55
-44
@@ -188,34 +188,40 @@ export class PostgresEngine implements BrainEngine {
|
||||
console.warn(`[gbrain] Warning: search limit clamped from ${opts.limit} to ${MAX_SEARCH_LIMIT}`);
|
||||
}
|
||||
|
||||
// CTE: rank pages by FTS score, then pick the best chunk per page in SQL
|
||||
const rows = await sql`
|
||||
WITH ranked_pages AS (
|
||||
SELECT p.id, p.slug, p.title, p.type,
|
||||
ts_rank(p.search_vector, websearch_to_tsquery('english', ${query})) AS score
|
||||
FROM pages p
|
||||
WHERE p.search_vector @@ websearch_to_tsquery('english', ${query})
|
||||
${type ? sql`AND p.type = ${type}` : sql``}
|
||||
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
||||
// Search-only timeout: prevents DoS via expensive queries without
|
||||
// affecting long-running operations like embed --all or bulk import
|
||||
await sql`SET statement_timeout = '8s'`;
|
||||
try {
|
||||
// CTE: rank pages by FTS score, then pick the best chunk per page in SQL
|
||||
const rows = await sql`
|
||||
WITH ranked_pages AS (
|
||||
SELECT p.id, p.slug, p.title, p.type,
|
||||
ts_rank(p.search_vector, websearch_to_tsquery('english', ${query})) AS score
|
||||
FROM pages p
|
||||
WHERE p.search_vector @@ websearch_to_tsquery('english', ${query})
|
||||
${type ? sql`AND p.type = ${type}` : sql``}
|
||||
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
||||
ORDER BY score DESC
|
||||
LIMIT ${limit}
|
||||
OFFSET ${offset}
|
||||
),
|
||||
best_chunks AS (
|
||||
SELECT DISTINCT ON (rp.slug)
|
||||
rp.slug, rp.id as page_id, rp.title, rp.type, rp.score,
|
||||
cc.chunk_text, cc.chunk_source
|
||||
FROM ranked_pages rp
|
||||
JOIN content_chunks cc ON cc.page_id = rp.id
|
||||
ORDER BY rp.slug, cc.chunk_index
|
||||
)
|
||||
SELECT slug, page_id, title, type, chunk_text, chunk_source, score,
|
||||
false AS stale
|
||||
FROM best_chunks
|
||||
ORDER BY score DESC
|
||||
LIMIT ${limit}
|
||||
OFFSET ${offset}
|
||||
),
|
||||
best_chunks AS (
|
||||
SELECT DISTINCT ON (rp.slug)
|
||||
rp.slug, rp.id as page_id, rp.title, rp.type, rp.score,
|
||||
cc.chunk_text, cc.chunk_source
|
||||
FROM ranked_pages rp
|
||||
JOIN content_chunks cc ON cc.page_id = rp.id
|
||||
ORDER BY rp.slug, cc.chunk_index
|
||||
)
|
||||
SELECT slug, page_id, title, type, chunk_text, chunk_source, score,
|
||||
false AS stale
|
||||
FROM best_chunks
|
||||
ORDER BY score DESC
|
||||
`;
|
||||
|
||||
return rows.map(rowToSearchResult);
|
||||
`;
|
||||
return rows.map(rowToSearchResult);
|
||||
} finally {
|
||||
await sql`SET statement_timeout = '0'`;
|
||||
}
|
||||
}
|
||||
|
||||
async searchVector(embedding: Float32Array, opts?: SearchOpts): Promise<SearchResult[]> {
|
||||
@@ -231,23 +237,28 @@ export class PostgresEngine implements BrainEngine {
|
||||
|
||||
const vecStr = '[' + Array.from(embedding).join(',') + ']';
|
||||
|
||||
const rows = await sql`
|
||||
SELECT
|
||||
p.slug, p.id as page_id, p.title, p.type,
|
||||
cc.chunk_text, cc.chunk_source,
|
||||
1 - (cc.embedding <=> ${vecStr}::vector) AS score,
|
||||
false AS stale
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NOT NULL
|
||||
${type ? sql`AND p.type = ${type}` : sql``}
|
||||
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
||||
ORDER BY cc.embedding <=> ${vecStr}::vector
|
||||
LIMIT ${limit}
|
||||
OFFSET ${offset}
|
||||
`;
|
||||
|
||||
return rows.map(rowToSearchResult);
|
||||
// Search-only timeout (see searchKeyword for rationale)
|
||||
await sql`SET statement_timeout = '8s'`;
|
||||
try {
|
||||
const rows = await sql`
|
||||
SELECT
|
||||
p.slug, p.id as page_id, p.title, p.type,
|
||||
cc.chunk_text, cc.chunk_source,
|
||||
1 - (cc.embedding <=> ${vecStr}::vector) AS score,
|
||||
false AS stale
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NOT NULL
|
||||
${type ? sql`AND p.type = ${type}` : sql``}
|
||||
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
||||
ORDER BY cc.embedding <=> ${vecStr}::vector
|
||||
LIMIT ${limit}
|
||||
OFFSET ${offset}
|
||||
`;
|
||||
return rows.map(rowToSearchResult);
|
||||
} finally {
|
||||
await sql`SET statement_timeout = '0'`;
|
||||
}
|
||||
}
|
||||
|
||||
// Chunks
|
||||
|
||||
@@ -210,8 +210,9 @@ describe('generateHtml', () => {
|
||||
expect(html).toContain('prefers-color-scheme: dark');
|
||||
});
|
||||
|
||||
test('includes marked.js CDN', () => {
|
||||
test('inlines marked.js (no CDN dependency)', () => {
|
||||
const html = generateHtml({ title: 'T', markdown: 'x' });
|
||||
expect(html).toContain('cdn.jsdelivr.net/npm/marked');
|
||||
expect(html).not.toContain('cdn.jsdelivr.net');
|
||||
expect(html).toContain('marked');
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user