mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
fix: statement_timeout scoped to search, upload-raw writes pointer, publish inlines marked.js
1. statement_timeout: 8s moved from global connection config to searchKeyword/searchVector only. Prevents DoS on search without killing embed --all or bulk imports that need longer than 8s. 2. upload-raw now writes the .redirect.yaml pointer file to disk (was creating the pointer object but never calling writeFileSync). 3. publish inlines marked.js from node_modules instead of loading from cdn.jsdelivr.net. Generated HTML is now truly self-contained with no external dependencies. 4. v0.9.1 migration doc updated with slug authority breaking change warning for brains that use frontmatter slug: overrides. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
fa62e61994
commit
004ac6c66f
@@ -10,6 +10,7 @@
|
|||||||
"@electric-sql/pglite": "^0.4.4",
|
"@electric-sql/pglite": "^0.4.4",
|
||||||
"@modelcontextprotocol/sdk": "^1.0.0",
|
"@modelcontextprotocol/sdk": "^1.0.0",
|
||||||
"gray-matter": "^4.0.3",
|
"gray-matter": "^4.0.3",
|
||||||
|
"marked": "^18.0.0",
|
||||||
"openai": "^4.0.0",
|
"openai": "^4.0.0",
|
||||||
"pgvector": "^0.2.0",
|
"pgvector": "^0.2.0",
|
||||||
"postgres": "^3.4.0",
|
"postgres": "^3.4.0",
|
||||||
@@ -358,6 +359,8 @@
|
|||||||
|
|
||||||
"kind-of": ["kind-of@6.0.3", "", {}, "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw=="],
|
"kind-of": ["kind-of@6.0.3", "", {}, "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw=="],
|
||||||
|
|
||||||
|
"marked": ["marked@18.0.0", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-2e7Qiv/HJSXj8rDEpgTvGKsP8yYtI9xXHKDnrftrmnrJPaFNM7VRb2YCzWaX4BP1iCJ/XPduzDJZMFoqTCcIMA=="],
|
||||||
|
|
||||||
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
|
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
|
||||||
|
|
||||||
"media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="],
|
"media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="],
|
||||||
|
|||||||
@@ -34,6 +34,7 @@
|
|||||||
"@electric-sql/pglite": "^0.4.4",
|
"@electric-sql/pglite": "^0.4.4",
|
||||||
"@modelcontextprotocol/sdk": "^1.0.0",
|
"@modelcontextprotocol/sdk": "^1.0.0",
|
||||||
"gray-matter": "^4.0.3",
|
"gray-matter": "^4.0.3",
|
||||||
|
"marked": "^18.0.0",
|
||||||
"openai": "^4.0.0",
|
"openai": "^4.0.0",
|
||||||
"pgvector": "^0.2.0",
|
"pgvector": "^0.2.0",
|
||||||
"postgres": "^3.4.0"
|
"postgres": "^3.4.0"
|
||||||
|
|||||||
@@ -20,6 +20,19 @@ The content hash now includes all page fields (title, type, frontmatter), not
|
|||||||
just compiled_truth + timeline. This means every existing page's hash will differ
|
just compiled_truth + timeline. This means every existing page's hash will differ
|
||||||
on next sync, triggering a full re-import. This is expected and correct.
|
on next sync, triggering a full re-import. This is expected and correct.
|
||||||
|
|
||||||
|
### Slug authority (BREAKING for some brains)
|
||||||
|
The file path is now the source of truth for slugs. If a markdown file has a
|
||||||
|
frontmatter `slug:` field that disagrees with its path, the import is REJECTED
|
||||||
|
(skipped with an error). This prevents page-hijack attacks in shared brains.
|
||||||
|
|
||||||
|
**If your brain uses frontmatter `slug:` overrides intentionally**, those pages
|
||||||
|
will be skipped on next sync. Fix by either:
|
||||||
|
- Removing the `slug:` line from frontmatter (let the path determine the slug)
|
||||||
|
- Moving the file to match the declared slug
|
||||||
|
|
||||||
|
Run `gbrain sync` and check for "Skipped ... does not match path-derived slug"
|
||||||
|
errors. Fix any before proceeding.
|
||||||
|
|
||||||
### Search limit ceiling
|
### Search limit ceiling
|
||||||
Search results are now capped at 100. If your agent requests more, it gets a
|
Search results are now capped at 100. If your agent requests more, it gets a
|
||||||
warning and 100 results. Use the new `--offset` flag to paginate.
|
warning and 100 results. Use the new `--offset` flag to paginate.
|
||||||
|
|||||||
@@ -240,9 +240,10 @@ async function uploadRaw(args: string[]) {
|
|||||||
uploaded: new Date().toISOString(),
|
uploaded: new Date().toISOString(),
|
||||||
...(fileType ? { type: fileType } : {}),
|
...(fileType ? { type: fileType } : {}),
|
||||||
});
|
});
|
||||||
// Write pointer next to the page that references it
|
// Write pointer next to the original file
|
||||||
pointerPath = `${pageSlug}/${filename}.redirect.yaml`;
|
pointerPath = filePath + '.redirect.yaml';
|
||||||
console.error(`Pointer: ${pointerPath}`);
|
writeFileSync(pointerPath, pointer);
|
||||||
|
console.error(`Pointer written: ${pointerPath}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record in DB
|
// Record in DB
|
||||||
|
|||||||
@@ -14,7 +14,12 @@
|
|||||||
|
|
||||||
import { readFileSync, writeFileSync, mkdirSync } from 'fs';
|
import { readFileSync, writeFileSync, mkdirSync } from 'fs';
|
||||||
import { randomBytes, createCipheriv, pbkdf2Sync } from 'crypto';
|
import { randomBytes, createCipheriv, pbkdf2Sync } from 'crypto';
|
||||||
import { dirname, basename } from 'path';
|
import { dirname, basename, join } from 'path';
|
||||||
|
import { createRequire } from 'module';
|
||||||
|
|
||||||
|
// Inline marked.js so published HTML is truly self-contained (no CDN dependency)
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
const MARKED_JS = readFileSync(join(dirname(require.resolve('marked')), 'marked.umd.js'), 'utf8');
|
||||||
|
|
||||||
// ── Content stripping ──────────────────────────────────────────────
|
// ── Content stripping ──────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -314,7 +319,7 @@ export function generateHtml({ title, markdown, encrypted }: GenerateHtmlOptions
|
|||||||
${passwordHtml}
|
${passwordHtml}
|
||||||
<div id="content"></div>
|
<div id="content"></div>
|
||||||
${encryptedVars}
|
${encryptedVars}
|
||||||
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"><\/script>
|
<script>${MARKED_JS}<\/script>
|
||||||
${contentScript}
|
${contentScript}
|
||||||
</body>
|
</body>
|
||||||
</html>`;
|
</html>`;
|
||||||
|
|||||||
@@ -39,7 +39,6 @@ export async function connect(config: EngineConfig): Promise<void> {
|
|||||||
max: 10,
|
max: 10,
|
||||||
idle_timeout: 20,
|
idle_timeout: 20,
|
||||||
connect_timeout: 10,
|
connect_timeout: 10,
|
||||||
connection: { statement_timeout: '8s' },
|
|
||||||
types: {
|
types: {
|
||||||
// Register pgvector type
|
// Register pgvector type
|
||||||
bigint: postgres.BigInt,
|
bigint: postgres.BigInt,
|
||||||
|
|||||||
+55
-44
@@ -188,34 +188,40 @@ export class PostgresEngine implements BrainEngine {
|
|||||||
console.warn(`[gbrain] Warning: search limit clamped from ${opts.limit} to ${MAX_SEARCH_LIMIT}`);
|
console.warn(`[gbrain] Warning: search limit clamped from ${opts.limit} to ${MAX_SEARCH_LIMIT}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// CTE: rank pages by FTS score, then pick the best chunk per page in SQL
|
// Search-only timeout: prevents DoS via expensive queries without
|
||||||
const rows = await sql`
|
// affecting long-running operations like embed --all or bulk import
|
||||||
WITH ranked_pages AS (
|
await sql`SET statement_timeout = '8s'`;
|
||||||
SELECT p.id, p.slug, p.title, p.type,
|
try {
|
||||||
ts_rank(p.search_vector, websearch_to_tsquery('english', ${query})) AS score
|
// CTE: rank pages by FTS score, then pick the best chunk per page in SQL
|
||||||
FROM pages p
|
const rows = await sql`
|
||||||
WHERE p.search_vector @@ websearch_to_tsquery('english', ${query})
|
WITH ranked_pages AS (
|
||||||
${type ? sql`AND p.type = ${type}` : sql``}
|
SELECT p.id, p.slug, p.title, p.type,
|
||||||
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
ts_rank(p.search_vector, websearch_to_tsquery('english', ${query})) AS score
|
||||||
|
FROM pages p
|
||||||
|
WHERE p.search_vector @@ websearch_to_tsquery('english', ${query})
|
||||||
|
${type ? sql`AND p.type = ${type}` : sql``}
|
||||||
|
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
||||||
|
ORDER BY score DESC
|
||||||
|
LIMIT ${limit}
|
||||||
|
OFFSET ${offset}
|
||||||
|
),
|
||||||
|
best_chunks AS (
|
||||||
|
SELECT DISTINCT ON (rp.slug)
|
||||||
|
rp.slug, rp.id as page_id, rp.title, rp.type, rp.score,
|
||||||
|
cc.chunk_text, cc.chunk_source
|
||||||
|
FROM ranked_pages rp
|
||||||
|
JOIN content_chunks cc ON cc.page_id = rp.id
|
||||||
|
ORDER BY rp.slug, cc.chunk_index
|
||||||
|
)
|
||||||
|
SELECT slug, page_id, title, type, chunk_text, chunk_source, score,
|
||||||
|
false AS stale
|
||||||
|
FROM best_chunks
|
||||||
ORDER BY score DESC
|
ORDER BY score DESC
|
||||||
LIMIT ${limit}
|
`;
|
||||||
OFFSET ${offset}
|
return rows.map(rowToSearchResult);
|
||||||
),
|
} finally {
|
||||||
best_chunks AS (
|
await sql`SET statement_timeout = '0'`;
|
||||||
SELECT DISTINCT ON (rp.slug)
|
}
|
||||||
rp.slug, rp.id as page_id, rp.title, rp.type, rp.score,
|
|
||||||
cc.chunk_text, cc.chunk_source
|
|
||||||
FROM ranked_pages rp
|
|
||||||
JOIN content_chunks cc ON cc.page_id = rp.id
|
|
||||||
ORDER BY rp.slug, cc.chunk_index
|
|
||||||
)
|
|
||||||
SELECT slug, page_id, title, type, chunk_text, chunk_source, score,
|
|
||||||
false AS stale
|
|
||||||
FROM best_chunks
|
|
||||||
ORDER BY score DESC
|
|
||||||
`;
|
|
||||||
|
|
||||||
return rows.map(rowToSearchResult);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async searchVector(embedding: Float32Array, opts?: SearchOpts): Promise<SearchResult[]> {
|
async searchVector(embedding: Float32Array, opts?: SearchOpts): Promise<SearchResult[]> {
|
||||||
@@ -231,23 +237,28 @@ export class PostgresEngine implements BrainEngine {
|
|||||||
|
|
||||||
const vecStr = '[' + Array.from(embedding).join(',') + ']';
|
const vecStr = '[' + Array.from(embedding).join(',') + ']';
|
||||||
|
|
||||||
const rows = await sql`
|
// Search-only timeout (see searchKeyword for rationale)
|
||||||
SELECT
|
await sql`SET statement_timeout = '8s'`;
|
||||||
p.slug, p.id as page_id, p.title, p.type,
|
try {
|
||||||
cc.chunk_text, cc.chunk_source,
|
const rows = await sql`
|
||||||
1 - (cc.embedding <=> ${vecStr}::vector) AS score,
|
SELECT
|
||||||
false AS stale
|
p.slug, p.id as page_id, p.title, p.type,
|
||||||
FROM content_chunks cc
|
cc.chunk_text, cc.chunk_source,
|
||||||
JOIN pages p ON p.id = cc.page_id
|
1 - (cc.embedding <=> ${vecStr}::vector) AS score,
|
||||||
WHERE cc.embedding IS NOT NULL
|
false AS stale
|
||||||
${type ? sql`AND p.type = ${type}` : sql``}
|
FROM content_chunks cc
|
||||||
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
JOIN pages p ON p.id = cc.page_id
|
||||||
ORDER BY cc.embedding <=> ${vecStr}::vector
|
WHERE cc.embedding IS NOT NULL
|
||||||
LIMIT ${limit}
|
${type ? sql`AND p.type = ${type}` : sql``}
|
||||||
OFFSET ${offset}
|
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
||||||
`;
|
ORDER BY cc.embedding <=> ${vecStr}::vector
|
||||||
|
LIMIT ${limit}
|
||||||
return rows.map(rowToSearchResult);
|
OFFSET ${offset}
|
||||||
|
`;
|
||||||
|
return rows.map(rowToSearchResult);
|
||||||
|
} finally {
|
||||||
|
await sql`SET statement_timeout = '0'`;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Chunks
|
// Chunks
|
||||||
|
|||||||
@@ -210,8 +210,9 @@ describe('generateHtml', () => {
|
|||||||
expect(html).toContain('prefers-color-scheme: dark');
|
expect(html).toContain('prefers-color-scheme: dark');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('includes marked.js CDN', () => {
|
test('inlines marked.js (no CDN dependency)', () => {
|
||||||
const html = generateHtml({ title: 'T', markdown: 'x' });
|
const html = generateHtml({ title: 'T', markdown: 'x' });
|
||||||
expect(html).toContain('cdn.jsdelivr.net/npm/marked');
|
expect(html).not.toContain('cdn.jsdelivr.net');
|
||||||
|
expect(html).toContain('marked');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user