From 659b6e9b4d656944c81208a1ac51d4a8bfd08606 Mon Sep 17 00:00:00 2001 From: Jaehwan Lee <51878645+irresi@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:26:43 -0700 Subject: [PATCH] fix(import): skip marked.lexer on fence-less pages to avoid bulk-import OOM (#2437) (#2440) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(import): skip marked.lexer on fence-less pages to avoid bulk-import OOM (#2437) extractFencedChunks() ran marked.lexer() on every page body. The lexer allocates transient memory proportional to page size even when there is no code fence to extract — a ~2MB table/doc page spikes ~110MB of heap to produce zero fenced chunks. During bulk import these per-page spikes stack on accumulated chunk/embedding memory and can OOM the worker; the existing try/catch cannot rescue an OOM (process death, not a throw). On a representative brain ~99% of importable pages have no fence, so the lexer pass is pure wasted work there. Add a fast-path that returns early when the body contains no fence marker. Matches both ``` and ~~~ so tilde fenced code still extracts. Scope: this removes the fence-less transient-allocation surface (the observed incident). It does not make marked.lexer safe for pages that DO contain a fence; an input-size/nesting cap is a sensible follow-up. Tests: add two regression cases — tilde-fenced code still extracts, and a large fence-less table page imports with zero fenced chunks. Co-Authored-By: Claude Opus 4.8 * fix(import): match marked's \r normalization in the fence fast-path (#2437) Self-review follow-up. marked normalizes `\r\n|\r → \n` before lexing, but the no-fence fast-path probed the raw body with `(^|\n)`. A CR-only (classic-Mac) line-ended page with a real fenced block would be skipped by the guard while marked would have extracted it — a lost fenced_code chunk. Widen the line-start class to `(^|[\r\n])` so the probe agrees with marked. CRLF was already covered. Add a regression test (CR-only fenced page still extracts); it fails on the old `(^|\n)` regex and passes on the fix. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- src/core/import-file.ts | 11 ++++++++++ test/fence-extraction.test.ts | 39 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/src/core/import-file.ts b/src/core/import-file.ts index cbf2d03a0..9ba0c574b 100644 --- a/src/core/import-file.ts +++ b/src/core/import-file.ts @@ -115,6 +115,17 @@ async function extractFencedChunks( startChunkIndex: number, ): Promise { const out: ChunkInput[] = []; + // Fast path: most pages (prose, tables, converted docs) contain no code + // fence at all, so there is nothing for this function to extract. marked's + // lexer still allocates transient memory proportional to page size on every + // call — a ~2MB table-heavy page spikes ~110MB of heap just to produce zero + // fenced chunks. During bulk import those per-page spikes stack on top of + // accumulated chunk/embedding memory and can OOM the worker, and the + // try/catch below cannot rescue an OOM (it is process death, not a throw). + // Skip the lexer entirely when no fence marker (``` or ~~~) is present. + // The `\r` in the line-start class mirrors marked's own `\r\n|\r → \n` + // normalization, so CR/CRLF-only documents don't lose a real fence. + if (!/(^|[\r\n])[ \t]{0,3}(```|~~~)/.test(markdown)) return out; let tokens: ReturnType; try { tokens = marked.lexer(markdown); diff --git a/test/fence-extraction.test.ts b/test/fence-extraction.test.ts index e65fe6ecd..366693ba2 100644 --- a/test/fence-extraction.test.ts +++ b/test/fence-extraction.test.ts @@ -136,4 +136,43 @@ echo hi const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code'); expect(fenceChunks.length).toBe(0); }); + + // #2437 — extractFencedChunks skips marked.lexer entirely when the body has + // no fence marker (the lexer transiently allocates ~60x the page size, which + // OOMs the import worker under memory pressure). These two guard that the + // fast-path neither breaks tilde fences nor lexes fence-less pages. + test('tilde-fenced (~~~) code is still extracted after the no-fence fast-path (#2437)', async () => { + const md = 'Docs.\n\n~~~ts\nexport const x = 1;\n~~~\n'; + await importFromContent(engine, 'guides/fence-tilde', md, { noEmbed: true }); + const chunks = await engine.getChunks('guides/fence-tilde'); + const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code'); + expect(fenceChunks.length).toBeGreaterThan(0); + expect(fenceChunks[0]!.language).toBe('typescript'); + }); + + test('CR-only (\\r) line endings still extract a fenced chunk (#2437)', async () => { + // marked normalizes \r → \n before lexing, so the fast-path's fence probe + // must too; otherwise a classic-Mac line-ended page loses its fenced code. + const md = 'intro\r```ts\rexport const x = 1;\r```\r'; + await importFromContent(engine, 'guides/fence-cr', md, { noEmbed: true }); + const chunks = await engine.getChunks('guides/fence-cr'); + const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code'); + expect(fenceChunks.length).toBeGreaterThan(0); + expect(fenceChunks[0]!.language).toBe('typescript'); + }); + + test('large fence-less table page imports with zero fenced chunks (no lexer pass) (#2437)', async () => { + const cols = 16; + const header = '| ' + Array.from({ length: cols }, (_, i) => 'col' + i).join(' | ') + ' |'; + const sep = '| ' + Array.from({ length: cols }, () => '---').join(' | ') + ' |'; + const body = Array.from({ length: 2000 }, (_, r) => + '| ' + Array.from({ length: cols }, (_, c) => 'v' + r + '_' + c).join(' | ') + ' |').join('\n'); + const md = '# Overview\n\n' + header + '\n' + sep + '\n' + body + '\n'; + // sanity: the page is genuinely fence-less, so the fast-path applies + expect(/(^|[\r\n])[ \t]{0,3}(```|~~~)/.test(md)).toBe(false); + await importFromContent(engine, 'guides/fence-less-table', md, { noEmbed: true }); + const chunks = await engine.getChunks('guides/fence-less-table'); + const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code'); + expect(fenceChunks.length).toBe(0); + }); });