diff --git a/src/core/import-file.ts b/src/core/import-file.ts index cbf2d03a0..9ba0c574b 100644 --- a/src/core/import-file.ts +++ b/src/core/import-file.ts @@ -115,6 +115,17 @@ async function extractFencedChunks( startChunkIndex: number, ): Promise { const out: ChunkInput[] = []; + // Fast path: most pages (prose, tables, converted docs) contain no code + // fence at all, so there is nothing for this function to extract. marked's + // lexer still allocates transient memory proportional to page size on every + // call — a ~2MB table-heavy page spikes ~110MB of heap just to produce zero + // fenced chunks. During bulk import those per-page spikes stack on top of + // accumulated chunk/embedding memory and can OOM the worker, and the + // try/catch below cannot rescue an OOM (it is process death, not a throw). + // Skip the lexer entirely when no fence marker (``` or ~~~) is present. + // The `\r` in the line-start class mirrors marked's own `\r\n|\r → \n` + // normalization, so CR/CRLF-only documents don't lose a real fence. + if (!/(^|[\r\n])[ \t]{0,3}(```|~~~)/.test(markdown)) return out; let tokens: ReturnType; try { tokens = marked.lexer(markdown); diff --git a/test/fence-extraction.test.ts b/test/fence-extraction.test.ts index e65fe6ecd..366693ba2 100644 --- a/test/fence-extraction.test.ts +++ b/test/fence-extraction.test.ts @@ -136,4 +136,43 @@ echo hi const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code'); expect(fenceChunks.length).toBe(0); }); + + // #2437 — extractFencedChunks skips marked.lexer entirely when the body has + // no fence marker (the lexer transiently allocates ~60x the page size, which + // OOMs the import worker under memory pressure). These two guard that the + // fast-path neither breaks tilde fences nor lexes fence-less pages. + test('tilde-fenced (~~~) code is still extracted after the no-fence fast-path (#2437)', async () => { + const md = 'Docs.\n\n~~~ts\nexport const x = 1;\n~~~\n'; + await importFromContent(engine, 'guides/fence-tilde', md, { noEmbed: true }); + const chunks = await engine.getChunks('guides/fence-tilde'); + const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code'); + expect(fenceChunks.length).toBeGreaterThan(0); + expect(fenceChunks[0]!.language).toBe('typescript'); + }); + + test('CR-only (\\r) line endings still extract a fenced chunk (#2437)', async () => { + // marked normalizes \r → \n before lexing, so the fast-path's fence probe + // must too; otherwise a classic-Mac line-ended page loses its fenced code. + const md = 'intro\r```ts\rexport const x = 1;\r```\r'; + await importFromContent(engine, 'guides/fence-cr', md, { noEmbed: true }); + const chunks = await engine.getChunks('guides/fence-cr'); + const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code'); + expect(fenceChunks.length).toBeGreaterThan(0); + expect(fenceChunks[0]!.language).toBe('typescript'); + }); + + test('large fence-less table page imports with zero fenced chunks (no lexer pass) (#2437)', async () => { + const cols = 16; + const header = '| ' + Array.from({ length: cols }, (_, i) => 'col' + i).join(' | ') + ' |'; + const sep = '| ' + Array.from({ length: cols }, () => '---').join(' | ') + ' |'; + const body = Array.from({ length: 2000 }, (_, r) => + '| ' + Array.from({ length: cols }, (_, c) => 'v' + r + '_' + c).join(' | ') + ' |').join('\n'); + const md = '# Overview\n\n' + header + '\n' + sep + '\n' + body + '\n'; + // sanity: the page is genuinely fence-less, so the fast-path applies + expect(/(^|[\r\n])[ \t]{0,3}(```|~~~)/.test(md)).toBe(false); + await importFromContent(engine, 'guides/fence-less-table', md, { noEmbed: true }); + const chunks = await engine.getChunks('guides/fence-less-table'); + const fenceChunks = chunks.filter(c => c.chunk_source === 'fenced_code'); + expect(fenceChunks.length).toBe(0); + }); });