From 0413c93e724bd6c859cdc4f21921a9f4acba8558 Mon Sep 17 00:00:00 2001 From: alkalide Date: Tue, 28 Jul 2026 09:03:31 +0800 Subject: [PATCH] feat: CJK entity extraction for Chinese/Japanese/Korean names (#1637) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: CJK entity extraction for Chinese/Japanese/Korean names - Add hasCJK() / cjkCharCount() detection helpers - Lower min name length for CJK entities from 4 to 2 chars - Fix tokenizeTitle() to handle pure CJK titles as single tokens (was returning [] for CJK-only titles, excluding them from gazetteer) - Add CJK substring matching pass in findMentionedEntities - NER extraction works without schema pack (plain mentions fallback) Verified: gbrain extract links --by-mention creates 27 links from 456 pages with 3 CJK entity pages in gazetteer. * feat: Chinese link type inference + timeline date formats Link types: - CN_FOUNDED_RE: 创立/创办/成立/创建 → founded - CN_INVESTED_RE: 投资/入股/融资 → invested_in - CN_ADVISES_RE: 顾问/咨询/指导 → advises - CN_WORKS_AT_RE: 任职/就职/担任 → works_at - CN_CITED_RE: 引用/提到/提及 → cited Timeline: - TIMELINE_LINE_RE_CN: YYYY年M月D日 | event - Auto-normalizes to YYYY-MM-DD format - Falls through to English format if CN doesn't match * fix: CJK tokenizer uses char-level tokens (reviewer feedback) Addresses all 4 concerns from review of PR #1637: 1. tokenizeForScan now emits CJK characters as individual tokens — normal scan path reaches CJK gazetteer entries naturally, eliminating the separate O(P×C×N) substring fallback pass. 2. tokenizeTitle splits pure CJK titles into individual chars — e.g. '纳瓦尔' → ['纳','瓦','尔'], matching body-level CJK tokens. 3. Removed O(P×C×N) CJK substring pass — no longer needed. Performance now O(P × N_tokens) for both ASCII and CJK. 4. Renamed CN_*_RE → ZH_*_RE in link-extraction.ts with a comment clarifying these are Chinese-only (entity NAME extraction in by-mention.ts covers CJK scripts, link TYPE extraction is zh only). Added 12 CJK-specific tests (10 pure + 2 engine integration). All 51 existing + new tests pass. * review-repair(#1637): scope CN timeline regex to 年月日, revert off-scope extract-ner no-pack change, cosmetics - TIMELINE_LINE_RE_CN required only [年-] separators, so non-bold ASCII dates (- 2020-01-02 - text) started parsing as timeline entries — an English-default regression. Now requires the 年/月 markers. - Dropped the dead 'm = cm as any' assignment. - src/core/extract-ner.ts reverted to origin/master: the no-pack → plain-mentions walk was off-scope for a CJK PR, duplicated the existing --by-mention pass, and hardcoded pack_unavailable:false (breaking the CLI hint). - by-mention.ts: fixed stray indentation + restored EOF newline. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Garry Tan Co-authored-by: Claude Fable 5 Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/core/by-mention.ts | 136 +++++++++++++++++++++++++++--- src/core/link-extraction.ts | 44 +++++++--- test/by-mention.test.ts | 163 +++++++++++++++++++++++++++++++++++- 3 files changed, 318 insertions(+), 25 deletions(-) diff --git a/src/core/by-mention.ts b/src/core/by-mention.ts index 5910ea7ab..61bac9469 100644 --- a/src/core/by-mention.ts +++ b/src/core/by-mention.ts @@ -40,6 +40,7 @@ export const LINKABLE_ENTITY_TYPES = ['person', 'company', 'organization', 'enti * types in. */ const MIN_NAME_LENGTH = 4; +const MIN_CJK_NAME_LENGTH = 2; /** * Built-in ignore list — common ambiguous tokens whose body-text mentions @@ -104,12 +105,12 @@ export interface FindMentionsOpts { // ============================================================ /** - * Token-only tokenizer. Returns `[token, offset]` pairs for every - * `[a-zA-Z0-9]+` run, lowercased. Non-ASCII (CJK, accented) is - * deliberately not tokenized in v1 — entity gazetteer is English-dominant - * in production today. Widening to `\p{L}+` is a future option once a - * real CJK entity catalog appears (filed under TODO-1 + a TODO for - * Unicode-aware tokenization). + * Token-only tokenizer. Returns `[token, offset]` pairs. + * + * ASCII: each `[a-zA-Z0-9]+` run is a single token, lowercased. + * CJK: each CJK character (Chinese/Japanese/Korean) is an individual + * token, lowercased. This allows the normal maximal-munch scan path + * to reach CJK gazetteer entries without a separate substring pass. * * Possessive "Acme's" tokenizes as ['acme', 's'] (single-quote breaks the * run) — single-word "Acme" lookup succeeds at offset 0; the trailing 's' @@ -127,18 +128,129 @@ function tokenizeForScan(text: string): ScannedToken[] { const out: ScannedToken[] = []; TOKEN_RE.lastIndex = 0; let m: RegExpExecArray | null; + + // Collect ASCII token spans first. + const asciiSpans: Array<{ start: number; end: number }> = []; while ((m = TOKEN_RE.exec(text)) !== null) { - out.push({ text: m[0].toLowerCase(), offset: m.index, length: m[0].length }); + asciiSpans.push({ start: m.index, end: m.index + m[0].length }); + } + + // Walk character-by-character: emit ASCII tokens at their start positions, + // then emit individual CJK characters for non-ASCII positions that fall + // outside ASCII token spans. + let asciiIdx = 0; + for (let i = 0; i < text.length;) { + const cp = text.codePointAt(i) ?? 0; + const isCJK = (cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) || + (cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) || + (cp >= 0xac00 && cp <= 0xd7af); + + // Advance asciiIdx past any spans that end before or at i. + while (asciiIdx < asciiSpans.length && asciiSpans[asciiIdx]!.end <= i) { + asciiIdx++; + } + + // If position i is inside an ASCII token span, emit the full ASCII token + // and jump past it. + if (asciiIdx < asciiSpans.length && i >= asciiSpans[asciiIdx]!.start && i < asciiSpans[asciiIdx]!.end) { + const span = asciiSpans[asciiIdx]!; + const token = text.slice(span.start, span.end); + out.push({ text: token.toLowerCase(), offset: span.start, length: token.length }); + i = span.end; + asciiIdx++; + continue; + } + + // CJK: emit as individual character token. + if (isCJK) { + const charLen = cp > 0xffff ? 2 : 1; // surrogate pair + const charStr = text.slice(i, i + charLen); + out.push({ text: charStr.toLowerCase(), offset: i, length: charLen }); + i += charLen; + } else { + i++; + } } return out; } +function hasCJK(s: string): boolean { + for (const ch of s) { + const cp = ch.codePointAt(0) ?? 0; + if ((cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) || + (cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) || + (cp >= 0xac00 && cp <= 0xd7af)) return true; + } + return false; +} + +function cjkCharCount(s: string): number { + let count = 0; + for (const ch of s) { + const cp = ch.codePointAt(0) ?? 0; + if ((cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) || + (cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) || + (cp >= 0xac00 && cp <= 0xd7af)) count++; + } + return count; +} + +/** + * Tokenize a page title for gazetteer insertion. + * + * ASCII titles: standard `[a-zA-Z0-9]+` tokenization, lowercased. + * CJK titles (no ASCII content): split into individual characters — + * e.g. "纳瓦尔" → ["纳","瓦","尔"]. This allows normal multi-token + * maximal-munch matching to work with character-level CJK tokens + * produced by `tokenizeForScan`. + * Mixed CJK+ASCII titles: ASCII parts tokenized normally, CJK parts + * split into individual characters. + */ function tokenizeTitle(title: string): string[] { const tokens: string[] = []; TOKEN_RE.lastIndex = 0; - let m: RegExpExecArray | null; - while ((m = TOKEN_RE.exec(title)) !== null) tokens.push(m[0].toLowerCase()); - return tokens; + const hasAscii = TOKEN_RE.test(title); + if (hasAscii) { + // Mixed ASCII+CJK or pure ASCII: tokenize ASCII normally, then + // append individual CJK characters in order. + TOKEN_RE.lastIndex = 0; + let m: RegExpExecArray | null; + const asciiSpans: Array<{ start: number; end: number; text: string }> = []; + while ((m = TOKEN_RE.exec(title)) !== null) { + asciiSpans.push({ start: m.index, end: m.index + m[0].length, text: m[0].toLowerCase() }); + } + let asciiIdx = 0; + for (let i = 0; i < title.length;) { + while (asciiIdx < asciiSpans.length && asciiSpans[asciiIdx]!.end <= i) asciiIdx++; + if (asciiIdx < asciiSpans.length && i >= asciiSpans[asciiIdx]!.start && i < asciiSpans[asciiIdx]!.end) { + tokens.push(asciiSpans[asciiIdx]!.text); + i = asciiSpans[asciiIdx]!.end; + asciiIdx++; + continue; + } + const cp = title.codePointAt(i) ?? 0; + if (hasCJK(title[i]!)) { + const charLen = cp > 0xffff ? 2 : 1; + tokens.push(title.slice(i, i + charLen).toLowerCase()); + i += charLen; + } else { + i++; + } + } + return tokens; + } + // Pure CJK (no ASCII content): split into individual characters. + if (hasCJK(title)) { + for (let i = 0; i < title.length;) { + const cp = title.codePointAt(i) ?? 0; + const charLen = cp > 0xffff ? 2 : 1; + tokens.push(title.slice(i, i + charLen).toLowerCase()); + i += charLen; + } + return tokens; + } + // Non-ASCII, non-CJK title (emoji, symbols, etc.) — empty set. + return []; } /** @@ -175,7 +287,9 @@ export async function buildGazetteer( const gazetteer: Gazetteer = new Map(); for (const row of rows) { - if (!row.title || row.title.length < MIN_NAME_LENGTH) continue; + if (!row.title) continue; + if (!hasCJK(row.title) && row.title.length < MIN_NAME_LENGTH) continue; + if (hasCJK(row.title) && cjkCharCount(row.title) < MIN_CJK_NAME_LENGTH) continue; if (ignoreSet.has(row.title) && !existingTitles.has(row.title)) continue; const tokens = tokenizeTitle(row.title); diff --git a/src/core/link-extraction.ts b/src/core/link-extraction.ts index 83c273693..88e150943 100644 --- a/src/core/link-extraction.ts +++ b/src/core/link-extraction.ts @@ -671,6 +671,17 @@ const FOUNDED_RE = /\b(?:founded|co-?founded|started the company|incorporated|fo // "security advisor to|at", "product advisor to|at", "industry advisor". const ADVISES_RE = /\b(?:advises|advised|advisor (?:to|at|for|of)|advisory (?:board|role|position|capacity|engagement|partnership|contract|relationship|work)|board advisor|on .{0,20} advisory board|joined .{0,20} advisory board|in an? advisory (?:capacity|role|position)|as an? (?:advisor|security advisor|technical advisor|strategic advisor|industry advisor|product advisor|board advisor|senior advisor)|(?:strategic|technical|security|product|industry|senior|board) advisor (?:to|at|for|of)|consults for|consulting role (?:at|with))\b/i; +// Chinese link type patterns for CJK entity mentions. +// NOTE: These patterns are Chinese-only (zh). Japanese and Korean link +// type extraction is not yet implemented. Entity NAME extraction in +// by-mention.ts covers all three scripts (CJK = Chinese/Japanese/Korean) +// via Unicode-aware tokenization. +const ZH_FOUNDED_RE = /(?:创立|创办|成立|创建|建立|开创|发起)(?:了|的)/; +const ZH_INVESTED_RE = /(?:投资|入股|融资|注资|参股)(?:了|的|了?于)/; +const ZH_ADVISES_RE = /(?:顾问|咨询|指导)(?:了|的)?/; +const ZH_WORKS_AT_RE = /(?:任职|就职|担任|供职|在.{0,10}(?:工作|上班|负责))(?:于|在|的)?/; +const ZH_CITED_RE = /(?:引用|援引|提到|提及|转述|摘录)(?:了|的|自)?/; + // Page-role detection: if the source page describes a partner/investor at // page level, that's a strong prior for outbound company refs being // invested_in even when per-edge context lacks explicit investment verbs. @@ -724,6 +735,12 @@ export function inferLinkType(pageType: PageType, context: string, globalContext if (INVESTED_RE.test(context)) return 'invested_in'; if (ADVISES_RE.test(context)) return 'advises'; if (WORKS_AT_RE.test(context)) return 'works_at'; + // Chinese link type patterns + if (ZH_FOUNDED_RE.test(context)) return 'founded'; + if (ZH_INVESTED_RE.test(context)) return 'invested_in'; + if (ZH_ADVISES_RE.test(context)) return 'advises'; + if (ZH_WORKS_AT_RE.test(context)) return 'works_at'; + if (ZH_CITED_RE.test(context)) return 'cited'; // Page-role prior: only fires for person -> company links. Concept pages // about VC topics naturally contain "venture capital" in their text, but // their company refs are mentions, not investments. Partner pages mentioning @@ -1174,6 +1191,10 @@ export interface TimelineCandidate { // Match: `- **YYYY-MM-DD** | summary` or `- **YYYY-MM-DD** -- summary` // or `- **YYYY-MM-DD** - summary` or just `**YYYY-MM-DD** | summary`. const TIMELINE_LINE_RE = /^\s*-?\s*\*\*(\d{4}-\d{2}-\d{2})\*\*\s*[|\-–—]+\s*(.+?)\s*$/; +// Chinese date lines: `- 2020年1月2日 | summary` (bold optional). Requires the +// 年/月 markers so plain ASCII `- 2020-01-02 - text` does NOT match — non-bold +// ASCII dates were never timeline entries and must stay that way. +const TIMELINE_LINE_RE_CN = /^\s*-?\s*(?:\*\*)?(\d{4})年(\d{1,2})月(\d{1,2})日?(?:\*\*)?\s*[|\-–—]+\s*(.+?)\s*$/; /** * Parse timeline entries from content. Looks at: @@ -1190,18 +1211,21 @@ export function parseTimelineEntries(content: string): TimelineCandidate[] { let i = 0; while (i < lines.length) { + // Try English format first, then Chinese const m = TIMELINE_LINE_RE.exec(lines[i]); - if (!m) { - i++; - continue; + let date: string; + let summary: string; + if (m) { + date = m[1]; + summary = m[2].trim(); + } else { + const cm = TIMELINE_LINE_RE_CN.exec(lines[i]); + if (!cm) { i++; continue; } + // Normalize Chinese date to YYYY-MM-DD + date = `${cm[1]}-${cm[2].padStart(2, '0')}-${cm[3].padStart(2, '0')}`; + summary = cm[4].trim(); } - const date = m[1]; - const summary = m[2].trim(); - if (!isValidDate(date) || summary.length === 0) { - i++; - continue; - } - + if (!isValidDate(date) || summary.length === 0) { i++; continue; } // Collect optional detail lines (indented, until next date or heading). const detailLines: string[] = []; let j = i + 1; diff --git a/test/by-mention.test.ts b/test/by-mention.test.ts index 21c7d8548..e76fcaf48 100644 --- a/test/by-mention.test.ts +++ b/test/by-mention.test.ts @@ -58,12 +58,23 @@ beforeEach(async () => { // Tiny gazetteer builder for pure-fn cases that don't need engine. function gazetteerFromEntries(entries: Omit[]): Gazetteer { const TOKEN_RE = /[a-zA-Z0-9]+/g; + const isCJK = (s: string): boolean => { + const cp = s.codePointAt(0) ?? 0; + return (cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) || + (cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) || + (cp >= 0xac00 && cp <= 0xd7af); + }; + const hasCJKTitle = (s: string): boolean => [...s].some(isCJK); const tokenize = (s: string): string[] => { TOKEN_RE.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = TOKEN_RE.exec(s)) !== null) out.push(m[0].toLowerCase()); - return out; + if (!hasCJKTitle(s)) { + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = TOKEN_RE.exec(s)) !== null) out.push(m[0].toLowerCase()); + return out; + } + // CJK: split into individual characters, lowercased. + return [...s].map(c => isCJK(c) ? c.toLowerCase() : '').filter(Boolean); }; const g: Gazetteer = new Map(); for (const raw of entries) { @@ -259,6 +270,128 @@ describe('findMentionedEntities — pure cases', () => { }); }); +// ============================================================ +// CJK — entity extraction tests +// ============================================================ + +describe('findMentionedEntities — CJK cases', () => { + test('CJK single-name match — "纳瓦尔" in body → matched', () => { + const g = gazetteerFromEntries([ + { slug: 'people/naval', source_id: 'default', title: '纳瓦尔' }, + ]); + const mentions = findMentionedEntities('我最近读了纳瓦尔的书。', g, { + fromSlug: 'writing/post-1', fromSourceId: 'default', + }); + expect(mentions).toHaveLength(1); + expect(mentions[0]!.slug).toBe('people/naval'); + expect(mentions[0]!.name).toBe('纳瓦尔'); + }); + + test('CJK multi-name — two different CJK entities in one body', () => { + const g = gazetteerFromEntries([ + { slug: 'people/naval', source_id: 'default', title: '纳瓦尔' }, + { slug: 'people/shuang-xuetao', source_id: 'default', title: '双雪涛' }, + ]); + const mentions = findMentionedEntities('纳瓦尔和双雪涛都是作家。', g, { + fromSlug: 'writing/post-1', fromSourceId: 'default', + }); + expect(mentions).toHaveLength(2); + const slugs = mentions.map(m => m.slug); + expect(slugs).toContain('people/naval'); + expect(slugs).toContain('people/shuang-xuetao'); + }); + + test('CJK first-mention-only — repeated name → single link', () => { + const g = gazetteerFromEntries([ + { slug: 'people/naval', source_id: 'default', title: '纳瓦尔' }, + ]); + const mentions = findMentionedEntities('纳瓦尔说过。然后纳瓦尔又说过。', g, { + fromSlug: 'writing/post-1', fromSourceId: 'default', + }); + expect(mentions).toHaveLength(1); + }); + + test('CJK self-link guard — entity page mentioning itself is skipped', () => { + const g = gazetteerFromEntries([ + { slug: 'people/naval', source_id: 'default', title: '纳瓦尔' }, + ]); + const mentions = findMentionedEntities('纳瓦尔是一位投资人。', g, { + fromSlug: 'people/naval', fromSourceId: 'default', + }); + expect(mentions).toEqual([]); + }); + + test('CJK cross-source guard — entity in different source skipped', () => { + const g = gazetteerFromEntries([ + { slug: 'people/naval', source_id: 'team-b', title: '纳瓦尔' }, + ]); + const mentions = findMentionedEntities('纳瓦尔写了这本书。', g, { + fromSlug: 'writing/post-1', fromSourceId: 'team-a', + }); + expect(mentions).toEqual([]); + }); + + test('CJK code-block stripping — CJK name inside ``` is skipped, outside matched', () => { + const g = gazetteerFromEntries([ + { slug: 'people/naval', source_id: 'default', title: '纳瓦尔' }, + ]); + // "纳瓦尔" only appears inside code block → should be skipped. + const body = '```\n纳瓦尔\n```\n只有代码块里面有。'; + const mentions = findMentionedEntities(body, g, { + fromSlug: 'writing/post-1', fromSourceId: 'default', + }); + expect(mentions).toHaveLength(0); + }); + + test('CJK determinism — same output across 10 calls', () => { + const g = gazetteerFromEntries([ + { slug: 'people/naval', source_id: 'default', title: '纳瓦尔' }, + { slug: 'people/shuang-xuetao', source_id: 'default', title: '双雪涛' }, + ]); + const body = '纳瓦尔和双雪涛。纳瓦尔再说一次。'; + const refs = new Set(); + for (let i = 0; i < 10; i++) { + const mentions = findMentionedEntities(body, g, { + fromSlug: 'writing/post-1', fromSourceId: 'default', + }); + refs.add(JSON.stringify(mentions)); + } + expect(refs.size).toBe(1); + }); + + test('CJK mixed body — CJK entity matched in body with ASCII around it', () => { + const g = gazetteerFromEntries([ + { slug: 'people/naval', source_id: 'default', title: '纳瓦尔' }, + { slug: 'companies/acme', source_id: 'default', title: 'Acme' }, + ]); + const mentions = findMentionedEntities('Acme was founded by 纳瓦尔 in 2020.', g, { + fromSlug: 'writing/post-1', fromSourceId: 'default', + }); + expect(mentions).toHaveLength(2); + const slugs = mentions.map(m => m.slug); + expect(slugs).toContain('people/naval'); + expect(slugs).toContain('companies/acme'); + }); + + test('CJK empty gazetteer — no false positives', () => { + const g: Gazetteer = new Map(); + const mentions = findMentionedEntities('纳瓦尔和双雪涛。', g, { + fromSlug: 'writing/post-1', fromSourceId: 'default', + }); + expect(mentions).toEqual([]); + }); + + test('CJK empty text → empty result', () => { + const g = gazetteerFromEntries([ + { slug: 'people/naval', source_id: 'default', title: '纳瓦尔' }, + ]); + const mentions = findMentionedEntities('', g, { + fromSlug: 'writing/post-1', fromSourceId: 'default', + }); + expect(mentions).toEqual([]); + }); +}); + // ============================================================ // buildGazetteer — engine-backed tests // ============================================================ @@ -366,4 +499,26 @@ describe('buildGazetteer — engine integration', () => { // forces a deliberate change (and a corresponding test update). expect(LINKABLE_ENTITY_TYPES).toEqual(['person', 'company', 'organization', 'entity']); }); + + // CJK — engine-backed tests + test('CJK entity with 2-char title enters gazetteer with char-level tokens', async () => { + await engine.putPage('people/naval', { + type: 'person', title: '纳瓦尔', compiled_truth: 'b', timeline: '', frontmatter: {}, + }); + const g = await buildGazetteer(engine); + // "纳瓦尔" tokenized as ["纳","瓦","尔"] → key is "纳" + expect(g.has('纳')).toBe(true); + const bucket = g.get('纳')!; + expect(bucket.length).toBe(1); + expect(bucket[0]!.tokens).toEqual(['纳', '瓦', '尔']); + expect(bucket[0]!.slug).toBe('people/naval'); + }); + + test('CJK single-char title (cjkCharCount < 2) excluded from gazetteer', async () => { + await engine.putPage('people/x', { + type: 'person', title: '谢', compiled_truth: 'b', timeline: '', frontmatter: {}, + }); + const g = await buildGazetteer(engine); + expect(g.size).toBe(0); + }); });