mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-28 14:59:47 +00:00
feat: CJK entity extraction for Chinese/Japanese/Korean names (#1637)
* feat: CJK entity extraction for Chinese/Japanese/Korean names - Add hasCJK() / cjkCharCount() detection helpers - Lower min name length for CJK entities from 4 to 2 chars - Fix tokenizeTitle() to handle pure CJK titles as single tokens (was returning [] for CJK-only titles, excluding them from gazetteer) - Add CJK substring matching pass in findMentionedEntities - NER extraction works without schema pack (plain mentions fallback) Verified: gbrain extract links --by-mention creates 27 links from 456 pages with 3 CJK entity pages in gazetteer. * feat: Chinese link type inference + timeline date formats Link types: - CN_FOUNDED_RE: 创立/创办/成立/创建 → founded - CN_INVESTED_RE: 投资/入股/融资 → invested_in - CN_ADVISES_RE: 顾问/咨询/指导 → advises - CN_WORKS_AT_RE: 任职/就职/担任 → works_at - CN_CITED_RE: 引用/提到/提及 → cited Timeline: - TIMELINE_LINE_RE_CN: YYYY年M月D日 | event - Auto-normalizes to YYYY-MM-DD format - Falls through to English format if CN doesn't match * fix: CJK tokenizer uses char-level tokens (reviewer feedback) Addresses all 4 concerns from review of PR #1637: 1. tokenizeForScan now emits CJK characters as individual tokens — normal scan path reaches CJK gazetteer entries naturally, eliminating the separate O(P×C×N) substring fallback pass. 2. tokenizeTitle splits pure CJK titles into individual chars — e.g. '纳瓦尔' → ['纳','瓦','尔'], matching body-level CJK tokens. 3. Removed O(P×C×N) CJK substring pass — no longer needed. Performance now O(P × N_tokens) for both ASCII and CJK. 4. Renamed CN_*_RE → ZH_*_RE in link-extraction.ts with a comment clarifying these are Chinese-only (entity NAME extraction in by-mention.ts covers CJK scripts, link TYPE extraction is zh only). Added 12 CJK-specific tests (10 pure + 2 engine integration). All 51 existing + new tests pass. * review-repair(#1637): scope CN timeline regex to 年月日, revert off-scope extract-ner no-pack change, cosmetics - TIMELINE_LINE_RE_CN required only [年-] separators, so non-bold ASCII dates (- 2020-01-02 - text) started parsing as timeline entries — an English-default regression. Now requires the 年/月 markers. - Dropped the dead 'm = cm as any' assignment. - src/core/extract-ner.ts reverted to origin/master: the no-pack → plain-mentions walk was off-scope for a CJK PR, duplicated the existing --by-mention pass, and hardcoded pack_unavailable:false (breaking the CLI hint). - by-mention.ts: fixed stray indentation + restored EOF newline. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Garry Tan <garrytan@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com>
This commit is contained in:
alkalide
co-authored by
Claude Fable 5
Garry Tan
Time Attakc
Claude Fable 5
Garry Tan
Time Attakc
parent
56aac51a08
commit
0413c93e72
+125
-11
@@ -40,6 +40,7 @@ export const LINKABLE_ENTITY_TYPES = ['person', 'company', 'organization', 'enti
|
||||
* types in.
|
||||
*/
|
||||
const MIN_NAME_LENGTH = 4;
|
||||
const MIN_CJK_NAME_LENGTH = 2;
|
||||
|
||||
/**
|
||||
* Built-in ignore list — common ambiguous tokens whose body-text mentions
|
||||
@@ -104,12 +105,12 @@ export interface FindMentionsOpts {
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Token-only tokenizer. Returns `[token, offset]` pairs for every
|
||||
* `[a-zA-Z0-9]+` run, lowercased. Non-ASCII (CJK, accented) is
|
||||
* deliberately not tokenized in v1 — entity gazetteer is English-dominant
|
||||
* in production today. Widening to `\p{L}+` is a future option once a
|
||||
* real CJK entity catalog appears (filed under TODO-1 + a TODO for
|
||||
* Unicode-aware tokenization).
|
||||
* Token-only tokenizer. Returns `[token, offset]` pairs.
|
||||
*
|
||||
* ASCII: each `[a-zA-Z0-9]+` run is a single token, lowercased.
|
||||
* CJK: each CJK character (Chinese/Japanese/Korean) is an individual
|
||||
* token, lowercased. This allows the normal maximal-munch scan path
|
||||
* to reach CJK gazetteer entries without a separate substring pass.
|
||||
*
|
||||
* Possessive "Acme's" tokenizes as ['acme', 's'] (single-quote breaks the
|
||||
* run) — single-word "Acme" lookup succeeds at offset 0; the trailing 's'
|
||||
@@ -127,18 +128,129 @@ function tokenizeForScan(text: string): ScannedToken[] {
|
||||
const out: ScannedToken[] = [];
|
||||
TOKEN_RE.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
|
||||
// Collect ASCII token spans first.
|
||||
const asciiSpans: Array<{ start: number; end: number }> = [];
|
||||
while ((m = TOKEN_RE.exec(text)) !== null) {
|
||||
out.push({ text: m[0].toLowerCase(), offset: m.index, length: m[0].length });
|
||||
asciiSpans.push({ start: m.index, end: m.index + m[0].length });
|
||||
}
|
||||
|
||||
// Walk character-by-character: emit ASCII tokens at their start positions,
|
||||
// then emit individual CJK characters for non-ASCII positions that fall
|
||||
// outside ASCII token spans.
|
||||
let asciiIdx = 0;
|
||||
for (let i = 0; i < text.length;) {
|
||||
const cp = text.codePointAt(i) ?? 0;
|
||||
const isCJK = (cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) ||
|
||||
(cp >= 0xac00 && cp <= 0xd7af);
|
||||
|
||||
// Advance asciiIdx past any spans that end before or at i.
|
||||
while (asciiIdx < asciiSpans.length && asciiSpans[asciiIdx]!.end <= i) {
|
||||
asciiIdx++;
|
||||
}
|
||||
|
||||
// If position i is inside an ASCII token span, emit the full ASCII token
|
||||
// and jump past it.
|
||||
if (asciiIdx < asciiSpans.length && i >= asciiSpans[asciiIdx]!.start && i < asciiSpans[asciiIdx]!.end) {
|
||||
const span = asciiSpans[asciiIdx]!;
|
||||
const token = text.slice(span.start, span.end);
|
||||
out.push({ text: token.toLowerCase(), offset: span.start, length: token.length });
|
||||
i = span.end;
|
||||
asciiIdx++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// CJK: emit as individual character token.
|
||||
if (isCJK) {
|
||||
const charLen = cp > 0xffff ? 2 : 1; // surrogate pair
|
||||
const charStr = text.slice(i, i + charLen);
|
||||
out.push({ text: charStr.toLowerCase(), offset: i, length: charLen });
|
||||
i += charLen;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function hasCJK(s: string): boolean {
|
||||
for (const ch of s) {
|
||||
const cp = ch.codePointAt(0) ?? 0;
|
||||
if ((cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) ||
|
||||
(cp >= 0xac00 && cp <= 0xd7af)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function cjkCharCount(s: string): number {
|
||||
let count = 0;
|
||||
for (const ch of s) {
|
||||
const cp = ch.codePointAt(0) ?? 0;
|
||||
if ((cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) ||
|
||||
(cp >= 0xac00 && cp <= 0xd7af)) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize a page title for gazetteer insertion.
|
||||
*
|
||||
* ASCII titles: standard `[a-zA-Z0-9]+` tokenization, lowercased.
|
||||
* CJK titles (no ASCII content): split into individual characters —
|
||||
* e.g. "纳瓦尔" → ["纳","瓦","尔"]. This allows normal multi-token
|
||||
* maximal-munch matching to work with character-level CJK tokens
|
||||
* produced by `tokenizeForScan`.
|
||||
* Mixed CJK+ASCII titles: ASCII parts tokenized normally, CJK parts
|
||||
* split into individual characters.
|
||||
*/
|
||||
function tokenizeTitle(title: string): string[] {
|
||||
const tokens: string[] = [];
|
||||
TOKEN_RE.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = TOKEN_RE.exec(title)) !== null) tokens.push(m[0].toLowerCase());
|
||||
return tokens;
|
||||
const hasAscii = TOKEN_RE.test(title);
|
||||
if (hasAscii) {
|
||||
// Mixed ASCII+CJK or pure ASCII: tokenize ASCII normally, then
|
||||
// append individual CJK characters in order.
|
||||
TOKEN_RE.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
const asciiSpans: Array<{ start: number; end: number; text: string }> = [];
|
||||
while ((m = TOKEN_RE.exec(title)) !== null) {
|
||||
asciiSpans.push({ start: m.index, end: m.index + m[0].length, text: m[0].toLowerCase() });
|
||||
}
|
||||
let asciiIdx = 0;
|
||||
for (let i = 0; i < title.length;) {
|
||||
while (asciiIdx < asciiSpans.length && asciiSpans[asciiIdx]!.end <= i) asciiIdx++;
|
||||
if (asciiIdx < asciiSpans.length && i >= asciiSpans[asciiIdx]!.start && i < asciiSpans[asciiIdx]!.end) {
|
||||
tokens.push(asciiSpans[asciiIdx]!.text);
|
||||
i = asciiSpans[asciiIdx]!.end;
|
||||
asciiIdx++;
|
||||
continue;
|
||||
}
|
||||
const cp = title.codePointAt(i) ?? 0;
|
||||
if (hasCJK(title[i]!)) {
|
||||
const charLen = cp > 0xffff ? 2 : 1;
|
||||
tokens.push(title.slice(i, i + charLen).toLowerCase());
|
||||
i += charLen;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
// Pure CJK (no ASCII content): split into individual characters.
|
||||
if (hasCJK(title)) {
|
||||
for (let i = 0; i < title.length;) {
|
||||
const cp = title.codePointAt(i) ?? 0;
|
||||
const charLen = cp > 0xffff ? 2 : 1;
|
||||
tokens.push(title.slice(i, i + charLen).toLowerCase());
|
||||
i += charLen;
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
// Non-ASCII, non-CJK title (emoji, symbols, etc.) — empty set.
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -175,7 +287,9 @@ export async function buildGazetteer(
|
||||
|
||||
const gazetteer: Gazetteer = new Map();
|
||||
for (const row of rows) {
|
||||
if (!row.title || row.title.length < MIN_NAME_LENGTH) continue;
|
||||
if (!row.title) continue;
|
||||
if (!hasCJK(row.title) && row.title.length < MIN_NAME_LENGTH) continue;
|
||||
if (hasCJK(row.title) && cjkCharCount(row.title) < MIN_CJK_NAME_LENGTH) continue;
|
||||
if (ignoreSet.has(row.title) && !existingTitles.has(row.title)) continue;
|
||||
|
||||
const tokens = tokenizeTitle(row.title);
|
||||
|
||||
+34
-10
@@ -671,6 +671,17 @@ const FOUNDED_RE = /\b(?:founded|co-?founded|started the company|incorporated|fo
|
||||
// "security advisor to|at", "product advisor to|at", "industry advisor".
|
||||
const ADVISES_RE = /\b(?:advises|advised|advisor (?:to|at|for|of)|advisory (?:board|role|position|capacity|engagement|partnership|contract|relationship|work)|board advisor|on .{0,20} advisory board|joined .{0,20} advisory board|in an? advisory (?:capacity|role|position)|as an? (?:advisor|security advisor|technical advisor|strategic advisor|industry advisor|product advisor|board advisor|senior advisor)|(?:strategic|technical|security|product|industry|senior|board) advisor (?:to|at|for|of)|consults for|consulting role (?:at|with))\b/i;
|
||||
|
||||
// Chinese link type patterns for CJK entity mentions.
|
||||
// NOTE: These patterns are Chinese-only (zh). Japanese and Korean link
|
||||
// type extraction is not yet implemented. Entity NAME extraction in
|
||||
// by-mention.ts covers all three scripts (CJK = Chinese/Japanese/Korean)
|
||||
// via Unicode-aware tokenization.
|
||||
const ZH_FOUNDED_RE = /(?:创立|创办|成立|创建|建立|开创|发起)(?:了|的)/;
|
||||
const ZH_INVESTED_RE = /(?:投资|入股|融资|注资|参股)(?:了|的|了?于)/;
|
||||
const ZH_ADVISES_RE = /(?:顾问|咨询|指导)(?:了|的)?/;
|
||||
const ZH_WORKS_AT_RE = /(?:任职|就职|担任|供职|在.{0,10}(?:工作|上班|负责))(?:于|在|的)?/;
|
||||
const ZH_CITED_RE = /(?:引用|援引|提到|提及|转述|摘录)(?:了|的|自)?/;
|
||||
|
||||
// Page-role detection: if the source page describes a partner/investor at
|
||||
// page level, that's a strong prior for outbound company refs being
|
||||
// invested_in even when per-edge context lacks explicit investment verbs.
|
||||
@@ -724,6 +735,12 @@ export function inferLinkType(pageType: PageType, context: string, globalContext
|
||||
if (INVESTED_RE.test(context)) return 'invested_in';
|
||||
if (ADVISES_RE.test(context)) return 'advises';
|
||||
if (WORKS_AT_RE.test(context)) return 'works_at';
|
||||
// Chinese link type patterns
|
||||
if (ZH_FOUNDED_RE.test(context)) return 'founded';
|
||||
if (ZH_INVESTED_RE.test(context)) return 'invested_in';
|
||||
if (ZH_ADVISES_RE.test(context)) return 'advises';
|
||||
if (ZH_WORKS_AT_RE.test(context)) return 'works_at';
|
||||
if (ZH_CITED_RE.test(context)) return 'cited';
|
||||
// Page-role prior: only fires for person -> company links. Concept pages
|
||||
// about VC topics naturally contain "venture capital" in their text, but
|
||||
// their company refs are mentions, not investments. Partner pages mentioning
|
||||
@@ -1174,6 +1191,10 @@ export interface TimelineCandidate {
|
||||
// Match: `- **YYYY-MM-DD** | summary` or `- **YYYY-MM-DD** -- summary`
|
||||
// or `- **YYYY-MM-DD** - summary` or just `**YYYY-MM-DD** | summary`.
|
||||
const TIMELINE_LINE_RE = /^\s*-?\s*\*\*(\d{4}-\d{2}-\d{2})\*\*\s*[|\-–—]+\s*(.+?)\s*$/;
|
||||
// Chinese date lines: `- 2020年1月2日 | summary` (bold optional). Requires the
|
||||
// 年/月 markers so plain ASCII `- 2020-01-02 - text` does NOT match — non-bold
|
||||
// ASCII dates were never timeline entries and must stay that way.
|
||||
const TIMELINE_LINE_RE_CN = /^\s*-?\s*(?:\*\*)?(\d{4})年(\d{1,2})月(\d{1,2})日?(?:\*\*)?\s*[|\-–—]+\s*(.+?)\s*$/;
|
||||
|
||||
/**
|
||||
* Parse timeline entries from content. Looks at:
|
||||
@@ -1190,18 +1211,21 @@ export function parseTimelineEntries(content: string): TimelineCandidate[] {
|
||||
|
||||
let i = 0;
|
||||
while (i < lines.length) {
|
||||
// Try English format first, then Chinese
|
||||
const m = TIMELINE_LINE_RE.exec(lines[i]);
|
||||
if (!m) {
|
||||
i++;
|
||||
continue;
|
||||
let date: string;
|
||||
let summary: string;
|
||||
if (m) {
|
||||
date = m[1];
|
||||
summary = m[2].trim();
|
||||
} else {
|
||||
const cm = TIMELINE_LINE_RE_CN.exec(lines[i]);
|
||||
if (!cm) { i++; continue; }
|
||||
// Normalize Chinese date to YYYY-MM-DD
|
||||
date = `${cm[1]}-${cm[2].padStart(2, '0')}-${cm[3].padStart(2, '0')}`;
|
||||
summary = cm[4].trim();
|
||||
}
|
||||
const date = m[1];
|
||||
const summary = m[2].trim();
|
||||
if (!isValidDate(date) || summary.length === 0) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!isValidDate(date) || summary.length === 0) { i++; continue; }
|
||||
// Collect optional detail lines (indented, until next date or heading).
|
||||
const detailLines: string[] = [];
|
||||
let j = i + 1;
|
||||
|
||||
+159
-4
@@ -58,12 +58,23 @@ beforeEach(async () => {
|
||||
// Tiny gazetteer builder for pure-fn cases that don't need engine.
|
||||
function gazetteerFromEntries(entries: Omit<GazetteerEntry, 'tokens'>[]): Gazetteer {
|
||||
const TOKEN_RE = /[a-zA-Z0-9]+/g;
|
||||
const isCJK = (s: string): boolean => {
|
||||
const cp = s.codePointAt(0) ?? 0;
|
||||
return (cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x3040 && cp <= 0x309f) || (cp >= 0x30a0 && cp <= 0x30ff) ||
|
||||
(cp >= 0xac00 && cp <= 0xd7af);
|
||||
};
|
||||
const hasCJKTitle = (s: string): boolean => [...s].some(isCJK);
|
||||
const tokenize = (s: string): string[] => {
|
||||
TOKEN_RE.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = TOKEN_RE.exec(s)) !== null) out.push(m[0].toLowerCase());
|
||||
return out;
|
||||
if (!hasCJKTitle(s)) {
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = TOKEN_RE.exec(s)) !== null) out.push(m[0].toLowerCase());
|
||||
return out;
|
||||
}
|
||||
// CJK: split into individual characters, lowercased.
|
||||
return [...s].map(c => isCJK(c) ? c.toLowerCase() : '').filter(Boolean);
|
||||
};
|
||||
const g: Gazetteer = new Map();
|
||||
for (const raw of entries) {
|
||||
@@ -259,6 +270,128 @@ describe('findMentionedEntities — pure cases', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// CJK — entity extraction tests
|
||||
// ============================================================
|
||||
|
||||
describe('findMentionedEntities — CJK cases', () => {
|
||||
test('CJK single-name match — "纳瓦尔" in body → matched', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('我最近读了纳瓦尔的书。', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toHaveLength(1);
|
||||
expect(mentions[0]!.slug).toBe('people/naval');
|
||||
expect(mentions[0]!.name).toBe('纳瓦尔');
|
||||
});
|
||||
|
||||
test('CJK multi-name — two different CJK entities in one body', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
{ slug: 'people/shuang-xuetao', source_id: 'default', title: '双雪涛' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('纳瓦尔和双雪涛都是作家。', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toHaveLength(2);
|
||||
const slugs = mentions.map(m => m.slug);
|
||||
expect(slugs).toContain('people/naval');
|
||||
expect(slugs).toContain('people/shuang-xuetao');
|
||||
});
|
||||
|
||||
test('CJK first-mention-only — repeated name → single link', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('纳瓦尔说过。然后纳瓦尔又说过。', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('CJK self-link guard — entity page mentioning itself is skipped', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('纳瓦尔是一位投资人。', g, {
|
||||
fromSlug: 'people/naval', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toEqual([]);
|
||||
});
|
||||
|
||||
test('CJK cross-source guard — entity in different source skipped', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'team-b', title: '纳瓦尔' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('纳瓦尔写了这本书。', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'team-a',
|
||||
});
|
||||
expect(mentions).toEqual([]);
|
||||
});
|
||||
|
||||
test('CJK code-block stripping — CJK name inside ``` is skipped, outside matched', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
]);
|
||||
// "纳瓦尔" only appears inside code block → should be skipped.
|
||||
const body = '```\n纳瓦尔\n```\n只有代码块里面有。';
|
||||
const mentions = findMentionedEntities(body, g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('CJK determinism — same output across 10 calls', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
{ slug: 'people/shuang-xuetao', source_id: 'default', title: '双雪涛' },
|
||||
]);
|
||||
const body = '纳瓦尔和双雪涛。纳瓦尔再说一次。';
|
||||
const refs = new Set<string>();
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const mentions = findMentionedEntities(body, g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
refs.add(JSON.stringify(mentions));
|
||||
}
|
||||
expect(refs.size).toBe(1);
|
||||
});
|
||||
|
||||
test('CJK mixed body — CJK entity matched in body with ASCII around it', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
{ slug: 'companies/acme', source_id: 'default', title: 'Acme' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('Acme was founded by 纳瓦尔 in 2020.', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toHaveLength(2);
|
||||
const slugs = mentions.map(m => m.slug);
|
||||
expect(slugs).toContain('people/naval');
|
||||
expect(slugs).toContain('companies/acme');
|
||||
});
|
||||
|
||||
test('CJK empty gazetteer — no false positives', () => {
|
||||
const g: Gazetteer = new Map();
|
||||
const mentions = findMentionedEntities('纳瓦尔和双雪涛。', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toEqual([]);
|
||||
});
|
||||
|
||||
test('CJK empty text → empty result', () => {
|
||||
const g = gazetteerFromEntries([
|
||||
{ slug: 'people/naval', source_id: 'default', title: '纳瓦尔' },
|
||||
]);
|
||||
const mentions = findMentionedEntities('', g, {
|
||||
fromSlug: 'writing/post-1', fromSourceId: 'default',
|
||||
});
|
||||
expect(mentions).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// buildGazetteer — engine-backed tests
|
||||
// ============================================================
|
||||
@@ -366,4 +499,26 @@ describe('buildGazetteer — engine integration', () => {
|
||||
// forces a deliberate change (and a corresponding test update).
|
||||
expect(LINKABLE_ENTITY_TYPES).toEqual(['person', 'company', 'organization', 'entity']);
|
||||
});
|
||||
|
||||
// CJK — engine-backed tests
|
||||
test('CJK entity with 2-char title enters gazetteer with char-level tokens', async () => {
|
||||
await engine.putPage('people/naval', {
|
||||
type: 'person', title: '纳瓦尔', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||||
});
|
||||
const g = await buildGazetteer(engine);
|
||||
// "纳瓦尔" tokenized as ["纳","瓦","尔"] → key is "纳"
|
||||
expect(g.has('纳')).toBe(true);
|
||||
const bucket = g.get('纳')!;
|
||||
expect(bucket.length).toBe(1);
|
||||
expect(bucket[0]!.tokens).toEqual(['纳', '瓦', '尔']);
|
||||
expect(bucket[0]!.slug).toBe('people/naval');
|
||||
});
|
||||
|
||||
test('CJK single-char title (cjkCharCount < 2) excluded from gazetteer', async () => {
|
||||
await engine.putPage('people/x', {
|
||||
type: 'person', title: '谢', compiled_truth: 'b', timeline: '', frontmatter: {},
|
||||
});
|
||||
const g = await buildGazetteer(engine);
|
||||
expect(g.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user