From 73eaed7691de4c2e1578a777efcfcb9faef704e0 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Thu, 14 May 2026 09:14:39 -0700 Subject: [PATCH] feat(v0.34 W7): per-op graph-traversal metrics module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/core/eval-capture-graph.ts — pure-function metrics module for comparing code_blast / code_flow / code_cluster_get result shapes across two runs (eval-replay's regression check). Per Codex finding #3 from the plan-review: page-slug Jaccard is the wrong metric for graph traversal. v0.34 W7 ships proper per-op metrics: - nodeSetJaccard(a, b): set Jaccard over (file, line, symbol) tuples. Right metric for code_blast/code_flow node sets. - depthGroupStability(a, b): 1 - (displaced / |union|). Catches the case where node membership is identical but nodes moved between depth buckets between runs. - truncationMatch(a, b): boolean match on the truncation enum. Discrete signal that pairs with Jaccard. - adjustedRandIndex(a, b): cluster-membership stability via ARI for code_cluster_get. v0.34.1 consumer; lands in W7 alongside the rest so the cluster-replay path is ready when clusters ship. - compareCodeWalk(a, b): convenience wrapper returning {jaccard, depth_stability, truncation_match} in one call. Hermetic — no engine, no DB, fully unit-testable. 20 test cases covering identical / disjoint / partial-overlap / empty / dedup / file+line-distinguished, depth-bucket reshuffles, truncation-enum matching, ARI identical-clustering recognition through label-rename, ARI singleton-vs-all-one expected-zero, equal-length contract, and combined compareCodeWalk envelope. Scope reduction from the original plan: extending src/core/eval-capture.ts capture wrapper with `tool` field + `result_shape` payload, and extending src/commands/eval-replay.ts to dispatch on tool — both deferred to v0.34.1. The metric MODULE is the load-bearing piece (Codex finding #3's primary fix); wiring it through the existing capture/replay surface is a follow-up that doesn't change production behavior until clusters ship. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/core/eval-capture-graph.ts | Bin 0 -> 6007 bytes test/code-intel/eval-capture-graph.test.ts | 150 +++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 src/core/eval-capture-graph.ts create mode 100644 test/code-intel/eval-capture-graph.test.ts diff --git a/src/core/eval-capture-graph.ts b/src/core/eval-capture-graph.ts new file mode 100644 index 0000000000000000000000000000000000000000..93fffc47121d811a73fca3b942c1cfe1abedcbcf GIT binary patch literal 6007 zcmbtY(Qf0$745UX;wBAZ(zYmj)1rWEuVFbwur1abcmw3YFuX%*Byml0SPp5eY8&WN z(dVK+(LVMQ`YHP*J@?K~q-5=GiUvVqac1t^IrrRi@62#-Z-@5i{xJIPH}v!G=;L4h zL8Yz+rleV=%K4zK)V;2($|%=$l_oaAd|vWnQ|ZC9DUv!h1?6VaWLlO_n=D&WQ>68v z*0xr0rVpqvq>EWvXezXx=t*nkwKFibm`o~x{0Nf;Mbu#zv#HOB$ahj)f=SVjhF>Z3YS6QPEyi13KMY>v~R!8)GYLiOW`j{$R znyQ8$Dg%->g)<)J)&(_r-&oAJ!ogn^Ky^e(#$T#IuJah2oTnu*zXY)^zJAv2Yz51G zs_5LCBeq9kFrq76H&tQjdT?|=M>oen{gmKKQKtzQ?=2ycwmZ%#pd&S_w9YY)V!bpl zn9Nm?FwgZ2u-$9eS(wUdpri^v8o4x!jyyDpDQuH7l@SL_a;LMz2NQA(cXsp##tCu9 zxu*ym{lHmy$-#TrA zv4~UYtZ#+03Lg+n0DZjkw6nu@FH~U)Mkf8UCgmtp0}PrgJLf#7JH3=mkW8EtVt%C& zOu0YmWl)T|Q-tlVavX32@t}9#JQR^j>8H_%f3Smec1F9q{sTMcKfG)GV^|>`gEn79 zCWPl+7FO}TRF?$N(s@~nQsM6a>7ol=sajGsQHULjAx@hKS_ybzX%4x-k!@m-IA3=O z;XIeTs^1}}uB zR=bjb)%=|~7dQ{@$V|oEki{(!yP!^R7vcjmyj{9~N=KdJB2749=h}YHr@`7%Ugh2M z%j6ViILT;jA_{;^=#%IBbh0VASknoe2(?#S@>`TW` z1;Ps9Szw2)u&ju=r3eMU5(K^S1U;`RwLE!n-nTqFmebnpY^A4{&`sv@|7l|@?O&_X z(;k1FI0b|N3n^a)ed7OunLU=ygr#AhxCMWjMI2Zgl7hX+BGr771x0v(Q9)3)-82?| z##sC%V=i^t9PnB0kNYyXj-p8Xf%4h_e&Dy5biZ%JA5Jzguq}&^B1uzD-;PwD$XziK ziuSasGEtnFdc+($aZ?NhsB<2LjbepzW|J3C$bG@73sV=AJ=k3^- zOry+pjmSV6k(_&%$hA^kB7CBE;cSu4g%DR1mm%fq!>xoIe@k$06Hsm_ITap5QM%4% zYf5$QNE7AL$q|(vAwLt1Mo~|Ej%*uPC%zZ9KuP6eFLy(c=^*ZZIm3xZEWWO6^TD=s zKdu>oa`n0#runMBOlzHUSIm0BHr^>~(^UdQ?bst zaYYBGCu|4psF*H|O}S5r=oNOdl%u&LRhC*+YpRRR1qIm4Er*efq(#|qzwXO(il$fF z67?|UHbnXy2(v6jHI`z`%dULGr+lcP?!NbPHxM8P%5Lodc{w5I81Ew{oglY_Id=rT zZqLm>IovuQmlWHdOhExSdxdz`kPD$Z zPV{O(u;Q|l;}9(klDZ?%_)C-5i7$;ww^NkiE{UrMOGwYA|J>?RJ+Js~hqmbVRb{GR z_t^nPz18Q!=$ky?xl zhcs@|47Ub@n?`}YGg*>cl!3gr>+A}j z@VCJK_V-&9y&qEVdgr`OZ@xO9pdDp>-xv*9k`6=Krz2k!ymK3-#%rd=QbsycGP-Gs z(FcYXW?+(}sVd;79n(Af5`(J-4#h3@1{`kw*7^4uv?o@9rPIF4%}8z!Rih`C&t&;4OAiVg2j(s83;E@e*f?+O6JB`{ep40 zYg;xsYH$aIz<&maPKcWJf_G#hVEqN3c(ZrE7g0Bnlt^?Mp`VGO?KcVTCC=UmGk2?v zmNILG&goNjn9HBBGhTOS{M?SizBaYW7$ggOt;2;l%z4@$<_p8Klyz2co5STsyY~nf z3V_x*fW-gCjACb|;}5zBd2Lut-(O7^(4A$xIhZ^xRE^uxDR&-j&4AXnWUJCX^=Gg7 zD^Rt4f{W#b4feCQTfa-VFnP=kiGL16MQyTrfUAAU>6Mo#QQa*A{g#Q}0}fiG&|=gS zE;^{+843&+VgFvZD#kNBi(pL&luQ=UPpHqndZQ$v2rcGmBDLKuRWG literal 0 HcmV?d00001 diff --git a/test/code-intel/eval-capture-graph.test.ts b/test/code-intel/eval-capture-graph.test.ts new file mode 100644 index 000000000..d1ed24c23 --- /dev/null +++ b/test/code-intel/eval-capture-graph.test.ts @@ -0,0 +1,150 @@ +/** + * v0.34 W7 — per-op graph metrics tests. + * Pure-function tests; no engine needed. + */ +import { describe, test, expect } from 'bun:test'; +import { + nodeSetJaccard, + depthGroupStability, + truncationMatch, + adjustedRandIndex, + compareCodeWalk, +} from '../../src/core/eval-capture-graph.ts'; + +describe('W7: nodeSetJaccard', () => { + test('identical sets → 1.0', () => { + const a = [{ symbol: 'foo' }, { symbol: 'bar' }]; + const b = [{ symbol: 'foo' }, { symbol: 'bar' }]; + expect(nodeSetJaccard(a, b)).toBe(1); + }); + + test('disjoint sets → 0', () => { + expect(nodeSetJaccard([{ symbol: 'a' }], [{ symbol: 'b' }])).toBe(0); + }); + + test('partial overlap (3 shared of 4 total)', () => { + const a = [{ symbol: 'x' }, { symbol: 'y' }, { symbol: 'z' }]; + const b = [{ symbol: 'x' }, { symbol: 'y' }, { symbol: 'w' }]; + // intersection=2, union=4 → 0.5 + expect(nodeSetJaccard(a, b)).toBe(0.5); + }); + + test('both empty → NaN (degenerate)', () => { + expect(Number.isNaN(nodeSetJaccard([], []))).toBe(true); + }); + + test('one empty → 0', () => { + expect(nodeSetJaccard([], [{ symbol: 'foo' }])).toBe(0); + }); + + test('file + line distinguish same-name symbols', () => { + const a = [{ symbol: 'foo', file: 'a.ts', line: 1 }]; + const b = [{ symbol: 'foo', file: 'a.ts', line: 2 }]; + expect(nodeSetJaccard(a, b)).toBe(0); + }); + + test('dedup within a single side', () => { + const a = [{ symbol: 'foo' }, { symbol: 'foo' }]; + const b = [{ symbol: 'foo' }]; + expect(nodeSetJaccard(a, b)).toBe(1); + }); +}); + +describe('W7: depthGroupStability', () => { + test('all nodes in same depth → 1.0', () => { + const a = [{ depth: 1, nodes: [{ symbol: 'a' }, { symbol: 'b' }] }]; + const b = [{ depth: 1, nodes: [{ symbol: 'a' }, { symbol: 'b' }] }]; + expect(depthGroupStability(a, b)).toBe(1); + }); + + test('one node moved buckets → 0.5 with 2 nodes', () => { + const a = [{ depth: 1, nodes: [{ symbol: 'a' }, { symbol: 'b' }] }]; + const b = [ + { depth: 1, nodes: [{ symbol: 'a' }] }, + { depth: 2, nodes: [{ symbol: 'b' }] }, + ]; + expect(depthGroupStability(a, b)).toBe(0.5); + }); + + test('both empty → 1.0', () => { + expect(depthGroupStability([], [])).toBe(1); + }); + + test('completely reshuffled → 0', () => { + const a = [{ depth: 1, nodes: [{ symbol: 'a' }] }]; + const b = [{ depth: 2, nodes: [{ symbol: 'a' }] }]; + expect(depthGroupStability(a, b)).toBe(0); + }); +}); + +describe('W7: truncationMatch', () => { + test('both none → 1', () => { + expect(truncationMatch('none', 'none')).toBe(1); + expect(truncationMatch(undefined, undefined)).toBe(1); + }); + + test('mismatch → 0', () => { + expect(truncationMatch('max_nodes', 'depth_cap')).toBe(0); + }); + + test('undefined treated as none', () => { + expect(truncationMatch('none', undefined)).toBe(1); + }); +}); + +describe('W7: adjustedRandIndex', () => { + test('identical clusterings → 1', () => { + const a = ['A', 'A', 'B', 'B']; + const b = ['X', 'X', 'Y', 'Y']; + // Same partition, different labels — ARI should be 1. + expect(adjustedRandIndex(a, b)).toBeCloseTo(1, 5); + }); + + test('all items in one cluster vs all in distinct → expected 0', () => { + const a = ['A', 'A', 'A', 'A']; + const b = ['W', 'X', 'Y', 'Z']; + // Singleton vs single-group clustering: ARI should be 0 (no agreement + // beyond chance). + const ari = adjustedRandIndex(a, b); + expect(ari).toBeCloseTo(0, 5); + }); + + test('equal-length contract enforced', () => { + expect(() => adjustedRandIndex(['A'], ['X', 'Y'])).toThrow(/equal length/); + }); + + test('singleton input → 1.0', () => { + expect(adjustedRandIndex(['A'], ['X'])).toBe(1); + }); +}); + +describe('W7: compareCodeWalk', () => { + test('shared depth_groups → high jaccard + stability', () => { + const a = { + depth_groups: [{ depth: 1, nodes: [{ symbol: 'x' }, { symbol: 'y' }] }], + truncation: 'none', + }; + const b = { + depth_groups: [{ depth: 1, nodes: [{ symbol: 'x' }, { symbol: 'y' }] }], + truncation: 'none', + }; + const cmp = compareCodeWalk(a, b); + expect(cmp.jaccard).toBe(1); + expect(cmp.depth_stability).toBe(1); + expect(cmp.truncation_match).toBe(1); + }); + + test('no overlap → low jaccard, full reshuffle → low stability', () => { + const a = { + depth_groups: [{ depth: 1, nodes: [{ symbol: 'x' }] }], + truncation: 'none', + }; + const b = { + depth_groups: [{ depth: 1, nodes: [{ symbol: 'y' }] }], + truncation: 'max_nodes', + }; + const cmp = compareCodeWalk(a, b); + expect(cmp.jaccard).toBe(0); + expect(cmp.truncation_match).toBe(0); + }); +});