feat(v0.34 W7): per-op graph-traversal metrics module

src/core/eval-capture-graph.ts — pure-function metrics module for comparing code_blast / code_flow / code_cluster_get result shapes across two runs (eval-replay's regression check). Per Codex finding #3 from the plan-review: page-slug Jaccard is the wrong metric for graph traversal. v0.34 W7 ships proper per-op metrics: - nodeSetJaccard(a, b): set Jaccard over (file, line, symbol) tuples. Right metric for code_blast/code_flow node sets. - depthGroupStability(a, b): 1 - (displaced / |union|). Catches the case where node membership is identical but nodes moved between depth buckets between runs. - truncationMatch(a, b): boolean match on the truncation enum. Discrete signal that pairs with Jaccard. - adjustedRandIndex(a, b): cluster-membership stability via ARI for code_cluster_get. v0.34.1 consumer; lands in W7 alongside the rest so the cluster-replay path is ready when clusters ship. - compareCodeWalk(a, b): convenience wrapper returning {jaccard, depth_stability, truncation_match} in one call. Hermetic — no engine, no DB, fully unit-testable. 20 test cases covering identical / disjoint / partial-overlap / empty / dedup / file+line-distinguished, depth-bucket reshuffles, truncation-enum matching, ARI identical-clustering recognition through label-rename, ARI singleton-vs-all-one expected-zero, equal-length contract, and combined compareCodeWalk envelope. Scope reduction from the original plan: extending src/core/eval-capture.ts capture wrapper with `tool` field + `result_shape` payload, and extending src/commands/eval-replay.ts to dispatch on tool — both deferred to v0.34.1. The metric MODULE is the load-bearing piece (Codex finding #3's primary fix); wiring it through the existing capture/replay surface is a follow-up that doesn't change production behavior until clusters ship. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-07-28 06:23:01 +00:00 · 2026-05-14 09:14:39 -07:00
parent e8396917e1
commit 73eaed7691
2 changed files with 150 additions and 0 deletions
@@ -0,0 +1,150 @@
+/**
+ * v0.34 W7 — per-op graph metrics tests.
+ * Pure-function tests; no engine needed.
+ */
+import { describe, test, expect } from 'bun:test';
+import {
+  nodeSetJaccard,
+  depthGroupStability,
+  truncationMatch,
+  adjustedRandIndex,
+  compareCodeWalk,
+} from '../../src/core/eval-capture-graph.ts';
+
+describe('W7: nodeSetJaccard', () => {
+  test('identical sets → 1.0', () => {
+    const a = [{ symbol: 'foo' }, { symbol: 'bar' }];
+    const b = [{ symbol: 'foo' }, { symbol: 'bar' }];
+    expect(nodeSetJaccard(a, b)).toBe(1);
+  });
+
+  test('disjoint sets → 0', () => {
+    expect(nodeSetJaccard([{ symbol: 'a' }], [{ symbol: 'b' }])).toBe(0);
+  });
+
+  test('partial overlap (3 shared of 4 total)', () => {
+    const a = [{ symbol: 'x' }, { symbol: 'y' }, { symbol: 'z' }];
+    const b = [{ symbol: 'x' }, { symbol: 'y' }, { symbol: 'w' }];
+    // intersection=2, union=4 → 0.5
+    expect(nodeSetJaccard(a, b)).toBe(0.5);
+  });
+
+  test('both empty → NaN (degenerate)', () => {
+    expect(Number.isNaN(nodeSetJaccard([], []))).toBe(true);
+  });
+
+  test('one empty → 0', () => {
+    expect(nodeSetJaccard([], [{ symbol: 'foo' }])).toBe(0);
+  });
+
+  test('file + line distinguish same-name symbols', () => {
+    const a = [{ symbol: 'foo', file: 'a.ts', line: 1 }];
+    const b = [{ symbol: 'foo', file: 'a.ts', line: 2 }];
+    expect(nodeSetJaccard(a, b)).toBe(0);
+  });
+
+  test('dedup within a single side', () => {
+    const a = [{ symbol: 'foo' }, { symbol: 'foo' }];
+    const b = [{ symbol: 'foo' }];
+    expect(nodeSetJaccard(a, b)).toBe(1);
+  });
+});
+
+describe('W7: depthGroupStability', () => {
+  test('all nodes in same depth → 1.0', () => {
+    const a = [{ depth: 1, nodes: [{ symbol: 'a' }, { symbol: 'b' }] }];
+    const b = [{ depth: 1, nodes: [{ symbol: 'a' }, { symbol: 'b' }] }];
+    expect(depthGroupStability(a, b)).toBe(1);
+  });
+
+  test('one node moved buckets → 0.5 with 2 nodes', () => {
+    const a = [{ depth: 1, nodes: [{ symbol: 'a' }, { symbol: 'b' }] }];
+    const b = [
+      { depth: 1, nodes: [{ symbol: 'a' }] },
+      { depth: 2, nodes: [{ symbol: 'b' }] },
+    ];
+    expect(depthGroupStability(a, b)).toBe(0.5);
+  });
+
+  test('both empty → 1.0', () => {
+    expect(depthGroupStability([], [])).toBe(1);
+  });
+
+  test('completely reshuffled → 0', () => {
+    const a = [{ depth: 1, nodes: [{ symbol: 'a' }] }];
+    const b = [{ depth: 2, nodes: [{ symbol: 'a' }] }];
+    expect(depthGroupStability(a, b)).toBe(0);
+  });
+});
+
+describe('W7: truncationMatch', () => {
+  test('both none → 1', () => {
+    expect(truncationMatch('none', 'none')).toBe(1);
+    expect(truncationMatch(undefined, undefined)).toBe(1);
+  });
+
+  test('mismatch → 0', () => {
+    expect(truncationMatch('max_nodes', 'depth_cap')).toBe(0);
+  });
+
+  test('undefined treated as none', () => {
+    expect(truncationMatch('none', undefined)).toBe(1);
+  });
+});
+
+describe('W7: adjustedRandIndex', () => {
+  test('identical clusterings → 1', () => {
+    const a = ['A', 'A', 'B', 'B'];
+    const b = ['X', 'X', 'Y', 'Y'];
+    // Same partition, different labels — ARI should be 1.
+    expect(adjustedRandIndex(a, b)).toBeCloseTo(1, 5);
+  });
+
+  test('all items in one cluster vs all in distinct → expected 0', () => {
+    const a = ['A', 'A', 'A', 'A'];
+    const b = ['W', 'X', 'Y', 'Z'];
+    // Singleton vs single-group clustering: ARI should be 0 (no agreement
+    // beyond chance).
+    const ari = adjustedRandIndex(a, b);
+    expect(ari).toBeCloseTo(0, 5);
+  });
+
+  test('equal-length contract enforced', () => {
+    expect(() => adjustedRandIndex(['A'], ['X', 'Y'])).toThrow(/equal length/);
+  });
+
+  test('singleton input → 1.0', () => {
+    expect(adjustedRandIndex(['A'], ['X'])).toBe(1);
+  });
+});
+
+describe('W7: compareCodeWalk', () => {
+  test('shared depth_groups → high jaccard + stability', () => {
+    const a = {
+      depth_groups: [{ depth: 1, nodes: [{ symbol: 'x' }, { symbol: 'y' }] }],
+      truncation: 'none',
+    };
+    const b = {
+      depth_groups: [{ depth: 1, nodes: [{ symbol: 'x' }, { symbol: 'y' }] }],
+      truncation: 'none',
+    };
+    const cmp = compareCodeWalk(a, b);
+    expect(cmp.jaccard).toBe(1);
+    expect(cmp.depth_stability).toBe(1);
+    expect(cmp.truncation_match).toBe(1);
+  });
+
+  test('no overlap → low jaccard, full reshuffle → low stability', () => {
+    const a = {
+      depth_groups: [{ depth: 1, nodes: [{ symbol: 'x' }] }],
+      truncation: 'none',
+    };
+    const b = {
+      depth_groups: [{ depth: 1, nodes: [{ symbol: 'y' }] }],
+      truncation: 'max_nodes',
+    };
+    const cmp = compareCodeWalk(a, b);
+    expect(cmp.jaccard).toBe(0);
+    expect(cmp.truncation_match).toBe(0);
+  });
+});