fix(webhook): extract links for incremental push syncs (#2850) (#3337)

* test(webhook): pin sync extraction contract (#2849)

* test(webhook): target the submitted sync payload (#2849)

* fix(webhook): run extraction in sync job (#2849)

* fix(sync): align push trigger extraction (#2849)

Co-authored-by: Song <patentsong@gmail.com>

src/commands/serve-http.ts

@@ -2249,8 +2249,10 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
   //     Other event types (ping, pull_request, etc.) return 202 'ignored'
   //     so GitHub doesn't retry.
   // D15.5: HMAC compare uses the shared safeHexEqual helper.
-  // D18: submits 'sync' job with auto_embed_backfill=true and priority -10
-  //     (above autopilot's 0).
+  // D18: submits 'sync' job with extraction + auto_embed_backfill enabled and
+  //     priority -10 (above autopilot's 0). This opts normal incremental pushes
+  //     into sync's inline extraction while pagesAffected still identifies the
+  //     changed pages. The sync core can still defer large (>100) changes.
   // ---------------------------------------------------------------------------
   const githubWebhookLimiter = rateLimit({
     windowMs: 60_000,
@@ -2370,6 +2372,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
           'sync',
           {
             sourceId: source.id,
+            noExtract: false,
             auto_embed_backfill: true,
             embed_reason: 'webhook',
           },

src/commands/sync.ts

@@ -1423,6 +1423,7 @@ See also:
     {
       sourceId: sourceIdArg,
       repoPath: source.local_path,
+      noExtract: false,
       auto_embed_backfill: true,
       embed_reason: 'sync_trigger',
     },

test/sources-webhook.test.ts

@@ -16,6 +16,7 @@
  */
 import { describe, test, expect } from 'bun:test';
 import { createHmac } from 'node:crypto';
+import { readFileSync } from 'node:fs';
 import { safeHexEqual } from '../src/core/timing-safe.ts';
 
 const GITHUB_SECRET = 'super-secret-webhook-key';
@@ -123,3 +124,25 @@ describe('Branch ref construction (D5)', () => {
     expect(pushedRef === `refs/heads/${trackedBranch}`).toBe(false);
   });
 });
+
+describe('Webhook sync job extraction contract', () => {
+  test('opts into extraction before the pushed commit is consumed', () => {
+    const serveSource = readFileSync(
+      new URL('../src/commands/serve-http.ts', import.meta.url),
+      'utf8',
+    );
+    const routeStart = serveSource.indexOf("'/webhooks/github'");
+    const queueStart = serveSource.indexOf('const job = await queue.add(', routeStart);
+    const responseStart = serveSource.indexOf('res.status(202)', queueStart);
+    expect(routeStart).toBeGreaterThanOrEqual(0);
+    expect(queueStart).toBeGreaterThan(routeStart);
+    expect(responseStart).toBeGreaterThan(queueStart);
+
+    const routeSource = serveSource.slice(queueStart, responseStart);
+    const payload = routeSource.match(
+      /queue\.add\(\s*'sync',\s*\{([\s\S]*?)\}\s*,\s*\{/,
+    );
+    expect(payload).not.toBeNull();
+    expect(payload?.[1]).toMatch(/\bnoExtract:\s*false\b/);
+  });
+});

test/sync-trigger-cli.test.ts

@@ -100,6 +100,7 @@ describe('runSyncTrigger', () => {
     const job = jobs[0];
     expect(job.priority).toBe(-10);
     expect((job.data as { sourceId: string }).sourceId).toBe('default');
+    expect((job.data as { noExtract: boolean }).noExtract).toBe(false);
     expect((job.data as { auto_embed_backfill: boolean }).auto_embed_backfill).toBe(true);
   });