fix(build): forward tokenjuice-treesitter + gate feature-forwarding drift in CI (#4918, #4919) (#4934)

Co-authored-by: M3gA-Mind <elvin@tinyhumans.ai>

.github/workflows/ci-lite.yml

@@ -665,6 +665,30 @@ jobs:
       - name: Run orchestration IP gate
         run: bash scripts/ci/orch-ip-gate.sh
 
+  feature-forwarding-gate:
+    name: Feature Forwarding Gate (shell forwards core defaults)
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      # The shell sets `default-features = false` on `openhuman_core`, so it does
+      # NOT inherit the core's default gates — each must be forwarded by hand.
+      # When that list drifts, the domain is compiled out of the shipped app with
+      # no build error and no failing test: that is how #4901 (voice — 56 users,
+      # ~93k Sentry events) and #4918 (tokenjuice-treesitter — silent) shipped.
+      #
+      # Deliberately NOT filtered on `changes`: drift can be introduced by
+      # editing either manifest, so a path filter watching one would miss it, and
+      # a skipped job counts as a pass in the gate below. The check is a few
+      # seconds of pure Node with no dependencies.
+      - name: Verify the desktop shell forwards every default-ON core gate
+        run: node scripts/ci/check-feature-forwarding.mjs
+
   pester-install:
     name: PowerShell Install Test (Pester)
     needs: [changes]
@@ -731,6 +755,7 @@ jobs:
       - pester-install
       - tinycortex-tests
       - orch-ip-gate
+      - feature-forwarding-gate
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 15
@@ -749,6 +774,7 @@ jobs:
             ["PowerShell Install Test"]="${{ needs['pester-install'].result }}"
             ["TinyCortex Memory Tests"]="${{ needs['tinycortex-tests'].result }}"
             ["Orchestration IP Gate"]="${{ needs['orch-ip-gate'].result }}"
+            ["Feature Forwarding Gate"]="${{ needs['feature-forwarding-gate'].result }}"
           )
 
           failed=0

AGENTS.md

@@ -222,6 +222,10 @@ Two independent runtime axes on `CoreBuilder` (`src/core/runtime/builder.rs`):
 
 Per-domain Cargo features drop whole domains **at compile time** (smaller binary, fewer deps), composing with the runtime `DomainSet` axis above. Each gate is **default-ON**, so the desktop build is byte-identical; slim builds opt out explicitly.
 
+> **Adding a default-ON gate? You must forward it to the desktop shell.**
+> `app/src-tauri/Cargo.toml` declares `openhuman_core` with `default-features = false` (set in #1061, before gates existed), so the shipped app does **not** inherit the core's `default` list. A gate you add to `default` but not to the shell's `features` list is **compiled out of the shipped desktop app** — with no build error and no failing test. This is not hypothetical: `voice` shipped missing from v0.58.19 to v0.61.x (56 users, ~93k Sentry events, #4901), and `tokenjuice-treesitter` was never forwarded once since #4123 and failed *soft*, silently degrading AST compression (#4918).
+> `scripts/ci/check-feature-forwarding.mjs` (the **Feature Forwarding Gate** lane) now fails CI on drift and covers new gates automatically. If a gate genuinely must not ship, add it to `INTENTIONALLY_NOT_FORWARDED` in that script **with a reason** — an explicit exclusion is the only way "deliberate" stays distinguishable from "forgotten".
+
 **Slim-profile convention** (no `full` meta-feature): build slim variants with `cargo build --no-default-features --features "<explicit list of gates you want>"`. This mirrors the existing standalone-feature style (`sandbox-landlock`, `browser-native`, …). Example — everything except voice:
 
 ```bash

app/src-tauri/Cargo.toml

@@ -152,11 +152,17 @@ cef = { version = "=146.4.1", default-features = false }
 #   the pre-gate desktop tool surface.
 # - `web3` — keeps the wallet/web3/x402 domains and their agent tools in the
 #   desktop build while allowing slim builds to omit the crypto-only deps.
+# - `tokenjuice-treesitter` — forwards `tinyjuice/tinyjuice-treesitter`, which
+#   pulls the tree-sitter Rust/TS/Python grammars. Without it TokenJuice falls
+#   back to the brace-depth heuristic, so the desktop app compresses code worse
+#   than intended. This failed *soft* (no error, no Sentry signal) and had never
+#   once been forwarded since the gate was added in #4123 (#4918).
 #
-# `tokenjuice-treesitter` is the remaining un-forwarded default — tracked in
-# #4918, deliberately not bundled here because dropping it may be intentional.
+# `scripts/ci/check-feature-forwarding.mjs` fails CI when this list drifts from
+# the core's `[features] default` (#4919) — do not hand-maintain it from memory.
 openhuman_core = { path = "../..", package = "openhuman", default-features = false, features = [
     "media",
+    "tokenjuice-treesitter",
     "voice",
     "tokenjuice-treesitter",
     "web3",

scripts/__tests__/feature-forwarding.test.mjs

@@ -0,0 +1,187 @@
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { test } from 'node:test';
+import { fileURLToPath } from 'node:url';
+
+import {
+  diffForwarding,
+  parseCoreDefaultFeatures,
+  parseShellForwardedFeatures,
+  stripComments,
+} from '../lib/feature-forwarding.mjs';
+
+const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
+const CHECKER = resolve(REPO_ROOT, 'scripts/ci/check-feature-forwarding.mjs');
+
+// ── parsing ────────────────────────────────────────────────────────────────
+
+test('parses the core default gate list', () => {
+  const toml = `
+[features]
+default = ["tokenjuice-treesitter", "voice", "media"]
+voice = ["dep:hound"]
+`;
+  assert.deepEqual(parseCoreDefaultFeatures(toml), ['tokenjuice-treesitter', 'voice', 'media']);
+});
+
+test('parses a multi-line default gate list', () => {
+  const toml = `
+[features]
+default = [
+    "voice",
+    "media",
+]
+`;
+  assert.deepEqual(parseCoreDefaultFeatures(toml), ['voice', 'media']);
+});
+
+test('ignores a default key belonging to another table', () => {
+  const toml = `
+[some-other-table]
+default = ["not-a-gate"]
+
+[features]
+default = ["voice"]
+`;
+  assert.deepEqual(parseCoreDefaultFeatures(toml), ['voice']);
+});
+
+test('parses the shell forwarded list across multiple lines', () => {
+  const toml = `
+openhuman_core = { path = "../..", package = "openhuman", default-features = false, features = [
+    "media",
+    "voice",
+] }
+`;
+  assert.deepEqual(parseShellForwardedFeatures(toml), {
+    defaultFeatures: false,
+    features: ['media', 'voice'],
+  });
+});
+
+test('detects when the shell inherits defaults instead of forwarding', () => {
+  const toml = 'openhuman_core = { path = "../..", package = "openhuman" }\n';
+  assert.deepEqual(parseShellForwardedFeatures(toml), { defaultFeatures: true, features: [] });
+});
+
+test('comment stripping does not truncate on a # inside a quoted value', () => {
+  const stripped = stripComments('a = "issue #4901"  # trailing comment\n');
+  assert.match(stripped, /issue #4901/);
+  assert.doesNotMatch(stripped, /trailing comment/);
+});
+
+test('a commented-out gate does not count as forwarded', () => {
+  const toml = `
+openhuman_core = { path = "../..", package = "openhuman", default-features = false, features = [
+    # "voice",
+    "media",
+] }
+`;
+  assert.deepEqual(parseShellForwardedFeatures(toml).features, ['media']);
+});
+
+// ── drift detection ────────────────────────────────────────────────────────
+
+test('passes when every default gate is forwarded', () => {
+  const result = diffForwarding({
+    coreDefaults: ['voice', 'media'],
+    shell: { defaultFeatures: false, features: ['media', 'voice'] },
+  });
+  assert.equal(result.ok, true);
+  assert.deepEqual(result.missing, []);
+});
+
+test('reproduces #4901: a dropped voice gate is reported missing', () => {
+  const result = diffForwarding({
+    coreDefaults: ['tokenjuice-treesitter', 'voice', 'media'],
+    shell: { defaultFeatures: false, features: ['media', 'tokenjuice-treesitter'] },
+  });
+  assert.equal(result.ok, false);
+  assert.deepEqual(result.missing, ['voice']);
+});
+
+test('reproduces #4918: a dropped tokenjuice-treesitter gate is reported missing', () => {
+  const result = diffForwarding({
+    coreDefaults: ['tokenjuice-treesitter', 'voice', 'media'],
+    shell: { defaultFeatures: false, features: ['media', 'voice'] },
+  });
+  assert.equal(result.ok, false);
+  assert.deepEqual(result.missing, ['tokenjuice-treesitter']);
+});
+
+test('a brand new default gate is covered automatically, with no per-gate wiring', () => {
+  const result = diffForwarding({
+    coreDefaults: ['voice', 'media', 'some-future-gate'],
+    shell: { defaultFeatures: false, features: ['voice', 'media'] },
+  });
+  assert.equal(result.ok, false);
+  assert.deepEqual(result.missing, ['some-future-gate']);
+});
+
+test('an allow-listed gate passes and is reported as intentional', () => {
+  const result = diffForwarding({
+    coreDefaults: ['voice', 'heavy-gate'],
+    shell: { defaultFeatures: false, features: ['voice'] },
+    allowlist: { 'heavy-gate': 'Adds 400MB of models to the bundle.' },
+  });
+  assert.equal(result.ok, true);
+  assert.deepEqual(result.allowed, ['heavy-gate']);
+  assert.deepEqual(result.missing, []);
+});
+
+test('an allow-list entry for a gate that IS forwarded is flagged as stale', () => {
+  const result = diffForwarding({
+    coreDefaults: ['voice'],
+    shell: { defaultFeatures: false, features: ['voice'] },
+    allowlist: { voice: 'stale entry' },
+  });
+  assert.equal(result.ok, false);
+  assert.deepEqual(result.stale, ['voice']);
+});
+
+test('inheriting defaults needs no forwarding', () => {
+  const result = diffForwarding({
+    coreDefaults: ['voice'],
+    shell: { defaultFeatures: true, features: [] },
+  });
+  assert.equal(result.ok, true);
+});
+
+test('a missing dependency fails rather than passing vacuously', () => {
+  const result = diffForwarding({ coreDefaults: ['voice'], shell: null });
+  assert.equal(result.ok, false);
+  assert.equal(result.reason, 'dependency-not-found');
+});
+
+// ── the real manifests + CLI ───────────────────────────────────────────────
+
+test('the checked-in manifests pass the guard', () => {
+  const out = execFileSync('node', [CHECKER], { encoding: 'utf8' });
+  assert.match(out, /every default-ON core gate is forwarded/);
+});
+
+test('--help exits 0', () => {
+  const out = execFileSync('node', [CHECKER, '--help'], { encoding: 'utf8' });
+  assert.match(out, /Usage:/);
+});
+
+test('the real shell manifest forwards every real core default', () => {
+  const coreDefaults = parseCoreDefaultFeatures(
+    readFileSync(resolve(REPO_ROOT, 'Cargo.toml'), 'utf8')
+  );
+  const shell = parseShellForwardedFeatures(
+    readFileSync(resolve(REPO_ROOT, 'app/src-tauri/Cargo.toml'), 'utf8')
+  );
+  // Guards the guard: if the parser silently returned nothing, the assertions
+  // below would pass against empty input and prove nothing.
+  assert.ok(coreDefaults.length > 0, 'expected to parse at least one core default gate');
+  assert.equal(shell.defaultFeatures, false, 'shell is expected to set default-features = false');
+  for (const gate of coreDefaults) {
+    assert.ok(
+      shell.features.includes(gate),
+      `core default gate not forwarded to the shell: ${gate}`
+    );
+  }
+});

scripts/ci/check-feature-forwarding.mjs

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+// Fails when the desktop shell does not forward a default-ON core Cargo gate.
+//
+// See scripts/lib/feature-forwarding.mjs for why this exists (#4919). Short
+// version: the shell sets `default-features = false` on `openhuman_core`, so
+// every default-ON gate must be forwarded by hand. When someone forgets, the
+// domain vanishes from the shipped app with no build error — that is how #4901
+// (voice, 56 users) and #4918 (tokenjuice-treesitter) shipped.
+//
+// Usage: check-feature-forwarding.mjs [core-manifest] [shell-manifest]
+import { readFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import {
+  diffForwarding,
+  formatReport,
+  parseCoreDefaultFeatures,
+  parseShellForwardedFeatures,
+} from '../lib/feature-forwarding.mjs';
+
+const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
+
+/**
+ * Gates the desktop shell intentionally does NOT forward, mapped to why.
+ *
+ * Empty by design: every current default-ON gate belongs in the shipped app.
+ * Adding an entry is a deliberate product decision, not a way to silence this
+ * check — the reason string is what a future reader (and reviewer) relies on to
+ * tell "excluded on purpose" from "forgotten". That ambiguity is exactly what
+ * let #4918 sit unnoticed since #4123.
+ */
+const INTENTIONALLY_NOT_FORWARDED = {
+  // 'some-gate': 'Reason it must not ship in the desktop build.',
+};
+
+function usage() {
+  return 'Usage: check-feature-forwarding.mjs [core-manifest] [shell-manifest]';
+}
+
+const [coreArg, shellArg, extra] = process.argv.slice(2);
+if (coreArg === '--help' || coreArg === '-h') {
+  console.log(usage());
+  process.exit(0);
+}
+if (extra) {
+  console.error(usage());
+  process.exit(2);
+}
+
+const corePath = coreArg ? resolve(coreArg) : resolve(REPO_ROOT, 'Cargo.toml');
+const shellPath = shellArg ? resolve(shellArg) : resolve(REPO_ROOT, 'app/src-tauri/Cargo.toml');
+
+let coreToml;
+let shellToml;
+try {
+  coreToml = readFileSync(corePath, 'utf8');
+  shellToml = readFileSync(shellPath, 'utf8');
+} catch (err) {
+  console.error(`Could not read manifests: ${err.message}`);
+  process.exit(2);
+}
+
+const coreDefaults = parseCoreDefaultFeatures(coreToml);
+const shell = parseShellForwardedFeatures(shellToml);
+
+// A parser that silently finds nothing would turn this guard into a rubber
+// stamp, which is worse than not having it. Treat "no defaults found" as a
+// failure of the check itself rather than a pass.
+if (coreDefaults.length === 0) {
+  console.error(
+    `FAIL: parsed zero default features from ${corePath}.\n` +
+      'Either the manifest changed shape or the parser is broken — refusing to pass vacuously.'
+  );
+  process.exit(2);
+}
+
+const result = diffForwarding({ coreDefaults, shell, allowlist: INTENTIONALLY_NOT_FORWARDED });
+console.log(formatReport(result, { coreDefaults, shell, allowlist: INTENTIONALLY_NOT_FORWARDED }));
+process.exit(result.ok ? 0 : 1);

scripts/lib/feature-forwarding.mjs

@@ -0,0 +1,189 @@
+// Detects drift between the core crate's default-ON Cargo gates and the gates
+// the Tauri shell forwards to its embedded copy of that crate.
+//
+// Why this exists (#4919): the shell declares `openhuman_core` with
+// `default-features = false`, so it does NOT inherit the core's `default` list.
+// Every default-ON gate must be forwarded by hand, and nothing enforced that.
+// When the two drift, the domain is compiled out of the shipped desktop app and
+// the failure is invisible — no build error, no failing test:
+//
+//   - `voice`  shipped missing from v0.58.19 to v0.61.x. Every `openhuman.voice_*`
+//              RPC answered "unknown method"; 56 users, ~93k Sentry events (#4901).
+//   - `tokenjuice-treesitter` was never forwarded once since #4123 and failed
+//              *soft* — AST compression silently degraded to a heuristic (#4918).
+//
+// Two of three gates were dropped, by two different authors, one of whom knew
+// about the trap and documented it in a comment. A comment is documentation,
+// not enforcement.
+//
+// This is a deliberately narrow TOML reader rather than a general parser: it
+// only needs two well-known shapes, and the repo has no TOML dependency for
+// Node. It is regex/scanner-based in the same spirit as `checklist-parser.mjs`.
+
+/**
+ * Strip TOML `#` comments while respecting quoted strings, so a `#` inside a
+ * value (or an issue number in a comment) can't truncate a real line.
+ */
+export function stripComments(text) {
+  const out = [];
+  for (const line of text.split(/\r?\n/)) {
+    let quote = null;
+    let cut = -1;
+    for (let i = 0; i < line.length; i++) {
+      const ch = line[i];
+      if (quote) {
+        if (ch === quote && line[i - 1] !== '\\') quote = null;
+      } else if (ch === '"' || ch === "'") {
+        quote = ch;
+      } else if (ch === '#') {
+        cut = i;
+        break;
+      }
+    }
+    out.push(cut === -1 ? line : line.slice(0, cut));
+  }
+  return out.join('\n');
+}
+
+/**
+ * Read a bracketed array starting at `open` (the index of `[`), returning its
+ * raw inner text. Scans for the balanced close so multi-line arrays work.
+ */
+function readArray(text, open) {
+  let depth = 0;
+  for (let i = open; i < text.length; i++) {
+    if (text[i] === '[') depth++;
+    else if (text[i] === ']') {
+      depth--;
+      if (depth === 0) return text.slice(open + 1, i);
+    }
+  }
+  return null;
+}
+
+/** Pull the quoted string items out of a raw TOML array body. */
+function arrayItems(raw) {
+  if (raw === null) return [];
+  return [...raw.matchAll(/"([^"]+)"/g)].map(m => m[1]);
+}
+
+/**
+ * The core crate's default-ON gates: `[features] default = [...]`.
+ *
+ * Scoped to the `[features]` table so an unrelated `default = [...]` in another
+ * table cannot be picked up by mistake.
+ */
+export function parseCoreDefaultFeatures(coreToml) {
+  const text = stripComments(coreToml);
+  // NOTE: `[ \t]` not `\s` — `\s` matches newlines, so `^\s*\[features\]` would
+  // happily anchor several lines early and slice the section to nothing.
+  const header = text.match(/^[ \t]*\[features\][ \t]*$/m);
+  if (!header) return [];
+  // Bound the search at the next table header so we stay inside [features].
+  const rest = text.slice(header.index + header[0].length);
+  const nextTable = rest.search(/^[ \t]*\[[^[\]]+\][ \t]*$/m);
+  const section = nextTable === -1 ? rest : rest.slice(0, nextTable);
+  const defaultAt = section.search(/^[ \t]*default[ \t]*=[ \t]*\[/m);
+  if (defaultAt === -1) return [];
+  return arrayItems(readArray(section, section.indexOf('[', defaultAt)));
+}
+
+/**
+ * What the shell forwards on its `openhuman_core` dependency.
+ *
+ * Returns `{ defaultFeatures, features }`. `defaultFeatures: true` means the
+ * shell inherits the core's defaults and forwarding is moot — there is nothing
+ * to drift.
+ */
+export function parseShellForwardedFeatures(shellToml, depName = 'openhuman_core') {
+  const text = stripComments(shellToml);
+  // `[ \t]` not `\s`, for the same newline-matching reason as above.
+  const declAt = text.search(new RegExp(`^[ \\t]*${depName}[ \\t]*=[ \\t]*\\{`, 'm'));
+  if (declAt === -1) return null;
+  const braceOpen = text.indexOf('{', declAt);
+  // Scan to the matching close brace; the inline table spans lines.
+  let depth = 0;
+  let braceClose = -1;
+  for (let i = braceOpen; i < text.length; i++) {
+    if (text[i] === '{') depth++;
+    else if (text[i] === '}') {
+      depth--;
+      if (depth === 0) {
+        braceClose = i;
+        break;
+      }
+    }
+  }
+  if (braceClose === -1) return null;
+  const decl = text.slice(braceOpen, braceClose + 1);
+  const defaultFeatures = !/default-features\s*=\s*false/.test(decl);
+  const featuresAt = decl.search(/features\s*=\s*\[/);
+  const features =
+    featuresAt === -1 ? [] : arrayItems(readArray(decl, decl.indexOf('[', featuresAt)));
+  return { defaultFeatures, features };
+}
+
+/**
+ * Compare the two lists.
+ *
+ * `allowlist` maps a gate name to the reason it is intentionally NOT forwarded.
+ * An intentional exclusion must be explicit and carry a reason, so that
+ * "deliberately excluded" and "forgotten" stop looking identical — which is the
+ * ambiguity that let #4918 sit unnoticed.
+ */
+export function diffForwarding({ coreDefaults, shell, allowlist = {} }) {
+  if (shell === null) {
+    return { ok: false, reason: 'dependency-not-found', missing: [], stale: [], allowed: [] };
+  }
+  // Inheriting defaults means there is no forwarding list to drift.
+  if (shell.defaultFeatures) {
+    return { ok: true, reason: 'inherits-defaults', missing: [], stale: [], allowed: [] };
+  }
+  const forwarded = new Set(shell.features);
+  const missing = [];
+  const allowed = [];
+  for (const gate of coreDefaults) {
+    if (forwarded.has(gate)) continue;
+    if (Object.prototype.hasOwnProperty.call(allowlist, gate)) allowed.push(gate);
+    else missing.push(gate);
+  }
+  // A gate that is allow-listed AND forwarded is a contradiction: the allow-list
+  // entry is stale and would mask a real drop if the gate were later removed.
+  const stale = Object.keys(allowlist).filter(gate => forwarded.has(gate));
+  return { ok: missing.length === 0 && stale.length === 0, reason: null, missing, stale, allowed };
+}
+
+export function formatReport(result, { coreDefaults, shell, allowlist = {} }) {
+  if (result.reason === 'dependency-not-found') {
+    return 'FAIL: could not find the `openhuman_core` dependency in the shell manifest.\nThe guard cannot verify forwarding — fix the parser or the manifest.';
+  }
+  if (result.reason === 'inherits-defaults') {
+    return 'OK: the shell inherits the core default features (no `default-features = false`), so no forwarding is required.';
+  }
+  const lines = [
+    `Core default gates (${coreDefaults.length}): ${coreDefaults.join(', ') || '(none)'}`,
+    `Shell forwards (${shell.features.length}): ${shell.features.join(', ') || '(none)'}`,
+  ];
+  for (const gate of result.allowed) {
+    lines.push(`  allowed: ${gate} — ${allowlist[gate]}`);
+  }
+  if (result.stale.length > 0) {
+    lines.push('', 'Stale allow-list entries (gate is forwarded, so the entry is wrong):');
+    for (const gate of result.stale) lines.push(`  - ${gate}`);
+  }
+  if (result.missing.length > 0) {
+    lines.push('', 'Default-ON core gates NOT forwarded by the desktop shell:');
+    for (const gate of result.missing) lines.push(`  - ${gate}`);
+    lines.push(
+      '',
+      'Each of these is compiled OUT of the shipped desktop app, silently.',
+      'Fix by adding the gate to the `openhuman_core` features list in',
+      'app/src-tauri/Cargo.toml — or, if the exclusion is deliberate, add it to',
+      'INTENTIONALLY_NOT_FORWARDED in scripts/ci/check-feature-forwarding.mjs',
+      'with a reason. See #4901 (voice) and #4918 (tokenjuice-treesitter).'
+    );
+  }
+  if (result.ok)
+    lines.push('', 'OK: every default-ON core gate is forwarded to the desktop shell.');
+  return lines.join('\n');
+}