perf: cut OpenHuman + Tauri compile/CI time (dev-profile deps, build-once E2E fanout, single-pass Linux packaging) (#3986)

This commit is contained in:
Mega Mind
2026-06-23 11:50:20 -07:00
committed by GitHub
parent 854634db17
commit 3ded18c165
5 changed files with 261 additions and 91 deletions
+38 -11
View File
@@ -434,19 +434,46 @@ jobs:
# copied into the AppDir at usr/lib/libcef.so, but the binary's
# RUNPATH does not include either location, so plain `ldd` reports
# `libcef.so => not found` and lib4bin aborts with the generic
# "is missing libraries! Aborting..." banner. To get past it we:
# 1. Run `cargo tauri build --no-bundle` first so the cef-dll-sys
# build script downloads libcef.so into the cache dir.
# 2. Re-invoke `cargo tauri build` (incremental — cargo skips the
# already-compiled artifacts) with that cache dir on
# LD_LIBRARY_PATH so the bundler's ldd call resolves libcef.so.
# macOS / Windows take the original single-call path.
# "is missing libraries! Aborting..." banner. libcef.so is downloaded
# by the cef-dll-sys build script into the CEF cache dir, which the
# "Cache CEF binary distribution" step restores across runs.
#
# Build-once (#3877): this used to compile the whole app twice — a
# throwaway `cargo tauri build --no-bundle` purely to fetch CEF, then
# the real bundling build — which also re-ran the Vite/bundler
# orchestration twice. We now resolve libcef.so cheaply and bundle in
# a SINGLE `cargo tauri build`:
# 1. Warm CEF cache (the common case): libcef.so is already restored,
# so we skip straight to the single bundling build.
# 2. Cold cache: prewarm with a TARGETED `cargo build -p cef-dll-sys`
# (compiles only the CEF downloader crate + its deps, not the
# whole app) to populate the cache.
# 3. If that still didn't yield libcef.so, fall back to the
# historical full `--no-bundle` prebuild — so correctness can
# never regress relative to the old double build.
# macOS / Windows take the single-call path unchanged.
find_cef_lib_dir() {
find "$HOME/.cache/tauri-cef" -name libcef.so -printf '%h\n' 2>/dev/null | head -1
}
if [ "${RUNNER_OS}" = "Linux" ]; then
echo "[appimage-fix] linux split build: compile first to fetch CEF"
NODE_OPTIONS="--max-old-space-size=8192" bash ../scripts/ci-cancel-aware.sh cargo tauri build --no-bundle $PROFILE_FLAG -c "$TAURI_CONFIG_OVERRIDE" $MATRIX_ARGS
CEF_LIB_DIR="$(find "$HOME/.cache/tauri-cef" -name libcef.so -printf '%h\n' 2>/dev/null | head -1)"
CEF_LIB_DIR="$(find_cef_lib_dir)"
if [ -z "$CEF_LIB_DIR" ]; then
echo "::error::libcef.so not found under ~/.cache/tauri-cef after --no-bundle compile; cannot satisfy lib4bin ldd resolution." >&2
echo "[appimage-fix] CEF not cached — targeted prewarm via cef-dll-sys build script"
# `cargo tauri build --debug` maps to cargo's default (dev)
# profile; release maps to `--release`. Match the profile so the
# prewarm's artifacts are reused by the bundling build below.
CARGO_BUILD_PROFILE="--release"
[ "$PROFILE_FLAG" = "--debug" ] && CARGO_BUILD_PROFILE=""
NODE_OPTIONS="--max-old-space-size=8192" bash ../scripts/ci-cancel-aware.sh cargo build -p cef-dll-sys $CARGO_BUILD_PROFILE || true
CEF_LIB_DIR="$(find_cef_lib_dir)"
fi
if [ -z "$CEF_LIB_DIR" ]; then
echo "[appimage-fix] targeted prewarm did not yield libcef.so — falling back to full --no-bundle prebuild"
NODE_OPTIONS="--max-old-space-size=8192" bash ../scripts/ci-cancel-aware.sh cargo tauri build --no-bundle $PROFILE_FLAG -c "$TAURI_CONFIG_OVERRIDE" $MATRIX_ARGS
CEF_LIB_DIR="$(find_cef_lib_dir)"
fi
if [ -z "$CEF_LIB_DIR" ]; then
echo "::error::libcef.so not found under ~/.cache/tauri-cef after prewarm; cannot satisfy lib4bin ldd resolution." >&2
exit 1
fi
echo "[appimage-fix] prepending CEF lib dir to LD_LIBRARY_PATH: $CEF_LIB_DIR"
+185 -80
View File
@@ -595,23 +595,20 @@ jobs:
# reusable-workflow-is-also-used-by-releases rationale.
# ---------------------------------------------------------------------------
# Full-suite macOS — sharded matrix mirroring e2e-linux-full.
# Full-suite macOS — build-once-then-fanout, mirroring build-linux-full.
#
# Previously e2e-macos-full was a 6-shard matrix where each shard restored an
# `actions/cache` of the .app bundle and rebuilt the full Tauri app on a cold
# cache. On a cold cache that means up to 6 parallel full Tauri builds (only
# the first shard to finish would populate the cache; the rest lose the race).
# `build-macos-full` now builds the .app once and uploads it as a per-run
# workflow artifact; the shards `needs:` it, download, adhoc-sign, and run.
# ---------------------------------------------------------------------------
e2e-macos-full:
build-macos-full:
if: inputs.run_macos && inputs.full
name: E2E (macOS full / ${{ matrix.shard.name }})
name: Build (macOS full)
runs-on: macos-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
shard:
- { name: foundation, suites: "auth,navigation,system" }
- { name: chat, suites: "chat,skills,journeys" }
- { name: providers, suites: "providers,notifications" }
- { name: webhooks, suites: "webhooks" }
- { name: connectors, suites: "connectors" }
- { name: commerce, suites: "payments,settings" }
steps:
- name: Checkout code
uses: actions/checkout@v5
@@ -662,13 +659,6 @@ jobs:
restore-keys: |
cef-aarch64-apple-darwin-
- name: Cache Appium global install
uses: actions/cache@v5
with:
path: |
~/.appium
key: appium3-chromium-${{ runner.os }}-v1
- name: Install JS dependencies
run: bash scripts/ci-cancel-aware.sh pnpm install --frozen-lockfile
@@ -677,6 +667,71 @@ jobs:
touch .env
touch app/.env
- name: Build E2E app
run: bash scripts/ci-cancel-aware.sh pnpm --filter openhuman-app test:e2e:build
- name: Package build artifact
run: |
# The macOS .app is self-contained (frontend assets + CEF Frameworks +
# the OpenHuman Helper.app are embedded by tauri-bundler), so the .app
# bundle alone is everything a shard needs. tar preserves the bundle's
# internal symlinks + executable bits; each shard re-signs after
# extraction (adhoc codesign is required on every runner regardless).
tar -czf e2e-build-macos.tar.gz \
-C app/src-tauri/target/debug/bundle/macos OpenHuman.app
ls -lh e2e-build-macos.tar.gz
- name: Upload build artifact
uses: actions/upload-artifact@v5
with:
name: e2e-build-macos-${{ github.run_id }}
path: e2e-build-macos.tar.gz
retention-days: 1
if-no-files-found: error
e2e-macos-full:
if: inputs.run_macos && inputs.full
needs: build-macos-full
name: E2E (macOS full / ${{ matrix.shard.name }})
runs-on: macos-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
shard:
- { name: foundation, suites: "auth,navigation,system" }
- { name: chat, suites: "chat,skills,journeys" }
- { name: providers, suites: "providers,notifications" }
- { name: webhooks, suites: "webhooks" }
- { name: connectors, suites: "connectors" }
- { name: commerce, suites: "payments,settings" }
steps:
- name: Checkout code
uses: actions/checkout@v5
with:
ref: ${{ inputs.ref }}
fetch-depth: 1
submodules: recursive
- name: Install pnpm
uses: pnpm/action-setup@v4
- name: Setup Node.js 24.x
uses: actions/setup-node@v5
with:
node-version: 24.x
cache: pnpm
- name: Cache Appium global install
uses: actions/cache@v5
with:
path: |
~/.appium
key: appium3-chromium-${{ runner.os }}-v1
- name: Install JS dependencies (for test harness only)
run: bash scripts/ci-cancel-aware.sh pnpm install --frozen-lockfile
- name: Install Appium and chromium driver
run: |
if ! command -v appium >/dev/null 2>&1; then
@@ -684,24 +739,24 @@ jobs:
fi
bash scripts/ci-cancel-aware.sh appium driver install --source=npm appium-chromium-driver >/dev/null 2>&1 || true
# Binary cache — see Linux full job for the rationale. Mac caches the
# entire .app bundle (self-contained including frontend assets + CEF
# Frameworks/OpenHuman Helper.app embedded by tauri-bundler).
- name: Cache built E2E binary (macOS)
id: e2e-binary-cache
uses: actions/cache@v5
- name: Download build artifact
uses: actions/download-artifact@v5
with:
path: |
app/src-tauri/target/debug/bundle/macos/OpenHuman.app
key: e2e-binary-${{ runner.os }}-${{ hashFiles('src/**/*.rs', 'app/src-tauri/src/**', 'app/src-tauri/build.rs', 'app/src-tauri/tauri.conf.json', 'Cargo.lock', 'app/src-tauri/Cargo.lock', 'app/src-tauri/vendor/tauri-cef/Cargo.lock', 'rust-toolchain.toml', 'app/src/**', 'app/index.html', 'app/vite.config.*', 'app/tailwind.config.*', 'app/postcss.config.*', 'app/package.json', 'pnpm-lock.yaml', 'app/scripts/e2e-build.sh') }}
name: e2e-build-macos-${{ github.run_id }}
path: .
- name: Build E2E app
if: steps.e2e-binary-cache.outputs.cache-hit != 'true'
run: bash scripts/ci-cancel-aware.sh pnpm --filter openhuman-app test:e2e:build
- name: Restore .app bundle into workspace
run: |
mkdir -p app/src-tauri/target/debug/bundle/macos
tar -xzf e2e-build-macos.tar.gz \
-C app/src-tauri/target/debug/bundle/macos
rm -f e2e-build-macos.tar.gz
ls -la app/src-tauri/target/debug/bundle/macos/OpenHuman.app | head
# Adhoc-sign runs unconditionally — codesign is idempotent and a
# restored .app bundle from cache also needs to be (re-)signed for
# macOS to load its dynamic frameworks on this runner.
# macOS rejects dynamic-framework loads from unsigned bundles — adhoc
# signing satisfies the loader without a real developer-ID cert. A
# bundle restored from the build artifact also needs (re-)signing on this
# runner; codesign is idempotent.
- name: Adhoc-sign the .app bundle
run: |
codesign --force --deep --sign - \
@@ -744,23 +799,16 @@ jobs:
fi
# ---------------------------------------------------------------------------
# Full-suite Windows — sharded matrix mirroring e2e-linux-full.
# Full-suite Windows — build-once-then-fanout, mirroring build-linux-full.
# `build-windows-full` builds the .exe + frontend dist + CEF runtime once and
# uploads them as a per-run workflow artifact; the shards `needs:` it and
# download, instead of each shard rebuilding on a cold binary cache.
# ---------------------------------------------------------------------------
e2e-windows-full:
build-windows-full:
if: inputs.run_windows && inputs.full
name: E2E (Windows full / ${{ matrix.shard.name }})
name: Build (Windows full)
runs-on: windows-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
shard:
- { name: foundation, suites: "auth,navigation,system" }
- { name: chat, suites: "chat,skills,journeys" }
- { name: providers, suites: "providers,notifications" }
- { name: webhooks, suites: "webhooks" }
- { name: connectors, suites: "connectors" }
- { name: commerce, suites: "payments,settings" }
steps:
- name: Checkout code
uses: actions/checkout@v5
@@ -791,26 +839,17 @@ jobs:
key: e2e-windows-unified
- name: Cache CEF binary distribution
id: cef-cache
uses: actions/cache@v5
with:
# ensure-tauri-cli.sh + e2e-build.sh both export
# CEF_PATH=$HOME/Library/Caches/tauri-cef regardless of OS; on
# Windows under Git Bash that resolves under the user profile and
# is the actual download target.
# ensure-tauri-cli.sh + e2e-build.sh export
# CEF_PATH=$HOME/Library/Caches/tauri-cef regardless of OS; under Git
# Bash on Windows that resolves under the user profile.
path: |
~/Library/Caches/tauri-cef
key: cef-x86_64-pc-windows-msvc-v2-${{ hashFiles('app/src-tauri/Cargo.toml') }}
restore-keys: |
cef-x86_64-pc-windows-msvc-v2-
- name: Cache Appium global install
uses: actions/cache@v5
with:
path: |
~/.appium
key: appium3-chromium-${{ runner.os }}-v1
- name: Install JS dependencies
run: bash scripts/ci-cancel-aware.sh pnpm install --frozen-lockfile
@@ -820,6 +859,76 @@ jobs:
touch .env
touch app/.env
- name: Build E2E app
run: bash scripts/ci-cancel-aware.sh pnpm --filter openhuman-app test:e2e:build
- name: Package build artifact
shell: bash
run: |
# Windows is NOT self-contained: the shard needs the .exe, the
# frontend dist, and the CEF runtime (exported via CEF_PATH at run
# time). Stage all three at the layout the shard restores into.
STAGE="$(mktemp -d)"
mkdir -p "$STAGE/repo/app/src-tauri/target/debug"
mkdir -p "$STAGE/repo/app/dist"
mkdir -p "$STAGE/home/Library/Caches"
cp -a app/src-tauri/target/debug/OpenHuman.exe "$STAGE/repo/app/src-tauri/target/debug/"
cp -a app/dist/. "$STAGE/repo/app/dist/"
cp -a "$HOME/Library/Caches/tauri-cef" "$STAGE/home/Library/Caches/tauri-cef"
tar -czf e2e-build-windows.tar.gz -C "$STAGE" repo home
ls -lh e2e-build-windows.tar.gz
- name: Upload build artifact
uses: actions/upload-artifact@v5
with:
name: e2e-build-windows-${{ github.run_id }}
path: e2e-build-windows.tar.gz
retention-days: 1
if-no-files-found: error
e2e-windows-full:
if: inputs.run_windows && inputs.full
needs: build-windows-full
name: E2E (Windows full / ${{ matrix.shard.name }})
runs-on: windows-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
shard:
- { name: foundation, suites: "auth,navigation,system" }
- { name: chat, suites: "chat,skills,journeys" }
- { name: providers, suites: "providers,notifications" }
- { name: webhooks, suites: "webhooks" }
- { name: connectors, suites: "connectors" }
- { name: commerce, suites: "payments,settings" }
steps:
- name: Checkout code
uses: actions/checkout@v5
with:
ref: ${{ inputs.ref }}
fetch-depth: 1
submodules: recursive
- name: Install pnpm
uses: pnpm/action-setup@v4
- name: Setup Node.js 24.x
uses: actions/setup-node@v5
with:
node-version: 24.x
cache: pnpm
- name: Cache Appium global install
uses: actions/cache@v5
with:
path: |
~/.appium
key: appium3-chromium-${{ runner.os }}-v1
- name: Install JS dependencies (for test harness only)
run: bash scripts/ci-cancel-aware.sh pnpm install --frozen-lockfile
- name: Install Appium and chromium driver
shell: bash
run: |
@@ -828,34 +937,30 @@ jobs:
fi
bash scripts/ci-cancel-aware.sh appium driver install --source=npm appium-chromium-driver >/dev/null 2>&1 || true
# Binary cache — see Linux full job for rationale. Windows is built
# with --debug --no-bundle so the .exe + frontend dist are what the
# runner needs at launch. CEF runtime DLLs come from the dedicated
# CEF cache step above (now correctly pointing at the actual download
# location, ~/Library/Caches/tauri-cef).
- name: Cache built E2E binary (Windows)
id: e2e-binary-cache
uses: actions/cache@v5
- name: Download build artifact
uses: actions/download-artifact@v5
with:
path: |
app/src-tauri/target/debug/OpenHuman.exe
app/dist
key: e2e-binary-${{ runner.os }}-${{ hashFiles('src/**/*.rs', 'app/src-tauri/src/**', 'app/src-tauri/build.rs', 'app/src-tauri/tauri.conf.json', 'Cargo.lock', 'app/src-tauri/Cargo.lock', 'app/src-tauri/vendor/tauri-cef/Cargo.lock', 'rust-toolchain.toml', 'app/src/**', 'app/index.html', 'app/vite.config.*', 'app/tailwind.config.*', 'app/postcss.config.*', 'app/package.json', 'pnpm-lock.yaml', 'app/scripts/e2e-build.sh') }}
name: e2e-build-windows-${{ github.run_id }}
path: .
# Skip the build only when BOTH the binary AND the CEF runtime caches
# hit (see Linux full job for the rationale).
- name: Build E2E app
if: steps.e2e-binary-cache.outputs.cache-hit != 'true' || steps.cef-cache.outputs.cache-hit != 'true'
run: bash scripts/ci-cancel-aware.sh pnpm --filter openhuman-app test:e2e:build
- name: Restore build artifact into workspace + $HOME
shell: bash
run: |
tar -xzf e2e-build-windows.tar.gz
mkdir -p app/src-tauri/target/debug app/dist "$HOME/Library/Caches"
cp -a repo/app/src-tauri/target/debug/OpenHuman.exe app/src-tauri/target/debug/
cp -a repo/app/dist/. app/dist/
cp -a home/Library/Caches/tauri-cef "$HOME/Library/Caches/"
rm -rf repo home e2e-build-windows.tar.gz
ls -la app/src-tauri/target/debug/OpenHuman.exe app/dist | head
- name: Run E2E shard (${{ matrix.shard.name }} — suites=${{ matrix.shard.suites }})
shell: bash
env:
E2E_BAIL_ON_FAILURE: ${{ vars.E2E_BAIL_ON_FAILURE || '' }}
run: |
# See Linux shard — binary cache can skip the build that would have
# exported CEF_PATH. e2e-build.sh + ensure-tauri-cli.sh always
# download CEF to $HOME/Library/Caches/tauri-cef regardless of OS.
# e2e-build.sh + ensure-tauri-cli.sh always download CEF to
# $HOME/Library/Caches/tauri-cef regardless of OS.
export CEF_PATH="$HOME/Library/Caches/tauri-cef"
BAIL_FLAG=""
if [[ "${E2E_BAIL_ON_FAILURE:-}" == "1" ]]; then
+4
View File
@@ -59,6 +59,10 @@ GGML_NATIVE=OFF cargo check --manifest-path Cargo.toml
`pnpm core:stage` is a no-op (sidecar removed).
**Build speed**: both `Cargo.toml` files set `[profile.dev.package."*"] debug = false` — dependencies compile without DWARF in `dev`/`test` (faster builds + smaller `target/`); our own crates keep full debuginfo so panics/backtraces still resolve to file:line. `release`/`ci` profiles are unchanged. Keep this stanza in sync across the root and `app/src-tauri/Cargo.toml` if you touch profiles.
**CI build topology**: full-suite E2E is **build-once-then-fanout** on all three OSes — `build-{linux,macos,windows}-full` compile/bundle the app once and upload it as a per-run workflow artifact, and the shard jobs (`e2e-*-full`) `needs:` that job and download it instead of each shard rebuilding on a cold cache (`.github/workflows/e2e-reusable.yml`). Linux desktop packaging (`build-desktop.yml`) does a **single** `cargo tauri build`: libcef.so is resolved from the restored CEF cache (or a targeted `cargo build -p cef-dll-sys` prewarm on a cold cache) rather than a throwaway `--no-bundle` full build. The root core crate and the Tauri shell are still **separate Cargo worlds** (two `Cargo.lock`, two `target/`); converging them into one workspace is tracked as follow-up in #3877.
**Tests**: `pnpm test` (Vitest) · `pnpm test:coverage` · `pnpm test:rust` (`scripts/test-rust-with-mock.sh`).
**Quality**: ESLint + Prettier + Husky. Pre-push hook runs `pnpm rust:check`.
+18
View File
@@ -308,3 +308,21 @@ lto = false
incremental = false
strip = true
debug = false
# Faster local + CI iteration (#3877): compile third-party dependencies in the
# dev/test profiles WITHOUT debuginfo. The dependency graph here is large (a
# local checkout showed root `target/` ~12G and Tauri `target/` ~4.4G), and
# DWARF generation + linking for every dependency is a dominant, repeated cost
# across `cargo build`, `cargo test`, `cargo clippy`, and `cargo llvm-cov`.
#
# Scope is intentionally narrow and low-risk:
# * `package."*"` targets dependencies only — our own crates keep full
# debuginfo, so panics/backtraces in OpenHuman code still resolve to
# file:line and a debugger can still step through our code.
# * Only the unoptimised `dev`/`test` profiles change. `release` and `ci`
# (which already set `debug = false` / `line-tables-only`) are untouched.
# * No artifact paths, features, or runtime behaviour change — this only
# reduces how much DWARF is emitted for dependencies, which also shrinks
# `target/` substantially.
[profile.dev.package."*"]
debug = false
+16
View File
@@ -257,3 +257,19 @@ lto = false
incremental = false
strip = true
debug = false
# Faster local + CI iteration (#3877): compile third-party dependencies in the
# dev/test profiles WITHOUT debuginfo. The Tauri shell embeds the full core
# crate, so its dev dependency graph is huge (a local checkout showed
# `app/src-tauri/target/` ~4.4G) and DWARF generation + linking for every
# dependency dominates `cargo build` / `cargo tauri dev` / `cargo llvm-cov`.
#
# Scope is intentionally narrow and low-risk (kept in sync with the root
# crate's `Cargo.toml` so both Cargo worlds get the same discipline):
# * `package."*"` targets dependencies only — the shell + core crates keep
# full debuginfo, so panics/backtraces still resolve to file:line.
# * Only the unoptimised `dev`/`test` profiles change; `release`/`ci` are
# untouched, so shipped bundles and Sentry symbolication are unaffected.
# * No artifact paths, features, or runtime behaviour change.
[profile.dev.package."*"]
debug = false