From 3ded18c165b26714da3aa469d96b2259da970971 Mon Sep 17 00:00:00 2001 From: Mega Mind <146339422+M3gA-Mind@users.noreply.github.com> Date: Wed, 24 Jun 2026 00:20:20 +0530 Subject: [PATCH] perf: cut OpenHuman + Tauri compile/CI time (dev-profile deps, build-once E2E fanout, single-pass Linux packaging) (#3986) --- .github/workflows/build-desktop.yml | 49 +++-- .github/workflows/e2e-reusable.yml | 265 +++++++++++++++++++--------- AGENTS.md | 4 + Cargo.toml | 18 ++ app/src-tauri/Cargo.toml | 16 ++ 5 files changed, 261 insertions(+), 91 deletions(-) diff --git a/.github/workflows/build-desktop.yml b/.github/workflows/build-desktop.yml index 1ca978909..294c6ee2b 100644 --- a/.github/workflows/build-desktop.yml +++ b/.github/workflows/build-desktop.yml @@ -434,19 +434,46 @@ jobs: # copied into the AppDir at usr/lib/libcef.so, but the binary's # RUNPATH does not include either location, so plain `ldd` reports # `libcef.so => not found` and lib4bin aborts with the generic - # "is missing libraries! Aborting..." banner. To get past it we: - # 1. Run `cargo tauri build --no-bundle` first so the cef-dll-sys - # build script downloads libcef.so into the cache dir. - # 2. Re-invoke `cargo tauri build` (incremental — cargo skips the - # already-compiled artifacts) with that cache dir on - # LD_LIBRARY_PATH so the bundler's ldd call resolves libcef.so. - # macOS / Windows take the original single-call path. + # "is missing libraries! Aborting..." banner. libcef.so is downloaded + # by the cef-dll-sys build script into the CEF cache dir, which the + # "Cache CEF binary distribution" step restores across runs. + # + # Build-once (#3877): this used to compile the whole app twice — a + # throwaway `cargo tauri build --no-bundle` purely to fetch CEF, then + # the real bundling build — which also re-ran the Vite/bundler + # orchestration twice. We now resolve libcef.so cheaply and bundle in + # a SINGLE `cargo tauri build`: + # 1. Warm CEF cache (the common case): libcef.so is already restored, + # so we skip straight to the single bundling build. + # 2. Cold cache: prewarm with a TARGETED `cargo build -p cef-dll-sys` + # (compiles only the CEF downloader crate + its deps, not the + # whole app) to populate the cache. + # 3. If that still didn't yield libcef.so, fall back to the + # historical full `--no-bundle` prebuild — so correctness can + # never regress relative to the old double build. + # macOS / Windows take the single-call path unchanged. + find_cef_lib_dir() { + find "$HOME/.cache/tauri-cef" -name libcef.so -printf '%h\n' 2>/dev/null | head -1 + } if [ "${RUNNER_OS}" = "Linux" ]; then - echo "[appimage-fix] linux split build: compile first to fetch CEF" - NODE_OPTIONS="--max-old-space-size=8192" bash ../scripts/ci-cancel-aware.sh cargo tauri build --no-bundle $PROFILE_FLAG -c "$TAURI_CONFIG_OVERRIDE" $MATRIX_ARGS - CEF_LIB_DIR="$(find "$HOME/.cache/tauri-cef" -name libcef.so -printf '%h\n' 2>/dev/null | head -1)" + CEF_LIB_DIR="$(find_cef_lib_dir)" if [ -z "$CEF_LIB_DIR" ]; then - echo "::error::libcef.so not found under ~/.cache/tauri-cef after --no-bundle compile; cannot satisfy lib4bin ldd resolution." >&2 + echo "[appimage-fix] CEF not cached — targeted prewarm via cef-dll-sys build script" + # `cargo tauri build --debug` maps to cargo's default (dev) + # profile; release maps to `--release`. Match the profile so the + # prewarm's artifacts are reused by the bundling build below. + CARGO_BUILD_PROFILE="--release" + [ "$PROFILE_FLAG" = "--debug" ] && CARGO_BUILD_PROFILE="" + NODE_OPTIONS="--max-old-space-size=8192" bash ../scripts/ci-cancel-aware.sh cargo build -p cef-dll-sys $CARGO_BUILD_PROFILE || true + CEF_LIB_DIR="$(find_cef_lib_dir)" + fi + if [ -z "$CEF_LIB_DIR" ]; then + echo "[appimage-fix] targeted prewarm did not yield libcef.so — falling back to full --no-bundle prebuild" + NODE_OPTIONS="--max-old-space-size=8192" bash ../scripts/ci-cancel-aware.sh cargo tauri build --no-bundle $PROFILE_FLAG -c "$TAURI_CONFIG_OVERRIDE" $MATRIX_ARGS + CEF_LIB_DIR="$(find_cef_lib_dir)" + fi + if [ -z "$CEF_LIB_DIR" ]; then + echo "::error::libcef.so not found under ~/.cache/tauri-cef after prewarm; cannot satisfy lib4bin ldd resolution." >&2 exit 1 fi echo "[appimage-fix] prepending CEF lib dir to LD_LIBRARY_PATH: $CEF_LIB_DIR" diff --git a/.github/workflows/e2e-reusable.yml b/.github/workflows/e2e-reusable.yml index 6d60a422f..a9385ba42 100644 --- a/.github/workflows/e2e-reusable.yml +++ b/.github/workflows/e2e-reusable.yml @@ -595,23 +595,20 @@ jobs: # reusable-workflow-is-also-used-by-releases rationale. # --------------------------------------------------------------------------- - # Full-suite macOS — sharded matrix mirroring e2e-linux-full. + # Full-suite macOS — build-once-then-fanout, mirroring build-linux-full. + # + # Previously e2e-macos-full was a 6-shard matrix where each shard restored an + # `actions/cache` of the .app bundle and rebuilt the full Tauri app on a cold + # cache. On a cold cache that means up to 6 parallel full Tauri builds (only + # the first shard to finish would populate the cache; the rest lose the race). + # `build-macos-full` now builds the .app once and uploads it as a per-run + # workflow artifact; the shards `needs:` it, download, adhoc-sign, and run. # --------------------------------------------------------------------------- - e2e-macos-full: + build-macos-full: if: inputs.run_macos && inputs.full - name: E2E (macOS full / ${{ matrix.shard.name }}) + name: Build (macOS full) runs-on: macos-latest timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: - - { name: foundation, suites: "auth,navigation,system" } - - { name: chat, suites: "chat,skills,journeys" } - - { name: providers, suites: "providers,notifications" } - - { name: webhooks, suites: "webhooks" } - - { name: connectors, suites: "connectors" } - - { name: commerce, suites: "payments,settings" } steps: - name: Checkout code uses: actions/checkout@v5 @@ -662,13 +659,6 @@ jobs: restore-keys: | cef-aarch64-apple-darwin- - - name: Cache Appium global install - uses: actions/cache@v5 - with: - path: | - ~/.appium - key: appium3-chromium-${{ runner.os }}-v1 - - name: Install JS dependencies run: bash scripts/ci-cancel-aware.sh pnpm install --frozen-lockfile @@ -677,6 +667,71 @@ jobs: touch .env touch app/.env + - name: Build E2E app + run: bash scripts/ci-cancel-aware.sh pnpm --filter openhuman-app test:e2e:build + + - name: Package build artifact + run: | + # The macOS .app is self-contained (frontend assets + CEF Frameworks + + # the OpenHuman Helper.app are embedded by tauri-bundler), so the .app + # bundle alone is everything a shard needs. tar preserves the bundle's + # internal symlinks + executable bits; each shard re-signs after + # extraction (adhoc codesign is required on every runner regardless). + tar -czf e2e-build-macos.tar.gz \ + -C app/src-tauri/target/debug/bundle/macos OpenHuman.app + ls -lh e2e-build-macos.tar.gz + + - name: Upload build artifact + uses: actions/upload-artifact@v5 + with: + name: e2e-build-macos-${{ github.run_id }} + path: e2e-build-macos.tar.gz + retention-days: 1 + if-no-files-found: error + + e2e-macos-full: + if: inputs.run_macos && inputs.full + needs: build-macos-full + name: E2E (macOS full / ${{ matrix.shard.name }}) + runs-on: macos-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shard: + - { name: foundation, suites: "auth,navigation,system" } + - { name: chat, suites: "chat,skills,journeys" } + - { name: providers, suites: "providers,notifications" } + - { name: webhooks, suites: "webhooks" } + - { name: connectors, suites: "connectors" } + - { name: commerce, suites: "payments,settings" } + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + fetch-depth: 1 + submodules: recursive + + - name: Install pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js 24.x + uses: actions/setup-node@v5 + with: + node-version: 24.x + cache: pnpm + + - name: Cache Appium global install + uses: actions/cache@v5 + with: + path: | + ~/.appium + key: appium3-chromium-${{ runner.os }}-v1 + + - name: Install JS dependencies (for test harness only) + run: bash scripts/ci-cancel-aware.sh pnpm install --frozen-lockfile + - name: Install Appium and chromium driver run: | if ! command -v appium >/dev/null 2>&1; then @@ -684,24 +739,24 @@ jobs: fi bash scripts/ci-cancel-aware.sh appium driver install --source=npm appium-chromium-driver >/dev/null 2>&1 || true - # Binary cache — see Linux full job for the rationale. Mac caches the - # entire .app bundle (self-contained including frontend assets + CEF - # Frameworks/OpenHuman Helper.app embedded by tauri-bundler). - - name: Cache built E2E binary (macOS) - id: e2e-binary-cache - uses: actions/cache@v5 + - name: Download build artifact + uses: actions/download-artifact@v5 with: - path: | - app/src-tauri/target/debug/bundle/macos/OpenHuman.app - key: e2e-binary-${{ runner.os }}-${{ hashFiles('src/**/*.rs', 'app/src-tauri/src/**', 'app/src-tauri/build.rs', 'app/src-tauri/tauri.conf.json', 'Cargo.lock', 'app/src-tauri/Cargo.lock', 'app/src-tauri/vendor/tauri-cef/Cargo.lock', 'rust-toolchain.toml', 'app/src/**', 'app/index.html', 'app/vite.config.*', 'app/tailwind.config.*', 'app/postcss.config.*', 'app/package.json', 'pnpm-lock.yaml', 'app/scripts/e2e-build.sh') }} + name: e2e-build-macos-${{ github.run_id }} + path: . - - name: Build E2E app - if: steps.e2e-binary-cache.outputs.cache-hit != 'true' - run: bash scripts/ci-cancel-aware.sh pnpm --filter openhuman-app test:e2e:build + - name: Restore .app bundle into workspace + run: | + mkdir -p app/src-tauri/target/debug/bundle/macos + tar -xzf e2e-build-macos.tar.gz \ + -C app/src-tauri/target/debug/bundle/macos + rm -f e2e-build-macos.tar.gz + ls -la app/src-tauri/target/debug/bundle/macos/OpenHuman.app | head - # Adhoc-sign runs unconditionally — codesign is idempotent and a - # restored .app bundle from cache also needs to be (re-)signed for - # macOS to load its dynamic frameworks on this runner. + # macOS rejects dynamic-framework loads from unsigned bundles — adhoc + # signing satisfies the loader without a real developer-ID cert. A + # bundle restored from the build artifact also needs (re-)signing on this + # runner; codesign is idempotent. - name: Adhoc-sign the .app bundle run: | codesign --force --deep --sign - \ @@ -744,23 +799,16 @@ jobs: fi # --------------------------------------------------------------------------- - # Full-suite Windows — sharded matrix mirroring e2e-linux-full. + # Full-suite Windows — build-once-then-fanout, mirroring build-linux-full. + # `build-windows-full` builds the .exe + frontend dist + CEF runtime once and + # uploads them as a per-run workflow artifact; the shards `needs:` it and + # download, instead of each shard rebuilding on a cold binary cache. # --------------------------------------------------------------------------- - e2e-windows-full: + build-windows-full: if: inputs.run_windows && inputs.full - name: E2E (Windows full / ${{ matrix.shard.name }}) + name: Build (Windows full) runs-on: windows-latest timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: - - { name: foundation, suites: "auth,navigation,system" } - - { name: chat, suites: "chat,skills,journeys" } - - { name: providers, suites: "providers,notifications" } - - { name: webhooks, suites: "webhooks" } - - { name: connectors, suites: "connectors" } - - { name: commerce, suites: "payments,settings" } steps: - name: Checkout code uses: actions/checkout@v5 @@ -791,26 +839,17 @@ jobs: key: e2e-windows-unified - name: Cache CEF binary distribution - id: cef-cache uses: actions/cache@v5 with: - # ensure-tauri-cli.sh + e2e-build.sh both export - # CEF_PATH=$HOME/Library/Caches/tauri-cef regardless of OS; on - # Windows under Git Bash that resolves under the user profile and - # is the actual download target. + # ensure-tauri-cli.sh + e2e-build.sh export + # CEF_PATH=$HOME/Library/Caches/tauri-cef regardless of OS; under Git + # Bash on Windows that resolves under the user profile. path: | ~/Library/Caches/tauri-cef key: cef-x86_64-pc-windows-msvc-v2-${{ hashFiles('app/src-tauri/Cargo.toml') }} restore-keys: | cef-x86_64-pc-windows-msvc-v2- - - name: Cache Appium global install - uses: actions/cache@v5 - with: - path: | - ~/.appium - key: appium3-chromium-${{ runner.os }}-v1 - - name: Install JS dependencies run: bash scripts/ci-cancel-aware.sh pnpm install --frozen-lockfile @@ -820,6 +859,76 @@ jobs: touch .env touch app/.env + - name: Build E2E app + run: bash scripts/ci-cancel-aware.sh pnpm --filter openhuman-app test:e2e:build + + - name: Package build artifact + shell: bash + run: | + # Windows is NOT self-contained: the shard needs the .exe, the + # frontend dist, and the CEF runtime (exported via CEF_PATH at run + # time). Stage all three at the layout the shard restores into. + STAGE="$(mktemp -d)" + mkdir -p "$STAGE/repo/app/src-tauri/target/debug" + mkdir -p "$STAGE/repo/app/dist" + mkdir -p "$STAGE/home/Library/Caches" + cp -a app/src-tauri/target/debug/OpenHuman.exe "$STAGE/repo/app/src-tauri/target/debug/" + cp -a app/dist/. "$STAGE/repo/app/dist/" + cp -a "$HOME/Library/Caches/tauri-cef" "$STAGE/home/Library/Caches/tauri-cef" + tar -czf e2e-build-windows.tar.gz -C "$STAGE" repo home + ls -lh e2e-build-windows.tar.gz + + - name: Upload build artifact + uses: actions/upload-artifact@v5 + with: + name: e2e-build-windows-${{ github.run_id }} + path: e2e-build-windows.tar.gz + retention-days: 1 + if-no-files-found: error + + e2e-windows-full: + if: inputs.run_windows && inputs.full + needs: build-windows-full + name: E2E (Windows full / ${{ matrix.shard.name }}) + runs-on: windows-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shard: + - { name: foundation, suites: "auth,navigation,system" } + - { name: chat, suites: "chat,skills,journeys" } + - { name: providers, suites: "providers,notifications" } + - { name: webhooks, suites: "webhooks" } + - { name: connectors, suites: "connectors" } + - { name: commerce, suites: "payments,settings" } + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + fetch-depth: 1 + submodules: recursive + + - name: Install pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js 24.x + uses: actions/setup-node@v5 + with: + node-version: 24.x + cache: pnpm + + - name: Cache Appium global install + uses: actions/cache@v5 + with: + path: | + ~/.appium + key: appium3-chromium-${{ runner.os }}-v1 + + - name: Install JS dependencies (for test harness only) + run: bash scripts/ci-cancel-aware.sh pnpm install --frozen-lockfile + - name: Install Appium and chromium driver shell: bash run: | @@ -828,34 +937,30 @@ jobs: fi bash scripts/ci-cancel-aware.sh appium driver install --source=npm appium-chromium-driver >/dev/null 2>&1 || true - # Binary cache — see Linux full job for rationale. Windows is built - # with --debug --no-bundle so the .exe + frontend dist are what the - # runner needs at launch. CEF runtime DLLs come from the dedicated - # CEF cache step above (now correctly pointing at the actual download - # location, ~/Library/Caches/tauri-cef). - - name: Cache built E2E binary (Windows) - id: e2e-binary-cache - uses: actions/cache@v5 + - name: Download build artifact + uses: actions/download-artifact@v5 with: - path: | - app/src-tauri/target/debug/OpenHuman.exe - app/dist - key: e2e-binary-${{ runner.os }}-${{ hashFiles('src/**/*.rs', 'app/src-tauri/src/**', 'app/src-tauri/build.rs', 'app/src-tauri/tauri.conf.json', 'Cargo.lock', 'app/src-tauri/Cargo.lock', 'app/src-tauri/vendor/tauri-cef/Cargo.lock', 'rust-toolchain.toml', 'app/src/**', 'app/index.html', 'app/vite.config.*', 'app/tailwind.config.*', 'app/postcss.config.*', 'app/package.json', 'pnpm-lock.yaml', 'app/scripts/e2e-build.sh') }} + name: e2e-build-windows-${{ github.run_id }} + path: . - # Skip the build only when BOTH the binary AND the CEF runtime caches - # hit (see Linux full job for the rationale). - - name: Build E2E app - if: steps.e2e-binary-cache.outputs.cache-hit != 'true' || steps.cef-cache.outputs.cache-hit != 'true' - run: bash scripts/ci-cancel-aware.sh pnpm --filter openhuman-app test:e2e:build + - name: Restore build artifact into workspace + $HOME + shell: bash + run: | + tar -xzf e2e-build-windows.tar.gz + mkdir -p app/src-tauri/target/debug app/dist "$HOME/Library/Caches" + cp -a repo/app/src-tauri/target/debug/OpenHuman.exe app/src-tauri/target/debug/ + cp -a repo/app/dist/. app/dist/ + cp -a home/Library/Caches/tauri-cef "$HOME/Library/Caches/" + rm -rf repo home e2e-build-windows.tar.gz + ls -la app/src-tauri/target/debug/OpenHuman.exe app/dist | head - name: Run E2E shard (${{ matrix.shard.name }} — suites=${{ matrix.shard.suites }}) shell: bash env: E2E_BAIL_ON_FAILURE: ${{ vars.E2E_BAIL_ON_FAILURE || '' }} run: | - # See Linux shard — binary cache can skip the build that would have - # exported CEF_PATH. e2e-build.sh + ensure-tauri-cli.sh always - # download CEF to $HOME/Library/Caches/tauri-cef regardless of OS. + # e2e-build.sh + ensure-tauri-cli.sh always download CEF to + # $HOME/Library/Caches/tauri-cef regardless of OS. export CEF_PATH="$HOME/Library/Caches/tauri-cef" BAIL_FLAG="" if [[ "${E2E_BAIL_ON_FAILURE:-}" == "1" ]]; then diff --git a/AGENTS.md b/AGENTS.md index 1de0a83fb..ce8a55650 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,6 +59,10 @@ GGML_NATIVE=OFF cargo check --manifest-path Cargo.toml `pnpm core:stage` is a no-op (sidecar removed). +**Build speed**: both `Cargo.toml` files set `[profile.dev.package."*"] debug = false` — dependencies compile without DWARF in `dev`/`test` (faster builds + smaller `target/`); our own crates keep full debuginfo so panics/backtraces still resolve to file:line. `release`/`ci` profiles are unchanged. Keep this stanza in sync across the root and `app/src-tauri/Cargo.toml` if you touch profiles. + +**CI build topology**: full-suite E2E is **build-once-then-fanout** on all three OSes — `build-{linux,macos,windows}-full` compile/bundle the app once and upload it as a per-run workflow artifact, and the shard jobs (`e2e-*-full`) `needs:` that job and download it instead of each shard rebuilding on a cold cache (`.github/workflows/e2e-reusable.yml`). Linux desktop packaging (`build-desktop.yml`) does a **single** `cargo tauri build`: libcef.so is resolved from the restored CEF cache (or a targeted `cargo build -p cef-dll-sys` prewarm on a cold cache) rather than a throwaway `--no-bundle` full build. The root core crate and the Tauri shell are still **separate Cargo worlds** (two `Cargo.lock`, two `target/`); converging them into one workspace is tracked as follow-up in #3877. + **Tests**: `pnpm test` (Vitest) · `pnpm test:coverage` · `pnpm test:rust` (`scripts/test-rust-with-mock.sh`). **Quality**: ESLint + Prettier + Husky. Pre-push hook runs `pnpm rust:check`. diff --git a/Cargo.toml b/Cargo.toml index 96ee3495c..3a8403f9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -308,3 +308,21 @@ lto = false incremental = false strip = true debug = false + +# Faster local + CI iteration (#3877): compile third-party dependencies in the +# dev/test profiles WITHOUT debuginfo. The dependency graph here is large (a +# local checkout showed root `target/` ~12G and Tauri `target/` ~4.4G), and +# DWARF generation + linking for every dependency is a dominant, repeated cost +# across `cargo build`, `cargo test`, `cargo clippy`, and `cargo llvm-cov`. +# +# Scope is intentionally narrow and low-risk: +# * `package."*"` targets dependencies only — our own crates keep full +# debuginfo, so panics/backtraces in OpenHuman code still resolve to +# file:line and a debugger can still step through our code. +# * Only the unoptimised `dev`/`test` profiles change. `release` and `ci` +# (which already set `debug = false` / `line-tables-only`) are untouched. +# * No artifact paths, features, or runtime behaviour change — this only +# reduces how much DWARF is emitted for dependencies, which also shrinks +# `target/` substantially. +[profile.dev.package."*"] +debug = false diff --git a/app/src-tauri/Cargo.toml b/app/src-tauri/Cargo.toml index f52293085..529147ddc 100644 --- a/app/src-tauri/Cargo.toml +++ b/app/src-tauri/Cargo.toml @@ -257,3 +257,19 @@ lto = false incremental = false strip = true debug = false + +# Faster local + CI iteration (#3877): compile third-party dependencies in the +# dev/test profiles WITHOUT debuginfo. The Tauri shell embeds the full core +# crate, so its dev dependency graph is huge (a local checkout showed +# `app/src-tauri/target/` ~4.4G) and DWARF generation + linking for every +# dependency dominates `cargo build` / `cargo tauri dev` / `cargo llvm-cov`. +# +# Scope is intentionally narrow and low-risk (kept in sync with the root +# crate's `Cargo.toml` so both Cargo worlds get the same discipline): +# * `package."*"` targets dependencies only — the shell + core crates keep +# full debuginfo, so panics/backtraces still resolve to file:line. +# * Only the unoptimised `dev`/`test` profiles change; `release`/`ci` are +# untouched, so shipped bundles and Sentry symbolication are unaffected. +# * No artifact paths, features, or runtime behaviour change. +[profile.dev.package."*"] +debug = false