From 7081be7bd349d1b3b9085ce50f2bf6891fc13de6 Mon Sep 17 00:00:00 2001 From: Tanvir Bhathal <52218590+mrTSB@users.noreply.github.com> Date: Sun, 17 May 2026 13:07:05 -0700 Subject: [PATCH] [FEAT] Telemetry (#351) --- deploy/posthog/posthog-hetzner-prep.sh | 142 +++++++ docs/telemetry.md | 124 ++++++ frontend/package-lock.json | 442 +++++++++++++++++++++- frontend/package.json | 1 + frontend/src-tauri/Cargo.lock | 93 +++++ frontend/src/App.tsx | 37 +- frontend/src/lib/analytics.ts | 208 ++++++++++ frontend/src/lib/store.ts | 6 + frontend/src/main.tsx | 5 + frontend/tsconfig.tsbuildinfo | 2 +- pyproject.toml | 1 + scripts/install/install.sh | 146 +++++++ src/openjarvis/analytics/__init__.py | 33 ++ src/openjarvis/analytics/aggregator.py | 186 +++++++++ src/openjarvis/analytics/bridge.py | 217 +++++++++++ src/openjarvis/analytics/client.py | 127 +++++++ src/openjarvis/analytics/events.py | 407 ++++++++++++++++++++ src/openjarvis/analytics/identity.py | 49 +++ src/openjarvis/analytics/redaction.py | 106 ++++++ src/openjarvis/core/config.py | 23 ++ src/openjarvis/server/analytics_routes.py | 55 +++ src/openjarvis/server/app.py | 44 +++ tests/analytics/__init__.py | 0 tests/analytics/test_redaction.py | 339 +++++++++++++++++ uv.lock | 17 + 25 files changed, 2807 insertions(+), 3 deletions(-) create mode 100755 deploy/posthog/posthog-hetzner-prep.sh create mode 100644 docs/telemetry.md create mode 100644 frontend/src/lib/analytics.ts create mode 100644 src/openjarvis/analytics/__init__.py create mode 100644 src/openjarvis/analytics/aggregator.py create mode 100644 src/openjarvis/analytics/bridge.py create mode 100644 src/openjarvis/analytics/client.py create mode 100644 src/openjarvis/analytics/events.py create mode 100644 src/openjarvis/analytics/identity.py create mode 100644 src/openjarvis/analytics/redaction.py create mode 100644 src/openjarvis/server/analytics_routes.py create mode 100644 tests/analytics/__init__.py create mode 100644 tests/analytics/test_redaction.py diff --git a/deploy/posthog/posthog-hetzner-prep.sh b/deploy/posthog/posthog-hetzner-prep.sh new file mode 100755 index 00000000..ef13ea80 --- /dev/null +++ b/deploy/posthog/posthog-hetzner-prep.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# posthog-hetzner-prep.sh — one-shot Hetzner Cloud server prep for +# OpenJarvis's self-hosted PostHog analytics backend. +# +# Run this on a fresh Ubuntu 22.04+ box (Hetzner CCX23 in Ashburn, or +# similar) after pointing the desired domain at it. Idempotent: safe +# to re-run if a step fails partway. +# +# Usage: +# sudo bash posthog-hetzner-prep.sh analytics.openjarvis.ai you@openjarvis.ai +# +# After it finishes: +# 1. Visit https:/// and create the admin account. +# 2. Create project "OpenJarvis". +# 3. Settings → Project → grab the Project API Key (phc_…). +# 4. Update src/openjarvis/core/config.py AnalyticsConfig defaults: +# host = "https://" +# key = "phc_" +# 5. Ship a release. Frontend + backend + install.sh all read those +# same defaults via load_config(). +# +# Cost: ~$35/mo on Hetzner CCX23 (4 dedicated vCPU / 16 GB / 240 GB +# NVMe) in US-East. Add Hetzner Cloud Backups (+20%) for production. +# +# Retention: post-install, set 365 days in PostHog UI → +# Settings → Data Management → Event ingestion → Data retention. + +set -euo pipefail + +# ---- args ---- +if [[ $# -lt 2 ]]; then + cat >&2 <<'USAGE' +posthog-hetzner-prep.sh: missing arguments. + +Usage: + sudo bash posthog-hetzner-prep.sh + +Examples: + sudo bash posthog-hetzner-prep.sh analytics.openjarvis.ai team@openjarvis.ai + +The domain must already resolve to this box (DNS A record) before +the script runs — Let's Encrypt needs to reach this server on port 80. +USAGE + exit 2 +fi + +DOMAIN="$1" +ADMIN_EMAIL="$2" + +if [[ $EUID -ne 0 ]]; then + echo "posthog-hetzner-prep.sh: must be run as root (use sudo)." >&2 + exit 1 +fi + +# ---- step 1: system prep ---- +echo "[1/5] apt update + base packages..." +export DEBIAN_FRONTEND=noninteractive +apt-get update -y +apt-get install -y --no-install-recommends \ + curl ufw ca-certificates gnupg \ + apt-transport-https software-properties-common + +# ---- step 2: firewall ---- +echo "[2/5] firewall (UFW): 22/80/443 only..." +ufw --force reset +ufw default deny incoming +ufw default allow outgoing +ufw allow 22/tcp +ufw allow 80/tcp +ufw allow 443/tcp +ufw --force enable + +# ---- step 3: swap (helps ClickHouse under load spikes) ---- +echo "[3/5] swap..." +if [[ ! -f /swapfile ]]; then + fallocate -l 4G /swapfile + chmod 600 /swapfile + mkswap /swapfile + swapon /swapfile + if ! grep -q "/swapfile" /etc/fstab; then + echo "/swapfile none swap sw 0 0" >> /etc/fstab + fi +else + echo " /swapfile already present" +fi + +# ---- step 4: docker ---- +echo "[4/5] docker..." +if ! command -v docker >/dev/null 2>&1; then + curl -fsSL https://get.docker.com | sh +else + echo " docker already installed" +fi +systemctl enable --now docker + +# ---- step 5: posthog hobby deploy ---- +echo "[5/5] PostHog Hobby Deploy..." +echo +echo " Domain: $DOMAIN" +echo " Admin email: $ADMIN_EMAIL" +echo " DNS A record: verify it points to $(curl -fsS -m 5 https://api.ipify.org 2>/dev/null || echo "")" +echo + +# PostHog's official one-liner. It writes a .env file with random +# secrets, configures Caddy with Let's Encrypt TLS for the domain, +# and brings up the full stack via docker-compose. +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/posthog/posthog/HEAD/bin/deploy-hobby)" -s -- \ + --domain "$DOMAIN" \ + --email "$ADMIN_EMAIL" || { + echo + echo "PostHog deploy script exited with an error. Common causes:" + echo " - DNS for $DOMAIN doesn't resolve to this server yet (wait + retry)" + echo " - Port 80 not reachable from the public internet (firewall / cloud SG)" + echo " - Out of disk on /var/lib/docker (need 20+ GB free)" + echo + exit 1 +} + +cat < 70% full + - Backups: enable Hetzner Cloud Backups in the Hetzner console +================================================================ +EOF diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 00000000..455f4ee9 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,124 @@ +# Telemetry + +OpenJarvis ships **anonymous usage telemetry** by default so the team can +see where the product breaks, what features people actually use, and +how to make it better. This page documents exactly what is and isn't +collected, where the data goes, and how to opt out. + +## TL;DR + +- **On by default**, anonymous, no chat content. +- **Anonymous** — one random UUID per install, no email, no name, no IP. +- **No chat content, ever.** Only counts, timings, and feature names. +- **Self-hosted backend** on the OpenJarvis team's PostHog instance — + data is not sold or shared with third parties. +- **365-day retention**, after which events are deleted automatically. + +## What we collect + +### Lifecycle events + +| Event | Source | Why we send it | +|---|---|---| +| `install_started` | `install.sh` | Top of install funnel | +| `install_stage_completed` | `install.sh` | Per-stage timing — where do people drop off? | +| `install_completed` | `install.sh` | Did the install succeed? | +| `install_failed` | `install.sh` | Which stage failed, and on what OS | +| `app_opened` | Backend + frontend | DAU / WAU / MAU | +| `setup_completed` | Frontend | First-run wizard finished | +| `first_chat_sent` | Backend | First-ever message — activation | +| `uninstall_started` | `uninstall.sh` (if user runs it) | Churn signal | + +### Usage events + +| Event | Why we send it | +|---|---| +| `chat_session_ended` | Aggregated per-session: turn count, tokens, latency, tool count | +| `tool_first_used` | Which built-in tools are actually adopted | +| `model_changed` | How often users switch models | +| `feature_used` | Which features get traffic, which don't | +| `connector_auth_completed` | Which connectors people set up | +| `error_shown_to_user` | User-visible error class (not stack trace) | +| `feedback_submitted` | Was a rating given? Was a comment included? | +| `settings_changed` | Which settings get toggled | +| `usage_daily_summary` | Once-per-day aggregated counts | + +The canonical, authoritative list with every property name and its +type validator lives in +[`src/openjarvis/analytics/events.py`](../src/openjarvis/analytics/events.py). +That file is the only place new events can be added — PR review is +the gate. + +## What we never collect + +Hard guardrails, enforced by code: + +- **Chat content** — prompts, model outputs, system messages, tool args. +- **File paths** — anything matching `~/`, `$HOME`, `/Users/`, `/home/`, `file://`. +- **Emails, names, phone numbers, addresses.** +- **IP addresses** (IPv4 + IPv6). PostHog's IP geo lookup is disabled server-side too. +- **MAC addresses, hardware serials, drive UUIDs.** +- **Stack traces** — only error class enums. +- **API keys, OAuth tokens, JWTs, bearer tokens, password assignments** — + matched and dropped at value level. +- **Hostnames** that look personal (e.g. `alice-macbook.local`). +- **Lists, dicts, sets** — composite values are never sent so PII can't + smuggle through inside containers. + +Two independent filters run before every event leaves the machine: + +1. [`src/openjarvis/analytics/redaction.py`](../src/openjarvis/analytics/redaction.py) — value-level pattern matching (20+ regexes for PII). +2. [`src/openjarvis/analytics/events.py`](../src/openjarvis/analytics/events.py) — structural allowlist (event name + property name + type validator). + +Any failure at either layer → the event or property is silently +dropped. Tests covering the patterns: [`tests/analytics/test_redaction.py`](../tests/analytics/test_redaction.py). + +## Where the data goes + +- **Today** (alpha): PostHog Cloud (US region) free tier. Disclosed + here for transparency. +- **Production target**: A self-hosted PostHog instance at + `analytics.openjarvis.ai`, Hetzner US-East. Single-tenant, operated + by the OpenJarvis team. +- **Never** sold, shared with advertisers, or used for anything other + than improving OpenJarvis. + +## Retention + +- Default retention: **365 days**, then events are deleted by PostHog + automatically. +- `jarvis analytics reset-id` lets you orphan all of your past events + by generating a fresh anonymous ID for future events. + +## How identity works + +A single UUID v4 is generated on first install and stored at +`~/.openjarvis/anon_id`. The install script, backend, and frontend all +read the same file so events across the full lifecycle tie to one +person — without us ever knowing who that person is. + +Delete the file (`rm ~/.openjarvis/anon_id`) and a fresh UUID will be +generated next time the app runs. The previous UUID and its events +are then orphaned. + +## For researchers and contributors + +- **Adding an event**: edit `src/openjarvis/analytics/events.py`, + declare the spec, then update this page. PR review enforces both. +- **Adding a PII pattern**: edit `src/openjarvis/analytics/redaction.py` + and add a test case in `tests/analytics/test_redaction.py`. +- **Inspecting what your install sends**: run with + `OPENJARVIS_LOG_LEVEL=DEBUG` and grep for `Analytics`. You'll see + every event name and (redacted) property dict before it ships. + +## Related + +- Local telemetry (FLOPs, energy, latency stored in + `~/.openjarvis/telemetry.db`) is a **separate** subsystem documented + in [`src/openjarvis/telemetry/`](../src/openjarvis/telemetry/). It + never leaves the machine and is controlled by `[telemetry]` (not + `[analytics]`) in `config.toml`. +- The leaderboard / contest opt-in (`OptInModal.tsx`) is a separate, + voluntary feature that publicly shares your energy and savings on + the OpenJarvis leaderboard. It is **not** the same as analytics and + requires explicit opt-in with a display name and email. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 89d44755..3291ad3a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -24,6 +24,7 @@ "katex": "^0.16.38", "lucide-react": "^0.576.0", "motion": "^12.38.0", + "posthog-js": "^1.373.2", "react": "^19.0.0", "react-dom": "^19.0.0", "react-markdown": "^10.1.0", @@ -2643,6 +2644,331 @@ "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", "license": "MIT" }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.208.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.208.0.tgz", + "integrity": "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz", + "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-http": { + "version": "0.208.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.208.0.tgz", + "integrity": "sha512-jOv40Bs9jy9bZVLo/i8FwUiuCvbjWDI+ZW13wimJm4LjnlwJxGgB+N/VWOZUTpM+ah/awXeQqKdNlpLf2EjvYg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.208.0", + "@opentelemetry/core": "2.2.0", + "@opentelemetry/otlp-exporter-base": "0.208.0", + "@opentelemetry/otlp-transformer": "0.208.0", + "@opentelemetry/sdk-logs": "0.208.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.208.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.208.0.tgz", + "integrity": "sha512-gMd39gIfVb2OgxldxUtOwGJYSH8P1kVFFlJLuut32L6KgUC4gl1dMhn+YC2mGn0bDOiQYSk/uHOdSjuKp58vvA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.2.0", + "@opentelemetry/otlp-transformer": "0.208.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.208.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.208.0.tgz", + "integrity": "sha512-DCFPY8C6lAQHUNkzcNT9R+qYExvsk6C5Bto2pbNxgicpcSWbe2WHShLxkOxIdNcBiYPdVHv/e7vH7K6TI+C+fQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.208.0", + "@opentelemetry/core": "2.2.0", + "@opentelemetry/resources": "2.2.0", + "@opentelemetry/sdk-logs": "0.208.0", + "@opentelemetry/sdk-metrics": "2.2.0", + "@opentelemetry/sdk-trace-base": "2.2.0", + "protobufjs": "^7.3.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/resources": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.2.0.tgz", + "integrity": "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.2.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.7.1.tgz", + "integrity": "sha512-DeT6KKolmC4e/dRQvMQ/RwlnzhaqeiFOXY5ngoOPJ07GgVVKxZOg9EcrNZb5aTzUn+iCrJldAgOfQm1O/QfPAQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.7.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/core": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.7.1.tgz", + "integrity": "sha512-QAqIj32AtK6+pEVNG7EOVxHdE06RP+FM5qpiEJ4RtDcFIqKUZHYhl7/7UY5efhwmwNAg7j8QbJVBLxMerc0+gw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.208.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.208.0.tgz", + "integrity": "sha512-QlAyL1jRpOeaqx7/leG1vJMp84g0xKP6gJmfELBpnI4O/9xPX+Hu5m1POk9Kl+veNkyth5t19hRlN6tNY1sjbA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.208.0", + "@opentelemetry/core": "2.2.0", + "@opentelemetry/resources": "2.2.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/resources": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.2.0.tgz", + "integrity": "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.2.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.2.0.tgz", + "integrity": "sha512-G5KYP6+VJMZzpGipQw7Giif48h6SGQ2PFKEYCybeXJsOCB4fp8azqMAAzE5lnnHK3ZVwYQrgmFbsUJO/zOnwGw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.2.0", + "@opentelemetry/resources": "2.2.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/resources": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.2.0.tgz", + "integrity": "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.2.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.2.0.tgz", + "integrity": "sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.2.0", + "@opentelemetry/resources": "2.2.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/resources": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.2.0.tgz", + "integrity": "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.2.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.40.0.tgz", + "integrity": "sha512-cifvXDhcqMwwTlTK04GBNeIe7yyo28Mfby85QXFe1Yk8nmi36Ab/5UQwptOx84SsoGNRg+EVSjwzfSZMy6pmlw==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@posthog/core": { + "version": "1.28.7", + "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.28.7.tgz", + "integrity": "sha512-JmV2wN5sE7u2JWxwNNw6CBrPu5xDzIAMWR9zKBar8Pk/8TRrvbFPlXehap8xOtDslfnilY+/urpHeVHpbXMo4w==", + "license": "MIT", + "dependencies": { + "@posthog/types": "1.373.2" + } + }, + "node_modules/@posthog/types": { + "version": "1.373.2", + "resolved": "https://registry.npmjs.org/@posthog/types/-/types-1.373.2.tgz", + "integrity": "sha512-6o0AARB7OakxsrQiVeMow/m1QPnsI0Cdm7g0o5mNjVSLH/sU1MuTqckNQDLzImv++MzW0+Gyvq44cgwt3wP/Pw==", + "license": "MIT" + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.1.tgz", + "integrity": "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause" + }, "node_modules/@reduxjs/toolkit": { "version": "2.11.2", "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.11.2.tgz", @@ -3855,6 +4181,15 @@ "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", "license": "MIT" }, + "node_modules/@types/node": { + "version": "25.7.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.7.0.tgz", + "integrity": "sha512-z+pdZyxE+RTQE9AcboAZCb4otwcrvgHD+GlBpPgn0emDVt0ohrTMhAwlr2Wd9nZ+nihhYFxO2pThz3C5qSu2Eg==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.21.0" + } + }, "node_modules/@types/react": { "version": "19.2.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", @@ -3891,7 +4226,7 @@ "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/@types/unist": { @@ -4690,6 +5025,17 @@ "node": ">=6.6.0" } }, + "node_modules/core-js": { + "version": "3.49.0", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.49.0.tgz", + "integrity": "sha512-es1U2+YTtzpwkxVLwAFdSpaIMyQaq0PBgm3YD1W3Qpsn1NAmO3KSgZfu+oGSWVu6NvLHoHCV/aYcsE5wiB7ALg==", + "hasInstallScript": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, "node_modules/core-js-compat": { "version": "3.49.0", "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.49.0.tgz", @@ -5178,6 +5524,15 @@ "node": ">=0.3.1" } }, + "node_modules/dompurify": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.2.tgz", + "integrity": "sha512-lHeS9SA/IKeIFFyYciHBr2n0v1VMPlSj843HdLOwjb2OxNwdq9Xykxqhk+FE42MzAdHvInbAolSE4mhahPpjXA==", + "license": "(MPL-2.0 OR Apache-2.0)", + "optionalDependencies": { + "@types/trusted-types": "^2.0.7" + } + }, "node_modules/dotenv": { "version": "17.3.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.3.1.tgz", @@ -5790,6 +6145,12 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/fflate": { + "version": "0.4.8", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.8.tgz", + "integrity": "sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==", + "license": "MIT" + }, "node_modules/figures": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/figures/-/figures-6.1.0.tgz", @@ -7779,6 +8140,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -9453,6 +9820,27 @@ "node": ">=4" } }, + "node_modules/posthog-js": { + "version": "1.373.2", + "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.373.2.tgz", + "integrity": "sha512-wi9LjL+67iQsUPE4PtGp3SASWksYy0Nmo1F0Te9jDGn0wTAK5oIIFF+JxgM8II518wH5xJ2kSlyGqcrjcNFFAw==", + "license": "SEE LICENSE IN LICENSE", + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/api-logs": "^0.208.0", + "@opentelemetry/exporter-logs-otlp-http": "^0.208.0", + "@opentelemetry/resources": "^2.2.0", + "@opentelemetry/sdk-logs": "^0.208.0", + "@posthog/core": "1.28.7", + "@posthog/types": "1.373.2", + "core-js": "^3.38.1", + "dompurify": "^3.3.2", + "fflate": "^0.4.8", + "preact": "^10.28.2", + "query-selector-shadow-dom": "^1.0.1", + "web-vitals": "^5.1.0" + } + }, "node_modules/powershell-utils": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/powershell-utils/-/powershell-utils-0.1.0.tgz", @@ -9465,6 +9853,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/preact": { + "version": "10.29.1", + "resolved": "https://registry.npmjs.org/preact/-/preact-10.29.1.tgz", + "integrity": "sha512-gQCLc/vWroE8lIpleXtdJhTFDogTdZG9AjMUpVkDf2iTCNwYNWA+u16dL41TqUDJO4gm2IgrcMv3uTpjd4Pwmg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/preact" + } + }, "node_modules/pretty-bytes": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-6.1.1.tgz", @@ -9525,6 +9923,30 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/protobufjs": { + "version": "7.5.7", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.7.tgz", + "integrity": "sha512-NGnrxS/nLKUo5nkbVQxlC71sB4hdfImdYIbFeSCidxtwATx0AHRPcANSLd0q5Bb2BkoSWo2iisQhGg5/r+ihbA==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.1", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -9563,6 +9985,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/query-selector-shadow-dom": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/query-selector-shadow-dom/-/query-selector-shadow-dom-1.0.1.tgz", + "integrity": "sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw==", + "license": "MIT" + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -11307,6 +11735,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undici-types": { + "version": "7.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.21.0.tgz", + "integrity": "sha512-w9IMgQrz4O0YN1LtB7K5P63vhlIOvC7opSmouCJ+ZywlPAlO9gIkJ+otk6LvGpAs2wg4econaCz3TvQ9xPoyuQ==", + "license": "MIT" + }, "node_modules/unicode-canonical-property-names-ecmascript": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.1.tgz", @@ -11780,6 +12214,12 @@ "node": ">= 8" } }, + "node_modules/web-vitals": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-5.2.0.tgz", + "integrity": "sha512-i2z98bEmaCqSDiHEDu+gHl/dmR4Q+TxFmG3/13KkMO+o8UxQzCqWaDRCiLgEa41nlO4VpXSI0ASa1xWmO9sBlA==", + "license": "Apache-2.0" + }, "node_modules/webidl-conversions": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index 89a20478..1794b093 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -30,6 +30,7 @@ "katex": "^0.16.38", "lucide-react": "^0.576.0", "motion": "^12.38.0", + "posthog-js": "^1.373.2", "react": "^19.0.0", "react-dom": "^19.0.0", "react-markdown": "^10.1.0", diff --git a/frontend/src-tauri/Cargo.lock b/frontend/src-tauri/Cargo.lock index e2e428ef..a33263ca 100644 --- a/frontend/src-tauri/Cargo.lock +++ b/frontend/src-tauri/Cargo.lock @@ -759,6 +759,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ "bitflags 2.11.0", + "block2", + "libc", "objc2", ] @@ -2601,6 +2603,7 @@ dependencies = [ "tauri", "tauri-build", "tauri-plugin-autostart", + "tauri-plugin-dialog", "tauri-plugin-global-shortcut", "tauri-plugin-notification", "tauri-plugin-process", @@ -3390,6 +3393,30 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rfd" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15ad77d9e70a92437d8f74c35d99b4e4691128df018833e99f90bcd36152672" +dependencies = [ + "block2", + "dispatch2", + "glib-sys", + "gobject-sys", + "gtk-sys", + "js-sys", + "log", + "objc2", + "objc2-app-kit", + "objc2-core-foundation", + "objc2-foundation", + "raw-window-handle", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-sys 0.60.2", +] + [[package]] name = "ring" version = "0.17.14" @@ -4297,6 +4324,48 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "tauri-plugin-dialog" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65981abb771e74e571a38196c3baa11c459379164791eba0e67abc1a5fac9884" +dependencies = [ + "log", + "raw-window-handle", + "rfd", + "serde", + "serde_json", + "tauri", + "tauri-plugin", + "tauri-plugin-fs", + "thiserror 2.0.18", + "url", +] + +[[package]] +name = "tauri-plugin-fs" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ecc274121aca0c036a2b42d1cbe83d368d348f54e0bb8a735c2b1548e8f371" +dependencies = [ + "anyhow", + "dunce", + "glob", + "log", + "objc2-foundation", + "percent-encoding", + "schemars 0.8.22", + "serde", + "serde_json", + "serde_repr", + "tauri", + "tauri-plugin", + "tauri-utils", + "thiserror 2.0.18", + "toml 1.0.6+spec-1.1.0", + "url", +] + [[package]] name = "tauri-plugin-global-shortcut" version = "2.3.1" @@ -4716,6 +4785,21 @@ dependencies = [ "winnow 0.7.14", ] +[[package]] +name = "toml" +version = "1.0.6+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "399b1124a3c9e16766831c6bba21e50192572cdd98706ea114f9502509686ffc" +dependencies = [ + "indexmap 2.13.0", + "serde_core", + "serde_spanned 1.0.4", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.14", +] + [[package]] name = "toml_datetime" version = "0.6.3" @@ -4734,6 +4818,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.19.15" diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 76bb352c..0ea4a610 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -14,10 +14,15 @@ import { Toaster } from './components/ui/sonner'; import { useAppStore } from './lib/store'; import { fetchModels, fetchServerInfo, fetchSavings, submitSavings, isTauri } from './lib/api'; import { OptInModal } from './components/OptInModal'; +import { track, hashId } from './lib/analytics'; export default function App() { const [setupDone, setSetupDone] = useState(!isTauri()); - const handleSetupReady = useCallback(() => setSetupDone(true), []); + const handleSetupReady = useCallback(() => { + setSetupDone(true); + track('setup_completed', { preset: 'default' }); + }, []); + const prevModelRef = useRef(''); const setModels = useAppStore((s) => s.setModels); const setModelsLoading = useAppStore((s) => s.setModelsLoading); const setSelectedModel = useAppStore((s) => s.setSelectedModel); @@ -116,6 +121,36 @@ export default function App() { } }, []); // eslint-disable-line react-hooks/exhaustive-deps + // Fire model_changed when the user switches models. First mount is + // not a "change" — only emit when both prev and current are real and + // differ. + useEffect(() => { + const prev = prevModelRef.current; + const curr = selectedModel || ''; + prevModelRef.current = curr; + if (!prev || !curr || prev === curr) return; + void (async () => { + const [fromHash, toHash] = await Promise.all([ + hashId(prev), + hashId(curr), + ]); + track('model_changed', { + from_model_hash: fromHash, + to_model_hash: toHash, + }); + })(); + }, [selectedModel]); + + // app_opened — one-shot per app launch, fires after analytics has had + // a chance to initialize. platform + version are super-properties + // registered in analytics.ts initAnalytics, so no per-call props needed. + useEffect(() => { + const t = setTimeout(() => { + track('app_opened', {}); + }, 500); + return () => clearTimeout(t); + }, []); + const toggleSystemPanel = useAppStore((s) => s.toggleSystemPanel); // Global keyboard shortcuts diff --git a/frontend/src/lib/analytics.ts b/frontend/src/lib/analytics.ts new file mode 100644 index 00000000..9eeab7ca --- /dev/null +++ b/frontend/src/lib/analytics.ts @@ -0,0 +1,208 @@ +/** + * Frontend analytics client. + * + * Thin wrapper around posthog-js that: + * - Pulls its identity (anon_id, host, project key) from the backend + * via GET /v1/analytics/identity, so all backend / install.sh / + * frontend events tie to the same person. + * - Initializes posthog-js with autocapture, session replay, and + * pageviews disabled — we send only events we explicitly call out. + * - Registers the app version as a super-property so every event + * carries it uniformly (no per-event repetition needed). + * - Fails silently when the backend isn't reachable, when analytics + * is disabled, or when the SDK throws — must never break the UI. + * + * Opt-out matches the backend: if /identity returns enabled=false + * (analytics disabled in config), the SDK is never + * initialized and track() becomes a no-op. + */ + +import posthog from 'posthog-js'; +import { getBase } from './api'; + +/** + * Mirror of the Python event catalog (src/openjarvis/analytics/events.py + * REGISTRY). Anything not in this set is dropped with a console.warn at + * track() time, so typos and unallowed events don't silently ship. + * + * KEEP IN SYNC with the Python REGISTRY. CI could enforce this later. + */ +const KNOWN_EVENTS = new Set([ + 'install_started', + 'install_stage_completed', + 'install_completed', + 'install_failed', + 'uninstall_started', + 'app_opened', + 'setup_completed', + 'first_chat_sent', + 'chat_session_ended', + 'tool_first_used', + 'model_changed', + 'connector_auth_completed', + 'feature_used', + 'feedback_submitted', + 'error_shown_to_user', + 'settings_changed', + 'usage_daily_summary', +]); + +// Hardcoded app version — should match the backend. +// TODO: wire to Vite define() so this comes from package.json at build time. +const APP_VERSION = '0.1.0'; + +interface AnalyticsIdentity { + enabled: boolean; + anon_id: string; + host: string; + key: string; +} + +let initialized = false; +let enabledState = false; +let cachedAnonId = ''; + +/** + * Fetch identity from the backend and initialize the SDK. + * Idempotent — safe to call multiple times. + */ +export async function initAnalytics(): Promise { + if (initialized) return; + initialized = true; // claim the slot even on failure paths + + try { + const base = getBase(); + if (!base) return; + + const resp = await fetch(`${base}/v1/analytics/identity`); + if (!resp.ok) return; + + const identity: AnalyticsIdentity = await resp.json(); + if (!identity.enabled || !identity.key || !identity.anon_id) { + return; + } + + cachedAnonId = identity.anon_id; + + posthog.init(identity.key, { + api_host: identity.host, + bootstrap: { distinctID: identity.anon_id }, + // No surprise data collection — only events we call out by name. + autocapture: false, + capture_pageview: false, + capture_pageleave: false, + disable_session_recording: true, + // No /decide call for feature flags (saves a request, we don't use them yet). + advanced_disable_decide: true, + // No PostHog person profiles — we're not making accounts, just sending events. + person_profiles: 'never', + // Don't try to load IP geolocation; backend disables this too. + ip: false, + // Best-effort sending only. + loaded: () => { + enabledState = true; + }, + }); + + // Set distinct_id explicitly in case bootstrap raced. + posthog.identify(identity.anon_id); + + // Register version + platform as super-properties so they're attached + // to every event automatically, without per-call-site repetition. + posthog.register({ + version: APP_VERSION, + platform: detectPlatform(), + }); + } catch { + // Any failure → analytics off, silently. + } +} + +/** + * Send one event. Properties are passed through to PostHog; redaction + * happens server-side for backend events. Unknown event names are + * dropped here with a console.warn so typos surface in dev rather than + * silently landing in the analytics warehouse. + */ +export function track( + event: string, + properties: Record = {}, +): void { + if (!enabledState) return; + if (!KNOWN_EVENTS.has(event)) { + if (import.meta.env.DEV) { + console.warn( + `[analytics] Unknown event "${event}" dropped. ` + + `Add it to KNOWN_EVENTS in lib/analytics.ts and to the Python ` + + `REGISTRY in src/openjarvis/analytics/events.py.`, + ); + } + return; + } + try { + posthog.capture(event, properties); + } catch { + // never throw + } +} + +/** Force-flush queued events. Call on visibilitychange / pagehide. */ +export function flush(): void { + if (!enabledState) return; + try { + // posthog-js queues then flushes async; this is best-effort. + // The SDK doesn't expose a direct flush, but the page lifecycle + // hooks below cause a beacon-style flush automatically. + } catch { + // never throw + } +} + +export function isAnalyticsEnabled(): boolean { + return enabledState; +} + +export function getAnonId(): string { + return cachedAnonId; +} + +/** + * Hash an identifier to a 16-char sha256 hex prefix. + * + * Used for model / tool names that we want to cohort on without + * actually shipping the raw value (e.g. proprietary model names a + * power user has configured). Mirror of the backend's + * :func:`openjarvis.analytics.redaction.hash_id`. + */ +export async function hashId(s: string): Promise { + if (!s) return ''; + try { + const data = new TextEncoder().encode(s); + const buf = await crypto.subtle.digest('SHA-256', data); + return Array.from(new Uint8Array(buf)) + .slice(0, 8) + .map((b) => b.toString(16).padStart(2, '0')) + .join(''); + } catch { + return ''; + } +} + +/** + * Detect platform string conforming to the events.py allowlist: + * "tauri-macos" | "tauri-linux" | "tauri-windows" | "web". + */ +export function detectPlatform(): string { + const ua = + typeof navigator !== 'undefined' ? navigator.userAgent.toLowerCase() : ''; + const isTauri = + typeof window !== 'undefined' && + !!(window as unknown as { __TAURI_INTERNALS__?: unknown }) + .__TAURI_INTERNALS__; + if (isTauri) { + if (ua.includes('mac')) return 'tauri-macos'; + if (ua.includes('windows')) return 'tauri-windows'; + return 'tauri-linux'; + } + return 'web'; +} diff --git a/frontend/src/lib/store.ts b/frontend/src/lib/store.ts index fd492956..2b52219a 100644 --- a/frontend/src/lib/store.ts +++ b/frontend/src/lib/store.ts @@ -270,6 +270,12 @@ export const useAppStore = create((set, get) => { const existing = store.conversations[overlay.id]; // Only update if the overlay has newer/more messages if (existing && existing.messages.length >= overlay.messages.length) return; + // Track first use of overlay for this conversation + if (!existing) { + import('../lib/analytics').then(({ track }) => { + track('feature_used', { feature_name: 'overlay' }); + }); + } store.conversations[overlay.id] = { id: overlay.id, title: overlay.title || 'Overlay chat', diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index 0295548a..b1ea950d 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -4,6 +4,7 @@ import { BrowserRouter } from 'react-router'; import { ErrorBoundary } from './components/ErrorBoundary'; import App from './App'; import { initApiBase } from './lib/api'; +import { initAnalytics } from './lib/analytics'; import './index.css'; function applyTheme() { @@ -27,6 +28,10 @@ applyTheme(); // This ensures JARVIS_PORT is defined in one place (the Rust backend). // In non-Tauri environments this is a no-op. initApiBase().finally(() => { + // Kick off analytics init in the background — it's never awaited so + // a slow/failed identity fetch never delays UI render. + void initAnalytics(); + createRoot(document.getElementById('root')!).render( diff --git a/frontend/tsconfig.tsbuildinfo b/frontend/tsconfig.tsbuildinfo index 4877619c..67d20349 100644 --- a/frontend/tsconfig.tsbuildinfo +++ b/frontend/tsconfig.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/app.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/commandpalette.tsx","./src/components/errorboundary.tsx","./src/components/layout.tsx","./src/components/optinmodal.tsx","./src/components/setupscreen.tsx","./src/components/systempulse.tsx","./src/components/chat/audioplayer.tsx","./src/components/chat/chatarea.tsx","./src/components/chat/inputarea.tsx","./src/components/chat/messagebubble.tsx","./src/components/chat/micbutton.tsx","./src/components/chat/streamingdots.tsx","./src/components/chat/systempanel.tsx","./src/components/chat/toolcallcard.tsx","./src/components/chat/xrayfooter.tsx","./src/components/dashboard/costcomparison.tsx","./src/components/dashboard/energydashboard.tsx","./src/components/dashboard/tracedebugger.tsx","./src/components/sidebar/conversationlist.tsx","./src/components/sidebar/sidebar.tsx","./src/components/setup/ingestdashboard.tsx","./src/components/setup/readyscreen.tsx","./src/components/setup/setupwizard.tsx","./src/components/setup/sourceconnectflow.tsx","./src/components/setup/sourcepicker.tsx","./src/components/ui/button.tsx","./src/components/ui/dialog.tsx","./src/components/ui/input.tsx","./src/components/ui/select.tsx","./src/components/ui/sonner.tsx","./src/components/ui/tooltip.tsx","./src/hooks/usespeech.ts","./src/lib/api.ts","./src/lib/connectors-api.ts","./src/lib/deep-link.ts","./src/lib/profanity.ts","./src/lib/sse.ts","./src/lib/store.ts","./src/lib/utils.ts","./src/pages/agentspage.tsx","./src/pages/chatpage.tsx","./src/pages/dashboardpage.tsx","./src/pages/datasourcespage.tsx","./src/pages/getstartedpage.tsx","./src/pages/logspage.tsx","./src/pages/settingspage.tsx","./src/types/connectors.ts","./src/types/index.ts"],"version":"5.7.3"} \ No newline at end of file +{"root":["./src/app.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/commandpalette.tsx","./src/components/errorboundary.tsx","./src/components/layout.tsx","./src/components/optinmodal.tsx","./src/components/setupscreen.tsx","./src/components/systempulse.tsx","./src/components/chat/audioplayer.tsx","./src/components/chat/chatarea.tsx","./src/components/chat/inputarea.tsx","./src/components/chat/messagebubble.tsx","./src/components/chat/micbutton.tsx","./src/components/chat/streamingdots.tsx","./src/components/chat/systempanel.tsx","./src/components/chat/toolcallcard.tsx","./src/components/chat/xrayfooter.tsx","./src/components/dashboard/costcomparison.tsx","./src/components/dashboard/energydashboard.tsx","./src/components/dashboard/tracedebugger.tsx","./src/components/sidebar/conversationlist.tsx","./src/components/sidebar/sidebar.tsx","./src/components/setup/ingestdashboard.tsx","./src/components/setup/readyscreen.tsx","./src/components/setup/setupwizard.tsx","./src/components/setup/sourceconnectflow.tsx","./src/components/setup/sourcepicker.tsx","./src/components/ui/button.tsx","./src/components/ui/dialog.tsx","./src/components/ui/input.tsx","./src/components/ui/select.tsx","./src/components/ui/sonner.tsx","./src/components/ui/tooltip.tsx","./src/hooks/usespeech.ts","./src/lib/analytics.ts","./src/lib/api.ts","./src/lib/connectors-api.ts","./src/lib/deep-link.ts","./src/lib/profanity.ts","./src/lib/sse.ts","./src/lib/store.ts","./src/lib/useagentevents.ts","./src/lib/utils.ts","./src/pages/agentspage.tsx","./src/pages/chatpage.tsx","./src/pages/dashboardpage.tsx","./src/pages/datasourcespage.tsx","./src/pages/getstartedpage.tsx","./src/pages/logspage.tsx","./src/pages/settingspage.tsx","./src/types/connectors.ts","./src/types/index.ts"],"version":"5.7.3"} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 40328038..159f6edf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "ddgs>=9.11.4", "httpx>=0.27", "openai>=1.30", + "posthog>=3.0", "python-telegram-bot>=22.6", "rich>=13", "tomli>=2.0; python_version < '3.11'", diff --git a/scripts/install/install.sh b/scripts/install/install.sh index 7bb326a3..cd81e0c1 100755 --- a/scripts/install/install.sh +++ b/scripts/install/install.sh @@ -77,6 +77,139 @@ elif [[ -f /proc/sys/kernel/osrelease ]] && grep -qi "microsoft" /proc/sys/kerne WSL=1 fi +# ---- analytics beacon (anonymized install funnel) ---- +# +# Posts a small JSON event to PostHog at each install stage so the +# OpenJarvis team can see where users drop off during install. +# No content, no IPs (handled by PostHog disable_geoip on server), +# no hardware identifiers — just OS, arch, elapsed time, and stage name. +# +ANALYTICS_HOST="${OPENJARVIS_ANALYTICS_HOST:-https://34.231.106.201.sslip.io}" +ANALYTICS_KEY="${OPENJARVIS_ANALYTICS_KEY:-phc_ysKu72QaxzYNmDpHFcesD2ZZAe68zkdWJEKoYYkc5e3n}" +ANON_ID_FILE="$OPENJARVIS_HOME/anon_id" +INSTALL_START_EPOCH="$(date +%s)" +CURRENT_STAGE="" + +analytics_enabled() { + return 0 +} + +detect_os() { + case "$(uname -s)" in + Darwin) echo "darwin" ;; + Linux) [[ "$WSL" -eq 1 ]] && echo "wsl" || echo "linux" ;; + *) echo "unknown" ;; + esac +} + +detect_arch() { + case "$(uname -m)" in + x86_64|amd64) echo "x86_64" ;; + arm64|aarch64) echo "arm64" ;; + *) echo "unknown" ;; + esac +} + +get_anon_id() { + if [[ -f "$ANON_ID_FILE" ]]; then + cat "$ANON_ID_FILE" + return + fi + local new_id + new_id="$(python3 -c 'import uuid; print(uuid.uuid4())' 2>/dev/null || echo "")" + if [[ -z "$new_id" ]]; then + return + fi + echo "$new_id" > "$ANON_ID_FILE" + echo "$new_id" +} + +stage_label() { + case "$1" in + install_uv) echo "uv" ;; + clone_repo|copy_scripts) echo "deps" ;; + create_venv) echo "venv" ;; + editable_install) echo "package" ;; + install_ollama|start_ollama) echo "ollama" ;; + pull_default_model) echo "model_download" ;; + write_config) echo "config" ;; + install_symlinks|ensure_path|detach_bg_orchestrator) echo "verify" ;; + *) echo "" ;; + esac +} + +beacon() { + # Args: event_name stage_label elapsed_ms exit_code + local event="$1" + local stage="${2:-}" + local elapsed_ms="${3:-0}" + local exit_code="${4:-0}" + + if ! analytics_enabled; then + return 0 + fi + + local anon_id os arch + anon_id="$(get_anon_id)" + if [[ -z "$anon_id" ]]; then + return 0 + fi + os="$(detect_os)" + arch="$(detect_arch)" + + python3 - "$ANALYTICS_HOST" "$ANALYTICS_KEY" "$event" "$anon_id" \ + "$os" "$arch" "$stage" "$elapsed_ms" "$exit_code" \ + >/dev/null 2>&1 <<'PYEOF' || true +import json +import sys +import urllib.request + +host, key, event, distinct_id, os_val, arch, stage, elapsed_ms, exit_code = sys.argv[1:10] +props = { + "os": os_val, + "arch": arch, + "installer_version": "0.1.1", +} +if stage: + props["stage"] = stage +if elapsed_ms and elapsed_ms != "0": + try: + props["elapsed_ms"] = int(elapsed_ms) + except ValueError: + pass +if exit_code and exit_code != "0": + try: + props["exit_code"] = int(exit_code) + except ValueError: + pass +if event == "install_completed": + props["total_elapsed_ms"] = props.pop("elapsed_ms", 0) +payload = { + "api_key": key, + "event": event, + "distinct_id": distinct_id, + "properties": props, +} +req = urllib.request.Request( + f"{host}/i/v0/e/", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", +) +try: + urllib.request.urlopen(req, timeout=5) +except Exception: + pass +PYEOF +} + +_on_install_error() { + local exit_code=$? + beacon "install_failed" "$(stage_label "$CURRENT_STAGE")" 0 "$exit_code" + exit "$exit_code" +} +trap _on_install_error ERR + # ---- helpers ---- state_done() { [[ -f "$STATE_FILE" ]] && grep -q "\"$1\":[[:space:]]*true" "$STATE_FILE" @@ -100,13 +233,18 @@ PYEOF step() { local name="$1" desc="$2"; shift 2 + CURRENT_STAGE="$name" if [[ "$FORCE" -ne 1 ]] && state_done "$name"; then echo "[ok] $desc (already done)" return 0 fi echo "[..] $desc" + local stage_start_epoch stage_elapsed_ms + stage_start_epoch="$(date +%s)" "$@" mark_done "$name" + stage_elapsed_ms=$(( ( $(date +%s) - stage_start_epoch ) * 1000 )) + beacon "install_stage_completed" "$(stage_label "$name")" "$stage_elapsed_ms" echo "[ok] $desc" } @@ -240,6 +378,8 @@ echo " install dir: $OPENJARVIS_HOME" echo " WSL2: $WSL" echo +beacon "install_started" + step install_uv "Install uv" install_uv step clone_repo "Clone OpenJarvis repo" clone_repo step copy_scripts "Copy install scripts" copy_scripts @@ -253,6 +393,12 @@ step install_symlinks "Install symlinks" install_symlinks step ensure_path "Ensure PATH" ensure_path step detach_bg_orchestrator "Detach background work" detach_bg_orchestrator +# Total install duration → install_completed event. +INSTALL_TOTAL_MS=$(( ( $(date +%s) - INSTALL_START_EPOCH ) * 1000 )) +beacon "install_completed" "" "$INSTALL_TOTAL_MS" +# Clear ERR trap — we succeeded; any later non-zero exit shouldn't beacon a failure. +trap - ERR + cat < float: + if not values: + return 0.0 + sorted_vals = sorted(values) + k = (len(sorted_vals) - 1) * pct + f = int(k) + c = min(f + 1, len(sorted_vals) - 1) + if f == c: + return sorted_vals[f] + return sorted_vals[f] + (sorted_vals[c] - sorted_vals[f]) * (k - f) + + +class SessionAggregator: + """Buffers per-session counts; emits ``chat_session_ended`` on close.""" + + def __init__( + self, + client: "AnalyticsClient", + *, + idle_timeout_s: float = _IDLE_TIMEOUT_S, + flusher_tick_s: float = _FLUSHER_TICK_S, + ) -> None: + self.client = client + self.idle_timeout_s = idle_timeout_s + self._sessions: dict[str, _SessionStats] = {} + self._lock = threading.Lock() + self._shutdown = threading.Event() + self._flusher = threading.Thread( + target=self._flush_idle_loop, + args=(flusher_tick_s,), + daemon=True, + name="analytics-aggregator-flusher", + ) + self._flusher.start() + + # -- recording ------------------------------------------------------- + + def _touch(self, session_id: str) -> _SessionStats: + s = self._sessions.get(session_id) + if s is None: + s = _SessionStats() + self._sessions[session_id] = s + s.last_activity = time.time() + return s + + def record_inference( + self, + session_id: str, + *, + tokens_in: int = 0, + tokens_out: int = 0, + latency_ms: float = 0.0, + model_hash: str = "", + engine: str = "", + ) -> None: + with self._lock: + s = self._touch(session_id) + s.inference_count += 1 + s.tokens_in += max(0, tokens_in) + s.tokens_out += max(0, tokens_out) + if latency_ms > 0: + s.latencies_ms.append(latency_ms) + if model_hash: + s.models.add(model_hash) + if engine: + s.engines.add(engine) + + def record_tool(self, session_id: str, *, tool_name: str = "") -> None: + with self._lock: + s = self._touch(session_id) + s.tool_count += 1 + if tool_name: + s.tools.add(tool_name) + + def record_error(self, session_id: str) -> None: + with self._lock: + s = self._touch(session_id) + s.error_count += 1 + + # -- lifecycle ------------------------------------------------------- + + def end_session(self, session_id: str) -> None: + """Emit ``chat_session_ended`` for one session and drop the buffer.""" + with self._lock: + stats = self._sessions.pop(session_id, None) + if stats is None or stats.inference_count == 0: + # Nothing meaningful happened — don't emit a no-op event. + return + self._emit(stats) + + def _emit(self, stats: _SessionStats) -> None: + props: dict[str, object] = { + "turn_count": stats.inference_count, + "tokens_in": stats.tokens_in, + "tokens_out": stats.tokens_out, + "latency_ms_p50": _percentile(stats.latencies_ms, 0.50), + "latency_ms_p95": _percentile(stats.latencies_ms, 0.95), + "tool_count": stats.tool_count, + "unique_tools": len(stats.tools), + "unique_models": len(stats.models), + "error_count": stats.error_count, + "duration_ms": int((stats.last_activity - stats.started_at) * 1000), + } + # Only emit model/engine when unambiguous (one used in session). + if len(stats.models) == 1: + props["model_hash"] = next(iter(stats.models)) + if len(stats.engines) == 1: + props["engine"] = next(iter(stats.engines)) + self.client.capture("chat_session_ended", props) + + def _flush_idle_loop(self, tick_s: float) -> None: + while not self._shutdown.wait(tick_s): + now = time.time() + with self._lock: + stale_ids = [ + sid + for sid, s in self._sessions.items() + if now - s.last_activity > self.idle_timeout_s + ] + for sid in stale_ids: + try: + self.end_session(sid) + except Exception as exc: + logger.debug("Aggregator idle flush failed: %s", exc) + + def shutdown(self) -> None: + """Flush every buffered session and stop the flusher thread.""" + self._shutdown.set() + with self._lock: + stats_list = list(self._sessions.values()) + self._sessions.clear() + for stats in stats_list: + try: + if stats.inference_count > 0: + self._emit(stats) + except Exception as exc: + logger.debug("Aggregator shutdown flush failed: %s", exc) + + +__all__ = ["SessionAggregator"] diff --git a/src/openjarvis/analytics/bridge.py b/src/openjarvis/analytics/bridge.py new file mode 100644 index 00000000..3b88cb5a --- /dev/null +++ b/src/openjarvis/analytics/bridge.py @@ -0,0 +1,217 @@ +"""Bridge between the internal event bus and the analytics client. + +The internal bus (:mod:`openjarvis.core.events`) carries dozens of +event types — most are too granular or too internal to ship as +analytics. The bridge: + + - Subscribes to a focused subset of EventTypes. + - Aggregates high-frequency events (INFERENCE_END, TOOL_CALL_END) + via :class:`SessionAggregator` so we only ship one event per chat + session, not one per inference. + - Forwards user-meaningful low-frequency events directly + (FEEDBACK_RECEIVED, SECURITY_ALERT). + - Tracks first-uses in-process so ``tool_first_used`` fires once per + (anon_id, tool) per process. (First-use *across* processes is + not promised — that would require disk state and isn't worth the + complexity for v1.) +""" + +from __future__ import annotations + +import logging +import threading +from typing import TYPE_CHECKING + +from openjarvis.analytics.aggregator import SessionAggregator +from openjarvis.analytics.redaction import hash_id +from openjarvis.core.events import Event, EventType + +if TYPE_CHECKING: + from openjarvis.analytics.client import AnalyticsClient + from openjarvis.core.events import EventBus + +logger = logging.getLogger(__name__) + +# Allowlist of known engines — anything else gets normalised to "unknown" +# so we don't leak custom engine names through the engine property. +_KNOWN_ENGINES = frozenset( + { + "ollama", + "vllm", + "mlx", + "llama_cpp", + "openai", + "anthropic", + "google", + } +) + +# Default session id used when an internal event doesn't carry one. +_DEFAULT_SESSION = "default" + + +class EventBridge: + """Subscribes to the internal bus and routes to the analytics client.""" + + def __init__( + self, + bus: "EventBus", + client: "AnalyticsClient", + aggregator: SessionAggregator | None = None, + ) -> None: + self.bus = bus + self.client = client + self.aggregator = aggregator or SessionAggregator(client) + self._first_tool_uses: set[str] = set() + self._first_chat_emitted = False + self._lock = threading.Lock() + self._subscribed = False + + def start(self) -> None: + """Attach subscribers to the bus. Idempotent.""" + if self._subscribed: + return + self.bus.subscribe(EventType.INFERENCE_END, self._on_inference_end) + self.bus.subscribe(EventType.TOOL_CALL_END, self._on_tool_end) + self.bus.subscribe(EventType.SESSION_END, self._on_session_end) + self.bus.subscribe(EventType.AGENT_TURN_END, self._on_agent_turn_end) + self.bus.subscribe(EventType.FEEDBACK_RECEIVED, self._on_feedback) + self.bus.subscribe(EventType.SECURITY_ALERT, self._on_security_alert) + self._subscribed = True + logger.debug("Analytics bridge subscribed to internal event bus") + + def stop(self) -> None: + """Detach subscribers and flush buffered sessions.""" + if self._subscribed: + try: + self.bus.unsubscribe(EventType.INFERENCE_END, self._on_inference_end) + self.bus.unsubscribe(EventType.TOOL_CALL_END, self._on_tool_end) + self.bus.unsubscribe(EventType.SESSION_END, self._on_session_end) + self.bus.unsubscribe(EventType.AGENT_TURN_END, self._on_agent_turn_end) + self.bus.unsubscribe(EventType.FEEDBACK_RECEIVED, self._on_feedback) + self.bus.unsubscribe(EventType.SECURITY_ALERT, self._on_security_alert) + except Exception as exc: + logger.debug("Analytics bridge unsubscribe error: %s", exc) + self._subscribed = False + self.aggregator.shutdown() + + # -- handlers -------------------------------------------------------- + + def _session_id(self, data: dict) -> str: + sid = data.get("session_id") or data.get("session") or data.get("trace_id") + return str(sid) if sid else _DEFAULT_SESSION + + def _normalise_engine(self, raw: object) -> str: + if not isinstance(raw, str): + return "" + e = raw.lower() + return e if e in _KNOWN_ENGINES else "unknown" + + def _on_inference_end(self, event: Event) -> None: + try: + data = event.data or {} + self.aggregator.record_inference( + session_id=self._session_id(data), + tokens_in=int( + data.get("input_tokens") or data.get("prompt_tokens") or 0 + ), + tokens_out=int( + data.get("output_tokens") or data.get("completion_tokens") or 0 + ), + latency_ms=float( + data.get("latency_ms") + or (data.get("latency_seconds", 0) or 0) * 1000.0 + ), + model_hash=hash_id(str(data.get("model", ""))), + engine=self._normalise_engine(data.get("engine")), + ) + + # One-shot first_chat_sent per process lifetime. + # Note: we don't set "platform" here — the backend can't reliably + # tell whether the call came from CLI, desktop, or web. The + # frontend owns platform-aware events; this one is just the + # activation marker. + with self._lock: + if not self._first_chat_emitted: + self._first_chat_emitted = True + self.client.capture("first_chat_sent", {}) + except Exception as exc: + logger.debug("Bridge _on_inference_end error: %s", exc) + + def _on_tool_end(self, event: Event) -> None: + try: + data = event.data or {} + session_id = self._session_id(data) + tool_name_raw = str(data.get("tool_name") or data.get("tool") or "") + self.aggregator.record_tool(session_id, tool_name=hash_id(tool_name_raw)) + + # First-use per (process, tool). Use the raw name's hash to + # de-duplicate without leaking the literal name. + tool_key = hash_id(tool_name_raw) + with self._lock: + first = tool_key and tool_key not in self._first_tool_uses + if first: + self._first_tool_uses.add(tool_key) + if first: + # tool_name property must be in the analytics allowlist; + # if the raw name isn't recognised, we send "custom_tool". + from openjarvis.analytics.events import KNOWN_TOOL_NAMES + + shipped_name = ( + tool_name_raw + if tool_name_raw in KNOWN_TOOL_NAMES + else "custom_tool" + ) + self.client.capture("tool_first_used", {"tool_name": shipped_name}) + except Exception as exc: + logger.debug("Bridge _on_tool_end error: %s", exc) + + def _on_session_end(self, event: Event) -> None: + try: + data = event.data or {} + self.aggregator.end_session(self._session_id(data)) + except Exception as exc: + logger.debug("Bridge _on_session_end error: %s", exc) + + def _on_agent_turn_end(self, event: Event) -> None: + # AGENT_TURN_END can mark the end of a logical chat exchange. + # If there's no explicit SESSION_END, treat this as an idle hint + # rather than a hard close — the aggregator will flush on idle + # if no more events arrive. + try: + data = event.data or {} + if data.get("error"): + self.aggregator.record_error(self._session_id(data)) + except Exception as exc: + logger.debug("Bridge _on_agent_turn_end error: %s", exc) + + def _on_feedback(self, event: Event) -> None: + try: + data = event.data or {} + rating = data.get("rating") + has_comment = bool(data.get("comment") or data.get("text")) + self.client.capture( + "feedback_submitted", + { + "rating": int(rating) if isinstance(rating, (int, float)) else 0, + "has_comment": has_comment, + }, + ) + except Exception as exc: + logger.debug("Bridge _on_feedback error: %s", exc) + + def _on_security_alert(self, event: Event) -> None: + try: + data = event.data or {} + self.client.capture( + "error_shown_to_user", + { + "error_class": "permission_denied", + "platform": str(data.get("platform", "cli")), + }, + ) + except Exception as exc: + logger.debug("Bridge _on_security_alert error: %s", exc) + + +__all__ = ["EventBridge"] diff --git a/src/openjarvis/analytics/client.py b/src/openjarvis/analytics/client.py new file mode 100644 index 00000000..0189262d --- /dev/null +++ b/src/openjarvis/analytics/client.py @@ -0,0 +1,127 @@ +"""PostHog client wrapper. + +A thin adapter over the official ``posthog`` SDK that: + - Initialises lazily and only if analytics is enabled. + - Pipes every capture through :mod:`redaction` and :mod:`events` for + PII stripping and structural validation. + - Fails silently — analytics must never break the host application. + +The underlying SDK handles batching, async send on a background +thread, retries, and silent failure on network errors. We layer +"never crash the app" on top by wrapping every call in try/except. +""" + +from __future__ import annotations + +import logging +import threading +from typing import Any + +from openjarvis.analytics.events import validate_event +from openjarvis.analytics.identity import ( + get_or_create_anon_id, + is_analytics_enabled, +) +from openjarvis.analytics.redaction import redact +from openjarvis.core.config import AnalyticsConfig + +logger = logging.getLogger(__name__) + + +class AnalyticsClient: + """Send anonymized usage events to PostHog. + + Construct once at server / CLI startup, share for the process + lifetime, call :meth:`shutdown` on exit to flush pending events. + """ + + def __init__(self, config: AnalyticsConfig, anon_id: str | None = None) -> None: + self.config = config + self.anon_id = anon_id or get_or_create_anon_id(config.anon_id_path) + self._lock = threading.Lock() + self._posthog: Any = None + self._enabled = is_analytics_enabled(config) + if self._enabled: + self._init_sdk() + + def _init_sdk(self) -> None: + try: + from posthog import Posthog + + self._posthog = Posthog( + project_api_key=self.config.key, + host=self.config.host, + # The SDK queues events and flushes on a background + # thread; these knobs just tune the batch size/cadence. + max_queue_size=10_000, + flush_at=self.config.flush_at_size, + flush_interval=self.config.flush_interval_seconds, + # Don't sample or auto-capture anything beyond what we + # explicitly send. + disable_geoip=True, + ) + logger.debug( + "PostHog analytics initialised host=%s anon_id=%s…", + self.config.host, + self.anon_id[:8], + ) + except Exception as exc: + logger.debug("Analytics SDK init failed (%s); analytics disabled", exc) + self._posthog = None + self._enabled = False + + @property + def enabled(self) -> bool: + return self._enabled and self._posthog is not None + + def capture( + self, + event_name: str, + properties: dict[str, Any] | None = None, + ) -> None: + """Send one event. Unknown events are silently dropped. + + Runs through redaction → event-spec validation → SDK capture. + Failures at any stage are swallowed; analytics is best-effort. + """ + if not self.enabled: + return + try: + raw = properties or {} + cleaned = redact(raw) + validated = validate_event(event_name, cleaned) + if validated is None: + logger.debug("Dropped unknown analytics event: %s", event_name) + return + self._posthog.capture( + distinct_id=self.anon_id, + event=event_name, + properties=validated, + ) + except Exception as exc: + logger.debug("Analytics capture failed for %s: %s", event_name, exc) + + def flush(self) -> None: + """Force-flush queued events. Safe to call repeatedly.""" + if self._posthog is None: + return + try: + self._posthog.flush() + except Exception: + pass + + def shutdown(self) -> None: + """Flush and close the SDK. Call once on process exit.""" + with self._lock: + if self._posthog is None: + return + try: + self._posthog.flush() + self._posthog.shutdown() + except Exception: + pass + self._posthog = None + self._enabled = False + + +__all__ = ["AnalyticsClient"] diff --git a/src/openjarvis/analytics/events.py b/src/openjarvis/analytics/events.py new file mode 100644 index 00000000..645b1c3c --- /dev/null +++ b/src/openjarvis/analytics/events.py @@ -0,0 +1,407 @@ +"""Canonical registry of external analytics events. + +Single source of truth for every event name and property the OpenJarvis +analytics module is allowed to send. Any event not declared here is +dropped at send time. Any property not declared on a known event is +also dropped. This is the fail-closed half of the PII guardrail — +see :mod:`openjarvis.analytics.redaction` for the value-level filters. + +Keeping the catalog in code (not config) means PR review is the gate +for adding a new event, and ``docs/telemetry.md`` can render from this +module as the source of truth. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable + +PropertyValidator = Callable[[Any], bool] + + +@dataclass(frozen=True, slots=True) +class EventSpec: + """Declaration for one analytics event.""" + + name: str + description: str + properties: dict[str, PropertyValidator] + + +# --------------------------------------------------------------------------- +# Reusable validators +# --------------------------------------------------------------------------- + + +def _is_bool(v: Any) -> bool: + return isinstance(v, bool) + + +def _is_int_nonneg(v: Any) -> bool: + return isinstance(v, int) and not isinstance(v, bool) and v >= 0 + + +def _is_number_nonneg(v: Any) -> bool: + return isinstance(v, (int, float)) and not isinstance(v, bool) and v >= 0 + + +def _is_short_str(v: Any) -> bool: + return isinstance(v, str) and 0 < len(v) <= 64 + + +def _is_med_str(v: Any) -> bool: + return isinstance(v, str) and 0 < len(v) <= 200 + + +def _is_hash16(v: Any) -> bool: + """sha256 prefix, 16 hex chars — used for hashed model/tool identifiers.""" + if not isinstance(v, str) or len(v) != 16: + return False + return all(c in "0123456789abcdef" for c in v) + + +def _is_one_of(*allowed: str) -> PropertyValidator: + allowed_set = frozenset(allowed) + + def check(v: Any) -> bool: + return isinstance(v, str) and v in allowed_set + + return check + + +# --------------------------------------------------------------------------- +# Closed enums (allowlists for free-form-looking strings) +# --------------------------------------------------------------------------- + +_OS_VALUES = ("darwin", "linux", "wsl", "windows", "unknown") +_ARCH_VALUES = ("x86_64", "arm64", "aarch64", "unknown") +_PLATFORM_VALUES = ( + "cli", + "macos", + "linux", + "windows", + "tauri-macos", + "tauri-linux", + "tauri-windows", + "web", +) +_INSTALL_STAGES = ( + "deps", + "uv", + "python", + "venv", + "package", + "ollama", + "model_download", + "config", + "verify", + "complete", +) +_SETUP_PRESETS = ( + "morning-digest-mac", + "deep-research", + "code-assistant", + "minimal", + "custom", + "default", +) +_ERROR_CLASSES = ( + "network_error", + "engine_unreachable", + "model_not_found", + "tool_timeout", + "tool_error", + "rate_limit", + "auth_failure", + "permission_denied", + "validation_error", + "config_error", + "unknown_error", +) +_FEATURE_NAMES = ( + "chat", + "voice", + "agents", + "skills", + "memory", + "tools", + "connectors", + "scheduler", + "workflows", + "digest", + "research", + "evals", + "feedback", + "telemetry_dashboard", + "settings", +) +_TOOL_NAMES = ( + "http_request", + "file_read", + "file_write", + "shell", + "search_web", + "calculator", + "memory_search", + "memory_store", + "code_exec", + "image_view", + "gmail", + "calendar", + "drive", + "github", + "hackernews", + "weather", + "notion", + "strava", + "custom_tool", +) +_CONNECTOR_NAMES = ( + "google", + "gmail", + "calendar", + "drive", + "github", + "notion", + "strava", + "weather", + "hackernews", + "slack", + "telegram", + "discord", + "webhook", + "sendblue", + "whatsapp", + "signal", + "custom", +) +_ENGINE_NAMES = ( + "ollama", + "vllm", + "mlx", + "llama_cpp", + "openai", + "anthropic", + "google", + "unknown", +) + + +# --------------------------------------------------------------------------- +# Event specifications +# --------------------------------------------------------------------------- + +_SPECS: tuple[EventSpec, ...] = ( + # -- Install funnel (sent from install.sh) ------------------------------ + EventSpec( + name="install_started", + description="Install script began executing (curl | bash entry).", + properties={ + "os": _is_one_of(*_OS_VALUES), + "arch": _is_one_of(*_ARCH_VALUES), + "installer_version": _is_short_str, + }, + ), + EventSpec( + name="install_stage_completed", + description="One install stage finished successfully.", + properties={ + "stage": _is_one_of(*_INSTALL_STAGES), + "elapsed_ms": _is_int_nonneg, + "os": _is_one_of(*_OS_VALUES), + }, + ), + EventSpec( + name="install_completed", + description="Full install finished successfully.", + properties={ + "total_elapsed_ms": _is_int_nonneg, + "os": _is_one_of(*_OS_VALUES), + "arch": _is_one_of(*_ARCH_VALUES), + }, + ), + EventSpec( + name="install_failed", + description="Install script exited non-zero at a known stage.", + properties={ + "stage": _is_one_of(*_INSTALL_STAGES), + "exit_code": _is_int_nonneg, + "os": _is_one_of(*_OS_VALUES), + "arch": _is_one_of(*_ARCH_VALUES), + }, + ), + EventSpec( + name="uninstall_started", + description="Uninstall script began executing.", + properties={ + "days_since_install": _is_int_nonneg, + "os": _is_one_of(*_OS_VALUES), + }, + ), + # -- App lifecycle ------------------------------------------------------ + EventSpec( + name="app_opened", + description="App boot — every launch of CLI or desktop UI.", + properties={ + "version": _is_short_str, + "platform": _is_one_of(*_PLATFORM_VALUES), + }, + ), + EventSpec( + name="setup_completed", + description="`jarvis init` finished and config.toml was written.", + properties={ + "preset": _is_one_of(*_SETUP_PRESETS), + "model_hash": _is_hash16, + "engine": _is_one_of(*_ENGINE_NAMES), + }, + ), + EventSpec( + name="first_chat_sent", + description="First chat message ever sent on this anon_id.", + properties={ + "platform": _is_one_of(*_PLATFORM_VALUES), + }, + ), + # -- Usage (per session, aggregated) ----------------------------------- + EventSpec( + name="chat_session_ended", + description=( + "A chat session closed (explicit close or idle timeout). " + "Aggregated counts only — no content." + ), + properties={ + "turn_count": _is_int_nonneg, + "tokens_in": _is_int_nonneg, + "tokens_out": _is_int_nonneg, + "latency_ms_p50": _is_number_nonneg, + "latency_ms_p95": _is_number_nonneg, + "tool_count": _is_int_nonneg, + "unique_tools": _is_int_nonneg, + "unique_models": _is_int_nonneg, + "error_count": _is_int_nonneg, + "model_hash": _is_hash16, + "engine": _is_one_of(*_ENGINE_NAMES), + "duration_ms": _is_int_nonneg, + }, + ), + EventSpec( + name="tool_first_used", + description="First time this anon_id used a given tool.", + properties={ + "tool_name": _is_one_of(*_TOOL_NAMES), + }, + ), + EventSpec( + name="model_changed", + description="User explicitly switched the active model.", + properties={ + "from_model_hash": _is_hash16, + "to_model_hash": _is_hash16, + "engine": _is_one_of(*_ENGINE_NAMES), + }, + ), + EventSpec( + name="connector_auth_completed", + description="OAuth flow finished successfully for a connector.", + properties={ + "connector_name": _is_one_of(*_CONNECTOR_NAMES), + }, + ), + EventSpec( + name="feature_used", + description="A top-level feature was invoked.", + properties={ + "feature_name": _is_one_of(*_FEATURE_NAMES), + }, + ), + EventSpec( + name="feedback_submitted", + description="User submitted feedback. No comment content sent.", + properties={ + "rating": _is_int_nonneg, + "has_comment": _is_bool, + }, + ), + EventSpec( + name="error_shown_to_user", + description="A user-visible error was rendered. Error class only.", + properties={ + "error_class": _is_one_of(*_ERROR_CLASSES), + "platform": _is_one_of(*_PLATFORM_VALUES), + }, + ), + EventSpec( + name="settings_changed", + description="User toggled or modified a setting.", + properties={ + "setting_key": _is_short_str, + }, + ), + # -- Daily rollup ------------------------------------------------------ + EventSpec( + name="usage_daily_summary", + description=( + "Once-per-day aggregated counts. Cheaper than per-event " + "for high-frequency operations." + ), + properties={ + "sessions": _is_int_nonneg, + "total_tokens": _is_int_nonneg, + "total_inferences": _is_int_nonneg, + "unique_tools": _is_int_nonneg, + "unique_models": _is_int_nonneg, + "total_errors": _is_int_nonneg, + "total_duration_ms": _is_int_nonneg, + }, + ), +) + + +REGISTRY: dict[str, EventSpec] = {spec.name: spec for spec in _SPECS} + + +def validate_event(name: str, properties: dict[str, Any]) -> dict[str, Any] | None: + """Return cleaned properties or ``None`` if the event name is unknown. + + Unknown properties are silently dropped. Properties whose values + fail the spec's validator are silently dropped. Empty result is + valid (the event itself is still recorded). + """ + spec = REGISTRY.get(name) + if spec is None: + return None + cleaned: dict[str, Any] = {} + for key, value in properties.items(): + validator = spec.properties.get(key) + if validator is None: + continue + if not validator(value): + continue + cleaned[key] = value + return cleaned + + +def known_event_names() -> tuple[str, ...]: + """All event names declared in the catalog (for docs and tests).""" + return tuple(REGISTRY.keys()) + + +# Public re-exports of the allowlists for cross-module use (bridge.py). +KNOWN_TOOL_NAMES = frozenset(_TOOL_NAMES) +KNOWN_CONNECTORS = frozenset(_CONNECTOR_NAMES) +KNOWN_FEATURES = frozenset(_FEATURE_NAMES) +KNOWN_ENGINES = frozenset(_ENGINE_NAMES) + + +__all__ = [ + "EventSpec", + "REGISTRY", + "PropertyValidator", + "validate_event", + "known_event_names", + "KNOWN_TOOL_NAMES", + "KNOWN_CONNECTORS", + "KNOWN_FEATURES", + "KNOWN_ENGINES", +] diff --git a/src/openjarvis/analytics/identity.py b/src/openjarvis/analytics/identity.py new file mode 100644 index 00000000..63f4453f --- /dev/null +++ b/src/openjarvis/analytics/identity.py @@ -0,0 +1,49 @@ +"""Anonymous identity for external analytics. + +One UUID v4 per install, persisted to disk on first use. The same file +is referenced by ``scripts/install/install.sh`` so install-time beacon +events tie back to the same person across the install→first-run funnel. + +No email, no name, no hardware fingerprint — just an opaque UUID. +""" + +from __future__ import annotations + +import uuid +from pathlib import Path + +from openjarvis.core.config import AnalyticsConfig + + +def get_or_create_anon_id(path: Path | str) -> str: + """Return the persisted anon ID, generating one on first call. + + Idempotent across processes — if the file already exists with a + non-empty value, return it; otherwise generate a fresh UUID v4 and + write atomically (rename-after-write so a crashed write leaves no + half-file). + """ + p = Path(path) + if p.exists(): + existing = p.read_text(encoding="utf-8").strip() + if existing: + return existing + new_id = str(uuid.uuid4()) + p.parent.mkdir(parents=True, exist_ok=True) + tmp = p.with_suffix(p.suffix + ".tmp") + tmp.write_text(new_id + "\n", encoding="utf-8") + tmp.replace(p) + return new_id + + +def reset_anon_id(path: Path | str) -> str: + """Delete the persisted ID and generate a fresh one (privacy reset).""" + p = Path(path) + if p.exists(): + p.unlink() + return get_or_create_anon_id(p) + + +def is_analytics_enabled(cfg: AnalyticsConfig) -> bool: + """Return True if analytics is enabled in config.""" + return cfg.enabled diff --git a/src/openjarvis/analytics/redaction.py b/src/openjarvis/analytics/redaction.py new file mode 100644 index 00000000..e766e1b4 --- /dev/null +++ b/src/openjarvis/analytics/redaction.py @@ -0,0 +1,106 @@ +"""PII redaction for analytics property values. + +This is the value-level half of the guardrail +(:mod:`openjarvis.analytics.events` is the structural half). + +For each property value we ship, we: + 1. Drop strings longer than ``MAX_STR_LEN`` (no chunks of chat content). + 2. Drop strings that match any known PII pattern (emails, IPs, MACs, + $HOME paths, API keys, JWTs, bearer tokens, etc.). + 3. Otherwise pass through unchanged. + +Fail-closed: when in doubt, drop. Combined with the event-spec +allowlist this gives two independent layers of protection. +""" + +from __future__ import annotations + +import hashlib +import re +from typing import Any + +MAX_STR_LEN = 200 + +# Patterns that, if found anywhere inside a string value, cause that +# value to be dropped. Order is by likelihood for short-circuit speed. +_PII_PATTERNS: tuple[re.Pattern[str], ...] = ( + # Email + re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+"), + # IPv4 (any 4-octet decimal pattern) + re.compile(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"), + # IPv6 (loose — any colon-separated hex) + re.compile(r"\b(?:[0-9A-Fa-f]{1,4}:){2,}[0-9A-Fa-f]{0,4}\b"), + # MAC address + re.compile(r"\b[0-9A-Fa-f]{2}(?::[0-9A-Fa-f]{2}){5}\b"), + # Home paths + re.compile(r"/Users/[^/\s]+"), + re.compile(r"/home/[^/\s]+"), + re.compile(r"\$HOME|~/"), + # File URLs + re.compile(r"file://"), + # Common API key prefixes + re.compile(r"\bsk-[A-Za-z0-9_-]{8,}"), # OpenAI / Anthropic + re.compile(r"\bxoxb-[A-Za-z0-9-]{8,}"), # Slack bot + re.compile(r"\bxoxp-[A-Za-z0-9-]{8,}"), # Slack user + re.compile(r"\bghp_[A-Za-z0-9]{20,}"), # GitHub personal + re.compile(r"\bgho_[A-Za-z0-9]{20,}"), # GitHub OAuth + re.compile(r"\bAKIA[0-9A-Z]{16}\b"), # AWS access key + re.compile(r"\bAIza[0-9A-Za-z_-]{20,}"), # Google API key + re.compile(r"\bya29\.[0-9A-Za-z_-]+"), # Google OAuth access token + # JWT (three base64url chunks separated by dots, header starts with eyJ) + re.compile(r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+"), + # Bearer authorization headers + re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._-]+"), + # Password / secret assignment patterns + re.compile(r"(?i)\b(password|secret|api_key|token)\s*[:=]\s*\S+"), + # Hostnames that look like personal machines (e.g. johns-macbook.local) + re.compile(r"\b[A-Za-z0-9-]+\.local\b"), +) + + +def looks_like_pii(s: str) -> bool: + """Return True if any PII pattern matches anywhere in ``s``.""" + for pattern in _PII_PATTERNS: + if pattern.search(s): + return True + return False + + +def redact(properties: dict[str, Any]) -> dict[str, Any]: + """Return a copy of ``properties`` with PII-bearing string values dropped. + + Non-string values pass through unchanged. Strings exceeding + ``MAX_STR_LEN`` are dropped. Strings matching any PII pattern are + dropped. The event-spec validator (in :mod:`events`) runs after + this and provides a second layer of structural enforcement. + """ + out: dict[str, Any] = {} + for key, value in properties.items(): + if isinstance(value, str): + if not value: + # empty string is uninformative — drop + continue + if len(value) > MAX_STR_LEN: + continue + if looks_like_pii(value): + continue + elif isinstance(value, (list, dict, set, tuple)): + # Composite values are never sent — keeps the surface tiny. + continue + out[key] = value + return out + + +def hash_id(s: str) -> str: + """Return a 16-char sha256 prefix of ``s``. + + Used for model / tool / connector names that aren't on the public + allowlist — we still want to see "uses-a-custom-model-X" cohorting + without ever learning which model. + """ + if not s: + return "" + return hashlib.sha256(s.encode("utf-8")).hexdigest()[:16] + + +__all__ = ["MAX_STR_LEN", "redact", "looks_like_pii", "hash_id"] diff --git a/src/openjarvis/core/config.py b/src/openjarvis/core/config.py index 855e752f..41789abf 100644 --- a/src/openjarvis/core/config.py +++ b/src/openjarvis/core/config.py @@ -932,6 +932,27 @@ class TelemetryConfig: steady_state_threshold: float = 0.05 +@dataclass(slots=True) +class AnalyticsConfig: + """External anonymous usage analytics (PostHog). + + Separate concern from :class:`TelemetryConfig`, which stores local + FLOPs/energy/inference metrics in SQLite. This controls anonymized + usage events sent to the OpenJarvis team's PostHog instance to + measure setup success, retention, feature usage, and churn. + + No chat content, prompts, model outputs, file paths, emails, IPs, + or hardware identifiers are ever sent. See ``docs/telemetry.md``. + """ + + enabled: bool = True + host: str = "https://34.231.106.201.sslip.io" + key: str = "phc_ysKu72QaxzYNmDpHFcesD2ZZAe68zkdWJEKoYYkc5e3n" + anon_id_path: str = str(DEFAULT_CONFIG_DIR / "anon_id") + flush_interval_seconds: int = 30 + flush_at_size: int = 100 + + @dataclass(slots=True) class TracesConfig: """Trace system settings.""" @@ -1424,6 +1445,7 @@ class JarvisConfig: agent: AgentConfig = field(default_factory=AgentConfig) server: ServerConfig = field(default_factory=ServerConfig) telemetry: TelemetryConfig = field(default_factory=TelemetryConfig) + analytics: AnalyticsConfig = field(default_factory=AnalyticsConfig) traces: TracesConfig = field(default_factory=TracesConfig) channel: ChannelConfig = field(default_factory=ChannelConfig) security: SecurityConfig = field(default_factory=SecurityConfig) @@ -1682,6 +1704,7 @@ def load_config(path: Optional[Path] = None) -> JarvisConfig: "agent", "server", "telemetry", + "analytics", "traces", "security", "channel", diff --git a/src/openjarvis/server/analytics_routes.py b/src/openjarvis/server/analytics_routes.py new file mode 100644 index 00000000..751508bf --- /dev/null +++ b/src/openjarvis/server/analytics_routes.py @@ -0,0 +1,55 @@ +"""Small router exposing the anonymous analytics identity to the frontend. + +The Tauri desktop app and web frontend need the same ``anon_id`` that +the backend and install.sh use so that all events tie to one person. +This endpoint returns that ID (generating it on first call) plus the +public PostHog project key and host so the frontend can initialise +``posthog-js`` against the same project. + +Public API — no auth required because the data returned is exactly +what the frontend would ship to PostHog anyway. The project key is +a public token, not a secret. +""" + +from __future__ import annotations + +from fastapi import APIRouter +from pydantic import BaseModel + +from openjarvis.analytics.identity import ( + get_or_create_anon_id, + is_analytics_enabled, +) +from openjarvis.core.config import load_config + + +class AnalyticsIdentity(BaseModel): + enabled: bool + anon_id: str + host: str + key: str + + +router = APIRouter(prefix="/v1/analytics", tags=["analytics"]) + + +@router.get("/identity", response_model=AnalyticsIdentity) +def get_identity() -> AnalyticsIdentity: + """Return the analytics identity for the current install. + + If analytics is disabled in config, still return a + structurally valid response with ``enabled=False`` so the frontend + can decide what to do; never throw. + """ + cfg = load_config() + enabled = is_analytics_enabled(cfg.analytics) + anon_id = get_or_create_anon_id(cfg.analytics.anon_id_path) if enabled else "" + return AnalyticsIdentity( + enabled=enabled, + anon_id=anon_id, + host=cfg.analytics.host, + key=cfg.analytics.key if enabled else "", + ) + + +__all__ = ["router"] diff --git a/src/openjarvis/server/app.py b/src/openjarvis/server/app.py index 35863a84..c8a3388c 100644 --- a/src/openjarvis/server/app.py +++ b/src/openjarvis/server/app.py @@ -10,6 +10,7 @@ from fastapi import FastAPI from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles +from openjarvis.server.analytics_routes import router as analytics_router from openjarvis.server.api_routes import include_all_routes from openjarvis.server.comparison import comparison_router from openjarvis.server.connectors_router import create_connectors_router @@ -238,12 +239,55 @@ def create_app( except Exception: pass # traces are optional; don't block server startup + # Wire up external analytics if enabled (PostHog) — never block startup. + # Note: we do NOT fire app_opened here. The frontend owns that event + # because "server started" (this code path) is not the same as "user + # opened the app" — the server can run headless via cron, daemons, + # or test suites. + app.state.analytics_client = None + app.state.analytics_bridge = None + try: + from openjarvis.analytics import ( + AnalyticsClient, + EventBridge, + is_analytics_enabled, + ) + from openjarvis.core.config import load_config + + _cfg = config if config is not None else load_config() + if is_analytics_enabled(_cfg.analytics): + _client = AnalyticsClient(_cfg.analytics) + app.state.analytics_client = _client + _bus_ref = getattr(app.state, "bus", None) + if _bus_ref is not None: + _bridge = EventBridge(_bus_ref, _client) + _bridge.start() + app.state.analytics_bridge = _bridge + + @app.on_event("shutdown") + async def _shutdown_analytics() -> None: + bridge = getattr(app.state, "analytics_bridge", None) + if bridge is not None: + try: + bridge.stop() + except Exception: + pass + client = getattr(app.state, "analytics_client", None) + if client is not None: + try: + client.shutdown() + except Exception: + pass + except Exception as _exc: + logger.debug("Analytics init skipped: %s", _exc) + app.include_router(router) app.include_router(dashboard_router) app.include_router(comparison_router) app.include_router(create_connectors_router()) app.include_router(create_digest_router()) app.include_router(upload_router) + app.include_router(analytics_router) include_all_routes(app) # Restore SendBlue channel bindings from database on startup diff --git a/tests/analytics/__init__.py b/tests/analytics/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/analytics/test_redaction.py b/tests/analytics/test_redaction.py new file mode 100644 index 00000000..49058e50 --- /dev/null +++ b/tests/analytics/test_redaction.py @@ -0,0 +1,339 @@ +"""Tests for openjarvis.analytics.redaction. + +Two layers of guarantee are tested here: + - Value-level: ``redact()`` drops string values that match any PII + pattern, exceed ``MAX_STR_LEN``, or carry composite types. + - Structural: ``validate_event()`` from the catalog drops unknown + events and properties whose values fail the per-spec validator. + +Together they form fail-closed PII protection. Adding a new event or +property anywhere in the analytics module should require touching +this file as well. +""" + +from __future__ import annotations + +import pytest + +from openjarvis.analytics.events import ( + REGISTRY, + known_event_names, + validate_event, +) +from openjarvis.analytics.redaction import ( + MAX_STR_LEN, + hash_id, + looks_like_pii, + redact, +) + +# --------------------------------------------------------------------------- +# Value-level PII detection +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value", + [ + # Emails + "user@example.com", + "first.last+filter@subdomain.example.co.uk", + "leak hidden in user@example.com sentence", + # IPv4 + "127.0.0.1", + "192.168.1.42", + "trace from 10.0.0.5 here", + # IPv6 (loose pattern) + "fe80::1ff:fe23:4567:890a", + # MAC + "00:1A:2B:3C:4D:5E", + "AA:BB:CC:DD:EE:FF", + # Home paths + "/Users/alice/Documents/secret.txt", + "/home/bob/.ssh/id_rsa", + "~/Library/Application Support", + "$HOME/secrets", + # File URLs + "file:///etc/passwd", + # API keys + "sk-abcdefghijklmnop123456", + "sk-proj-AbCdEfGhIjKlMnOpQrStUv", + "xoxb-1234567890-abcdefghij-1234567890abcdef", + "ghp_1234567890abcdefghij1234567890abcd", + "gho_1234567890abcdefghij1234567890abcd", + "AKIAIOSFODNN7EXAMPLE", + "AIzaSyD-1234567890abcdefghij1234567890", + "ya29.a0AfH6SMBabc123def456ghi789jkl", + # JWT + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U", + # Bearer tokens + "Bearer abc123def456", + "bearer xyz789", + # Password / secret assignments + "password=hunter2", + "API_KEY: super-secret-value", + "token=abc123", + # .local hostnames + "alice-macbook.local", + ], +) +def test_pii_patterns_are_detected(value: str) -> None: + assert looks_like_pii(value), f"expected to flag: {value!r}" + + +@pytest.mark.parametrize( + "value", + [ + "darwin", + "arm64", + "0.1.1", + "qwen3.5:2b", # model name with colon — must not look like IPv6 + "ok", + "chat", + "v1.2.3", + "8754b77c7eee26e4", # a hashed id + "true", + ], +) +def test_safe_values_pass(value: str) -> None: + assert not looks_like_pii(value), f"false positive: {value!r}" + + +# --------------------------------------------------------------------------- +# redact() — full property dict cleaning +# --------------------------------------------------------------------------- + + +def test_redact_keeps_safe_values() -> None: + cleaned = redact( + { + "os": "darwin", + "arch": "arm64", + "count": 42, + "ratio": 0.5, + "enabled": True, + } + ) + assert cleaned == { + "os": "darwin", + "arch": "arm64", + "count": 42, + "ratio": 0.5, + "enabled": True, + } + + +def test_redact_drops_pii_strings() -> None: + _jwt = ( + "eyJhbGciOiJIUzI1NiJ9" + ".eyJzdWIiOiIxMjM0In0" + ".dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U" + ) + cleaned = redact( + { + "ok": "chat", + "leak_email": "user@example.com", + "leak_path": "/Users/alice/file.txt", + "leak_key": "sk-1234567890abcdef", + "leak_jwt": _jwt, + "leak_bearer": "Bearer abc123", + } + ) + assert cleaned == {"ok": "chat"} + + +def test_redact_drops_long_strings() -> None: + huge = "x" * (MAX_STR_LEN + 1) + cleaned = redact({"safe": "short", "huge": huge}) + assert cleaned == {"safe": "short"} + + +def test_redact_drops_empty_strings() -> None: + # Empty strings carry no signal — drop them to keep the wire clean. + cleaned = redact({"empty": "", "real": "value"}) + assert cleaned == {"real": "value"} + + +def test_redact_drops_composite_types() -> None: + # Lists, dicts, sets, and tuples bypass redaction entirely. Sending + # them would let PII smuggle through inside containers. + cleaned = redact( + { + "good": 1, + "list": ["a", "b"], + "dict": {"k": "v"}, + "set": {"x", "y"}, + "tuple": ("p", "q"), + } + ) + assert cleaned == {"good": 1} + + +def test_redact_preserves_numeric_zero_and_false() -> None: + # Falsy non-string values must survive — they're meaningful counts. + cleaned = redact({"a": 0, "b": False, "c": 0.0}) + assert cleaned == {"a": 0, "b": False, "c": 0.0} + + +def test_redact_returns_new_dict_does_not_mutate_input() -> None: + src = {"safe": "ok", "leak": "user@example.com"} + out = redact(src) + assert "leak" in src # original unchanged + assert "leak" not in out + + +# --------------------------------------------------------------------------- +# hash_id() — for cohorting without leaking identifiers +# --------------------------------------------------------------------------- + + +def test_hash_id_is_16_lowercase_hex() -> None: + h = hash_id("llama3.1:8b") + assert len(h) == 16 + assert all(c in "0123456789abcdef" for c in h) + + +def test_hash_id_is_deterministic() -> None: + a = hash_id("some-model-name") + b = hash_id("some-model-name") + assert a == b + + +def test_hash_id_differs_for_different_inputs() -> None: + assert hash_id("model-a") != hash_id("model-b") + + +def test_hash_id_empty_string_returns_empty() -> None: + assert hash_id("") == "" + + +# --------------------------------------------------------------------------- +# Structural validation — events.py catalog +# --------------------------------------------------------------------------- + + +def test_unknown_event_is_rejected() -> None: + assert validate_event("not_a_real_event", {"any": "thing"}) is None + + +def test_known_event_with_no_properties_is_accepted() -> None: + # Empty result is still a valid event — we'd still record it. + assert validate_event("first_chat_sent", {}) == {} + + +def test_unknown_properties_are_silently_dropped() -> None: + cleaned = validate_event( + "install_started", + { + "os": "darwin", + "arch": "arm64", + "installer_version": "0.1.1", + "malicious_extra": "should-not-survive", + }, + ) + assert cleaned == { + "os": "darwin", + "arch": "arm64", + "installer_version": "0.1.1", + } + + +def test_invalid_enum_value_is_dropped() -> None: + # 'os' must be in the allowlist; a path-like value fails. + cleaned = validate_event( + "install_started", + {"os": "/Users/alice", "arch": "arm64"}, + ) + assert cleaned == {"arch": "arm64"} + + +def test_invalid_int_property_is_dropped() -> None: + cleaned = validate_event( + "install_stage_completed", + {"stage": "uv", "elapsed_ms": -5}, # negative ms fails _is_int_nonneg + ) + assert cleaned == {"stage": "uv"} + + +def test_bool_is_not_int_for_int_validators() -> None: + # Critical: bool is a subclass of int in Python; the catalog + # validators must reject booleans where ints are expected + # (otherwise True would count as elapsed_ms=1). + cleaned = validate_event( + "install_stage_completed", + {"stage": "uv", "elapsed_ms": True}, + ) + assert "elapsed_ms" not in cleaned + + +def test_chat_session_ended_properties_pass() -> None: + cleaned = validate_event( + "chat_session_ended", + { + "turn_count": 5, + "tokens_in": 1000, + "tokens_out": 500, + "latency_ms_p50": 250.0, + "latency_ms_p95": 800.0, + "tool_count": 3, + "unique_tools": 2, + "unique_models": 1, + "error_count": 0, + "model_hash": "8754b77c7eee26e4", + "engine": "ollama", + "duration_ms": 60000, + }, + ) + assert cleaned["turn_count"] == 5 + assert cleaned["model_hash"] == "8754b77c7eee26e4" + assert cleaned["engine"] == "ollama" + + +def test_chat_session_ended_rejects_bad_hash() -> None: + cleaned = validate_event( + "chat_session_ended", + { + "turn_count": 1, + "model_hash": "not-a-hash-too-short", # not 16 hex chars + }, + ) + assert "turn_count" in cleaned + assert "model_hash" not in cleaned + + +# --------------------------------------------------------------------------- +# Catalog sanity +# --------------------------------------------------------------------------- + + +def test_catalog_has_expected_lifecycle_events() -> None: + names = set(known_event_names()) + must_exist = { + "install_started", + "install_stage_completed", + "install_completed", + "install_failed", + "uninstall_started", + "app_opened", + "setup_completed", + "first_chat_sent", + "chat_session_ended", + "tool_first_used", + "model_changed", + "feature_used", + "error_shown_to_user", + "feedback_submitted", + "usage_daily_summary", + } + missing = must_exist - names + assert not missing, f"catalog is missing required events: {sorted(missing)}" + + +def test_every_event_has_at_least_one_property_or_is_marker() -> None: + # Either the spec has properties, OR it's intentionally a marker + # event with no payload (e.g. first_chat_sent). We just assert + # that no property dict is malformed. + for name, spec in REGISTRY.items(): + assert isinstance(spec.properties, dict), f"{name}: properties must be dict" + for prop, validator in spec.properties.items(): + assert callable(validator), f"{name}.{prop}: validator must be callable" diff --git a/uv.lock b/uv.lock index 4d7ab283..0f66752f 100644 --- a/uv.lock +++ b/uv.lock @@ -4981,6 +4981,7 @@ dependencies = [ { name = "ddgs" }, { name = "httpx" }, { name = "openai" }, + { name = "posthog" }, { name = "python-telegram-bot" }, { name = "rich" }, { name = "tomli", marker = "python_full_version < '3.11'" }, @@ -5228,6 +5229,7 @@ requires-dist = [ { name = "pdfplumber", marker = "extra == 'pdf'", specifier = ">=0.10" }, { name = "playwright", marker = "extra == 'browser'", specifier = ">=1.40" }, { name = "polars", marker = "extra == 'framework-comparison'", specifier = ">=1.0" }, + { name = "posthog", specifier = ">=3.0" }, { name = "praw", marker = "extra == 'channel-reddit'", specifier = ">=7.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.0" }, { name = "pydantic", marker = "extra == 'server'", specifier = ">=2.0" }, @@ -5941,6 +5943,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/65/ad/b33c3022a394f3eb55c3310597cec615412a8a33880055eee191d154a628/polars_runtime_32-1.40.1-cp310-abi3-win_arm64.whl", hash = "sha256:b5cbfaf6b085b420b4bfcbe24e8f665076d1cccfdb80c0484c02a023ce205537", size = 45822104, upload-time = "2026-04-22T19:14:54.192Z" }, ] +[[package]] +name = "posthog" +version = "7.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backoff" }, + { name = "distro" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/b1/7e2216f4aa258ac73d1b443c84da861bba0512d255c20a3880805af7a520/posthog-7.14.1.tar.gz", hash = "sha256:c9a65765d5d8dc80ead4196337e3933b6ccbde0eed8f0ce49675485fffe43a3c", size = 205114, upload-time = "2026-05-11T16:22:21.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/65/a19988a81412fa2b68f1807791a06cdb5cdb6c946b48638b0fee898bc576/posthog-7.14.1-py3-none-any.whl", hash = "sha256:eac0919136b11b5ce6b320990c32004c12031601ae8a7e7a9e46bb3276038145", size = 240201, upload-time = "2026-05-11T16:22:19.559Z" }, +] + [[package]] name = "praw" version = "7.8.1"