diff --git a/.github/workflows/uptime-monitor.yml b/.github/workflows/uptime-monitor.yml new file mode 100644 index 000000000..459e6d32e --- /dev/null +++ b/.github/workflows/uptime-monitor.yml @@ -0,0 +1,127 @@ +name: Uptime Monitor + +on: + schedule: + - cron: '*/5 * * * *' # Run every 5 minutes + workflow_dispatch: # Allow manual trigger + +concurrency: + group: uptime-monitor + cancel-in-progress: false + +permissions: + issues: write + +jobs: + check-uptime: + name: Check Backend Health + runs-on: ubuntu-latest + steps: + - name: Uptime Check Script + uses: actions/github-script@v7 + env: + ALERT_WEBHOOK_URL: ${{ secrets.ALERT_WEBHOOK_URL }} + with: + script: | + const endpoints = [ + { env: "Production", url: "https://api.tinyhumans.ai/health", timeout: 10000 }, + { env: "Staging", url: "https://staging-api.tinyhumans.ai/health", timeout: 15000 } + ]; + + const maxRetries = 3; + const retryDelay = 5000; + const issueTitle = "CRITICAL: Backend Outage Detected"; + + async function fetchWithRetry(url, timeout, retries) { + for (let i = 0; i <= retries; i++) { + try { + const controller = new AbortController(); + const id = setTimeout(() => controller.abort(), timeout); + const response = await fetch(url, { signal: controller.signal }); + clearTimeout(id); + + if (response.status === 200) return { ok: true, status: response.status }; + if (i === retries) return { ok: false, status: response.status }; + } catch (error) { + if (i === retries) return { ok: false, error: error.message }; + } + await new Promise(resolve => setTimeout(resolve, retryDelay)); + } + } + + let allHealthy = true; + let failureDetails = []; + + for (const ep of endpoints) { + const result = await fetchWithRetry(ep.url, ep.timeout, maxRetries); + if (!result.ok) { + allHealthy = false; + failureDetails.push(`- **${ep.env}**: ${ep.url} (Status: ${result.status || result.error}) at ${new Date().toISOString()}`); + } + } + + const { data: issues } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + creator: 'github-actions[bot]' + }); + const openIssue = issues.find(i => i.title === issueTitle); + + const webhookUrl = process.env.ALERT_WEBHOOK_URL; + async function sendWebhook(message) { + if (webhookUrl) { + try { + // Discord uses { content } and Slack uses { text }; detect by URL + const payload = webhookUrl.includes('discord') + ? { content: message } + : { text: message }; + await fetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + } catch (e) { + console.error("Failed to send webhook", e); + } + } + } + + if (allHealthy) { + if (openIssue) { + const recoveryMsg = `✅ **RESOLVED**: All backend endpoints are now healthy and reachable.\n\nMonitors reported recovery at ${new Date().toISOString()}.`; + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: openIssue.number, + body: recoveryMsg + }); + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: openIssue.number, + state: 'closed' + }); + await sendWebhook(recoveryMsg); + console.log("Services recovered. Issue closed."); + } else { + console.log("All services healthy."); + } + } else { + const issueBody = `The automated uptime monitor detected an outage in the backend.\n\n### Failing Endpoints:\n${failureDetails.join('\n')}\n\nPlease check the logs and follow the runbook in \`docs/OPERATIONS.md\`.`; + + if (!openIssue) { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: issueTitle, + body: issueBody, + labels: ['bug', 'critical', 'ops'] + }); + await sendWebhook(`🚨 **${issueTitle}**\n${issueBody}`); + core.setFailed("Backend health check failed."); + } else { + console.log("Issue already exists, skipping duplicate creation."); + core.setFailed("Backend health check failed (ongoing)."); + } + } diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 000000000..9565c03dd --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,65 @@ +# Operations and Monitoring + +This document describes the monitoring strategy, alert policies, and incident response procedures for the OpenHuman backend. + +## Uptime Monitoring + +OpenHuman uses external uptime monitors to ensure that critical backend services are available and performing within acceptable thresholds. + +### Critical Endpoints + +The following endpoints are monitored for uptime: + +| Environment | Endpoint | Purpose | Health Signal | +|-------------|----------|---------|---------------| +| **Production** | `https://api.tinyhumans.ai/health` | Public API liveness | HTTP 200 = healthy; HTTP 503 = one or more components in error state (alert) | +| **Staging** | `https://staging-api.tinyhumans.ai/health` | Staging API liveness | HTTP 200 = healthy; HTTP 503 = one or more components in error state (alert) | + +### Monitoring Providers + +1. **Pingdom (Planned — not yet configured)**: + - Planned to hit the `/health` endpoints every 1 minute from multiple regions (US, EU, Asia). + - Alerts to be triggered after 2 consecutive failures. + - No Pingdom configuration currently exists in this repository; no alerts will be sent until it is set up. + +2. **GitHub Actions (Active)**: + - Scheduled workflow (`.github/workflows/uptime-monitor.yml`) runs every 5 minutes. + - Serves as an independent signal from the deployment pipeline. + - On outage detection, automatically creates a labeled GitHub Issue (`bug`, `critical`, `ops`) titled **"CRITICAL: Backend Outage Detected"** and closes it when services recover, providing a durable incident log in the repository. + +## Alerting and Escalation + +### Alert Destinations + +- **Slack/Discord**: Alerts are sent to the configured webhook (e.g. `#ops-alerts`) when the `ALERT_WEBHOOK_URL` GitHub secret is set. Set this secret in the repository settings pointing to your Slack incoming webhook or Discord server webhook URL. Alerts are skipped silently if the secret is not configured. +- **Email** *(planned)*: Email alerting to `ops@tinyhumans.ai` is not yet wired into the automated workflow. Until an email integration is added, the `ALERT_WEBHOOK_URL` webhook is the only active notification channel. + +### Escalation Path + +1. **Level 1 (Immediate)**: Notification to `#ops-alerts`. On-call engineer acknowledges. +2. **Level 2 (15 minutes)**: Page to the lead backend engineer. +3. **Level 3 (30 minutes)**: Escalation to the CTO. + +## Incident Response (Runbook) + +When a monitor fires: + +1. **Verify the outage**: Check the endpoint manually or via `curl -I `. +2. **Check Cloud Status**: Check [DigitalOcean Status](https://status.digitalocean.com/) or other upstream providers. +3. **Review Logs**: Access runtime logs via the DigitalOcean console or your container runtime (e.g. `docker logs ` or Kubernetes pod logs for containers sourced from `ghcr.io`). +4. **Determine Scope**: Is it a total outage or degraded performance? Is it specific to a region? +5. **Mitigation**: Restart the service via the cloud console or redeploy the last known healthy tag. +6. **Communication**: Update the internal status and notify stakeholders if the outage exceeds 5 minutes. + +## Testing Alerts + +To test the GitHub Actions alert pipeline safely without causing a real outage: +1. In `.github/workflows/uptime-monitor.yml`, temporarily change one endpoint URL to a non-existent path (e.g., `/health-test-trigger`) and trigger the workflow manually via `workflow_dispatch`. +2. Verify that a GitHub Issue is created and an alert is received in the `#ops-alerts` channel. +3. Revert the URL change and trigger the workflow again; verify the issue is closed and a recovery notification is sent. + +To test the Pingdom monitor, use Pingdom's built-in test-alert feature from the dashboard rather than changing the monitored URL. + +## Maintenance + +During planned maintenance, monitors should be paused to avoid false positives. This is handled via the provider's "Maintenance Mode" or by disabling the GitHub Action temporarily. diff --git a/src/core/jsonrpc.rs b/src/core/jsonrpc.rs index 3621579d4..442409290 100644 --- a/src/core/jsonrpc.rs +++ b/src/core/jsonrpc.rs @@ -636,7 +636,34 @@ fn with_cors_headers(mut response: Response) -> Response { /// Handler for the health check endpoint. async fn health_handler() -> impl IntoResponse { - (StatusCode::OK, Json(json!({ "ok": true }))) + let snapshot = crate::openhuman::health::snapshot(); + let unhealthy: Vec<&str> = snapshot + .components + .iter() + .filter_map(|(name, c)| { + if c.status == "ok" || c.status == "starting" { + None + } else { + Some(name.as_str()) + } + }) + .collect(); + let is_ok = unhealthy.is_empty(); + + let status = if is_ok { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + + tracing::debug!( + "[health] status={} components={} unhealthy={:?}", + status.as_u16(), + snapshot.components.len(), + unhealthy + ); + + (status, Json(snapshot)) } /// Handler for the schema discovery endpoint. diff --git a/src/core/jsonrpc_tests.rs b/src/core/jsonrpc_tests.rs index 3eed02cf0..2669ab63a 100644 --- a/src/core/jsonrpc_tests.rs +++ b/src/core/jsonrpc_tests.rs @@ -908,3 +908,41 @@ async fn invoke_method_core_version_via_tier1_reflects_state() { .expect("core.version should succeed"); assert_eq!(result, json!({ "version": "0.0.1-abc" })); } + +#[tokio::test] +async fn test_http_health_handler_returns_correct_status() { + use axum::body::to_bytes; + use axum::http::StatusCode; + use axum::response::IntoResponse; + + // Call the handler once and derive both the status and expected status from + // the same response — avoids a TOCTOU race where a separate snapshot() + // call before/after the handler could observe different component state. + let resp = super::health_handler().await.into_response(); + let status = resp.status(); + + let body = to_bytes(resp.into_body(), usize::MAX) + .await + .expect("failed to read body"); + let snapshot: serde_json::Value = + serde_json::from_slice(&body).expect("failed to deserialize health snapshot"); + + let components = snapshot["components"] + .as_object() + .expect("components should be an object"); + + // Derive the expected HTTP status solely from the response body so the + // test asserts internal consistency of the handler rather than racing on + // live component state. + let body_says_ok = components.values().all(|c| { + let s = c["status"].as_str().unwrap_or(""); + s == "ok" || s == "starting" + }); + let expected_status = if body_says_ok { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + + assert_eq!(status, expected_status); +}