mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
feat(browser): add Playwright backend (#4140)
This commit is contained in:
Steven Enamakel's Droid
@@ -373,9 +373,18 @@ jobs:
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: . -> target
|
||||
cache-on-failure: true
|
||||
cache-on-failure: false
|
||||
shared-key: pr-rust-core-cov
|
||||
|
||||
- name: Prune stale coverage test executables
|
||||
run: |
|
||||
# The restored target cache can contain prior llvm-cov integration-test
|
||||
# executables. Those are enormous after coverage instrumentation and
|
||||
# can exhaust the runner before the current cargo invocation finishes.
|
||||
find target/llvm-cov-target -type f -path '*/debug/deps/*' -perm -111 -delete 2>/dev/null || true
|
||||
echo "Disk after coverage target prune:"
|
||||
df -h /__w || true
|
||||
|
||||
- name: Install cargo-llvm-cov
|
||||
uses: taiki-e/install-action@cargo-llvm-cov
|
||||
|
||||
@@ -384,6 +393,13 @@ jobs:
|
||||
env:
|
||||
CARGO_BUILD_JOBS: "1"
|
||||
|
||||
- name: Prune coverage test executables before cache save
|
||||
if: always()
|
||||
run: |
|
||||
find target/llvm-cov-target -type f -path '*/debug/deps/*' -perm -111 -delete 2>/dev/null || true
|
||||
echo "Disk after post-coverage prune:"
|
||||
df -h /__w || true
|
||||
|
||||
- name: Upload core lcov
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
|
||||
@@ -303,6 +303,9 @@ whatsapp-web = ["dep:whatsapp-rust", "dep:whatsapp-rust-tokio-transport", "dep:w
|
||||
# this feature so the wipe RPC isn't even registered, let alone reachable.
|
||||
e2e-test-support = []
|
||||
|
||||
[lints.rust]
|
||||
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage)'] }
|
||||
|
||||
# Fix whisper-rs-sys CRT mismatch on Windows MSVC (LNK2038).
|
||||
# Upstream cmake build defaults to /MD but Rust uses /MT.
|
||||
# This fork adds config.static_crt(true) to the build script.
|
||||
|
||||
@@ -90,6 +90,7 @@ const AutocompleteDebugPanel = () => {
|
||||
|
||||
// Live logs
|
||||
const [logs, setLogs] = useState<string[]>([]);
|
||||
const isMountedRef = useRef(true);
|
||||
const previousStatusRef = useRef<AutocompleteStatus | null>(null);
|
||||
|
||||
// Personalization history
|
||||
@@ -103,6 +104,7 @@ const AutocompleteDebugPanel = () => {
|
||||
|
||||
const appendLogs = (entries: string[]) => {
|
||||
if (entries.length === 0) return;
|
||||
if (!isMountedRef.current) return;
|
||||
const now = new Date();
|
||||
const stamp = `${now.toLocaleTimeString()}.${String(now.getMilliseconds()).padStart(3, '0')}`;
|
||||
setLogs(current =>
|
||||
@@ -179,22 +181,30 @@ const AutocompleteDebugPanel = () => {
|
||||
|
||||
const loadHistory = async (): Promise<AcceptedCompletion[]> => {
|
||||
if (!isTauri()) return [];
|
||||
if (!isMountedRef.current) return [];
|
||||
setIsHistoryLoading(true);
|
||||
try {
|
||||
const response = await openhumanAutocompleteHistory({ limit: 20 });
|
||||
if (!isMountedRef.current) return response.result.entries;
|
||||
setHistoryEntries(response.result.entries);
|
||||
return response.result.entries;
|
||||
} catch {
|
||||
// Non-critical — silently ignore
|
||||
return [];
|
||||
} finally {
|
||||
setIsHistoryLoading(false);
|
||||
if (isMountedRef.current) {
|
||||
setIsHistoryLoading(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
isMountedRef.current = true;
|
||||
void load();
|
||||
void loadHistory();
|
||||
return () => {
|
||||
isMountedRef.current = false;
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
|
||||
@@ -210,6 +220,7 @@ const AutocompleteDebugPanel = () => {
|
||||
}
|
||||
try {
|
||||
const response = await openhumanAutocompleteStatus();
|
||||
if (!isMountedRef.current) return response.result;
|
||||
setStatus(response.result);
|
||||
trackStatusChanges(response.result);
|
||||
if (showSpinner) {
|
||||
@@ -344,7 +355,10 @@ const AutocompleteDebugPanel = () => {
|
||||
return;
|
||||
}
|
||||
if (attempt < maxAttempts - 1) {
|
||||
await new Promise(resolve => window.setTimeout(resolve, 180));
|
||||
await new Promise(resolve => globalThis.setTimeout(resolve, 180));
|
||||
}
|
||||
if (!isMountedRef.current) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -359,6 +373,7 @@ const AutocompleteDebugPanel = () => {
|
||||
suggestion: status?.suggestion?.value ?? undefined,
|
||||
skip_apply: true,
|
||||
});
|
||||
if (!isMountedRef.current) return;
|
||||
appendLogs(response.logs);
|
||||
if (response.result.accepted && response.result.value) {
|
||||
setMessage(
|
||||
|
||||
@@ -150,6 +150,7 @@ export interface RuntimeSettingsUpdate {
|
||||
|
||||
export interface BrowserSettingsUpdate {
|
||||
enabled?: boolean | null;
|
||||
backend?: 'agent_browser' | 'playwright' | 'rust_native' | 'computer_use' | 'auto' | null;
|
||||
}
|
||||
|
||||
export interface ScreenIntelligenceSettingsUpdate {
|
||||
|
||||
@@ -12,6 +12,7 @@ use super::loader::{fallback_workspace_dir, load_config_with_timeout, snapshot_c
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct BrowserSettingsPatch {
|
||||
pub enabled: Option<bool>,
|
||||
pub backend: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
@@ -102,9 +103,18 @@ pub async fn apply_browser_settings(
|
||||
config: &mut Config,
|
||||
update: BrowserSettingsPatch,
|
||||
) -> Result<RpcOutcome<serde_json::Value>, String> {
|
||||
let normalized_backend = update
|
||||
.backend
|
||||
.as_deref()
|
||||
.map(normalize_browser_backend)
|
||||
.transpose()?;
|
||||
|
||||
if let Some(enabled) = update.enabled {
|
||||
config.browser.enabled = enabled;
|
||||
}
|
||||
if let Some(backend) = normalized_backend {
|
||||
config.browser.backend = backend;
|
||||
}
|
||||
config.save().await.map_err(|e| e.to_string())?;
|
||||
let snapshot = snapshot_config_json(config)?;
|
||||
Ok(RpcOutcome::new(
|
||||
@@ -116,6 +126,20 @@ pub async fn apply_browser_settings(
|
||||
))
|
||||
}
|
||||
|
||||
fn normalize_browser_backend(raw: &str) -> Result<String, String> {
|
||||
let key = raw.trim().to_ascii_lowercase().replace('-', "_");
|
||||
match key.as_str() {
|
||||
"agent_browser" | "agentbrowser" => Ok("agent_browser".to_string()),
|
||||
"playwright" => Ok("playwright".to_string()),
|
||||
"rust_native" | "native" => Ok("rust_native".to_string()),
|
||||
"computer_use" | "computeruse" => Ok("computer_use".to_string()),
|
||||
"auto" => Ok("auto".to_string()),
|
||||
_ => Err(format!(
|
||||
"Unsupported browser backend '{raw}'. Use agent_browser, playwright, rust_native, computer_use, or auto"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads the configuration, applies browser settings updates, and saves it.
|
||||
pub async fn load_and_apply_browser_settings(
|
||||
update: BrowserSettingsPatch,
|
||||
|
||||
@@ -971,6 +971,7 @@ async fn apply_browser_settings_updates_enabled_flag() {
|
||||
&mut cfg,
|
||||
BrowserSettingsPatch {
|
||||
enabled: Some(true),
|
||||
backend: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -978,6 +979,47 @@ async fn apply_browser_settings_updates_enabled_flag() {
|
||||
assert!(cfg.browser.enabled);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn apply_browser_settings_updates_backend() {
|
||||
let tmp = tempdir().unwrap();
|
||||
let mut cfg = tmp_config(&tmp);
|
||||
cfg.browser.backend = "agent_browser".into();
|
||||
|
||||
apply_browser_settings(
|
||||
&mut cfg,
|
||||
BrowserSettingsPatch {
|
||||
enabled: None,
|
||||
backend: Some("playwright".into()),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("apply");
|
||||
|
||||
assert_eq!(cfg.browser.backend, "playwright");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn apply_browser_settings_rejects_unknown_backend() {
|
||||
let tmp = tempdir().unwrap();
|
||||
let mut cfg = tmp_config(&tmp);
|
||||
cfg.browser.enabled = false;
|
||||
cfg.browser.backend = "agent_browser".into();
|
||||
|
||||
let err = apply_browser_settings(
|
||||
&mut cfg,
|
||||
BrowserSettingsPatch {
|
||||
enabled: Some(true),
|
||||
backend: Some("netscape".into()),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("unknown backend should fail");
|
||||
|
||||
assert!(err.contains("Unsupported browser backend"));
|
||||
assert!(!cfg.browser.enabled);
|
||||
assert_eq!(cfg.browser.backend, "agent_browser");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn apply_local_ai_settings_updates_lm_studio_provider_fields() {
|
||||
let tmp = tempdir().unwrap();
|
||||
|
||||
@@ -73,7 +73,7 @@ fn default_true() -> bool {
|
||||
}
|
||||
|
||||
fn default_browser_backend() -> String {
|
||||
"agent_browser".into()
|
||||
"auto".into()
|
||||
}
|
||||
|
||||
fn default_browser_webdriver_url() -> String {
|
||||
|
||||
@@ -507,6 +507,7 @@ fn handle_update_browser_settings(params: Map<String, Value>) -> ControllerFutur
|
||||
let update = deserialize_params::<BrowserSettingsUpdate>(params)?;
|
||||
let patch = config_rpc::BrowserSettingsPatch {
|
||||
enabled: update.enabled,
|
||||
backend: update.backend,
|
||||
};
|
||||
to_json(config_rpc::load_and_apply_browser_settings(patch).await?)
|
||||
})
|
||||
|
||||
@@ -94,6 +94,7 @@ pub(super) struct RuntimeSettingsUpdate {
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct BrowserSettingsUpdate {
|
||||
pub(super) enabled: Option<bool>,
|
||||
pub(super) backend: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
|
||||
@@ -253,7 +253,13 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
||||
namespace: "config",
|
||||
function: "update_browser_settings",
|
||||
description: "Update browser automation settings.",
|
||||
inputs: vec![optional_bool("enabled", "Enable browser integration.")],
|
||||
inputs: vec![
|
||||
optional_bool("enabled", "Enable browser integration."),
|
||||
optional_string(
|
||||
"backend",
|
||||
"Browser backend: agent_browser, playwright, rust_native, computer_use, or auto.",
|
||||
),
|
||||
],
|
||||
outputs: vec![json_output("snapshot", "Updated config snapshot.")],
|
||||
},
|
||||
"update_local_ai_settings" => ControllerSchema {
|
||||
|
||||
@@ -195,6 +195,7 @@ pub(crate) fn is_computer_use_only_action(action: &str) -> bool {
|
||||
pub(crate) fn backend_name(backend: ResolvedBackend) -> &'static str {
|
||||
match backend {
|
||||
ResolvedBackend::AgentBrowser => "agent_browser",
|
||||
ResolvedBackend::Playwright => "playwright",
|
||||
ResolvedBackend::RustNative => "rust_native",
|
||||
ResolvedBackend::ComputerUse => "computer_use",
|
||||
}
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
//! Browser automation tool with pluggable backends.
|
||||
//!
|
||||
//! By default this uses Vercel's `agent-browser` tool for automation.
|
||||
//! Optionally, a Rust-native backend can be enabled at build time via
|
||||
//! `--features browser-native` and selected through config.
|
||||
//! Playwright is also supported as a local Node-backed backend. Optionally, a
|
||||
//! Rust-native backend can be enabled at build time via `--features
|
||||
//! browser-native` and selected through config.
|
||||
//! Computer-use (OS-level) actions are supported via an optional sidecar endpoint.
|
||||
|
||||
#[path = "action_parser.rs"]
|
||||
@@ -15,6 +16,7 @@ mod security;
|
||||
#[path = "types.rs"]
|
||||
mod types;
|
||||
|
||||
use super::playwright_backend;
|
||||
#[allow(unused_imports)]
|
||||
pub(super) use action_parser::{
|
||||
backend_name, is_computer_use_only_action, is_supported_browser_action, parse_browser_action,
|
||||
@@ -50,6 +52,7 @@ pub struct BrowserTool {
|
||||
native_webdriver_url: String,
|
||||
native_chrome_path: Option<String>,
|
||||
computer_use: ComputerUseConfig,
|
||||
playwright_state: tokio::sync::Mutex<playwright_backend::PlaywrightBrowserState>,
|
||||
#[cfg(feature = "browser-native")]
|
||||
native_state: tokio::sync::Mutex<native_backend::NativeBrowserState>,
|
||||
}
|
||||
@@ -92,6 +95,9 @@ impl BrowserTool {
|
||||
native_webdriver_url,
|
||||
native_chrome_path,
|
||||
computer_use,
|
||||
playwright_state: tokio::sync::Mutex::new(
|
||||
playwright_backend::PlaywrightBrowserState::default(),
|
||||
),
|
||||
#[cfg(feature = "browser-native")]
|
||||
native_state: tokio::sync::Mutex::new(native_backend::NativeBrowserState::default()),
|
||||
}
|
||||
@@ -114,6 +120,11 @@ impl BrowserTool {
|
||||
Self::is_agent_browser_available().await
|
||||
}
|
||||
|
||||
/// Check if the local Playwright package is available to Node.js.
|
||||
pub async fn is_playwright_available() -> bool {
|
||||
playwright_backend::PlaywrightBrowserState::is_available().await
|
||||
}
|
||||
|
||||
fn configured_backend(&self) -> anyhow::Result<BrowserBackendKind> {
|
||||
BrowserBackendKind::parse(&self.backend)
|
||||
}
|
||||
@@ -197,6 +208,15 @@ impl BrowserTool {
|
||||
)
|
||||
}
|
||||
}
|
||||
BrowserBackendKind::Playwright => {
|
||||
if Self::is_playwright_available().await {
|
||||
Ok(ResolvedBackend::Playwright)
|
||||
} else {
|
||||
anyhow::bail!(
|
||||
"browser.backend='playwright' but Playwright is unavailable. Ensure app/node_modules is installed, run `pnpm --filter openhuman-app exec playwright install chromium-headless-shell`, or set OPENHUMAN_PLAYWRIGHT_CWD."
|
||||
)
|
||||
}
|
||||
}
|
||||
BrowserBackendKind::RustNative => {
|
||||
if !Self::rust_native_compiled() {
|
||||
anyhow::bail!(
|
||||
@@ -222,6 +242,9 @@ impl BrowserTool {
|
||||
if Self::rust_native_compiled() && self.rust_native_available() {
|
||||
return Ok(ResolvedBackend::RustNative);
|
||||
}
|
||||
if Self::is_playwright_available().await {
|
||||
return Ok(ResolvedBackend::Playwright);
|
||||
}
|
||||
if Self::is_agent_browser_available().await {
|
||||
return Ok(ResolvedBackend::AgentBrowser);
|
||||
}
|
||||
@@ -235,22 +258,22 @@ impl BrowserTool {
|
||||
if Self::rust_native_compiled() {
|
||||
if let Some(err) = computer_use_err {
|
||||
anyhow::bail!(
|
||||
"browser.backend='auto' found no usable backend (agent-browser missing, rust-native unavailable, computer-use invalid: {err})"
|
||||
"browser.backend='auto' found no usable backend (playwright missing, agent-browser missing, rust-native unavailable, computer-use invalid: {err})"
|
||||
);
|
||||
}
|
||||
anyhow::bail!(
|
||||
"browser.backend='auto' found no usable backend (agent-browser missing, rust-native unavailable, computer-use sidecar unreachable)"
|
||||
"browser.backend='auto' found no usable backend (playwright missing, agent-browser missing, rust-native unavailable, computer-use sidecar unreachable)"
|
||||
)
|
||||
}
|
||||
|
||||
if let Some(err) = computer_use_err {
|
||||
anyhow::bail!(
|
||||
"browser.backend='auto' needs agent-browser tool, browser-native, or valid computer-use sidecar (error: {err})"
|
||||
"browser.backend='auto' needs Playwright, agent-browser tool, browser-native, or valid computer-use sidecar (error: {err})"
|
||||
);
|
||||
}
|
||||
|
||||
anyhow::bail!(
|
||||
"browser.backend='auto' needs agent-browser tool, browser-native, or computer-use sidecar"
|
||||
"browser.backend='auto' needs Playwright, agent-browser tool, browser-native, or computer-use sidecar"
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -515,6 +538,30 @@ impl BrowserTool {
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_playwright_action(&self, action: BrowserAction) -> anyhow::Result<ToolResult> {
|
||||
if let BrowserAction::Open { url } = &action {
|
||||
debug!("[browser] validating playwright open url before dispatch");
|
||||
self.validate_url(url)?;
|
||||
}
|
||||
|
||||
let mut state = self.playwright_state.lock().await;
|
||||
let output = state
|
||||
.execute_action(
|
||||
action,
|
||||
self.native_headless,
|
||||
Some(playwright_backend::BrowserUrlPolicy {
|
||||
allowed_domains: self.allowed_domains.clone(),
|
||||
allow_all: allow_all_browser_domains(),
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.context("Playwright browser action failed")?;
|
||||
|
||||
Ok(ToolResult::success(
|
||||
serde_json::to_string_pretty(&output).unwrap_or_default(),
|
||||
))
|
||||
}
|
||||
|
||||
fn validate_coordinate(&self, key: &str, value: i64, max: Option<i64>) -> anyhow::Result<()> {
|
||||
if value < 0 {
|
||||
anyhow::bail!("'{key}' must be >= 0")
|
||||
@@ -679,6 +726,7 @@ impl BrowserTool {
|
||||
) -> anyhow::Result<ToolResult> {
|
||||
match backend {
|
||||
ResolvedBackend::AgentBrowser => self.execute_agent_browser_action(action).await,
|
||||
ResolvedBackend::Playwright => self.execute_playwright_action(action).await,
|
||||
ResolvedBackend::RustNative => self.execute_rust_native_action(action).await,
|
||||
ResolvedBackend::ComputerUse => anyhow::bail!(
|
||||
"Internal error: computer_use backend must be handled before BrowserAction parsing"
|
||||
@@ -708,7 +756,7 @@ impl Tool for BrowserTool {
|
||||
|
||||
fn description(&self) -> &str {
|
||||
concat!(
|
||||
"Web/browser automation with pluggable backends (agent-browser, rust-native, computer_use). ",
|
||||
"Web/browser automation with pluggable backends (playwright, agent-browser, rust-native, computer_use). ",
|
||||
"Supports DOM actions plus optional OS-level actions (mouse_move, mouse_click, mouse_drag, ",
|
||||
"key_type, key_press, screen_capture) through a computer-use sidecar. Use 'snapshot' to map ",
|
||||
"interactive elements to refs (@e1, @e2). Enforces browser.allowed_domains for open actions."
|
||||
|
||||
@@ -128,6 +128,10 @@ fn browser_backend_parser_accepts_supported_values() {
|
||||
BrowserBackendKind::parse("agent_browser").unwrap(),
|
||||
BrowserBackendKind::AgentBrowser
|
||||
);
|
||||
assert_eq!(
|
||||
BrowserBackendKind::parse("playwright").unwrap(),
|
||||
BrowserBackendKind::Playwright
|
||||
);
|
||||
assert_eq!(
|
||||
BrowserBackendKind::parse("rust-native").unwrap(),
|
||||
BrowserBackendKind::RustNative
|
||||
@@ -144,11 +148,11 @@ fn browser_backend_parser_accepts_supported_values() {
|
||||
|
||||
#[test]
|
||||
fn browser_backend_parser_rejects_unknown_values() {
|
||||
assert!(BrowserBackendKind::parse("playwright").is_err());
|
||||
assert!(BrowserBackendKind::parse("netscape").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browser_tool_default_backend_is_agent_browser() {
|
||||
fn browser_tool_default_backend_is_agent_browser_for_legacy_constructor() {
|
||||
let security = Arc::new(SecurityPolicy::default());
|
||||
let tool = BrowserTool::new(security, vec!["example.com".into()], None);
|
||||
assert_eq!(
|
||||
@@ -173,6 +177,53 @@ fn browser_tool_accepts_auto_backend_config() {
|
||||
assert_eq!(tool.configured_backend().unwrap(), BrowserBackendKind::Auto);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browser_tool_accepts_playwright_backend_config() {
|
||||
let security = Arc::new(SecurityPolicy::default());
|
||||
let tool = BrowserTool::new_with_backend(
|
||||
security,
|
||||
vec!["example.com".into()],
|
||||
None,
|
||||
"playwright".into(),
|
||||
true,
|
||||
"http://127.0.0.1:9515".into(),
|
||||
None,
|
||||
ComputerUseConfig::default(),
|
||||
);
|
||||
assert_eq!(
|
||||
tool.configured_backend().unwrap(),
|
||||
BrowserBackendKind::Playwright
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn playwright_backend_validates_open_url_before_runner() {
|
||||
let security = Arc::new(SecurityPolicy::default());
|
||||
let tool = BrowserTool::new_with_backend(
|
||||
security,
|
||||
vec!["example.com".into()],
|
||||
None,
|
||||
"playwright".into(),
|
||||
true,
|
||||
"http://127.0.0.1:9515".into(),
|
||||
None,
|
||||
ComputerUseConfig::default(),
|
||||
);
|
||||
|
||||
let err = tool
|
||||
.execute_action(
|
||||
BrowserAction::Open {
|
||||
url: "file:///tmp/secret.txt".into(),
|
||||
},
|
||||
ResolvedBackend::Playwright,
|
||||
)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
|
||||
assert!(err.contains("file:// URLs are not allowed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browser_tool_accepts_computer_use_backend_config() {
|
||||
let security = Arc::new(SecurityPolicy::default());
|
||||
@@ -606,6 +657,7 @@ fn supported_action_detection_is_exhaustive() {
|
||||
#[test]
|
||||
fn browser_backend_kind_as_str_roundtrips() {
|
||||
assert_eq!(BrowserBackendKind::AgentBrowser.as_str(), "agent_browser");
|
||||
assert_eq!(BrowserBackendKind::Playwright.as_str(), "playwright");
|
||||
assert_eq!(BrowserBackendKind::RustNative.as_str(), "rust_native");
|
||||
assert_eq!(BrowserBackendKind::ComputerUse.as_str(), "computer_use");
|
||||
assert_eq!(BrowserBackendKind::Auto.as_str(), "auto");
|
||||
@@ -749,6 +801,7 @@ fn validate_coordinate_no_limit_allows_any_non_negative() {
|
||||
#[test]
|
||||
fn backend_name_covers_all_variants() {
|
||||
assert_eq!(backend_name(ResolvedBackend::AgentBrowser), "agent_browser");
|
||||
assert_eq!(backend_name(ResolvedBackend::Playwright), "playwright");
|
||||
assert_eq!(backend_name(ResolvedBackend::RustNative), "rust_native");
|
||||
assert_eq!(backend_name(ResolvedBackend::ComputerUse), "computer_use");
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ mod browser;
|
||||
mod browser_open;
|
||||
mod image_info;
|
||||
mod image_output;
|
||||
mod playwright_backend;
|
||||
mod screenshot;
|
||||
|
||||
pub use browser::{BrowserAction, BrowserTool, ComputerUseConfig};
|
||||
|
||||
@@ -0,0 +1,407 @@
|
||||
use super::BrowserAction;
|
||||
use anyhow::{Context, Result};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, Value};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::process::{Child, ChildStdin, ChildStdout, Command};
|
||||
use tokio::time::{timeout, Duration};
|
||||
|
||||
const RUNNER_JS: &str = include_str!("playwright_runner.mjs");
|
||||
const PLAYWRIGHT_PROBE_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
const PLAYWRIGHT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct PlaywrightBrowserState {
|
||||
daemon: Option<PlaywrightDaemon>,
|
||||
next_id: u64,
|
||||
}
|
||||
|
||||
struct PlaywrightDaemon {
|
||||
child: Child,
|
||||
stdin: ChildStdin,
|
||||
stdout: BufReader<ChildStdout>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct PlaywrightResponse {
|
||||
#[serde(default)]
|
||||
id: Option<u64>,
|
||||
success: bool,
|
||||
#[serde(default)]
|
||||
data: Option<Value>,
|
||||
#[serde(default)]
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct BrowserUrlPolicy {
|
||||
pub(crate) allowed_domains: Vec<String>,
|
||||
pub(crate) allow_all: bool,
|
||||
}
|
||||
|
||||
impl Drop for PlaywrightDaemon {
|
||||
fn drop(&mut self) {
|
||||
tracing::debug!("[browser::playwright] dropping backend daemon");
|
||||
let _ = self.child.start_kill();
|
||||
}
|
||||
}
|
||||
|
||||
impl PlaywrightBrowserState {
|
||||
pub async fn is_available() -> bool {
|
||||
tracing::debug!("[browser::playwright] probing playwright runtime availability");
|
||||
let mut command = node_command();
|
||||
command
|
||||
.args([
|
||||
"-e",
|
||||
"const load=()=>{try{return require('playwright')}catch(_){return require('@playwright/test')}};(async()=>{const {chromium}=load();const browser=await chromium.launch({headless:true});await browser.close();})().then(()=>process.exit(0)).catch(()=>process.exit(1));",
|
||||
])
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null());
|
||||
apply_node_cwd(&mut command);
|
||||
|
||||
match timeout(PLAYWRIGHT_PROBE_TIMEOUT, command.status()).await {
|
||||
Ok(Ok(status)) => {
|
||||
let available = status.success();
|
||||
tracing::debug!(
|
||||
available,
|
||||
status = ?status.code(),
|
||||
"[browser::playwright] runtime availability probe finished"
|
||||
);
|
||||
available
|
||||
}
|
||||
Ok(Err(error)) => {
|
||||
tracing::debug!(
|
||||
error = %error,
|
||||
"[browser::playwright] runtime availability probe failed to start"
|
||||
);
|
||||
false
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::debug!(
|
||||
timeout_ms = PLAYWRIGHT_PROBE_TIMEOUT.as_millis() as u64,
|
||||
"[browser::playwright] runtime availability probe timed out"
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn execute_action(
|
||||
&mut self,
|
||||
action: BrowserAction,
|
||||
headless: bool,
|
||||
url_policy: Option<BrowserUrlPolicy>,
|
||||
) -> Result<Value> {
|
||||
tracing::trace!("[browser::playwright] preparing action request");
|
||||
let args = action_to_args(action, url_policy);
|
||||
self.execute_args(args, headless).await
|
||||
}
|
||||
|
||||
async fn execute_args(&mut self, args: Value, headless: bool) -> Result<Value> {
|
||||
if self.daemon.is_none() {
|
||||
tracing::debug!("[browser::playwright] starting playwright backend daemon");
|
||||
self.daemon = Some(start_daemon(headless).await?);
|
||||
}
|
||||
|
||||
let id = self.next_id;
|
||||
self.next_id = self.next_id.saturating_add(1);
|
||||
let action = args
|
||||
.get("action")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("<unknown>")
|
||||
.to_string();
|
||||
|
||||
let request = json!({
|
||||
"id": id,
|
||||
"args": args,
|
||||
});
|
||||
let line = serde_json::to_vec(&request).context("Failed to encode Playwright request")?;
|
||||
tracing::debug!(
|
||||
request_id = id,
|
||||
action = %action,
|
||||
"[browser::playwright] dispatching action"
|
||||
);
|
||||
|
||||
let daemon = self.daemon.as_mut().expect("daemon just initialized");
|
||||
if let Err(error) = write_request(daemon, &line).await {
|
||||
tracing::debug!(
|
||||
error = %error,
|
||||
request_id = id,
|
||||
"[browser::playwright] daemon write failed; restarting once"
|
||||
);
|
||||
self.daemon = Some(start_daemon(headless).await?);
|
||||
let daemon = self.daemon.as_mut().expect("daemon restarted");
|
||||
if let Err(error) = write_request(daemon, &line).await {
|
||||
tracing::debug!(
|
||||
error = %error,
|
||||
request_id = id,
|
||||
"[browser::playwright] daemon write retry failed; dropping daemon"
|
||||
);
|
||||
self.daemon = None;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
|
||||
let daemon = self.daemon.as_mut().expect("daemon available");
|
||||
let response = match timeout(PLAYWRIGHT_RESPONSE_TIMEOUT, read_response(daemon)).await {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(error)) => {
|
||||
tracing::debug!(
|
||||
error = %error,
|
||||
request_id = id,
|
||||
"[browser::playwright] daemon read failed; dropping daemon"
|
||||
);
|
||||
self.daemon = None;
|
||||
return Err(error).context("Failed to read Playwright response");
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::debug!(
|
||||
request_id = id,
|
||||
timeout_ms = PLAYWRIGHT_RESPONSE_TIMEOUT.as_millis() as u64,
|
||||
"[browser::playwright] daemon response timed out; dropping daemon"
|
||||
);
|
||||
self.daemon = None;
|
||||
anyhow::bail!("Timed out waiting for Playwright response");
|
||||
}
|
||||
};
|
||||
|
||||
if response.id != Some(id) {
|
||||
tracing::debug!(
|
||||
expected_id = id,
|
||||
response_id = ?response.id,
|
||||
"[browser::playwright] daemon response id mismatch; dropping daemon"
|
||||
);
|
||||
self.daemon = None;
|
||||
anyhow::bail!("Playwright daemon response id mismatch");
|
||||
}
|
||||
|
||||
if response.success {
|
||||
tracing::debug!(
|
||||
request_id = id,
|
||||
action = %action,
|
||||
"[browser::playwright] action completed"
|
||||
);
|
||||
Ok(response.data.unwrap_or_else(|| json!({ "ok": true })))
|
||||
} else {
|
||||
tracing::debug!(
|
||||
request_id = id,
|
||||
action = %action,
|
||||
error = ?response.error,
|
||||
"[browser::playwright] action failed"
|
||||
);
|
||||
anyhow::bail!(
|
||||
"{}",
|
||||
response
|
||||
.error
|
||||
.unwrap_or_else(|| "Playwright backend failed".to_string())
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_request(daemon: &mut PlaywrightDaemon, line: &[u8]) -> Result<()> {
|
||||
tracing::trace!("[browser::playwright] writing request to daemon");
|
||||
daemon
|
||||
.stdin
|
||||
.write_all(line)
|
||||
.await
|
||||
.context("Failed to write Playwright request")?;
|
||||
daemon
|
||||
.stdin
|
||||
.write_all(b"\n")
|
||||
.await
|
||||
.context("Failed to terminate Playwright request")?;
|
||||
daemon
|
||||
.stdin
|
||||
.flush()
|
||||
.await
|
||||
.context("Failed to flush Playwright request")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_response(daemon: &mut PlaywrightDaemon) -> Result<PlaywrightResponse> {
|
||||
tracing::trace!("[browser::playwright] reading response from daemon");
|
||||
let mut line = String::new();
|
||||
let read = daemon
|
||||
.stdout
|
||||
.read_line(&mut line)
|
||||
.await
|
||||
.context("Failed to read Playwright stdout")?;
|
||||
if read == 0 {
|
||||
anyhow::bail!("Playwright daemon exited without a response");
|
||||
}
|
||||
tracing::trace!("[browser::playwright] received response from daemon");
|
||||
serde_json::from_str(&line).context("Playwright daemon returned invalid JSON")
|
||||
}
|
||||
|
||||
async fn start_daemon(headless: bool) -> Result<PlaywrightDaemon> {
|
||||
tracing::debug!(
|
||||
headless,
|
||||
"[browser::playwright] spawning playwright backend daemon"
|
||||
);
|
||||
let mut command = node_command();
|
||||
command
|
||||
.arg("-e")
|
||||
.arg(RUNNER_JS)
|
||||
.env(
|
||||
"OPENHUMAN_PLAYWRIGHT_HEADLESS",
|
||||
if headless { "1" } else { "0" },
|
||||
)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
apply_node_cwd(&mut command);
|
||||
|
||||
let mut child = command
|
||||
.spawn()
|
||||
.map_err(|error| {
|
||||
tracing::debug!(
|
||||
error = %error,
|
||||
"[browser::playwright] failed to spawn backend daemon"
|
||||
);
|
||||
error
|
||||
})
|
||||
.context(
|
||||
"Failed to start Playwright backend. Ensure Node.js and the Playwright package are installed.",
|
||||
)?;
|
||||
let stdin = child
|
||||
.stdin
|
||||
.take()
|
||||
.ok_or_else(|| anyhow::anyhow!("Playwright daemon stdin unavailable"))?;
|
||||
let stdout = child
|
||||
.stdout
|
||||
.take()
|
||||
.ok_or_else(|| anyhow::anyhow!("Playwright daemon stdout unavailable"))?;
|
||||
|
||||
if let Some(stderr) = child.stderr.take() {
|
||||
tokio::spawn(async move {
|
||||
let mut lines = BufReader::new(stderr).lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
tracing::debug!("[browser::playwright] stderr: {line}");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
tracing::debug!("[browser::playwright] backend daemon spawned");
|
||||
Ok(PlaywrightDaemon {
|
||||
child,
|
||||
stdin,
|
||||
stdout: BufReader::new(stdout),
|
||||
})
|
||||
}
|
||||
|
||||
fn node_command() -> Command {
|
||||
let binary = std::env::var("OPENHUMAN_PLAYWRIGHT_NODE").unwrap_or_else(|_| "node".to_string());
|
||||
Command::new(binary)
|
||||
}
|
||||
|
||||
fn apply_node_cwd(command: &mut Command) {
|
||||
if let Some(cwd) = playwright_node_cwd() {
|
||||
command.current_dir(cwd);
|
||||
}
|
||||
}
|
||||
|
||||
fn playwright_node_cwd() -> Option<PathBuf> {
|
||||
if let Ok(raw) = std::env::var("OPENHUMAN_PLAYWRIGHT_CWD") {
|
||||
let path = PathBuf::from(raw);
|
||||
if path.exists() {
|
||||
return Some(path);
|
||||
}
|
||||
}
|
||||
|
||||
let app = Path::new("app");
|
||||
if app.join("node_modules").exists() {
|
||||
return Some(app.to_path_buf());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn action_to_args(action: BrowserAction, url_policy: Option<BrowserUrlPolicy>) -> Value {
|
||||
match action {
|
||||
BrowserAction::Open { url } => {
|
||||
let policy = url_policy.expect("playwright open actions require a URL policy");
|
||||
json!({
|
||||
"action": "open",
|
||||
"url": url,
|
||||
"url_policy": {
|
||||
"allowed_domains": policy.allowed_domains,
|
||||
"allow_all": policy.allow_all,
|
||||
},
|
||||
})
|
||||
}
|
||||
BrowserAction::Snapshot {
|
||||
interactive_only,
|
||||
compact,
|
||||
depth,
|
||||
} => json!({
|
||||
"action": "snapshot",
|
||||
"interactive_only": interactive_only,
|
||||
"compact": compact,
|
||||
"depth": depth,
|
||||
}),
|
||||
BrowserAction::Click { selector } => json!({ "action": "click", "selector": selector }),
|
||||
BrowserAction::Fill { selector, value } => {
|
||||
json!({ "action": "fill", "selector": selector, "value": value })
|
||||
}
|
||||
BrowserAction::Type { selector, text } => {
|
||||
json!({ "action": "type", "selector": selector, "text": text })
|
||||
}
|
||||
BrowserAction::GetText { selector } => {
|
||||
json!({ "action": "get_text", "selector": selector })
|
||||
}
|
||||
BrowserAction::GetTitle => json!({ "action": "get_title" }),
|
||||
BrowserAction::GetUrl => json!({ "action": "get_url" }),
|
||||
BrowserAction::Screenshot { path, full_page } => {
|
||||
json!({ "action": "screenshot", "path": path, "full_page": full_page })
|
||||
}
|
||||
BrowserAction::Wait { selector, ms, text } => {
|
||||
json!({ "action": "wait", "selector": selector, "ms": ms, "text": text })
|
||||
}
|
||||
BrowserAction::Press { key } => json!({ "action": "press", "key": key }),
|
||||
BrowserAction::Hover { selector } => json!({ "action": "hover", "selector": selector }),
|
||||
BrowserAction::Scroll { direction, pixels } => {
|
||||
json!({ "action": "scroll", "direction": direction, "pixels": pixels })
|
||||
}
|
||||
BrowserAction::IsVisible { selector } => {
|
||||
json!({ "action": "is_visible", "selector": selector })
|
||||
}
|
||||
BrowserAction::Close => json!({ "action": "close" }),
|
||||
BrowserAction::Find {
|
||||
by,
|
||||
value,
|
||||
action,
|
||||
fill_value,
|
||||
} => json!({
|
||||
"action": "find",
|
||||
"by": by,
|
||||
"value": value,
|
||||
"find_action": action,
|
||||
"fill_value": fill_value,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn action_to_args_preserves_find_shape() {
|
||||
let args = action_to_args(
|
||||
BrowserAction::Find {
|
||||
by: "label".into(),
|
||||
value: "Email".into(),
|
||||
action: "fill".into(),
|
||||
fill_value: Some("a@example.com".into()),
|
||||
},
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(args["action"], "find");
|
||||
assert_eq!(args["find_action"], "fill");
|
||||
assert_eq!(args["fill_value"], "a@example.com");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,422 @@
|
||||
const dns = require('node:dns').promises;
|
||||
const fs = require('node:fs');
|
||||
const net = require('node:net');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const readline = require('node:readline');
|
||||
|
||||
let chromium;
|
||||
try {
|
||||
({ chromium } = require('playwright'));
|
||||
} catch (error) {
|
||||
({ chromium } = require('@playwright/test'));
|
||||
}
|
||||
|
||||
const userDataDir =
|
||||
process.env.OPENHUMAN_PLAYWRIGHT_USER_DATA_DIR ||
|
||||
path.join(os.tmpdir(), 'openhuman-playwright-browser');
|
||||
const headless = process.env.OPENHUMAN_PLAYWRIGHT_HEADLESS !== '0';
|
||||
|
||||
let context = null;
|
||||
let page = null;
|
||||
let refs = new Map();
|
||||
|
||||
function write(message) {
|
||||
process.stdout.write(`${JSON.stringify(message)}\n`);
|
||||
}
|
||||
|
||||
async function ensurePage() {
|
||||
if (!context) {
|
||||
fs.mkdirSync(userDataDir, { recursive: true });
|
||||
context = await chromium.launchPersistentContext(userDataDir, {
|
||||
headless,
|
||||
viewport: { width: 1365, height: 900 },
|
||||
});
|
||||
}
|
||||
|
||||
if (!page || page.isClosed()) {
|
||||
page = context.pages()[0] || (await context.newPage());
|
||||
}
|
||||
|
||||
return page;
|
||||
}
|
||||
|
||||
function cssPath(el) {
|
||||
if (!el || el.nodeType !== Node.ELEMENT_NODE) return '';
|
||||
const parts = [];
|
||||
let node = el;
|
||||
while (node && node.nodeType === Node.ELEMENT_NODE && node !== document.body) {
|
||||
const tag = node.tagName.toLowerCase();
|
||||
if (node.id) {
|
||||
parts.unshift(`${tag}#${CSS.escape(node.id)}`);
|
||||
break;
|
||||
}
|
||||
const parent = node.parentElement;
|
||||
if (!parent) break;
|
||||
const siblings = Array.from(parent.children).filter(child => child.tagName === node.tagName);
|
||||
const index = siblings.indexOf(node) + 1;
|
||||
parts.unshift(siblings.length > 1 ? `${tag}:nth-of-type(${index})` : tag);
|
||||
node = parent;
|
||||
}
|
||||
return parts.length ? parts.join(' > ') : 'body';
|
||||
}
|
||||
|
||||
async function snapshot(interactiveOnly, compact, depth) {
|
||||
return await page.evaluate(
|
||||
({ interactiveOnly, compact, depth }) => {
|
||||
const isInteractive = el => {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const role = el.getAttribute('role');
|
||||
return (
|
||||
['a', 'button', 'input', 'select', 'textarea', 'summary'].includes(tag) ||
|
||||
['button', 'link', 'checkbox', 'menuitem', 'option', 'radio', 'switch', 'tab'].includes(role || '') ||
|
||||
el.hasAttribute('onclick') ||
|
||||
el.tabIndex >= 0
|
||||
);
|
||||
};
|
||||
|
||||
const nameFor = el =>
|
||||
(el.getAttribute('aria-label') ||
|
||||
el.getAttribute('alt') ||
|
||||
el.getAttribute('title') ||
|
||||
el.getAttribute('placeholder') ||
|
||||
el.innerText ||
|
||||
el.textContent ||
|
||||
'')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.slice(0, 180);
|
||||
|
||||
const cssPath = el => {
|
||||
if (!el || el.nodeType !== Node.ELEMENT_NODE) return '';
|
||||
const parts = [];
|
||||
let node = el;
|
||||
while (node && node.nodeType === Node.ELEMENT_NODE && node !== document.body) {
|
||||
const tag = node.tagName.toLowerCase();
|
||||
if (node.id) {
|
||||
parts.unshift(`${tag}#${CSS.escape(node.id)}`);
|
||||
break;
|
||||
}
|
||||
const parent = node.parentElement;
|
||||
if (!parent) break;
|
||||
const siblings = Array.from(parent.children).filter(child => child.tagName === node.tagName);
|
||||
const index = siblings.indexOf(node) + 1;
|
||||
parts.unshift(siblings.length > 1 ? `${tag}:nth-of-type(${index})` : tag);
|
||||
node = parent;
|
||||
}
|
||||
return parts.length ? parts.join(' > ') : 'body';
|
||||
};
|
||||
|
||||
const elements = [];
|
||||
const walker = document.createTreeWalker(document.body || document.documentElement, NodeFilter.SHOW_ELEMENT);
|
||||
let node = walker.currentNode;
|
||||
let id = 1;
|
||||
while (node) {
|
||||
const rect = node.getBoundingClientRect();
|
||||
const visible = rect.width > 0 && rect.height > 0;
|
||||
const interactive = isInteractive(node);
|
||||
const name = nameFor(node);
|
||||
if (visible && (!interactiveOnly || interactive) && (!compact || name || interactive)) {
|
||||
elements.push({
|
||||
ref: `@e${id++}`,
|
||||
tag: node.tagName.toLowerCase(),
|
||||
role: node.getAttribute('role') || undefined,
|
||||
name,
|
||||
selector: cssPath(node),
|
||||
});
|
||||
}
|
||||
if (depth && elements.length >= depth * 80) break;
|
||||
node = walker.nextNode();
|
||||
}
|
||||
return {
|
||||
title: document.title,
|
||||
url: location.href,
|
||||
elements,
|
||||
};
|
||||
},
|
||||
{ interactiveOnly, compact, depth },
|
||||
);
|
||||
}
|
||||
|
||||
async function selectorFor(input) {
|
||||
if (typeof input === 'string' && input.startsWith('@')) {
|
||||
const selector = refs.get(input);
|
||||
if (!selector) throw new Error(`Unknown element ref: ${input}. Run snapshot first.`);
|
||||
return selector;
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
async function runFind(args) {
|
||||
const current = await ensurePage();
|
||||
const value = args.value || '';
|
||||
let locator;
|
||||
switch (args.by) {
|
||||
case 'role':
|
||||
locator = current.getByRole(value);
|
||||
break;
|
||||
case 'text':
|
||||
locator = current.getByText(value);
|
||||
break;
|
||||
case 'label':
|
||||
locator = current.getByLabel(value);
|
||||
break;
|
||||
case 'placeholder':
|
||||
locator = current.getByPlaceholder(value);
|
||||
break;
|
||||
case 'testid':
|
||||
locator = current.getByTestId(value);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unsupported find locator: ${args.by}`);
|
||||
}
|
||||
|
||||
const first = locator.first();
|
||||
switch (args.find_action) {
|
||||
case 'click':
|
||||
await first.click();
|
||||
return { action: 'find', by: args.by, find_action: 'click' };
|
||||
case 'fill':
|
||||
await first.fill(args.fill_value || '');
|
||||
return { action: 'find', by: args.by, find_action: 'fill' };
|
||||
case 'hover':
|
||||
await first.hover();
|
||||
return { action: 'find', by: args.by, find_action: 'hover' };
|
||||
case 'text':
|
||||
return { action: 'find', by: args.by, find_action: 'text', text: await first.innerText() };
|
||||
case 'check':
|
||||
await first.check();
|
||||
return { action: 'find', by: args.by, find_action: 'check' };
|
||||
default:
|
||||
throw new Error(`Unsupported find action: ${args.find_action}`);
|
||||
}
|
||||
}
|
||||
|
||||
function ipv4Parts(host) {
|
||||
const parts = host.split('.');
|
||||
if (parts.length !== 4) return null;
|
||||
const nums = parts.map(part => Number(part));
|
||||
if (nums.some(num => !Number.isInteger(num) || num < 0 || num > 255)) return null;
|
||||
return nums;
|
||||
}
|
||||
|
||||
function isPrivateIpv4(host) {
|
||||
const parts = ipv4Parts(host);
|
||||
if (!parts) return false;
|
||||
const [a, b] = parts;
|
||||
return (
|
||||
a === 0 ||
|
||||
a === 10 ||
|
||||
a === 127 ||
|
||||
(a === 169 && b === 254) ||
|
||||
(a === 172 && b >= 16 && b <= 31) ||
|
||||
(a === 192 && b === 168) ||
|
||||
(a === 100 && b >= 64 && b <= 127) ||
|
||||
(a === 192 && b === 0 && parts[2] === 0) ||
|
||||
(a === 192 && b === 0 && parts[2] === 2) ||
|
||||
(a === 198 && (b === 18 || b === 19)) ||
|
||||
(a === 198 && b === 51 && parts[2] === 100) ||
|
||||
(a === 203 && b === 0 && parts[2] === 113) ||
|
||||
a >= 224
|
||||
);
|
||||
}
|
||||
|
||||
function mappedIpv4FromIpv6Tail(tail) {
|
||||
if (tail.includes('.')) return tail;
|
||||
const groups = tail.split(':').filter(Boolean);
|
||||
if (groups.length < 2) return null;
|
||||
const high = Number.parseInt(groups[groups.length - 2], 16);
|
||||
const low = Number.parseInt(groups[groups.length - 1], 16);
|
||||
if (!Number.isInteger(high) || !Number.isInteger(low) || high < 0 || high > 0xffff || low < 0 || low > 0xffff) {
|
||||
return null;
|
||||
}
|
||||
return `${(high >> 8) & 0xff}.${high & 0xff}.${(low >> 8) & 0xff}.${low & 0xff}`;
|
||||
}
|
||||
|
||||
function isPrivateIpv6(host) {
|
||||
const normalized = host.toLowerCase().replace(/^\[|\]$/g, '');
|
||||
if (normalized === '::1' || normalized === '::') return true;
|
||||
if (normalized.startsWith('::ffff:')) {
|
||||
const mapped = mappedIpv4FromIpv6Tail(normalized.slice('::ffff:'.length));
|
||||
return !mapped || isPrivateIpv4(mapped);
|
||||
}
|
||||
return (
|
||||
normalized.startsWith('fc') ||
|
||||
normalized.startsWith('fd') ||
|
||||
normalized.startsWith('fe8') ||
|
||||
normalized.startsWith('fe9') ||
|
||||
normalized.startsWith('fea') ||
|
||||
normalized.startsWith('feb')
|
||||
);
|
||||
}
|
||||
|
||||
function isPrivateHost(host) {
|
||||
const normalized = String(host || '').toLowerCase().replace(/^\[|\]$/g, '');
|
||||
if (!normalized) return true;
|
||||
if (normalized === 'localhost' || normalized.endsWith('.localhost') || normalized.endsWith('.local')) return true;
|
||||
const ipVersion = net.isIP(normalized);
|
||||
if (ipVersion === 4) return isPrivateIpv4(normalized);
|
||||
if (ipVersion === 6) return isPrivateIpv6(normalized);
|
||||
return false;
|
||||
}
|
||||
|
||||
async function assertDnsDoesNotResolvePrivate(host) {
|
||||
if (net.isIP(host)) return;
|
||||
let addresses;
|
||||
try {
|
||||
addresses = await dns.lookup(host, { all: true, verbatim: true });
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to resolve host '${host}': ${error.message}`);
|
||||
}
|
||||
for (const entry of addresses) {
|
||||
if (isPrivateHost(entry.address)) {
|
||||
throw new Error(`Host '${host}' resolves to blocked local/private address: ${entry.address}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function hostMatchesAllowlist(host, allowedDomains) {
|
||||
const normalizedHost = String(host || '').toLowerCase();
|
||||
return allowedDomains.some(domain => {
|
||||
const allowed = String(domain || '').trim().toLowerCase();
|
||||
if (!allowed) return false;
|
||||
if (allowed === '*') return true;
|
||||
if (allowed.startsWith('*.')) {
|
||||
const suffix = allowed.slice(2);
|
||||
return normalizedHost === suffix || normalizedHost.endsWith(`.${suffix}`);
|
||||
}
|
||||
return normalizedHost === allowed || normalizedHost.endsWith(`.${allowed}`);
|
||||
});
|
||||
}
|
||||
|
||||
async function authorizeUrl(rawUrl, policy) {
|
||||
if (!policy) throw new Error('Missing browser URL policy');
|
||||
const trimmed = String(rawUrl || '').trim();
|
||||
if (!trimmed) throw new Error('URL cannot be empty');
|
||||
if (trimmed.startsWith('file://')) throw new Error('file:// URLs are not allowed in browser automation');
|
||||
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(trimmed);
|
||||
} catch (error) {
|
||||
throw new Error(`Invalid URL: ${error.message}`);
|
||||
}
|
||||
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
||||
throw new Error('Only http:// and https:// URLs are allowed');
|
||||
}
|
||||
|
||||
const host = parsed.hostname.toLowerCase();
|
||||
if (isPrivateHost(host)) throw new Error(`Blocked local/private host: ${host}`);
|
||||
await assertDnsDoesNotResolvePrivate(host);
|
||||
|
||||
const allowedDomains = Array.isArray(policy.allowed_domains) ? policy.allowed_domains : [];
|
||||
if (allowedDomains.length === 0 && !policy.allow_all) {
|
||||
throw new Error('Browser tool enabled but no allowed_domains configured');
|
||||
}
|
||||
if (!policy.allow_all && allowedDomains.length > 0 && !hostMatchesAllowlist(host, allowedDomains)) {
|
||||
throw new Error(`Host '${host}' not in browser.allowed_domains`);
|
||||
}
|
||||
}
|
||||
|
||||
async function run(args) {
|
||||
const current = await ensurePage();
|
||||
switch (args.action) {
|
||||
case 'open':
|
||||
await authorizeUrl(args.url, args.url_policy);
|
||||
await current.goto(args.url, { waitUntil: 'domcontentloaded' });
|
||||
try {
|
||||
await authorizeUrl(current.url(), args.url_policy);
|
||||
} catch (error) {
|
||||
await current.goto('about:blank').catch(() => {});
|
||||
throw new Error(`Redirect blocked: ${error.message}`);
|
||||
}
|
||||
return { backend: 'playwright', action: 'open', url: current.url(), title: await current.title() };
|
||||
case 'snapshot': {
|
||||
const data = await snapshot(args.interactive_only ?? true, args.compact ?? true, args.depth);
|
||||
refs = new Map(data.elements.map(element => [element.ref, element.selector]));
|
||||
return { backend: 'playwright', action: 'snapshot', ...data };
|
||||
}
|
||||
case 'click':
|
||||
await current.locator(await selectorFor(args.selector)).first().click();
|
||||
return { backend: 'playwright', action: 'click', selector: args.selector };
|
||||
case 'fill':
|
||||
await current.locator(await selectorFor(args.selector)).first().fill(args.value || '');
|
||||
return { backend: 'playwright', action: 'fill', selector: args.selector };
|
||||
case 'type':
|
||||
await current.locator(await selectorFor(args.selector)).first().type(args.text || '');
|
||||
return { backend: 'playwright', action: 'type', selector: args.selector, typed: (args.text || '').length };
|
||||
case 'get_text':
|
||||
return {
|
||||
backend: 'playwright',
|
||||
action: 'get_text',
|
||||
selector: args.selector,
|
||||
text: await current.locator(await selectorFor(args.selector)).first().innerText(),
|
||||
};
|
||||
case 'get_title':
|
||||
return { backend: 'playwright', action: 'get_title', title: await current.title() };
|
||||
case 'get_url':
|
||||
return { backend: 'playwright', action: 'get_url', url: current.url() };
|
||||
case 'screenshot': {
|
||||
const png = await current.screenshot({ fullPage: Boolean(args.full_page) });
|
||||
if (args.path) {
|
||||
throw new Error('Playwright screenshot path writes require Rust-side path validation and are disabled');
|
||||
}
|
||||
return { backend: 'playwright', action: 'screenshot', png_base64: png.toString('base64'), bytes: png.length };
|
||||
}
|
||||
case 'wait':
|
||||
if (args.selector) await current.locator(await selectorFor(args.selector)).first().waitFor();
|
||||
else if (args.text) await current.getByText(args.text).first().waitFor();
|
||||
else await current.waitForTimeout(args.ms || 1000);
|
||||
return { backend: 'playwright', action: 'wait' };
|
||||
case 'press':
|
||||
await current.keyboard.press(args.key);
|
||||
return { backend: 'playwright', action: 'press', key: args.key };
|
||||
case 'hover':
|
||||
await current.locator(await selectorFor(args.selector)).first().hover();
|
||||
return { backend: 'playwright', action: 'hover', selector: args.selector };
|
||||
case 'scroll':
|
||||
await current.mouse.wheel(
|
||||
args.direction === 'left' ? -(args.pixels || 600) : args.direction === 'right' ? args.pixels || 600 : 0,
|
||||
args.direction === 'up' ? -(args.pixels || 600) : args.direction === 'down' ? args.pixels || 600 : 0,
|
||||
);
|
||||
return { backend: 'playwright', action: 'scroll', direction: args.direction };
|
||||
case 'is_visible':
|
||||
return {
|
||||
backend: 'playwright',
|
||||
action: 'is_visible',
|
||||
selector: args.selector,
|
||||
visible: await current.locator(await selectorFor(args.selector)).first().isVisible(),
|
||||
};
|
||||
case 'find':
|
||||
return { backend: 'playwright', ...(await runFind(args)) };
|
||||
case 'close':
|
||||
if (context) await context.close();
|
||||
context = null;
|
||||
page = null;
|
||||
refs = new Map();
|
||||
return { backend: 'playwright', action: 'close' };
|
||||
default:
|
||||
throw new Error(`Unsupported action: ${args.action}`);
|
||||
}
|
||||
}
|
||||
|
||||
const rl = readline.createInterface({ input: process.stdin });
|
||||
rl.on('line', async line => {
|
||||
let request;
|
||||
try {
|
||||
request = JSON.parse(line);
|
||||
const data = await run(request.args || {});
|
||||
write({ id: request.id, success: true, data });
|
||||
} catch (error) {
|
||||
write({ id: request && request.id, success: false, error: error && error.message ? error.message : String(error) });
|
||||
}
|
||||
});
|
||||
|
||||
process.on('SIGTERM', async () => {
|
||||
try {
|
||||
if (context) await context.close();
|
||||
} finally {
|
||||
process.exit(0);
|
||||
}
|
||||
});
|
||||
@@ -44,6 +44,7 @@ impl Default for ComputerUseConfig {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum BrowserBackendKind {
|
||||
AgentBrowser,
|
||||
Playwright,
|
||||
RustNative,
|
||||
ComputerUse,
|
||||
Auto,
|
||||
@@ -52,6 +53,7 @@ pub(crate) enum BrowserBackendKind {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum ResolvedBackend {
|
||||
AgentBrowser,
|
||||
Playwright,
|
||||
RustNative,
|
||||
ComputerUse,
|
||||
}
|
||||
@@ -61,11 +63,12 @@ impl BrowserBackendKind {
|
||||
let key = raw.trim().to_ascii_lowercase().replace('-', "_");
|
||||
match key.as_str() {
|
||||
"agent_browser" | "agentbrowser" => Ok(Self::AgentBrowser),
|
||||
"playwright" => Ok(Self::Playwright),
|
||||
"rust_native" | "native" => Ok(Self::RustNative),
|
||||
"computer_use" | "computeruse" => Ok(Self::ComputerUse),
|
||||
"auto" => Ok(Self::Auto),
|
||||
_ => anyhow::bail!(
|
||||
"Unsupported browser backend '{raw}'. Use 'agent_browser', 'rust_native', 'computer_use', or 'auto'"
|
||||
"Unsupported browser backend '{raw}'. Use 'agent_browser', 'playwright', 'rust_native', 'computer_use', or 'auto'"
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -73,6 +76,7 @@ impl BrowserBackendKind {
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::AgentBrowser => "agent_browser",
|
||||
Self::Playwright => "playwright",
|
||||
Self::RustNative => "rust_native",
|
||||
Self::ComputerUse => "computer_use",
|
||||
Self::Auto => "auto",
|
||||
|
||||
@@ -101,6 +101,11 @@
|
||||
//! Assertion is implicit: cargo reports the test as failed when the
|
||||
//! tokio runtime aborts with stack overflow.
|
||||
|
||||
// This stress target is still compiled by normal cargo test. Under llvm-cov,
|
||||
// coverage instrumentation makes the already-large integration-test binary
|
||||
// trip CI's linker with SIGBUS before the regression can run.
|
||||
#![cfg(not(coverage))]
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use openhuman_core::openhuman::agent::harness::definition::{AgentDefinitionRegistry, ModelSpec};
|
||||
|
||||
@@ -39,7 +39,7 @@ use openhuman_core::openhuman::agent::harness::definition::{
|
||||
};
|
||||
use openhuman_core::openhuman::agent::harness::subagent_runner::{
|
||||
autonomous_iter_cap, with_autonomous_iter_cap, SubagentMode, SubagentRunError,
|
||||
SubagentRunOptions, SubagentRunOutcome, SubagentRunStatus,
|
||||
SubagentRunOptions, SubagentRunOutcome, SubagentRunStatus, SubagentUsage,
|
||||
};
|
||||
use openhuman_core::openhuman::agent::harness::{
|
||||
check_interrupt, current_sandbox_mode, with_current_sandbox_mode, InterruptFence,
|
||||
@@ -4704,7 +4704,7 @@ async fn agent_subagent_public_types_cover_task_local_and_error_display_paths()
|
||||
mode: SubagentMode::Typed,
|
||||
status: SubagentRunStatus::Completed,
|
||||
final_history: Vec::new(),
|
||||
usage: Default::default(),
|
||||
usage: SubagentUsage::default(),
|
||||
};
|
||||
assert_eq!(outcome.mode.as_str(), "typed");
|
||||
assert_eq!(outcome.elapsed.as_millis(), 12);
|
||||
|
||||
@@ -981,6 +981,76 @@ fn ensure_test_rpc_auth() {
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn json_rpc_config_update_browser_settings_persists_backend() {
|
||||
let _env_lock = json_rpc_e2e_env_lock();
|
||||
let tmp = tempdir().expect("tempdir");
|
||||
let home = tmp.path();
|
||||
let openhuman_home = home.join(".openhuman");
|
||||
|
||||
let _home_guard = EnvVarGuard::set_to_path("HOME", home);
|
||||
let _workspace_guard = EnvVarGuard::unset("OPENHUMAN_WORKSPACE");
|
||||
let _backend_url_guard = EnvVarGuard::unset("BACKEND_URL");
|
||||
let _vite_backend_guard = EnvVarGuard::unset("VITE_BACKEND_URL");
|
||||
|
||||
write_min_config(&openhuman_home, "http://127.0.0.1:9");
|
||||
|
||||
let (rpc_addr, rpc_join) = serve_on_ephemeral(build_core_http_router(false)).await;
|
||||
let rpc_base = format!("http://{rpc_addr}");
|
||||
|
||||
let updated = post_json_rpc(
|
||||
&rpc_base,
|
||||
4124_1,
|
||||
"openhuman.config_update_browser_settings",
|
||||
json!({
|
||||
"enabled": true,
|
||||
"backend": "playwright"
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let updated_result =
|
||||
assert_no_jsonrpc_error(&updated, "config_update_browser_settings backend");
|
||||
let updated_snapshot = peel_logs_envelope(updated_result);
|
||||
assert_eq!(
|
||||
updated_snapshot
|
||||
.pointer("/config/browser/enabled")
|
||||
.and_then(Value::as_bool),
|
||||
Some(true),
|
||||
"browser enabled flag should persist in update response: {updated_snapshot}"
|
||||
);
|
||||
assert_eq!(
|
||||
updated_snapshot
|
||||
.pointer("/config/browser/backend")
|
||||
.and_then(Value::as_str),
|
||||
Some("playwright"),
|
||||
"browser backend should persist in update response: {updated_snapshot}"
|
||||
);
|
||||
|
||||
let get = post_json_rpc(&rpc_base, 4124_2, "openhuman.config_get", json!({})).await;
|
||||
let get_result = assert_no_jsonrpc_error(&get, "config_get");
|
||||
let snapshot = peel_logs_envelope(get_result);
|
||||
assert_eq!(
|
||||
snapshot
|
||||
.pointer("/config/browser/backend")
|
||||
.and_then(Value::as_str),
|
||||
Some("playwright"),
|
||||
"browser backend should persist in config_get response: {snapshot}"
|
||||
);
|
||||
|
||||
let invalid = post_json_rpc(
|
||||
&rpc_base,
|
||||
4124_3,
|
||||
"openhuman.config_update_browser_settings",
|
||||
json!({
|
||||
"backend": "netscape"
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_jsonrpc_error(&invalid, "invalid browser backend");
|
||||
|
||||
rpc_join.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn json_rpc_tokenjuice_detect_and_cache_stats() {
|
||||
let _env_lock = json_rpc_e2e_env_lock();
|
||||
|
||||
Reference in New Issue
Block a user