From c122e1ddd77a138482995ee1749b06c670bc2565 Mon Sep 17 00:00:00 2001 From: NextDoorLaoHuang-HF <57834279+NextDoorLaoHuang-HF@users.noreply.github.com> Date: Sun, 15 Mar 2026 22:40:09 +0800 Subject: [PATCH] test merge * Improve OpenClaw provider alias migration compatibility * Fix local provider env mapping regression in migration * test(migrate): cover json5 default_model provider/env mapping * test(migrate): add JSON5 agent provider mapping integration tests * test(migrate): add legacy YAML provider alias integration coverage * fix(migrate): harden JSON5 provider catalog resolution * chore(migrate): scope split_model_ref helper to tests --------- Co-authored-by: root --- crates/openfang-migrate/src/openclaw.rs | 380 +++++++++++++++++- .../tests/provider_json5_agents.rs | 137 +++++++ .../tests/provider_json5_default_model.rs | 64 +++ .../tests/provider_json5_provider_catalog.rs | 277 +++++++++++++ .../tests/provider_legacy_yaml.rs | 126 ++++++ 5 files changed, 962 insertions(+), 22 deletions(-) create mode 100644 crates/openfang-migrate/tests/provider_json5_agents.rs create mode 100644 crates/openfang-migrate/tests/provider_json5_default_model.rs create mode 100644 crates/openfang-migrate/tests/provider_json5_provider_catalog.rs create mode 100644 crates/openfang-migrate/tests/provider_legacy_yaml.rs diff --git a/crates/openfang-migrate/src/openclaw.rs b/crates/openfang-migrate/src/openclaw.rs index bf9ccd73..891186e1 100644 --- a/crates/openfang-migrate/src/openclaw.rs +++ b/crates/openfang-migrate/src/openclaw.rs @@ -549,15 +549,43 @@ fn build_channel_table( toml::Value::Table(table) } +#[derive(Debug, Clone)] +struct ResolvedModelRef { + provider: String, + model: String, + base_url: Option, +} + /// Split an OpenClaw model reference like `"provider/model"` into `(provider, model)`. /// If there's no slash, returns `("anthropic", input)` as a fallback. +#[cfg(test)] fn split_model_ref(model_ref: &str) -> (String, String) { + let resolved = split_model_ref_with_context(model_ref, None); + (resolved.provider, resolved.model) +} + +/// Split a model ref and resolve provider/base URL using optional OpenClaw +/// `models.providers` metadata for higher-fidelity migration. +fn split_model_ref_with_context( + model_ref: &str, + provider_catalog: Option<&serde_json::Map>, +) -> ResolvedModelRef { if let Some(pos) = model_ref.find('/') { - let provider = &model_ref[..pos]; + let raw_provider = &model_ref[..pos]; let model = &model_ref[pos + 1..]; - (map_provider(provider), model.to_string()) + let (provider, base_url) = + resolve_provider_with_models_context(raw_provider, provider_catalog); + ResolvedModelRef { + provider, + model: model.to_string(), + base_url, + } } else { - ("anthropic".to_string(), model_ref.to_string()) + ResolvedModelRef { + provider: "anthropic".to_string(), + model: model_ref.to_string(), + base_url: None, + } } } @@ -663,10 +691,27 @@ fn map_provider(openclaw_provider: &str) -> String { "together" => "together".to_string(), "mistral" => "mistral".to_string(), "fireworks" => "fireworks".to_string(), + // Gemini aliases "google" | "gemini" => "google".to_string(), + // Chinese provider aliases (including common OpenClaw custom IDs) + "qwen" | "dashscope" | "qwencode" => "qwen".to_string(), + "moonshot" | "kimi" | "kimicode" => "moonshot".to_string(), + "minimax" => "minimax".to_string(), + "zhipu" | "glm" => "zhipu".to_string(), + "zhipu_coding" | "codegeex" => "zhipu_coding".to_string(), + "qianfan" | "baidu" => "qianfan".to_string(), "xai" | "grok" => "xai".to_string(), "cerebras" => "cerebras".to_string(), "sambanova" => "sambanova".to_string(), + // Additional OpenFang-supported providers and aliases + "perplexity" => "perplexity".to_string(), + "cohere" => "cohere".to_string(), + "ai21" => "ai21".to_string(), + "huggingface" => "huggingface".to_string(), + "replicate" => "replicate".to_string(), + "github-copilot" | "copilot" => "github-copilot".to_string(), + "vllm" => "vllm".to_string(), + "lmstudio" => "lmstudio".to_string(), other => other.to_string(), } } @@ -683,14 +728,156 @@ fn default_api_key_env(provider: &str) -> String { "mistral" => "MISTRAL_API_KEY".to_string(), "fireworks" => "FIREWORKS_API_KEY".to_string(), "google" => "GOOGLE_API_KEY".to_string(), + "qwen" => "DASHSCOPE_API_KEY".to_string(), + "moonshot" => "MOONSHOT_API_KEY".to_string(), + "minimax" => "MINIMAX_API_KEY".to_string(), + "zhipu" | "zhipu_coding" => "ZHIPU_API_KEY".to_string(), + "qianfan" => "QIANFAN_API_KEY".to_string(), "xai" => "XAI_API_KEY".to_string(), "cerebras" => "CEREBRAS_API_KEY".to_string(), "sambanova" => "SAMBANOVA_API_KEY".to_string(), + "perplexity" => "PERPLEXITY_API_KEY".to_string(), + "cohere" => "COHERE_API_KEY".to_string(), + "ai21" => "AI21_API_KEY".to_string(), + "huggingface" => "HF_API_KEY".to_string(), + "replicate" => "REPLICATE_API_TOKEN".to_string(), + "github-copilot" => "GITHUB_TOKEN".to_string(), "ollama" => String::new(), // Ollama doesn't need an API key + // Keep explicit env names for local OpenAI-compatible providers to avoid + // falling back to default_model.api_key_env in kernel driver resolution. + "vllm" => "VLLM_API_KEY".to_string(), + "lmstudio" => "LMSTUDIO_API_KEY".to_string(), _ => format!("{}_API_KEY", provider.to_uppercase()), } } +fn is_known_openfang_provider_id(provider: &str) -> bool { + matches!( + provider, + "anthropic" + | "claude" + | "openai" + | "gpt" + | "groq" + | "grok" + | "openrouter" + | "deepseek" + | "together" + | "mistral" + | "fireworks" + | "google" + | "gemini" + | "ollama" + | "vllm" + | "lmstudio" + | "perplexity" + | "cohere" + | "ai21" + | "cerebras" + | "sambanova" + | "huggingface" + | "xai" + | "replicate" + | "github-copilot" + | "copilot" + | "moonshot" + | "kimi" + | "qwen" + | "dashscope" + | "minimax" + | "zhipu" + | "glm" + | "zhipu_coding" + | "codegeex" + | "qianfan" + | "baidu" + ) +} + +fn json_get_string_case_insensitive( + obj: &serde_json::Map, + key: &str, +) -> Option { + obj.iter() + .find(|(k, _)| k.eq_ignore_ascii_case(key)) + .and_then(|(_, v)| v.as_str().map(|s| s.to_string())) +} + +fn lookup_json5_provider_entry<'a>( + catalog: Option<&'a serde_json::Map>, + provider: &str, +) -> Option<&'a serde_json::Map> { + let catalog = catalog?; + if let Some(exact) = catalog + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(provider)) + .and_then(|(_, v)| v.as_object()) + { + return Some(exact); + } + + // Alias/canonical fallback: allow model refs that use aliases (e.g. `gpt`) + // while provider catalog keys use canonical IDs (e.g. `openai`), and vice versa. + let canonical = map_provider(&provider.to_lowercase()); + catalog + .iter() + .find(|(k, _)| map_provider(&k.to_lowercase()) == canonical) + .and_then(|(_, v)| v.as_object()) +} + +fn resolve_provider_with_models_context( + openclaw_provider: &str, + provider_catalog: Option<&serde_json::Map>, +) -> (String, Option) { + let raw = openclaw_provider.to_lowercase(); + let mapped = map_provider(&raw); + let Some(entry) = lookup_json5_provider_entry(provider_catalog, openclaw_provider) else { + return (mapped, None); + }; + + let base_url = json_get_string_case_insensitive(entry, "baseUrl") + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()); + let api_hint = json_get_string_case_insensitive(entry, "api") + .unwrap_or_default() + .to_lowercase(); + + let raw_is_known = is_known_openfang_provider_id(&raw); + let mapped_is_known = is_known_openfang_provider_id(&mapped); + + let provider = if api_hint.contains("anthropic") { + "anthropic".to_string() + } else if api_hint.contains("gemini") || api_hint.contains("google") { + "google".to_string() + } else if api_hint.contains("openai") { + if raw_is_known { + mapped.clone() + } else if base_url.is_some() { + // Preserve custom IDs only when runtime can route them via base_url. + raw.clone() + } else if mapped_is_known { + mapped.clone() + } else { + // Unknown custom ID without base_url would fail at runtime; default to + // OpenAI-compatible driver. + "openai".to_string() + } + } else if raw_is_known { + mapped.clone() + } else if base_url.is_some() { + // Unknown API type + explicit custom provider metadata: + // keep custom provider id and rely on base_url for runtime routing. + raw.clone() + } else if mapped_is_known { + mapped.clone() + } else { + // No API hint and no base URL; choose a known OpenAI-compatible driver. + "openai".to_string() + }; + + (provider, base_url) +} + /// Derive capability grants from the tool list. fn derive_capabilities(tools: &[String]) -> AgentCapabilities { let mut caps = AgentCapabilities::default(); @@ -816,15 +1003,21 @@ fn scan_from_json5(base: &Path, config_path: &Path, result: &mut ScanResult) { Err(_) => return, }; + let provider_catalog = root.models.as_ref().and_then(|m| m.providers.as_ref()); + // Agents from JSON config if let Some(ref agents) = root.agents { for entry in &agents.list { let id = entry.id.clone(); let name = entry.name.clone().unwrap_or_else(|| id.clone()); - let (provider, model) = extract_primary_model(entry, agents.defaults.as_ref()) - .map(|m| split_model_ref(&m)) - .unwrap_or_else(|| ("anthropic".to_string(), String::new())); + let resolved = extract_primary_model(entry, agents.defaults.as_ref()) + .map(|m| split_model_ref_with_context(&m, provider_catalog)) + .unwrap_or(ResolvedModelRef { + provider: "anthropic".to_string(), + model: String::new(), + base_url: None, + }); let tool_count = entry .tools @@ -853,8 +1046,8 @@ fn scan_from_json5(base: &Path, config_path: &Path, result: &mut ScanResult) { result.agents.push(ScannedAgent { name, description: String::new(), - provider, - model, + provider: resolved.provider, + model: resolved.model, tool_count, has_memory, has_sessions, @@ -1159,8 +1352,10 @@ fn migrate_config_from_json( dry_run: bool, report: &mut MigrationReport, ) -> Result<(), MigrateError> { + let provider_catalog = root.models.as_ref().and_then(|m| m.providers.as_ref()); + // Extract default model from agents.defaults.model - let (provider, model) = root + let resolved = root .agents .as_ref() .and_then(|a| a.defaults.as_ref()) @@ -1169,25 +1364,24 @@ fn migrate_config_from_json( OpenClawAgentModel::Simple(s) => Some(s.clone()), OpenClawAgentModel::Detailed(d) => d.primary.clone(), }) - .map(|m| split_model_ref(&m)) - .unwrap_or_else(|| { - ( - "anthropic".to_string(), - "claude-sonnet-4-20250514".to_string(), - ) + .map(|m| split_model_ref_with_context(&m, provider_catalog)) + .unwrap_or_else(|| ResolvedModelRef { + provider: "anthropic".to_string(), + model: "claude-sonnet-4-20250514".to_string(), + base_url: None, }); - let api_key_env = default_api_key_env(&provider); + let api_key_env = default_api_key_env(&resolved.provider); // Extract channels (writes secrets.env) let channels = migrate_channels_from_json(root, target, dry_run, report); let of_config = OpenFangConfig { default_model: OpenFangModelConfig { - provider, - model, + provider: resolved.provider, + model: resolved.model, api_key_env, - base_url: None, + base_url: resolved.base_url, }, memory: OpenFangMemorySection { decay_rate: 0.05 }, network: OpenFangNetworkSection { @@ -1731,6 +1925,7 @@ fn migrate_agents_from_json( }; let defaults = agents.defaults.as_ref(); + let provider_catalog = root.models.as_ref().and_then(|m| m.providers.as_ref()); for entry in &agents.list { let id = &entry.id; @@ -1738,7 +1933,7 @@ fn migrate_agents_from_json( continue; } - match convert_agent_from_json(entry, defaults) { + match convert_agent_from_json(entry, defaults, provider_catalog) { Ok((toml_str, unmapped_tools)) => { let dest_dir = target.join("agents").join(id); let dest_file = dest_dir.join("agent.toml"); @@ -1779,6 +1974,7 @@ fn migrate_agents_from_json( fn convert_agent_from_json( entry: &OpenClawAgentEntry, defaults: Option<&OpenClawAgentDefaults>, + provider_catalog: Option<&serde_json::Map>, ) -> Result<(String, Vec), MigrateError> { let id = &entry.id; let display_name = entry.name.clone().unwrap_or_else(|| id.clone()); @@ -1786,7 +1982,11 @@ fn convert_agent_from_json( // Resolve model let primary_ref = extract_primary_model(entry, defaults) .unwrap_or_else(|| "anthropic/claude-sonnet-4-20250514".to_string()); - let (provider, model) = split_model_ref(&primary_ref); + let ResolvedModelRef { + provider, + model, + base_url: primary_base_url, + } = split_model_ref_with_context(&primary_ref, provider_catalog); // Resolve fallback models let fallbacks = extract_fallback_models(entry, defaults); @@ -1879,10 +2079,15 @@ fn convert_agent_from_json( if let Some(ref api_key) = api_key_env { toml_str.push_str(&format!("api_key_env = \"{api_key}\"\n")); } + if let Some(base_url) = primary_base_url { + toml_str.push_str(&format!("base_url = \"{base_url}\"\n")); + } // Fallback models for fb in &fallbacks { - let (fb_provider, fb_model) = split_model_ref(fb); + let fallback = split_model_ref_with_context(fb, provider_catalog); + let fb_provider = fallback.provider; + let fb_model = fallback.model; let fb_api_key = default_api_key_env(&fb_provider); toml_str.push_str("\n[[fallback_models]]\n"); toml_str.push_str(&format!("provider = \"{fb_provider}\"\n")); @@ -1890,6 +2095,9 @@ fn convert_agent_from_json( if !fb_api_key.is_empty() { toml_str.push_str(&format!("api_key_env = \"{fb_api_key}\"\n")); } + if let Some(base_url) = fallback.base_url { + toml_str.push_str(&format!("base_url = \"{base_url}\"\n")); + } } // Capabilities section @@ -3070,6 +3278,7 @@ fn copy_dir_recursive(src: &Path, dst: &Path) -> Result<(), std::io::Error> { #[cfg(test)] mod tests { use super::*; + use serde_json::json; use tempfile::TempDir; // ===== Helper: create legacy YAML workspace ===== @@ -3807,6 +4016,102 @@ mod tests { assert_eq!(m, ""); } + #[test] + fn test_model_ref_split_with_catalog_preserves_custom_openai_provider() { + let providers = json!({ + "qwencode": { + "baseUrl": "https://coding.dashscope.aliyuncs.com/v1", + "api": "openai-completions" + } + }); + + let catalog = providers.as_object().unwrap(); + let resolved = split_model_ref_with_context("qwencode/glm-5", Some(catalog)); + + assert_eq!(resolved.provider, "qwencode"); + assert_eq!(resolved.model, "glm-5"); + assert_eq!( + resolved.base_url.as_deref(), + Some("https://coding.dashscope.aliyuncs.com/v1") + ); + } + + #[test] + fn test_model_ref_split_with_catalog_uses_api_hint_for_driver() { + let providers = json!({ + "kimicode": { + "baseUrl": "https://api.kimi.com/coding", + "api": "anthropic-messages" + } + }); + + let catalog = providers.as_object().unwrap(); + let resolved = split_model_ref_with_context("kimicode/kimi-k2.5", Some(catalog)); + + assert_eq!(resolved.provider, "anthropic"); + assert_eq!(resolved.model, "kimi-k2.5"); + assert_eq!( + resolved.base_url.as_deref(), + Some("https://api.kimi.com/coding") + ); + } + + #[test] + fn test_model_ref_split_with_catalog_unknown_openai_without_base_url_falls_back_to_openai() { + let providers = json!({ + "mycompany": { + "api": "openai-completions" + } + }); + + let catalog = providers.as_object().unwrap(); + let resolved = split_model_ref_with_context("mycompany/custom-model", Some(catalog)); + + assert_eq!(resolved.provider, "openai"); + assert_eq!(resolved.model, "custom-model"); + assert_eq!(resolved.base_url, None); + } + + #[test] + fn test_model_ref_split_with_catalog_alias_gpt_maps_to_openai() { + let providers = json!({ + "gpt": { + "baseUrl": "https://api.openai.com/v1", + "api": "openai-completions" + } + }); + + let catalog = providers.as_object().unwrap(); + let resolved = split_model_ref_with_context("gpt/gpt-4.1-mini", Some(catalog)); + + assert_eq!(resolved.provider, "openai"); + assert_eq!(resolved.model, "gpt-4.1-mini"); + assert_eq!( + resolved.base_url.as_deref(), + Some("https://api.openai.com/v1") + ); + } + + #[test] + fn test_model_ref_split_with_catalog_alias_key_mismatch_uses_catalog_metadata() { + let providers = json!({ + "openai": { + "baseUrl": "https://proxy.example/v1", + "api": "openai-completions" + } + }); + + let catalog = providers.as_object().unwrap(); + let resolved = split_model_ref_with_context("gpt/gpt-4.1-mini", Some(catalog)); + + assert_eq!(resolved.provider, "openai"); + assert_eq!(resolved.model, "gpt-4.1-mini"); + assert_eq!( + resolved.base_url.as_deref(), + Some("https://proxy.example/v1") + ); + } + #[test] fn test_json5_unknown_provider_passthrough() { let source = TempDir::new().unwrap(); @@ -3954,6 +4259,37 @@ mod tests { assert_eq!(map_provider("gemini"), "google"); assert_eq!(map_provider("xai"), "xai"); assert_eq!(map_provider("grok"), "xai"); + assert_eq!(map_provider("qwen"), "qwen"); + assert_eq!(map_provider("dashscope"), "qwen"); + assert_eq!(map_provider("qwencode"), "qwen"); + assert_eq!(map_provider("moonshot"), "moonshot"); + assert_eq!(map_provider("kimi"), "moonshot"); + assert_eq!(map_provider("kimicode"), "moonshot"); + assert_eq!(map_provider("zhipu"), "zhipu"); + assert_eq!(map_provider("glm"), "zhipu"); + assert_eq!(map_provider("codegeex"), "zhipu_coding"); + assert_eq!(map_provider("baidu"), "qianfan"); + assert_eq!(map_provider("copilot"), "github-copilot"); + assert_eq!(map_provider("github-copilot"), "github-copilot"); + } + + #[test] + fn test_default_api_key_env_mapping() { + assert_eq!(default_api_key_env("qwen"), "DASHSCOPE_API_KEY"); + assert_eq!(default_api_key_env("moonshot"), "MOONSHOT_API_KEY"); + assert_eq!(default_api_key_env("minimax"), "MINIMAX_API_KEY"); + assert_eq!(default_api_key_env("zhipu"), "ZHIPU_API_KEY"); + assert_eq!(default_api_key_env("zhipu_coding"), "ZHIPU_API_KEY"); + assert_eq!(default_api_key_env("qianfan"), "QIANFAN_API_KEY"); + assert_eq!(default_api_key_env("perplexity"), "PERPLEXITY_API_KEY"); + assert_eq!(default_api_key_env("cohere"), "COHERE_API_KEY"); + assert_eq!(default_api_key_env("ai21"), "AI21_API_KEY"); + assert_eq!(default_api_key_env("huggingface"), "HF_API_KEY"); + assert_eq!(default_api_key_env("replicate"), "REPLICATE_API_TOKEN"); + assert_eq!(default_api_key_env("github-copilot"), "GITHUB_TOKEN"); + assert!(default_api_key_env("ollama").is_empty()); + assert_eq!(default_api_key_env("vllm"), "VLLM_API_KEY"); + assert_eq!(default_api_key_env("lmstudio"), "LMSTUDIO_API_KEY"); } #[test] diff --git a/crates/openfang-migrate/tests/provider_json5_agents.rs b/crates/openfang-migrate/tests/provider_json5_agents.rs new file mode 100644 index 00000000..5a6d6dde --- /dev/null +++ b/crates/openfang-migrate/tests/provider_json5_agents.rs @@ -0,0 +1,137 @@ +use openfang_migrate::{run_migration, MigrateOptions, MigrateSource}; +use tempfile::TempDir; + +fn migrate_with_json5(json5_content: &str) -> (TempDir, TempDir) { + let source = TempDir::new().expect("create source tempdir"); + let target = TempDir::new().expect("create target tempdir"); + + std::fs::write(source.path().join("openclaw.json"), json5_content) + .expect("write openclaw.json"); + + let options = MigrateOptions { + source: MigrateSource::OpenClaw, + source_dir: source.path().to_path_buf(), + target_dir: target.path().to_path_buf(), + dry_run: false, + }; + + run_migration(&options).expect("run migration"); + (source, target) +} + +#[test] +fn json5_agent_provider_and_fallback_api_key_env_mapping() { + let json5_content = r#"{ + agents: { + list: [ + { + id: "provider-case", + model: { + primary: "qwencode/glm-5", + fallbacks: [ + "kimicode/kimi-k2.5", + "copilot/gpt-4.1", + "vllm/llama3" + ] + } + } + ] + } +}"#; + + let (_source, target) = migrate_with_json5(json5_content); + + let agent_toml = std::fs::read_to_string( + target + .path() + .join("agents") + .join("provider-case") + .join("agent.toml"), + ) + .expect("read migrated agent.toml"); + + let parsed: toml::Value = toml::from_str(&agent_toml).expect("parse agent.toml"); + + let model = parsed + .get("model") + .and_then(toml::Value::as_table) + .expect("[model] exists"); + + assert_eq!( + model.get("provider").and_then(toml::Value::as_str), + Some("qwen") + ); + assert_eq!( + model.get("api_key_env").and_then(toml::Value::as_str), + Some("DASHSCOPE_API_KEY") + ); + + let fallback_models = parsed + .get("fallback_models") + .and_then(toml::Value::as_array) + .expect("[[fallback_models]] exists"); + + assert_eq!(fallback_models.len(), 3); + + let expected = [ + ("moonshot", "MOONSHOT_API_KEY"), + ("github-copilot", "GITHUB_TOKEN"), + ("vllm", "VLLM_API_KEY"), + ]; + + for (index, (expected_provider, expected_api_key_env)) in expected.iter().enumerate() { + let fallback = fallback_models[index] + .as_table() + .expect("fallback entry is table"); + + assert_eq!( + fallback.get("provider").and_then(toml::Value::as_str), + Some(*expected_provider) + ); + assert_eq!( + fallback.get("api_key_env").and_then(toml::Value::as_str), + Some(*expected_api_key_env) + ); + } +} + +#[test] +fn json5_lmstudio_provider_api_key_env_mapping() { + let json5_content = r#"{ + agents: { + list: [ + { + id: "lmstudio-case", + model: "lmstudio/llama3" + } + ] + } +}"#; + + let (_source, target) = migrate_with_json5(json5_content); + + let agent_toml = std::fs::read_to_string( + target + .path() + .join("agents") + .join("lmstudio-case") + .join("agent.toml"), + ) + .expect("read migrated lmstudio agent.toml"); + + let parsed: toml::Value = toml::from_str(&agent_toml).expect("parse agent.toml"); + + let model = parsed + .get("model") + .and_then(toml::Value::as_table) + .expect("[model] exists"); + + assert_eq!( + model.get("provider").and_then(toml::Value::as_str), + Some("lmstudio") + ); + assert_eq!( + model.get("api_key_env").and_then(toml::Value::as_str), + Some("LMSTUDIO_API_KEY") + ); +} diff --git a/crates/openfang-migrate/tests/provider_json5_default_model.rs b/crates/openfang-migrate/tests/provider_json5_default_model.rs new file mode 100644 index 00000000..e892e07e --- /dev/null +++ b/crates/openfang-migrate/tests/provider_json5_default_model.rs @@ -0,0 +1,64 @@ +use openfang_migrate::{openclaw, MigrateOptions, MigrateSource}; +use tempfile::TempDir; + +fn assert_default_model_mapping( + model_ref: &str, + expected_provider: &str, + expected_api_key_env: &str, +) { + let source = TempDir::new().unwrap(); + let target = TempDir::new().unwrap(); + + let openclaw_json = format!( + r#"{{ + agents: {{ + defaults: {{ model: "{model_ref}" }}, + list: [ + {{ id: "tester" }} + ] + }} +}}"# + ); + std::fs::write(source.path().join("openclaw.json"), openclaw_json).unwrap(); + + let options = MigrateOptions { + source: MigrateSource::OpenClaw, + source_dir: source.path().to_path_buf(), + target_dir: target.path().to_path_buf(), + dry_run: false, + }; + openclaw::migrate(&options).unwrap(); + + let config_toml = std::fs::read_to_string(target.path().join("config.toml")).unwrap(); + let parsed: toml::Value = toml::from_str(&config_toml).unwrap(); + let default_model = parsed + .get("default_model") + .and_then(toml::Value::as_table) + .expect("config.toml should contain [default_model]"); + + assert_eq!( + default_model.get("provider").and_then(toml::Value::as_str), + Some(expected_provider) + ); + assert_eq!( + default_model + .get("api_key_env") + .and_then(toml::Value::as_str), + Some(expected_api_key_env) + ); +} + +#[test] +fn json5_default_model_qwencode_maps_to_qwen_and_dashscope_key() { + assert_default_model_mapping("qwencode/glm-5", "qwen", "DASHSCOPE_API_KEY"); +} + +#[test] +fn json5_default_model_kimicode_maps_to_moonshot_and_moonshot_key() { + assert_default_model_mapping("kimicode/kimi-k2.5", "moonshot", "MOONSHOT_API_KEY"); +} + +#[test] +fn json5_default_model_copilot_maps_to_github_copilot_and_github_token() { + assert_default_model_mapping("copilot/gpt-4.1", "github-copilot", "GITHUB_TOKEN"); +} diff --git a/crates/openfang-migrate/tests/provider_json5_provider_catalog.rs b/crates/openfang-migrate/tests/provider_json5_provider_catalog.rs new file mode 100644 index 00000000..316c2361 --- /dev/null +++ b/crates/openfang-migrate/tests/provider_json5_provider_catalog.rs @@ -0,0 +1,277 @@ +use openfang_migrate::{run_migration, MigrateOptions, MigrateSource}; +use tempfile::TempDir; + +#[test] +fn json5_catalog_preserves_custom_openai_provider_and_base_url() { + let source = TempDir::new().expect("create source tempdir"); + let target = TempDir::new().expect("create target tempdir"); + + let json5 = r#"{ + models: { + providers: { + qwencode: { + baseUrl: "https://coding.dashscope.aliyuncs.com/v1", + api: "openai-completions" + } + } + }, + agents: { + defaults: { + model: "qwencode/glm-5" + }, + list: [ + { id: "coder" } + ] + } +}"#; + + std::fs::write(source.path().join("openclaw.json"), json5).expect("write openclaw.json"); + + let options = MigrateOptions { + source: MigrateSource::OpenClaw, + source_dir: source.path().to_path_buf(), + target_dir: target.path().to_path_buf(), + dry_run: false, + }; + run_migration(&options).expect("migration succeeds"); + + let cfg = std::fs::read_to_string(target.path().join("config.toml")).expect("read config.toml"); + let cfg_val: toml::Value = toml::from_str(&cfg).expect("parse config.toml"); + let dm = cfg_val + .get("default_model") + .and_then(toml::Value::as_table) + .expect("default_model table"); + + assert_eq!( + dm.get("provider").and_then(toml::Value::as_str), + Some("qwencode") + ); + assert_eq!(dm.get("model").and_then(toml::Value::as_str), Some("glm-5")); + assert_eq!( + dm.get("api_key_env").and_then(toml::Value::as_str), + Some("QWENCODE_API_KEY") + ); + assert_eq!( + dm.get("base_url").and_then(toml::Value::as_str), + Some("https://coding.dashscope.aliyuncs.com/v1") + ); + + let agent = std::fs::read_to_string(target.path().join("agents/coder/agent.toml")) + .expect("read migrated agent"); + let agent_val: toml::Value = toml::from_str(&agent).expect("parse agent.toml"); + let model = agent_val + .get("model") + .and_then(toml::Value::as_table) + .expect("model table"); + + assert_eq!( + model.get("provider").and_then(toml::Value::as_str), + Some("qwencode") + ); + assert_eq!( + model.get("base_url").and_then(toml::Value::as_str), + Some("https://coding.dashscope.aliyuncs.com/v1") + ); +} + +#[test] +fn json5_catalog_api_hint_maps_custom_provider_to_anthropic_driver() { + let source = TempDir::new().expect("create source tempdir"); + let target = TempDir::new().expect("create target tempdir"); + + let json5 = r#"{ + models: { + providers: { + kimicode: { + baseUrl: "https://api.kimi.com/coding", + api: "anthropic-messages" + } + } + }, + agents: { + list: [ + { + id: "writer", + model: { + primary: "kimicode/kimi-k2.5", + fallbacks: ["kimicode/kimi-k2.5"] + } + } + ] + } +}"#; + + std::fs::write(source.path().join("openclaw.json"), json5).expect("write openclaw.json"); + + let options = MigrateOptions { + source: MigrateSource::OpenClaw, + source_dir: source.path().to_path_buf(), + target_dir: target.path().to_path_buf(), + dry_run: false, + }; + run_migration(&options).expect("migration succeeds"); + + let agent = std::fs::read_to_string(target.path().join("agents/writer/agent.toml")) + .expect("read migrated agent"); + let agent_val: toml::Value = toml::from_str(&agent).expect("parse agent.toml"); + + let model = agent_val + .get("model") + .and_then(toml::Value::as_table) + .expect("model table"); + assert_eq!( + model.get("provider").and_then(toml::Value::as_str), + Some("anthropic") + ); + assert_eq!( + model.get("api_key_env").and_then(toml::Value::as_str), + Some("ANTHROPIC_API_KEY") + ); + assert_eq!( + model.get("base_url").and_then(toml::Value::as_str), + Some("https://api.kimi.com/coding") + ); + + let fallback_models = agent_val + .get("fallback_models") + .and_then(toml::Value::as_array) + .expect("fallback models exist"); + let fb = fallback_models + .first() + .and_then(toml::Value::as_table) + .expect("first fallback table"); + + assert_eq!( + fb.get("provider").and_then(toml::Value::as_str), + Some("anthropic") + ); + assert_eq!( + fb.get("base_url").and_then(toml::Value::as_str), + Some("https://api.kimi.com/coding") + ); +} + +#[test] +fn json5_catalog_unknown_openai_without_base_url_falls_back_to_openai() { + let source = TempDir::new().expect("create source tempdir"); + let target = TempDir::new().expect("create target tempdir"); + + let json5 = r#"{ + models: { + providers: { + mygateway: { + api: "openai-completions" + } + } + }, + agents: { + defaults: { + model: "mygateway/gpt-like" + }, + list: [ + { id: "fallback-check" } + ] + } +}"#; + + std::fs::write(source.path().join("openclaw.json"), json5).expect("write openclaw.json"); + + let options = MigrateOptions { + source: MigrateSource::OpenClaw, + source_dir: source.path().to_path_buf(), + target_dir: target.path().to_path_buf(), + dry_run: false, + }; + run_migration(&options).expect("migration succeeds"); + + let cfg = std::fs::read_to_string(target.path().join("config.toml")).expect("read config.toml"); + let cfg_val: toml::Value = toml::from_str(&cfg).expect("parse config.toml"); + let dm = cfg_val + .get("default_model") + .and_then(toml::Value::as_table) + .expect("default_model table"); + + assert_eq!( + dm.get("provider").and_then(toml::Value::as_str), + Some("openai") + ); + assert_eq!( + dm.get("model").and_then(toml::Value::as_str), + Some("gpt-like") + ); + assert!(dm.get("base_url").is_none()); + + let agent = std::fs::read_to_string(target.path().join("agents/fallback-check/agent.toml")) + .expect("read migrated agent"); + let agent_val: toml::Value = toml::from_str(&agent).expect("parse agent.toml"); + let model = agent_val + .get("model") + .and_then(toml::Value::as_table) + .expect("model table"); + + assert_eq!( + model.get("provider").and_then(toml::Value::as_str), + Some("openai") + ); + assert_eq!( + model.get("api_key_env").and_then(toml::Value::as_str), + Some("OPENAI_API_KEY") + ); + assert!(model.get("base_url").is_none()); +} + +#[test] +fn json5_catalog_alias_key_mismatch_still_uses_catalog_metadata() { + let source = TempDir::new().expect("create source tempdir"); + let target = TempDir::new().expect("create target tempdir"); + + let json5 = r#"{ + models: { + providers: { + openai: { + baseUrl: "https://proxy.example/v1", + api: "openai-completions" + } + } + }, + agents: { + list: [ + { + id: "alias-check", + model: "gpt/gpt-4.1-mini" + } + ] + } +}"#; + + std::fs::write(source.path().join("openclaw.json"), json5).expect("write openclaw.json"); + + let options = MigrateOptions { + source: MigrateSource::OpenClaw, + source_dir: source.path().to_path_buf(), + target_dir: target.path().to_path_buf(), + dry_run: false, + }; + run_migration(&options).expect("migration succeeds"); + + let agent = std::fs::read_to_string(target.path().join("agents/alias-check/agent.toml")) + .expect("read migrated agent"); + let agent_val: toml::Value = toml::from_str(&agent).expect("parse agent.toml"); + let model = agent_val + .get("model") + .and_then(toml::Value::as_table) + .expect("model table"); + + assert_eq!( + model.get("provider").and_then(toml::Value::as_str), + Some("openai") + ); + assert_eq!( + model.get("api_key_env").and_then(toml::Value::as_str), + Some("OPENAI_API_KEY") + ); + assert_eq!( + model.get("base_url").and_then(toml::Value::as_str), + Some("https://proxy.example/v1") + ); +} diff --git a/crates/openfang-migrate/tests/provider_legacy_yaml.rs b/crates/openfang-migrate/tests/provider_legacy_yaml.rs new file mode 100644 index 00000000..adb06028 --- /dev/null +++ b/crates/openfang-migrate/tests/provider_legacy_yaml.rs @@ -0,0 +1,126 @@ +use std::fs; +use std::path::Path; + +use openfang_migrate::{run_migration, MigrateOptions, MigrateSource}; +use tempfile::TempDir; + +fn create_legacy_workspace( + source_dir: &Path, + config_provider: &str, + config_model: &str, + agent_id: &str, + agent_provider: &str, + agent_model: &str, +) { + fs::write( + source_dir.join("config.yaml"), + format!("provider: {config_provider}\nmodel: {config_model}\n"), + ) + .expect("write config.yaml"); + + let agent_dir = source_dir.join("agents").join(agent_id); + fs::create_dir_all(&agent_dir).expect("create agent dir"); + fs::write( + agent_dir.join("agent.yaml"), + format!( + "name: {agent_id}\n\ + description: provider alias mapping test\n\ + provider: {agent_provider}\n\ + model: {agent_model}\n" + ), + ) + .expect("write agent.yaml"); +} + +fn migrate_legacy_workspace(source_dir: &Path, target_dir: &Path) { + let options = MigrateOptions { + source: MigrateSource::OpenClaw, + source_dir: source_dir.to_path_buf(), + target_dir: target_dir.to_path_buf(), + dry_run: false, + }; + run_migration(&options).expect("legacy YAML migration should succeed"); +} + +fn assert_config_model_mapping( + target_dir: &Path, + expected_provider: &str, + expected_api_key_env: &str, +) { + let config_toml = fs::read_to_string(target_dir.join("config.toml")).expect("read config.toml"); + let config: toml::Value = toml::from_str(&config_toml).expect("parse config.toml"); + let model = config + .get("default_model") + .expect("config.toml should have [default_model]"); + + assert_eq!( + model.get("provider").and_then(toml::Value::as_str), + Some(expected_provider) + ); + assert_eq!( + model.get("api_key_env").and_then(toml::Value::as_str), + Some(expected_api_key_env) + ); +} + +fn assert_agent_model_mapping( + target_dir: &Path, + agent_id: &str, + expected_provider: &str, + expected_api_key_env: &str, +) { + let agent_toml = + fs::read_to_string(target_dir.join("agents").join(agent_id).join("agent.toml")) + .expect("read agent.toml"); + let agent: toml::Value = toml::from_str(&agent_toml).expect("parse agent.toml"); + let model = agent.get("model").expect("agent.toml should have [model]"); + + assert_eq!( + model.get("provider").and_then(toml::Value::as_str), + Some(expected_provider) + ); + assert_eq!( + model.get("api_key_env").and_then(toml::Value::as_str), + Some(expected_api_key_env) + ); +} + +#[test] +fn legacy_yaml_provider_alias_mapping_kimicode_and_copilot() { + let source = TempDir::new().expect("create source tempdir"); + let target = TempDir::new().expect("create target tempdir"); + let agent_id = "coder"; + + create_legacy_workspace( + source.path(), + "kimicode", + "kimi-k2", + agent_id, + "copilot", + "gpt-4.1", + ); + migrate_legacy_workspace(source.path(), target.path()); + + assert_config_model_mapping(target.path(), "moonshot", "MOONSHOT_API_KEY"); + assert_agent_model_mapping(target.path(), agent_id, "github-copilot", "GITHUB_TOKEN"); +} + +#[test] +fn legacy_yaml_provider_alias_mapping_qwencode_and_lmstudio() { + let source = TempDir::new().expect("create source tempdir"); + let target = TempDir::new().expect("create target tempdir"); + let agent_id = "assistant"; + + create_legacy_workspace( + source.path(), + "qwencode", + "qwen-plus", + agent_id, + "lmstudio", + "local-model", + ); + migrate_legacy_workspace(source.path(), target.path()); + + assert_config_model_mapping(target.path(), "qwen", "DASHSCOPE_API_KEY"); + assert_agent_model_mapping(target.path(), agent_id, "lmstudio", "LMSTUDIO_API_KEY"); +}