mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
feat(media-generation): image & video agents + GMI media tools (#4244)
This commit is contained in:
@@ -79,3 +79,38 @@ The module has focused Rust tests for:
|
||||
|
||||
Future runtime PRs should add provider-specific execution tests next to the
|
||||
runtime adapter, not in the hosted contract module.
|
||||
|
||||
## Media generation (GMI) — image & video tools
|
||||
|
||||
Separate from the high-level `image_generation` contract above, the
|
||||
`src/openhuman/media_generation/` domain ships **wired, executing** tools that
|
||||
generate images and video through the OpenHuman backend's `media_generation`
|
||||
provider (GMI Cloud — Seedream, SeedEdit, Seedance, Veo).
|
||||
|
||||
| Tool | Purpose | Permission | Output |
|
||||
| --- | --- | --- | --- |
|
||||
| `media_generate_image` | Text-to-image / image-to-image via GMI. | Execute | Local file path under `generated-media/`. |
|
||||
| `media_generate_video` | Text-to-video / image-to-video via GMI. | Execute | Local file path under `generated-media/`. |
|
||||
| `media_list_models` | List the curated model catalog (and optionally GMI's live list). | Read-only | Model ids + pricing. |
|
||||
|
||||
How it works:
|
||||
|
||||
- Generation is asynchronous. The tool submits to the backend (which charges on
|
||||
submit and returns a request id), then **blocks with progress**, polling until
|
||||
the request reaches a terminal state.
|
||||
- GMI returns expiring signed URLs; the tool downloads each artifact into the
|
||||
agent's `generated-media/` directory and returns a stable local file path.
|
||||
- The backend owns provider keys, billing, and rate limiting
|
||||
(`/agent-integrations/media-generation/*`, see `backend/docs/media-generation.md`).
|
||||
|
||||
### Image & video sub-agents
|
||||
|
||||
Two specialist sub-agents wrap these tools and are reachable from the
|
||||
orchestrator via delegation:
|
||||
|
||||
- **`image_agent`** (`delegate_create_image`) — owns prompt craft, model
|
||||
selection, and saving generated images. Rides the multimodal `vision-v1` tier
|
||||
so it can inspect what it produces.
|
||||
- **`video_agent`** (`delegate_create_video`) — owns text-to-video and
|
||||
image-to-video; sets expectations that generation can take minutes and blocks
|
||||
until the clip is saved.
|
||||
|
||||
@@ -22,6 +22,14 @@ const IMAGE_TO_BACKEND: Option<CapabilityPrivacy> = Some(CapabilityPrivacy {
|
||||
destinations: &["OpenHuman backend", "TinyHumans Neocortex"],
|
||||
});
|
||||
|
||||
// Media generation sends the prompt (and any reference image URL) to GMI Cloud
|
||||
// via the OpenHuman backend; generated media is downloaded back to the device.
|
||||
const MEDIA_GEN_TO_BACKEND: Option<CapabilityPrivacy> = Some(CapabilityPrivacy {
|
||||
leaves_device: true,
|
||||
data_kind: PrivacyDataKind::Raw,
|
||||
destinations: &["OpenHuman backend", "GMI Cloud"],
|
||||
});
|
||||
|
||||
const LOCAL_CREDENTIALS: Option<CapabilityPrivacy> = Some(CapabilityPrivacy {
|
||||
leaves_device: false,
|
||||
data_kind: PrivacyDataKind::Credentials,
|
||||
@@ -261,6 +269,26 @@ pub(super) const CAPABILITIES: &[Capability] = &[
|
||||
status: CapabilityStatus::Beta,
|
||||
privacy: IMAGE_TO_BACKEND,
|
||||
},
|
||||
Capability {
|
||||
id: "intelligence.image_generation",
|
||||
name: "Image Generation",
|
||||
domain: "agent",
|
||||
category: CapabilityCategory::Intelligence,
|
||||
description: "Delegate image creation to a dedicated image sub-agent — generate images from a text prompt, or edit/restyle reference images, using hosted GMI models (Seedream / SeedEdit). Results are saved to the workspace.",
|
||||
how_to: "Ask the assistant to generate, draw, or edit an image",
|
||||
status: CapabilityStatus::Beta,
|
||||
privacy: MEDIA_GEN_TO_BACKEND,
|
||||
},
|
||||
Capability {
|
||||
id: "intelligence.video_generation",
|
||||
name: "Video Generation",
|
||||
domain: "agent",
|
||||
category: CapabilityCategory::Intelligence,
|
||||
description: "Delegate short-video creation to a dedicated video sub-agent — text-to-video or animate a reference image using hosted GMI models (Seedance / Veo). Generation is asynchronous; the finished clip is saved to the workspace.",
|
||||
how_to: "Ask the assistant to generate a video or animate an image",
|
||||
status: CapabilityStatus::Beta,
|
||||
privacy: MEDIA_GEN_TO_BACKEND,
|
||||
},
|
||||
Capability {
|
||||
id: "conversation.label_filter",
|
||||
name: "Thread Label Filters",
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
id = "image_agent"
|
||||
display_name = "Image Creator"
|
||||
delegate_name = "create_image"
|
||||
when_to_use = "Image-generation specialist — create new images from a text prompt, or edit/restyle reference images, using the hosted GMI models (Seedream / SeedEdit). Route here for make/generate/draw an image, create a logo/illustration/photo, or edit this image requests. It rides the multimodal `vision-v1` tier, so it can also inspect the images it produces. It saves each result to the workspace and returns the local file path."
|
||||
temperature = 0.6
|
||||
max_iterations = 8
|
||||
agent_tier = "worker"
|
||||
omit_identity = true
|
||||
omit_memory_context = true
|
||||
omit_safety_preamble = false
|
||||
omit_skills_catalog = true
|
||||
omit_profile = true
|
||||
omit_memory_md = true
|
||||
|
||||
# Multimodal tier so the agent can review the images it generates (and any
|
||||
# reference images) via the image_info / inline-image path, then iterate.
|
||||
[model]
|
||||
hint = "vision"
|
||||
|
||||
[tools]
|
||||
# media_generate_image submits the generation and returns a saved local path;
|
||||
# media_list_models surfaces the catalog so the agent can pick a model; the
|
||||
# remaining tools let it inspect reference images and read prior outputs.
|
||||
named = [
|
||||
"media_generate_image",
|
||||
"media_list_models",
|
||||
"image_info",
|
||||
"file_read",
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
pub mod prompt;
|
||||
@@ -0,0 +1,38 @@
|
||||
# Image-generation specialist
|
||||
|
||||
You are a focused **image-creation** sub-agent. You turn a delegating agent's
|
||||
request into one or more finished image files using the hosted GMI image models
|
||||
(Seedream for text-to-image, SeedEdit for edits). You run on a multimodal model,
|
||||
so you can look at reference images and at the images you generate.
|
||||
|
||||
## Your job
|
||||
|
||||
- **Create** images from a text prompt (`media_generate_image`).
|
||||
- **Edit / restyle** a supplied image by passing its URL(s) as `input_images`.
|
||||
- **Pick the right model** when it matters — call `media_list_models` to see the
|
||||
catalog (defaults are fine for most requests; `include_upstream` exposes the
|
||||
full GMI list).
|
||||
|
||||
## How to work
|
||||
|
||||
- Write a vivid, specific prompt. Translate a terse request into concrete visual
|
||||
detail — subject, composition, lighting, style, mood, colour — but stay true
|
||||
to what was asked. Don't invent requirements the user didn't state.
|
||||
- Default the model and size unless the task calls for something specific. Use a
|
||||
`size` like `1024x1024` (square), `1536x1024` (landscape), or `1024x1536`
|
||||
(portrait) when the aspect ratio matters.
|
||||
- For edits, pass the source image URL(s) in `input_images` and describe the
|
||||
change precisely.
|
||||
- Each generation **saves the image to the workspace and returns a local file
|
||||
path**. Always report that path back so the deck/answer can reference the
|
||||
concrete artifact. Do not paste raw base64 or invent URLs.
|
||||
- Generation is billed. Don't loop on near-identical prompts — generate, inspect
|
||||
the result, and only re-run if it materially misses the brief.
|
||||
|
||||
## Boundaries
|
||||
|
||||
- Report results to the delegating agent — you are not talking to the end user.
|
||||
- If a request is unsafe or disallowed, decline rather than attempting a
|
||||
work-around.
|
||||
- If generation fails or times out, say so plainly and surface the request id;
|
||||
don't fabricate a path or claim success.
|
||||
@@ -0,0 +1,71 @@
|
||||
//! System prompt builder for the `image_agent` built-in agent.
|
||||
//!
|
||||
//! Returns the final, fully-assembled system prompt — archetype body (from the
|
||||
//! sibling `prompt.md`) plus the shared section helpers every agent uses.
|
||||
|
||||
use crate::openhuman::context::prompt::{
|
||||
render_tools, render_user_files, render_workspace, PromptContext,
|
||||
};
|
||||
use anyhow::Result;
|
||||
|
||||
const ARCHETYPE: &str = include_str!("prompt.md");
|
||||
|
||||
pub fn build(ctx: &PromptContext<'_>) -> Result<String> {
|
||||
let mut out = String::with_capacity(4096);
|
||||
out.push_str(ARCHETYPE.trim_end());
|
||||
out.push_str("\n\n");
|
||||
|
||||
let user_files = render_user_files(ctx)?;
|
||||
if !user_files.trim().is_empty() {
|
||||
out.push_str(user_files.trim_end());
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
|
||||
let tools = render_tools(ctx)?;
|
||||
if !tools.trim().is_empty() {
|
||||
out.push_str(tools.trim_end());
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
|
||||
let workspace = render_workspace(ctx)?;
|
||||
if !workspace.trim().is_empty() {
|
||||
out.push_str(workspace.trim_end());
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::openhuman::context::prompt::{LearnedContextData, ToolCallFormat};
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[test]
|
||||
fn build_returns_nonempty_body() {
|
||||
let visible: HashSet<String> = HashSet::new();
|
||||
let ctx = PromptContext {
|
||||
workspace_dir: std::path::Path::new("."),
|
||||
model_name: "vision-v1",
|
||||
agent_id: "image_agent",
|
||||
tools: &[],
|
||||
workflows: &[],
|
||||
dispatcher_instructions: "",
|
||||
learned: LearnedContextData::default(),
|
||||
visible_tool_names: &visible,
|
||||
tool_call_format: ToolCallFormat::PFormat,
|
||||
connected_integrations: &[],
|
||||
connected_identities_md: String::new(),
|
||||
include_profile: false,
|
||||
include_memory_md: false,
|
||||
curated_snapshot: None,
|
||||
user_identity: None,
|
||||
personality_soul_md: None,
|
||||
personality_memory_md: None,
|
||||
personality_roster: vec![],
|
||||
};
|
||||
let body = build(&ctx).unwrap();
|
||||
assert!(body.contains("Image-generation specialist"));
|
||||
}
|
||||
}
|
||||
@@ -169,6 +169,16 @@ pub const BUILTINS: &[BuiltinAgent] = &[
|
||||
toml: include_str!("vision_agent/agent.toml"),
|
||||
prompt_fn: super::vision_agent::prompt::build,
|
||||
},
|
||||
BuiltinAgent {
|
||||
id: "image_agent",
|
||||
toml: include_str!("image_agent/agent.toml"),
|
||||
prompt_fn: super::image_agent::prompt::build,
|
||||
},
|
||||
BuiltinAgent {
|
||||
id: "video_agent",
|
||||
toml: include_str!("video_agent/agent.toml"),
|
||||
prompt_fn: super::video_agent::prompt::build,
|
||||
},
|
||||
BuiltinAgent {
|
||||
id: "archivist",
|
||||
toml: include_str!("archivist/agent.toml"),
|
||||
|
||||
@@ -13,6 +13,7 @@ pub mod crypto_agent;
|
||||
pub mod desktop_control_agent;
|
||||
pub mod goals_agent;
|
||||
pub mod help;
|
||||
pub mod image_agent;
|
||||
pub mod integrations_agent;
|
||||
pub mod markets_agent;
|
||||
pub mod mcp_agent;
|
||||
@@ -33,6 +34,7 @@ pub mod tool_maker;
|
||||
pub mod tools_agent;
|
||||
pub mod trigger_reactor;
|
||||
pub mod trigger_triage;
|
||||
pub mod video_agent;
|
||||
pub mod vision_agent;
|
||||
|
||||
pub use loader::{load_builtins, validate_tier_hierarchy, BuiltinAgent, BUILTINS};
|
||||
|
||||
@@ -85,6 +85,14 @@ allowlist = [
|
||||
# of an attached image / screenshot / on-disk image file here — it rides the
|
||||
# multimodal `vision-v1` tier, so it can actually see the image.
|
||||
"vision_agent",
|
||||
# Image-generation specialist. Synthesised into a `delegate_create_image`
|
||||
# tool. Route make/generate/edit an image requests here — it owns prompt
|
||||
# craft, model selection, and saving the result to the workspace.
|
||||
"image_agent",
|
||||
# Video-generation specialist. Synthesised into a `delegate_create_video`
|
||||
# tool. Route make/generate a video or animate this image requests here —
|
||||
# generation is async (minutes) and the agent blocks until the clip is saved.
|
||||
"video_agent",
|
||||
"skill_creator",
|
||||
"critic",
|
||||
"archivist",
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
id = "video_agent"
|
||||
display_name = "Video Creator"
|
||||
delegate_name = "create_video"
|
||||
when_to_use = "Video-generation specialist — create short video clips from a text prompt (text-to-video) or animate a supplied image (image-to-video) using the hosted GMI models (Seedance / Veo). Route here for make/generate a video, animate this image, or short clip requests. Generation can take a few minutes; it blocks until the clip is ready, saves it to the workspace, and returns the local file path."
|
||||
temperature = 0.6
|
||||
max_iterations = 8
|
||||
agent_tier = "worker"
|
||||
omit_identity = true
|
||||
omit_memory_context = true
|
||||
omit_safety_preamble = false
|
||||
omit_skills_catalog = true
|
||||
omit_profile = true
|
||||
omit_memory_md = true
|
||||
|
||||
# Multimodal tier so the agent can inspect a reference/first-frame image or the
|
||||
# returned thumbnail when shaping an image-to-video request.
|
||||
[model]
|
||||
hint = "vision"
|
||||
|
||||
[tools]
|
||||
# media_generate_video submits the generation and returns a saved local path;
|
||||
# media_list_models surfaces the catalog; image_info / file_read let it inspect
|
||||
# a reference image before animating it.
|
||||
named = [
|
||||
"media_generate_video",
|
||||
"media_list_models",
|
||||
"image_info",
|
||||
"file_read",
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
pub mod prompt;
|
||||
@@ -0,0 +1,38 @@
|
||||
# Video-generation specialist
|
||||
|
||||
You are a focused **video-creation** sub-agent. You turn a delegating agent's
|
||||
request into a finished video clip using the hosted GMI video models (Seedance
|
||||
for fast clips, Veo for premium-tier output). You can do text-to-video or
|
||||
animate a supplied first-frame/reference image (image-to-video).
|
||||
|
||||
## Your job
|
||||
|
||||
- **Create** a clip from a text prompt (`media_generate_video`).
|
||||
- **Animate** a supplied image by passing its URL as `input_image`.
|
||||
- **Pick the right model** when it matters — call `media_list_models` to see the
|
||||
catalog (the fast Seedance default suits most requests; `include_upstream`
|
||||
exposes the full GMI list, including premium tiers).
|
||||
|
||||
## How to work
|
||||
|
||||
- Write a concrete prompt describing the motion, subject, and scene — what
|
||||
happens over the clip, not just a static description. Mention camera movement,
|
||||
pacing, and style when relevant.
|
||||
- Use `duration_seconds` and `aspect_ratio` (e.g. `16:9`, `9:16`, `1:1`) when the
|
||||
task specifies them; otherwise let the model default.
|
||||
- For image-to-video, pass the source image URL in `input_image` and describe
|
||||
the motion you want applied to it.
|
||||
- Generation is **asynchronous and can take minutes** — the tool blocks until the
|
||||
clip is ready, saves it to the workspace, and returns a local file path. Report
|
||||
that path back. Set expectations: tell the delegating agent it may take a
|
||||
little while.
|
||||
- Generation is billed and slow. Don't re-run on near-identical prompts — only
|
||||
iterate if the result materially misses the brief.
|
||||
|
||||
## Boundaries
|
||||
|
||||
- Report results to the delegating agent — you are not talking to the end user.
|
||||
- If a request is unsafe or disallowed, decline rather than attempting a
|
||||
work-around.
|
||||
- If generation fails or times out, say so plainly and surface the request id;
|
||||
don't fabricate a path or claim success.
|
||||
@@ -0,0 +1,71 @@
|
||||
//! System prompt builder for the `video_agent` built-in agent.
|
||||
//!
|
||||
//! Returns the final, fully-assembled system prompt — archetype body (from the
|
||||
//! sibling `prompt.md`) plus the shared section helpers every agent uses.
|
||||
|
||||
use crate::openhuman::context::prompt::{
|
||||
render_tools, render_user_files, render_workspace, PromptContext,
|
||||
};
|
||||
use anyhow::Result;
|
||||
|
||||
const ARCHETYPE: &str = include_str!("prompt.md");
|
||||
|
||||
pub fn build(ctx: &PromptContext<'_>) -> Result<String> {
|
||||
let mut out = String::with_capacity(4096);
|
||||
out.push_str(ARCHETYPE.trim_end());
|
||||
out.push_str("\n\n");
|
||||
|
||||
let user_files = render_user_files(ctx)?;
|
||||
if !user_files.trim().is_empty() {
|
||||
out.push_str(user_files.trim_end());
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
|
||||
let tools = render_tools(ctx)?;
|
||||
if !tools.trim().is_empty() {
|
||||
out.push_str(tools.trim_end());
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
|
||||
let workspace = render_workspace(ctx)?;
|
||||
if !workspace.trim().is_empty() {
|
||||
out.push_str(workspace.trim_end());
|
||||
out.push('\n');
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::openhuman::context::prompt::{LearnedContextData, ToolCallFormat};
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[test]
|
||||
fn build_returns_nonempty_body() {
|
||||
let visible: HashSet<String> = HashSet::new();
|
||||
let ctx = PromptContext {
|
||||
workspace_dir: std::path::Path::new("."),
|
||||
model_name: "vision-v1",
|
||||
agent_id: "video_agent",
|
||||
tools: &[],
|
||||
workflows: &[],
|
||||
dispatcher_instructions: "",
|
||||
learned: LearnedContextData::default(),
|
||||
visible_tool_names: &visible,
|
||||
tool_call_format: ToolCallFormat::PFormat,
|
||||
connected_integrations: &[],
|
||||
connected_identities_md: String::new(),
|
||||
include_profile: false,
|
||||
include_memory_md: false,
|
||||
curated_snapshot: None,
|
||||
user_identity: None,
|
||||
personality_soul_md: None,
|
||||
personality_memory_md: None,
|
||||
personality_roster: vec![],
|
||||
};
|
||||
let body = build(&ctx).unwrap();
|
||||
assert!(body.contains("Video-generation specialist"));
|
||||
}
|
||||
}
|
||||
@@ -133,6 +133,18 @@ const RESOURCE_CATALOG: &[PromptResource] = &[
|
||||
description: "Multimodal worker that analyses attached images for the vision tier.",
|
||||
content: include_str!("../agent_registry/agents/vision_agent/prompt.md"),
|
||||
},
|
||||
PromptResource {
|
||||
uri: "openhuman://prompts/agents/image_agent",
|
||||
name: "image_agent",
|
||||
description: "Worker that generates or edits images via GMI and saves them to the workspace.",
|
||||
content: include_str!("../agent_registry/agents/image_agent/prompt.md"),
|
||||
},
|
||||
PromptResource {
|
||||
uri: "openhuman://prompts/agents/video_agent",
|
||||
name: "video_agent",
|
||||
description: "Worker that generates short videos via GMI and saves them to the workspace.",
|
||||
content: include_str!("../agent_registry/agents/video_agent/prompt.md"),
|
||||
},
|
||||
PromptResource {
|
||||
uri: "openhuman://prompts/agents/archivist",
|
||||
name: "archivist",
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
//! Persist generated media to the agent's action directory.
|
||||
//!
|
||||
//! GMI returns expiring signed URLs; we download the bytes and write them under
|
||||
//! a `generated-media/` root inside `action_dir` so final answers can reference
|
||||
//! a stable local file path (per the `image_generation` contract). The action
|
||||
//! directory is the agent's canonical read/write root.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use super::types::MediaItem;
|
||||
|
||||
/// Subdirectory (under `action_dir`) where generated artifacts are stored.
|
||||
const GENERATED_MEDIA_DIR: &str = "generated-media";
|
||||
|
||||
/// A downloaded artifact and where it landed on disk.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PersistedArtifact {
|
||||
pub kind: String,
|
||||
pub path: PathBuf,
|
||||
pub source_url: String,
|
||||
pub thumbnail_url: Option<String>,
|
||||
}
|
||||
|
||||
/// Pick a file extension from the artifact kind + content type / URL.
|
||||
fn extension_for(kind: &str, content_type: Option<&str>, url: &str) -> String {
|
||||
if let Some(ct) = content_type {
|
||||
let ct = ct.to_ascii_lowercase();
|
||||
if ct.contains("png") {
|
||||
return "png".to_string();
|
||||
}
|
||||
if ct.contains("webp") {
|
||||
return "webp".to_string();
|
||||
}
|
||||
if ct.contains("jpeg") || ct.contains("jpg") {
|
||||
return "jpg".to_string();
|
||||
}
|
||||
if ct.contains("mp4") {
|
||||
return "mp4".to_string();
|
||||
}
|
||||
if ct.contains("webm") {
|
||||
return "webm".to_string();
|
||||
}
|
||||
}
|
||||
// Fall back to the URL path suffix, then a per-kind default.
|
||||
let lower = url.split('?').next().unwrap_or(url).to_ascii_lowercase();
|
||||
for ext in ["png", "webp", "jpg", "jpeg", "mp4", "webm"] {
|
||||
if lower.ends_with(&format!(".{ext}")) {
|
||||
return if ext == "jpeg" {
|
||||
"jpg".to_string()
|
||||
} else {
|
||||
ext.to_string()
|
||||
};
|
||||
}
|
||||
}
|
||||
if kind.eq_ignore_ascii_case("video") {
|
||||
"mp4".to_string()
|
||||
} else {
|
||||
"png".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Download a single media URL into `dir`, returning the written path.
|
||||
async fn download_one(
|
||||
http: &reqwest::Client,
|
||||
dir: &Path,
|
||||
item: &MediaItem,
|
||||
request_id: &str,
|
||||
index: usize,
|
||||
) -> Result<PersistedArtifact> {
|
||||
tracing::info!(
|
||||
"[media_generation] downloading {} artifact {} for request={}",
|
||||
item.kind,
|
||||
index,
|
||||
request_id
|
||||
);
|
||||
let resp = http
|
||||
.get(&item.url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("failed to fetch generated media from {}", item.url))?
|
||||
.error_for_status()
|
||||
.with_context(|| format!("generated media URL returned an error: {}", item.url))?;
|
||||
|
||||
let content_type = resp
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|s| s.to_string());
|
||||
let ext = extension_for(&item.kind, content_type.as_deref(), &item.url);
|
||||
|
||||
let bytes = resp
|
||||
.bytes()
|
||||
.await
|
||||
.with_context(|| format!("failed to read generated media body from {}", item.url))?;
|
||||
|
||||
// Sanitize the request id for use in a filename (it is a UUID from GMI, but
|
||||
// be defensive against path separators).
|
||||
let safe_id: String = request_id
|
||||
.chars()
|
||||
.map(|c| {
|
||||
if c.is_ascii_alphanumeric() || c == '-' {
|
||||
c
|
||||
} else {
|
||||
'_'
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let filename = format!("{safe_id}-{index}.{ext}");
|
||||
let path = dir.join(&filename);
|
||||
tokio::fs::write(&path, &bytes)
|
||||
.await
|
||||
.with_context(|| format!("failed to write generated media to {}", path.display()))?;
|
||||
|
||||
Ok(PersistedArtifact {
|
||||
kind: item.kind.clone(),
|
||||
path,
|
||||
source_url: item.url.clone(),
|
||||
thumbnail_url: item.thumbnail_url.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Download + persist all media items for a request under
|
||||
/// `{action_dir}/generated-media/`. Returns the written artifacts.
|
||||
pub async fn persist_media(
|
||||
action_dir: &Path,
|
||||
request_id: &str,
|
||||
items: &[MediaItem],
|
||||
) -> Result<Vec<PersistedArtifact>> {
|
||||
if items.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let dir = action_dir.join(GENERATED_MEDIA_DIR);
|
||||
tokio::fs::create_dir_all(&dir)
|
||||
.await
|
||||
.with_context(|| format!("failed to create generated-media dir {}", dir.display()))?;
|
||||
|
||||
let http = reqwest::Client::new();
|
||||
let mut out = Vec::with_capacity(items.len());
|
||||
for (i, item) in items.iter().enumerate() {
|
||||
out.push(download_one(&http, &dir, item, request_id, i).await?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn extension_prefers_content_type() {
|
||||
assert_eq!(
|
||||
extension_for("image", Some("image/png"), "https://x/y"),
|
||||
"png"
|
||||
);
|
||||
assert_eq!(
|
||||
extension_for("image", Some("image/webp"), "https://x/y"),
|
||||
"webp"
|
||||
);
|
||||
assert_eq!(
|
||||
extension_for("video", Some("video/mp4"), "https://x/y"),
|
||||
"mp4"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_falls_back_to_url_then_kind() {
|
||||
assert_eq!(
|
||||
extension_for("image", None, "https://x/y/a.webp?sig=1"),
|
||||
"webp"
|
||||
);
|
||||
assert_eq!(extension_for("video", None, "https://x/y/clip"), "mp4");
|
||||
assert_eq!(extension_for("image", None, "https://x/y/clip"), "png");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
//! Media generation domain — agent tools for image/video generation backed by
|
||||
//! GMI via the OpenHuman backend's `media_generation` provider.
|
||||
//!
|
||||
//! The backend (`/agent-integrations/media-generation/*`) owns provider keys,
|
||||
//! billing, and the standardized contract; these tools submit a request, block
|
||||
//! with progress until it completes, download the resulting media into the
|
||||
//! agent's `generated-media/` root, and return local file paths.
|
||||
|
||||
pub mod download;
|
||||
pub mod tools;
|
||||
pub mod types;
|
||||
|
||||
pub use tools::{
|
||||
build_media_tools, MediaGenerateImageTool, MediaGenerateVideoTool, MediaListModelsTool,
|
||||
};
|
||||
@@ -0,0 +1,417 @@
|
||||
//! Agent-facing media-generation tools (image + video) backed by GMI via the
|
||||
//! OpenHuman backend's `media_generation` provider.
|
||||
//!
|
||||
//! **Endpoints** (see `backend/docs/media-generation.md`):
|
||||
//! - `POST /agent-integrations/media-generation/images`
|
||||
//! - `POST /agent-integrations/media-generation/videos`
|
||||
//! - `GET /agent-integrations/media-generation/requests/{requestId}`
|
||||
//! - `GET /agent-integrations/media-generation/models`
|
||||
//!
|
||||
//! Generation is asynchronous. These tools **block with progress**: they submit
|
||||
//! (`wait:false`, so the backend charges + returns a request id immediately),
|
||||
//! then poll the request until it reaches a terminal state, download each
|
||||
//! resulting artifact into the agent's `generated-media/` root, and return the
|
||||
//! local file paths. The backend owns GMI keys, billing, and rate limiting.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::openhuman::config::Config;
|
||||
use crate::openhuman::integrations::IntegrationClient;
|
||||
use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCategory, ToolResult};
|
||||
|
||||
use super::download::persist_media;
|
||||
use super::types::MediaResponse;
|
||||
|
||||
const IMAGES_PATH: &str = "/agent-integrations/media-generation/images";
|
||||
const VIDEOS_PATH: &str = "/agent-integrations/media-generation/videos";
|
||||
const MODELS_PATH: &str = "/agent-integrations/media-generation/models";
|
||||
|
||||
/// Poll cadence + caps. Images are fast; video can take minutes.
|
||||
const POLL_INTERVAL: Duration = Duration::from_secs(4);
|
||||
const IMAGE_MAX_WAIT_SECS: u64 = 180;
|
||||
const VIDEO_MAX_WAIT_SECS: u64 = 420;
|
||||
|
||||
/// Shared submit-then-poll-then-persist flow for both modalities.
|
||||
async fn generate_and_persist(
|
||||
client: &IntegrationClient,
|
||||
action_dir: &std::path::Path,
|
||||
submit_path: &str,
|
||||
body: Value,
|
||||
max_wait_secs: u64,
|
||||
) -> ToolResult {
|
||||
// Submit without server-side blocking; the backend charges on submit and
|
||||
// returns a request id we poll ourselves (so the core owns the progress UX).
|
||||
let submitted: MediaResponse = match client.post::<MediaResponse>(submit_path, &body).await {
|
||||
Ok(resp) => resp,
|
||||
Err(e) => return ToolResult::error(format!("Media generation submit failed: {e}")),
|
||||
};
|
||||
|
||||
let request_id = submitted.request_id.clone();
|
||||
tracing::info!(
|
||||
"[media_generation] submitted request={} status={} cost=${:.4}",
|
||||
request_id,
|
||||
submitted.status,
|
||||
submitted.cost_usd
|
||||
);
|
||||
|
||||
let status_path = format!(
|
||||
"/agent-integrations/media-generation/requests/{}",
|
||||
request_id
|
||||
);
|
||||
|
||||
let mut latest = submitted;
|
||||
let deadline = Instant::now() + Duration::from_secs(max_wait_secs);
|
||||
while !latest.is_terminal() {
|
||||
if Instant::now() >= deadline {
|
||||
tracing::warn!(
|
||||
"[media_generation] wait budget elapsed for request={} (status={})",
|
||||
request_id,
|
||||
latest.status
|
||||
);
|
||||
return ToolResult::success(format!(
|
||||
"Media generation is still {} after {}s. It was accepted (request_id: {}) and \
|
||||
billed; it may finish shortly — check again later.",
|
||||
latest.status, max_wait_secs, request_id
|
||||
));
|
||||
}
|
||||
tokio::time::sleep(POLL_INTERVAL).await;
|
||||
match client.get::<MediaResponse>(&status_path).await {
|
||||
Ok(resp) => {
|
||||
tracing::debug!(
|
||||
"[media_generation] poll request={} status={}",
|
||||
request_id,
|
||||
resp.status
|
||||
);
|
||||
latest = resp;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"[media_generation] poll error for request={}: {e}",
|
||||
request_id
|
||||
);
|
||||
// Transient poll failures shouldn't abort a paid generation —
|
||||
// keep polling until the deadline.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if latest.is_failed() {
|
||||
return ToolResult::error(format!(
|
||||
"Media generation failed (request_id: {request_id})."
|
||||
));
|
||||
}
|
||||
|
||||
if latest.media.is_empty() {
|
||||
return ToolResult::error(format!(
|
||||
"Media generation reported success but returned no media (request_id: {request_id})."
|
||||
));
|
||||
}
|
||||
|
||||
match persist_media(action_dir, &request_id, &latest.media).await {
|
||||
Ok(artifacts) => {
|
||||
let mut lines = vec![format!(
|
||||
"Generated {} artifact(s) (request_id: {}, model: {}):",
|
||||
artifacts.len(),
|
||||
request_id,
|
||||
latest.model
|
||||
)];
|
||||
for art in &artifacts {
|
||||
lines.push(format!("- {} → {}", art.kind, art.path.display()));
|
||||
if let Some(thumb) = &art.thumbnail_url {
|
||||
lines.push(format!(" thumbnail: {thumb}"));
|
||||
}
|
||||
}
|
||||
lines.push(format!("\nCost: ${:.4}", latest.cost_usd));
|
||||
let payload = json!({
|
||||
"request_id": request_id,
|
||||
"model": latest.model,
|
||||
"cost_usd": latest.cost_usd,
|
||||
"artifacts": artifacts.iter().map(|a| json!({
|
||||
"type": a.kind,
|
||||
"path": a.path.display().to_string(),
|
||||
"source_url": a.source_url,
|
||||
"thumbnail_url": a.thumbnail_url,
|
||||
})).collect::<Vec<_>>(),
|
||||
});
|
||||
ToolResult::success_with_markdown(payload, lines.join("\n"))
|
||||
}
|
||||
Err(e) => ToolResult::error(format!(
|
||||
"Generation succeeded but persisting media failed (request_id: {request_id}): {e}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
// ── MediaGenerateImageTool ──────────────────────────────────────────
|
||||
|
||||
pub struct MediaGenerateImageTool {
|
||||
client: Arc<IntegrationClient>,
|
||||
action_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl MediaGenerateImageTool {
|
||||
pub fn new(client: Arc<IntegrationClient>, action_dir: PathBuf) -> Self {
|
||||
Self { client, action_dir }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for MediaGenerateImageTool {
|
||||
fn name(&self) -> &str {
|
||||
"media_generate_image"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Generate or edit an image from a text prompt using GMI (Seedream / SeedEdit). \
|
||||
Optionally pass reference image URLs to edit/condition (image-to-image). \
|
||||
Blocks until the image is ready and saves it under the workspace \
|
||||
generated-media folder, returning the local file path. Cost is billed by the backend."
|
||||
}
|
||||
|
||||
fn parameters_schema(&self) -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": { "type": "string", "description": "Detailed visual prompt or edit instruction" },
|
||||
"model": { "type": "string", "description": "Optional GMI model id (default: seedream-4-0-250828). Use media_list_models to discover." },
|
||||
"size": { "type": "string", "description": "Optional output size, e.g. 1024x1024 or 1536x1024" },
|
||||
"n": { "type": "integer", "minimum": 1, "maximum": 8, "description": "Number of images (default 1)" },
|
||||
"input_images": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" },
|
||||
"description": "Optional reference image URLs for edit / image-to-image"
|
||||
},
|
||||
"seed": { "type": "integer", "description": "Optional seed for reproducibility" }
|
||||
},
|
||||
"required": ["prompt"]
|
||||
})
|
||||
}
|
||||
|
||||
fn permission_level(&self) -> PermissionLevel {
|
||||
PermissionLevel::Execute
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
ToolCategory::Workflow
|
||||
}
|
||||
|
||||
async fn execute(&self, args: Value) -> anyhow::Result<ToolResult> {
|
||||
let prompt = match args.get("prompt").and_then(|v| v.as_str()) {
|
||||
Some(p) if !p.trim().is_empty() => p,
|
||||
_ => return Ok(ToolResult::error("prompt is required")),
|
||||
};
|
||||
|
||||
let mut body = json!({ "prompt": prompt, "wait": false });
|
||||
if let Some(model) = args.get("model").and_then(|v| v.as_str()) {
|
||||
body["model"] = json!(model);
|
||||
}
|
||||
if let Some(size) = args.get("size").and_then(|v| v.as_str()) {
|
||||
body["size"] = json!(size);
|
||||
}
|
||||
if let Some(n) = args.get("n").and_then(|v| v.as_u64()) {
|
||||
body["n"] = json!(n.clamp(1, 8));
|
||||
}
|
||||
if let Some(imgs) = args.get("input_images").and_then(|v| v.as_array()) {
|
||||
let urls: Vec<&str> = imgs.iter().filter_map(|v| v.as_str()).collect();
|
||||
if !urls.is_empty() {
|
||||
body["inputImages"] = json!(urls);
|
||||
}
|
||||
}
|
||||
if let Some(seed) = args.get("seed").and_then(|v| v.as_i64()) {
|
||||
body["seed"] = json!(seed);
|
||||
}
|
||||
|
||||
tracing::info!("[media_generate_image] prompt_len={}", prompt.len());
|
||||
Ok(generate_and_persist(
|
||||
&self.client,
|
||||
&self.action_dir,
|
||||
IMAGES_PATH,
|
||||
body,
|
||||
IMAGE_MAX_WAIT_SECS,
|
||||
)
|
||||
.await)
|
||||
}
|
||||
}
|
||||
|
||||
// ── MediaGenerateVideoTool ──────────────────────────────────────────
|
||||
|
||||
pub struct MediaGenerateVideoTool {
|
||||
client: Arc<IntegrationClient>,
|
||||
action_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl MediaGenerateVideoTool {
|
||||
pub fn new(client: Arc<IntegrationClient>, action_dir: PathBuf) -> Self {
|
||||
Self { client, action_dir }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for MediaGenerateVideoTool {
|
||||
fn name(&self) -> &str {
|
||||
"media_generate_video"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Generate a short video from a text prompt using GMI (Seedance / Veo). \
|
||||
Optionally pass a first-frame/reference image URL for image-to-video. \
|
||||
Video can take a few minutes; this blocks until it is ready, saves the \
|
||||
clip under the workspace generated-media folder, and returns the local \
|
||||
file path. Cost is billed by the backend."
|
||||
}
|
||||
|
||||
fn parameters_schema(&self) -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": { "type": "string", "description": "Detailed description of the video to generate" },
|
||||
"model": { "type": "string", "description": "Optional GMI model id (default: seedance-1-0-pro-fast-251015). Use media_list_models to discover." },
|
||||
"input_image": { "type": "string", "description": "Optional first-frame / reference image URL for image-to-video" },
|
||||
"duration_seconds": { "type": "integer", "minimum": 1, "maximum": 60, "description": "Optional clip duration in seconds" },
|
||||
"aspect_ratio": { "type": "string", "description": "Optional aspect ratio, e.g. 16:9, 9:16, 1:1" },
|
||||
"negative_prompt": { "type": "string", "description": "Optional description of what to avoid" },
|
||||
"seed": { "type": "integer", "description": "Optional seed for reproducibility" }
|
||||
},
|
||||
"required": ["prompt"]
|
||||
})
|
||||
}
|
||||
|
||||
fn permission_level(&self) -> PermissionLevel {
|
||||
PermissionLevel::Execute
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
ToolCategory::Workflow
|
||||
}
|
||||
|
||||
async fn execute(&self, args: Value) -> anyhow::Result<ToolResult> {
|
||||
let prompt = match args.get("prompt").and_then(|v| v.as_str()) {
|
||||
Some(p) if !p.trim().is_empty() => p,
|
||||
_ => return Ok(ToolResult::error("prompt is required")),
|
||||
};
|
||||
|
||||
let mut body = json!({ "prompt": prompt, "wait": false });
|
||||
if let Some(model) = args.get("model").and_then(|v| v.as_str()) {
|
||||
body["model"] = json!(model);
|
||||
}
|
||||
if let Some(img) = args.get("input_image").and_then(|v| v.as_str()) {
|
||||
body["inputImage"] = json!(img);
|
||||
}
|
||||
if let Some(d) = args.get("duration_seconds").and_then(|v| v.as_u64()) {
|
||||
body["durationSeconds"] = json!(d.clamp(1, 60));
|
||||
}
|
||||
if let Some(ar) = args.get("aspect_ratio").and_then(|v| v.as_str()) {
|
||||
body["aspectRatio"] = json!(ar);
|
||||
}
|
||||
if let Some(np) = args.get("negative_prompt").and_then(|v| v.as_str()) {
|
||||
body["negativePrompt"] = json!(np);
|
||||
}
|
||||
if let Some(seed) = args.get("seed").and_then(|v| v.as_i64()) {
|
||||
body["seed"] = json!(seed);
|
||||
}
|
||||
|
||||
tracing::info!("[media_generate_video] prompt_len={}", prompt.len());
|
||||
Ok(generate_and_persist(
|
||||
&self.client,
|
||||
&self.action_dir,
|
||||
VIDEOS_PATH,
|
||||
body,
|
||||
VIDEO_MAX_WAIT_SECS,
|
||||
)
|
||||
.await)
|
||||
}
|
||||
}
|
||||
|
||||
// ── MediaListModelsTool ─────────────────────────────────────────────
|
||||
|
||||
pub struct MediaListModelsTool {
|
||||
client: Arc<IntegrationClient>,
|
||||
}
|
||||
|
||||
impl MediaListModelsTool {
|
||||
pub fn new(client: Arc<IntegrationClient>) -> Self {
|
||||
Self { client }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for MediaListModelsTool {
|
||||
fn name(&self) -> &str {
|
||||
"media_list_models"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"List available image/video generation models — a curated catalog with \
|
||||
pricing, plus (with include_upstream) GMI's full live model list. Use to \
|
||||
pick a `model` id for media_generate_image / media_generate_video."
|
||||
}
|
||||
|
||||
fn parameters_schema(&self) -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"include_upstream": {
|
||||
"type": "boolean",
|
||||
"description": "Also fetch GMI's full live model list (default false)"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn category(&self) -> ToolCategory {
|
||||
ToolCategory::Workflow
|
||||
}
|
||||
|
||||
async fn execute(&self, args: Value) -> anyhow::Result<ToolResult> {
|
||||
let include_upstream = args
|
||||
.get("include_upstream")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
let path = if include_upstream {
|
||||
format!("{MODELS_PATH}?includeUpstream=true")
|
||||
} else {
|
||||
MODELS_PATH.to_string()
|
||||
};
|
||||
match self.client.get::<Value>(&path).await {
|
||||
Ok(resp) => Ok(ToolResult::success_with_markdown(
|
||||
resp.clone(),
|
||||
serde_json::to_string_pretty(&resp).unwrap_or_else(|_| resp.to_string()),
|
||||
)),
|
||||
Err(e) => Ok(ToolResult::error(format!(
|
||||
"Failed to list media models: {e}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Builder ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Build the media-generation tool surface. Returns empty when no integration
|
||||
/// client is configured (no backend URL / not signed in), mirroring the other
|
||||
/// backend-proxied tool families.
|
||||
pub fn build_media_tools(root_config: &Config, action_dir: &std::path::Path) -> Vec<Box<dyn Tool>> {
|
||||
let Some(client) = crate::openhuman::integrations::build_client(root_config) else {
|
||||
tracing::debug!("[media_generation] no integration client — media tools skipped");
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let action_dir = action_dir.to_path_buf();
|
||||
let tools: Vec<Box<dyn Tool>> = vec![
|
||||
Box::new(MediaGenerateImageTool::new(
|
||||
Arc::clone(&client),
|
||||
action_dir.clone(),
|
||||
)),
|
||||
Box::new(MediaGenerateVideoTool::new(Arc::clone(&client), action_dir)),
|
||||
Box::new(MediaListModelsTool::new(Arc::clone(&client))),
|
||||
];
|
||||
tracing::debug!("[media_generation] registered {} media tools", tools.len());
|
||||
tools
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tools_tests.rs"]
|
||||
mod tools_tests;
|
||||
@@ -0,0 +1,249 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use super::{MediaGenerateImageTool, MediaGenerateVideoTool, MediaListModelsTool};
|
||||
use crate::openhuman::integrations::IntegrationClient;
|
||||
use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCategory};
|
||||
|
||||
fn dummy_client() -> Arc<IntegrationClient> {
|
||||
// No requests are made in these tests; the URL/token are placeholders.
|
||||
Arc::new(IntegrationClient::new(
|
||||
"http://127.0.0.1:0".to_string(),
|
||||
"test-token".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_tool_schema_and_metadata() {
|
||||
let tool = MediaGenerateImageTool::new(dummy_client(), PathBuf::from("/tmp"));
|
||||
assert_eq!(tool.name(), "media_generate_image");
|
||||
assert_eq!(tool.permission_level(), PermissionLevel::Execute);
|
||||
assert_eq!(tool.category(), ToolCategory::Workflow);
|
||||
|
||||
let schema = tool.parameters_schema();
|
||||
assert_eq!(schema["required"], json!(["prompt"]));
|
||||
let props = schema["properties"].as_object().unwrap();
|
||||
for key in ["prompt", "model", "size", "n", "input_images", "seed"] {
|
||||
assert!(props.contains_key(key), "missing image property {key}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_tool_schema_and_metadata() {
|
||||
let tool = MediaGenerateVideoTool::new(dummy_client(), PathBuf::from("/tmp"));
|
||||
assert_eq!(tool.name(), "media_generate_video");
|
||||
assert_eq!(tool.permission_level(), PermissionLevel::Execute);
|
||||
assert_eq!(tool.category(), ToolCategory::Workflow);
|
||||
|
||||
let schema = tool.parameters_schema();
|
||||
assert_eq!(schema["required"], json!(["prompt"]));
|
||||
let props = schema["properties"].as_object().unwrap();
|
||||
for key in [
|
||||
"prompt",
|
||||
"model",
|
||||
"input_image",
|
||||
"duration_seconds",
|
||||
"aspect_ratio",
|
||||
"negative_prompt",
|
||||
"seed",
|
||||
] {
|
||||
assert!(props.contains_key(key), "missing video property {key}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_models_tool_metadata() {
|
||||
let tool = MediaListModelsTool::new(dummy_client());
|
||||
assert_eq!(tool.name(), "media_list_models");
|
||||
assert_eq!(tool.category(), ToolCategory::Workflow);
|
||||
assert!(tool.parameters_schema()["properties"]
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.contains_key("include_upstream"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_tool_rejects_empty_prompt_without_network() {
|
||||
let tool = MediaGenerateImageTool::new(dummy_client(), PathBuf::from("/tmp"));
|
||||
let result = tool.execute(json!({ "prompt": " " })).await.unwrap();
|
||||
assert!(result.is_error);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_tool_rejects_missing_prompt_without_network() {
|
||||
let tool = MediaGenerateVideoTool::new(dummy_client(), PathBuf::from("/tmp"));
|
||||
let result = tool.execute(json!({ "model": "x" })).await.unwrap();
|
||||
assert!(result.is_error);
|
||||
}
|
||||
|
||||
// ── End-to-end flow against a mock backend (wiremock) ───────────────
|
||||
|
||||
use wiremock::matchers::{method, path, path_regex};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
fn client_for(server: &MockServer) -> std::sync::Arc<IntegrationClient> {
|
||||
std::sync::Arc::new(IntegrationClient::new(server.uri(), "tok".to_string()))
|
||||
}
|
||||
|
||||
/// Mount a media download endpoint that returns `bytes` for the given path.
|
||||
async fn mount_media(server: &MockServer, p: &str, content_type: &str, bytes: &[u8]) {
|
||||
Mock::given(method("GET"))
|
||||
.and(path(p.to_string()))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(bytes.to_vec(), content_type))
|
||||
.mount(server)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_tool_submits_downloads_and_persists_local_artifact() {
|
||||
let server = MockServer::start().await;
|
||||
let media_url = format!("{}/media/out.png", server.uri());
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/agent-integrations/media-generation/images"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(
|
||||
serde_json::json!({ "success": true, "data": {
|
||||
"requestId": "req-1",
|
||||
"status": "success",
|
||||
"model": "seedream-4-0-250828",
|
||||
"media": [{ "type": "image", "url": media_url }],
|
||||
"costUsd": 0.039
|
||||
} }),
|
||||
))
|
||||
.mount(&server)
|
||||
.await;
|
||||
mount_media(&server, "/media/out.png", "image/png", b"PNGBYTES").await;
|
||||
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let tool = MediaGenerateImageTool::new(client_for(&server), tmp.path().to_path_buf());
|
||||
let res = tool
|
||||
.execute(json!({ "prompt": "a fox", "size": "1024x1024" }))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!res.is_error, "expected success, got {res:?}");
|
||||
let dir = tmp.path().join("generated-media");
|
||||
let files: Vec<_> = std::fs::read_dir(&dir)
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.collect();
|
||||
assert_eq!(files.len(), 1, "exactly one artifact should be persisted");
|
||||
assert_eq!(std::fs::read(files[0].path()).unwrap(), b"PNGBYTES");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_tool_persists_clip_with_image_to_video_payload() {
|
||||
let server = MockServer::start().await;
|
||||
let media_url = format!("{}/media/clip.mp4", server.uri());
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/agent-integrations/media-generation/videos"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(
|
||||
serde_json::json!({ "success": true, "data": {
|
||||
"requestId": "vid-1",
|
||||
"status": "success",
|
||||
"model": "seedance-1-0-pro-fast-251015",
|
||||
"media": [{ "type": "video", "url": media_url, "thumbnailUrl": "https://x/t.png" }],
|
||||
"costUsd": 0.13
|
||||
} }),
|
||||
))
|
||||
.mount(&server)
|
||||
.await;
|
||||
mount_media(&server, "/media/clip.mp4", "video/mp4", b"MP4BYTES").await;
|
||||
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let tool = MediaGenerateVideoTool::new(client_for(&server), tmp.path().to_path_buf());
|
||||
let res = tool
|
||||
.execute(
|
||||
json!({ "prompt": "a wave", "input_image": "https://in/f.png", "duration_seconds": 6 }),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!res.is_error, "expected success, got {res:?}");
|
||||
let dir = tmp.path().join("generated-media");
|
||||
let files: Vec<_> = std::fs::read_dir(&dir)
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.collect();
|
||||
assert_eq!(files.len(), 1);
|
||||
assert!(files[0].path().extension().is_some_and(|e| e == "mp4"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_tool_polls_until_terminal_then_persists() {
|
||||
let server = MockServer::start().await;
|
||||
let media_url = format!("{}/media/p.png", server.uri());
|
||||
// Submit returns a non-terminal status; the tool must poll the status endpoint.
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/agent-integrations/media-generation/images"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(
|
||||
serde_json::json!({ "success": true, "data": {
|
||||
"requestId": "req-2", "status": "queued", "model": "seedream-4-0-250828", "media": []
|
||||
} }),
|
||||
))
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path_regex(
|
||||
r"^/agent-integrations/media-generation/requests/.+",
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(
|
||||
serde_json::json!({ "success": true, "data": {
|
||||
"requestId": "req-2",
|
||||
"status": "success",
|
||||
"model": "seedream-4-0-250828",
|
||||
"media": [{ "type": "image", "url": media_url }],
|
||||
"costUsd": 0.039
|
||||
} }),
|
||||
))
|
||||
.mount(&server)
|
||||
.await;
|
||||
mount_media(&server, "/media/p.png", "image/png", b"POLLED").await;
|
||||
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let tool = MediaGenerateImageTool::new(client_for(&server), tmp.path().to_path_buf());
|
||||
let res = tool.execute(json!({ "prompt": "a fox" })).await.unwrap();
|
||||
assert!(!res.is_error, "expected success after poll, got {res:?}");
|
||||
assert_eq!(
|
||||
std::fs::read_dir(tmp.path().join("generated-media"))
|
||||
.unwrap()
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_tool_reports_failed_terminal_status() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/agent-integrations/media-generation/images"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(
|
||||
serde_json::json!({ "success": true, "data": {
|
||||
"requestId": "req-3", "status": "failed", "model": "seedream-4-0-250828", "media": []
|
||||
} }),
|
||||
))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let tool = MediaGenerateImageTool::new(client_for(&server), tmp.path().to_path_buf());
|
||||
let res = tool.execute(json!({ "prompt": "a fox" })).await.unwrap();
|
||||
assert!(res.is_error);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_models_tool_returns_backend_catalog() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/agent-integrations/media-generation/models"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(
|
||||
serde_json::json!({ "success": true, "data": {
|
||||
"curated": [{ "id": "seedream-4-0-250828", "modality": "image" }]
|
||||
} }),
|
||||
))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let tool = MediaListModelsTool::new(client_for(&server));
|
||||
let res = tool.execute(json!({})).await.unwrap();
|
||||
assert!(!res.is_error, "expected success, got {res:?}");
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
//! Shared types for the `media_generation` agent tools.
|
||||
//!
|
||||
//! These mirror the backend's standardized `media_generation` contract
|
||||
//! (`/agent-integrations/media-generation/*`) — see
|
||||
//! `backend/docs/media-generation.md`. The backend normalizes GMI's per-model
|
||||
//! payload/outcome shapes; the core only depends on this stable envelope.
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
/// A single generated artifact as returned by the backend. The `url` is an
|
||||
/// expiring signed URL — the core downloads + persists it locally.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct MediaItem {
|
||||
#[serde(rename = "type")]
|
||||
pub kind: String,
|
||||
pub url: String,
|
||||
#[serde(rename = "thumbnailUrl", default)]
|
||||
pub thumbnail_url: Option<String>,
|
||||
}
|
||||
|
||||
/// Standardized media-generation response envelope.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct MediaResponse {
|
||||
#[serde(rename = "requestId")]
|
||||
pub request_id: String,
|
||||
pub status: String,
|
||||
#[serde(default)]
|
||||
pub model: String,
|
||||
#[serde(default)]
|
||||
pub media: Vec<MediaItem>,
|
||||
#[serde(rename = "costUsd", default)]
|
||||
pub cost_usd: f64,
|
||||
}
|
||||
|
||||
impl MediaResponse {
|
||||
pub fn is_success(&self) -> bool {
|
||||
self.status.eq_ignore_ascii_case("success")
|
||||
}
|
||||
|
||||
pub fn is_failed(&self) -> bool {
|
||||
self.status.eq_ignore_ascii_case("failed")
|
||||
}
|
||||
|
||||
pub fn is_terminal(&self) -> bool {
|
||||
self.is_success() || self.is_failed()
|
||||
}
|
||||
}
|
||||
@@ -65,6 +65,7 @@ pub mod mcp_audit;
|
||||
pub mod mcp_client;
|
||||
pub mod mcp_registry;
|
||||
pub mod mcp_server;
|
||||
pub mod media_generation;
|
||||
pub mod meet;
|
||||
pub mod meet_agent;
|
||||
pub mod memory;
|
||||
|
||||
@@ -745,6 +745,13 @@ pub fn all_tools_with_runtime(
|
||||
|
||||
tools.extend(crate::openhuman::search::build_search_tools(root_config));
|
||||
|
||||
// Media generation (image/video via GMI through the backend). Skipped when
|
||||
// no integration client is configured; artifacts land under `action_dir`.
|
||||
tools.extend(crate::openhuman::media_generation::build_media_tools(
|
||||
root_config,
|
||||
action_dir,
|
||||
));
|
||||
|
||||
// High-level web3 tools (swaps / bridges / dapp calls) built on the wallet.
|
||||
// They call the backend deBridge proxy per-invocation and error gracefully
|
||||
// when the user is not signed in, so they register unconditionally.
|
||||
|
||||
Reference in New Issue
Block a user