diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 000000000..0c03c6a34 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,6 @@ +{ + "attribution": { + "commit": "", + "pr": "" + } +} \ No newline at end of file diff --git a/.github/Dockerfile b/.github/Dockerfile index ce5711ac5..bd23fe970 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -45,6 +45,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ webkit2gtk-driver \ && rm -rf /var/lib/apt/lists/* +# Install system dependencies (cmake, ALSA, X11) +RUN apt-get update && apt-get install -y --no-install-recommends cmake libasound2-dev libxdo-dev libxtst-dev libx11-dev libevdev-dev && rm -rf /var/lib/apt/lists/* + # sccache (pre-installed so the action only needs to configure it) RUN curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz \ | tar xz -C /usr/local/bin --strip-components=1 sccache-v0.10.0-x86_64-unknown-linux-musl/sccache \ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7bad546a7..972fc7320 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -85,8 +85,8 @@ jobs: app/src-tauri -> target cache-on-failure: true - - name: Install cmake (for whisper-rs) - run: apt-get update && apt-get install -y --no-install-recommends cmake && rm -rf /var/lib/apt/lists/* + - name: Install system dependencies (cmake, ALSA, X11) + run: apt-get update && apt-get install -y --no-install-recommends cmake libasound2-dev libxdo-dev libxtst-dev libx11-dev libevdev-dev && rm -rf /var/lib/apt/lists/* - name: Check formatting (cargo fmt) run: cargo fmt --all -- --check @@ -136,7 +136,8 @@ jobs: libgtk-3-dev libwebkit2gtk-4.1-dev libappindicator3-dev \ librsvg2-dev patchelf \ xvfb at-spi2-core dbus-x11 \ - webkit2gtk-driver + webkit2gtk-driver \ + libasound2-dev libxdo-dev libxtst-dev libx11-dev libevdev-dev - name: Cargo.lock fingerprint (deps only) id: cargo-lock-fingerprint diff --git a/Cargo.lock b/Cargo.lock index b3b0a954a..f3e662482 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.11.0", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -611,6 +633,12 @@ dependencies = [ "cpufeatures 0.3.0", ] +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + [[package]] name = "block-buffer" version = "0.10.4" @@ -638,13 +666,22 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f" +dependencies = [ + "objc2 0.5.2", +] + [[package]] name = "block2" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" dependencies = [ - "objc2", + "objc2 0.6.4", ] [[package]] @@ -942,6 +979,21 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "cocoa" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "667fdc068627a2816b9ff831201dd9864249d6ee8d190b9532357f1fc0f61ea7" +dependencies = [ + "bitflags 1.3.2", + "block", + "core-foundation 0.9.4", + "core-graphics 0.21.0", + "foreign-types 0.3.2", + "libc", + "objc", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -1090,13 +1142,23 @@ dependencies = [ "url", ] +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys 0.7.0", + "libc", +] + [[package]] name = "core-foundation" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", ] @@ -1106,16 +1168,70 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", ] +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core-graphics" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3889374e6ea6ab25dba90bb5d96202f61108058361f6dc72e8b03e6f8bbe923" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.7.0", + "foreign-types 0.3.2", + "libc", +] + +[[package]] +name = "core-graphics" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a67c4378cf203eace8fb6567847eb641fd6ff933c1145a115c6ee820ebb978" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "foreign-types 0.3.2", + "libc", +] + +[[package]] +name = "core-graphics" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa95a34622365fa5bbf40b20b75dba8dfa8c94c734aea8ac9a5ca38af14316f1" +dependencies = [ + "bitflags 2.11.0", + "core-foundation 0.10.1", + "core-graphics-types", + "foreign-types 0.5.0", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" +dependencies = [ + "bitflags 2.11.0", + "core-foundation 0.10.1", + "libc", +] + [[package]] name = "core_maths" version = "0.1.1" @@ -1125,6 +1241,49 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys 0.8.7", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceec7a6067e62d6f931a2baf6f3a751f4a892595bcec1461a3c94ef9949864b6" +dependencies = [ + "bindgen", +] + +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys 0.8.7", + "coreaudio-rs", + "dasp_sample", + "jni", + "js-sys", + "libc", + "mach2 0.4.3", + "ndk", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1342,6 +1501,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.10.0" @@ -1640,7 +1805,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ "bitflags 2.11.0", - "objc2", + "objc2 0.6.4", ] [[package]] @@ -1800,6 +1965,25 @@ dependencies = [ "tower-service", ] +[[package]] +name = "enigo" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cf6f550bbbdd5fe66f39d429cb2604bcdacbf00dca0f5bbe2e9306a0009b7c6" +dependencies = [ + "core-foundation 0.10.1", + "core-graphics 0.24.0", + "foreign-types-shared 0.3.1", + "libc", + "log", + "objc2 0.5.2", + "objc2-app-kit", + "objc2-foundation 0.2.2", + "windows 0.58.0", + "xkbcommon", + "xkeysym", +] + [[package]] name = "enumflags2" version = "0.7.12" @@ -2125,7 +2309,28 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" dependencies = [ - "foreign-types-shared", + "foreign-types-shared 0.1.1", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared 0.3.1", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] @@ -2134,6 +2339,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -2604,6 +2815,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "html5ever" version = "0.35.0" @@ -2779,7 +2996,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "iana-time-zone-haiku", "js-sys", "log", @@ -3109,7 +3326,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "617ee6cf8e3f66f3b4ea67a4058564628cde41901316e19f559e14c7c72c5e7b" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "mach2 0.4.3", ] @@ -3119,7 +3336,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06d3a048d09fbb6597dbf7c69f40d14df4a49487db1487191618c893fc3b1c26" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "mach2 0.5.0", ] @@ -3607,6 +3824,15 @@ dependencies = [ "hashify", ] +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + [[package]] name = "maplit" version = "1.0.2" @@ -3984,6 +4210,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "miette" version = "7.6.0" @@ -4159,6 +4394,35 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.11.0", + "jni-sys 0.3.1", + "log", + "ndk-sys", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -4291,6 +4555,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -4319,6 +4594,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -4333,7 +4630,7 @@ checksum = "2f861541f15de120eae5982923d073bfc0c1a65466561988c82d6e197734c19e" dependencies = [ "atomic-waker", "core-foundation 0.9.4", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "futures-core", "io-kit-sys 0.4.1", "libc", @@ -4351,7 +4648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a330b3bc7f8b4fc729a4c63164b3927eeeaced198222a3ce6b8b6e034851b7a" dependencies = [ "core-foundation 0.10.1", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "futures-core", "io-kit-sys 0.5.0", "linux-raw-sys 0.12.1", @@ -4382,6 +4679,31 @@ dependencies = [ "url", ] +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", +] + +[[package]] +name = "objc-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" + +[[package]] +name = "objc2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a785d4eeff09c14c487497c162e92766fbb3e4059a71840cecc03d9a50b804" +dependencies = [ + "objc-sys", + "objc2-encode", +] + [[package]] name = "objc2" version = "0.6.4" @@ -4391,6 +4713,22 @@ dependencies = [ "objc2-encode", ] +[[package]] +name = "objc2-app-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "libc", + "objc2 0.5.2", + "objc2-core-data 0.2.2", + "objc2-core-image 0.2.2", + "objc2-foundation 0.2.2", + "objc2-quartz-core 0.2.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.3.2" @@ -4398,8 +4736,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" dependencies = [ "bitflags 2.11.0", - "objc2", - "objc2-foundation", + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-core-data" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", ] [[package]] @@ -4408,8 +4758,8 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b402a653efbb5e82ce4df10683b6b28027616a2715e90009947d50b8dd298fa" dependencies = [ - "objc2", - "objc2-foundation", + "objc2 0.6.4", + "objc2-foundation 0.3.2", ] [[package]] @@ -4420,7 +4770,7 @@ checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.11.0", "dispatch2", - "objc2", + "objc2 0.6.4", ] [[package]] @@ -4431,19 +4781,31 @@ checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ "bitflags 2.11.0", "dispatch2", - "objc2", + "objc2 0.6.4", "objc2-core-foundation", "objc2-io-surface", ] +[[package]] +name = "objc2-core-image" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55260963a527c99f1819c4f8e3b47fe04f9650694ef348ffd2227e8196d34c80" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", + "objc2-metal", +] + [[package]] name = "objc2-core-image" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d563b38d2b97209f8e861173de434bd0214cf020e3423a52624cd1d989f006" dependencies = [ - "objc2", - "objc2-foundation", + "objc2 0.6.4", + "objc2-foundation 0.3.2", ] [[package]] @@ -4452,8 +4814,8 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca347214e24bc973fc025fd0d36ebb179ff30536ed1f80252706db19ee452009" dependencies = [ - "objc2", - "objc2-foundation", + "objc2 0.6.4", + "objc2-foundation 0.3.2", ] [[package]] @@ -4463,7 +4825,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" dependencies = [ "bitflags 2.11.0", - "objc2", + "objc2 0.6.4", "objc2-core-foundation", "objc2-core-graphics", ] @@ -4474,6 +4836,18 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" +[[package]] +name = "objc2-foundation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "libc", + "objc2 0.5.2", +] + [[package]] name = "objc2-foundation" version = "0.3.2" @@ -4481,9 +4855,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ "bitflags 2.11.0", - "block2", + "block2 0.6.2", "libc", - "objc2", + "objc2 0.6.4", "objc2-core-foundation", ] @@ -4494,10 +4868,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ "bitflags 2.11.0", - "objc2", + "objc2 0.6.4", "objc2-core-foundation", ] +[[package]] +name = "objc2-metal" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", + "objc2-metal", +] + [[package]] name = "objc2-quartz-core" version = "0.3.2" @@ -4505,9 +4904,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" dependencies = [ "bitflags 2.11.0", - "objc2", + "objc2 0.6.4", "objc2-core-foundation", - "objc2-foundation", + "objc2-foundation 0.3.2", ] [[package]] @@ -4526,17 +4925,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d87d638e33c06f577498cbcc50491496a3ed4246998a7fbba7ccb98b1e7eab22" dependencies = [ "bitflags 2.11.0", - "block2", - "objc2", + "block2 0.6.2", + "objc2 0.6.4", "objc2-cloud-kit", - "objc2-core-data", + "objc2-core-data 0.3.2", "objc2-core-foundation", "objc2-core-graphics", - "objc2-core-image", + "objc2-core-image 0.3.2", "objc2-core-location", "objc2-core-text", - "objc2-foundation", - "objc2-quartz-core", + "objc2-foundation 0.3.2", + "objc2-quartz-core 0.3.2", "objc2-user-notifications", ] @@ -4546,8 +4945,8 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9df9128cbbfef73cda168416ccf7f837b62737d748333bfe9ab71c245d76613e" dependencies = [ - "objc2", - "objc2-foundation", + "objc2 0.6.4", + "objc2-foundation 0.3.2", ] [[package]] @@ -4570,6 +4969,29 @@ dependencies = [ "ruzstd", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -4627,10 +5049,12 @@ dependencies = [ "clap", "clap_complete", "console 0.16.3", + "cpal", "cron", "dialoguer", "directories", "dirs 5.0.1", + "enigo", "env_logger", "fantoccini", "fastembed", @@ -4640,6 +5064,7 @@ dependencies = [ "hex", "hmac 0.12.1", "hostname", + "hound", "image", "landlock", "lettre", @@ -4661,6 +5086,7 @@ dependencies = [ "prometheus", "prost 0.14.3", "rand 0.9.2", + "rdev", "regex", "reqwest 0.12.28", "ring", @@ -4714,7 +5140,7 @@ checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" dependencies = [ "bitflags 2.11.0", "cfg-if", - "foreign-types", + "foreign-types 0.3.2", "libc", "once_cell", "openssl-macros", @@ -4854,8 +5280,8 @@ dependencies = [ "android_system_properties", "log", "nix 0.30.1", - "objc2", - "objc2-foundation", + "objc2 0.6.4", + "objc2-foundation 0.3.2", "objc2-ui-kit", "serde", "windows-sys 0.61.2", @@ -5716,6 +6142,22 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rdev" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00552ca2dc2f93b84cd7b5581de49549411e4e41d89e1c691bcb93dc4be360c3" +dependencies = [ + "cocoa", + "core-foundation 0.7.0", + "core-foundation-sys 0.7.0", + "core-graphics 0.19.2", + "lazy_static", + "libc", + "winapi", + "x11", +] + [[package]] name = "readlock" version = "0.1.11" @@ -6252,7 +6694,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ "core-foundation 0.10.1", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "jni", "log", "once_cell", @@ -6412,7 +6854,7 @@ checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags 2.11.0", "core-foundation 0.10.1", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", "security-framework-sys", ] @@ -6423,7 +6865,7 @@ version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", ] @@ -6711,7 +7153,7 @@ dependencies = [ "bitflags 2.11.0", "cfg-if", "core-foundation 0.10.1", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "io-kit-sys 0.4.1", "mach2 0.4.3", "nix 0.26.4", @@ -7069,11 +7511,11 @@ version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", "memchr", "ntapi", - "windows", + "windows 0.57.0", ] [[package]] @@ -7093,7 +7535,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", ] @@ -8659,6 +9101,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.57.0" @@ -8669,6 +9121,26 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.57.0" @@ -8681,6 +9153,19 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement 0.58.0", + "windows-interface 0.58.0", + "windows-result 0.2.0", + "windows-strings 0.1.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -8691,7 +9176,7 @@ dependencies = [ "windows-interface 0.59.3", "windows-link", "windows-result 0.4.1", - "windows-strings", + "windows-strings 0.5.1", ] [[package]] @@ -8705,6 +9190,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -8727,6 +9223,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "windows-interface" version = "0.59.3" @@ -8752,7 +9259,7 @@ checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ "windows-link", "windows-result 0.4.1", - "windows-strings", + "windows-strings 0.5.1", ] [[package]] @@ -8764,6 +9271,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -8773,6 +9289,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result 0.2.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-strings" version = "0.5.1" @@ -9203,6 +9729,16 @@ dependencies = [ "tap", ] +[[package]] +name = "x11" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "502da5464ccd04011667b11c435cb992822c2c0dbde1770c988480d312a0db2e" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "x25519-dalek" version = "2.0.1" @@ -9215,6 +9751,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "xkbcommon" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d66ca9352cbd4eecbbc40871d8a11b4ac8107cfc528a6e14d7c19c69d0e1ac9" +dependencies = [ + "libc", + "memmap2", + "xkeysym", +] + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + [[package]] name = "xxhash-rust" version = "0.8.15" diff --git a/Cargo.toml b/Cargo.toml index 250cdec80..528c81e40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,10 @@ socketioxide = { version = "0.15", features = ["extensions"] } whisper-rs = "0.16" image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } tempfile = "3" +cpal = "0.15" +hound = "3.5" +enigo = "0.3" +rdev = "0.5" matrix-sdk = { version = "0.16", optional = true, default-features = false, features = ["e2e-encryption", "rustls-tls", "markdown"] } fantoccini = { version = "0.22.0", optional = true, default-features = false, features = ["rustls-tls"] } diff --git a/app/src/App.tsx b/app/src/App.tsx index 31bb56172..5aa7682ec 100644 --- a/app/src/App.tsx +++ b/app/src/App.tsx @@ -6,6 +6,7 @@ import { PersistGate } from 'redux-persist/integration/react'; import AppRoutes from './AppRoutes'; import BottomTabBar from './components/BottomTabBar'; import ServiceBlockingGate from './components/daemon/ServiceBlockingGate'; +import DictationHotkeyManager from './components/DictationHotkeyManager'; import ErrorFallbackScreen from './components/ErrorFallbackScreen'; import LocalAIDownloadSnackbar from './components/LocalAIDownloadSnackbar'; import MeshGradient from './components/MeshGradient'; @@ -40,6 +41,7 @@ function App() { + diff --git a/app/src/components/DictationHotkeyManager.tsx b/app/src/components/DictationHotkeyManager.tsx new file mode 100644 index 000000000..374402af6 --- /dev/null +++ b/app/src/components/DictationHotkeyManager.tsx @@ -0,0 +1,31 @@ +/** + * DictationHotkeyManager + * + * Headless component that auto-registers the global dictation hotkey on mount + * and logs toggle events. Mount inside ServiceBlockingGate so the core RPC + * is available when it initialises. + */ +import { useEffect } from 'react'; + +import { useDictationHotkey } from '../hooks/useDictationHotkey'; + +export default function DictationHotkeyManager() { + const { dictationEnabled, hotkeyRegistered, toggleCount, hotkey } = useDictationHotkey(); + + useEffect(() => { + if (toggleCount === 0) return; + console.debug(`[dictation] toggle #${toggleCount} — dictation overlay should show/hide`); + }, [toggleCount]); + + useEffect(() => { + if (hotkeyRegistered) { + console.debug(`[dictation] global hotkey active: ${hotkey}`); + } + }, [hotkeyRegistered, hotkey]); + + useEffect(() => { + console.debug(`[dictation] enabled=${dictationEnabled}`); + }, [dictationEnabled]); + + return null; +} diff --git a/app/src/hooks/useDictationHotkey.ts b/app/src/hooks/useDictationHotkey.ts new file mode 100644 index 000000000..f2f763b79 --- /dev/null +++ b/app/src/hooks/useDictationHotkey.ts @@ -0,0 +1,112 @@ +/** + * useDictationHotkey + * + * Fetches dictation config from the core RPC on mount and listens for + * `dictation:toggle` Socket.IO events emitted by the Rust core when + * the global hotkey is pressed. The hotkey listener runs in the core + * process (via rdev), not in the Tauri shell. + * + * Consumers receive: + * - `dictationEnabled`: whether dictation is configured on + * - `hotkeyRegistered`: true once the core confirms the hotkey is active + * - `toggleCount`: increments each time the hotkey fires (use to trigger effects) + * - `activationMode`: "toggle" or "push" + * - `hotkey`: the configured hotkey string + */ +import { useEffect, useState } from 'react'; + +import { callCoreRpc } from '../services/coreRpcClient'; +import { socketService } from '../services/socketService'; + +interface DictationSettings { + enabled: boolean; + hotkey: string; + activation_mode: string; + llm_refinement: boolean; + streaming: boolean; + streaming_interval_ms: number; +} + +export interface DictationHotkeyState { + /** Whether dictation is enabled in the core config. */ + dictationEnabled: boolean; + /** Whether the core hotkey listener is active. */ + hotkeyRegistered: boolean; + /** Increments each time the hotkey is pressed (consumers can use as a trigger). */ + toggleCount: number; + /** The configured activation mode ("toggle" or "push"). */ + activationMode: string; + /** The configured hotkey string. */ + hotkey: string; +} + +export function useDictationHotkey(): DictationHotkeyState { + const [dictationEnabled, setDictationEnabled] = useState(false); + const [hotkeyRegistered, setHotkeyRegistered] = useState(false); + const [toggleCount, setToggleCount] = useState(0); + const [activationMode, setActivationMode] = useState('toggle'); + const [hotkey, setHotkey] = useState(''); + + // Fetch config from core RPC on mount. + useEffect(() => { + let disposed = false; + + const init = async () => { + try { + const settings = await callCoreRpc({ + method: 'openhuman.config_get_dictation_settings', + }); + + if (disposed) return; + + if (!settings || typeof settings !== 'object') { + console.debug('[dictation] no dictation settings from core'); + return; + } + + // Handle RpcOutcome wrapper — the result may be nested in .result + const s = ( + 'result' in settings ? (settings as Record).result : settings + ) as DictationSettings; + + setDictationEnabled(s.enabled); + setActivationMode(s.activation_mode ?? 'toggle'); + setHotkey(s.hotkey ?? ''); + + if (s.enabled && s.hotkey) { + // The core process registers the hotkey via rdev — we just note it. + setHotkeyRegistered(true); + console.debug(`[dictation] core hotkey active: ${s.hotkey}`); + } else { + console.debug('[dictation] dictation disabled or no hotkey configured'); + } + } catch (err) { + console.warn('[dictation] failed to fetch dictation settings', err); + } + }; + + void init(); + + return () => { + disposed = true; + }; + }, []); + + // Listen for hotkey events from the core via Socket.IO. + useEffect(() => { + const handleToggle = () => { + console.debug('[dictation] hotkey toggle event received via socket'); + setToggleCount(c => c + 1); + }; + + socketService.on('dictation:toggle', handleToggle); + socketService.on('dictation_toggle', handleToggle); + + return () => { + socketService.off('dictation:toggle', handleToggle); + socketService.off('dictation_toggle', handleToggle); + }; + }, []); + + return { dictationEnabled, hotkeyRegistered, toggleCount, activationMode, hotkey }; +} diff --git a/app/src/pages/Conversations.tsx b/app/src/pages/Conversations.tsx index 11f82be29..28a7ac9e6 100644 --- a/app/src/pages/Conversations.tsx +++ b/app/src/pages/Conversations.tsx @@ -331,9 +331,8 @@ const Conversations = () => { let cancelled = false; void (async () => { try { - const resp = await openhumanVoiceStatus(); + const status = await openhumanVoiceStatus(); if (cancelled) return; - const status = resp.result; if (!status.stt_available) { setVoiceStatus( 'Speech-to-text unavailable: whisper-cli binary or STT model not found. Check Settings > Local Models.' @@ -688,7 +687,7 @@ const Conversations = () => { : undefined; const result = await openhumanVoiceTranscribeBytes(audioBytes, extension, context); - const transcript = result.result.text.trim(); + const transcript = result.text.trim(); if (!transcript) { setVoiceStatus('No speech detected. Try again.'); @@ -790,7 +789,7 @@ const Conversations = () => { const ttsResult = await openhumanVoiceTts(latestAgentMessage.content); if (cancelled) return; - const audioSrc = convertFileSrc(ttsResult.result.output_path); + const audioSrc = convertFileSrc(ttsResult.output_path); const audio = new window.Audio(audioSrc); replyAudioRef.current?.pause(); replyAudioRef.current = audio; diff --git a/app/src/utils/tauriCommands.ts b/app/src/utils/tauriCommands.ts index e7a5792e8..b71e49cf1 100644 --- a/app/src/utils/tauriCommands.ts +++ b/app/src/utils/tauriCommands.ts @@ -2456,19 +2456,16 @@ export interface VoiceStatus { llm_cleanup_enabled: boolean; } -export async function openhumanVoiceStatus(): Promise> { - return await callCoreRpc>({ - method: 'openhuman.voice_status', - params: {}, - }); +export async function openhumanVoiceStatus(): Promise { + return await callCoreRpc({ method: 'openhuman.voice_status', params: {} }); } export async function openhumanVoiceTranscribe( audioPath: string, context?: string, skipCleanup?: boolean -): Promise> { - return await callCoreRpc>({ +): Promise { + return await callCoreRpc({ method: 'openhuman.voice_transcribe', params: { audio_path: audioPath, context, skip_cleanup: skipCleanup }, }); @@ -2479,8 +2476,8 @@ export async function openhumanVoiceTranscribeBytes( extension?: string, context?: string, skipCleanup?: boolean -): Promise> { - return await callCoreRpc>({ +): Promise { + return await callCoreRpc({ method: 'openhuman.voice_transcribe_bytes', params: { audio_bytes: audioBytes, extension, context, skip_cleanup: skipCleanup }, }); @@ -2489,8 +2486,8 @@ export async function openhumanVoiceTranscribeBytes( export async function openhumanVoiceTts( text: string, outputPath?: string -): Promise> { - return await callCoreRpc>({ +): Promise { + return await callCoreRpc({ method: 'openhuman.voice_tts', params: { text, output_path: outputPath }, }); diff --git a/src/core/cli.rs b/src/core/cli.rs index f66776462..980be4b7f 100644 --- a/src/core/cli.rs +++ b/src/core/cli.rs @@ -58,6 +58,7 @@ pub fn run_from_cli_args(args: &[String]) -> Result<()> { "screen-intelligence" => { crate::core::screen_intelligence_cli::run_screen_intelligence_command(&args[1..]) } + "voice" | "dictate" => run_voice_server_command(&args[1..]), "text-input" => crate::core::text_input_cli::run_text_input_command(&args[1..]), namespace => run_namespace_command(namespace, &args[1..], &grouped), } @@ -189,6 +190,98 @@ fn run_call_command(args: &[String]) -> Result<()> { Ok(()) } +/// Handles the `voice` subcommand to run the standalone voice dictation server. +/// +/// Listens for a hotkey, records audio, transcribes via whisper, and inserts +/// the result into the active text field. + +fn run_voice_server_command(args: &[String]) -> Result<()> { + use crate::openhuman::voice::hotkey::ActivationMode; + use crate::openhuman::voice::server::{run_standalone, VoiceServerConfig}; + + let mut hotkey: Option = None; + let mut mode: Option = None; + let mut skip_cleanup = false; + let mut verbose = false; + let mut i = 0usize; + + while i < args.len() { + match args[i].as_str() { + "--hotkey" => { + hotkey = Some( + args.get(i + 1) + .ok_or_else(|| anyhow::anyhow!("missing value for --hotkey"))? + .clone(), + ); + i += 2; + } + "--mode" => { + mode = Some( + args.get(i + 1) + .ok_or_else(|| anyhow::anyhow!("missing value for --mode"))? + .clone(), + ); + i += 2; + } + "--skip-cleanup" => { + skip_cleanup = true; + i += 1; + } + "-v" | "--verbose" => { + verbose = true; + i += 1; + } + "-h" | "--help" => { + println!("Usage: openhuman voice [--hotkey ] [--mode ] [--skip-cleanup] [-v]"); + println!(); + println!(" --hotkey Key combination (default: ctrl+shift+space)"); + println!( + " --mode Activation: tap to toggle, push to hold (default: tap)" + ); + println!(" --skip-cleanup Skip LLM post-processing on transcriptions"); + println!(" -v, --verbose Enable debug logging"); + println!(); + println!("Standalone voice dictation server. Press the hotkey to dictate,"); + println!("transcribed text is inserted into the active text field."); + return Ok(()); + } + other => return Err(anyhow::anyhow!("unknown voice arg: {other}")), + } + } + + crate::core::logging::init_for_cli_run(verbose); + + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + + rt.block_on(async { + let mut config = crate::openhuman::config::Config::load_or_init() + .await + .unwrap_or_default(); + config.apply_env_overrides(); + + let activation_mode = match mode.as_deref() { + Some("push") => ActivationMode::Push, + _ => ActivationMode::Tap, + }; + + let server_config = VoiceServerConfig { + hotkey: hotkey.unwrap_or_else(|| config.voice_server.hotkey.clone()), + activation_mode, + skip_cleanup, + context: None, + min_duration_secs: config.voice_server.min_duration_secs, + }; + + run_standalone(config, server_config) + .await + .map_err(anyhow::Error::msg) + })?; + + Ok(()) +} + /// Dispatches commands that fall under a specific namespace (e.g., `openhuman `). /// /// It looks up the function schema for validation and executes the request. @@ -430,6 +523,7 @@ fn print_general_help(grouped: &BTreeMap>) { println!(" openhuman repl [--verbose] [--eval ''] [--batch]"); println!(" openhuman call --method [--params '']"); println!(" openhuman skills [options] (skill development runtime)"); + println!(" openhuman voice [--hotkey ] [--mode ] (voice dictation server)"); println!(" openhuman [--param value ...]\n"); println!("Available namespaces:"); for namespace in grouped.keys() { diff --git a/src/core/jsonrpc.rs b/src/core/jsonrpc.rs index 1ce6f7ff0..e9abe72f5 100644 --- a/src/core/jsonrpc.rs +++ b/src/core/jsonrpc.rs @@ -6,7 +6,9 @@ //! - SSE (Server-Sent Events) for real-time event streaming. //! - Helper routes for health checks, schema discovery, and Telegram authentication. -use axum::extract::{Query, State}; +use std::sync::Arc; + +use axum::extract::{Query, State, WebSocketUpgrade}; use axum::http::{header, HeaderValue, Method, StatusCode}; use axum::middleware::{self, Next}; use axum::response::sse::{Event, KeepAlive, Sse}; @@ -326,6 +328,21 @@ async fn telegram_auth_handler(Query(query): Query) -> impl I html_response(StatusCode::OK, success_html()) } +/// WebSocket upgrade handler for streaming voice dictation. +async fn dictation_ws_handler(ws: WebSocketUpgrade) -> Response { + log::info!("[ws] dictation WebSocket upgrade requested"); + ws.on_upgrade(|socket| async move { + let config = match crate::openhuman::config::rpc::load_config_with_timeout().await { + Ok(c) => Arc::new(c), + Err(e) => { + log::error!("[ws] failed to load config for dictation: {e}"); + return; + } + }; + crate::openhuman::voice::streaming::handle_dictation_ws(socket, config).await; + }) +} + /// Builds the main Axum router for the core HTTP server. /// /// Includes routes for health, schema, SSE events, JSON-RPC, and Telegram auth. @@ -338,6 +355,7 @@ pub fn build_core_http_router(socketio_enabled: bool) -> Router { .route("/events", get(events_handler)) .route("/events/webhooks", get(webhook_events_handler)) .route("/rpc", post(rpc_handler)) + .route("/ws/dictation", get(dictation_ws_handler)) .route("/auth/telegram", get(telegram_auth_handler)) .fallback(not_found_handler) .layer(middleware::from_fn(http_request_log_middleware)) @@ -628,6 +646,10 @@ pub async fn run_server( } else { log::info!("[overlay] overlay disabled by config (overlay_enabled = false)"); } + + // Start the global dictation hotkey listener (rdev-based, core-side). + + crate::openhuman::voice::dictation_listener::start_if_enabled(&config).await; } Err(err) => { log::warn!("[core] config load failed, skipping local-ai and overlay: {err}"); diff --git a/src/core/socketio.rs b/src/core/socketio.rs index 457b73081..8b97a107b 100644 --- a/src/core/socketio.rs +++ b/src/core/socketio.rs @@ -196,6 +196,8 @@ pub fn attach_socketio() -> (socketioxide::layer::SocketIoLayer, SocketIo) { } pub fn spawn_web_channel_bridge(io: SocketIo) { + // Web channel events → per-client rooms. + let io_web = io.clone(); tokio::spawn(async move { let mut rx = crate::openhuman::channels::providers::web::subscribe_web_channel_events(); loop { @@ -211,10 +213,35 @@ pub fn spawn_web_channel_bridge(io: SocketIo) { Err(tokio::sync::broadcast::error::RecvError::Closed) => break, }; - emit_web_channel_event(&io, event); + emit_web_channel_event(&io_web, event); } log::debug!("[socketio] web_channel bridge stopped"); }); + + // Dictation hotkey events → broadcast to all connected clients. + tokio::spawn(async move { + let mut rx = crate::openhuman::voice::dictation_listener::subscribe_dictation_events(); + loop { + let event = match rx.recv().await { + Ok(event) => event, + Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { + log::warn!("[socketio] dropped {} dictation events due to lag", skipped); + continue; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + }; + + if let Ok(payload) = serde_json::to_value(&event) { + log::debug!( + "[socketio] broadcast dictation:{} to all clients", + event.event_type + ); + let _ = io.emit("dictation:toggle", &payload); + let _ = io.emit("dictation_toggle", &payload); + } + } + log::debug!("[socketio] dictation bridge stopped"); + }); } fn emit_web_channel_event(io: &SocketIo, event: WebChannelEvent) { diff --git a/src/openhuman/config/mod.rs b/src/openhuman/config/mod.rs index b12c74fb3..b2038ca5c 100644 --- a/src/openhuman/config/mod.rs +++ b/src/openhuman/config/mod.rs @@ -26,17 +26,17 @@ pub use schema::{ build_runtime_proxy_client_with_timeouts, runtime_proxy_config, set_runtime_proxy_config, AgentConfig, ArchetypeConfig, AuditConfig, AutocompleteConfig, AutonomyConfig, BrowserComputerUseConfig, BrowserConfig, ChannelsConfig, ClassificationRule, ComposioConfig, - Config, CostConfig, CronConfig, DelegateAgentConfig, DiscordConfig, DockerRuntimeConfig, - EmbeddingRouteConfig, HardwareConfig, HardwareTransport, HeartbeatConfig, HttpRequestConfig, - IMessageConfig, IdentityConfig, IntegrationToggle, IntegrationsConfig, LarkConfig, - LearningConfig, LocalAiConfig, MatrixConfig, MemoryConfig, ModelRouteConfig, MultimodalConfig, - ObservabilityConfig, OrchestratorConfig, PeripheralBoardConfig, PeripheralsConfig, ProxyConfig, - ProxyScope, QueryClassificationConfig, ReflectionSource, ReliabilityConfig, - ResourceLimitsConfig, RuntimeConfig, SandboxBackend, SandboxConfig, SchedulerConfig, - ScreenIntelligenceConfig, SecretsConfig, SecurityConfig, SlackConfig, StorageConfig, - StorageProviderConfig, StorageProviderSection, StreamMode, TelegramConfig, UpdateConfig, - WebSearchConfig, WebhookConfig, DEFAULT_MODEL, MODEL_AGENTIC_V1, MODEL_CODING_V1, - MODEL_REASONING_V1, + Config, CostConfig, CronConfig, DelegateAgentConfig, DictationActivationMode, DictationConfig, + DiscordConfig, DockerRuntimeConfig, EmbeddingRouteConfig, HardwareConfig, HardwareTransport, + HeartbeatConfig, HttpRequestConfig, IMessageConfig, IdentityConfig, IntegrationToggle, + IntegrationsConfig, LarkConfig, LearningConfig, LocalAiConfig, MatrixConfig, MemoryConfig, + ModelRouteConfig, MultimodalConfig, ObservabilityConfig, OrchestratorConfig, + PeripheralBoardConfig, PeripheralsConfig, ProxyConfig, ProxyScope, QueryClassificationConfig, + ReflectionSource, ReliabilityConfig, ResourceLimitsConfig, RuntimeConfig, SandboxBackend, + SandboxConfig, SchedulerConfig, ScreenIntelligenceConfig, SecretsConfig, SecurityConfig, + SlackConfig, StorageConfig, StorageProviderConfig, StorageProviderSection, StreamMode, + TelegramConfig, UpdateConfig, VoiceActivationMode, VoiceServerConfig, WebSearchConfig, + WebhookConfig, DEFAULT_MODEL, MODEL_AGENTIC_V1, MODEL_CODING_V1, MODEL_REASONING_V1, }; pub use schema::{ clear_active_user, default_root_openhuman_dir, read_active_user_id, user_openhuman_dir, diff --git a/src/openhuman/config/ops.rs b/src/openhuman/config/ops.rs index d34bd4c7b..fa0bdb499 100644 --- a/src/openhuman/config/ops.rs +++ b/src/openhuman/config/ops.rs @@ -534,6 +534,80 @@ pub async fn set_onboarding_completed(value: bool) -> Result, S )) } +// ── Dictation settings ─────────────────────────────────────────────── + +pub struct DictationSettingsPatch { + pub enabled: Option, + pub hotkey: Option, + pub activation_mode: Option, + pub llm_refinement: Option, + pub streaming: Option, + pub streaming_interval_ms: Option, +} + +pub async fn get_dictation_settings() -> Result, String> { + let config = load_config_with_timeout().await?; + let result = json!({ + "enabled": config.dictation.enabled, + "hotkey": config.dictation.hotkey, + "activation_mode": config.dictation.activation_mode, + "llm_refinement": config.dictation.llm_refinement, + "streaming": config.dictation.streaming, + "streaming_interval_ms": config.dictation.streaming_interval_ms, + }); + Ok(RpcOutcome::new( + result, + vec!["dictation settings read".to_string()], + )) +} + +pub async fn load_and_apply_dictation_settings( + update: DictationSettingsPatch, +) -> Result, String> { + let mut config = load_config_with_timeout().await?; + if let Some(enabled) = update.enabled { + config.dictation.enabled = enabled; + } + if let Some(hotkey) = update.hotkey { + config.dictation.hotkey = hotkey; + } + if let Some(mode) = update.activation_mode { + match mode.as_str() { + "toggle" => { + config.dictation.activation_mode = + crate::openhuman::config::DictationActivationMode::Toggle; + } + "push" => { + config.dictation.activation_mode = + crate::openhuman::config::DictationActivationMode::Push; + } + _ => { + return Err(format!( + "invalid activation_mode: {mode} (valid: toggle, push)" + )) + } + } + } + if let Some(llm_refinement) = update.llm_refinement { + config.dictation.llm_refinement = llm_refinement; + } + if let Some(streaming) = update.streaming { + config.dictation.streaming = streaming; + } + if let Some(interval) = update.streaming_interval_ms { + config.dictation.streaming_interval_ms = interval; + } + config.save().await.map_err(|e| e.to_string())?; + let snapshot = snapshot_config_json(&config)?; + Ok(RpcOutcome::new( + snapshot, + vec![format!( + "dictation settings saved to {}", + config.config_path.display() + )], + )) +} + pub fn agent_server_status() -> RpcOutcome { let running = crate::openhuman::service::mock::mock_agent_running().unwrap_or(true); log::info!("[config] agent_server_status requested: running={running}"); diff --git a/src/openhuman/config/schema/dictation.rs b/src/openhuman/config/schema/dictation.rs new file mode 100644 index 000000000..4f91445b3 --- /dev/null +++ b/src/openhuman/config/schema/dictation.rs @@ -0,0 +1,81 @@ +//! Voice dictation configuration. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Activation mode for the dictation hotkey. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum DictationActivationMode { + /// Press once to start, press again to stop. + Toggle, + /// Hold to record, release to stop (push-to-talk). + Push, +} + +impl Default for DictationActivationMode { + fn default() -> Self { + Self::Toggle + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct DictationConfig { + /// Whether voice dictation is enabled. + #[serde(default = "default_enabled")] + pub enabled: bool, + + /// Global hotkey for activating dictation (e.g. "CmdOrCtrl+Shift+D"). + #[serde(default = "default_hotkey")] + pub hotkey: String, + + /// Activation mode: "toggle" (press to start/stop) or "push" (hold to record). + #[serde(default)] + pub activation_mode: DictationActivationMode, + + /// Whether to refine raw transcription through a local LLM for grammar/punctuation. + #[serde(default = "default_llm_refinement")] + pub llm_refinement: bool, + + /// Whether to use WebSocket streaming transcription (chunks sent in real-time) + /// instead of batch transcription after recording stops. + #[serde(default = "default_streaming")] + pub streaming: bool, + + /// Interval in milliseconds between streaming inference passes on accumulated audio. + #[serde(default = "default_streaming_interval_ms")] + pub streaming_interval_ms: u64, +} + +fn default_enabled() -> bool { + true +} + +fn default_hotkey() -> String { + "CmdOrCtrl+Shift+D".to_string() +} + +fn default_llm_refinement() -> bool { + true +} + +fn default_streaming() -> bool { + true +} + +fn default_streaming_interval_ms() -> u64 { + 2000 +} + +impl Default for DictationConfig { + fn default() -> Self { + Self { + enabled: default_enabled(), + hotkey: default_hotkey(), + activation_mode: DictationActivationMode::default(), + llm_refinement: default_llm_refinement(), + streaming: default_streaming(), + streaming_interval_ms: default_streaming_interval_ms(), + } + } +} diff --git a/src/openhuman/config/schema/load.rs b/src/openhuman/config/schema/load.rs index 5d52fd3a0..c359eebb8 100644 --- a/src/openhuman/config/schema/load.rs +++ b/src/openhuman/config/schema/load.rs @@ -776,6 +776,62 @@ impl Config { } } + // Dictation overrides + if let Ok(flag) = std::env::var("OPENHUMAN_DICTATION_ENABLED") { + let normalized = flag.trim().to_ascii_lowercase(); + match normalized.as_str() { + "1" | "true" | "yes" | "on" => self.dictation.enabled = true, + "0" | "false" | "no" | "off" => self.dictation.enabled = false, + _ => {} + } + } + if let Ok(hotkey) = std::env::var("OPENHUMAN_DICTATION_HOTKEY") { + let hotkey = hotkey.trim(); + if !hotkey.is_empty() { + self.dictation.hotkey = hotkey.to_string(); + } + } + if let Ok(mode) = std::env::var("OPENHUMAN_DICTATION_ACTIVATION_MODE") { + let normalized = mode.trim().to_ascii_lowercase(); + match normalized.as_str() { + "toggle" => { + self.dictation.activation_mode = + crate::openhuman::config::DictationActivationMode::Toggle + } + "push" => { + self.dictation.activation_mode = + crate::openhuman::config::DictationActivationMode::Push + } + _ => { + tracing::warn!( + mode = %mode, + "ignoring invalid OPENHUMAN_DICTATION_ACTIVATION_MODE (valid: toggle, push)" + ); + } + } + } + if let Ok(flag) = std::env::var("OPENHUMAN_DICTATION_LLM_REFINEMENT") { + let normalized = flag.trim().to_ascii_lowercase(); + match normalized.as_str() { + "1" | "true" | "yes" | "on" => self.dictation.llm_refinement = true, + "0" | "false" | "no" | "off" => self.dictation.llm_refinement = false, + _ => {} + } + } + if let Ok(flag) = std::env::var("OPENHUMAN_DICTATION_STREAMING") { + let normalized = flag.trim().to_ascii_lowercase(); + match normalized.as_str() { + "1" | "true" | "yes" | "on" => self.dictation.streaming = true, + "0" | "false" | "no" | "off" => self.dictation.streaming = false, + _ => {} + } + } + if let Ok(val) = std::env::var("OPENHUMAN_DICTATION_STREAMING_INTERVAL_MS") { + if let Ok(ms) = val.trim().parse::() { + self.dictation.streaming_interval_ms = ms; + } + } + if let Ok(flag) = std::env::var("OPENHUMAN_OVERLAY_ENABLED") { let normalized = flag.trim().to_ascii_lowercase(); match normalized.as_str() { diff --git a/src/openhuman/config/schema/mod.rs b/src/openhuman/config/schema/mod.rs index 3fbc6d722..c2611bd9c 100644 --- a/src/openhuman/config/schema/mod.rs +++ b/src/openhuman/config/schema/mod.rs @@ -9,6 +9,7 @@ mod autonomy; mod channels; mod defaults; pub(crate) use defaults::default_true; +mod dictation; mod hardware; mod heartbeat_cron; mod identity_cost; @@ -38,6 +39,7 @@ pub use channels::{ SandboxBackend, SandboxConfig, SecurityConfig, SignalConfig, SlackConfig, StreamMode, TelegramConfig, WebhookConfig, WhatsAppConfig, }; +pub use dictation::{DictationActivationMode, DictationConfig}; pub use hardware::{HardwareConfig, HardwareTransport}; pub use heartbeat_cron::{CronConfig, HeartbeatConfig}; pub use identity_cost::{ @@ -64,5 +66,7 @@ pub use tools::{ IntegrationsConfig, MultimodalConfig, SecretsConfig, WebSearchConfig, }; pub use update::UpdateConfig; +mod voice_server; +pub use voice_server::{VoiceActivationMode, VoiceServerConfig}; mod types; pub use types::*; diff --git a/src/openhuman/config/schema/types.rs b/src/openhuman/config/schema/types.rs index 5412fb188..0bb93df28 100644 --- a/src/openhuman/config/schema/types.rs +++ b/src/openhuman/config/schema/types.rs @@ -112,6 +112,9 @@ pub struct Config { #[serde(default)] pub local_ai: LocalAiConfig, + #[serde(default)] + pub voice_server: VoiceServerConfig, + #[serde(default)] pub integrations: IntegrationsConfig, @@ -124,6 +127,9 @@ pub struct Config { #[serde(default)] pub update: UpdateConfig, + #[serde(default)] + pub dictation: DictationConfig, + /// Whether to launch the overlay Tauri app (floating debug/voice panel) /// when the core RPC server starts. Defaults to `true`. #[serde(default = "default_true")] @@ -175,11 +181,13 @@ impl Default for Config { agents: HashMap::new(), hardware: HardwareConfig::default(), local_ai: LocalAiConfig::default(), + voice_server: VoiceServerConfig::default(), query_classification: QueryClassificationConfig::default(), integrations: IntegrationsConfig::default(), learning: LearningConfig::default(), orchestrator: OrchestratorConfig::default(), update: UpdateConfig::default(), + dictation: DictationConfig::default(), overlay_enabled: true, onboarding_completed: false, } diff --git a/src/openhuman/config/schema/voice_server.rs b/src/openhuman/config/schema/voice_server.rs new file mode 100644 index 000000000..bb9e06046 --- /dev/null +++ b/src/openhuman/config/schema/voice_server.rs @@ -0,0 +1,65 @@ +//! Voice server configuration. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Activation mode for the voice server hotkey. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum VoiceActivationMode { + /// Single press toggles recording on/off. + Tap, + /// Hold to record, release to stop. + Push, +} + +impl Default for VoiceActivationMode { + fn default() -> Self { + Self::Tap + } +} + +/// Configuration for the voice dictation server. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct VoiceServerConfig { + /// Whether the voice server should start automatically with the core. + #[serde(default)] + pub auto_start: bool, + + /// Hotkey combination to trigger recording (e.g. "ctrl+shift+space"). + #[serde(default = "default_hotkey")] + pub hotkey: String, + + /// Activation mode: "tap" (toggle) or "push" (hold-to-record). + #[serde(default)] + pub activation_mode: VoiceActivationMode, + + /// Skip LLM post-processing for transcriptions. + #[serde(default)] + pub skip_cleanup: bool, + + /// Minimum recording duration in seconds. Recordings shorter than + /// this are discarded. + #[serde(default = "default_min_duration")] + pub min_duration_secs: f32, +} + +fn default_hotkey() -> String { + "ctrl+shift+space".to_string() +} + +fn default_min_duration() -> f32 { + 0.3 +} + +impl Default for VoiceServerConfig { + fn default() -> Self { + Self { + auto_start: false, + hotkey: default_hotkey(), + activation_mode: VoiceActivationMode::default(), + skip_cleanup: false, + min_duration_secs: default_min_duration(), + } + } +} diff --git a/src/openhuman/config/schemas.rs b/src/openhuman/config/schemas.rs index 5eb93d2e7..75c8e20c7 100644 --- a/src/openhuman/config/schemas.rs +++ b/src/openhuman/config/schemas.rs @@ -76,6 +76,16 @@ struct OnboardingCompletedSetParams { value: bool, } +#[derive(Debug, Deserialize)] +struct DictationSettingsUpdate { + enabled: Option, + hotkey: Option, + activation_mode: Option, + llm_refinement: Option, + streaming: Option, + streaming_interval_ms: Option, +} + pub fn all_controller_schemas() -> Vec { vec![ schemas("get_config"), @@ -95,6 +105,8 @@ pub fn all_controller_schemas() -> Vec { schemas("reset_local_data"), schemas("get_onboarding_completed"), schemas("set_onboarding_completed"), + schemas("get_dictation_settings"), + schemas("update_dictation_settings"), ] } @@ -168,6 +180,14 @@ pub fn all_registered_controllers() -> Vec { schema: schemas("set_onboarding_completed"), handler: handle_set_onboarding_completed, }, + RegisteredController { + schema: schemas("get_dictation_settings"), + handler: handle_get_dictation_settings, + }, + RegisteredController { + schema: schemas("update_dictation_settings"), + handler: handle_update_dictation_settings, + }, ] } @@ -410,6 +430,32 @@ pub fn schemas(function: &str) -> ControllerSchema { required: true, }], }, + "get_dictation_settings" => ControllerSchema { + namespace: "config", + function: "get_dictation_settings", + description: "Read current voice dictation settings.", + inputs: vec![], + outputs: vec![json_output("settings", "Dictation settings payload.")], + }, + "update_dictation_settings" => ControllerSchema { + namespace: "config", + function: "update_dictation_settings", + description: "Update voice dictation settings.", + inputs: vec![ + optional_bool("enabled", "Enable voice dictation."), + optional_string("hotkey", "Global hotkey string (e.g. CmdOrCtrl+Shift+D)."), + optional_string("activation_mode", "Activation mode: toggle or push."), + optional_bool("llm_refinement", "Enable LLM post-processing of transcription."), + optional_bool("streaming", "Enable WebSocket streaming transcription."), + FieldSchema { + name: "streaming_interval_ms", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Interval between streaming inference passes (ms).", + required: false, + }, + ], + outputs: vec![json_output("snapshot", "Updated config snapshot.")], + }, "set_onboarding_completed" => ControllerSchema { namespace: "config", function: "set_onboarding_completed", @@ -589,6 +635,25 @@ fn handle_get_onboarding_completed(_params: Map) -> ControllerFut Box::pin(async { to_json(config_rpc::get_onboarding_completed().await?) }) } +fn handle_get_dictation_settings(_params: Map) -> ControllerFuture { + Box::pin(async { to_json(config_rpc::get_dictation_settings().await?) }) +} + +fn handle_update_dictation_settings(params: Map) -> ControllerFuture { + Box::pin(async move { + let update = deserialize_params::(params)?; + let patch = config_rpc::DictationSettingsPatch { + enabled: update.enabled, + hotkey: update.hotkey, + activation_mode: update.activation_mode, + llm_refinement: update.llm_refinement, + streaming: update.streaming, + streaming_interval_ms: update.streaming_interval_ms, + }; + to_json(config_rpc::load_and_apply_dictation_settings(patch).await?) + }) +} + fn handle_set_onboarding_completed(params: Map) -> ControllerFuture { Box::pin(async move { let payload = deserialize_params::(params)?; diff --git a/src/openhuman/voice/audio_capture.rs b/src/openhuman/voice/audio_capture.rs new file mode 100644 index 000000000..25630e94a --- /dev/null +++ b/src/openhuman/voice/audio_capture.rs @@ -0,0 +1,387 @@ +//! Microphone audio capture using cpal. +//! +//! Records audio from the default input device and produces 16-kHz mono WAV +//! bytes suitable for whisper transcription. + +use std::io::Cursor; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use cpal::{SampleFormat, SampleRate, StreamConfig}; +use hound::{SampleFormat as HoundFormat, WavSpec, WavWriter}; +use log::{debug, error, info, warn}; +use tokio::sync::oneshot; + +const LOG_PREFIX: &str = "[voice_capture]"; + +/// Target sample rate for whisper (16 kHz mono). +const TARGET_SAMPLE_RATE: u32 = 16_000; + +/// Result of a completed recording. +#[derive(Debug, Clone)] +pub struct RecordingResult { + /// WAV-encoded audio bytes (16 kHz, mono, 16-bit PCM). + pub wav_bytes: Vec, + /// Duration of the recording in seconds. + pub duration_secs: f32, + /// Number of samples captured. + pub sample_count: usize, +} + +/// Handle to a recording in progress. Drop or call `stop()` to end recording. +pub struct RecordingHandle { + stop_flag: Arc, + result_rx: Option>>, +} + +impl RecordingHandle { + /// Signal the recording to stop and return the captured audio. + pub async fn stop(mut self) -> Result { + self.stop_flag.store(true, Ordering::SeqCst); + debug!("{LOG_PREFIX} stop signal sent"); + + match self.result_rx.take() { + Some(rx) => rx + .await + .map_err(|_| "recording task dropped before completing".to_string())?, + None => Err("recording already stopped".to_string()), + } + } +} + +/// Start recording from the default microphone. +/// +/// Returns a `RecordingHandle` that must be `.stop().await`-ed to get +/// the captured audio. Recording runs on a dedicated OS thread because +/// `cpal::Stream` is `!Send` (it must be created and dropped on the +/// same thread). +pub fn start_recording() -> Result { + let stop_flag = Arc::new(AtomicBool::new(false)); + let stop_flag_clone = stop_flag.clone(); + let (result_tx, result_rx) = oneshot::channel(); + + // Use a oneshot to report whether stream setup succeeded. + let (setup_tx, setup_rx) = std::sync::mpsc::sync_channel::>(1); + + std::thread::Builder::new() + .name("voice-capture".into()) + .spawn(move || { + // All cpal objects are created and used on this thread. + let result = record_on_thread(stop_flag_clone, setup_tx); + let _ = result_tx.send(result); + }) + .map_err(|e| format!("failed to spawn capture thread: {e}"))?; + + // Wait for the stream to be set up (or an error). + match setup_rx.recv() { + Ok(Ok(())) => { + info!("{LOG_PREFIX} recording started"); + Ok(RecordingHandle { + stop_flag, + result_rx: Some(result_rx), + }) + } + Ok(Err(e)) => Err(e), + Err(_) => Err("capture thread exited before signalling readiness".to_string()), + } +} + +/// Runs the entire recording lifecycle on a single thread (cpal requirement). +fn record_on_thread( + stop_flag: Arc, + setup_tx: std::sync::mpsc::SyncSender>, +) -> Result { + let host = cpal::default_host(); + let device = host + .default_input_device() + .ok_or_else(|| "no default audio input device found".to_string())?; + + let device_name = device.name().unwrap_or_else(|_| "".into()); + info!("{LOG_PREFIX} using input device: {device_name}"); + + let supported_configs = device + .supported_input_configs() + .map_err(|e| format!("failed to query input configs: {e}"))?; + + let config = find_best_config(supported_configs)?; + let source_sample_rate = config.sample_rate().0; + let source_channels = config.channels() as usize; + + debug!( + "{LOG_PREFIX} recording config: rate={source_sample_rate} channels={source_channels} format={:?}", + config.sample_format() + ); + + let samples: Arc>> = Arc::new(parking_lot::Mutex::new( + Vec::with_capacity(TARGET_SAMPLE_RATE as usize * 30), + )); + + let sample_format = config.sample_format(); + let stream_config: StreamConfig = config.into(); + + let stream = { + let samples_writer = samples.clone(); + match sample_format { + SampleFormat::F32 => device + .build_input_stream( + &stream_config, + move |data: &[f32], _: &cpal::InputCallbackInfo| { + let mono = to_mono(data, source_channels); + samples_writer.lock().extend_from_slice(&mono); + }, + |err| error!("{LOG_PREFIX} audio stream error: {err}"), + None, + ) + .map_err(|e| format!("failed to build f32 input stream: {e}")), + SampleFormat::I16 => device + .build_input_stream( + &stream_config, + move |data: &[i16], _: &cpal::InputCallbackInfo| { + let floats: Vec = data.iter().map(|&s| s as f32 / 32768.0).collect(); + let mono = to_mono(&floats, source_channels); + samples_writer.lock().extend_from_slice(&mono); + }, + |err| error!("{LOG_PREFIX} audio stream error: {err}"), + None, + ) + .map_err(|e| format!("failed to build i16 input stream: {e}")), + SampleFormat::U16 => device + .build_input_stream( + &stream_config, + move |data: &[u16], _: &cpal::InputCallbackInfo| { + let floats: Vec = data + .iter() + .map(|&s| (s as f32 - 32768.0) / 32768.0) + .collect(); + let mono = to_mono(&floats, source_channels); + samples_writer.lock().extend_from_slice(&mono); + }, + |err| error!("{LOG_PREFIX} audio stream error: {err}"), + None, + ) + .map_err(|e| format!("failed to build u16 input stream: {e}")), + other => Err(format!("unsupported sample format: {other:?}")), + } + }; + + let stream = match stream { + Ok(s) => s, + Err(e) => { + let _ = setup_tx.send(Err(e.clone())); + return Err(e); + } + }; + + if let Err(e) = stream.play() { + let msg = format!("failed to start audio stream: {e}"); + let _ = setup_tx.send(Err(msg.clone())); + return Err(msg); + } + + // Signal success so start_recording() returns. + let _ = setup_tx.send(Ok(())); + + // Poll stop flag while keeping the stream alive on this thread. + while !stop_flag.load(Ordering::SeqCst) { + std::thread::sleep(std::time::Duration::from_millis(50)); + } + + debug!("{LOG_PREFIX} stop flag detected, finalizing recording"); + drop(stream); + + let raw_samples = samples.lock().clone(); + finalize_recording(raw_samples, source_sample_rate) +} + +/// List available input devices. +pub fn list_input_devices() -> Result, String> { + let host = cpal::default_host(); + let devices = host + .input_devices() + .map_err(|e| format!("failed to enumerate input devices: {e}"))?; + + let names: Vec = devices.filter_map(|d| d.name().ok()).collect(); + + debug!("{LOG_PREFIX} found {} input devices", names.len()); + Ok(names) +} + +/// Convert interleaved multi-channel samples to mono by averaging channels. +fn to_mono(samples: &[f32], channels: usize) -> Vec { + if channels <= 1 { + return samples.to_vec(); + } + + samples + .chunks_exact(channels) + .map(|frame| frame.iter().sum::() / channels as f32) + .collect() +} + +/// Resample mono f32 samples from `source_rate` to `TARGET_SAMPLE_RATE` using +/// linear interpolation. Good enough for voice dictation quality. +fn resample(samples: &[f32], source_rate: u32) -> Vec { + if source_rate == TARGET_SAMPLE_RATE { + return samples.to_vec(); + } + + let ratio = source_rate as f64 / TARGET_SAMPLE_RATE as f64; + let output_len = (samples.len() as f64 / ratio).ceil() as usize; + let mut output = Vec::with_capacity(output_len); + + for i in 0..output_len { + let src_idx = i as f64 * ratio; + let idx0 = src_idx.floor() as usize; + let idx1 = (idx0 + 1).min(samples.len().saturating_sub(1)); + let frac = (src_idx - idx0 as f64) as f32; + output.push(samples[idx0] * (1.0 - frac) + samples[idx1] * frac); + } + + output +} + +/// Finalize recorded samples into a 16-kHz mono WAV. +fn finalize_recording( + raw_samples: Vec, + source_sample_rate: u32, +) -> Result { + if raw_samples.is_empty() { + warn!("{LOG_PREFIX} no audio samples captured"); + return Err("no audio samples captured".to_string()); + } + + let resampled = resample(&raw_samples, source_sample_rate); + let sample_count = resampled.len(); + let duration_secs = sample_count as f32 / TARGET_SAMPLE_RATE as f32; + + debug!( + "{LOG_PREFIX} finalizing: {sample_count} samples, {duration_secs:.1}s, \ + resampled from {source_sample_rate} to {TARGET_SAMPLE_RATE}" + ); + + let spec = WavSpec { + channels: 1, + sample_rate: TARGET_SAMPLE_RATE, + bits_per_sample: 16, + sample_format: HoundFormat::Int, + }; + + let mut buf = Cursor::new(Vec::new()); + { + let mut writer = + WavWriter::new(&mut buf, spec).map_err(|e| format!("WAV writer error: {e}"))?; + + for &sample in &resampled { + let clamped = sample.clamp(-1.0, 1.0); + let i16_sample = (clamped * 32767.0) as i16; + writer + .write_sample(i16_sample) + .map_err(|e| format!("WAV write error: {e}"))?; + } + + writer + .finalize() + .map_err(|e| format!("WAV finalize error: {e}"))?; + } + + let wav_bytes = buf.into_inner(); + info!( + "{LOG_PREFIX} recording finalized: {duration_secs:.1}s, {} bytes WAV", + wav_bytes.len() + ); + + Ok(RecordingResult { + wav_bytes, + duration_secs, + sample_count, + }) +} + +/// Find the best input config — prefer 16 kHz mono, else closest match. +fn find_best_config( + configs: impl Iterator, +) -> Result { + let mut configs_vec: Vec = configs.collect(); + if configs_vec.is_empty() { + return Err("no supported audio input configurations found".to_string()); + } + + // Sort: prefer configs whose range includes 16kHz, then by fewer channels. + configs_vec.sort_by(|a, b| { + let a_has_target = a.min_sample_rate().0 <= TARGET_SAMPLE_RATE + && a.max_sample_rate().0 >= TARGET_SAMPLE_RATE; + let b_has_target = b.min_sample_rate().0 <= TARGET_SAMPLE_RATE + && b.max_sample_rate().0 >= TARGET_SAMPLE_RATE; + + b_has_target + .cmp(&a_has_target) + .then(a.channels().cmp(&b.channels())) + }); + + let best = &configs_vec[0]; + let rate = if best.min_sample_rate().0 <= TARGET_SAMPLE_RATE + && best.max_sample_rate().0 >= TARGET_SAMPLE_RATE + { + SampleRate(TARGET_SAMPLE_RATE) + } else { + // Use the maximum supported rate and resample later. + best.max_sample_rate() + }; + + Ok(best.clone().with_sample_rate(rate)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn to_mono_passthrough_single_channel() { + let input = vec![0.1, 0.2, 0.3]; + assert_eq!(to_mono(&input, 1), input); + } + + #[test] + fn to_mono_averages_stereo() { + let input = vec![0.0, 1.0, 0.5, 0.5]; + let mono = to_mono(&input, 2); + assert_eq!(mono.len(), 2); + assert!((mono[0] - 0.5).abs() < 1e-6); + assert!((mono[1] - 0.5).abs() < 1e-6); + } + + #[test] + fn resample_same_rate_passthrough() { + let input = vec![0.1, 0.2, 0.3]; + let output = resample(&input, TARGET_SAMPLE_RATE); + assert_eq!(output, input); + } + + #[test] + fn resample_downsamples() { + // 32kHz -> 16kHz should roughly halve the samples. + let input: Vec = (0..3200).map(|i| (i as f32 / 3200.0).sin()).collect(); + let output = resample(&input, 32_000); + // Should be approximately 1600 samples. + assert!(output.len() >= 1590 && output.len() <= 1610); + } + + #[test] + fn finalize_produces_valid_wav() { + let samples: Vec = (0..16000) + .map(|i| (i as f32 * 440.0 * 2.0 * std::f32::consts::PI / 16000.0).sin()) + .collect(); + let result = finalize_recording(samples, 16_000).unwrap(); + assert!(result.wav_bytes.len() > 44); // WAV header is 44 bytes + assert!((result.duration_secs - 1.0).abs() < 0.1); + // Check WAV magic bytes. + assert_eq!(&result.wav_bytes[..4], b"RIFF"); + } + + #[test] + fn finalize_empty_samples_errors() { + let result = finalize_recording(vec![], 16_000); + assert!(result.is_err()); + } +} diff --git a/src/openhuman/voice/dictation_listener.rs b/src/openhuman/voice/dictation_listener.rs new file mode 100644 index 000000000..58ea4f533 --- /dev/null +++ b/src/openhuman/voice/dictation_listener.rs @@ -0,0 +1,181 @@ +//! Core-side dictation hotkey listener. +//! +//! Reads the `DictationConfig` from config, starts an `rdev`-based global +//! hotkey listener on the core process, and broadcasts `dictation:toggle` +//! events over a `tokio::sync::broadcast` channel that the Socket.IO +//! bridge subscribes to — so the frontend receives hotkey presses without +//! any Tauri-side shortcut registration. + +use once_cell::sync::Lazy; +use serde::Serialize; +use tokio::sync::broadcast; + +use crate::openhuman::config::Config; +use crate::openhuman::voice::hotkey::{self, ActivationMode, HotkeyEvent}; + +const LOG_PREFIX: &str = "[dictation_listener]"; + +// ── Broadcast channel for dictation events ──────────────────────────── + +/// A dictation event broadcast to Socket.IO clients. +#[derive(Debug, Clone, Serialize)] +pub struct DictationEvent { + /// Event type: `"pressed"` or `"released"`. + #[serde(rename = "type")] + pub event_type: String, + /// The hotkey that triggered this event. + pub hotkey: String, + /// The activation mode in use. + pub activation_mode: String, +} + +static DICTATION_BUS: Lazy> = Lazy::new(|| { + let (tx, _rx) = broadcast::channel(64); + tx +}); + +/// Subscribe to dictation events (used by the Socket.IO bridge). +pub fn subscribe_dictation_events() -> broadcast::Receiver { + DICTATION_BUS.subscribe() +} + +fn publish_dictation_event(event: DictationEvent) { + let _ = DICTATION_BUS.send(event); +} + +// ── Listener lifecycle ──────────────────────────────────────────────── + +/// Start the dictation hotkey listener if enabled in config. +/// +/// Intended to be called once from `run_server()` as a background task. +/// Reads the `dictation` config section and registers the global hotkey. +/// When the hotkey fires, publishes a `DictationEvent` to the broadcast +/// channel that the Socket.IO bridge forwards to all connected clients. +pub async fn start_if_enabled(config: &Config) { + if !config.dictation.enabled { + log::info!("{LOG_PREFIX} dictation disabled in config, skipping hotkey listener"); + return; + } + + let hotkey_str = config.dictation.hotkey.clone(); + if hotkey_str.is_empty() { + log::warn!("{LOG_PREFIX} dictation enabled but no hotkey configured"); + return; + } + + // Map DictationActivationMode to our hotkey ActivationMode. + let mode = match config.dictation.activation_mode { + crate::openhuman::config::DictationActivationMode::Push => ActivationMode::Push, + crate::openhuman::config::DictationActivationMode::Toggle => ActivationMode::Tap, + }; + + // Normalize the hotkey string for rdev (CmdOrCtrl → cmd on macOS, ctrl on others). + let normalized = normalize_hotkey_for_rdev(&hotkey_str); + + log::info!( + "{LOG_PREFIX} starting dictation hotkey listener: hotkey={normalized} (raw={hotkey_str}) mode={mode:?}" + ); + + let combo = match hotkey::parse_hotkey(&normalized) { + Ok(c) => c, + Err(e) => { + log::error!("{LOG_PREFIX} failed to parse hotkey '{normalized}': {e}"); + return; + } + }; + + let (listener_handle, mut hotkey_rx) = match hotkey::start_listener(combo, mode) { + Ok(pair) => pair, + Err(e) => { + log::error!("{LOG_PREFIX} failed to start hotkey listener: {e}"); + return; + } + }; + + let mode_str = match mode { + ActivationMode::Tap => "toggle", + ActivationMode::Push => "push", + }; + + log::info!("{LOG_PREFIX} dictation hotkey active: {normalized}"); + + // Forward hotkey events to the broadcast channel. + tokio::spawn(async move { + // Keep the listener handle alive for the lifetime of this task. + let _handle = listener_handle; + + while let Some(event) = hotkey_rx.recv().await { + let event_type = match event { + HotkeyEvent::Pressed => "pressed", + HotkeyEvent::Released => "released", + }; + + log::debug!("{LOG_PREFIX} hotkey {event_type}"); + + publish_dictation_event(DictationEvent { + event_type: event_type.to_string(), + hotkey: normalized.clone(), + activation_mode: mode_str.to_string(), + }); + } + + log::warn!("{LOG_PREFIX} hotkey event channel closed, listener stopping"); + }); +} + +/// Normalize a Tauri-style hotkey string to rdev-compatible format. +/// +/// Converts `CmdOrCtrl+Shift+D` → `cmd+shift+d` (macOS) or `ctrl+shift+d` (other). +fn normalize_hotkey_for_rdev(hotkey: &str) -> String { + let parts: Vec<&str> = hotkey.split('+').map(|s| s.trim()).collect(); + let mut result = Vec::new(); + + for part in parts { + let lower = part.to_lowercase(); + let mapped = match lower.as_str() { + "cmdorctrl" | "commandorcontrol" => { + if cfg!(target_os = "macos") { + "cmd" + } else { + "ctrl" + } + } + "cmd" | "command" => "cmd", + "ctrl" | "control" => "ctrl", + other => other, + }; + result.push(mapped.to_string()); + } + + result.join("+") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_cmdorctrl_macos() { + let result = normalize_hotkey_for_rdev("CmdOrCtrl+Shift+D"); + if cfg!(target_os = "macos") { + assert_eq!(result, "cmd+shift+d"); + } else { + assert_eq!(result, "ctrl+shift+d"); + } + } + + #[test] + fn normalize_plain_keys() { + assert_eq!(normalize_hotkey_for_rdev("Ctrl+Space"), "ctrl+space"); + } + + #[test] + fn normalize_preserves_structure() { + assert_eq!(normalize_hotkey_for_rdev("Alt+Shift+F5"), "alt+shift+f5"); + } + + #[test] + fn subscribe_returns_receiver() { + let _rx = subscribe_dictation_events(); + } +} diff --git a/src/openhuman/voice/hotkey.rs b/src/openhuman/voice/hotkey.rs new file mode 100644 index 000000000..29fe23582 --- /dev/null +++ b/src/openhuman/voice/hotkey.rs @@ -0,0 +1,345 @@ +//! Global hotkey listener using rdev. +//! +//! Monitors keyboard events system-wide and fires callbacks when a +//! configurable key combination is pressed/released. Supports two +//! activation modes: **tap** (toggle on press) and **push** (hold to +//! record, release to stop). + +use std::collections::HashSet; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use log::{debug, error, info, warn}; +use parking_lot::Mutex; +use rdev::{listen, Event, EventType, Key}; +use tokio::sync::mpsc; + +const LOG_PREFIX: &str = "[voice_hotkey]"; + +/// Activation mode for the voice hotkey. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum ActivationMode { + /// Single press toggles recording on/off. + Tap, + /// Hold to record, release to stop. + Push, +} + +impl Default for ActivationMode { + fn default() -> Self { + Self::Tap + } +} + +/// Events emitted by the hotkey listener. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HotkeyEvent { + /// The hotkey was pressed (start recording). + Pressed, + /// The hotkey was released (stop recording — only relevant in Push mode). + Released, +} + +/// Parsed hotkey combination (e.g. Ctrl+Shift+Space). +#[derive(Debug, Clone)] +pub struct HotkeyCombination { + /// Modifier keys that must be held. + pub modifiers: HashSet, + /// The primary trigger key. + pub trigger: Key, +} + +/// Handle to a running hotkey listener. Drop to stop. +pub struct HotkeyListenerHandle { + stop_flag: Arc, + _thread: Option>, +} + +impl HotkeyListenerHandle { + /// Signal the listener to ignore further events. + /// + /// Note: this does **not** terminate the listener thread. `rdev::listen` + /// blocks in the platform event loop and provides no cancellation API + /// (rdev 0.5). The thread stays alive until the process exits; the + /// stop flag merely causes the callback to discard all events. + pub fn stop(&self) { + self.stop_flag.store(true, Ordering::SeqCst); + info!("{LOG_PREFIX} hotkey listener signaled to skip events"); + } +} + +impl Drop for HotkeyListenerHandle { + fn drop(&mut self) { + self.stop_flag.store(true, Ordering::SeqCst); + } +} + +/// Parse a hotkey string like "ctrl+shift+space" into a `HotkeyCombination`. +pub fn parse_hotkey(hotkey_str: &str) -> Result { + let parts: Vec<&str> = hotkey_str + .split('+') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .collect(); + + if parts.is_empty() { + return Err("hotkey string is empty".to_string()); + } + + let mut modifiers = HashSet::new(); + let mut trigger = None; + + for (i, part) in parts.iter().enumerate() { + let key = string_to_key(part)?; + if i < parts.len() - 1 { + // Everything except the last part is a modifier. + modifiers.insert(key); + } else { + trigger = Some(key); + } + } + + let trigger = trigger.ok_or_else(|| "no trigger key specified".to_string())?; + + debug!( + "{LOG_PREFIX} parsed hotkey: modifiers={:?} trigger={:?}", + modifiers, trigger + ); + + Ok(HotkeyCombination { modifiers, trigger }) +} + +/// Start the global hotkey listener. +/// +/// Returns a handle (drop to stop) and a receiver for hotkey events. +/// The listener runs on a dedicated OS thread since rdev::listen is blocking. +pub fn start_listener( + hotkey: HotkeyCombination, + mode: ActivationMode, +) -> Result<(HotkeyListenerHandle, mpsc::UnboundedReceiver), String> { + let stop_flag = Arc::new(AtomicBool::new(false)); + let (tx, rx) = mpsc::unbounded_channel(); + + let stop_flag_clone = stop_flag.clone(); + let pressed_keys: Arc>> = Arc::new(Mutex::new(HashSet::new())); + let is_active = Arc::new(AtomicBool::new(false)); + + info!( + "{LOG_PREFIX} starting hotkey listener, mode={mode:?}, trigger={:?}, modifiers={:?}", + hotkey.trigger, hotkey.modifiers + ); + + let thread = std::thread::Builder::new() + .name("voice-hotkey".into()) + .spawn(move || { + let callback = move |event: Event| { + if stop_flag_clone.load(Ordering::SeqCst) { + return; + } + + match event.event_type { + EventType::KeyPress(key) => { + let mut keys = pressed_keys.lock(); + keys.insert(key); + + // Check if all modifiers + trigger are held. + if key == hotkey.trigger + && hotkey.modifiers.iter().all(|m| keys.contains(m)) + { + match mode { + ActivationMode::Tap => { + let was_active = is_active.fetch_xor(true, Ordering::SeqCst); + let event = if was_active { + HotkeyEvent::Released + } else { + HotkeyEvent::Pressed + }; + debug!("{LOG_PREFIX} tap hotkey → {event:?}"); + if tx.send(event).is_err() { + warn!("{LOG_PREFIX} event receiver dropped"); + } + } + ActivationMode::Push => { + if !is_active.swap(true, Ordering::SeqCst) { + debug!("{LOG_PREFIX} push hotkey → Pressed"); + if tx.send(HotkeyEvent::Pressed).is_err() { + warn!("{LOG_PREFIX} event receiver dropped"); + } + } + } + } + } + } + EventType::KeyRelease(key) => { + let mut keys = pressed_keys.lock(); + keys.remove(&key); + + // In push mode, release when the trigger key is released. + if mode == ActivationMode::Push + && key == hotkey.trigger + && is_active.swap(false, Ordering::SeqCst) + { + debug!("{LOG_PREFIX} push hotkey → Released"); + if tx.send(HotkeyEvent::Released).is_err() { + warn!("{LOG_PREFIX} event receiver dropped"); + } + } + } + _ => {} + } + }; + + // rdev::listen blocks in the platform event loop and has no + // graceful cancellation API (rdev 0.5.3). The thread remains + // alive until the process exits; the stop_flag only causes the + // callback above to discard events. This is a known limitation. + if let Err(e) = listen(callback) { + error!("{LOG_PREFIX} rdev listen error: {e:?}"); + } + }) + .map_err(|e| format!("failed to spawn hotkey listener thread: {e}"))?; + + Ok(( + HotkeyListenerHandle { + stop_flag, + _thread: Some(thread), + }, + rx, + )) +} + +/// Convert a string key name to an rdev Key. +fn string_to_key(s: &str) -> Result { + match s.to_lowercase().as_str() { + // Modifiers + "ctrl" | "control" | "leftcontrol" => Ok(Key::ControlLeft), + "rctrl" | "rightcontrol" => Ok(Key::ControlRight), + "shift" | "leftshift" => Ok(Key::ShiftLeft), + "rshift" | "rightshift" => Ok(Key::ShiftRight), + "alt" | "option" | "leftalt" => Ok(Key::Alt), + "ralt" | "rightaltoption" => Ok(Key::AltGr), + "meta" | "super" | "cmd" | "command" | "leftmeta" => Ok(Key::MetaLeft), + "rmeta" | "rsuper" | "rcmd" | "rightmeta" => Ok(Key::MetaRight), + + // Common keys + "space" => Ok(Key::Space), + "enter" | "return" => Ok(Key::Return), + "tab" => Ok(Key::Tab), + "escape" | "esc" => Ok(Key::Escape), + "backspace" => Ok(Key::Backspace), + "delete" | "del" => Ok(Key::Delete), + "capslock" => Ok(Key::CapsLock), + + // F-keys + "f1" => Ok(Key::F1), + "f2" => Ok(Key::F2), + "f3" => Ok(Key::F3), + "f4" => Ok(Key::F4), + "f5" => Ok(Key::F5), + "f6" => Ok(Key::F6), + "f7" => Ok(Key::F7), + "f8" => Ok(Key::F8), + "f9" => Ok(Key::F9), + "f10" => Ok(Key::F10), + "f11" => Ok(Key::F11), + "f12" => Ok(Key::F12), + + // Navigation + "up" | "uparrow" => Ok(Key::UpArrow), + "down" | "downarrow" => Ok(Key::DownArrow), + "left" | "leftarrow" => Ok(Key::LeftArrow), + "right" | "rightarrow" => Ok(Key::RightArrow), + "home" => Ok(Key::Home), + "end" => Ok(Key::End), + "pageup" | "pgup" => Ok(Key::PageUp), + "pagedown" | "pgdn" => Ok(Key::PageDown), + "insert" | "ins" => Ok(Key::Insert), + + // Letters + "a" => Ok(Key::KeyA), + "b" => Ok(Key::KeyB), + "c" => Ok(Key::KeyC), + "d" => Ok(Key::KeyD), + "e" => Ok(Key::KeyE), + "f" => Ok(Key::KeyF), + "g" => Ok(Key::KeyG), + "h" => Ok(Key::KeyH), + "i" => Ok(Key::KeyI), + "j" => Ok(Key::KeyJ), + "k" => Ok(Key::KeyK), + "l" => Ok(Key::KeyL), + "m" => Ok(Key::KeyM), + "n" => Ok(Key::KeyN), + "o" => Ok(Key::KeyO), + "p" => Ok(Key::KeyP), + "q" => Ok(Key::KeyQ), + "r" => Ok(Key::KeyR), + "s" => Ok(Key::KeyS), + "t" => Ok(Key::KeyT), + "u" => Ok(Key::KeyU), + "v" => Ok(Key::KeyV), + "w" => Ok(Key::KeyW), + "x" => Ok(Key::KeyX), + "y" => Ok(Key::KeyY), + "z" => Ok(Key::KeyZ), + + // Numbers + "0" => Ok(Key::Num0), + "1" => Ok(Key::Num1), + "2" => Ok(Key::Num2), + "3" => Ok(Key::Num3), + "4" => Ok(Key::Num4), + "5" => Ok(Key::Num5), + "6" => Ok(Key::Num6), + "7" => Ok(Key::Num7), + "8" => Ok(Key::Num8), + "9" => Ok(Key::Num9), + + other => Err(format!("unknown key: '{other}'")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_simple_hotkey() { + let combo = parse_hotkey("ctrl+shift+space").unwrap(); + assert_eq!(combo.trigger, Key::Space); + assert!(combo.modifiers.contains(&Key::ControlLeft)); + assert!(combo.modifiers.contains(&Key::ShiftLeft)); + } + + #[test] + fn parse_single_key() { + let combo = parse_hotkey("f5").unwrap(); + assert_eq!(combo.trigger, Key::F5); + assert!(combo.modifiers.is_empty()); + } + + #[test] + fn parse_cmd_key() { + let combo = parse_hotkey("cmd+space").unwrap(); + assert_eq!(combo.trigger, Key::Space); + assert!(combo.modifiers.contains(&Key::MetaLeft)); + } + + #[test] + fn parse_empty_errors() { + assert!(parse_hotkey("").is_err()); + } + + #[test] + fn parse_unknown_key_errors() { + assert!(parse_hotkey("ctrl+unknownkey").is_err()); + } + + #[test] + fn activation_mode_default_is_tap() { + assert_eq!(ActivationMode::default(), ActivationMode::Tap); + } +} diff --git a/src/openhuman/voice/mod.rs b/src/openhuman/voice/mod.rs index bf1e1be43..881094a44 100644 --- a/src/openhuman/voice/mod.rs +++ b/src/openhuman/voice/mod.rs @@ -1,11 +1,18 @@ //! Voice domain — speech-to-text (whisper.cpp) and text-to-speech (piper). //! //! Provides RPC endpoints under the `openhuman.voice_*` namespace for -//! transcription, synthesis, and proactive availability checking. +//! transcription, synthesis, proactive availability checking, and a +//! standalone voice dictation server (hotkey → record → transcribe → insert). +pub mod audio_capture; +pub mod dictation_listener; +pub mod hotkey; mod ops; mod postprocess; mod schemas; +pub mod server; +pub mod streaming; +pub mod text_input; mod types; pub use ops::*; diff --git a/src/openhuman/voice/postprocess.rs b/src/openhuman/voice/postprocess.rs index aca890afd..308adbc08 100644 --- a/src/openhuman/voice/postprocess.rs +++ b/src/openhuman/voice/postprocess.rs @@ -20,6 +20,13 @@ nothing else."; /// Clean up raw transcription text using a local LLM. /// +/// Cleanup is enabled when **either** of these conditions holds: +/// - `config.local_ai.voice_llm_cleanup_enabled` is `true` (default), **or** +/// - the local LLM state is `"ready"` or `"degraded"`. +/// +/// Even when enabled by config, cleanup is **skipped** if the LLM is not +/// in a ready/degraded state (i.e. not yet downloaded or bootstrapped). +/// /// Returns the cleaned text on success, or the original raw text if the /// LLM is unavailable or cleanup fails (graceful degradation). pub async fn cleanup_transcription( @@ -31,13 +38,27 @@ pub async fn cleanup_transcription( return raw_text.to_string(); } - if !config.local_ai.voice_llm_cleanup_enabled { - debug!("{LOG_PREFIX} LLM cleanup disabled in config"); + let service = local_ai::global(config); + let llm_state = service.status.lock().state.clone(); + let llm_ready = matches!(llm_state.as_str(), "ready" | "degraded"); + + // Enable cleanup when: + // 1. Explicitly enabled in config (default: true), OR + // 2. The local LLM is already downloaded and ready. + let should_cleanup = config.local_ai.voice_llm_cleanup_enabled || llm_ready; + + if !should_cleanup { + debug!("{LOG_PREFIX} LLM cleanup skipped: config disabled and LLM not ready (state={llm_state})"); + return raw_text.to_string(); + } + + if !llm_ready { + debug!("{LOG_PREFIX} LLM cleanup enabled but LLM not ready (state={llm_state}), skipping"); return raw_text.to_string(); } debug!( - "{LOG_PREFIX} cleaning up transcription ({} chars, context={})", + "{LOG_PREFIX} cleaning up transcription ({} chars, context={}, llm_state={llm_state})", raw_text.len(), conversation_context.is_some() ); @@ -52,8 +73,6 @@ pub async fn cleanup_transcription( _ => raw_text.to_string(), }; - let service = local_ai::global(config); - let result: Result = service .inference(config, CLEANUP_SYSTEM_PROMPT, &prompt, Some(512), true) .await; diff --git a/src/openhuman/voice/schemas.rs b/src/openhuman/voice/schemas.rs index 1d1ee237f..91c4f09be 100644 --- a/src/openhuman/voice/schemas.rs +++ b/src/openhuman/voice/schemas.rs @@ -54,6 +54,9 @@ pub fn all_voice_controller_schemas() -> Vec { voice_schemas("voice_transcribe"), voice_schemas("voice_transcribe_bytes"), voice_schemas("voice_tts"), + voice_schemas("voice_server_start"), + voice_schemas("voice_server_stop"), + voice_schemas("voice_server_status"), ] } @@ -75,6 +78,18 @@ pub fn all_voice_registered_controllers() -> Vec { schema: voice_schemas("voice_tts"), handler: handle_voice_tts, }, + RegisteredController { + schema: voice_schemas("voice_server_start"), + handler: handle_voice_server_start, + }, + RegisteredController { + schema: voice_schemas("voice_server_stop"), + handler: handle_voice_server_stop, + }, + RegisteredController { + schema: voice_schemas("voice_server_status"), + handler: handle_voice_server_status, + }, ] } @@ -139,6 +154,35 @@ pub fn voice_schemas(function: &str) -> ControllerSchema { ], outputs: vec![json_output("tts", "TTS result with output path.")], }, + "voice_server_start" => ControllerSchema { + namespace: "voice", + function: "server_start", + description: + "Start the voice dictation server (hotkey → record → transcribe → insert text).", + inputs: vec![ + optional_string("hotkey", "Hotkey combination (default: ctrl+shift+space)."), + optional_string( + "activation_mode", + "Activation mode: tap or push (default: tap).", + ), + optional_bool("skip_cleanup", "Skip LLM post-processing."), + ], + outputs: vec![json_output("status", "Voice server status after start.")], + }, + "voice_server_stop" => ControllerSchema { + namespace: "voice", + function: "server_stop", + description: "Stop the voice dictation server.", + inputs: vec![], + outputs: vec![json_output("status", "Voice server status after stop.")], + }, + "voice_server_status" => ControllerSchema { + namespace: "voice", + function: "server_status", + description: "Get the current voice dictation server status.", + inputs: vec![], + outputs: vec![json_output("status", "Current voice server status.")], + }, _ => ControllerSchema { namespace: "voice", function: "unknown", @@ -208,6 +252,127 @@ fn handle_voice_tts(params: Map) -> ControllerFuture { }) } +fn handle_voice_server_start(params: Map) -> ControllerFuture { + Box::pin(async move { + use crate::openhuman::voice::hotkey::ActivationMode; + use crate::openhuman::voice::server::{global_server, VoiceServerConfig}; + + let config = config_rpc::load_config_with_timeout().await?; + + let hotkey = params + .get("hotkey") + .and_then(|v| v.as_str()) + .unwrap_or(&config.voice_server.hotkey) + .to_string(); + + let activation_mode = match params.get("activation_mode").and_then(|v| v.as_str()) { + Some("push") => ActivationMode::Push, + Some("tap") => ActivationMode::Tap, + Some(other) => { + log::warn!( + "[voice_server] unrecognized activation_mode '{}', defaulting to Tap", + other + ); + ActivationMode::Tap + } + None => match config.voice_server.activation_mode { + crate::openhuman::config::VoiceActivationMode::Push => ActivationMode::Push, + crate::openhuman::config::VoiceActivationMode::Tap => ActivationMode::Tap, + }, + }; + + let skip_cleanup = params + .get("skip_cleanup") + .and_then(|v| v.as_bool()) + .unwrap_or(config.voice_server.skip_cleanup); + + let server_config = VoiceServerConfig { + hotkey, + activation_mode, + skip_cleanup, + context: None, + min_duration_secs: config.voice_server.min_duration_secs, + }; + + // Check if a server is already running with a different config. + if let Some(existing) = crate::openhuman::voice::server::try_global_server() { + let existing_status = existing.status().await; + if existing_status.state != crate::openhuman::voice::server::ServerState::Stopped { + if existing_status.hotkey != server_config.hotkey + || existing_status.activation_mode != server_config.activation_mode + { + return Err(format!( + "voice server already running (hotkey={}, mode={:?}); \ + stop it first before starting with different config", + existing_status.hotkey, existing_status.activation_mode + )); + } + // Same config, already running — return current status. + return serde_json::to_value(existing_status) + .map_err(|e| format!("serialize error: {e}")); + } + } + + let server = global_server(server_config); + let config_clone = config.clone(); + + tokio::spawn(async move { + if let Err(e) = server.run(&config_clone).await { + log::error!("[voice_server] server exited with error: {e}"); + } + }); + + // Give the server a moment to start. + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + + if let Some(s) = crate::openhuman::voice::server::try_global_server() { + let status = s.status().await; + serde_json::to_value(status).map_err(|e| format!("serialize error: {e}")) + } else { + Err("voice server failed to initialize".to_string()) + } + }) +} + +fn handle_voice_server_stop(_params: Map) -> ControllerFuture { + Box::pin(async move { + if let Some(server) = crate::openhuman::voice::server::try_global_server() { + server.stop().await; + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + let status = server.status().await; + serde_json::to_value(status).map_err(|e| format!("serialize error: {e}")) + } else { + // Not running — return a stopped status rather than an error. + let status = crate::openhuman::voice::server::VoiceServerStatus { + state: crate::openhuman::voice::server::ServerState::Stopped, + hotkey: String::new(), + activation_mode: crate::openhuman::voice::hotkey::ActivationMode::Tap, + transcription_count: 0, + last_error: None, + }; + serde_json::to_value(status).map_err(|e| format!("serialize error: {e}")) + } + }) +} + +fn handle_voice_server_status(_params: Map) -> ControllerFuture { + Box::pin(async move { + if let Some(server) = crate::openhuman::voice::server::try_global_server() { + let status = server.status().await; + serde_json::to_value(status).map_err(|e| format!("serialize error: {e}")) + } else { + let status = crate::openhuman::voice::server::VoiceServerStatus { + state: crate::openhuman::voice::server::ServerState::Stopped, + hotkey: String::new(), + activation_mode: crate::openhuman::voice::hotkey::ActivationMode::Tap, + transcription_count: 0, + last_error: None, + }; + serde_json::to_value(status).map_err(|e| format!("serialize error: {e}")) + } + }) +} + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- diff --git a/src/openhuman/voice/server.rs b/src/openhuman/voice/server.rs new file mode 100644 index 000000000..c634cff3e --- /dev/null +++ b/src/openhuman/voice/server.rs @@ -0,0 +1,360 @@ +//! Standalone voice server — hotkey → record → transcribe → insert text. +//! +//! Can run as part of the core process or independently via the CLI. +//! The server listens for a configurable hotkey, records audio from the +//! microphone, transcribes via whisper, and inserts the result into the +//! active text field. + +use std::sync::atomic::Ordering; +use std::sync::Arc; + +use log::{debug, error, info, warn}; +use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; + +use crate::openhuman::config::Config; + +use super::audio_capture::{self, RecordingHandle}; +use super::hotkey::{self, ActivationMode, HotkeyEvent}; +use super::text_input; + +const LOG_PREFIX: &str = "[voice_server]"; + +/// Running state of the voice server. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ServerState { + /// Server is not running. + Stopped, + /// Server is running and idle, waiting for hotkey. + Idle, + /// Actively recording audio. + Recording, + /// Transcribing recorded audio. + Transcribing, +} + +/// Status snapshot of the voice server. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct VoiceServerStatus { + pub state: ServerState, + pub hotkey: String, + pub activation_mode: ActivationMode, + pub transcription_count: u64, + pub last_error: Option, +} + +/// Configuration for the voice server. +#[derive(Debug, Clone)] +pub struct VoiceServerConfig { + pub hotkey: String, + pub activation_mode: ActivationMode, + /// Skip LLM post-processing on transcriptions. + pub skip_cleanup: bool, + /// Optional conversation context for better transcription accuracy. + pub context: Option, + /// Minimum recording duration in seconds. Shorter recordings are discarded. + pub min_duration_secs: f32, +} + +impl Default for VoiceServerConfig { + fn default() -> Self { + Self { + hotkey: "ctrl+shift+space".to_string(), + activation_mode: ActivationMode::Tap, + skip_cleanup: false, + context: None, + min_duration_secs: 0.3, + } + } +} + +/// The voice server runtime. +pub struct VoiceServer { + state: Arc>, + cancel: CancellationToken, + config: VoiceServerConfig, + transcription_count: Arc, + last_error: Arc>>, +} + +impl VoiceServer { + pub fn new(config: VoiceServerConfig) -> Self { + Self { + state: Arc::new(Mutex::new(ServerState::Stopped)), + cancel: CancellationToken::new(), + config, + transcription_count: Arc::new(std::sync::atomic::AtomicU64::new(0)), + last_error: Arc::new(Mutex::new(None)), + } + } + + /// Get the current server status. + pub async fn status(&self) -> VoiceServerStatus { + VoiceServerStatus { + state: *self.state.lock().await, + hotkey: self.config.hotkey.clone(), + activation_mode: self.config.activation_mode, + transcription_count: self.transcription_count.load(Ordering::Relaxed), + last_error: self.last_error.lock().await.clone(), + } + } + + /// Run the voice server. Blocks until stopped. + /// + /// This is the main entry point for both embedded and standalone modes. + pub async fn run(&self, app_config: &Config) -> Result<(), String> { + info!( + "{LOG_PREFIX} starting voice server: hotkey={} mode={:?}", + self.config.hotkey, self.config.activation_mode + ); + + let combo = hotkey::parse_hotkey(&self.config.hotkey)?; + let (listener_handle, mut hotkey_rx) = + hotkey::start_listener(combo, self.config.activation_mode)?; + + *self.state.lock().await = ServerState::Idle; + + info!("{LOG_PREFIX} voice server ready, listening for hotkey"); + + let mut recording: Option = None; + + loop { + let event = tokio::select! { + ev = hotkey_rx.recv() => { + match ev { + Some(e) => e, + None => { + warn!("{LOG_PREFIX} hotkey channel closed"); + break; + } + } + } + _ = self.cancel.cancelled() => { + debug!("{LOG_PREFIX} cancellation received"); + break; + } + }; + + match event { + HotkeyEvent::Pressed => { + if recording.is_some() { + // In tap mode, second press stops recording. + debug!("{LOG_PREFIX} hotkey pressed while recording → stopping"); + if let Some(handle) = recording.take() { + self.process_recording(handle, app_config).await; + } + } else { + debug!("{LOG_PREFIX} hotkey pressed → starting recording"); + match audio_capture::start_recording() { + Ok(handle) => { + *self.state.lock().await = ServerState::Recording; + recording = Some(handle); + info!("{LOG_PREFIX} recording started"); + } + Err(e) => { + error!("{LOG_PREFIX} failed to start recording: {e}"); + *self.last_error.lock().await = Some(e); + } + } + } + } + HotkeyEvent::Released => { + // In push mode, release stops recording. + if let Some(handle) = recording.take() { + debug!("{LOG_PREFIX} hotkey released → stopping recording"); + self.process_recording(handle, app_config).await; + } + } + } + } + + listener_handle.stop(); + *self.state.lock().await = ServerState::Stopped; + info!("{LOG_PREFIX} voice server stopped"); + + Ok(()) + } + + /// Stop the voice server. + pub async fn stop(&self) { + info!("{LOG_PREFIX} stopping voice server"); + self.cancel.cancel(); + } + + /// Process a completed recording: transcribe and insert text. + async fn process_recording(&self, handle: RecordingHandle, config: &Config) { + *self.state.lock().await = ServerState::Transcribing; + + match handle.stop().await { + Ok(result) => { + info!( + "{LOG_PREFIX} recording stopped: {:.1}s, {} bytes", + result.duration_secs, + result.wav_bytes.len() + ); + + if result.duration_secs < self.config.min_duration_secs { + warn!( + "{LOG_PREFIX} recording too short ({:.1}s), skipping", + result.duration_secs + ); + *self.state.lock().await = ServerState::Idle; + return; + } + + match crate::openhuman::voice::voice_transcribe_bytes( + config, + &result.wav_bytes, + Some("wav".to_string()), + self.config.context.as_deref(), + self.config.skip_cleanup, + ) + .await + { + Ok(outcome) => { + let text = &outcome.value.text; + info!( + "{LOG_PREFIX} transcription: '{}' ({} chars)", + truncate_for_log(text, 80), + text.len() + ); + + if !text.trim().is_empty() { + if let Err(e) = text_input::insert_text(text) { + error!("{LOG_PREFIX} failed to insert text: {e}"); + *self.last_error.lock().await = Some(e); + } else { + self.transcription_count.fetch_add(1, Ordering::Relaxed); + info!("{LOG_PREFIX} text inserted into active field"); + } + } else { + debug!("{LOG_PREFIX} transcription was empty, nothing to insert"); + } + } + Err(e) => { + error!("{LOG_PREFIX} transcription failed: {e}"); + *self.last_error.lock().await = Some(e); + } + } + } + Err(e) => { + error!("{LOG_PREFIX} failed to stop recording: {e}"); + *self.last_error.lock().await = Some(e); + } + } + + *self.state.lock().await = ServerState::Idle; + } +} + +/// Global voice server instance, lazily initialized. +static VOICE_SERVER: once_cell::sync::OnceCell> = once_cell::sync::OnceCell::new(); + +/// Get or initialize the global voice server instance. +pub fn global_server(config: VoiceServerConfig) -> Arc { + VOICE_SERVER + .get_or_init(|| Arc::new(VoiceServer::new(config))) + .clone() +} + +/// Get the global voice server if already initialized. +pub fn try_global_server() -> Option> { + VOICE_SERVER.get().cloned() +} + +/// Run the voice server standalone (blocking). Intended for CLI usage. +/// +/// Creates a fresh `VoiceServer` that is **not** registered in the global +/// singleton used by `voice_server_status` RPC. This keeps CLI-started +/// instances isolated from the core RPC lifecycle. +pub async fn run_standalone( + app_config: Config, + server_config: VoiceServerConfig, +) -> Result<(), String> { + info!("{LOG_PREFIX} starting standalone voice server"); + info!("{LOG_PREFIX} hotkey: {}", server_config.hotkey); + info!("{LOG_PREFIX} mode: {:?}", server_config.activation_mode); + info!("{LOG_PREFIX} press the hotkey to start dictating"); + + let server = VoiceServer::new(server_config); + + // Handle Ctrl+C gracefully. + let server_arc = Arc::new(server); + let server_for_signal = server_arc.clone(); + + tokio::spawn(async move { + if let Ok(()) = tokio::signal::ctrl_c().await { + info!("{LOG_PREFIX} Ctrl+C received, shutting down"); + server_for_signal.stop().await; + } + }); + + // This is safe because we hold the Arc and nothing else moves it. + // The server.run() borrows &self, and we await it to completion. + server_arc.run(&app_config).await +} + +fn truncate_for_log(s: &str, max: usize) -> String { + let truncated: String = s.chars().take(max).collect(); + if truncated.len() < s.len() { + format!("{truncated}...") + } else { + truncated + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_server_config() { + let cfg = VoiceServerConfig::default(); + assert_eq!(cfg.hotkey, "ctrl+shift+space"); + assert_eq!(cfg.activation_mode, ActivationMode::Tap); + assert!(!cfg.skip_cleanup); + assert!(cfg.context.is_none()); + } + + #[tokio::test] + async fn server_status_initial() { + let server = VoiceServer::new(VoiceServerConfig::default()); + let status = server.status().await; + assert_eq!(status.state, ServerState::Stopped); + assert_eq!(status.transcription_count, 0); + assert!(status.last_error.is_none()); + } + + #[test] + fn server_state_serializes() { + let json = serde_json::to_string(&ServerState::Recording).unwrap(); + assert_eq!(json, "\"recording\""); + } + + #[test] + fn voice_server_status_serializes() { + let status = VoiceServerStatus { + state: ServerState::Idle, + hotkey: "ctrl+shift+space".into(), + activation_mode: ActivationMode::Tap, + transcription_count: 5, + last_error: None, + }; + let v = serde_json::to_value(&status).unwrap(); + assert_eq!(v["state"], "idle"); + assert_eq!(v["transcription_count"], 5); + } + + #[test] + fn truncate_for_log_short() { + assert_eq!(truncate_for_log("hello", 10), "hello"); + } + + #[test] + fn truncate_for_log_long() { + let result = truncate_for_log("hello world this is long", 10); + assert!(result.ends_with("...")); + assert!(result.len() <= 14); // 10 + "..." + } +} diff --git a/src/openhuman/voice/streaming.rs b/src/openhuman/voice/streaming.rs new file mode 100644 index 000000000..92c4308d7 --- /dev/null +++ b/src/openhuman/voice/streaming.rs @@ -0,0 +1,214 @@ +//! WebSocket streaming transcription endpoint. +//! +//! Accepts a WebSocket connection that receives PCM16 audio chunks (16kHz mono) +//! and periodically runs whisper inference on the accumulated buffer, sending +//! back partial transcription results as JSON messages. +//! +//! Protocol: +//! Client → Server: binary frames containing PCM16 LE audio bytes (16kHz mono) +//! Server → Client: JSON text frames: +//! { "type": "partial", "text": "..." } — interim transcription +//! { "type": "final", "text": "...", "raw_text": "..." } — after client sends +//! `{"type":"stop"}` text frame +//! { "type": "error", "message": "..." } — on error +//! Client → Server: text frame `{"type":"stop"}` — end recording, get final result + +use std::sync::Arc; + +use axum::extract::ws::{Message, WebSocket}; +use serde::Deserialize; +use tokio::sync::Mutex; + +use super::postprocess; +use crate::openhuman::config::Config; +use crate::openhuman::local_ai; +use crate::openhuman::local_ai::whisper_engine; + +const LOG_PREFIX: &str = "[voice-stream]"; + +#[derive(Debug, Deserialize)] +struct ClientCommand { + #[serde(rename = "type")] + cmd_type: String, +} + +/// Handle an upgraded WebSocket connection for streaming dictation. +pub async fn handle_dictation_ws(mut socket: WebSocket, config: Arc) { + log::info!("{LOG_PREFIX} new streaming dictation connection"); + + let audio_buf: Arc>> = Arc::new(Mutex::new(Vec::new())); + let interval_ms = config.dictation.streaming_interval_ms; + let do_streaming = config.dictation.streaming; + + // Periodic inference task — runs every `interval_ms` on the accumulated buffer + let buf_clone = audio_buf.clone(); + let config_clone = config.clone(); + let (partial_tx, mut partial_rx) = tokio::sync::mpsc::channel::(8); + + let inference_handle = if do_streaming { + let handle = tokio::spawn(async move { + let mut interval = + tokio::time::interval(std::time::Duration::from_millis(interval_ms.max(500))); + let mut last_len: usize = 0; + + loop { + interval.tick().await; + + let samples: Vec = { + let guard = buf_clone.lock().await; + if guard.len() <= last_len || guard.len() < 8000 { + // No new data or less than 0.5s of audio — skip + continue; + } + last_len = guard.len(); + guard.clone() + }; + + let service = local_ai::global(&config_clone); + match whisper_engine::transcribe_pcm_i16(&service.whisper, &samples, None) { + Ok(text) => { + let trimmed = text.trim().to_string(); + if !trimmed.is_empty() { + log::debug!( + "{LOG_PREFIX} partial transcription ({} samples): {}", + samples.len(), + &trimmed[..trimmed.len().min(80)] + ); + if partial_tx.send(trimmed).await.is_err() { + break; // receiver dropped + } + } + } + Err(e) => { + log::warn!("{LOG_PREFIX} partial inference error: {e}"); + } + } + } + }); + Some(handle) + } else { + None + }; + + loop { + tokio::select! { + // Forward partial results to the client + Some(partial_text) = partial_rx.recv() => { + let msg = serde_json::json!({ + "type": "partial", + "text": partial_text, + }); + if socket.send(Message::Text(msg.to_string().into())).await.is_err() { + log::debug!("{LOG_PREFIX} client disconnected while sending partial"); + break; + } + } + + // Receive audio data or commands from the client + msg = socket.recv() => { + match msg { + Some(Ok(Message::Binary(data))) => { + // PCM16 LE bytes → i16 samples + if data.len() % 2 != 0 { + log::warn!("{LOG_PREFIX} received odd-length binary frame, skipping"); + continue; + } + let samples: Vec = data + .chunks_exact(2) + .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]])) + .collect(); + + let mut buf = audio_buf.lock().await; + buf.extend_from_slice(&samples); + log::trace!( + "{LOG_PREFIX} buffered {} new samples, total {}", + samples.len(), + buf.len() + ); + } + + Some(Ok(Message::Text(text))) => { + if let Ok(cmd) = serde_json::from_str::(&text) { + if cmd.cmd_type == "stop" { + log::info!("{LOG_PREFIX} stop command received, running final inference"); + break; // fall through to final transcription + } + } + } + + Some(Ok(Message::Close(_))) | None => { + log::info!("{LOG_PREFIX} client disconnected"); + if let Some(h) = inference_handle { + h.abort(); + } + return; + } + + Some(Err(e)) => { + log::warn!("{LOG_PREFIX} websocket error: {e}"); + if let Some(h) = inference_handle { + h.abort(); + } + return; + } + + _ => {} + } + } + } + } + + // Stop the periodic inference task + if let Some(h) = inference_handle { + h.abort(); + } + + // Run final transcription on the complete buffer + let final_samples = audio_buf.lock().await.clone(); + if final_samples.is_empty() { + let msg = serde_json::json!({ + "type": "final", + "text": "", + "raw_text": "", + }); + let _ = socket.send(Message::Text(msg.to_string().into())).await; + return; + } + + log::info!( + "{LOG_PREFIX} running final inference on {} samples ({:.1}s)", + final_samples.len(), + final_samples.len() as f64 / 16000.0 + ); + + let service = local_ai::global(&config); + let raw_text = match whisper_engine::transcribe_pcm_i16(&service.whisper, &final_samples, None) + { + Ok(text) => text.trim().to_string(), + Err(e) => { + log::error!("{LOG_PREFIX} final inference error: {e}"); + let msg = serde_json::json!({ + "type": "error", + "message": format!("Transcription failed: {e}"), + }); + let _ = socket.send(Message::Text(msg.to_string().into())).await; + return; + } + }; + + // LLM refinement if enabled + let refined_text = if config.dictation.llm_refinement && !raw_text.is_empty() { + postprocess::cleanup_transcription(&config, &raw_text, None).await + } else { + raw_text.clone() + }; + + let msg = serde_json::json!({ + "type": "final", + "text": refined_text, + "raw_text": raw_text, + }); + let _ = socket.send(Message::Text(msg.to_string().into())).await; + log::info!("{LOG_PREFIX} streaming session complete"); + // Socket is dropped here, which sends a close frame automatically +} diff --git a/src/openhuman/voice/text_input.rs b/src/openhuman/voice/text_input.rs new file mode 100644 index 000000000..fffe03eb0 --- /dev/null +++ b/src/openhuman/voice/text_input.rs @@ -0,0 +1,51 @@ +//! Text insertion into the currently active text field. +//! +//! Uses enigo to simulate keyboard input so that transcribed text +//! appears in whatever application has focus. + +use log::{debug, info, warn}; + +use enigo::{Enigo, Keyboard, Settings}; + +const LOG_PREFIX: &str = "[voice_input]"; + +/// Insert text into the currently active text field via enigo. +/// +/// Skips empty or whitespace-only input. Uses enigo's `text()` method +/// which handles Unicode and platform-appropriate input simulation. +pub fn insert_text(text: &str) -> Result<(), String> { + if text.trim().is_empty() { + warn!("{LOG_PREFIX} transcription was empty/whitespace, skipping insertion"); + return Ok(()); + } + + info!( + "{LOG_PREFIX} inserting {} chars into active field", + text.len() + ); + + let mut enigo = Enigo::new(&Settings::default()) + .map_err(|e| format!("failed to create enigo instance: {e}"))?; + + enigo + .text(text) + .map_err(|e| format!("failed to insert text: {e}"))?; + + debug!("{LOG_PREFIX} text inserted successfully"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_text_is_noop() { + assert!(insert_text("").is_ok()); + } + + #[test] + fn whitespace_only_skips_insertion() { + assert!(insert_text(" ").is_ok()); + } +}