From 2cd0bc33278524a1381da0589e42ec256de76f05 Mon Sep 17 00:00:00 2001 From: jimersylee Date: Mon, 10 Aug 2026 09:28:32 +0800 Subject: [PATCH 01/10] feat: add selectable local ASR backends --- .gitmodules | 3 + openless-all/app/package.json | 1 + .../scripts/check-macos-metal-toolchain.mjs | 25 + openless-all/app/src-tauri/Cargo.lock | 854 +++++++++++++++++- openless-all/app/src-tauri/Cargo.toml | 3 + openless-all/app/src-tauri/build.rs | 26 +- .../app/src-tauri/src/asr/local/cache.rs | 55 +- .../app/src-tauri/src/asr/local/download.rs | 12 +- .../src-tauri/src/asr/local/foundry_native.rs | 6 +- .../src-tauri/src/asr/local/local_provider.rs | 84 +- .../src/asr/local/mlx_qwen_engine.rs | 100 ++ .../app/src-tauri/src/asr/local/mod.rs | 94 +- .../app/src-tauri/src/asr/local/models.rs | 98 +- .../src/asr/local/sherpa_download.rs | 8 +- .../app/src-tauri/src/asr/local/test_run.rs | 111 ++- .../src/asr/local/whisper_provider.rs | 242 +++++ .../app/src-tauri/src/commands/credentials.rs | 46 +- .../app/src-tauri/src/commands/local_asr.rs | 11 +- .../app/src-tauri/src/commands/mod.rs | 34 +- .../app/src-tauri/src/commands/providers.rs | 28 +- openless-all/app/src-tauri/src/coordinator.rs | 58 +- .../src-tauri/src/coordinator/asr_wiring.rs | 139 ++- .../src-tauri/src/coordinator/dictation.rs | 134 ++- .../src-tauri/src/coordinator/qa_session.rs | 45 +- .../src-tauri/src/coordinator/resources.rs | 4 +- .../app/src-tauri/vendor/qwen3-asr-rs | 1 + openless-all/app/src/i18n/en.ts | 9 + openless-all/app/src/i18n/ja.ts | 9 + openless-all/app/src/i18n/ko.ts | 9 + openless-all/app/src/i18n/zh-CN.ts | 9 + openless-all/app/src/i18n/zh-TW.ts | 9 + openless-all/app/src/lib/localAsr.ts | 4 +- openless-all/app/src/pages/History.tsx | 7 +- .../app/src/pages/LocalAsr/components.tsx | 72 +- openless-all/app/src/pages/LocalAsr/index.tsx | 87 +- .../src/pages/settings/ChannelList.test.ts | 8 +- .../app/src/pages/settings/ChannelList.tsx | 3 +- .../src/pages/settings/ProvidersSection.tsx | 8 +- .../app/src/pages/settings/shared.tsx | 4 +- 39 files changed, 2211 insertions(+), 249 deletions(-) create mode 100644 openless-all/app/scripts/check-macos-metal-toolchain.mjs create mode 100644 openless-all/app/src-tauri/src/asr/local/mlx_qwen_engine.rs create mode 100644 openless-all/app/src-tauri/src/asr/local/whisper_provider.rs create mode 160000 openless-all/app/src-tauri/vendor/qwen3-asr-rs diff --git a/.gitmodules b/.gitmodules index 1c75230a9..083570cba 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "openless-all/app/src-tauri/vendor/qwen-asr"] path = openless-all/app/src-tauri/vendor/qwen-asr url = https://github.com/Open-Less/qwen-asr.git +[submodule "openless-all/app/src-tauri/vendor/qwen3-asr-rs"] + path = openless-all/app/src-tauri/vendor/qwen3-asr-rs + url = git@github.com:jimersylee/qwen3_asr_rs.git diff --git a/openless-all/app/package.json b/openless-all/app/package.json index c5b0d7184..e5912836a 100644 --- a/openless-all/app/package.json +++ b/openless-all/app/package.json @@ -10,6 +10,7 @@ "prebuild": "node scripts/android-ipc-import-boundary.test.mjs", "build": "tsc && vite build", "preview": "vite preview", + "pretauri": "node scripts/check-macos-metal-toolchain.mjs", "tauri": "tauri", "tauri:android:init": "tauri android init", "tauri:android:dev": "tauri android dev", diff --git a/openless-all/app/scripts/check-macos-metal-toolchain.mjs b/openless-all/app/scripts/check-macos-metal-toolchain.mjs new file mode 100644 index 000000000..b9b58c3e0 --- /dev/null +++ b/openless-all/app/scripts/check-macos-metal-toolchain.mjs @@ -0,0 +1,25 @@ +import { spawnSync } from "node:child_process" + +if (process.platform !== "darwin") process.exit(0) + +const result = spawnSync("xcrun", ["--find", "metal"], { + encoding: "utf8", +}) + +if (result.status === 0 && result.stdout?.trim()) process.exit(0) + +console.error(` +OpenLess 的 Qwen3-ASR MLX 后端需要 Apple MetalToolchain。 +当前 Xcode 未找到 Metal 编译器，请先执行： + + xcodebuild -downloadComponent MetalToolchain + +完成后验证： + + xcrun --find metal + +然后重新运行： + + pnpm tauri dev +`) +process.exit(1) diff --git a/openless-all/app/src-tauri/Cargo.lock b/openless-all/app/src-tauri/Cargo.lock index ff8097590..8e99c5883 100644 --- a/openless-all/app/src-tauri/Cargo.lock +++ b/openless-all/app/src-tauri/Cargo.lock @@ -19,6 +19,20 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -160,6 +174,12 @@ dependencies = [ "x11rb", ] +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-broadcast" version = "0.7.2" @@ -370,10 +390,13 @@ dependencies = [ "matchit", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "rustversion", "serde", + "serde_json", + "serde_path_to_error", "serde_urlencoded", "sha1", "sync_wrapper", @@ -382,6 +405,7 @@ dependencies = [ "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -402,8 +426,15 @@ dependencies = [ "sync_wrapper", "tower-layer", "tower-service", + "tracing", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -416,6 +447,26 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags 2.13.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.117", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -425,7 +476,7 @@ dependencies = [ "bitflags 2.13.0", "cexpr", "clang-sys", - "itertools", + "itertools 0.13.0", "proc-macro2", "quote", "regex", @@ -720,6 +771,15 @@ dependencies = [ "toml 0.9.12+spec-1.1.0", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cbc" version = "0.1.2" @@ -835,6 +895,46 @@ dependencies = [ "libloading 0.8.9", ] +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "clipboard-win" version = "5.4.1" @@ -844,6 +944,15 @@ dependencies = [ "error-code", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cocoa" version = "0.26.1" @@ -889,6 +998,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -898,6 +1022,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -1018,7 +1155,7 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9b4739a805a62757a83e5654fa3faabec0442666b263bb2287d5a8185bfd953" dependencies = [ - "bindgen", + "bindgen 0.72.1", ] [[package]] @@ -1095,6 +1232,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1225,6 +1381,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "dasp_sample" version = "0.11.0" @@ -1541,6 +1706,12 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ef6b89e5b37196644d8796de5268852ff179b44e96276cf4290264843743bb7" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -1650,6 +1821,15 @@ version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +dependencies = [ + "cc", +] + [[package]] name = "event-listener" version = "5.4.1" @@ -1671,6 +1851,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fastrand" version = "2.4.1" @@ -1853,6 +2039,12 @@ dependencies = [ "zip 2.4.2", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fst" version = "0.4.7" @@ -2373,6 +2565,12 @@ dependencies = [ "digest", ] +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "html5ever" version = "0.38.0" @@ -2703,6 +2901,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + [[package]] name = "infer" version = "0.19.0" @@ -2762,6 +2973,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2960,6 +3180,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -3118,6 +3344,22 @@ dependencies = [ "libc", ] +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + [[package]] name = "malloc_buf" version = "0.0.6" @@ -3138,6 +3380,15 @@ dependencies = [ "web_atoms", ] +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.7.3" @@ -3227,6 +3478,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "moxcms" version = "0.8.1" @@ -3258,6 +3531,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + [[package]] name = "munge" version = "0.4.7" @@ -3415,6 +3705,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num" version = "0.4.3" @@ -3536,6 +3835,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "objc" version = "0.2.7" @@ -3905,6 +4210,28 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags 2.13.0", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "open" version = "5.3.5" @@ -3959,6 +4286,7 @@ dependencies = [ "objc2-foundation 0.2.2", "once_cell", "parking_lot", + "qwen3-asr-rs", "raw-window-handle", "rcgen", "reqwest 0.12.28", @@ -3982,12 +4310,14 @@ dependencies = [ "tauri-plugin-single-instance", "tauri-plugin-updater", "thiserror 1.0.69", + "tokenizers", "tokio", "tokio-rustls", "tokio-tungstenite", "tower", "url", "uuid", + "whisper-rs", "window-vibrancy 0.7.1", "windows 0.58.0", "winreg 0.52.0", @@ -4131,6 +4461,18 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pathdiff" version = "0.2.3" @@ -4358,6 +4700,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -4526,6 +4877,29 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "qwen3-asr-rs" +version = "0.2.0" +dependencies = [ + "anyhow", + "axum", + "clap", + "cmake", + "hound", + "rubato", + "safetensors", + "serde", + "serde_json", + "symphonia", + "tempfile", + "thiserror 2.0.18", + "tokenizers", + "tokio", + "tower-http 0.5.2", + "tracing", + "tracing-subscriber", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -4554,13 +4928,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", - "rand_chacha", + "rand_chacha 0.3.1", "rand_core 0.6.4", ] [[package]] name = "rand" -version = "0.10.2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ @@ -4579,6 +4963,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -4588,6 +4982,15 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "rand_core" version = "0.10.1" @@ -4609,6 +5012,37 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools 0.14.0", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rcgen" version = "0.13.2" @@ -4622,6 +5056,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -4749,7 +5192,7 @@ dependencies = [ "tokio-rustls", "tokio-util", "tower", - "tower-http", + "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", @@ -4789,7 +5232,7 @@ dependencies = [ "tokio-rustls", "tokio-util", "tower", - "tower-http", + "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", @@ -4866,6 +5309,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "rubato" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5258099699851cfd0082aeb645feb9c084d9a5e1f1b8d5372086b989fc5e56a1" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "realfft", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -4881,6 +5336,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.4" @@ -4981,6 +5450,16 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safetensors" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc0cdb7198d738a111f6df8fef42cb175412c311d0c4ac9126ff4e550ad1a0e8" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "same-file" version = "1.0.6" @@ -5206,6 +5685,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -5332,6 +5822,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shared_child" version = "1.1.1" @@ -5517,6 +6016,24 @@ dependencies = [ "system-deps", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -5529,6 +6046,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "string_cache" version = "0.9.0" @@ -5576,6 +6099,164 @@ dependencies = [ "serde_json", ] +[[package]] +name = "symphonia" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-adpcm", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dddc50e2bbea4cfe027441eece77c46b9f319748605ab8f3443350129ddd07f" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73" +dependencies = [ + "log", + "symphonia-core", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122d786d2c43a49beb6f397551b4a050d8229eaa54c7ddf9ee4b98899b8742d0" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-utils-xiph" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16" +dependencies = [ + "symphonia-core", + "symphonia-metadata", +] + [[package]] name = "syn" version = "1.0.109" @@ -5597,6 +6278,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -6161,6 +6853,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + [[package]] name = "tiff" version = "0.11.3" @@ -6233,6 +6934,40 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "indicatif", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.5", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.52.3" @@ -6454,6 +7189,24 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.13.0", + "bytes", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", ] [[package]] @@ -6492,6 +7245,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -6515,6 +7269,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", ] [[package]] @@ -6652,18 +7446,39 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "untrusted" version = "0.9.0" @@ -6782,6 +7597,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -7191,6 +8012,27 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "whisper-rs" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d2eac0a371f8ae667a5ee15ae4130553ea3004e7572544d1ce546c81ea8874b" +dependencies = [ + "whisper-rs-sys", +] + +[[package]] +name = "whisper-rs-sys" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c86f1b993f216594b1ad9a9bb00a26014fb7c512e12664a2d401c7897d2ef7d" +dependencies = [ + "bindgen 0.71.1", + "cfg-if", + "cmake", + "fs_extra", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/openless-all/app/src-tauri/Cargo.toml b/openless-all/app/src-tauri/Cargo.toml index dd5d8e542..13e49bee0 100644 --- a/openless-all/app/src-tauri/Cargo.toml +++ b/openless-all/app/src-tauri/Cargo.toml @@ -117,6 +117,9 @@ minisign-verify = "0.2" [target.'cfg(target_os = "macos")'.dependencies] block2 = "0.5" core-foundation = "0.10" +qwen3-asr-rs = { path = "vendor/qwen3-asr-rs", default-features = false, features = ["mlx"] } +tokenizers = "0.21" +whisper-rs = { version = "0.14", features = ["metal"] } core-graphics = "0.24" # issue #470：托盘麦克风设备变更改用 CoreAudio AudioObjectAddPropertyListener 原生通知 # （替代 10s 轮询，空闲零唤醒）。符号本就在依赖树（cpal → coreaudio-sys 0.2），提升为 diff --git a/openless-all/app/src-tauri/build.rs b/openless-all/app/src-tauri/build.rs index c7332edb9..d73d56615 100644 --- a/openless-all/app/src-tauri/build.rs +++ b/openless-all/app/src-tauri/build.rs @@ -2,8 +2,8 @@ fn main() { #[cfg(target_os = "windows")] link_windows_common_controls_v6_manifest_dependency(); - #[cfg(target_os = "macos")] - build_qwen_asr_macos(); + #[cfg(any(target_os = "macos", target_os = "linux"))] + build_qwen_asr(); if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("android") { link_android_cpp_runtime(); @@ -40,14 +40,14 @@ int openless_common_controls_v6_manifest_dependency_anchor = 0; ); } -/// 编译 vendored Open-Less/qwen-asr 的 C 源（仅 macOS）。 +/// 编译 vendored Open-Less/qwen-asr 的 C 源（macOS/Linux）。 /// /// 上游 Makefile `make blas` 等价配置：BLAS 加速通过 Accelerate framework， /// `USE_BLAS` + `ACCELERATE_NEW_LAPACK` 是必要宏。 /// `-march=native` 这里**不**用——分发二进制要可移植，cc crate 在 release 下 /// 默认带 `-O2`，加上 `-O3` 提一档；NEON/AVX 在源码里有 `#ifdef` 自动分派。 -#[cfg(target_os = "macos")] -fn build_qwen_asr_macos() { +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn build_qwen_asr() { const VENDOR: &str = "vendor/qwen-asr"; const SOURCES: &[&str] = &[ "qwen_asr.c", @@ -65,8 +65,6 @@ fn build_qwen_asr_macos() { let mut build = cc::Build::new(); build .include(VENDOR) - .define("USE_BLAS", None) - .define("ACCELERATE_NEW_LAPACK", None) .flag("-O3") .flag("-ffast-math") // 上游开 `-Wall -Wextra`；我们把 qwen-asr 的代码当三方依赖，把无关警告压成静默 @@ -77,6 +75,11 @@ fn build_qwen_asr_macos() { .flag("-Wno-sign-compare") .warnings(false); + #[cfg(target_os = "macos")] + build + .define("USE_BLAS", None) + .define("ACCELERATE_NEW_LAPACK", None); + for src in SOURCES { let path = format!("{}/{}", VENDOR, src); println!("cargo:rerun-if-changed={}", path); @@ -87,9 +90,18 @@ fn build_qwen_asr_macos() { build.compile("qwen_asr"); // BLAS = Accelerate + #[cfg(target_os = "macos")] println!("cargo:rustc-link-lib=framework=Accelerate"); + // Linux 不依赖发行版的 OpenBLAS 开发包，先走 C 引擎自带的通用 CPU kernels。 + #[cfg(target_os = "linux")] + { + println!("cargo:rustc-link-lib=m"); + println!("cargo:rustc-link-lib=pthread"); + } + // Apple Speech 本地 ASR（issue #574）：apple_speech_provider 用 // SFSpeechRecognizer / SFSpeechURLRecognitionRequest，符号在 Speech.framework。 + #[cfg(target_os = "macos")] println!("cargo:rustc-link-lib=framework=Speech"); } diff --git a/openless-all/app/src-tauri/src/asr/local/cache.rs b/openless-all/app/src-tauri/src/asr/local/cache.rs index 43c043b5b..7efe20b23 100644 --- a/openless-all/app/src-tauri/src/asr/local/cache.rs +++ b/openless-all/app/src-tauri/src/asr/local/cache.rs @@ -18,20 +18,21 @@ use std::time::{Duration, Instant}; use anyhow::Result; use parking_lot::Mutex; -#[cfg(target_os = "macos")] -use super::QwenAsrEngine; +#[cfg(any(target_os = "macos", target_os = "linux"))] +use super::{LocalQwenEngine, QwenBackend}; pub struct LocalAsrCache { - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] inner: Mutex>, - #[cfg(not(target_os = "macos"))] + #[cfg(not(any(target_os = "macos", target_os = "linux")))] _phantom: (), } -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] struct CachedEngine { model_id: String, - engine: Arc, + backend: QwenBackend, + engine: Arc, last_used: Instant, } @@ -44,21 +45,26 @@ impl Default for LocalAsrCache { impl LocalAsrCache { pub fn new() -> Self { Self { - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] inner: Mutex::new(None), - #[cfg(not(target_os = "macos"))] + #[cfg(not(any(target_os = "macos", target_os = "linux")))] _phantom: (), } } /// 取已缓存的同 id 引擎，没有就加载（**阻塞、可能数秒**——调用方应放 /// `spawn_blocking`）。模型 id 不同则把旧的 drop 再加载新的。 - #[cfg(target_os = "macos")] - pub fn get_or_load(&self, model_id: &str, model_dir: &Path) -> Result> { + #[cfg(any(target_os = "macos", target_os = "linux"))] + pub fn get_or_load( + &self, + backend: QwenBackend, + model_id: &str, + model_dir: &Path, + ) -> Result> { { let mut slot = self.inner.lock(); if let Some(cached) = slot.as_mut() { - if cached.model_id == model_id { + if cached.model_id == model_id && cached.backend == backend { cached.last_used = Instant::now(); log::info!("[local-asr cache] reuse engine: {model_id}"); return Ok(Arc::clone(&cached.engine)); @@ -72,13 +78,15 @@ impl LocalAsrCache { } } log::info!( - "[local-asr cache] loading {model_id} from {}", + "[local-asr cache] loading {}:{model_id} from {}", + backend.cache_key(), model_dir.display() ); - let engine = Arc::new(QwenAsrEngine::load(model_dir)?); + let engine = Arc::new(LocalQwenEngine::load(backend, model_dir)?); let mut slot = self.inner.lock(); *slot = Some(CachedEngine { model_id: model_id.to_string(), + backend, engine: Arc::clone(&engine), last_used: Instant::now(), }); @@ -89,7 +97,7 @@ impl LocalAsrCache { /// 标记最近使用时间——end_session 在调过 transcribe 之后调一下， /// 让 release 计时器从这一刻重新算。 pub fn touch(&self) { - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] { if let Some(cached) = self.inner.lock().as_mut() { cached.last_used = Instant::now(); @@ -99,7 +107,7 @@ impl LocalAsrCache { /// 如果空闲时长 ≥ threshold，释放引擎。返回是否真释放了。 pub fn release_if_idle(&self, idle_threshold: Duration) -> bool { - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] { let taken = { let mut slot = self.inner.lock(); @@ -117,7 +125,7 @@ impl LocalAsrCache { }; if let Some(cached) = taken { drop(cached); - pressure_relief_macos(); + pressure_relief(); return true; } } @@ -127,7 +135,7 @@ impl LocalAsrCache { /// 立刻释放（用户点"立即释放"、切走 provider、删模型时调）。 pub fn release_now(&self) { - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] { let taken = self.inner.lock().take(); if let Some(cached) = taken { @@ -136,26 +144,29 @@ impl LocalAsrCache { cached.model_id ); drop(cached); - pressure_relief_macos(); + pressure_relief(); } } } pub fn loaded_model_id(&self) -> Option { - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] { return self.inner.lock().as_ref().map(|c| c.model_id.clone()); } - #[cfg(not(target_os = "macos"))] + #[cfg(not(any(target_os = "macos", target_os = "linux")))] None } } -/// drop QwenAsrEngine 后调一次：让 macOS libmalloc 把 freelist 上的物理页归还内核。 +#[cfg(target_os = "linux")] +fn pressure_relief() {} + +/// drop MLX Qwen 引擎后调一次：让 macOS libmalloc 把 freelist 上的物理页归还内核。 /// 不调的话，encoder f32 weights 那 ~几百 MB 的 free 不会立刻反映到 RSS，活动监视器 /// 看起来"释放按钮没生效"。decoder bf16 走 mmap，munmap 时已立即生效，不依赖这个调用。 #[cfg(target_os = "macos")] -fn pressure_relief_macos() { +fn pressure_relief() { // SAFETY: 系统 API；NULL zone + goal=0 = 对所有 zone 尽量多地归还，无内存安全风险。 let freed = unsafe { malloc_zone_pressure_relief(std::ptr::null_mut(), 0) }; log::info!( diff --git a/openless-all/app/src-tauri/src/asr/local/download.rs b/openless-all/app/src-tauri/src/asr/local/download.rs index a5c8a2019..5f95795e9 100644 --- a/openless-all/app/src-tauri/src/asr/local/download.rs +++ b/openless-all/app/src-tauri/src/asr/local/download.rs @@ -101,7 +101,7 @@ struct HfTreeEntry { pub async fn fetch_remote_info(model_id: ModelId, mirror: Mirror) -> Result { let client = build_client()?; - let files = fetch_file_list(&client, model_id.hf_repo(), mirror).await?; + let files = fetch_file_list(&client, model_id, mirror).await?; let total_bytes = files.iter().map(|f| f.size).sum(); Ok(RemoteInfo { model_id: model_id.as_str().into(), @@ -113,9 +113,10 @@ pub async fn fetch_remote_info(model_id: ModelId, mirror: Mirror) -> Result Result> { + let repo = model_id.hf_repo(); let url = format!("{}/api/models/{}/tree/main", mirror.base_url(), repo); let resp = client .get(&url) @@ -131,7 +132,7 @@ async fn fetch_file_list( .with_context(|| format!("HF tree JSON 解码失败: {url}"))?; let files: Vec = entries .into_iter() - .filter(|e| e.entry_type == "file" && keep_file(&e.path)) + .filter(|e| e.entry_type == "file" && keep_file(&e.path, model_id)) .map(|e| RemoteFile { path: e.path, size: e.size.unwrap_or(0), @@ -143,7 +144,10 @@ async fn fetch_file_list( Ok(files) } -fn keep_file(path: &str) -> bool { +fn keep_file(path: &str, model_id: ModelId) -> bool { + if let Some(file_name) = model_id.file_name() { + return path == file_name; + } if path.starts_with('.') { return false; } diff --git a/openless-all/app/src-tauri/src/asr/local/foundry_native.rs b/openless-all/app/src-tauri/src/asr/local/foundry_native.rs index cc9ebf808..17534fca1 100644 --- a/openless-all/app/src-tauri/src/asr/local/foundry_native.rs +++ b/openless-all/app/src-tauri/src/asr/local/foundry_native.rs @@ -228,11 +228,7 @@ mod imp { })?; } return Err(replace_error).with_context(|| { - format!( - "move {} to {}", - staging_dir.display(), - target_dir.display() - ) + format!("move {} to {}", staging_dir.display(), target_dir.display()) }); } diff --git a/openless-all/app/src-tauri/src/asr/local/local_provider.rs b/openless-all/app/src-tauri/src/asr/local/local_provider.rs index bd868e131..bc2e256d2 100644 --- a/openless-all/app/src-tauri/src/asr/local/local_provider.rs +++ b/openless-all/app/src-tauri/src/asr/local/local_provider.rs @@ -1,49 +1,35 @@ //! 本地 Qwen3-ASR 在 dictation 路径上的适配器。 //! //! 与 `WhisperBatchASR` 形状对齐：实现 `AudioConsumer` 缓冲 PCM，stop 时 -//! 调 `transcribe_stream`，期间每个稳定 token 通过 Tauri 事件 -//! `local-asr-token` 推到前端胶囊做实时显示。 +//! 调本地 Qwen3-ASR 的 batch 解码，不向前端发送中间 token。 //! //! engine 现在由 `LocalAsrCache` 提供——Coordinator 在 build_local_qwen3 里 //! 取已缓存的引擎再传进来，避免每次会话都重加载 1.2GB+ 模型。 -#[cfg(target_os = "macos")] -use std::sync::atomic::{AtomicBool, Ordering}; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] use std::sync::Arc; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] +use super::LocalQwenEngine; +#[cfg(any(target_os = "macos", target_os = "linux"))] +use crate::asr::RawTranscript; +#[cfg(any(target_os = "macos", target_os = "linux"))] use anyhow::{Context, Result}; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] use parking_lot::Mutex; -#[cfg(target_os = "macos")] -use tauri::{AppHandle, Emitter}; - -#[cfg(target_os = "macos")] -use super::QwenAsrEngine; -#[cfg(target_os = "macos")] -use crate::asr::RawTranscript; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] pub struct LocalQwenAsr { - engine: Arc, - /// `spawn_blocking` 已经启动后不能被 JoinHandle 强制终止；取消先关闭 - /// 当前会话的 token 门控，确保 native worker 返回前不会向新 UI 会话泄漏旧 token。 - cancelled: Arc, - /// 16-bit LE PCM 字节缓冲（recorder 推什么我们存什么），在 transcribe 时再 - /// 转 f32 喂给 C 端。一次会话最多几 MB，clone 一次成本可接受。 + engine: Arc, buffer: Mutex>, - app: AppHandle, } -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] impl LocalQwenAsr { - pub fn new(app: AppHandle, engine: Arc) -> Self { + pub fn new(engine: Arc) -> Self { Self { engine, - cancelled: Arc::new(AtomicBool::new(false)), buffer: Mutex::new(Vec::new()), - app, } } @@ -54,10 +40,8 @@ impl LocalQwenAsr { (self.buffer.lock().len() as u64 / 2) * 1000 / 16_000 } - /// stop 时调用：把 buffer 的 i16 PCM 转 f32，跑流式转写，token 实时 - /// 通过事件吐到前端胶囊；最终文本一起返回供 polish/insert。 + /// stop 时调用：把 PCM 转 f32，整段执行一次 batch 解码。 pub async fn transcribe(self: Arc) -> Result { - self.cancelled.store(false, Ordering::Release); let pcm_bytes = std::mem::take(&mut *self.buffer.lock()); if pcm_bytes.is_empty() { return Ok(RawTranscript { @@ -66,56 +50,30 @@ impl LocalQwenAsr { }); } let duration_ms = (pcm_bytes.len() as u64 / 2) * 1000 / 16_000; - let mut samples_f32 = i16_le_bytes_to_f32(&pcm_bytes); - // `transcribe_stream` 内部按 2s chunk 切片；末 chunk < 2s 且缓冲没有 - // 静默尾巴时，C 引擎不会把它当作"语音已结束"，该 chunk 的转写结果 - // 会被丢弃，导致末段内容消失。这里追加 0.5s 静默（@16kHz = 8000 个 - // f32 零值）作为收尾信号。`duration_ms` 仍按原始缓冲长度计算（上面 - // 一行），padding 不计入。 - samples_f32.extend(std::iter::repeat(0.0f32).take(8_000)); - - // 注册 token 回调：每个稳定 token 抛 `local-asr-token` 事件。 - // capsule 前端按 sessionId 累积显示。回调安装、native 调用和解绑 - // 在 QwenAsrEngine 的同一把 context 锁内完成。 - let app = self.app.clone(); - let cancelled = Arc::clone(&self.cancelled); - - // qwen_transcribe_stream 是阻塞调用；用 spawn_blocking 防止占住 tokio runtime。 - // 用 tauri::async_runtime::spawn_blocking 而非 tokio 的 —— 同 download.rs 注释， - // 走 Tauri 持有的 runtime handle，不依赖调用方上下文（虽然这里目前都在 async 路径上调， - // 但保持一致更稳）。 + let samples_f32 = i16_le_bytes_to_f32(&pcm_bytes); let engine = Arc::clone(&self.engine); - let text = tauri::async_runtime::spawn_blocking(move || { - engine.transcribe_stream_with_handler(&samples_f32, move |piece: &str| { - if cancelled.load(Ordering::Acquire) { - return; - } - if let Err(e) = app.emit("local-asr-token", piece.to_string()) { - log::warn!("[local-asr] emit token failed: {e}"); - } - }) - }) - .await - .context("transcribe spawn_blocking join 失败")? - .context("qwen_transcribe_stream 失败")?; + let text = + tauri::async_runtime::spawn_blocking(move || engine.transcribe_pcm(&samples_f32)) + .await + .context("transcribe spawn_blocking join 失败")? + .context("本地 Qwen3-ASR batch 解码失败")?; Ok(RawTranscript { text, duration_ms }) } pub fn cancel(&self) { - self.cancelled.store(true, Ordering::Release); self.buffer.lock().clear(); } } -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] impl crate::recorder::AudioConsumer for LocalQwenAsr { fn consume_pcm_chunk(&self, pcm: &[u8]) { self.buffer.lock().extend_from_slice(pcm); } } -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] fn i16_le_bytes_to_f32(bytes: &[u8]) -> Vec { bytes .chunks_exact(2) diff --git a/openless-all/app/src-tauri/src/asr/local/mlx_qwen_engine.rs b/openless-all/app/src-tauri/src/asr/local/mlx_qwen_engine.rs new file mode 100644 index 000000000..f6499ba79 --- /dev/null +++ b/openless-all/app/src-tauri/src/asr/local/mlx_qwen_engine.rs @@ -0,0 +1,100 @@ +//! qwen3_asr_rs 的 MLX/Metal 包装。 +//! +//! 上游库目前以音频文件作为输入。OpenLess 的录音器产生的是 16 kHz、单声道、 +//! 16-bit PCM，因此这里只做一次临时 WAV 封装；模型本身保持驻留并跨会话复用。 + +use std::path::Path; +use std::sync::Mutex; + +use anyhow::{Context, Result}; +use qwen3_asr_rs::inference::AsrInference; +use qwen3_asr_rs::tensor::Device; + +pub struct MlxQwenAsrEngine { + inference: Mutex, +} + +impl MlxQwenAsrEngine { + pub fn load(model_dir: &Path) -> Result { + ensure_tokenizer_json(model_dir)?; + // qwen3_asr_rs 的 CLI 会在加载模型前做这一步；OpenLess 直接调用库 API， + // 必须自行初始化全局 MLX stream，否则首次创建张量会 panic。 + qwen3_asr_rs::backend::mlx::stream::init_mlx(true); + log::info!( + "[local-qwen3-mlx] loading model from {}", + model_dir.display() + ); + let inference = AsrInference::load(model_dir, Device::gpu()) + .with_context(|| format!("加载 Qwen3-ASR MLX 模型失败: {}", model_dir.display()))?; + Ok(Self { + inference: Mutex::new(inference), + }) + } + + pub fn transcribe_pcm(&self, samples: &[f32]) -> Result { + let path = + std::env::temp_dir().join(format!("openless-qwen3-{}.wav", uuid::Uuid::new_v4())); + let pcm: Vec = samples + .iter() + .map(|sample| (sample.clamp(-1.0, 1.0) * 32767.0) as i16) + .collect(); + std::fs::write(&path, crate::asr::wav::encode_wav_16k_mono(&pcm)) + .with_context(|| format!("写入临时 Qwen3-ASR 音频失败: {}", path.display()))?; + + let path_string = path.to_string_lossy().into_owned(); + let result = self + .inference + .lock() + .map_err(|_| anyhow::anyhow!("Qwen3-ASR MLX 引擎锁已中毒"))? + .transcribe(&path_string, None) + .context("Qwen3-ASR MLX batch 解码失败"); + let _ = std::fs::remove_file(&path); + result.map(|output| output.text.trim().to_string()) + } +} + +/// Qwen 官方 ASR 权重通常只有 `vocab.json` + `merges.txt`，而 qwen3_asr_rs +/// 使用 HuggingFace 的统一 `tokenizer.json`。这里在首次加载时本地生成一次， +/// 避免要求用户安装 Python/Transformers；如果模型包已经带 tokenizer.json，则直接复用。 +fn ensure_tokenizer_json(model_dir: &Path) -> Result<()> { + let tokenizer_path = model_dir.join("tokenizer.json"); + if tokenizer_path.is_file() { + return Ok(()); + } + let vocab = model_dir.join("vocab.json"); + let merges = model_dir.join("merges.txt"); + if !vocab.is_file() || !merges.is_file() { + anyhow::bail!( + "Qwen3-ASR MLX 模型缺少 tokenizer.json、vocab.json 或 merges.txt: {}", + model_dir.display() + ); + } + let vocab = vocab + .to_str() + .ok_or_else(|| anyhow::anyhow!("Qwen3-ASR vocab 路径不是有效 UTF-8"))?; + let merges = merges + .to_str() + .ok_or_else(|| anyhow::anyhow!("Qwen3-ASR merges 路径不是有效 UTF-8"))?; + let model = tokenizers::models::bpe::BPE::from_file(vocab, merges) + .build() + .map_err(|error| anyhow::anyhow!("生成 Qwen3-ASR BPE tokenizer 失败: {error}"))?; + let mut tokenizer = tokenizers::Tokenizer::new(model); + tokenizer.with_pre_tokenizer(Some( + tokenizers::pre_tokenizers::byte_level::ByteLevel::default(), + )); + tokenizer.with_decoder(Some(tokenizers::decoders::byte_level::ByteLevel::default())); + let temporary = tokenizer_path.with_extension("json.partial"); + let tokenizer_json = tokenizer + .to_string(false) + .map_err(|error| anyhow::anyhow!("序列化 Qwen3-ASR tokenizer 失败: {error}"))?; + std::fs::write(&temporary, tokenizer_json) + .with_context(|| format!("写入 Qwen3-ASR tokenizer 失败: {}", temporary.display()))?; + std::fs::rename(&temporary, &tokenizer_path).with_context(|| { + format!( + "提交 Qwen3-ASR tokenizer 失败: {} -> {}", + temporary.display(), + tokenizer_path.display() + ) + })?; + Ok(()) +} diff --git a/openless-all/app/src-tauri/src/asr/local/mod.rs b/openless-all/app/src-tauri/src/asr/local/mod.rs index cb60aec92..3d3792630 100644 --- a/openless-all/app/src-tauri/src/asr/local/mod.rs +++ b/openless-all/app/src-tauri/src/asr/local/mod.rs @@ -1,7 +1,8 @@ //! 本地 ASR 引擎入口。 //! //! 当前本地引擎： -//! - **macOS**：`antirez/qwen-asr` 纯 C + Accelerate（`local_provider` / `qwen_engine`） +//! - **macOS**：Qwen3-ASR 可选 MLX/Metal 或 C/CPU； +//! - **Linux**：Qwen3-ASR C/CPU； //! - **Windows**：Foundry Local Whisper（`foundry_*`），以及 sherpa-onnx-local //! 实验 provider（`sherpa*`，offline batch + online streaming） @@ -11,6 +12,7 @@ pub mod foundry; pub mod foundry_native; pub mod foundry_provider; pub mod foundry_runtime; +#[cfg(any(target_os = "macos", target_os = "linux"))] mod local_provider; pub mod models; pub mod sherpa; @@ -19,6 +21,9 @@ pub mod sherpa_provider; pub mod sherpa_runtime; pub mod test_run; +#[cfg(target_os = "macos")] +mod whisper_provider; + pub use cache::LocalAsrCache; #[allow(unused_imports)] pub use foundry_provider::FoundryLocalWhisperAsr; @@ -32,26 +37,105 @@ pub use sherpa_runtime::SherpaOnnxRuntime; #[cfg(target_os = "macos")] mod apple_speech_provider; #[cfg(target_os = "macos")] +mod mlx_qwen_engine; +#[cfg(any(target_os = "macos", target_os = "linux"))] mod qwen_engine; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] mod qwen_ffi; #[cfg(target_os = "macos")] #[allow(unused_imports)] pub use apple_speech_provider::{native_name_to_apple_locale, AppleSpeechAsr}; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] pub use local_provider::LocalQwenAsr; #[cfg(target_os = "macos")] -pub use qwen_engine::QwenAsrEngine; +pub use mlx_qwen_engine::MlxQwenAsrEngine; +#[cfg(target_os = "macos")] +pub use whisper_provider::MODEL_ID as WHISPER_MODEL_ID; +#[cfg(target_os = "macos")] +pub use whisper_provider::{ + model_path_for_model as whisper_model_path_for_model, + model_ready_for_model as whisper_model_ready_for_model, +}; +#[cfg(target_os = "macos")] +pub use whisper_provider::{LocalWhisperAsr, LocalWhisperCache}; pub use download::{DownloadManager, Mirror}; pub use models::{ModelId, ModelStatus}; /// 本地 Qwen3-ASR 在 active_asr 字段里的标识；与前端 ASR_PRESETS 的 id 对齐。 +/// 旧版本的本地 Qwen3-ASR provider id。macOS 映射到 MLX，Linux 映射到 C， +/// 仅用于兼容已经保存的渠道配置；新渠道请使用下方两个明确后端 id。 pub const PROVIDER_ID: &str = "local-qwen3"; +pub const LOCAL_QWEN3_MLX_PROVIDER_ID: &str = "local-qwen3-mlx"; +pub const LOCAL_QWEN3_C_PROVIDER_ID: &str = "local-qwen3-c"; + +pub const LOCAL_WHISPER_PROVIDER_ID: &str = "local-whisper"; + +pub fn is_local_whisper(id: &str) -> bool { + id == LOCAL_WHISPER_PROVIDER_ID +} pub fn is_local_qwen3(id: &str) -> bool { - id == PROVIDER_ID + matches!( + id, + PROVIDER_ID | LOCAL_QWEN3_MLX_PROVIDER_ID | LOCAL_QWEN3_C_PROVIDER_ID + ) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum QwenBackend { + #[cfg(target_os = "macos")] + Mlx, + C, +} + +impl QwenBackend { + pub fn cache_key(self) -> &'static str { + match self { + #[cfg(target_os = "macos")] + Self::Mlx => "mlx", + Self::C => "c", + } + } +} + +pub fn qwen_backend_for_provider(id: &str) -> Option { + match id { + #[cfg(target_os = "macos")] + PROVIDER_ID | LOCAL_QWEN3_MLX_PROVIDER_ID => Some(QwenBackend::Mlx), + #[cfg(target_os = "linux")] + PROVIDER_ID | LOCAL_QWEN3_C_PROVIDER_ID => Some(QwenBackend::C), + #[cfg(target_os = "macos")] + LOCAL_QWEN3_C_PROVIDER_ID => Some(QwenBackend::C), + _ => None, + } +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +pub enum LocalQwenEngine { + #[cfg(target_os = "macos")] + Mlx(MlxQwenAsrEngine), + C(qwen_engine::QwenAsrEngine), +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +impl LocalQwenEngine { + pub fn load(backend: QwenBackend, model_dir: &std::path::Path) -> anyhow::Result { + match backend { + #[cfg(target_os = "macos")] + QwenBackend::Mlx => Ok(Self::Mlx(MlxQwenAsrEngine::load(model_dir)?)), + QwenBackend::C => Ok(Self::C(qwen_engine::QwenAsrEngine::load(model_dir)?)), + } + } + + pub fn transcribe_pcm(&self, samples: &[f32]) -> anyhow::Result { + match self { + #[cfg(target_os = "macos")] + Self::Mlx(engine) => engine.transcribe_pcm(samples), + Self::C(engine) => engine.transcribe_audio(samples), + } + } } /// Apple Speech（SFSpeechRecognizer）本地 ASR 的 provider id；与前端 diff --git a/openless-all/app/src-tauri/src/asr/local/models.rs b/openless-all/app/src-tauri/src/asr/local/models.rs index 9ae76f164..bdd3d9df8 100644 --- a/openless-all/app/src-tauri/src/asr/local/models.rs +++ b/openless-all/app/src-tauri/src/asr/local/models.rs @@ -18,6 +18,12 @@ pub(super) const READY_SENTINEL: &str = ".openless-asr-ready"; pub enum ModelId { Small06b, Large17b, + WhisperBase, + WhisperSmall, + WhisperMedium, + WhisperLargeV3, + WhisperLargeV3Turbo, + WhisperLargeV3TurboQ5, } impl ModelId { @@ -25,6 +31,12 @@ impl ModelId { match self { ModelId::Small06b => "qwen3-asr-0.6b", ModelId::Large17b => "qwen3-asr-1.7b", + ModelId::WhisperBase => "whisper-base", + ModelId::WhisperSmall => "whisper-small", + ModelId::WhisperMedium => "whisper-medium", + ModelId::WhisperLargeV3 => "whisper-large-v3", + ModelId::WhisperLargeV3Turbo => "whisper-large-v3-turbo", + ModelId::WhisperLargeV3TurboQ5 => "whisper-large-v3-turbo-q5", } } @@ -32,12 +44,27 @@ impl ModelId { match s { "qwen3-asr-0.6b" => Some(ModelId::Small06b), "qwen3-asr-1.7b" => Some(ModelId::Large17b), + "whisper-base" => Some(ModelId::WhisperBase), + "whisper-small" => Some(ModelId::WhisperSmall), + "whisper-medium" => Some(ModelId::WhisperMedium), + "whisper-large-v3" => Some(ModelId::WhisperLargeV3), + "whisper-large-v3-turbo" => Some(ModelId::WhisperLargeV3Turbo), + "whisper-large-v3-turbo-q5" => Some(ModelId::WhisperLargeV3TurboQ5), _ => None, } } pub fn all() -> &'static [ModelId] { - &[ModelId::Small06b, ModelId::Large17b] + &[ + ModelId::Small06b, + ModelId::Large17b, + ModelId::WhisperBase, + ModelId::WhisperSmall, + ModelId::WhisperMedium, + ModelId::WhisperLargeV3, + ModelId::WhisperLargeV3Turbo, + ModelId::WhisperLargeV3TurboQ5, + ] } /// HuggingFace repo id（用于拼 API + 下载 URL）。 @@ -45,22 +72,70 @@ impl ModelId { match self { ModelId::Small06b => "Qwen/Qwen3-ASR-0.6B", ModelId::Large17b => "Qwen/Qwen3-ASR-1.7B", + ModelId::WhisperBase + | ModelId::WhisperSmall + | ModelId::WhisperMedium + | ModelId::WhisperLargeV3 + | ModelId::WhisperLargeV3Turbo + | ModelId::WhisperLargeV3TurboQ5 => "ggerganov/whisper.cpp", + } + } + + pub fn is_whisper(self) -> bool { + matches!( + self, + ModelId::WhisperBase + | ModelId::WhisperSmall + | ModelId::WhisperMedium + | ModelId::WhisperLargeV3 + | ModelId::WhisperLargeV3Turbo + | ModelId::WhisperLargeV3TurboQ5 + ) + } + + pub fn is_qwen(self) -> bool { + matches!(self, ModelId::Small06b | ModelId::Large17b) + } + + pub fn file_name(self) -> Option<&'static str> { + match self { + ModelId::WhisperBase => Some("ggml-base.bin"), + ModelId::WhisperSmall => Some("ggml-small.bin"), + ModelId::WhisperMedium => Some("ggml-medium.bin"), + ModelId::WhisperLargeV3 => Some("ggml-large-v3.bin"), + ModelId::WhisperLargeV3Turbo => Some("ggml-large-v3-turbo.bin"), + ModelId::WhisperLargeV3TurboQ5 => Some("ggml-large-v3-turbo-q5_0.bin"), + _ => None, } } } /// 模型在本地的根目录（可能不存在）。 pub fn model_dir(id: ModelId) -> Result { - Ok(persistence::local_models_root()?.join(id.as_str())) + if id.is_whisper() { + // Whisper 与 Qwen 共用模型根目录，但各自独立子目录；Turbo 的全精度与 + // Q5 量化文件放同一目录，兼容之前手动迁移的 q5_0 文件。 + let dir_name = if matches!(id, ModelId::WhisperLargeV3TurboQ5) { + ModelId::WhisperLargeV3Turbo.as_str() + } else { + id.as_str() + }; + Ok(persistence::models_root()?.join(dir_name)) + } else { + Ok(persistence::local_models_root()?.join(id.as_str())) + } } -/// 完整且可加载？= 哨兵存在。 +/// 判断模型是否完整且可加载：Whisper 看目标文件，Qwen 看完成哨兵。 /// 比"枚举所有应有文件"稳：HF 仓库改文件名 / 加新文件时不会误报缺失。 pub fn is_downloaded(id: ModelId) -> bool { let dir = match model_dir(id) { Ok(d) => d, Err(_) => return false, }; + if let Some(file_name) = id.file_name() { + return dir.join(file_name).is_file(); + } dir.join(READY_SENTINEL).exists() } @@ -70,6 +145,13 @@ pub fn downloaded_bytes(id: ModelId) -> u64 { Ok(d) => d, Err(_) => return 0, }; + if let Some(file_name) = id.file_name() { + let dest = dir.join(file_name); + if let Ok(meta) = std::fs::metadata(&dest) { + return meta.len(); + } + return super::download::partial_actual_size(&dest.with_extension("partial")); + } let mut total: u64 = 0; walk_files(&dir, &mut |size| total += size); total @@ -130,6 +212,16 @@ pub fn list_status() -> Vec { /// 删除本地模型目录（用户在 UI 主动删）。 pub fn delete_model(id: ModelId) -> Result<()> { let dir = model_dir(id)?; + if let Some(file_name) = id.file_name() { + let dest = dir.join(file_name); + let _ = std::fs::remove_file(&dest); + let _ = std::fs::remove_file(dest.with_extension("partial")); + let _ = std::fs::remove_file(dest.with_extension("partial.idx")); + if dir.exists() && dir.read_dir()?.next().is_none() { + let _ = std::fs::remove_dir(&dir); + } + return Ok(()); + } if dir.exists() { std::fs::remove_dir_all(&dir)?; } diff --git a/openless-all/app/src-tauri/src/asr/local/sherpa_download.rs b/openless-all/app/src-tauri/src/asr/local/sherpa_download.rs index 772e7de04..36727eb3a 100644 --- a/openless-all/app/src-tauri/src/asr/local/sherpa_download.rs +++ b/openless-all/app/src-tauri/src/asr/local/sherpa_download.rs @@ -12,8 +12,8 @@ use sha2::{Digest, Sha256}; use tauri::{AppHandle, Emitter}; use super::download::{ - build_client, download_one, now_millis, partial_actual_size, DownloadPhase, - DownloadProgress, Mirror, PROGRESS_EMIT_MIN_INTERVAL_MS, + build_client, download_one, now_millis, partial_actual_size, DownloadPhase, DownloadProgress, + Mirror, PROGRESS_EMIT_MIN_INTERVAL_MS, }; use super::sherpa; @@ -424,9 +424,7 @@ async fn run_download( // 节流（同 download.rs）：每 HTTP chunk 回调一次，全量转发会 // 高频刷前端进度条；in_flight 照常累计，只按 ≥150ms 转发最新值。 let now = now_millis(); - if now - last_emit.load(Ordering::Relaxed) - < PROGRESS_EMIT_MIN_INTERVAL_MS - { + if now - last_emit.load(Ordering::Relaxed) < PROGRESS_EMIT_MIN_INTERVAL_MS { return; } last_emit.store(now, Ordering::Relaxed); diff --git a/openless-all/app/src-tauri/src/asr/local/test_run.rs b/openless-all/app/src-tauri/src/asr/local/test_run.rs index 930bebdee..3bd48042f 100644 --- a/openless-all/app/src-tauri/src/asr/local/test_run.rs +++ b/openless-all/app/src-tauri/src/asr/local/test_run.rs @@ -5,26 +5,26 @@ //! 1. 用 antirez 项目自带的 `samples/test_speech.wav` 作输入（编进二进制） //! 2. WAV 解析（16kHz mono 16-bit PCM，但 fmt 后面可能有 LIST/INFO 等 //! 非 data chunk，必须按 RIFF 标准走 chunk 链找 "data"，不能 +44 硬偏移） -//! 3. 加载模型，跑 transcribe_audio，分别记录 load_ms / transcribe_ms +//! 3. 加载模型，跑 batch transcribe，分别记录 load_ms / transcribe_ms //! 4. 给前端用：用户点击「加载并测试」按钮立即知道模型是否能跑、有多快、识别什么 -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] use std::path::Path; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] use std::sync::Arc; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] use std::time::Instant; use anyhow::Result; use serde::Serialize; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] use super::models::model_dir; use super::models::ModelId; /// 内嵌测试音频。原始文件 `vendor/qwen-asr/samples/test_speech.wav` /// 内容："Hello. This is a test of the Voxtrail speech-to-text system." -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] const TEST_WAV: &[u8] = include_bytes!("../../../vendor/qwen-asr/samples/test_speech.wav"); /// 测试结果给前端展示。 @@ -40,17 +40,28 @@ pub struct TestResult { pub transcribe_ms: u64, } -#[cfg(target_os = "macos")] -pub async fn run_test(model_id: ModelId) -> Result { +#[cfg(any(target_os = "macos", target_os = "linux"))] +pub async fn run_test( + model_id: ModelId, + backend: Option, +) -> Result { + if model_id.is_whisper() { + #[cfg(target_os = "macos")] + return run_whisper_test(model_id).await; + #[cfg(target_os = "linux")] + anyhow::bail!("本地 Whisper 测试仅支持 macOS"); + } + let backend = + backend.ok_or_else(|| anyhow::anyhow!("当前系统不支持所选的本地 Qwen3-ASR 后端"))?; let dir = model_dir(model_id)?; if !dir.exists() { anyhow::bail!("模型目录不存在：{}（请先下载）", dir.display()); } // ── 模型文件完整性检查 ──────────────────────────────────────────── - // 在调 C FFI 之前先检查关键文件是否齐全、尺寸是否合理，避免因下载不完整 - // 或文件损坏导致 C 端 qwen_load / qwen_transcribe_audio segfault 杀死进程。 - // 需要的文件清单见 QwenAsrEngine::load() 注释。 + // 在调 native 引擎之前先检查关键文件是否齐全、尺寸是否合理，避免因下载不完整 + // 或文件损坏导致模型加载失败。tokenizer.json 会在 MLX 引擎首次加载时从 + // vocab.json / merges.txt 本地生成。 let required_files = ["config.json", "vocab.json", "merges.txt"]; for fname in &required_files { let path = dir.join(fname); @@ -90,25 +101,70 @@ pub async fn run_test(model_id: ModelId) -> Result { let samples = decode_wav_16k_mono(TEST_WAV)?; let audio_ms = (samples.len() as u64) * 1000 / 16_000; - // qwen_load 是同步阻塞调用且较慢（数秒）；扔到 spawn_blocking 不阻塞 tokio runtime。 + // 本地模型加载是同步阻塞调用且较慢（数秒）；扔到 spawn_blocking 不阻塞 tokio runtime。 let load_start = Instant::now(); let dir_for_blocking = dir.clone(); - let engine = tauri::async_runtime::spawn_blocking(move || load_engine(&dir_for_blocking)) + let engine = + tauri::async_runtime::spawn_blocking(move || load_engine(backend, &dir_for_blocking)) + .await + .map_err(|e| anyhow::anyhow!("spawn_blocking join failed: {e:#}"))??; + let load_ms = load_start.elapsed().as_millis() as u64; + + // batch transcribe 也是阻塞 + 重活，同样扔到 blocking pool。 + let trans_start = Instant::now(); + let engine_clone = Arc::clone(&engine); + let text = tauri::async_runtime::spawn_blocking(move || engine_clone.transcribe_pcm(&samples)) .await .map_err(|e| anyhow::anyhow!("spawn_blocking join failed: {e:#}"))??; + let transcribe_ms = trans_start.elapsed().as_millis() as u64; + + Ok(TestResult { + backend: match backend { + #[cfg(target_os = "macos")] + super::QwenBackend::Mlx => "MLX Metal (Apple Silicon)", + super::QwenBackend::C => "C CPU", + } + .into(), + model_id: model_id.as_str().into(), + expected_text: "Hello. This is a test of the Voxtrail speech-to-text system.".into(), + transcribed_text: text, + audio_ms, + load_ms, + transcribe_ms, + }) +} + +#[cfg(target_os = "macos")] +async fn run_whisper_test(model_id: ModelId) -> Result { + use super::whisper_provider::{LocalWhisperCache, WhisperEngine}; + + let path = super::whisper_provider::model_path_for_model(model_id.as_str())?; + if !path.is_file() { + anyhow::bail!("模型文件不存在：{}（请先下载）", path.display()); + } + let samples = decode_wav_16k_mono(TEST_WAV)?; + let audio_ms = samples.len() as u64 * 1000 / 16_000; + let cache = LocalWhisperCache::new(); + let load_start = Instant::now(); + let path_for_blocking = path.clone(); + let model_name = model_id.as_str().to_string(); + let engine = tauri::async_runtime::spawn_blocking(move || { + cache.get_or_load(&model_name, &path_for_blocking) + }) + .await + .map_err(|e| anyhow::anyhow!("spawn_blocking join failed: {e:#}"))??; let load_ms = load_start.elapsed().as_millis() as u64; - // transcribe_audio 也是阻塞 + 重活，同样扔到 blocking pool。 let trans_start = Instant::now(); - let engine_clone = Arc::clone(&engine); - let text = - tauri::async_runtime::spawn_blocking(move || engine_clone.transcribe_audio(&samples)) - .await - .map_err(|e| anyhow::anyhow!("spawn_blocking join failed: {e:#}"))??; + let text = tauri::async_runtime::spawn_blocking(move || { + WhisperEngine::transcribe(&engine, &samples, "en") + }) + .await + .map_err(|e| anyhow::anyhow!("spawn_blocking join failed: {e:#}"))??; let transcribe_ms = trans_start.elapsed().as_millis() as u64; Ok(TestResult { - backend: "Apple Accelerate (AMX/NEON, CPU)".into(), + backend: "whisper.cpp (Metal/CPU)".into(), model_id: model_id.as_str().into(), expected_text: "Hello. This is a test of the Voxtrail speech-to-text system.".into(), transcribed_text: text, @@ -118,14 +174,17 @@ pub async fn run_test(model_id: ModelId) -> Result { }) } -#[cfg(not(target_os = "macos"))] -pub async fn run_test(_model_id: ModelId) -> Result { - anyhow::bail!("本地 ASR 引擎本期仅 macOS 可用（见 issue #256）") +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +pub async fn run_test( + _model_id: ModelId, + _backend: Option, +) -> Result { + anyhow::bail!("本地 Qwen3-ASR C 后端目前仅支持 macOS/Linux；MLX 后端仅支持 macOS") } -#[cfg(target_os = "macos")] -fn load_engine(dir: &Path) -> Result> { - let engine = super::QwenAsrEngine::load(dir)?; +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn load_engine(backend: super::QwenBackend, dir: &Path) -> Result> { + let engine = super::LocalQwenEngine::load(backend, dir)?; Ok(Arc::new(engine)) } diff --git a/openless-all/app/src-tauri/src/asr/local/whisper_provider.rs b/openless-all/app/src-tauri/src/asr/local/whisper_provider.rs new file mode 100644 index 000000000..7d4c368c5 --- /dev/null +++ b/openless-all/app/src-tauri/src/asr/local/whisper_provider.rs @@ -0,0 +1,242 @@ +//! macOS 本地 Whisper Large-v3 Turbo：录音结束后整段 batch 解码。 + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use parking_lot::Mutex; +use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters}; + +use crate::asr::RawTranscript; + +pub const MODEL_ID: &str = "whisper-large-v3-turbo"; +const QUANTIZED_MODEL_FILE: &str = "ggml-large-v3-turbo-q5_0.bin"; + +/// 模型默认放 OpenLess 的统一模型目录；也允许用环境变量临时指定现有模型文件， +/// 便于把 Input0 已下载的文件迁移过来而不复制 1.5GB 数据。 +pub fn model_path() -> Result { + if let Some(path) = std::env::var_os("OPENLESS_WHISPER_MODEL_PATH") { + return Ok(PathBuf::from(path)); + } + model_path_for_model(MODEL_ID) +} + +pub fn model_ready() -> bool { + model_path().map(|path| path.is_file()).unwrap_or(false) +} + +pub fn model_path_for_model(model_id: &str) -> Result { + let id = crate::asr::local::ModelId::from_str(model_id) + .filter(|id| id.is_whisper()) + .ok_or_else(|| anyhow::anyhow!("未知的本地 Whisper 模型: {model_id}"))?; + let dir = crate::asr::local::models::model_dir(id)?; + let file_name = id + .file_name() + .ok_or_else(|| anyhow::anyhow!("本地 Whisper 模型没有文件名: {model_id}"))?; + let path = dir.join(file_name); + if id == crate::asr::local::ModelId::WhisperLargeV3Turbo && !path.exists() { + let quantized = dir.join(QUANTIZED_MODEL_FILE); + if quantized.exists() { + return Ok(quantized); + } + } + Ok(path) +} + +pub fn model_ready_for_model(model_id: &str) -> bool { + model_path_for_model(model_id) + .map(|path| path.is_file()) + .unwrap_or(false) +} + +pub struct LocalWhisperCache { + inner: Mutex>, +} + +struct CachedEngine { + model_id: String, + engine: Arc, + last_used: Instant, +} + +impl Default for LocalWhisperCache { + fn default() -> Self { + Self::new() + } +} + +impl LocalWhisperCache { + pub fn new() -> Self { + Self { + inner: Mutex::new(None), + } + } + + pub fn get_or_load(&self, model_id: &str, path: &Path) -> Result> { + { + let mut slot = self.inner.lock(); + if let Some(cached) = slot.as_mut() { + if cached.model_id == model_id { + cached.last_used = Instant::now(); + return Ok(Arc::clone(&cached.engine)); + } + slot.take(); + } + } + + let path_str = path + .to_str() + .ok_or_else(|| anyhow::anyhow!("Whisper 模型路径不是有效的 UTF-8"))?; + log::info!("[local-whisper] loading model from {}", path.display()); + let context = + WhisperContext::new_with_params(path_str, WhisperContextParameters::default()) + .map_err(|error| anyhow::anyhow!("加载 Whisper 模型失败: {error}"))?; + let engine = Arc::new(WhisperEngine { + context: Mutex::new(context), + }); + self.inner.lock().replace(CachedEngine { + model_id: model_id.to_string(), + engine: Arc::clone(&engine), + last_used: Instant::now(), + }); + Ok(engine) + } + + pub fn touch(&self) { + if let Some(cached) = self.inner.lock().as_mut() { + cached.last_used = Instant::now(); + } + } + + pub fn release_if_idle(&self, threshold: Duration) -> bool { + let should_release = self + .inner + .lock() + .as_ref() + .map(|cached| cached.last_used.elapsed() >= threshold) + .unwrap_or(false); + if should_release { + self.inner.lock().take(); + return true; + } + false + } + + pub fn release_now(&self) { + self.inner.lock().take(); + } +} + +pub struct WhisperEngine { + context: Mutex, +} + +impl WhisperEngine { + pub(crate) fn transcribe(&self, audio: &[f32], language: &str) -> Result { + let mut context = self.context.lock(); + let mut state = context + .create_state() + .map_err(|error| anyhow::anyhow!("创建 Whisper 状态失败: {error}"))?; + let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 }); + match language { + "auto" | "" => params.set_language(None), + language => params.set_language(Some(language)), + } + if language == "zh" { + params.set_initial_prompt("以下是普通话的句子。"); + } + params.set_translate(false); + params.set_no_timestamps(true); + params.set_print_progress(false); + params.set_print_realtime(false); + params.set_print_special(false); + state + .full(params, audio) + .map_err(|error| anyhow::anyhow!("Whisper batch 解码失败: {error}"))?; + + let count = state + .full_n_segments() + .map_err(|error| anyhow::anyhow!("读取 Whisper 分段数失败: {error}"))?; + let mut text = String::new(); + for index in 0..count { + text.push_str( + &state + .full_get_segment_text(index) + .map_err(|error| anyhow::anyhow!("读取 Whisper 分段失败: {error}"))?, + ); + } + Ok(text.trim().to_string()) + } +} + +pub struct LocalWhisperAsr { + engine: Arc, + language: String, + buffer: Mutex>, +} + +impl LocalWhisperAsr { + pub fn new(engine: Arc, language: String) -> Self { + Self { + engine, + language, + buffer: Mutex::new(Vec::new()), + } + } + + pub fn buffer_duration_ms(&self) -> u64 { + (self.buffer.lock().len() as u64 / 2) * 1000 / 16_000 + } + + pub async fn transcribe(self: Arc) -> Result { + let pcm = std::mem::take(&mut *self.buffer.lock()); + let duration_ms = (pcm.len() as u64 / 2) * 1000 / 16_000; + if pcm.is_empty() { + return Ok(RawTranscript { + text: String::new(), + duration_ms: 0, + }); + } + let audio = pcm_to_f32(&pcm); + let engine = Arc::clone(&self.engine); + let language = self.language.clone(); + let text = + tauri::async_runtime::spawn_blocking(move || engine.transcribe(&audio, &language)) + .await + .context("Whisper batch 解码任务异常")??; + Ok(RawTranscript { text, duration_ms }) + } + + pub fn cancel(&self) { + self.buffer.lock().clear(); + } +} + +impl crate::recorder::AudioConsumer for LocalWhisperAsr { + fn consume_pcm_chunk(&self, pcm: &[u8]) { + self.buffer.lock().extend_from_slice(pcm); + } +} + +fn pcm_to_f32(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(2) + .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]]) as f32 / 32768.0) + .collect() +} + +#[cfg(test)] +mod tests { + use super::pcm_to_f32; + + #[test] + fn converts_recorder_pcm_to_whisper_samples() { + let bytes = [0x00, 0x80, 0x00, 0x40, 0xff, 0x7f]; + let samples = pcm_to_f32(&bytes); + assert_eq!(samples.len(), 3); + assert!((samples[0] + 1.0).abs() < f32::EPSILON); + assert!((samples[1] - 0.5).abs() < 0.0001); + assert!(samples[2] > 0.99); + } +} diff --git a/openless-all/app/src-tauri/src/commands/credentials.rs b/openless-all/app/src-tauri/src/commands/credentials.rs index 1866e611f..b883e3293 100644 --- a/openless-all/app/src-tauri/src/commands/credentials.rs +++ b/openless-all/app/src-tauri/src/commands/credentials.rs @@ -52,18 +52,47 @@ fn volcengine_configured(snap: &CredentialsSnapshot) -> bool { } pub(crate) fn asr_configured_for_provider(provider: &str, snap: &CredentialsSnapshot) -> bool { + if crate::asr::local::is_local_whisper(provider) { + #[cfg(target_os = "macos")] + { + let model_id = crate::persistence::PreferencesStore::new() + .ok() + .map(|store| store.get().local_asr_active_model) + .filter(|id| { + crate::asr::local::ModelId::from_str(id) + .map(|model| model.is_whisper()) + .unwrap_or(false) + }) + .unwrap_or_else(|| crate::asr::local::WHISPER_MODEL_ID.to_string()); + return crate::asr::local::whisper_model_ready_for_model(&model_id); + } + #[cfg(not(target_os = "macos"))] + { + return false; + } + } // 本地 / 无凭据引擎不属于云端分类枚举（ActiveAsrProviderKind），由平台 cfg 门 // 在此单独判定；移动端上这些引擎不可用直接判未配置。 if cfg!(mobile) - && (provider == crate::asr::local::PROVIDER_ID + && (crate::asr::local::is_local_qwen3(provider) + || crate::asr::local::is_local_whisper(provider) || provider == crate::asr::local::sherpa::PROVIDER_ID || provider == crate::asr::local::foundry::PROVIDER_ID || provider == crate::asr::local::APPLE_SPEECH_PROVIDER_ID) { return false; } - if provider == crate::asr::local::PROVIDER_ID - || active_apple_speech_asr_is_supported(provider) + if crate::asr::local::is_local_qwen3(provider) { + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + return crate::asr::local::qwen_backend_for_provider(provider).is_some(); + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + return false; + } + } + if active_apple_speech_asr_is_supported(provider) || active_foundry_asr_is_supported(provider) || active_sherpa_asr_is_supported(provider) { @@ -167,7 +196,7 @@ pub(crate) struct LocalAsrReleasePlan { #[cfg(not(mobile))] pub(crate) fn local_asr_release_plan_for_provider(provider: &str) -> LocalAsrReleasePlan { LocalAsrReleasePlan { - qwen: provider != crate::asr::local::PROVIDER_ID, + qwen: crate::asr::local::qwen_backend_for_provider(provider).is_none(), foundry: provider != FOUNDRY_LOCAL_PROVIDER_ID, sherpa: provider != crate::asr::local::sherpa::PROVIDER_ID, } @@ -263,7 +292,7 @@ pub async fn set_active_asr_provider( _coord: CoordinatorState<'_>, provider: String, ) -> Result<(), String> { - if provider == crate::asr::local::PROVIDER_ID + if crate::asr::local::is_local_qwen3(&provider) || provider == crate::asr::local::sherpa::PROVIDER_ID || provider == crate::asr::local::foundry::PROVIDER_ID || provider == crate::asr::local::APPLE_SPEECH_PROVIDER_ID @@ -284,6 +313,11 @@ pub async fn set_active_asr_provider( sherpa_runtime: State<'_, Arc>, provider: String, ) -> Result<(), String> { + if crate::asr::local::is_local_qwen3(&provider) + && crate::asr::local::qwen_backend_for_provider(&provider).is_none() + { + return Err("所选本地 Qwen3-ASR 后端不支持当前系统".to_string()); + } if provider == FOUNDRY_LOCAL_PROVIDER_ID && !active_foundry_asr_is_supported(&provider) { return Err("Foundry Local Whisper is only available on Windows".to_string()); } @@ -302,7 +336,7 @@ pub async fn set_active_asr_provider( } CredentialsVault::set_active_asr_provider(&provider).map_err(|e| e.to_string())?; let release_plan = local_asr_release_plan_for_provider(&provider); - if provider == crate::asr::local::PROVIDER_ID { + if crate::asr::local::is_local_qwen3(&provider) { // 切到本地 ASR → 后台预加载模型，下次按 hotkey 时不必等数秒。 coord.preload_local_asr_in_background(); } diff --git a/openless-all/app/src-tauri/src/commands/local_asr.rs b/openless-all/app/src-tauri/src/commands/local_asr.rs index 4b01bc4b8..466ed21dc 100644 --- a/openless-all/app/src-tauri/src/commands/local_asr.rs +++ b/openless-all/app/src-tauri/src/commands/local_asr.rs @@ -13,7 +13,7 @@ pub struct LocalAsrSettings { pub mirror: String, pub models_base_dir: Option, pub models_root_dir: String, - /// macOS 才编入引擎；Windows 端 UI 需要据此把"开始下载"按钮灰掉。 + /// macOS/Linux 编入本地 Qwen3-ASR C 引擎；MLX 仅在 macOS 可用。 pub engine_available: bool, } @@ -30,7 +30,7 @@ pub fn local_asr_get_settings(coord: CoordinatorState<'_>) -> LocalAsrSettings { mirror: prefs.local_asr_mirror, models_base_dir, models_root_dir, - engine_available: cfg!(target_os = "macos"), + engine_available: cfg!(any(target_os = "macos", target_os = "linux")), } } @@ -239,7 +239,7 @@ pub fn local_asr_delete_model(coord: CoordinatorState<'_>, model_id: String) -> let id = ModelId::from_str(&model_id).ok_or_else(|| format!("unknown model id: {model_id}"))?; // 如果内存里加载的就是要删的这个模型，先释放：否则 mmap 残留指向已 unlink 的文件， // 且 RAM 直到下次切模型 / 用户手动按"释放"才回收。 - if coord.local_asr_loaded_model().as_deref() == Some(id.as_str()) { + if id.is_whisper() || coord.local_asr_loaded_model().as_deref() == Some(id.as_str()) { coord.release_local_asr_engine(); } crate::asr::local::models::delete_model(id).map_err(|e| e.to_string()) @@ -272,10 +272,13 @@ pub fn local_asr_reveal_models_root(coord: CoordinatorState<'_>) -> Result<(), S #[tauri::command] pub async fn local_asr_test_model( + coord: CoordinatorState<'_>, model_id: String, ) -> Result { let id = ModelId::from_str(&model_id).ok_or_else(|| format!("unknown model id: {model_id}"))?; - crate::asr::local::test_run::run_test(id) + let backend = + crate::asr::local::qwen_backend_for_provider(&coord.prefs().get().active_asr_provider); + crate::asr::local::test_run::run_test(id, backend) .await .map_err(|e| format!("{e:#}")) } diff --git a/openless-all/app/src-tauri/src/commands/mod.rs b/openless-all/app/src-tauri/src/commands/mod.rs index 85dc6efde..ada6b9c22 100644 --- a/openless-all/app/src-tauri/src/commands/mod.rs +++ b/openless-all/app/src-tauri/src/commands/mod.rs @@ -347,10 +347,27 @@ mod tests { &whisper_keyless_ready )); - assert!(asr_configured_for_provider( + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + assert!(asr_configured_for_provider( + crate::asr::local::PROVIDER_ID, + &snapshot() + )); + assert!(asr_configured_for_provider( + crate::asr::local::LOCAL_QWEN3_C_PROVIDER_ID, + &snapshot() + )); + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + assert!(!asr_configured_for_provider( crate::asr::local::PROVIDER_ID, &snapshot() )); + #[cfg(target_os = "macos")] + assert!(asr_configured_for_provider( + crate::asr::local::LOCAL_QWEN3_MLX_PROVIDER_ID, + &snapshot() + )); #[cfg(target_os = "windows")] assert!(asr_configured_for_provider( crate::asr::local::foundry::PROVIDER_ID, @@ -396,6 +413,14 @@ mod tests { assert!(active_asr_is_keyless_for_validation( crate::asr::local::PROVIDER_ID )); + #[cfg(any(target_os = "macos", target_os = "linux"))] + assert!(active_asr_is_keyless_for_validation( + crate::asr::local::LOCAL_QWEN3_C_PROVIDER_ID + )); + #[cfg(target_os = "macos")] + assert!(active_asr_is_keyless_for_validation( + crate::asr::local::LOCAL_QWEN3_MLX_PROVIDER_ID + )); #[cfg(target_os = "windows")] assert!(active_asr_is_keyless_for_validation( crate::asr::local::foundry::PROVIDER_ID @@ -423,6 +448,13 @@ mod tests { assert!(qwen.foundry); assert!(qwen.sherpa); + let qwen_c = + local_asr_release_plan_for_provider(crate::asr::local::LOCAL_QWEN3_C_PROVIDER_ID); + #[cfg(any(target_os = "macos", target_os = "linux"))] + assert!(!qwen_c.qwen); + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + assert!(qwen_c.qwen); + let foundry = local_asr_release_plan_for_provider(crate::asr::local::foundry::PROVIDER_ID); assert!(foundry.qwen); assert!(!foundry.foundry); diff --git a/openless-all/app/src-tauri/src/commands/providers.rs b/openless-all/app/src-tauri/src/commands/providers.rs index a772e4fe7..ad1987b9b 100644 --- a/openless-all/app/src-tauri/src/commands/providers.rs +++ b/openless-all/app/src-tauri/src/commands/providers.rs @@ -380,6 +380,31 @@ async fn validate_omni_provider() -> Result<(), String> { async fn validate_asr_provider(scope: &ProviderScope) -> Result<(), String> { let active_asr = scope.provider_type(); + if crate::asr::local::is_local_whisper(&active_asr) { + #[cfg(not(target_os = "macos"))] + { + return Err("本地 Whisper 当前仅支持 macOS".to_string()); + } + #[cfg(target_os = "macos")] + { + let model_id = crate::persistence::PreferencesStore::new() + .ok() + .map(|store| store.get().local_asr_active_model) + .filter(|id| { + crate::asr::local::ModelId::from_str(id) + .map(|model| model.is_whisper()) + .unwrap_or(false) + }) + .unwrap_or_else(|| crate::asr::local::WHISPER_MODEL_ID.to_string()); + let path = crate::asr::local::whisper_model_path_for_model(&model_id) + .map_err(|e| e.to_string())?; + return if path.is_file() { + Ok(()) + } else { + Err(format!("本地 Whisper 模型不存在: {}", path.display())) + }; + } + } if active_asr_is_keyless_for_validation(&active_asr) { return Ok(()); } @@ -794,7 +819,8 @@ pub(crate) fn active_asr_is_keyless_for_validation(provider: &str) -> bool { if cfg!(mobile) { return false; } - provider == crate::asr::local::PROVIDER_ID + crate::asr::local::qwen_backend_for_provider(provider).is_some() + || crate::asr::local::is_local_whisper(provider) || active_apple_speech_asr_is_supported(provider) || active_foundry_asr_is_supported(provider) || active_sherpa_asr_is_supported(provider) diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index a52c51ea1..9693fed04 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -358,9 +358,12 @@ enum ActiveAsr { /// Windows sherpa-onnx 本地 ASR（offline batch + 实验 online streaming）。 #[cfg(target_os = "windows")] SherpaOnnxLocal(Arc), - /// 本地 Qwen3-ASR；只在 macOS + 模型已下载时可达。 - #[cfg(target_os = "macos")] + /// 本地 Qwen3-ASR；macOS 可选 MLX/C，Linux 使用 C。 + #[cfg(any(target_os = "macos", target_os = "linux"))] Local(Arc), + /// 本地 Whisper Large-v3 Turbo；只在 macOS + 模型已迁移时可达。 + #[cfg(target_os = "macos")] + LocalWhisper(Arc), /// Apple Speech（SFSpeechRecognizer）系统本地 ASR；只在 macOS 可达。 #[cfg(target_os = "macos")] AppleSpeech(Arc), @@ -374,6 +377,8 @@ fn asr_transcribe_uses_global_timeout(asr: &ActiveAsr) -> bool { // COORDINATOR_GLOBAL_TIMEOUT；各 provider 自己里面控制細粒度超时。 #[cfg(target_os = "windows")] ActiveAsr::SherpaOnnxLocal(_) => false, + #[cfg(target_os = "macos")] + ActiveAsr::LocalWhisper(_) => false, _ => true, } } @@ -724,9 +729,11 @@ struct Inner { /// `multimodal_pipeline_enabled && pipeline_mode == multimodal` 时使用， /// 与 asr 槽互斥——同一会话二者有且仅有一个。 omni_pcm: Mutex>>>, - /// 本地 Qwen3-ASR 引擎缓存。跨会话复用，避免每次重加载 1.2GB+ 模型。 + /// 本地 Qwen3-ASR MLX 引擎缓存。跨会话复用，避免每次重加载 1.2GB+ 模型。 /// 释放时机由 prefs.local_asr_keep_loaded_secs 决定。 local_asr_cache: Arc, + #[cfg(target_os = "macos")] + local_whisper_cache: Arc, #[cfg(target_os = "windows")] foundry_local_runtime: Arc, /// Windows sherpa-onnx 本地 ASR runtime。与 Foundry 同处一个 @@ -1068,6 +1075,8 @@ impl Coordinator { qa_recorder: Mutex::new(None), qa_stream_cancelled: Arc::new(AtomicBool::new(false)), local_asr_cache: Arc::new(crate::asr::local::LocalAsrCache::new()), + #[cfg(target_os = "macos")] + local_whisper_cache: Arc::new(crate::asr::local::LocalWhisperCache::new()), shutdown: AtomicBool::new(false), #[cfg(not(mobile))] remote_audio_sink: Mutex::new(None), @@ -1193,6 +1202,8 @@ impl Coordinator { qa_recorder: Mutex::new(None), qa_stream_cancelled: Arc::new(AtomicBool::new(false)), local_asr_cache: Arc::new(crate::asr::local::LocalAsrCache::new()), + #[cfg(target_os = "macos")] + local_whisper_cache: Arc::new(crate::asr::local::LocalWhisperCache::new()), foundry_local_runtime, sherpa_onnx_runtime, shutdown: AtomicBool::new(false), @@ -1215,11 +1226,11 @@ impl Coordinator { } } - /// 后台预加载本地 ASR 引擎；当用户在 UI 切到 local-qwen3 provider 时调一次。 + /// 后台预加载当前本地 Qwen3-ASR 后端；当用户在 UI 切到对应 provider 时调一次。 /// 加载是阻塞且数秒，所以放 spawn_blocking 里，不影响 UI 响应。 - /// 模型未下载或不在 macOS 上时静默跳过。 + /// 模型未下载或当前平台不支持该后端时静默跳过。 pub fn preload_local_asr_in_background(self: &Arc) { - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] { let inner = Arc::clone(&self.inner); tauri::async_runtime::spawn(async move { @@ -1229,6 +1240,16 @@ impl Coordinator { Some(m) => m, None => return, }; + if !model_id.is_qwen() + || !crate::asr::local::is_local_qwen3(&prefs.active_asr_provider) + { + return; + } + let Some(backend) = + crate::asr::local::qwen_backend_for_provider(&prefs.active_asr_provider) + else { + return; + }; if !crate::asr::local::models::is_downloaded(model_id) { log::info!( "[coord] local ASR preload skipped: model {} not downloaded", @@ -1243,7 +1264,7 @@ impl Coordinator { let cache = Arc::clone(&inner.local_asr_cache); let mid = model_id.as_str().to_string(); let _ = tauri::async_runtime::spawn_blocking(move || { - if let Err(e) = cache.get_or_load(&mid, &dir) { + if let Err(e) = cache.get_or_load(backend, &mid, &dir) { log::warn!("[coord] local ASR preload failed: {e:#}"); } }) @@ -1261,6 +1282,8 @@ impl Coordinator { /// 释放当前缓存的本地 ASR 引擎（用户主动点 / 或 删除模型时调）。 pub fn release_local_asr_engine(&self) { self.inner.local_asr_cache.release_now(); + #[cfg(target_os = "macos")] + self.inner.local_whisper_cache.release_now(); emit_local_asr_engine_status(&self.inner); } @@ -2518,7 +2541,7 @@ impl Coordinator { .await .map_err(|e| e.to_string())? } - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] ActiveAsr::Local(local) => { let dur = local_qwen_transcribe_timeout((local.buffer_duration_ms() as f64) / 1000.0); @@ -2531,6 +2554,18 @@ impl Coordinator { out } #[cfg(target_os = "macos")] + ActiveAsr::LocalWhisper(local) => { + let dur = + local_whisper_transcribe_timeout((local.buffer_duration_ms() as f64) / 1000.0); + inner.local_whisper_cache.touch(); + let out = tokio::time::timeout(dur, local.transcribe()) + .await + .map_err(|_| "重新转录超时".to_string())? + .map_err(|e| e.to_string())?; + schedule_local_whisper_release(inner); + out + } + #[cfg(target_os = "macos")] ActiveAsr::AppleSpeech(local) => tokio::time::timeout(timeout, local.transcribe()) .await .map_err(|_| "重新转录超时".to_string())? @@ -5306,6 +5341,13 @@ fn local_qwen_transcribe_timeout(audio_secs: f64) -> std::time::Duration { std::time::Duration::from_secs(secs) } +fn local_whisper_transcribe_timeout(audio_secs: f64) -> std::time::Duration { + let secs = ((audio_secs * 0.5).ceil() as u64) + .saturating_add(10) + .max(15); + std::time::Duration::from_secs(secs) +} + /// Whisper / OpenRouter 云端 batch ASR 的动态转写超时。OpenRouter 按 30s /// 分片，每片是一次 HTTP round-trip；网络抖动、排队、base64 body 都会 /// 拉长耗时。公式 max(30, ceil(audio_s × 0.5) + 20)：30s 是全局兜底； diff --git a/openless-all/app/src-tauri/src/coordinator/asr_wiring.rs b/openless-all/app/src-tauri/src/coordinator/asr_wiring.rs index 5af4d0e62..5082fc7f9 100644 --- a/openless-all/app/src-tauri/src/coordinator/asr_wiring.rs +++ b/openless-all/app/src-tauri/src/coordinator/asr_wiring.rs @@ -67,15 +67,31 @@ pub(super) fn ensure_microphone_permission(_inner: &Arc) -> Result<(), St pub(super) fn ensure_asr_credentials() -> Result<(), String> { let active_asr = CredentialsVault::get_active_asr(); - // 本地 Qwen3-ASR 没有"凭据"概念，但需要：(a) macOS 平台 (b) 模型已下载。 + // 本地 Qwen3-ASR 没有"凭据"概念，但需要：(a) 当前渠道的后端可用 (b) 模型已下载。 if crate::asr::local::is_local_qwen3(&active_asr) { + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + if crate::asr::local::qwen_backend_for_provider(&active_asr).is_none() { + return Err(format!("本地 Qwen3-ASR 渠道 {active_asr} 不支持当前系统")); + } + return ensure_local_qwen3_model_ready(); + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + return Err( + "本地 Qwen3-ASR C 后端目前支持 macOS/Linux；MLX 后端仅支持 macOS".to_string(), + ); + } + } + + if crate::asr::local::is_local_whisper(&active_asr) { #[cfg(not(target_os = "macos"))] { - return Err("本地 ASR 当前仅支持 macOS（Windows 见 issue #256）".to_string()); + return Err("本地 Whisper 当前仅支持 macOS".to_string()); } #[cfg(target_os = "macos")] { - return ensure_local_qwen3_model_ready(); + return ensure_local_whisper_model_ready(); } } @@ -185,7 +201,7 @@ pub(super) fn require_openai_compatible_fields(endpoint: &str, model: &str) -> R #[cfg(test)] pub(super) fn is_keyless_local_asr_provider(id: &str) -> bool { if crate::asr::local::is_local_qwen3(id) { - return true; + return crate::asr::local::qwen_backend_for_provider(id).is_some(); } #[cfg(target_os = "macos")] if crate::asr::local::is_apple_speech(id) { @@ -203,7 +219,7 @@ pub(super) fn is_keyless_local_asr_provider(id: &str) -> bool { } } -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] pub(super) fn ensure_local_qwen3_model_ready() -> Result<(), String> { let prefs = || -> Result { // 这里没法拿到 inner，直接读 preferences.json 即可（Coordinator 写盘后总是同步的）。 @@ -213,6 +229,12 @@ pub(super) fn ensure_local_qwen3_model_ready() -> Result<(), String> { }()?; let model_id = crate::asr::local::ModelId::from_str(&prefs.local_asr_active_model) .ok_or_else(|| format!("未知的本地模型 id: {}", prefs.local_asr_active_model))?; + if !model_id.is_qwen() { + return Err(format!( + "当前模型 {} 不属于本地 Qwen3-ASR", + model_id.as_str() + )); + } if !crate::asr::local::models::is_downloaded(model_id) { return Err(format!( "本地模型 {} 未下载完整，请到 设置 → 模型设置 中下载", @@ -222,6 +244,29 @@ pub(super) fn ensure_local_qwen3_model_ready() -> Result<(), String> { Ok(()) } +#[cfg(target_os = "macos")] +pub(super) fn ensure_local_whisper_model_ready() -> Result<(), String> { + let model_id = crate::persistence::PreferencesStore::new() + .map(|store| store.get().local_asr_active_model) + .ok() + .filter(|id| { + crate::asr::local::ModelId::from_str(id) + .map(|model| model.is_whisper()) + .unwrap_or(false) + }) + .unwrap_or_else(|| crate::asr::local::WHISPER_MODEL_ID.to_string()); + if crate::asr::local::whisper_model_ready_for_model(&model_id) { + return Ok(()); + } + let path = + crate::asr::local::whisper_model_path_for_model(&model_id).map_err(|e| e.to_string())?; + Err(format!( + "本地 Whisper 模型 {} 不存在，请到 设置 → 本地模型 下载，或将模型文件放到 {}", + model_id, + path.display() + )) +} + /// 引擎加载/释放/keepLoadedSecs 变化时主动推给前端，前端 listen /// `local-asr:engine-changed` 即可零轮询同步 UI（issue #470 / #6）。 /// 只反映 Qwen3 这一路（loaded_model_id / prefs），不碰 Foundry / Sherpa。 @@ -266,6 +311,21 @@ pub(super) fn schedule_local_asr_release(inner: &Arc) { }); } +#[cfg(target_os = "macos")] +pub(super) fn schedule_local_whisper_release(inner: &Arc) { + let keep_secs = inner.prefs.get().local_asr_keep_loaded_secs; + let cache = Arc::clone(&inner.local_whisper_cache); + if keep_secs == 0 { + cache.release_now(); + return; + } + let threshold = std::time::Duration::from_secs(keep_secs as u64); + tauri::async_runtime::spawn(async move { + tokio::time::sleep(threshold).await; + cache.release_if_idle(threshold); + }); +} + #[cfg(target_os = "windows")] pub(super) fn foundry_local_asr_release_keep_secs(inner: &Arc) -> u32 { inner.prefs.get().foundry_local_asr_keep_loaded_secs @@ -279,7 +339,10 @@ pub(super) enum AsrReleaseSession { } #[cfg(target_os = "windows")] -pub(super) fn asr_release_session_is_current(inner: &Arc, session: AsrReleaseSession) -> bool { +pub(super) fn asr_release_session_is_current( + inner: &Arc, + session: AsrReleaseSession, +) -> bool { match session { AsrReleaseSession::Dictation(session_id) => inner.state.lock().session_id == session_id, AsrReleaseSession::Qa(session_id) => inner.qa_state.lock().session_id == session_id, @@ -329,37 +392,66 @@ pub(super) fn schedule_sherpa_onnx_release(inner: &Arc, session: AsrRelea }); } -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] /// 返回 (provider, 实际加载的模型 id)。模型 id 是 ModelId 校验归一后的值，调用方 /// 直接用它做历史归因（构建时快照，PR #826 review）。 pub(super) async fn build_local_qwen3( inner: &Arc, + provider_id: &str, ) -> anyhow::Result<(Arc, String)> { + let backend = crate::asr::local::qwen_backend_for_provider(provider_id) + .ok_or_else(|| anyhow::anyhow!("本地 Qwen3-ASR 渠道 {provider_id} 不支持当前系统"))?; let prefs = inner.prefs.get(); let model_id = crate::asr::local::ModelId::from_str(&prefs.local_asr_active_model) + .filter(|id| id.is_qwen()) .ok_or_else(|| anyhow::anyhow!("未知本地模型 id: {}", prefs.local_asr_active_model))?; let dir = crate::asr::local::models::model_dir(model_id)?; - let app = inner - .app - .lock() - .clone() - .ok_or_else(|| anyhow::anyhow!("AppHandle 未绑定"))?; // 走缓存：如果已有同 id 的引擎在内存里就直接复用，避免每次会话都重加载 // 1.2GB+ 模型。第一次加载阻塞数秒，spawn_blocking 不卡 tokio runtime。 let cache = Arc::clone(&inner.local_asr_cache); let mid = model_id.as_str().to_string(); - let engine = tauri::async_runtime::spawn_blocking(move || cache.get_or_load(&mid, &dir)) - .await - .map_err(|e| anyhow::anyhow!("spawn_blocking join failed: {e:#}"))??; + let engine = + tauri::async_runtime::spawn_blocking(move || cache.get_or_load(backend, &mid, &dir)) + .await + .map_err(|e| anyhow::anyhow!("spawn_blocking join failed: {e:#}"))??; // 加载完成（含缓存命中刷新 last_used）后推一次状态，前端零轮询更新「已加载」。 emit_local_asr_engine_status(inner); let model_label = model_id.as_str().to_string(); Ok(( - Arc::new(crate::asr::local::LocalQwenAsr::new(app, engine)), + Arc::new(crate::asr::local::LocalQwenAsr::new(engine)), model_label, )) } +#[cfg(target_os = "macos")] +pub(super) async fn build_local_whisper( + inner: &Arc, +) -> anyhow::Result<(Arc, String)> { + let model_id = crate::asr::local::ModelId::from_str(&inner.prefs.get().local_asr_active_model) + .filter(|id| id.is_whisper()) + .map(|id| id.as_str().to_string()) + .unwrap_or_else(|| crate::asr::local::WHISPER_MODEL_ID.to_string()); + let path = crate::asr::local::whisper_model_path_for_model(&model_id)?; + let cache = Arc::clone(&inner.local_whisper_cache); + let cache_model_id = model_id.clone(); + let engine = + tauri::async_runtime::spawn_blocking(move || cache.get_or_load(&cache_model_id, &path)) + .await + .map_err(|e| anyhow::anyhow!("spawn_blocking join failed: {e:#}"))??; + let language = inner + .prefs + .get() + .working_languages + .first() + .and_then(|name| crate::asr::local::native_name_to_apple_locale(name)) + .map(|locale| locale.split('-').next().unwrap_or("auto").to_string()) + .unwrap_or_else(|| "auto".to_string()); + Ok(( + Arc::new(crate::asr::local::LocalWhisperAsr::new(engine, language)), + model_id, + )) +} + #[cfg(target_os = "macos")] pub(super) fn build_apple_speech( prefs: &crate::types::UserPreferences, @@ -712,13 +804,24 @@ pub(super) async fn build_qa_asr_start( } #[cfg(target_os = "macos")] + if crate::asr::local::is_local_whisper(active_asr) { + let (local, model) = build_local_whisper(inner) + .await + .map_err(|e| format!("local Whisper init failed: {e}"))?; + let active = ActiveAsr::LocalWhisper(Arc::clone(&local)); + let consumer: Arc = local; + let label = AsrCallLabel::new(crate::asr::local::LOCAL_WHISPER_PROVIDER_ID, Some(model)); + return Ok((QaAsrStart::Ready { active, consumer }, label)); + } + + #[cfg(any(target_os = "macos", target_os = "linux"))] if crate::asr::local::is_local_qwen3(active_asr) { - let (local, model) = build_local_qwen3(inner) + let (local, model) = build_local_qwen3(inner, active_asr) .await .map_err(|e| format!("local ASR init failed: {e}"))?; let active = ActiveAsr::Local(Arc::clone(&local)); let consumer: Arc = local; - let label = AsrCallLabel::new(crate::asr::local::PROVIDER_ID, Some(model)); + let label = AsrCallLabel::new(active_asr, Some(model)); return Ok((QaAsrStart::Ready { active, consumer }, label)); } diff --git a/openless-all/app/src-tauri/src/coordinator/dictation.rs b/openless-all/app/src-tauri/src/coordinator/dictation.rs index 45ab08417..d428823c3 100644 --- a/openless-all/app/src-tauri/src/coordinator/dictation.rs +++ b/openless-all/app/src-tauri/src/coordinator/dictation.rs @@ -32,22 +32,30 @@ pub(super) const COMBO_ARBITRATION_GRACE: std::time::Duration = std::time::Duration::from_millis(150); const STREAMING_INSERT_FLUSH_INTERVAL: std::time::Duration = std::time::Duration::from_millis(12); -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum MacosKeylessDictationProvider { +enum DesktopKeylessDictationProvider { LocalQwen3, + #[cfg(target_os = "macos")] + LocalWhisper, + #[cfg(target_os = "macos")] AppleSpeech, } -#[cfg(target_os = "macos")] -fn macos_keyless_dictation_provider(active_asr: &str) -> Option { - if crate::asr::local::is_local_qwen3(active_asr) { - Some(MacosKeylessDictationProvider::LocalQwen3) - } else if crate::asr::local::is_apple_speech(active_asr) { - Some(MacosKeylessDictationProvider::AppleSpeech) - } else { - None +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn desktop_keyless_dictation_provider(active_asr: &str) -> Option { + if crate::asr::local::qwen_backend_for_provider(active_asr).is_some() { + return Some(DesktopKeylessDictationProvider::LocalQwen3); + } + #[cfg(target_os = "macos")] + if crate::asr::local::is_local_whisper(active_asr) { + return Some(DesktopKeylessDictationProvider::LocalWhisper); + } + #[cfg(target_os = "macos")] + if crate::asr::local::is_apple_speech(active_asr) { + return Some(DesktopKeylessDictationProvider::AppleSpeech); } + None } /// Less Computer 浮窗的 Tauri 事件名（前端 LessComputerPanel 订阅）。 @@ -2045,11 +2053,11 @@ pub(super) async fn begin_session_as(inner: &Arc, voice_agent: bool) -> R return Ok(()); } - #[cfg(target_os = "macos")] - if let Some(provider) = macos_keyless_dictation_provider(&active_asr) { + #[cfg(any(target_os = "macos", target_os = "linux"))] + if let Some(provider) = desktop_keyless_dictation_provider(&active_asr) { match provider { - MacosKeylessDictationProvider::LocalQwen3 => { - let (local, local_model) = match build_local_qwen3(inner).await { + DesktopKeylessDictationProvider::LocalQwen3 => { + let (local, local_model) = match build_local_qwen3(inner, &active_asr).await { Ok(l) => l, Err(e) => { log::error!("[coord] 本地 Qwen3-ASR 初始化失败: {e:#}"); @@ -2071,7 +2079,7 @@ pub(super) async fn begin_session_as(inner: &Arc, voice_agent: bool) -> R inner, current_session_id, ActiveAsr::Local(Arc::clone(&local)), - AsrCallLabel::new(crate::asr::local::PROVIDER_ID, Some(local_model)), + AsrCallLabel::new(active_asr.clone(), Some(local_model)), ); let consumer: Arc = local; start_recorder_and_enter_listening( @@ -2082,7 +2090,8 @@ pub(super) async fn begin_session_as(inner: &Arc, voice_agent: bool) -> R ) .await?; } - MacosKeylessDictationProvider::AppleSpeech => { + #[cfg(target_os = "macos")] + DesktopKeylessDictationProvider::AppleSpeech => { let local = build_apple_speech(&inner.prefs.get()); store_asr_for_session( inner, @@ -2100,6 +2109,41 @@ pub(super) async fn begin_session_as(inner: &Arc, voice_agent: bool) -> R ) .await?; } + #[cfg(target_os = "macos")] + DesktopKeylessDictationProvider::LocalWhisper => { + let (local, model) = match build_local_whisper(inner).await { + Ok(value) => value, + Err(error) => { + log::error!("[coord] 本地 Whisper 初始化失败: {error:#}"); + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + 0, + Some(format!("本地模型初始化失败: {error}")), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + inner.state.lock().phase = SessionPhase::Idle; + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + return Err(format!("local Whisper init failed: {error}")); + } + }; + store_asr_for_session( + inner, + current_session_id, + ActiveAsr::LocalWhisper(Arc::clone(&local)), + AsrCallLabel::new(crate::asr::local::LOCAL_WHISPER_PROVIDER_ID, Some(model)), + ); + let consumer: Arc = local; + start_recorder_and_enter_listening( + inner, + current_session_id, + &active_asr, + consumer, + ) + .await?; + } } return Ok(()); } @@ -3157,11 +3201,16 @@ pub(super) fn schedule_cancelled_asr_release( ActiveAsr::SherpaOnnxLocal(_) => { schedule_sherpa_onnx_release(inner, AsrReleaseSession::Dictation(session_id)); } - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] ActiveAsr::Local(_) => { inner.local_asr_cache.touch(); schedule_local_asr_release(inner); } + #[cfg(target_os = "macos")] + ActiveAsr::LocalWhisper(_) => { + inner.local_whisper_cache.touch(); + schedule_local_whisper_release(inner); + } _ => {} } } @@ -3643,7 +3692,7 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { } } } - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] ActiveAsr::Local(local) => { debug_assert!(uses_global_timeout); // 缓存命中时 transcribe 不含 load 时间；冷启动 load 已在 build_local_qwen3 @@ -3718,6 +3767,31 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { } } } + #[cfg(target_os = "macos")] + ActiveAsr::LocalWhisper(local) => { + debug_assert!(!uses_global_timeout); + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + let timeout_duration = local_whisper_transcribe_timeout(audio_secs); + log::info!( + "[coord] local Whisper transcribe: audio={:.2}s timeout={}s", + audio_secs, + timeout_duration.as_secs() + ); + let result = tokio::time::timeout(timeout_duration, local.transcribe()).await; + inner.local_whisper_cache.touch(); + schedule_local_whisper_release(inner); + match result { + Ok(Ok(raw)) => Ok(raw), + Ok(Err(error)) => Err(TranscribeFail::new( + format!("本地识别失败: {error}"), + error.to_string(), + )), + Err(_) => Err(TranscribeFail::new( + "识别超时".to_string(), + "local whisper timeout".to_string(), + )), + } + } }; transcribe_outcome }; @@ -4813,8 +4887,8 @@ mod tests { append_cursor_context_to_multimodal_prompt, pcm_duration_ms, pcm_from_wav_bytes, should_arm_edit_watch, should_read_cursor_context, streaming_insert_eligible, }; - #[cfg(target_os = "macos")] - use super::{macos_keyless_dictation_provider, MacosKeylessDictationProvider}; + #[cfg(any(target_os = "macos", target_os = "linux"))] + use super::{desktop_keyless_dictation_provider, DesktopKeylessDictationProvider}; use crate::types::{ ChineseScriptPreference, CorrectionRule, DictationSession, InsertStatus, PolishMode, }; @@ -5134,16 +5208,24 @@ mod tests { #[cfg(target_os = "macos")] #[test] - fn macos_keyless_dictation_provider_routes_apple_speech_locally() { + fn desktop_keyless_dictation_provider_routes_apple_speech_locally() { + assert_eq!( + desktop_keyless_dictation_provider(crate::asr::local::APPLE_SPEECH_PROVIDER_ID), + Some(DesktopKeylessDictationProvider::AppleSpeech) + ); + assert_eq!( + desktop_keyless_dictation_provider(crate::asr::local::PROVIDER_ID), + Some(DesktopKeylessDictationProvider::LocalQwen3) + ); assert_eq!( - macos_keyless_dictation_provider(crate::asr::local::APPLE_SPEECH_PROVIDER_ID), - Some(MacosKeylessDictationProvider::AppleSpeech) + desktop_keyless_dictation_provider(crate::asr::local::LOCAL_QWEN3_MLX_PROVIDER_ID), + Some(DesktopKeylessDictationProvider::LocalQwen3) ); assert_eq!( - macos_keyless_dictation_provider(crate::asr::local::PROVIDER_ID), - Some(MacosKeylessDictationProvider::LocalQwen3) + desktop_keyless_dictation_provider(crate::asr::local::LOCAL_QWEN3_C_PROVIDER_ID), + Some(DesktopKeylessDictationProvider::LocalQwen3) ); - assert_eq!(macos_keyless_dictation_provider("volcengine"), None); + assert_eq!(desktop_keyless_dictation_provider("volcengine"), None); } #[test] diff --git a/openless-all/app/src-tauri/src/coordinator/qa_session.rs b/openless-all/app/src-tauri/src/coordinator/qa_session.rs index 37c17f330..963feba27 100644 --- a/openless-all/app/src-tauri/src/coordinator/qa_session.rs +++ b/openless-all/app/src-tauri/src/coordinator/qa_session.rs @@ -590,7 +590,7 @@ pub(super) async fn transcribe_overlay_dictation_asr( } } } - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] ActiveAsr::Local(local) => { debug_assert!(uses_global_timeout); let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; @@ -605,6 +605,21 @@ pub(super) async fn transcribe_overlay_dictation_asr( } } #[cfg(target_os = "macos")] + ActiveAsr::LocalWhisper(local) => { + debug_assert!(!uses_global_timeout); + let timeout_duration = local_whisper_transcribe_timeout( + (local.buffer_duration_ms() as f64) / 1000.0, + ); + let result = tokio::time::timeout(timeout_duration, local.transcribe()).await; + _inner.local_whisper_cache.touch(); + schedule_local_whisper_release(_inner); + match result { + Ok(Ok(raw)) => Ok(raw), + Ok(Err(error)) => Err(error.to_string()), + Err(_) => Err("local whisper transcribe timeout".to_string()), + } + } + #[cfg(target_os = "macos")] ActiveAsr::AppleSpeech(local) => { debug_assert!(uses_global_timeout); match tokio::time::timeout( @@ -1397,7 +1412,7 @@ pub(super) async fn end_qa_session(inner: &Arc) -> Result<(), String> { } } } - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] ActiveAsr::Local(local) => { debug_assert!(uses_global_timeout); let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; @@ -1432,6 +1447,32 @@ pub(super) async fn end_qa_session(inner: &Arc) -> Result<(), String> { } } #[cfg(target_os = "macos")] + ActiveAsr::LocalWhisper(local) => { + debug_assert!(!uses_global_timeout); + let timeout_duration = local_whisper_transcribe_timeout( + (local.buffer_duration_ms() as f64) / 1000.0, + ); + let result = tokio::time::timeout(timeout_duration, local.transcribe()).await; + inner.local_whisper_cache.touch(); + schedule_local_whisper_release(inner); + match result { + Ok(Ok(raw)) => raw, + Ok(Err(error)) => { + log::error!("[coord] QA local Whisper transcribe failed: {error:#}"); + finish_qa_with_error_if_current( + inner, + session_id, + format!("本地识别失败: {error}"), + ); + return Err(error.to_string()); + } + Err(_) => { + finish_qa_with_error_if_current(inner, session_id, "本地识别超时".to_string()); + return Err("local whisper transcribe timeout".to_string()); + } + } + } + #[cfg(target_os = "macos")] ActiveAsr::AppleSpeech(local) => { debug_assert!(uses_global_timeout); let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); diff --git a/openless-all/app/src-tauri/src/coordinator/resources.rs b/openless-all/app/src-tauri/src/coordinator/resources.rs index 55c742c61..4b5352196 100644 --- a/openless-all/app/src-tauri/src/coordinator/resources.rs +++ b/openless-all/app/src-tauri/src/coordinator/resources.rs @@ -164,9 +164,11 @@ pub(super) fn cancel_active_asr(asr: ActiveAsr) { ActiveAsr::FoundryLocalWhisper(local) => local.cancel(), #[cfg(target_os = "windows")] ActiveAsr::SherpaOnnxLocal(local) => local.cancel(), - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] ActiveAsr::Local(local) => local.cancel(), #[cfg(target_os = "macos")] + ActiveAsr::LocalWhisper(local) => local.cancel(), + #[cfg(target_os = "macos")] ActiveAsr::AppleSpeech(local) => local.cancel(), } } diff --git a/openless-all/app/src-tauri/vendor/qwen3-asr-rs b/openless-all/app/src-tauri/vendor/qwen3-asr-rs new file mode 160000 index 000000000..88f7f4ae2 --- /dev/null +++ b/openless-all/app/src-tauri/vendor/qwen3-asr-rs @@ -0,0 +1 @@ +Subproject commit 88f7f4ae26a50f01cf40d9a30819cd006d3f1d8e diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index da32ac693..2167b70ae 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -968,7 +968,10 @@ export const en: typeof zhCN = { asrElevenLabs: 'ElevenLabs Scribe', asrSherpaOnnxLocal: 'Local sherpa-onnx (experimental)', asrFoundryLocalWhisper: 'Local Whisper (Foundry Local)', + asrLocalWhisper: 'Local Whisper (batch)', asrLocalQwen3: 'Local Qwen3-ASR', + asrLocalQwen3Mlx: 'Local Qwen3-ASR (MLX / Metal)', + asrLocalQwen3C: 'Local Qwen3-ASR (C / CPU)', asrAppleSpeech: 'Apple Speech (macOS)', omniOpenai: 'OpenAI (audio-capable)', omniGemini: 'Google Gemini', @@ -1599,6 +1602,12 @@ export const en: typeof zhCN = { sizeLoading: 'Fetching size…', sizeUnknown: 'Size unknown', performanceWarning: 'Local ASR is best for offline or privacy-sensitive use. First use requires model download.', + metalToolchainTitle: 'macOS MLX first-build requirement', + metalToolchainDesc: 'If the development build reports a missing MetalToolchain, install the Xcode component first.', + metalToolchainStep: 'Run this command in Terminal, then restart OpenLess:', + metalToolchainCopy: 'Copy command', + metalToolchainCopied: 'Copied', + metalToolchainVerify: 'Verify it with xcrun --find metal.', test: 'Load & Test', testRunning: 'Testing…', testHeading: 'Built-in audio test', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index ea61f4f0c..26f999a73 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -970,7 +970,10 @@ export const ja: typeof zhCN = { asrElevenLabs: 'ElevenLabs Scribe', asrSherpaOnnxLocal: 'ローカル sherpa-onnx（実験的）', asrFoundryLocalWhisper: 'ローカル Whisper（Foundry Local）', + asrLocalWhisper: 'ローカル Whisper（バッチ）', asrLocalQwen3: 'ローカル Qwen3-ASR', + asrLocalQwen3Mlx: 'ローカル Qwen3-ASR（MLX / Metal）', + asrLocalQwen3C: 'ローカル Qwen3-ASR（C / CPU）', asrAppleSpeech: 'Apple 音声認識 (macOS)', omniOpenai: 'OpenAI（音声対応）', omniGemini: 'Google Gemini', @@ -1567,6 +1570,12 @@ export const ja: typeof zhCN = { sizeLoading: 'サイズ問い合わせ中…', sizeUnknown: 'サイズ不明', performanceWarning: 'ローカル ASR はオフラインやプライバシー重視のシーンに最適。初回使用時にモデルのダウンロードが必要。', + metalToolchainTitle: 'macOS MLX 初回ビルド要件', + metalToolchainDesc: '開発版の初回ビルドで MetalToolchain が不足している場合は、先に Xcode コンポーネントをインストールしてください。', + metalToolchainStep: 'ターミナルで次のコマンドを実行してから OpenLess を再起動してください：', + metalToolchainCopy: 'コマンドをコピー', + metalToolchainCopied: 'コピーしました', + metalToolchainVerify: 'xcrun --find metal で確認できます。', test: 'ロードしてテスト', testRunning: 'テスト中…', testHeading: '内蔵オーディオテスト', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index eac09ce27..3c5cb8637 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -970,7 +970,10 @@ export const ko: typeof zhCN = { asrElevenLabs: 'ElevenLabs Scribe', asrSherpaOnnxLocal: '로컬 sherpa-onnx(실험적)', asrFoundryLocalWhisper: '로컬 Whisper(Foundry Local)', + asrLocalWhisper: '로컬 Whisper(배치)', asrLocalQwen3: '로컬 Qwen3-ASR', + asrLocalQwen3Mlx: '로컬 Qwen3-ASR(MLX / Metal)', + asrLocalQwen3C: '로컬 Qwen3-ASR(C / CPU)', asrAppleSpeech: 'Apple 음성 (macOS)', omniOpenai: 'OpenAI (오디오 지원)', omniGemini: 'Google Gemini', @@ -1567,6 +1570,12 @@ export const ko: typeof zhCN = { sizeLoading: '크기 조회 중…', sizeUnknown: '크기 알 수 없음', performanceWarning: '로컬 ASR 은 오프라인 또는 개인정보 보호 시나리오에 적합. 첫 사용 시 모델 다운로드 필요.', + metalToolchainTitle: 'macOS MLX 최초 빌드 요구 사항', + metalToolchainDesc: '개발 버전 최초 빌드에서 MetalToolchain이 없다고 표시되면 먼저 Xcode 구성 요소를 설치하세요.', + metalToolchainStep: '터미널에서 아래 명령을 실행한 후 OpenLess를 다시 시작하세요:', + metalToolchainCopy: '명령 복사', + metalToolchainCopied: '복사됨', + metalToolchainVerify: 'xcrun --find metal로 설치를 확인할 수 있습니다.', test: '로드하여 테스트', testRunning: '테스트 중…', testHeading: '내장 오디오 테스트', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index bbe80ef39..e46c9f3ab 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -966,7 +966,10 @@ export const zhCN = { asrElevenLabs: 'ElevenLabs Scribe', asrSherpaOnnxLocal: '本地 sherpa-onnx（实验性）', asrFoundryLocalWhisper: '本地 Whisper（Foundry Local）', + asrLocalWhisper: '本地 Whisper（批量解码）', asrLocalQwen3: '本地 Qwen3-ASR', + asrLocalQwen3Mlx: '本地 Qwen3-ASR（MLX / Metal）', + asrLocalQwen3C: '本地 Qwen3-ASR（C / CPU）', asrAppleSpeech: 'Apple 语音（macOS）', omniOpenai: 'OpenAI（支持音频）', omniGemini: 'Google Gemini', @@ -1597,6 +1600,12 @@ export const zhCN = { sizeLoading: '正在查询尺寸…', sizeUnknown: '尺寸未知', performanceWarning: '本地 ASR 适合离线或隐私敏感场景，首次使用需下载模型。', + metalToolchainTitle: 'macOS MLX 首次构建要求', + metalToolchainDesc: '如果首次运行开发版时提示 MetalToolchain 缺失，请先安装 Xcode 组件。', + metalToolchainStep: '在终端执行下面的命令，完成后重新启动 OpenLess：', + metalToolchainCopy: '复制命令', + metalToolchainCopied: '已复制', + metalToolchainVerify: '可用 xcrun --find metal 验证安装是否成功。', test: '加载并测试', testRunning: '测试中…', testHeading: '内置音频测试', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index 0f94b4d2b..465c0c35f 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -968,7 +968,10 @@ export const zhTW: typeof zhCN = { asrElevenLabs: 'ElevenLabs Scribe', asrSherpaOnnxLocal: '本地 sherpa-onnx（實驗性）', asrFoundryLocalWhisper: '本地 Whisper（Foundry Local）', + asrLocalWhisper: '本地 Whisper（批次解碼）', asrLocalQwen3: '本地 Qwen3-ASR', + asrLocalQwen3Mlx: '本地 Qwen3-ASR（MLX / Metal）', + asrLocalQwen3C: '本地 Qwen3-ASR（C / CPU）', asrAppleSpeech: 'Apple 語音（macOS）', omniOpenai: 'OpenAI（支援音訊）', omniGemini: 'Google Gemini', @@ -1565,6 +1568,12 @@ export const zhTW: typeof zhCN = { sizeLoading: '正在查詢尺寸…', sizeUnknown: '尺寸未知', performanceWarning: '本地 ASR 適合離線或隱私敏感場景，首次使用需下載模型。', + metalToolchainTitle: 'macOS MLX 首次建置需求', + metalToolchainDesc: '如果首次執行開發版時提示缺少 MetalToolchain，請先安裝 Xcode 元件。', + metalToolchainStep: '在終端機執行以下命令，完成後重新啟動 OpenLess：', + metalToolchainCopy: '複製命令', + metalToolchainCopied: '已複製', + metalToolchainVerify: '可使用 xcrun --find metal 驗證安裝是否成功。', test: '加載並測試', testRunning: '測試中…', testHeading: '內置音頻測試', diff --git a/openless-all/app/src/lib/localAsr.ts b/openless-all/app/src/lib/localAsr.ts index 19becca8e..509572923 100644 --- a/openless-all/app/src/lib/localAsr.ts +++ b/openless-all/app/src/lib/localAsr.ts @@ -16,7 +16,7 @@ export interface LocalAsrSettings { mirror: string modelsBaseDir: string | null modelsRootDir: string - /** macOS 才编入 vendored Open-Less/qwen-asr 引擎；Win 端 UI 据此把"开始"按钮灰掉。 */ + /** macOS/Linux 编入 C 引擎；MLX 仅在 macOS 可用。 */ engineAvailable: boolean } @@ -177,7 +177,7 @@ const MOCK_FOUNDRY_CATALOG: FoundryLocalAsrCatalogModel[] = [ ] const MOCK_SETTINGS: LocalAsrSettings = { - providerId: "local-qwen3", + providerId: "local-qwen3-mlx", activeModel: "qwen3-asr-0.6b", mirror: "huggingface", modelsBaseDir: null, diff --git a/openless-all/app/src/pages/History.tsx b/openless-all/app/src/pages/History.tsx index 2963dc1d7..84910df28 100644 --- a/openless-all/app/src/pages/History.tsx +++ b/openless-all/app/src/pages/History.tsx @@ -431,7 +431,12 @@ export function History() { )} {(!mobile || mobileDetailOpen) && ( - + {item ? ( <> {mobile && ( diff --git a/openless-all/app/src/pages/LocalAsr/components.tsx b/openless-all/app/src/pages/LocalAsr/components.tsx index 412e456be..b0842428d 100644 --- a/openless-all/app/src/pages/LocalAsr/components.tsx +++ b/openless-all/app/src/pages/LocalAsr/components.tsx @@ -2,7 +2,7 @@ // LocalAsr/index.tsx (behavior-preserving move). All are props-driven and // stateless beyond local render memoization. -import { useMemo } from "react" +import { useMemo, useState } from "react" import { createPortal } from "react-dom" import { useTranslation } from "react-i18next" import { @@ -12,11 +12,75 @@ import { type LocalAsrModelStatus, type LocalAsrTestResult, } from "../../lib/localAsr" -import { Btn, Card, Pill } from "../_atoms" +import { Btn, Card, Collapsible, Pill } from "../_atoms" import { Icon } from "../../components/Icon" import { formatBytes } from "./helpers" import type { RemoteSize } from "./types" +export function MetalToolchainGuide() { + const { t } = useTranslation() + const [copied, setCopied] = useState(false) + const command = "xcodebuild -downloadComponent MetalToolchain" + + const copyCommand = async () => { + try { + await navigator.clipboard.writeText(command) + setCopied(true) + window.setTimeout(() => setCopied(false), 1600) + } catch { + // 命令仍然直接显示在 code block 中，剪贴板不可用时不阻断引导。 + } + } + + return ( + +
+
+ {t("localAsr.metalToolchainStep")} +
+
+ + {command} + + void copyCommand()}> + {copied + ? t("localAsr.metalToolchainCopied") + : t("localAsr.metalToolchainCopy")} + +
+
+ {t("localAsr.metalToolchainVerify")} +
+
+ + ) +} + export function FoundryPrepareProgressBlock({ progress, modelCached, @@ -558,7 +622,7 @@ export function TestResultBlock({ // 纯展示组件；数据与动作由 LocalAsr/index.tsx 组装后传入。 // ───────────────────────────────────────────────────────────────────── -/** 侧栏统一条目：三套本地引擎（Qwen3 / sherpa-onnx / foundry）归一化。 */ +/** 侧栏统一条目：本地引擎（Qwen3 / Whisper / sherpa-onnx / foundry）归一化。 */ export interface SidebarModelEntry { id: string /** 展示名（如 qwen3-asr-0.6b / whisper-small）。 */ @@ -576,7 +640,7 @@ export interface SidebarModelEntry { /** 当前激活（设为默认的本地模型）。 */ isActive: boolean /** 引擎标识，决定右侧动作按钮分派。 */ - engine: "qwen3" | "sherpa" | "foundry" + engine: "qwen3" | "whisper" | "sherpa" | "foundry" } /** 左侧模型选择栏：竖排条目，选中高亮；底部预留「下载新模型」按钮位。 */ diff --git a/openless-all/app/src/pages/LocalAsr/index.tsx b/openless-all/app/src/pages/LocalAsr/index.tsx index af68f92df..1a268030c 100644 --- a/openless-all/app/src/pages/LocalAsr/index.tsx +++ b/openless-all/app/src/pages/LocalAsr/index.tsx @@ -104,6 +104,7 @@ import { import { DownloadProgressBlock, FoundryPrepareProgressBlock, + MetalToolchainGuide, ModelDetailPanel, ModelSidebar, type SidebarModelEntry, @@ -1497,21 +1498,28 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { } } - // 「加载并测试」（qwen3）：先设为当前模型（含把 active provider 切到本地 - // —— 与 ProvidersSection 的本地模型下拉一致），再跑内置音频测试。不再单独 - // 提供「设为默认」按钮：激活 = 在 ASR 语音转写里选择本地模型供应商。 - const handleTest = async (modelId: string) => { + // 先设为当前模型（含把 active provider 切到对应的本地引擎），再跑内置音频 + // 测试。这样 Qwen3 与 Whisper 可以在同一页切换并比较加载/转写耗时。 + const handleTest = async ( + modelId: string, + provider: "local-qwen3-mlx" | "local-qwen3-c" | "local-whisper" = + prefs?.activeAsrProvider === "local-qwen3-c" + ? "local-qwen3-c" + : IS_MAC + ? "local-qwen3-mlx" + : "local-qwen3-c", + ) => { try { await setLocalAsrActiveModel(modelId) - await ensureLocalAsrChannel("local-qwen3") - await setActiveAsrProvider("local-qwen3") + await ensureLocalAsrChannel(provider) + await setActiveAsrProvider(provider) await updatePrefs((current) => - current.activeAsrProvider === "local-qwen3" && + current.activeAsrProvider === provider && current.localAsrActiveModel === modelId ? current : { ...current, - activeAsrProvider: "local-qwen3", + activeAsrProvider: provider, localAsrActiveModel: modelId, }, ) @@ -1774,8 +1782,10 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { // 「＋ 下载新模型」弹窗获取）。 const allSidebarEntries = useMemo(() => { const entries: SidebarModelEntry[] = [] - // macOS：Qwen3 引擎 + // macOS：Qwen3 / Whisper 引擎 for (const m of models) { + const isWhisper = m.id.startsWith("whisper-") + if (isWhisper && !IS_MAC) continue const isDownloading = Boolean(progress[m.id]) && (progress[m.id]?.phase === "started" || @@ -1797,8 +1807,14 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { : null, isActive: settings?.activeModel === m.id && - prefs?.activeAsrProvider === "local-qwen3", - engine: "qwen3", + (isWhisper + ? prefs?.activeAsrProvider === "local-whisper" + : [ + "local-qwen3", + "local-qwen3-mlx", + "local-qwen3-c", + ].includes(prefs?.activeAsrProvider ?? "")), + engine: isWhisper ? "whisper" : "qwen3", }) } // Windows：sherpa-onnx + foundry @@ -1916,6 +1932,10 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { if (action === "download") void handleDownload(entry.id) else if (action === "delete") void handleDelete(entry.id) else if (action === "reveal") void handleRevealModelDir(entry.id) + } else if (entry.engine === "whisper") { + if (action === "download") void handleDownload(entry.id) + else if (action === "delete") void handleDelete(entry.id) + else if (action === "reveal") void handleRevealModelDir(entry.id) } else if (entry.engine === "sherpa") { const alias = entry.id as SherpaOnnxModelAlias if (action === "download") { @@ -1955,12 +1975,14 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { const selectedEntryRemote = selectedEntry ? selectedEntry.engine === "qwen3" ? remoteSizes[selectedEntry.id] - : selectedEntry.engine === "sherpa" - ? sherpaRemoteSizes[selectedEntry.id] + : selectedEntry.engine === "whisper" || selectedEntry.engine === "sherpa" + ? selectedEntry.engine === "whisper" + ? remoteSizes[selectedEntry.id] + : sherpaRemoteSizes[selectedEntry.id] : null : null const selectedEntryProgress = - selectedEntry?.engine === "qwen3" + selectedEntry?.engine === "qwen3" || selectedEntry?.engine === "whisper" ? progress[selectedEntry.id] : selectedEntry?.engine === "sherpa" ? sherpaDownloadProgress[selectedEntry.id] @@ -2006,6 +2028,8 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { )} + {IS_MAC && } + {/* ─── 模型管理看板：左侧模型选择（竖排，已下载打绿勾），右侧详情 （HF 实时抓取的尺寸/文件数）+ 操作。全平台归一化（Qwen3 / sherpa-onnx / foundry），作为设置里的单独板块而非独立窗口。 ─── */} @@ -2053,7 +2077,8 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { entry={selectedEntry} fileCount={selectedEntryRemote?.fileCount ?? null} mirrorLabel={ - selectedEntry?.engine === "qwen3" + selectedEntry?.engine === "qwen3" || + selectedEntry?.engine === "whisper" ? settings?.mirror === "hf-mirror" ? "hf-mirror" : "huggingface" @@ -2069,7 +2094,10 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { } onCancel={() => { if (!selectedEntry) return - if (selectedEntry.engine === "qwen3") + if ( + selectedEntry.engine === "qwen3" || + selectedEntry.engine === "whisper" + ) void handleCancel(selectedEntry.id) else if (selectedEntry.engine === "sherpa") void handleCancelSherpaDownload() @@ -2080,18 +2108,33 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { onReveal={() => selectedEntry && dispatchEntryAction(selectedEntry, "reveal") } - onTest={() => - selectedEntry?.engine === "qwen3" && - void handleTest(selectedEntry.id) + onTest={() => { + if ( + selectedEntry?.engine === "qwen3" || + selectedEntry?.engine === "whisper" + ) { + void handleTest( + selectedEntry.id, + selectedEntry.engine === "whisper" + ? "local-whisper" + : IS_MAC + ? "local-qwen3-mlx" + : "local-qwen3-c", + ) + } + }} + showTest={ + selectedEntry?.engine === "qwen3" || + selectedEntry?.engine === "whisper" } - showTest={selectedEntry?.engine === "qwen3"} testResult={ selectedEntry ? (testResults[selectedEntry.id] ?? null) : null } testing={ - selectedEntry?.engine === "qwen3" && + (selectedEntry?.engine === "qwen3" || + selectedEntry?.engine === "whisper") && testingModelId === selectedEntry.id } /> @@ -2420,7 +2463,7 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { const entry = allSidebarEntries.find((e) => e.id === id) if (!entry) return null const remote = - entry.engine === "qwen3" + entry.engine === "qwen3" || entry.engine === "whisper" ? remoteSizes[id] : entry.engine === "sherpa" ? sherpaRemoteSizes[id] diff --git a/openless-all/app/src/pages/settings/ChannelList.test.ts b/openless-all/app/src/pages/settings/ChannelList.test.ts index f340235a1..e8fa07d6f 100644 --- a/openless-all/app/src/pages/settings/ChannelList.test.ts +++ b/openless-all/app/src/pages/settings/ChannelList.test.ts @@ -2,16 +2,18 @@ import type { OS } from '../../components/WindowChrome'; import { presetsFor, shouldRecycleDraft } from './ChannelList'; const localProviders = [ - 'local-qwen3', + 'local-qwen3-mlx', + 'local-qwen3-c', + 'local-whisper', 'apple-speech', 'foundry-local-whisper', 'sherpa-onnx-local', ] as const; const expectedByPlatform: Record = { - mac: ['local-qwen3', 'apple-speech'], + mac: ['local-qwen3-mlx', 'local-qwen3-c', 'local-whisper', 'apple-speech'], win: ['foundry-local-whisper', 'sherpa-onnx-local'], - linux: [], + linux: ['local-qwen3-c'], android: [], }; diff --git a/openless-all/app/src/pages/settings/ChannelList.tsx b/openless-all/app/src/pages/settings/ChannelList.tsx index 5113d6ecc..839c1eaf9 100644 --- a/openless-all/app/src/pages/settings/ChannelList.tsx +++ b/openless-all/app/src/pages/settings/ChannelList.tsx @@ -55,7 +55,8 @@ export function presetsFor(kind: ChannelKind, os: OS): PresetOption[] { } return ASR_PRESETS.filter(p => { // 本地引擎严格按其实际支持的平台暴露；Linux / Android 不展示桌面专有实现。 - if (p.id === 'local-qwen3' || p.id === 'apple-speech') return os === 'mac'; + if (p.id === 'local-qwen3-mlx' || p.id === 'local-whisper' || p.id === 'apple-speech') return os === 'mac'; + if (p.id === 'local-qwen3-c') return os === 'mac' || os === 'linux'; if (p.id === 'foundry-local-whisper' || p.id === 'sherpa-onnx-local') { return os === 'win'; } diff --git a/openless-all/app/src/pages/settings/ProvidersSection.tsx b/openless-all/app/src/pages/settings/ProvidersSection.tsx index 8a4789256..b4aee7ff3 100644 --- a/openless-all/app/src/pages/settings/ProvidersSection.tsx +++ b/openless-all/app/src/pages/settings/ProvidersSection.tsx @@ -43,7 +43,9 @@ import { // 本地模型供应商：在主下拉里标注「本地」后缀，与云端供应商区分开。 const LOCAL_ASR_PRESET_IDS: ReadonlySet = new Set([ - 'local-qwen3', + 'local-qwen3-mlx', + 'local-qwen3-c', + 'local-whisper', 'foundry-local-whisper', 'sherpa-onnx-local', ]); @@ -233,7 +235,9 @@ const ASR_DEFAULT_RESOURCE_ID = 'volc.seedasr.sauc.duration'; /// 无 key / 无地址的本地引擎：卡片编辑里没有凭据字段，模型下载仍在「高级 → 本地模型」。 export const LOCAL_ASR_PROVIDER_IDS: string[] = [ - 'local-qwen3', + 'local-qwen3-mlx', + 'local-qwen3-c', + 'local-whisper', 'sherpa-onnx-local', 'foundry-local-whisper', 'apple-speech', diff --git a/openless-all/app/src/pages/settings/shared.tsx b/openless-all/app/src/pages/settings/shared.tsx index cf9ec9b1f..204e5f4c8 100644 --- a/openless-all/app/src/pages/settings/shared.tsx +++ b/openless-all/app/src/pages/settings/shared.tsx @@ -252,10 +252,12 @@ export const ASR_PRESETS = [ // 两字段（asr/xfyun.rs）；音频 16k/16bit/mono，与 recorder 输出一致。 { id: 'iflytek', nameKey: 'asrIflytek', baseUrl: '', model: '' }, { id: 'foundry-local-whisper', nameKey: 'asrFoundryLocalWhisper', baseUrl: '', model: '' }, + { id: 'local-whisper', nameKey: 'asrLocalWhisper', baseUrl: '', model: '' }, // 本地引擎（Foundry / sherpa-onnx / Qwen3）：无 baseUrl/model 配置， // 模型在「高级 → 本地模型」里下载与切换。 { id: 'sherpa-onnx-local', nameKey: 'asrSherpaOnnxLocal', baseUrl: '', model: '' }, - { id: 'local-qwen3', nameKey: 'asrLocalQwen3', baseUrl: '', model: '' }, + { id: 'local-qwen3-mlx', nameKey: 'asrLocalQwen3Mlx', baseUrl: '', model: '' }, + { id: 'local-qwen3-c', nameKey: 'asrLocalQwen3C', baseUrl: '', model: '' }, // Apple 系统语音识别（macOS）：无 baseUrl/model、无下载、无凭据。 { id: 'apple-speech', nameKey: 'asrAppleSpeech', baseUrl: '', model: '' }, ] as const; From dbe086da097af2d0a907a1aa0aa2f118fb203521 Mon Sep 17 00:00:00 2001 From: jimersylee Date: Tue, 11 Aug 2026 21:46:39 +0800 Subject: [PATCH 02/10] =?UTF-8?q?fix(build):=20=E4=BF=AE=E5=A4=8D=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=20ASR=20=E5=A4=9A=E5=B9=B3=E5=8F=B0=E6=9E=84=E5=BB=BA?= =?UTF-8?q?=E4=B8=8E=E6=97=A7=E6=B8=A0=E9=81=93=E5=85=BC=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitmodules | 2 +- .../scripts/check-macos-metal-toolchain.mjs | 4 ++- openless-all/app/src-tauri/Cargo.toml | 8 ++++-- .../app/src-tauri/src/asr/local/mod.rs | 18 +++++++------ .../app/src-tauri/src/asr/local/test_run.rs | 2 +- .../app/src-tauri/src/commands/mod.rs | 4 +-- openless-all/app/src-tauri/src/types.rs | 10 +++++++- .../app/src-tauri/vendor/qwen3-asr-rs | 2 +- openless-all/app/src/lib/platform.ts | 7 +++++- openless-all/app/src/lib/types.ts | 1 + openless-all/app/src/pages/LocalAsr/index.tsx | 25 ++++++++++++------- .../src/pages/settings/ChannelList.test.ts | 9 +++++++ .../app/src/pages/settings/ChannelList.tsx | 14 ++++++++--- .../src/pages/settings/ProvidersSection.tsx | 1 + .../app/src/pages/settings/shared.tsx | 2 ++ 15 files changed, 79 insertions(+), 30 deletions(-) diff --git a/.gitmodules b/.gitmodules index 083570cba..4862d20df 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = https://github.com/Open-Less/qwen-asr.git [submodule "openless-all/app/src-tauri/vendor/qwen3-asr-rs"] path = openless-all/app/src-tauri/vendor/qwen3-asr-rs - url = git@github.com:jimersylee/qwen3_asr_rs.git + url = https://github.com/jimersylee/qwen3_asr_rs.git diff --git a/openless-all/app/scripts/check-macos-metal-toolchain.mjs b/openless-all/app/scripts/check-macos-metal-toolchain.mjs index b9b58c3e0..fb41bcc09 100644 --- a/openless-all/app/scripts/check-macos-metal-toolchain.mjs +++ b/openless-all/app/scripts/check-macos-metal-toolchain.mjs @@ -1,6 +1,8 @@ import { spawnSync } from "node:child_process" -if (process.platform !== "darwin") process.exit(0) +// MLX is only compiled into Apple Silicon builds. Intel macOS uses the C/CPU +// Qwen backend and Whisper, so it must not be blocked by a Metal toolchain check. +if (process.platform !== "darwin" || process.arch !== "arm64") process.exit(0) const result = spawnSync("xcrun", ["--find", "metal"], { encoding: "utf8", diff --git a/openless-all/app/src-tauri/Cargo.toml b/openless-all/app/src-tauri/Cargo.toml index 13e49bee0..1eba82d5a 100644 --- a/openless-all/app/src-tauri/Cargo.toml +++ b/openless-all/app/src-tauri/Cargo.toml @@ -117,7 +117,6 @@ minisign-verify = "0.2" [target.'cfg(target_os = "macos")'.dependencies] block2 = "0.5" core-foundation = "0.10" -qwen3-asr-rs = { path = "vendor/qwen3-asr-rs", default-features = false, features = ["mlx"] } tokenizers = "0.21" whisper-rs = { version = "0.14", features = ["metal"] } core-graphics = "0.24" @@ -128,6 +127,9 @@ coreaudio-sys = "0.2" objc2 = { version = "0.5", features = ["exception"] } objc2-foundation = "0.2" objc2-app-kit = "0.2" + +[target.'cfg(all(target_os = "macos", target_arch = "aarch64"))'.dependencies] +qwen3-asr-rs = { path = "vendor/qwen3-asr-rs", default-features = false, features = ["mlx"] } # 把胶囊窗口转成「非激活 NSPanel」，使其能叠在别的 app 的全屏 Space 之上。 # 普通 NSWindow 即便设了 collectionBehavior 也做不到（tauri#9556 / #11488）—— 必须是 # NSPanel。该插件做窗口 NSWindow→NSPanel subclass 转换；仅 macOS。 @@ -174,7 +176,9 @@ custom-protocol = ["tauri/custom-protocol"] opt-level = 3 lto = "thin" codegen-units = 1 -strip = "symbols" +# `symbols` corrupts macOS proc-macro dylibs with the Homebrew Rust toolchain +# (rustc reports "mis-aligned LINKEDIT string pool" when loading them). +strip = "debuginfo" # issue #801: wayland-scanner 0.31.10 still pins vulnerable quick-xml 0.39. # This local copy backports upstream security commit d07c4f91f28b without diff --git a/openless-all/app/src-tauri/src/asr/local/mod.rs b/openless-all/app/src-tauri/src/asr/local/mod.rs index 3d3792630..3a6cfefb3 100644 --- a/openless-all/app/src-tauri/src/asr/local/mod.rs +++ b/openless-all/app/src-tauri/src/asr/local/mod.rs @@ -36,7 +36,7 @@ pub use sherpa_runtime::SherpaOnnxRuntime; #[cfg(target_os = "macos")] mod apple_speech_provider; -#[cfg(target_os = "macos")] +#[cfg(all(target_os = "macos", target_arch = "aarch64"))] mod mlx_qwen_engine; #[cfg(any(target_os = "macos", target_os = "linux"))] mod qwen_engine; @@ -48,7 +48,7 @@ mod qwen_ffi; pub use apple_speech_provider::{native_name_to_apple_locale, AppleSpeechAsr}; #[cfg(any(target_os = "macos", target_os = "linux"))] pub use local_provider::LocalQwenAsr; -#[cfg(target_os = "macos")] +#[cfg(all(target_os = "macos", target_arch = "aarch64"))] pub use mlx_qwen_engine::MlxQwenAsrEngine; #[cfg(target_os = "macos")] pub use whisper_provider::MODEL_ID as WHISPER_MODEL_ID; @@ -85,7 +85,7 @@ pub fn is_local_qwen3(id: &str) -> bool { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum QwenBackend { - #[cfg(target_os = "macos")] + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] Mlx, C, } @@ -93,7 +93,7 @@ pub enum QwenBackend { impl QwenBackend { pub fn cache_key(self) -> &'static str { match self { - #[cfg(target_os = "macos")] + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] Self::Mlx => "mlx", Self::C => "c", } @@ -102,19 +102,21 @@ impl QwenBackend { pub fn qwen_backend_for_provider(id: &str) -> Option { match id { - #[cfg(target_os = "macos")] + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] PROVIDER_ID | LOCAL_QWEN3_MLX_PROVIDER_ID => Some(QwenBackend::Mlx), #[cfg(target_os = "linux")] PROVIDER_ID | LOCAL_QWEN3_C_PROVIDER_ID => Some(QwenBackend::C), #[cfg(target_os = "macos")] LOCAL_QWEN3_C_PROVIDER_ID => Some(QwenBackend::C), + #[cfg(all(target_os = "macos", not(target_arch = "aarch64")))] + PROVIDER_ID => Some(QwenBackend::C), _ => None, } } #[cfg(any(target_os = "macos", target_os = "linux"))] pub enum LocalQwenEngine { - #[cfg(target_os = "macos")] + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] Mlx(MlxQwenAsrEngine), C(qwen_engine::QwenAsrEngine), } @@ -123,7 +125,7 @@ pub enum LocalQwenEngine { impl LocalQwenEngine { pub fn load(backend: QwenBackend, model_dir: &std::path::Path) -> anyhow::Result { match backend { - #[cfg(target_os = "macos")] + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] QwenBackend::Mlx => Ok(Self::Mlx(MlxQwenAsrEngine::load(model_dir)?)), QwenBackend::C => Ok(Self::C(qwen_engine::QwenAsrEngine::load(model_dir)?)), } @@ -131,7 +133,7 @@ impl LocalQwenEngine { pub fn transcribe_pcm(&self, samples: &[f32]) -> anyhow::Result { match self { - #[cfg(target_os = "macos")] + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] Self::Mlx(engine) => engine.transcribe_pcm(samples), Self::C(engine) => engine.transcribe_audio(samples), } diff --git a/openless-all/app/src-tauri/src/asr/local/test_run.rs b/openless-all/app/src-tauri/src/asr/local/test_run.rs index 3bd48042f..2bc7b6966 100644 --- a/openless-all/app/src-tauri/src/asr/local/test_run.rs +++ b/openless-all/app/src-tauri/src/asr/local/test_run.rs @@ -120,7 +120,7 @@ pub async fn run_test( Ok(TestResult { backend: match backend { - #[cfg(target_os = "macos")] + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] super::QwenBackend::Mlx => "MLX Metal (Apple Silicon)", super::QwenBackend::C => "C CPU", } diff --git a/openless-all/app/src-tauri/src/commands/mod.rs b/openless-all/app/src-tauri/src/commands/mod.rs index ada6b9c22..4c65d998c 100644 --- a/openless-all/app/src-tauri/src/commands/mod.rs +++ b/openless-all/app/src-tauri/src/commands/mod.rs @@ -363,7 +363,7 @@ mod tests { crate::asr::local::PROVIDER_ID, &snapshot() )); - #[cfg(target_os = "macos")] + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] assert!(asr_configured_for_provider( crate::asr::local::LOCAL_QWEN3_MLX_PROVIDER_ID, &snapshot() @@ -417,7 +417,7 @@ mod tests { assert!(active_asr_is_keyless_for_validation( crate::asr::local::LOCAL_QWEN3_C_PROVIDER_ID )); - #[cfg(target_os = "macos")] + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] assert!(active_asr_is_keyless_for_validation( crate::asr::local::LOCAL_QWEN3_MLX_PROVIDER_ID )); diff --git a/openless-all/app/src-tauri/src/types.rs b/openless-all/app/src-tauri/src/types.rs index d842b4bb2..15b484277 100644 --- a/openless-all/app/src-tauri/src/types.rs +++ b/openless-all/app/src-tauri/src/types.rs @@ -3013,6 +3013,7 @@ pub struct PlatformCapabilities { pub supports_desktop_hotkey: bool, pub supports_tray: bool, pub supports_local_asr: bool, + pub supports_local_qwen3_mlx: bool, pub supports_in_app_dictation: bool, pub supports_auto_update: bool, } @@ -3028,6 +3029,7 @@ impl PlatformCapabilities { supports_desktop_hotkey: false, supports_tray: false, supports_local_asr: false, + supports_local_qwen3_mlx: false, supports_in_app_dictation: true, supports_auto_update: true, } @@ -3045,6 +3047,7 @@ impl PlatformCapabilities { supports_desktop_hotkey: false, supports_tray: false, supports_local_asr: false, + supports_local_qwen3_mlx: false, supports_in_app_dictation: false, supports_auto_update: false, } @@ -3058,7 +3061,12 @@ impl PlatformCapabilities { supports_overlay: true, supports_desktop_hotkey: true, supports_tray: true, - supports_local_asr: cfg!(any(target_os = "macos", target_os = "windows")), + supports_local_asr: cfg!(any( + target_os = "macos", + target_os = "linux", + target_os = "windows" + )), + supports_local_qwen3_mlx: cfg!(all(target_os = "macos", target_arch = "aarch64")), supports_in_app_dictation: false, supports_auto_update: true, } diff --git a/openless-all/app/src-tauri/vendor/qwen3-asr-rs b/openless-all/app/src-tauri/vendor/qwen3-asr-rs index 88f7f4ae2..9a17974ce 160000 --- a/openless-all/app/src-tauri/vendor/qwen3-asr-rs +++ b/openless-all/app/src-tauri/vendor/qwen3-asr-rs @@ -1 +1 @@ -Subproject commit 88f7f4ae26a50f01cf40d9a30819cd006d3f1d8e +Subproject commit 9a17974ce8d5026fdb36913de0f03198e31737fb diff --git a/openless-all/app/src/lib/platform.ts b/openless-all/app/src/lib/platform.ts index 1414dcecd..49ea0670b 100644 --- a/openless-all/app/src/lib/platform.ts +++ b/openless-all/app/src/lib/platform.ts @@ -34,6 +34,7 @@ const MOBILE_UNAVAILABLE: PlatformCapabilities = { supportsOverlay: false, supportsImeInput: false, supportsLocalAsr: false, + supportsLocalQwen3Mlx: false, supportsInAppDictation: false, supportsAutoUpdate: false, }; @@ -67,6 +68,7 @@ export function inferPlatformCapabilities(): PlatformCapabilities { supportsOverlay: true, supportsImeInput: false, supportsLocalAsr: false, + supportsLocalQwen3Mlx: false, supportsInAppDictation: true, supportsAutoUpdate: true, }; @@ -83,7 +85,10 @@ export function inferPlatformCapabilities(): PlatformCapabilities { supportsTray: true, supportsOverlay: true, supportsImeInput: os === 'win', - supportsLocalAsr: os === 'mac' || os === 'win', + supportsLocalAsr: os === 'mac' || os === 'linux' || os === 'win', + // Tauri returns the authoritative architecture-aware value. The browser fallback + // keeps MLX visible on macOS until the native capability query is available. + supportsLocalQwen3Mlx: os === 'mac', supportsInAppDictation: false, supportsAutoUpdate: true, }; diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index 8d19393f2..d33c9b581 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -699,6 +699,7 @@ export interface PlatformCapabilities { supportsOverlay: boolean; supportsImeInput: boolean; supportsLocalAsr: boolean; + supportsLocalQwen3Mlx: boolean; supportsInAppDictation: boolean; supportsAutoUpdate: boolean; } diff --git a/openless-all/app/src/pages/LocalAsr/index.tsx b/openless-all/app/src/pages/LocalAsr/index.tsx index 1a268030c..f1bddb583 100644 --- a/openless-all/app/src/pages/LocalAsr/index.tsx +++ b/openless-all/app/src/pages/LocalAsr/index.tsx @@ -89,6 +89,7 @@ import { } from "../../lib/localAsr" import { useHotkeySettings } from "../../state/HotkeySettingsContext" import { detectOS } from "../../components/WindowChrome" +import { getPlatformCapabilities } from "../../lib/platform" import { SelectLite } from "../../components/ui/SelectLite" import { Btn, Card, Collapsible, PageHeader, Pill } from "../_atoms" import { @@ -138,12 +139,11 @@ async function ensureLocalAsrChannel(providerType: string): Promise { // 非 Windows 平台 runtime 是 stub 永远 unavailable。前端这一页对应的卡片、状态拉取、 // 事件订阅都必须按 OS 隔离，避免 macOS / Linux 用户看到 Windows 专属的 UI。 // -// 同理 Qwen3-ASR 后端只在 macOS 编译实体（qwen_engine / cache / local_provider 全是 -// `#[cfg(target_os = "macos")]`），Qwen3 模型管理 UI 也按 IS_MAC 守严——之前用 -// `!IS_WINDOWS` 会让假设的 Linux 渲染路径暴露死 UI（pr_agent #403 'Linux regression' -// 修法）。 +// Qwen3-ASR 的 MLX 实体只在 Apple Silicon 编译，C/CPU 实体覆盖 macOS / Linux； +// Qwen3 模型管理 UI 仍按桌面端守严，具体后端由平台能力与渠道选择决定。 const IS_WINDOWS = detectOS() === "win" const IS_MAC = detectOS() === "mac" +const IS_QWEN_PLATFORM = IS_MAC || detectOS() === "linux" interface LocalAsrProps { /// `embedded=true` 表示作为子组件嵌入「高级」设置页（Settings → Advanced）； @@ -158,6 +158,7 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { const { t } = useTranslation() const { prefs, updatePrefs } = useHotkeySettings() const [settings, setSettings] = useState(null) + const [supportsQwen3Mlx, setSupportsQwen3Mlx] = useState(IS_MAC) const [models, setModels] = useState([]) // 两栏看板：右侧当前选中的模型（默认选第一个已下载的）。 const [selectedModelId, setSelectedModelId] = useState(null) @@ -244,6 +245,12 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { const scrollGuardTimer = useRef(null) const scrollGuardCleanup = useRef<(() => void) | null>(null) + useEffect(() => { + void getPlatformCapabilities().then(caps => + setSupportsQwen3Mlx(caps.supportsLocalQwen3Mlx), + ) + }, []) + const restoreScrollGuard = () => { const guard = scrollGuard.current if (!guard) return @@ -1505,7 +1512,7 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { provider: "local-qwen3-mlx" | "local-qwen3-c" | "local-whisper" = prefs?.activeAsrProvider === "local-qwen3-c" ? "local-qwen3-c" - : IS_MAC + : supportsQwen3Mlx ? "local-qwen3-mlx" : "local-qwen3-c", ) => { @@ -2028,7 +2035,7 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { )} - {IS_MAC && } + {supportsQwen3Mlx && } {/* ─── 模型管理看板：左侧模型选择（竖排，已下载打绿勾），右侧详情 （HF 实时抓取的尺寸/文件数）+ 操作。全平台归一化（Qwen3 / @@ -2117,7 +2124,7 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { selectedEntry.id, selectedEntry.engine === "whisper" ? "local-whisper" - : IS_MAC + : supportsQwen3Mlx ? "local-qwen3-mlx" : "local-qwen3-c", ) @@ -2149,7 +2156,7 @@ export function LocalAsr({ embedded = false }: LocalAsrProps = {}) { title={t("localAsr.downloadSettingsTitle")} desc={t("localAsr.downloadSettingsDesc")} > - {IS_MAC && ( + {IS_QWEN_PLATFORM && ( <>