From 83fd48b9104bef762b8751210569445a196b5ad9 Mon Sep 17 00:00:00 2001 From: xiangsilian Date: Sun, 16 Aug 2026 11:40:47 +0800 Subject: [PATCH] =?UTF-8?q?feat(app):=20AI=20=E5=AF=B9=E8=AF=9D=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E8=BE=93=E5=85=A5=EF=BC=88=E5=BE=AE=E4=BF=A1=E5=BC=8F?= =?UTF-8?q?=EF=BC=89+=20=E5=9B=9E=E5=A4=8D=E8=AF=AD=E9=9F=B3=E6=9C=97?= =?UTF-8?q?=E8=AF=BB=E2=80=94=E2=80=94ASR=20V2=20=E5=BC=95=E6=93=8E?= =?UTF-8?q?=E8=87=AA=E9=87=87=E4=BF=AE=E5=A4=8D=201002200006=20busy?= =?UTF-8?q?=EF=BC=88recognitionMode=200=20=E4=B8=8D=E5=86=8D=E8=87=AA?= =?UTF-8?q?=E5=BB=BA=E9=87=87=E9=9B=86=E5=99=A8=EF=BC=89+=20busy=20?= =?UTF-8?q?=E8=87=AA=E6=84=88=EF=BC=9BTTS=20V2=20=E9=95=BF=E6=96=87?= =?UTF-8?q?=E5=88=86=E6=AE=B5=E4=B8=B2=E8=A1=8C=E6=9C=97=E8=AF=BB=EF=BC=88?= =?UTF-8?q?=E5=8D=95=E6=AC=A1=20500=20=E5=AD=97=E7=AC=A6=E4=B8=8A=E9=99=90?= =?UTF-8?q?=EF=BC=89+=20=E7=8A=B6=E6=80=81=E5=9B=9E=E8=B0=83=EF=BC=9B?= =?UTF-8?q?=E6=8C=89=E4=BD=8F=E8=AF=B4=E8=AF=9D/=E6=9D=BE=E5=BC=80?= =?UTF-8?q?=E5=8F=91=E9=80=81/=E4=B8=8A=E6=BB=91=E5=8F=96=E6=B6=88?= =?UTF-8?q?=EF=BC=8C=E9=BA=A6=E5=85=8B=E9=A3=8E=E6=9D=83=E9=99=90=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E6=97=B6=E7=94=B3=E8=AF=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/ets/components/f4/TtsEngine.ets | 239 ++++++++++ .../ets/components/f4/VoiceInputEngine.ets | 310 ++++++++++++ app/entry/src/main/ets/pages/ChatPage.ets | 442 +++++++++++++----- app/entry/src/main/module.json5 | 12 +- .../main/resources/base/element/string.json | 8 +- 5 files changed, 900 insertions(+), 111 deletions(-) create mode 100644 app/entry/src/main/ets/components/f4/TtsEngine.ets create mode 100644 app/entry/src/main/ets/components/f4/VoiceInputEngine.ets diff --git a/app/entry/src/main/ets/components/f4/TtsEngine.ets b/app/entry/src/main/ets/components/f4/TtsEngine.ets new file mode 100644 index 0000000..0159ebb --- /dev/null +++ b/app/entry/src/main/ets/components/f4/TtsEngine.ets @@ -0,0 +1,239 @@ +/** + * TtsEngine —— AI 回复语音朗读（CoreSpeechKit textToSpeech）。 + * + * V2（真机修复版）： + * - 长文本分段：单次 speak 文本上限 500 字符（1002300001），超长直接失败。 + * 按句子边界切 ≤400 字符/段，在 onComplete(type=1 播报完成) 回调里串行 + * 播下一段，150 秒兜底定时器防回调丢失卡死链条 + * - 不依赖 queueMode：官方 API 文档（0=flush）与 Codelabs（0=排队）说法 + * 互相矛盾，新朗读前显式 engine.stop() 打断，语义自持 + * - 播报状态回调 onStateChanged：页面据此复位 🔊 图标 + * + * 模型：offline 中文引擎（online: 1，系统内置）；模拟器通常无此服务，须真机。 + * 日志 tag MEIDA_TTS：hdc shell hilog | grep MEIDA_TTS + */ + +import { textToSpeech } from '@kit.CoreSpeechKit' + +const SEGMENT_MAX_CHARS: number = 400 +const SEGMENT_FALLBACK_MS: number = 150000 + +/** 播报状态变化回调（speaking=false 表示本轮朗读完全结束或被停止）。 */ +export type TtsStateChanged = (speaking: boolean) => void + +function splitSegments(text: string, maxChars: number): string[] { + const segments: string[] = [] + let current: string = '' + const sentenceEnds: string[] = ['。', '！', '？', '；', '\n'] + for (let i = 0; i < text.length; i++) { + current += text[i] + const atSentenceEnd: boolean = sentenceEnds.indexOf(text[i]) >= 0 + if ((atSentenceEnd && current.length >= 16) || current.length >= maxChars) { + segments.push(current) + current = '' + } + } + if (current !== '') { + segments.push(current) + } + return segments +} + +export class TtsEngine { + private engine: textToSpeech.TextToSpeechEngine | null = null + private queue: string[] = [] + private currentRequestId: string = '' + private requestSeq: number = 0 + private speaking: boolean = false + private fallbackTimer: number = -1 + private stateChanged: TtsStateChanged | null = null + + /** 页面注入状态回调（如复位朗读高亮图标）。 */ + setOnStateChanged(cb: TtsStateChanged): void { + this.stateChanged = cb + } + + private async ensureEngine(): Promise { + if (this.engine !== null) { + return + } + const extraParam: Record = { + 'style': 'interaction-broadcast', + 'locate': 'CN', + 'name': 'MeiDaTTS' + } + const createParams: textToSpeech.CreateEngineParams = { + language: 'zh-CN', + person: 0, + online: 1, + extraParams: extraParam + } + this.engine = await textToSpeech.createEngine(createParams) + console.info('MEIDA_TTS engine created') + this.attachListener() + } + + /** + * 朗读入口：切段入队并立即播首段。引擎创建失败会抛 BusinessError， + * 由调用方决定是否提示（自动朗读静默、手动点击提示）。 + */ + async speak(text: string): Promise { + const trimmed: string = text.trim() + if (trimmed.length === 0) { + return + } + await this.ensureEngine() + this.stop() + this.queue = splitSegments(trimmed, SEGMENT_MAX_CHARS) + console.info(`MEIDA_TTS speak start: ${trimmed.length} chars, ${this.queue.length} segment(s)`) + this.speakNext() + } + + /** 串行驱动：每段播完（onComplete type=1）再播下一段。 */ + private speakNext(): void { + if (this.queue.length === 0) { + this.finishSpeaking() + return + } + if (this.engine === null) { + this.finishSpeaking() + return + } + const segment: string = this.queue.shift() ?? '' + if (segment === '') { + this.finishSpeaking() + return + } + this.requestSeq += 1 + this.currentRequestId = `meida-tts-${this.requestSeq}-${Date.now()}` + this.speaking = true + if (this.stateChanged !== null) { + this.stateChanged(true) + } + const speakExtra: Record = { + 'queueMode': 0, + 'speed': 1, + 'volume': 2, + 'pitch': 1, + 'languageContext': 'zh-CN', + 'audioType': 'pcm', + 'soundChannel': 3, + 'playType': 1 + } + const speakParams: textToSpeech.SpeakParams = { + requestId: this.currentRequestId, + extraParams: speakExtra + } + this.armFallback() + try { + this.engine.speak(segment, speakParams) + } catch (e) { + console.error(`MEIDA_TTS segment speak throw: ${String(e)}`) + this.clearFallback() + this.finishSpeaking() + } + } + + /** 停止朗读并清空待播分段。幂等：未在播/引擎为空也安全。 */ + stop(): void { + this.queue = [] + this.clearFallback() + if (this.engine !== null) { + try { + this.engine.stop() + } catch (_e) { + // 空闲态调用 stop 可能抛错：忽略 + } + } + if (this.speaking) { + this.speaking = false + if (this.stateChanged !== null) { + this.stateChanged(false) + } + } + } + + isSpeaking(): boolean { + return this.speaking + } + + release(): void { + this.stop() + const engine = this.engine + this.engine = null + if (engine !== null) { + try { + engine.shutdown() + } catch (_e) { + // 页面销毁路径，失败即随对象丢弃 + } + } + } + + private finishSpeaking(): void { + this.queue = [] + this.clearFallback() + if (this.speaking) { + this.speaking = false + console.info('MEIDA_TTS all segments done') + if (this.stateChanged !== null) { + this.stateChanged(false) + } + } + } + + /** 某段回调丢失时强制推进，避免长文朗读链条卡死。 */ + private armFallback(): void { + this.clearFallback() + this.fallbackTimer = setTimeout(() => { + console.warn(`MEIDA_TTS segment fallback fire, ${this.queue.length} segment(s) left`) + this.speakNext() + }, SEGMENT_FALLBACK_MS) + } + + private clearFallback(): void { + if (this.fallbackTimer >= 0) { + clearTimeout(this.fallbackTimer) + this.fallbackTimer = -1 + } + } + + private attachListener(): void { + const engine = this.engine + if (engine === null) { + return + } + engine.setListener({ + onStart: (requestId: string, response: textToSpeech.StartResponse) => { + console.info(`MEIDA_TTS onStart rid=${requestId}`) + }, + onComplete: (requestId: string, response: textToSpeech.CompleteResponse) => { + if (requestId !== this.currentRequestId) { + return + } + // type=0 合成完成、type=1 播报完成：只等播报完成才推进下一段 + if (response.type === 1) { + console.info(`MEIDA_TTS segment done, ${this.queue.length} segment(s) left`) + this.clearFallback() + this.speakNext() + } + }, + onStop: (requestId: string, response: textToSpeech.StopResponse) => { + console.info(`MEIDA_TTS onStop rid=${requestId}`) + if (requestId !== this.currentRequestId) { + return + } + this.finishSpeaking() + }, + onData: (requestId: string, audio: ArrayBuffer, response: textToSpeech.SynthesisResponse) => { + }, + onError: (requestId: string, errorCode: number, errorMessage: string) => { + console.error(`MEIDA_TTS onError rid=${requestId} code=${errorCode} msg=${errorMessage}`) + if (requestId !== this.currentRequestId) { + return + } + this.finishSpeaking() + } + }) + } +} diff --git a/app/entry/src/main/ets/components/f4/VoiceInputEngine.ets b/app/entry/src/main/ets/components/f4/VoiceInputEngine.ets new file mode 100644 index 0000000..01d3342 --- /dev/null +++ b/app/entry/src/main/ets/components/f4/VoiceInputEngine.ets @@ -0,0 +1,310 @@ +/** + * VoiceInputEngine —— 语音输入封装（CoreSpeechKit speechRecognizer）。 + * + * V2（真机修复版，V1/中间版按住即报 1002200006 busy）： + * - 根因：recognitionMode 0 是「引擎自采麦克风」模式，之前却自建 AudioCapturer + * 喂 writeAudio（那对应 mode 1），双采集方冲突 → 引擎 busy。 + * V2 改为纯 mode 0：startListening 后引擎自己录音，本类不再碰 AudioKit， + * PCM 分块写入层（writeAudio 仅收 640/1280 字节）随之整体删除 + * - busy 自愈：start 前 isBusy() 轮询等待；超时判定卡死 → shutdown 重建 + * （引擎全系统单实例，其他应用占用时也可能 busy） + * - finish 后等 onResult/isLast 或 onComplete 交付结果，5 秒兜底防回调丢失； + * onError 后复位会话状态，坏状态不跨会话存活 + * + * 权限/模型：ohos.permission.MICROPHONE（user_grant，运行时申请）； + * online: 1 = 离线中文引擎（系统内置，无需下载；模拟器通常无此服务，须真机）。 + * 日志 tag MEIDA_ASR，真机排查：hdc shell hilog | grep MEIDA_ASR + */ + +import { speechRecognizer } from '@kit.CoreSpeechKit' +import { abilityAccessCtrl, Permissions } from '@kit.AbilityKit' +import { ContextHolder } from '../../core/ContextHolder' + +const FINISH_FALLBACK_MS: number = 5000 +const BUSY_WAIT_MAX_MS: number = 1500 +const BUSY_WAIT_STEP_MS: number = 100 +const RECREATE_WAIT_MS: number = 500 + +/** 引擎回调（由 ChatPage 注入）。 */ +export interface VoiceInputCallbacks { + onPartial: (text: string) => void + onFinal: (text: string) => void + onError: (message: string) => void +} + +export class VoiceInputEngine { + private engine: speechRecognizer.SpeechRecognitionEngine | null = null + private callbacks: VoiceInputCallbacks | null = null + private sessionId: string = '' + private sessionSeq: number = 0 + private recording: boolean = false + private finalized: boolean = true + private lastText: string = '' + + setCallbacks(cb: VoiceInputCallbacks): void { + this.callbacks = cb + } + + /** 麦克风权限（user_grant）：未授权时弹系统授权弹窗，拒绝/失败返回 false。 */ + static async ensureMicrophonePermission(): Promise { + try { + const atManager = abilityAccessCtrl.createAtManager() + const perms: Array = ['ohos.permission.MICROPHONE'] + const result = await atManager.requestPermissionsFromUser(ContextHolder.get(), perms) + return result.authResults.length > 0 && result.authResults[0] === 0 + } catch (_e) { + return false + } + } + + /** + * 按下开始一次识别。引擎常驻复用（省去每次按下的冷启动延迟）； + * 失败抛 BusinessError，由调用方 catch 后提示。 + */ + async start(): Promise { + if (this.recording) { + return + } + if (this.engine === null) { + this.engine = await VoiceInputEngine.createEngineWithRetry() + this.attachListener() + } + await this.waitEngineIdle() + if (this.engine === null) { + throw new Error('asr engine lost after idle wait') + } + this.sessionSeq += 1 + this.sessionId = `meida-asr-${this.sessionSeq}-${Date.now()}` + this.lastText = '' + this.finalized = false + const audioInfo: speechRecognizer.AudioInfo = { + audioType: 'pcm', + sampleRate: 16000, + soundChannel: 1, + sampleBit: 16 + } + const startExtra: Record = { + 'recognitionMode': 0, + 'maxAudioDuration': 60000 + } + const startParams: speechRecognizer.StartParams = { + sessionId: this.sessionId, + audioInfo: audioInfo, + extraParams: startExtra + } + console.info(`MEIDA_ASR startListening sid=${this.sessionId}`) + try { + this.engine.startListening(startParams) + } catch (e) { + // 会话未建立：立即复位，坏状态不跨会话存活 + console.error(`MEIDA_ASR startListening throw: ${String(e)}`) + this.finalized = true + throw e as Error + } + this.recording = true + } + + /** 松手：结束识别，最终结果经 onFinal 回调（无结果时回调空串）。 */ + finish(): void { + if (!this.recording) { + return + } + this.recording = false + const sid: string = this.sessionId + // 回调丢失兜底：5 秒后强制交付已有文本（期间已开新一轮会话则作废） + setTimeout(() => { + if (!this.finalized && this.sessionId === sid) { + console.warn(`MEIDA_ASR finish fallback fire sid=${sid}`) + this.deliverFinal(this.lastText) + } + }, FINISH_FALLBACK_MS) + const engine = this.engine + if (engine === null) { + this.deliverFinal(this.lastText) + return + } + try { + console.info(`MEIDA_ASR finish sid=${sid}`) + engine.finish(sid) + } catch (_e) { + // 会话已被引擎收尾（VAD 自动完成等）后再 finish 会同步抛错：用已有结果兜底 + this.deliverFinal(this.lastText) + } + } + + /** 上滑取消：丢弃本次识别，不回调结果。 */ + cancel(): void { + if (!this.recording) { + return + } + this.recording = false + this.finalized = true + this.lastText = '' + const engine = this.engine + if (engine !== null) { + try { + console.info(`MEIDA_ASR cancel sid=${this.sessionId}`) + engine.cancel(this.sessionId) + } catch (_e) { + // finalized 已置位，晚到的结果不会上屏 + } + } + } + + /** 页面销毁时释放引擎。 */ + release(): void { + this.cancel() + this.recording = false + this.finalized = true + const engine = this.engine + this.engine = null + if (engine !== null) { + try { + console.info('MEIDA_ASR shutdown on release') + engine.shutdown() + } catch (_e) { + // 页面销毁路径，失败即随对象丢弃 + } + } + } + + /** 建引擎；busy/销毁中（其他应用占用、刚 shutdown）等半秒重试一次。 */ + private static async createEngineWithRetry(): Promise { + const extra: Record = { 'locate': 'CN', 'recognizerMode': 'short' } + const params: speechRecognizer.CreateEngineParams = { + language: 'zh-CN', + online: 1, + extraParams: extra + } + try { + const engine = await speechRecognizer.createEngine(params) + console.info('MEIDA_ASR engine created') + return engine + } catch (e) { + console.error(`MEIDA_ASR createEngine failed, retry in ${RECREATE_WAIT_MS}ms: ${String(e)}`) + await VoiceInputEngine.sleep(RECREATE_WAIT_MS) + const engine = await speechRecognizer.createEngine(params) + console.info('MEIDA_ASR engine created on retry') + return engine + } + } + + /** + * 引擎全系统单实例：上一会话未收尾或其他应用占用时 isBusy。 + * 等待空闲；超过 1.5s 判定卡死 → shutdown 重建（shutdown 后立即 create 会撞 + * 1002200008 销毁中窗口，重建内含半秒等待 + 重试）。 + */ + private async waitEngineIdle(): Promise { + const engine = this.engine + if (engine === null || !VoiceInputEngine.isEngineBusy(engine)) { + return + } + console.warn('MEIDA_ASR engine busy, waiting idle...') + const deadline: number = Date.now() + BUSY_WAIT_MAX_MS + while (VoiceInputEngine.isEngineBusy(engine)) { + if (Date.now() >= deadline) { + console.warn('MEIDA_ASR engine stuck busy, recreate') + try { + engine.shutdown() + } catch (_e) { + // 忙/销毁中 shutdown 抛错：直接走重建 + } + this.engine = await VoiceInputEngine.createEngineWithRetry() + this.attachListener() + return + } + await VoiceInputEngine.sleep(BUSY_WAIT_STEP_MS) + } + } + + private static isEngineBusy(engine: speechRecognizer.SpeechRecognitionEngine): boolean { + try { + return engine.isBusy() + } catch (_e) { + return false + } + } + + private static sleep(ms: number): Promise { + return new Promise((resolve: () => void) => { + setTimeout(() => { + resolve() + }, ms) + }) + } + + private attachListener(): void { + const engine = this.engine + if (engine === null) { + return + } + engine.setListener({ + onStart: (sessionId: string, eventMessage: string) => { + console.info(`MEIDA_ASR onStart sid=${sessionId}`) + }, + onEvent: (sessionId: string, eventCode: number, eventMessage: string) => { + console.info(`MEIDA_ASR onEvent sid=${sessionId} code=${eventCode}`) + }, + onResult: (sessionId: string, result: speechRecognizer.SpeechRecognitionResult) => { + if (result.result !== '') { + this.lastText = result.result + } + if (this.finalized || sessionId !== this.sessionId) { + return + } + if (this.callbacks !== null) { + this.callbacks.onPartial(this.lastText) + } + if (result.isLast) { + console.info(`MEIDA_ASR final(isLast) len=${this.lastText.length}`) + this.deliverFinal(this.lastText) + } + }, + onComplete: (sessionId: string, eventMessage: string) => { + console.info(`MEIDA_ASR onComplete sid=${sessionId}`) + if (sessionId !== this.sessionId) { + return + } + // 手指仍按着但引擎已收尾（VAD 判定说完/达到时长上限）：提前交付， + // 松手时 finish() 因 recording=false 空转，不会二次发送 + this.recording = false + this.deliverFinal(this.lastText) + }, + onError: (sessionId: string, errorCode: number, errorMessage: string) => { + console.error(`MEIDA_ASR onError sid=${sessionId} code=${errorCode} msg=${errorMessage}`) + if (sessionId !== this.sessionId) { + return + } + this.recording = false + const alreadyFinal: boolean = this.finalized + this.finalized = true + if (!alreadyFinal && this.callbacks !== null) { + this.callbacks.onError(VoiceInputEngine.friendlyError(errorCode, errorMessage)) + } + } + }) + } + + private deliverFinal(text: string): void { + if (this.finalized) { + return + } + this.finalized = true + if (this.callbacks !== null) { + this.callbacks.onFinal(text) + } + } + + private static friendlyError(code: number, msg: string): string { + if (code === 1002200006) { + return '语音引擎正忙，请稍等 1 秒再试' + } + if (code === 1002200001) { + return '当前设备不支持语音识别（缺少离线语音服务，请用真机测试）' + } + if (code === 1002200008) { + return '语音引擎正在重启，请再试一次' + } + return `语音识别失败（${code}）：${msg}` + } +} diff --git a/app/entry/src/main/ets/pages/ChatPage.ets b/app/entry/src/main/ets/pages/ChatPage.ets index 5f56eb3..dcb14b2 100644 --- a/app/entry/src/main/ets/pages/ChatPage.ets +++ b/app/entry/src/main/ets/pages/ChatPage.ets @@ -1,21 +1,9 @@ -/** - * ChatPage —— AI 穿搭对话页（F4）。 - * - * 任务契约：docs/开发任务包/F4-AI对话页.md - * - 消息列表 @State ChatMsg[]：本地内存态，退出即失（V1 不做会话持久化， - * 后端 ag_conversation 会话落库属后续任务） - * - 发送流程：push 用户消息 → AI 侧"正在思考…"占位 → POST /api/v1/agent/chat - * → 成功替换占位（meta = "模型 · 耗时"）并更新顶部配额薄条 quota_remaining - * - 失败态：只 catch ApiError（40301 额度用尽 / 60003 今日服务上限 / 50000 网络错 - * 与未知异常，均已在 ApiClient 内包成 ApiError），统一在 AI 气泡位展示 - * message + hint 次行小字，"正在思考…"占位必撤 - * - 发送中禁用输入防连点；空消息不发送；首次进入插入本地静态欢迎语（不调接口） - */ import { router } from '@kit.ArkUI' import { ApiClient } from '../core/ApiClient' import { ApiError } from '../core/ApiTypes' +import { VoiceInputEngine } from '../components/f4/VoiceInputEngine' +import { TtsEngine } from '../components/f4/TtsEngine' -/** wire 响应：POST /api/v1/agent/chat 的 data.usage（字段名对齐后端 JSON） */ interface ChatUsage { tokens_in: number tokens_out: number @@ -24,7 +12,6 @@ interface ChatUsage { cost_cents: number } -/** wire 响应：POST /api/v1/agent/chat 的 data（字段名对齐后端 JSON） */ interface ChatResp { reply: string provider: string @@ -36,7 +23,6 @@ interface ChatResp { quota_remaining: number } -/** 列表项：meta 存 AI 消息底部小字（成功为"模型 · 耗时"，失败为 e.hint） */ interface ChatMsg { role: 'user' | 'ai' text: string @@ -48,12 +34,14 @@ const ROLE_AI = 'ai' const THINKING_TEXT = '正在思考…' const WELCOME_TEXT = '你好，我是你的穿搭顾问，可以问我面试怎么穿、约会怎么搭～' -/** 单条消息气泡：用户右对齐主色底，AI 左对齐白底；meta 为底部小字 */ @Component struct MessageBubble { @Prop role: string = ROLE_AI @Prop text: string = '' @Prop meta: string = '' + @Prop isSpeaking: boolean = false + @Prop msgIdx: number = -1 + speakHandler: (idx: number) => void = () => {} private isUser(): boolean { return this.role === ROLE_USER @@ -66,12 +54,25 @@ struct MessageBubble { .fontSize(15) .lineHeight(22) .fontColor(this.isUser() ? '#FFFFFF' : '#333333') - if (this.meta.length > 0) { - Text(this.meta) - .fontSize(11) - .margin({ top: 4 }) - .fontColor(this.isUser() ? '#FFDCD8' : '#999999') + Row() { + if (this.meta.length > 0) { + Text(this.meta) + .fontSize(11) + .fontColor(this.isUser() ? '#FFDCD8' : '#999999') + } + if (!this.isUser()) { + Blank() + Text(this.isSpeaking ? '🔊' : '🔈') + .fontSize(14) + .fontColor('#999999') + .margin({ left: 4 }) + .onClick(() => { + this.speakHandler(this.msgIdx) + }) + } } + .width('100%') + .margin({ top: 4 }) } .alignItems(HorizontalAlign.Start) .padding({ left: 12, right: 12, top: 10, bottom: 10 }) @@ -88,20 +89,81 @@ struct MessageBubble { @Entry @Component struct ChatPage { - /** 本地内存态：不做会话持久化（后端 ag_conversation 属后续任务） */ @State messages: ChatMsg[] = [] @State inputText: string = '' @State sending: boolean = false - /** -1 = 尚未从任何响应拿到配额 → 薄条不显示 */ @State quotaRemaining: number = -1 + @State voiceMode: boolean = false + @State holdState: string = '' + @State partialText: string = '' + @State voiceHint: string = '' + @State speakingMsgIdx: number = -1 + private fingerDown: boolean = false + private holdStartY: number = 0 + private voiceEngine: VoiceInputEngine = new VoiceInputEngine() + private ttsEngine: TtsEngine = new TtsEngine() private scroller: Scroller = new Scroller() + private autoSpeak: boolean = true aboutToAppear(): void { const welcome: ChatMsg = { role: ROLE_AI, text: WELCOME_TEXT } this.messages.push(welcome) + // 朗读结束（播完/被停/出错）→ 复位消息气泡的 🔊 高亮 + this.ttsEngine.setOnStateChanged((speaking: boolean) => { + if (!speaking) { + this.speakingMsgIdx = -1 + } + }) + this.voiceEngine.setCallbacks({ + onPartial: (text: string): void => { + this.partialText = text + }, + onFinal: (text: string): void => { + const finalText: string = text.trim() + if (finalText.length === 0) { + this.showVoiceHint('没听清，请再按住说一次') + return + } + this.inputText = finalText + this.send() + }, + onError: (message: string): void => { + this.showVoiceHint(message) + } + }) + } + + aboutToDisappear(): void { + this.voiceEngine.release() + this.ttsEngine.release() + } + + private async speakText(text: string, msgIdx: number, manual: boolean): Promise { + if (this.speakingMsgIdx === msgIdx && this.ttsEngine.isSpeaking()) { + this.ttsEngine.stop() + this.speakingMsgIdx = -1 + return + } + this.speakingMsgIdx = msgIdx + try { + await this.ttsEngine.speak(text) + } catch (e) { + this.speakingMsgIdx = -1 + console.error(`MEIDA_TTS speak failed: ${String(e)}`) + if (manual) { + this.showVoiceHint('语音朗读启动失败（模拟器无语音服务，请用真机）') + } + } + } + + private onSpeakToggle(idx: number): void { + if (idx < 0 || idx >= this.messages.length) { + return + } + const msg: ChatMsg = this.messages[idx] + this.speakText(msg.text, idx, true) } - /** 发送入口（回车/按钮共用）：空消息不发，发送中防连点 */ private send(): void { const text: string = this.inputText.trim() if (text.length === 0 || this.sending) { @@ -113,6 +175,8 @@ struct ChatPage { private async doSend(text: string): Promise { this.inputText = '' this.sending = true + this.ttsEngine.stop() + this.speakingMsgIdx = -1 const userMsg: ChatMsg = { role: ROLE_USER, text: text } this.messages.push(userMsg) const thinkingMsg: ChatMsg = { role: ROLE_AI, text: THINKING_TEXT } @@ -124,9 +188,11 @@ struct ChatPage { const meta: string = `${data.model} · ${data.usage.latency_ms}ms` this.replaceThinking(data.reply, meta) this.quotaRemaining = data.quota_remaining + if (this.autoSpeak) { + const aiIdx: number = this.messages.length - 1 + this.speakText(data.reply, aiIdx, false) + } } catch (e) { - // 只 catch ApiError：40301 / 60003 / 网络错（50000）均包成 ApiError， - // 统一气泡位展示 message + hint；占位在 replaceThinking 里必撤 if (e instanceof ApiError) { this.replaceThinking(e.message, e.hint ?? '') } @@ -135,7 +201,6 @@ struct ChatPage { } } - /** 把末尾"正在思考…"占位替换为结果；占位缺失时兜底补一条，确保占位一定被撤掉 */ private replaceThinking(text: string, meta: string): void { const replaced: ChatMsg = { role: ROLE_AI, text: text, meta: meta } const idx: number = this.messages.length - 1 @@ -153,91 +218,252 @@ struct ChatPage { }, 50) } + private showVoiceHint(message: string): void { + this.voiceHint = message + setTimeout(() => { + if (this.voiceHint === message) { + this.voiceHint = '' + } + }, 3000) + } + + private async startHold(): Promise { + this.fingerDown = true + this.partialText = '' + this.ttsEngine.stop() + this.speakingMsgIdx = -1 + const granted = await VoiceInputEngine.ensureMicrophonePermission() + if (!this.fingerDown) { + return + } + if (!granted) { + this.showVoiceHint('未获得麦克风权限，无法语音输入') + return + } + try { + await this.voiceEngine.start() + } catch (e) { + console.error(`MEIDA_ASR startHold failed: ${String(e)}`) + this.showVoiceHint('语音引擎启动失败，请重试') + return + } + if (!this.fingerDown) { + this.voiceEngine.cancel() + return + } + this.holdState = 'rec' + } + + private endHold(wantCancel: boolean): void { + this.fingerDown = false + if (this.holdState === '') { + return + } + this.holdState = '' + this.partialText = '' + if (wantCancel) { + this.voiceEngine.cancel() + } else { + this.voiceEngine.finish() + } + } + + private onHoldTouch(event: TouchEvent): void { + if (event.type === TouchType.Down) { + if (event.touches.length === 0) { + return + } + this.holdStartY = event.touches[0].displayY + this.startHold() + } else if (event.type === TouchType.Move) { + if (this.holdState === '' || event.touches.length === 0) { + return + } + const dy: number = this.holdStartY - event.touches[0].displayY + this.holdState = dy > 80 ? 'cancel' : 'rec' + } else if (event.type === TouchType.Up) { + this.endHold(this.holdState === 'cancel') + } else if (event.type === TouchType.Cancel) { + this.endHold(true) + } + } + + @Builder + VoiceOverlay() { + Column() + .width('100%') + .height('100%') + .backgroundColor('#66000000') + + Column({ space: 10 }) { + Text(this.holdState === 'cancel' ? '✕' : '🎙') + .fontSize(40) + .fontColor(Color.White) + Text(this.holdState === 'cancel' ? '松开 取消' : '松开 发送') + .fontSize(15) + .fontWeight(FontWeight.Medium) + .fontColor(Color.White) + Text(this.holdState === 'cancel' ? '' : (this.partialText !== '' ? this.partialText : '正在聆听…')) + .fontSize(13) + .fontColor('#DDDDDD') + .maxLines(2) + .textAlign(TextAlign.Center) + Text('↑ 上滑取消') + .fontSize(11) + .fontColor('#999999') + } + .width('72%') + .padding({ top: 24, bottom: 24, left: 16, right: 16 }) + .borderRadius(20) + .backgroundColor('#CC1F1F1F') + .alignItems(HorizontalAlign.Center) + } + build() { - Column() { - // 顶部栏：返回 + 标题 - Row() { - Text('‹') - .fontSize(26) - .fontColor('#333333') - .width(32) - .height(32) - .textAlign(TextAlign.Center) - .onClick(() => { - router.back() - }) - Text('AI 穿搭顾问') - .fontSize(18) - .fontWeight(FontWeight.Bold) - .fontColor('#1F1F1F') - .margin({ left: 4 }) - Blank() - } - .width('100%') - .height(56) - .padding({ left: 12, right: 12 }) - .backgroundColor('#FFFFFF') - - // 配额薄条：拿到首个响应后展示"今日剩余 N 次" - if (this.quotaRemaining >= 0) { - Text(`今日剩余 ${this.quotaRemaining} 次`) - .fontSize(12) - .fontColor('#B3554D') - .backgroundColor('#FBEAE8') - .width('100%') - .textAlign(TextAlign.Center) - .padding({ top: 4, bottom: 4 }) - } - - // 消息列表（key 带内容：占位被替换时 key 变化触发该项重渲染） - List({ scroller: this.scroller }) { - ForEach(this.messages, (msg: ChatMsg, idx: number) => { - ListItem() { - MessageBubble({ - role: msg.role, - text: msg.text, - meta: msg.meta ?? '' + Stack() { + Column() { + Row() { + Text('‹') + .fontSize(26) + .fontColor('#333333') + .width(32) + .height(32) + .textAlign(TextAlign.Center) + .onClick(() => { + router.back() }) - } - }, (msg: ChatMsg, idx: number) => `${idx}_${msg.role}_${msg.text}`) - } - .layoutWeight(1) - .width('100%') - .backgroundColor('#F5F5F7') - .scrollBar(BarState.Off) + Text('AI 穿搭顾问') + .fontSize(18) + .fontWeight(FontWeight.Bold) + .fontColor('#1F1F1F') + .margin({ left: 4 }) + Blank() + Text(this.autoSpeak ? '🔊' : '🔇') + .fontSize(18) + .fontColor('#666666') + .width(32) + .height(32) + .textAlign(TextAlign.Center) + .onClick(() => { + this.autoSpeak = !this.autoSpeak + if (!this.autoSpeak) { + this.ttsEngine.stop() + this.speakingMsgIdx = -1 + } + }) + } + .width('100%') + .height(56) + .padding({ left: 12, right: 12 }) + .backgroundColor('#FFFFFF') - // 底部输入栏：发送中禁用输入框与按钮防连点 - Row() { - TextInput({ - text: this.inputText, - placeholder: this.sending ? 'AI 正在回复…' : '聊聊穿搭，比如：面试怎么穿？' - }) - .layoutWeight(1) - .height(40) - .fontSize(14) - .backgroundColor('#F2F2F4') - .borderRadius(8) - .enabled(!this.sending) - .enterKeyType(EnterKeyType.Send) - .onChange((value: string) => { - this.inputText = value - }) - .onSubmit(() => { - this.send() - }) - Button(this.sending ? '回复中' : '发送') - .fontSize(14) - .height(36) - .margin({ left: 8 }) - .backgroundColor(this.sending ? '#C9C9CD' : '#D9605A') - .enabled(!this.sending) - .onClick(() => { - this.send() - }) + if (this.quotaRemaining >= 0) { + Text(`今日剩余 ${this.quotaRemaining} 次`) + .fontSize(12) + .fontColor('#B3554D') + .backgroundColor('#FBEAE8') + .width('100%') + .textAlign(TextAlign.Center) + .padding({ top: 4, bottom: 4 }) + } + + List({ scroller: this.scroller }) { + ForEach(this.messages, (msg: ChatMsg, idx: number) => { + ListItem() { + MessageBubble({ + role: msg.role, + text: msg.text, + meta: msg.meta ?? '', + isSpeaking: this.speakingMsgIdx === idx, + msgIdx: idx as number, + speakHandler: (i: number) => { + this.onSpeakToggle(i) + } + }) + } + }, (msg: ChatMsg, idx: number) => `${idx}_${msg.role}_${msg.text}`) + } + .layoutWeight(1) + .width('100%') + .backgroundColor('#F5F5F7') + .scrollBar(BarState.Off) + + if (this.voiceHint !== '') { + Text(this.voiceHint) + .fontSize(12) + .fontColor('#C0392B') + .backgroundColor('#FDECEC') + .width('100%') + .textAlign(TextAlign.Center) + .padding({ top: 4, bottom: 4 }) + } + + Row() { + Text(this.voiceMode ? '⌨' : '🎤') + .fontSize(20) + .fontColor(this.voiceMode ? '#D9605A' : '#666666') + .width(36) + .height(40) + .textAlign(TextAlign.Center) + .align(Alignment.Center) + .onClick(() => { + if (this.holdState !== '') { + this.endHold(true) + } + this.voiceMode = !this.voiceMode + }) + + if (this.voiceMode) { + Text(this.holdState === 'cancel' ? '松开 取消' : (this.holdState === 'rec' ? '松开 发送' : '按住 说话')) + .layoutWeight(1) + .height(40) + .fontSize(14) + .textAlign(TextAlign.Center) + .fontColor(this.holdState === 'cancel' ? '#C0392B' : '#333333') + .backgroundColor(this.holdState === 'cancel' ? '#FDECEC' : '#F2F2F4') + .borderRadius(8) + .onTouch((event: TouchEvent) => { + this.onHoldTouch(event) + }) + } else { + TextInput({ + text: this.inputText, + placeholder: this.sending ? 'AI 正在回复…' : '聊聊穿搭，比如：面试怎么穿？' + }) + .layoutWeight(1) + .height(40) + .fontSize(14) + .backgroundColor('#F2F2F4') + .borderRadius(8) + .enabled(!this.sending) + .enterKeyType(EnterKeyType.Send) + .onChange((value: string) => { + this.inputText = value + }) + .onSubmit(() => { + this.send() + }) + Button(this.sending ? '回复中' : '发送') + .fontSize(14) + .height(36) + .margin({ left: 8 }) + .backgroundColor(this.sending ? '#C9C9CD' : '#D9605A') + .enabled(!this.sending) + .onClick(() => { + this.send() + }) + } + } + .width('100%') + .padding({ left: 12, right: 12, top: 8, bottom: 8 }) + .backgroundColor('#FFFFFF') } .width('100%') - .padding({ left: 12, right: 12, top: 8, bottom: 8 }) - .backgroundColor('#FFFFFF') + .height('100%') + + if (this.holdState !== '') { + this.VoiceOverlay() + } } .width('100%') .height('100%') diff --git a/app/entry/src/main/module.json5 b/app/entry/src/main/module.json5 index f658a45..b705182 100644 --- a/app/entry/src/main/module.json5 +++ b/app/entry/src/main/module.json5 @@ -9,7 +9,17 @@ ], "deliveryWithInstall": true, "requestPermissions": [ - { "name": "ohos.permission.INTERNET" } + { "name": "ohos.permission.INTERNET" }, + { + "name": "ohos.permission.MICROPHONE", + "reason": "$string:microphone_permission_reason", + "usedScene": { + "abilities": [ + "EntryAbility" + ], + "when": "inuse" + } + } ], "installationFree": false, "pages": "$profile:main_pages", diff --git a/app/entry/src/main/resources/base/element/string.json b/app/entry/src/main/resources/base/element/string.json index f945955..0cad1cb 100644 --- a/app/entry/src/main/resources/base/element/string.json +++ b/app/entry/src/main/resources/base/element/string.json @@ -10,7 +10,11 @@ }, { "name": "EntryAbility_label", - "value": "label" + "value": "美搭" + }, + { + "name": "microphone_permission_reason", + "value": "语音输入需要使用麦克风" } ] -} \ No newline at end of file +}