spike(llm): T18 端侧小模型——MNN native 骨架 + 局域网模型服务
- cpp/：liblocalllm.so NAPI 骨架（loadModel/chat/benchmark/release），预置 libMNN.so 3.6.1 鸿蒙包（已验证含 createLLM 符号；.gitignore 挡预编译库与 models/，获取方式见任务文档） - LocalLlm.ets：native 调用的 ets 封装；LlmTestPage：模型从局域网服务器下载→加载→首 token/速度 benchmark - scripts/serve-models.mjs：开发期局域网模型文件服务器（Qwen3-0.6B-MNN 456MB，SHA256 与 ModelScope 官方清单逐文件核对） - build-profile.json5 接 externalNativeOptions（arm64-v8a）；ProfileContent 加「端侧LLM调试」入口（内部调试用） - docs/开发任务包/T18-端侧小模型SPIKE.md：目标/验证项/回退线；真机 benchmark 数据回来后写 T18-REPORT 三问三答
This commit is contained in:
@@ -432,6 +432,19 @@ export struct ProfileContent {
|
||||
.backgroundColor('#FFFFFF')
|
||||
.alignItems(VerticalAlign.Center)
|
||||
|
||||
// T18 SPIKE 调试入口（内部用，正式版移除）
|
||||
Text('端侧LLM调试')
|
||||
.fontSize(13)
|
||||
.fontColor('#9A9A9A')
|
||||
.width('100%')
|
||||
.padding(12)
|
||||
.borderRadius(12)
|
||||
.backgroundColor('#FFFFFF')
|
||||
.textAlign(TextAlign.Center)
|
||||
.onClick(() => {
|
||||
router.pushUrl({ url: 'pages/LlmTestPage' })
|
||||
})
|
||||
|
||||
ErrorBanner({
|
||||
message: this.bannerMessage,
|
||||
hint: this.bannerHint,
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* LocalLlm：端侧小模型管理（T18 端侧小模型 SPIKE）。
|
||||
*
|
||||
* 职责：模型文件的下载/校验/加载状态机 + native（liblocalllm.so）调用的 ArkTS 封装。
|
||||
* 调试期模型从局域网服务器下载（scripts/serve-models.mjs）；
|
||||
* 商用阶段把 baseUrl 换成 COS/CDN 域名即可，代码不变。
|
||||
*
|
||||
* 纪律：
|
||||
* - 模型放 filesDir（cacheDir 有系统配额会被清理，不能放）；
|
||||
* - SPIKE 只校验文件大小，SHA256 校验留待 P1（crypto 全量哈希 450MB 太慢，需流式）；
|
||||
* - 推理只在前台进行（无计算类长时任务，退后台进程会被冻结）。
|
||||
*/
|
||||
|
||||
import localllm from 'liblocalllm.so'
|
||||
import { ContextHolder } from './ContextHolder'
|
||||
import request from '@ohos.request'
|
||||
import fs from '@ohos.file.fs'
|
||||
|
||||
export interface ManifestFile {
|
||||
name: string
|
||||
size: number
|
||||
sha256: string
|
||||
}
|
||||
|
||||
export interface ModelManifest {
|
||||
version: string
|
||||
model: string
|
||||
files: ManifestFile[]
|
||||
}
|
||||
|
||||
export interface GenerateResult {
|
||||
text: string
|
||||
first_token_ms: number
|
||||
total_ms: number
|
||||
chars: number
|
||||
error?: string
|
||||
}
|
||||
|
||||
export type ModelState = 'none' | 'downloading' | 'ready' | 'loaded' | 'error'
|
||||
|
||||
const MODEL_DIR_NAME: string = 'qwen3-0.6b-mnn'
|
||||
|
||||
export class LocalLlm {
|
||||
/** 调试期默认地址由测试页传入（形如 http://192.168.x.x:8080），这里不硬编码。 */
|
||||
|
||||
static modelDir(): string {
|
||||
return `${ContextHolder.get().filesDir}/models/${MODEL_DIR_NAME}`
|
||||
}
|
||||
|
||||
/** 本地模型文件是否齐全（按 manifest 尺寸校验）。 */
|
||||
static checkFiles(manifest: ModelManifest): boolean {
|
||||
const dir: string = LocalLlm.modelDir()
|
||||
for (const f of manifest.files) {
|
||||
const path: string = `${dir}/${f.name}`
|
||||
if (!fs.accessSync(path)) {
|
||||
return false
|
||||
}
|
||||
try {
|
||||
const st = fs.statSync(path)
|
||||
if (Number(st.size) !== f.size) {
|
||||
return false
|
||||
}
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/** 从 baseUrl 逐个下载模型文件（整包下载，断点续传留待商用阶段）。 */
|
||||
static async downloadAll(baseUrl: string, manifest: ModelManifest,
|
||||
onProgress?: (fileName: string, index: number, total: number) => void): Promise<void> {
|
||||
const dir: string = LocalLlm.modelDir()
|
||||
if (!fs.accessSync(dir)) {
|
||||
fs.mkdirSync(dir, true)
|
||||
}
|
||||
const ctx: Context = ContextHolder.get()
|
||||
for (let i = 0; i < manifest.files.length; i++) {
|
||||
const f: ManifestFile = manifest.files[i]
|
||||
onProgress?.(f.name, i + 1, manifest.files.length)
|
||||
const target: string = `${dir}/${f.name}`
|
||||
if (fs.accessSync(target) && Number(fs.statSync(target).size) === f.size) {
|
||||
continue // 已下好，跳过（断点续传的乞丐版：按文件粒度续）
|
||||
}
|
||||
const task = await request.downloadFile(ctx, {
|
||||
url: `${baseUrl}/${MODEL_DIR_NAME}/${f.name}`,
|
||||
filePath: target
|
||||
})
|
||||
await LocalLlm.waitTask(task)
|
||||
if (Number(fs.statSync(target).size) !== f.size) {
|
||||
throw new Error(`文件尺寸不符: ${f.name}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 加载模型到 native（耗时，走 async work 线程）。返回空串=成功。 */
|
||||
static async load(): Promise<string> {
|
||||
const dir: string = LocalLlm.modelDir()
|
||||
const tmp: string = `${dir}/.tmp`
|
||||
if (!fs.accessSync(tmp)) {
|
||||
fs.mkdirSync(tmp, true)
|
||||
}
|
||||
return localllm.loadModel(dir)
|
||||
}
|
||||
|
||||
static isLoaded(): boolean {
|
||||
return localllm.isLoaded()
|
||||
}
|
||||
|
||||
static unload(): void {
|
||||
localllm.unloadModel()
|
||||
}
|
||||
|
||||
/** 生成。native 侧已完成计时；解析失败视为 error。 */
|
||||
static async generate(prompt: string, maxNewTokens: number): Promise<GenerateResult> {
|
||||
const raw: string = await localllm.generate(prompt, maxNewTokens)
|
||||
try {
|
||||
return JSON.parse(raw) as GenerateResult
|
||||
} catch {
|
||||
return { text: '', first_token_ms: -1, total_ms: -1, chars: 0, error: `bad json: ${raw}` }
|
||||
}
|
||||
}
|
||||
|
||||
private static waitTask(task: request.DownloadTask): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
task.on('complete', () => resolve())
|
||||
task.on('fail', (err: number) => reject(new Error(`download fail: ${err}`)))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
/**
|
||||
* LlmTestPage：T18 端侧小模型 SPIKE 的调试/基准页。
|
||||
*
|
||||
* 用途（仅限内部调试，不对真实用户开放）：
|
||||
* 1. 从局域网服务器下载模型 → 加载 → 跑 benchmark（首 token 延迟 / 总耗时 / 字符速度）；
|
||||
* 2. 意图分类 smoke（固定 prompt + 固定样例，人工核对输出 JSON）；
|
||||
* 3. hidebug 内存采样（加载前后 native 堆对比，验收口径：峰值 <1GB）。
|
||||
*
|
||||
* 验收门槛见 docs/开发任务包/T18-端侧小模型SPIKE.md。
|
||||
*/
|
||||
|
||||
import { LocalLlm, GenerateResult, ModelManifest } from '../core/LocalLlm'
|
||||
import hidebug from '@ohos.hidebug'
|
||||
import http from '@ohos.net.http'
|
||||
|
||||
const DEFAULT_BASE: string = 'http://192.168.3.19:8080' // 开发机局域网 IP（模型服务器）
|
||||
const MODEL_SUBPATH: string = 'qwen3-0.6b-mnn'
|
||||
|
||||
/** 意图分类 smoke 用的系统 prompt（标签集对齐《穿搭Agent-需求文档》§3.2，P2 的"穿搭评价"先占位）。 */
|
||||
const INTENT_PROMPT: string =
|
||||
'你是穿搭助手的意图分类器。把用户输入分类到以下之一：' +
|
||||
'场景咨询/单品搭配/风格建议/色彩咨询/身材修饰/天气适配/发型建议/穿搭评价/闲聊/越界。' +
|
||||
'只输出 JSON：{"intent":"<类别>"}，不要输出任何其它内容。用户输入：'
|
||||
|
||||
const INTENT_CASES: string[] = [
|
||||
'面试怎么穿？',
|
||||
'白衬衫配什么裤子？',
|
||||
'我这种矮胖身材是不是穿什么都难看？顺便告诉我怎么减肥',
|
||||
'你好呀',
|
||||
]
|
||||
|
||||
const BENCH_PROMPTS: string[] = [
|
||||
'你好，简单介绍一下你自己，一句话以内。',
|
||||
INTENT_PROMPT + '约会穿什么？',
|
||||
'夏天通勤穿搭有什么通用原则？100字以内回答。',
|
||||
]
|
||||
|
||||
@Entry
|
||||
@Component
|
||||
struct LlmTestPage {
|
||||
@State baseUrl: string = DEFAULT_BASE
|
||||
@State logText: string = '就绪。步骤：① 填开发机IP ② 下载模型 ③ 加载模型 ④ 跑 Benchmark\n'
|
||||
@State busy: boolean = false
|
||||
|
||||
private log(line: string): void {
|
||||
this.logText += `${line}\n`
|
||||
}
|
||||
|
||||
private heapMb(): string {
|
||||
const bytes: bigint = hidebug.getNativeHeapSize()
|
||||
return (Number(bytes) / 1024 / 1024).toFixed(0)
|
||||
}
|
||||
|
||||
private async fetchManifest(): Promise<ModelManifest> {
|
||||
const client = http.createHttp()
|
||||
try {
|
||||
const resp = await client.request(`${this.baseUrl}/${MODEL_SUBPATH}/manifest.json`,
|
||||
{ connectTimeout: 5000, readTimeout: 10000 })
|
||||
if (resp.responseCode !== 200) {
|
||||
throw new Error(`manifest http ${resp.responseCode}`)
|
||||
}
|
||||
return JSON.parse(resp.result as string) as ModelManifest
|
||||
} finally {
|
||||
client.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
private async onDownload(): Promise<void> {
|
||||
this.busy = true
|
||||
try {
|
||||
this.log(`连接 ${this.baseUrl} ...`)
|
||||
const manifest = await this.fetchManifest()
|
||||
this.log(`manifest v${manifest.version}，${manifest.files.length} 个文件`)
|
||||
if (LocalLlm.checkFiles(manifest)) {
|
||||
this.log('本地文件已齐全，跳过下载')
|
||||
return
|
||||
}
|
||||
await LocalLlm.downloadAll(this.baseUrl, manifest, (name: string, idx: number, total: number) => {
|
||||
this.log(`[${idx}/${total}] ${name}`)
|
||||
})
|
||||
if (!LocalLlm.checkFiles(manifest)) {
|
||||
throw new Error('下载后校验失败')
|
||||
}
|
||||
this.log('下载完成，校验通过')
|
||||
} catch (e) {
|
||||
this.log(`下载失败: ${(e as Error).message}`)
|
||||
} finally {
|
||||
this.busy = false
|
||||
}
|
||||
}
|
||||
|
||||
private async onLoad(): Promise<void> {
|
||||
this.busy = true
|
||||
try {
|
||||
this.log(`加载前 native 堆: ${this.heapMb()}MB`)
|
||||
const t0: number = Date.now()
|
||||
const err: string = await LocalLlm.load()
|
||||
if (err.length > 0) {
|
||||
this.log(`加载失败: ${err}`)
|
||||
return
|
||||
}
|
||||
this.log(`加载完成，耗时 ${Date.now() - t0}ms，加载后 native 堆: ${this.heapMb()}MB`)
|
||||
} catch (e) {
|
||||
this.log(`加载异常: ${(e as Error).message}`)
|
||||
} finally {
|
||||
this.busy = false
|
||||
}
|
||||
}
|
||||
|
||||
private async runOne(prompt: string, maxNewTokens: number): Promise<void> {
|
||||
const r: GenerateResult = await LocalLlm.generate(prompt, maxNewTokens)
|
||||
if (r.error !== undefined) {
|
||||
this.log(` 生成失败: ${r.error}`)
|
||||
return
|
||||
}
|
||||
const speed: string = r.total_ms > 0 ? (r.chars * 1000 / r.total_ms).toFixed(1) : '?'
|
||||
this.log(` 首token ${r.first_token_ms}ms / 总 ${r.total_ms}ms / ${r.chars}字 (${speed}字/s) / 堆 ${this.heapMb()}MB`)
|
||||
this.log(` 输出: ${r.text.substring(0, 120)}`)
|
||||
}
|
||||
|
||||
private async onBench(): Promise<void> {
|
||||
if (!LocalLlm.isLoaded()) {
|
||||
this.log('请先加载模型')
|
||||
return
|
||||
}
|
||||
this.busy = true
|
||||
try {
|
||||
for (let i = 0; i < BENCH_PROMPTS.length; i++) {
|
||||
this.log(`[bench ${i + 1}/${BENCH_PROMPTS.length}] ${BENCH_PROMPTS[i].substring(0, 20)}...`)
|
||||
await this.runOne(BENCH_PROMPTS[i], 128) // 预热
|
||||
await this.runOne(BENCH_PROMPTS[i], 128)
|
||||
await this.runOne(BENCH_PROMPTS[i], 128)
|
||||
}
|
||||
this.log('benchmark 完成')
|
||||
} finally {
|
||||
this.busy = false
|
||||
}
|
||||
}
|
||||
|
||||
private async onIntentSmoke(): Promise<void> {
|
||||
if (!LocalLlm.isLoaded()) {
|
||||
this.log('请先加载模型')
|
||||
return
|
||||
}
|
||||
this.busy = true
|
||||
try {
|
||||
for (const c of INTENT_CASES) {
|
||||
this.log(`[意图] 输入: ${c}`)
|
||||
await this.runOne(INTENT_PROMPT + c, 32)
|
||||
}
|
||||
} finally {
|
||||
this.busy = false
|
||||
}
|
||||
}
|
||||
|
||||
build() {
|
||||
Column({ space: 12 }) {
|
||||
Text('端侧小模型调试（T18 SPIKE）').fontSize(18).fontWeight(FontWeight.Bold)
|
||||
TextInput({ text: this.baseUrl, placeholder: '开发机地址，如 http://192.168.1.100:8080' })
|
||||
.onChange((v: string) => { this.baseUrl = v })
|
||||
Row({ space: 8 }) {
|
||||
Button('① 下载模型').enabled(!this.busy).onClick(() => { this.onDownload() })
|
||||
Button('② 加载模型').enabled(!this.busy).onClick(() => { this.onLoad() })
|
||||
}
|
||||
Row({ space: 8 }) {
|
||||
Button('③ Benchmark').enabled(!this.busy).onClick(() => { this.onBench() })
|
||||
Button('④ 意图smoke').enabled(!this.busy).onClick(() => { this.onIntentSmoke() })
|
||||
Button('卸载').enabled(!this.busy).onClick(() => {
|
||||
LocalLlm.unload()
|
||||
this.log(`已卸载，native 堆: ${this.heapMb()}MB`)
|
||||
})
|
||||
}
|
||||
Scroll() {
|
||||
Text(this.logText).fontSize(12).fontFamily('monospace')
|
||||
}
|
||||
.layoutWeight(1)
|
||||
.align(Alignment.TopStart)
|
||||
.scrollBar(BarState.Auto)
|
||||
}
|
||||
.padding(16)
|
||||
.width('100%')
|
||||
.height('100%')
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user