spike(llm): T18 端侧小模型——MNN native 骨架 + 局域网模型服务

- cpp/：liblocalllm.so NAPI 骨架（loadModel/chat/benchmark/release），预置 libMNN.so 3.6.1 鸿蒙包（已验证含 createLLM 符号；.gitignore 挡预编译库与 models/，获取方式见任务文档） - LocalLlm.ets：native 调用的 ets 封装；LlmTestPage：模型从局域网服务器下载→加载→首 token/速度 benchmark - scripts/serve-models.mjs：开发期局域网模型文件服务器（Qwen3-0.6B-MNN 456MB，SHA256 与 ModelScope 官方清单逐文件核对） - build-profile.json5 接 externalNativeOptions（arm64-v8a）；ProfileContent 加「端侧LLM调试」入口（内部调试用） - docs/开发任务包/T18-端侧小模型SPIKE.md：目标/验证项/回退线；真机 benchmark 数据回来后写 T18-REPORT 三问三答
2026-08-30 13:18:03 +08:00
parent 120bbeceff
commit b31c7490f2
36 changed files with 6789 additions and 0 deletions
@@ -0,0 +1,148 @@
+#include "local_llm.h"
+
+#include <chrono>
+#include <sstream>
+#include <streambuf>
+
+#include "llm/llm.hpp"
+#include <hilog/log.h>
+
+#define LOG_TAG "LocalLlm"
+#define LOGI(...) OH_LOG_INFO(LOG_APP, __VA_ARGS__)
+#define LOGE(...) OH_LOG_ERROR(LOG_APP, __VA_ARGS__)
+
+using MNN::Transformer::ChatMessages;
+using MNN::Transformer::Llm;
+
+namespace {
+
+// 计时流：接住 Llm::response 的流式输出，记录首 chunk 时间并累计全文。
+class TimingStreamBuf : public std::streambuf {
+public:
+    std::string text;
+    long firstTokenMs = -1;
+    std::chrono::steady_clock::time_point start;
+
+    void reset(std::chrono::steady_clock::time_point s) {
+        text.clear();
+        firstTokenMs = -1;
+        start = s;
+    }
+
+protected:
+    int overflow(int c) override {
+        if (c == EOF) {
+            return 0;
+        }
+        markFirst();
+        text += static_cast<char>(c);
+        return c;
+    }
+
+    std::streamsize xsputn(const char *s, std::streamsize n) override {
+        if (n > 0) {
+            markFirst();
+            text.append(s, static_cast<size_t>(n));
+        }
+        return n;
+    }
+
+private:
+    void markFirst() {
+        if (firstTokenMs < 0) {
+            firstTokenMs = std::chrono::duration_cast<std::chrono::milliseconds>(
+                std::chrono::steady_clock::now() - start)
+                .count();
+        }
+    }
+};
+
+std::string jsonEscape(const std::string &in) {
+    std::string out;
+    out.reserve(in.size() + 16);
+    for (char c : in) {
+        switch (c) {
+        case '"': out += "\\\""; break;
+        case '\\': out += "\\\\"; break;
+        case '\n': out += "\\n"; break;
+        case '\r': out += "\\r"; break;
+        case '\t': out += "\\t"; break;
+        default: out += c; break;
+        }
+    }
+    return out;
+}
+
+} // namespace
+
+LocalLlm &LocalLlm::instance() {
+    static LocalLlm inst;
+    return inst;
+}
+
+bool LocalLlm::loaded() const {
+    return llm_ != nullptr;
+}
+
+std::string LocalLlm::load(const std::string &modelDir) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    unload();
+    const std::string configPath = modelDir + "/config.json";
+    Llm *llm = Llm::createLLM(configPath);
+    if (llm == nullptr) {
+        LOGE("createLLM failed: %{public}s", configPath.c_str());
+        return "createLLM failed: " + configPath;
+    }
+    // use_mmap 手机端必开（权重走 mmap 防 OOM）；tmp_path 必须指向沙箱可写目录，
+    // 由 ArkTS 侧在调用前 mkdir 好（modelDir/.tmp）。
+    const std::string cfg = std::string("{\"tmp_path\":\"") + modelDir + "/.tmp" +
+                            "\",\"use_mmap\":true,\"thread_num\":4,"
+                            "\"precision\":\"low\",\"memory\":\"low\","
+                            "\"reuse_kv\":true,\"max_new_tokens\":256}";
+    llm->set_config(cfg);
+    if (!llm->load()) {
+        LOGE("llm load failed: %{public}s", configPath.c_str());
+        Llm::destroy(llm);
+        return "llm load failed: " + configPath;
+    }
+    llm_ = llm;
+    modelDir_ = modelDir;
+    LOGI("model loaded: %{public}s", configPath.c_str());
+    return "";
+}
+
+std::string LocalLlm::generate(const std::string &userPrompt, int maxNewTokens) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (llm_ == nullptr) {
+        return "{\"error\":\"model not loaded\"}";
+    }
+    TimingStreamBuf buf;
+    std::ostream os(&buf);
+    ChatMessages msgs;
+    msgs.emplace_back("user", userPrompt);
+    const auto start = std::chrono::steady_clock::now();
+    buf.reset(start);
+    try {
+        llm_->response(msgs, &os, nullptr, maxNewTokens);
+    } catch (const std::exception &e) {
+        LOGE("response exception: %{public}s", e.what());
+        return std::string("{\"error\":\"") + jsonEscape(e.what()) + "\"}";
+    }
+    const long totalMs = std::chrono::duration_cast<std::chrono::milliseconds>(
+        std::chrono::steady_clock::now() - start)
+        .count();
+    LOGI("generate: first=%{public}ldms total=%{public}ldms chars=%{public}zu",
+         buf.firstTokenMs, totalMs, buf.text.size());
+    std::ostringstream out;
+    out << "{\"text\":\"" << jsonEscape(buf.text) << "\",\"first_token_ms\":" << buf.firstTokenMs
+        << ",\"total_ms\":" << totalMs << ",\"chars\":" << buf.text.size() << "}";
+    return out.str();
+}
+
+void LocalLlm::unload() {
+    if (llm_ != nullptr) {
+        Llm::destroy(llm_);
+        llm_ = nullptr;
+        modelDir_.clear();
+    }
+}