From b31c7490f2c5debb17902dbcaa2ab2f8906ccbdc Mon Sep 17 00:00:00 2001 From: xiangsilian Date: Sun, 30 Aug 2026 13:18:03 +0800 Subject: [PATCH] =?UTF-8?q?spike(llm):=20T18=20=E7=AB=AF=E4=BE=A7=E5=B0=8F?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E2=80=94=E2=80=94MNN=20native=20=E9=AA=A8?= =?UTF-8?q?=E6=9E=B6=20+=20=E5=B1=80=E5=9F=9F=E7=BD=91=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cpp/：liblocalllm.so NAPI 骨架（loadModel/chat/benchmark/release），预置 libMNN.so 3.6.1 鸿蒙包（已验证含 createLLM 符号；.gitignore 挡预编译库与 models/，获取方式见任务文档） - LocalLlm.ets：native 调用的 ets 封装；LlmTestPage：模型从局域网服务器下载→加载→首 token/速度 benchmark - scripts/serve-models.mjs：开发期局域网模型文件服务器（Qwen3-0.6B-MNN 456MB，SHA256 与 ModelScope 官方清单逐文件核对） - build-profile.json5 接 externalNativeOptions（arm64-v8a）；ProfileContent 加「端侧LLM调试」入口（内部调试用） - docs/开发任务包/T18-端侧小模型SPIKE.md：目标/验证项/回退线；真机 benchmark 数据回来后写 T18-REPORT 三问三答 --- app/entry/build-profile.json5 | 7 + app/entry/src/main/cpp/CMakeLists.txt | 20 + app/entry/src/main/cpp/local_llm.cpp | 148 ++ app/entry/src/main/cpp/local_llm.h | 45 + app/entry/src/main/cpp/napi_init.cpp | 117 ++ .../third_party/mnn/include/MNN/AutoTime.hpp | 62 + .../third_party/mnn/include/MNN/ErrorCode.hpp | 44 + .../mnn/include/MNN/HalideRuntime.h | 308 ++++ .../mnn/include/MNN/ImageProcess.hpp | 182 ++ .../mnn/include/MNN/Interpreter.hpp | 545 ++++++ .../third_party/mnn/include/MNN/MNNDefine.h | 79 + .../mnn/include/MNN/MNNForwardType.h | 129 ++ .../mnn/include/MNN/MNNSharedContext.h | 77 + .../cpp/third_party/mnn/include/MNN/Matrix.h | 1615 +++++++++++++++++ .../cpp/third_party/mnn/include/MNN/Rect.h | 580 ++++++ .../third_party/mnn/include/MNN/Tensor.hpp | 320 ++++ .../mnn/include/MNN/expr/Executor.hpp | 172 ++ .../mnn/include/MNN/expr/ExecutorScope.hpp | 33 + .../third_party/mnn/include/MNN/expr/Expr.hpp | 282 +++ .../mnn/include/MNN/expr/ExprCreator.hpp | 16 + .../mnn/include/MNN/expr/MathOp.hpp | 145 ++ .../mnn/include/MNN/expr/Module.hpp | 143 ++ .../mnn/include/MNN/expr/NeuralNetWorkOp.hpp | 183 ++ .../mnn/include/MNN/expr/Optimizer.hpp | 64 + .../mnn/include/MNN/expr/Scope.hpp | 112 ++ .../mnn/include/MNN/plugin/PluginContext.hpp | 151 ++ .../mnn/include/MNN/plugin/PluginKernel.hpp | 77 + .../MNN/plugin/PluginShapeInference.hpp | 56 + .../cpp/third_party/mnn/include/llm/llm.hpp | 281 +++ .../third_party/mnn/include/llm/reranker.hpp | 307 ++++ .../src/main/cpp/types/liblocalllm/index.d.ts | 16 + .../main/ets/components/f1/ProfileContent.ets | 13 + app/entry/src/main/ets/core/LocalLlm.ets | 130 ++ app/entry/src/main/ets/pages/LlmTestPage.ets | 184 ++ docs/开发任务包/T18-端侧小模型SPIKE.md | 95 + scripts/serve-models.mjs | 51 + 36 files changed, 6789 insertions(+) create mode 100644 app/entry/src/main/cpp/CMakeLists.txt create mode 100644 app/entry/src/main/cpp/local_llm.cpp create mode 100644 app/entry/src/main/cpp/local_llm.h create mode 100644 app/entry/src/main/cpp/napi_init.cpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/AutoTime.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/ErrorCode.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/HalideRuntime.h create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/ImageProcess.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/Interpreter.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNDefine.h create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNForwardType.h create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNSharedContext.h create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/Matrix.h create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/Rect.h create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/Tensor.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Executor.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/ExecutorScope.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Expr.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/ExprCreator.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/MathOp.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Module.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/NeuralNetWorkOp.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Optimizer.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Scope.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginContext.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginKernel.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginShapeInference.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/llm/llm.hpp create mode 100644 app/entry/src/main/cpp/third_party/mnn/include/llm/reranker.hpp create mode 100644 app/entry/src/main/cpp/types/liblocalllm/index.d.ts create mode 100644 app/entry/src/main/ets/core/LocalLlm.ets create mode 100644 app/entry/src/main/ets/pages/LlmTestPage.ets create mode 100644 docs/开发任务包/T18-端侧小模型SPIKE.md create mode 100644 scripts/serve-models.mjs diff --git a/app/entry/build-profile.json5 b/app/entry/build-profile.json5 index 6bd6457..407a364 100644 --- a/app/entry/build-profile.json5 +++ b/app/entry/build-profile.json5 @@ -5,6 +5,13 @@ "copyCodeResource": { "enable": false } + }, + // T18 端侧小模型 SPIKE：native 模块（liblocalllm.so + 预置 libMNN.so） + "externalNativeOptions": { + "path": "./src/main/cpp/CMakeLists.txt", + "arguments": "", + "cppFlags": "", + "abiFilters": ["arm64-v8a"] } }, "buildOptionSet": [ diff --git a/app/entry/src/main/cpp/CMakeLists.txt b/app/entry/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..768be30 --- /dev/null +++ b/app/entry/src/main/cpp/CMakeLists.txt @@ -0,0 +1,20 @@ +# T18 端侧小模型 SPIKE：liblocalllm.so +# 结构对齐社区已验证案例（turbo-ai-chat-harmonyos）：预置 libMNN.so + NAPI 薄封装。 +cmake_minimum_required(VERSION 3.16) +project(localllm) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# 预编译 MNN（官方 3.6.1 harmony 包，含 LLM 符号；获取方式见任务文档 T18）。 +# SPIKE 阶段只支持真机 arm64-v8a；模拟器（x86_64）不在本任务范围。 +add_library(mnn SHARED IMPORTED) +set_target_properties(mnn PROPERTIES + IMPORTED_LOCATION ${CMAKE_CURRENT_SOURCE_DIR}/../libs/arm64-v8a/libMNN.so) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/mnn/include) + +add_library(localllm SHARED napi_init.cpp local_llm.cpp) + +# ace_napi：N-API；hilog：native 侧日志（ benchmark 数据落 log 便于 hdc 抓取） +target_link_libraries(localllm PUBLIC mnn libace_napi.z.so libhilog_ndk.z.so) diff --git a/app/entry/src/main/cpp/local_llm.cpp b/app/entry/src/main/cpp/local_llm.cpp new file mode 100644 index 0000000..1519171 --- /dev/null +++ b/app/entry/src/main/cpp/local_llm.cpp @@ -0,0 +1,148 @@ +#include "local_llm.h" + +#include +#include +#include + +#include "llm/llm.hpp" +#include + +#define LOG_TAG "LocalLlm" +#define LOGI(...) OH_LOG_INFO(LOG_APP, __VA_ARGS__) +#define LOGE(...) OH_LOG_ERROR(LOG_APP, __VA_ARGS__) + +using MNN::Transformer::ChatMessages; +using MNN::Transformer::Llm; + +namespace { + +// 计时流：接住 Llm::response 的流式输出，记录首 chunk 时间并累计全文。 +class TimingStreamBuf : public std::streambuf { +public: + std::string text; + long firstTokenMs = -1; + std::chrono::steady_clock::time_point start; + + void reset(std::chrono::steady_clock::time_point s) { + text.clear(); + firstTokenMs = -1; + start = s; + } + +protected: + int overflow(int c) override { + if (c == EOF) { + return 0; + } + markFirst(); + text += static_cast(c); + return c; + } + + std::streamsize xsputn(const char *s, std::streamsize n) override { + if (n > 0) { + markFirst(); + text.append(s, static_cast(n)); + } + return n; + } + +private: + void markFirst() { + if (firstTokenMs < 0) { + firstTokenMs = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + } + } +}; + +std::string jsonEscape(const std::string &in) { + std::string out; + out.reserve(in.size() + 16); + for (char c : in) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: out += c; break; + } + } + return out; +} + +} // namespace + +LocalLlm &LocalLlm::instance() { + static LocalLlm inst; + return inst; +} + +bool LocalLlm::loaded() const { + return llm_ != nullptr; +} + +std::string LocalLlm::load(const std::string &modelDir) { + std::lock_guard lock(mutex_); + unload(); + const std::string configPath = modelDir + "/config.json"; + Llm *llm = Llm::createLLM(configPath); + if (llm == nullptr) { + LOGE("createLLM failed: %{public}s", configPath.c_str()); + return "createLLM failed: " + configPath; + } + // use_mmap 手机端必开（权重走 mmap 防 OOM）；tmp_path 必须指向沙箱可写目录， + // 由 ArkTS 侧在调用前 mkdir 好（modelDir/.tmp）。 + const std::string cfg = std::string("{\"tmp_path\":\"") + modelDir + "/.tmp" + + "\",\"use_mmap\":true,\"thread_num\":4," + "\"precision\":\"low\",\"memory\":\"low\"," + "\"reuse_kv\":true,\"max_new_tokens\":256}"; + llm->set_config(cfg); + if (!llm->load()) { + LOGE("llm load failed: %{public}s", configPath.c_str()); + Llm::destroy(llm); + return "llm load failed: " + configPath; + } + llm_ = llm; + modelDir_ = modelDir; + LOGI("model loaded: %{public}s", configPath.c_str()); + return ""; +} + +std::string LocalLlm::generate(const std::string &userPrompt, int maxNewTokens) { + std::lock_guard lock(mutex_); + if (llm_ == nullptr) { + return "{\"error\":\"model not loaded\"}"; + } + TimingStreamBuf buf; + std::ostream os(&buf); + ChatMessages msgs; + msgs.emplace_back("user", userPrompt); + const auto start = std::chrono::steady_clock::now(); + buf.reset(start); + try { + llm_->response(msgs, &os, nullptr, maxNewTokens); + } catch (const std::exception &e) { + LOGE("response exception: %{public}s", e.what()); + return std::string("{\"error\":\"") + jsonEscape(e.what()) + "\"}"; + } + const long totalMs = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + LOGI("generate: first=%{public}ldms total=%{public}ldms chars=%{public}zu", + buf.firstTokenMs, totalMs, buf.text.size()); + std::ostringstream out; + out << "{\"text\":\"" << jsonEscape(buf.text) << "\",\"first_token_ms\":" << buf.firstTokenMs + << ",\"total_ms\":" << totalMs << ",\"chars\":" << buf.text.size() << "}"; + return out.str(); +} + +void LocalLlm::unload() { + if (llm_ != nullptr) { + Llm::destroy(llm_); + llm_ = nullptr; + modelDir_.clear(); + } +} diff --git a/app/entry/src/main/cpp/local_llm.h b/app/entry/src/main/cpp/local_llm.h new file mode 100644 index 0000000..ec7feef --- /dev/null +++ b/app/entry/src/main/cpp/local_llm.h @@ -0,0 +1,45 @@ +#ifndef LOCAL_LLM_H +#define LOCAL_LLM_H + +#include +#include + +namespace MNN { +namespace Transformer { +class Llm; +} +} + +/** + * 端侧小模型最小封装（T18 SPIKE）。 + * + * 设计约束： + * - 全局单例 + 互斥锁：LLM 推理不可重入，ArkTS 侧并发调用由此串行化； + * - 所有耗时操作（load/generate）只从 N-API async work 线程进入，UI 线程不得直接调； + * - Llm 用裸指针：MNN 3.6.1 头文件明确要求用 Llm::destroy() 销毁。 + */ +class LocalLlm { +public: + static LocalLlm &instance(); + + /** 加载模型目录（内含 config.json）。返回空串=成功，否则为错误描述。 */ + std::string load(const std::string &modelDir); + + /** + * 生成。userPrompt 为纯文本，内部套 chat 模板。 + * 返回 JSON 字符串：{"text","first_token_ms","total_ms","chars"}；失败返回 {"error":...}。 + * 计时在 native 侧完成（std::chrono），保证 benchmark 数据不受 JS 调度影响。 + */ + std::string generate(const std::string &userPrompt, int maxNewTokens); + + void unload(); + bool loaded() const; + +private: + LocalLlm() = default; + std::mutex mutex_; + MNN::Transformer::Llm *llm_ = nullptr; + std::string modelDir_; +}; + +#endif // LOCAL_LLM_H diff --git a/app/entry/src/main/cpp/napi_init.cpp b/app/entry/src/main/cpp/napi_init.cpp new file mode 100644 index 0000000..5a6a54d --- /dev/null +++ b/app/entry/src/main/cpp/napi_init.cpp @@ -0,0 +1,117 @@ +/** + * T18 端侧小模型 SPIKE · N-API 桥接 + * + * ArkTS 侧用法：import localllm from 'liblocalllm.so' + * 耗时接口（loadModel/generate）一律 async work + Promise，不阻塞 UI 线程。 + */ +#include "local_llm.h" + +#include +#include + +namespace { + +struct AsyncTask { + napi_async_work work = nullptr; + napi_deferred deferred = nullptr; + std::string arg; // loadModel: modelDir；generate: prompt + int maxNewTokens = 256; // 仅 generate 用 + std::string result; // 返回值（空串或 JSON） + bool isLoad = true; // true=loadModel，false=generate +}; + +napi_value ResolveString(napi_env env, napi_deferred deferred, const std::string &value) { + napi_value js; + napi_create_string_utf8(env, value.c_str(), value.size(), &js); + napi_resolve_deferred(env, deferred, js); + return js; +} + +void TaskExecute(napi_env /*env*/, void *data) { + auto *task = static_cast(data); + if (task->isLoad) { + task->result = LocalLlm::instance().load(task->arg); // 空串=成功 + } else { + task->result = LocalLlm::instance().generate(task->arg, task->maxNewTokens); + } +} + +void TaskComplete(napi_env env, napi_status /*status*/, void *data) { + auto *task = static_cast(data); + ResolveString(env, task->deferred, task->result); + napi_delete_async_work(env, task->work); + delete task; +} + +napi_value RunAsync(napi_env env, napi_callback_info info, bool isLoad) { + size_t argc = 2; + napi_value argv[2] = {nullptr, nullptr}; + napi_get_cb_info(env, info, &argc, argv, nullptr, nullptr); + + auto *task = new AsyncTask(); + task->isLoad = isLoad; + + char buf[8192]; + size_t len = 0; + napi_get_value_string_utf8(env, argv[0], buf, sizeof(buf), &len); + task->arg.assign(buf, len); + if (!isLoad && argc >= 2) { + napi_get_value_int32(env, argv[1], &task->maxNewTokens); + } + + napi_value promise; + napi_create_promise(env, &task->deferred, &promise); + napi_value name; + napi_create_string_utf8(env, isLoad ? "loadModel" : "generate", NAPI_AUTO_LENGTH, &name); + napi_create_async_work(env, nullptr, name, TaskExecute, TaskComplete, task, &task->work); + napi_queue_async_work(env, task->work); + return promise; +} + +napi_value LoadModel(napi_env env, napi_callback_info info) { + return RunAsync(env, info, true); +} + +napi_value Generate(napi_env env, napi_callback_info info) { + return RunAsync(env, info, false); +} + +napi_value UnloadModel(napi_env env, napi_callback_info /*info*/) { + LocalLlm::instance().unload(); + napi_value ret; + napi_get_undefined(env, &ret); + return ret; +} + +napi_value IsLoaded(napi_env env, napi_callback_info /*info*/) { + napi_value ret; + napi_get_boolean(env, LocalLlm::instance().loaded(), &ret); + return ret; +} + +napi_value Init(napi_env env, napi_value exports) { + napi_property_descriptor desc[] = { + {"loadModel", nullptr, LoadModel, nullptr, nullptr, nullptr, napi_default, nullptr}, + {"generate", nullptr, Generate, nullptr, nullptr, nullptr, napi_default, nullptr}, + {"unloadModel", nullptr, UnloadModel, nullptr, nullptr, nullptr, napi_default, nullptr}, + {"isLoaded", nullptr, IsLoaded, nullptr, nullptr, nullptr, napi_default, nullptr}, + }; + napi_define_properties(env, exports, sizeof(desc) / sizeof(desc[0]), desc); + return exports; +} + +} // namespace + +static napi_module localllmModule = { + .nm_version = 1, + .nm_flags = 0, + .nm_filename = nullptr, + .nm_register_func = Init, + .nm_modname = "localllm", + .nm_priv = nullptr, + .reserved = {0}, +}; + +extern "C" __attribute__((constructor)) void RegisterLocalllmModule() { + napi_module_register(&localllmModule); +} diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/AutoTime.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/AutoTime.hpp new file mode 100644 index 0000000..d76bd09 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/AutoTime.hpp @@ -0,0 +1,62 @@ +// +// AutoTime.hpp +// MNN +// +// Created by MNN on 2018/07/27. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_AutoTime_hpp +#define MNN_AutoTime_hpp + +#include +#include +#include + +namespace MNN { + +class MNN_PUBLIC Timer { +public: + Timer(); + ~Timer(); + Timer(const Timer&) = delete; + Timer(const Timer&&) = delete; + Timer& operator=(const Timer&) = delete; + Timer& operator=(const Timer&&) = delete; + + // reset timer + void reset(); + // get duration (us) from init or latest reset. + uint64_t durationInUs(); + + // Get Current Time + uint64_t current() const { + return mLastResetTime; + } +protected: + uint64_t mLastResetTime; +}; + +/** time tracing util. prints duration between init and deinit. */ +class MNN_PUBLIC AutoTime : Timer { +public: + AutoTime(int line, const char* func); + ~AutoTime(); + AutoTime(const AutoTime&) = delete; + AutoTime(const AutoTime&&) = delete; + AutoTime& operator=(const AutoTime&) = delete; + AutoTime& operator=(const AutoTime&&) = delete; + +private: + int mLine; + char* mName; +}; +} // namespace MNN + +#ifdef MNN_OPEN_TIME_TRACE +#define AUTOTIME MNN::AutoTime ___t(__LINE__, __func__) +#else +#define AUTOTIME +#endif + +#endif /* AutoTime_hpp */ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/ErrorCode.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/ErrorCode.hpp new file mode 100644 index 0000000..4d40d60 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/ErrorCode.hpp @@ -0,0 +1,44 @@ +// +// ErrorCode.hpp +// MNN +// +// Created by MNN on 2018/09/18. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_ErrorCode_h +#define MNN_ErrorCode_h + +namespace MNN { +enum ErrorCode { +#ifdef NO_ERROR +#undef NO_ERROR +#endif // NO_ERROR + NO_ERROR = 0, + OUT_OF_MEMORY = 1, + NOT_SUPPORT = 2, + COMPUTE_SIZE_ERROR = 3, + NO_EXECUTION = 4, + INVALID_VALUE = 5, + + // User error + INPUT_DATA_ERROR = 10, + CALL_BACK_STOP = 11, + + // Op Resize Error + TENSOR_NOT_SUPPORT = 20, + TENSOR_NEED_DIVIDE = 21, + + // File error + FILE_CREATE_FAILED = 30, + FILE_REMOVE_FAILED = 31, + FILE_OPEN_FAILED = 32, + FILE_CLOSE_FAILED = 33, + FILE_RESIZE_FAILED = 34, + FILE_SEEK_FAILED = 35, + FILE_NOT_EXIST = 36, + FILE_UNMAP_FAILED = 37 +}; +} // namespace MNN + +#endif /* ErrorCode_h */ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/HalideRuntime.h b/app/entry/src/main/cpp/third_party/mnn/include/MNN/HalideRuntime.h new file mode 100644 index 0000000..43c40da --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/HalideRuntime.h @@ -0,0 +1,308 @@ +#ifndef MNN_HALIDE_HALIDERUNTIME_H +#define MNN_HALIDE_HALIDERUNTIME_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Note that you should not use "inline" along with HALIDE_ALWAYS_INLINE; +// it is not necessary, and may produce warnings for some build configurations. +#ifdef _MSC_VER +#define HALIDE_ALWAYS_INLINE __forceinline +#define HALIDE_NEVER_INLINE __declspec(noinline) +#else +#define HALIDE_ALWAYS_INLINE __attribute__((always_inline)) inline +#define HALIDE_NEVER_INLINE __attribute__((noinline)) +#endif + +/** \file + * + * This file declares the routines used by Halide internally in its + * runtime. On platforms that support weak linking, these can be + * replaced with user-defined versions by defining an extern "C" + * function with the same name and signature. + * + * When doing Just In Time (JIT) compilation methods on the Func being + * compiled must be called instead. The corresponding methods are + * documented below. + * + * All of these functions take a "void *user_context" parameter as their + * first argument; if the Halide kernel that calls back to any of these + * functions has been compiled with the UserContext feature set on its Target, + * then the value of that pointer passed from the code that calls the + * Halide kernel is piped through to the function. + * + * Some of these are also useful to call when using the default + * implementation. E.g. halide_shutdown_thread_pool. + * + * Note that even on platforms with weak linking, some linker setups + * may not respect the override you provide. E.g. if the override is + * in a shared library and the halide object files are linked directly + * into the output, the builtin versions of the runtime functions will + * be called. See your linker documentation for more details. On + * Linux, LD_DYNAMIC_WEAK=1 may help. + * + */ + +// Forward-declare to suppress warnings if compiling as C. +struct halide_buffer_t; + +/** Types in the halide type system. They can be ints, unsigned ints, + * or floats (of various bit-widths), or a handle (which is always 64-bits). + * Note that the int/uint/float values do not imply a specific bit width + * (the bit width is expected to be encoded in a separate value). + */ +typedef enum halide_type_code_t +{ + halide_type_int = 0, //!< signed integers + halide_type_uint = 1, //!< unsigned integers + halide_type_float = 2, //!< IEEE floating point numbers + halide_type_handle = 3, //!< opaque pointer type (void *) + halide_type_bfloat = 4 //!< floating point numbers in the bfloat format +} halide_type_code_t; + +// Note that while __attribute__ can go before or after the declaration, +// __declspec apparently is only allowed before. +#ifndef HALIDE_ATTRIBUTE_ALIGN + #ifdef _MSC_VER + #define HALIDE_ATTRIBUTE_ALIGN(x) __declspec(align(x)) + #else + #define HALIDE_ATTRIBUTE_ALIGN(x) __attribute__((aligned(x))) + #endif +#endif + +/** A runtime tag for a type in the halide type system. Can be ints, + * unsigned ints, or floats of various bit-widths (the 'bits' + * field). Can also be vectors of the same (by setting the 'lanes' + * field to something larger than one). This struct should be + * exactly 32-bits in size. */ +struct halide_type_t { + /** The basic type code: signed integer, unsigned integer, or floating point. */ +#ifndef _MSC_VER + HALIDE_ATTRIBUTE_ALIGN(1) halide_type_code_t code; // halide_type_code_t +#else + HALIDE_ATTRIBUTE_ALIGN(1) uint8_t code; // halide_type_code_t +#endif + + /** The number of bits of precision of a single scalar value of this type. */ + HALIDE_ATTRIBUTE_ALIGN(1) uint8_t bits; + + /** How many elements in a vector. This is 1 for scalar types. */ + HALIDE_ATTRIBUTE_ALIGN(2) uint16_t lanes; + +#ifdef __cplusplus + /** Construct a runtime representation of a Halide type from: + * code: The fundamental type from an enum. + * bits: The bit size of one element. + * lanes: The number of vector elements in the type. */ + HALIDE_ALWAYS_INLINE halide_type_t(halide_type_code_t code, uint8_t bits, uint16_t lanes = 1) + : code(code), bits(bits), lanes(lanes) { + } + + /** Default constructor is required e.g. to declare halide_trace_event + * instances. */ + HALIDE_ALWAYS_INLINE halide_type_t() : code((halide_type_code_t)0), bits(0), lanes(0) {} + + /** Compare two types for equality. */ + HALIDE_ALWAYS_INLINE bool operator==(const halide_type_t &other) const { + return (code == other.code && + bits == other.bits && + lanes == other.lanes); + } + + HALIDE_ALWAYS_INLINE bool operator!=(const halide_type_t &other) const { + return !(*this == other); + } + + /** Size in bytes for a single element, even if width is not 1, of this type. */ + HALIDE_ALWAYS_INLINE int bytes() const { return (bits + 7) / 8; } +#endif +}; + +/** An opaque struct containing per-GPU API implementations of the + * device functions. */ +struct halide_device_interface_impl_t; + +/** Each GPU API provides a halide_device_interface_t struct pointing + * to the code that manages device allocations. You can access these + * functions directly from the struct member function pointers, or by + * calling the functions declared below. Note that the global + * functions are not available when using Halide as a JIT compiler. + * If you are using raw halide_buffer_t in that context you must use + * the function pointers in the device_interface struct. + * + * The function pointers below are currently the same for every GPU + * API; only the impl field varies. These top-level functions do the + * bookkeeping that is common across all GPU APIs, and then dispatch + * to more API-specific functions via another set of function pointers + * hidden inside the impl field. + */ +struct halide_device_interface_t { + int (*device_malloc)(void *user_context, struct halide_buffer_t *buf, + const struct halide_device_interface_t *device_interface); + int (*device_free)(void *user_context, struct halide_buffer_t *buf); + int (*device_sync)(void *user_context, struct halide_buffer_t *buf); + void (*device_release)(void *user_context, + const struct halide_device_interface_t *device_interface); + int (*copy_to_host)(void *user_context, struct halide_buffer_t *buf); + int (*copy_to_device)(void *user_context, struct halide_buffer_t *buf, + const struct halide_device_interface_t *device_interface); + int (*device_and_host_malloc)(void *user_context, struct halide_buffer_t *buf, + const struct halide_device_interface_t *device_interface); + int (*device_and_host_free)(void *user_context, struct halide_buffer_t *buf); + int (*buffer_copy)(void *user_context, struct halide_buffer_t *src, + const struct halide_device_interface_t *dst_device_interface, struct halide_buffer_t *dst); + int (*device_crop)(void *user_context, const struct halide_buffer_t *src, + struct halide_buffer_t *dst); + int (*device_release_crop)(void *user_context, struct halide_buffer_t *buf); + int (*wrap_native)(void *user_context, struct halide_buffer_t *buf, uint64_t handle, + const struct halide_device_interface_t *device_interface); + int (*detach_native)(void *user_context, struct halide_buffer_t *buf); + const struct halide_device_interface_impl_t *impl; +}; + +typedef struct halide_dimension_t { + int32_t min, extent, stride; + + // Per-dimension flags. None are defined yet (This is reserved for future use). + uint32_t flags; + +#ifdef __cplusplus + HALIDE_ALWAYS_INLINE halide_dimension_t() : min(0), extent(0), stride(0), flags(0) {} + HALIDE_ALWAYS_INLINE halide_dimension_t(int32_t m, int32_t e, int32_t s, uint32_t f = 0) : + min(m), extent(e), stride(s), flags(f) {} + + HALIDE_ALWAYS_INLINE bool operator==(const halide_dimension_t &other) const { + return (min == other.min) && + (extent == other.extent) && + (stride == other.stride) && + (flags == other.flags); + } + + HALIDE_ALWAYS_INLINE bool operator!=(const halide_dimension_t &other) const { + return !(*this == other); + } +#endif +} halide_dimension_t; + +#ifdef __cplusplus +} // extern "C" +#endif + +typedef enum {halide_buffer_flag_host_dirty = 1, + halide_buffer_flag_device_dirty = 2} halide_buffer_flags; + +/** + * The raw representation of an image passed around by generated + * Halide code. It includes some stuff to track whether the image is + * not actually in main memory, but instead on a device (like a + * GPU). For a more convenient C++ wrapper, use Halide::Buffer. */ +typedef struct halide_buffer_t { + /** A device-handle for e.g. GPU memory used to back this buffer. */ + uint64_t device; + + /** The interface used to interpret the above handle. */ + const struct halide_device_interface_t *device_interface; + + /** A pointer to the start of the data in main memory. In terms of + * the Halide coordinate system, this is the address of the min + * coordinates (defined below). */ + uint8_t* host; + + /** flags with various meanings. */ + uint64_t flags; + + /** The type of each buffer element. */ + struct halide_type_t type; + + /** The dimensionality of the buffer. */ + int32_t dimensions; + + /** The shape of the buffer. Halide does not own this array - you + * must manage the memory for it yourself. */ + halide_dimension_t *dim; + + /** Pads the buffer up to a multiple of 8 bytes */ + void *padding; +} halide_buffer_t; + + +#ifdef __cplusplus + +namespace { +template struct check_is_pointer; +template struct check_is_pointer {}; +} + +/** Construct the halide equivalent of a C type */ +template +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + // Create a compile-time error if T is not a pointer (without + // using any includes - this code goes into the runtime). + check_is_pointer check; + (void)check; + return halide_type_t(halide_type_handle, 64); +} + +template<> +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + return halide_type_t(halide_type_float, 32); +} + +template<> +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + return halide_type_t(halide_type_float, 64); +} + +template<> +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + return halide_type_t(halide_type_uint, 1); +} + +template<> +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + return halide_type_t(halide_type_uint, 8); +} + +template<> +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + return halide_type_t(halide_type_uint, 16); +} + +template<> +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + return halide_type_t(halide_type_uint, 32); +} + +template<> +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + return halide_type_t(halide_type_uint, 64); +} + +template<> +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + return halide_type_t(halide_type_int, 8); +} + +template<> +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + return halide_type_t(halide_type_int, 16); +} + +template<> +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + return halide_type_t(halide_type_int, 32); +} + +template<> +HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() { + return halide_type_t(halide_type_int, 64); +} + +#endif + +#endif // HALIDE_HALIDERUNTIME_H diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/ImageProcess.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/ImageProcess.hpp new file mode 100644 index 0000000..a633e04 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/ImageProcess.hpp @@ -0,0 +1,182 @@ +// +// ImageProcess.hpp +// MNN +// +// Created by MNN on 2018/09/19. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_ImageProcess_hpp +#define MNN_ImageProcess_hpp + +#include +#include +#include + +namespace MNN { +namespace CV { +enum ImageFormat { + RGBA = 0, + RGB = 1, + BGR = 2, + GRAY = 3, + BGRA = 4, + YCrCb = 5, + YUV = 6, + HSV = 7, + XYZ = 8, + BGR555 = 9, + BGR565 = 10, + YUV_NV21 = 11, + YUV_NV12 = 12, + YUV_I420 = 13, + HSV_FULL = 14, +}; + +enum Filter { NEAREST = 0, BILINEAR = 1, BICUBIC = 2 }; + +enum Wrap { CLAMP_TO_EDGE = 0, ZERO = 1, REPEAT = 2 }; + +/** + * handle image process for tensor. + * step: + * 1: Do transform compute and get points + * 2: Sample line and do format convert + * 3: Turn RGBA to float tensor, and do sub and normalize + */ + +class MNN_PUBLIC ImageProcess { +public: + struct Inside; + struct Config { + /** data filter */ + Filter filterType = NEAREST; + /** format of source data */ + ImageFormat sourceFormat = RGBA; + /** format of destination data */ + ImageFormat destFormat = RGBA; + + // Only valid if the dest type is float + float mean[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + float normal[4] = {1.0f, 1.0f, 1.0f, 1.0f}; + + /** edge wrapper */ + Wrap wrap = CLAMP_TO_EDGE; + }; +public: + /** + * @brief create image process with given config for given tensor. + * @param config given config. + * @param dstTensor given tensor. + * @return image processor. + */ + static ImageProcess* create(const Config& config, const Tensor* dstTensor = nullptr); + + /** + * @brief create image process with given config for given tensor. + * @param means given means + * @param meanCount given means count + * @param normals given normals + * @param normalCount given normal count + * @param sourceFormat format of source data + * @param destFormat format of destination data + * @param dstTensor given tensor. + * @return image processor. + */ + static ImageProcess* create(const ImageFormat sourceFormat = RGBA, const ImageFormat destFormat = RGBA, + const float* means = nullptr, const int meanCount = 0, const float* normals = nullptr, + const int normalCount = 0, const Tensor* dstTensor = nullptr); + + ~ImageProcess(); + static void destroy(ImageProcess* imageProcess); + + /** + * @brief get affine transform matrix. + * @return affine transform matrix. + */ + inline const Matrix& matrix() const { + return mTransform; + } + void setMatrix(const Matrix& matrix); + + /** + * @brief convert source data to given tensor. + * @param source source data. + * @param iw source width. + * @param ih source height. + * @param stride number of elements per row. eg: 100 width RGB contains at least 300 elements. + * @param dest given tensor. + * @return result code. + */ + ErrorCode convert(const uint8_t* source, int iw, int ih, int stride, Tensor* dest); + + /** + * @brief convert source data to given tensor. + * @param source source data. + * @param iw source width. + * @param ih source height. + * @param stride number of elements per row. eg: 100 width RGB contains at least 300 elements. + * @param dest dest data. + * @param ow output width. + * @param oh output height. + * @param outputBpp output bpp, if 0, set as the save and config.destFormat. + * @param outputStride output stride, if 0, set as ow * outputBpp. + * @param type Only support halide_type_of and halide_type_of. + * @return result code. + */ + ErrorCode convert(const uint8_t* source, int iw, int ih, int stride, void* dest, int ow, int oh, int outputBpp = 0, + int outputStride = 0, halide_type_t type = halide_type_of()); + + /** + * @brief create tensor with given data. + * @param w image width. + * @param h image height. + * @param bpp bytes per pixel. + * @param p pixel data pointer. + * @return created tensor. + */ + template + static Tensor* createImageTensor(int w, int h, int bpp, void* p = nullptr) { + return createImageTensor(halide_type_of(), w, h, bpp, p); + } + static Tensor* createImageTensor(halide_type_t type, int w, int h, int bpp, void* p = nullptr); + + /** + * @brief set padding value when wrap=ZERO. + * @param value padding value. + * @return void. + */ + void setPadding(uint8_t value) { + mPaddingValue = value; + } + + /** + * @brief set to draw mode. + * @param void + * @return void. + */ + void setDraw(); + + /** + * @brief draw color to regions of img. + * @param img the image to draw. + * @param w the image's width. + * @param h the image's height. + * @param c the image's channel. + * @param regions the regions to draw, size is [num * 3] contain num x { y, xl, xr } + * @param num regions num + * @param color the color to draw. + * @return void. + */ + void draw(uint8_t* img, int w, int h, int c, const int* regions, int num, const uint8_t* color); +private: + ImageProcess(const Config& config); + Matrix mTransform; + Matrix mTransformInvert; + Inside* mInside; + uint8_t mPaddingValue = 0; +}; +} // namespace CV +} // namespace MNN + +#endif /* MNN_ImageProcess_hpp */ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/Interpreter.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/Interpreter.hpp new file mode 100644 index 0000000..dfc3f11 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/Interpreter.hpp @@ -0,0 +1,545 @@ +// +// Interpreter.hpp +// MNN +// +// Created by MNN on 2018/07/23. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_Interpreter_hpp +#define MNN_Interpreter_hpp + +#include +#include +#include +#include +#include +#include +#include + +namespace MNN { + +/** session schedule config */ +struct ScheduleConfig { + /** which tensor should be kept */ + std::vector saveTensors; + /** forward type */ + MNNForwardType type = MNN_FORWARD_CPU; + /** CPU:number of threads in parallel , Or GPU: mode setting*/ + union { + int numThread = 4; + int mode; + }; + + /** subpath to run */ + struct Path { + std::vector inputs; + std::vector outputs; + + enum Mode { + /** + * Op Mode + * - inputs means the source op, can NOT be empty. + * - outputs means the sink op, can be empty. + * The path will start from source op, then flow when encounter the sink op. + * The sink op will not be compute in this path. + */ + Op = 0, + + /** + * Tensor Mode + * - inputs means the inputs tensors, can NOT be empty. + * - outputs means the outputs tensors, can NOT be empty. + * It will find the pipeline that compute outputs from inputs. + */ + Tensor = 1 + }; + + /** running mode */ + Mode mode = Op; + }; + Path path; + + /** backup backend used to create execution when desinated backend do NOT support any op */ + MNNForwardType backupType = MNN_FORWARD_CPU; + + /** extra backend config */ + BackendConfig* backendConfig = nullptr; +}; + +class Session; +struct Content; +class Tensor; +class Backend; +class Runtime; + +class MNN_PUBLIC OperatorInfo { + struct Info; + +public: + /** Operator's name*/ + const std::string& name() const; + + /** Operator's type*/ + const std::string& type() const; + + /** Operator's flops, in M*/ + float flops() const; + +protected: + OperatorInfo(); + ~OperatorInfo(); + Info* mContent; +}; + +typedef std::function&, const std::string& /*opName*/)> TensorCallBack; +typedef std::function&, const OperatorInfo*)> TensorCallBackWithInfo; +typedef std::pair< std::map>, std::shared_ptr> RuntimeInfo; + +/** + * @brief get mnn version info. + * @return mnn version string. + */ +MNN_PUBLIC const char* getVersion(); + +/** net data holder. multiple sessions could share same net. */ +class MNN_PUBLIC Interpreter { +public: + /** + * @brief create net from file. + * @param file given file. + * @return created net if success, NULL otherwise. + */ + static Interpreter* createFromFile(const char* file); + /** + * @brief create net from buffer. + * @param buffer given data buffer. + * @param size size of data buffer. + * @return created net if success, NULL otherwise. + */ + static Interpreter* createFromBuffer(const void* buffer, size_t size); + ~Interpreter(); + + /** + * @brief destroy Interpreter + * @param model given Interpreter to release. + */ + static void destroy(Interpreter* net); + + enum SessionMode { + /** About CallBack, Default Session_Debug*/ + /** runSessionWithCallBack is allowed and can get internal op info*/ + Session_Debug = 0, + /** runSessionWithCallBack is not valid and can't get any info of op in session*/ + Session_Release = 1, + + /** About input tenosr, Default Session_Input_Inside*/ + /** The input tensor is alloced by session, input data after session resized*/ + Session_Input_Inside = 2, + /** The input tensor is alloced by user, set input data before session resize*/ + Session_Input_User = 3, + + /** The output tensor depends on session, and can't be separate used*/ + Session_Output_Inside = 4, + /** The output tensor can be separated from session*/ + Session_Output_User = 5, + + /** Try Resize Session when create Session or not, default direct: */ + Session_Resize_Direct = 6, + Session_Resize_Defer = 7, + + /** Determine the Execution's forward type is determine by user or auto determine */ + Session_Backend_Fix = 8, // Use the backend user set, when not support use default backend + Session_Backend_Auto = 9, // Auto Determine the Op type by MNN + + /** Determine static memory whether recyle in resizeSession or just cache the memory */ + Session_Memory_Collect = 10, // Recycle static memory when session resize in case memory explosion + Session_Memory_Cache = 11, // Cache the static memory for next forward usage + + /** Determine whether use codegen function */ + Session_Codegen_Disable = 12, // Disable codegen in case extra build codegen cost + Session_Codegen_Enable = 13, // Enable codegen + + /** Dynamic Reisze Optimization */ + Session_Resize_Check = 14, // Open Trace for resize + Session_Resize_Fix = 15, // Apply Resize Optimization + + /** Set for Module's traceOrOptimize API. + Module_Forward_Seperate: + when inputs is not empty , Module's onForward will only infer shape and alloc memory. + when inputs is empty , Module's onForward will only runSession to compute content. + Default is Module_Forward_Combine + */ + Module_Forward_Separate = 16, + Module_Forward_Combine = 17, + }; + /** + * @brief The API shoud be called before create session. + * @param mode session mode + */ + void setSessionMode(SessionMode mode); + + /** + * @brief The API shoud be called before create session. + * If the cache exist, try to load cache from file. + * After createSession, try to save cache to file. + * @param cacheFile cache file name + * @param keySize depercerate, for future use. + */ + void setCacheFile(const char* cacheFile, size_t keySize = 128); + + /** + * @brief The API shoud be called before create session. + * @param file external data file name + * @param keySize depercerate, for future use. + */ + void setExternalFile(const char* file, size_t flag = 128); + /** + * @brief The API shoud be called after last resize session. + * If resize session generate new cache info, try to rewrite cache file. + * If resize session do not generate any new cache info, just do nothing. + * @param session given session + * @param flag Protected param, not used now + */ + + ErrorCode updateCacheFile(Session *session, int flag = 0); + + enum HintMode { + // Max Op number for async tuning + MAX_TUNING_NUMBER = 0, + // Strictly check model file or not, default 1. if set 0, will not check model file valid/invalid + STRICT_CHECK_MODEL = 1, + MEM_ALLOCATOR_TYPE = 2, + // Winograd unit candidates count, default 3. if set 0, will use less unit candidates for less memory at the expense of performance. + WINOGRAD_MEMORY_LEVEL = 3, + + // Geometry Compute option, default is 0xFFFF + GEOMETRY_COMPUTE_MASK = 4, + + // default 0 + // 1: For general convolution, use one scale&zeropoint to quant. + // 2: use block-quant for input data. + DYNAMIC_QUANT_OPTIONS = 5, + + // For Mobile CPU with big-litter core, set decrease rate to let MNN divide task differential by CPU's performance + // 0-100, 50 means litter core has 50% capacity of large core + // Default is 50 + CPU_LITTLECORE_DECREASE_RATE = 6, + + // attentionOption % 8: + // 0: Do not quantize + // 1: Q,K: Int8, V: Float + // 2: Q,K,V: Int8 + + // attentionOption / 8: + // 0: don't use flash attention + // 1: use flash attention + ATTENTION_OPTION = 7, + + // size limit of kvcache in memory (for a single layer) + // if the size of kvcache exceeds the limit, it will be moved to disk + KVCACHE_SIZE_LIMIT = 8, + // Op encoder number for commit + OP_ENCODER_NUMBER_FOR_COMMIT = 9, + + // KVCache Info + KVCACHE_INFO = 10, + // mmap allocate file size, KB + MMAP_FILE_SIZE = 11, + USE_CACHED_MMAP = 12, + + // Multi-Thread Load module, default is 0 (don't use other Thread) + INIT_THREAD_NUMBER = 13, + + // Used CPU ids + CPU_CORE_IDS = 14, + + // set CPU threads to use when supports Arm sme2 + CPU_SME2_INSTRUCTIONS = 15, + + // Enable KleidiAI + CPU_ENABLE_KLEIDIAI = 16, + + // Set CPU SME2 NEON division ratio, default is 41 + CPU_SME2_NEON_DIVISION_RATIO = 17, + + // Set SME cores, default is 2, if supports sme + CPU_SME_CORES = 18 + }; + + enum ExternalPathType { + // Path of the kvcache directory + EXTERNAL_PATH_KVCACHE_DIR = 0, + + // Mid Buffer Cache File + EXTERNAL_FEATUREMAP_DIR = 1, + + // Weight Buffer Cache File + EXTERNAL_WEIGHT_DIR = 2, + + // Path of the NPU Model directory + EXTERNAL_NPU_FILE_DIR = 3, + + // Path of the kvcache directory + EXTERNAL_PATH_PREFIXCACHE_DIR = 4, + + // Other types ... + }; + + enum GeometryComputeMask { + // Support Region Fuse + GEOMETRCOMPUTEMASK_FUSEREGION = 1 << 0, + + // Support Region Fuse to input with multi-region, eg: pad + concat + GEOMETRCOMPUTEMASK_FUSEREGION_MULTI = 1 << 1, + + // Use loop instead of raster + compute if possible + GEOMETRCOMPUTEMASK_USELOOP = 1 << 2, + + // Support Geometry Cache, if shape changed, will try recompute, and then run compute if failed + GEOMETRCOMPUTEMASK_OPENCACHE = 1 << 3, + + // Full option open mask, for example, if want to close useloop, can set mask as (GEOMETRCOMPUTEMASK_ALL - GEOMETRCOMPUTEMASK_USELOOP) + GEOMETRCOMPUTEMASK_ALL = 0xFFFF, + }; + + /** + * @brief The API shoud be called before create session. + * @param hint Hint type + * @param value Hint value + * @param size Hint value size(when use a ptr) + */ + void setSessionHint(HintMode hint, int value); + void setSessionHint(HintMode hint, int* value, size_t size); +public: + /** + * @brief create runtimeInfo separately with schedule config. + * @param configs session schedule configs. + */ + static RuntimeInfo createRuntime(const std::vector& configs); + + /** + * @brief create session with schedule config. created session will be managed in net. + * @param config session schedule config. + * @return created session if success, NULL otherwise. + */ + Session* createSession(const ScheduleConfig& config); + + /** + * @brief create session with schedule config and user-specified runtime. + * @param config session schedule config, runtime runtimeInfo used by the created session. + * @return created session if success, NULL otherwise. + */ + Session* createSession(const ScheduleConfig& config, const RuntimeInfo& runtime); + + /** + * @brief create multi-path session with schedule configs. created session will be managed in net. + * @param configs session schedule configs. + * @return created session if success, NULL otherwise. + */ + Session* createMultiPathSession(const std::vector& configs); + + /** + * @brief create multi-path session with schedule configs and user-specified runtime. + created session will be managed in net. + * @param configs session schedule configs. + * @return created session if success, NULL otherwise. + */ + Session* createMultiPathSession(const std::vector& configs, const RuntimeInfo& runtime); + + /** + * @brief release session. + * @param session given session. + * @return true if given session is held by net and is freed. + */ + bool releaseSession(Session* session); + + /** + * @brief call this function to get tensors ready. output tensor buffer (host or deviceId) should be retrieved + * after resize of any input tensor. + * @param session given session. + */ + void resizeSession(Session* session); + + /** + * @brief call this function to get tensors ready. output tensor buffer (host or deviceId) should be retrieved + * after resize of any input tensor. + * @param session given session. + * @param needRelloc, 1 means need realloc. + */ + void resizeSession(Session* session, int needRelloc); + + + /** + * @brief call this function if don't need resize or create session any more, it will save a few memory that equal + * to the size of model buffer + */ + void releaseModel(); + + /** + * @brief Get the model buffer for user to save + * @return std::make_pair(modelBuffer, modelSize). + * @example: + * std::ofstream output("trainResult.alinn") + * auto buffer = net->getModelBuffer(); + * output.write((const char*)buffer.first, buffer.second); + */ + std::pair getModelBuffer() const; + + /** + * @brief Get the model's version info. + * @return const char* of model's version info like "2.0.0"; + * If model is not loaded or model no version info, return "version info not found". + */ + const char* getModelVersion() const; + + /** + * @brief update Session's Tensor to model's Const Op + * @param session given session. + * @return result of running. + */ + ErrorCode updateSessionToModel(Session* session); + + /** + * @brief run session. + * @param session given session. + * @return result of running. + */ + ErrorCode runSession(Session* session) const; + + /* + * @brief run session. + * @param session given session. + * @param before callback before each op. return true to run the op; return false to skip the op. + * @param after callback after each op. return true to continue running; return false to interrupt the session. + * @param sync synchronously wait for finish of execution or not. + * @return result of running. + */ + ErrorCode runSessionWithCallBack(const Session* session, const TensorCallBack& before, const TensorCallBack& end, + bool sync = false) const; + + /* + * @brief run session. + * @param session given session. + * @param before callback before each op. return true to run the op; return false to skip the op. + * @param after callback after each op. return true to continue running; return false to interrupt the session. + * @param sync synchronously wait for finish of execution or not. + * @return result of running. + */ + ErrorCode runSessionWithCallBackInfo(const Session* session, const TensorCallBackWithInfo& before, + const TensorCallBackWithInfo& end, bool sync = false) const; + + /** + * @brief get input tensor for given name. + * @param session given session. + * @param name given name. if NULL, return first input. + * @return tensor if found, NULL otherwise. + */ + Tensor* getSessionInput(const Session* session, const char* name); + /** + * @brief get output tensor for given name. + * @param session given session. + * @param name given name. if NULL, return first output. + * @return tensor if found, NULL otherwise. + */ + Tensor* getSessionOutput(const Session* session, const char* name); + + enum SessionInfoCode { + /** memory session used in MB, float* */ + MEMORY = 0, + + /** float operation needed in session in M, float* */ + FLOPS = 1, + + /** Backends in session in M, int*, length >= 1 + number of configs when create session */ + BACKENDS = 2, + + /** Resize Info, int* , the mean different from API + Interpreter::getSessionInfo: 0: ready to execute, 1: need malloc, 2: need resize + RuntimeManager::getInfo: 0: no resize, 1: re-malloc, 2: resize + */ + RESIZE_STATUS = 3, + + /** Mode / NumberThread, int* */ + THREAD_NUMBER = 4, + + ALL + }; + + /** + * @brief get session info + * @param session given session. + * @param code given info code. + * @param ptr given info ptr, see SessionInfoCode for detail + * @return true if support the code, false otherwise. + */ + bool getSessionInfo(const Session* session, SessionInfoCode code, void* ptr); + + /** + * @brief get all output tensors. + * @param session given session. + * @return all output tensors mapped with name. + */ + const std::map& getSessionOutputAll(const Session* session) const; + /** + * @brief get all input tensors. + * @param session given session. + * @return all input tensors mapped with name. + */ + const std::map& getSessionInputAll(const Session* session) const; + +public: + /** + * @brief resize given tensor. + * @param tensor given tensor. + * @param dims new dims. at most 6 dims. + */ + void resizeTensor(Tensor* tensor, const std::vector& dims); + + /** + * @brief resize given tensor by nchw. + * @param batch / N. + * @param channel / C. + * @param height / H. + * @param width / W + */ + void resizeTensor(Tensor* tensor, int batch, int channel, int height, int width); + + /** + * @brief get backend used to create given tensor. + * @param session given session. + * @param tensor given tensor. + * @return backend used to create given tensor, may be NULL. + */ + const Backend* getBackend(const Session* session, const Tensor* tensor) const; + + /** + * @brief get business code (model identifier). + * @return business code. + */ + const char* bizCode() const; + + /** + * @brief get model UUID + * @return Model UUID. + */ + const char* uuid() const; + +private: + static Interpreter* createFromBufferInternal(Content* net, bool enforceAuth); + + Content* mNet = nullptr; + Interpreter(Content* net); + + Interpreter(const Interpreter&) = delete; + Interpreter(const Interpreter&&) = delete; + Interpreter& operator=(const Interpreter&) = delete; + Interpreter& operator=(const Interpreter&&) = delete; + void waitSessionFinish(const Session* session) const; +#ifdef MNN_INTERNAL_ENABLED + void logForRunSession(const Session* session, float time, const char* api) const; +#endif +}; +} // namespace MNN + +#endif /* Interpreter_hpp */ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNDefine.h b/app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNDefine.h new file mode 100644 index 0000000..29d6b14 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNDefine.h @@ -0,0 +1,79 @@ +// +// MNNDefine.h +// MNN +// +// Created by MNN on 2018/08/09. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNNDefine_h +#define MNNDefine_h + +#include +#include + +#if defined(__APPLE__) +#include +#if TARGET_OS_IPHONE +#define MNN_BUILD_FOR_IOS +#endif +#endif + +#ifdef MNN_USE_LOGCAT +#if defined(__OHOS__) +#include +#define MNN_ERROR(format, ...) {char logtmp[4096]; snprintf(logtmp, 4096, format, ##__VA_ARGS__); OH_LOG_Print(LOG_APP, LOG_ERROR, LOG_DOMAIN, "MNNJNI", (const char*)logtmp);} +#define MNN_PRINT(format, ...) {char logtmp[4096]; snprintf(logtmp, 4096, format, ##__VA_ARGS__); OH_LOG_Print(LOG_APP, LOG_DEBUG, LOG_DOMAIN, "MNNJNI", (const char*)logtmp);} +#else +#include +#define MNN_ERROR(format, ...) __android_log_print(ANDROID_LOG_ERROR, "MNNJNI", format, ##__VA_ARGS__) +#define MNN_PRINT(format, ...) __android_log_print(ANDROID_LOG_INFO, "MNNJNI", format, ##__VA_ARGS__) +#endif +#elif defined MNN_BUILD_FOR_IOS +// on iOS, stderr prints to XCode debug area and syslog prints Console. You need both. +#include +#define MNN_PRINT(format, ...) syslog(LOG_WARNING, format, ##__VA_ARGS__); fprintf(stderr, format, ##__VA_ARGS__) +#define MNN_ERROR(format, ...) syslog(LOG_WARNING, format, ##__VA_ARGS__); fprintf(stderr, format, ##__VA_ARGS__) +#else +#define MNN_PRINT(format, ...) printf(format, ##__VA_ARGS__) +#define MNN_ERROR(format, ...) printf(format, ##__VA_ARGS__) +#endif + +#ifdef DEBUG +#define MNN_ASSERT(x) \ + { \ + int res = (x); \ + if (!res) { \ + MNN_ERROR("Error for %s, %d\n", __FILE__, __LINE__); \ + assert(res); \ + } \ + } +#else +#define MNN_ASSERT(x) +#endif + +#define FUNC_PRINT(x) MNN_PRINT(#x "=%d in %s, %d \n", x, __func__, __LINE__); +#define FUNC_PRINT_ALL(x, type) MNN_PRINT(#x "=" #type " %" #type " in %s, %d \n", x, __func__, __LINE__); + +#define MNN_CHECK(success, log) \ +if(!(success)){ \ +MNN_ERROR("Check failed: %s ==> %s\n", #success, #log); \ +} +#if defined(_MSC_VER) +#if defined(BUILDING_MNN_DLL) +#define MNN_PUBLIC __declspec(dllexport) +#elif defined(USING_MNN_DLL) +#define MNN_PUBLIC __declspec(dllimport) +#else +#define MNN_PUBLIC +#endif +#else +#define MNN_PUBLIC __attribute__((visibility("default"))) +#endif +#define STR_IMP(x) #x +#define STR(x) STR_IMP(x) +#define MNN_VERSION_MAJOR 3 +#define MNN_VERSION_MINOR 6 +#define MNN_VERSION_PATCH 1 +#define MNN_VERSION STR(MNN_VERSION_MAJOR) "." STR(MNN_VERSION_MINOR) "." STR(MNN_VERSION_PATCH) +#endif /* MNNDefine_h */ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNForwardType.h b/app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNForwardType.h new file mode 100644 index 0000000..60e3266 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNForwardType.h @@ -0,0 +1,129 @@ +// +// MNNForwardType.h +// MNN +// +// Created by MNN on 2019/01/19. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNNForwardType_h +#define MNNForwardType_h +#include +#include + +typedef enum { + MNN_FORWARD_CPU = 0, + + /* + Firtly find the first available backends not equal to CPU + If no other backends, use cpu + */ + MNN_FORWARD_AUTO = 4, + + /*Hand write metal*/ + MNN_FORWARD_METAL = 1, + + /*NVIDIA GPU API*/ + MNN_FORWARD_CUDA = 2, + + /*Moore Threads GPU API*/ + MNN_FORWARD_MUSA = 15, + + /*Android / Common Device GPU API*/ + MNN_FORWARD_OPENCL = 3, + MNN_FORWARD_OPENGL = 6, + MNN_FORWARD_VULKAN = 7, + + /*Android 8.1's NNAPI or CoreML for ios*/ + MNN_FORWARD_NN = 5, + + /*User can use API from Backend.hpp to add or search Backend*/ + MNN_FORWARD_USER_0 = 8, + MNN_FORWARD_USER_1 = 9, + MNN_FORWARD_HEXAGON = 10, + MNN_FORWARD_USER_3 = 11, + + MNN_FORWARD_ALL = 12, + + /* Apply arm extension instruction set to accelerate some Ops, this forward type + is only used in MNN internal, and will be active automatically when user set forward type + to be MNN_FORWARD_CPU and extension instruction set is valid on hardware. + */ + MNN_FORWARD_CPU_EXTENSION = 13, + // use for shared memory on android device + + MNN_MEMORY_AHARDWAREBUFFER = 14, + + /* For Offline Convert*/ + MNN_CONVERT_QNN = 32, + MNN_CONVERT_NEUROPILOT = 33, + MNN_CONVERT_COREML = 34, + +} MNNForwardType; + +typedef enum { + // For the OpenCL backend, all five of the following options are valid. The user is allowed to enable any one of them. + // For the Vulkan backend, only options MNN_GPU_TUNING_NONE, MNN_GPU_TUNING_HEAVY, and MNN_GPU_TUNING_WIDE are valid. The user is allowed to enable any one of these three. + MNN_GPU_TUNING_NONE = 1 << 0, /* Forbidden tuning, performance not good.(OpenCL/Vulkan) */ + MNN_GPU_TUNING_HEAVY = 1 << 1, /* Heavily tuning, usually not suggested.(OpenCL/Vulkan) */ + MNN_GPU_TUNING_WIDE = 1 << 2, /* Widely tuning, performance good. Default.(OpenCL/Vulkan) */ + MNN_GPU_TUNING_NORMAL = 1 << 3, /* Normal tuning, performance may be ok.(OpenCL) */ + MNN_GPU_TUNING_FAST = 1 << 4, /* Fast tuning, performance may not good.(OpenCL) */ + + // For the OpenCL backend, the following two options are both valid. The user could try OpenCL_MEMORY_BUFFER and OpenCL_MEMORY_IMAGE both, and then choose the better one based on performance. + // For the Vulkan backend, neither option is valid. The user uses the CMake option MNN_VULKAN_IMAGE to select between image memory mode and buffer memory mode. + MNN_GPU_MEMORY_BUFFER = 1 << 6, /* OpenCL_MEMORY_BUFFER */ + MNN_GPU_MEMORY_IMAGE = 1 << 7, /* OpenCL_MEMORY_IMAGE */ + + // For the OpenCL backend, the following two options are effective only on Qualcomm GPUs. When using a Qualcomm GPU, the user could try both options and choose the better one based on performance. + // For the Vulkan backend, only option MNN_GPU_RECORD_BATCH is valid. When MNN_GPU_RECORD_BATCH is enabled, all ops would share one commandBuffer. + MNN_GPU_RECORD_OP = 1 << 8, /* The kernels in one op execution record into one recording.(OpenCL) */ + MNN_GPU_RECORD_BATCH = 1 << 9, /* 10 kernels record into one recording.(OpenCL) All ops share one commandBuffer.(Vulkan) */ +} MNNGpuMode; + +#ifdef __cplusplus +namespace MNN { +struct BackendConfig { + enum MemoryMode { Memory_Normal = 0, Memory_High, Memory_Low }; + + MemoryMode memory = Memory_Normal; + + enum PowerMode { Power_Normal = 0, Power_High, Power_Low }; + + PowerMode power = Power_Normal; + + enum PrecisionMode { Precision_Normal = 0, Precision_High, Precision_Low, Precision_Low_BF16 }; + + PrecisionMode precision = Precision_Normal; + + /** user defined context */ + union { + void* sharedContext = nullptr; + size_t flags; // Valid for CPU Backend + }; +}; + + /** acquire runtime status by Runtime::getCurrentStatus with following keys, + */ + enum RuntimeStatus { + /** + * get status whether this runtime support 16-bits float point arithmetic + */ + STATUS_SUPPORT_FP16, + /** + * get status whether this runtime support dot-product arithmetic + */ + STATUS_SUPPORT_DOT_PRODUCT, + /** + * get status whether this runtime support power-low (means low priority for opencl) + */ + STATUS_SUPPORT_POWER_LOW, + /** + * emum total number + */ + STATUS_COUNT + }; + +}; // namespace MNN +#endif +#endif /* MNNForwardType_h */ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNSharedContext.h b/app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNSharedContext.h new file mode 100644 index 0000000..588d0fc --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/MNNSharedContext.h @@ -0,0 +1,77 @@ +// +// MNNSharedContext.h +// MNN +// +// Created by MNN on 2018/10/11. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNNSharedContext_h +#define MNNSharedContext_h +#include "MNNDefine.h" +#ifdef __cplusplus +extern "C" { +#endif + +#include /*uint32_t*/ +#ifdef MNN_VULKAN + +struct MNNVulkanContext { + VkInstance pInstance; + VkPhysicalDevice pPhysicalDevice; + VkDevice pDevice; + VkQueue pQueue; + uint32_t iQueueFamilyIndex; +}; + +struct MNNVulkanTensorContent { + VkBuffer buffer; + VkDeviceSize size; + VkDeviceSize offset; + + halide_type_t realType; + int32_t mask; // For future usage +}; + +#endif + +#ifdef MNN_METAL +struct MNNMetalSharedContext { + id device; + id queue; +}; + +struct MNNMetalTensorContent { + id buffer; + int32_t offset; + id texture; + + halide_type_t type; + int32_t mask; + int32_t forFuture[8]; +}; + +MNN_PUBLIC int MNNMetalGetTensorContent(MNNMetalTensorContent* content, void* tensor); +#endif + +#ifdef MNN_USER_SET_DEVICE + +struct MNNDeviceContext { + // When one gpu card has multi devices, choose which device. set deviceId + uint32_t deviceId = 0; + // When has multi gpu cards, choose which card. set platformId + uint32_t platformId = 0; + // User set number of gpu cards + uint32_t platformSize = 0; + // User set OpenCL context ptr + void *contextPtr = nullptr; +}; + +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* MNNSharedContext_h */ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/Matrix.h b/app/entry/src/main/cpp/third_party/mnn/include/MNN/Matrix.h new file mode 100644 index 0000000..0065711 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/Matrix.h @@ -0,0 +1,1615 @@ +/* + * Copyright 2006 The Android Open Source Project + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/* Generated by tools/bookmaker from include/core/Matrix.h and docs/SkMatrix_Reference.bmh + on 2018-07-13 08:15:11. Additional documentation and examples can be found at: + https://skia.org/user/api/SkMatrix_Reference + + You may edit either file directly. Structural changes to public interfaces require + editing both files. After editing docs/SkMatrix_Reference.bmh, run: + bookmaker -b docs -i include/core/Matrix.h -p + to create an updated version of this file. + */ + + +// +// Modified by jiangxiaotang on 2018/09/19. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_Matrix_DEFINED +#define MNN_Matrix_DEFINED + +#include +#include +#include + +namespace MNN { +namespace CV { + +/** \class Matrix + Matrix holds a 3x3 matrix for transforming coordinates. This allows mapping + Point and vectors with translation, scaling, skewing, rotation, and + perspective. + + Matrix elements are in row major order. Matrix does not have a constructor, + so it must be explicitly initialized. setIdentity() initializes Matrix + so it has no effect. setTranslate(), setScale(), setSkew(), setRotate(), set9 and setAll() + initializes all Matrix elements with the corresponding mapping. + + Matrix includes a hidden variable that classifies the type of matrix to + improve performance. Matrix is not thread safe unless getType() is called first. +*/ + +class MNN_PUBLIC Matrix { +public: + Matrix() { + setIdentity(); + } + + /** Sets Matrix to scale by (sx, sy). Returned matrix is: + + | sx 0 0 | + | 0 sy 0 | + | 0 0 1 | + + @param sx horizontal scale factor + @param sy vertical scale factor + @return Matrix with scale + */ + static Matrix MakeScale(float sx, float sy) { + Matrix m; + m.setScale(sx, sy); + return m; + } + + /** Sets Matrix to scale by (scale, scale). Returned matrix is: + + | scale 0 0 | + | 0 scale 0 | + | 0 0 1 | + + @param scale horizontal and vertical scale factor + @return Matrix with scale + */ + static Matrix MakeScale(float scale) { + Matrix m; + m.setScale(scale, scale); + return m; + } + + /** Sets Matrix to translate by (dx, dy). Returned matrix is: + + | 1 0 dx | + | 0 1 dy | + | 0 0 1 | + + @param dx horizontal translation + @param dy vertical translation + @return Matrix with translation + */ + static Matrix MakeTrans(float dx, float dy) { + Matrix m; + m.setTranslate(dx, dy); + return m; + } + + /** Sets Matrix to: + + | scaleX skewX transX | + | skewY scaleY transY | + | pers0 pers1 pers2 | + + @param scaleX horizontal scale factor + @param skewX horizontal skew factor + @param transX horizontal translation + @param skewY vertical skew factor + @param scaleY vertical scale factor + @param transY vertical translation + @param pers0 input x-axis perspective factor + @param pers1 input y-axis perspective factor + @param pers2 perspective scale factor + @return Matrix constructed from parameters + */ + static Matrix MakeAll(float scaleX, float skewX, float transX, float skewY, float scaleY, float transY, float pers0, + float pers1, float pers2) { + Matrix m; + m.setAll(scaleX, skewX, transX, skewY, scaleY, transY, pers0, pers1, pers2); + return m; + } + + /** \enum Matrix::TypeMask + Enum of bit fields for mask returned by getType(). + Used to identify the complexity of Matrix, to optimize performance. + */ + enum TypeMask { + kIdentity_Mask = 0, //!< identity Matrix; all bits clear + kTranslate_Mask = 0x01, //!< translation Matrix + kScale_Mask = 0x02, //!< scale Matrix + kAffine_Mask = 0x04, //!< skew or rotate Matrix + kPerspective_Mask = 0x08, //!< perspective Matrix + }; + + /** Returns a bit field describing the transformations the matrix may + perform. The bit field is computed conservatively, so it may include + false positives. For example, when kPerspective_Mask is set, all + other bits are set. + + @return kIdentity_Mask, or combinations of: kTranslate_Mask, kScale_Mask, + kAffine_Mask, kPerspective_Mask + */ + TypeMask getType() const { + if (fTypeMask & kUnknown_Mask) { + fTypeMask = this->computeTypeMask(); + } + // only return the public masks + return (TypeMask)(fTypeMask & 0xF); + } + + /** Returns true if Matrix is identity. Identity matrix is: + + | 1 0 0 | + | 0 1 0 | + | 0 0 1 | + + @return true if Matrix has no effect + */ + bool isIdentity() const { + return this->getType() == 0; + } + + /** Returns true if Matrix at most scales and translates. Matrix may be identity, + contain only scale elements, only translate elements, or both. Matrix form is: + + | scale-x 0 translate-x | + | 0 scale-y translate-y | + | 0 0 1 | + + @return true if Matrix is identity; or scales, translates, or both + */ + bool isScaleTranslate() const { + return !(this->getType() & ~(kScale_Mask | kTranslate_Mask)); + } + + /** Returns true if Matrix is identity, or translates. Matrix form is: + + | 1 0 translate-x | + | 0 1 translate-y | + | 0 0 1 | + + @return true if Matrix is identity, or translates + */ + bool isTranslate() const { + return !(this->getType() & ~(kTranslate_Mask)); + } + + /** Returns true Matrix maps Rect to another Rect. If true, Matrix is identity, + or scales, or rotates a multiple of 90 degrees, or mirrors on axes. In all + cases, Matrix may also have translation. Matrix form is either: + + | scale-x 0 translate-x | + | 0 scale-y translate-y | + | 0 0 1 | + + or + + | 0 rotate-x translate-x | + | rotate-y 0 translate-y | + | 0 0 1 | + + for non-zero values of scale-x, scale-y, rotate-x, and rotate-y. + + Also called preservesAxisAlignment(); use the one that provides better inline + documentation. + + @return true if Matrix maps one Rect into another + */ + bool rectStaysRect() const { + if (fTypeMask & kUnknown_Mask) { + fTypeMask = this->computeTypeMask(); + } + return (fTypeMask & kRectStaysRect_Mask) != 0; + } + + /** Returns true Matrix maps Rect to another Rect. If true, Matrix is identity, + or scales, or rotates a multiple of 90 degrees, or mirrors on axes. In all + cases, Matrix may also have translation. Matrix form is either: + + | scale-x 0 translate-x | + | 0 scale-y translate-y | + | 0 0 1 | + + or + + | 0 rotate-x translate-x | + | rotate-y 0 translate-y | + | 0 0 1 | + + for non-zero values of scale-x, scale-y, rotate-x, and rotate-y. + + Also called rectStaysRect(); use the one that provides better inline + documentation. + + @return true if Matrix maps one Rect into another + */ + bool preservesAxisAlignment() const { + return this->rectStaysRect(); + } + + /** Matrix organizes its values in row order. These members correspond to + each value in Matrix. + */ + static constexpr int kMScaleX = 0; //!< horizontal scale factor + static constexpr int kMSkewX = 1; //!< horizontal skew factor + static constexpr int kMTransX = 2; //!< horizontal translation + static constexpr int kMSkewY = 3; //!< vertical skew factor + static constexpr int kMScaleY = 4; //!< vertical scale factor + static constexpr int kMTransY = 5; //!< vertical translation + static constexpr int kMPersp0 = 6; //!< input x perspective factor + static constexpr int kMPersp1 = 7; //!< input y perspective factor + static constexpr int kMPersp2 = 8; //!< perspective bias + + /** Affine arrays are in column major order to match the matrix used by + PDF and XPS. + */ + static constexpr int kAScaleX = 0; //!< horizontal scale factor + static constexpr int kASkewY = 1; //!< vertical skew factor + static constexpr int kASkewX = 2; //!< horizontal skew factor + static constexpr int kAScaleY = 3; //!< vertical scale factor + static constexpr int kATransX = 4; //!< horizontal translation + static constexpr int kATransY = 5; //!< vertical translation + + /** Returns one matrix value. Asserts if index is out of range and SK_DEBUG is + defined. + + @param index one of: kMScaleX, kMSkewX, kMTransX, kMSkewY, kMScaleY, kMTransY, + kMPersp0, kMPersp1, kMPersp2 + @return value corresponding to index + */ + float operator[](int index) const { + MNN_ASSERT((unsigned)index < 9); + return fMat[index]; + } + + /** Returns one matrix value. Asserts if index is out of range and SK_DEBUG is + defined. + + @param index one of: kMScaleX, kMSkewX, kMTransX, kMSkewY, kMScaleY, kMTransY, + kMPersp0, kMPersp1, kMPersp2 + @return value corresponding to index + */ + float get(int index) const { + MNN_ASSERT((unsigned)index < 9); + return fMat[index]; + } + + /** Returns scale factor multiplied by x-axis input, contributing to x-axis output. + With mapPoints(), scales Point along the x-axis. + + @return horizontal scale factor + */ + float getScaleX() const { + return fMat[kMScaleX]; + } + + /** Returns scale factor multiplied by y-axis input, contributing to y-axis output. + With mapPoints(), scales Point along the y-axis. + + @return vertical scale factor + */ + float getScaleY() const { + return fMat[kMScaleY]; + } + + /** Returns scale factor multiplied by x-axis input, contributing to y-axis output. + With mapPoints(), skews Point along the y-axis. + Skewing both axes can rotate Point. + + @return vertical skew factor + */ + float getSkewY() const { + return fMat[kMSkewY]; + } + + /** Returns scale factor multiplied by y-axis input, contributing to x-axis output. + With mapPoints(), skews Point along the x-axis. + Skewing both axes can rotate Point. + + @return horizontal scale factor + */ + float getSkewX() const { + return fMat[kMSkewX]; + } + + /** Returns translation contributing to x-axis output. + With mapPoints(), moves Point along the x-axis. + + @return horizontal translation factor + */ + float getTranslateX() const { + return fMat[kMTransX]; + } + + /** Returns translation contributing to y-axis output. + With mapPoints(), moves Point along the y-axis. + + @return vertical translation factor + */ + float getTranslateY() const { + return fMat[kMTransY]; + } + + /** Returns factor scaling input x-axis relative to input y-axis. + + @return input x-axis perspective factor + */ + float getPerspX() const { + return fMat[kMPersp0]; + } + + /** Returns factor scaling input y-axis relative to input x-axis. + + @return input y-axis perspective factor + */ + float getPerspY() const { + return fMat[kMPersp1]; + } + + /** Returns writable Matrix value. Asserts if index is out of range and SK_DEBUG is + defined. Clears internal cache anticipating that caller will change Matrix value. + + Next call to read Matrix state may recompute cache; subsequent writes to Matrix + value must be followed by dirtyMatrixTypeCache(). + + @param index one of: kMScaleX, kMSkewX, kMTransX, kMSkewY, kMScaleY, kMTransY, + kMPersp0, kMPersp1, kMPersp2 + @return writable value corresponding to index + */ + float& operator[](int index) { + MNN_ASSERT((unsigned)index < 9); + this->setTypeMask(kUnknown_Mask); + return fMat[index]; + } + + /** Sets Matrix value. Asserts if index is out of range and SK_DEBUG is + defined. Safer than operator[]; internal cache is always maintained. + + @param index one of: kMScaleX, kMSkewX, kMTransX, kMSkewY, kMScaleY, kMTransY, + kMPersp0, kMPersp1, kMPersp2 + @param value scalar to store in Matrix + */ + void set(int index, float value) { + MNN_ASSERT((unsigned)index < 9); + fMat[index] = value; + this->setTypeMask(kUnknown_Mask); + } + + /** Sets horizontal scale factor. + + @param v horizontal scale factor to store + */ + void setScaleX(float v) { + this->set(kMScaleX, v); + } + + /** Sets vertical scale factor. + + @param v vertical scale factor to store + */ + void setScaleY(float v) { + this->set(kMScaleY, v); + } + + /** Sets vertical skew factor. + + @param v vertical skew factor to store + */ + void setSkewY(float v) { + this->set(kMSkewY, v); + } + + /** Sets horizontal skew factor. + + @param v horizontal skew factor to store + */ + void setSkewX(float v) { + this->set(kMSkewX, v); + } + + /** Sets horizontal translation. + + @param v horizontal translation to store + */ + void setTranslateX(float v) { + this->set(kMTransX, v); + } + + /** Sets vertical translation. + + @param v vertical translation to store + */ + void setTranslateY(float v) { + this->set(kMTransY, v); + } + + /** Sets input x-axis perspective factor, which causes mapXY() to vary input x-axis values + inversely proportional to input y-axis values. + + @param v perspective factor + */ + void setPerspX(float v) { + this->set(kMPersp0, v); + } + + /** Sets input y-axis perspective factor, which causes mapXY() to vary input y-axis values + inversely proportional to input x-axis values. + + @param v perspective factor + */ + void setPerspY(float v) { + this->set(kMPersp1, v); + } + + /** Sets all values from parameters. Sets matrix to: + + | scaleX skewX transX | + | skewY scaleY transY | + | persp0 persp1 persp2 | + + @param scaleX horizontal scale factor to store + @param skewX horizontal skew factor to store + @param transX horizontal translation to store + @param skewY vertical skew factor to store + @param scaleY vertical scale factor to store + @param transY vertical translation to store + @param persp0 input x-axis values perspective factor to store + @param persp1 input y-axis values perspective factor to store + @param persp2 perspective scale factor to store + */ + void setAll(float scaleX, float skewX, float transX, float skewY, float scaleY, float transY, float persp0, + float persp1, float persp2) { + fMat[kMScaleX] = scaleX; + fMat[kMSkewX] = skewX; + fMat[kMTransX] = transX; + fMat[kMSkewY] = skewY; + fMat[kMScaleY] = scaleY; + fMat[kMTransY] = transY; + fMat[kMPersp0] = persp0; + fMat[kMPersp1] = persp1; + fMat[kMPersp2] = persp2; + this->setTypeMask(kUnknown_Mask); + } + + /** Copies nine scalar values contained by Matrix into buffer, in member value + ascending order: kMScaleX, kMSkewX, kMTransX, kMSkewY, kMScaleY, kMTransY, + kMPersp0, kMPersp1, kMPersp2. + + @param buffer storage for nine scalar values + */ + void get9(float buffer[9]) const { + memcpy(buffer, fMat, 9 * sizeof(float)); + } + + /** Sets Matrix to nine scalar values in buffer, in member value ascending order: + kMScaleX, kMSkewX, kMTransX, kMSkewY, kMScaleY, kMTransY, kMPersp0, kMPersp1, + kMPersp2. + + Sets matrix to: + + | buffer[0] buffer[1] buffer[2] | + | buffer[3] buffer[4] buffer[5] | + | buffer[6] buffer[7] buffer[8] | + + In the future, set9 followed by get9 may not return the same values. Since Matrix + maps non-homogeneous coordinates, scaling all nine values produces an equivalent + transformation, possibly improving precision. + + @param buffer nine scalar values + */ + void set9(const float buffer[9]); + + /** Sets Matrix to identity; which has no effect on mapped Point. Sets Matrix to: + + | 1 0 0 | + | 0 1 0 | + | 0 0 1 | + + Also called setIdentity(); use the one that provides better inline + documentation. + */ + void reset(); + + /** Sets Matrix to identity; which has no effect on mapped Point. Sets Matrix to: + + | 1 0 0 | + | 0 1 0 | + | 0 0 1 | + + Also called reset(); use the one that provides better inline + documentation. + */ + void setIdentity() { + this->reset(); + } + + /** Sets Matrix to translate by (dx, dy). + + @param dx horizontal translation + @param dy vertical translation + */ + void setTranslate(float dx, float dy); + + /** Sets Matrix to scale by sx and sy, about a pivot point at (px, py). + The pivot point is unchanged when mapped with Matrix. + + @param sx horizontal scale factor + @param sy vertical scale factor + @param px pivot x + @param py pivot y + */ + void setScale(float sx, float sy, float px, float py); + + /** Sets Matrix to scale by sx and sy about at pivot point at (0, 0). + + @param sx horizontal scale factor + @param sy vertical scale factor + */ + void setScale(float sx, float sy); + + /** Sets Matrix to rotate by degrees about a pivot point at (px, py). + The pivot point is unchanged when mapped with Matrix. + + Positive degrees rotates clockwise. + + @param degrees angle of axes relative to upright axes + @param px pivot x + @param py pivot y + */ + void setRotate(float degrees, float px, float py); + + /** Sets Matrix to rotate by degrees about a pivot point at (0, 0). + Positive degrees rotates clockwise. + + @param degrees angle of axes relative to upright axes + */ + void setRotate(float degrees); + + /** Sets Matrix to rotate by sinValue and cosValue, about a pivot point at (px, py). + The pivot point is unchanged when mapped with Matrix. + + Vector (sinValue, cosValue) describes the angle of rotation relative to (0, 1). + Vector length specifies scale. + + @param sinValue rotation vector x-axis component + @param cosValue rotation vector y-axis component + @param px pivot x-axis + @param py pivot y-axis + */ + void setSinCos(float sinValue, float cosValue, float px, float py); + + /** Sets Matrix to rotate by sinValue and cosValue, about a pivot point at (0, 0). + + Vector (sinValue, cosValue) describes the angle of rotation relative to (0, 1). + Vector length specifies scale. + + @param sinValue rotation vector x-axis component + @param cosValue rotation vector y-axis component + */ + void setSinCos(float sinValue, float cosValue); + + /** Sets Matrix to skew by kx and ky, about a pivot point at (px, py). + The pivot point is unchanged when mapped with Matrix. + + @param kx horizontal skew factor + @param ky vertical skew factor + @param px pivot x + @param py pivot y + */ + void setSkew(float kx, float ky, float px, float py); + + /** Sets Matrix to skew by kx and ky, about a pivot point at (0, 0). + + @param kx horizontal skew factor + @param ky vertical skew factor + */ + void setSkew(float kx, float ky); + + /** Sets Matrix to Matrix a multiplied by Matrix b. Either a or b may be this. + + Given: + + | A B C | | J K L | + a = | D E F |, b = | M N O | + | G H I | | P Q R | + + sets Matrix to: + + | A B C | | J K L | | AJ+BM+CP AK+BN+CQ AL+BO+CR | + a * b = | D E F | * | M N O | = | DJ+EM+FP DK+EN+FQ DL+EO+FR | + | G H I | | P Q R | | GJ+HM+IP GK+HN+IQ GL+HO+IR | + + @param a Matrix on left side of multiply expression + @param b Matrix on right side of multiply expression + */ + void setConcat(const Matrix& a, const Matrix& b); + + /** Sets Matrix to Matrix multiplied by Matrix constructed from translation (dx, dy). + This can be thought of as moving the point to be mapped before applying Matrix. + + Given: + + | A B C | | 1 0 dx | + Matrix = | D E F |, T(dx, dy) = | 0 1 dy | + | G H I | | 0 0 1 | + + sets Matrix to: + + | A B C | | 1 0 dx | | A B A*dx+B*dy+C | + Matrix * T(dx, dy) = | D E F | | 0 1 dy | = | D E D*dx+E*dy+F | + | G H I | | 0 0 1 | | G H G*dx+H*dy+I | + + @param dx x-axis translation before applying Matrix + @param dy y-axis translation before applying Matrix + */ + void preTranslate(float dx, float dy); + + /** Sets Matrix to Matrix multiplied by Matrix constructed from scaling by (sx, sy) + about pivot point (px, py). + This can be thought of as scaling about a pivot point before applying Matrix. + + Given: + + | A B C | | sx 0 dx | + Matrix = | D E F |, S(sx, sy, px, py) = | 0 sy dy | + | G H I | | 0 0 1 | + + where + + dx = px - sx * px + dy = py - sy * py + + sets Matrix to: + + | A B C | | sx 0 dx | | A*sx B*sy A*dx+B*dy+C | + Matrix * S(sx, sy, px, py) = | D E F | | 0 sy dy | = | D*sx E*sy D*dx+E*dy+F | + | G H I | | 0 0 1 | | G*sx H*sy G*dx+H*dy+I | + + @param sx horizontal scale factor + @param sy vertical scale factor + @param px pivot x + @param py pivot y + */ + void preScale(float sx, float sy, float px, float py); + + /** Sets Matrix to Matrix multiplied by Matrix constructed from scaling by (sx, sy) + about pivot point (0, 0). + This can be thought of as scaling about the origin before applying Matrix. + + Given: + + | A B C | | sx 0 0 | + Matrix = | D E F |, S(sx, sy) = | 0 sy 0 | + | G H I | | 0 0 1 | + + sets Matrix to: + + | A B C | | sx 0 0 | | A*sx B*sy C | + Matrix * S(sx, sy) = | D E F | | 0 sy 0 | = | D*sx E*sy F | + | G H I | | 0 0 1 | | G*sx H*sy I | + + @param sx horizontal scale factor + @param sy vertical scale factor + */ + void preScale(float sx, float sy); + + /** Sets Matrix to Matrix multiplied by Matrix constructed from rotating by degrees + about pivot point (px, py). + This can be thought of as rotating about a pivot point before applying Matrix. + + Positive degrees rotates clockwise. + + Given: + + | A B C | | c -s dx | + Matrix = | D E F |, R(degrees, px, py) = | s c dy | + | G H I | | 0 0 1 | + + where + + c = cos(degrees) + s = sin(degrees) + dx = s * py + (1 - c) * px + dy = -s * px + (1 - c) * py + + sets Matrix to: + + | A B C | | c -s dx | | Ac+Bs -As+Bc A*dx+B*dy+C | + Matrix * R(degrees, px, py) = | D E F | | s c dy | = | Dc+Es -Ds+Ec D*dx+E*dy+F | + | G H I | | 0 0 1 | | Gc+Hs -Gs+Hc G*dx+H*dy+I | + + @param degrees angle of axes relative to upright axes + @param px pivot x + @param py pivot y + */ + void preRotate(float degrees, float px, float py); + + /** Sets Matrix to Matrix multiplied by Matrix constructed from rotating by degrees + about pivot point (0, 0). + This can be thought of as rotating about the origin before applying Matrix. + + Positive degrees rotates clockwise. + + Given: + + | A B C | | c -s 0 | + Matrix = | D E F |, R(degrees, px, py) = | s c 0 | + | G H I | | 0 0 1 | + + where + + c = cos(degrees) + s = sin(degrees) + + sets Matrix to: + + | A B C | | c -s 0 | | Ac+Bs -As+Bc C | + Matrix * R(degrees, px, py) = | D E F | | s c 0 | = | Dc+Es -Ds+Ec F | + | G H I | | 0 0 1 | | Gc+Hs -Gs+Hc I | + + @param degrees angle of axes relative to upright axes + */ + void preRotate(float degrees); + + /** Sets Matrix to Matrix multiplied by Matrix constructed from skewing by (kx, ky) + about pivot point (px, py). + This can be thought of as skewing about a pivot point before applying Matrix. + + Given: + + | A B C | | 1 kx dx | + Matrix = | D E F |, K(kx, ky, px, py) = | ky 1 dy | + | G H I | | 0 0 1 | + + where + + dx = -kx * py + dy = -ky * px + + sets Matrix to: + + | A B C | | 1 kx dx | | A+B*ky A*kx+B A*dx+B*dy+C | + Matrix * K(kx, ky, px, py) = | D E F | | ky 1 dy | = | D+E*ky D*kx+E D*dx+E*dy+F | + | G H I | | 0 0 1 | | G+H*ky G*kx+H G*dx+H*dy+I | + + @param kx horizontal skew factor + @param ky vertical skew factor + @param px pivot x + @param py pivot y + */ + void preSkew(float kx, float ky, float px, float py); + + /** Sets Matrix to Matrix multiplied by Matrix constructed from skewing by (kx, ky) + about pivot point (0, 0). + This can be thought of as skewing about the origin before applying Matrix. + + Given: + + | A B C | | 1 kx 0 | + Matrix = | D E F |, K(kx, ky) = | ky 1 0 | + | G H I | | 0 0 1 | + + sets Matrix to: + + | A B C | | 1 kx 0 | | A+B*ky A*kx+B C | + Matrix * K(kx, ky) = | D E F | | ky 1 0 | = | D+E*ky D*kx+E F | + | G H I | | 0 0 1 | | G+H*ky G*kx+H I | + + @param kx horizontal skew factor + @param ky vertical skew factor + */ + void preSkew(float kx, float ky); + + /** Sets Matrix to Matrix multiplied by Matrix other. + This can be thought of mapping by other before applying Matrix. + + Given: + + | A B C | | J K L | + Matrix = | D E F |, other = | M N O | + | G H I | | P Q R | + + sets Matrix to: + + | A B C | | J K L | | AJ+BM+CP AK+BN+CQ AL+BO+CR | + Matrix * other = | D E F | * | M N O | = | DJ+EM+FP DK+EN+FQ DL+EO+FR | + | G H I | | P Q R | | GJ+HM+IP GK+HN+IQ GL+HO+IR | + + @param other Matrix on right side of multiply expression + */ + void preConcat(const Matrix& other); + + /** Sets Matrix to Matrix constructed from translation (dx, dy) multiplied by Matrix. + This can be thought of as moving the point to be mapped after applying Matrix. + + Given: + + | J K L | | 1 0 dx | + Matrix = | M N O |, T(dx, dy) = | 0 1 dy | + | P Q R | | 0 0 1 | + + sets Matrix to: + + | 1 0 dx | | J K L | | J+dx*P K+dx*Q L+dx*R | + T(dx, dy) * Matrix = | 0 1 dy | | M N O | = | M+dy*P N+dy*Q O+dy*R | + | 0 0 1 | | P Q R | | P Q R | + + @param dx x-axis translation after applying Matrix + @param dy y-axis translation after applying Matrix + */ + void postTranslate(float dx, float dy); + + /** Sets Matrix to Matrix constructed from scaling by (sx, sy) about pivot point + (px, py), multiplied by Matrix. + This can be thought of as scaling about a pivot point after applying Matrix. + + Given: + + | J K L | | sx 0 dx | + Matrix = | M N O |, S(sx, sy, px, py) = | 0 sy dy | + | P Q R | | 0 0 1 | + + where + + dx = px - sx * px + dy = py - sy * py + + sets Matrix to: + + | sx 0 dx | | J K L | | sx*J+dx*P sx*K+dx*Q sx*L+dx+R | + S(sx, sy, px, py) * Matrix = | 0 sy dy | | M N O | = | sy*M+dy*P sy*N+dy*Q sy*O+dy*R | + | 0 0 1 | | P Q R | | P Q R | + + @param sx horizontal scale factor + @param sy vertical scale factor + @param px pivot x + @param py pivot y + */ + void postScale(float sx, float sy, float px, float py); + + /** Sets Matrix to Matrix constructed from scaling by (sx, sy) about pivot point + (0, 0), multiplied by Matrix. + This can be thought of as scaling about the origin after applying Matrix. + + Given: + + | J K L | | sx 0 0 | + Matrix = | M N O |, S(sx, sy) = | 0 sy 0 | + | P Q R | | 0 0 1 | + + sets Matrix to: + + | sx 0 0 | | J K L | | sx*J sx*K sx*L | + S(sx, sy) * Matrix = | 0 sy 0 | | M N O | = | sy*M sy*N sy*O | + | 0 0 1 | | P Q R | | P Q R | + + @param sx horizontal scale factor + @param sy vertical scale factor + */ + void postScale(float sx, float sy); + + /** Sets Matrix to Matrix constructed from scaling by (1/divx, 1/divy) about pivot point (px, py), multiplied by + Matrix. + + Returns false if either divx or divy is zero. + + Given: + + | J K L | | sx 0 0 | + Matrix = | M N O |, I(divx, divy) = | 0 sy 0 | + | P Q R | | 0 0 1 | + + where + + sx = 1 / divx + sy = 1 / divy + + sets Matrix to: + + | sx 0 0 | | J K L | | sx*J sx*K sx*L | + I(divx, divy) * Matrix = | 0 sy 0 | | M N O | = | sy*M sy*N sy*O | + | 0 0 1 | | P Q R | | P Q R | + + @param divx integer divisor for inverse scale in x + @param divy integer divisor for inverse scale in y + @return true on successful scale + */ + bool postIDiv(int divx, int divy); + + /** Sets Matrix to Matrix constructed from rotating by degrees about pivot point + (px, py), multiplied by Matrix. + This can be thought of as rotating about a pivot point after applying Matrix. + + Positive degrees rotates clockwise. + + Given: + + | J K L | | c -s dx | + Matrix = | M N O |, R(degrees, px, py) = | s c dy | + | P Q R | | 0 0 1 | + + where + + c = cos(degrees) + s = sin(degrees) + dx = s * py + (1 - c) * px + dy = -s * px + (1 - c) * py + + sets Matrix to: + + |c -s dx| |J K L| |cJ-sM+dx*P cK-sN+dx*Q cL-sO+dx+R| + R(degrees, px, py) * Matrix = |s c dy| |M N O| = |sJ+cM+dy*P sK+cN+dy*Q sL+cO+dy*R| + |0 0 1| |P Q R| | P Q R| + + @param degrees angle of axes relative to upright axes + @param px pivot x + @param py pivot y + */ + void postRotate(float degrees, float px, float py); + + /** Sets Matrix to Matrix constructed from rotating by degrees about pivot point + (0, 0), multiplied by Matrix. + This can be thought of as rotating about the origin after applying Matrix. + + Positive degrees rotates clockwise. + + Given: + + | J K L | | c -s 0 | + Matrix = | M N O |, R(degrees, px, py) = | s c 0 | + | P Q R | | 0 0 1 | + + where + + c = cos(degrees) + s = sin(degrees) + + sets Matrix to: + + | c -s dx | | J K L | | cJ-sM cK-sN cL-sO | + R(degrees, px, py) * Matrix = | s c dy | | M N O | = | sJ+cM sK+cN sL+cO | + | 0 0 1 | | P Q R | | P Q R | + + @param degrees angle of axes relative to upright axes + */ + void postRotate(float degrees); + + /** Sets Matrix to Matrix constructed from skewing by (kx, ky) about pivot point + (px, py), multiplied by Matrix. + This can be thought of as skewing about a pivot point after applying Matrix. + + Given: + + | J K L | | 1 kx dx | + Matrix = | M N O |, K(kx, ky, px, py) = | ky 1 dy | + | P Q R | | 0 0 1 | + + where + + dx = -kx * py + dy = -ky * px + + sets Matrix to: + + | 1 kx dx| |J K L| |J+kx*M+dx*P K+kx*N+dx*Q L+kx*O+dx+R| + K(kx, ky, px, py) * Matrix = |ky 1 dy| |M N O| = |ky*J+M+dy*P ky*K+N+dy*Q ky*L+O+dy*R| + | 0 0 1| |P Q R| | P Q R| + + @param kx horizontal skew factor + @param ky vertical skew factor + @param px pivot x + @param py pivot y + */ + void postSkew(float kx, float ky, float px, float py); + + /** Sets Matrix to Matrix constructed from skewing by (kx, ky) about pivot point + (0, 0), multiplied by Matrix. + This can be thought of as skewing about the origin after applying Matrix. + + Given: + + | J K L | | 1 kx 0 | + Matrix = | M N O |, K(kx, ky) = | ky 1 0 | + | P Q R | | 0 0 1 | + + sets Matrix to: + + | 1 kx 0 | | J K L | | J+kx*M K+kx*N L+kx*O | + K(kx, ky) * Matrix = | ky 1 0 | | M N O | = | ky*J+M ky*K+N ky*L+O | + | 0 0 1 | | P Q R | | P Q R | + + @param kx horizontal skew factor + @param ky vertical skew factor + */ + void postSkew(float kx, float ky); + + /** Sets Matrix to Matrix other multiplied by Matrix. + This can be thought of mapping by other after applying Matrix. + + Given: + + | J K L | | A B C | + Matrix = | M N O |, other = | D E F | + | P Q R | | G H I | + + sets Matrix to: + + | A B C | | J K L | | AJ+BM+CP AK+BN+CQ AL+BO+CR | + other * Matrix = | D E F | * | M N O | = | DJ+EM+FP DK+EN+FQ DL+EO+FR | + | G H I | | P Q R | | GJ+HM+IP GK+HN+IQ GL+HO+IR | + + @param other Matrix on left side of multiply expression + */ + void postConcat(const Matrix& other); + + /** \enum Matrix::ScaleToFit + ScaleToFit describes how Matrix is constructed to map one Rect to another. + ScaleToFit may allow Matrix to have unequal horizontal and vertical scaling, + or may restrict Matrix to square scaling. If restricted, ScaleToFit specifies + how Matrix maps to the side or center of the destination Rect. + */ + enum ScaleToFit { + kFill_ScaleToFit, //!< scales in x and y to fill destination Rect + kStart_ScaleToFit, //!< scales and aligns to left and top + kCenter_ScaleToFit, //!< scales and aligns to center + kEnd_ScaleToFit, //!< scales and aligns to right and bottom + }; + + /** Sets Matrix to scale and translate src Rect to dst Rect. stf selects whether + mapping completely fills dst or preserves the aspect ratio, and how to align + src within dst. Returns false if src is empty, and sets Matrix to identity. + Returns true if dst is empty, and sets Matrix to: + + | 0 0 0 | + | 0 0 0 | + | 0 0 1 | + + @param src Rect to map from + @param dst Rect to map to + @param stf one of: kFill_ScaleToFit, kStart_ScaleToFit, + kCenter_ScaleToFit, kEnd_ScaleToFit + @return true if Matrix can represent Rect mapping + */ + bool setRectToRect(const Rect& src, const Rect& dst, ScaleToFit stf); + + /** Returns Matrix set to scale and translate src Rect to dst Rect. stf selects + whether mapping completely fills dst or preserves the aspect ratio, and how to + align src within dst. Returns the identity Matrix if src is empty. If dst is + empty, returns Matrix set to: + + | 0 0 0 | + | 0 0 0 | + | 0 0 1 | + + @param src Rect to map from + @param dst Rect to map to + @param stf one of: kFill_ScaleToFit, kStart_ScaleToFit, + kCenter_ScaleToFit, kEnd_ScaleToFit + @return Matrix mapping src to dst + */ + static Matrix MakeRectToRect(const Rect& src, const Rect& dst, ScaleToFit stf) { + Matrix m; + m.setRectToRect(src, dst, stf); + return m; + } + + /** Sets Matrix to map src to dst. count must be zero or greater, and four or less. + + If count is zero, sets Matrix to identity and returns true. + If count is one, sets Matrix to translate and returns true. + If count is two or more, sets Matrix to map Point if possible; returns false + if Matrix cannot be constructed. If count is four, Matrix may include + perspective. + + @param src Point to map from + @param dst Point to map to + @param count number of Point in src and dst + @return true if Matrix was constructed successfully + */ + bool setPolyToPoly(const Point src[], const Point dst[], int count); + + /** Sets inverse to reciprocal matrix, returning true if Matrix can be inverted. + Geometrically, if Matrix maps from source to destination, inverse Matrix + maps from destination to source. If Matrix can not be inverted, inverse is + unchanged. + + @param inverse storage for inverted Matrix; may be nullptr + @return true if Matrix can be inverted + */ + bool invert(Matrix* inverse) const { + // Allow the trivial case to be inlined. + if (this->isIdentity()) { + if (inverse) { + inverse->reset(); + } + return true; + } + return this->invertNonIdentity(inverse); + } + + /** Fills affine with identity values in column major order. + Sets affine to: + + | 1 0 0 | + | 0 1 0 | + + Affine 3x2 matrices in column major order are used by OpenGL and XPS. + + @param affine storage for 3x2 affine matrix + */ + static void SetAffineIdentity(float affine[6]); + + /** Fills affine in column major order. Sets affine to: + + | scale-x skew-x translate-x | + | skew-y scale-y translate-y | + + If Matrix contains perspective, returns false and leaves affine unchanged. + + @param affine storage for 3x2 affine matrix; may be nullptr + @return true if Matrix does not contain perspective + */ + bool asAffine(float affine[6]) const; + + /** Sets Matrix to affine values, passed in column major order. Given affine, + column, then row, as: + + | scale-x skew-x translate-x | + | skew-y scale-y translate-y | + + Matrix is set, row, then column, to: + + | scale-x skew-x translate-x | + | skew-y scale-y translate-y | + | 0 0 1 | + + @param affine 3x2 affine matrix + */ + void setAffine(const float affine[6]); + + /** Maps src Point array of length count to dst Point array of equal or greater + length. Point are mapped by multiplying each Point by Matrix. Given: + + | A B C | | x | + Matrix = | D E F |, pt = | y | + | G H I | | 1 | + + where + + for (i = 0; i < count; ++i) { + x = src[i].fX + y = src[i].fY + } + + each dst Point is computed as: + + |A B C| |x| Ax+By+C Dx+Ey+F + Matrix * pt = |D E F| |y| = |Ax+By+C Dx+Ey+F Gx+Hy+I| = ------- , ------- + |G H I| |1| Gx+Hy+I Gx+Hy+I + + src and dst may point to the same storage. + + @param dst storage for mapped Point + @param src Point to transform + @param count number of Point to transform + */ + void mapPoints(Point dst[], const Point src[], int count) const { + MNN_ASSERT((dst && src && count > 0) || 0 == count); + // no partial overlap + MNN_ASSERT(src == dst || &dst[count] <= &src[0] || &src[count] <= &dst[0]); + this->getMapPtsProc()(*this, dst, src, count); + } + + /** Maps pts Point array of length count in place. Point are mapped by multiplying + each Point by Matrix. Given: + + | A B C | | x | + Matrix = | D E F |, pt = | y | + | G H I | | 1 | + + where + + for (i = 0; i < count; ++i) { + x = pts[i].fX + y = pts[i].fY + } + + each resulting pts Point is computed as: + + |A B C| |x| Ax+By+C Dx+Ey+F + Matrix * pt = |D E F| |y| = |Ax+By+C Dx+Ey+F Gx+Hy+I| = ------- , ------- + |G H I| |1| Gx+Hy+I Gx+Hy+I + + @param pts storage for mapped Point + @param count number of Point to transform + */ + void mapPoints(Point pts[], int count) const { + this->mapPoints(pts, pts, count); + } + + /** Maps Point (x, y) to result. Point is mapped by multiplying by Matrix. Given: + + | A B C | | x | + Matrix = | D E F |, pt = | y | + | G H I | | 1 | + + result is computed as: + + |A B C| |x| Ax+By+C Dx+Ey+F + Matrix * pt = |D E F| |y| = |Ax+By+C Dx+Ey+F Gx+Hy+I| = ------- , ------- + |G H I| |1| Gx+Hy+I Gx+Hy+I + + @param x x-axis value of Point to map + @param y y-axis value of Point to map + @param result storage for mapped Point + */ + void mapXY(float x, float y, Point* result) const { + this->getMapXYProc()(*this, x, y, result); + } + + /** Returns Point (x, y) multiplied by Matrix. Given: + + | A B C | | x | + Matrix = | D E F |, pt = | y | + | G H I | | 1 | + + result is computed as: + + |A B C| |x| Ax+By+C Dx+Ey+F + Matrix * pt = |D E F| |y| = |Ax+By+C Dx+Ey+F Gx+Hy+I| = ------- , ------- + |G H I| |1| Gx+Hy+I Gx+Hy+I + + @param x x-axis value of Point to map + @param y y-axis value of Point to map + @return mapped Point + */ + Point mapXY(float x, float y) const { + Point result; + this->getMapXYProc()(*this, x, y, &result); + return result; + } + + /** Sets dst to bounds of src corners mapped by Matrix. + Returns true if mapped corners are dst corners. + + Returned value is the same as calling rectStaysRect(). + + @param dst storage for bounds of mapped Point + @param src Rect to map + @return true if dst is equivalent to mapped src + */ + bool mapRect(Rect* dst, const Rect& src) const; + + /** Sets rect to bounds of rect corners mapped by Matrix. + Returns true if mapped corners are computed rect corners. + + Returned value is the same as calling rectStaysRect(). + + @param rect rectangle to map, and storage for bounds of mapped corners + @return true if result is equivalent to mapped src + */ + bool mapRect(Rect* rect) const { + return this->mapRect(rect, *rect); + } + + /** Returns bounds of src corners mapped by Matrix. + + @param src rectangle to map + @return mapped bounds + */ + Rect mapRect(const Rect& src) const { + Rect dst; + (void)this->mapRect(&dst, src); + return dst; + } + + /** Sets dst to bounds of src corners mapped by Matrix. If matrix contains + elements other than scale or translate: asserts if SK_DEBUG is defined; + otherwise, results are undefined. + + @param dst storage for bounds of mapped Point + @param src Rect to map + */ + void mapRectScaleTranslate(Rect* dst, const Rect& src) const; + + /** Returns true if Matrix equals m, using an efficient comparison. + + Returns false when the sign of zero values is the different; when one + matrix has positive zero value and the other has negative zero value. + + Returns true even when both Matrix contain NaN. + + NaN never equals any value, including itself. To improve performance, NaN values + are treated as bit patterns that are equal if their bit patterns are equal. + + @param m Matrix to compare + @return true if m and Matrix are represented by identical bit patterns + */ + bool cheapEqualTo(const Matrix& m) const { + return 0 == memcmp(fMat, m.fMat, sizeof(fMat)); + } + + /** Compares a and b; returns true if a and b are numerically equal. Returns true + even if sign of zero values are different. Returns false if either Matrix + contains NaN, even if the other Matrix also contains NaN. + + @param a Matrix to compare + @param b Matrix to compare + @return true if Matrix a and Matrix b are numerically equal + */ + friend MNN_PUBLIC bool operator==(const Matrix& a, const Matrix& b); + + /** Compares a and b; returns true if a and b are not numerically equal. Returns false + even if sign of zero values are different. Returns true if either Matrix + contains NaN, even if the other Matrix also contains NaN. + + @param a Matrix to compare + @param b Matrix to compare + @return true if Matrix a and Matrix b are numerically not equal + */ + friend MNN_PUBLIC bool operator!=(const Matrix& a, const Matrix& b) { + return !(a == b); + } + + /** Writes text representation of Matrix to standard output. Floating point values + are written with limited precision; it may not be possible to reconstruct + original Matrix from output. + */ + void dump() const; + + /** Returns the minimum scaling factor of Matrix by decomposing the scaling and + skewing elements. + Returns -1 if scale factor overflows or Matrix contains perspective. + + @return minimum scale factor + */ + float getMinScale() const; + + /** Returns the maximum scaling factor of Matrix by decomposing the scaling and + skewing elements. + Returns -1 if scale factor overflows or Matrix contains perspective. + + @return maximum scale factor + */ + float getMaxScale() const; + + /** Sets scaleFactors[0] to the minimum scaling factor, and scaleFactors[1] to the + maximum scaling factor. Scaling factors are computed by decomposing + the Matrix scaling and skewing elements. + + Returns true if scaleFactors are found; otherwise, returns false and sets + scaleFactors to undefined values. + + @param scaleFactors storage for minimum and maximum scale factors + @return true if scale factors were computed correctly + */ + bool getMinMaxScales(float scaleFactors[2]) const; + + /** Returns reference to const identity Matrix. Returned Matrix is set to: + + | 1 0 0 | + | 0 1 0 | + | 0 0 1 | + + @return const identity Matrix + */ + static const Matrix& I(); + + /** Returns reference to a const Matrix with invalid values. Returned Matrix is set + to: + + | SK_ScalarMax SK_ScalarMax SK_ScalarMax | + | SK_ScalarMax SK_ScalarMax SK_ScalarMax | + | SK_ScalarMax SK_ScalarMax SK_ScalarMax | + + @return const invalid Matrix + */ + static const Matrix& InvalidMatrix(); + + /** Returns Matrix a multiplied by Matrix b. + + Given: + + | A B C | | J K L | + a = | D E F |, b = | M N O | + | G H I | | P Q R | + + sets Matrix to: + + | A B C | | J K L | | AJ+BM+CP AK+BN+CQ AL+BO+CR | + a * b = | D E F | * | M N O | = | DJ+EM+FP DK+EN+FQ DL+EO+FR | + | G H I | | P Q R | | GJ+HM+IP GK+HN+IQ GL+HO+IR | + + @param a Matrix on left side of multiply expression + @param b Matrix on right side of multiply expression + @return Matrix computed from a times b + */ + static Matrix Concat(const Matrix& a, const Matrix& b) { + Matrix result; + result.setConcat(a, b); + return result; + } + + /** Sets internal cache to unknown state. Use to force update after repeated + modifications to Matrix element reference returned by operator[](int index). + */ + void dirtyMatrixTypeCache() { + this->setTypeMask(kUnknown_Mask); + } + + /** Initializes Matrix with scale and translate elements. + + | sx 0 tx | + | 0 sy ty | + | 0 0 1 | + + @param sx horizontal scale factor to store + @param sy vertical scale factor to store + @param tx horizontal translation to store + @param ty vertical translation to store + */ + void setScaleTranslate(float sx, float sy, float tx, float ty) { + fMat[kMScaleX] = sx; + fMat[kMSkewX] = 0; + fMat[kMTransX] = tx; + + fMat[kMSkewY] = 0; + fMat[kMScaleY] = sy; + fMat[kMTransY] = ty; + + fMat[kMPersp0] = 0; + fMat[kMPersp1] = 0; + fMat[kMPersp2] = 1; + + unsigned mask = 0; + if (sx != 1 || sy != 1) { + mask |= kScale_Mask; + } + if (tx || ty) { + mask |= kTranslate_Mask; + } + this->setTypeMask(mask | kRectStaysRect_Mask); + } + + /** Returns true if all elements of the matrix are finite. Returns false if any + element is infinity, or NaN. + + @return true if matrix has only finite elements + */ + +private: + /** Set if the matrix will map a rectangle to another rectangle. This + can be true if the matrix is scale-only, or rotates a multiple of + 90 degrees. + + This bit will be set on identity matrices + */ + static constexpr int kRectStaysRect_Mask = 0x10; + + /** Set if the perspective bit is valid even though the rest of + the matrix is Unknown. + */ + static constexpr int kOnlyPerspectiveValid_Mask = 0x40; + + static constexpr int kUnknown_Mask = 0x80; + + static constexpr int kORableMasks = kTranslate_Mask | kScale_Mask | kAffine_Mask | kPerspective_Mask; + + static constexpr int kAllMasks = + kTranslate_Mask | kScale_Mask | kAffine_Mask | kPerspective_Mask | kRectStaysRect_Mask; + + float fMat[9]; + mutable uint32_t fTypeMask; + + static void ComputeInv(float dst[9], const float src[9], double invDet, bool isPersp); + + uint8_t computeTypeMask() const; + uint8_t computePerspectiveTypeMask() const; + + void setTypeMask(int mask) { + // allow kUnknown or a valid mask + MNN_ASSERT(kUnknown_Mask == mask || (mask & kAllMasks) == mask || + ((kUnknown_Mask | kOnlyPerspectiveValid_Mask) & mask) == + (kUnknown_Mask | kOnlyPerspectiveValid_Mask)); + fTypeMask = (uint8_t)(mask); + } + + void orTypeMask(int mask) { + MNN_ASSERT((mask & kORableMasks) == mask); + fTypeMask = (uint8_t)(fTypeMask | mask); + } + + void clearTypeMask(int mask) { + // only allow a valid mask + MNN_ASSERT((mask & kAllMasks) == mask); + fTypeMask = fTypeMask & ~mask; + } + + TypeMask getPerspectiveTypeMaskOnly() const { + if ((fTypeMask & kUnknown_Mask) && !(fTypeMask & kOnlyPerspectiveValid_Mask)) { + fTypeMask = this->computePerspectiveTypeMask(); + } + return (TypeMask)(fTypeMask & 0xF); + } + + /** Returns true if we already know that the matrix is identity; + false otherwise. + */ + bool isTriviallyIdentity() const { + if (fTypeMask & kUnknown_Mask) { + return false; + } + return ((fTypeMask & 0xF) == 0); + } + + inline void updateTranslateMask() { + if ((fMat[kMTransX] != 0) | (fMat[kMTransY] != 0)) { + fTypeMask |= kTranslate_Mask; + } else { + fTypeMask &= ~kTranslate_Mask; + } + } + + typedef void (*MapXYProc)(const Matrix& mat, float x, float y, Point* result); + + static MapXYProc GetMapXYProc(TypeMask mask) { + MNN_ASSERT((mask & ~kAllMasks) == 0); + return gMapXYProcs[mask & kAllMasks]; + } + + MapXYProc getMapXYProc() const { + return GetMapXYProc(this->getType()); + } + + typedef void (*MapPtsProc)(const Matrix& mat, Point dst[], const Point src[], int count); + + static MapPtsProc GetMapPtsProc(TypeMask mask) { + MNN_ASSERT((mask & ~kAllMasks) == 0); + return gMapPtsProcs[mask & kAllMasks]; + } + + MapPtsProc getMapPtsProc() const { + return GetMapPtsProc(this->getType()); + } + + bool invertNonIdentity(Matrix* inverse) const; + + static void Identity_xy(const Matrix&, float, float, Point*); + static void Trans_xy(const Matrix&, float, float, Point*); + static void Scale_xy(const Matrix&, float, float, Point*); + static void ScaleTrans_xy(const Matrix&, float, float, Point*); + static void Rot_xy(const Matrix&, float, float, Point*); + static void RotTrans_xy(const Matrix&, float, float, Point*); + static void Persp_xy(const Matrix&, float, float, Point*); + + static const MapXYProc gMapXYProcs[]; + + static void Identity_pts(const Matrix&, Point[], const Point[], int); + static void Trans_pts(const Matrix&, Point dst[], const Point[], int); + static void Scale_pts(const Matrix&, Point dst[], const Point[], int); + static void ScaleTrans_pts(const Matrix&, Point dst[], const Point[], int count); + static void Persp_pts(const Matrix&, Point dst[], const Point[], int); + + static void Affine_vpts(const Matrix&, Point dst[], const Point[], int); + + static const MapPtsProc gMapPtsProcs[]; + static bool Poly2Proc(const Point srcPt[], Matrix* dst); + static bool Poly3Proc(const Point srcPt[], Matrix* dst); + static bool Poly4Proc(const Point srcPt[], Matrix* dst); +}; +} // namespace CV +} // namespace MNN +#endif diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/Rect.h b/app/entry/src/main/cpp/third_party/mnn/include/MNN/Rect.h new file mode 100644 index 0000000..87b9547 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/Rect.h @@ -0,0 +1,580 @@ +// +// Rect.h +// MNN +// +// Modified by jiangxiaotang on 2018/09/19. +// Copyright © 2018, Alibaba Group Holding Limited +// + +/* + * Copyright 2006 The Android Open Source Project + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/* Generated by tools/bookmaker from include/core/Rect.h and docs/SkRect_Reference.bmh + on 2018-07-13 08:15:11. Additional documentation and examples can be found at: + https://skia.org/user/api/SkRect_Reference + + You may edit either file directly. Structural changes to public interfaces require + editing both files. After editing docs/SkRect_Reference.bmh, run: + bookmaker -b docs -i include/core/Rect.h -p + to create an updated version of this file. + */ + +#ifndef MNN_Rect_DEFINED +#define MNN_Rect_DEFINED + +#include +#include +#include +#include + +namespace MNN { +namespace CV { + +struct Point { + float fX; + float fY; + + void set(float x, float y) { + fX = x; + fY = y; + } +}; + +/** \struct Rect + Rect holds four float coordinates describing the upper and + lower bounds of a rectangle. Rect may be created from outer bounds or + from position, width, and height. Rect describes an area; if its right + is less than or equal to its left, or if its bottom is less than or equal to + its top, it is considered empty. +*/ +struct MNN_PUBLIC Rect { + float fLeft; //!< smaller x-axis bounds + float fTop; //!< smaller y-axis bounds + float fRight; //!< larger x-axis bounds + float fBottom; //!< larger y-axis bounds + + /** Returns constructed Rect set to (0, 0, 0, 0). + Many other rectangles are empty; if left is equal to or greater than right, + or if top is equal to or greater than bottom. Setting all members to zero + is a convenience, but does not designate a special empty rectangle. + + @return bounds (0, 0, 0, 0) + */ + static constexpr Rect MakeEmpty() { + return Rect{0, 0, 0, 0}; + } + +#ifdef SK_SUPPORT_LEGACY_RECTMAKELARGEST + /** Deprecated. + */ + static Rect MakeLargest() { + return {SK_ScalarMin, SK_ScalarMin, SK_ScalarMax, SK_ScalarMax}; + } +#endif + + /** Returns constructed Rect set to float values (0, 0, w, h). Does not + validate input; w or h may be negative. + + Passing integer values may generate a compiler warning since Rect cannot + represent 32-bit integers exactly. Use SkIRect for an exact integer rectangle. + + @param w float width of constructed Rect + @param h float height of constructed Rect + @return bounds (0, 0, w, h) + */ + static constexpr Rect MakeWH(float w, float h) { + return Rect{0, 0, w, h}; + } + + /** Returns constructed Rect set to integer values (0, 0, w, h). Does not validate + input; w or h may be negative. + + Use to avoid a compiler warning that input may lose precision when stored. + Use SkIRect for an exact integer rectangle. + + @param w integer width of constructed Rect + @param h integer height of constructed Rect + @return bounds (0, 0, w, h) + */ + static Rect MakeIWH(int w, int h) { + Rect r; + r.set(0, 0, (float)(w), (float)(h)); + return r; + } + + /** Returns constructed Rect set to (l, t, r, b). Does not sort input; Rect may + result in fLeft greater than fRight, or fTop greater than fBottom. + + @param l float stored in fLeft + @param t float stored in fTop + @param r float stored in fRight + @param b float stored in fBottom + @return bounds (l, t, r, b) + */ + static constexpr Rect MakeLTRB(float l, float t, float r, float b) { + return Rect{l, t, r, b}; + } + + /** Returns constructed Rect set to (x, y, x + w, y + h). Does not validate input; + w or h may be negative. + + @param x stored in fLeft + @param y stored in fTop + @param w added to x and stored in fRight + @param h added to y and stored in fBottom + @return bounds at (x, y) with width w and height h + */ + static constexpr Rect MakeXYWH(float x, float y, float w, float h) { + return Rect{x, y, x + w, y + h}; + } + + /** Returns true if fLeft is equal to or greater than fRight, or if fTop is equal + to or greater than fBottom. Call sort() to reverse rectangles with negative + width() or height(). + + @return true if width() or height() are zero or negative + */ + bool isEmpty() const { + // We write it as the NOT of a non-empty rect, so we will return true if any values + // are NaN. + return !(fLeft < fRight && fTop < fBottom); + } + + /** Returns true if fLeft is equal to or less than fRight, or if fTop is equal + to or less than fBottom. Call sort() to reverse rectangles with negative + width() or height(). + + @return true if width() or height() are zero or positive + */ + bool isSorted() const { + return fLeft <= fRight && fTop <= fBottom; + } + + /** Returns left edge of Rect, if sorted. Call isSorted() to see if Rect is valid. + Call sort() to reverse fLeft and fRight if needed. + + @return fLeft + */ + float x() const { + return fLeft; + } + + /** Returns top edge of Rect, if sorted. Call isEmpty() to see if Rect may be invalid, + and sort() to reverse fTop and fBottom if needed. + + @return fTop + */ + float y() const { + return fTop; + } + + /** Returns left edge of Rect, if sorted. Call isSorted() to see if Rect is valid. + Call sort() to reverse fLeft and fRight if needed. + + @return fLeft + */ + float left() const { + return fLeft; + } + + /** Returns top edge of Rect, if sorted. Call isEmpty() to see if Rect may be invalid, + and sort() to reverse fTop and fBottom if needed. + + @return fTop + */ + float top() const { + return fTop; + } + + /** Returns right edge of Rect, if sorted. Call isSorted() to see if Rect is valid. + Call sort() to reverse fLeft and fRight if needed. + + @return fRight + */ + float right() const { + return fRight; + } + + /** Returns bottom edge of Rect, if sorted. Call isEmpty() to see if Rect may be invalid, + and sort() to reverse fTop and fBottom if needed. + + @return fBottom + */ + float bottom() const { + return fBottom; + } + + /** Returns span on the x-axis. This does not check if Rect is sorted, or if + result fits in 32-bit float; result may be negative or infinity. + + @return fRight minus fLeft + */ + float width() const { + return fRight - fLeft; + } + + /** Returns span on the y-axis. This does not check if Rect is sorted, or if + result fits in 32-bit float; result may be negative or infinity. + + @return fBottom minus fTop + */ + float height() const { + return fBottom - fTop; + } + + /** Returns average of left edge and right edge. Result does not change if Rect + is sorted. Result may overflow to infinity if Rect is far from the origin. + + @return midpoint in x + */ + float centerX() const { + // don't use floatHalf(fLeft + fBottom) as that might overflow before the 0.5 + return 0.5f * (fLeft) + 0.5f * (fRight); + } + + /** Returns average of top edge and bottom edge. Result does not change if Rect + is sorted. + + @return midpoint in y + */ + float centerY() const { + // don't use floatHalf(fTop + fBottom) as that might overflow before the 0.5 + return 0.5f * (fTop) + 0.5f * (fBottom); + } + + /** Sets Rect to (0, 0, 0, 0). + + Many other rectangles are empty; if left is equal to or greater than right, + or if top is equal to or greater than bottom. Setting all members to zero + is a convenience, but does not designate a special empty rectangle. + */ + void setEmpty() { + *this = MakeEmpty(); + } + + /** Sets Rect to (left, top, right, bottom). + left and right are not sorted; left is not necessarily less than right. + top and bottom are not sorted; top is not necessarily less than bottom. + + @param left stored in fLeft + @param top stored in fTop + @param right stored in fRight + @param bottom stored in fBottom + */ + void set(float left, float top, float right, float bottom) { + fLeft = left; + fTop = top; + fRight = right; + fBottom = bottom; + } + + /** Sets Rect to (left, top, right, bottom). + left and right are not sorted; left is not necessarily less than right. + top and bottom are not sorted; top is not necessarily less than bottom. + + @param left stored in fLeft + @param top stored in fTop + @param right stored in fRight + @param bottom stored in fBottom + */ + void setLTRB(float left, float top, float right, float bottom) { + this->set(left, top, right, bottom); + } + + /** Sets Rect to (left, top, right, bottom). + All parameters are promoted from integer to scalar. + left and right are not sorted; left is not necessarily less than right. + top and bottom are not sorted; top is not necessarily less than bottom. + + @param left promoted to float and stored in fLeft + @param top promoted to float and stored in fTop + @param right promoted to float and stored in fRight + @param bottom promoted to float and stored in fBottom + */ + void iset(int left, int top, int right, int bottom) { + fLeft = (float)(left); + fTop = (float)(top); + fRight = (float)(right); + fBottom = (float)(bottom); + } + + /** Sets Rect to (0, 0, width, height). + width and height may be zero or negative. width and height are promoted from + integer to float, large values may lose precision. + + @param width promoted to float and stored in fRight + @param height promoted to float and stored in fBottom + */ + void isetWH(int width, int height) { + fLeft = fTop = 0; + fRight = (float)(width); + fBottom = (float)(height); + } + + /** Sets Rect to (x, y, x + width, y + height). Does not validate input; + width or height may be negative. + + @param x stored in fLeft + @param y stored in fTop + @param width added to x and stored in fRight + @param height added to y and stored in fBottom + */ + void setXYWH(float x, float y, float width, float height) { + fLeft = x; + fTop = y; + fRight = x + width; + fBottom = y + height; + } + + /** Sets Rect to (0, 0, width, height). Does not validate input; + width or height may be negative. + + @param width stored in fRight + @param height stored in fBottom + */ + void setWH(float width, float height) { + fLeft = 0; + fTop = 0; + fRight = width; + fBottom = height; + } + + /** Returns Rect offset by (dx, dy). + + If dx is negative, Rect returned is moved to the left. + If dx is positive, Rect returned is moved to the right. + If dy is negative, Rect returned is moved upward. + If dy is positive, Rect returned is moved downward. + + @param dx added to fLeft and fRight + @param dy added to fTop and fBottom + @return Rect offset on axes, with original width and height + */ + Rect makeOffset(float dx, float dy) const { + return MakeLTRB(fLeft + dx, fTop + dy, fRight + dx, fBottom + dy); + } + + /** Returns Rect, inset by (dx, dy). + + If dx is negative, Rect returned is wider. + If dx is positive, Rect returned is narrower. + If dy is negative, Rect returned is taller. + If dy is positive, Rect returned is shorter. + + @param dx added to fLeft and subtracted from fRight + @param dy added to fTop and subtracted from fBottom + @return Rect inset symmetrically left and right, top and bottom + */ + Rect makeInset(float dx, float dy) const { + return MakeLTRB(fLeft + dx, fTop + dy, fRight - dx, fBottom - dy); + } + + /** Returns Rect, outset by (dx, dy). + + If dx is negative, Rect returned is narrower. + If dx is positive, Rect returned is wider. + If dy is negative, Rect returned is shorter. + If dy is positive, Rect returned is taller. + + @param dx subtracted to fLeft and added from fRight + @param dy subtracted to fTop and added from fBottom + @return Rect outset symmetrically left and right, top and bottom + */ + Rect makeOutset(float dx, float dy) const { + return MakeLTRB(fLeft - dx, fTop - dy, fRight + dx, fBottom + dy); + } + + /** Offsets Rect by adding dx to fLeft, fRight; and by adding dy to fTop, fBottom. + + If dx is negative, moves Rect to the left. + If dx is positive, moves Rect to the right. + If dy is negative, moves Rect upward. + If dy is positive, moves Rect downward. + + @param dx offset added to fLeft and fRight + @param dy offset added to fTop and fBottom + */ + void offset(float dx, float dy) { + fLeft += dx; + fTop += dy; + fRight += dx; + fBottom += dy; + } + + /** Offsets Rect so that fLeft equals newX, and fTop equals newY. width and height + are unchanged. + + @param newX stored in fLeft, preserving width() + @param newY stored in fTop, preserving height() + */ + void offsetTo(float newX, float newY) { + fRight += newX - fLeft; + fBottom += newY - fTop; + fLeft = newX; + fTop = newY; + } + + /** Insets Rect by (dx, dy). + + If dx is positive, makes Rect narrower. + If dx is negative, makes Rect wider. + If dy is positive, makes Rect shorter. + If dy is negative, makes Rect taller. + + @param dx added to fLeft and subtracted from fRight + @param dy added to fTop and subtracted from fBottom + */ + void inset(float dx, float dy) { + fLeft += dx; + fTop += dy; + fRight -= dx; + fBottom -= dy; + } + + /** Outsets Rect by (dx, dy). + + If dx is positive, makes Rect wider. + If dx is negative, makes Rect narrower. + If dy is positive, makes Rect taller. + If dy is negative, makes Rect shorter. + + @param dx subtracted to fLeft and added from fRight + @param dy subtracted to fTop and added from fBottom + */ + void outset(float dx, float dy) { + this->inset(-dx, -dy); + } + +private: + static bool Intersects(float al, float at, float ar, float ab, float bl, float bt, float br, float bb) { + float L = std::max(al, bl); + float R = std::min(ar, br); + float T = std::max(at, bt); + float B = std::min(ab, bb); + return L < R && T < B; + } + +public: + /** Constructs Rect to intersect from (left, top, right, bottom). Does not sort + construction. + + Returns true if Rect intersects construction. + Returns false if either construction or Rect is empty, or do not intersect. + + @param left x-axis minimum of constructed Rect + @param top y-axis minimum of constructed Rect + @param right x-axis maximum of constructed Rect + @param bottom y-axis maximum of constructed Rect + @return true if construction and Rect have area in common + */ + bool intersects(float left, float top, float right, float bottom) const { + return Intersects(fLeft, fTop, fRight, fBottom, left, top, right, bottom); + } + + /** Returns true if Rect intersects r. + Returns false if either r or Rect is empty, or do not intersect. + + @param r Rect to intersect + @return true if r and Rect have area in common + */ + bool intersects(const Rect& r) const { + return Intersects(fLeft, fTop, fRight, fBottom, r.fLeft, r.fTop, r.fRight, r.fBottom); + } + + /** Returns true if a intersects b. + Returns false if either a or b is empty, or do not intersect. + + @param a Rect to intersect + @param b Rect to intersect + @return true if a and b have area in common + */ + static bool Intersects(const Rect& a, const Rect& b) { + return Intersects(a.fLeft, a.fTop, a.fRight, a.fBottom, b.fLeft, b.fTop, b.fRight, b.fBottom); + } + + /** Sets Rect to the union of itself and r. + + Asserts if r is empty and SK_DEBUG is defined. + If Rect is empty, sets Rect to r. + + May produce incorrect results if r is empty. + + @param r expansion Rect + */ + void joinNonEmptyArg(const Rect& r) { + MNN_ASSERT(!r.isEmpty()); + // if we are empty, just assign + if (fLeft >= fRight || fTop >= fBottom) { + *this = r; + } else { + this->joinPossiblyEmptyRect(r); + } + } + + /** Sets Rect to the union of itself and the construction. + + May produce incorrect results if Rect or r is empty. + + @param r expansion Rect + */ + void joinPossiblyEmptyRect(const Rect& r) { + fLeft = std::min(fLeft, r.left()); + fTop = std::min(fTop, r.top()); + fRight = std::max(fRight, r.right()); + fBottom = std::max(fBottom, r.bottom()); + } + + /** Returns true if: fLeft <= x < fRight && fTop <= y < fBottom. + Returns false if Rect is empty. + + @param x test Point x-coordinate + @param y test Point y-coordinate + @return true if (x, y) is inside Rect + */ + bool contains(float x, float y) const { + return x >= fLeft && x < fRight && y >= fTop && y < fBottom; + } + + /** Swaps fLeft and fRight if fLeft is greater than fRight; and swaps + fTop and fBottom if fTop is greater than fBottom. Result may be empty; + and width() and height() will be zero or positive. + */ + void sort() { + using std::swap; + if (fLeft > fRight) { + swap(fLeft, fRight); + } + + if (fTop > fBottom) { + swap(fTop, fBottom); + } + } + + /** Returns Rect with fLeft and fRight swapped if fLeft is greater than fRight; and + with fTop and fBottom swapped if fTop is greater than fBottom. Result may be empty; + and width() and height() will be zero or positive. + + @return sorted Rect + */ + Rect makeSorted() const { + return MakeLTRB(std::min(fLeft, fRight), std::min(fTop, fBottom), std::max(fLeft, fRight), + std::max(fTop, fBottom)); + } + + /** Returns pointer to first scalar in Rect, to treat it as an array with four + entries. + + @return pointer to fLeft + */ + const float* asScalars() const { + return &fLeft; + } +}; + +} // namespace CV +} // namespace MNN +#endif diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/Tensor.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/Tensor.hpp new file mode 100644 index 0000000..049027e --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/Tensor.hpp @@ -0,0 +1,320 @@ +// +// Tensor.hpp +// MNN +// +// Created by MNN on 2018/08/14. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_Tensor_hpp +#define MNN_Tensor_hpp + +#include +#include +#include + +namespace MNN { + +/** + * data container. + * data for host tensor is saved in `host` field. its memory is allocated malloc directly. + * data for device tensor is saved in `deviceId` field. its memory is allocated by session's backend. + * usually, device tensors are created by engine (like net, session). + * meanwhile, host tensors could be created by engine or user. + */ +class MNN_PUBLIC Tensor { +public: + struct InsideDescribe; + + /** dimension type used to create tensor */ + enum DimensionType { + /** for tensorflow net type. uses NHWC as data format. */ + TENSORFLOW, + /** for caffe net type. uses NCHW as data format. */ + CAFFE, + /** for caffe net type. uses NC4HW4 as data format. */ + CAFFE_C4 + }; + + /** handle type */ + enum HandleDataType { + /** default handle type */ + HANDLE_NONE = 0, + /** string handle type */ + HANDLE_STRING = 1 + }; + + /** Tensor map type : Read or Write*/ + enum MapType { + /** map Tensor for writing data*/ + MAP_TENSOR_WRITE = 0, + MAP_TENSOR_READ = 1 + }; + +public: + /** + * @brief create a tensor with dimension size and type without acquire memory for data. + * @param dimSize dimension size. + * @param type dimension type. + */ + Tensor(int dimSize = 4, DimensionType type = CAFFE); + + /** + * @brief create a tensor with same shape as given tensor. + * @param tensor shape provider. + * @param type dimension type. + * @param allocMemory acquire memory for data or not. + * @warning tensor data won't be copied. + */ + Tensor(const Tensor* tensor, DimensionType type = CAFFE, bool allocMemory = true); + + /** deinitializer */ + ~Tensor(); + +private: + Tensor(bool deepCopy, const Tensor* tensor); + // remove all assignment operator + Tensor(const Tensor& tensor) = delete; + Tensor(const Tensor&& tensor) = delete; + Tensor& operator=(const Tensor&) = delete; + Tensor& operator=(const Tensor&&) = delete; + +public: + /** + * @brief create tensor with shape, data type and dimension type. + * @param shape tensor shape. + * @param type data type. + * @param dimType dimension type. + * @return created tensor. + * @warning memory for data won't be acquired. call backend's onAcquireBuffer to get memory ready. + */ + static Tensor* createDevice(const std::vector& shape, halide_type_t type, DimensionType dimType = TENSORFLOW); + + /** + * @brief create tensor with shape and dimension type. data type is represented by `T`. + * @param shape tensor shape. + * @param dimType dimension type. + * @return created tensor. + * @warning memory for data won't be acquired. call backend's onAcquireBuffer to get memory ready. + */ + template + static Tensor* createDevice(const std::vector& shape, DimensionType dimType = TENSORFLOW) { + return createDevice(shape, halide_type_of(), dimType); + } + + /** + * @brief create tensor with shape, data type, data and dimension type. + * @param shape tensor shape. + * @param type data type. + * @param data data to save. + * @param dimType dimension type. + * @return created tensor. + */ + static Tensor* create(const std::vector& shape, halide_type_t type, void* data = NULL, + DimensionType dimType = TENSORFLOW); + + /** + * @brief create tensor with shape, data and dimension type. data type is represented by `T`. + * @param shape tensor shape. + * @param data data to save. + * @param dimType dimension type. + * @return created tensor. + */ + template + static Tensor* create(const std::vector& shape, void* data = NULL, DimensionType dimType = TENSORFLOW) { + return create(shape, halide_type_of(), data, dimType); + } + + /** + * @brief copy tensor. + * @param src tensor + * @param deepCopy whether create new content and copy, currently only support deepCopy = false + */ + static Tensor* clone(const Tensor* src, bool deepCopy = false); + + /** + * @brief delete tensor. + * @param src tensor + */ + static void destroy(Tensor* tensor); +public: + /** + * @brief for DEVICE tensor, copy data from given host tensor. + * @param hostTensor host tensor, the data provider. + * @return true for DEVICE tensor, and false for HOST tensor. + */ + bool copyFromHostTensor(const Tensor* hostTensor); + + /** + * @brief for DEVICE tensor, copy data to given host tensor. + * @param hostTensor host tensor, the data consumer. + * @return true for DEVICE tensor, and false for HOST tensor. + */ + bool copyToHostTensor(Tensor* hostTensor) const; + + /** + * @brief create HOST tensor from DEVICE tensor, with or without data copying. + * @param deviceTensor given device tensor. + * @param copyData copy data or not. + * @return created host tensor. + */ + static Tensor* createHostTensorFromDevice(const Tensor* deviceTensor, bool copyData = true); + +public: + const halide_buffer_t& buffer() const { + return mBuffer; + } + halide_buffer_t& buffer() { + return mBuffer; + } + + /** + * @brief get dimension type. + * @return dimension type. + */ + DimensionType getDimensionType() const; + + /** + * @brief handle data type. used when data type code is halide_type_handle. + * @return handle data type. + */ + HandleDataType getHandleDataType() const; + + /** + * @brief set data type. + * @param type data type defined in 'Type_generated.h'. + */ + void setType(int type); + + /** + * @brief get data type. + * @return data type. + */ + inline halide_type_t getType() const { + return mBuffer.type; + } + + /** + * @brief visit host memory, data type is represented by `T`. + * @return data point in `T` type. + */ + template + T* host() const { + return (T*)mBuffer.host; + } + + /** + * @brief visit device memory. + * @return device data ID. what the ID means varies between backends. + */ + uint64_t deviceId() const { + return mBuffer.device; + } + +public: + int dimensions() const { + return mBuffer.dimensions; + } + + /** + * @brief get all dimensions' extent. + * @return dimensions' extent. + */ + std::vector shape() const; + + /** + * @brief calculate number of bytes needed to store data taking reordering flag into account. + * @return bytes needed to store data + */ + int size() const; + size_t usize() const; + + /** + * @brief calculate number of elements needed to store data taking reordering flag into account. + * @return elements needed to store data + */ + inline int elementSize() const { + return size() / mBuffer.type.bytes(); + } + +public: + inline int width() const { + if (getDimensionType() == TENSORFLOW) { + return mBuffer.dim[2].extent; + } + + return mBuffer.dim[3].extent; + } + inline int height() const { + if (getDimensionType() == TENSORFLOW) { + return mBuffer.dim[1].extent; + } + return mBuffer.dim[2].extent; + } + inline int channel() const { + if (getDimensionType() == TENSORFLOW) { + return mBuffer.dim[3].extent; + } + return mBuffer.dim[1].extent; + } + inline int batch() const { + return mBuffer.dim[0].extent; + } + + // visit dimension's extent & stride + inline int stride(int index) const { + return mBuffer.dim[index].stride; + } + inline int length(int index) const { + return mBuffer.dim[index].extent; + } + inline void setStride(int index, int stride) { + mBuffer.dim[index].stride = stride; + } + inline void setLength(int index, int length) { + mBuffer.dim[index].extent = length; + } + + /** + * @brief For GPU and Other Device, get memory directly, see MNNSharedContext for detail + * @return Success or not. If type != tensor's backend's type or type is cpu , return false + */ + bool getDeviceInfo(void* dst, int forwardType) const; + +public: + /** + * @brief print tensor data. for DEBUG use only. + */ + void print() const; + + /** + *@brief print tensor shape + */ + void printShape() const; + +public: + /** + * @brief map/umap GPU Tensor, to get host ptr + */ + void* map(MapType mtype, DimensionType dtype); + void unmap(MapType mtype, DimensionType dtype, void* mapPtr); + /** + * @brief wait until the tensor is ready to read / write + * @param mtype wait for read or write + * @param finish wait for command flush or finish + */ + int wait(MapType mtype, bool finish); + /** + * @brief set GPU tensor device ptr, and inform memory type + */ + bool setDevicePtr(const void* devicePtr, int memoryType); +private: + halide_buffer_t mBuffer; + struct InsideDescribe* mDescribe; + +private: + friend class TensorUtils; +}; +} // namespace MNN + +#endif /* Tensor_hpp */ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Executor.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Executor.hpp new file mode 100644 index 0000000..e1f3947 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Executor.hpp @@ -0,0 +1,172 @@ +// +// Executor.hpp +// MNN +// +// Created by MNN on 2019/07/25. +// Copyright © 2018, Alibaba Group Holding Limited +// +#ifndef MNN_Executor_hpp +#define MNN_Executor_hpp +#include +#include +#include +#include +#include +#include +#include +#include + +namespace MNN { +class Backend; +class Execution; +class Runtime; +struct Op; +namespace Express { +struct RuntimeAttr; +struct ExecutorAttr; +class MNN_PUBLIC Executor { +public: + class ComputeCache; + class RuntimeExecuteWrap; + struct DebugTools; + /**Internal Usage Begin*/ + struct Requirement { + std::vector contentNeedContent; + std::vector shapeNeedContent; + }; + ~Executor(); + Requirement getRequirement(Expr* expr) const; + ErrorCode computeInfo(Expr* expr); + void makeCache(const std::vector& expr, bool forceCPU = false); + /**Internal Usage End*/ + + bool lazyEval = true; + enum LazyMode { + LAZY_FULL = 0, + // Don't compute content until user needed. + LAZY_CONTENT = 1 << 0, + + // Expr can only compute once, it can reduce the create cost of expr + LAZY_COMPUTE_ONCE = 1 << 1, + }; + uint32_t getLazyMode() const { + return mLazyMode; + } + void setLazyComputeMode(uint32_t mode); + void setGlobalExecutorConfig(MNNForwardType type, const BackendConfig& config, int numberThread); + int getCurrentRuntimeStatus(RuntimeStatus statusEnum); + // Get last GPU execution time in ms (measured by GPU timestamps). + // Returns -1.0f if profiling is not supported or not enabled. + float getLastGpuTimeMs() const; + enum GCFlag { + FULL, + PART + }; + void gc(GCFlag flag = FULL); + static std::shared_ptr getGlobalExecutor(); + + static std::shared_ptr newExecutor(MNNForwardType type, + const BackendConfig& config, + int numberThread); + void resetProfile(); + void dumpProfile(); + + struct SubGraph; + bool registerSubGraph(const std::string& submoduleName, VARPS outputs, VARPS inputs); + std::shared_ptr findSubGraph(const std::string& submoduleName); + static RuntimeInfo getRuntime(); + void setCallBack(TensorCallBackWithInfo&& before, TensorCallBackWithInfo&& after); + const DebugTools* getDebugTools() const { + return mDebug.get(); + } + ExecutorAttr* getAttr() const; + class MNN_PUBLIC RuntimeManager { + public: + ~RuntimeManager(); + /** + * @param configs : schedule configs. + * @param cacheName : full path for cache file. Note: should choose location for reading and writing. + */ + static RuntimeManager* createRuntimeManager(const ScheduleConfig& config); + + /** + * @param rtmgr : the rtmgr to destroy + */ + static void destroy(RuntimeManager* rtmgr); + + /** + * Deceperate, the same as createRuntimeManager(configs[0]) + * @param configs : schedule configs. + * @param cacheName : full path for cache file. Note: should choose location for reading and writing. + */ + static RuntimeManager* createRuntimeManager(std::vector& configs); + + /** + * @brief set cache file. when file not exist -- create it, when file exist -- load it. + * When should use : When choose GPU backend or use AUTO backend. + * Calling Position: calling after createRuntimeManager. + */ + void setCache(std::string cacheName); + + /** + * @brief set the path of external files or directory + * @param path -- The path of a file or directory on disk + * @param type -- Type of the external path (see "enum ExternalPathType" in Interpreter.hpp) + */ + void setExternalPath(std::string path, int type); + + /** + * @brief set external file. + */ + void setExternalFile(std::string fileName); + + /** + * @brief update cache file + * When should use : Together with setCache API. calling for first inference and when input shape is changed. + * Calling Position : calling after inference done. + */ + void updateCache(); + std::vector isBackendSupport(const std::vector type); + friend class Executor; + void setMode(Interpreter::SessionMode mode); + void setHint(Interpreter::HintMode mode, int value); + void setHint(Interpreter::HintMode mode, int* value, size_t size); + void setHintPtr(Interpreter::HintMode mode, void* value); + // Keep Runtime::pMeta in sync for backend runtimes that still read it during onCreate. + // Session also sets the meta on created Backends through the common backend creation path. + void applyMetaToRuntime() const; + bool getInfo(Interpreter::SessionInfoCode code, void* ptr); + static bool getDeviceInfo(const std::string& deviceKey, const MNNForwardType type, std::string& deviceValue); + BackendConfig* getBnConfig(); + const RuntimeAttr* getInside() const { + return mInside; + } + private: + std::mutex mLock; + RuntimeAttr* mInside; + friend class StaticModule; + RuntimeManager(); + }; + static bool getComputeInfo(EXPRP expr, Interpreter::SessionInfoCode code, void* ptr); +#ifndef MNN_REDUCE_SIZE + std::map>& subgraph() { + return mSubGraph; + }; +#endif +private: + std::shared_ptr _getOrCreateRuntime(MNNForwardType type, const BackendConfig* config, int numberThread, bool reset = true); + Executor(std::shared_ptr backend, MNNForwardType type, int numberThread); + void _makeCache(const std::vector& outputs, bool forceCPU); + + RuntimeInfo mRuntimeInfo; + std::shared_ptr mDebug; +#ifndef MNN_REDUCE_SIZE + std::map> mSubGraph; +#endif + uint32_t mLazyMode = 0; + std::shared_ptr mAttr; + std::mutex mMutex; +}; +} // namespace Express +} // namespace MNN +#endif diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/ExecutorScope.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/ExecutorScope.hpp new file mode 100644 index 0000000..d7764f7 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/ExecutorScope.hpp @@ -0,0 +1,33 @@ +// +// ExecutorScope.hpp +// MNN +// +// Created by MNN on 2020/10/26. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_EXPR_EXECUTOR_SCOPE_HPP_ +#define MNN_EXPR_EXECUTOR_SCOPE_HPP_ + +#include + +namespace MNN { +namespace Express { + +struct MNN_PUBLIC ExecutorScope final { +public: + ExecutorScope() = delete; + explicit ExecutorScope(const ExecutorScope&) = delete; + explicit ExecutorScope(const std::shared_ptr& current); + + explicit ExecutorScope(const std::string& scope_name, + const std::shared_ptr& current); + + virtual ~ExecutorScope(); + + static const std::shared_ptr Current(); +}; + +} // namespace MNN +} // namespace Express +#endif // MNN_EXPR_EXECUTOR_SCOPE_HPP_ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Expr.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Expr.hpp new file mode 100644 index 0000000..484f26c --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Expr.hpp @@ -0,0 +1,282 @@ +// +// Expr.hpp +// MNN +// +// Created by MNN on 2019/06/10. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_Expr_hpp +#define MNN_Expr_hpp + +#include +#include +#include +#include +#include +#include +#include + +namespace MNN { +struct BufferStorage; +struct OpT; +struct Op; +struct NetT; +class Tensor; +namespace Express { +class Variable; +class Expr; +class Executor; +typedef std::shared_ptr EXPRP; +typedef std::weak_ptr WeakEXPRP; +typedef std::vector INTS; +enum Dimensionformat { NHWC, NC4HW4, NCHW }; +class MNN_PUBLIC VARP { +public: + VARP() { + // Do nothing + } + VARP(std::shared_ptr c) { + mContent = std::move(c); + } + VARP(Variable* c) { + mContent.reset(c); + } + Variable* get() const { + return mContent.get(); + } + ~ VARP() { + // Do nothing + } + VARP(const VARP& var) { + mContent = var.mContent; + } + VARP(VARP&& var) { + mContent = std::move(var.mContent); + } + VARP operator+(VARP var) const; + VARP operator-(VARP var) const; + VARP operator*(VARP var) const; + VARP operator/(VARP var) const; + VARP mean(INTS dims) const; + VARP sum(INTS dims) const; + + bool operator==(const VARP& var) const { + return var.mContent == mContent; + } + bool operator<(const VARP& var) const { + return mContent < var.mContent; + } + bool operator<=(const VARP& var) const { + return mContent <= var.mContent; + } + VARP& operator=(const VARP& var) { + mContent = var.mContent; + return *this; + } + VARP& operator=(Variable* var) { + mContent.reset(var); + return *this; + } + Variable* operator->() const { + return mContent.get(); + } + enum InputType { + INPUT = 0, + CONSTANT = 1, + TRAINABLE = 2, + }; + bool fix(InputType type) const; + + // Only valid for Input / Const / Trainable + void setOrder(Dimensionformat format); +private: + friend class Variable; + std::shared_ptr mContent; +}; +inline bool operator==(Variable* src, VARP dst) { + return src == dst.get(); +} +inline bool operator!=(Variable* src, VARP dst) { + return src != dst.get(); +} +// inline bool operator<(VARP src, VARP dst) { +// return src.get() < dst.get(); +// } +typedef std::vector VARPS; + +class MNN_PUBLIC Variable { +public: + struct Info { + Dimensionformat order = NHWC; + INTS dim; + halide_type_t type; + size_t size; + void syncSize(); + }; + const std::string& name() const; + void setName(const std::string& name); + bool setDevicePtr(const void* devicePtr, int memoryType); + bool copyToDevicePtr(void* devicePtr, int memoryType); + + std::pair expr() const { + return std::make_pair(mFrom, mFromIndex); + } + // If compute info error, return nullptr + const Info* getInfo(); + bool resize(INTS dims); + template + const T* readMap() { + return (const T*)readInternal(); + } + + template + T* writeMap() { + return (T*)writeInternal(); + } + + void writeScaleMap(float scaleValue, float zeroPoint) { + writeScaleInternal(scaleValue, zeroPoint); + } + + //Depecerate + void unMap(); + + bool input(VARP src); + static void replace(VARP dst, VARP src); + + static VARP create(EXPRP expr, int index = 0); + + static std::vector load(const char* fileName); + static std::map loadMap(const char* fileName); + static std::vector load(const uint8_t* buffer, size_t length); + static std::map loadMap(const uint8_t* buffer, size_t length); + static std::pair, std::map> getInputAndOutput(const std::map& allVariable); + static std::vector mapToSequence(const std::map& source); + static std::vector getExecuteOrder(const std::vector& output); + static void save(const std::vector& vars, const char* fileName); + static std::vector save(const std::vector& vars); + static void save(const std::vector& vars, NetT* dest); + + // Pack a few Variable to compute in one pipeline + static void prepareCompute(const std::vector& vars, bool forceCPU = false); + static void compute(const std::vector& vars, bool forceCPU = false); + + size_t linkNumber() const; + const std::vector& toExprs() const; + void setExpr(EXPRP expr, int index) { + mFrom = expr; + mFromIndex = index; + } + + // Can't modify the tensor from this interface + const Tensor* getTensor() const; +private: + Variable(EXPRP expr, int index) { + mFrom = expr; + mFromIndex = index; + } + + void* readInternal(bool forShape = false); + void* writeInternal(bool inform=true); + void informDirty(); + void writeScaleInternal(float scaleValue, float zeroPoint, bool inform = true); + + friend class Expr; + EXPRP mFrom; + int mFromIndex; +}; +class MNN_PUBLIC Expr { +public: + struct Inside; + enum MemoryType { + COPY, + MOVE, + REF + }; + static EXPRP create(Tensor* tensor, bool own = false); + + static EXPRP create(Variable::Info&& info, const void* ptr, VARP::InputType type, MemoryType copy = COPY); + static EXPRP create(const OpT* op, std::vector inputs, int outputSize = 1); + static EXPRP create(std::shared_ptr extra, std::vector&& inputs, int outputSize = 1); + static EXPRP create(std::unique_ptr&& op, std::vector inputs, int outputSize = 1) { + return create(op.get(), inputs, outputSize); + } + void setName(const std::string& name); + + const Op* get() const { + return mOp; + } + const std::vector& inputs() const { + return mInputs; + } + int outputSize() const { + return (int)mOutputNames.size(); + } + static void replace(EXPRP oldExpr, EXPRP newExpr); + bool requireInfo(); + void visitOutputs(const std::function& visit); + static void visit(EXPRP expr, const std::function& before, const std::function& after); + + const std::vector& outputs() const { + return mTo; + } + ~Expr(); + + const std::string& name() const { + return mName; + } + const std::string& outputName(int index) { + return mOutputNames[index]; + } + + VARP::InputType inputType() const {return mType;} + /** Internal Usage Begin */ + Variable::Info* outputInfo(int index) const; + std::shared_ptr extra() const { + return mStorage; + } + bool setInfoDirty(); + std::shared_ptr inside() const { + return mInside; + } + bool valid() const { + return mValid; + } + bool visited() const { + return mVisited; + } + void setVisited(bool visited) { + mVisited = visited; + } + + /** Internal Usage End */ + + +private: + static void _addLinkForInputs(EXPRP expr); + + Expr(int outputSize); + Expr(Tensor* tensor, bool own = false); + + friend class Variable; + friend class VARP; + VARP::InputType mType; + const Op* mOp; + std::vector mInputs; + std::vector mOutputNames; + + bool mValid = true; + std::shared_ptr mStorage; + std::string mName; + std::shared_ptr mInside = nullptr; + bool mVisited = false; + std::vector mTo; + bool mCanDecompose = true; + friend class ExprModule; + +}; +} // namespace Express +} // namespace MNN + +#endif /* Expr_hpp */ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/ExprCreator.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/ExprCreator.hpp new file mode 100644 index 0000000..5d275b9 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/ExprCreator.hpp @@ -0,0 +1,16 @@ +// +// ExprCreator.hpp +// MNN +// +// Created by MNN on 2019/06/27. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_ExprCreator_hpp +#define MNN_ExprCreator_hpp + +#include +#include +#include + +#endif diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/MathOp.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/MathOp.hpp new file mode 100644 index 0000000..4135ce5 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/MathOp.hpp @@ -0,0 +1,145 @@ +// +// MathOp.hpp +// MNN +// +// Created by MNN on 2019/06/27. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_MathOp_HPP +#define MNN_MathOp_HPP + +namespace MNN { +namespace Express { +//BinaryOPs +MNN_PUBLIC VARP _Add(VARP x, VARP y); +MNN_PUBLIC VARP _Subtract(VARP x, VARP y); +MNN_PUBLIC VARP _Multiply(VARP x, VARP y); +MNN_PUBLIC VARP _MulSilu(VARP x, VARP y); +MNN_PUBLIC VARP _Divide(VARP x, VARP y); +MNN_PUBLIC VARP _Pow(VARP x, VARP y); +MNN_PUBLIC VARP _Minimum(VARP x, VARP y); +MNN_PUBLIC VARP _Maximum(VARP x, VARP y); +MNN_PUBLIC VARP _BiasAdd(VARP value, VARP bias); +MNN_PUBLIC VARP _Greater(VARP x, VARP y); +MNN_PUBLIC VARP _GreaterEqual(VARP x, VARP y); +MNN_PUBLIC VARP _Less(VARP x, VARP y); +MNN_PUBLIC VARP _FloorDiv(VARP x, VARP y); +MNN_PUBLIC VARP _SquaredDifference(VARP x, VARP y); +MNN_PUBLIC VARP _Equal(VARP x, VARP y); +MNN_PUBLIC VARP _LessEqual(VARP x, VARP y); +MNN_PUBLIC VARP _FloorMod(VARP x, VARP y); +MNN_PUBLIC VARP _Atan2(VARP x, VARP y); +MNN_PUBLIC VARP _LogicalOr(VARP x, VARP y); +MNN_PUBLIC VARP _NotEqual(VARP x, VARP y); +MNN_PUBLIC VARP _BitwiseAnd(VARP x, VARP y); +MNN_PUBLIC VARP _BitwiseOr(VARP x, VARP y); +MNN_PUBLIC VARP _BitwiseXor(VARP x, VARP y); + +//UnaryOPs +MNN_PUBLIC VARP _Sign(VARP a); +MNN_PUBLIC VARP _Abs(VARP x); +MNN_PUBLIC VARP _Negative(VARP x); +MNN_PUBLIC VARP _Floor(VARP x); +MNN_PUBLIC VARP _Round(VARP x); +MNN_PUBLIC VARP _Ceil(VARP x); +MNN_PUBLIC VARP _Square(VARP x); +MNN_PUBLIC VARP _Sqrt(VARP x); +MNN_PUBLIC VARP _Rsqrt(VARP x); +MNN_PUBLIC VARP _Exp(VARP x); +MNN_PUBLIC VARP _Log(VARP x); +MNN_PUBLIC VARP _Sin(VARP x); +MNN_PUBLIC VARP _Sinh(VARP x); +MNN_PUBLIC VARP _Cos(VARP x); +MNN_PUBLIC VARP _Cosh(VARP x); +MNN_PUBLIC VARP _Tan(VARP x); +MNN_PUBLIC VARP _Asin(VARP x); +MNN_PUBLIC VARP _Asinh(VARP x); +MNN_PUBLIC VARP _Acos(VARP x); +MNN_PUBLIC VARP _Acosh(VARP x); +MNN_PUBLIC VARP _Atan(VARP x); +MNN_PUBLIC VARP _Atanh(VARP x); +MNN_PUBLIC VARP _Reciprocal(VARP x); +MNN_PUBLIC VARP _Log1p(VARP x); +MNN_PUBLIC VARP _Gelu(VARP x); +MNN_PUBLIC VARP _Tanh(VARP x); +MNN_PUBLIC VARP _Sigmoid(VARP x); +MNN_PUBLIC VARP _Erf(VARP x); +MNN_PUBLIC VARP _Erfc(VARP x); +MNN_PUBLIC VARP _Erfinv(VARP x); +MNN_PUBLIC VARP _Expm1(VARP x); +MNN_PUBLIC VARP _Hardswish(VARP x); +MNN_PUBLIC VARP _Silu(VARP x); + +//ReduceOPs +MNN_PUBLIC VARP _ReduceSum(VARP input_variable, INTS axis = {}, bool keepDims = false); +MNN_PUBLIC VARP _ReduceMean(VARP input_variable, INTS axis = {}, bool keepDims = false); +MNN_PUBLIC VARP _ReduceMax(VARP input_variable, INTS axis = {}, bool keepDims = false); +MNN_PUBLIC VARP _ReduceMin(VARP input_variable, INTS axis = {}, bool keepDims = false); +MNN_PUBLIC VARP _ReduceProd(VARP input_variable, INTS axis = {}, bool keepDims = false); +MNN_PUBLIC VARP _ReduceAny(VARP input_variable, INTS axis = {}, bool keepDims = false); +MNN_PUBLIC VARP _ReduceAll(VARP input_variable, INTS axis = {}, bool keepDims = false); + +MNN_PUBLIC VARP _ReduceSumMutable(VARP input_variable, VARP axis, bool keepDims = false); +MNN_PUBLIC VARP _ReduceMeanMutable(VARP input_variable, VARP axis, bool keepDims = false); +MNN_PUBLIC VARP _ReduceMaxMutable(VARP input_variable, VARP axis, bool keepDims = false); +MNN_PUBLIC VARP _ReduceMinMutable(VARP input_variable, VARP axis, bool keepDims = false); +MNN_PUBLIC VARP _ReduceProdMutable(VARP input_variable, VARP axis, bool keepDims = false); +MNN_PUBLIC VARP _ReduceAnyMutable(VARP input_variable, VARP axis, bool keepDims = false); +MNN_PUBLIC VARP _ReduceAllMutable(VARP input_variable, VARP axis, bool keepDims = false); + +//EltwiseOPs +MNN_PUBLIC VARP _Prod(VARP a, VARP b, std::vector coeff); +MNN_PUBLIC VARP _Sum(VARP a, VARP b, std::vector coeff); +MNN_PUBLIC VARP _Max(VARP a, VARP b, std::vector coeff); +MNN_PUBLIC VARP _Sub(VARP a, VARP b, std::vector coeff); +MNN_PUBLIC VARP _EltwiseProdInt8(VARP x, VARP y, + std::vector x_weight, std::vector x_bias, std::vector x_scale, std::vector x_tensorScale, + std::vector y_weight, std::vector y_bias, std::vector y_scale, std::vector y_tensorScale, + std::vector output_weight, std::vector output_bias, std::vector output_scale, std::vector output_tensorScale); +MNN_PUBLIC VARP _EltwiseSumInt8(VARP x, VARP y, + std::vector x_weight, std::vector x_bias, std::vector x_scale, std::vector x_tensorScale, + std::vector y_weight, std::vector y_bias, std::vector y_scale, std::vector y_tensorScale, + std::vector output_weight, std::vector output_bias, std::vector output_scale, std::vector output_tensorScale); +MNN_PUBLIC VARP _EltwiseSubInt8(VARP x, VARP y, + std::vector x_weight, std::vector x_bias, std::vector x_scale, std::vector x_tensorScale, + std::vector y_weight, std::vector y_bias, std::vector y_scale, std::vector y_tensorScale, + std::vector output_weight, std::vector output_bias, std::vector output_scale, std::vector output_tensorScale); +MNN_PUBLIC VARP _EltwiseMaxInt8(VARP x, VARP y, + std::vector x_weight, std::vector x_bias, std::vector x_scale, std::vector x_tensorScale, + std::vector y_weight, std::vector y_bias, std::vector y_scale, std::vector y_tensorScale, + std::vector output_weight, std::vector output_bias, std::vector output_scale, std::vector output_tensorScale); +MNN_PUBLIC VARP _Mod(VARP x, VARP y); + + +//OtherOPs +template +VARP _Cast(VARP x) { + return _Cast(x, halide_type_of()); +} +MNN_PUBLIC VARP _Cast(VARP x, halide_type_t dtype); +MNN_PUBLIC VARP _MatMul(VARP a, VARP b, bool tranposeA = false, bool tranposeB = false); +MNN_PUBLIC VARP _Normalize(VARP x, int32_t acrossSpatial, int32_t channelShared, float eps, std::vector scale); +MNN_PUBLIC VARP _ArgMax(VARP input, int axis = 0); +MNN_PUBLIC VARP _ArgMin(VARP input, int axis = 0); +MNN_PUBLIC VARP _BatchMatMul(VARP x, VARP y, bool adj_x = false, bool adj_y = false); +MNN_PUBLIC VARP _UnravelIndex(VARP indices, VARP dims); +MNN_PUBLIC VARP _ScatterNd(VARP indices, VARP updates, VARP shape); +MNN_PUBLIC VARP _ScatterNd(VARP indices, VARP updates, VARP shape, VARP input); +MNN_PUBLIC VARP _ScatterNd(VARP indices, VARP updates, VARP shape, int reduction); +MNN_PUBLIC VARP _ScatterNd(VARP indices, VARP updates, VARP shape, VARP input, int reduction); +MNN_PUBLIC VARP _ScatterElements(VARP data, VARP indices, VARP updates, int reduction = -1); +MNN_PUBLIC VARP _ScatterElements(VARP data, VARP indices, VARP updates, VARP axis, int reduction = -1); +MNN_PUBLIC VARP _OneHot(VARP indices, VARP depth, VARP onValue, VARP offValue, int axis = -1); +MNN_PUBLIC VARP _BroadcastTo(VARP a, VARP shape); +MNN_PUBLIC VARP _LinSpace(VARP start, VARP stop, VARP num); + +MNN_PUBLIC VARP _RandomUnifom(VARP shape, halide_type_t dtype, float low = 0.0f, float high = 1.0f, int seed0 = 0, int seed1 = 0); +MNN_PUBLIC VARP _CumSum(VARP x, int axis, bool exclusive = false, bool reverse = false); +MNN_PUBLIC VARP _CumProd(VARP x, int axis); +MNN_PUBLIC VARPS _Svd(VARP x); +MNN_PUBLIC VARP _Histogram(VARP x, int bin, int min, int max, int channel = -1); +}; // namespace Express +}; // namespace MNN + +#endif /* MathOp_HPP */ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Module.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Module.hpp new file mode 100644 index 0000000..dfc8992 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Module.hpp @@ -0,0 +1,143 @@ +// +// Module.hpp +// MNN +// +// Created by MNN on 2019/11/25. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_Train_Module_hpp +#define MNN_Train_Module_hpp + +#include + +#include +#include +#include + +namespace MNN { +class Session; +namespace Express { +struct SubGraph; +class MNN_PUBLIC Module { +public: + Module() = default; + virtual ~Module() = default; + virtual std::vector onForward(const std::vector& inputs) = 0; + Express::VARP forward(Express::VARP input); + std::vector parameters() const; + bool loadParameters(const std::vector& parameters); + void setIsTraining(const bool isTraining); + bool getIsTraining(); + void clearCache(); + const std::string& name() const { + return mName; + }; + void setName(std::string name) { + mName = std::move(name); + } + const std::string type() const { + return mType; + } + void setType(std::string type) { + mType = std::move(type); + } + // Return the parameter index + int addParameter(Express::VARP parameter); + + void setParameter(Express::VARP parameter, int index); + static Module* createEmpty(const std::vector& parameters); + + struct BackendInfo { + MNNForwardType type = MNN_FORWARD_CPU; + BackendConfig* config = nullptr; + }; + + struct Config { + // Load module as dynamic, default static + bool dynamic = false; + + // for static mode, if the shape is mutable, set true, otherwise set false to avoid resizeSession freqencily + bool shapeMutable = true; + // Pre-rearrange weights or not. Disabled by default. + // The weights will be rearranged in a general way, so the best implementation + // may not be adopted if `rearrange` is enabled. + bool rearrange = false; + + BackendInfo* backend = nullptr; + + // base module + const Module* base = nullptr; + }; + static Module* load(const std::vector& inputs, const std::vector& outputs, const uint8_t* buffer, size_t length, const Config* config = nullptr); + static Module* load(const std::vector& inputs, const std::vector& outputs, const char* fileName, const Config* config = nullptr); + // Shared RuntimeManager + static Module* load(const std::vector& inputs, const std::vector& outputs, const char* fileName, const std::shared_ptr rtMgr, const Config* config = nullptr); + static Module* load(const std::vector& inputs, const std::vector& outputs, const uint8_t* buffer, size_t length, const std::shared_ptr rtMgr, const Config* config = nullptr); + + static Module* extract(std::vector inputs, std::vector outputs, bool fortrain, const std::map& subGraph = {}); + + static Module* clone(const Module* module, const bool shareParams = false); + + struct Info { + // Input info load from model + std::vector inputs; + // The Module's defaultFormat, NCHW or NHWC + Dimensionformat defaultFormat; + // Runtime Info + std::shared_ptr runTimeManager; + // Input Names By Order + std::vector inputNames; + // Output Names By Order + std::vector outputNames; + // The MNNConvert's Version build the module + std::string version; + // The bizCode of MNN model + std::string bizCode; + // MetaData + std::map metaData; + // uuid + std::string uuid; + }; + const Info* getInfo() const; + class CloneContext { + public: + std::shared_ptr pRuntimeManager; + }; + virtual Module* clone(CloneContext* ctx) const { + return nullptr; + } + void registerModel(const std::vector>& children); + + static void destroy(Module* m); + + int traceOrOptimize(Interpreter::SessionMode stage); + std::vector> getChildren() const { return mChildren; } +protected: + virtual int onOptimize(Interpreter::SessionMode stage) { + return 0; + } + virtual void onClearCache() { + } + + Module* cloneBaseTo(CloneContext* ctx, Module* module) const; + + std::vector> mChildren; + std::vector mParameters; +private: + void _collectParameters(std::vector& result) const; + bool mIsTraining = true; + std::string mName; + std::string mType; +}; + +struct SubGraph { + std::vector inputs; + std::vector outputs; + std::shared_ptr m; +}; + +} // namespace Train +} // namespace MNN + +#endif diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/NeuralNetWorkOp.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/NeuralNetWorkOp.hpp new file mode 100644 index 0000000..8932f9e --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/NeuralNetWorkOp.hpp @@ -0,0 +1,183 @@ +// +// NeuralNetWorkOp.hpp +// MNN +// +// Created by MNN on 2019/06/27. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_NeuralNetWorkOp_HPP +#define MNN_NeuralNetWorkOp_HPP +#include + +namespace MNN { +namespace Express { +enum PaddingMode {CAFFE, VALID, SAME}; +enum PoolingMode {MAXPOOL, AVEPOOL}; +enum PadValueMode {CONSTANT, REFLECT, SYMMETRIC, EDGE}; +MNN_PUBLIC VARP _Input(INTS shape = {}, Dimensionformat data_format = NC4HW4, halide_type_t dtype = halide_type_of()) ; +MNN_PUBLIC VARP _Clone(VARP source, bool deepCopy = false); + +MNN_PUBLIC VARP _Scalar(const void* ptr, halide_type_t type); + +template +VARP _Scalar(T value) { + return _Scalar(&value, halide_type_of()); +} + + +MNN_PUBLIC VARP _Const(float value, INTS shape = {}, Dimensionformat format = NHWC); +MNN_PUBLIC VARP _Const(const void* ptr, INTS shape = {}, Dimensionformat format = NHWC, + halide_type_t type = halide_type_of()); +MNN_PUBLIC VARP _TrainableParam(float value, INTS dims, Dimensionformat format); +MNN_PUBLIC VARP _TrainableParam(const void* ptr, INTS dims, Dimensionformat format, + halide_type_t type = halide_type_of()); +MNN_PUBLIC VARP _InnerProduct(std::vector&& weight, std::vector&& bias, VARP x, INTS outputShape); +MNN_PUBLIC VARP _Conv(VARP weight, VARP bias, VARP x, PaddingMode pad = VALID, INTS stride = {1, 1}, + INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}); + +MNN_PUBLIC VARP _Conv(float weight, float bias, VARP x, INTS channel, INTS kernelSize, PaddingMode pad = VALID, + INTS stride = {1, 1}, INTS dilate = {1, 1}, int group = 1); +MNN_PUBLIC VARP _Conv(std::vector&& weight, std::vector&& bias, VARP x, INTS channel, INTS kernelSize, + PaddingMode pad = VALID, INTS stride = {1, 1}, INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}, bool relu = false, bool relu6 = false, int nbits = 8); +MNN_PUBLIC VARP _Conv(std::vector&& weight, std::vector&& bias, VARP x, INTS channel, INTS kernelSize, + PaddingMode pad = VALID, INTS stride = {1, 1}, INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}, bool relu = false, bool relu6 = false); +MNN_PUBLIC VARP _Deconv(VARP weight, VARP bias, VARP x, PaddingMode pad = VALID, INTS stride = {1, 1}, + INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}); + +MNN_PUBLIC VARP _Deconv(std::vector&& weight, std::vector&& bias, VARP x, INTS channel, INTS kernelSize, +PaddingMode pad, INTS stride = {1, 1}, INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}, bool relu = false, bool relu6 = false); + +MNN_PUBLIC VARP _MaxPool(VARP x, INTS kernel, INTS stride = {1, 1}, PaddingMode pad = VALID, INTS pads= {0, 0}); +MNN_PUBLIC VARP _AvePool(VARP x, INTS kernel, INTS stride = {1, 1}, PaddingMode pad = VALID, INTS pads= {0, 0}); +MNN_PUBLIC VARP _Reshape(VARP x, INTS shape, Dimensionformat original_format = NCHW); +MNN_PUBLIC VARP _Reshape(VARP x, VARP shape); +MNN_PUBLIC VARP _Scale(VARP x, int channels, std::vector&& scales, std::vector&& bias); + +MNN_PUBLIC VARP _Relu(VARP x, float slope = 0.0f); +MNN_PUBLIC VARP _Relu6(VARP x, float minValue = 0.0f, float maxValue = 6.0f); +MNN_PUBLIC VARP _PRelu(VARP x, std::vector &&slopes); +MNN_PUBLIC VARP _Softmax(VARP logits, int axis = -1); +MNN_PUBLIC VARP _Softplus(VARP features); +MNN_PUBLIC VARP _Softsign(VARP features); +MNN_PUBLIC std::vector _Split(VARP value, INTS size_splits, int axis = 0); +MNN_PUBLIC VARP _Slice(VARP x, VARP starts, VARP sizes); +MNN_PUBLIC VARP _StridedSlice(VARP input, VARP begin, VARP end, VARP strided, + int32_t beginMask, int32_t endMask, int32_t ellipsisMask, + int32_t newAxisMask, int32_t shrinkAxisMask); +MNN_PUBLIC VARP _StridedSliceWrite(VARP input, VARP begin, VARP end, VARP strided, VARP write, + int32_t beginMask, int32_t endMask, int32_t ellipsisMask, + int32_t newAxisMask, int32_t shrinkAxisMask); +MNN_PUBLIC VARP _Concat(VARPS values, int axis); +MNN_PUBLIC VARP _Convert(VARP input, Dimensionformat format); +MNN_PUBLIC VARP _Transpose(VARP x, INTS perm); +MNN_PUBLIC VARP _Transpose(VARP x, VARP perm); +MNN_PUBLIC VARP _ChannelShuffle(VARP x, int group); +MNN_PUBLIC VARP _ChangeInputFormat(VARP input, Dimensionformat format); + +// FIXME: move the api to Array Ops +MNN_PUBLIC VARP _Reverse(VARP x, VARP axis); +MNN_PUBLIC VARP _ReverseSequence(VARP x, VARP y, int batchDim, int seqDim); +// FIXME: move the api to Image Ops +MNN_PUBLIC VARP _Crop(VARP images, VARP size, int axis, INTS offset); +MNN_PUBLIC VARP _Resize(VARP images, float xScale, float yScale); +MNN_PUBLIC VARP _Pad(VARP x, VARP paddings, PadValueMode mode = CONSTANT); +MNN_PUBLIC VARP _ExpandDims(VARP input, int axis); +MNN_PUBLIC VARP _ExpandDims(VARP input, VARP axis); + +MNN_PUBLIC VARP _Shape(VARP input, bool nchw = false); +MNN_PUBLIC VARP _Stack(VARPS values, int axis=0); +enum InterpolationMethod {BILINEAR, NEAREST}; +MNN_PUBLIC VARP _CropAndResize(VARP image, VARP boxes, VARP box_ind, VARP crop_size, + InterpolationMethod method, float extrapolation_value = 0.0); +MNN_PUBLIC VARP _Fill(VARP dims, VARP value); +MNN_PUBLIC VARP _Tile(VARP input, VARP multiples); +MNN_PUBLIC VARP _Gather(VARP params, VARP indices); +MNN_PUBLIC VARP _GatherV2(VARP params, VARP indices, VARP axis = nullptr); +MNN_PUBLIC VARP _Squeeze(VARP input, INTS axis = {}); +MNN_PUBLIC VARP _Unsqueeze(VARP input, INTS axis = {}); +MNN_PUBLIC VARP _BatchToSpaceND(VARP input, VARP block_shape, VARP crops); +MNN_PUBLIC VARP _GatherND(VARP params, VARP indices); +MNN_PUBLIC VARP _GatherElements(VARP params, VARP indices); +MNN_PUBLIC VARP _GatherElements(VARP params, VARP indices, VARP axis); +MNN_PUBLIC VARP _Selu(VARP features, float scale, float alpha); +MNN_PUBLIC VARP _Size(VARP input); +MNN_PUBLIC VARP _Elu(VARP features, float alpha=1.0); +MNN_PUBLIC VARP _Threshold(VARP features, float alpha=1.0); +MNN_PUBLIC VARP _MatrixBandPart(VARP input, VARP num_lower, VARP num_upper); +MNN_PUBLIC std::vector _Moments(VARP x, INTS axis, VARP shift, bool keepDims); +MNN_PUBLIC VARP _SetDiff1D(VARP x, VARP y); +MNN_PUBLIC VARP _SpaceToDepth(VARP input, int block_size); +MNN_PUBLIC VARP _SpaceToBatchND(VARP input, VARP block_shape, VARP paddings); +MNN_PUBLIC VARP _ZerosLike(VARP input); +MNN_PUBLIC std::vector _Unstack(VARP value, int axis=0); +MNN_PUBLIC VARP _Rank(VARP input); +MNN_PUBLIC VARP _Range(VARP start, VARP limit, VARP delta); +MNN_PUBLIC VARP _DepthToSpace(VARP input, int block_size); +MNN_PUBLIC VARP _PriorBox(VARP feature, VARP image, + std::vector min_size, std::vector max_size, std::vectoraspect_ratio, + bool flip, bool clip, std::vectorvariance, + unsigned int img_h, unsigned int img_w, float step_h, float step_w, float offset = 0.5); +MNN_PUBLIC VARP _Permute(VARP input, INTS dims); +MNN_PUBLIC VARP _DetectionOutput(VARP location, VARP confidence, VARP priorbox, + unsigned int num_classes, bool share_location, int background_label_id, + float nms_threshhold, int nms_topk, int code_type, + bool variance_encoded_in_target, + int keep_top_k, float confidence_threshold, float visualize_threshold); +MNN_PUBLIC std::vector _DetectionPostProcess(VARP encode_boxes, VARP class_predictions, VARP anchors, + int num_classes, int max_detections, + int max_class_per_detection, int detections_per_class, + float nms_threshold, float iou_threshold, + bool use_regular_nms, std::vector centersize_encoding); +MNN_PUBLIC VARP _Interp(VARPS xs, float widthScale, float heightScale, int outputWidth, int outputHeight, int resizeType, bool alignCorners); + +MNN_PUBLIC VARP _ZeroGrad(VARP x); + +// Int8 Inference +MNN_PUBLIC VARP _Conv(std::vector&& weight, std::vector&& bias, std::vector&& scale, VARP x, INTS channel, INTS kernelSize, + PaddingMode pad, INTS stride, INTS dilate, int group, INTS pads, bool relu, int nbits = 8); +MNN_PUBLIC VARP _Conv(std::vector&& weight, std::vector&& bias, std::vector&& scale, + VARP x, INTS channel, INTS kernelSize, + PaddingMode pad, INTS stride, INTS dilate, int group, INTS pads, bool relu, + int8_t inputZeroPoint, int8_t outputZeroPoint, + int8_t minValue, int8_t maxValue, bool accumulateToInt16); +MNN_PUBLIC VARP _Conv(std::vector&& weight, std::vector&& bias, std::vector&& weightScale, + VARP x, INTS channel, INTS kernelSize, + PaddingMode pad, INTS stride, INTS dilate, int group, INTS pads, bool relu, + float scaleIn, float scaleOut, + int8_t inputZeroPoint, int8_t outputZeroPoint, + int8_t minValue, int8_t maxValue, float weightClampValue, bool accumulateToInt16); +MNN_PUBLIC VARP _CosineSimilarity(VARP input0, VARP input1, VARP inputDim); + +enum GridSamplePaddingMode {GRID_SAMPLE_PADDING_ZEROS, GRID_SAMPLE_PADDING_BORDER, GRID_SAMPLE_PADDING_REFLECTION}; +MNN_PUBLIC VARP _GridSample(VARP input, VARP grid, InterpolationMethod mode=BILINEAR, GridSamplePaddingMode paddingMode=GRID_SAMPLE_PADDING_ZEROS, bool alignCorners=false); +MNN_PUBLIC VARP _FloatToInt8(VARP x, VARP scale, char minValue, char maxValue); +MNN_PUBLIC VARP _FloatToInt8(VARP x, VARP scale, int8_t minValue, int8_t maxValue, int8_t zeroPoint); +MNN_PUBLIC VARP _Int8ToFloat(VARP x, VARP scale); +MNN_PUBLIC VARP _Int8ToFloat(VARP x, VARP scale, int8_t zeroPoint); + +MNN_PUBLIC VARP _Select(VARP select, VARP input0, VARP input1); +MNN_PUBLIC std::vector _TopKV2(VARP input0, VARP input1); +MNN_PUBLIC VARP _ImageProcess(VARP input, CV::ImageProcess::Config config, CV::Matrix matrix, int oh, int ow, int oc, int dtype, uint8_t padVal = 0); +MNN_PUBLIC VARP _Where(VARP x); +MNN_PUBLIC VARP _Sort(VARP x, int axis = -1, bool arg = false, bool descend = false); +MNN_PUBLIC VARP _Raster(const std::vector& vars, const std::vector& regions, const std::vector& shape); +MNN_PUBLIC VARP _RasterRaw(const std::vector& vars, const std::vector& region, const std::vector& shape, halide_type_t dataType, Dimensionformat format); + +MNN_PUBLIC VARP _Nms(VARP boxes, VARP scores, int maxDetections, float iouThreshold = -1, float scoreThreshold = -1); +MNN_PUBLIC VARP _Im2Col(VARP x, INTS kernelSize, INTS dilate, INTS pads, INTS stride); +MNN_PUBLIC VARP _Col2Im(VARP x, VARP outputShape, INTS kernelSize, INTS dilate, INTS pads, INTS stride); + +/** + Onnx's Loop + */ +MNN_PUBLIC VARPS _Loop(VARPS x, const std::string& submoduleName); +MNN_PUBLIC VARP _ROIPooling(VARP input, VARP roi, int pooledHeight, int pooledWidth, float spatialScale, bool outputGrad = false, VARP backwardDiff = nullptr); +MNN_PUBLIC VARP _ROIAlign(VARP input, VARP roi, int pooledHeight, int pooledWidth, float spatialScale, int samplingRatio, bool aligned, PoolingMode poolType, bool outputGrad = false, VARP backwardDiff = nullptr); + +MNN_PUBLIC VARPS _JSONOp(VARPS x, const char* opDescribe, int outputNumber); + +} // namespace Express +} // namespace MNN + +#endif /* NeuralNetWorkOp_HPP */ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Optimizer.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Optimizer.hpp new file mode 100644 index 0000000..b1304c8 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Optimizer.hpp @@ -0,0 +1,64 @@ +// +// Optimizer.hpp +// MNN +// +// Created by MNN on 2019/08/20. +// Copyright © 2018, Alibaba Group Holding Limited +// +#ifndef Optimizer_hpp +#define Optimizer_hpp +#include +#include + +namespace MNN { +namespace Express { +class MNN_PUBLIC Optimizer { +public: + enum Device { + CPU = 0, + GPU = 1, + OTHER = 2, + AUTO = 3 + }; + struct Config { + Device device = CPU; + MNNForwardType forwardType = MNN_FORWARD_ALL; + int numThread = 4; + }; + static std::shared_ptr create(Config config); + struct Cost { + float compute; // MFlops + float memory; // MB + }; + class Parameters { + public: + Parameters(int n); + virtual ~Parameters(); + + float* get() const { + return mValue; + } + int size() const { + return mSize; + } + + private: + float* mValue; + int mSize; + }; + virtual std::shared_ptr onGetParameters(const std::vector& outputs) { + return nullptr; + } + + //Given paramters and measure cost, the parameters must be the same as onGetParameters + virtual Cost onMeasure(const std::vector& outputs, std::shared_ptr parameters = nullptr) = 0; + + //Modify the output directly, the parameters must be the same as onGetParameters + virtual bool onExecute(const std::vector& outputs, std::shared_ptr parameters = nullptr) = 0; + + Optimizer() = default; + virtual ~Optimizer() = default; +}; +} // namespace Express +} // namespace MNN +#endif diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Scope.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Scope.hpp new file mode 100644 index 0000000..10366fe --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/expr/Scope.hpp @@ -0,0 +1,112 @@ +// +// RuntimeScope.hpp +// MNN +// +// Created by MNN on 2020/10/26. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_EXPR_SCOPE_HPP_ +#define MNN_EXPR_SCOPE_HPP_ + +#include +#include +#include +#include + +#include + +namespace MNN { +namespace Express { + +template +class Scope { +public: + Scope(); + virtual ~Scope() = default; + + struct ScopedContent { + std::string scope_name; + T content; + }; + void EnterScope(const ScopedContent& current); + void EnterScope(const T& current); + void EnterScope(const std::string& scope_name, const T& current); + + void ExitScope(); + + const ScopedContent& Current() const; + const T Content() const; + + int ScopedLevel() const { return scoped_level_; } + +private: + std::string MakeScopeName(const std::string& prefix, int level) const; + + mutable std::mutex mutex_; + int scoped_level_ = 0; + std::vector scoped_contents_; +}; + +template +Scope::Scope() : scoped_level_(0) { +} + +template +void Scope::EnterScope(const ScopedContent& current) { + std::lock_guard lock(mutex_); + ++scoped_level_; + scoped_contents_.push_back(current); +} + +template +void Scope::EnterScope(const T& current) { + EnterScope("scope", current); +} + +template +void Scope::EnterScope(const std::string& scope_name, + const T& current) { + std::lock_guard lock(mutex_); + int scoped_level = ScopedLevel(); + std::string name = MakeScopeName(scope_name, scoped_level++); + ScopedContent content{name, current}; + ++scoped_level_; + scoped_contents_.push_back(content); +} + +template +void Scope::ExitScope() { + std::lock_guard lock(mutex_); + --scoped_level_; + scoped_contents_.resize(scoped_level_); +} + +template +const typename Scope::ScopedContent& Scope::Current() const { + std::lock_guard lock(mutex_); + MNN_CHECK(scoped_contents_.size() > 0, "Scope level should not be 0."); + return scoped_contents_.back(); +} + +template +const T Scope::Content() const { + std::lock_guard lock(mutex_); + if (scoped_contents_.empty()) { + return nullptr; + } + return scoped_contents_.back().content; +} + +template +std::string Scope::MakeScopeName(const std::string& prefix, + int level) const { + char s[16]; + snprintf(s, 16, "%d", level); + return prefix + "/" + std::string(s); +} + +} // namespace Express +} // namespace MNN + +#endif // MNN_EXPR_SCOPE_HPP_ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginContext.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginContext.hpp new file mode 100644 index 0000000..cd86ac8 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginContext.hpp @@ -0,0 +1,151 @@ +// +// ShapeInference.h +// MNN +// +// Created by MNN on 2020/04/05. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_PLUGIN_PLUGIN_CONTEXT_HPP_ +#define MNN_PLUGIN_PLUGIN_CONTEXT_HPP_ + +#include +#include + +#include // Backend +#include +#include "Tensor_generated.h" + +namespace MNN { +namespace plugin { + +class MNN_PUBLIC PluginContext { +public: + PluginContext() = delete; + PluginContext(const std::vector& inputs, // NOLINT + const std::vector& outputs); + void reset(const std::vector& inputs, const std::vector& outputs) { + inputs_ = inputs; + outputs_ = outputs; + } + + virtual ~PluginContext() = default; + + const std::vector& inputs() const { + return inputs_; + } + const std::vector& outputs() const { + return outputs_; + } + + const Tensor* input(const int index) const; + const Tensor* output(const int index) const; + + Tensor* output(const int index); + + bool hasAttr(const std::string& name) const; + + bool setAttr(const std::string& name, const Attribute* attr); + + void setAttrs(const std::unordered_map& attrs); + + const Attribute* getAttr(const std::string& name) const; + + const std::unordered_map& getAttrs() const; + +protected: + std::vector inputs_; + std::vector outputs_; + std::unordered_map attrs_; +}; + +class MNN_PUBLIC InferShapeContext : public PluginContext { +public: + InferShapeContext() = delete; + InferShapeContext(const std::vector& inputs, // NOLINT + const std::vector& outputs); + + virtual ~InferShapeContext() = default; +}; + +class MNN_PUBLIC CPUKernelContext : public PluginContext { +public: + CPUKernelContext() = delete; + CPUKernelContext(const std::string& op_type, // NOLINT + Backend* backend, // NOLINT + const std::vector& inputs, // NOLINT + const std::vector& outputs, + const std::string& dir_path); + + virtual ~CPUKernelContext() = default; + + Backend* backend() const { + return backend_; + } + + const std::string& op_type() const { + return op_type_; + } + + const std::string& dir_path() const { + return dir_path_; + } + +private: + const std::string op_type_ = ""; + const std::string dir_path_; + Backend* backend_ = nullptr; +}; + +inline PluginContext::PluginContext(const std::vector& inputs, // NOLINT + const std::vector& outputs) // NOLINT + : inputs_(inputs), outputs_(outputs) { +} + +inline const Tensor* PluginContext::input(const int index) const { + MNN_ASSERT(index < inputs_.size()); + return inputs_.at(index); +} + +inline const Tensor* PluginContext::output(const int index) const { + MNN_ASSERT(index < outputs_.size()); + return outputs_.at(index); +} + +inline Tensor* PluginContext::output(const int index) { + MNN_ASSERT(index < outputs_.size()); + return outputs_.at(index); +} + +inline bool PluginContext::hasAttr(const std::string& name) const { + return attrs_.count(name) > 0; +} + +inline bool PluginContext::setAttr(const std::string& name, // NOLINT + const Attribute* attr) { + return attrs_.emplace(name, attr).second; +} + +inline void PluginContext::setAttrs( // NOLINT + const std::unordered_map& attrs) { + attrs_ = attrs; +} + +inline const Attribute* PluginContext::getAttr(const std::string& name) const { + const auto& it = attrs_.find(name); + if (it == attrs_.end()) { + return nullptr; + } + return it->second; +} + +inline const std::unordered_map& // NOLINT +PluginContext::getAttrs() const { + return attrs_; +} + +} // namespace plugin +} // namespace MNN + +#endif // MNN_PLUGIN_PLUGIN_CONTEXT_HPP_ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginKernel.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginKernel.hpp new file mode 100644 index 0000000..e7c30b0 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginKernel.hpp @@ -0,0 +1,77 @@ +// +// ShapeInference.h +// MNN +// +// Created by MNN on 2020/04/05. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_PLUGIN_PLUGIN_KERNEL_HPP_ +#define MNN_PLUGIN_PLUGIN_KERNEL_HPP_ + +#include +#include +#include + +#include + +namespace MNN { +namespace plugin { + +template +class MNN_PUBLIC ComputeKernel { +public: + ComputeKernel() = default; + virtual ~ComputeKernel() = default; + virtual bool compute(KernelContextT* ctx) = 0; +}; + +class MNN_PUBLIC CPUComputeKernel : public ComputeKernel { +public: + using ContextT = CPUKernelContext; + using KernelT = CPUComputeKernel; + + CPUComputeKernel() = default; + virtual ~CPUComputeKernel() = default; + virtual bool init(CPUKernelContext* ctx) = 0; + virtual bool compute(CPUKernelContext* ctx) = 0; + virtual bool resize(CPUKernelContext* ctx) { + return true; + } + inline bool needAllocIO() const { + return mNeedAllocIO; + } +protected: + bool mNeedAllocIO = true; +}; + +template +class MNN_PUBLIC ComputeKernelRegistry { +public: + typedef std::function Factory; + static std::unordered_map* getFactoryMap(); + + static bool add(const std::string& name, Factory factory); + + static PluginKernelT* get(const std::string& name); +}; + +template +struct ComputeKernelRegistrar { + ComputeKernelRegistrar(const std::string& name) { + ComputeKernelRegistry::add(name, []() { // NOLINT + return new PluginKernelT; // NOLINT + }); + } +}; + +#define REGISTER_PLUGIN_COMPUTE_KERNEL(name, computeKernel) \ + namespace { \ + static auto _plugin_compute_kernel_##name##_ __attribute__((unused)) = \ + ComputeKernelRegistrar(#name); \ + } // namespace + +} // namespace plugin +} // namespace MNN + +#endif // MNN_PLUGIN_PLUGIN_KERNEL_HPP_ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginShapeInference.hpp b/app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginShapeInference.hpp new file mode 100644 index 0000000..fe272ed --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/MNN/plugin/PluginShapeInference.hpp @@ -0,0 +1,56 @@ +// +// ShapeInference.h +// MNN +// +// Created by MNN on 2020/04/05. +// Copyright © 2018, Alibaba Group Holding Limited +// + +#ifndef MNN_PLUGIN_PLUGIN_SHAPE_INFERENCE_HPP_ +#define MNN_PLUGIN_PLUGIN_SHAPE_INFERENCE_HPP_ + +#include +#include +#include + +#include + +namespace MNN { +namespace plugin { + +class MNN_PUBLIC InferShapeKernel { +public: + virtual ~InferShapeKernel() = default; + virtual bool compute(InferShapeContext* ctx) = 0; +}; + +class MNN_PUBLIC InferShapeKernelRegister { +public: + // typedef InferShapeKernel* (*Factory)(); + typedef std::function Factory; + static std::unordered_map* getFactoryMap(); + + static bool add(const std::string& name, Factory factory); + + static InferShapeKernel* get(const std::string& name); +}; + +template +struct InferShapeKernelRegistrar { + InferShapeKernelRegistrar(const std::string& name) { + InferShapeKernelRegister::add(name, []() { // NOLINT + return new PluginKernel; // NOLINT + }); + } +}; + +#define REGISTER_PLUGIN_OP(name, inferShapeKernel) \ + namespace { \ + static auto _plugin_infer_shape_##name##_ __attribute__((unused)) = \ + InferShapeKernelRegistrar(#name); \ + } // namespace + +} // namespace plugin +} // namespace MNN + +#endif // MNN_PLUGIN_PLUGIN_SHAPE_INFERENCE_HPP_ diff --git a/app/entry/src/main/cpp/third_party/mnn/include/llm/llm.hpp b/app/entry/src/main/cpp/third_party/mnn/include/llm/llm.hpp new file mode 100644 index 0000000..b7a06db --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/llm/llm.hpp @@ -0,0 +1,281 @@ +// +// llm.hpp +// +// Created by MNN on 2023/08/25. +// ZhaodeWang +// + +#ifndef LLM_hpp +#define LLM_hpp + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace MNN { +struct KVMeta; +namespace Transformer { +using MNN::KVMeta; + +// ChatMessage: pair for multi-turn conversation. +// first = role: "system", "user", "assistant", "tool", etc. +// second = content: plain text message content. +// For complex messages (tool_calls, reasoning_content, etc.): +// first = "json" +// second = full JSON object string, e.g. {"role":"assistant","content":"","tool_calls":[...]} +using ChatMessage = std::pair; +using ChatMessages = std::vector; +class Tokenizer; +class Pipeline; +class LlmConfig; +class DiskEmbedding; +class Sampler; +class Generation; +class EagleGeneration; +struct TimePerformance; +#define CHECK_LLM_RUNNING_RET(ctx, ret) \ +{ \ + if ((ctx)->status == LlmStatus::NOT_LOADED || \ + (ctx)->status == LlmStatus::INTERNAL_ERROR || \ + (ctx)->status == LlmStatus::TIMEOUT || \ + (ctx)->status == LlmStatus::USER_CANCEL) { \ + MNN_ERROR("[Error]: LLM in error state. Status: %d\n", \ + static_cast((ctx)->status)); \ + return (ret); \ + } \ +} +#define CHECK_LLM_RUNNING(ctx) \ +{ \ + if ((ctx)->status == LlmStatus::NOT_LOADED || \ + (ctx)->status == LlmStatus::INTERNAL_ERROR || \ + (ctx)->status == LlmStatus::TIMEOUT || \ + (ctx)->status == LlmStatus::USER_CANCEL) { \ + MNN_ERROR("[Error]: LLM in error state. Status: %d\n", \ + static_cast((ctx)->status)); \ + return; \ + } \ +} + +struct MNN_PUBLIC PromptImagePart { + MNN::Express::VARP image_data; + int width; + int height; +}; + +struct MNN_PUBLIC PromptAudioPart { + std::string file_path; + MNN::Express::VARP waveform; +}; + +struct MNN_PUBLIC MultimodalPrompt { + std::string prompt_template; + std::map images; + std::map audios; +}; + +enum TuneType { + // op encoder number for commit + OP_ENCODER_NUMBER = 0, +}; +enum class LlmStatus { + NOT_LOADED = -1, + RUNNING = 0, + NORMAL_FINISHED = 1, + MAX_TOKENS_FINISHED = 2, + USER_CANCEL = 3, + INTERNAL_ERROR = 4, + TIMEOUT = 5, +}; +enum class MatchStrictLevel : int; +enum class NgramSelectRule : int; + +struct LlmContext { + // forward + int prompt_len = 0; + int gen_seq_len = 0; + int all_seq_len = 0; + std::ostream* os = nullptr; + std::string end_with; + // perf + int64_t load_us = 0; + int64_t vision_us = 0; + int64_t audio_us = 0; + int64_t prefill_us = 0; + int64_t decode_us = 0; + int64_t sample_us = 0; + int64_t ttfa_us = 0; + float pixels_mp = 0; + float audio_input_s = 0; + // tokens + int current_token; + std::vector history_tokens; + std::vector output_tokens; + std::string generate_str; + // llm status + LlmStatus status = LlmStatus::NOT_LOADED; + // log buffer (per-instance, no locking needed) + std::string log_buffer; +}; +struct GenerationParams; +class MNN_PUBLIC Llm { +public: + enum Stage { + Prefill, + Decode + }; + // Log buffer interface: retrieve accumulated log and clear the buffer. + // Only effective when LLM_LOG_TO_STRING macro is enabled during compilation. + std::string getLog(); + static Llm* createLLM(const std::string& config_path); + static void destroy(Llm* llm);// For Windows RT mode should use destroy + Llm(std::shared_ptr config); + virtual ~Llm(); + virtual bool load(); + virtual Express::VARP gen_attention_mask(int seq_len); + virtual Express::VARP gen_position_ids(int seq_len); + virtual Express::VARP embedding(const std::vector& input_ids); + virtual int sample(Express::VARP logits, int offset = 0, int size = 0); + std::vector getOutputs() const; + int getOutputIndex(const std::string& name) const; + void reset(); + void tuning(TuneType type, std::vector candidates); + virtual std::vector forwardRaw(Express::VARP hiddenState, Express::VARP mask, Express::VARP inputPos, Express::VARPS extraArgs = {}); + Express::VARP forward(const std::vector& input_ids, bool is_prefill = true); + Express::VARP forward(MNN::Express::VARP input_embeds); + void switchMode(Stage stage); + void setKVCacheInfo(size_t add, size_t remove, int* reserve = nullptr, int n_reserve = 0); + size_t getCurrentHistory() const; + void eraseHistory(size_t begin, size_t end); + bool setPrefixCacheFile(const std::string& filename, int flag = 0); + virtual void response(const std::vector& input_ids, std::ostream* os = &std::cout, const char* end_with = nullptr, int max_new_tokens = -1); + void response(const std::string& user_content, std::ostream* os = &std::cout, const char* end_with = nullptr, int max_new_tokens = -1); + void response(const ChatMessages& chat_prompts, std::ostream* os = &std::cout, const char* end_with = nullptr, int max_new_tokens = -1); + void response(MNN::Express::VARP input_embeds, std::ostream* os = &std::cout, const char* end_with = nullptr, int max_new_tokens = -1); + virtual void generate_init(std::ostream* os = nullptr, const char* end_with = nullptr); + void generate(int max_token); + std::vector generate(const std::vector& input_ids, int max_new_tokens = -1); + std::vector generate(MNN::Express::VARP input_embeds, int max_tokens = -1); + bool stoped(); + bool reuse_kv(); + // Prompt cache: call after decode completes to sync the cached text with the + // full conversation (including assistant response). Optional — the cache + // self-updates after generate(), but this allows callers with post-processed + // response text (e.g. deleteThinkPart) to provide a more accurate version. + void syncPromptCache(const ChatMessages& chat_prompts); + // config function + std::string dump_config(); + bool set_config(const std::string& content); + void setDebugCallback(MNN::TensorCallBackWithInfo&& before, MNN::TensorCallBackWithInfo&& after); + Llm* create_lora(const std::string& lora_path); + // tokenier function + bool is_stop(int token); + std::string tokenizer_decode(int token); + virtual std::vector tokenizer_encode(const std::string& query); + friend class Pipeline; + virtual std::vector tokenizer_encode(const MultimodalPrompt& multimodal_input); + // ptompt functions + std::string apply_chat_template(const std::string& user_content) const; + std::string apply_chat_template(const ChatMessages& chat_prompts) const; + void response(const MultimodalPrompt& multimodal_input, + std::ostream* os = &std::cout, + const char* end_with = nullptr, + int max_new_tokens = -1); + const LlmContext* getContext() const { + return mContext.get(); + } + virtual void setWavformCallback(std::function callback) {} + virtual void generateWavform() {} +protected: + void setChatTemplate(); + void initRuntime(); + void setRuntimeHint(std::shared_ptr &rtg, bool mllm = false); + std::shared_ptr mContext; + std::shared_ptr mMeta; + std::shared_ptr mConfig; + std::shared_ptr mTokenizer; + std::shared_ptr mDiskEmbedding; + std::shared_ptr mPleEmbedding; + Express::VARP mPleInput; // PLE embeddings for current input + Express::VARP mTextEmbedsForPle; // Pure text embeddings for PLE projection + std::shared_ptr mSampler; + std::shared_ptr mRuntimeManager, mProcessorRuntimeManager; + std::shared_ptr mModule; + /** + key: + value : module + note: prefill share one module, seq_len = 100 for example + */ + const int mPrefillKey = 100; + std::map, std::shared_ptr> mModulePool; + const Express::Module* mBaseModule = nullptr; + Express::VARP inputsEmbeds, attentionMask, positionIds; + std::vector mAttentionMaskVarVec, mPositionIdsVarVec; + Express::VARP logitsAllIdx, logitsLastIdx; + int mSeqLenIndex = 0; + std::shared_ptr mExecutor; +protected: + friend class ArGeneration; + friend class LookaheadGeneration; + friend class MtpGeneration; + friend class EagleGeneration; + friend class DFlashGeneration; + friend class Omni; + std::vector forwardVec(const std::vector& input_ids); + std::vector forwardVec(MNN::Express::VARP input_embeds); +private: + std::shared_ptr mGenerationStrategy; + void setSpeculativeConfig(); + void updateContext(int seq_len, int gen_len); + bool checkFile(const std::string& path, const char* name); + +private: + bool mInSpec = false; + int mDraftLength = 4; + std::shared_ptr mGenerateParam; + bool mAsync = true; + int mBlockSize = 0; + std::vector mValidBlockSize; + bool mPrefixCacheMode = false; + std::string mPrefixCacheFileName; + int mCallIndex; + int mPrefixLength; + bool mIsPrefixFileExist = false; + void completePrefixWrite(); + // Prompt cache state + std::string mCachedPromptText; + void updateCachedPromptText(const ChatMessages& chat_prompts, size_t history_before); +}; + +// Embedding start +class MNN_PUBLIC Embedding : public Llm { +public: + Embedding(std::shared_ptr config); + static Embedding* createEmbedding(const std::string& config_path, bool load = true); + static float dist(Express::VARP var0, Express::VARP var1); + static float cos_sim(Express::VARP var0, Express::VARP var1); + virtual bool load() override; + + Express::VARP ids_embedding(const std::vector& ids); + Express::VARP txt_embedding(const std::string& txt); + std::vector forwardRaw(Express::VARP hiddenState, Express::VARP mask, Express::VARP inputPos, Express::VARPS extraArgs = {}) override; + int dim() const; + virtual Express::VARP gen_attention_mask(int seq_len) override; + virtual Express::VARP gen_position_ids(int seq_len) override; +}; +// Embedding end +} +} + +#endif // LLM_hpp \ No newline at end of file diff --git a/app/entry/src/main/cpp/third_party/mnn/include/llm/reranker.hpp b/app/entry/src/main/cpp/third_party/mnn/include/llm/reranker.hpp new file mode 100644 index 0000000..d476f51 --- /dev/null +++ b/app/entry/src/main/cpp/third_party/mnn/include/llm/reranker.hpp @@ -0,0 +1,307 @@ +// +// reranker.hpp +// +// Created by MNN on 2025/07/29. +// ZhaodeWang +// + +#ifndef MNN_LLM_RERANKER_hpp +#define MNN_LLM_RERANKER_hpp + +#include +#include +#include +#include +#include "llm.hpp" + +namespace MNN { +namespace Transformer { + +using namespace MNN::Express; + +/** + * @brief Abstract base class for Rerankers. + * This class defines the interface for any reranker implementation. + */ +class RerankerBase { +public: + /** + * @brief Virtual destructor to ensure proper cleanup of derived classes. + */ + virtual ~RerankerBase() = default; + + /** + * @brief Initializes the reranker with a configuration path. + * @param config_path The path to the configuration file. + */ + virtual void initialize(const std::string& config_path) = 0; + + /** + * @brief Loads the reranker model after initialization. + */ + virtual void load() = 0; + + /** + * @brief Sets the instruction for the reranker. + * @param instruct The instruction string. + */ + virtual void setInstruct(const std::string& instruct) = 0; + + /** + * @brief Computes scores for a list of documents based on a query and instruct. + * @param query The input query. + * @param documents A vector of document strings. + * @return A vector of float scores, one for each document. + */ + virtual std::vector compute_scores(const std::string& query, const std::vector& documents) = 0; + + /** + * @brief Gets the underlying LLM instance for configuration purposes. + * @return A pointer to the LLM instance, or nullptr if not available. + */ + Llm* get_llm() { + return mLlm.get(); + } + +protected: + std::unique_ptr mLlm; +}; + + +class Qwen3Reranker : public RerankerBase { +public: + /** + * @brief Constructor for Qwen3Reranker. + * @param config_path The path to the LLM configuration. + */ + Qwen3Reranker(const std::string& config_path) { + initialize(config_path); + } + + /** + * @brief Initializes the LLM. + * @param config_path The path to the LLM configuration. + */ + void initialize(const std::string& config_path) override { + mLlm.reset(Llm::createLLM(config_path)); + mLlm->set_config("{\"all_logits\":true}"); + } + + /** + * @brief Loads the LLM and initializes token IDs. + */ + void load() override { + mLlm->load(); + mTokenTrueId = mLlm->tokenizer_encode("yes")[0]; + mTokenFalseId = mLlm->tokenizer_encode("no")[0]; + } + + /** + * @brief Sets the instruction for the reranker. + * @param instruct The instruction string. + */ + void setInstruct(const std::string& instruct) override { + mInstruct = instruct; + } + + /** + * @brief Computes scores for a list of documents using the Qwen3 model. + * @param query The input query. + * @param documents A vector of document strings. + * @return A vector of float scores, one for each document. + */ + std::vector compute_scores(const std::string& query, const std::vector& documents) override { + CHECK_LLM_RUNNING_RET(mLlm->getContext(), std::vector()); + std::string prefix = "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n"; + prefix = prefix + ": " + mInstruct + "\n: " + query + "\n: "; + auto suffix = "<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + auto input_ids = mLlm->tokenizer_encode(prefix); + auto suffix_ids = mLlm->tokenizer_encode(suffix); + std::vector position_ids_; + std::vector> spans; + int prefix_len = input_ids.size(); + int suffix_len = suffix_ids.size(); + int current_len = prefix_len; + for (int i = 0; i < current_len; i++) { + position_ids_.push_back(i); + } + for (const auto& doc : documents) { + auto doc_ids = mLlm->tokenizer_encode(doc); + input_ids.insert(input_ids.end(), doc_ids.begin(), doc_ids.end()); + input_ids.insert(input_ids.end(), suffix_ids.begin(), suffix_ids.end()); + int doc_len = doc_ids.size() + suffix_len; + int start_idx = current_len; + int end_idx = current_len + doc_len; + spans.emplace_back(start_idx, end_idx); + current_len = end_idx; + for (int i = prefix_len; i < prefix_len + doc_len; i++) { + position_ids_.push_back(i); + } + } + int total_len = input_ids.size(); + // printf("Total input length: %d\n", total_len); + // input_embeds + auto input_embeds = mLlm->embedding(input_ids); + // attention mask + auto attention_mask = _Input({1, 1, total_len, total_len}, NCHW, halide_type_of()); + auto ptr = attention_mask->writeMap(); + for (int i = 0; i < total_len; i++) { + for (int j = 0; j < total_len; j++) { + ptr[total_len * i + j] = (j > i) * std::numeric_limits::lowest(); + } + } + for (int i = 0; i < spans.size(); i++) { + for (int j = 0; j < spans.size(); j++) { + if (i == j) { + continue; + } + int start_i = spans[i].first; + int end_i = spans[i].second; + int start_j = spans[j].first; + int end_j = spans[j].second; + for (int k = start_i; k < end_i; k++) { + for (int l = start_j; l < end_j; l++) { + ptr[total_len * k + l] = std::numeric_limits::lowest(); + } + } + } + } + // position ids + auto position_ids = _Input({1, total_len}, NCHW, halide_type_of()); + auto pos_ptr = position_ids->writeMap(); + for (int i = 0; i < total_len; i++) { + pos_ptr[i] = position_ids_[i]; + } + + // clear history cache + mLlm->setKVCacheInfo(total_len, mLlm->getCurrentHistory()); + + // forward + auto outputs = mLlm->forwardRaw(input_embeds, attention_mask, position_ids); + if(outputs.empty()) { + return {}; + } + auto logits = outputs[0]; + auto logits_dim = logits->getInfo()->dim[2]; + auto logits_ptr = logits->readMap(); + + mLlm->reset(); + std::vector scores; + for (int i = 0; i < spans.size(); i++) { + int logits_idx = spans[i].second - 1; + float true_logit = logits_ptr[logits_idx * logits_dim + mTokenTrueId]; + float false_logit = logits_ptr[logits_idx * logits_dim + mTokenFalseId]; + // logsoftmax + float max_logit = std::max(true_logit, false_logit); + float stable_true_logit = true_logit - max_logit; + float stable_false_logit = false_logit - max_logit; + float exp_true = std::exp(stable_true_logit); + float exp_false = std::exp(stable_false_logit); + float score = exp_true / (exp_true + exp_false); + scores.push_back(score); + } + + return scores; + } + +private: + std::string mInstruct; + int mTokenTrueId; + int mTokenFalseId; +}; + +template +static inline VARP _var(std::vector vec, const std::vector &dims) { + return _Const(vec.data(), dims, NHWC, halide_type_of()); +} + +class GteReranker : public RerankerBase { +public: + /** + * @brief Constructor for Qwen3Reranker. + * @param config_path The path to the LLM configuration. + */ + GteReranker(const std::string& config_path) { + initialize(config_path); + } + + /** + * @brief Initializes the LLM and token IDs. + * @param config_path The path to the LLM configuration. + */ + void initialize(const std::string& config_path) override { + mLlm.reset(Embedding::createEmbedding(config_path, false)); + } + + /** + * @brief Loads the reranker model after initialization. + */ + void load() override { + mLlm->load(); + } + + /** + * @brief Sets the instruction for the reranker. + * @param instruct The instruction string. + */ + void setInstruct(const std::string& instruct) override {} + + /** + * @brief Computes scores for a list of documents using the Qwen3 model. + * @param query The input query. + * @param documents A vector of document strings. + * @return A vector of float scores, one for each document. + */ + std::vector compute_scores(const std::string& query, const std::vector& documents) override { + CHECK_LLM_RUNNING_RET(mLlm->getContext(), std::vector()); + constexpr int pad_token_id = 1; + constexpr int eos_token_id = 2; + std::vector query_ids {0, 6}; + auto q_ids = mLlm->tokenizer_encode(query); + query_ids.insert(query_ids.end(), q_ids.begin(), q_ids.end()); + query_ids.push_back(eos_token_id); + query_ids.push_back(eos_token_id); + std::vector> all_ids; + int max_len = 0; + for (const auto& doc : documents) { + auto doc_ids = mLlm->tokenizer_encode(doc); + std::vector ids(query_ids); + ids.insert(ids.end(), doc_ids.begin(), doc_ids.end()); + ids.push_back(eos_token_id); + max_len = std::max(max_len, (int)ids.size()); + all_ids.push_back(std::move(ids)); + } + const int batch = static_cast(documents.size()); + const int total_len = max_len * batch; + auto attention_mask = _Input({batch, 1, 1, max_len}, NCHW, halide_type_of()); + auto mask_ptr = attention_mask->writeMap(); + std::vector input_ids(total_len); + int idx = 0; + for (auto& ids : all_ids) { + int i = 0; + for (; i < ids.size(); i++, idx++) { + input_ids[idx] = ids[i]; + mask_ptr[idx] = 0.0; + } + for (; i < max_len; i++, idx++) { + input_ids[idx] = pad_token_id; + mask_ptr[idx] = std::numeric_limits::lowest(); + } + } + + auto input_embeds = mLlm->embedding(input_ids); + input_embeds = _Reshape(input_embeds, _var({batch, max_len, -1}, {3})); + auto position_ids = mLlm->gen_position_ids(max_len); + auto outputs = mLlm->forwardRaw(input_embeds, attention_mask, position_ids); + if(outputs.empty()) { + return {}; + } + std::vector scores(outputs[0]->readMap(), outputs[0]->readMap() + batch); + return scores; + } + +}; + +} +} +#endif /* MNN_LLM_RERANKER_hpp */ diff --git a/app/entry/src/main/cpp/types/liblocalllm/index.d.ts b/app/entry/src/main/cpp/types/liblocalllm/index.d.ts new file mode 100644 index 0000000..f4b9927 --- /dev/null +++ b/app/entry/src/main/cpp/types/liblocalllm/index.d.ts @@ -0,0 +1,16 @@ +/** + * T18 端侧小模型 SPIKE · native 接口声明（liblocalllm.so） + * 实现见 entry/src/main/cpp/napi_init.cpp。 + */ + +/** 加载模型目录（内含 config.json），resolve 空串=成功，否则为错误描述。 */ +export const loadModel: (modelDir: string) => Promise; + +/** + * 生成。resolve JSON 字符串： + * {"text","first_token_ms","total_ms","chars"}；失败 {"error":...}。 + */ +export const generate: (prompt: string, maxNewTokens: number) => Promise; + +export const unloadModel: () => void; +export const isLoaded: () => boolean; diff --git a/app/entry/src/main/ets/components/f1/ProfileContent.ets b/app/entry/src/main/ets/components/f1/ProfileContent.ets index 9fb3743..ac18368 100644 --- a/app/entry/src/main/ets/components/f1/ProfileContent.ets +++ b/app/entry/src/main/ets/components/f1/ProfileContent.ets @@ -432,6 +432,19 @@ export struct ProfileContent { .backgroundColor('#FFFFFF') .alignItems(VerticalAlign.Center) + // T18 SPIKE 调试入口（内部用，正式版移除） + Text('端侧LLM调试') + .fontSize(13) + .fontColor('#9A9A9A') + .width('100%') + .padding(12) + .borderRadius(12) + .backgroundColor('#FFFFFF') + .textAlign(TextAlign.Center) + .onClick(() => { + router.pushUrl({ url: 'pages/LlmTestPage' }) + }) + ErrorBanner({ message: this.bannerMessage, hint: this.bannerHint, diff --git a/app/entry/src/main/ets/core/LocalLlm.ets b/app/entry/src/main/ets/core/LocalLlm.ets new file mode 100644 index 0000000..f1fab10 --- /dev/null +++ b/app/entry/src/main/ets/core/LocalLlm.ets @@ -0,0 +1,130 @@ +/** + * LocalLlm：端侧小模型管理（T18 端侧小模型 SPIKE）。 + * + * 职责：模型文件的下载/校验/加载状态机 + native（liblocalllm.so）调用的 ArkTS 封装。 + * 调试期模型从局域网服务器下载（scripts/serve-models.mjs）； + * 商用阶段把 baseUrl 换成 COS/CDN 域名即可，代码不变。 + * + * 纪律： + * - 模型放 filesDir（cacheDir 有系统配额会被清理，不能放）； + * - SPIKE 只校验文件大小，SHA256 校验留待 P1（crypto 全量哈希 450MB 太慢，需流式）； + * - 推理只在前台进行（无计算类长时任务，退后台进程会被冻结）。 + */ + +import localllm from 'liblocalllm.so' +import { ContextHolder } from './ContextHolder' +import request from '@ohos.request' +import fs from '@ohos.file.fs' + +export interface ManifestFile { + name: string + size: number + sha256: string +} + +export interface ModelManifest { + version: string + model: string + files: ManifestFile[] +} + +export interface GenerateResult { + text: string + first_token_ms: number + total_ms: number + chars: number + error?: string +} + +export type ModelState = 'none' | 'downloading' | 'ready' | 'loaded' | 'error' + +const MODEL_DIR_NAME: string = 'qwen3-0.6b-mnn' + +export class LocalLlm { + /** 调试期默认地址由测试页传入（形如 http://192.168.x.x:8080），这里不硬编码。 */ + + static modelDir(): string { + return `${ContextHolder.get().filesDir}/models/${MODEL_DIR_NAME}` + } + + /** 本地模型文件是否齐全（按 manifest 尺寸校验）。 */ + static checkFiles(manifest: ModelManifest): boolean { + const dir: string = LocalLlm.modelDir() + for (const f of manifest.files) { + const path: string = `${dir}/${f.name}` + if (!fs.accessSync(path)) { + return false + } + try { + const st = fs.statSync(path) + if (Number(st.size) !== f.size) { + return false + } + } catch { + return false + } + } + return true + } + + /** 从 baseUrl 逐个下载模型文件（整包下载，断点续传留待商用阶段）。 */ + static async downloadAll(baseUrl: string, manifest: ModelManifest, + onProgress?: (fileName: string, index: number, total: number) => void): Promise { + const dir: string = LocalLlm.modelDir() + if (!fs.accessSync(dir)) { + fs.mkdirSync(dir, true) + } + const ctx: Context = ContextHolder.get() + for (let i = 0; i < manifest.files.length; i++) { + const f: ManifestFile = manifest.files[i] + onProgress?.(f.name, i + 1, manifest.files.length) + const target: string = `${dir}/${f.name}` + if (fs.accessSync(target) && Number(fs.statSync(target).size) === f.size) { + continue // 已下好，跳过（断点续传的乞丐版：按文件粒度续） + } + const task = await request.downloadFile(ctx, { + url: `${baseUrl}/${MODEL_DIR_NAME}/${f.name}`, + filePath: target + }) + await LocalLlm.waitTask(task) + if (Number(fs.statSync(target).size) !== f.size) { + throw new Error(`文件尺寸不符: ${f.name}`) + } + } + } + + /** 加载模型到 native（耗时，走 async work 线程）。返回空串=成功。 */ + static async load(): Promise { + const dir: string = LocalLlm.modelDir() + const tmp: string = `${dir}/.tmp` + if (!fs.accessSync(tmp)) { + fs.mkdirSync(tmp, true) + } + return localllm.loadModel(dir) + } + + static isLoaded(): boolean { + return localllm.isLoaded() + } + + static unload(): void { + localllm.unloadModel() + } + + /** 生成。native 侧已完成计时；解析失败视为 error。 */ + static async generate(prompt: string, maxNewTokens: number): Promise { + const raw: string = await localllm.generate(prompt, maxNewTokens) + try { + return JSON.parse(raw) as GenerateResult + } catch { + return { text: '', first_token_ms: -1, total_ms: -1, chars: 0, error: `bad json: ${raw}` } + } + } + + private static waitTask(task: request.DownloadTask): Promise { + return new Promise((resolve, reject) => { + task.on('complete', () => resolve()) + task.on('fail', (err: number) => reject(new Error(`download fail: ${err}`))) + }) + } +} diff --git a/app/entry/src/main/ets/pages/LlmTestPage.ets b/app/entry/src/main/ets/pages/LlmTestPage.ets new file mode 100644 index 0000000..1feceac --- /dev/null +++ b/app/entry/src/main/ets/pages/LlmTestPage.ets @@ -0,0 +1,184 @@ +/** + * LlmTestPage：T18 端侧小模型 SPIKE 的调试/基准页。 + * + * 用途（仅限内部调试，不对真实用户开放）： + * 1. 从局域网服务器下载模型 → 加载 → 跑 benchmark（首 token 延迟 / 总耗时 / 字符速度）； + * 2. 意图分类 smoke（固定 prompt + 固定样例，人工核对输出 JSON）； + * 3. hidebug 内存采样（加载前后 native 堆对比，验收口径：峰值 <1GB）。 + * + * 验收门槛见 docs/开发任务包/T18-端侧小模型SPIKE.md。 + */ + +import { LocalLlm, GenerateResult, ModelManifest } from '../core/LocalLlm' +import hidebug from '@ohos.hidebug' +import http from '@ohos.net.http' + +const DEFAULT_BASE: string = 'http://192.168.3.19:8080' // 开发机局域网 IP（模型服务器） +const MODEL_SUBPATH: string = 'qwen3-0.6b-mnn' + +/** 意图分类 smoke 用的系统 prompt（标签集对齐《穿搭Agent-需求文档》§3.2，P2 的"穿搭评价"先占位）。 */ +const INTENT_PROMPT: string = + '你是穿搭助手的意图分类器。把用户输入分类到以下之一：' + + '场景咨询/单品搭配/风格建议/色彩咨询/身材修饰/天气适配/发型建议/穿搭评价/闲聊/越界。' + + '只输出 JSON：{"intent":"<类别>"}，不要输出任何其它内容。用户输入：' + +const INTENT_CASES: string[] = [ + '面试怎么穿？', + '白衬衫配什么裤子？', + '我这种矮胖身材是不是穿什么都难看？顺便告诉我怎么减肥', + '你好呀', +] + +const BENCH_PROMPTS: string[] = [ + '你好，简单介绍一下你自己，一句话以内。', + INTENT_PROMPT + '约会穿什么？', + '夏天通勤穿搭有什么通用原则？100字以内回答。', +] + +@Entry +@Component +struct LlmTestPage { + @State baseUrl: string = DEFAULT_BASE + @State logText: string = '就绪。步骤：① 填开发机IP ② 下载模型 ③ 加载模型 ④ 跑 Benchmark\n' + @State busy: boolean = false + + private log(line: string): void { + this.logText += `${line}\n` + } + + private heapMb(): string { + const bytes: bigint = hidebug.getNativeHeapSize() + return (Number(bytes) / 1024 / 1024).toFixed(0) + } + + private async fetchManifest(): Promise { + const client = http.createHttp() + try { + const resp = await client.request(`${this.baseUrl}/${MODEL_SUBPATH}/manifest.json`, + { connectTimeout: 5000, readTimeout: 10000 }) + if (resp.responseCode !== 200) { + throw new Error(`manifest http ${resp.responseCode}`) + } + return JSON.parse(resp.result as string) as ModelManifest + } finally { + client.destroy() + } + } + + private async onDownload(): Promise { + this.busy = true + try { + this.log(`连接 ${this.baseUrl} ...`) + const manifest = await this.fetchManifest() + this.log(`manifest v${manifest.version}，${manifest.files.length} 个文件`) + if (LocalLlm.checkFiles(manifest)) { + this.log('本地文件已齐全，跳过下载') + return + } + await LocalLlm.downloadAll(this.baseUrl, manifest, (name: string, idx: number, total: number) => { + this.log(`[${idx}/${total}] ${name}`) + }) + if (!LocalLlm.checkFiles(manifest)) { + throw new Error('下载后校验失败') + } + this.log('下载完成，校验通过') + } catch (e) { + this.log(`下载失败: ${(e as Error).message}`) + } finally { + this.busy = false + } + } + + private async onLoad(): Promise { + this.busy = true + try { + this.log(`加载前 native 堆: ${this.heapMb()}MB`) + const t0: number = Date.now() + const err: string = await LocalLlm.load() + if (err.length > 0) { + this.log(`加载失败: ${err}`) + return + } + this.log(`加载完成，耗时 ${Date.now() - t0}ms，加载后 native 堆: ${this.heapMb()}MB`) + } catch (e) { + this.log(`加载异常: ${(e as Error).message}`) + } finally { + this.busy = false + } + } + + private async runOne(prompt: string, maxNewTokens: number): Promise { + const r: GenerateResult = await LocalLlm.generate(prompt, maxNewTokens) + if (r.error !== undefined) { + this.log(` 生成失败: ${r.error}`) + return + } + const speed: string = r.total_ms > 0 ? (r.chars * 1000 / r.total_ms).toFixed(1) : '?' + this.log(` 首token ${r.first_token_ms}ms / 总 ${r.total_ms}ms / ${r.chars}字 (${speed}字/s) / 堆 ${this.heapMb()}MB`) + this.log(` 输出: ${r.text.substring(0, 120)}`) + } + + private async onBench(): Promise { + if (!LocalLlm.isLoaded()) { + this.log('请先加载模型') + return + } + this.busy = true + try { + for (let i = 0; i < BENCH_PROMPTS.length; i++) { + this.log(`[bench ${i + 1}/${BENCH_PROMPTS.length}] ${BENCH_PROMPTS[i].substring(0, 20)}...`) + await this.runOne(BENCH_PROMPTS[i], 128) // 预热 + await this.runOne(BENCH_PROMPTS[i], 128) + await this.runOne(BENCH_PROMPTS[i], 128) + } + this.log('benchmark 完成') + } finally { + this.busy = false + } + } + + private async onIntentSmoke(): Promise { + if (!LocalLlm.isLoaded()) { + this.log('请先加载模型') + return + } + this.busy = true + try { + for (const c of INTENT_CASES) { + this.log(`[意图] 输入: ${c}`) + await this.runOne(INTENT_PROMPT + c, 32) + } + } finally { + this.busy = false + } + } + + build() { + Column({ space: 12 }) { + Text('端侧小模型调试（T18 SPIKE）').fontSize(18).fontWeight(FontWeight.Bold) + TextInput({ text: this.baseUrl, placeholder: '开发机地址，如 http://192.168.1.100:8080' }) + .onChange((v: string) => { this.baseUrl = v }) + Row({ space: 8 }) { + Button('① 下载模型').enabled(!this.busy).onClick(() => { this.onDownload() }) + Button('② 加载模型').enabled(!this.busy).onClick(() => { this.onLoad() }) + } + Row({ space: 8 }) { + Button('③ Benchmark').enabled(!this.busy).onClick(() => { this.onBench() }) + Button('④ 意图smoke').enabled(!this.busy).onClick(() => { this.onIntentSmoke() }) + Button('卸载').enabled(!this.busy).onClick(() => { + LocalLlm.unload() + this.log(`已卸载，native 堆: ${this.heapMb()}MB`) + }) + } + Scroll() { + Text(this.logText).fontSize(12).fontFamily('monospace') + } + .layoutWeight(1) + .align(Alignment.TopStart) + .scrollBar(BarState.Auto) + } + .padding(16) + .width('100%') + .height('100%') + } +} diff --git a/docs/开发任务包/T18-端侧小模型SPIKE.md b/docs/开发任务包/T18-端侧小模型SPIKE.md new file mode 100644 index 0000000..3be19f7 --- /dev/null +++ b/docs/开发任务包/T18-端侧小模型SPIKE.md @@ -0,0 +1,95 @@ +# T18 · 端侧小模型 SPIKE·MNN + Qwen3-0.6B 真机可行性与意图分类原型 + +> 对应需求：端云分工方案（弱网兜底 + 意图识别下沉端侧，见本文 §0 背景）｜ 分支：`task/t18-ondevice-llm-spike` +> 先读《README-FE批次.md》，ArkTS 严格模式铁律全部适用。**本任务是 SPIKE：产出是可运行的调试页 + 真机基准报告 + 结论，不是生产代码。** + +## 0. 背景与分工结论（决策依据，勿复述进代码） + +- 端侧只做「确定性高、判错代价低」的事：意图分类/越界拦截/离线兜底；需要记忆、知识库、规则引擎的生成永远留云端。 +- 端侧结论默认是「建议」，云端权威。本 SPIKE 只验证可行性，**不接生产链路**（ApiClient fallback、hint 上行是 P1~P3 的事）。 +- 框架选型：MNN 3.6.1（官方 harmony 产物，CPU-only，Apache-2.0）+ ModelScope 官方导出 `MNN/Qwen3-0.6B-MNN`（Apache-2.0，权重 450,810,338 B）。鸿蒙上 GPU/NPU 加速均不可用，按 CPU 规划。 +- 内存红线：官方性能体验建议前台峰值 ≤1500MB；本任务验收按峰值 <1GB（加载后 native 堆）。 + +## 1. 目标（三问三答） + +1. **跑不跑得动**：Qwen3-0.6B int4 在真机（麒麟芯片）上的首 token 延迟、生成速度、峰值内存——门槛：意图分类单次（输出 ≤32 token）**总耗时 <3s**、加载后 native 堆 **<1GB**、连续 9 次推理不因降频/内存崩掉； +2. **够不够准**：意图分类 smoke（标签集对齐《穿搭Agent-需求文档》§3.2 九类 + 越界）输出受 JSON schema 约束——门槛：20 条自建样例人工核对 **≥85%**（验证门槛；≥90% 上线门槛留待 P2 用服务端评测集）； +3. **链路通不通**：局域网下载 → 校验 → 加载 → 生成全链路可重复执行——门槛：新机器按本文档 §3 能在 30 分钟内复现。 + +**不达标就换方案（换 1.7B / 只做离线兜底 / 放弃端侧），不要硬做。** + +## 2. 已就绪的契约（协调提交已提供，直接用） + +- **MNN 预编译库**：`app/entry/src/main/libs/arm64-v8a/libMNN.so`（5.85MB，已验证含 `createLLM` 符号；官方 zip 只有 .so 没有头文件）。**不入库**（.gitignore），获取命令见 §3.1； +- **头文件**：`app/entry/src/main/cpp/third_party/mnn/include/`（MNN/* + llm/llm.hpp，来自 3.6.1 tag 稀疏克隆，**版本与 .so 严格绑定，禁止混版本**——C4 图融合新旧不匹配会静默算错）； +- **模型**：`models/qwen3-0.6b-mnn/`（6 个文件 + manifest.json，SHA256 已逐个与 ModelScope 官方清单核对）。**不入库**； +- **native 模块**：`app/entry/src/main/cpp/`（CMakeLists.txt / napi_init.cpp / local_llm.h/.cpp / types/liblocalllm/index.d.ts）。接口：`loadModel(dir)→Promise`（空串=成功）、`generate(prompt,maxNewTokens)→Promise`（JSON：text/first_token_ms/total_ms/chars）、`unloadModel()`、`isLoaded()`。计时在 native 侧（std::chrono）； +- **ArkTS 封装**：`ets/core/LocalLlm.ets`（下载/校验/加载状态机；SPIKE 只校文件尺寸，SHA256 流式校验留 P1）； +- **调试页**：`ets/pages/LlmTestPage.ets`（已注册 main_pages，入口在「我的」页底部「端侧LLM调试」，正式版移除）； +- **局域网服务器**：`scripts/serve-models.mjs`（零依赖，`node scripts/serve-models.mjs 8080`，打印局域网 IP）； +- **构建配置**：`entry/build-profile.json5` 已加 externalNativeOptions（仅 arm64-v8a；模拟器 x86_64 不在本任务范围——MNN 无官方 x86 鸿蒙包，且模拟器性能不代表真机）。 + +**FE 批次规程破例清单**（本任务新增/改动，超出 F 批次禁改范围，特此登记）： +`entry/build-profile.json5`（+externalNativeOptions）、`ets/core/LocalLlm.ets`（新文件）、`main_pages.json`（+1 页）、`components/f1/ProfileContent.ets`（+调试入口）、根 `.gitignore`（+models/ 与 libs/）。 + +## 3. 实现要点 + +### 3.1 环境复现（新机器 30 分钟口径） + +```bash +# ① MNN 库（2MB） +mkdir -p models/mnn && cd models/mnn +curl -LO https://github.com/alibaba/MNN/releases/download/3.6.1/mnn_3.6.1_harmony_arm64_cpu.zip +unzip mnn_3.6.1_harmony_arm64_cpu.zip +cp mnn_3.6.1_harmony_arm64_cpu/arm64-v8a/libMNN.so ../../app/entry/src/main/libs/arm64-v8a/ +# 头文件（与 .so 同 tag）： +git clone --depth 1 --branch 3.6.1 --filter=blob:none --sparse https://github.com/alibaba/MNN.git mnn-src +cd mnn-src && git sparse-checkout set include transformers/llm/engine/include +cp -r include ../../app/entry/src/main/cpp/third_party/mnn/ +cp -r transformers/llm/engine/include/llm ../../app/entry/src/main/cpp/third_party/mnn/include/ + +# ② 模型（456MB） +mkdir -p models/qwen3-0.6b-mnn && cd models/qwen3-0.6b-mnn +for f in config.json configuration.json llm_config.json llm.mnn tokenizer.txt llm.mnn.weight; do + curl -LO "https://modelscope.cn/models/MNN/Qwen3-0.6B-MNN/resolve/master/$f" +done +sha256sum -c <<< "953afb7e0165818add34a7a6caf0af5d0ed9428da102eb22c9b98ee9da292e9f llm.mnn.weight" + +# ③ 局域网服务（手机与电脑同 Wi-Fi） +node scripts/serve-models.mjs 8080 # 打印本机局域网 IP +``` + +### 3.2 真机验证步骤（用户在 DevEco 做，agent 不跑） + +1. `app/.build-gate.bat`（或同等 hvigor 命令）构建 → 签名安装到真机（现有 debug 签名配置可直接用）； +2. App「我的」→「端侧LLM调试」→ 填开发机 IP → ① 下载 → ② 加载（记录加载耗时与 native 堆）→ ③ Benchmark（3 prompt × 3 次，首个为预热）→ ④ 意图 smoke（4 条样例人工核对 JSON 输出）； +3. 全程页面保持前台亮屏（无计算类长时任务，退后台会被冻结——这是产品约束不是 bug）； +4. 局域网 HTTP 明文：Network Kit 自 API 18 起默认支持明文 HTTP（本项目 targetSdk 6.1.1(24)，预期零配置可用；开发后端本身也是 `http://`）。真机若仍拦截，建 `resources/base/profile/network_config.json`（network-security-config/domain-config 白名单，仅调试，不得提交 release）；兜底改用 `hdc file send models/qwen3-0.6b-mnn /data/app/el2/100/base//haps/entry/files/models/`（先装 App 才有目录；注意物理路径与沙箱映射路径的区别）； +5. 若 `loadModel` 报 tokenizer 相关错误：该模型仓库用的是旧格式 `tokenizer.txt`（文档默认新格式 `tokenizer.mtok`），引擎回退行为未证实——在下载目录的 `config.json` 里加一行 `"tokenizer_file": "tokenizer.txt"` 再试（只改设备上的副本，不改 manifest）。 + +### 3.3 报告（`docs/开发任务包/T18-REPORT.md`，核心交付物） + +结构对齐 T9：结论先行（三问三答：通过 / 有条件通过 / 不达标换方案）→ 基准数字表（机型/芯片/内存档位，首 token、总耗时、字/s、加载耗时、加载前后 native 堆、连续 9 次的降频曲线）→ 意图 smoke 明细 → 偏差与坑 → 对 P1（离线兜底）/P2（意图 shadow）的建议。 + +## 4. 验收标准 + +1. `assembleHap` 构建绿（.build-gate.bat）； +2. 真机三问三答有数字、有结论； +3. 复现文档（§3.1+§3.2）可独立执行； +4. 不入库的东西没入库：models/、libs/、任何模型权重； +5. 不改生产链路：ApiClient/ChatPage 的云端对话路径零改动（离线 fallback 是 P1 任务）。 + +## 5. 禁改清单 + +`ets/core/ApiClient.ets`、`AuthStore.ets`、`ApiTypes.ets`、`ContextHolder.ets`、除 LlmTestPage/ProfileContent 调试入口外的全部页面与组件、`module.json5`、后端仓库一切文件。 + +## 6. 提交信息 + +``` +spike(ondevice-llm): MNN+Qwen3-0.6B harmony pipeline, bench page, LAN model serving + +T18 可行性验证：native 骨架（liblocalllm+MNN 3.6.1）+ 局域网模型下载 + +LlmTestPage 基准/意图 smoke + T18-REPORT 三问三答（P1~P3 的决策依据）。 + +偏差说明：<写在这里，无则写"无"> +``` diff --git a/scripts/serve-models.mjs b/scripts/serve-models.mjs new file mode 100644 index 0000000..cefe9d6 --- /dev/null +++ b/scripts/serve-models.mjs @@ -0,0 +1,51 @@ +#!/usr/bin/env node +/** + * T18 调试期局域网模型服务器（零依赖，Node 18+）。 + * + * 用法：node scripts/serve-models.mjs [端口=8080] + * 手机与电脑连同一 Wi-Fi，App 内模型地址填 http://<电脑局域网IP>:<端口> + * + * 服务内容：models/ 目录下的 manifest.json 与模型文件。 + * 调试阶段一次性整包下载即可；断点续传（Range）留待商用阶段接 CDN 时再做。 + */ +import { createServer } from 'node:http'; +import { createReadStream, existsSync, statSync } from 'node:fs'; +import { networkInterfaces } from 'node:os'; +import { extname, join, normalize } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = normalize(join(fileURLToPath(import.meta.url), '..', '..', 'models')); +const port = Number(process.argv[2] ?? 8080); + +const MIME = { + '.json': 'application/json; charset=utf-8', + '.txt': 'text/plain; charset=utf-8', + '.mnn': 'application/octet-stream', + '.weight': 'application/octet-stream', +}; + +const server = createServer((req, res) => { + const urlPath = decodeURIComponent((req.url ?? '/').split('?')[0]); + // 防目录穿越：规范化后必须仍落在 root 内 + const filePath = normalize(join(root, urlPath)); + if (!filePath.startsWith(root) || !existsSync(filePath) || !statSync(filePath).isFile()) { + res.writeHead(404).end('not found'); + return; + } + res.writeHead(200, { + 'Content-Type': MIME[extname(filePath)] ?? 'application/octet-stream', + 'Content-Length': statSync(filePath).size, + }); + createReadStream(filePath).pipe(res); +}); + +server.listen(port, '0.0.0.0', () => { + console.log(`serving ${root}`); + for (const [name, addrs] of Object.entries(networkInterfaces())) { + for (const a of addrs ?? []) { + if (a.family === 'IPv4' && !a.internal) { + console.log(` http://${a.address}:${port}/ (${name})`); + } + } + } +});