spike(llm): T18 端侧小模型——MNN native 骨架 + 局域网模型服务
- cpp/：liblocalllm.so NAPI 骨架（loadModel/chat/benchmark/release），预置 libMNN.so 3.6.1 鸿蒙包（已验证含 createLLM 符号；.gitignore 挡预编译库与 models/，获取方式见任务文档） - LocalLlm.ets：native 调用的 ets 封装；LlmTestPage：模型从局域网服务器下载→加载→首 token/速度 benchmark - scripts/serve-models.mjs：开发期局域网模型文件服务器（Qwen3-0.6B-MNN 456MB，SHA256 与 ModelScope 官方清单逐文件核对） - build-profile.json5 接 externalNativeOptions（arm64-v8a）；ProfileContent 加「端侧LLM调试」入口（内部调试用） - docs/开发任务包/T18-端侧小模型SPIKE.md：目标/验证项/回退线；真机 benchmark 数据回来后写 T18-REPORT 三问三答
This commit is contained in:
@@ -5,6 +5,13 @@
|
||||
"copyCodeResource": {
|
||||
"enable": false
|
||||
}
|
||||
},
|
||||
// T18 端侧小模型 SPIKE：native 模块（liblocalllm.so + 预置 libMNN.so）
|
||||
"externalNativeOptions": {
|
||||
"path": "./src/main/cpp/CMakeLists.txt",
|
||||
"arguments": "",
|
||||
"cppFlags": "",
|
||||
"abiFilters": ["arm64-v8a"]
|
||||
}
|
||||
},
|
||||
"buildOptionSet": [
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# T18 端侧小模型 SPIKE：liblocalllm.so
|
||||
# 结构对齐社区已验证案例（turbo-ai-chat-harmonyos）：预置 libMNN.so + NAPI 薄封装。
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
project(localllm)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
# 预编译 MNN（官方 3.6.1 harmony 包，含 LLM 符号；获取方式见任务文档 T18）。
|
||||
# SPIKE 阶段只支持真机 arm64-v8a；模拟器（x86_64）不在本任务范围。
|
||||
add_library(mnn SHARED IMPORTED)
|
||||
set_target_properties(mnn PROPERTIES
|
||||
IMPORTED_LOCATION ${CMAKE_CURRENT_SOURCE_DIR}/../libs/arm64-v8a/libMNN.so)
|
||||
|
||||
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/mnn/include)
|
||||
|
||||
add_library(localllm SHARED napi_init.cpp local_llm.cpp)
|
||||
|
||||
# ace_napi：N-API；hilog：native 侧日志（ benchmark 数据落 log 便于 hdc 抓取）
|
||||
target_link_libraries(localllm PUBLIC mnn libace_napi.z.so libhilog_ndk.z.so)
|
||||
@@ -0,0 +1,148 @@
|
||||
#include "local_llm.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <sstream>
|
||||
#include <streambuf>
|
||||
|
||||
#include "llm/llm.hpp"
|
||||
#include <hilog/log.h>
|
||||
|
||||
#define LOG_TAG "LocalLlm"
|
||||
#define LOGI(...) OH_LOG_INFO(LOG_APP, __VA_ARGS__)
|
||||
#define LOGE(...) OH_LOG_ERROR(LOG_APP, __VA_ARGS__)
|
||||
|
||||
using MNN::Transformer::ChatMessages;
|
||||
using MNN::Transformer::Llm;
|
||||
|
||||
namespace {
|
||||
|
||||
// 计时流：接住 Llm::response 的流式输出，记录首 chunk 时间并累计全文。
|
||||
class TimingStreamBuf : public std::streambuf {
|
||||
public:
|
||||
std::string text;
|
||||
long firstTokenMs = -1;
|
||||
std::chrono::steady_clock::time_point start;
|
||||
|
||||
void reset(std::chrono::steady_clock::time_point s) {
|
||||
text.clear();
|
||||
firstTokenMs = -1;
|
||||
start = s;
|
||||
}
|
||||
|
||||
protected:
|
||||
int overflow(int c) override {
|
||||
if (c == EOF) {
|
||||
return 0;
|
||||
}
|
||||
markFirst();
|
||||
text += static_cast<char>(c);
|
||||
return c;
|
||||
}
|
||||
|
||||
std::streamsize xsputn(const char *s, std::streamsize n) override {
|
||||
if (n > 0) {
|
||||
markFirst();
|
||||
text.append(s, static_cast<size_t>(n));
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
private:
|
||||
void markFirst() {
|
||||
if (firstTokenMs < 0) {
|
||||
firstTokenMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - start)
|
||||
.count();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
std::string jsonEscape(const std::string &in) {
|
||||
std::string out;
|
||||
out.reserve(in.size() + 16);
|
||||
for (char c : in) {
|
||||
switch (c) {
|
||||
case '"': out += "\\\""; break;
|
||||
case '\\': out += "\\\\"; break;
|
||||
case '\n': out += "\\n"; break;
|
||||
case '\r': out += "\\r"; break;
|
||||
case '\t': out += "\\t"; break;
|
||||
default: out += c; break;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
LocalLlm &LocalLlm::instance() {
|
||||
static LocalLlm inst;
|
||||
return inst;
|
||||
}
|
||||
|
||||
bool LocalLlm::loaded() const {
|
||||
return llm_ != nullptr;
|
||||
}
|
||||
|
||||
std::string LocalLlm::load(const std::string &modelDir) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
unload();
|
||||
const std::string configPath = modelDir + "/config.json";
|
||||
Llm *llm = Llm::createLLM(configPath);
|
||||
if (llm == nullptr) {
|
||||
LOGE("createLLM failed: %{public}s", configPath.c_str());
|
||||
return "createLLM failed: " + configPath;
|
||||
}
|
||||
// use_mmap 手机端必开（权重走 mmap 防 OOM）；tmp_path 必须指向沙箱可写目录，
|
||||
// 由 ArkTS 侧在调用前 mkdir 好（modelDir/.tmp）。
|
||||
const std::string cfg = std::string("{\"tmp_path\":\"") + modelDir + "/.tmp" +
|
||||
"\",\"use_mmap\":true,\"thread_num\":4,"
|
||||
"\"precision\":\"low\",\"memory\":\"low\","
|
||||
"\"reuse_kv\":true,\"max_new_tokens\":256}";
|
||||
llm->set_config(cfg);
|
||||
if (!llm->load()) {
|
||||
LOGE("llm load failed: %{public}s", configPath.c_str());
|
||||
Llm::destroy(llm);
|
||||
return "llm load failed: " + configPath;
|
||||
}
|
||||
llm_ = llm;
|
||||
modelDir_ = modelDir;
|
||||
LOGI("model loaded: %{public}s", configPath.c_str());
|
||||
return "";
|
||||
}
|
||||
|
||||
std::string LocalLlm::generate(const std::string &userPrompt, int maxNewTokens) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (llm_ == nullptr) {
|
||||
return "{\"error\":\"model not loaded\"}";
|
||||
}
|
||||
TimingStreamBuf buf;
|
||||
std::ostream os(&buf);
|
||||
ChatMessages msgs;
|
||||
msgs.emplace_back("user", userPrompt);
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
buf.reset(start);
|
||||
try {
|
||||
llm_->response(msgs, &os, nullptr, maxNewTokens);
|
||||
} catch (const std::exception &e) {
|
||||
LOGE("response exception: %{public}s", e.what());
|
||||
return std::string("{\"error\":\"") + jsonEscape(e.what()) + "\"}";
|
||||
}
|
||||
const long totalMs = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - start)
|
||||
.count();
|
||||
LOGI("generate: first=%{public}ldms total=%{public}ldms chars=%{public}zu",
|
||||
buf.firstTokenMs, totalMs, buf.text.size());
|
||||
std::ostringstream out;
|
||||
out << "{\"text\":\"" << jsonEscape(buf.text) << "\",\"first_token_ms\":" << buf.firstTokenMs
|
||||
<< ",\"total_ms\":" << totalMs << ",\"chars\":" << buf.text.size() << "}";
|
||||
return out.str();
|
||||
}
|
||||
|
||||
void LocalLlm::unload() {
|
||||
if (llm_ != nullptr) {
|
||||
Llm::destroy(llm_);
|
||||
llm_ = nullptr;
|
||||
modelDir_.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
#ifndef LOCAL_LLM_H
|
||||
#define LOCAL_LLM_H
|
||||
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
namespace MNN {
|
||||
namespace Transformer {
|
||||
class Llm;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 端侧小模型最小封装（T18 SPIKE）。
|
||||
*
|
||||
* 设计约束：
|
||||
* - 全局单例 + 互斥锁：LLM 推理不可重入，ArkTS 侧并发调用由此串行化；
|
||||
* - 所有耗时操作（load/generate）只从 N-API async work 线程进入，UI 线程不得直接调；
|
||||
* - Llm 用裸指针：MNN 3.6.1 头文件明确要求用 Llm::destroy() 销毁。
|
||||
*/
|
||||
class LocalLlm {
|
||||
public:
|
||||
static LocalLlm &instance();
|
||||
|
||||
/** 加载模型目录（内含 config.json）。返回空串=成功，否则为错误描述。 */
|
||||
std::string load(const std::string &modelDir);
|
||||
|
||||
/**
|
||||
* 生成。userPrompt 为纯文本，内部套 chat 模板。
|
||||
* 返回 JSON 字符串：{"text","first_token_ms","total_ms","chars"}；失败返回 {"error":...}。
|
||||
* 计时在 native 侧完成（std::chrono），保证 benchmark 数据不受 JS 调度影响。
|
||||
*/
|
||||
std::string generate(const std::string &userPrompt, int maxNewTokens);
|
||||
|
||||
void unload();
|
||||
bool loaded() const;
|
||||
|
||||
private:
|
||||
LocalLlm() = default;
|
||||
std::mutex mutex_;
|
||||
MNN::Transformer::Llm *llm_ = nullptr;
|
||||
std::string modelDir_;
|
||||
};
|
||||
|
||||
#endif // LOCAL_LLM_H
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* T18 端侧小模型 SPIKE · N-API 桥接
|
||||
*
|
||||
* ArkTS 侧用法：import localllm from 'liblocalllm.so'
|
||||
* 耗时接口（loadModel/generate）一律 async work + Promise，不阻塞 UI 线程。
|
||||
*/
|
||||
#include "local_llm.h"
|
||||
|
||||
#include <napi/native_api.h>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
|
||||
struct AsyncTask {
|
||||
napi_async_work work = nullptr;
|
||||
napi_deferred deferred = nullptr;
|
||||
std::string arg; // loadModel: modelDir；generate: prompt
|
||||
int maxNewTokens = 256; // 仅 generate 用
|
||||
std::string result; // 返回值（空串或 JSON）
|
||||
bool isLoad = true; // true=loadModel，false=generate
|
||||
};
|
||||
|
||||
napi_value ResolveString(napi_env env, napi_deferred deferred, const std::string &value) {
|
||||
napi_value js;
|
||||
napi_create_string_utf8(env, value.c_str(), value.size(), &js);
|
||||
napi_resolve_deferred(env, deferred, js);
|
||||
return js;
|
||||
}
|
||||
|
||||
void TaskExecute(napi_env /*env*/, void *data) {
|
||||
auto *task = static_cast<AsyncTask *>(data);
|
||||
if (task->isLoad) {
|
||||
task->result = LocalLlm::instance().load(task->arg); // 空串=成功
|
||||
} else {
|
||||
task->result = LocalLlm::instance().generate(task->arg, task->maxNewTokens);
|
||||
}
|
||||
}
|
||||
|
||||
void TaskComplete(napi_env env, napi_status /*status*/, void *data) {
|
||||
auto *task = static_cast<AsyncTask *>(data);
|
||||
ResolveString(env, task->deferred, task->result);
|
||||
napi_delete_async_work(env, task->work);
|
||||
delete task;
|
||||
}
|
||||
|
||||
napi_value RunAsync(napi_env env, napi_callback_info info, bool isLoad) {
|
||||
size_t argc = 2;
|
||||
napi_value argv[2] = {nullptr, nullptr};
|
||||
napi_get_cb_info(env, info, &argc, argv, nullptr, nullptr);
|
||||
|
||||
auto *task = new AsyncTask();
|
||||
task->isLoad = isLoad;
|
||||
|
||||
char buf[8192];
|
||||
size_t len = 0;
|
||||
napi_get_value_string_utf8(env, argv[0], buf, sizeof(buf), &len);
|
||||
task->arg.assign(buf, len);
|
||||
if (!isLoad && argc >= 2) {
|
||||
napi_get_value_int32(env, argv[1], &task->maxNewTokens);
|
||||
}
|
||||
|
||||
napi_value promise;
|
||||
napi_create_promise(env, &task->deferred, &promise);
|
||||
napi_value name;
|
||||
napi_create_string_utf8(env, isLoad ? "loadModel" : "generate", NAPI_AUTO_LENGTH, &name);
|
||||
napi_create_async_work(env, nullptr, name, TaskExecute, TaskComplete, task, &task->work);
|
||||
napi_queue_async_work(env, task->work);
|
||||
return promise;
|
||||
}
|
||||
|
||||
napi_value LoadModel(napi_env env, napi_callback_info info) {
|
||||
return RunAsync(env, info, true);
|
||||
}
|
||||
|
||||
napi_value Generate(napi_env env, napi_callback_info info) {
|
||||
return RunAsync(env, info, false);
|
||||
}
|
||||
|
||||
napi_value UnloadModel(napi_env env, napi_callback_info /*info*/) {
|
||||
LocalLlm::instance().unload();
|
||||
napi_value ret;
|
||||
napi_get_undefined(env, &ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
napi_value IsLoaded(napi_env env, napi_callback_info /*info*/) {
|
||||
napi_value ret;
|
||||
napi_get_boolean(env, LocalLlm::instance().loaded(), &ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
napi_value Init(napi_env env, napi_value exports) {
|
||||
napi_property_descriptor desc[] = {
|
||||
{"loadModel", nullptr, LoadModel, nullptr, nullptr, nullptr, napi_default, nullptr},
|
||||
{"generate", nullptr, Generate, nullptr, nullptr, nullptr, napi_default, nullptr},
|
||||
{"unloadModel", nullptr, UnloadModel, nullptr, nullptr, nullptr, napi_default, nullptr},
|
||||
{"isLoaded", nullptr, IsLoaded, nullptr, nullptr, nullptr, napi_default, nullptr},
|
||||
};
|
||||
napi_define_properties(env, exports, sizeof(desc) / sizeof(desc[0]), desc);
|
||||
return exports;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
static napi_module localllmModule = {
|
||||
.nm_version = 1,
|
||||
.nm_flags = 0,
|
||||
.nm_filename = nullptr,
|
||||
.nm_register_func = Init,
|
||||
.nm_modname = "localllm",
|
||||
.nm_priv = nullptr,
|
||||
.reserved = {0},
|
||||
};
|
||||
|
||||
extern "C" __attribute__((constructor)) void RegisterLocalllmModule() {
|
||||
napi_module_register(&localllmModule);
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//
|
||||
// AutoTime.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2018/07/27.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_AutoTime_hpp
|
||||
#define MNN_AutoTime_hpp
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <MNN/MNNDefine.h>
|
||||
|
||||
namespace MNN {
|
||||
|
||||
class MNN_PUBLIC Timer {
|
||||
public:
|
||||
Timer();
|
||||
~Timer();
|
||||
Timer(const Timer&) = delete;
|
||||
Timer(const Timer&&) = delete;
|
||||
Timer& operator=(const Timer&) = delete;
|
||||
Timer& operator=(const Timer&&) = delete;
|
||||
|
||||
// reset timer
|
||||
void reset();
|
||||
// get duration (us) from init or latest reset.
|
||||
uint64_t durationInUs();
|
||||
|
||||
// Get Current Time
|
||||
uint64_t current() const {
|
||||
return mLastResetTime;
|
||||
}
|
||||
protected:
|
||||
uint64_t mLastResetTime;
|
||||
};
|
||||
|
||||
/** time tracing util. prints duration between init and deinit. */
|
||||
class MNN_PUBLIC AutoTime : Timer {
|
||||
public:
|
||||
AutoTime(int line, const char* func);
|
||||
~AutoTime();
|
||||
AutoTime(const AutoTime&) = delete;
|
||||
AutoTime(const AutoTime&&) = delete;
|
||||
AutoTime& operator=(const AutoTime&) = delete;
|
||||
AutoTime& operator=(const AutoTime&&) = delete;
|
||||
|
||||
private:
|
||||
int mLine;
|
||||
char* mName;
|
||||
};
|
||||
} // namespace MNN
|
||||
|
||||
#ifdef MNN_OPEN_TIME_TRACE
|
||||
#define AUTOTIME MNN::AutoTime ___t(__LINE__, __func__)
|
||||
#else
|
||||
#define AUTOTIME
|
||||
#endif
|
||||
|
||||
#endif /* AutoTime_hpp */
|
||||
@@ -0,0 +1,44 @@
|
||||
//
|
||||
// ErrorCode.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2018/09/18.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_ErrorCode_h
|
||||
#define MNN_ErrorCode_h
|
||||
|
||||
namespace MNN {
|
||||
enum ErrorCode {
|
||||
#ifdef NO_ERROR
|
||||
#undef NO_ERROR
|
||||
#endif // NO_ERROR
|
||||
NO_ERROR = 0,
|
||||
OUT_OF_MEMORY = 1,
|
||||
NOT_SUPPORT = 2,
|
||||
COMPUTE_SIZE_ERROR = 3,
|
||||
NO_EXECUTION = 4,
|
||||
INVALID_VALUE = 5,
|
||||
|
||||
// User error
|
||||
INPUT_DATA_ERROR = 10,
|
||||
CALL_BACK_STOP = 11,
|
||||
|
||||
// Op Resize Error
|
||||
TENSOR_NOT_SUPPORT = 20,
|
||||
TENSOR_NEED_DIVIDE = 21,
|
||||
|
||||
// File error
|
||||
FILE_CREATE_FAILED = 30,
|
||||
FILE_REMOVE_FAILED = 31,
|
||||
FILE_OPEN_FAILED = 32,
|
||||
FILE_CLOSE_FAILED = 33,
|
||||
FILE_RESIZE_FAILED = 34,
|
||||
FILE_SEEK_FAILED = 35,
|
||||
FILE_NOT_EXIST = 36,
|
||||
FILE_UNMAP_FAILED = 37
|
||||
};
|
||||
} // namespace MNN
|
||||
|
||||
#endif /* ErrorCode_h */
|
||||
@@ -0,0 +1,308 @@
|
||||
#ifndef MNN_HALIDE_HALIDERUNTIME_H
|
||||
#define MNN_HALIDE_HALIDERUNTIME_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// Note that you should not use "inline" along with HALIDE_ALWAYS_INLINE;
|
||||
// it is not necessary, and may produce warnings for some build configurations.
|
||||
#ifdef _MSC_VER
|
||||
#define HALIDE_ALWAYS_INLINE __forceinline
|
||||
#define HALIDE_NEVER_INLINE __declspec(noinline)
|
||||
#else
|
||||
#define HALIDE_ALWAYS_INLINE __attribute__((always_inline)) inline
|
||||
#define HALIDE_NEVER_INLINE __attribute__((noinline))
|
||||
#endif
|
||||
|
||||
/** \file
|
||||
*
|
||||
* This file declares the routines used by Halide internally in its
|
||||
* runtime. On platforms that support weak linking, these can be
|
||||
* replaced with user-defined versions by defining an extern "C"
|
||||
* function with the same name and signature.
|
||||
*
|
||||
* When doing Just In Time (JIT) compilation methods on the Func being
|
||||
* compiled must be called instead. The corresponding methods are
|
||||
* documented below.
|
||||
*
|
||||
* All of these functions take a "void *user_context" parameter as their
|
||||
* first argument; if the Halide kernel that calls back to any of these
|
||||
* functions has been compiled with the UserContext feature set on its Target,
|
||||
* then the value of that pointer passed from the code that calls the
|
||||
* Halide kernel is piped through to the function.
|
||||
*
|
||||
* Some of these are also useful to call when using the default
|
||||
* implementation. E.g. halide_shutdown_thread_pool.
|
||||
*
|
||||
* Note that even on platforms with weak linking, some linker setups
|
||||
* may not respect the override you provide. E.g. if the override is
|
||||
* in a shared library and the halide object files are linked directly
|
||||
* into the output, the builtin versions of the runtime functions will
|
||||
* be called. See your linker documentation for more details. On
|
||||
* Linux, LD_DYNAMIC_WEAK=1 may help.
|
||||
*
|
||||
*/
|
||||
|
||||
// Forward-declare to suppress warnings if compiling as C.
|
||||
struct halide_buffer_t;
|
||||
|
||||
/** Types in the halide type system. They can be ints, unsigned ints,
|
||||
* or floats (of various bit-widths), or a handle (which is always 64-bits).
|
||||
* Note that the int/uint/float values do not imply a specific bit width
|
||||
* (the bit width is expected to be encoded in a separate value).
|
||||
*/
|
||||
typedef enum halide_type_code_t
|
||||
{
|
||||
halide_type_int = 0, //!< signed integers
|
||||
halide_type_uint = 1, //!< unsigned integers
|
||||
halide_type_float = 2, //!< IEEE floating point numbers
|
||||
halide_type_handle = 3, //!< opaque pointer type (void *)
|
||||
halide_type_bfloat = 4 //!< floating point numbers in the bfloat format
|
||||
} halide_type_code_t;
|
||||
|
||||
// Note that while __attribute__ can go before or after the declaration,
|
||||
// __declspec apparently is only allowed before.
|
||||
#ifndef HALIDE_ATTRIBUTE_ALIGN
|
||||
#ifdef _MSC_VER
|
||||
#define HALIDE_ATTRIBUTE_ALIGN(x) __declspec(align(x))
|
||||
#else
|
||||
#define HALIDE_ATTRIBUTE_ALIGN(x) __attribute__((aligned(x)))
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/** A runtime tag for a type in the halide type system. Can be ints,
|
||||
* unsigned ints, or floats of various bit-widths (the 'bits'
|
||||
* field). Can also be vectors of the same (by setting the 'lanes'
|
||||
* field to something larger than one). This struct should be
|
||||
* exactly 32-bits in size. */
|
||||
struct halide_type_t {
|
||||
/** The basic type code: signed integer, unsigned integer, or floating point. */
|
||||
#ifndef _MSC_VER
|
||||
HALIDE_ATTRIBUTE_ALIGN(1) halide_type_code_t code; // halide_type_code_t
|
||||
#else
|
||||
HALIDE_ATTRIBUTE_ALIGN(1) uint8_t code; // halide_type_code_t
|
||||
#endif
|
||||
|
||||
/** The number of bits of precision of a single scalar value of this type. */
|
||||
HALIDE_ATTRIBUTE_ALIGN(1) uint8_t bits;
|
||||
|
||||
/** How many elements in a vector. This is 1 for scalar types. */
|
||||
HALIDE_ATTRIBUTE_ALIGN(2) uint16_t lanes;
|
||||
|
||||
#ifdef __cplusplus
|
||||
/** Construct a runtime representation of a Halide type from:
|
||||
* code: The fundamental type from an enum.
|
||||
* bits: The bit size of one element.
|
||||
* lanes: The number of vector elements in the type. */
|
||||
HALIDE_ALWAYS_INLINE halide_type_t(halide_type_code_t code, uint8_t bits, uint16_t lanes = 1)
|
||||
: code(code), bits(bits), lanes(lanes) {
|
||||
}
|
||||
|
||||
/** Default constructor is required e.g. to declare halide_trace_event
|
||||
* instances. */
|
||||
HALIDE_ALWAYS_INLINE halide_type_t() : code((halide_type_code_t)0), bits(0), lanes(0) {}
|
||||
|
||||
/** Compare two types for equality. */
|
||||
HALIDE_ALWAYS_INLINE bool operator==(const halide_type_t &other) const {
|
||||
return (code == other.code &&
|
||||
bits == other.bits &&
|
||||
lanes == other.lanes);
|
||||
}
|
||||
|
||||
HALIDE_ALWAYS_INLINE bool operator!=(const halide_type_t &other) const {
|
||||
return !(*this == other);
|
||||
}
|
||||
|
||||
/** Size in bytes for a single element, even if width is not 1, of this type. */
|
||||
HALIDE_ALWAYS_INLINE int bytes() const { return (bits + 7) / 8; }
|
||||
#endif
|
||||
};
|
||||
|
||||
/** An opaque struct containing per-GPU API implementations of the
|
||||
* device functions. */
|
||||
struct halide_device_interface_impl_t;
|
||||
|
||||
/** Each GPU API provides a halide_device_interface_t struct pointing
|
||||
* to the code that manages device allocations. You can access these
|
||||
* functions directly from the struct member function pointers, or by
|
||||
* calling the functions declared below. Note that the global
|
||||
* functions are not available when using Halide as a JIT compiler.
|
||||
* If you are using raw halide_buffer_t in that context you must use
|
||||
* the function pointers in the device_interface struct.
|
||||
*
|
||||
* The function pointers below are currently the same for every GPU
|
||||
* API; only the impl field varies. These top-level functions do the
|
||||
* bookkeeping that is common across all GPU APIs, and then dispatch
|
||||
* to more API-specific functions via another set of function pointers
|
||||
* hidden inside the impl field.
|
||||
*/
|
||||
struct halide_device_interface_t {
|
||||
int (*device_malloc)(void *user_context, struct halide_buffer_t *buf,
|
||||
const struct halide_device_interface_t *device_interface);
|
||||
int (*device_free)(void *user_context, struct halide_buffer_t *buf);
|
||||
int (*device_sync)(void *user_context, struct halide_buffer_t *buf);
|
||||
void (*device_release)(void *user_context,
|
||||
const struct halide_device_interface_t *device_interface);
|
||||
int (*copy_to_host)(void *user_context, struct halide_buffer_t *buf);
|
||||
int (*copy_to_device)(void *user_context, struct halide_buffer_t *buf,
|
||||
const struct halide_device_interface_t *device_interface);
|
||||
int (*device_and_host_malloc)(void *user_context, struct halide_buffer_t *buf,
|
||||
const struct halide_device_interface_t *device_interface);
|
||||
int (*device_and_host_free)(void *user_context, struct halide_buffer_t *buf);
|
||||
int (*buffer_copy)(void *user_context, struct halide_buffer_t *src,
|
||||
const struct halide_device_interface_t *dst_device_interface, struct halide_buffer_t *dst);
|
||||
int (*device_crop)(void *user_context, const struct halide_buffer_t *src,
|
||||
struct halide_buffer_t *dst);
|
||||
int (*device_release_crop)(void *user_context, struct halide_buffer_t *buf);
|
||||
int (*wrap_native)(void *user_context, struct halide_buffer_t *buf, uint64_t handle,
|
||||
const struct halide_device_interface_t *device_interface);
|
||||
int (*detach_native)(void *user_context, struct halide_buffer_t *buf);
|
||||
const struct halide_device_interface_impl_t *impl;
|
||||
};
|
||||
|
||||
typedef struct halide_dimension_t {
|
||||
int32_t min, extent, stride;
|
||||
|
||||
// Per-dimension flags. None are defined yet (This is reserved for future use).
|
||||
uint32_t flags;
|
||||
|
||||
#ifdef __cplusplus
|
||||
HALIDE_ALWAYS_INLINE halide_dimension_t() : min(0), extent(0), stride(0), flags(0) {}
|
||||
HALIDE_ALWAYS_INLINE halide_dimension_t(int32_t m, int32_t e, int32_t s, uint32_t f = 0) :
|
||||
min(m), extent(e), stride(s), flags(f) {}
|
||||
|
||||
HALIDE_ALWAYS_INLINE bool operator==(const halide_dimension_t &other) const {
|
||||
return (min == other.min) &&
|
||||
(extent == other.extent) &&
|
||||
(stride == other.stride) &&
|
||||
(flags == other.flags);
|
||||
}
|
||||
|
||||
HALIDE_ALWAYS_INLINE bool operator!=(const halide_dimension_t &other) const {
|
||||
return !(*this == other);
|
||||
}
|
||||
#endif
|
||||
} halide_dimension_t;
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
typedef enum {halide_buffer_flag_host_dirty = 1,
|
||||
halide_buffer_flag_device_dirty = 2} halide_buffer_flags;
|
||||
|
||||
/**
|
||||
* The raw representation of an image passed around by generated
|
||||
* Halide code. It includes some stuff to track whether the image is
|
||||
* not actually in main memory, but instead on a device (like a
|
||||
* GPU). For a more convenient C++ wrapper, use Halide::Buffer<T>. */
|
||||
typedef struct halide_buffer_t {
|
||||
/** A device-handle for e.g. GPU memory used to back this buffer. */
|
||||
uint64_t device;
|
||||
|
||||
/** The interface used to interpret the above handle. */
|
||||
const struct halide_device_interface_t *device_interface;
|
||||
|
||||
/** A pointer to the start of the data in main memory. In terms of
|
||||
* the Halide coordinate system, this is the address of the min
|
||||
* coordinates (defined below). */
|
||||
uint8_t* host;
|
||||
|
||||
/** flags with various meanings. */
|
||||
uint64_t flags;
|
||||
|
||||
/** The type of each buffer element. */
|
||||
struct halide_type_t type;
|
||||
|
||||
/** The dimensionality of the buffer. */
|
||||
int32_t dimensions;
|
||||
|
||||
/** The shape of the buffer. Halide does not own this array - you
|
||||
* must manage the memory for it yourself. */
|
||||
halide_dimension_t *dim;
|
||||
|
||||
/** Pads the buffer up to a multiple of 8 bytes */
|
||||
void *padding;
|
||||
} halide_buffer_t;
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
namespace {
|
||||
template<typename T> struct check_is_pointer;
|
||||
template<typename T> struct check_is_pointer<T *> {};
|
||||
}
|
||||
|
||||
/** Construct the halide equivalent of a C type */
|
||||
template<typename T>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of() {
|
||||
// Create a compile-time error if T is not a pointer (without
|
||||
// using any includes - this code goes into the runtime).
|
||||
check_is_pointer<T> check;
|
||||
(void)check;
|
||||
return halide_type_t(halide_type_handle, 64);
|
||||
}
|
||||
|
||||
template<>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of<float>() {
|
||||
return halide_type_t(halide_type_float, 32);
|
||||
}
|
||||
|
||||
template<>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of<double>() {
|
||||
return halide_type_t(halide_type_float, 64);
|
||||
}
|
||||
|
||||
template<>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of<bool>() {
|
||||
return halide_type_t(halide_type_uint, 1);
|
||||
}
|
||||
|
||||
template<>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of<uint8_t>() {
|
||||
return halide_type_t(halide_type_uint, 8);
|
||||
}
|
||||
|
||||
template<>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of<uint16_t>() {
|
||||
return halide_type_t(halide_type_uint, 16);
|
||||
}
|
||||
|
||||
template<>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of<uint32_t>() {
|
||||
return halide_type_t(halide_type_uint, 32);
|
||||
}
|
||||
|
||||
template<>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of<uint64_t>() {
|
||||
return halide_type_t(halide_type_uint, 64);
|
||||
}
|
||||
|
||||
template<>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of<int8_t>() {
|
||||
return halide_type_t(halide_type_int, 8);
|
||||
}
|
||||
|
||||
template<>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of<int16_t>() {
|
||||
return halide_type_t(halide_type_int, 16);
|
||||
}
|
||||
|
||||
template<>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of<int32_t>() {
|
||||
return halide_type_t(halide_type_int, 32);
|
||||
}
|
||||
|
||||
template<>
|
||||
HALIDE_ALWAYS_INLINE halide_type_t halide_type_of<int64_t>() {
|
||||
return halide_type_t(halide_type_int, 64);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif // HALIDE_HALIDERUNTIME_H
|
||||
@@ -0,0 +1,182 @@
|
||||
//
|
||||
// ImageProcess.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2018/09/19.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_ImageProcess_hpp
|
||||
#define MNN_ImageProcess_hpp
|
||||
|
||||
#include <MNN/ErrorCode.hpp>
|
||||
#include <MNN/Matrix.h>
|
||||
#include <MNN/Tensor.hpp>
|
||||
|
||||
namespace MNN {
|
||||
namespace CV {
|
||||
enum ImageFormat {
|
||||
RGBA = 0,
|
||||
RGB = 1,
|
||||
BGR = 2,
|
||||
GRAY = 3,
|
||||
BGRA = 4,
|
||||
YCrCb = 5,
|
||||
YUV = 6,
|
||||
HSV = 7,
|
||||
XYZ = 8,
|
||||
BGR555 = 9,
|
||||
BGR565 = 10,
|
||||
YUV_NV21 = 11,
|
||||
YUV_NV12 = 12,
|
||||
YUV_I420 = 13,
|
||||
HSV_FULL = 14,
|
||||
};
|
||||
|
||||
enum Filter { NEAREST = 0, BILINEAR = 1, BICUBIC = 2 };
|
||||
|
||||
enum Wrap { CLAMP_TO_EDGE = 0, ZERO = 1, REPEAT = 2 };
|
||||
|
||||
/**
|
||||
* handle image process for tensor.
|
||||
* step:
|
||||
* 1: Do transform compute and get points
|
||||
* 2: Sample line and do format convert
|
||||
* 3: Turn RGBA to float tensor, and do sub and normalize
|
||||
*/
|
||||
|
||||
class MNN_PUBLIC ImageProcess {
|
||||
public:
|
||||
struct Inside;
|
||||
struct Config {
|
||||
/** data filter */
|
||||
Filter filterType = NEAREST;
|
||||
/** format of source data */
|
||||
ImageFormat sourceFormat = RGBA;
|
||||
/** format of destination data */
|
||||
ImageFormat destFormat = RGBA;
|
||||
|
||||
// Only valid if the dest type is float
|
||||
float mean[4] = {0.0f, 0.0f, 0.0f, 0.0f};
|
||||
float normal[4] = {1.0f, 1.0f, 1.0f, 1.0f};
|
||||
|
||||
/** edge wrapper */
|
||||
Wrap wrap = CLAMP_TO_EDGE;
|
||||
};
|
||||
public:
|
||||
/**
|
||||
* @brief create image process with given config for given tensor.
|
||||
* @param config given config.
|
||||
* @param dstTensor given tensor.
|
||||
* @return image processor.
|
||||
*/
|
||||
static ImageProcess* create(const Config& config, const Tensor* dstTensor = nullptr);
|
||||
|
||||
/**
|
||||
* @brief create image process with given config for given tensor.
|
||||
* @param means given means
|
||||
* @param meanCount given means count
|
||||
* @param normals given normals
|
||||
* @param normalCount given normal count
|
||||
* @param sourceFormat format of source data
|
||||
* @param destFormat format of destination data
|
||||
* @param dstTensor given tensor.
|
||||
* @return image processor.
|
||||
*/
|
||||
static ImageProcess* create(const ImageFormat sourceFormat = RGBA, const ImageFormat destFormat = RGBA,
|
||||
const float* means = nullptr, const int meanCount = 0, const float* normals = nullptr,
|
||||
const int normalCount = 0, const Tensor* dstTensor = nullptr);
|
||||
|
||||
~ImageProcess();
|
||||
static void destroy(ImageProcess* imageProcess);
|
||||
|
||||
/**
|
||||
* @brief get affine transform matrix.
|
||||
* @return affine transform matrix.
|
||||
*/
|
||||
inline const Matrix& matrix() const {
|
||||
return mTransform;
|
||||
}
|
||||
void setMatrix(const Matrix& matrix);
|
||||
|
||||
/**
|
||||
* @brief convert source data to given tensor.
|
||||
* @param source source data.
|
||||
* @param iw source width.
|
||||
* @param ih source height.
|
||||
* @param stride number of elements per row. eg: 100 width RGB contains at least 300 elements.
|
||||
* @param dest given tensor.
|
||||
* @return result code.
|
||||
*/
|
||||
ErrorCode convert(const uint8_t* source, int iw, int ih, int stride, Tensor* dest);
|
||||
|
||||
/**
|
||||
* @brief convert source data to given tensor.
|
||||
* @param source source data.
|
||||
* @param iw source width.
|
||||
* @param ih source height.
|
||||
* @param stride number of elements per row. eg: 100 width RGB contains at least 300 elements.
|
||||
* @param dest dest data.
|
||||
* @param ow output width.
|
||||
* @param oh output height.
|
||||
* @param outputBpp output bpp, if 0, set as the save and config.destFormat.
|
||||
* @param outputStride output stride, if 0, set as ow * outputBpp.
|
||||
* @param type Only support halide_type_of<uint8_t> and halide_type_of<float>.
|
||||
* @return result code.
|
||||
*/
|
||||
ErrorCode convert(const uint8_t* source, int iw, int ih, int stride, void* dest, int ow, int oh, int outputBpp = 0,
|
||||
int outputStride = 0, halide_type_t type = halide_type_of<float>());
|
||||
|
||||
/**
|
||||
* @brief create tensor with given data.
|
||||
* @param w image width.
|
||||
* @param h image height.
|
||||
* @param bpp bytes per pixel.
|
||||
* @param p pixel data pointer.
|
||||
* @return created tensor.
|
||||
*/
|
||||
template <typename T>
|
||||
static Tensor* createImageTensor(int w, int h, int bpp, void* p = nullptr) {
|
||||
return createImageTensor(halide_type_of<T>(), w, h, bpp, p);
|
||||
}
|
||||
static Tensor* createImageTensor(halide_type_t type, int w, int h, int bpp, void* p = nullptr);
|
||||
|
||||
/**
|
||||
* @brief set padding value when wrap=ZERO.
|
||||
* @param value padding value.
|
||||
* @return void.
|
||||
*/
|
||||
void setPadding(uint8_t value) {
|
||||
mPaddingValue = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief set to draw mode.
|
||||
* @param void
|
||||
* @return void.
|
||||
*/
|
||||
void setDraw();
|
||||
|
||||
/**
|
||||
* @brief draw color to regions of img.
|
||||
* @param img the image to draw.
|
||||
* @param w the image's width.
|
||||
* @param h the image's height.
|
||||
* @param c the image's channel.
|
||||
* @param regions the regions to draw, size is [num * 3] contain num x { y, xl, xr }
|
||||
* @param num regions num
|
||||
* @param color the color to draw.
|
||||
* @return void.
|
||||
*/
|
||||
void draw(uint8_t* img, int w, int h, int c, const int* regions, int num, const uint8_t* color);
|
||||
private:
|
||||
ImageProcess(const Config& config);
|
||||
Matrix mTransform;
|
||||
Matrix mTransformInvert;
|
||||
Inside* mInside;
|
||||
uint8_t mPaddingValue = 0;
|
||||
};
|
||||
} // namespace CV
|
||||
} // namespace MNN
|
||||
|
||||
#endif /* MNN_ImageProcess_hpp */
|
||||
@@ -0,0 +1,545 @@
|
||||
//
|
||||
// Interpreter.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2018/07/23.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_Interpreter_hpp
|
||||
#define MNN_Interpreter_hpp
|
||||
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <MNN/ErrorCode.hpp>
|
||||
#include <MNN/MNNForwardType.h>
|
||||
#include <MNN/Tensor.hpp>
|
||||
|
||||
namespace MNN {
|
||||
|
||||
/** session schedule config */
|
||||
struct ScheduleConfig {
|
||||
/** which tensor should be kept */
|
||||
std::vector<std::string> saveTensors;
|
||||
/** forward type */
|
||||
MNNForwardType type = MNN_FORWARD_CPU;
|
||||
/** CPU:number of threads in parallel , Or GPU: mode setting*/
|
||||
union {
|
||||
int numThread = 4;
|
||||
int mode;
|
||||
};
|
||||
|
||||
/** subpath to run */
|
||||
struct Path {
|
||||
std::vector<std::string> inputs;
|
||||
std::vector<std::string> outputs;
|
||||
|
||||
enum Mode {
|
||||
/**
|
||||
* Op Mode
|
||||
* - inputs means the source op, can NOT be empty.
|
||||
* - outputs means the sink op, can be empty.
|
||||
* The path will start from source op, then flow when encounter the sink op.
|
||||
* The sink op will not be compute in this path.
|
||||
*/
|
||||
Op = 0,
|
||||
|
||||
/**
|
||||
* Tensor Mode
|
||||
* - inputs means the inputs tensors, can NOT be empty.
|
||||
* - outputs means the outputs tensors, can NOT be empty.
|
||||
* It will find the pipeline that compute outputs from inputs.
|
||||
*/
|
||||
Tensor = 1
|
||||
};
|
||||
|
||||
/** running mode */
|
||||
Mode mode = Op;
|
||||
};
|
||||
Path path;
|
||||
|
||||
/** backup backend used to create execution when desinated backend do NOT support any op */
|
||||
MNNForwardType backupType = MNN_FORWARD_CPU;
|
||||
|
||||
/** extra backend config */
|
||||
BackendConfig* backendConfig = nullptr;
|
||||
};
|
||||
|
||||
class Session;
|
||||
struct Content;
|
||||
class Tensor;
|
||||
class Backend;
|
||||
class Runtime;
|
||||
|
||||
class MNN_PUBLIC OperatorInfo {
|
||||
struct Info;
|
||||
|
||||
public:
|
||||
/** Operator's name*/
|
||||
const std::string& name() const;
|
||||
|
||||
/** Operator's type*/
|
||||
const std::string& type() const;
|
||||
|
||||
/** Operator's flops, in M*/
|
||||
float flops() const;
|
||||
|
||||
protected:
|
||||
OperatorInfo();
|
||||
~OperatorInfo();
|
||||
Info* mContent;
|
||||
};
|
||||
|
||||
typedef std::function<bool(const std::vector<Tensor*>&, const std::string& /*opName*/)> TensorCallBack;
|
||||
typedef std::function<bool(const std::vector<Tensor*>&, const OperatorInfo*)> TensorCallBackWithInfo;
|
||||
typedef std::pair< std::map<MNNForwardType, std::shared_ptr<Runtime>>, std::shared_ptr<Runtime>> RuntimeInfo;
|
||||
|
||||
/**
|
||||
* @brief get mnn version info.
|
||||
* @return mnn version string.
|
||||
*/
|
||||
MNN_PUBLIC const char* getVersion();
|
||||
|
||||
/** net data holder. multiple sessions could share same net. */
|
||||
class MNN_PUBLIC Interpreter {
|
||||
public:
|
||||
/**
|
||||
* @brief create net from file.
|
||||
* @param file given file.
|
||||
* @return created net if success, NULL otherwise.
|
||||
*/
|
||||
static Interpreter* createFromFile(const char* file);
|
||||
/**
|
||||
* @brief create net from buffer.
|
||||
* @param buffer given data buffer.
|
||||
* @param size size of data buffer.
|
||||
* @return created net if success, NULL otherwise.
|
||||
*/
|
||||
static Interpreter* createFromBuffer(const void* buffer, size_t size);
|
||||
~Interpreter();
|
||||
|
||||
/**
|
||||
* @brief destroy Interpreter
|
||||
* @param model given Interpreter to release.
|
||||
*/
|
||||
static void destroy(Interpreter* net);
|
||||
|
||||
enum SessionMode {
|
||||
/** About CallBack, Default Session_Debug*/
|
||||
/** runSessionWithCallBack is allowed and can get internal op info*/
|
||||
Session_Debug = 0,
|
||||
/** runSessionWithCallBack is not valid and can't get any info of op in session*/
|
||||
Session_Release = 1,
|
||||
|
||||
/** About input tenosr, Default Session_Input_Inside*/
|
||||
/** The input tensor is alloced by session, input data after session resized*/
|
||||
Session_Input_Inside = 2,
|
||||
/** The input tensor is alloced by user, set input data before session resize*/
|
||||
Session_Input_User = 3,
|
||||
|
||||
/** The output tensor depends on session, and can't be separate used*/
|
||||
Session_Output_Inside = 4,
|
||||
/** The output tensor can be separated from session*/
|
||||
Session_Output_User = 5,
|
||||
|
||||
/** Try Resize Session when create Session or not, default direct: */
|
||||
Session_Resize_Direct = 6,
|
||||
Session_Resize_Defer = 7,
|
||||
|
||||
/** Determine the Execution's forward type is determine by user or auto determine */
|
||||
Session_Backend_Fix = 8, // Use the backend user set, when not support use default backend
|
||||
Session_Backend_Auto = 9, // Auto Determine the Op type by MNN
|
||||
|
||||
/** Determine static memory whether recyle in resizeSession or just cache the memory */
|
||||
Session_Memory_Collect = 10, // Recycle static memory when session resize in case memory explosion
|
||||
Session_Memory_Cache = 11, // Cache the static memory for next forward usage
|
||||
|
||||
/** Determine whether use codegen function */
|
||||
Session_Codegen_Disable = 12, // Disable codegen in case extra build codegen cost
|
||||
Session_Codegen_Enable = 13, // Enable codegen
|
||||
|
||||
/** Dynamic Reisze Optimization */
|
||||
Session_Resize_Check = 14, // Open Trace for resize
|
||||
Session_Resize_Fix = 15, // Apply Resize Optimization
|
||||
|
||||
/** Set for Module's traceOrOptimize API.
|
||||
Module_Forward_Seperate:
|
||||
when inputs is not empty , Module's onForward will only infer shape and alloc memory.
|
||||
when inputs is empty , Module's onForward will only runSession to compute content.
|
||||
Default is Module_Forward_Combine
|
||||
*/
|
||||
Module_Forward_Separate = 16,
|
||||
Module_Forward_Combine = 17,
|
||||
};
|
||||
/**
|
||||
* @brief The API shoud be called before create session.
|
||||
* @param mode session mode
|
||||
*/
|
||||
void setSessionMode(SessionMode mode);
|
||||
|
||||
/**
|
||||
* @brief The API shoud be called before create session.
|
||||
* If the cache exist, try to load cache from file.
|
||||
* After createSession, try to save cache to file.
|
||||
* @param cacheFile cache file name
|
||||
* @param keySize depercerate, for future use.
|
||||
*/
|
||||
void setCacheFile(const char* cacheFile, size_t keySize = 128);
|
||||
|
||||
/**
|
||||
* @brief The API shoud be called before create session.
|
||||
* @param file external data file name
|
||||
* @param keySize depercerate, for future use.
|
||||
*/
|
||||
void setExternalFile(const char* file, size_t flag = 128);
|
||||
/**
|
||||
* @brief The API shoud be called after last resize session.
|
||||
* If resize session generate new cache info, try to rewrite cache file.
|
||||
* If resize session do not generate any new cache info, just do nothing.
|
||||
* @param session given session
|
||||
* @param flag Protected param, not used now
|
||||
*/
|
||||
|
||||
ErrorCode updateCacheFile(Session *session, int flag = 0);
|
||||
|
||||
enum HintMode {
|
||||
// Max Op number for async tuning
|
||||
MAX_TUNING_NUMBER = 0,
|
||||
// Strictly check model file or not, default 1. if set 0, will not check model file valid/invalid
|
||||
STRICT_CHECK_MODEL = 1,
|
||||
MEM_ALLOCATOR_TYPE = 2,
|
||||
// Winograd unit candidates count, default 3. if set 0, will use less unit candidates for less memory at the expense of performance.
|
||||
WINOGRAD_MEMORY_LEVEL = 3,
|
||||
|
||||
// Geometry Compute option, default is 0xFFFF
|
||||
GEOMETRY_COMPUTE_MASK = 4,
|
||||
|
||||
// default 0
|
||||
// 1: For general convolution, use one scale&zeropoint to quant.
|
||||
// 2: use block-quant for input data.
|
||||
DYNAMIC_QUANT_OPTIONS = 5,
|
||||
|
||||
// For Mobile CPU with big-litter core, set decrease rate to let MNN divide task differential by CPU's performance
|
||||
// 0-100, 50 means litter core has 50% capacity of large core
|
||||
// Default is 50
|
||||
CPU_LITTLECORE_DECREASE_RATE = 6,
|
||||
|
||||
// attentionOption % 8:
|
||||
// 0: Do not quantize
|
||||
// 1: Q,K: Int8, V: Float
|
||||
// 2: Q,K,V: Int8
|
||||
|
||||
// attentionOption / 8:
|
||||
// 0: don't use flash attention
|
||||
// 1: use flash attention
|
||||
ATTENTION_OPTION = 7,
|
||||
|
||||
// size limit of kvcache in memory (for a single layer)
|
||||
// if the size of kvcache exceeds the limit, it will be moved to disk
|
||||
KVCACHE_SIZE_LIMIT = 8,
|
||||
// Op encoder number for commit
|
||||
OP_ENCODER_NUMBER_FOR_COMMIT = 9,
|
||||
|
||||
// KVCache Info
|
||||
KVCACHE_INFO = 10,
|
||||
// mmap allocate file size, KB
|
||||
MMAP_FILE_SIZE = 11,
|
||||
USE_CACHED_MMAP = 12,
|
||||
|
||||
// Multi-Thread Load module, default is 0 (don't use other Thread)
|
||||
INIT_THREAD_NUMBER = 13,
|
||||
|
||||
// Used CPU ids
|
||||
CPU_CORE_IDS = 14,
|
||||
|
||||
// set CPU threads to use when supports Arm sme2
|
||||
CPU_SME2_INSTRUCTIONS = 15,
|
||||
|
||||
// Enable KleidiAI
|
||||
CPU_ENABLE_KLEIDIAI = 16,
|
||||
|
||||
// Set CPU SME2 NEON division ratio, default is 41
|
||||
CPU_SME2_NEON_DIVISION_RATIO = 17,
|
||||
|
||||
// Set SME cores, default is 2, if supports sme
|
||||
CPU_SME_CORES = 18
|
||||
};
|
||||
|
||||
enum ExternalPathType {
|
||||
// Path of the kvcache directory
|
||||
EXTERNAL_PATH_KVCACHE_DIR = 0,
|
||||
|
||||
// Mid Buffer Cache File
|
||||
EXTERNAL_FEATUREMAP_DIR = 1,
|
||||
|
||||
// Weight Buffer Cache File
|
||||
EXTERNAL_WEIGHT_DIR = 2,
|
||||
|
||||
// Path of the NPU Model directory
|
||||
EXTERNAL_NPU_FILE_DIR = 3,
|
||||
|
||||
// Path of the kvcache directory
|
||||
EXTERNAL_PATH_PREFIXCACHE_DIR = 4,
|
||||
|
||||
// Other types ...
|
||||
};
|
||||
|
||||
enum GeometryComputeMask {
|
||||
// Support Region Fuse
|
||||
GEOMETRCOMPUTEMASK_FUSEREGION = 1 << 0,
|
||||
|
||||
// Support Region Fuse to input with multi-region, eg: pad + concat
|
||||
GEOMETRCOMPUTEMASK_FUSEREGION_MULTI = 1 << 1,
|
||||
|
||||
// Use loop instead of raster + compute if possible
|
||||
GEOMETRCOMPUTEMASK_USELOOP = 1 << 2,
|
||||
|
||||
// Support Geometry Cache, if shape changed, will try recompute, and then run compute if failed
|
||||
GEOMETRCOMPUTEMASK_OPENCACHE = 1 << 3,
|
||||
|
||||
// Full option open mask, for example, if want to close useloop, can set mask as (GEOMETRCOMPUTEMASK_ALL - GEOMETRCOMPUTEMASK_USELOOP)
|
||||
GEOMETRCOMPUTEMASK_ALL = 0xFFFF,
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief The API shoud be called before create session.
|
||||
* @param hint Hint type
|
||||
* @param value Hint value
|
||||
* @param size Hint value size(when use a ptr)
|
||||
*/
|
||||
void setSessionHint(HintMode hint, int value);
|
||||
void setSessionHint(HintMode hint, int* value, size_t size);
|
||||
public:
|
||||
/**
|
||||
* @brief create runtimeInfo separately with schedule config.
|
||||
* @param configs session schedule configs.
|
||||
*/
|
||||
static RuntimeInfo createRuntime(const std::vector<ScheduleConfig>& configs);
|
||||
|
||||
/**
|
||||
* @brief create session with schedule config. created session will be managed in net.
|
||||
* @param config session schedule config.
|
||||
* @return created session if success, NULL otherwise.
|
||||
*/
|
||||
Session* createSession(const ScheduleConfig& config);
|
||||
|
||||
/**
|
||||
* @brief create session with schedule config and user-specified runtime.
|
||||
* @param config session schedule config, runtime runtimeInfo used by the created session.
|
||||
* @return created session if success, NULL otherwise.
|
||||
*/
|
||||
Session* createSession(const ScheduleConfig& config, const RuntimeInfo& runtime);
|
||||
|
||||
/**
|
||||
* @brief create multi-path session with schedule configs. created session will be managed in net.
|
||||
* @param configs session schedule configs.
|
||||
* @return created session if success, NULL otherwise.
|
||||
*/
|
||||
Session* createMultiPathSession(const std::vector<ScheduleConfig>& configs);
|
||||
|
||||
/**
|
||||
* @brief create multi-path session with schedule configs and user-specified runtime.
|
||||
created session will be managed in net.
|
||||
* @param configs session schedule configs.
|
||||
* @return created session if success, NULL otherwise.
|
||||
*/
|
||||
Session* createMultiPathSession(const std::vector<ScheduleConfig>& configs, const RuntimeInfo& runtime);
|
||||
|
||||
/**
|
||||
* @brief release session.
|
||||
* @param session given session.
|
||||
* @return true if given session is held by net and is freed.
|
||||
*/
|
||||
bool releaseSession(Session* session);
|
||||
|
||||
/**
|
||||
* @brief call this function to get tensors ready. output tensor buffer (host or deviceId) should be retrieved
|
||||
* after resize of any input tensor.
|
||||
* @param session given session.
|
||||
*/
|
||||
void resizeSession(Session* session);
|
||||
|
||||
/**
|
||||
* @brief call this function to get tensors ready. output tensor buffer (host or deviceId) should be retrieved
|
||||
* after resize of any input tensor.
|
||||
* @param session given session.
|
||||
* @param needRelloc, 1 means need realloc.
|
||||
*/
|
||||
void resizeSession(Session* session, int needRelloc);
|
||||
|
||||
|
||||
/**
|
||||
* @brief call this function if don't need resize or create session any more, it will save a few memory that equal
|
||||
* to the size of model buffer
|
||||
*/
|
||||
void releaseModel();
|
||||
|
||||
/**
|
||||
* @brief Get the model buffer for user to save
|
||||
* @return std::make_pair(modelBuffer, modelSize).
|
||||
* @example:
|
||||
* std::ofstream output("trainResult.alinn")
|
||||
* auto buffer = net->getModelBuffer();
|
||||
* output.write((const char*)buffer.first, buffer.second);
|
||||
*/
|
||||
std::pair<const void*, size_t> getModelBuffer() const;
|
||||
|
||||
/**
|
||||
* @brief Get the model's version info.
|
||||
* @return const char* of model's version info like "2.0.0";
|
||||
* If model is not loaded or model no version info, return "version info not found".
|
||||
*/
|
||||
const char* getModelVersion() const;
|
||||
|
||||
/**
|
||||
* @brief update Session's Tensor to model's Const Op
|
||||
* @param session given session.
|
||||
* @return result of running.
|
||||
*/
|
||||
ErrorCode updateSessionToModel(Session* session);
|
||||
|
||||
/**
|
||||
* @brief run session.
|
||||
* @param session given session.
|
||||
* @return result of running.
|
||||
*/
|
||||
ErrorCode runSession(Session* session) const;
|
||||
|
||||
/*
|
||||
* @brief run session.
|
||||
* @param session given session.
|
||||
* @param before callback before each op. return true to run the op; return false to skip the op.
|
||||
* @param after callback after each op. return true to continue running; return false to interrupt the session.
|
||||
* @param sync synchronously wait for finish of execution or not.
|
||||
* @return result of running.
|
||||
*/
|
||||
ErrorCode runSessionWithCallBack(const Session* session, const TensorCallBack& before, const TensorCallBack& end,
|
||||
bool sync = false) const;
|
||||
|
||||
/*
|
||||
* @brief run session.
|
||||
* @param session given session.
|
||||
* @param before callback before each op. return true to run the op; return false to skip the op.
|
||||
* @param after callback after each op. return true to continue running; return false to interrupt the session.
|
||||
* @param sync synchronously wait for finish of execution or not.
|
||||
* @return result of running.
|
||||
*/
|
||||
ErrorCode runSessionWithCallBackInfo(const Session* session, const TensorCallBackWithInfo& before,
|
||||
const TensorCallBackWithInfo& end, bool sync = false) const;
|
||||
|
||||
/**
|
||||
* @brief get input tensor for given name.
|
||||
* @param session given session.
|
||||
* @param name given name. if NULL, return first input.
|
||||
* @return tensor if found, NULL otherwise.
|
||||
*/
|
||||
Tensor* getSessionInput(const Session* session, const char* name);
|
||||
/**
|
||||
* @brief get output tensor for given name.
|
||||
* @param session given session.
|
||||
* @param name given name. if NULL, return first output.
|
||||
* @return tensor if found, NULL otherwise.
|
||||
*/
|
||||
Tensor* getSessionOutput(const Session* session, const char* name);
|
||||
|
||||
enum SessionInfoCode {
|
||||
/** memory session used in MB, float* */
|
||||
MEMORY = 0,
|
||||
|
||||
/** float operation needed in session in M, float* */
|
||||
FLOPS = 1,
|
||||
|
||||
/** Backends in session in M, int*, length >= 1 + number of configs when create session */
|
||||
BACKENDS = 2,
|
||||
|
||||
/** Resize Info, int* , the mean different from API
|
||||
Interpreter::getSessionInfo: 0: ready to execute, 1: need malloc, 2: need resize
|
||||
RuntimeManager::getInfo: 0: no resize, 1: re-malloc, 2: resize
|
||||
*/
|
||||
RESIZE_STATUS = 3,
|
||||
|
||||
/** Mode / NumberThread, int* */
|
||||
THREAD_NUMBER = 4,
|
||||
|
||||
ALL
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief get session info
|
||||
* @param session given session.
|
||||
* @param code given info code.
|
||||
* @param ptr given info ptr, see SessionInfoCode for detail
|
||||
* @return true if support the code, false otherwise.
|
||||
*/
|
||||
bool getSessionInfo(const Session* session, SessionInfoCode code, void* ptr);
|
||||
|
||||
/**
|
||||
* @brief get all output tensors.
|
||||
* @param session given session.
|
||||
* @return all output tensors mapped with name.
|
||||
*/
|
||||
const std::map<std::string, Tensor*>& getSessionOutputAll(const Session* session) const;
|
||||
/**
|
||||
* @brief get all input tensors.
|
||||
* @param session given session.
|
||||
* @return all input tensors mapped with name.
|
||||
*/
|
||||
const std::map<std::string, Tensor*>& getSessionInputAll(const Session* session) const;
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief resize given tensor.
|
||||
* @param tensor given tensor.
|
||||
* @param dims new dims. at most 6 dims.
|
||||
*/
|
||||
void resizeTensor(Tensor* tensor, const std::vector<int>& dims);
|
||||
|
||||
/**
|
||||
* @brief resize given tensor by nchw.
|
||||
* @param batch / N.
|
||||
* @param channel / C.
|
||||
* @param height / H.
|
||||
* @param width / W
|
||||
*/
|
||||
void resizeTensor(Tensor* tensor, int batch, int channel, int height, int width);
|
||||
|
||||
/**
|
||||
* @brief get backend used to create given tensor.
|
||||
* @param session given session.
|
||||
* @param tensor given tensor.
|
||||
* @return backend used to create given tensor, may be NULL.
|
||||
*/
|
||||
const Backend* getBackend(const Session* session, const Tensor* tensor) const;
|
||||
|
||||
/**
|
||||
* @brief get business code (model identifier).
|
||||
* @return business code.
|
||||
*/
|
||||
const char* bizCode() const;
|
||||
|
||||
/**
|
||||
* @brief get model UUID
|
||||
* @return Model UUID.
|
||||
*/
|
||||
const char* uuid() const;
|
||||
|
||||
private:
|
||||
static Interpreter* createFromBufferInternal(Content* net, bool enforceAuth);
|
||||
|
||||
Content* mNet = nullptr;
|
||||
Interpreter(Content* net);
|
||||
|
||||
Interpreter(const Interpreter&) = delete;
|
||||
Interpreter(const Interpreter&&) = delete;
|
||||
Interpreter& operator=(const Interpreter&) = delete;
|
||||
Interpreter& operator=(const Interpreter&&) = delete;
|
||||
void waitSessionFinish(const Session* session) const;
|
||||
#ifdef MNN_INTERNAL_ENABLED
|
||||
void logForRunSession(const Session* session, float time, const char* api) const;
|
||||
#endif
|
||||
};
|
||||
} // namespace MNN
|
||||
|
||||
#endif /* Interpreter_hpp */
|
||||
@@ -0,0 +1,79 @@
|
||||
//
|
||||
// MNNDefine.h
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2018/08/09.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNNDefine_h
|
||||
#define MNNDefine_h
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#if defined(__APPLE__)
|
||||
#include <TargetConditionals.h>
|
||||
#if TARGET_OS_IPHONE
|
||||
#define MNN_BUILD_FOR_IOS
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef MNN_USE_LOGCAT
|
||||
#if defined(__OHOS__)
|
||||
#include <hilog/log.h>
|
||||
#define MNN_ERROR(format, ...) {char logtmp[4096]; snprintf(logtmp, 4096, format, ##__VA_ARGS__); OH_LOG_Print(LOG_APP, LOG_ERROR, LOG_DOMAIN, "MNNJNI", (const char*)logtmp);}
|
||||
#define MNN_PRINT(format, ...) {char logtmp[4096]; snprintf(logtmp, 4096, format, ##__VA_ARGS__); OH_LOG_Print(LOG_APP, LOG_DEBUG, LOG_DOMAIN, "MNNJNI", (const char*)logtmp);}
|
||||
#else
|
||||
#include <android/log.h>
|
||||
#define MNN_ERROR(format, ...) __android_log_print(ANDROID_LOG_ERROR, "MNNJNI", format, ##__VA_ARGS__)
|
||||
#define MNN_PRINT(format, ...) __android_log_print(ANDROID_LOG_INFO, "MNNJNI", format, ##__VA_ARGS__)
|
||||
#endif
|
||||
#elif defined MNN_BUILD_FOR_IOS
|
||||
// on iOS, stderr prints to XCode debug area and syslog prints Console. You need both.
|
||||
#include <syslog.h>
|
||||
#define MNN_PRINT(format, ...) syslog(LOG_WARNING, format, ##__VA_ARGS__); fprintf(stderr, format, ##__VA_ARGS__)
|
||||
#define MNN_ERROR(format, ...) syslog(LOG_WARNING, format, ##__VA_ARGS__); fprintf(stderr, format, ##__VA_ARGS__)
|
||||
#else
|
||||
#define MNN_PRINT(format, ...) printf(format, ##__VA_ARGS__)
|
||||
#define MNN_ERROR(format, ...) printf(format, ##__VA_ARGS__)
|
||||
#endif
|
||||
|
||||
#ifdef DEBUG
|
||||
#define MNN_ASSERT(x) \
|
||||
{ \
|
||||
int res = (x); \
|
||||
if (!res) { \
|
||||
MNN_ERROR("Error for %s, %d\n", __FILE__, __LINE__); \
|
||||
assert(res); \
|
||||
} \
|
||||
}
|
||||
#else
|
||||
#define MNN_ASSERT(x)
|
||||
#endif
|
||||
|
||||
#define FUNC_PRINT(x) MNN_PRINT(#x "=%d in %s, %d \n", x, __func__, __LINE__);
|
||||
#define FUNC_PRINT_ALL(x, type) MNN_PRINT(#x "=" #type " %" #type " in %s, %d \n", x, __func__, __LINE__);
|
||||
|
||||
#define MNN_CHECK(success, log) \
|
||||
if(!(success)){ \
|
||||
MNN_ERROR("Check failed: %s ==> %s\n", #success, #log); \
|
||||
}
|
||||
#if defined(_MSC_VER)
|
||||
#if defined(BUILDING_MNN_DLL)
|
||||
#define MNN_PUBLIC __declspec(dllexport)
|
||||
#elif defined(USING_MNN_DLL)
|
||||
#define MNN_PUBLIC __declspec(dllimport)
|
||||
#else
|
||||
#define MNN_PUBLIC
|
||||
#endif
|
||||
#else
|
||||
#define MNN_PUBLIC __attribute__((visibility("default")))
|
||||
#endif
|
||||
#define STR_IMP(x) #x
|
||||
#define STR(x) STR_IMP(x)
|
||||
#define MNN_VERSION_MAJOR 3
|
||||
#define MNN_VERSION_MINOR 6
|
||||
#define MNN_VERSION_PATCH 1
|
||||
#define MNN_VERSION STR(MNN_VERSION_MAJOR) "." STR(MNN_VERSION_MINOR) "." STR(MNN_VERSION_PATCH)
|
||||
#endif /* MNNDefine_h */
|
||||
@@ -0,0 +1,129 @@
|
||||
//
|
||||
// MNNForwardType.h
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2019/01/19.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNNForwardType_h
|
||||
#define MNNForwardType_h
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
|
||||
typedef enum {
|
||||
MNN_FORWARD_CPU = 0,
|
||||
|
||||
/*
|
||||
Firtly find the first available backends not equal to CPU
|
||||
If no other backends, use cpu
|
||||
*/
|
||||
MNN_FORWARD_AUTO = 4,
|
||||
|
||||
/*Hand write metal*/
|
||||
MNN_FORWARD_METAL = 1,
|
||||
|
||||
/*NVIDIA GPU API*/
|
||||
MNN_FORWARD_CUDA = 2,
|
||||
|
||||
/*Moore Threads GPU API*/
|
||||
MNN_FORWARD_MUSA = 15,
|
||||
|
||||
/*Android / Common Device GPU API*/
|
||||
MNN_FORWARD_OPENCL = 3,
|
||||
MNN_FORWARD_OPENGL = 6,
|
||||
MNN_FORWARD_VULKAN = 7,
|
||||
|
||||
/*Android 8.1's NNAPI or CoreML for ios*/
|
||||
MNN_FORWARD_NN = 5,
|
||||
|
||||
/*User can use API from Backend.hpp to add or search Backend*/
|
||||
MNN_FORWARD_USER_0 = 8,
|
||||
MNN_FORWARD_USER_1 = 9,
|
||||
MNN_FORWARD_HEXAGON = 10,
|
||||
MNN_FORWARD_USER_3 = 11,
|
||||
|
||||
MNN_FORWARD_ALL = 12,
|
||||
|
||||
/* Apply arm extension instruction set to accelerate some Ops, this forward type
|
||||
is only used in MNN internal, and will be active automatically when user set forward type
|
||||
to be MNN_FORWARD_CPU and extension instruction set is valid on hardware.
|
||||
*/
|
||||
MNN_FORWARD_CPU_EXTENSION = 13,
|
||||
// use for shared memory on android device
|
||||
|
||||
MNN_MEMORY_AHARDWAREBUFFER = 14,
|
||||
|
||||
/* For Offline Convert*/
|
||||
MNN_CONVERT_QNN = 32,
|
||||
MNN_CONVERT_NEUROPILOT = 33,
|
||||
MNN_CONVERT_COREML = 34,
|
||||
|
||||
} MNNForwardType;
|
||||
|
||||
typedef enum {
|
||||
// For the OpenCL backend, all five of the following options are valid. The user is allowed to enable any one of them.
|
||||
// For the Vulkan backend, only options MNN_GPU_TUNING_NONE, MNN_GPU_TUNING_HEAVY, and MNN_GPU_TUNING_WIDE are valid. The user is allowed to enable any one of these three.
|
||||
MNN_GPU_TUNING_NONE = 1 << 0, /* Forbidden tuning, performance not good.(OpenCL/Vulkan) */
|
||||
MNN_GPU_TUNING_HEAVY = 1 << 1, /* Heavily tuning, usually not suggested.(OpenCL/Vulkan) */
|
||||
MNN_GPU_TUNING_WIDE = 1 << 2, /* Widely tuning, performance good. Default.(OpenCL/Vulkan) */
|
||||
MNN_GPU_TUNING_NORMAL = 1 << 3, /* Normal tuning, performance may be ok.(OpenCL) */
|
||||
MNN_GPU_TUNING_FAST = 1 << 4, /* Fast tuning, performance may not good.(OpenCL) */
|
||||
|
||||
// For the OpenCL backend, the following two options are both valid. The user could try OpenCL_MEMORY_BUFFER and OpenCL_MEMORY_IMAGE both, and then choose the better one based on performance.
|
||||
// For the Vulkan backend, neither option is valid. The user uses the CMake option MNN_VULKAN_IMAGE to select between image memory mode and buffer memory mode.
|
||||
MNN_GPU_MEMORY_BUFFER = 1 << 6, /* OpenCL_MEMORY_BUFFER */
|
||||
MNN_GPU_MEMORY_IMAGE = 1 << 7, /* OpenCL_MEMORY_IMAGE */
|
||||
|
||||
// For the OpenCL backend, the following two options are effective only on Qualcomm GPUs. When using a Qualcomm GPU, the user could try both options and choose the better one based on performance.
|
||||
// For the Vulkan backend, only option MNN_GPU_RECORD_BATCH is valid. When MNN_GPU_RECORD_BATCH is enabled, all ops would share one commandBuffer.
|
||||
MNN_GPU_RECORD_OP = 1 << 8, /* The kernels in one op execution record into one recording.(OpenCL) */
|
||||
MNN_GPU_RECORD_BATCH = 1 << 9, /* 10 kernels record into one recording.(OpenCL) All ops share one commandBuffer.(Vulkan) */
|
||||
} MNNGpuMode;
|
||||
|
||||
#ifdef __cplusplus
|
||||
namespace MNN {
|
||||
struct BackendConfig {
|
||||
enum MemoryMode { Memory_Normal = 0, Memory_High, Memory_Low };
|
||||
|
||||
MemoryMode memory = Memory_Normal;
|
||||
|
||||
enum PowerMode { Power_Normal = 0, Power_High, Power_Low };
|
||||
|
||||
PowerMode power = Power_Normal;
|
||||
|
||||
enum PrecisionMode { Precision_Normal = 0, Precision_High, Precision_Low, Precision_Low_BF16 };
|
||||
|
||||
PrecisionMode precision = Precision_Normal;
|
||||
|
||||
/** user defined context */
|
||||
union {
|
||||
void* sharedContext = nullptr;
|
||||
size_t flags; // Valid for CPU Backend
|
||||
};
|
||||
};
|
||||
|
||||
/** acquire runtime status by Runtime::getCurrentStatus with following keys,
|
||||
*/
|
||||
enum RuntimeStatus {
|
||||
/**
|
||||
* get status whether this runtime support 16-bits float point arithmetic
|
||||
*/
|
||||
STATUS_SUPPORT_FP16,
|
||||
/**
|
||||
* get status whether this runtime support dot-product arithmetic
|
||||
*/
|
||||
STATUS_SUPPORT_DOT_PRODUCT,
|
||||
/**
|
||||
* get status whether this runtime support power-low (means low priority for opencl)
|
||||
*/
|
||||
STATUS_SUPPORT_POWER_LOW,
|
||||
/**
|
||||
* emum total number
|
||||
*/
|
||||
STATUS_COUNT
|
||||
};
|
||||
|
||||
}; // namespace MNN
|
||||
#endif
|
||||
#endif /* MNNForwardType_h */
|
||||
@@ -0,0 +1,77 @@
|
||||
//
|
||||
// MNNSharedContext.h
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2018/10/11.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNNSharedContext_h
|
||||
#define MNNSharedContext_h
|
||||
#include "MNNDefine.h"
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdint.h> /*uint32_t*/
|
||||
#ifdef MNN_VULKAN
|
||||
|
||||
struct MNNVulkanContext {
|
||||
VkInstance pInstance;
|
||||
VkPhysicalDevice pPhysicalDevice;
|
||||
VkDevice pDevice;
|
||||
VkQueue pQueue;
|
||||
uint32_t iQueueFamilyIndex;
|
||||
};
|
||||
|
||||
struct MNNVulkanTensorContent {
|
||||
VkBuffer buffer;
|
||||
VkDeviceSize size;
|
||||
VkDeviceSize offset;
|
||||
|
||||
halide_type_t realType;
|
||||
int32_t mask; // For future usage
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef MNN_METAL
|
||||
struct MNNMetalSharedContext {
|
||||
id<MTLDevice> device;
|
||||
id<MTLCommandQueue> queue;
|
||||
};
|
||||
|
||||
struct MNNMetalTensorContent {
|
||||
id<MTLBuffer> buffer;
|
||||
int32_t offset;
|
||||
id<MTLTexture> texture;
|
||||
|
||||
halide_type_t type;
|
||||
int32_t mask;
|
||||
int32_t forFuture[8];
|
||||
};
|
||||
|
||||
MNN_PUBLIC int MNNMetalGetTensorContent(MNNMetalTensorContent* content, void* tensor);
|
||||
#endif
|
||||
|
||||
#ifdef MNN_USER_SET_DEVICE
|
||||
|
||||
struct MNNDeviceContext {
|
||||
// When one gpu card has multi devices, choose which device. set deviceId
|
||||
uint32_t deviceId = 0;
|
||||
// When has multi gpu cards, choose which card. set platformId
|
||||
uint32_t platformId = 0;
|
||||
// User set number of gpu cards
|
||||
uint32_t platformSize = 0;
|
||||
// User set OpenCL context ptr
|
||||
void *contextPtr = nullptr;
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MNNSharedContext_h */
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,580 @@
|
||||
//
|
||||
// Rect.h
|
||||
// MNN
|
||||
//
|
||||
// Modified by jiangxiaotang on 2018/09/19.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
/*
|
||||
* Copyright 2006 The Android Open Source Project
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license that can be
|
||||
* found in the LICENSE file.
|
||||
*/
|
||||
|
||||
/* Generated by tools/bookmaker from include/core/Rect.h and docs/SkRect_Reference.bmh
|
||||
on 2018-07-13 08:15:11. Additional documentation and examples can be found at:
|
||||
https://skia.org/user/api/SkRect_Reference
|
||||
|
||||
You may edit either file directly. Structural changes to public interfaces require
|
||||
editing both files. After editing docs/SkRect_Reference.bmh, run:
|
||||
bookmaker -b docs -i include/core/Rect.h -p
|
||||
to create an updated version of this file.
|
||||
*/
|
||||
|
||||
#ifndef MNN_Rect_DEFINED
|
||||
#define MNN_Rect_DEFINED
|
||||
|
||||
#include <math.h>
|
||||
#include <algorithm>
|
||||
#include <utility>
|
||||
#include <MNN/MNNDefine.h>
|
||||
|
||||
namespace MNN {
|
||||
namespace CV {
|
||||
|
||||
struct Point {
|
||||
float fX;
|
||||
float fY;
|
||||
|
||||
void set(float x, float y) {
|
||||
fX = x;
|
||||
fY = y;
|
||||
}
|
||||
};
|
||||
|
||||
/** \struct Rect
|
||||
Rect holds four float coordinates describing the upper and
|
||||
lower bounds of a rectangle. Rect may be created from outer bounds or
|
||||
from position, width, and height. Rect describes an area; if its right
|
||||
is less than or equal to its left, or if its bottom is less than or equal to
|
||||
its top, it is considered empty.
|
||||
*/
|
||||
struct MNN_PUBLIC Rect {
|
||||
float fLeft; //!< smaller x-axis bounds
|
||||
float fTop; //!< smaller y-axis bounds
|
||||
float fRight; //!< larger x-axis bounds
|
||||
float fBottom; //!< larger y-axis bounds
|
||||
|
||||
/** Returns constructed Rect set to (0, 0, 0, 0).
|
||||
Many other rectangles are empty; if left is equal to or greater than right,
|
||||
or if top is equal to or greater than bottom. Setting all members to zero
|
||||
is a convenience, but does not designate a special empty rectangle.
|
||||
|
||||
@return bounds (0, 0, 0, 0)
|
||||
*/
|
||||
static constexpr Rect MakeEmpty() {
|
||||
return Rect{0, 0, 0, 0};
|
||||
}
|
||||
|
||||
#ifdef SK_SUPPORT_LEGACY_RECTMAKELARGEST
|
||||
/** Deprecated.
|
||||
*/
|
||||
static Rect MakeLargest() {
|
||||
return {SK_ScalarMin, SK_ScalarMin, SK_ScalarMax, SK_ScalarMax};
|
||||
}
|
||||
#endif
|
||||
|
||||
/** Returns constructed Rect set to float values (0, 0, w, h). Does not
|
||||
validate input; w or h may be negative.
|
||||
|
||||
Passing integer values may generate a compiler warning since Rect cannot
|
||||
represent 32-bit integers exactly. Use SkIRect for an exact integer rectangle.
|
||||
|
||||
@param w float width of constructed Rect
|
||||
@param h float height of constructed Rect
|
||||
@return bounds (0, 0, w, h)
|
||||
*/
|
||||
static constexpr Rect MakeWH(float w, float h) {
|
||||
return Rect{0, 0, w, h};
|
||||
}
|
||||
|
||||
/** Returns constructed Rect set to integer values (0, 0, w, h). Does not validate
|
||||
input; w or h may be negative.
|
||||
|
||||
Use to avoid a compiler warning that input may lose precision when stored.
|
||||
Use SkIRect for an exact integer rectangle.
|
||||
|
||||
@param w integer width of constructed Rect
|
||||
@param h integer height of constructed Rect
|
||||
@return bounds (0, 0, w, h)
|
||||
*/
|
||||
static Rect MakeIWH(int w, int h) {
|
||||
Rect r;
|
||||
r.set(0, 0, (float)(w), (float)(h));
|
||||
return r;
|
||||
}
|
||||
|
||||
/** Returns constructed Rect set to (l, t, r, b). Does not sort input; Rect may
|
||||
result in fLeft greater than fRight, or fTop greater than fBottom.
|
||||
|
||||
@param l float stored in fLeft
|
||||
@param t float stored in fTop
|
||||
@param r float stored in fRight
|
||||
@param b float stored in fBottom
|
||||
@return bounds (l, t, r, b)
|
||||
*/
|
||||
static constexpr Rect MakeLTRB(float l, float t, float r, float b) {
|
||||
return Rect{l, t, r, b};
|
||||
}
|
||||
|
||||
/** Returns constructed Rect set to (x, y, x + w, y + h). Does not validate input;
|
||||
w or h may be negative.
|
||||
|
||||
@param x stored in fLeft
|
||||
@param y stored in fTop
|
||||
@param w added to x and stored in fRight
|
||||
@param h added to y and stored in fBottom
|
||||
@return bounds at (x, y) with width w and height h
|
||||
*/
|
||||
static constexpr Rect MakeXYWH(float x, float y, float w, float h) {
|
||||
return Rect{x, y, x + w, y + h};
|
||||
}
|
||||
|
||||
/** Returns true if fLeft is equal to or greater than fRight, or if fTop is equal
|
||||
to or greater than fBottom. Call sort() to reverse rectangles with negative
|
||||
width() or height().
|
||||
|
||||
@return true if width() or height() are zero or negative
|
||||
*/
|
||||
bool isEmpty() const {
|
||||
// We write it as the NOT of a non-empty rect, so we will return true if any values
|
||||
// are NaN.
|
||||
return !(fLeft < fRight && fTop < fBottom);
|
||||
}
|
||||
|
||||
/** Returns true if fLeft is equal to or less than fRight, or if fTop is equal
|
||||
to or less than fBottom. Call sort() to reverse rectangles with negative
|
||||
width() or height().
|
||||
|
||||
@return true if width() or height() are zero or positive
|
||||
*/
|
||||
bool isSorted() const {
|
||||
return fLeft <= fRight && fTop <= fBottom;
|
||||
}
|
||||
|
||||
/** Returns left edge of Rect, if sorted. Call isSorted() to see if Rect is valid.
|
||||
Call sort() to reverse fLeft and fRight if needed.
|
||||
|
||||
@return fLeft
|
||||
*/
|
||||
float x() const {
|
||||
return fLeft;
|
||||
}
|
||||
|
||||
/** Returns top edge of Rect, if sorted. Call isEmpty() to see if Rect may be invalid,
|
||||
and sort() to reverse fTop and fBottom if needed.
|
||||
|
||||
@return fTop
|
||||
*/
|
||||
float y() const {
|
||||
return fTop;
|
||||
}
|
||||
|
||||
/** Returns left edge of Rect, if sorted. Call isSorted() to see if Rect is valid.
|
||||
Call sort() to reverse fLeft and fRight if needed.
|
||||
|
||||
@return fLeft
|
||||
*/
|
||||
float left() const {
|
||||
return fLeft;
|
||||
}
|
||||
|
||||
/** Returns top edge of Rect, if sorted. Call isEmpty() to see if Rect may be invalid,
|
||||
and sort() to reverse fTop and fBottom if needed.
|
||||
|
||||
@return fTop
|
||||
*/
|
||||
float top() const {
|
||||
return fTop;
|
||||
}
|
||||
|
||||
/** Returns right edge of Rect, if sorted. Call isSorted() to see if Rect is valid.
|
||||
Call sort() to reverse fLeft and fRight if needed.
|
||||
|
||||
@return fRight
|
||||
*/
|
||||
float right() const {
|
||||
return fRight;
|
||||
}
|
||||
|
||||
/** Returns bottom edge of Rect, if sorted. Call isEmpty() to see if Rect may be invalid,
|
||||
and sort() to reverse fTop and fBottom if needed.
|
||||
|
||||
@return fBottom
|
||||
*/
|
||||
float bottom() const {
|
||||
return fBottom;
|
||||
}
|
||||
|
||||
/** Returns span on the x-axis. This does not check if Rect is sorted, or if
|
||||
result fits in 32-bit float; result may be negative or infinity.
|
||||
|
||||
@return fRight minus fLeft
|
||||
*/
|
||||
float width() const {
|
||||
return fRight - fLeft;
|
||||
}
|
||||
|
||||
/** Returns span on the y-axis. This does not check if Rect is sorted, or if
|
||||
result fits in 32-bit float; result may be negative or infinity.
|
||||
|
||||
@return fBottom minus fTop
|
||||
*/
|
||||
float height() const {
|
||||
return fBottom - fTop;
|
||||
}
|
||||
|
||||
/** Returns average of left edge and right edge. Result does not change if Rect
|
||||
is sorted. Result may overflow to infinity if Rect is far from the origin.
|
||||
|
||||
@return midpoint in x
|
||||
*/
|
||||
float centerX() const {
|
||||
// don't use floatHalf(fLeft + fBottom) as that might overflow before the 0.5
|
||||
return 0.5f * (fLeft) + 0.5f * (fRight);
|
||||
}
|
||||
|
||||
/** Returns average of top edge and bottom edge. Result does not change if Rect
|
||||
is sorted.
|
||||
|
||||
@return midpoint in y
|
||||
*/
|
||||
float centerY() const {
|
||||
// don't use floatHalf(fTop + fBottom) as that might overflow before the 0.5
|
||||
return 0.5f * (fTop) + 0.5f * (fBottom);
|
||||
}
|
||||
|
||||
/** Sets Rect to (0, 0, 0, 0).
|
||||
|
||||
Many other rectangles are empty; if left is equal to or greater than right,
|
||||
or if top is equal to or greater than bottom. Setting all members to zero
|
||||
is a convenience, but does not designate a special empty rectangle.
|
||||
*/
|
||||
void setEmpty() {
|
||||
*this = MakeEmpty();
|
||||
}
|
||||
|
||||
/** Sets Rect to (left, top, right, bottom).
|
||||
left and right are not sorted; left is not necessarily less than right.
|
||||
top and bottom are not sorted; top is not necessarily less than bottom.
|
||||
|
||||
@param left stored in fLeft
|
||||
@param top stored in fTop
|
||||
@param right stored in fRight
|
||||
@param bottom stored in fBottom
|
||||
*/
|
||||
void set(float left, float top, float right, float bottom) {
|
||||
fLeft = left;
|
||||
fTop = top;
|
||||
fRight = right;
|
||||
fBottom = bottom;
|
||||
}
|
||||
|
||||
/** Sets Rect to (left, top, right, bottom).
|
||||
left and right are not sorted; left is not necessarily less than right.
|
||||
top and bottom are not sorted; top is not necessarily less than bottom.
|
||||
|
||||
@param left stored in fLeft
|
||||
@param top stored in fTop
|
||||
@param right stored in fRight
|
||||
@param bottom stored in fBottom
|
||||
*/
|
||||
void setLTRB(float left, float top, float right, float bottom) {
|
||||
this->set(left, top, right, bottom);
|
||||
}
|
||||
|
||||
/** Sets Rect to (left, top, right, bottom).
|
||||
All parameters are promoted from integer to scalar.
|
||||
left and right are not sorted; left is not necessarily less than right.
|
||||
top and bottom are not sorted; top is not necessarily less than bottom.
|
||||
|
||||
@param left promoted to float and stored in fLeft
|
||||
@param top promoted to float and stored in fTop
|
||||
@param right promoted to float and stored in fRight
|
||||
@param bottom promoted to float and stored in fBottom
|
||||
*/
|
||||
void iset(int left, int top, int right, int bottom) {
|
||||
fLeft = (float)(left);
|
||||
fTop = (float)(top);
|
||||
fRight = (float)(right);
|
||||
fBottom = (float)(bottom);
|
||||
}
|
||||
|
||||
/** Sets Rect to (0, 0, width, height).
|
||||
width and height may be zero or negative. width and height are promoted from
|
||||
integer to float, large values may lose precision.
|
||||
|
||||
@param width promoted to float and stored in fRight
|
||||
@param height promoted to float and stored in fBottom
|
||||
*/
|
||||
void isetWH(int width, int height) {
|
||||
fLeft = fTop = 0;
|
||||
fRight = (float)(width);
|
||||
fBottom = (float)(height);
|
||||
}
|
||||
|
||||
/** Sets Rect to (x, y, x + width, y + height). Does not validate input;
|
||||
width or height may be negative.
|
||||
|
||||
@param x stored in fLeft
|
||||
@param y stored in fTop
|
||||
@param width added to x and stored in fRight
|
||||
@param height added to y and stored in fBottom
|
||||
*/
|
||||
void setXYWH(float x, float y, float width, float height) {
|
||||
fLeft = x;
|
||||
fTop = y;
|
||||
fRight = x + width;
|
||||
fBottom = y + height;
|
||||
}
|
||||
|
||||
/** Sets Rect to (0, 0, width, height). Does not validate input;
|
||||
width or height may be negative.
|
||||
|
||||
@param width stored in fRight
|
||||
@param height stored in fBottom
|
||||
*/
|
||||
void setWH(float width, float height) {
|
||||
fLeft = 0;
|
||||
fTop = 0;
|
||||
fRight = width;
|
||||
fBottom = height;
|
||||
}
|
||||
|
||||
/** Returns Rect offset by (dx, dy).
|
||||
|
||||
If dx is negative, Rect returned is moved to the left.
|
||||
If dx is positive, Rect returned is moved to the right.
|
||||
If dy is negative, Rect returned is moved upward.
|
||||
If dy is positive, Rect returned is moved downward.
|
||||
|
||||
@param dx added to fLeft and fRight
|
||||
@param dy added to fTop and fBottom
|
||||
@return Rect offset on axes, with original width and height
|
||||
*/
|
||||
Rect makeOffset(float dx, float dy) const {
|
||||
return MakeLTRB(fLeft + dx, fTop + dy, fRight + dx, fBottom + dy);
|
||||
}
|
||||
|
||||
/** Returns Rect, inset by (dx, dy).
|
||||
|
||||
If dx is negative, Rect returned is wider.
|
||||
If dx is positive, Rect returned is narrower.
|
||||
If dy is negative, Rect returned is taller.
|
||||
If dy is positive, Rect returned is shorter.
|
||||
|
||||
@param dx added to fLeft and subtracted from fRight
|
||||
@param dy added to fTop and subtracted from fBottom
|
||||
@return Rect inset symmetrically left and right, top and bottom
|
||||
*/
|
||||
Rect makeInset(float dx, float dy) const {
|
||||
return MakeLTRB(fLeft + dx, fTop + dy, fRight - dx, fBottom - dy);
|
||||
}
|
||||
|
||||
/** Returns Rect, outset by (dx, dy).
|
||||
|
||||
If dx is negative, Rect returned is narrower.
|
||||
If dx is positive, Rect returned is wider.
|
||||
If dy is negative, Rect returned is shorter.
|
||||
If dy is positive, Rect returned is taller.
|
||||
|
||||
@param dx subtracted to fLeft and added from fRight
|
||||
@param dy subtracted to fTop and added from fBottom
|
||||
@return Rect outset symmetrically left and right, top and bottom
|
||||
*/
|
||||
Rect makeOutset(float dx, float dy) const {
|
||||
return MakeLTRB(fLeft - dx, fTop - dy, fRight + dx, fBottom + dy);
|
||||
}
|
||||
|
||||
/** Offsets Rect by adding dx to fLeft, fRight; and by adding dy to fTop, fBottom.
|
||||
|
||||
If dx is negative, moves Rect to the left.
|
||||
If dx is positive, moves Rect to the right.
|
||||
If dy is negative, moves Rect upward.
|
||||
If dy is positive, moves Rect downward.
|
||||
|
||||
@param dx offset added to fLeft and fRight
|
||||
@param dy offset added to fTop and fBottom
|
||||
*/
|
||||
void offset(float dx, float dy) {
|
||||
fLeft += dx;
|
||||
fTop += dy;
|
||||
fRight += dx;
|
||||
fBottom += dy;
|
||||
}
|
||||
|
||||
/** Offsets Rect so that fLeft equals newX, and fTop equals newY. width and height
|
||||
are unchanged.
|
||||
|
||||
@param newX stored in fLeft, preserving width()
|
||||
@param newY stored in fTop, preserving height()
|
||||
*/
|
||||
void offsetTo(float newX, float newY) {
|
||||
fRight += newX - fLeft;
|
||||
fBottom += newY - fTop;
|
||||
fLeft = newX;
|
||||
fTop = newY;
|
||||
}
|
||||
|
||||
/** Insets Rect by (dx, dy).
|
||||
|
||||
If dx is positive, makes Rect narrower.
|
||||
If dx is negative, makes Rect wider.
|
||||
If dy is positive, makes Rect shorter.
|
||||
If dy is negative, makes Rect taller.
|
||||
|
||||
@param dx added to fLeft and subtracted from fRight
|
||||
@param dy added to fTop and subtracted from fBottom
|
||||
*/
|
||||
void inset(float dx, float dy) {
|
||||
fLeft += dx;
|
||||
fTop += dy;
|
||||
fRight -= dx;
|
||||
fBottom -= dy;
|
||||
}
|
||||
|
||||
/** Outsets Rect by (dx, dy).
|
||||
|
||||
If dx is positive, makes Rect wider.
|
||||
If dx is negative, makes Rect narrower.
|
||||
If dy is positive, makes Rect taller.
|
||||
If dy is negative, makes Rect shorter.
|
||||
|
||||
@param dx subtracted to fLeft and added from fRight
|
||||
@param dy subtracted to fTop and added from fBottom
|
||||
*/
|
||||
void outset(float dx, float dy) {
|
||||
this->inset(-dx, -dy);
|
||||
}
|
||||
|
||||
private:
|
||||
static bool Intersects(float al, float at, float ar, float ab, float bl, float bt, float br, float bb) {
|
||||
float L = std::max(al, bl);
|
||||
float R = std::min(ar, br);
|
||||
float T = std::max(at, bt);
|
||||
float B = std::min(ab, bb);
|
||||
return L < R && T < B;
|
||||
}
|
||||
|
||||
public:
|
||||
/** Constructs Rect to intersect from (left, top, right, bottom). Does not sort
|
||||
construction.
|
||||
|
||||
Returns true if Rect intersects construction.
|
||||
Returns false if either construction or Rect is empty, or do not intersect.
|
||||
|
||||
@param left x-axis minimum of constructed Rect
|
||||
@param top y-axis minimum of constructed Rect
|
||||
@param right x-axis maximum of constructed Rect
|
||||
@param bottom y-axis maximum of constructed Rect
|
||||
@return true if construction and Rect have area in common
|
||||
*/
|
||||
bool intersects(float left, float top, float right, float bottom) const {
|
||||
return Intersects(fLeft, fTop, fRight, fBottom, left, top, right, bottom);
|
||||
}
|
||||
|
||||
/** Returns true if Rect intersects r.
|
||||
Returns false if either r or Rect is empty, or do not intersect.
|
||||
|
||||
@param r Rect to intersect
|
||||
@return true if r and Rect have area in common
|
||||
*/
|
||||
bool intersects(const Rect& r) const {
|
||||
return Intersects(fLeft, fTop, fRight, fBottom, r.fLeft, r.fTop, r.fRight, r.fBottom);
|
||||
}
|
||||
|
||||
/** Returns true if a intersects b.
|
||||
Returns false if either a or b is empty, or do not intersect.
|
||||
|
||||
@param a Rect to intersect
|
||||
@param b Rect to intersect
|
||||
@return true if a and b have area in common
|
||||
*/
|
||||
static bool Intersects(const Rect& a, const Rect& b) {
|
||||
return Intersects(a.fLeft, a.fTop, a.fRight, a.fBottom, b.fLeft, b.fTop, b.fRight, b.fBottom);
|
||||
}
|
||||
|
||||
/** Sets Rect to the union of itself and r.
|
||||
|
||||
Asserts if r is empty and SK_DEBUG is defined.
|
||||
If Rect is empty, sets Rect to r.
|
||||
|
||||
May produce incorrect results if r is empty.
|
||||
|
||||
@param r expansion Rect
|
||||
*/
|
||||
void joinNonEmptyArg(const Rect& r) {
|
||||
MNN_ASSERT(!r.isEmpty());
|
||||
// if we are empty, just assign
|
||||
if (fLeft >= fRight || fTop >= fBottom) {
|
||||
*this = r;
|
||||
} else {
|
||||
this->joinPossiblyEmptyRect(r);
|
||||
}
|
||||
}
|
||||
|
||||
/** Sets Rect to the union of itself and the construction.
|
||||
|
||||
May produce incorrect results if Rect or r is empty.
|
||||
|
||||
@param r expansion Rect
|
||||
*/
|
||||
void joinPossiblyEmptyRect(const Rect& r) {
|
||||
fLeft = std::min(fLeft, r.left());
|
||||
fTop = std::min(fTop, r.top());
|
||||
fRight = std::max(fRight, r.right());
|
||||
fBottom = std::max(fBottom, r.bottom());
|
||||
}
|
||||
|
||||
/** Returns true if: fLeft <= x < fRight && fTop <= y < fBottom.
|
||||
Returns false if Rect is empty.
|
||||
|
||||
@param x test Point x-coordinate
|
||||
@param y test Point y-coordinate
|
||||
@return true if (x, y) is inside Rect
|
||||
*/
|
||||
bool contains(float x, float y) const {
|
||||
return x >= fLeft && x < fRight && y >= fTop && y < fBottom;
|
||||
}
|
||||
|
||||
/** Swaps fLeft and fRight if fLeft is greater than fRight; and swaps
|
||||
fTop and fBottom if fTop is greater than fBottom. Result may be empty;
|
||||
and width() and height() will be zero or positive.
|
||||
*/
|
||||
void sort() {
|
||||
using std::swap;
|
||||
if (fLeft > fRight) {
|
||||
swap(fLeft, fRight);
|
||||
}
|
||||
|
||||
if (fTop > fBottom) {
|
||||
swap(fTop, fBottom);
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns Rect with fLeft and fRight swapped if fLeft is greater than fRight; and
|
||||
with fTop and fBottom swapped if fTop is greater than fBottom. Result may be empty;
|
||||
and width() and height() will be zero or positive.
|
||||
|
||||
@return sorted Rect
|
||||
*/
|
||||
Rect makeSorted() const {
|
||||
return MakeLTRB(std::min(fLeft, fRight), std::min(fTop, fBottom), std::max(fLeft, fRight),
|
||||
std::max(fTop, fBottom));
|
||||
}
|
||||
|
||||
/** Returns pointer to first scalar in Rect, to treat it as an array with four
|
||||
entries.
|
||||
|
||||
@return pointer to fLeft
|
||||
*/
|
||||
const float* asScalars() const {
|
||||
return &fLeft;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace CV
|
||||
} // namespace MNN
|
||||
#endif
|
||||
@@ -0,0 +1,320 @@
|
||||
//
|
||||
// Tensor.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2018/08/14.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_Tensor_hpp
|
||||
#define MNN_Tensor_hpp
|
||||
|
||||
#include <vector>
|
||||
#include <MNN/HalideRuntime.h>
|
||||
#include <MNN/MNNDefine.h>
|
||||
|
||||
namespace MNN {
|
||||
|
||||
/**
|
||||
* data container.
|
||||
* data for host tensor is saved in `host` field. its memory is allocated malloc directly.
|
||||
* data for device tensor is saved in `deviceId` field. its memory is allocated by session's backend.
|
||||
* usually, device tensors are created by engine (like net, session).
|
||||
* meanwhile, host tensors could be created by engine or user.
|
||||
*/
|
||||
class MNN_PUBLIC Tensor {
|
||||
public:
|
||||
struct InsideDescribe;
|
||||
|
||||
/** dimension type used to create tensor */
|
||||
enum DimensionType {
|
||||
/** for tensorflow net type. uses NHWC as data format. */
|
||||
TENSORFLOW,
|
||||
/** for caffe net type. uses NCHW as data format. */
|
||||
CAFFE,
|
||||
/** for caffe net type. uses NC4HW4 as data format. */
|
||||
CAFFE_C4
|
||||
};
|
||||
|
||||
/** handle type */
|
||||
enum HandleDataType {
|
||||
/** default handle type */
|
||||
HANDLE_NONE = 0,
|
||||
/** string handle type */
|
||||
HANDLE_STRING = 1
|
||||
};
|
||||
|
||||
/** Tensor map type : Read or Write*/
|
||||
enum MapType {
|
||||
/** map Tensor for writing data*/
|
||||
MAP_TENSOR_WRITE = 0,
|
||||
MAP_TENSOR_READ = 1
|
||||
};
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief create a tensor with dimension size and type without acquire memory for data.
|
||||
* @param dimSize dimension size.
|
||||
* @param type dimension type.
|
||||
*/
|
||||
Tensor(int dimSize = 4, DimensionType type = CAFFE);
|
||||
|
||||
/**
|
||||
* @brief create a tensor with same shape as given tensor.
|
||||
* @param tensor shape provider.
|
||||
* @param type dimension type.
|
||||
* @param allocMemory acquire memory for data or not.
|
||||
* @warning tensor data won't be copied.
|
||||
*/
|
||||
Tensor(const Tensor* tensor, DimensionType type = CAFFE, bool allocMemory = true);
|
||||
|
||||
/** deinitializer */
|
||||
~Tensor();
|
||||
|
||||
private:
|
||||
Tensor(bool deepCopy, const Tensor* tensor);
|
||||
// remove all assignment operator
|
||||
Tensor(const Tensor& tensor) = delete;
|
||||
Tensor(const Tensor&& tensor) = delete;
|
||||
Tensor& operator=(const Tensor&) = delete;
|
||||
Tensor& operator=(const Tensor&&) = delete;
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief create tensor with shape, data type and dimension type.
|
||||
* @param shape tensor shape.
|
||||
* @param type data type.
|
||||
* @param dimType dimension type.
|
||||
* @return created tensor.
|
||||
* @warning memory for data won't be acquired. call backend's onAcquireBuffer to get memory ready.
|
||||
*/
|
||||
static Tensor* createDevice(const std::vector<int>& shape, halide_type_t type, DimensionType dimType = TENSORFLOW);
|
||||
|
||||
/**
|
||||
* @brief create tensor with shape and dimension type. data type is represented by `T`.
|
||||
* @param shape tensor shape.
|
||||
* @param dimType dimension type.
|
||||
* @return created tensor.
|
||||
* @warning memory for data won't be acquired. call backend's onAcquireBuffer to get memory ready.
|
||||
*/
|
||||
template <typename T>
|
||||
static Tensor* createDevice(const std::vector<int>& shape, DimensionType dimType = TENSORFLOW) {
|
||||
return createDevice(shape, halide_type_of<T>(), dimType);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief create tensor with shape, data type, data and dimension type.
|
||||
* @param shape tensor shape.
|
||||
* @param type data type.
|
||||
* @param data data to save.
|
||||
* @param dimType dimension type.
|
||||
* @return created tensor.
|
||||
*/
|
||||
static Tensor* create(const std::vector<int>& shape, halide_type_t type, void* data = NULL,
|
||||
DimensionType dimType = TENSORFLOW);
|
||||
|
||||
/**
|
||||
* @brief create tensor with shape, data and dimension type. data type is represented by `T`.
|
||||
* @param shape tensor shape.
|
||||
* @param data data to save.
|
||||
* @param dimType dimension type.
|
||||
* @return created tensor.
|
||||
*/
|
||||
template <typename T>
|
||||
static Tensor* create(const std::vector<int>& shape, void* data = NULL, DimensionType dimType = TENSORFLOW) {
|
||||
return create(shape, halide_type_of<T>(), data, dimType);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief copy tensor.
|
||||
* @param src tensor
|
||||
* @param deepCopy whether create new content and copy, currently only support deepCopy = false
|
||||
*/
|
||||
static Tensor* clone(const Tensor* src, bool deepCopy = false);
|
||||
|
||||
/**
|
||||
* @brief delete tensor.
|
||||
* @param src tensor
|
||||
*/
|
||||
static void destroy(Tensor* tensor);
|
||||
public:
|
||||
/**
|
||||
* @brief for DEVICE tensor, copy data from given host tensor.
|
||||
* @param hostTensor host tensor, the data provider.
|
||||
* @return true for DEVICE tensor, and false for HOST tensor.
|
||||
*/
|
||||
bool copyFromHostTensor(const Tensor* hostTensor);
|
||||
|
||||
/**
|
||||
* @brief for DEVICE tensor, copy data to given host tensor.
|
||||
* @param hostTensor host tensor, the data consumer.
|
||||
* @return true for DEVICE tensor, and false for HOST tensor.
|
||||
*/
|
||||
bool copyToHostTensor(Tensor* hostTensor) const;
|
||||
|
||||
/**
|
||||
* @brief create HOST tensor from DEVICE tensor, with or without data copying.
|
||||
* @param deviceTensor given device tensor.
|
||||
* @param copyData copy data or not.
|
||||
* @return created host tensor.
|
||||
*/
|
||||
static Tensor* createHostTensorFromDevice(const Tensor* deviceTensor, bool copyData = true);
|
||||
|
||||
public:
|
||||
const halide_buffer_t& buffer() const {
|
||||
return mBuffer;
|
||||
}
|
||||
halide_buffer_t& buffer() {
|
||||
return mBuffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief get dimension type.
|
||||
* @return dimension type.
|
||||
*/
|
||||
DimensionType getDimensionType() const;
|
||||
|
||||
/**
|
||||
* @brief handle data type. used when data type code is halide_type_handle.
|
||||
* @return handle data type.
|
||||
*/
|
||||
HandleDataType getHandleDataType() const;
|
||||
|
||||
/**
|
||||
* @brief set data type.
|
||||
* @param type data type defined in 'Type_generated.h'.
|
||||
*/
|
||||
void setType(int type);
|
||||
|
||||
/**
|
||||
* @brief get data type.
|
||||
* @return data type.
|
||||
*/
|
||||
inline halide_type_t getType() const {
|
||||
return mBuffer.type;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief visit host memory, data type is represented by `T`.
|
||||
* @return data point in `T` type.
|
||||
*/
|
||||
template <typename T>
|
||||
T* host() const {
|
||||
return (T*)mBuffer.host;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief visit device memory.
|
||||
* @return device data ID. what the ID means varies between backends.
|
||||
*/
|
||||
uint64_t deviceId() const {
|
||||
return mBuffer.device;
|
||||
}
|
||||
|
||||
public:
|
||||
int dimensions() const {
|
||||
return mBuffer.dimensions;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief get all dimensions' extent.
|
||||
* @return dimensions' extent.
|
||||
*/
|
||||
std::vector<int> shape() const;
|
||||
|
||||
/**
|
||||
* @brief calculate number of bytes needed to store data taking reordering flag into account.
|
||||
* @return bytes needed to store data
|
||||
*/
|
||||
int size() const;
|
||||
size_t usize() const;
|
||||
|
||||
/**
|
||||
* @brief calculate number of elements needed to store data taking reordering flag into account.
|
||||
* @return elements needed to store data
|
||||
*/
|
||||
inline int elementSize() const {
|
||||
return size() / mBuffer.type.bytes();
|
||||
}
|
||||
|
||||
public:
|
||||
inline int width() const {
|
||||
if (getDimensionType() == TENSORFLOW) {
|
||||
return mBuffer.dim[2].extent;
|
||||
}
|
||||
|
||||
return mBuffer.dim[3].extent;
|
||||
}
|
||||
inline int height() const {
|
||||
if (getDimensionType() == TENSORFLOW) {
|
||||
return mBuffer.dim[1].extent;
|
||||
}
|
||||
return mBuffer.dim[2].extent;
|
||||
}
|
||||
inline int channel() const {
|
||||
if (getDimensionType() == TENSORFLOW) {
|
||||
return mBuffer.dim[3].extent;
|
||||
}
|
||||
return mBuffer.dim[1].extent;
|
||||
}
|
||||
inline int batch() const {
|
||||
return mBuffer.dim[0].extent;
|
||||
}
|
||||
|
||||
// visit dimension's extent & stride
|
||||
inline int stride(int index) const {
|
||||
return mBuffer.dim[index].stride;
|
||||
}
|
||||
inline int length(int index) const {
|
||||
return mBuffer.dim[index].extent;
|
||||
}
|
||||
inline void setStride(int index, int stride) {
|
||||
mBuffer.dim[index].stride = stride;
|
||||
}
|
||||
inline void setLength(int index, int length) {
|
||||
mBuffer.dim[index].extent = length;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief For GPU and Other Device, get memory directly, see MNNSharedContext for detail
|
||||
* @return Success or not. If type != tensor's backend's type or type is cpu , return false
|
||||
*/
|
||||
bool getDeviceInfo(void* dst, int forwardType) const;
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief print tensor data. for DEBUG use only.
|
||||
*/
|
||||
void print() const;
|
||||
|
||||
/**
|
||||
*@brief print tensor shape
|
||||
*/
|
||||
void printShape() const;
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief map/umap GPU Tensor, to get host ptr
|
||||
*/
|
||||
void* map(MapType mtype, DimensionType dtype);
|
||||
void unmap(MapType mtype, DimensionType dtype, void* mapPtr);
|
||||
/**
|
||||
* @brief wait until the tensor is ready to read / write
|
||||
* @param mtype wait for read or write
|
||||
* @param finish wait for command flush or finish
|
||||
*/
|
||||
int wait(MapType mtype, bool finish);
|
||||
/**
|
||||
* @brief set GPU tensor device ptr, and inform memory type
|
||||
*/
|
||||
bool setDevicePtr(const void* devicePtr, int memoryType);
|
||||
private:
|
||||
halide_buffer_t mBuffer;
|
||||
struct InsideDescribe* mDescribe;
|
||||
|
||||
private:
|
||||
friend class TensorUtils;
|
||||
};
|
||||
} // namespace MNN
|
||||
|
||||
#endif /* Tensor_hpp */
|
||||
@@ -0,0 +1,172 @@
|
||||
//
|
||||
// Executor.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2019/07/25.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
#ifndef MNN_Executor_hpp
|
||||
#define MNN_Executor_hpp
|
||||
#include <MNN/ErrorCode.hpp>
|
||||
#include <MNN/expr/Expr.hpp>
|
||||
#include <MNN/Tensor.hpp>
|
||||
#include <MNN/Interpreter.hpp>
|
||||
#include <vector>
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <MNN/MNNForwardType.h>
|
||||
|
||||
namespace MNN {
|
||||
class Backend;
|
||||
class Execution;
|
||||
class Runtime;
|
||||
struct Op;
|
||||
namespace Express {
|
||||
struct RuntimeAttr;
|
||||
struct ExecutorAttr;
|
||||
class MNN_PUBLIC Executor {
|
||||
public:
|
||||
class ComputeCache;
|
||||
class RuntimeExecuteWrap;
|
||||
struct DebugTools;
|
||||
/**Internal Usage Begin*/
|
||||
struct Requirement {
|
||||
std::vector<bool> contentNeedContent;
|
||||
std::vector<bool> shapeNeedContent;
|
||||
};
|
||||
~Executor();
|
||||
Requirement getRequirement(Expr* expr) const;
|
||||
ErrorCode computeInfo(Expr* expr);
|
||||
void makeCache(const std::vector<EXPRP>& expr, bool forceCPU = false);
|
||||
/**Internal Usage End*/
|
||||
|
||||
bool lazyEval = true;
|
||||
enum LazyMode {
|
||||
LAZY_FULL = 0,
|
||||
// Don't compute content until user needed.
|
||||
LAZY_CONTENT = 1 << 0,
|
||||
|
||||
// Expr can only compute once, it can reduce the create cost of expr
|
||||
LAZY_COMPUTE_ONCE = 1 << 1,
|
||||
};
|
||||
uint32_t getLazyMode() const {
|
||||
return mLazyMode;
|
||||
}
|
||||
void setLazyComputeMode(uint32_t mode);
|
||||
void setGlobalExecutorConfig(MNNForwardType type, const BackendConfig& config, int numberThread);
|
||||
int getCurrentRuntimeStatus(RuntimeStatus statusEnum);
|
||||
// Get last GPU execution time in ms (measured by GPU timestamps).
|
||||
// Returns -1.0f if profiling is not supported or not enabled.
|
||||
float getLastGpuTimeMs() const;
|
||||
enum GCFlag {
|
||||
FULL,
|
||||
PART
|
||||
};
|
||||
void gc(GCFlag flag = FULL);
|
||||
static std::shared_ptr<Executor> getGlobalExecutor();
|
||||
|
||||
static std::shared_ptr<Executor> newExecutor(MNNForwardType type,
|
||||
const BackendConfig& config,
|
||||
int numberThread);
|
||||
void resetProfile();
|
||||
void dumpProfile();
|
||||
|
||||
struct SubGraph;
|
||||
bool registerSubGraph(const std::string& submoduleName, VARPS outputs, VARPS inputs);
|
||||
std::shared_ptr<SubGraph> findSubGraph(const std::string& submoduleName);
|
||||
static RuntimeInfo getRuntime();
|
||||
void setCallBack(TensorCallBackWithInfo&& before, TensorCallBackWithInfo&& after);
|
||||
const DebugTools* getDebugTools() const {
|
||||
return mDebug.get();
|
||||
}
|
||||
ExecutorAttr* getAttr() const;
|
||||
class MNN_PUBLIC RuntimeManager {
|
||||
public:
|
||||
~RuntimeManager();
|
||||
/**
|
||||
* @param configs : schedule configs.
|
||||
* @param cacheName : full path for cache file. Note: should choose location for reading and writing.
|
||||
*/
|
||||
static RuntimeManager* createRuntimeManager(const ScheduleConfig& config);
|
||||
|
||||
/**
|
||||
* @param rtmgr : the rtmgr to destroy
|
||||
*/
|
||||
static void destroy(RuntimeManager* rtmgr);
|
||||
|
||||
/**
|
||||
* Deceperate, the same as createRuntimeManager(configs[0])
|
||||
* @param configs : schedule configs.
|
||||
* @param cacheName : full path for cache file. Note: should choose location for reading and writing.
|
||||
*/
|
||||
static RuntimeManager* createRuntimeManager(std::vector<ScheduleConfig>& configs);
|
||||
|
||||
/**
|
||||
* @brief set cache file. when file not exist -- create it, when file exist -- load it.
|
||||
* When should use : When choose GPU backend or use AUTO backend.
|
||||
* Calling Position: calling after createRuntimeManager.
|
||||
*/
|
||||
void setCache(std::string cacheName);
|
||||
|
||||
/**
|
||||
* @brief set the path of external files or directory
|
||||
* @param path -- The path of a file or directory on disk
|
||||
* @param type -- Type of the external path (see "enum ExternalPathType" in Interpreter.hpp)
|
||||
*/
|
||||
void setExternalPath(std::string path, int type);
|
||||
|
||||
/**
|
||||
* @brief set external file.
|
||||
*/
|
||||
void setExternalFile(std::string fileName);
|
||||
|
||||
/**
|
||||
* @brief update cache file
|
||||
* When should use : Together with setCache API. calling for first inference and when input shape is changed.
|
||||
* Calling Position : calling after inference done.
|
||||
*/
|
||||
void updateCache();
|
||||
std::vector<bool> isBackendSupport(const std::vector<MNNForwardType> type);
|
||||
friend class Executor;
|
||||
void setMode(Interpreter::SessionMode mode);
|
||||
void setHint(Interpreter::HintMode mode, int value);
|
||||
void setHint(Interpreter::HintMode mode, int* value, size_t size);
|
||||
void setHintPtr(Interpreter::HintMode mode, void* value);
|
||||
// Keep Runtime::pMeta in sync for backend runtimes that still read it during onCreate.
|
||||
// Session also sets the meta on created Backends through the common backend creation path.
|
||||
void applyMetaToRuntime() const;
|
||||
bool getInfo(Interpreter::SessionInfoCode code, void* ptr);
|
||||
static bool getDeviceInfo(const std::string& deviceKey, const MNNForwardType type, std::string& deviceValue);
|
||||
BackendConfig* getBnConfig();
|
||||
const RuntimeAttr* getInside() const {
|
||||
return mInside;
|
||||
}
|
||||
private:
|
||||
std::mutex mLock;
|
||||
RuntimeAttr* mInside;
|
||||
friend class StaticModule;
|
||||
RuntimeManager();
|
||||
};
|
||||
static bool getComputeInfo(EXPRP expr, Interpreter::SessionInfoCode code, void* ptr);
|
||||
#ifndef MNN_REDUCE_SIZE
|
||||
std::map<std::string, std::shared_ptr<SubGraph>>& subgraph() {
|
||||
return mSubGraph;
|
||||
};
|
||||
#endif
|
||||
private:
|
||||
std::shared_ptr<Runtime> _getOrCreateRuntime(MNNForwardType type, const BackendConfig* config, int numberThread, bool reset = true);
|
||||
Executor(std::shared_ptr<Runtime> backend, MNNForwardType type, int numberThread);
|
||||
void _makeCache(const std::vector<EXPRP>& outputs, bool forceCPU);
|
||||
|
||||
RuntimeInfo mRuntimeInfo;
|
||||
std::shared_ptr<DebugTools> mDebug;
|
||||
#ifndef MNN_REDUCE_SIZE
|
||||
std::map<std::string, std::shared_ptr<SubGraph>> mSubGraph;
|
||||
#endif
|
||||
uint32_t mLazyMode = 0;
|
||||
std::shared_ptr<ExecutorAttr> mAttr;
|
||||
std::mutex mMutex;
|
||||
};
|
||||
} // namespace Express
|
||||
} // namespace MNN
|
||||
#endif
|
||||
@@ -0,0 +1,33 @@
|
||||
//
|
||||
// ExecutorScope.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2020/10/26.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_EXPR_EXECUTOR_SCOPE_HPP_
|
||||
#define MNN_EXPR_EXECUTOR_SCOPE_HPP_
|
||||
|
||||
#include <MNN/expr/Executor.hpp>
|
||||
|
||||
namespace MNN {
|
||||
namespace Express {
|
||||
|
||||
struct MNN_PUBLIC ExecutorScope final {
|
||||
public:
|
||||
ExecutorScope() = delete;
|
||||
explicit ExecutorScope(const ExecutorScope&) = delete;
|
||||
explicit ExecutorScope(const std::shared_ptr<Executor>& current);
|
||||
|
||||
explicit ExecutorScope(const std::string& scope_name,
|
||||
const std::shared_ptr<Executor>& current);
|
||||
|
||||
virtual ~ExecutorScope();
|
||||
|
||||
static const std::shared_ptr<Executor> Current();
|
||||
};
|
||||
|
||||
} // namespace MNN
|
||||
} // namespace Express
|
||||
#endif // MNN_EXPR_EXECUTOR_SCOPE_HPP_
|
||||
@@ -0,0 +1,282 @@
|
||||
//
|
||||
// Expr.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2019/06/10.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_Expr_hpp
|
||||
#define MNN_Expr_hpp
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <MNN/HalideRuntime.h>
|
||||
#include <MNN/MNNDefine.h>
|
||||
|
||||
namespace MNN {
|
||||
struct BufferStorage;
|
||||
struct OpT;
|
||||
struct Op;
|
||||
struct NetT;
|
||||
class Tensor;
|
||||
namespace Express {
|
||||
class Variable;
|
||||
class Expr;
|
||||
class Executor;
|
||||
typedef std::shared_ptr<Expr> EXPRP;
|
||||
typedef std::weak_ptr<Expr> WeakEXPRP;
|
||||
typedef std::vector<int> INTS;
|
||||
enum Dimensionformat { NHWC, NC4HW4, NCHW };
|
||||
class MNN_PUBLIC VARP {
|
||||
public:
|
||||
VARP() {
|
||||
// Do nothing
|
||||
}
|
||||
VARP(std::shared_ptr<Variable> c) {
|
||||
mContent = std::move(c);
|
||||
}
|
||||
VARP(Variable* c) {
|
||||
mContent.reset(c);
|
||||
}
|
||||
Variable* get() const {
|
||||
return mContent.get();
|
||||
}
|
||||
~ VARP() {
|
||||
// Do nothing
|
||||
}
|
||||
VARP(const VARP& var) {
|
||||
mContent = var.mContent;
|
||||
}
|
||||
VARP(VARP&& var) {
|
||||
mContent = std::move(var.mContent);
|
||||
}
|
||||
VARP operator+(VARP var) const;
|
||||
VARP operator-(VARP var) const;
|
||||
VARP operator*(VARP var) const;
|
||||
VARP operator/(VARP var) const;
|
||||
VARP mean(INTS dims) const;
|
||||
VARP sum(INTS dims) const;
|
||||
|
||||
bool operator==(const VARP& var) const {
|
||||
return var.mContent == mContent;
|
||||
}
|
||||
bool operator<(const VARP& var) const {
|
||||
return mContent < var.mContent;
|
||||
}
|
||||
bool operator<=(const VARP& var) const {
|
||||
return mContent <= var.mContent;
|
||||
}
|
||||
VARP& operator=(const VARP& var) {
|
||||
mContent = var.mContent;
|
||||
return *this;
|
||||
}
|
||||
VARP& operator=(Variable* var) {
|
||||
mContent.reset(var);
|
||||
return *this;
|
||||
}
|
||||
Variable* operator->() const {
|
||||
return mContent.get();
|
||||
}
|
||||
enum InputType {
|
||||
INPUT = 0,
|
||||
CONSTANT = 1,
|
||||
TRAINABLE = 2,
|
||||
};
|
||||
bool fix(InputType type) const;
|
||||
|
||||
// Only valid for Input / Const / Trainable
|
||||
void setOrder(Dimensionformat format);
|
||||
private:
|
||||
friend class Variable;
|
||||
std::shared_ptr<Variable> mContent;
|
||||
};
|
||||
inline bool operator==(Variable* src, VARP dst) {
|
||||
return src == dst.get();
|
||||
}
|
||||
inline bool operator!=(Variable* src, VARP dst) {
|
||||
return src != dst.get();
|
||||
}
|
||||
// inline bool operator<(VARP src, VARP dst) {
|
||||
// return src.get() < dst.get();
|
||||
// }
|
||||
typedef std::vector<VARP> VARPS;
|
||||
|
||||
class MNN_PUBLIC Variable {
|
||||
public:
|
||||
struct Info {
|
||||
Dimensionformat order = NHWC;
|
||||
INTS dim;
|
||||
halide_type_t type;
|
||||
size_t size;
|
||||
void syncSize();
|
||||
};
|
||||
const std::string& name() const;
|
||||
void setName(const std::string& name);
|
||||
bool setDevicePtr(const void* devicePtr, int memoryType);
|
||||
bool copyToDevicePtr(void* devicePtr, int memoryType);
|
||||
|
||||
std::pair<EXPRP, int> expr() const {
|
||||
return std::make_pair(mFrom, mFromIndex);
|
||||
}
|
||||
// If compute info error, return nullptr
|
||||
const Info* getInfo();
|
||||
bool resize(INTS dims);
|
||||
template <typename T>
|
||||
const T* readMap() {
|
||||
return (const T*)readInternal();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T* writeMap() {
|
||||
return (T*)writeInternal();
|
||||
}
|
||||
|
||||
void writeScaleMap(float scaleValue, float zeroPoint) {
|
||||
writeScaleInternal(scaleValue, zeroPoint);
|
||||
}
|
||||
|
||||
//Depecerate
|
||||
void unMap();
|
||||
|
||||
bool input(VARP src);
|
||||
static void replace(VARP dst, VARP src);
|
||||
|
||||
static VARP create(EXPRP expr, int index = 0);
|
||||
|
||||
static std::vector<VARP> load(const char* fileName);
|
||||
static std::map<std::string, VARP> loadMap(const char* fileName);
|
||||
static std::vector<VARP> load(const uint8_t* buffer, size_t length);
|
||||
static std::map<std::string, VARP> loadMap(const uint8_t* buffer, size_t length);
|
||||
static std::pair<std::map<std::string, VARP>, std::map<std::string, VARP>> getInputAndOutput(const std::map<std::string, VARP>& allVariable);
|
||||
static std::vector<VARP> mapToSequence(const std::map<std::string, VARP>& source);
|
||||
static std::vector<EXPRP> getExecuteOrder(const std::vector<VARP>& output);
|
||||
static void save(const std::vector<VARP>& vars, const char* fileName);
|
||||
static std::vector<int8_t> save(const std::vector<VARP>& vars);
|
||||
static void save(const std::vector<VARP>& vars, NetT* dest);
|
||||
|
||||
// Pack a few Variable to compute in one pipeline
|
||||
static void prepareCompute(const std::vector<VARP>& vars, bool forceCPU = false);
|
||||
static void compute(const std::vector<VARP>& vars, bool forceCPU = false);
|
||||
|
||||
size_t linkNumber() const;
|
||||
const std::vector<WeakEXPRP>& toExprs() const;
|
||||
void setExpr(EXPRP expr, int index) {
|
||||
mFrom = expr;
|
||||
mFromIndex = index;
|
||||
}
|
||||
|
||||
// Can't modify the tensor from this interface
|
||||
const Tensor* getTensor() const;
|
||||
private:
|
||||
Variable(EXPRP expr, int index) {
|
||||
mFrom = expr;
|
||||
mFromIndex = index;
|
||||
}
|
||||
|
||||
void* readInternal(bool forShape = false);
|
||||
void* writeInternal(bool inform=true);
|
||||
void informDirty();
|
||||
void writeScaleInternal(float scaleValue, float zeroPoint, bool inform = true);
|
||||
|
||||
friend class Expr;
|
||||
EXPRP mFrom;
|
||||
int mFromIndex;
|
||||
};
|
||||
class MNN_PUBLIC Expr {
|
||||
public:
|
||||
struct Inside;
|
||||
enum MemoryType {
|
||||
COPY,
|
||||
MOVE,
|
||||
REF
|
||||
};
|
||||
static EXPRP create(Tensor* tensor, bool own = false);
|
||||
|
||||
static EXPRP create(Variable::Info&& info, const void* ptr, VARP::InputType type, MemoryType copy = COPY);
|
||||
static EXPRP create(const OpT* op, std::vector<VARP> inputs, int outputSize = 1);
|
||||
static EXPRP create(std::shared_ptr<BufferStorage> extra, std::vector<VARP>&& inputs, int outputSize = 1);
|
||||
static EXPRP create(std::unique_ptr<OpT>&& op, std::vector<VARP> inputs, int outputSize = 1) {
|
||||
return create(op.get(), inputs, outputSize);
|
||||
}
|
||||
void setName(const std::string& name);
|
||||
|
||||
const Op* get() const {
|
||||
return mOp;
|
||||
}
|
||||
const std::vector<VARP>& inputs() const {
|
||||
return mInputs;
|
||||
}
|
||||
int outputSize() const {
|
||||
return (int)mOutputNames.size();
|
||||
}
|
||||
static void replace(EXPRP oldExpr, EXPRP newExpr);
|
||||
bool requireInfo();
|
||||
void visitOutputs(const std::function<bool(EXPRP, int)>& visit);
|
||||
static void visit(EXPRP expr, const std::function<bool(EXPRP)>& before, const std::function<bool(EXPRP)>& after);
|
||||
|
||||
const std::vector<WeakEXPRP>& outputs() const {
|
||||
return mTo;
|
||||
}
|
||||
~Expr();
|
||||
|
||||
const std::string& name() const {
|
||||
return mName;
|
||||
}
|
||||
const std::string& outputName(int index) {
|
||||
return mOutputNames[index];
|
||||
}
|
||||
|
||||
VARP::InputType inputType() const {return mType;}
|
||||
/** Internal Usage Begin */
|
||||
Variable::Info* outputInfo(int index) const;
|
||||
std::shared_ptr<BufferStorage> extra() const {
|
||||
return mStorage;
|
||||
}
|
||||
bool setInfoDirty();
|
||||
std::shared_ptr<Inside> inside() const {
|
||||
return mInside;
|
||||
}
|
||||
bool valid() const {
|
||||
return mValid;
|
||||
}
|
||||
bool visited() const {
|
||||
return mVisited;
|
||||
}
|
||||
void setVisited(bool visited) {
|
||||
mVisited = visited;
|
||||
}
|
||||
|
||||
/** Internal Usage End */
|
||||
|
||||
|
||||
private:
|
||||
static void _addLinkForInputs(EXPRP expr);
|
||||
|
||||
Expr(int outputSize);
|
||||
Expr(Tensor* tensor, bool own = false);
|
||||
|
||||
friend class Variable;
|
||||
friend class VARP;
|
||||
VARP::InputType mType;
|
||||
const Op* mOp;
|
||||
std::vector<VARP> mInputs;
|
||||
std::vector<std::string> mOutputNames;
|
||||
|
||||
bool mValid = true;
|
||||
std::shared_ptr<BufferStorage> mStorage;
|
||||
std::string mName;
|
||||
std::shared_ptr<Inside> mInside = nullptr;
|
||||
bool mVisited = false;
|
||||
std::vector<WeakEXPRP> mTo;
|
||||
bool mCanDecompose = true;
|
||||
friend class ExprModule;
|
||||
|
||||
};
|
||||
} // namespace Express
|
||||
} // namespace MNN
|
||||
|
||||
#endif /* Expr_hpp */
|
||||
@@ -0,0 +1,16 @@
|
||||
//
|
||||
// ExprCreator.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2019/06/27.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_ExprCreator_hpp
|
||||
#define MNN_ExprCreator_hpp
|
||||
|
||||
#include <MNN/expr/Expr.hpp>
|
||||
#include <MNN/expr/MathOp.hpp>
|
||||
#include <MNN/expr/NeuralNetWorkOp.hpp>
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,145 @@
|
||||
//
|
||||
// MathOp.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2019/06/27.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_MathOp_HPP
|
||||
#define MNN_MathOp_HPP
|
||||
|
||||
namespace MNN {
|
||||
namespace Express {
|
||||
//BinaryOPs
|
||||
MNN_PUBLIC VARP _Add(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _Subtract(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _Multiply(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _MulSilu(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _Divide(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _Pow(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _Minimum(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _Maximum(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _BiasAdd(VARP value, VARP bias);
|
||||
MNN_PUBLIC VARP _Greater(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _GreaterEqual(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _Less(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _FloorDiv(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _SquaredDifference(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _Equal(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _LessEqual(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _FloorMod(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _Atan2(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _LogicalOr(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _NotEqual(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _BitwiseAnd(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _BitwiseOr(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _BitwiseXor(VARP x, VARP y);
|
||||
|
||||
//UnaryOPs
|
||||
MNN_PUBLIC VARP _Sign(VARP a);
|
||||
MNN_PUBLIC VARP _Abs(VARP x);
|
||||
MNN_PUBLIC VARP _Negative(VARP x);
|
||||
MNN_PUBLIC VARP _Floor(VARP x);
|
||||
MNN_PUBLIC VARP _Round(VARP x);
|
||||
MNN_PUBLIC VARP _Ceil(VARP x);
|
||||
MNN_PUBLIC VARP _Square(VARP x);
|
||||
MNN_PUBLIC VARP _Sqrt(VARP x);
|
||||
MNN_PUBLIC VARP _Rsqrt(VARP x);
|
||||
MNN_PUBLIC VARP _Exp(VARP x);
|
||||
MNN_PUBLIC VARP _Log(VARP x);
|
||||
MNN_PUBLIC VARP _Sin(VARP x);
|
||||
MNN_PUBLIC VARP _Sinh(VARP x);
|
||||
MNN_PUBLIC VARP _Cos(VARP x);
|
||||
MNN_PUBLIC VARP _Cosh(VARP x);
|
||||
MNN_PUBLIC VARP _Tan(VARP x);
|
||||
MNN_PUBLIC VARP _Asin(VARP x);
|
||||
MNN_PUBLIC VARP _Asinh(VARP x);
|
||||
MNN_PUBLIC VARP _Acos(VARP x);
|
||||
MNN_PUBLIC VARP _Acosh(VARP x);
|
||||
MNN_PUBLIC VARP _Atan(VARP x);
|
||||
MNN_PUBLIC VARP _Atanh(VARP x);
|
||||
MNN_PUBLIC VARP _Reciprocal(VARP x);
|
||||
MNN_PUBLIC VARP _Log1p(VARP x);
|
||||
MNN_PUBLIC VARP _Gelu(VARP x);
|
||||
MNN_PUBLIC VARP _Tanh(VARP x);
|
||||
MNN_PUBLIC VARP _Sigmoid(VARP x);
|
||||
MNN_PUBLIC VARP _Erf(VARP x);
|
||||
MNN_PUBLIC VARP _Erfc(VARP x);
|
||||
MNN_PUBLIC VARP _Erfinv(VARP x);
|
||||
MNN_PUBLIC VARP _Expm1(VARP x);
|
||||
MNN_PUBLIC VARP _Hardswish(VARP x);
|
||||
MNN_PUBLIC VARP _Silu(VARP x);
|
||||
|
||||
//ReduceOPs
|
||||
MNN_PUBLIC VARP _ReduceSum(VARP input_variable, INTS axis = {}, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceMean(VARP input_variable, INTS axis = {}, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceMax(VARP input_variable, INTS axis = {}, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceMin(VARP input_variable, INTS axis = {}, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceProd(VARP input_variable, INTS axis = {}, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceAny(VARP input_variable, INTS axis = {}, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceAll(VARP input_variable, INTS axis = {}, bool keepDims = false);
|
||||
|
||||
MNN_PUBLIC VARP _ReduceSumMutable(VARP input_variable, VARP axis, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceMeanMutable(VARP input_variable, VARP axis, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceMaxMutable(VARP input_variable, VARP axis, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceMinMutable(VARP input_variable, VARP axis, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceProdMutable(VARP input_variable, VARP axis, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceAnyMutable(VARP input_variable, VARP axis, bool keepDims = false);
|
||||
MNN_PUBLIC VARP _ReduceAllMutable(VARP input_variable, VARP axis, bool keepDims = false);
|
||||
|
||||
//EltwiseOPs
|
||||
MNN_PUBLIC VARP _Prod(VARP a, VARP b, std::vector<float> coeff);
|
||||
MNN_PUBLIC VARP _Sum(VARP a, VARP b, std::vector<float> coeff);
|
||||
MNN_PUBLIC VARP _Max(VARP a, VARP b, std::vector<float> coeff);
|
||||
MNN_PUBLIC VARP _Sub(VARP a, VARP b, std::vector<float> coeff);
|
||||
MNN_PUBLIC VARP _EltwiseProdInt8(VARP x, VARP y,
|
||||
std::vector<int8_t> x_weight, std::vector<int32_t> x_bias, std::vector<float> x_scale, std::vector<float> x_tensorScale,
|
||||
std::vector<int8_t> y_weight, std::vector<int32_t> y_bias, std::vector<float> y_scale, std::vector<float> y_tensorScale,
|
||||
std::vector<int8_t> output_weight, std::vector<int32_t> output_bias, std::vector<float> output_scale, std::vector<float> output_tensorScale);
|
||||
MNN_PUBLIC VARP _EltwiseSumInt8(VARP x, VARP y,
|
||||
std::vector<int8_t> x_weight, std::vector<int32_t> x_bias, std::vector<float> x_scale, std::vector<float> x_tensorScale,
|
||||
std::vector<int8_t> y_weight, std::vector<int32_t> y_bias, std::vector<float> y_scale, std::vector<float> y_tensorScale,
|
||||
std::vector<int8_t> output_weight, std::vector<int32_t> output_bias, std::vector<float> output_scale, std::vector<float> output_tensorScale);
|
||||
MNN_PUBLIC VARP _EltwiseSubInt8(VARP x, VARP y,
|
||||
std::vector<int8_t> x_weight, std::vector<int32_t> x_bias, std::vector<float> x_scale, std::vector<float> x_tensorScale,
|
||||
std::vector<int8_t> y_weight, std::vector<int32_t> y_bias, std::vector<float> y_scale, std::vector<float> y_tensorScale,
|
||||
std::vector<int8_t> output_weight, std::vector<int32_t> output_bias, std::vector<float> output_scale, std::vector<float> output_tensorScale);
|
||||
MNN_PUBLIC VARP _EltwiseMaxInt8(VARP x, VARP y,
|
||||
std::vector<int8_t> x_weight, std::vector<int32_t> x_bias, std::vector<float> x_scale, std::vector<float> x_tensorScale,
|
||||
std::vector<int8_t> y_weight, std::vector<int32_t> y_bias, std::vector<float> y_scale, std::vector<float> y_tensorScale,
|
||||
std::vector<int8_t> output_weight, std::vector<int32_t> output_bias, std::vector<float> output_scale, std::vector<float> output_tensorScale);
|
||||
MNN_PUBLIC VARP _Mod(VARP x, VARP y);
|
||||
|
||||
|
||||
//OtherOPs
|
||||
template<typename T>
|
||||
VARP _Cast(VARP x) {
|
||||
return _Cast(x, halide_type_of<T>());
|
||||
}
|
||||
MNN_PUBLIC VARP _Cast(VARP x, halide_type_t dtype);
|
||||
MNN_PUBLIC VARP _MatMul(VARP a, VARP b, bool tranposeA = false, bool tranposeB = false);
|
||||
MNN_PUBLIC VARP _Normalize(VARP x, int32_t acrossSpatial, int32_t channelShared, float eps, std::vector<float> scale);
|
||||
MNN_PUBLIC VARP _ArgMax(VARP input, int axis = 0);
|
||||
MNN_PUBLIC VARP _ArgMin(VARP input, int axis = 0);
|
||||
MNN_PUBLIC VARP _BatchMatMul(VARP x, VARP y, bool adj_x = false, bool adj_y = false);
|
||||
MNN_PUBLIC VARP _UnravelIndex(VARP indices, VARP dims);
|
||||
MNN_PUBLIC VARP _ScatterNd(VARP indices, VARP updates, VARP shape);
|
||||
MNN_PUBLIC VARP _ScatterNd(VARP indices, VARP updates, VARP shape, VARP input);
|
||||
MNN_PUBLIC VARP _ScatterNd(VARP indices, VARP updates, VARP shape, int reduction);
|
||||
MNN_PUBLIC VARP _ScatterNd(VARP indices, VARP updates, VARP shape, VARP input, int reduction);
|
||||
MNN_PUBLIC VARP _ScatterElements(VARP data, VARP indices, VARP updates, int reduction = -1);
|
||||
MNN_PUBLIC VARP _ScatterElements(VARP data, VARP indices, VARP updates, VARP axis, int reduction = -1);
|
||||
MNN_PUBLIC VARP _OneHot(VARP indices, VARP depth, VARP onValue, VARP offValue, int axis = -1);
|
||||
MNN_PUBLIC VARP _BroadcastTo(VARP a, VARP shape);
|
||||
MNN_PUBLIC VARP _LinSpace(VARP start, VARP stop, VARP num);
|
||||
|
||||
MNN_PUBLIC VARP _RandomUnifom(VARP shape, halide_type_t dtype, float low = 0.0f, float high = 1.0f, int seed0 = 0, int seed1 = 0);
|
||||
MNN_PUBLIC VARP _CumSum(VARP x, int axis, bool exclusive = false, bool reverse = false);
|
||||
MNN_PUBLIC VARP _CumProd(VARP x, int axis);
|
||||
MNN_PUBLIC VARPS _Svd(VARP x);
|
||||
MNN_PUBLIC VARP _Histogram(VARP x, int bin, int min, int max, int channel = -1);
|
||||
}; // namespace Express
|
||||
}; // namespace MNN
|
||||
|
||||
#endif /* MathOp_HPP */
|
||||
@@ -0,0 +1,143 @@
|
||||
//
|
||||
// Module.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2019/11/25.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_Train_Module_hpp
|
||||
#define MNN_Train_Module_hpp
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include <MNN/expr/Expr.hpp>
|
||||
#include <MNN/expr/Executor.hpp>
|
||||
#include <MNN/MNNForwardType.h>
|
||||
|
||||
namespace MNN {
|
||||
class Session;
|
||||
namespace Express {
|
||||
struct SubGraph;
|
||||
class MNN_PUBLIC Module {
|
||||
public:
|
||||
Module() = default;
|
||||
virtual ~Module() = default;
|
||||
virtual std::vector<Express::VARP> onForward(const std::vector<Express::VARP>& inputs) = 0;
|
||||
Express::VARP forward(Express::VARP input);
|
||||
std::vector<Express::VARP> parameters() const;
|
||||
bool loadParameters(const std::vector<Express::VARP>& parameters);
|
||||
void setIsTraining(const bool isTraining);
|
||||
bool getIsTraining();
|
||||
void clearCache();
|
||||
const std::string& name() const {
|
||||
return mName;
|
||||
};
|
||||
void setName(std::string name) {
|
||||
mName = std::move(name);
|
||||
}
|
||||
const std::string type() const {
|
||||
return mType;
|
||||
}
|
||||
void setType(std::string type) {
|
||||
mType = std::move(type);
|
||||
}
|
||||
// Return the parameter index
|
||||
int addParameter(Express::VARP parameter);
|
||||
|
||||
void setParameter(Express::VARP parameter, int index);
|
||||
static Module* createEmpty(const std::vector<Express::VARP>& parameters);
|
||||
|
||||
struct BackendInfo {
|
||||
MNNForwardType type = MNN_FORWARD_CPU;
|
||||
BackendConfig* config = nullptr;
|
||||
};
|
||||
|
||||
struct Config {
|
||||
// Load module as dynamic, default static
|
||||
bool dynamic = false;
|
||||
|
||||
// for static mode, if the shape is mutable, set true, otherwise set false to avoid resizeSession freqencily
|
||||
bool shapeMutable = true;
|
||||
// Pre-rearrange weights or not. Disabled by default.
|
||||
// The weights will be rearranged in a general way, so the best implementation
|
||||
// may not be adopted if `rearrange` is enabled.
|
||||
bool rearrange = false;
|
||||
|
||||
BackendInfo* backend = nullptr;
|
||||
|
||||
// base module
|
||||
const Module* base = nullptr;
|
||||
};
|
||||
static Module* load(const std::vector<std::string>& inputs, const std::vector<std::string>& outputs, const uint8_t* buffer, size_t length, const Config* config = nullptr);
|
||||
static Module* load(const std::vector<std::string>& inputs, const std::vector<std::string>& outputs, const char* fileName, const Config* config = nullptr);
|
||||
// Shared RuntimeManager
|
||||
static Module* load(const std::vector<std::string>& inputs, const std::vector<std::string>& outputs, const char* fileName, const std::shared_ptr<MNN::Express::Executor::RuntimeManager> rtMgr, const Config* config = nullptr);
|
||||
static Module* load(const std::vector<std::string>& inputs, const std::vector<std::string>& outputs, const uint8_t* buffer, size_t length, const std::shared_ptr<MNN::Express::Executor::RuntimeManager> rtMgr, const Config* config = nullptr);
|
||||
|
||||
static Module* extract(std::vector<Express::VARP> inputs, std::vector<Express::VARP> outputs, bool fortrain, const std::map<std::string, SubGraph>& subGraph = {});
|
||||
|
||||
static Module* clone(const Module* module, const bool shareParams = false);
|
||||
|
||||
struct Info {
|
||||
// Input info load from model
|
||||
std::vector<Variable::Info> inputs;
|
||||
// The Module's defaultFormat, NCHW or NHWC
|
||||
Dimensionformat defaultFormat;
|
||||
// Runtime Info
|
||||
std::shared_ptr<MNN::Express::Executor::RuntimeManager> runTimeManager;
|
||||
// Input Names By Order
|
||||
std::vector<std::string> inputNames;
|
||||
// Output Names By Order
|
||||
std::vector<std::string> outputNames;
|
||||
// The MNNConvert's Version build the module
|
||||
std::string version;
|
||||
// The bizCode of MNN model
|
||||
std::string bizCode;
|
||||
// MetaData
|
||||
std::map<std::string, std::string> metaData;
|
||||
// uuid
|
||||
std::string uuid;
|
||||
};
|
||||
const Info* getInfo() const;
|
||||
class CloneContext {
|
||||
public:
|
||||
std::shared_ptr<Executor::RuntimeManager> pRuntimeManager;
|
||||
};
|
||||
virtual Module* clone(CloneContext* ctx) const {
|
||||
return nullptr;
|
||||
}
|
||||
void registerModel(const std::vector<std::shared_ptr<Module>>& children);
|
||||
|
||||
static void destroy(Module* m);
|
||||
|
||||
int traceOrOptimize(Interpreter::SessionMode stage);
|
||||
std::vector<std::shared_ptr<Module>> getChildren() const { return mChildren; }
|
||||
protected:
|
||||
virtual int onOptimize(Interpreter::SessionMode stage) {
|
||||
return 0;
|
||||
}
|
||||
virtual void onClearCache() {
|
||||
}
|
||||
|
||||
Module* cloneBaseTo(CloneContext* ctx, Module* module) const;
|
||||
|
||||
std::vector<std::shared_ptr<Module>> mChildren;
|
||||
std::vector<Express::VARP> mParameters;
|
||||
private:
|
||||
void _collectParameters(std::vector<Express::VARP>& result) const;
|
||||
bool mIsTraining = true;
|
||||
std::string mName;
|
||||
std::string mType;
|
||||
};
|
||||
|
||||
struct SubGraph {
|
||||
std::vector<std::string> inputs;
|
||||
std::vector<std::string> outputs;
|
||||
std::shared_ptr<Module> m;
|
||||
};
|
||||
|
||||
} // namespace Train
|
||||
} // namespace MNN
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,183 @@
|
||||
//
|
||||
// NeuralNetWorkOp.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2019/06/27.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_NeuralNetWorkOp_HPP
|
||||
#define MNN_NeuralNetWorkOp_HPP
|
||||
#include <MNN/ImageProcess.hpp>
|
||||
|
||||
namespace MNN {
|
||||
namespace Express {
|
||||
enum PaddingMode {CAFFE, VALID, SAME};
|
||||
enum PoolingMode {MAXPOOL, AVEPOOL};
|
||||
enum PadValueMode {CONSTANT, REFLECT, SYMMETRIC, EDGE};
|
||||
MNN_PUBLIC VARP _Input(INTS shape = {}, Dimensionformat data_format = NC4HW4, halide_type_t dtype = halide_type_of<float>()) ;
|
||||
MNN_PUBLIC VARP _Clone(VARP source, bool deepCopy = false);
|
||||
|
||||
MNN_PUBLIC VARP _Scalar(const void* ptr, halide_type_t type);
|
||||
|
||||
template <typename T>
|
||||
VARP _Scalar(T value) {
|
||||
return _Scalar(&value, halide_type_of<T>());
|
||||
}
|
||||
|
||||
|
||||
MNN_PUBLIC VARP _Const(float value, INTS shape = {}, Dimensionformat format = NHWC);
|
||||
MNN_PUBLIC VARP _Const(const void* ptr, INTS shape = {}, Dimensionformat format = NHWC,
|
||||
halide_type_t type = halide_type_of<float>());
|
||||
MNN_PUBLIC VARP _TrainableParam(float value, INTS dims, Dimensionformat format);
|
||||
MNN_PUBLIC VARP _TrainableParam(const void* ptr, INTS dims, Dimensionformat format,
|
||||
halide_type_t type = halide_type_of<float>());
|
||||
MNN_PUBLIC VARP _InnerProduct(std::vector<float>&& weight, std::vector<float>&& bias, VARP x, INTS outputShape);
|
||||
MNN_PUBLIC VARP _Conv(VARP weight, VARP bias, VARP x, PaddingMode pad = VALID, INTS stride = {1, 1},
|
||||
INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0});
|
||||
|
||||
MNN_PUBLIC VARP _Conv(float weight, float bias, VARP x, INTS channel, INTS kernelSize, PaddingMode pad = VALID,
|
||||
INTS stride = {1, 1}, INTS dilate = {1, 1}, int group = 1);
|
||||
MNN_PUBLIC VARP _Conv(std::vector<int8_t>&& weight, std::vector<float>&& bias, VARP x, INTS channel, INTS kernelSize,
|
||||
PaddingMode pad = VALID, INTS stride = {1, 1}, INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}, bool relu = false, bool relu6 = false, int nbits = 8);
|
||||
MNN_PUBLIC VARP _Conv(std::vector<float>&& weight, std::vector<float>&& bias, VARP x, INTS channel, INTS kernelSize,
|
||||
PaddingMode pad = VALID, INTS stride = {1, 1}, INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}, bool relu = false, bool relu6 = false);
|
||||
MNN_PUBLIC VARP _Deconv(VARP weight, VARP bias, VARP x, PaddingMode pad = VALID, INTS stride = {1, 1},
|
||||
INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0});
|
||||
|
||||
MNN_PUBLIC VARP _Deconv(std::vector<float>&& weight, std::vector<float>&& bias, VARP x, INTS channel, INTS kernelSize,
|
||||
PaddingMode pad, INTS stride = {1, 1}, INTS dilate = {1, 1}, int group = 1, INTS pads = {0, 0}, bool relu = false, bool relu6 = false);
|
||||
|
||||
MNN_PUBLIC VARP _MaxPool(VARP x, INTS kernel, INTS stride = {1, 1}, PaddingMode pad = VALID, INTS pads= {0, 0});
|
||||
MNN_PUBLIC VARP _AvePool(VARP x, INTS kernel, INTS stride = {1, 1}, PaddingMode pad = VALID, INTS pads= {0, 0});
|
||||
MNN_PUBLIC VARP _Reshape(VARP x, INTS shape, Dimensionformat original_format = NCHW);
|
||||
MNN_PUBLIC VARP _Reshape(VARP x, VARP shape);
|
||||
MNN_PUBLIC VARP _Scale(VARP x, int channels, std::vector<float>&& scales, std::vector<float>&& bias);
|
||||
|
||||
MNN_PUBLIC VARP _Relu(VARP x, float slope = 0.0f);
|
||||
MNN_PUBLIC VARP _Relu6(VARP x, float minValue = 0.0f, float maxValue = 6.0f);
|
||||
MNN_PUBLIC VARP _PRelu(VARP x, std::vector<float> &&slopes);
|
||||
MNN_PUBLIC VARP _Softmax(VARP logits, int axis = -1);
|
||||
MNN_PUBLIC VARP _Softplus(VARP features);
|
||||
MNN_PUBLIC VARP _Softsign(VARP features);
|
||||
MNN_PUBLIC std::vector<VARP> _Split(VARP value, INTS size_splits, int axis = 0);
|
||||
MNN_PUBLIC VARP _Slice(VARP x, VARP starts, VARP sizes);
|
||||
MNN_PUBLIC VARP _StridedSlice(VARP input, VARP begin, VARP end, VARP strided,
|
||||
int32_t beginMask, int32_t endMask, int32_t ellipsisMask,
|
||||
int32_t newAxisMask, int32_t shrinkAxisMask);
|
||||
MNN_PUBLIC VARP _StridedSliceWrite(VARP input, VARP begin, VARP end, VARP strided, VARP write,
|
||||
int32_t beginMask, int32_t endMask, int32_t ellipsisMask,
|
||||
int32_t newAxisMask, int32_t shrinkAxisMask);
|
||||
MNN_PUBLIC VARP _Concat(VARPS values, int axis);
|
||||
MNN_PUBLIC VARP _Convert(VARP input, Dimensionformat format);
|
||||
MNN_PUBLIC VARP _Transpose(VARP x, INTS perm);
|
||||
MNN_PUBLIC VARP _Transpose(VARP x, VARP perm);
|
||||
MNN_PUBLIC VARP _ChannelShuffle(VARP x, int group);
|
||||
MNN_PUBLIC VARP _ChangeInputFormat(VARP input, Dimensionformat format);
|
||||
|
||||
// FIXME: move the api to Array Ops
|
||||
MNN_PUBLIC VARP _Reverse(VARP x, VARP axis);
|
||||
MNN_PUBLIC VARP _ReverseSequence(VARP x, VARP y, int batchDim, int seqDim);
|
||||
// FIXME: move the api to Image Ops
|
||||
MNN_PUBLIC VARP _Crop(VARP images, VARP size, int axis, INTS offset);
|
||||
MNN_PUBLIC VARP _Resize(VARP images, float xScale, float yScale);
|
||||
MNN_PUBLIC VARP _Pad(VARP x, VARP paddings, PadValueMode mode = CONSTANT);
|
||||
MNN_PUBLIC VARP _ExpandDims(VARP input, int axis);
|
||||
MNN_PUBLIC VARP _ExpandDims(VARP input, VARP axis);
|
||||
|
||||
MNN_PUBLIC VARP _Shape(VARP input, bool nchw = false);
|
||||
MNN_PUBLIC VARP _Stack(VARPS values, int axis=0);
|
||||
enum InterpolationMethod {BILINEAR, NEAREST};
|
||||
MNN_PUBLIC VARP _CropAndResize(VARP image, VARP boxes, VARP box_ind, VARP crop_size,
|
||||
InterpolationMethod method, float extrapolation_value = 0.0);
|
||||
MNN_PUBLIC VARP _Fill(VARP dims, VARP value);
|
||||
MNN_PUBLIC VARP _Tile(VARP input, VARP multiples);
|
||||
MNN_PUBLIC VARP _Gather(VARP params, VARP indices);
|
||||
MNN_PUBLIC VARP _GatherV2(VARP params, VARP indices, VARP axis = nullptr);
|
||||
MNN_PUBLIC VARP _Squeeze(VARP input, INTS axis = {});
|
||||
MNN_PUBLIC VARP _Unsqueeze(VARP input, INTS axis = {});
|
||||
MNN_PUBLIC VARP _BatchToSpaceND(VARP input, VARP block_shape, VARP crops);
|
||||
MNN_PUBLIC VARP _GatherND(VARP params, VARP indices);
|
||||
MNN_PUBLIC VARP _GatherElements(VARP params, VARP indices);
|
||||
MNN_PUBLIC VARP _GatherElements(VARP params, VARP indices, VARP axis);
|
||||
MNN_PUBLIC VARP _Selu(VARP features, float scale, float alpha);
|
||||
MNN_PUBLIC VARP _Size(VARP input);
|
||||
MNN_PUBLIC VARP _Elu(VARP features, float alpha=1.0);
|
||||
MNN_PUBLIC VARP _Threshold(VARP features, float alpha=1.0);
|
||||
MNN_PUBLIC VARP _MatrixBandPart(VARP input, VARP num_lower, VARP num_upper);
|
||||
MNN_PUBLIC std::vector<VARP> _Moments(VARP x, INTS axis, VARP shift, bool keepDims);
|
||||
MNN_PUBLIC VARP _SetDiff1D(VARP x, VARP y);
|
||||
MNN_PUBLIC VARP _SpaceToDepth(VARP input, int block_size);
|
||||
MNN_PUBLIC VARP _SpaceToBatchND(VARP input, VARP block_shape, VARP paddings);
|
||||
MNN_PUBLIC VARP _ZerosLike(VARP input);
|
||||
MNN_PUBLIC std::vector<VARP> _Unstack(VARP value, int axis=0);
|
||||
MNN_PUBLIC VARP _Rank(VARP input);
|
||||
MNN_PUBLIC VARP _Range(VARP start, VARP limit, VARP delta);
|
||||
MNN_PUBLIC VARP _DepthToSpace(VARP input, int block_size);
|
||||
MNN_PUBLIC VARP _PriorBox(VARP feature, VARP image,
|
||||
std::vector<float> min_size, std::vector<float> max_size, std::vector<float>aspect_ratio,
|
||||
bool flip, bool clip, std::vector<float>variance,
|
||||
unsigned int img_h, unsigned int img_w, float step_h, float step_w, float offset = 0.5);
|
||||
MNN_PUBLIC VARP _Permute(VARP input, INTS dims);
|
||||
MNN_PUBLIC VARP _DetectionOutput(VARP location, VARP confidence, VARP priorbox,
|
||||
unsigned int num_classes, bool share_location, int background_label_id,
|
||||
float nms_threshhold, int nms_topk, int code_type,
|
||||
bool variance_encoded_in_target,
|
||||
int keep_top_k, float confidence_threshold, float visualize_threshold);
|
||||
MNN_PUBLIC std::vector<VARP> _DetectionPostProcess(VARP encode_boxes, VARP class_predictions, VARP anchors,
|
||||
int num_classes, int max_detections,
|
||||
int max_class_per_detection, int detections_per_class,
|
||||
float nms_threshold, float iou_threshold,
|
||||
bool use_regular_nms, std::vector<float> centersize_encoding);
|
||||
MNN_PUBLIC VARP _Interp(VARPS xs, float widthScale, float heightScale, int outputWidth, int outputHeight, int resizeType, bool alignCorners);
|
||||
|
||||
MNN_PUBLIC VARP _ZeroGrad(VARP x);
|
||||
|
||||
// Int8 Inference
|
||||
MNN_PUBLIC VARP _Conv(std::vector<int8_t>&& weight, std::vector<int>&& bias, std::vector<float>&& scale, VARP x, INTS channel, INTS kernelSize,
|
||||
PaddingMode pad, INTS stride, INTS dilate, int group, INTS pads, bool relu, int nbits = 8);
|
||||
MNN_PUBLIC VARP _Conv(std::vector<int8_t>&& weight, std::vector<int>&& bias, std::vector<float>&& scale,
|
||||
VARP x, INTS channel, INTS kernelSize,
|
||||
PaddingMode pad, INTS stride, INTS dilate, int group, INTS pads, bool relu,
|
||||
int8_t inputZeroPoint, int8_t outputZeroPoint,
|
||||
int8_t minValue, int8_t maxValue, bool accumulateToInt16);
|
||||
MNN_PUBLIC VARP _Conv(std::vector<int8_t>&& weight, std::vector<float>&& bias, std::vector<float>&& weightScale,
|
||||
VARP x, INTS channel, INTS kernelSize,
|
||||
PaddingMode pad, INTS stride, INTS dilate, int group, INTS pads, bool relu,
|
||||
float scaleIn, float scaleOut,
|
||||
int8_t inputZeroPoint, int8_t outputZeroPoint,
|
||||
int8_t minValue, int8_t maxValue, float weightClampValue, bool accumulateToInt16);
|
||||
MNN_PUBLIC VARP _CosineSimilarity(VARP input0, VARP input1, VARP inputDim);
|
||||
|
||||
enum GridSamplePaddingMode {GRID_SAMPLE_PADDING_ZEROS, GRID_SAMPLE_PADDING_BORDER, GRID_SAMPLE_PADDING_REFLECTION};
|
||||
MNN_PUBLIC VARP _GridSample(VARP input, VARP grid, InterpolationMethod mode=BILINEAR, GridSamplePaddingMode paddingMode=GRID_SAMPLE_PADDING_ZEROS, bool alignCorners=false);
|
||||
MNN_PUBLIC VARP _FloatToInt8(VARP x, VARP scale, char minValue, char maxValue);
|
||||
MNN_PUBLIC VARP _FloatToInt8(VARP x, VARP scale, int8_t minValue, int8_t maxValue, int8_t zeroPoint);
|
||||
MNN_PUBLIC VARP _Int8ToFloat(VARP x, VARP scale);
|
||||
MNN_PUBLIC VARP _Int8ToFloat(VARP x, VARP scale, int8_t zeroPoint);
|
||||
|
||||
MNN_PUBLIC VARP _Select(VARP select, VARP input0, VARP input1);
|
||||
MNN_PUBLIC std::vector<VARP> _TopKV2(VARP input0, VARP input1);
|
||||
MNN_PUBLIC VARP _ImageProcess(VARP input, CV::ImageProcess::Config config, CV::Matrix matrix, int oh, int ow, int oc, int dtype, uint8_t padVal = 0);
|
||||
MNN_PUBLIC VARP _Where(VARP x);
|
||||
MNN_PUBLIC VARP _Sort(VARP x, int axis = -1, bool arg = false, bool descend = false);
|
||||
MNN_PUBLIC VARP _Raster(const std::vector<VARP>& vars, const std::vector<int>& regions, const std::vector<int>& shape);
|
||||
MNN_PUBLIC VARP _RasterRaw(const std::vector<VARP>& vars, const std::vector<int>& region, const std::vector<int>& shape, halide_type_t dataType, Dimensionformat format);
|
||||
|
||||
MNN_PUBLIC VARP _Nms(VARP boxes, VARP scores, int maxDetections, float iouThreshold = -1, float scoreThreshold = -1);
|
||||
MNN_PUBLIC VARP _Im2Col(VARP x, INTS kernelSize, INTS dilate, INTS pads, INTS stride);
|
||||
MNN_PUBLIC VARP _Col2Im(VARP x, VARP outputShape, INTS kernelSize, INTS dilate, INTS pads, INTS stride);
|
||||
|
||||
/**
|
||||
Onnx's Loop
|
||||
*/
|
||||
MNN_PUBLIC VARPS _Loop(VARPS x, const std::string& submoduleName);
|
||||
MNN_PUBLIC VARP _ROIPooling(VARP input, VARP roi, int pooledHeight, int pooledWidth, float spatialScale, bool outputGrad = false, VARP backwardDiff = nullptr);
|
||||
MNN_PUBLIC VARP _ROIAlign(VARP input, VARP roi, int pooledHeight, int pooledWidth, float spatialScale, int samplingRatio, bool aligned, PoolingMode poolType, bool outputGrad = false, VARP backwardDiff = nullptr);
|
||||
|
||||
MNN_PUBLIC VARPS _JSONOp(VARPS x, const char* opDescribe, int outputNumber);
|
||||
|
||||
} // namespace Express
|
||||
} // namespace MNN
|
||||
|
||||
#endif /* NeuralNetWorkOp_HPP */
|
||||
@@ -0,0 +1,64 @@
|
||||
//
|
||||
// Optimizer.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2019/08/20.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
#ifndef Optimizer_hpp
|
||||
#define Optimizer_hpp
|
||||
#include <MNN/expr/Expr.hpp>
|
||||
#include <MNN/MNNForwardType.h>
|
||||
|
||||
namespace MNN {
|
||||
namespace Express {
|
||||
class MNN_PUBLIC Optimizer {
|
||||
public:
|
||||
enum Device {
|
||||
CPU = 0,
|
||||
GPU = 1,
|
||||
OTHER = 2,
|
||||
AUTO = 3
|
||||
};
|
||||
struct Config {
|
||||
Device device = CPU;
|
||||
MNNForwardType forwardType = MNN_FORWARD_ALL;
|
||||
int numThread = 4;
|
||||
};
|
||||
static std::shared_ptr<Optimizer> create(Config config);
|
||||
struct Cost {
|
||||
float compute; // MFlops
|
||||
float memory; // MB
|
||||
};
|
||||
class Parameters {
|
||||
public:
|
||||
Parameters(int n);
|
||||
virtual ~Parameters();
|
||||
|
||||
float* get() const {
|
||||
return mValue;
|
||||
}
|
||||
int size() const {
|
||||
return mSize;
|
||||
}
|
||||
|
||||
private:
|
||||
float* mValue;
|
||||
int mSize;
|
||||
};
|
||||
virtual std::shared_ptr<Parameters> onGetParameters(const std::vector<VARP>& outputs) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//Given paramters and measure cost, the parameters must be the same as onGetParameters
|
||||
virtual Cost onMeasure(const std::vector<VARP>& outputs, std::shared_ptr<Parameters> parameters = nullptr) = 0;
|
||||
|
||||
//Modify the output directly, the parameters must be the same as onGetParameters
|
||||
virtual bool onExecute(const std::vector<VARP>& outputs, std::shared_ptr<Parameters> parameters = nullptr) = 0;
|
||||
|
||||
Optimizer() = default;
|
||||
virtual ~Optimizer() = default;
|
||||
};
|
||||
} // namespace Express
|
||||
} // namespace MNN
|
||||
#endif
|
||||
@@ -0,0 +1,112 @@
|
||||
//
|
||||
// RuntimeScope.hpp
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2020/10/26.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_EXPR_SCOPE_HPP_
|
||||
#define MNN_EXPR_SCOPE_HPP_
|
||||
|
||||
#include <cstdio>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
|
||||
#include <MNN/Interpreter.hpp>
|
||||
|
||||
namespace MNN {
|
||||
namespace Express {
|
||||
|
||||
template <typename T>
|
||||
class Scope {
|
||||
public:
|
||||
Scope();
|
||||
virtual ~Scope() = default;
|
||||
|
||||
struct ScopedContent {
|
||||
std::string scope_name;
|
||||
T content;
|
||||
};
|
||||
void EnterScope(const ScopedContent& current);
|
||||
void EnterScope(const T& current);
|
||||
void EnterScope(const std::string& scope_name, const T& current);
|
||||
|
||||
void ExitScope();
|
||||
|
||||
const ScopedContent& Current() const;
|
||||
const T Content() const;
|
||||
|
||||
int ScopedLevel() const { return scoped_level_; }
|
||||
|
||||
private:
|
||||
std::string MakeScopeName(const std::string& prefix, int level) const;
|
||||
|
||||
mutable std::mutex mutex_;
|
||||
int scoped_level_ = 0;
|
||||
std::vector<ScopedContent> scoped_contents_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
Scope<T>::Scope() : scoped_level_(0) {
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void Scope<T>::EnterScope(const ScopedContent& current) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
++scoped_level_;
|
||||
scoped_contents_.push_back(current);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void Scope<T>::EnterScope(const T& current) {
|
||||
EnterScope("scope", current);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void Scope<T>::EnterScope(const std::string& scope_name,
|
||||
const T& current) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
int scoped_level = ScopedLevel();
|
||||
std::string name = MakeScopeName(scope_name, scoped_level++);
|
||||
ScopedContent content{name, current};
|
||||
++scoped_level_;
|
||||
scoped_contents_.push_back(content);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void Scope<T>::ExitScope() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
--scoped_level_;
|
||||
scoped_contents_.resize(scoped_level_);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
const typename Scope<T>::ScopedContent& Scope<T>::Current() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
MNN_CHECK(scoped_contents_.size() > 0, "Scope level should not be 0.");
|
||||
return scoped_contents_.back();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
const T Scope<T>::Content() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (scoped_contents_.empty()) {
|
||||
return nullptr;
|
||||
}
|
||||
return scoped_contents_.back().content;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::string Scope<T>::MakeScopeName(const std::string& prefix,
|
||||
int level) const {
|
||||
char s[16];
|
||||
snprintf(s, 16, "%d", level);
|
||||
return prefix + "/" + std::string(s);
|
||||
}
|
||||
|
||||
} // namespace Express
|
||||
} // namespace MNN
|
||||
|
||||
#endif // MNN_EXPR_SCOPE_HPP_
|
||||
@@ -0,0 +1,151 @@
|
||||
//
|
||||
// ShapeInference.h
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2020/04/05.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_PLUGIN_PLUGIN_CONTEXT_HPP_
|
||||
#define MNN_PLUGIN_PLUGIN_CONTEXT_HPP_
|
||||
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <MNN/Interpreter.hpp> // Backend
|
||||
#include <MNN/Tensor.hpp>
|
||||
#include "Tensor_generated.h"
|
||||
|
||||
namespace MNN {
|
||||
namespace plugin {
|
||||
|
||||
class MNN_PUBLIC PluginContext {
|
||||
public:
|
||||
PluginContext() = delete;
|
||||
PluginContext(const std::vector<Tensor*>& inputs, // NOLINT
|
||||
const std::vector<Tensor*>& outputs);
|
||||
void reset(const std::vector<Tensor*>& inputs, const std::vector<Tensor*>& outputs) {
|
||||
inputs_ = inputs;
|
||||
outputs_ = outputs;
|
||||
}
|
||||
|
||||
virtual ~PluginContext() = default;
|
||||
|
||||
const std::vector<Tensor*>& inputs() const {
|
||||
return inputs_;
|
||||
}
|
||||
const std::vector<Tensor*>& outputs() const {
|
||||
return outputs_;
|
||||
}
|
||||
|
||||
const Tensor* input(const int index) const;
|
||||
const Tensor* output(const int index) const;
|
||||
|
||||
Tensor* output(const int index);
|
||||
|
||||
bool hasAttr(const std::string& name) const;
|
||||
|
||||
bool setAttr(const std::string& name, const Attribute* attr);
|
||||
|
||||
void setAttrs(const std::unordered_map<std::string, // NOLINT
|
||||
const Attribute*>& attrs);
|
||||
|
||||
const Attribute* getAttr(const std::string& name) const;
|
||||
|
||||
const std::unordered_map<std::string, const Attribute*>& getAttrs() const;
|
||||
|
||||
protected:
|
||||
std::vector<Tensor*> inputs_;
|
||||
std::vector<Tensor*> outputs_;
|
||||
std::unordered_map<std::string, const Attribute*> attrs_;
|
||||
};
|
||||
|
||||
class MNN_PUBLIC InferShapeContext : public PluginContext {
|
||||
public:
|
||||
InferShapeContext() = delete;
|
||||
InferShapeContext(const std::vector<Tensor*>& inputs, // NOLINT
|
||||
const std::vector<Tensor*>& outputs);
|
||||
|
||||
virtual ~InferShapeContext() = default;
|
||||
};
|
||||
|
||||
class MNN_PUBLIC CPUKernelContext : public PluginContext {
|
||||
public:
|
||||
CPUKernelContext() = delete;
|
||||
CPUKernelContext(const std::string& op_type, // NOLINT
|
||||
Backend* backend, // NOLINT
|
||||
const std::vector<Tensor*>& inputs, // NOLINT
|
||||
const std::vector<Tensor*>& outputs,
|
||||
const std::string& dir_path);
|
||||
|
||||
virtual ~CPUKernelContext() = default;
|
||||
|
||||
Backend* backend() const {
|
||||
return backend_;
|
||||
}
|
||||
|
||||
const std::string& op_type() const {
|
||||
return op_type_;
|
||||
}
|
||||
|
||||
const std::string& dir_path() const {
|
||||
return dir_path_;
|
||||
}
|
||||
|
||||
private:
|
||||
const std::string op_type_ = "";
|
||||
const std::string dir_path_;
|
||||
Backend* backend_ = nullptr;
|
||||
};
|
||||
|
||||
inline PluginContext::PluginContext(const std::vector<Tensor*>& inputs, // NOLINT
|
||||
const std::vector<Tensor*>& outputs) // NOLINT
|
||||
: inputs_(inputs), outputs_(outputs) {
|
||||
}
|
||||
|
||||
inline const Tensor* PluginContext::input(const int index) const {
|
||||
MNN_ASSERT(index < inputs_.size());
|
||||
return inputs_.at(index);
|
||||
}
|
||||
|
||||
inline const Tensor* PluginContext::output(const int index) const {
|
||||
MNN_ASSERT(index < outputs_.size());
|
||||
return outputs_.at(index);
|
||||
}
|
||||
|
||||
inline Tensor* PluginContext::output(const int index) {
|
||||
MNN_ASSERT(index < outputs_.size());
|
||||
return outputs_.at(index);
|
||||
}
|
||||
|
||||
inline bool PluginContext::hasAttr(const std::string& name) const {
|
||||
return attrs_.count(name) > 0;
|
||||
}
|
||||
|
||||
inline bool PluginContext::setAttr(const std::string& name, // NOLINT
|
||||
const Attribute* attr) {
|
||||
return attrs_.emplace(name, attr).second;
|
||||
}
|
||||
|
||||
inline void PluginContext::setAttrs( // NOLINT
|
||||
const std::unordered_map<std::string, const Attribute*>& attrs) {
|
||||
attrs_ = attrs;
|
||||
}
|
||||
|
||||
inline const Attribute* PluginContext::getAttr(const std::string& name) const {
|
||||
const auto& it = attrs_.find(name);
|
||||
if (it == attrs_.end()) {
|
||||
return nullptr;
|
||||
}
|
||||
return it->second;
|
||||
}
|
||||
|
||||
inline const std::unordered_map<std::string, const Attribute*>& // NOLINT
|
||||
PluginContext::getAttrs() const {
|
||||
return attrs_;
|
||||
}
|
||||
|
||||
} // namespace plugin
|
||||
} // namespace MNN
|
||||
|
||||
#endif // MNN_PLUGIN_PLUGIN_CONTEXT_HPP_
|
||||
@@ -0,0 +1,77 @@
|
||||
//
|
||||
// ShapeInference.h
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2020/04/05.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_PLUGIN_PLUGIN_KERNEL_HPP_
|
||||
#define MNN_PLUGIN_PLUGIN_KERNEL_HPP_
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <MNN/plugin/PluginContext.hpp>
|
||||
|
||||
namespace MNN {
|
||||
namespace plugin {
|
||||
|
||||
template <typename KernelContextT>
|
||||
class MNN_PUBLIC ComputeKernel {
|
||||
public:
|
||||
ComputeKernel() = default;
|
||||
virtual ~ComputeKernel() = default;
|
||||
virtual bool compute(KernelContextT* ctx) = 0;
|
||||
};
|
||||
|
||||
class MNN_PUBLIC CPUComputeKernel : public ComputeKernel<CPUKernelContext> {
|
||||
public:
|
||||
using ContextT = CPUKernelContext;
|
||||
using KernelT = CPUComputeKernel;
|
||||
|
||||
CPUComputeKernel() = default;
|
||||
virtual ~CPUComputeKernel() = default;
|
||||
virtual bool init(CPUKernelContext* ctx) = 0;
|
||||
virtual bool compute(CPUKernelContext* ctx) = 0;
|
||||
virtual bool resize(CPUKernelContext* ctx) {
|
||||
return true;
|
||||
}
|
||||
inline bool needAllocIO() const {
|
||||
return mNeedAllocIO;
|
||||
}
|
||||
protected:
|
||||
bool mNeedAllocIO = true;
|
||||
};
|
||||
|
||||
template <typename PluginKernelT>
|
||||
class MNN_PUBLIC ComputeKernelRegistry {
|
||||
public:
|
||||
typedef std::function<PluginKernelT*()> Factory;
|
||||
static std::unordered_map<std::string, Factory>* getFactoryMap();
|
||||
|
||||
static bool add(const std::string& name, Factory factory);
|
||||
|
||||
static PluginKernelT* get(const std::string& name);
|
||||
};
|
||||
|
||||
template <typename PluginKernelT>
|
||||
struct ComputeKernelRegistrar {
|
||||
ComputeKernelRegistrar(const std::string& name) {
|
||||
ComputeKernelRegistry<typename PluginKernelT::KernelT>::add(name, []() { // NOLINT
|
||||
return new PluginKernelT; // NOLINT
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
#define REGISTER_PLUGIN_COMPUTE_KERNEL(name, computeKernel) \
|
||||
namespace { \
|
||||
static auto _plugin_compute_kernel_##name##_ __attribute__((unused)) = \
|
||||
ComputeKernelRegistrar<computeKernel>(#name); \
|
||||
} // namespace
|
||||
|
||||
} // namespace plugin
|
||||
} // namespace MNN
|
||||
|
||||
#endif // MNN_PLUGIN_PLUGIN_KERNEL_HPP_
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
//
|
||||
// ShapeInference.h
|
||||
// MNN
|
||||
//
|
||||
// Created by MNN on 2020/04/05.
|
||||
// Copyright © 2018, Alibaba Group Holding Limited
|
||||
//
|
||||
|
||||
#ifndef MNN_PLUGIN_PLUGIN_SHAPE_INFERENCE_HPP_
|
||||
#define MNN_PLUGIN_PLUGIN_SHAPE_INFERENCE_HPP_
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <MNN/plugin/PluginContext.hpp>
|
||||
|
||||
namespace MNN {
|
||||
namespace plugin {
|
||||
|
||||
class MNN_PUBLIC InferShapeKernel {
|
||||
public:
|
||||
virtual ~InferShapeKernel() = default;
|
||||
virtual bool compute(InferShapeContext* ctx) = 0;
|
||||
};
|
||||
|
||||
class MNN_PUBLIC InferShapeKernelRegister {
|
||||
public:
|
||||
// typedef InferShapeKernel* (*Factory)();
|
||||
typedef std::function<InferShapeKernel*()> Factory;
|
||||
static std::unordered_map<std::string, Factory>* getFactoryMap();
|
||||
|
||||
static bool add(const std::string& name, Factory factory);
|
||||
|
||||
static InferShapeKernel* get(const std::string& name);
|
||||
};
|
||||
|
||||
template <typename PluginKernel>
|
||||
struct InferShapeKernelRegistrar {
|
||||
InferShapeKernelRegistrar(const std::string& name) {
|
||||
InferShapeKernelRegister::add(name, []() { // NOLINT
|
||||
return new PluginKernel; // NOLINT
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
#define REGISTER_PLUGIN_OP(name, inferShapeKernel) \
|
||||
namespace { \
|
||||
static auto _plugin_infer_shape_##name##_ __attribute__((unused)) = \
|
||||
InferShapeKernelRegistrar<inferShapeKernel>(#name); \
|
||||
} // namespace
|
||||
|
||||
} // namespace plugin
|
||||
} // namespace MNN
|
||||
|
||||
#endif // MNN_PLUGIN_PLUGIN_SHAPE_INFERENCE_HPP_
|
||||
@@ -0,0 +1,281 @@
|
||||
//
|
||||
// llm.hpp
|
||||
//
|
||||
// Created by MNN on 2023/08/25.
|
||||
// ZhaodeWang
|
||||
//
|
||||
|
||||
#ifndef LLM_hpp
|
||||
#define LLM_hpp
|
||||
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <iostream>
|
||||
#include <streambuf>
|
||||
#include <functional>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <MNN/expr/Expr.hpp>
|
||||
#include <MNN/expr/Module.hpp>
|
||||
#include <MNN/expr/MathOp.hpp>
|
||||
#include <MNN/expr/NeuralNetWorkOp.hpp>
|
||||
#include <MNN/expr/ExecutorScope.hpp>
|
||||
|
||||
namespace MNN {
|
||||
struct KVMeta;
|
||||
namespace Transformer {
|
||||
using MNN::KVMeta;
|
||||
|
||||
// ChatMessage: pair<role, content> for multi-turn conversation.
|
||||
// first = role: "system", "user", "assistant", "tool", etc.
|
||||
// second = content: plain text message content.
|
||||
// For complex messages (tool_calls, reasoning_content, etc.):
|
||||
// first = "json"
|
||||
// second = full JSON object string, e.g. {"role":"assistant","content":"","tool_calls":[...]}
|
||||
using ChatMessage = std::pair<std::string, std::string>;
|
||||
using ChatMessages = std::vector<ChatMessage>;
|
||||
class Tokenizer;
|
||||
class Pipeline;
|
||||
class LlmConfig;
|
||||
class DiskEmbedding;
|
||||
class Sampler;
|
||||
class Generation;
|
||||
class EagleGeneration;
|
||||
struct TimePerformance;
|
||||
#define CHECK_LLM_RUNNING_RET(ctx, ret) \
|
||||
{ \
|
||||
if ((ctx)->status == LlmStatus::NOT_LOADED || \
|
||||
(ctx)->status == LlmStatus::INTERNAL_ERROR || \
|
||||
(ctx)->status == LlmStatus::TIMEOUT || \
|
||||
(ctx)->status == LlmStatus::USER_CANCEL) { \
|
||||
MNN_ERROR("[Error]: LLM in error state. Status: %d\n", \
|
||||
static_cast<int>((ctx)->status)); \
|
||||
return (ret); \
|
||||
} \
|
||||
}
|
||||
#define CHECK_LLM_RUNNING(ctx) \
|
||||
{ \
|
||||
if ((ctx)->status == LlmStatus::NOT_LOADED || \
|
||||
(ctx)->status == LlmStatus::INTERNAL_ERROR || \
|
||||
(ctx)->status == LlmStatus::TIMEOUT || \
|
||||
(ctx)->status == LlmStatus::USER_CANCEL) { \
|
||||
MNN_ERROR("[Error]: LLM in error state. Status: %d\n", \
|
||||
static_cast<int>((ctx)->status)); \
|
||||
return; \
|
||||
} \
|
||||
}
|
||||
|
||||
struct MNN_PUBLIC PromptImagePart {
|
||||
MNN::Express::VARP image_data;
|
||||
int width;
|
||||
int height;
|
||||
};
|
||||
|
||||
struct MNN_PUBLIC PromptAudioPart {
|
||||
std::string file_path;
|
||||
MNN::Express::VARP waveform;
|
||||
};
|
||||
|
||||
struct MNN_PUBLIC MultimodalPrompt {
|
||||
std::string prompt_template;
|
||||
std::map<std::string, PromptImagePart> images;
|
||||
std::map<std::string, PromptAudioPart> audios;
|
||||
};
|
||||
|
||||
enum TuneType {
|
||||
// op encoder number for commit
|
||||
OP_ENCODER_NUMBER = 0,
|
||||
};
|
||||
enum class LlmStatus {
|
||||
NOT_LOADED = -1,
|
||||
RUNNING = 0,
|
||||
NORMAL_FINISHED = 1,
|
||||
MAX_TOKENS_FINISHED = 2,
|
||||
USER_CANCEL = 3,
|
||||
INTERNAL_ERROR = 4,
|
||||
TIMEOUT = 5,
|
||||
};
|
||||
enum class MatchStrictLevel : int;
|
||||
enum class NgramSelectRule : int;
|
||||
|
||||
struct LlmContext {
|
||||
// forward
|
||||
int prompt_len = 0;
|
||||
int gen_seq_len = 0;
|
||||
int all_seq_len = 0;
|
||||
std::ostream* os = nullptr;
|
||||
std::string end_with;
|
||||
// perf
|
||||
int64_t load_us = 0;
|
||||
int64_t vision_us = 0;
|
||||
int64_t audio_us = 0;
|
||||
int64_t prefill_us = 0;
|
||||
int64_t decode_us = 0;
|
||||
int64_t sample_us = 0;
|
||||
int64_t ttfa_us = 0;
|
||||
float pixels_mp = 0;
|
||||
float audio_input_s = 0;
|
||||
// tokens
|
||||
int current_token;
|
||||
std::vector<int> history_tokens;
|
||||
std::vector<int> output_tokens;
|
||||
std::string generate_str;
|
||||
// llm status
|
||||
LlmStatus status = LlmStatus::NOT_LOADED;
|
||||
// log buffer (per-instance, no locking needed)
|
||||
std::string log_buffer;
|
||||
};
|
||||
struct GenerationParams;
|
||||
class MNN_PUBLIC Llm {
|
||||
public:
|
||||
enum Stage {
|
||||
Prefill,
|
||||
Decode
|
||||
};
|
||||
// Log buffer interface: retrieve accumulated log and clear the buffer.
|
||||
// Only effective when LLM_LOG_TO_STRING macro is enabled during compilation.
|
||||
std::string getLog();
|
||||
static Llm* createLLM(const std::string& config_path);
|
||||
static void destroy(Llm* llm);// For Windows RT mode should use destroy
|
||||
Llm(std::shared_ptr<LlmConfig> config);
|
||||
virtual ~Llm();
|
||||
virtual bool load();
|
||||
virtual Express::VARP gen_attention_mask(int seq_len);
|
||||
virtual Express::VARP gen_position_ids(int seq_len);
|
||||
virtual Express::VARP embedding(const std::vector<int>& input_ids);
|
||||
virtual int sample(Express::VARP logits, int offset = 0, int size = 0);
|
||||
std::vector<Express::VARP> getOutputs() const;
|
||||
int getOutputIndex(const std::string& name) const;
|
||||
void reset();
|
||||
void tuning(TuneType type, std::vector<int> candidates);
|
||||
virtual std::vector<Express::VARP> forwardRaw(Express::VARP hiddenState, Express::VARP mask, Express::VARP inputPos, Express::VARPS extraArgs = {});
|
||||
Express::VARP forward(const std::vector<int>& input_ids, bool is_prefill = true);
|
||||
Express::VARP forward(MNN::Express::VARP input_embeds);
|
||||
void switchMode(Stage stage);
|
||||
void setKVCacheInfo(size_t add, size_t remove, int* reserve = nullptr, int n_reserve = 0);
|
||||
size_t getCurrentHistory() const;
|
||||
void eraseHistory(size_t begin, size_t end);
|
||||
bool setPrefixCacheFile(const std::string& filename, int flag = 0);
|
||||
virtual void response(const std::vector<int>& input_ids, std::ostream* os = &std::cout, const char* end_with = nullptr, int max_new_tokens = -1);
|
||||
void response(const std::string& user_content, std::ostream* os = &std::cout, const char* end_with = nullptr, int max_new_tokens = -1);
|
||||
void response(const ChatMessages& chat_prompts, std::ostream* os = &std::cout, const char* end_with = nullptr, int max_new_tokens = -1);
|
||||
void response(MNN::Express::VARP input_embeds, std::ostream* os = &std::cout, const char* end_with = nullptr, int max_new_tokens = -1);
|
||||
virtual void generate_init(std::ostream* os = nullptr, const char* end_with = nullptr);
|
||||
void generate(int max_token);
|
||||
std::vector<int> generate(const std::vector<int>& input_ids, int max_new_tokens = -1);
|
||||
std::vector<int> generate(MNN::Express::VARP input_embeds, int max_tokens = -1);
|
||||
bool stoped();
|
||||
bool reuse_kv();
|
||||
// Prompt cache: call after decode completes to sync the cached text with the
|
||||
// full conversation (including assistant response). Optional — the cache
|
||||
// self-updates after generate(), but this allows callers with post-processed
|
||||
// response text (e.g. deleteThinkPart) to provide a more accurate version.
|
||||
void syncPromptCache(const ChatMessages& chat_prompts);
|
||||
// config function
|
||||
std::string dump_config();
|
||||
bool set_config(const std::string& content);
|
||||
void setDebugCallback(MNN::TensorCallBackWithInfo&& before, MNN::TensorCallBackWithInfo&& after);
|
||||
Llm* create_lora(const std::string& lora_path);
|
||||
// tokenier function
|
||||
bool is_stop(int token);
|
||||
std::string tokenizer_decode(int token);
|
||||
virtual std::vector<int> tokenizer_encode(const std::string& query);
|
||||
friend class Pipeline;
|
||||
virtual std::vector<int> tokenizer_encode(const MultimodalPrompt& multimodal_input);
|
||||
// ptompt functions
|
||||
std::string apply_chat_template(const std::string& user_content) const;
|
||||
std::string apply_chat_template(const ChatMessages& chat_prompts) const;
|
||||
void response(const MultimodalPrompt& multimodal_input,
|
||||
std::ostream* os = &std::cout,
|
||||
const char* end_with = nullptr,
|
||||
int max_new_tokens = -1);
|
||||
const LlmContext* getContext() const {
|
||||
return mContext.get();
|
||||
}
|
||||
virtual void setWavformCallback(std::function<bool(const float*, size_t, bool)> callback) {}
|
||||
virtual void generateWavform() {}
|
||||
protected:
|
||||
void setChatTemplate();
|
||||
void initRuntime();
|
||||
void setRuntimeHint(std::shared_ptr<Express::Executor::RuntimeManager> &rtg, bool mllm = false);
|
||||
std::shared_ptr<LlmContext> mContext;
|
||||
std::shared_ptr<KVMeta> mMeta;
|
||||
std::shared_ptr<LlmConfig> mConfig;
|
||||
std::shared_ptr<Tokenizer> mTokenizer;
|
||||
std::shared_ptr<DiskEmbedding> mDiskEmbedding;
|
||||
std::shared_ptr<DiskEmbedding> mPleEmbedding;
|
||||
Express::VARP mPleInput; // PLE embeddings for current input
|
||||
Express::VARP mTextEmbedsForPle; // Pure text embeddings for PLE projection
|
||||
std::shared_ptr<Sampler> mSampler;
|
||||
std::shared_ptr<Express::Executor::RuntimeManager> mRuntimeManager, mProcessorRuntimeManager;
|
||||
std::shared_ptr<Express::Module> mModule;
|
||||
/**
|
||||
key: <seq_len, all_logists>
|
||||
value : module
|
||||
note: prefill share one module, seq_len = 100 for example
|
||||
*/
|
||||
const int mPrefillKey = 100;
|
||||
std::map<std::pair<int, bool>, std::shared_ptr<Express::Module>> mModulePool;
|
||||
const Express::Module* mBaseModule = nullptr;
|
||||
Express::VARP inputsEmbeds, attentionMask, positionIds;
|
||||
std::vector<Express::VARP> mAttentionMaskVarVec, mPositionIdsVarVec;
|
||||
Express::VARP logitsAllIdx, logitsLastIdx;
|
||||
int mSeqLenIndex = 0;
|
||||
std::shared_ptr<MNN::Express::Executor> mExecutor;
|
||||
protected:
|
||||
friend class ArGeneration;
|
||||
friend class LookaheadGeneration;
|
||||
friend class MtpGeneration;
|
||||
friend class EagleGeneration;
|
||||
friend class DFlashGeneration;
|
||||
friend class Omni;
|
||||
std::vector<Express::VARP> forwardVec(const std::vector<int>& input_ids);
|
||||
std::vector<Express::VARP> forwardVec(MNN::Express::VARP input_embeds);
|
||||
private:
|
||||
std::shared_ptr<Generation> mGenerationStrategy;
|
||||
void setSpeculativeConfig();
|
||||
void updateContext(int seq_len, int gen_len);
|
||||
bool checkFile(const std::string& path, const char* name);
|
||||
|
||||
private:
|
||||
bool mInSpec = false;
|
||||
int mDraftLength = 4;
|
||||
std::shared_ptr<GenerationParams> mGenerateParam;
|
||||
bool mAsync = true;
|
||||
int mBlockSize = 0;
|
||||
std::vector<int> mValidBlockSize;
|
||||
bool mPrefixCacheMode = false;
|
||||
std::string mPrefixCacheFileName;
|
||||
int mCallIndex;
|
||||
int mPrefixLength;
|
||||
bool mIsPrefixFileExist = false;
|
||||
void completePrefixWrite();
|
||||
// Prompt cache state
|
||||
std::string mCachedPromptText;
|
||||
void updateCachedPromptText(const ChatMessages& chat_prompts, size_t history_before);
|
||||
};
|
||||
|
||||
// Embedding start
|
||||
class MNN_PUBLIC Embedding : public Llm {
|
||||
public:
|
||||
Embedding(std::shared_ptr<LlmConfig> config);
|
||||
static Embedding* createEmbedding(const std::string& config_path, bool load = true);
|
||||
static float dist(Express::VARP var0, Express::VARP var1);
|
||||
static float cos_sim(Express::VARP var0, Express::VARP var1);
|
||||
virtual bool load() override;
|
||||
|
||||
Express::VARP ids_embedding(const std::vector<int>& ids);
|
||||
Express::VARP txt_embedding(const std::string& txt);
|
||||
std::vector<Express::VARP> forwardRaw(Express::VARP hiddenState, Express::VARP mask, Express::VARP inputPos, Express::VARPS extraArgs = {}) override;
|
||||
int dim() const;
|
||||
virtual Express::VARP gen_attention_mask(int seq_len) override;
|
||||
virtual Express::VARP gen_position_ids(int seq_len) override;
|
||||
};
|
||||
// Embedding end
|
||||
}
|
||||
}
|
||||
|
||||
#endif // LLM_hpp
|
||||
@@ -0,0 +1,307 @@
|
||||
//
|
||||
// reranker.hpp
|
||||
//
|
||||
// Created by MNN on 2025/07/29.
|
||||
// ZhaodeWang
|
||||
//
|
||||
|
||||
#ifndef MNN_LLM_RERANKER_hpp
|
||||
#define MNN_LLM_RERANKER_hpp
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include "llm.hpp"
|
||||
|
||||
namespace MNN {
|
||||
namespace Transformer {
|
||||
|
||||
using namespace MNN::Express;
|
||||
|
||||
/**
|
||||
* @brief Abstract base class for Rerankers.
|
||||
* This class defines the interface for any reranker implementation.
|
||||
*/
|
||||
class RerankerBase {
|
||||
public:
|
||||
/**
|
||||
* @brief Virtual destructor to ensure proper cleanup of derived classes.
|
||||
*/
|
||||
virtual ~RerankerBase() = default;
|
||||
|
||||
/**
|
||||
* @brief Initializes the reranker with a configuration path.
|
||||
* @param config_path The path to the configuration file.
|
||||
*/
|
||||
virtual void initialize(const std::string& config_path) = 0;
|
||||
|
||||
/**
|
||||
* @brief Loads the reranker model after initialization.
|
||||
*/
|
||||
virtual void load() = 0;
|
||||
|
||||
/**
|
||||
* @brief Sets the instruction for the reranker.
|
||||
* @param instruct The instruction string.
|
||||
*/
|
||||
virtual void setInstruct(const std::string& instruct) = 0;
|
||||
|
||||
/**
|
||||
* @brief Computes scores for a list of documents based on a query and instruct.
|
||||
* @param query The input query.
|
||||
* @param documents A vector of document strings.
|
||||
* @return A vector of float scores, one for each document.
|
||||
*/
|
||||
virtual std::vector<float> compute_scores(const std::string& query, const std::vector<std::string>& documents) = 0;
|
||||
|
||||
/**
|
||||
* @brief Gets the underlying LLM instance for configuration purposes.
|
||||
* @return A pointer to the LLM instance, or nullptr if not available.
|
||||
*/
|
||||
Llm* get_llm() {
|
||||
return mLlm.get();
|
||||
}
|
||||
|
||||
protected:
|
||||
std::unique_ptr<Llm> mLlm;
|
||||
};
|
||||
|
||||
|
||||
class Qwen3Reranker : public RerankerBase {
|
||||
public:
|
||||
/**
|
||||
* @brief Constructor for Qwen3Reranker.
|
||||
* @param config_path The path to the LLM configuration.
|
||||
*/
|
||||
Qwen3Reranker(const std::string& config_path) {
|
||||
initialize(config_path);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Initializes the LLM.
|
||||
* @param config_path The path to the LLM configuration.
|
||||
*/
|
||||
void initialize(const std::string& config_path) override {
|
||||
mLlm.reset(Llm::createLLM(config_path));
|
||||
mLlm->set_config("{\"all_logits\":true}");
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Loads the LLM and initializes token IDs.
|
||||
*/
|
||||
void load() override {
|
||||
mLlm->load();
|
||||
mTokenTrueId = mLlm->tokenizer_encode("yes")[0];
|
||||
mTokenFalseId = mLlm->tokenizer_encode("no")[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Sets the instruction for the reranker.
|
||||
* @param instruct The instruction string.
|
||||
*/
|
||||
void setInstruct(const std::string& instruct) override {
|
||||
mInstruct = instruct;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Computes scores for a list of documents using the Qwen3 model.
|
||||
* @param query The input query.
|
||||
* @param documents A vector of document strings.
|
||||
* @return A vector of float scores, one for each document.
|
||||
*/
|
||||
std::vector<float> compute_scores(const std::string& query, const std::vector<std::string>& documents) override {
|
||||
CHECK_LLM_RUNNING_RET(mLlm->getContext(), std::vector<float>());
|
||||
std::string prefix = "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n";
|
||||
prefix = prefix + "<Instruct>: " + mInstruct + "\n<Query>: " + query + "\n<Document>: ";
|
||||
auto suffix = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n";
|
||||
auto input_ids = mLlm->tokenizer_encode(prefix);
|
||||
auto suffix_ids = mLlm->tokenizer_encode(suffix);
|
||||
std::vector<int> position_ids_;
|
||||
std::vector<std::pair<int, int>> spans;
|
||||
int prefix_len = input_ids.size();
|
||||
int suffix_len = suffix_ids.size();
|
||||
int current_len = prefix_len;
|
||||
for (int i = 0; i < current_len; i++) {
|
||||
position_ids_.push_back(i);
|
||||
}
|
||||
for (const auto& doc : documents) {
|
||||
auto doc_ids = mLlm->tokenizer_encode(doc);
|
||||
input_ids.insert(input_ids.end(), doc_ids.begin(), doc_ids.end());
|
||||
input_ids.insert(input_ids.end(), suffix_ids.begin(), suffix_ids.end());
|
||||
int doc_len = doc_ids.size() + suffix_len;
|
||||
int start_idx = current_len;
|
||||
int end_idx = current_len + doc_len;
|
||||
spans.emplace_back(start_idx, end_idx);
|
||||
current_len = end_idx;
|
||||
for (int i = prefix_len; i < prefix_len + doc_len; i++) {
|
||||
position_ids_.push_back(i);
|
||||
}
|
||||
}
|
||||
int total_len = input_ids.size();
|
||||
// printf("Total input length: %d\n", total_len);
|
||||
// input_embeds
|
||||
auto input_embeds = mLlm->embedding(input_ids);
|
||||
// attention mask
|
||||
auto attention_mask = _Input({1, 1, total_len, total_len}, NCHW, halide_type_of<float>());
|
||||
auto ptr = attention_mask->writeMap<float>();
|
||||
for (int i = 0; i < total_len; i++) {
|
||||
for (int j = 0; j < total_len; j++) {
|
||||
ptr[total_len * i + j] = (j > i) * std::numeric_limits<float>::lowest();
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < spans.size(); i++) {
|
||||
for (int j = 0; j < spans.size(); j++) {
|
||||
if (i == j) {
|
||||
continue;
|
||||
}
|
||||
int start_i = spans[i].first;
|
||||
int end_i = spans[i].second;
|
||||
int start_j = spans[j].first;
|
||||
int end_j = spans[j].second;
|
||||
for (int k = start_i; k < end_i; k++) {
|
||||
for (int l = start_j; l < end_j; l++) {
|
||||
ptr[total_len * k + l] = std::numeric_limits<float>::lowest();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// position ids
|
||||
auto position_ids = _Input({1, total_len}, NCHW, halide_type_of<int>());
|
||||
auto pos_ptr = position_ids->writeMap<int>();
|
||||
for (int i = 0; i < total_len; i++) {
|
||||
pos_ptr[i] = position_ids_[i];
|
||||
}
|
||||
|
||||
// clear history cache
|
||||
mLlm->setKVCacheInfo(total_len, mLlm->getCurrentHistory());
|
||||
|
||||
// forward
|
||||
auto outputs = mLlm->forwardRaw(input_embeds, attention_mask, position_ids);
|
||||
if(outputs.empty()) {
|
||||
return {};
|
||||
}
|
||||
auto logits = outputs[0];
|
||||
auto logits_dim = logits->getInfo()->dim[2];
|
||||
auto logits_ptr = logits->readMap<float>();
|
||||
|
||||
mLlm->reset();
|
||||
std::vector<float> scores;
|
||||
for (int i = 0; i < spans.size(); i++) {
|
||||
int logits_idx = spans[i].second - 1;
|
||||
float true_logit = logits_ptr[logits_idx * logits_dim + mTokenTrueId];
|
||||
float false_logit = logits_ptr[logits_idx * logits_dim + mTokenFalseId];
|
||||
// logsoftmax
|
||||
float max_logit = std::max(true_logit, false_logit);
|
||||
float stable_true_logit = true_logit - max_logit;
|
||||
float stable_false_logit = false_logit - max_logit;
|
||||
float exp_true = std::exp(stable_true_logit);
|
||||
float exp_false = std::exp(stable_false_logit);
|
||||
float score = exp_true / (exp_true + exp_false);
|
||||
scores.push_back(score);
|
||||
}
|
||||
|
||||
return scores;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string mInstruct;
|
||||
int mTokenTrueId;
|
||||
int mTokenFalseId;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
static inline VARP _var(std::vector<T> vec, const std::vector<int> &dims) {
|
||||
return _Const(vec.data(), dims, NHWC, halide_type_of<T>());
|
||||
}
|
||||
|
||||
class GteReranker : public RerankerBase {
|
||||
public:
|
||||
/**
|
||||
* @brief Constructor for Qwen3Reranker.
|
||||
* @param config_path The path to the LLM configuration.
|
||||
*/
|
||||
GteReranker(const std::string& config_path) {
|
||||
initialize(config_path);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Initializes the LLM and token IDs.
|
||||
* @param config_path The path to the LLM configuration.
|
||||
*/
|
||||
void initialize(const std::string& config_path) override {
|
||||
mLlm.reset(Embedding::createEmbedding(config_path, false));
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Loads the reranker model after initialization.
|
||||
*/
|
||||
void load() override {
|
||||
mLlm->load();
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Sets the instruction for the reranker.
|
||||
* @param instruct The instruction string.
|
||||
*/
|
||||
void setInstruct(const std::string& instruct) override {}
|
||||
|
||||
/**
|
||||
* @brief Computes scores for a list of documents using the Qwen3 model.
|
||||
* @param query The input query.
|
||||
* @param documents A vector of document strings.
|
||||
* @return A vector of float scores, one for each document.
|
||||
*/
|
||||
std::vector<float> compute_scores(const std::string& query, const std::vector<std::string>& documents) override {
|
||||
CHECK_LLM_RUNNING_RET(mLlm->getContext(), std::vector<float>());
|
||||
constexpr int pad_token_id = 1;
|
||||
constexpr int eos_token_id = 2;
|
||||
std::vector<int> query_ids {0, 6};
|
||||
auto q_ids = mLlm->tokenizer_encode(query);
|
||||
query_ids.insert(query_ids.end(), q_ids.begin(), q_ids.end());
|
||||
query_ids.push_back(eos_token_id);
|
||||
query_ids.push_back(eos_token_id);
|
||||
std::vector<std::vector<int>> all_ids;
|
||||
int max_len = 0;
|
||||
for (const auto& doc : documents) {
|
||||
auto doc_ids = mLlm->tokenizer_encode(doc);
|
||||
std::vector<int> ids(query_ids);
|
||||
ids.insert(ids.end(), doc_ids.begin(), doc_ids.end());
|
||||
ids.push_back(eos_token_id);
|
||||
max_len = std::max(max_len, (int)ids.size());
|
||||
all_ids.push_back(std::move(ids));
|
||||
}
|
||||
const int batch = static_cast<int>(documents.size());
|
||||
const int total_len = max_len * batch;
|
||||
auto attention_mask = _Input({batch, 1, 1, max_len}, NCHW, halide_type_of<float>());
|
||||
auto mask_ptr = attention_mask->writeMap<float>();
|
||||
std::vector<int> input_ids(total_len);
|
||||
int idx = 0;
|
||||
for (auto& ids : all_ids) {
|
||||
int i = 0;
|
||||
for (; i < ids.size(); i++, idx++) {
|
||||
input_ids[idx] = ids[i];
|
||||
mask_ptr[idx] = 0.0;
|
||||
}
|
||||
for (; i < max_len; i++, idx++) {
|
||||
input_ids[idx] = pad_token_id;
|
||||
mask_ptr[idx] = std::numeric_limits<float>::lowest();
|
||||
}
|
||||
}
|
||||
|
||||
auto input_embeds = mLlm->embedding(input_ids);
|
||||
input_embeds = _Reshape(input_embeds, _var<int>({batch, max_len, -1}, {3}));
|
||||
auto position_ids = mLlm->gen_position_ids(max_len);
|
||||
auto outputs = mLlm->forwardRaw(input_embeds, attention_mask, position_ids);
|
||||
if(outputs.empty()) {
|
||||
return {};
|
||||
}
|
||||
std::vector<float> scores(outputs[0]->readMap<float>(), outputs[0]->readMap<float>() + batch);
|
||||
return scores;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
}
|
||||
#endif /* MNN_LLM_RERANKER_hpp */
|
||||
@@ -0,0 +1,16 @@
|
||||
/**
|
||||
* T18 端侧小模型 SPIKE · native 接口声明（liblocalllm.so）
|
||||
* 实现见 entry/src/main/cpp/napi_init.cpp。
|
||||
*/
|
||||
|
||||
/** 加载模型目录（内含 config.json），resolve 空串=成功，否则为错误描述。 */
|
||||
export const loadModel: (modelDir: string) => Promise<string>;
|
||||
|
||||
/**
|
||||
* 生成。resolve JSON 字符串：
|
||||
* {"text","first_token_ms","total_ms","chars"}；失败 {"error":...}。
|
||||
*/
|
||||
export const generate: (prompt: string, maxNewTokens: number) => Promise<string>;
|
||||
|
||||
export const unloadModel: () => void;
|
||||
export const isLoaded: () => boolean;
|
||||
@@ -432,6 +432,19 @@ export struct ProfileContent {
|
||||
.backgroundColor('#FFFFFF')
|
||||
.alignItems(VerticalAlign.Center)
|
||||
|
||||
// T18 SPIKE 调试入口（内部用，正式版移除）
|
||||
Text('端侧LLM调试')
|
||||
.fontSize(13)
|
||||
.fontColor('#9A9A9A')
|
||||
.width('100%')
|
||||
.padding(12)
|
||||
.borderRadius(12)
|
||||
.backgroundColor('#FFFFFF')
|
||||
.textAlign(TextAlign.Center)
|
||||
.onClick(() => {
|
||||
router.pushUrl({ url: 'pages/LlmTestPage' })
|
||||
})
|
||||
|
||||
ErrorBanner({
|
||||
message: this.bannerMessage,
|
||||
hint: this.bannerHint,
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* LocalLlm：端侧小模型管理（T18 端侧小模型 SPIKE）。
|
||||
*
|
||||
* 职责：模型文件的下载/校验/加载状态机 + native（liblocalllm.so）调用的 ArkTS 封装。
|
||||
* 调试期模型从局域网服务器下载（scripts/serve-models.mjs）；
|
||||
* 商用阶段把 baseUrl 换成 COS/CDN 域名即可，代码不变。
|
||||
*
|
||||
* 纪律：
|
||||
* - 模型放 filesDir（cacheDir 有系统配额会被清理，不能放）；
|
||||
* - SPIKE 只校验文件大小，SHA256 校验留待 P1（crypto 全量哈希 450MB 太慢，需流式）；
|
||||
* - 推理只在前台进行（无计算类长时任务，退后台进程会被冻结）。
|
||||
*/
|
||||
|
||||
import localllm from 'liblocalllm.so'
|
||||
import { ContextHolder } from './ContextHolder'
|
||||
import request from '@ohos.request'
|
||||
import fs from '@ohos.file.fs'
|
||||
|
||||
export interface ManifestFile {
|
||||
name: string
|
||||
size: number
|
||||
sha256: string
|
||||
}
|
||||
|
||||
export interface ModelManifest {
|
||||
version: string
|
||||
model: string
|
||||
files: ManifestFile[]
|
||||
}
|
||||
|
||||
export interface GenerateResult {
|
||||
text: string
|
||||
first_token_ms: number
|
||||
total_ms: number
|
||||
chars: number
|
||||
error?: string
|
||||
}
|
||||
|
||||
export type ModelState = 'none' | 'downloading' | 'ready' | 'loaded' | 'error'
|
||||
|
||||
const MODEL_DIR_NAME: string = 'qwen3-0.6b-mnn'
|
||||
|
||||
export class LocalLlm {
|
||||
/** 调试期默认地址由测试页传入（形如 http://192.168.x.x:8080），这里不硬编码。 */
|
||||
|
||||
static modelDir(): string {
|
||||
return `${ContextHolder.get().filesDir}/models/${MODEL_DIR_NAME}`
|
||||
}
|
||||
|
||||
/** 本地模型文件是否齐全（按 manifest 尺寸校验）。 */
|
||||
static checkFiles(manifest: ModelManifest): boolean {
|
||||
const dir: string = LocalLlm.modelDir()
|
||||
for (const f of manifest.files) {
|
||||
const path: string = `${dir}/${f.name}`
|
||||
if (!fs.accessSync(path)) {
|
||||
return false
|
||||
}
|
||||
try {
|
||||
const st = fs.statSync(path)
|
||||
if (Number(st.size) !== f.size) {
|
||||
return false
|
||||
}
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/** 从 baseUrl 逐个下载模型文件（整包下载，断点续传留待商用阶段）。 */
|
||||
static async downloadAll(baseUrl: string, manifest: ModelManifest,
|
||||
onProgress?: (fileName: string, index: number, total: number) => void): Promise<void> {
|
||||
const dir: string = LocalLlm.modelDir()
|
||||
if (!fs.accessSync(dir)) {
|
||||
fs.mkdirSync(dir, true)
|
||||
}
|
||||
const ctx: Context = ContextHolder.get()
|
||||
for (let i = 0; i < manifest.files.length; i++) {
|
||||
const f: ManifestFile = manifest.files[i]
|
||||
onProgress?.(f.name, i + 1, manifest.files.length)
|
||||
const target: string = `${dir}/${f.name}`
|
||||
if (fs.accessSync(target) && Number(fs.statSync(target).size) === f.size) {
|
||||
continue // 已下好，跳过（断点续传的乞丐版：按文件粒度续）
|
||||
}
|
||||
const task = await request.downloadFile(ctx, {
|
||||
url: `${baseUrl}/${MODEL_DIR_NAME}/${f.name}`,
|
||||
filePath: target
|
||||
})
|
||||
await LocalLlm.waitTask(task)
|
||||
if (Number(fs.statSync(target).size) !== f.size) {
|
||||
throw new Error(`文件尺寸不符: ${f.name}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 加载模型到 native（耗时，走 async work 线程）。返回空串=成功。 */
|
||||
static async load(): Promise<string> {
|
||||
const dir: string = LocalLlm.modelDir()
|
||||
const tmp: string = `${dir}/.tmp`
|
||||
if (!fs.accessSync(tmp)) {
|
||||
fs.mkdirSync(tmp, true)
|
||||
}
|
||||
return localllm.loadModel(dir)
|
||||
}
|
||||
|
||||
static isLoaded(): boolean {
|
||||
return localllm.isLoaded()
|
||||
}
|
||||
|
||||
static unload(): void {
|
||||
localllm.unloadModel()
|
||||
}
|
||||
|
||||
/** 生成。native 侧已完成计时；解析失败视为 error。 */
|
||||
static async generate(prompt: string, maxNewTokens: number): Promise<GenerateResult> {
|
||||
const raw: string = await localllm.generate(prompt, maxNewTokens)
|
||||
try {
|
||||
return JSON.parse(raw) as GenerateResult
|
||||
} catch {
|
||||
return { text: '', first_token_ms: -1, total_ms: -1, chars: 0, error: `bad json: ${raw}` }
|
||||
}
|
||||
}
|
||||
|
||||
private static waitTask(task: request.DownloadTask): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
task.on('complete', () => resolve())
|
||||
task.on('fail', (err: number) => reject(new Error(`download fail: ${err}`)))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
/**
|
||||
* LlmTestPage：T18 端侧小模型 SPIKE 的调试/基准页。
|
||||
*
|
||||
* 用途（仅限内部调试，不对真实用户开放）：
|
||||
* 1. 从局域网服务器下载模型 → 加载 → 跑 benchmark（首 token 延迟 / 总耗时 / 字符速度）；
|
||||
* 2. 意图分类 smoke（固定 prompt + 固定样例，人工核对输出 JSON）；
|
||||
* 3. hidebug 内存采样（加载前后 native 堆对比，验收口径：峰值 <1GB）。
|
||||
*
|
||||
* 验收门槛见 docs/开发任务包/T18-端侧小模型SPIKE.md。
|
||||
*/
|
||||
|
||||
import { LocalLlm, GenerateResult, ModelManifest } from '../core/LocalLlm'
|
||||
import hidebug from '@ohos.hidebug'
|
||||
import http from '@ohos.net.http'
|
||||
|
||||
const DEFAULT_BASE: string = 'http://192.168.3.19:8080' // 开发机局域网 IP（模型服务器）
|
||||
const MODEL_SUBPATH: string = 'qwen3-0.6b-mnn'
|
||||
|
||||
/** 意图分类 smoke 用的系统 prompt（标签集对齐《穿搭Agent-需求文档》§3.2，P2 的"穿搭评价"先占位）。 */
|
||||
const INTENT_PROMPT: string =
|
||||
'你是穿搭助手的意图分类器。把用户输入分类到以下之一：' +
|
||||
'场景咨询/单品搭配/风格建议/色彩咨询/身材修饰/天气适配/发型建议/穿搭评价/闲聊/越界。' +
|
||||
'只输出 JSON：{"intent":"<类别>"}，不要输出任何其它内容。用户输入：'
|
||||
|
||||
const INTENT_CASES: string[] = [
|
||||
'面试怎么穿？',
|
||||
'白衬衫配什么裤子？',
|
||||
'我这种矮胖身材是不是穿什么都难看？顺便告诉我怎么减肥',
|
||||
'你好呀',
|
||||
]
|
||||
|
||||
const BENCH_PROMPTS: string[] = [
|
||||
'你好，简单介绍一下你自己，一句话以内。',
|
||||
INTENT_PROMPT + '约会穿什么？',
|
||||
'夏天通勤穿搭有什么通用原则？100字以内回答。',
|
||||
]
|
||||
|
||||
@Entry
|
||||
@Component
|
||||
struct LlmTestPage {
|
||||
@State baseUrl: string = DEFAULT_BASE
|
||||
@State logText: string = '就绪。步骤：① 填开发机IP ② 下载模型 ③ 加载模型 ④ 跑 Benchmark\n'
|
||||
@State busy: boolean = false
|
||||
|
||||
private log(line: string): void {
|
||||
this.logText += `${line}\n`
|
||||
}
|
||||
|
||||
private heapMb(): string {
|
||||
const bytes: bigint = hidebug.getNativeHeapSize()
|
||||
return (Number(bytes) / 1024 / 1024).toFixed(0)
|
||||
}
|
||||
|
||||
private async fetchManifest(): Promise<ModelManifest> {
|
||||
const client = http.createHttp()
|
||||
try {
|
||||
const resp = await client.request(`${this.baseUrl}/${MODEL_SUBPATH}/manifest.json`,
|
||||
{ connectTimeout: 5000, readTimeout: 10000 })
|
||||
if (resp.responseCode !== 200) {
|
||||
throw new Error(`manifest http ${resp.responseCode}`)
|
||||
}
|
||||
return JSON.parse(resp.result as string) as ModelManifest
|
||||
} finally {
|
||||
client.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
private async onDownload(): Promise<void> {
|
||||
this.busy = true
|
||||
try {
|
||||
this.log(`连接 ${this.baseUrl} ...`)
|
||||
const manifest = await this.fetchManifest()
|
||||
this.log(`manifest v${manifest.version}，${manifest.files.length} 个文件`)
|
||||
if (LocalLlm.checkFiles(manifest)) {
|
||||
this.log('本地文件已齐全，跳过下载')
|
||||
return
|
||||
}
|
||||
await LocalLlm.downloadAll(this.baseUrl, manifest, (name: string, idx: number, total: number) => {
|
||||
this.log(`[${idx}/${total}] ${name}`)
|
||||
})
|
||||
if (!LocalLlm.checkFiles(manifest)) {
|
||||
throw new Error('下载后校验失败')
|
||||
}
|
||||
this.log('下载完成，校验通过')
|
||||
} catch (e) {
|
||||
this.log(`下载失败: ${(e as Error).message}`)
|
||||
} finally {
|
||||
this.busy = false
|
||||
}
|
||||
}
|
||||
|
||||
private async onLoad(): Promise<void> {
|
||||
this.busy = true
|
||||
try {
|
||||
this.log(`加载前 native 堆: ${this.heapMb()}MB`)
|
||||
const t0: number = Date.now()
|
||||
const err: string = await LocalLlm.load()
|
||||
if (err.length > 0) {
|
||||
this.log(`加载失败: ${err}`)
|
||||
return
|
||||
}
|
||||
this.log(`加载完成，耗时 ${Date.now() - t0}ms，加载后 native 堆: ${this.heapMb()}MB`)
|
||||
} catch (e) {
|
||||
this.log(`加载异常: ${(e as Error).message}`)
|
||||
} finally {
|
||||
this.busy = false
|
||||
}
|
||||
}
|
||||
|
||||
private async runOne(prompt: string, maxNewTokens: number): Promise<void> {
|
||||
const r: GenerateResult = await LocalLlm.generate(prompt, maxNewTokens)
|
||||
if (r.error !== undefined) {
|
||||
this.log(` 生成失败: ${r.error}`)
|
||||
return
|
||||
}
|
||||
const speed: string = r.total_ms > 0 ? (r.chars * 1000 / r.total_ms).toFixed(1) : '?'
|
||||
this.log(` 首token ${r.first_token_ms}ms / 总 ${r.total_ms}ms / ${r.chars}字 (${speed}字/s) / 堆 ${this.heapMb()}MB`)
|
||||
this.log(` 输出: ${r.text.substring(0, 120)}`)
|
||||
}
|
||||
|
||||
private async onBench(): Promise<void> {
|
||||
if (!LocalLlm.isLoaded()) {
|
||||
this.log('请先加载模型')
|
||||
return
|
||||
}
|
||||
this.busy = true
|
||||
try {
|
||||
for (let i = 0; i < BENCH_PROMPTS.length; i++) {
|
||||
this.log(`[bench ${i + 1}/${BENCH_PROMPTS.length}] ${BENCH_PROMPTS[i].substring(0, 20)}...`)
|
||||
await this.runOne(BENCH_PROMPTS[i], 128) // 预热
|
||||
await this.runOne(BENCH_PROMPTS[i], 128)
|
||||
await this.runOne(BENCH_PROMPTS[i], 128)
|
||||
}
|
||||
this.log('benchmark 完成')
|
||||
} finally {
|
||||
this.busy = false
|
||||
}
|
||||
}
|
||||
|
||||
private async onIntentSmoke(): Promise<void> {
|
||||
if (!LocalLlm.isLoaded()) {
|
||||
this.log('请先加载模型')
|
||||
return
|
||||
}
|
||||
this.busy = true
|
||||
try {
|
||||
for (const c of INTENT_CASES) {
|
||||
this.log(`[意图] 输入: ${c}`)
|
||||
await this.runOne(INTENT_PROMPT + c, 32)
|
||||
}
|
||||
} finally {
|
||||
this.busy = false
|
||||
}
|
||||
}
|
||||
|
||||
build() {
|
||||
Column({ space: 12 }) {
|
||||
Text('端侧小模型调试（T18 SPIKE）').fontSize(18).fontWeight(FontWeight.Bold)
|
||||
TextInput({ text: this.baseUrl, placeholder: '开发机地址，如 http://192.168.1.100:8080' })
|
||||
.onChange((v: string) => { this.baseUrl = v })
|
||||
Row({ space: 8 }) {
|
||||
Button('① 下载模型').enabled(!this.busy).onClick(() => { this.onDownload() })
|
||||
Button('② 加载模型').enabled(!this.busy).onClick(() => { this.onLoad() })
|
||||
}
|
||||
Row({ space: 8 }) {
|
||||
Button('③ Benchmark').enabled(!this.busy).onClick(() => { this.onBench() })
|
||||
Button('④ 意图smoke').enabled(!this.busy).onClick(() => { this.onIntentSmoke() })
|
||||
Button('卸载').enabled(!this.busy).onClick(() => {
|
||||
LocalLlm.unload()
|
||||
this.log(`已卸载，native 堆: ${this.heapMb()}MB`)
|
||||
})
|
||||
}
|
||||
Scroll() {
|
||||
Text(this.logText).fontSize(12).fontFamily('monospace')
|
||||
}
|
||||
.layoutWeight(1)
|
||||
.align(Alignment.TopStart)
|
||||
.scrollBar(BarState.Auto)
|
||||
}
|
||||
.padding(16)
|
||||
.width('100%')
|
||||
.height('100%')
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user