292 lines
18 KiB
JavaScript
292 lines
18 KiB
JavaScript
// Local tensor-only face detection. All coordinates are in the supplied raw frame
|
|
// pixels. No mirroring, rotation, DOM, native AI API, upload or photo storage.
|
|
var faceLuma = require('./face-luma');
|
|
var shared = { face: null, ready: {}, users: 0, busy: false, loaded: false, cpu: null,
|
|
backendKind: '', backendReady: null, gpu: null, warmed: {}, queue: Promise.resolve() };
|
|
function failure(code, text) { var error = new Error(text); error.code = code; return error; }
|
|
function integer(value) { return typeof value === 'number' && isFinite(value) && Math.floor(value) === value; }
|
|
function format(value) {
|
|
if (typeof value !== 'string') return '';
|
|
var match = /^(RGBA|BGRA|ARGB|ABGR)(?:8888)?$/.exec(value.toUpperCase());
|
|
return match ? match[1] : '';
|
|
}
|
|
function decodeBase64(value) {
|
|
var alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', padding = value.slice(-2) === '==' ? 2 : value.slice(-1) === '=' ? 1 : 0;
|
|
var data = new Uint8Array(value.length / 4 * 3 - padding), offset = 0;
|
|
for (var index = 0; index < value.length; index += 4) {
|
|
var a = alphabet.indexOf(value.charAt(index)), b = alphabet.indexOf(value.charAt(index + 1));
|
|
var c = alphabet.indexOf(value.charAt(index + 2)), d = alphabet.indexOf(value.charAt(index + 3));
|
|
var packed = (a << 18) | (b << 12) | ((c < 0 ? 0 : c) << 6) | (d < 0 ? 0 : d);
|
|
if (offset < data.length) data[offset++] = packed >> 16 & 255;
|
|
if (offset < data.length) data[offset++] = packed >> 8 & 255;
|
|
if (offset < data.length) data[offset++] = packed & 255;
|
|
}
|
|
return data.buffer;
|
|
}
|
|
function unloadModels() {
|
|
if (!shared.face) return;
|
|
var nets = shared.face.nets;
|
|
[nets.tinyFaceDetector, nets.faceLandmark68Net, nets.faceLandmark68TinyNet].forEach(function (net) {
|
|
var guarded = [];
|
|
try {
|
|
if (!net.isLoaded && !(net.faceFeatureExtractor && net.faceFeatureExtractor.isLoaded)) return;
|
|
// A single native Tensor.dispose failure must not abort the network's
|
|
// own remaining parameter loop or prevent it clearing its loaded state.
|
|
function guard(network) {
|
|
if (!network || !network.isLoaded) return;
|
|
// The landmark model owns a separate feature-extractor network; its
|
|
// parameters are not included in the parent's getParamList().
|
|
if (network.faceFeatureExtractor) guard(network.faceFeatureExtractor);
|
|
network.getParamList().forEach(function (parameter) {
|
|
var tensor = parameter.tensor, original = tensor.dispose;
|
|
var safe = function () { try { return original.apply(tensor, arguments); } catch (_) {} };
|
|
try { tensor.dispose = safe; if (tensor.dispose === safe) guarded.push({ tensor: tensor, original: original }); } catch (_) {}
|
|
});
|
|
}
|
|
guard(net);
|
|
net.dispose(false);
|
|
} catch (_) {}
|
|
finally { guarded.forEach(function (entry) { try { entry.tensor.dispose = entry.original; } catch (_) {} }); }
|
|
});
|
|
shared.loaded = false; shared.ready = {};
|
|
}
|
|
function enqueue(work) {
|
|
var operation = shared.queue.catch(function () {}).then(work);
|
|
// Keep only a completion barrier globally; the caller owns any detection
|
|
// result. A resolved shared queue must not retain the last face coordinates.
|
|
shared.queue = operation.then(function () {}, function () {}); return operation;
|
|
}
|
|
function releaseSession() {
|
|
try { unloadModels(); }
|
|
finally {
|
|
try { if (shared.gpu) shared.gpu.dispose(); } catch (_) {}
|
|
shared.gpu = null; shared.backendKind = ''; shared.backendReady = null; shared.warmed = {};
|
|
}
|
|
}
|
|
function ensureBackend(backendKind, platform) {
|
|
if (!shared.backendReady) {
|
|
shared.backendReady = Promise.resolve().then(function () {
|
|
var face = shared.face || (shared.face = require('../vendor/native-face/face-api'));
|
|
if (backendKind === 'webgl') {
|
|
shared.gpu = require('./native-face-webgl')(face, platform);
|
|
return shared.gpu.initialize().then(function () { return face; }, function (error) { shared.gpu.dispose(); throw error; });
|
|
}
|
|
return Promise.resolve(face.tf.setBackend('cpu')).then(function () {
|
|
if (face.tf.getBackend() !== 'cpu') throw failure('NATIVE_FACE_BACKEND', '本地人脸引擎无法启用 CPU');
|
|
if (!shared.cpu) shared.cpu = require('./native-face-cpu')(face);
|
|
return face;
|
|
});
|
|
});
|
|
}
|
|
return shared.backendReady;
|
|
}
|
|
function loadModels(landmarkModel, backendKind, platform) {
|
|
if (!shared.ready[landmarkModel]) {
|
|
shared.ready[landmarkModel] = ensureBackend(backendKind, platform).then(function (face) {
|
|
var weights = require('../vendor/native-face/weights');
|
|
function load(net, group) {
|
|
var map = face.tf.io.decodeWeights(decodeBase64(group.base64), group.weights);
|
|
try { net.loadFromWeightMap(map); }
|
|
catch (error) { Object.keys(map).forEach(function (key) { try { map[key].dispose(); } catch (_) {} }); throw error; }
|
|
}
|
|
if (!face.nets.tinyFaceDetector.isLoaded) load(face.nets.tinyFaceDetector, weights.tiny);
|
|
var net = landmarkModel === 'full' ? face.nets.faceLandmark68Net : face.nets.faceLandmark68TinyNet;
|
|
if (!net.isLoaded) load(net, landmarkModel === 'full' ? weights.landmark : weights.landmarkTiny);
|
|
shared.loaded = true;
|
|
return face;
|
|
}).catch(function (error) { delete shared.ready[landmarkModel]; throw error; });
|
|
}
|
|
return shared.ready[landmarkModel];
|
|
}
|
|
function warmup(face, landmarkModel, inputSize, maxSide) {
|
|
if (!shared.gpu) return Promise.resolve();
|
|
var key = landmarkModel + ':' + inputSize + ':' + maxSide;
|
|
if (shared.warmed[key]) return Promise.resolve();
|
|
var tensors = [], started = Date.now(), gpu = shared.gpu;
|
|
gpu.warmupStart();
|
|
// Match the typical portrait aspect ratio (small camera 360x640 ->180x320).
|
|
// Explicitly run the landmark net too: black detector input has no face.
|
|
var operation = Promise.resolve().then(function () {
|
|
gpu.check();
|
|
var tensor = face.tf.zeros([maxSide, Math.round(maxSide * 9 / 16), 3], 'int32'); tensors.push(tensor);
|
|
return Promise.resolve(face.detectAllFaces(new face.NetInput([tensor]), new face.TinyFaceDetectorOptions({ inputSize: inputSize, scoreThreshold: 0.35 })));
|
|
}).then(function () {
|
|
gpu.check(); var crop = face.tf.zeros([192, 192, 3], 'int32'); tensors.push(crop);
|
|
var net = landmarkModel === 'full' ? face.nets.faceLandmark68Net : face.nets.faceLandmark68TinyNet;
|
|
return net.detectLandmarks(new face.NetInput([crop]));
|
|
}).then(function (points) {
|
|
gpu.check();
|
|
if (!points || !points.positions || points.positions.length !== 68 || !points.positions.every(function (point) { return isFinite(point.x) && isFinite(point.y); })) throw gpu.warmupFailed();
|
|
shared.warmed[key] = true; gpu.warmupDone(Date.now() - started);
|
|
});
|
|
function cleanup() { tensors.forEach(function (tensor) { try { tensor.dispose(); } catch (_) {} }); }
|
|
return operation.then(function () { cleanup(); }, function (error) {
|
|
cleanup();
|
|
if (error && /^NATIVE_FACE_WEBGL_/.test(error.code || '')) throw error;
|
|
gpu.check(); throw gpu.warmupFailed();
|
|
});
|
|
}
|
|
function rawFrame(frame, defaultFormat, maxSide) {
|
|
if (!frame || !integer(frame.width) || !integer(frame.height) || frame.width < 1 || frame.height < 1 ||
|
|
frame.width > 4096 || frame.height > 4096) throw failure('NATIVE_FACE_FRAME', '相机帧尺寸无效');
|
|
var width = frame.width, height = frame.height;
|
|
var metadata = frame.pixelFormat !== undefined && frame.pixelFormat !== null && frame.pixelFormat !== '' ? frame.pixelFormat : frame.format;
|
|
var hasFormat = metadata !== undefined && metadata !== null && metadata !== '';
|
|
var layout = format(hasFormat ? metadata : defaultFormat);
|
|
if (!layout) throw failure('NATIVE_FACE_FORMAT', '本地人脸检测仅支持 RGBA/BGRA/ARGB/ABGR 四通道帧');
|
|
if ((frame.stride !== undefined && frame.stride !== width * 4) || (frame.bytesPerRow !== undefined && frame.bytesPerRow !== width * 4)) {
|
|
throw failure('NATIVE_FACE_STRIDE', '暂不支持带额外行填充的相机帧');
|
|
}
|
|
if (frame.rotation !== undefined && frame.rotation !== 0) throw failure('NATIVE_FACE_ROTATION', '暂不支持带旋转标记的相机帧');
|
|
var value = frame.data, bytes;
|
|
if (Object.prototype.toString.call(value) === '[object ArrayBuffer]') bytes = new Uint8Array(value);
|
|
else if (value && Object.prototype.toString.call(value.buffer) === '[object ArrayBuffer]' && integer(value.byteOffset) && integer(value.byteLength)) {
|
|
bytes = new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
|
|
}
|
|
if (!bytes || bytes.length !== width * height * 4) throw failure('NATIVE_FACE_FORMAT', '相机帧不是紧密排列的四通道数据');
|
|
var scale = Math.min(1, maxSide / Math.max(width, height));
|
|
var w = Math.max(1, Math.round(width * scale)), h = Math.max(1, Math.round(height * scale));
|
|
var rgb = new Int32Array(w * h * 3);
|
|
var r = layout === 'BGRA' ? 2 : layout === 'ARGB' ? 1 : layout === 'ABGR' ? 3 : 0;
|
|
var g = layout === 'ARGB' || layout === 'ABGR' ? 2 : 1;
|
|
var b = layout === 'BGRA' ? 0 : layout === 'ARGB' ? 3 : layout === 'ABGR' ? 1 : 2;
|
|
for (var y = 0; y < h; y++) for (var x = 0; x < w; x++) {
|
|
var source = (Math.min(height - 1, Math.floor(y * height / h)) * width + Math.min(width - 1, Math.floor(x * width / w))) * 4;
|
|
var target = (y * w + x) * 3;
|
|
rgb[target] = bytes[source + r]; rgb[target + 1] = bytes[source + g]; rgb[target + 2] = bytes[source + b];
|
|
}
|
|
return { rgb: rgb, width: w, height: h, originalWidth: width, originalHeight: height };
|
|
}
|
|
function create(options) {
|
|
options = options || {};
|
|
var maxSide = integer(options.maxSide) && options.maxSide >= 128 && options.maxSide <= 320 ? options.maxSide : 320;
|
|
var inputSize = integer(options.inputSize) && options.inputSize >= 128 && options.inputSize <= 320 && options.inputSize % 32 === 0 ? options.inputSize : 160;
|
|
var landmarkModel = options.landmarkModel === 'full' ? 'full' : 'tiny';
|
|
var cpuOptimization = options.cpuOptimization !== false;
|
|
var defaultFormat = options.defaultPixelFormat === undefined ? 'RGBA' : options.defaultPixelFormat;
|
|
var scoreThreshold = typeof options.scoreThreshold === 'number' && options.scoreThreshold > 0 && options.scoreThreshold < 1 ? options.scoreThreshold : 0.35;
|
|
var backendKind = options.backend === 'webgl' ? 'webgl' : 'cpu';
|
|
var conflict = shared.users > 0 && shared.backendKind !== backendKind;
|
|
var disposed = false, released = false, pending = 0, busy = false, gpuHandle = null, actualBackend = '';
|
|
if (!conflict) { if (!shared.users) shared.backendKind = backendKind; shared.users++; }
|
|
function check() {
|
|
if (disposed) throw failure('NATIVE_FACE_DISPOSED', '本地人脸引擎已关闭');
|
|
if (conflict) throw failure('NATIVE_FACE_BACKEND_CONFLICT', '请先关闭上一种本地检测引擎');
|
|
if (gpuHandle) gpuHandle.check();
|
|
}
|
|
function gpuSummary() {
|
|
var handle = gpuHandle || (backendKind === 'webgl' && !conflict ? shared.gpu : null);
|
|
return handle ? handle.summary() : { requested: backendKind === 'webgl', selftest: 'not_run', stage: 'uninitialized', code: 'NONE', contextLost: false,
|
|
profile: 'default', attemptCount: 0, matmulDraws: 0, convDraws: 0, pixelReads: 0, floatReads: 0, byteReads: 0,
|
|
selftestMs: 0, warmupMs: 0, matmul: null, conv: null };
|
|
}
|
|
function prepare() {
|
|
check();
|
|
return loadModels(landmarkModel, backendKind, options.platform).then(function (face) {
|
|
gpuHandle = shared.gpu; check();
|
|
return warmup(face, landmarkModel, inputSize, maxSide).then(function () { check(); actualBackend = backendKind; return face; });
|
|
}, function (error) {
|
|
gpuHandle = shared.gpu;
|
|
if (backendKind === 'webgl' && gpuHandle && !/^NATIVE_FACE_WEBGL_/.test(error && error.code || '')) throw gpuHandle.runtimeFailed();
|
|
throw error;
|
|
});
|
|
}
|
|
function release() {
|
|
if (!disposed || pending || released) return;
|
|
released = true;
|
|
if (conflict) return;
|
|
shared.users--;
|
|
if (!shared.users) releaseSession();
|
|
}
|
|
function settle() { pending--; release(); }
|
|
function init() {
|
|
if (disposed) return Promise.reject(failure('NATIVE_FACE_DISPOSED', '本地人脸引擎已关闭'));
|
|
if (conflict) return Promise.reject(failure('NATIVE_FACE_BACKEND_CONFLICT', '请先关闭上一种本地检测引擎'));
|
|
pending++;
|
|
return enqueue(prepare).then(function () { check(); return { backend: actualBackend, inputSize: inputSize, maxSide: maxSide, landmarkModel: landmarkModel,
|
|
cpuOptimization: backendKind === 'cpu' && cpuOptimization ? 'pad-copy' : 'none', gpu: gpuSummary() }; })
|
|
.then(function (result) { settle(); return result; }, function (error) { settle(); throw error; });
|
|
}
|
|
function detect(frame) {
|
|
if (disposed) return Promise.reject(failure('NATIVE_FACE_DISPOSED', '本地人脸引擎已关闭'));
|
|
if (conflict) return Promise.reject(failure('NATIVE_FACE_BACKEND_CONFLICT', '请先关闭上一种本地检测引擎'));
|
|
if (busy || shared.busy) return Promise.reject(failure('NATIVE_FACE_BUSY', '本地人脸检测仍在处理中'));
|
|
busy = true; shared.busy = true; pending++;
|
|
var tensors = [], started = Date.now(), raw;
|
|
var timings = { snapshotMs: 0, modelMs: 0, tensorMs: 0, detectorMs: 0, landmarkMs: 0, totalMs: 0 };
|
|
function cleanup() {
|
|
try { tensors.forEach(function (tensor) { try { tensor.dispose(); } catch (_) {} }); }
|
|
finally {
|
|
try { if (shared.cpu) shared.cpu.setEnabled(false); } catch (_) {}
|
|
busy = false; shared.busy = false; settle();
|
|
}
|
|
}
|
|
// Native camera buffers may be recycled after this call returns. Snapshot
|
|
// only the downsampled RGB (at most 320px), before any Promise boundary.
|
|
try { raw = rawFrame(frame, defaultFormat, maxSide); }
|
|
catch (error) { cleanup(); return Promise.reject(error); }
|
|
frame = null;
|
|
timings.snapshotMs = Date.now() - started;
|
|
var modelStart = Date.now();
|
|
var operation = enqueue(function () { return prepare().then(function (face) {
|
|
check();
|
|
if (shared.cpu) shared.cpu.setEnabled(backendKind === 'cpu' && cpuOptimization);
|
|
timings.modelMs = Date.now() - modelStart;
|
|
var tensorStart = Date.now();
|
|
var tensor = face.tf.tensor3d(raw.rgb, [raw.height, raw.width, 3], 'int32'); tensors.push(tensor);
|
|
// High-level withFaceLandmarks() calls Canvas even for tensor input.
|
|
// Keep both networks and face crops on the explicit tensor-only path.
|
|
var input = new face.NetInput([tensor]);
|
|
timings.tensorMs = Date.now() - tensorStart;
|
|
var detectorStart = Date.now();
|
|
return Promise.resolve(face.detectAllFaces(input, new face.TinyFaceDetectorOptions({ inputSize: inputSize, scoreThreshold: scoreThreshold }))).then(function (found) {
|
|
check();
|
|
timings.detectorMs = Date.now() - detectorStart;
|
|
var detections = [], luma = null, chain = Promise.resolve();
|
|
// Two faces are enough to reject multi-person captures without running
|
|
// an unbounded number of landmark networks in one camera frame.
|
|
found.slice(0, 2).forEach(function (detection) {
|
|
chain = chain.then(function () {
|
|
check();
|
|
var box = detection.box.floor().clipAtImageBorders(raw.width, raw.height);
|
|
if (!(box.width > 0 && box.height > 0)) return;
|
|
var landmarkStart = Date.now();
|
|
return face.extractFaceTensors(tensor, [box]).then(function (crops) {
|
|
var crop = crops[0]; tensors.push(crop); check();
|
|
var net = landmarkModel === 'full' ? face.nets.faceLandmark68Net : face.nets.faceLandmark68TinyNet;
|
|
return net.detectLandmarks(new face.NetInput([crop])).then(function (landmarks) {
|
|
check();
|
|
timings.landmarkMs += Date.now() - landmarkStart;
|
|
var sx = raw.originalWidth / raw.width, sy = raw.originalHeight / raw.height;
|
|
if (!detections.length) luma = faceLuma.sample(raw.rgb, raw.width, raw.height, box);
|
|
detections.push({ detection: { score: detection.score, box: { x: detection.box.x * sx, y: detection.box.y * sy,
|
|
width: detection.box.width * sx, height: detection.box.height * sy } },
|
|
landmarks: { positions: landmarks.positions.map(function (point) { return { x: (point.x + box.x) * sx, y: (point.y + box.y) * sy }; }) } });
|
|
});
|
|
});
|
|
});
|
|
});
|
|
return chain.then(function () { check(); timings.totalMs = Date.now() - started; return { width: raw.originalWidth, height: raw.originalHeight, detections: detections, luma: luma,
|
|
faceCount: found.length, elapsedMs: timings.totalMs, backend: actualBackend, inputSize: inputSize, maxSide: maxSide, landmarkModel: landmarkModel,
|
|
cpuOptimization: backendKind === 'cpu' && cpuOptimization ? 'pad-copy' : 'none', gpu: gpuSummary(), timings: timings }; });
|
|
});
|
|
}); });
|
|
return operation.then(function (result) { cleanup(); return result; }, function (error) {
|
|
if (backendKind === 'webgl' && gpuHandle && !/^NATIVE_FACE_(?:WEBGL_|DISPOSED|BACKEND_CONFLICT)/.test(error && error.code || '')) {
|
|
try { error = gpuHandle.runtimeFailed(); } catch (lostError) { error = lostError; }
|
|
}
|
|
cleanup(); throw error;
|
|
});
|
|
}
|
|
return { init: init, detect: detect,
|
|
dispose: function () { disposed = true; release(); },
|
|
stats: function () {
|
|
var memory = {};
|
|
try { if (shared.face) memory = shared.face.tf.memory(); } catch (_) {}
|
|
return { backend: actualBackend, inputSize: inputSize, maxSide: maxSide, landmarkModel: landmarkModel,
|
|
cpuOptimization: backendKind === 'cpu' && cpuOptimization ? 'pad-copy' : 'none', gpu: gpuSummary(), modelsLoaded: !conflict && shared.loaded, busy: busy,
|
|
disposed: disposed, numTensors: memory.numTensors || 0, numBytes: memory.numBytes || 0 };
|
|
} };
|
|
}
|
|
module.exports = { create: create };
|