// Local tensor-only face detection. All coordinates are in the supplied raw frame // pixels. No mirroring, rotation, DOM, native AI API, upload or photo storage. var faceLuma = require('./face-luma'); var shared = { face: null, ready: {}, users: 0, busy: false, loaded: false, cpu: null, backendKind: '', backendReady: null, gpu: null, warmed: {}, queue: Promise.resolve() }; function failure(code, text) { var error = new Error(text); error.code = code; return error; } function integer(value) { return typeof value === 'number' && isFinite(value) && Math.floor(value) === value; } function format(value) { if (typeof value !== 'string') return ''; var match = /^(RGBA|BGRA|ARGB|ABGR)(?:8888)?$/.exec(value.toUpperCase()); return match ? match[1] : ''; } function decodeBase64(value) { var alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', padding = value.slice(-2) === '==' ? 2 : value.slice(-1) === '=' ? 1 : 0; var data = new Uint8Array(value.length / 4 * 3 - padding), offset = 0; for (var index = 0; index < value.length; index += 4) { var a = alphabet.indexOf(value.charAt(index)), b = alphabet.indexOf(value.charAt(index + 1)); var c = alphabet.indexOf(value.charAt(index + 2)), d = alphabet.indexOf(value.charAt(index + 3)); var packed = (a << 18) | (b << 12) | ((c < 0 ? 0 : c) << 6) | (d < 0 ? 0 : d); if (offset < data.length) data[offset++] = packed >> 16 & 255; if (offset < data.length) data[offset++] = packed >> 8 & 255; if (offset < data.length) data[offset++] = packed & 255; } return data.buffer; } function unloadModels() { if (!shared.face) return; var nets = shared.face.nets; [nets.tinyFaceDetector, nets.faceLandmark68Net, nets.faceLandmark68TinyNet].forEach(function (net) { var guarded = []; try { if (!net.isLoaded && !(net.faceFeatureExtractor && net.faceFeatureExtractor.isLoaded)) return; // A single native Tensor.dispose failure must not abort the network's // own remaining parameter loop or prevent it clearing its loaded state. function guard(network) { if (!network || !network.isLoaded) return; // The landmark model owns a separate feature-extractor network; its // parameters are not included in the parent's getParamList(). if (network.faceFeatureExtractor) guard(network.faceFeatureExtractor); network.getParamList().forEach(function (parameter) { var tensor = parameter.tensor, original = tensor.dispose; var safe = function () { try { return original.apply(tensor, arguments); } catch (_) {} }; try { tensor.dispose = safe; if (tensor.dispose === safe) guarded.push({ tensor: tensor, original: original }); } catch (_) {} }); } guard(net); net.dispose(false); } catch (_) {} finally { guarded.forEach(function (entry) { try { entry.tensor.dispose = entry.original; } catch (_) {} }); } }); shared.loaded = false; shared.ready = {}; } function enqueue(work) { var operation = shared.queue.catch(function () {}).then(work); // Keep only a completion barrier globally; the caller owns any detection // result. A resolved shared queue must not retain the last face coordinates. shared.queue = operation.then(function () {}, function () {}); return operation; } function releaseSession() { try { unloadModels(); } finally { try { if (shared.gpu) shared.gpu.dispose(); } catch (_) {} shared.gpu = null; shared.backendKind = ''; shared.backendReady = null; shared.warmed = {}; } } function ensureBackend(backendKind, platform) { if (!shared.backendReady) { shared.backendReady = Promise.resolve().then(function () { var face = shared.face || (shared.face = require('../vendor/native-face/face-api')); if (backendKind === 'webgl') { shared.gpu = require('./native-face-webgl')(face, platform); return shared.gpu.initialize().then(function () { return face; }, function (error) { shared.gpu.dispose(); throw error; }); } return Promise.resolve(face.tf.setBackend('cpu')).then(function () { if (face.tf.getBackend() !== 'cpu') throw failure('NATIVE_FACE_BACKEND', '本地人脸引擎无法启用 CPU'); if (!shared.cpu) shared.cpu = require('./native-face-cpu')(face); return face; }); }); } return shared.backendReady; } function loadModels(landmarkModel, backendKind, platform) { if (!shared.ready[landmarkModel]) { shared.ready[landmarkModel] = ensureBackend(backendKind, platform).then(function (face) { var weights = require('../vendor/native-face/weights'); function load(net, group) { var map = face.tf.io.decodeWeights(decodeBase64(group.base64), group.weights); try { net.loadFromWeightMap(map); } catch (error) { Object.keys(map).forEach(function (key) { try { map[key].dispose(); } catch (_) {} }); throw error; } } if (!face.nets.tinyFaceDetector.isLoaded) load(face.nets.tinyFaceDetector, weights.tiny); var net = landmarkModel === 'full' ? face.nets.faceLandmark68Net : face.nets.faceLandmark68TinyNet; if (!net.isLoaded) load(net, landmarkModel === 'full' ? weights.landmark : weights.landmarkTiny); shared.loaded = true; return face; }).catch(function (error) { delete shared.ready[landmarkModel]; throw error; }); } return shared.ready[landmarkModel]; } function warmup(face, landmarkModel, inputSize, maxSide) { if (!shared.gpu) return Promise.resolve(); var key = landmarkModel + ':' + inputSize + ':' + maxSide; if (shared.warmed[key]) return Promise.resolve(); var tensors = [], started = Date.now(), gpu = shared.gpu; gpu.warmupStart(); // Match the typical portrait aspect ratio (small camera 360x640 ->180x320). // Explicitly run the landmark net too: black detector input has no face. var operation = Promise.resolve().then(function () { gpu.check(); var tensor = face.tf.zeros([maxSide, Math.round(maxSide * 9 / 16), 3], 'int32'); tensors.push(tensor); return Promise.resolve(face.detectAllFaces(new face.NetInput([tensor]), new face.TinyFaceDetectorOptions({ inputSize: inputSize, scoreThreshold: 0.35 }))); }).then(function () { gpu.check(); var crop = face.tf.zeros([192, 192, 3], 'int32'); tensors.push(crop); var net = landmarkModel === 'full' ? face.nets.faceLandmark68Net : face.nets.faceLandmark68TinyNet; return net.detectLandmarks(new face.NetInput([crop])); }).then(function (points) { gpu.check(); if (!points || !points.positions || points.positions.length !== 68 || !points.positions.every(function (point) { return isFinite(point.x) && isFinite(point.y); })) throw gpu.warmupFailed(); shared.warmed[key] = true; gpu.warmupDone(Date.now() - started); }); function cleanup() { tensors.forEach(function (tensor) { try { tensor.dispose(); } catch (_) {} }); } return operation.then(function () { cleanup(); }, function (error) { cleanup(); if (error && /^NATIVE_FACE_WEBGL_/.test(error.code || '')) throw error; gpu.check(); throw gpu.warmupFailed(); }); } function rawFrame(frame, defaultFormat, maxSide) { if (!frame || !integer(frame.width) || !integer(frame.height) || frame.width < 1 || frame.height < 1 || frame.width > 4096 || frame.height > 4096) throw failure('NATIVE_FACE_FRAME', '相机帧尺寸无效'); var width = frame.width, height = frame.height; var metadata = frame.pixelFormat !== undefined && frame.pixelFormat !== null && frame.pixelFormat !== '' ? frame.pixelFormat : frame.format; var hasFormat = metadata !== undefined && metadata !== null && metadata !== ''; var layout = format(hasFormat ? metadata : defaultFormat); if (!layout) throw failure('NATIVE_FACE_FORMAT', '本地人脸检测仅支持 RGBA/BGRA/ARGB/ABGR 四通道帧'); if ((frame.stride !== undefined && frame.stride !== width * 4) || (frame.bytesPerRow !== undefined && frame.bytesPerRow !== width * 4)) { throw failure('NATIVE_FACE_STRIDE', '暂不支持带额外行填充的相机帧'); } if (frame.rotation !== undefined && frame.rotation !== 0) throw failure('NATIVE_FACE_ROTATION', '暂不支持带旋转标记的相机帧'); var value = frame.data, bytes; if (Object.prototype.toString.call(value) === '[object ArrayBuffer]') bytes = new Uint8Array(value); else if (value && Object.prototype.toString.call(value.buffer) === '[object ArrayBuffer]' && integer(value.byteOffset) && integer(value.byteLength)) { bytes = new Uint8Array(value.buffer, value.byteOffset, value.byteLength); } if (!bytes || bytes.length !== width * height * 4) throw failure('NATIVE_FACE_FORMAT', '相机帧不是紧密排列的四通道数据'); var scale = Math.min(1, maxSide / Math.max(width, height)); var w = Math.max(1, Math.round(width * scale)), h = Math.max(1, Math.round(height * scale)); var rgb = new Int32Array(w * h * 3); var r = layout === 'BGRA' ? 2 : layout === 'ARGB' ? 1 : layout === 'ABGR' ? 3 : 0; var g = layout === 'ARGB' || layout === 'ABGR' ? 2 : 1; var b = layout === 'BGRA' ? 0 : layout === 'ARGB' ? 3 : layout === 'ABGR' ? 1 : 2; for (var y = 0; y < h; y++) for (var x = 0; x < w; x++) { var source = (Math.min(height - 1, Math.floor(y * height / h)) * width + Math.min(width - 1, Math.floor(x * width / w))) * 4; var target = (y * w + x) * 3; rgb[target] = bytes[source + r]; rgb[target + 1] = bytes[source + g]; rgb[target + 2] = bytes[source + b]; } return { rgb: rgb, width: w, height: h, originalWidth: width, originalHeight: height }; } function create(options) { options = options || {}; var maxSide = integer(options.maxSide) && options.maxSide >= 128 && options.maxSide <= 320 ? options.maxSide : 320; var inputSize = integer(options.inputSize) && options.inputSize >= 128 && options.inputSize <= 320 && options.inputSize % 32 === 0 ? options.inputSize : 160; var landmarkModel = options.landmarkModel === 'full' ? 'full' : 'tiny'; var cpuOptimization = options.cpuOptimization !== false; var defaultFormat = options.defaultPixelFormat === undefined ? 'RGBA' : options.defaultPixelFormat; var scoreThreshold = typeof options.scoreThreshold === 'number' && options.scoreThreshold > 0 && options.scoreThreshold < 1 ? options.scoreThreshold : 0.35; var backendKind = options.backend === 'webgl' ? 'webgl' : 'cpu'; var conflict = shared.users > 0 && shared.backendKind !== backendKind; var disposed = false, released = false, pending = 0, busy = false, gpuHandle = null, actualBackend = ''; if (!conflict) { if (!shared.users) shared.backendKind = backendKind; shared.users++; } function check() { if (disposed) throw failure('NATIVE_FACE_DISPOSED', '本地人脸引擎已关闭'); if (conflict) throw failure('NATIVE_FACE_BACKEND_CONFLICT', '请先关闭上一种本地检测引擎'); if (gpuHandle) gpuHandle.check(); } function gpuSummary() { var handle = gpuHandle || (backendKind === 'webgl' && !conflict ? shared.gpu : null); return handle ? handle.summary() : { requested: backendKind === 'webgl', selftest: 'not_run', stage: 'uninitialized', code: 'NONE', contextLost: false, profile: 'default', attemptCount: 0, matmulDraws: 0, convDraws: 0, pixelReads: 0, floatReads: 0, byteReads: 0, selftestMs: 0, warmupMs: 0, matmul: null, conv: null }; } function prepare() { check(); return loadModels(landmarkModel, backendKind, options.platform).then(function (face) { gpuHandle = shared.gpu; check(); return warmup(face, landmarkModel, inputSize, maxSide).then(function () { check(); actualBackend = backendKind; return face; }); }, function (error) { gpuHandle = shared.gpu; if (backendKind === 'webgl' && gpuHandle && !/^NATIVE_FACE_WEBGL_/.test(error && error.code || '')) throw gpuHandle.runtimeFailed(); throw error; }); } function release() { if (!disposed || pending || released) return; released = true; if (conflict) return; shared.users--; if (!shared.users) releaseSession(); } function settle() { pending--; release(); } function init() { if (disposed) return Promise.reject(failure('NATIVE_FACE_DISPOSED', '本地人脸引擎已关闭')); if (conflict) return Promise.reject(failure('NATIVE_FACE_BACKEND_CONFLICT', '请先关闭上一种本地检测引擎')); pending++; return enqueue(prepare).then(function () { check(); return { backend: actualBackend, inputSize: inputSize, maxSide: maxSide, landmarkModel: landmarkModel, cpuOptimization: backendKind === 'cpu' && cpuOptimization ? 'pad-copy' : 'none', gpu: gpuSummary() }; }) .then(function (result) { settle(); return result; }, function (error) { settle(); throw error; }); } function detect(frame) { if (disposed) return Promise.reject(failure('NATIVE_FACE_DISPOSED', '本地人脸引擎已关闭')); if (conflict) return Promise.reject(failure('NATIVE_FACE_BACKEND_CONFLICT', '请先关闭上一种本地检测引擎')); if (busy || shared.busy) return Promise.reject(failure('NATIVE_FACE_BUSY', '本地人脸检测仍在处理中')); busy = true; shared.busy = true; pending++; var tensors = [], started = Date.now(), raw; var timings = { snapshotMs: 0, modelMs: 0, tensorMs: 0, detectorMs: 0, landmarkMs: 0, totalMs: 0 }; function cleanup() { try { tensors.forEach(function (tensor) { try { tensor.dispose(); } catch (_) {} }); } finally { try { if (shared.cpu) shared.cpu.setEnabled(false); } catch (_) {} busy = false; shared.busy = false; settle(); } } // Native camera buffers may be recycled after this call returns. Snapshot // only the downsampled RGB (at most 320px), before any Promise boundary. try { raw = rawFrame(frame, defaultFormat, maxSide); } catch (error) { cleanup(); return Promise.reject(error); } frame = null; timings.snapshotMs = Date.now() - started; var modelStart = Date.now(); var operation = enqueue(function () { return prepare().then(function (face) { check(); if (shared.cpu) shared.cpu.setEnabled(backendKind === 'cpu' && cpuOptimization); timings.modelMs = Date.now() - modelStart; var tensorStart = Date.now(); var tensor = face.tf.tensor3d(raw.rgb, [raw.height, raw.width, 3], 'int32'); tensors.push(tensor); // High-level withFaceLandmarks() calls Canvas even for tensor input. // Keep both networks and face crops on the explicit tensor-only path. var input = new face.NetInput([tensor]); timings.tensorMs = Date.now() - tensorStart; var detectorStart = Date.now(); return Promise.resolve(face.detectAllFaces(input, new face.TinyFaceDetectorOptions({ inputSize: inputSize, scoreThreshold: scoreThreshold }))).then(function (found) { check(); timings.detectorMs = Date.now() - detectorStart; var detections = [], luma = null, chain = Promise.resolve(); // Two faces are enough to reject multi-person captures without running // an unbounded number of landmark networks in one camera frame. found.slice(0, 2).forEach(function (detection) { chain = chain.then(function () { check(); var box = detection.box.floor().clipAtImageBorders(raw.width, raw.height); if (!(box.width > 0 && box.height > 0)) return; var landmarkStart = Date.now(); return face.extractFaceTensors(tensor, [box]).then(function (crops) { var crop = crops[0]; tensors.push(crop); check(); var net = landmarkModel === 'full' ? face.nets.faceLandmark68Net : face.nets.faceLandmark68TinyNet; return net.detectLandmarks(new face.NetInput([crop])).then(function (landmarks) { check(); timings.landmarkMs += Date.now() - landmarkStart; var sx = raw.originalWidth / raw.width, sy = raw.originalHeight / raw.height; if (!detections.length) luma = faceLuma.sample(raw.rgb, raw.width, raw.height, box); detections.push({ detection: { score: detection.score, box: { x: detection.box.x * sx, y: detection.box.y * sy, width: detection.box.width * sx, height: detection.box.height * sy } }, landmarks: { positions: landmarks.positions.map(function (point) { return { x: (point.x + box.x) * sx, y: (point.y + box.y) * sy }; }) } }); }); }); }); }); return chain.then(function () { check(); timings.totalMs = Date.now() - started; return { width: raw.originalWidth, height: raw.originalHeight, detections: detections, luma: luma, faceCount: found.length, elapsedMs: timings.totalMs, backend: actualBackend, inputSize: inputSize, maxSide: maxSide, landmarkModel: landmarkModel, cpuOptimization: backendKind === 'cpu' && cpuOptimization ? 'pad-copy' : 'none', gpu: gpuSummary(), timings: timings }; }); }); }); }); return operation.then(function (result) { cleanup(); return result; }, function (error) { if (backendKind === 'webgl' && gpuHandle && !/^NATIVE_FACE_(?:WEBGL_|DISPOSED|BACKEND_CONFLICT)/.test(error && error.code || '')) { try { error = gpuHandle.runtimeFailed(); } catch (lostError) { error = lostError; } } cleanup(); throw error; }); } return { init: init, detect: detect, dispose: function () { disposed = true; release(); }, stats: function () { var memory = {}; try { if (shared.face) memory = shared.face.tf.memory(); } catch (_) {} return { backend: actualBackend, inputSize: inputSize, maxSide: maxSide, landmarkModel: landmarkModel, cpuOptimization: backendKind === 'cpu' && cpuOptimization ? 'pad-copy' : 'none', gpu: gpuSummary(), modelsLoaded: !conflict && shared.loaded, busy: busy, disposed: disposed, numTensors: memory.numTensors || 0, numBytes: memory.numBytes || 0 }; } }; } module.exports = { create: create };