feat(web): T-WEB-006 BVH/FBX → MediaPipe33 关键点预处理
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { basename, extname, join } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export const ASSIMP_CONVERT_TIMEOUT_MS = 120_000;
|
||||
|
||||
/**
|
||||
* 使用外部 Assimp 可执行文件把 FBX / glTF 转换为 BVH。
|
||||
* Assimp 是**可选**依赖（契约 model-formats.md §2）：
|
||||
* 未安装时抛出可读错误，由调用方落为 preprocessState=failed，不影响 BVH 通路。
|
||||
*/
|
||||
export async function convertToBvhWithAssimp(
|
||||
inputPath: string,
|
||||
outputDir: string,
|
||||
assimpBin: string,
|
||||
): Promise<string> {
|
||||
if (!existsSync(inputPath)) {
|
||||
throw new BadRequestException(`源文件不存在：${inputPath}`);
|
||||
}
|
||||
const outPath = join(outputDir, `${basename(inputPath, extname(inputPath))}-assimp.bvh`);
|
||||
|
||||
try {
|
||||
await execFileAsync(assimpBin, ['export', inputPath, outPath], {
|
||||
timeout: ASSIMP_CONVERT_TIMEOUT_MS,
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
});
|
||||
} catch (error) {
|
||||
const err = error as NodeJS.ErrnoException & { stderr?: string };
|
||||
if (err.code === 'ENOENT') {
|
||||
throw new BadRequestException(
|
||||
`未找到 Assimp 可执行文件「${assimpBin}」。FBX / glTF 动作预处理需要 Assimp（BSD-3-Clause），` +
|
||||
'请安装后设置 ASSIMP_BIN 环境变量；BVH 动作无需 Assimp。',
|
||||
);
|
||||
}
|
||||
const detail = (err.stderr ?? err.message ?? '').toString().trim().slice(0, 500);
|
||||
throw new BadRequestException(`Assimp 转换失败：${detail || '未知错误'}`);
|
||||
}
|
||||
|
||||
if (!existsSync(outPath)) {
|
||||
throw new BadRequestException('Assimp 未产出 BVH 文件，可能源文件中不含骨骼动画');
|
||||
}
|
||||
return outPath;
|
||||
}
|
||||
|
||||
/** 探测 Assimp 是否可用（用于 /health 与部署自检） */
|
||||
export async function detectAssimp(assimpBin: string): Promise<{ available: boolean; version?: string; error?: string }> {
|
||||
try {
|
||||
const { stdout } = await execFileAsync(assimpBin, ['version'], { timeout: 10_000 });
|
||||
const version = stdout.split(/\r?\n/).find((line) => line.toLowerCase().includes('assimp')) ?? stdout.trim();
|
||||
return { available: true, version: version.trim() };
|
||||
} catch (error) {
|
||||
const err = error as NodeJS.ErrnoException;
|
||||
return {
|
||||
available: false,
|
||||
error: err.code === 'ENOENT' ? `未安装（${assimpBin}）` : (err.message ?? '未知错误'),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { Mat3, Vec3, composeRotation, IDENTITY3, matMul, matVec } from './math3d';
|
||||
|
||||
export interface BvhJoint {
|
||||
name: string;
|
||||
offset: Vec3;
|
||||
channels: string[];
|
||||
/** 在该帧通道值数组中的起始下标 */
|
||||
channelIndex: number;
|
||||
parent: BvhJoint | null;
|
||||
children: BvhJoint[];
|
||||
}
|
||||
|
||||
export interface BvhMotion {
|
||||
root: BvhJoint;
|
||||
/** 先序展开的全部关节（不含 End Site） */
|
||||
joints: BvhJoint[];
|
||||
frameCount: number;
|
||||
frameTime: number;
|
||||
/** frames[frameIndex][channelIndex] = 通道值 */
|
||||
frames: number[][];
|
||||
channelCount: number;
|
||||
/** 关节名（小写）→ 关节 */
|
||||
byName: Map<string, BvhJoint>;
|
||||
}
|
||||
|
||||
const CHANNEL_AXIS = /^([XYZ])(position|rotation)$/i;
|
||||
|
||||
function tokenize(text: string): string[] {
|
||||
const tokens = text.match(/[^\s{}]+|[{}]/g);
|
||||
return tokens ?? [];
|
||||
}
|
||||
|
||||
/**
|
||||
* BVH 解析器（纯 TypeScript，无原生依赖）。
|
||||
* 仅支持标准 BVH：HIERARCHY + MOTION，End Site 视为叶子不产生关节。
|
||||
*/
|
||||
export function parseBvh(text: string): BvhMotion {
|
||||
const tokens = tokenize(text);
|
||||
let pos = 0;
|
||||
|
||||
const peek = (): string | undefined => tokens[pos];
|
||||
const next = (): string => {
|
||||
const token = tokens[pos];
|
||||
if (token === undefined) throw new BadRequestException('BVH 解析失败：内容意外结束');
|
||||
pos += 1;
|
||||
return token;
|
||||
};
|
||||
const expect = (value: string): void => {
|
||||
const token = next();
|
||||
if (token !== value) {
|
||||
throw new BadRequestException(`BVH 解析失败：期望 "${value}"，实际 "${token}"`);
|
||||
}
|
||||
};
|
||||
|
||||
if ((peek() ?? '').toUpperCase() !== 'HIERARCHY') {
|
||||
throw new BadRequestException('BVH 解析失败：缺少 HIERARCHY 段');
|
||||
}
|
||||
next();
|
||||
|
||||
const joints: BvhJoint[] = [];
|
||||
let channelCursor = 0;
|
||||
|
||||
const parseNode = (parent: BvhJoint | null): BvhJoint => {
|
||||
const kind = next().toUpperCase();
|
||||
if (kind !== 'ROOT' && kind !== 'JOINT') {
|
||||
throw new BadRequestException(`BVH 解析失败：期望 ROOT/JOINT，实际 ${kind}`);
|
||||
}
|
||||
const name = next();
|
||||
const joint: BvhJoint = {
|
||||
name,
|
||||
offset: [0, 0, 0],
|
||||
channels: [],
|
||||
channelIndex: channelCursor,
|
||||
parent,
|
||||
children: [],
|
||||
};
|
||||
if (parent) parent.children.push(joint);
|
||||
joints.push(joint);
|
||||
|
||||
expect('{');
|
||||
for (;;) {
|
||||
const token = next();
|
||||
const upper = token.toUpperCase();
|
||||
if (upper === '}') break;
|
||||
if (upper === 'OFFSET') {
|
||||
joint.offset = [Number(next()), Number(next()), Number(next())];
|
||||
if (joint.offset.some((n) => !Number.isFinite(n))) {
|
||||
throw new BadRequestException(`BVH 解析失败：关节 ${name} 的 OFFSET 非法`);
|
||||
}
|
||||
} else if (upper === 'CHANNELS') {
|
||||
const count = Number(next());
|
||||
if (!Number.isInteger(count) || count <= 0) {
|
||||
throw new BadRequestException(`BVH 解析失败：关节 ${name} 的通道数非法`);
|
||||
}
|
||||
const list: string[] = [];
|
||||
for (let i = 0; i < count; i += 1) {
|
||||
const channel = next();
|
||||
if (!CHANNEL_AXIS.test(channel)) {
|
||||
throw new BadRequestException(`BVH 解析失败：未知通道 ${channel}`);
|
||||
}
|
||||
list.push(channel);
|
||||
}
|
||||
joint.channels = list;
|
||||
joint.channelIndex = channelCursor;
|
||||
channelCursor += count;
|
||||
} else if (upper === 'JOINT') {
|
||||
pos -= 1;
|
||||
parseNode(joint);
|
||||
} else if (upper === 'END') {
|
||||
// End Site { OFFSET x y z }
|
||||
expect('Site');
|
||||
expect('{');
|
||||
for (;;) {
|
||||
const inner = next();
|
||||
if (inner === '}') break;
|
||||
if (inner.toUpperCase() === 'OFFSET') {
|
||||
next();
|
||||
next();
|
||||
next();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw new BadRequestException(`BVH 解析失败：关节 ${name} 中出现未知标记 ${token}`);
|
||||
}
|
||||
}
|
||||
return joint;
|
||||
};
|
||||
|
||||
const root = parseNode(null);
|
||||
|
||||
expect('MOTION');
|
||||
expect('Frames:');
|
||||
const frameCount = Number(next());
|
||||
expect('Frame');
|
||||
expect('Time:');
|
||||
const frameTime = Number(next());
|
||||
if (!Number.isInteger(frameCount) || frameCount <= 0) {
|
||||
throw new BadRequestException('BVH 解析失败：Frames 数量非法');
|
||||
}
|
||||
if (!Number.isFinite(frameTime) || frameTime <= 0) {
|
||||
throw new BadRequestException('BVH 解析失败：Frame Time 非法');
|
||||
}
|
||||
|
||||
const flat: number[] = [];
|
||||
while (pos < tokens.length) {
|
||||
const value = Number(tokens[pos]);
|
||||
pos += 1;
|
||||
if (Number.isFinite(value)) flat.push(value);
|
||||
}
|
||||
|
||||
const expected = frameCount * channelCursor;
|
||||
if (flat.length < expected) {
|
||||
throw new BadRequestException(
|
||||
`BVH 解析失败：期望 ${expected} 个通道值（${frameCount} 帧 × ${channelCursor} 通道），实际 ${flat.length}`,
|
||||
);
|
||||
}
|
||||
|
||||
const frames: number[][] = [];
|
||||
for (let f = 0; f < frameCount; f += 1) {
|
||||
frames.push(flat.slice(f * channelCursor, (f + 1) * channelCursor));
|
||||
}
|
||||
|
||||
const byName = new Map<string, BvhJoint>();
|
||||
for (const joint of joints) {
|
||||
const key = joint.name.toLowerCase();
|
||||
if (!byName.has(key)) byName.set(key, joint);
|
||||
}
|
||||
|
||||
return { root, joints, frameCount, frameTime, frames, channelCount: channelCursor, byName };
|
||||
}
|
||||
|
||||
export interface JointWorld {
|
||||
name: string;
|
||||
position: Vec3;
|
||||
rotation: Mat3;
|
||||
}
|
||||
|
||||
/** 读取关节在某帧的通道值，并按通道名拆分为平移/旋转 */
|
||||
function readChannels(joint: BvhJoint, frame: number[]): { translation: Vec3; rotations: Array<{ axis: string; value: number }> } {
|
||||
const translation: Vec3 = [0, 0, 0];
|
||||
const rotations: Array<{ axis: string; value: number }> = [];
|
||||
joint.channels.forEach((channel, i) => {
|
||||
const match = CHANNEL_AXIS.exec(channel);
|
||||
const value = frame[joint.channelIndex + i] ?? 0;
|
||||
if (!match) return;
|
||||
const axis = match[1].toUpperCase();
|
||||
if (match[2].toLowerCase() === 'position') {
|
||||
if (axis === 'X') translation[0] = value;
|
||||
else if (axis === 'Y') translation[1] = value;
|
||||
else translation[2] = value;
|
||||
} else {
|
||||
rotations.push({ axis, value });
|
||||
}
|
||||
});
|
||||
return { translation, rotations };
|
||||
}
|
||||
|
||||
/**
|
||||
* 逐帧前向运动学，返回关节世界坐标。
|
||||
* 世界变换 = 父世界变换 × T(offset + 自身平移) × R(通道旋转)。
|
||||
*/
|
||||
export function forwardKinematics(motion: BvhMotion, frameIndex: number): Map<string, JointWorld> {
|
||||
const frame = motion.frames[frameIndex];
|
||||
const out = new Map<string, JointWorld>();
|
||||
|
||||
const walk = (joint: BvhJoint, parentR: Mat3, parentT: Vec3): void => {
|
||||
const { translation, rotations } = readChannels(joint, frame);
|
||||
const localR = composeRotation(rotations);
|
||||
// 旋转矩阵相乘：worldR = parentR × localR
|
||||
const worldR = matMul(parentR, localR);
|
||||
const offset = joint.offset;
|
||||
const shifted: Vec3 = [
|
||||
offset[0] + translation[0],
|
||||
offset[1] + translation[1],
|
||||
offset[2] + translation[2],
|
||||
];
|
||||
const rotated = matVec(parentR, shifted);
|
||||
const worldT: Vec3 = [parentT[0] + rotated[0], parentT[1] + rotated[1], parentT[2] + rotated[2]];
|
||||
|
||||
out.set(joint.name.toLowerCase(), { name: joint.name, position: worldT, rotation: worldR });
|
||||
for (const child of joint.children) walk(child, worldR, worldT);
|
||||
};
|
||||
|
||||
walk(motion.root, IDENTITY3, [0, 0, 0]);
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { MP } from './mediapipe33';
|
||||
|
||||
/**
|
||||
* 默认 BVH 关节名映射 —— 契约 model-formats.md §2.2。
|
||||
* 按序匹配（不区分大小写），第一个命中的关节被采用。
|
||||
* 实际项目中可在本表追加；追加不改变契约，仅提升映射覆盖率。
|
||||
*/
|
||||
export const DEFAULT_JOINT_MAP: Readonly<Record<number, readonly string[]>> = Object.freeze({
|
||||
[MP.NOSE]: ['head', 'head_nub', 'neck1', 'skull'],
|
||||
[MP.LEFT_SHOULDER]: ['leftshoulder', 'leftarm', 'lshoulder', 'shoulder_l', 'l_arm'],
|
||||
[MP.RIGHT_SHOULDER]: ['rightshoulder', 'rightarm', 'rshoulder', 'shoulder_r', 'r_arm'],
|
||||
[MP.LEFT_ELBOW]: ['leftforearm', 'leftelbow', 'lelbow', 'forearm_l'],
|
||||
[MP.RIGHT_ELBOW]: ['rightforearm', 'rightelbow', 'relbow', 'forearm_r'],
|
||||
[MP.LEFT_WRIST]: ['lefthand', 'leftwrist', 'lhand', 'wrist_l'],
|
||||
[MP.RIGHT_WRIST]: ['righthand', 'rightwrist', 'rhand', 'wrist_r'],
|
||||
[MP.LEFT_HIP]: ['leftupleg', 'lefthip', 'lhipjoint', 'hip_l', 'l_thigh'],
|
||||
[MP.RIGHT_HIP]: ['rightupleg', 'righthip', 'rhipjoint', 'hip_r', 'r_thigh'],
|
||||
[MP.LEFT_KNEE]: ['leftleg', 'leftknee', 'lknee', 'knee_l', 'l_shin'],
|
||||
[MP.RIGHT_KNEE]: ['rightleg', 'rightknee', 'rknee', 'knee_r', 'r_shin'],
|
||||
[MP.LEFT_ANKLE]: ['leftfoot', 'leftankle', 'lankle', 'ankle_l'],
|
||||
[MP.RIGHT_ANKLE]: ['rightfoot', 'rightankle', 'rankle', 'ankle_r'],
|
||||
[MP.LEFT_FOOT_INDEX]: ['lefttoebase', 'lefttoe', 'ltoe', 'toe_l'],
|
||||
[MP.RIGHT_FOOT_INDEX]: ['righttoebase', 'righttoe', 'rtoe', 'toe_r'],
|
||||
});
|
||||
|
||||
/** 直接映射的置信度 */
|
||||
export const CONF_DIRECT = 1.0;
|
||||
/** 由父节点推算的置信度 —— 契约 model-formats.md §2.2 */
|
||||
export const CONF_DERIVED = 0.5;
|
||||
/** 完全无法推算 */
|
||||
export const CONF_MISSING = 0.0;
|
||||
@@ -0,0 +1,147 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { KEYPOINT_COUNT, KEYPOINT_FORMAT, KeypointFrame } from './mediapipe33';
|
||||
|
||||
/** 二进制帧总长度 —— 契约 websocket-keypoints.md §2（冻结） */
|
||||
export const BINARY_FRAME_BYTES = 239;
|
||||
export const MAGIC = 0x4f44; // 'OD'
|
||||
export const PROTOCOL_VERSION = 0x01;
|
||||
const SCALE = 32767;
|
||||
|
||||
export interface DecodedPoseFrame {
|
||||
flags: number;
|
||||
frameId: number;
|
||||
points: Array<[number, number, number, number]>;
|
||||
}
|
||||
|
||||
const clampInt16 = (v: number): number => (v < -32768 ? -32768 : v > 32767 ? 32767 : Math.round(v));
|
||||
|
||||
/** 编码 239 字节二进制帧 —— Android 端参考实现 */
|
||||
export function encodeBinaryFrame(frameId: number, points: Array<[number, number, number, number]>, flags = 0): Buffer {
|
||||
if (points.length !== KEYPOINT_COUNT) {
|
||||
throw new BadRequestException(`关键点数量必须为 ${KEYPOINT_COUNT}，实际 ${points.length}`);
|
||||
}
|
||||
const buf = Buffer.alloc(BINARY_FRAME_BYTES);
|
||||
buf.writeUInt16LE(MAGIC, 0);
|
||||
buf.writeUInt8(PROTOCOL_VERSION, 2);
|
||||
buf.writeUInt8(flags & 0xff, 3);
|
||||
buf.writeUInt32LE(frameId >>> 0, 4);
|
||||
for (let i = 0; i < KEYPOINT_COUNT; i += 1) {
|
||||
const offset = 8 + i * 7;
|
||||
const [x, y, z, conf] = points[i];
|
||||
buf.writeInt16LE(clampInt16(x * SCALE), offset);
|
||||
buf.writeInt16LE(clampInt16(y * SCALE), offset + 2);
|
||||
buf.writeInt16LE(clampInt16(z * SCALE), offset + 4);
|
||||
buf.writeUInt8(Math.max(0, Math.min(255, Math.round(conf * 255))), offset + 6);
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
/** 解码 239 字节二进制帧 —— Windows 端参考实现（T-WIN-004） */
|
||||
export function decodeBinaryFrame(buf: Buffer): DecodedPoseFrame {
|
||||
if (buf.length !== BINARY_FRAME_BYTES) {
|
||||
throw new BadRequestException(`帧长度必须为 ${BINARY_FRAME_BYTES} 字节，实际 ${buf.length}`);
|
||||
}
|
||||
const magic = buf.readUInt16LE(0);
|
||||
if (magic !== MAGIC) throw new BadRequestException('帧 magic 校验失败');
|
||||
const version = buf.readUInt8(2);
|
||||
if (version !== PROTOCOL_VERSION) {
|
||||
throw new BadRequestException(`不支持的协议版本 ${version}`);
|
||||
}
|
||||
const flags = buf.readUInt8(3);
|
||||
const frameId = buf.readUInt32LE(4);
|
||||
const points: Array<[number, number, number, number]> = [];
|
||||
for (let i = 0; i < KEYPOINT_COUNT; i += 1) {
|
||||
const offset = 8 + i * 7;
|
||||
points.push([
|
||||
buf.readInt16LE(offset) / SCALE,
|
||||
buf.readInt16LE(offset + 2) / SCALE,
|
||||
buf.readInt16LE(offset + 4) / SCALE,
|
||||
buf.readUInt8(offset + 6) / 255,
|
||||
]);
|
||||
}
|
||||
return { flags, frameId, points };
|
||||
}
|
||||
|
||||
/** 关键点序列 → NDJSON（契约 model-formats.md §2.1） */
|
||||
export function serializeFramesNdjson(frames: KeypointFrame[]): string {
|
||||
return frames
|
||||
.map((frame) =>
|
||||
JSON.stringify({
|
||||
frame: frame.frame,
|
||||
timeMs: frame.timeMs,
|
||||
points: frame.points,
|
||||
angles: frame.angles,
|
||||
}),
|
||||
)
|
||||
.join('\n')
|
||||
.concat('\n');
|
||||
}
|
||||
|
||||
export function parseFramesNdjson(text: string): KeypointFrame[] {
|
||||
return text
|
||||
.split(/\r?\n/)
|
||||
.filter((line) => line.trim().length > 0)
|
||||
.map((line, index) => {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(line);
|
||||
} catch {
|
||||
throw new BadRequestException(`NDJSON 第 ${index + 1} 行不是合法 JSON`);
|
||||
}
|
||||
return validateFrame(parsed, index);
|
||||
});
|
||||
}
|
||||
|
||||
function validateFrame(input: unknown, index: number): KeypointFrame {
|
||||
const obj = input as Partial<KeypointFrame>;
|
||||
if (!obj || typeof obj !== 'object') {
|
||||
throw new BadRequestException(`第 ${index} 帧不是对象`);
|
||||
}
|
||||
if (!Array.isArray(obj.points) || obj.points.length !== KEYPOINT_COUNT) {
|
||||
throw new BadRequestException(`第 ${index} 帧的 points 必须为 ${KEYPOINT_COUNT} 项`);
|
||||
}
|
||||
const points = obj.points.map((point, i) => {
|
||||
if (!Array.isArray(point) || point.length !== 4 || point.some((v) => typeof v !== 'number')) {
|
||||
throw new BadRequestException(`第 ${index} 帧第 ${i} 个点必须为 [x, y, z, confidence] 四个数字`);
|
||||
}
|
||||
return [point[0], point[1], point[2], point[3]] as [number, number, number, number];
|
||||
});
|
||||
const angles = Array.isArray(obj.angles) && obj.angles.every((v) => typeof v === 'number')
|
||||
? (obj.angles as number[])
|
||||
: new Array(9).fill(0);
|
||||
return {
|
||||
frame: typeof obj.frame === 'number' ? obj.frame : index,
|
||||
timeMs: typeof obj.timeMs === 'number' ? obj.timeMs : 0,
|
||||
points,
|
||||
angles,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验并规范化「已是 mediapipe33 结构」的 JSON 关键点文件。
|
||||
* 接受两种形态：{ format, fps, frames: [...] } 或裸数组。
|
||||
*/
|
||||
export function parseKeypointsJson(text: string): { frames: KeypointFrame[]; fps?: number } {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(text);
|
||||
} catch {
|
||||
throw new BadRequestException('关键点 JSON 解析失败');
|
||||
}
|
||||
|
||||
if (Array.isArray(parsed)) {
|
||||
return { frames: parsed.map((f, i) => validateFrame(f, i)) };
|
||||
}
|
||||
|
||||
const obj = parsed as { format?: string; fps?: number; frames?: unknown };
|
||||
if (!Array.isArray(obj.frames)) {
|
||||
throw new BadRequestException('关键点 JSON 必须为数组或包含 frames 数组的对象');
|
||||
}
|
||||
if (obj.format && obj.format !== KEYPOINT_FORMAT) {
|
||||
throw new BadRequestException(`不支持的 keypointFormat：${obj.format}，期望 ${KEYPOINT_FORMAT}`);
|
||||
}
|
||||
return {
|
||||
frames: obj.frames.map((f, i) => validateFrame(f, i)),
|
||||
fps: typeof obj.fps === 'number' ? obj.fps : undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
export type Vec3 = [number, number, number];
|
||||
|
||||
/** 3×3 行主序矩阵 */
|
||||
export type Mat3 = [number, number, number, number, number, number, number, number, number];
|
||||
|
||||
export const IDENTITY3: Mat3 = [1, 0, 0, 0, 1, 0, 0, 0, 1];
|
||||
|
||||
export function vAdd(a: Vec3, b: Vec3): Vec3 {
|
||||
return [a[0] + b[0], a[1] + b[1], a[2] + b[2]];
|
||||
}
|
||||
|
||||
export function vSub(a: Vec3, b: Vec3): Vec3 {
|
||||
return [a[0] - b[0], a[1] - b[1], a[2] - b[2]];
|
||||
}
|
||||
|
||||
export function vScale(a: Vec3, s: number): Vec3 {
|
||||
return [a[0] * s, a[1] * s, a[2] * s];
|
||||
}
|
||||
|
||||
export function vMid(a: Vec3, b: Vec3): Vec3 {
|
||||
return [(a[0] + b[0]) / 2, (a[1] + b[1]) / 2, (a[2] + b[2]) / 2];
|
||||
}
|
||||
|
||||
export function vLen(a: Vec3): number {
|
||||
return Math.hypot(a[0], a[1], a[2]);
|
||||
}
|
||||
|
||||
export function vNorm(a: Vec3): Vec3 {
|
||||
const len = vLen(a);
|
||||
if (len < 1e-9) return [0, 0, 0];
|
||||
return [a[0] / len, a[1] / len, a[2] / len];
|
||||
}
|
||||
|
||||
export function vDot(a: Vec3, b: Vec3): number {
|
||||
return a[0] * b[0] + a[1] * b[1] + a[2] * b[2];
|
||||
}
|
||||
|
||||
/** 向量 ba 与 bc 夹角的余弦，落在 [-1, 1] —— 契约 skeleton-topology.md §5 */
|
||||
export function angleCos(a: Vec3, b: Vec3, c: Vec3): number {
|
||||
const ba = vSub(a, b);
|
||||
const bc = vSub(c, b);
|
||||
const denom = vLen(ba) * vLen(bc);
|
||||
if (denom < 1e-9) return 0;
|
||||
return Math.max(-1, Math.min(1, vDot(ba, bc) / denom));
|
||||
}
|
||||
|
||||
export function matMul(a: Mat3, b: Mat3): Mat3 {
|
||||
const out = new Array<number>(9);
|
||||
for (let r = 0; r < 3; r += 1) {
|
||||
for (let c = 0; c < 3; c += 1) {
|
||||
out[r * 3 + c] = a[r * 3] * b[c] + a[r * 3 + 1] * b[3 + c] + a[r * 3 + 2] * b[6 + c];
|
||||
}
|
||||
}
|
||||
return out as Mat3;
|
||||
}
|
||||
|
||||
export function matVec(m: Mat3, v: Vec3): Vec3 {
|
||||
return [
|
||||
m[0] * v[0] + m[1] * v[1] + m[2] * v[2],
|
||||
m[3] * v[0] + m[4] * v[1] + m[5] * v[2],
|
||||
m[6] * v[0] + m[7] * v[1] + m[8] * v[2],
|
||||
];
|
||||
}
|
||||
|
||||
const DEG2RAD = Math.PI / 180;
|
||||
|
||||
/** 绕单轴旋转矩阵，axis ∈ {X, Y, Z} */
|
||||
export function rotationMatrix(axis: string, degrees: number): Mat3 {
|
||||
const rad = degrees * DEG2RAD;
|
||||
const c = Math.cos(rad);
|
||||
const s = Math.sin(rad);
|
||||
switch (axis.toUpperCase()) {
|
||||
case 'X':
|
||||
return [1, 0, 0, 0, c, -s, 0, s, c];
|
||||
case 'Y':
|
||||
return [c, 0, s, 0, 1, 0, -s, 0, c];
|
||||
case 'Z':
|
||||
return [c, -s, 0, s, c, 0, 0, 0, 1];
|
||||
default:
|
||||
return IDENTITY3;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 由通道序列合成局部旋转矩阵。
|
||||
* BVH 约定：按通道声明顺序依次左乘，即 R = R1 × R2 × R3（内旋）。
|
||||
*/
|
||||
export function composeRotation(channels: ReadonlyArray<{ axis: string; value: number }>): Mat3 {
|
||||
let out: Mat3 = IDENTITY3;
|
||||
for (const { axis, value } of channels) {
|
||||
out = matMul(out, rotationMatrix(axis, value));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** 世界变换：旋转 + 平移 */
|
||||
export interface Transform {
|
||||
r: Mat3;
|
||||
t: Vec3;
|
||||
}
|
||||
|
||||
export const IDENTITY_TRANSFORM: Transform = { r: IDENTITY3, t: [0, 0, 0] };
|
||||
|
||||
export function applyTransform(parent: Transform, offset: Vec3, localR: Mat3, extraT?: Vec3): Transform {
|
||||
return {
|
||||
r: matMul(parent.r, localR),
|
||||
t: vAdd(parent.t, matVec(parent.r, vAdd(offset, extraT ?? [0, 0, 0]))),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
/** 33 点索引常量 —— 契约 skeleton-topology.md §1 */
|
||||
export const MP = {
|
||||
NOSE: 0,
|
||||
LEFT_EYE_INNER: 1,
|
||||
LEFT_EYE: 2,
|
||||
LEFT_EYE_OUTER: 3,
|
||||
RIGHT_EYE_INNER: 4,
|
||||
RIGHT_EYE: 5,
|
||||
RIGHT_EYE_OUTER: 6,
|
||||
LEFT_EAR: 7,
|
||||
RIGHT_EAR: 8,
|
||||
MOUTH_LEFT: 9,
|
||||
MOUTH_RIGHT: 10,
|
||||
LEFT_SHOULDER: 11,
|
||||
RIGHT_SHOULDER: 12,
|
||||
LEFT_ELBOW: 13,
|
||||
RIGHT_ELBOW: 14,
|
||||
LEFT_WRIST: 15,
|
||||
RIGHT_WRIST: 16,
|
||||
LEFT_PINKY: 17,
|
||||
RIGHT_PINKY: 18,
|
||||
LEFT_INDEX: 19,
|
||||
RIGHT_INDEX: 20,
|
||||
LEFT_THUMB: 21,
|
||||
RIGHT_THUMB: 22,
|
||||
LEFT_HIP: 23,
|
||||
RIGHT_HIP: 24,
|
||||
LEFT_KNEE: 25,
|
||||
RIGHT_KNEE: 26,
|
||||
LEFT_ANKLE: 27,
|
||||
RIGHT_ANKLE: 28,
|
||||
LEFT_HEEL: 29,
|
||||
RIGHT_HEEL: 30,
|
||||
LEFT_FOOT_INDEX: 31,
|
||||
RIGHT_FOOT_INDEX: 32,
|
||||
} as const;
|
||||
|
||||
export const KEYPOINT_COUNT = 33;
|
||||
|
||||
/** 虚拟关节索引 —— skeleton-topology.md §2 */
|
||||
export const VIRTUAL = { NECK: 33, PELVIS: 34 } as const;
|
||||
|
||||
/** 父节点表 —— skeleton-topology.md §3，索引到 34 */
|
||||
export const PARENT_TABLE: readonly number[] = [
|
||||
33, // 0 nose
|
||||
0, // 1 left_eye_inner
|
||||
0, // 2 left_eye
|
||||
0, // 3 left_eye_outer
|
||||
0, // 4 right_eye_inner
|
||||
0, // 5 right_eye
|
||||
0, // 6 right_eye_outer
|
||||
3, // 7 left_ear
|
||||
6, // 8 right_ear
|
||||
0, // 9 mouth_left
|
||||
0, // 10 mouth_right
|
||||
33, // 11 left_shoulder
|
||||
33, // 12 right_shoulder
|
||||
11, // 13 left_elbow
|
||||
12, // 14 right_elbow
|
||||
13, // 15 left_wrist
|
||||
14, // 16 right_wrist
|
||||
15, // 17 left_pinky
|
||||
16, // 18 right_pinky
|
||||
15, // 19 left_index
|
||||
16, // 20 right_index
|
||||
15, // 21 left_thumb
|
||||
16, // 22 right_thumb
|
||||
34, // 23 left_hip
|
||||
34, // 24 right_hip
|
||||
23, // 25 left_knee
|
||||
24, // 26 right_knee
|
||||
25, // 27 left_ankle
|
||||
26, // 28 right_ankle
|
||||
27, // 29 left_heel
|
||||
28, // 30 right_heel
|
||||
29, // 31 left_foot_index
|
||||
30, // 32 right_foot_index
|
||||
34, // 33 neck
|
||||
-1, // 34 pelvis (world root)
|
||||
];
|
||||
|
||||
/** 关节角特征使用的 (a, b, c) 三元组，顺序冻结 —— skeleton-topology.md §5 */
|
||||
export const ANGLE_TRIPLETS: ReadonlyArray<readonly [number, number, number]> = [
|
||||
[MP.LEFT_ELBOW, MP.LEFT_SHOULDER, MP.LEFT_HIP], // shoulder_L
|
||||
[MP.RIGHT_ELBOW, MP.RIGHT_SHOULDER, MP.RIGHT_HIP], // shoulder_R
|
||||
[MP.LEFT_SHOULDER, MP.LEFT_ELBOW, MP.LEFT_WRIST], // elbow_L
|
||||
[MP.RIGHT_SHOULDER, MP.RIGHT_ELBOW, MP.RIGHT_WRIST], // elbow_R
|
||||
[MP.LEFT_SHOULDER, MP.LEFT_HIP, MP.LEFT_KNEE], // hip_L
|
||||
[MP.RIGHT_SHOULDER, MP.RIGHT_HIP, MP.RIGHT_KNEE], // hip_R
|
||||
[MP.LEFT_HIP, MP.LEFT_KNEE, MP.LEFT_ANKLE], // knee_L
|
||||
[MP.RIGHT_HIP, MP.RIGHT_KNEE, MP.RIGHT_ANKLE], // knee_R
|
||||
[MP.NOSE, VIRTUAL.NECK, VIRTUAL.PELVIS], // spine/neck
|
||||
];
|
||||
|
||||
export const ANGLE_NAMES = [
|
||||
'shoulder_L',
|
||||
'shoulder_R',
|
||||
'elbow_L',
|
||||
'elbow_R',
|
||||
'hip_L',
|
||||
'hip_R',
|
||||
'knee_L',
|
||||
'knee_R',
|
||||
'spine',
|
||||
] as const;
|
||||
|
||||
/** 单帧关键点：[x, y, z, confidence] */
|
||||
export type Keypoint = [number, number, number, number];
|
||||
|
||||
export interface KeypointFrame {
|
||||
frame: number;
|
||||
timeMs: number;
|
||||
points: Keypoint[];
|
||||
angles: number[];
|
||||
}
|
||||
|
||||
export const KEYPOINT_FORMAT = 'mediapipe33@0.1.0';
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { Motion } from '../entities/motion.entity';
|
||||
import { StorageModule } from '../storage/storage.module';
|
||||
import { PreprocessService } from './preprocess.service';
|
||||
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([Motion]), StorageModule],
|
||||
providers: [PreprocessService],
|
||||
exports: [PreprocessService],
|
||||
})
|
||||
export class PreprocessModule {}
|
||||
@@ -0,0 +1,144 @@
|
||||
import { BadRequestException, Injectable, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { Repository } from 'typeorm';
|
||||
import { AssetKind, MotionFormat, PreprocessState } from '../common/enums';
|
||||
import { AppConfig } from '../config/configuration';
|
||||
import { Motion } from '../entities/motion.entity';
|
||||
import { StorageService } from '../storage/storage.service';
|
||||
import { parseBvh } from './bvh-parser';
|
||||
import { parseKeypointsJson, serializeFramesNdjson } from './keypoint-codec';
|
||||
import { KEYPOINT_FORMAT, KeypointFrame } from './mediapipe33';
|
||||
import { bvhToKeypointFrames } from './retarget';
|
||||
import { convertToBvhWithAssimp } from './assimp';
|
||||
|
||||
export interface PreprocessOutcome {
|
||||
frames: number;
|
||||
fps: number;
|
||||
durationMs: number;
|
||||
format: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* T-WEB-006：BVH / FBX → MediaPipe 33 点关键点序列预处理。
|
||||
*
|
||||
* - BVH：内置纯 TS 解析器 + FK + 重定向，无外部依赖
|
||||
* - glTF / GLB / FBX：调用可选的外部 Assimp 转为 BVH 后再走同一通路
|
||||
* - keypoints-json：直通校验与规范化
|
||||
*/
|
||||
@Injectable()
|
||||
export class PreprocessService {
|
||||
private readonly logger = new Logger(PreprocessService.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(Motion) private readonly motions: Repository<Motion>,
|
||||
private readonly storage: StorageService,
|
||||
private readonly config: AppConfig,
|
||||
) {}
|
||||
|
||||
/** 执行预处理并落库；异常会被记录到 preprocessError 后重新抛出 */
|
||||
async run(motion: Motion): Promise<Motion> {
|
||||
const started = Date.now();
|
||||
motion.preprocessState = PreprocessState.Running;
|
||||
motion.preprocessError = null;
|
||||
await this.motions.save(motion);
|
||||
|
||||
try {
|
||||
const result = await this.convert(motion);
|
||||
const ndjson = serializeFramesNdjson(result.frames);
|
||||
|
||||
const stored = await this.storage.persistBuffer(
|
||||
AssetKind.Keypoints,
|
||||
ndjson,
|
||||
`motion-${motion.id}.ndjson`,
|
||||
'application/x-ndjson',
|
||||
);
|
||||
|
||||
motion.keypointFormat = KEYPOINT_FORMAT;
|
||||
motion.keypointAssetId = stored.asset.id;
|
||||
motion.keypointFrames = result.frames.length;
|
||||
motion.fps = result.fps;
|
||||
motion.frameCount = result.frames.length;
|
||||
motion.durationMs = result.durationMs;
|
||||
motion.preprocessState = PreprocessState.Done;
|
||||
motion.preprocessError = null;
|
||||
|
||||
const saved = await this.motions.save(motion);
|
||||
this.logger.log(
|
||||
`动作 #${motion.id} 预处理完成：${result.frames.length} 帧，${Date.now() - started}ms`,
|
||||
);
|
||||
return saved;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
motion.preprocessState = PreprocessState.Failed;
|
||||
motion.preprocessError = message.slice(0, 2000);
|
||||
await this.motions.save(motion);
|
||||
this.logger.error(`动作 #${motion.id} 预处理失败：${message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** 读取已完成预处理的关键点 NDJSON */
|
||||
async readKeypointsNdjson(motion: Motion): Promise<string> {
|
||||
if (motion.preprocessState !== PreprocessState.Done || !motion.keypointAssetId) {
|
||||
throw new BadRequestException('该动作尚未完成关键点预处理，请先调用 /preprocess');
|
||||
}
|
||||
const asset = await this.storage.findById(motion.keypointAssetId);
|
||||
if (!asset) throw new NotFoundException('关键点文件已丢失');
|
||||
return this.storage.readAssetText(asset);
|
||||
}
|
||||
|
||||
private async convert(
|
||||
motion: Motion,
|
||||
): Promise<{ frames: KeypointFrame[]; fps: number; durationMs: number }> {
|
||||
const asset = await this.storage.findById(motion.fileAssetId);
|
||||
if (!asset) throw new BadRequestException('动作源文件缺失，无法预处理');
|
||||
const sourcePath = this.storage.absolutePath(asset);
|
||||
|
||||
let workDir: string | null = null;
|
||||
try {
|
||||
if (motion.format === MotionFormat.Bvh) {
|
||||
const text = await readFile(sourcePath, 'utf8');
|
||||
return this.fromBvh(text, motion.fps ?? undefined);
|
||||
}
|
||||
|
||||
if (motion.format === MotionFormat.KeypointsJson) {
|
||||
const text = await readFile(sourcePath, 'utf8');
|
||||
const { frames, fps } = parseKeypointsJson(text);
|
||||
if (frames.length === 0) throw new BadRequestException('关键点 JSON 中没有任何帧');
|
||||
const effectiveFps = fps ?? motion.fps ?? 30;
|
||||
return {
|
||||
frames: frames.map((frame, index) => ({
|
||||
...frame,
|
||||
frame: index,
|
||||
timeMs: frame.timeMs || Math.round((index / effectiveFps) * 1000 * 1000) / 1000,
|
||||
})),
|
||||
fps: effectiveFps,
|
||||
durationMs: Math.round((frames.length / effectiveFps) * 1000),
|
||||
};
|
||||
}
|
||||
|
||||
// FBX / glTF / GLB：交给可选的外部 Assimp
|
||||
workDir = await mkdtemp(join(tmpdir(), 'opendance-assimp-'));
|
||||
const bvhPath = await convertToBvhWithAssimp(sourcePath, workDir, this.config.assimpBin);
|
||||
const text = await readFile(bvhPath, 'utf8');
|
||||
return this.fromBvh(text, motion.fps ?? undefined);
|
||||
} finally {
|
||||
if (workDir) await rm(workDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
private fromBvh(text: string, fpsOverride?: number): { frames: KeypointFrame[]; fps: number; durationMs: number } {
|
||||
const motion = parseBvh(text);
|
||||
if (motion.frameCount === 0) throw new BadRequestException('BVH 中没有帧');
|
||||
const frames = bvhToKeypointFrames(motion, { fps: fpsOverride });
|
||||
const fps = fpsOverride && fpsOverride > 0 ? fpsOverride : 1 / motion.frameTime;
|
||||
return {
|
||||
frames,
|
||||
fps: Math.round(fps * 1000) / 1000,
|
||||
durationMs: Math.round((frames.length / fps) * 1000),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
import { Vec3, vAdd, vMid, vNorm, vScale, vSub, angleCos, vLen } from './math3d';
|
||||
import { BvhMotion, forwardKinematics } from './bvh-parser';
|
||||
import { CONF_DERIVED, CONF_DIRECT, CONF_MISSING, DEFAULT_JOINT_MAP } from './joint-map';
|
||||
import {
|
||||
ANGLE_TRIPLETS,
|
||||
KEYPOINT_COUNT,
|
||||
KeypointFrame,
|
||||
MP,
|
||||
VIRTUAL,
|
||||
} from './mediapipe33';
|
||||
|
||||
export interface RetargetOptions {
|
||||
/** 源帧率；缺省由 frameTime 推导 */
|
||||
fps?: number;
|
||||
/** 自适应缩放系数：投影后关键点范围为 ±scaleFactor */
|
||||
scaleFactor?: number;
|
||||
/** 关节名映射覆盖（可选） */
|
||||
jointMap?: Record<number, readonly string[]>;
|
||||
}
|
||||
|
||||
interface RawPoint {
|
||||
p: Vec3;
|
||||
conf: number;
|
||||
}
|
||||
|
||||
const clamp01 = (v: number): number => (v < 0 ? 0 : v > 1 ? 1 : v);
|
||||
const clampPm1 = (v: number): number => (v < -1 ? -1 : v > 1 ? 1 : v);
|
||||
|
||||
/** 规范化单人骨骼体积内的合理躯干长度，避免退化输入导致除零 */
|
||||
const MIN_TORSO = 1e-3;
|
||||
|
||||
/**
|
||||
* BVH 动作 → MediaPipe 33 点关键点序列。
|
||||
*
|
||||
* 流程：逐帧 FK → 直接映射 15 个主关节点 → 按层级推算其余点 →
|
||||
* 骨盆归零 → 全局自适应缩放 → 投影到 MediaPipe 归一化图像空间 → 计算 9 个关节角。
|
||||
*/
|
||||
export function bvhToKeypointFrames(motion: BvhMotion, options: RetargetOptions = {}): KeypointFrame[] {
|
||||
const jointMap = options.jointMap ?? DEFAULT_JOINT_MAP;
|
||||
const scaleFactor = options.scaleFactor ?? 0.35;
|
||||
const fps = options.fps && options.fps > 0 ? options.fps : 1 / motion.frameTime;
|
||||
|
||||
// 第一遍：逐帧解算原始 3D 位置（骨盆归零前）
|
||||
const rawFrames: RawPoint[][] = [];
|
||||
let maxAbs = 0;
|
||||
|
||||
for (let f = 0; f < motion.frameCount; f += 1) {
|
||||
const world = forwardKinematics(motion, f);
|
||||
const positions = new Map<string, Vec3>();
|
||||
for (const [name, joint] of world) positions.set(name, joint.position);
|
||||
|
||||
const points = resolveFrame(positions, jointMap);
|
||||
|
||||
// 骨盆归零
|
||||
const pelvis = points[VIRTUAL.PELVIS].p;
|
||||
for (const point of points) point.p = vSub(point.p, pelvis);
|
||||
|
||||
for (const point of points) {
|
||||
maxAbs = Math.max(maxAbs, Math.abs(point.p[0]), Math.abs(point.p[1]), Math.abs(point.p[2]));
|
||||
}
|
||||
rawFrames.push(points);
|
||||
}
|
||||
|
||||
const scale = maxAbs > 1e-9 ? scaleFactor / maxAbs : 1;
|
||||
|
||||
// 第二遍：投影 + 关节角
|
||||
const frames: KeypointFrame[] = [];
|
||||
for (let f = 0; f < rawFrames.length; f += 1) {
|
||||
const points = rawFrames[f];
|
||||
const positions = points.map((entry) => entry.p);
|
||||
|
||||
// points 含 33 个真实点 + 2 个虚拟关节（neck=33 / pelvis=34），
|
||||
// 虚拟关节只参与关节角计算，不进入输出序列。
|
||||
const projected = points.slice(0, KEYPOINT_COUNT).map(({ p, conf }): [number, number, number, number] => [
|
||||
clamp01(0.5 + p[0] * scale),
|
||||
clamp01(0.5 - p[1] * scale),
|
||||
clampPm1(-p[2] * scale),
|
||||
conf,
|
||||
]);
|
||||
|
||||
const angles = ANGLE_TRIPLETS.map(([a, b, c]) => {
|
||||
const pa = positions[a];
|
||||
const pb = positions[b];
|
||||
const pc = positions[c];
|
||||
if (!pa || !pb || !pc) return 0;
|
||||
return angleCos(pa, pb, pc);
|
||||
});
|
||||
|
||||
frames.push({
|
||||
frame: f,
|
||||
timeMs: Math.round((f / fps) * 1000 * 1000) / 1000,
|
||||
points: projected,
|
||||
angles,
|
||||
});
|
||||
}
|
||||
|
||||
return frames;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解算单帧的 **35** 项：索引 0..32 为 MediaPipe 33 点，33 = neck、34 = pelvis 为虚拟关节。
|
||||
* 调用方只输出前 33 项，虚拟关节用于关节角特征。
|
||||
* 依赖顺序：躯干 → 肩髋 → 肘/膝 → 腕/踝 → 头/手足末端。
|
||||
*/
|
||||
function resolveFrame(
|
||||
world: Map<string, Vec3>,
|
||||
jointMap: Record<number, readonly string[]>,
|
||||
): RawPoint[] {
|
||||
const lookup = (index: number): Vec3 | null => {
|
||||
const candidates = jointMap[index];
|
||||
if (!candidates) return null;
|
||||
for (const name of candidates) {
|
||||
const position = world.get(name);
|
||||
if (position) return position;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const points: Array<RawPoint | null> = new Array<RawPoint | null>(VIRTUAL.PELVIS + 1).fill(null);
|
||||
const put = (index: number, p: Vec3, conf: number): void => {
|
||||
if (!points[index]) points[index] = { p, conf };
|
||||
};
|
||||
|
||||
// ---- 1. 直接映射 ----
|
||||
for (const key of Object.keys(jointMap)) {
|
||||
const index = Number(key);
|
||||
const p = lookup(index);
|
||||
if (p) put(index, p, CONF_DIRECT);
|
||||
}
|
||||
|
||||
// 根关节兜底（多数 BVH 的 Hips）
|
||||
const rootFallback =
|
||||
world.get('hips') ?? world.get('root') ?? world.get('pelvis') ?? firstValue(world);
|
||||
if (!rootFallback) throw new Error('BVH 中找不到任何关节位置');
|
||||
|
||||
// ---- 2. 躯干 ----
|
||||
if (!points[MP.LEFT_HIP]) put(MP.LEFT_HIP, vAdd(rootFallback, [torsoGuess(world) * 0.09, 0, 0]), CONF_DERIVED);
|
||||
if (!points[MP.RIGHT_HIP]) put(MP.RIGHT_HIP, vAdd(rootFallback, [-torsoGuess(world) * 0.09, 0, 0]), CONF_DERIVED);
|
||||
|
||||
const pelvis = vMid(points[MP.LEFT_HIP]!.p, points[MP.RIGHT_HIP]!.p);
|
||||
put(VIRTUAL.PELVIS, pelvis, Math.min(points[MP.LEFT_HIP]!.conf, points[MP.RIGHT_HIP]!.conf));
|
||||
|
||||
// 肩：缺失时按直立姿态从骨盆向上、向两侧推算
|
||||
const rawTorso = vLen(vSub(pelvis, vMid(
|
||||
points[MP.LEFT_SHOULDER]?.p ?? pelvis,
|
||||
points[MP.RIGHT_SHOULDER]?.p ?? pelvis,
|
||||
)));
|
||||
const torso = rawTorso > MIN_TORSO ? rawTorso : torsoGuess(world);
|
||||
|
||||
if (!points[MP.LEFT_SHOULDER]) {
|
||||
put(MP.LEFT_SHOULDER, vAdd(pelvis, [torso * 0.22, torso * 0.95, 0]), CONF_DERIVED);
|
||||
}
|
||||
if (!points[MP.RIGHT_SHOULDER]) {
|
||||
put(MP.RIGHT_SHOULDER, vAdd(pelvis, [-torso * 0.22, torso * 0.95, 0]), CONF_DERIVED);
|
||||
}
|
||||
|
||||
const neck = vMid(points[MP.LEFT_SHOULDER]!.p, points[MP.RIGHT_SHOULDER]!.p);
|
||||
put(
|
||||
VIRTUAL.NECK,
|
||||
neck,
|
||||
Math.min(points[MP.LEFT_SHOULDER]!.conf, points[MP.RIGHT_SHOULDER]!.conf),
|
||||
);
|
||||
|
||||
const up: Vec3 = (() => {
|
||||
const candidate = vNorm(vSub(neck, pelvis));
|
||||
return vLen(candidate) > 0 ? candidate : [0, 1, 0];
|
||||
})();
|
||||
|
||||
// ---- 3. 头 ----
|
||||
if (!points[MP.NOSE]) {
|
||||
put(MP.NOSE, vAdd(neck, vScale(up, torso * 0.45)), CONF_DERIVED);
|
||||
}
|
||||
const nose = points[MP.NOSE]!.p;
|
||||
const headScale = torso * 0.30;
|
||||
const headOffsets: Array<[number, [number, number, number]]> = [
|
||||
[MP.LEFT_EYE_INNER, [0.16, 0.06, -0.20]],
|
||||
[MP.LEFT_EYE, [0.26, 0.06, -0.16]],
|
||||
[MP.LEFT_EYE_OUTER, [0.36, 0.06, -0.10]],
|
||||
[MP.RIGHT_EYE_INNER, [-0.16, 0.06, -0.20]],
|
||||
[MP.RIGHT_EYE, [-0.26, 0.06, -0.16]],
|
||||
[MP.RIGHT_EYE_OUTER, [-0.36, 0.06, -0.10]],
|
||||
[MP.LEFT_EAR, [0.48, 0.0, 0.04]],
|
||||
[MP.RIGHT_EAR, [-0.48, 0.0, 0.04]],
|
||||
[MP.MOUTH_LEFT, [0.16, -0.18, -0.24]],
|
||||
[MP.MOUTH_RIGHT, [-0.16, -0.18, -0.24]],
|
||||
];
|
||||
for (const [index, offset] of headOffsets) {
|
||||
if (!points[index]) {
|
||||
put(
|
||||
index,
|
||||
vAdd(nose, [offset[0] * headScale, offset[1] * headScale, offset[2] * headScale]),
|
||||
CONF_DERIVED,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 4. 四肢：肘 / 膝（缺失时沿规范方向推算）----
|
||||
const armLen = { upper: torso * 0.62, fore: torso * 0.55, hand: torso * 0.22 };
|
||||
const legLen = { thigh: torso * 0.92, shin: torso * 0.88, foot: torso * 0.38 };
|
||||
|
||||
const limbChains: Array<{
|
||||
proximal: number;
|
||||
mid: number;
|
||||
distal: number;
|
||||
end: number;
|
||||
sign: number;
|
||||
lens: { a: number; b: number; c: number };
|
||||
}> = [
|
||||
{
|
||||
proximal: MP.LEFT_SHOULDER,
|
||||
mid: MP.LEFT_ELBOW,
|
||||
distal: MP.LEFT_WRIST,
|
||||
end: MP.LEFT_INDEX,
|
||||
sign: 1,
|
||||
lens: { a: armLen.upper, b: armLen.fore, c: armLen.hand },
|
||||
},
|
||||
{
|
||||
proximal: MP.RIGHT_SHOULDER,
|
||||
mid: MP.RIGHT_ELBOW,
|
||||
distal: MP.RIGHT_WRIST,
|
||||
end: MP.RIGHT_INDEX,
|
||||
sign: -1,
|
||||
lens: { a: armLen.upper, b: armLen.fore, c: armLen.hand },
|
||||
},
|
||||
{
|
||||
proximal: MP.LEFT_HIP,
|
||||
mid: MP.LEFT_KNEE,
|
||||
distal: MP.LEFT_ANKLE,
|
||||
end: MP.LEFT_FOOT_INDEX,
|
||||
sign: 1,
|
||||
lens: { a: legLen.thigh, b: legLen.shin, c: legLen.foot },
|
||||
},
|
||||
{
|
||||
proximal: MP.RIGHT_HIP,
|
||||
mid: MP.RIGHT_KNEE,
|
||||
distal: MP.RIGHT_ANKLE,
|
||||
end: MP.RIGHT_FOOT_INDEX,
|
||||
sign: -1,
|
||||
lens: { a: legLen.thigh, b: legLen.shin, c: legLen.foot },
|
||||
},
|
||||
];
|
||||
|
||||
for (const chain of limbChains) {
|
||||
if (!points[chain.mid]) {
|
||||
const dir = vNorm([chain.sign * 0.22, -1, 0]);
|
||||
put(chain.mid, vAdd(points[chain.proximal]!.p, vScale(dir, chain.lens.a)), CONF_DERIVED);
|
||||
}
|
||||
if (!points[chain.distal]) {
|
||||
const dir = vNorm(vSub(points[chain.mid]!.p, points[chain.proximal]!.p));
|
||||
const fallback: Vec3 = vLen(dir) > 0 ? dir : [chain.sign * 0.2, -1, 0];
|
||||
put(chain.distal, vAdd(points[chain.mid]!.p, vScale(fallback, chain.lens.b)), CONF_DERIVED);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 5. 手部末端（小指 / 食指 / 拇指）----
|
||||
const handChains: Array<{ wrist: number; elbow: number; sign: number; pinky: number; index: number; thumb: number }> = [
|
||||
{ wrist: MP.LEFT_WRIST, elbow: MP.LEFT_ELBOW, sign: 1, pinky: MP.LEFT_PINKY, index: MP.LEFT_INDEX, thumb: MP.LEFT_THUMB },
|
||||
{ wrist: MP.RIGHT_WRIST, elbow: MP.RIGHT_ELBOW, sign: -1, pinky: MP.RIGHT_PINKY, index: MP.RIGHT_INDEX, thumb: MP.RIGHT_THUMB },
|
||||
];
|
||||
for (const chain of handChains) {
|
||||
const wrist = points[chain.wrist]!.p;
|
||||
const elbow = points[chain.elbow]!.p;
|
||||
const dir = vNorm(vSub(wrist, elbow));
|
||||
const forward: Vec3 = vLen(dir) > 0 ? dir : [chain.sign * 0.2, -1, 0];
|
||||
const hand = torso * 0.20;
|
||||
if (!points[chain.pinky]) {
|
||||
put(chain.pinky, vAdd(wrist, vAdd(vScale(forward, hand * 0.35), [chain.sign * hand * 0.25, -hand * 0.2, 0])), CONF_DERIVED);
|
||||
}
|
||||
if (!points[chain.index]) {
|
||||
put(chain.index, vAdd(wrist, vAdd(vScale(forward, hand * 0.55), [chain.sign * hand * 0.1, -hand * 0.08, 0])), CONF_DERIVED);
|
||||
}
|
||||
if (!points[chain.thumb]) {
|
||||
put(chain.thumb, vAdd(wrist, vAdd(vScale(forward, hand * 0.25), [chain.sign * -hand * 0.3, -hand * 0.1, 0])), CONF_DERIVED);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 6. 足跟（足尖已在链尾解算）----
|
||||
const footChains: Array<{ ankle: number; toe: number; heel: number; sign: number }> = [
|
||||
{ ankle: MP.LEFT_ANKLE, toe: MP.LEFT_FOOT_INDEX, heel: MP.LEFT_HEEL, sign: 1 },
|
||||
{ ankle: MP.RIGHT_ANKLE, toe: MP.RIGHT_FOOT_INDEX, heel: MP.RIGHT_HEEL, sign: -1 },
|
||||
];
|
||||
for (const chain of footChains) {
|
||||
if (!points[chain.toe]) {
|
||||
const ankle = points[chain.ankle]!.p;
|
||||
put(chain.toe, vAdd(ankle, [chain.sign * torso * 0.04, -legLen.foot * 0.5, -legLen.foot * 0.6]), CONF_DERIVED);
|
||||
}
|
||||
if (!points[chain.heel]) {
|
||||
const ankle = points[chain.ankle]!.p;
|
||||
put(chain.heel, vAdd(ankle, [chain.sign * torso * 0.03, -legLen.foot * 0.5, legLen.foot * 0.35]), CONF_DERIVED);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 7. 兜底 ----
|
||||
for (let i = 0; i <= VIRTUAL.PELVIS; i += 1) {
|
||||
if (!points[i]) {
|
||||
points[i] = { p: [...pelvis] as Vec3, conf: CONF_MISSING };
|
||||
}
|
||||
}
|
||||
|
||||
return points as RawPoint[];
|
||||
}
|
||||
|
||||
function firstValue(map: Map<string, Vec3>): Vec3 | undefined {
|
||||
for (const value of map.values()) return value;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** 无躯干信息时的尺度兜底：取根关节到最远关节的距离 */
|
||||
function torsoGuess(world: Map<string, Vec3>): number {
|
||||
const root = world.get('hips') ?? world.get('root') ?? world.get('pelvis');
|
||||
if (!root) return 1;
|
||||
let maxDistance = 0;
|
||||
for (const position of world.values()) {
|
||||
maxDistance = Math.max(maxDistance, vLen(vSub(position, root)));
|
||||
}
|
||||
return maxDistance > MIN_TORSO ? maxDistance : 1;
|
||||
}
|
||||
Reference in New Issue
Block a user