feat(motions): 动作预览——骨架缩略图 + 管理端 3D 回放(契约 data-models 0.6.0)

按《DSH 开发文档 v1.6》4.11(T-WEB-014,用户标注「重要」):

服务端:
- 抽出公共 PNG 编码器 src/common/png.ts(模型缩略图改为复用,零原生依赖不变)
- 新增 src/motions/motion-thumbnail.ts:关键点序列 -> 正视图骨架 PNG
  (复用 33 点 PARENT_TABLE 连线,包围盒按整段计算,低置信度点不画)
- motions 增 thumbnail_asset_id(迁移 AddMotionThumbnail)+ AssetKind.MotionThumbnail
- /preprocess 生成缩略图并落库,失败只记 warning;MotionDto 增 thumbnailUrl(契约 0.6.0)

管理端:
- 新增 MotionPreview:懒加载 NDJSON -> 骨架线框 + 关节实例球体
  支持播放/暂停、上一帧/下一帧、时间轴拖动、0.25x-2x 变速、循环
  叠加帧号/当前时间/总时长,按整段包围盒自动取景
  three.js 单独分包(lazy + Suspense),主包 241 kB 不涨
- 动作列表新增 40px 骨架缩略图列(点击开预览)+ 预览按钮 + 内联预览面板
- 新增 admin/src/lib/keypoint-sequence.ts 与 lib/skeleton-topology.ts

测试:新增 8 项;规模 18 文件 / 237 项。版本 0.5.0。
This commit is contained in:
opendance
2026-09-17 00:21:09 +08:00
parent 9eae6f3e43
commit aad60cbd2e
25 changed files with 1158 additions and 89 deletions
+2
View File
@@ -70,6 +70,8 @@ export enum AssetKind {
/** 上传的模型压缩包(zip)原件,仅用于溯源;入库的是转换后的模型文件 */
ModelArchive = 'model-archive',
Thumbnail = 'thumbnail',
/** 动作关键点序列的骨架缩略图(T-WEB-014) */
MotionThumbnail = 'motion-thumbnail',
Keypoints = 'keypoints',
}
+65
View File
@@ -0,0 +1,65 @@
/**
* 极简 PNG 编码(8 位真彩 RGB、无隔行、filter=0、手写 CRC32)。
*
* 模型缩略图与动作缩略图共用:**零原生依赖**(只用 node:zlib 的 deflate),
* 避免为了出图引入 canvas / sharp / headless-gl。
* 解码见 `models/model-thumbnail.ts` 内的 decodePng(同样零依赖)。
*/
import { deflateSync } from 'node:zlib';
const CRC_TABLE = (() => {
const table = new Uint32Array(256);
for (let n = 0; n < 256; n += 1) {
let c = n;
for (let k = 0; k < 8; k += 1) {
c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
}
table[n] = c >>> 0;
}
return table;
})();
function crc32(buffer: Buffer): number {
let c = 0xffffffff;
for (let i = 0; i < buffer.length; i += 1) {
c = CRC_TABLE[(c ^ buffer[i]) & 0xff] ^ (c >>> 8);
}
return (c ^ 0xffffffff) >>> 0;
}
function pngChunk(type: string, data: Buffer): Buffer {
const typeBuffer = Buffer.from(type, 'ascii');
const length = Buffer.alloc(4);
length.writeUInt32BE(data.length, 0);
const crc = Buffer.alloc(4);
crc.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])), 0);
return Buffer.concat([length, typeBuffer, data, crc]);
}
/** 把 RGB 像素(行优先,长度 = width*height*3)编码为 PNG */
export function encodePngRgb(rgb: Buffer, width: number, height: number): Buffer {
const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
const ihdr = Buffer.alloc(13);
ihdr.writeUInt32BE(width, 0);
ihdr.writeUInt32BE(height, 4);
ihdr[8] = 8; // bit depth
ihdr[9] = 2; // color type: truecolor RGB
ihdr[10] = 0; // compression
ihdr[11] = 0; // filter method
ihdr[12] = 0; // interlace: none
const stride = width * 3;
const raw = Buffer.alloc(height * (stride + 1));
for (let y = 0; y < height; y += 1) {
raw[y * (stride + 1)] = 0; // filter type 0
rgb.copy(raw, y * (stride + 1) + 1, y * stride, (y + 1) * stride);
}
return Buffer.concat([
signature,
pngChunk('IHDR', ihdr),
pngChunk('IDAT', deflateSync(raw)),
pngChunk('IEND', Buffer.alloc(0)),
]);
}
+1 -1
View File
@@ -11,7 +11,7 @@ import { join } from 'node:path';
export const CONTRACT_VERSIONS: Readonly<Record<string, string>> = Object.freeze({
'rest-api': '0.4.0',
'websocket-keypoints': '0.1.0',
'data-models': '0.5.0',
'data-models': '0.6.0',
'skeleton-topology': '0.2.0',
'model-formats': '0.5.0',
database: '0.3.0',
+4
View File
@@ -57,6 +57,10 @@ export class Motion {
@Column({ name: 'keypoint_frames', type: 'int', nullable: true })
keypointFrames!: number | null;
/** 骨架缩略图资产(预处理时生成;失败不影响预处理) */
@Column({ name: 'thumbnail_asset_id', type: 'bigint', nullable: true, transformer: bigintNumber })
thumbnailAssetId!: number | null;
@Column({ name: 'preprocess_state', type: 'varchar', length: 16, default: PreprocessState.None })
preprocessState!: PreprocessState;
@@ -0,0 +1,19 @@
import { MigrationInterface, QueryRunner } from "typeorm";
/**
* 动作预览(T-WEB-014):`motions.thumbnail_asset_id` 存骨架缩略图资产。
*
* 缩略图在 `/api/motions/:id/preprocess` 时生成(关键点序列 -> 正视图骨架 PNG),
* 供动作管理列表展示;生成失败不影响预处理结果。
*/
export class AddMotionThumbnail1789600000000 implements MigrationInterface {
name = 'AddMotionThumbnail1789600000000'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE \`motions\` ADD \`thumbnail_asset_id\` bigint NULL`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE \`motions\` DROP COLUMN \`thumbnail_asset_id\``);
}
}
+3 -63
View File
@@ -1,7 +1,8 @@
import { readFile } from 'node:fs/promises';
import { inflateSync } from 'node:zlib';
import { encodePngRgb } from '../common/png';
import { dirname, resolve as resolvePath } from 'node:path';
import { deflateSync } from 'node:zlib';
/**
* 服务端模型缩略图生成(纯 TypeScript 软件光栅化)。
@@ -1355,67 +1356,6 @@ function decodePng(buffer: Buffer): DecodedImage | null {
return { width, height, rgba: out };
}
// ---------------------------------------------------------------------------
// PNG 编码(8 位 RGB、无隔行、filter=0;不引第三方库)
// ---------------------------------------------------------------------------
const CRC_TABLE = (() => {
const table = new Uint32Array(256);
for (let n = 0; n < 256; n += 1) {
let c = n;
for (let k = 0; k < 8; k += 1) {
c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
}
table[n] = c >>> 0;
}
return table;
})();
function crc32(buffer: Buffer): number {
let c = 0xffffffff;
for (let i = 0; i < buffer.length; i += 1) {
c = CRC_TABLE[(c ^ buffer[i]) & 0xff] ^ (c >>> 8);
}
return (c ^ 0xffffffff) >>> 0;
}
function pngChunk(type: string, data: Buffer): Buffer {
const typeBuffer = Buffer.from(type, 'ascii');
const length = Buffer.alloc(4);
length.writeUInt32BE(data.length, 0);
const crc = Buffer.alloc(4);
crc.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])), 0);
return Buffer.concat([length, typeBuffer, data, crc]);
}
/** 把 RGB 像素编码为 PNG(每行前置 filter 字节 0,整块 deflate) */
function encodePng(rgb: Buffer, width: number, height: number): Buffer {
const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
const ihdr = Buffer.alloc(13);
ihdr.writeUInt32BE(width, 0);
ihdr.writeUInt32BE(height, 4);
ihdr[8] = 8; // bit depth
ihdr[9] = 2; // color type: truecolor RGB
ihdr[10] = 0; // compression
ihdr[11] = 0; // filter method
ihdr[12] = 0; // interlace: none
const stride = width * 3;
const raw = Buffer.alloc(height * (stride + 1));
for (let y = 0; y < height; y += 1) {
raw[y * (stride + 1)] = 0; // filter type 0
rgb.copy(raw, y * (stride + 1) + 1, y * stride, (y + 1) * stride);
}
return Buffer.concat([
signature,
pngChunk('IHDR', ihdr),
pngChunk('IDAT', deflateSync(raw)),
pngChunk('IEND', Buffer.alloc(0)),
]);
}
// ---------------------------------------------------------------------------
// 对外入口
// ---------------------------------------------------------------------------
@@ -1476,7 +1416,7 @@ export async function renderModelThumbnail(
if (output.drawn === 0) return null;
return {
png: encodePng(output.pixels, size, size),
png: encodePngRgb(output.pixels, size, size),
width: size,
height: size,
trianglesDrawn: output.drawn,
+177
View File
@@ -0,0 +1,177 @@
/**
* 动作缩略图:把关键点序列画成一张骨架 PNG(零原生依赖)。
*
* 用途:动作管理列表每行显示首帧骨架缩略图(T-WEB-014),随预处理一起生成并落库
* (`motions.thumbnail_asset_id`)。与模型缩略图共用 `common/png.ts` 的编码器。
*
* 画法取舍:
* * 只做**正视图**(X/Y 平面)——动作的"姿态"在正视图里最容易辨认,且不需要相机与光照;
* * 取**代表帧**(默认整段中间帧)绘制,包围盒按**整段**计算,保证不同动作构图一致;
* * 置信度低于阈值的点不画(与其相连的骨骼也跳过),避免出现"飘线"。
*/
import { PARENT_TABLE } from '../preprocess/mediapipe33';
import type { Keypoint, KeypointFrame } from '../preprocess/mediapipe33';
import { encodePngRgb } from '../common/png';
export interface MotionThumbnailOptions {
/** 输出边长(正方形),默认 256 */
size?: number;
/** 背景色 RGB,默认深灰 [24,26,32](与模型缩略图一致) */
background?: [number, number, number];
/** 骨骼颜色 RGB */
boneColor?: [number, number, number];
/** 关节颜色 RGB */
jointColor?: [number, number, number];
/** 置信度阈值,低于该值的点不绘制,默认 0.3 */
minVisibility?: number;
/** 指定绘制第几帧(默认中间帧);越界会夹到范围内 */
frameIndex?: number;
}
const DEFAULT_SIZE = 256;
const MIN_SIZE = 32;
const MAX_SIZE = 1024;
const DEFAULT_BACKGROUND: [number, number, number] = [24, 26, 32];
const DEFAULT_BONE: [number, number, number] = [96, 165, 250];
const DEFAULT_JOINT: [number, number, number] = [248, 250, 252];
const DEFAULT_MIN_VISIBILITY = 0.3;
interface Point2 {
x: number;
y: number;
}
/** 画一条粗线(Bresenham + 方形笔刷);坐标已为像素空间 */
function drawLine(
pixels: Uint8Array,
size: number,
from: Point2,
to: Point2,
color: [number, number, number],
thickness: number,
): void {
const half = Math.max(0, Math.floor(thickness / 2));
const steps = Math.max(Math.abs(to.x - from.x), Math.abs(to.y - from.y));
const count = Math.max(1, Math.ceil(steps));
for (let step = 0; step <= count; step += 1) {
const t = count === 0 ? 0 : step / count;
const cx = Math.round(from.x + (to.x - from.x) * t);
const cy = Math.round(from.y + (to.y - from.y) * t);
for (let dy = -half; dy <= half; dy += 1) {
for (let dx = -half; dx <= half; dx += 1) {
const x = cx + dx;
const y = cy + dy;
if (x < 0 || y < 0 || x >= size || y >= size) continue;
const index = (y * size + x) * 3;
pixels[index] = color[0];
pixels[index + 1] = color[1];
pixels[index + 2] = color[2];
}
}
}
}
/** 画一个实心圆(关节) */
function drawDisc(
pixels: Uint8Array,
size: number,
center: Point2,
radius: number,
color: [number, number, number],
): void {
const r = Math.max(1, radius);
const r2 = r * r;
for (let dy = -r; dy <= r; dy += 1) {
for (let dx = -r; dx <= r; dx += 1) {
if (dx * dx + dy * dy > r2) continue;
const x = Math.round(center.x) + dx;
const y = Math.round(center.y) + dy;
if (x < 0 || y < 0 || x >= size || y >= size) continue;
const index = (y * size + x) * 3;
pixels[index] = color[0];
pixels[index + 1] = color[1];
pixels[index + 2] = color[2];
}
}
}
/**
* 渲染动作骨架缩略图。
* 关键点为空 / 全部低于置信度 / 退化包围盒时返回 null(调用方按"未生成"处理,不抛错)。
*/
export function renderMotionThumbnail(
frames: readonly KeypointFrame[],
options: MotionThumbnailOptions = {},
): Buffer | null {
if (frames.length === 0) return null;
const rawSize = options.size ?? DEFAULT_SIZE;
if (!Number.isFinite(rawSize)) throw new Error('缩略图尺寸非法:size 必须是有限数字');
const size = Math.min(MAX_SIZE, Math.max(MIN_SIZE, Math.round(rawSize)));
const background = options.background ?? DEFAULT_BACKGROUND;
const boneColor = options.boneColor ?? DEFAULT_BONE;
const jointColor = options.jointColor ?? DEFAULT_JOINT;
const minVisibility = options.minVisibility ?? DEFAULT_MIN_VISIBILITY;
// 包围盒按整段计算(所有可见点),保证同一动作族的构图一致
let minX = Number.POSITIVE_INFINITY;
let minY = Number.POSITIVE_INFINITY;
let maxX = Number.NEGATIVE_INFINITY;
let maxY = Number.NEGATIVE_INFINITY;
let visible = 0;
for (const frame of frames) {
for (const point of frame.points) {
// Keypoint = [x, y, z, visibility]
const [x, y, , visibility] = point;
if (visibility < minVisibility) continue;
if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
if (x < minX) minX = x;
if (x > maxX) maxX = x;
if (y < minY) minY = y;
if (y > maxY) maxY = y;
visible += 1;
}
}
if (visible === 0 || !Number.isFinite(minX) || !Number.isFinite(minY)) return null;
const width = Math.max(maxX - minX, 1e-3);
const height = Math.max(maxY - minY, 1e-3);
const scale = (size * 0.86) / Math.max(width, height);
const offsetX = size / 2 - ((minX + maxX) / 2) * scale;
// glTF / 关键点坐标是 Y 向上,图像 Y 向下 —— 这里翻转,保证头在上
const offsetY = size / 2 + ((minY + maxY) / 2) * scale;
const project = (point: Keypoint): Point2 => ({
x: point[0] * scale + offsetX,
y: offsetY - point[1] * scale,
});
const pixels = new Uint8Array(size * size * 3);
for (let i = 0; i < size * size; i += 1) {
pixels[i * 3] = background[0];
pixels[i * 3 + 1] = background[1];
pixels[i * 3 + 2] = background[2];
}
const requested = options.frameIndex ?? Math.floor(frames.length / 2);
const frame = frames[Math.min(frames.length - 1, Math.max(0, requested))];
const thickness = Math.max(2, Math.round(size / 96));
const jointRadius = Math.max(1, Math.round(size / 128));
// 骨骼:按 33 点父表连线(父索引 33/34 是虚拟关节,映射到 0)
for (let index = 0; index < PARENT_TABLE.length && index < frame.points.length; index += 1) {
const rawParent = PARENT_TABLE[index];
const parentIndex = rawParent === 33 || rawParent === 34 ? 0 : rawParent;
if (parentIndex < 0 || parentIndex >= frame.points.length) continue;
const child = frame.points[index];
const parent = frame.points[parentIndex];
if (child[3] < minVisibility || parent[3] < minVisibility) continue;
drawLine(pixels, size, project(parent), project(child), boneColor, thickness);
}
// 关节
for (const point of frame.points) {
if (point[3] < minVisibility) continue;
drawDisc(pixels, size, project(point), jointRadius, jointColor);
}
return encodePngRgb(Buffer.from(pixels.buffer, pixels.byteOffset, pixels.length), size, size);
}
@@ -14,6 +14,8 @@ export interface MotionDto {
durationMs: number | null;
keypointFormat: string | null;
keypointUrl: string | null;
/** 骨架缩略图(预处理时生成;T-WEB-014) */
thumbnailUrl: string | null;
keypointFrames: number | null;
preprocessState: PreprocessState;
preprocessError: string | null;
@@ -31,6 +33,7 @@ export interface MotionUrlLookup {
fileUrl: string | null;
keypointUrl: string | null;
fileSize?: number | null;
thumbnailUrl: string | null;
}
export function toMotionDto(motion: Motion, urls: MotionUrlLookup): MotionDto {
@@ -46,6 +49,7 @@ export function toMotionDto(motion: Motion, urls: MotionUrlLookup): MotionDto {
durationMs: motion.durationMs,
keypointFormat: motion.keypointFormat,
keypointUrl: urls.keypointUrl,
thumbnailUrl: urls.thumbnailUrl,
keypointFrames: motion.keypointFrames,
preprocessState: motion.preprocessState,
preprocessError: motion.preprocessError,
+3 -1
View File
@@ -204,14 +204,16 @@ export class MotionsService {
async toDtos(rows: Motion[]): Promise<MotionDto[]> {
const assets = await this.storage.findMany(
rows.flatMap((row) => [row.fileAssetId, row.keypointAssetId]),
rows.flatMap((row) => [row.fileAssetId, row.keypointAssetId, row.thumbnailAssetId]),
);
return rows.map((row) => {
const file = row.fileAssetId ? assets.get(row.fileAssetId) : undefined;
const keypoints = row.keypointAssetId ? assets.get(row.keypointAssetId) : undefined;
const thumbnail = row.thumbnailAssetId ? assets.get(row.thumbnailAssetId) : undefined;
return toMotionDto(row, {
fileUrl: file ? this.storage.url(file) : null,
keypointUrl: keypoints ? this.storage.url(keypoints) : null,
thumbnailUrl: thumbnail ? this.storage.url(thumbnail) : null,
fileSize: file?.size ?? null,
});
});
@@ -1,4 +1,5 @@
import { BadRequestException, Injectable, Logger, NotFoundException } from '@nestjs/common';
import { renderMotionThumbnail } from '../motions/motion-thumbnail';
import { InjectRepository } from '@nestjs/typeorm';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
@@ -63,6 +64,26 @@ export class PreprocessService {
motion.keypointFormat = KEYPOINT_FORMAT;
motion.keypointAssetId = stored.asset.id;
// 骨架缩略图(T-WEB-014):动作管理列表展示;失败只记 warning,不影响预处理结果
try {
const thumbnail = renderMotionThumbnail(result.frames);
if (thumbnail) {
const storedThumb = await this.storage.persistBuffer(
AssetKind.MotionThumbnail,
thumbnail,
`motion-${motion.id}-thumb.png`,
'image/png',
);
motion.thumbnailAssetId = storedThumb.asset.id;
} else {
this.logger.warn(`动作 #${motion.id} 缩略图未生成(关键点不可用)`);
}
} catch (error) {
this.logger.warn(
`动作 #${motion.id} 缩略图生成失败:${error instanceof Error ? error.message : String(error)}`,
);
}
motion.keypointFrames = result.frames.length;
motion.fps = result.fps;
motion.frameCount = result.frames.length;