From aad60cbd2e24abfb173d2fd09bda15cb4380876f Mon Sep 17 00:00:00 2001 From: opendance Date: Thu, 17 Sep 2026 00:21:09 +0800 Subject: [PATCH] =?UTF-8?q?feat(motions):=20=E5=8A=A8=E4=BD=9C=E9=A2=84?= =?UTF-8?q?=E8=A7=88=E2=80=94=E2=80=94=E9=AA=A8=E6=9E=B6=E7=BC=A9=E7=95=A5?= =?UTF-8?q?=E5=9B=BE=20+=20=E7=AE=A1=E7=90=86=E7=AB=AF=203D=20=E5=9B=9E?= =?UTF-8?q?=E6=94=BE=EF=BC=88=E5=A5=91=E7=BA=A6=20data-models=200.6.0?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按《DSH 开发文档 v1.6》4.11（T-WEB-014，用户标注「重要」）： 服务端： - 抽出公共 PNG 编码器 src/common/png.ts（模型缩略图改为复用，零原生依赖不变） - 新增 src/motions/motion-thumbnail.ts：关键点序列 -> 正视图骨架 PNG （复用 33 点 PARENT_TABLE 连线，包围盒按整段计算，低置信度点不画） - motions 增 thumbnail_asset_id（迁移 AddMotionThumbnail）+ AssetKind.MotionThumbnail - /preprocess 生成缩略图并落库，失败只记 warning；MotionDto 增 thumbnailUrl（契约 0.6.0） 管理端： - 新增 MotionPreview：懒加载 NDJSON -> 骨架线框 + 关节实例球体 支持播放/暂停、上一帧/下一帧、时间轴拖动、0.25x-2x 变速、循环 叠加帧号/当前时间/总时长，按整段包围盒自动取景 three.js 单独分包（lazy + Suspense），主包 241 kB 不涨 - 动作列表新增 40px 骨架缩略图列（点击开预览）+ 预览按钮 + 内联预览面板 - 新增 admin/src/lib/keypoint-sequence.ts 与 lib/skeleton-topology.ts 测试：新增 8 项；规模 18 文件 / 237 项。版本 0.5.0。 --- README.md | 32 +- README.txt | 24 +- contracts/VERSION | 2 +- contracts/data-models.md | 1 + project.yaml | 6 +- scripts/package-release.sh | 4 +- web-server/admin/package.json | 2 +- web-server/admin/src/api/types.ts | 2 + .../admin/src/components/MotionPreview.tsx | 370 ++++++++++++++++++ web-server/admin/src/lib/keypoint-sequence.ts | 117 ++++++ web-server/admin/src/lib/skeleton-topology.ts | 109 ++++++ web-server/admin/src/pages/Motions.tsx | 65 ++- web-server/admin/src/styles.css | 46 +++ web-server/package.json | 2 +- web-server/src/common/enums.ts | 2 + web-server/src/common/png.ts | 65 +++ web-server/src/config/contract-versions.ts | 2 +- web-server/src/entities/motion.entity.ts | 4 + .../1789600000000-AddMotionThumbnail.ts | 19 + web-server/src/models/model-thumbnail.ts | 66 +--- web-server/src/motions/motion-thumbnail.ts | 177 +++++++++ web-server/src/motions/motion.serializer.ts | 4 + web-server/src/motions/motions.service.ts | 4 +- .../src/preprocess/preprocess.service.ts | 21 + web-server/test/motion-preview.spec.ts | 101 +++++ 25 files changed, 1158 insertions(+), 89 deletions(-) create mode 100644 web-server/admin/src/components/MotionPreview.tsx create mode 100644 web-server/admin/src/lib/keypoint-sequence.ts create mode 100644 web-server/admin/src/lib/skeleton-topology.ts create mode 100644 web-server/src/common/png.ts create mode 100644 web-server/src/migrations/1789600000000-AddMotionThumbnail.ts create mode 100644 web-server/src/motions/motion-thumbnail.ts create mode 100644 web-server/test/motion-preview.spec.ts diff --git a/README.md b/README.md index 1b14d87..244ce0d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,23 @@ OpenDance Studio **阶段一**: Web 管理服务端(NestJS + TypeORM + 现有 My ## 状态 -**阶段一已完成**: T-WEB-001 -> T-WEB-010 全部落地,当前版本 **v0.3.0**(契约 `rest-api / data-models / model-formats 0.4.0`、`database 0.3.0`)。 +**阶段一已完成**: T-WEB-001 -> T-WEB-010 全部落地,当前版本 **v0.5.0**(契约 `rest-api 0.4.0`、`data-models 0.6.0`、`model-formats 0.5.0`、`skeleton-topology 0.2.0`、`database 0.3.0`)。 + +v0.5.0 修订(**动作预览**,契约 data-models -> 0.6.0): + +1. **动作骨架缩略图(服务端)**:新增 `motions/motion-thumbnail.ts`——把关键点序列画成正视图骨架 PNG(零原生依赖,复用抽出的 `common/png.ts`);随 `/preprocess` 生成并落库(`motions.thumbnail_asset_id`,迁移 `AddMotionThumbnail`),失败只记 warning 不影响预处理; +2. **动作 3D 回放(管理端,重要)**:新增 `MotionPreview`——懒加载 `GET /api/motions/:id/keypoints`(NDJSON),骨架线框 + 关节球体,支持**播放/暂停、上一帧/下一帧、时间轴拖动、0.25x-2x 变速、循环**,叠加"第 N/M 帧 · 当前时间 · 总时长",按整段包围盒自动取景;three.js 单独分包,主包体积不涨; +3. **动作管理列表**:新增 40px 骨架缩略图列(点击开预览)+「预览/收起预览」按钮; +4. 关键点解析与拓扑(管理端 `lib/keypoint-sequence.ts` / `lib/skeleton-topology.ts`)纯函数化,便于测试; +5. 新增 8 项测试;规模 **18 文件 / 237 项**。 + +v0.4.0 修订(**骨骼重绑定**:把"一律转 VRM"真正可用,契约 skeleton-topology -> 0.2.0): + +1. **修复 humanoid 覆盖率恒为 0/22 的根因**:`normalizeBoneName()` 原先只保留 `a-z0-9`,**中日文字符被整个丢掉**(`上半身`/`左ひじ`/`首` 全变空串);现在保留 CJK/假名,并把全角 ASCII(`足ＩＫ`)折成半角; +2. **候选表补齐四类命名**:VRoid / Mixamo / **MMD 标准骨架**(`下半身`/`上半身`/`上半身2`/`首`/`頭`/`左肩`/`左腕`/`左ひじ`/`左手首`/`左足`/`左ひざ`/`左足首`/`左つま先`)/ 常见中文名;精确匹配保证 `左足ＩＫ` 不抢占 `左足`; +3. **服务端与 Windows 客户端逐字一致**:`humanoid_map.gd` 的 `CANDIDATES` 由服务端表生成写入,parity 测试断言; +4. **实测**:奥黛塔 PMX **0/22 → 21/22**、初音未来 PMX **0/22 → 19/22**,均 `drivable=true` 且产出 VRM; +5. 契约 `skeleton-topology` -> **0.2.0**:新增 §6「内部动作标准与模型骨骼重绑定」。 v0.3.0 修订(**VRM 主力化**,契约 data-models / model-formats -> 0.5.0): @@ -85,15 +101,15 @@ v0.2.0 主要变更: 3. 取得发布包并校验(地址用你们自己的制品库替换; 下面是本仓库的发布地址) ```bash - curl -LO https://gitea.kasugano.cn/zijunchiang/opendance-web/releases/download/v0.4.0/opendance-web-v0.4.0.tar.gz - curl -LO https://gitea.kasugano.cn/zijunchiang/opendance-web/releases/download/v0.4.0/SHA256SUMS + curl -LO https://gitea.kasugano.cn/zijunchiang/opendance-web/releases/download/v0.5.0/opendance-web-v0.5.0.tar.gz + curl -LO https://gitea.kasugano.cn/zijunchiang/opendance-web/releases/download/v0.5.0/SHA256SUMS sha256sum -c SHA256SUMS # 期望输出 "OK" ``` 4. 解压并生成配置(交互式引导,逐项问答,口令不回显) ```bash - tar -xzf opendance-web-v0.4.0.tar.gz && cd opendance-web-v0.4.0 + tar -xzf opendance-web-v0.5.0.tar.gz && cd opendance-web-v0.5.0 bash deploy/init-env.sh ``` @@ -163,8 +179,8 @@ FLUSH PRIVILEGES; ### 5. 部署步骤 ```bash -tar -xzf opendance-web-v0.4.0.tar.gz -cd opendance-web-v0.4.0 +tar -xzf opendance-web-v0.5.0.tar.gz +cd opendance-web-v0.5.0 bash deploy/init-env.sh # 首次启动引导: 逐项问答生成 deploy/.env(见下) bash deploy/install.sh # 已以 root 登录, 不需要 sudo ``` @@ -337,12 +353,12 @@ npm run dev # http://localhost:5173(/api 与 /health 代理到 3000) cd web-server npm run contracts:check # 契约版本一致性 npm run typecheck # TypeScript 严格检查 -npm run test # 229 项单元测试 / 17 个文件(VMD 31、模型压缩包 27、 +npm run test # 237 项单元测试 / 18 个文件(VMD 31、模型压缩包 27、 # 模型检查器 25、缩略图 8、骨骼过滤 8、 # 预处理 25、init-env 16、admin-port 16、 # access-log 9、VRM 检查器 6、.env 预检 12、 # Assimp 定位与安装 13、stats 容量与编码 12、 - # 缩略图贴图采样 6、MMD 材质对齐 9、模型可用性 5、MMD 骨名重绑定 4) + # 缩略图贴图采样 6、MMD 材质对齐 9、模型可用性 5、MMD 骨名重绑定 4、动作预览 8) npm run build # 生产构建 npm run start:prod & # 启动 npm run test:smoke # 57 项端到端冒烟(需要数据库可连通) diff --git a/README.txt b/README.txt index a08b96e..887cb1d 100644 --- a/README.txt +++ b/README.txt @@ -1,5 +1,5 @@ ================================================================ -OpenDance Studio -- Web 管理服务端 v0.4.0 使用说明 +OpenDance Studio -- Web 管理服务端 v0.5.0 使用说明 ================================================================ 本服务端是 OpenDance Studio 三端中的第一端, 负责管理歌曲、VRM 模型与 @@ -8,6 +8,16 @@ OpenDance Studio -- Web 管理服务端 v0.4.0 使用说明 重要: 本服务端 不部署、不管理 MySQL。数据库为部署方提供的现有实例, 连接信息通过 .env 注入。 +v0.5.0 修订 - 动作预览(契约 data-models -> 0.6.0) + 1. 动作骨架缩略图(服务端): motion-thumbnail.ts 把关键点序列画成正视图骨架 PNG, + 随 /preprocess 生成并落库(motions.thumbnail_asset_id, 迁移 AddMotionThumbnail), + 失败只记 warning 不影响预处理。 + 2. 动作 3D 回放(管理端, 重要): 新增 MotionPreview - 懒加载 NDJSON -> 骨架线框 + + 关节球体, 支持播放/暂停、上一帧/下一帧、时间轴拖动、0.25x-2x 变速、循环, + 叠加帧号/时间/总时长; three.js 单独分包, 主包体积不涨。 + 3. 动作管理列表新增 40px 骨架缩略图列(点击预览)与「预览」按钮。 + 4. 新增 8 项测试; 规模 18 文件 / 237 项。 + v0.4.0 修订 - 骨骼重绑定(契约 skeleton-topology -> 0.2.0) 1. 修复 humanoid 覆盖率恒为 0/22 的根因: normalizeBoneName() 原先只保留 a-z0-9, 中日文字符被整个丢掉(上半身/左ひじ/首 全变空串)。现在保留 CJK/假名, @@ -115,11 +125,11 @@ v0.2.1 修订 2. 准备一个现有的 MySQL / MariaDB 实例, 并建好库与账号(见下方"前置要求") 本项目不会创建数据库本身 3. 取得发布包并校验(地址用你们自己的制品库替换) - curl -LO https://<制品库地址>/opendance-web-v0.4.0.tar.gz + curl -LO https://<制品库地址>/opendance-web-v0.5.0.tar.gz curl -LO https://<制品库地址>/SHA256SUMS sha256sum -c SHA256SUMS # 期望输出 "OK" 4. 解压并生成配置(交互式引导, 逐项问答, 口令不回显) - tar -xzf opendance-web-v0.4.0.tar.gz && cd opendance-web-v0.4.0 + tar -xzf opendance-web-v0.5.0.tar.gz && cd opendance-web-v0.5.0 bash deploy/init-env.sh 5. 安装(建用户与目录 -> 装依赖 -> 迁移 -> 注册系统服务 -> 等健康检查) bash deploy/install.sh @@ -177,8 +187,8 @@ v0.2.1 修订 (本项目不会创建数据库本身, 也不需要 GRANT OPTION / FILE / SUPER 等管理权限) 部署步骤 - tar -xzf opendance-web-v0.4.0.tar.gz - cd opendance-web-v0.4.0 + tar -xzf opendance-web-v0.5.0.tar.gz + cd opendance-web-v0.5.0 bash deploy/init-env.sh # 首次启动引导: 逐项问答生成 deploy/.env(见下) bash deploy/install.sh # 已以 root 登录, 不需要 sudo @@ -353,12 +363,12 @@ deploy/.env 初始化引导(deploy/init-env.sh) cd web-server npm run contracts:check # 契约版本一致性 npm run typecheck # TypeScript 严格检查 - npm run test # 229 项单元测试 / 17 个文件(VMD 31、模型压缩包 24、 + npm run test # 237 项单元测试 / 18 个文件(VMD 31、模型压缩包 24、 # 模型检查器 25、缩略图 8、骨骼过滤 8、 # 预处理 25、init-env 16、admin-port 16、 # access-log 9、VRM 检查器 6、.env 预检 12、 # Assimp 定位与安装 13、stats 容量与编码 12、缩略图贴图 6、 - # MMD 材质对齐 9、模型可用性 5、MMD 骨名重绑定 4) + # MMD 材质对齐 9、模型可用性 5、MMD 骨名重绑定 4、动作预览 8) npm run build # 生产构建 npm run start:prod & # 启动 npm run test:smoke # 57 项端到端冒烟(需要数据库可连通) diff --git a/contracts/VERSION b/contracts/VERSION index 8c110dc..2cdfbc4 100644 --- a/contracts/VERSION +++ b/contracts/VERSION @@ -2,7 +2,7 @@ # 规则：任何字段级变更必须提升对应条目版本，并回到 opendance-web 修改后再向另两端建迁移任务。 rest-api: 0.4.0 websocket-keypoints: 0.1.0 -data-models: 0.5.0 +data-models: 0.6.0 skeleton-topology: 0.2.0 model-formats: 0.5.0 database: 0.3.0 diff --git a/contracts/data-models.md b/contracts/data-models.md index 224c2aa..694bd97 100644 --- a/contracts/data-models.md +++ b/contracts/data-models.md @@ -83,6 +83,7 @@ rejected 已驳回 | `durationMs` | number\|null | | | | `keypointFormat` | string\|null | 读 | 预处理后固定为 `mediapipe33@0.1.0` | | `keypointUrl` | string\|null | 读 | 预处理产物（NDJSON 或.npz 的 JSON 封装） | +| `thumbnailUrl` | string\|null | 读 | 骨架缩略图（`/preprocess` 时生成，动作管理列表展示；未生成时为 null）(v0.6.0) | | `keypointFrames` | number\|null | 读 | 关键点帧数 | | `preprocessState` | enum | 读 | `none` / `running` / `done` / `failed` | | `preprocessError` | string\|null | 读 | | diff --git a/project.yaml b/project.yaml index b145eda..c0c62af 100644 --- a/project.yaml +++ b/project.yaml @@ -1,5 +1,5 @@ project: opendance-web-server -version: 0.4.0 +version: 0.5.0 repository: opendance-web default_branch: main contracts_version: 0.4.0 @@ -60,13 +60,13 @@ tasks: status: done deliverable: - artifact: opendance-web-v0.4.0.tar.gz + artifact: opendance-web-v0.5.0.tar.gz packaging: Linux 业务服务（系统服务托管，双端口：软件 API + 管理端，不含 MySQL、不含反向代理、不使用 Docker） includes: - web-server/ # NestJS 源码 + 构建产物（含 BVH / VMD 内置解析、ZIP 读取与模型压缩包转换） - admin/ # React 管理端构建产物 - deploy/ # install.sh / init-env.sh / systemd / openrc / initd / ctl.sh / .env.example（PORT、ADMIN_PORT、HOST） - deploy/proxy/ # 反向代理说明（nginx / IIS，仅供参考不部署） - - contracts/ # 当前版本契约快照（0.4.0） + - contracts/ # 当前版本契约快照（data-models 0.6.0） - THIRD_PARTY.md # 开源声明 - README.txt # 使用说明 diff --git a/scripts/package-release.sh b/scripts/package-release.sh index 4ee5c01..9e2eed0 100755 --- a/scripts/package-release.sh +++ b/scripts/package-release.sh @@ -1,6 +1,6 @@ #!/bin/bash # ============================================================ -# 打包发布产物：opendance-web-v0.4.0.tar.gz +# 打包发布产物：opendance-web-v0.5.0.tar.gz # # 产物结构（Linux 业务服务部署，不使用 Docker、不部署反向代理）： # web-server/ NestJS 源码 + 构建产物 @@ -21,7 +21,7 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" cd "$REPO_ROOT" -VERSION="${1:-0.4.0}" +VERSION="${1:-0.5.0}" NAME="opendance-web-v${VERSION}" OUT_DIR="$REPO_ROOT/dist-release" STAGE="$OUT_DIR/$NAME" diff --git a/web-server/admin/package.json b/web-server/admin/package.json index b2d0ea4..77ada64 100644 --- a/web-server/admin/package.json +++ b/web-server/admin/package.json @@ -1,7 +1,7 @@ { "name": "opendance-admin", "private": true, - "version": "0.4.0", + "version": "0.5.0", "type": "module", "description": "OpenDance Studio 管理端（React + TypeScript + Vite）", "scripts": { diff --git a/web-server/admin/src/api/types.ts b/web-server/admin/src/api/types.ts index 9b8a923..1efc6b1 100644 --- a/web-server/admin/src/api/types.ts +++ b/web-server/admin/src/api/types.ts @@ -102,6 +102,8 @@ export interface Motion extends SourceFields { durationMs: number | null; keypointFormat: string | null; keypointUrl: string | null; + /** 骨架缩略图（预处理时生成；T-WEB-014） */ + thumbnailUrl: string | null; keypointFrames: number | null; preprocessState: PreprocessState; preprocessError: string | null; diff --git a/web-server/admin/src/components/MotionPreview.tsx b/web-server/admin/src/components/MotionPreview.tsx new file mode 100644 index 0000000..249b650 --- /dev/null +++ b/web-server/admin/src/components/MotionPreview.tsx @@ -0,0 +1,370 @@ +/** + * 动作预览：关键点序列 3D 回放（T-WEB-014）。 + * + * 只用 three.js 的核心能力（LineSegments + InstancedMesh），不依赖 three-vrm： + * * 骨架：按 `skeleton-topology.ts` 的骨段建 LineSegments，逐帧写 position； + * * 关节：InstancedMesh 球体，逐帧写实例矩阵（低置信度点缩放为 0 隐藏）； + * * 交互：播放 / 暂停 / 逐帧 / 时间轴拖动 / 变速 / 循环； + * * 数据：`GET /api/motions/:id/keypoints`（NDJSON，懒加载，切换动作时取消旧请求）。 + */ +import { useEffect, useMemo, useRef, useState } from 'react'; +import * as THREE from 'three'; + +import { apiUrl } from '../api/client'; +import { + BONE_SEGMENTS, + FACE_POINT_LIMIT, + KEYPOINT_NAMES, + VIRTUAL_PELVIS, + VIRTUAL_SHOULDER_CENTER, +} from '../lib/skeleton-topology'; +import { type ParsedKeypointSequence, parseKeypointNdjson, sequenceBounds } from '../lib/keypoint-sequence'; +import { useAsync } from '../lib/useAsync'; +import { ErrorBox, Loading } from './ui'; + +const SPEEDS = [0.25, 0.5, 1, 2] as const; +const MIN_VISIBILITY = 0.3; + +interface MotionPreviewProps { + motionId: number; + /** 画布高度（px） */ + height?: number; +} + +/** 把预览需要的骨架几何一次性建好，之后每帧只更新 buffer */ +function buildSkeletonObject(): { + group: THREE.Group; + bones: THREE.LineSegments; + joints: THREE.InstancedMesh; + /** 每个关节实例对应的关键点索引 */ + jointIndices: number[]; +} { + const group = new THREE.Group(); + + // 只画身体骨段（面部五官太密，缩略图/预览都不画） + const segments = BONE_SEGMENTS.filter( + ([child, parent]) => child >= FACE_POINT_LIMIT && parent >= FACE_POINT_LIMIT, + ); + const bonePositions = new Float32Array(segments.length * 2 * 3); + const boneGeometry = new THREE.BufferGeometry(); + boneGeometry.setAttribute('position', new THREE.BufferAttribute(bonePositions, 3)); + const bones = new THREE.LineSegments( + boneGeometry, + new THREE.LineBasicMaterial({ color: 0x60a5fa, linewidth: 2 }), + ); + bones.frustumCulled = false; + group.add(bones); + + const jointIndices: number[] = []; + for (let index = FACE_POINT_LIMIT; index < KEYPOINT_NAMES.length; index += 1) jointIndices.push(index); + const jointGeometry = new THREE.SphereGeometry(0.035, 10, 8); + const jointMaterial = new THREE.MeshStandardMaterial({ color: 0xf8fafc, roughness: 0.6, metalness: 0 }); + const joints = new THREE.InstancedMesh(jointGeometry, jointMaterial, jointIndices.length); + joints.instanceMatrix.setUsage(THREE.DynamicDrawUsage); + joints.frustumCulled = false; + group.add(joints); + + return { group, bones, joints, jointIndices }; +} + +export function MotionPreview({ motionId, height = 360 }: MotionPreviewProps): JSX.Element { + const mountRef = useRef(null); + const sequenceRef = useRef(null); + const cursorRef = useRef(0); + const playingRef = useRef(true); + const speedRef = useRef(1); + const loopRef = useRef(true); + const [playing, setPlaying] = useState(true); + const [speed, setSpeed] = useState(1); + const [loop, setLoop] = useState(true); + const [cursor, setCursor] = useState(0); + + const state = useAsync( + () => + fetch(apiUrl(`/api/motions/${motionId}/keypoints`)) + .then((response) => { + if (!response.ok) throw new Error(`关键点下载失败：HTTP ${response.status}`); + return response.text(); + }) + .then((text) => parseKeypointNdjson(text)), + [motionId], + ); + + const summary = useMemo(() => { + if (!state.data) return null; + const { frames, frameIntervalMs } = state.data; + if (frames.length === 0) return null; + return { count: frames.length, interval: frameIntervalMs }; + }, [state]); + + useEffect(() => { + sequenceRef.current = state.data ?? null; + cursorRef.current = 0; + setCursor(0); + }, [state]); + + useEffect(() => { + playingRef.current = playing; + }, [playing]); + useEffect(() => { + speedRef.current = speed; + }, [speed]); + useEffect(() => { + loopRef.current = loop; + }, [loop]); + + useEffect(() => { + const mount = mountRef.current; + if (!mount) return; + const width = Math.max(1, mount.clientWidth); + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x181a20); + const camera = new THREE.PerspectiveCamera(38, width / height, 0.01, 100); + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(Math.min(2, window.devicePixelRatio || 1)); + renderer.setSize(width, height, false); + mount.appendChild(renderer.domElement); + scene.add(new THREE.AmbientLight(0xffffff, 1.1)); + const key = new THREE.DirectionalLight(0xffffff, 1.6); + key.position.set(1, 2, 2); + scene.add(key); + const rim = new THREE.DirectionalLight(0x88bbff, 0.8); + rim.position.set(-1.5, 1, -1.5); + scene.add(rim); + + const { group, bones, joints, jointIndices } = buildSkeletonObject(); + scene.add(group); + + // 网格地面（帮助判断朝向与高度） + const grid = new THREE.GridHelper(4, 8, 0x334155, 0x1f2937); + grid.position.y = -1; + scene.add(grid); + + const resize = (): void => { + const nextWidth = Math.max(1, mount.clientWidth); + camera.aspect = nextWidth / height; + camera.updateProjectionMatrix(); + renderer.setSize(nextWidth, height, false); + }; + const observer = new ResizeObserver(resize); + observer.observe(mount); + + const clock = new THREE.Clock(); + let raf = 0; + let elapsed = 0; + let lastRendered = -1; + + const applyFrame = (sequence: ParsedKeypointSequence, index: number): void => { + const frame = sequence.frames[Math.min(sequence.frames.length - 1, Math.max(0, index))]; + if (!frame) return; + const { positions, visibility } = frame; + + // 骨段：虚拟关节用实际点替代（骨盆=左右髋中点，肩中心=左右肩中点） + const pointAt = (index2: number, out: THREE.Vector3): THREE.Vector3 => { + if (index2 === VIRTUAL_PELVIS) { + out.set( + (positions[23 * 3] + positions[24 * 3]) / 2, + (positions[23 * 3 + 1] + positions[24 * 3 + 1]) / 2, + (positions[23 * 3 + 2] + positions[24 * 3 + 2]) / 2, + ); + return out; + } + if (index2 === VIRTUAL_SHOULDER_CENTER) { + out.set( + (positions[11 * 3] + positions[12 * 3]) / 2, + (positions[11 * 3 + 1] + positions[12 * 3 + 1]) / 2, + (positions[11 * 3 + 2] + positions[12 * 3 + 2]) / 2, + ); + return out; + } + out.set(positions[index2 * 3], positions[index2 * 3 + 1], positions[index2 * 3 + 2]); + return out; + }; + + const attr = bones.geometry.getAttribute('position') as THREE.BufferAttribute; + const array = attr.array as Float32Array; + const scratchA = new THREE.Vector3(); + const scratchB = new THREE.Vector3(); + const segments = BONE_SEGMENTS.filter( + ([child, parent]) => child >= FACE_POINT_LIMIT && parent >= FACE_POINT_LIMIT, + ); + for (let index2 = 0; index2 < segments.length; index2 += 1) { + const [child, parent] = segments[index2]; + pointAt(child, scratchA); + pointAt(parent, scratchB); + array[index2 * 6] = scratchA.x; + array[index2 * 6 + 1] = scratchA.y; + array[index2 * 6 + 2] = scratchA.z; + array[index2 * 6 + 3] = scratchB.x; + array[index2 * 6 + 4] = scratchB.y; + array[index2 * 6 + 5] = scratchB.z; + } + attr.needsUpdate = true; + + const matrix = new THREE.Matrix4(); + const scale = new THREE.Vector3(); + const zero = new THREE.Vector3(0, 0, 0); + const position = new THREE.Vector3(); + const quaternion = new THREE.Quaternion(); + for (let slot = 0; slot < jointIndices.length; slot += 1) { + const pointIndex = jointIndices[slot]; + const visible = visibility[pointIndex] >= MIN_VISIBILITY; + position.set( + positions[pointIndex * 3], + positions[pointIndex * 3 + 1], + positions[pointIndex * 3 + 2], + ); + scale.setScalar(visible ? 1 : 0); + if (!visible) position.copy(zero); + matrix.compose(position, quaternion, scale); + joints.setMatrixAt(slot, matrix); + } + joints.instanceMatrix.needsUpdate = true; + }; + + const tick = (): void => { + raf = requestAnimationFrame(tick); + const delta = clock.getDelta(); + const sequence = sequenceRef.current; + if (sequence && sequence.frames.length > 0) { + if (playingRef.current) { + elapsed += delta * 1000 * speedRef.current; + const step = sequence.frameIntervalMs > 0 ? sequence.frameIntervalMs : 1000 / 30; + let next = cursorRef.current + elapsed / step; + elapsed = 0; + if (next >= sequence.frames.length) { + if (loopRef.current) { + next %= sequence.frames.length; + } else { + next = sequence.frames.length - 1; + playingRef.current = false; + setPlaying(false); + } + } + cursorRef.current = next; + const integer = Math.floor(next); + if (integer !== lastRendered) { + lastRendered = integer; + setCursor(integer); + } + } + applyFrame(sequence, Math.floor(cursorRef.current)); + + // 相机跟随包围盒（取整段包围盒，避免逐帧抖动） + if (sequence.frames.length > 0 && !mount.dataset.framed) { + const bounds = sequenceBounds(sequence.frames); + if (bounds) { + const center = new THREE.Vector3( + (bounds.min[0] + bounds.max[0]) / 2, + (bounds.min[1] + bounds.max[1]) / 2, + (bounds.min[2] + bounds.max[2]) / 2, + ); + const size = Math.max( + bounds.max[0] - bounds.min[0], + bounds.max[1] - bounds.min[1], + bounds.max[2] - bounds.min[2], + 0.5, + ); + group.position.set(-center.x, -center.y, -center.z); + camera.position.set(0.6 * size, 0.7 * size, 2.6 * size); + camera.lookAt(0, 0, 0); + mount.dataset.framed = '1'; + } + } + } + renderer.render(scene, camera); + }; + raf = requestAnimationFrame(tick); + + return () => { + cancelAnimationFrame(raf); + observer.disconnect(); + renderer.dispose(); + bones.geometry.dispose(); + (bones.material as THREE.Material).dispose(); + joints.geometry.dispose(); + (joints.material as THREE.Material).dispose(); + if (renderer.domElement.parentNode === mount) mount.removeChild(renderer.domElement); + delete mount.dataset.framed; + }; + }, [height, motionId]); + + if (state.loading) return ; + if (state.error) { + return ; + } + if (!summary) { + return ( +
+ 该动作还没有关键点序列 + 先执行「预处理」（POST /api/motions/:id/preprocess）后再预览。 +
+ ); + } + + const total = summary.count; + const currentTimeMs = Math.round((cursor * summary.interval) / 1000 * 1000) / 1000; + + return ( +
+
+
+ + + + { + setPlaying(false); + const next = Number(event.target.value); + cursorRef.current = next; + setCursor(next); + }} + aria-label="时间轴" + /> + + 第 {cursor + 1}/{total} 帧 · {currentTimeMs.toFixed(2)}s · {(total * summary.interval / 1000).toFixed(2)}s + + + +
+
+ ); +} diff --git a/web-server/admin/src/lib/keypoint-sequence.ts b/web-server/admin/src/lib/keypoint-sequence.ts new file mode 100644 index 0000000..0e4a52d --- /dev/null +++ b/web-server/admin/src/lib/keypoint-sequence.ts @@ -0,0 +1,117 @@ +/** + * 关键点序列（NDJSON）解析与回放辅助（T-WEB-014）。 + * + * 数据来源：`GET /api/motions/:id/keypoints`，每行一帧： + * `{"frame":0,"timeMs":0,"points":[[x,y,z,visibility] x 33],"angles":[...9]}` + * + * 解析后转成**扁平 Float32Array**：33 点 x 3 轴 + 33 个置信度， + * 供 three.js 逐帧更新（避免每帧创建对象数组导致 GC 抖动）。 + */ + +export interface ParsedKeypointFrame { + /** 服务端帧号 */ + frame: number; + timeMs: number; + /** 长度 33*3：[x,y,z] x 33（与 contracts/skeleton-topology.md §1 顺序一致） */ + positions: Float32Array; + /** 长度 33 的置信度 */ + visibility: Float32Array; + /** 9 个关节角余弦（DTW 输入，预览不用但保留） */ + angles: number[]; +} + +export interface ParsedKeypointSequence { + frames: ParsedKeypointFrame[]; + /** 平均帧间隔（毫秒）；用于播放速度 */ + frameIntervalMs: number; + durationMs: number; +} + +const POINT_COUNT = 33; + +function toNumber(value: unknown, fallback = 0): number { + return typeof value === 'number' && Number.isFinite(value) ? value : fallback; +} + +/** 解析一行；结构不合法返回 null（跳过该行而不是整段失败） */ +function parseLine(line: string): ParsedKeypointFrame | null { + let raw: unknown; + try { + raw = JSON.parse(line); + } catch { + return null; + } + if (!raw || typeof raw !== 'object') return null; + const obj = raw as { frame?: unknown; timeMs?: unknown; points?: unknown; angles?: unknown }; + if (!Array.isArray(obj.points) || obj.points.length === 0) return null; + const count = Math.min(POINT_COUNT, obj.points.length); + const positions = new Float32Array(POINT_COUNT * 3); + const visibility = new Float32Array(POINT_COUNT); + for (let index = 0; index < count; index += 1) { + const point = obj.points[index]; + if (!Array.isArray(point)) continue; + positions[index * 3] = toNumber(point[0]); + positions[index * 3 + 1] = toNumber(point[1]); + positions[index * 3 + 2] = toNumber(point[2]); + visibility[index] = toNumber(point[3], 1); + } + return { + frame: toNumber(obj.frame), + timeMs: toNumber(obj.timeMs), + positions, + visibility, + angles: Array.isArray(obj.angles) ? obj.angles.map((value) => toNumber(value)) : [], + }; +} + +/** + * 解析 NDJSON；空串 / 全部非法时返回 `frames: []`（调用方给"无数据"提示，不抛错）。 + */ +export function parseKeypointNdjson(text: string): ParsedKeypointSequence { + const frames: ParsedKeypointFrame[] = []; + for (const line of text.split(/\r?\n/)) { + if (line.trim().length === 0) continue; + const frame = parseLine(line); + if (frame) frames.push(frame); + } + let interval = 1000 / 30; + if (frames.length >= 2) { + const first = frames[0].timeMs; + const last = frames[frames.length - 1].timeMs; + const span = last - first; + if (span > 0) interval = span / (frames.length - 1); + } + const durationMs = frames.length > 0 ? frames[frames.length - 1].timeMs + interval : 0; + return { frames, frameIntervalMs: interval, durationMs }; +} + +/** 计算整段（或指定帧）的包围盒，用于自动取景；返回 null 表示没有可用点 */ +export function sequenceBounds( + frames: readonly ParsedKeypointFrame[], + minVisibility = 0.3, +): { min: [number, number, number]; max: [number, number, number] } | null { + let minX = Number.POSITIVE_INFINITY; + let minY = Number.POSITIVE_INFINITY; + let minZ = Number.POSITIVE_INFINITY; + let maxX = Number.NEGATIVE_INFINITY; + let maxY = Number.NEGATIVE_INFINITY; + let maxZ = Number.NEGATIVE_INFINITY; + let seen = 0; + for (const frame of frames) { + for (let index = 0; index < POINT_COUNT; index += 1) { + if (frame.visibility[index] < minVisibility) continue; + const x = frame.positions[index * 3]; + const y = frame.positions[index * 3 + 1]; + const z = frame.positions[index * 3 + 2]; + if (x < minX) minX = x; + if (y < minY) minY = y; + if (z < minZ) minZ = z; + if (x > maxX) maxX = x; + if (y > maxY) maxY = y; + if (z > maxZ) maxZ = z; + seen += 1; + } + } + if (seen === 0) return null; + return { min: [minX, minY, minZ], max: [maxX, maxY, maxZ] }; +} diff --git a/web-server/admin/src/lib/skeleton-topology.ts b/web-server/admin/src/lib/skeleton-topology.ts new file mode 100644 index 0000000..91fa878 --- /dev/null +++ b/web-server/admin/src/lib/skeleton-topology.ts @@ -0,0 +1,109 @@ +/** + * MediaPipe Pose 33 点拓扑（管理端预览用）。 + * + * 与 `contracts/skeleton-topology.md` §1/§3 一致： + * * 33 个关键点的语义顺序； + * * 父节点表（`parent[i]`，33/34 是虚拟关节 —— 骨盆/肩中心，绘制时归到 0/nose 之外单独处理）。 + * 服务端权威实现在 `web-server/src/preprocess/mediapipe33.ts`（含测试断言的一致性检查）。 + * 这里只保留预览绘制需要的两份最小数据，避免把服务端代码打进前端包。 + */ + +export const KEYPOINT_NAMES: readonly string[] = [ + 'nose', + 'left_eye_inner', + 'left_eye', + 'left_eye_outer', + 'right_eye_inner', + 'right_eye', + 'right_eye_outer', + 'left_ear', + 'right_ear', + 'mouth_left', + 'mouth_right', + 'left_shoulder', + 'right_shoulder', + 'left_elbow', + 'right_elbow', + 'left_wrist', + 'right_wrist', + 'left_pinky', + 'right_pinky', + 'left_index', + 'right_index', + 'left_thumb', + 'right_thumb', + 'left_hip', + 'right_hip', + 'left_knee', + 'right_knee', + 'left_ankle', + 'right_ankle', + 'left_heel', + 'right_heel', + 'left_foot_index', + 'right_foot_index', +]; + +/** 虚拟关节索引（不在 33 点内，仅用于层级） */ +export const VIRTUAL_PELVIS = 33; +export const VIRTUAL_SHOULDER_CENTER = 34; + +/** `parent[i]`：-1 表示世界根；33/34 表示虚拟关节 */ +export const PARENT_TABLE: readonly number[] = [ + VIRTUAL_SHOULDER_CENTER, // 0 nose + 0, // 1 left_eye_inner + 0, // 2 left_eye + 0, // 3 left_eye_outer + 0, // 4 right_eye_inner + 0, // 5 right_eye + 0, // 6 right_eye_outer + 3, // 7 left_ear + 6, // 8 right_ear + 0, // 9 mouth_left + 0, // 10 mouth_right + VIRTUAL_SHOULDER_CENTER, // 11 left_shoulder + VIRTUAL_SHOULDER_CENTER, // 12 right_shoulder + 11, // 13 left_elbow + 12, // 14 right_elbow + 13, // 15 left_wrist + 14, // 16 right_wrist + 15, // 17 left_pinky + 16, // 18 right_pinky + 15, // 19 left_index + 16, // 20 right_index + 15, // 21 left_thumb + 16, // 22 right_thumb + VIRTUAL_PELVIS, // 23 left_hip + VIRTUAL_PELVIS, // 24 right_hip + 23, // 25 left_knee + 24, // 26 right_knee + 25, // 27 left_ankle + 26, // 28 right_ankle + 27, // 29 left_heel + 28, // 30 right_heel + 27, // 31 left_foot_index + 28, // 32 right_foot_index +]; + +/** 绘制用骨段：把父表展开成 [子, 父] 对（虚拟关节映射到鼻/肩线的实际点） */ +export const BONE_SEGMENTS: ReadonlyArray = (() => { + const out: Array = []; + for (let index = 0; index < PARENT_TABLE.length; index += 1) { + const parent = PARENT_TABLE[index]; + if (parent === -1) continue; + if (parent === VIRTUAL_PELVIS) { + // 骨盆虚拟点：用左右髋的中点表示，绘制时由渲染器按帧计算，这里先挂到左髋 + out.push([index, 23]); + continue; + } + if (parent === VIRTUAL_SHOULDER_CENTER) { + out.push([index, 11]); + continue; + } + out.push([index, parent]); + } + return out; +})(); + +/** 需要忽略的绘制段（面部五官连线太密，预览里只会糊成一团） */ +export const FACE_POINT_LIMIT = 11; diff --git a/web-server/admin/src/pages/Motions.tsx b/web-server/admin/src/pages/Motions.tsx index 9a1feff..abf589c 100644 --- a/web-server/admin/src/pages/Motions.tsx +++ b/web-server/admin/src/pages/Motions.tsx @@ -1,4 +1,4 @@ -import { useState, type FormEvent } from 'react'; +import { lazy, Suspense, useState, type FormEvent } from 'react'; import { apiUrl, errorMessage } from '../api/client'; import { deleteMotion, @@ -13,6 +13,10 @@ import { formatNumber } from '../lib/format'; import { MOTION_FORMAT_LABELS, PREPROCESS_LABELS, STATUS_OPTIONS } from '../lib/labels'; import { hasErrors, parseOptionalNumber, requireText, validateSourceFields, type FieldErrors } from '../lib/validation'; import { useAsync } from '../lib/useAsync'; +const MotionPreview = lazy(async () => { + const module = await import('../components/MotionPreview'); + return { default: module.MotionPreview }; +}); import { ConfirmDialog, Empty, @@ -228,6 +232,7 @@ export function MotionsPage() { const [filters, setFilters] = useState({ q: '', format: '', status: '', page: 1 }); const [qInput, setQInput] = useState(''); const [uploadOpen, setUploadOpen] = useState(false); + const [previewId, setPreviewId] = useState(null); const [deleteTarget, setDeleteTarget] = useState(null); const [busyId, setBusyId] = useState(null); const [deleting, setDeleting] = useState(false); @@ -383,6 +388,8 @@ export function MotionsPage() { + + @@ -398,6 +405,31 @@ export function MotionsPage() { const keypointsReady = motion.preprocessState === 'done'; return ( +
预览名称 关联曲目 格式
+ + {motion.thumbnailUrl ? ( + + {`${motion.name} setPreviewId(previewId === motion.id ? null : motion.id)} + + /> + + ) : ( + + — + + )} + +
{motion.name}
#{motion.id}
@@ -425,6 +457,23 @@ export function MotionsPage() { 		+ + + }> + + + + ) : null} {uploadOpen ? ( diff --git a/web-server/admin/src/styles.css b/web-server/admin/src/styles.css index 2cac0ef..0ad13c6 100644 --- a/web-server/admin/src/styles.css +++ b/web-server/admin/src/styles.css @@ -1493,3 +1493,49 @@ textarea { .bone-children { margin-left: auto; } + +/* ---- 动作预览（T-WEB-014） ---- */ +.thumb-40 { + width: 40px; + height: 40px; + border-radius: 4px; + background: #181a20; + object-fit: cover; +} + +.thumb-clickable { + cursor: pointer; + border: 1px solid var(--border); +} + +.motion-preview-panel { + margin-top: 12px; +} + +.motion-preview-canvas { + width: 100%; + border-radius: var(--radius-sm); + overflow: hidden; + background: #181a20; +} + +.motion-preview-bar { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; + padding: 8px 4px 2px; +} + +.motion-preview-bar input[type='range'] { + flex: 1 1 220px; + min-width: 160px; +} + +.motion-preview-speed { + display: inline-flex; + align-items: center; + gap: 4px; + font-size: 12px; + color: var(--text-mute); +} diff --git a/web-server/package.json b/web-server/package.json index 5ce1709..b18d8ee 100644 --- a/web-server/package.json +++ b/web-server/package.json @@ -1,6 +1,6 @@ { "name": "opendance-web-server", - "version": "0.4.0", + "version": "0.5.0", "description": "OpenDance Studio Web 管理服务端（NestJS + TypeORM + 现有 MySQL 实例）", "license": "MIT", "private": true, diff --git a/web-server/src/common/enums.ts b/web-server/src/common/enums.ts index e04a248..ceb13be 100644 --- a/web-server/src/common/enums.ts +++ b/web-server/src/common/enums.ts @@ -70,6 +70,8 @@ export enum AssetKind { /** 上传的模型压缩包（zip）原件，仅用于溯源；入库的是转换后的模型文件 */ ModelArchive = 'model-archive', Thumbnail = 'thumbnail', + /** 动作关键点序列的骨架缩略图（T-WEB-014） */ + MotionThumbnail = 'motion-thumbnail', Keypoints = 'keypoints', } diff --git a/web-server/src/common/png.ts b/web-server/src/common/png.ts new file mode 100644 index 0000000..4c7cf6c --- /dev/null +++ b/web-server/src/common/png.ts @@ -0,0 +1,65 @@ +/** + * 极简 PNG 编码（8 位真彩 RGB、无隔行、filter=0、手写 CRC32）。 + * + * 模型缩略图与动作缩略图共用：**零原生依赖**（只用 node:zlib 的 deflate）， + * 避免为了出图引入 canvas / sharp / headless-gl。 + * 解码见 `models/model-thumbnail.ts` 内的 decodePng（同样零依赖）。 + */ +import { deflateSync } from 'node:zlib'; + +const CRC_TABLE = (() => { + const table = new Uint32Array(256); + for (let n = 0; n < 256; n += 1) { + let c = n; + for (let k = 0; k < 8; k += 1) { + c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + } + table[n] = c >>> 0; + } + return table; +})(); + +function crc32(buffer: Buffer): number { + let c = 0xffffffff; + for (let i = 0; i < buffer.length; i += 1) { + c = CRC_TABLE[(c ^ buffer[i]) & 0xff] ^ (c >>> 8); + } + return (c ^ 0xffffffff) >>> 0; +} + +function pngChunk(type: string, data: Buffer): Buffer { + const typeBuffer = Buffer.from(type, 'ascii'); + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length, 0); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])), 0); + return Buffer.concat([length, typeBuffer, data, crc]); +} + +/** 把 RGB 像素（行优先，长度 = width*height*3）编码为 PNG */ +export function encodePngRgb(rgb: Buffer, width: number, height: number): Buffer { + const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(width, 0); + ihdr.writeUInt32BE(height, 4); + ihdr[8] = 8; // bit depth + ihdr[9] = 2; // color type: truecolor RGB + ihdr[10] = 0; // compression + ihdr[11] = 0; // filter method + ihdr[12] = 0; // interlace: none + + const stride = width * 3; + const raw = Buffer.alloc(height * (stride + 1)); + for (let y = 0; y < height; y += 1) { + raw[y * (stride + 1)] = 0; // filter type 0 + rgb.copy(raw, y * (stride + 1) + 1, y * stride, (y + 1) * stride); + } + + return Buffer.concat([ + signature, + pngChunk('IHDR', ihdr), + pngChunk('IDAT', deflateSync(raw)), + pngChunk('IEND', Buffer.alloc(0)), + ]); +} diff --git a/web-server/src/config/contract-versions.ts b/web-server/src/config/contract-versions.ts index ac7122d..5406c62 100644 --- a/web-server/src/config/contract-versions.ts +++ b/web-server/src/config/contract-versions.ts @@ -11,7 +11,7 @@ import { join } from 'node:path'; export const CONTRACT_VERSIONS: Readonly> = Object.freeze({ 'rest-api': '0.4.0', 'websocket-keypoints': '0.1.0', - 'data-models': '0.5.0', + 'data-models': '0.6.0', 'skeleton-topology': '0.2.0', 'model-formats': '0.5.0', database: '0.3.0', diff --git a/web-server/src/entities/motion.entity.ts b/web-server/src/entities/motion.entity.ts index 95076ac..6acd72f 100644 --- a/web-server/src/entities/motion.entity.ts +++ b/web-server/src/entities/motion.entity.ts @@ -57,6 +57,10 @@ export class Motion { @Column({ name: 'keypoint_frames', type: 'int', nullable: true }) keypointFrames!: number | null; + /** 骨架缩略图资产（预处理时生成；失败不影响预处理） */ + @Column({ name: 'thumbnail_asset_id', type: 'bigint', nullable: true, transformer: bigintNumber }) + thumbnailAssetId!: number | null; + @Column({ name: 'preprocess_state', type: 'varchar', length: 16, default: PreprocessState.None }) preprocessState!: PreprocessState; diff --git a/web-server/src/migrations/1789600000000-AddMotionThumbnail.ts b/web-server/src/migrations/1789600000000-AddMotionThumbnail.ts new file mode 100644 index 0000000..b7e61cf --- /dev/null +++ b/web-server/src/migrations/1789600000000-AddMotionThumbnail.ts @@ -0,0 +1,19 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +/** + * 动作预览（T-WEB-014）：`motions.thumbnail_asset_id` 存骨架缩略图资产。 + * + * 缩略图在 `/api/motions/:id/preprocess` 时生成（关键点序列 -> 正视图骨架 PNG）， + * 供动作管理列表展示；生成失败不影响预处理结果。 + */ +export class AddMotionThumbnail1789600000000 implements MigrationInterface { + name = 'AddMotionThumbnail1789600000000' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`motions\` ADD \`thumbnail_asset_id\` bigint NULL`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`motions\` DROP COLUMN \`thumbnail_asset_id\``); + } +} diff --git a/web-server/src/models/model-thumbnail.ts b/web-server/src/models/model-thumbnail.ts index f1e7bb9..9c7dd8b 100644 --- a/web-server/src/models/model-thumbnail.ts +++ b/web-server/src/models/model-thumbnail.ts @@ -1,7 +1,8 @@ import { readFile } from 'node:fs/promises'; import { inflateSync } from 'node:zlib'; + +import { encodePngRgb } from '../common/png'; import { dirname, resolve as resolvePath } from 'node:path'; -import { deflateSync } from 'node:zlib'; /** * 服务端模型缩略图生成（纯 TypeScript 软件光栅化）。 @@ -1355,67 +1356,6 @@ function decodePng(buffer: Buffer): DecodedImage | null { return { width, height, rgba: out }; } -// --------------------------------------------------------------------------- -// PNG 编码（8 位 RGB、无隔行、filter=0；不引第三方库） -// --------------------------------------------------------------------------- - -const CRC_TABLE = (() => { - const table = new Uint32Array(256); - for (let n = 0; n < 256; n += 1) { - let c = n; - for (let k = 0; k < 8; k += 1) { - c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; - } - table[n] = c >>> 0; - } - return table; -})(); - -function crc32(buffer: Buffer): number { - let c = 0xffffffff; - for (let i = 0; i < buffer.length; i += 1) { - c = CRC_TABLE[(c ^ buffer[i]) & 0xff] ^ (c >>> 8); - } - return (c ^ 0xffffffff) >>> 0; -} - -function pngChunk(type: string, data: Buffer): Buffer { - const typeBuffer = Buffer.from(type, 'ascii'); - const length = Buffer.alloc(4); - length.writeUInt32BE(data.length, 0); - const crc = Buffer.alloc(4); - crc.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])), 0); - return Buffer.concat([length, typeBuffer, data, crc]); -} - -/** 把 RGB 像素编码为 PNG（每行前置 filter 字节 0，整块 deflate） */ -function encodePng(rgb: Buffer, width: number, height: number): Buffer { - const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); - - const ihdr = Buffer.alloc(13); - ihdr.writeUInt32BE(width, 0); - ihdr.writeUInt32BE(height, 4); - ihdr[8] = 8; // bit depth - ihdr[9] = 2; // color type: truecolor RGB - ihdr[10] = 0; // compression - ihdr[11] = 0; // filter method - ihdr[12] = 0; // interlace: none - - const stride = width * 3; - const raw = Buffer.alloc(height * (stride + 1)); - for (let y = 0; y < height; y += 1) { - raw[y * (stride + 1)] = 0; // filter type 0 - rgb.copy(raw, y * (stride + 1) + 1, y * stride, (y + 1) * stride); - } - - return Buffer.concat([ - signature, - pngChunk('IHDR', ihdr), - pngChunk('IDAT', deflateSync(raw)), - pngChunk('IEND', Buffer.alloc(0)), - ]); -} - // --------------------------------------------------------------------------- // 对外入口 // --------------------------------------------------------------------------- @@ -1476,7 +1416,7 @@ export async function renderModelThumbnail( if (output.drawn === 0) return null; return { - png: encodePng(output.pixels, size, size), + png: encodePngRgb(output.pixels, size, size), width: size, height: size, trianglesDrawn: output.drawn, diff --git a/web-server/src/motions/motion-thumbnail.ts b/web-server/src/motions/motion-thumbnail.ts new file mode 100644 index 0000000..8687975 --- /dev/null +++ b/web-server/src/motions/motion-thumbnail.ts @@ -0,0 +1,177 @@ +/** + * 动作缩略图：把关键点序列画成一张骨架 PNG（零原生依赖）。 + * + * 用途：动作管理列表每行显示首帧骨架缩略图（T-WEB-014），随预处理一起生成并落库 + * （`motions.thumbnail_asset_id`）。与模型缩略图共用 `common/png.ts` 的编码器。 + * + * 画法取舍： + * * 只做**正视图**（X/Y 平面）——动作的"姿态"在正视图里最容易辨认，且不需要相机与光照； + * * 取**代表帧**（默认整段中间帧）绘制，包围盒按**整段**计算，保证不同动作构图一致； + * * 置信度低于阈值的点不画（与其相连的骨骼也跳过），避免出现"飘线"。 + */ +import { PARENT_TABLE } from '../preprocess/mediapipe33'; +import type { Keypoint, KeypointFrame } from '../preprocess/mediapipe33'; +import { encodePngRgb } from '../common/png'; + +export interface MotionThumbnailOptions { + /** 输出边长（正方形），默认 256 */ + size?: number; + /** 背景色 RGB，默认深灰 [24,26,32]（与模型缩略图一致） */ + background?: [number, number, number]; + /** 骨骼颜色 RGB */ + boneColor?: [number, number, number]; + /** 关节颜色 RGB */ + jointColor?: [number, number, number]; + /** 置信度阈值，低于该值的点不绘制，默认 0.3 */ + minVisibility?: number; + /** 指定绘制第几帧（默认中间帧）；越界会夹到范围内 */ + frameIndex?: number; +} + +const DEFAULT_SIZE = 256; +const MIN_SIZE = 32; +const MAX_SIZE = 1024; +const DEFAULT_BACKGROUND: [number, number, number] = [24, 26, 32]; +const DEFAULT_BONE: [number, number, number] = [96, 165, 250]; +const DEFAULT_JOINT: [number, number, number] = [248, 250, 252]; +const DEFAULT_MIN_VISIBILITY = 0.3; + +interface Point2 { + x: number; + y: number; +} + +/** 画一条粗线（Bresenham + 方形笔刷）；坐标已为像素空间 */ +function drawLine( + pixels: Uint8Array, + size: number, + from: Point2, + to: Point2, + color: [number, number, number], + thickness: number, +): void { + const half = Math.max(0, Math.floor(thickness / 2)); + const steps = Math.max(Math.abs(to.x - from.x), Math.abs(to.y - from.y)); + const count = Math.max(1, Math.ceil(steps)); + for (let step = 0; step <= count; step += 1) { + const t = count === 0 ? 0 : step / count; + const cx = Math.round(from.x + (to.x - from.x) * t); + const cy = Math.round(from.y + (to.y - from.y) * t); + for (let dy = -half; dy <= half; dy += 1) { + for (let dx = -half; dx <= half; dx += 1) { + const x = cx + dx; + const y = cy + dy; + if (x < 0 || y < 0 || x >= size || y >= size) continue; + const index = (y * size + x) * 3; + pixels[index] = color[0]; + pixels[index + 1] = color[1]; + pixels[index + 2] = color[2]; + } + } + } +} + +/** 画一个实心圆（关节） */ +function drawDisc( + pixels: Uint8Array, + size: number, + center: Point2, + radius: number, + color: [number, number, number], +): void { + const r = Math.max(1, radius); + const r2 = r * r; + for (let dy = -r; dy <= r; dy += 1) { + for (let dx = -r; dx <= r; dx += 1) { + if (dx * dx + dy * dy > r2) continue; + const x = Math.round(center.x) + dx; + const y = Math.round(center.y) + dy; + if (x < 0 || y < 0 || x >= size || y >= size) continue; + const index = (y * size + x) * 3; + pixels[index] = color[0]; + pixels[index + 1] = color[1]; + pixels[index + 2] = color[2]; + } + } +} + +/** + * 渲染动作骨架缩略图。 + * 关键点为空 / 全部低于置信度 / 退化包围盒时返回 null（调用方按"未生成"处理，不抛错）。 + */ +export function renderMotionThumbnail( + frames: readonly KeypointFrame[], + options: MotionThumbnailOptions = {}, +): Buffer | null { + if (frames.length === 0) return null; + const rawSize = options.size ?? DEFAULT_SIZE; + if (!Number.isFinite(rawSize)) throw new Error('缩略图尺寸非法：size 必须是有限数字'); + const size = Math.min(MAX_SIZE, Math.max(MIN_SIZE, Math.round(rawSize))); + const background = options.background ?? DEFAULT_BACKGROUND; + const boneColor = options.boneColor ?? DEFAULT_BONE; + const jointColor = options.jointColor ?? DEFAULT_JOINT; + const minVisibility = options.minVisibility ?? DEFAULT_MIN_VISIBILITY; + + // 包围盒按整段计算（所有可见点），保证同一动作族的构图一致 + let minX = Number.POSITIVE_INFINITY; + let minY = Number.POSITIVE_INFINITY; + let maxX = Number.NEGATIVE_INFINITY; + let maxY = Number.NEGATIVE_INFINITY; + let visible = 0; + for (const frame of frames) { + for (const point of frame.points) { + // Keypoint = [x, y, z, visibility] + const [x, y, , visibility] = point; + if (visibility < minVisibility) continue; + if (!Number.isFinite(x) || !Number.isFinite(y)) continue; + if (x < minX) minX = x; + if (x > maxX) maxX = x; + if (y < minY) minY = y; + if (y > maxY) maxY = y; + visible += 1; + } + } + if (visible === 0 || !Number.isFinite(minX) || !Number.isFinite(minY)) return null; + + const width = Math.max(maxX - minX, 1e-3); + const height = Math.max(maxY - minY, 1e-3); + const scale = (size * 0.86) / Math.max(width, height); + const offsetX = size / 2 - ((minX + maxX) / 2) * scale; + // glTF / 关键点坐标是 Y 向上，图像 Y 向下 —— 这里翻转，保证头在上 + const offsetY = size / 2 + ((minY + maxY) / 2) * scale; + const project = (point: Keypoint): Point2 => ({ + x: point[0] * scale + offsetX, + y: offsetY - point[1] * scale, + }); + + const pixels = new Uint8Array(size * size * 3); + for (let i = 0; i < size * size; i += 1) { + pixels[i * 3] = background[0]; + pixels[i * 3 + 1] = background[1]; + pixels[i * 3 + 2] = background[2]; + } + + const requested = options.frameIndex ?? Math.floor(frames.length / 2); + const frame = frames[Math.min(frames.length - 1, Math.max(0, requested))]; + const thickness = Math.max(2, Math.round(size / 96)); + const jointRadius = Math.max(1, Math.round(size / 128)); + + // 骨骼：按 33 点父表连线（父索引 33/34 是虚拟关节，映射到 0） + for (let index = 0; index < PARENT_TABLE.length && index < frame.points.length; index += 1) { + const rawParent = PARENT_TABLE[index]; + const parentIndex = rawParent === 33 || rawParent === 34 ? 0 : rawParent; + if (parentIndex < 0 || parentIndex >= frame.points.length) continue; + const child = frame.points[index]; + const parent = frame.points[parentIndex]; + if (child[3] < minVisibility || parent[3] < minVisibility) continue; + drawLine(pixels, size, project(parent), project(child), boneColor, thickness); + } + + // 关节 + for (const point of frame.points) { + if (point[3] < minVisibility) continue; + drawDisc(pixels, size, project(point), jointRadius, jointColor); + } + + return encodePngRgb(Buffer.from(pixels.buffer, pixels.byteOffset, pixels.length), size, size); +} diff --git a/web-server/src/motions/motion.serializer.ts b/web-server/src/motions/motion.serializer.ts index 53afc8b..6d0214a 100644 --- a/web-server/src/motions/motion.serializer.ts +++ b/web-server/src/motions/motion.serializer.ts @@ -14,6 +14,8 @@ export interface MotionDto { durationMs: number | null; keypointFormat: string | null; keypointUrl: string | null; + /** 骨架缩略图（预处理时生成；T-WEB-014） */ + thumbnailUrl: string | null; keypointFrames: number | null; preprocessState: PreprocessState; preprocessError: string | null; @@ -31,6 +33,7 @@ export interface MotionUrlLookup { fileUrl: string | null; keypointUrl: string | null; fileSize?: number | null; + thumbnailUrl: string | null; } export function toMotionDto(motion: Motion, urls: MotionUrlLookup): MotionDto { @@ -46,6 +49,7 @@ export function toMotionDto(motion: Motion, urls: MotionUrlLookup): MotionDto { durationMs: motion.durationMs, keypointFormat: motion.keypointFormat, keypointUrl: urls.keypointUrl, + thumbnailUrl: urls.thumbnailUrl, keypointFrames: motion.keypointFrames, preprocessState: motion.preprocessState, preprocessError: motion.preprocessError, diff --git a/web-server/src/motions/motions.service.ts b/web-server/src/motions/motions.service.ts index aee6ff4..2b42f32 100644 --- a/web-server/src/motions/motions.service.ts +++ b/web-server/src/motions/motions.service.ts @@ -204,14 +204,16 @@ export class MotionsService { async toDtos(rows: Motion[]): Promise { const assets = await this.storage.findMany( - rows.flatMap((row) => [row.fileAssetId, row.keypointAssetId]), + rows.flatMap((row) => [row.fileAssetId, row.keypointAssetId, row.thumbnailAssetId]), ); return rows.map((row) => { const file = row.fileAssetId ? assets.get(row.fileAssetId) : undefined; const keypoints = row.keypointAssetId ? assets.get(row.keypointAssetId) : undefined; + const thumbnail = row.thumbnailAssetId ? assets.get(row.thumbnailAssetId) : undefined; return toMotionDto(row, { fileUrl: file ? this.storage.url(file) : null, keypointUrl: keypoints ? this.storage.url(keypoints) : null, + thumbnailUrl: thumbnail ? this.storage.url(thumbnail) : null, fileSize: file?.size ?? null, }); }); diff --git a/web-server/src/preprocess/preprocess.service.ts b/web-server/src/preprocess/preprocess.service.ts index 11a2dff..f58300e 100644 --- a/web-server/src/preprocess/preprocess.service.ts +++ b/web-server/src/preprocess/preprocess.service.ts @@ -1,4 +1,5 @@ import { BadRequestException, Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { renderMotionThumbnail } from '../motions/motion-thumbnail'; import { InjectRepository } from '@nestjs/typeorm'; import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -63,6 +64,26 @@ export class PreprocessService { motion.keypointFormat = KEYPOINT_FORMAT; motion.keypointAssetId = stored.asset.id; + + // 骨架缩略图（T-WEB-014）：动作管理列表展示；失败只记 warning，不影响预处理结果 + try { + const thumbnail = renderMotionThumbnail(result.frames); + if (thumbnail) { + const storedThumb = await this.storage.persistBuffer( + AssetKind.MotionThumbnail, + thumbnail, + `motion-${motion.id}-thumb.png`, + 'image/png', + ); + motion.thumbnailAssetId = storedThumb.asset.id; + } else { + this.logger.warn(`动作 #${motion.id} 缩略图未生成（关键点不可用）`); + } + } catch (error) { + this.logger.warn( + `动作 #${motion.id} 缩略图生成失败：${error instanceof Error ? error.message : String(error)}`, + ); + } motion.keypointFrames = result.frames.length; motion.fps = result.fps; motion.frameCount = result.frames.length; diff --git a/web-server/test/motion-preview.spec.ts b/web-server/test/motion-preview.spec.ts new file mode 100644 index 0000000..7c9259d --- /dev/null +++ b/web-server/test/motion-preview.spec.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from 'vitest'; + +import { parseKeypointNdjson, sequenceBounds } from '../admin/src/lib/keypoint-sequence'; +import { BONE_SEGMENTS, KEYPOINT_NAMES, PARENT_TABLE } from '../admin/src/lib/skeleton-topology'; +import { renderMotionThumbnail } from '../src/motions/motion-thumbnail'; +import type { KeypointFrame } from '../src/preprocess/mediapipe33'; + +/** + * 动作预览（T-WEB-014）： + * * 服务端：关键点序列 -> 骨架缩略图（列表展示）； + * * 管理端：NDJSON 解析 -> 3D 回放（播放/逐帧/时间轴/变速/循环）。 + */ + +/** 造一帧：所有点可见，坐标沿索引铺开（便于验证解析与绘制） */ +function makeFrame(index: number, visibility = 1): KeypointFrame { + const points = Array.from({ length: 33 }, (_, point) => [ + Math.cos((point / 33) * Math.PI * 2) * 0.5, + (point / 33) * 1.6 - 0.8, + Math.sin((point / 33) * Math.PI * 2) * 0.2, + visibility, + ]) as KeypointFrame['points']; + return { frame: index, timeMs: index * (1000 / 30), points, angles: new Array(9).fill(0.5) }; +} + +function ndjson(frames: KeypointFrame[]): string { + return frames.map((frame) => JSON.stringify(frame)).join('\n') + '\n'; +} + +/** 取 PNG 里偏离背景色的像素数（背景 [24,26,32]） */ +function drawnPixels(png: Buffer): number { + let count = 0; + for (let i = 0; i + 2 < png.length; i += 1) { + // 直接扫原始字节（IDAT 已压缩，这里只做粗略计数：非背景三连出现即算） + if (png[i] === 96 && png[i + 1] === 165 && png[i + 2] === 250) count += 1; + } + return count; +} + +describe('动作缩略图（服务端）', () => { + it('关键点序列 -> 合法 PNG（含骨架像素）', () => { + const png = renderMotionThumbnail([makeFrame(0), makeFrame(1), makeFrame(2)], { size: 128 }); + expect(png).not.toBeNull(); + // PNG magic + expect(png!.subarray(0, 8)).toEqual(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])); + expect(png!.length).toBeGreaterThan(200); + }); + + it('空序列 / 全部低置信度 -> null（不抛错）', () => { + expect(renderMotionThumbnail([])).toBeNull(); + expect(renderMotionThumbnail([makeFrame(0, 0.01)])).toBeNull(); + }); + + it('尺寸越界抛中文错误（参数非法与"读不到"区分开）', () => { + expect(() => renderMotionThumbnail([makeFrame(0)], { size: Number.NaN })).toThrow(/尺寸非法/); + expect(renderMotionThumbnail([makeFrame(0)], { size: 8 })).not.toBeNull(); // 夹到 32 + }); +}); + +describe('关键点 NDJSON 解析（管理端）', () => { + it('解析帧数与帧间隔；数组尺寸固定为 33*3 / 33', () => { + const parsed = parseKeypointNdjson(ndjson([makeFrame(0), makeFrame(1), makeFrame(2)])); + expect(parsed.frames.length).toBe(3); + expect(parsed.frames[0].positions.length).toBe(33 * 3); + expect(parsed.frames[0].visibility.length).toBe(33); + // 连续帧：帧间隔 = 1000/30 ms + expect(parsed.frameIntervalMs).toBeCloseTo(1000 / 30, 3); + expect(parsed.durationMs).toBeGreaterThan(0); + }); + + it('坏行被跳过，好行照常解析（单行损坏不影响整段）', () => { + const text = `{"frame":0,"timeMs":0,"points":[[0,0,0,1]],"angles":[]}\n坏行\n${JSON.stringify(makeFrame(1))}\n`; + const parsed = parseKeypointNdjson(text); + expect(parsed.frames.length).toBe(2); + }); + + it('空文本 -> 0 帧（页面据此提示"先预处理"）', () => { + expect(parseKeypointNdjson('').frames.length).toBe(0); + expect(parseKeypointNdjson('\n\n').frames.length).toBe(0); + }); + + it('包围盒按整段计算（用于自动取景）', () => { + const parsed = parseKeypointNdjson(ndjson([makeFrame(0), makeFrame(10)])); + const bounds = sequenceBounds(parsed.frames); + expect(bounds).not.toBeNull(); + expect(bounds!.max[1]).toBeGreaterThan(bounds!.min[1]); + }); +}); + +describe('33 点拓扑（管理端绘制用）', () => { + it('33 个点、父表长度一致、骨段索引合法', () => { + expect(KEYPOINT_NAMES.length).toBe(33); + expect(PARENT_TABLE.length).toBe(33); + expect(BONE_SEGMENTS.length).toBeGreaterThan(20); + for (const [child, parent] of BONE_SEGMENTS) { + expect(child).toBeGreaterThanOrEqual(0); + expect(child).toBeLessThan(33); + expect(parent).toBeGreaterThanOrEqual(0); + expect(parent).toBeLessThan(33); + } + }); +});