feat(web): T-WEB-006 BVH/FBX → MediaPipe33 关键点预处理

2026-09-14 19:00:55 +08:00
parent 73e793318e
commit a2e3282603
12 changed files with 1799 additions and 0 deletions
@@ -0,0 +1,144 @@
+import { BadRequestException, Injectable, Logger, NotFoundException } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Repository } from 'typeorm';
+import { AssetKind, MotionFormat, PreprocessState } from '../common/enums';
+import { AppConfig } from '../config/configuration';
+import { Motion } from '../entities/motion.entity';
+import { StorageService } from '../storage/storage.service';
+import { parseBvh } from './bvh-parser';
+import { parseKeypointsJson, serializeFramesNdjson } from './keypoint-codec';
+import { KEYPOINT_FORMAT, KeypointFrame } from './mediapipe33';
+import { bvhToKeypointFrames } from './retarget';
+import { convertToBvhWithAssimp } from './assimp';
+
+export interface PreprocessOutcome {
+  frames: number;
+  fps: number;
+  durationMs: number;
+  format: string;
+}
+
+/**
+ * T-WEB-006：BVH / FBX → MediaPipe 33 点关键点序列预处理。
+ *
+ * - BVH：内置纯 TS 解析器 + FK + 重定向，无外部依赖
+ * - glTF / GLB / FBX：调用可选的外部 Assimp 转为 BVH 后再走同一通路
+ * - keypoints-json：直通校验与规范化
+ */
+@Injectable()
+export class PreprocessService {
+  private readonly logger = new Logger(PreprocessService.name);
+
+  constructor(
+    @InjectRepository(Motion) private readonly motions: Repository<Motion>,
+    private readonly storage: StorageService,
+    private readonly config: AppConfig,
+  ) {}
+
+  /** 执行预处理并落库；异常会被记录到 preprocessError 后重新抛出 */
+  async run(motion: Motion): Promise<Motion> {
+    const started = Date.now();
+    motion.preprocessState = PreprocessState.Running;
+    motion.preprocessError = null;
+    await this.motions.save(motion);
+
+    try {
+      const result = await this.convert(motion);
+      const ndjson = serializeFramesNdjson(result.frames);
+
+      const stored = await this.storage.persistBuffer(
+        AssetKind.Keypoints,
+        ndjson,
+        `motion-${motion.id}.ndjson`,
+        'application/x-ndjson',
+      );
+
+      motion.keypointFormat = KEYPOINT_FORMAT;
+      motion.keypointAssetId = stored.asset.id;
+      motion.keypointFrames = result.frames.length;
+      motion.fps = result.fps;
+      motion.frameCount = result.frames.length;
+      motion.durationMs = result.durationMs;
+      motion.preprocessState = PreprocessState.Done;
+      motion.preprocessError = null;
+
+      const saved = await this.motions.save(motion);
+      this.logger.log(
+        `动作 #${motion.id} 预处理完成：${result.frames.length} 帧，${Date.now() - started}ms`,
+      );
+      return saved;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      motion.preprocessState = PreprocessState.Failed;
+      motion.preprocessError = message.slice(0, 2000);
+      await this.motions.save(motion);
+      this.logger.error(`动作 #${motion.id} 预处理失败：${message}`);
+      throw error;
+    }
+  }
+
+  /** 读取已完成预处理的关键点 NDJSON */
+  async readKeypointsNdjson(motion: Motion): Promise<string> {
+    if (motion.preprocessState !== PreprocessState.Done || !motion.keypointAssetId) {
+      throw new BadRequestException('该动作尚未完成关键点预处理，请先调用 /preprocess');
+    }
+    const asset = await this.storage.findById(motion.keypointAssetId);
+    if (!asset) throw new NotFoundException('关键点文件已丢失');
+    return this.storage.readAssetText(asset);
+  }
+
+  private async convert(
+    motion: Motion,
+  ): Promise<{ frames: KeypointFrame[]; fps: number; durationMs: number }> {
+    const asset = await this.storage.findById(motion.fileAssetId);
+    if (!asset) throw new BadRequestException('动作源文件缺失，无法预处理');
+    const sourcePath = this.storage.absolutePath(asset);
+
+    let workDir: string | null = null;
+    try {
+      if (motion.format === MotionFormat.Bvh) {
+        const text = await readFile(sourcePath, 'utf8');
+        return this.fromBvh(text, motion.fps ?? undefined);
+      }
+
+      if (motion.format === MotionFormat.KeypointsJson) {
+        const text = await readFile(sourcePath, 'utf8');
+        const { frames, fps } = parseKeypointsJson(text);
+        if (frames.length === 0) throw new BadRequestException('关键点 JSON 中没有任何帧');
+        const effectiveFps = fps ?? motion.fps ?? 30;
+        return {
+          frames: frames.map((frame, index) => ({
+            ...frame,
+            frame: index,
+            timeMs: frame.timeMs || Math.round((index / effectiveFps) * 1000 * 1000) / 1000,
+          })),
+          fps: effectiveFps,
+          durationMs: Math.round((frames.length / effectiveFps) * 1000),
+        };
+      }
+
+      // FBX / glTF / GLB：交给可选的外部 Assimp
+      workDir = await mkdtemp(join(tmpdir(), 'opendance-assimp-'));
+      const bvhPath = await convertToBvhWithAssimp(sourcePath, workDir, this.config.assimpBin);
+      const text = await readFile(bvhPath, 'utf8');
+      return this.fromBvh(text, motion.fps ?? undefined);
+    } finally {
+      if (workDir) await rm(workDir, { recursive: true, force: true });
+    }
+  }
+
+  private fromBvh(text: string, fpsOverride?: number): { frames: KeypointFrame[]; fps: number; durationMs: number } {
+    const motion = parseBvh(text);
+    if (motion.frameCount === 0) throw new BadRequestException('BVH 中没有帧');
+    const frames = bvhToKeypointFrames(motion, { fps: fpsOverride });
+    const fps = fpsOverride && fpsOverride > 0 ? fpsOverride : 1 / motion.frameTime;
+    return {
+      frames,
+      fps: Math.round(fps * 1000) / 1000,
+      durationMs: Math.round((frames.length / fps) * 1000),
+    };
+  }
+}