opendance-web/web-server/test/real-samples.spec.ts

/**
 * 真实样例素材的端到端集成测试（**opt-in**）。
 *
 * 为什么单独一个文件：`model-archive.spec.ts` / `mmd-vmd.spec.ts` 用的都是
 * 合成的 PMX / GLB / VMD 字节与假 Assimp 夹具，能覆盖解析边界，但证明不了
 * 「**真的** MMD 素材能出片」——而 ①「所有模型 → VRM」与 ②「动作内部标准 +
 * 骨骼重绑定」的验收标准恰恰是真实素材。
 *
 * 门禁：找不到样例目录或 Assimp 时整个 describe 跳过，所以在没有素材的开发机
 * 与 CI 上 `npm run test` 依旧全绿；素材与 Assimp 齐备时会真跑一遍。
 *
 *   OPENDANCE_SAMPLES_DIR=/path/to/示例文件  ASSIMP_BIN=/usr/bin/assimp npm test
 *
 * 样例不随仓库分发（体积 + 版权），因此这里只断言**结构性不变量**，不断言
 * 模型的具体骨骼名或贴图内容。
 */
import { existsSync, readFileSync } from 'node:fs';
import { join } from 'node:path';
import { describe, expect, it } from 'vitest';

import { parseGlbChunks } from '../src/common/glb-chunks';
import { resolveAssimpBinary } from '../src/common/assimp';
import { ANGLE_NAMES, KEYPOINT_COUNT, KEYPOINT_FORMAT, MP } from '../src/preprocess/mediapipe33';
import { LEG_IK, MMD_SKELETON } from '../src/preprocess/mmd-skeleton';
import { vmdToWorldFrames } from '../src/preprocess/mmd-pose';
import { serializeFramesNdjson, parseFramesNdjson } from '../src/preprocess/keypoint-codec';
import { worldFramesToKeypointFrames } from '../src/preprocess/retarget';
import { VMD_FPS, parseVmd, trackHasPositionMotion } from '../src/preprocess/vmd-parser';
import { convertModelArchive } from '../src/models/model-convert';
import { inspectModelFile } from '../src/models/model-inspector';
import { renderMotionThumbnail } from '../src/motions/motion-thumbnail';

/* ------------------------------------------------------------------ 样例定位 */

/** 仓库根目录下的样例目录（web-server/test/../../.. = 仓库根） */
const SAMPLES_DIR =
  process.env.OPENDANCE_SAMPLES_DIR ?? join(__dirname, '..', '..', '..', '示例文件');

const MODEL_SMALL = '模型_初音未来Hatsune Miku_by_Boku_Wa_Luka.zip';
const MODEL_LARGE = '【奥黛塔】_by_原神.zip';
const MOTION_VMD = '【动作】霜雪千年.vmd';

const samplePath = (name: string): string => join(SAMPLES_DIR, name);
const hasSample = (name: string): boolean => existsSync(samplePath(name));
// 与 configuration.ts 的默认值保持一致（ASSIMP_BIN 未设时按 PATH 找 `assimp`）。
// 注意不能传 undefined：resolveAssimpBinary 会对入参调 `.includes()`。
const assimp = resolveAssimpBinary(process.env.ASSIMP_BIN ?? 'assimp');

const MODELS_READY = hasSample(MODEL_SMALL) && hasSample(MODEL_LARGE) && assimp.found;
const VMD_READY = hasSample(MOTION_VMD);

/** 只在需要时报出的跳过原因，方便排障 */
if (!MODELS_READY) {
  // eslint-disable-next-line no-console
  console.warn(
    `[real-samples] 跳过模型转换用例：样例目录 ${SAMPLES_DIR} 缺失文件或未找到 Assimp（${assimp.searched.join(' -> ')}）`,
  );
}
if (!VMD_READY) {
  // eslint-disable-next-line no-console
  console.warn(`[real-samples] 跳过 VMD 用例：缺少 ${samplePath(MOTION_VMD)}`);
}

/* ------------------------------------------------------- 访问器字节级读取工具 */

const COMPONENTS: Record<number, { size: number; get: (v: DataView, o: number) => number }> = {
  5120: { size: 1, get: (v, o) => v.getInt8(o) },
  5121: { size: 1, get: (v, o) => v.getUint8(o) },
  5122: { size: 2, get: (v, o) => v.getInt16(o, true) },
  5123: { size: 2, get: (v, o) => v.getUint16(o, true) },
  5125: { size: 4, get: (v, o) => v.getUint32(o, true) },
  5126: { size: 4, get: (v, o) => v.getFloat32(o, true) },
};

const COMPONENT_COUNT: Record<string, number> = {
  SCALAR: 1,
  VEC2: 2,
  VEC3: 3,
  VEC4: 4,
  MAT4: 16,
};

interface GltfDocument {
  json: Record<string, any>;
  bin: Buffer;
}

/**
 * 逐元素读出访问器覆盖的**真实** min/max。
 * 必须按 `byteStride` 走（Assimp 会给交错缓冲写 byteStride，忽略它会把
 * JOINTS_0 之类的 u16 数据读成乱码）。
 */
function actualBounds(gltf: GltfDocument, accessorIndex: number) {
  const a = gltf.json.accessors[accessorIndex];
  const view = gltf.json.bufferViews[a.bufferView];
  const component = COMPONENTS[a.componentType];
  const width = COMPONENT_COUNT[a.type];
  const stride = a.byteStride ?? component.size * width;
  const base = gltf.bin.byteOffset + (view.byteOffset ?? 0) + (a.byteOffset ?? 0);
  const dv = new DataView(gltf.bin.buffer, gltf.bin.byteOffset, gltf.bin.byteLength);

  const min = new Array<number>(width).fill(Infinity);
  const max = new Array<number>(width).fill(-Infinity);
  for (let e = 0; e < a.count; e += 1) {
    for (let c = 0; c < width; c += 1) {
      const value = component.get(dv, base + e * stride + c * component.size);
      if (value < min[c]) min[c] = value;
      if (value > max[c]) max[c] = value;
    }
  }
  return { min, max, accessor: a, component, width, stride, base };
}

/** POSITION 访问器的语义标记 = 4（ARRAY_BUFFER）且被某个 primitive 引用为 POSITION */
function positionAccessors(gltf: GltfDocument): number[] {
  const found = new Set<number>();
  for (const mesh of gltf.json.meshes ?? []) {
    for (const primitive of mesh.primitives ?? []) {
      const index = primitive.attributes?.POSITION;
      if (typeof index === 'number') found.add(index);
    }
  }
  return [...found].sort((a, b) => a - b);
}

/** 所有 JOINTS_0 访问器里出现的最大骨骼下标 */
function maxJointIndex(gltf: GltfDocument): number {
  let worst = -1;
  for (const mesh of gltf.json.meshes ?? []) {
    for (const primitive of mesh.primitives ?? []) {
      const index = primitive.attributes?.JOINTS_0;
      if (typeof index !== 'number') continue;
      const a = gltf.json.accessors[index];
      const view = gltf.json.bufferViews[a.bufferView];
      const component = COMPONENTS[a.componentType];
      const width = COMPONENT_COUNT[a.type];
      const stride = a.byteStride ?? component.size * width;
      const base = gltf.bin.byteOffset + (view.byteOffset ?? 0) + (a.byteOffset ?? 0);
      const dv = new DataView(gltf.bin.buffer, gltf.bin.byteOffset, gltf.bin.byteLength);
      for (let e = 0; e < a.count; e += 1) {
        for (let c = 0; c < width; c += 1) {
          const value = component.get(dv, base + e * stride + c * component.size);
          if (value > worst) worst = value;
        }
      }
    }
  }
  return worst;
}

/* ----------------------------------------------------------------- 模型 → VRM */

describe.skipIf(!MODELS_READY)('真实样例：PMX 压缩包 → VRM（T-WEB-003/T-WEB-011）', () => {
  /** 转换结果按样例名缓存，两个 describe 之间不重复跑 Assimp */
  const cache = new Map<string, { converted: any; gltf: GltfDocument }>();

  async function convert(name: string) {
    const cached = cache.get(name);
    if (cached) return cached;
    const archive = readFileSync(samplePath(name));
    const converted = await convertModelArchive(archive, {
      assimpBin: assimp.path,
      timeoutMs: 600_000,
      preferFormat: 'vrm',
    });
    const { json, bin } = parseGlbChunks<Record<string, any>>(converted.buffer);
    expect(bin, 'VRM 必须内嵌 BIN 块（贴图与顶点数据都在里面）').not.toBeNull();
    const entry = { converted, gltf: { json, bin: bin as Buffer } };
    cache.set(name, entry);
    return entry;
  }

  /** 两个真实模型的公共不变量 */
  async function assertVrmInvariants(name: string) {
    const { converted, gltf } = await convert(name);
    const { json } = gltf;

    expect(converted.format).toBe('vrm');
    expect(json.asset?.version).toBe('2.0');
    expect(json.extensions?.VRMC_vrm?.specVersion).toBe('1.0');
    // 元信息里必须写清楚源文件与处理摘要，运营要能据此判断为什么「不可用」
    expect(converted.sourceModelPath.toLowerCase()).toMatch(/\.pmx$/);
    expect(converted.note.length).toBeGreaterThan(0);

    // 贴图必须全部内嵌：外链贴图一旦离开原目录就是坏图
    const images = json.images ?? [];
    expect(images.length).toBeGreaterThan(0);
    expect(images.filter((i: any) => i.bufferView !== undefined)).toHaveLength(images.length);
    expect(images.filter((i: any) => typeof i.uri === 'string')).toHaveLength(0);

    // MMD 专属的材质扩展不能泄漏到标准 PBR 产物里（客户端不认这些键）
    const mmdOnly = [
      'KHR_materials_specular',
      'KHR_materials_volume',
      'KHR_materials_transmission',
      'KHR_materials_ior',
      'KHR_materials_emissive_strength',
    ];
    const leftovers = new Set<string>();
    for (const material of json.materials ?? []) {
      for (const key of Object.keys(material.extensions ?? {})) {
        if (mmdOnly.includes(key)) leftovers.add(key);
      }
    }
    expect([...leftovers]).toEqual([]);

    // 顶点不能被写成 NaN / Inf —— 一个坏顶点就能让客户端整个网格消失
    for (const index of positionAccessors(gltf)) {
      const { min, max } = actualBounds(gltf, index);
      for (const v of [...min, ...max]) expect(Number.isFinite(v)).toBe(true);
      expect(max[0]).toBeGreaterThan(min[0]);
      expect(max[1]).toBeGreaterThan(min[1]);
      expect(max[2]).toBeGreaterThan(min[2]);

      // POSITION 的 min/max 是 glTF 规范**唯一强制**要求填写的，且必须等于真实
      // 包围盒：客户端拿它做视锥剔除与 AABB，填错会导致模型被整体剔除。
      const declared = json.accessors[index];
      expect(declared.min).toHaveLength(3);
      expect(declared.max).toHaveLength(3);
      for (let c = 0; c < 3; c += 1) {
        expect(declared.min[c]).toBeCloseTo(min[c], 4);
        expect(declared.max[c]).toBeCloseTo(max[c], 4);
      }
    }

    // 蒙皮下标必须落在 skin.joints 范围内，否则客户端会取到越界矩阵
    for (const skin of json.skins ?? []) {
      expect(maxJointIndex(gltf)).toBeLessThan(skin.joints.length);
    }

    // 分析器（审核界面看到的统计）与产物本身一致
    return { converted, gltf };
  }

  it(
    '小体积 PMX（初音未来，1 MB）转出可驱动的 VRM',
    async () => {
      const { converted, gltf } = await assertVrmInvariants(MODEL_SMALL);
      const { json } = gltf;

      expect(json.materials?.length ?? 0).toBeGreaterThanOrEqual(1);
      expect(json.skins?.length ?? 0).toBeGreaterThanOrEqual(1);

      expect(converted.note).toContain('VRM');
      expect(converted.buffer.length).toBeGreaterThan(100_000);
    },
    600_000,
  );

  it(
    '大体积 PMX（原神奥黛塔，28 MB / 18 贴图 / 36 双面材质）保留全部资源',
    async () => {
      const { converted, gltf } = await assertVrmInvariants(MODEL_LARGE);
      const { json } = gltf;

      const images = json.images ?? [];
      const materials = json.materials ?? [];
      expect(images.length).toBeGreaterThanOrEqual(10);
      expect(materials.length).toBeGreaterThanOrEqual(20);

      // MMD 的双面标志必须在转换时补回 GLB（否则裙摆/头发穿帮）
      const doubleSided = materials.filter((m: any) => m.doubleSided === true).length;
      expect(doubleSided).toBeGreaterThanOrEqual(20);

      // 产物不能比源压缩包还小一个量级（贴图丢了会）
      expect(converted.buffer.length).toBeGreaterThan(5_000_000);
    },
    900_000,
  );

  it(
    'REAL-SAMPLE humanoid 覆盖与可驱动性（真实骨架分析）',
    async () => {
      for (const name of [MODEL_SMALL, MODEL_LARGE]) {
        const { converted } = await convert(name);
        const { writeFileSync, mkdtempSync } = await import('node:fs');
        const { tmpdir } = await import('node:os');
        const dir = mkdtempSync(join(tmpdir(), 'opendance-real-'));
        const file = join(dir, 'model.glb');
        writeFileSync(file, converted.buffer);

        const stats = await inspectModelFile(file, converted.buffer.length);
        expect(stats.container).toBe('vrm');
        expect(stats.vrmVersion).toMatch(/^1\.0/);
        expect(stats.humanoidTotal).toBe(22);
        // 真实 MMD 素材的 humanoid 覆盖率下限：实测初音 19/22、奥黛塔 21/22
        expect(stats.humanoidMapped).toBeGreaterThanOrEqual(19);
        // 33 点驱动所需的 16 根必须齐全
        expect(stats.missingRequiredBones).toEqual([]);
        expect(stats.drivable).toBe(true);
        expect(stats.triangles).toBeGreaterThan(1_000);
        expect(stats.textures).toBe(stats.images);
        expect(stats.warnings.filter((w) => w.includes('无法'))).toEqual([]);
      }
    },
    900_000,
  );
});

/* ------------------------------------------------------- VMD → 动作内部标准 */

describe.skipIf(!VMD_READY)('真实样例：VMD → 动作内部标准 + 骨骼重绑定（T-WEB-006）', () => {
  const vmd = () => parseVmd(readFileSync(samplePath(MOTION_VMD)));

  it('解析真实 VMD：标准骨架 31 根全命中，双侧腿部 IK 链完整', () => {
    const motion = vmd();

    expect(motion.maxFrame).toBeGreaterThan(1000);
    expect(motion.tracks.size).toBeGreaterThanOrEqual(31);

    const standard = MMD_SKELETON.map((bone) => bone.name);
    expect(standard).toHaveLength(31);
    const missing = standard.filter((name) => !motion.tracks.has(name));
    expect(missing).toEqual([]);

    for (const side of ['左', '右'] as const) {
      const ik = LEG_IK[side];
      expect(motion.tracks.has(ik.ik), `${side}足ＩＫ`).toBe(true);
      expect(motion.tracks.has(ik.knee), `${side}ひざ`).toBe(true);
      expect(motion.tracks.has(ik.ankle), `${side}足首`).toBe(true);
      expect(motion.tracks.has(ik.toe), `${side}つま先`).toBe(true);
    }

    // 真实舞蹈数据里躯干有位移、腿部由 IK 目标驱动
    expect(trackHasPositionMotion(motion.tracks.get('センター'))).toBe(true);
    expect(trackHasPositionMotion(motion.tracks.get(LEG_IK.左.ik))).toBe(true);
  }, 120_000);

  it('骨骼重绑定：15 个直接关节命中真实骨骼（置信度 1.0），其余按层级推算（0.5）', () => {
    const motion = vmd();
    const { frames } = vmdToWorldFrames(motion, { fps: VMD_FPS });
    expect(frames).toHaveLength(motion.maxFrame + 1);

    // vmdToWorldFrames 把 MMD 骨骼名改写成 joint-map 的候选名（重绑定核心）
    const keys = [...frames[0].keys()].sort();
    expect(keys).toEqual(
      [
        'head',
        'leftarm',
        'leftfoot',
        'leftforearm',
        'lefthand',
        'leftleg',
        'lefttoebase',
        'leftupleg',
        'rightarm',
        'rightfoot',
        'rightforearm',
        'righthand',
        'rightleg',
        'righttoebase',
        'rightupleg',
      ].sort(),
    );
    // 逐元素断言在 2901 帧 × 33 点上跑会拖满 vitest 的超时，因此这里先聚合
    // 违规项、最后只 assert 一次；失败信息仍然带得出具体帧与下标。
    const violations: string[] = [];
    for (const [f, frame] of frames.entries()) {
      for (const key of keys) {
        const p = frame.get(key) as [number, number, number];
        if (!p.every((v) => Number.isFinite(v))) violations.push(`帧${f} ${key} 非有限`);
      }
    }
    expect(violations.slice(0, 5)).toEqual([]);

    const keypoints = worldFramesToKeypointFrames(frames, VMD_FPS);
    expect(keypoints).toHaveLength(frames.length);

    // 显式标注 number[]：MP 是 as const，直接从它推出的数组是字面量联合，
    // 会让 `DIRECT.includes(index)` 这类按 number 查询的调用过不了类型检查。
    const DIRECT: number[] = [MP.NOSE, MP.LEFT_SHOULDER, MP.RIGHT_SHOULDER, MP.LEFT_ELBOW, MP.RIGHT_ELBOW,
      MP.LEFT_WRIST, MP.RIGHT_WRIST, MP.LEFT_HIP, MP.RIGHT_HIP, MP.LEFT_KNEE, MP.RIGHT_KNEE,
      MP.LEFT_ANKLE, MP.RIGHT_ANKLE, MP.LEFT_FOOT_INDEX, MP.RIGHT_FOOT_INDEX];

    const confidence = new Map<number, Set<number>>();
    for (const [f, frame] of keypoints.entries()) {
      if (frame.points.length !== KEYPOINT_COUNT) violations.push(`帧${f} 点数 ${frame.points.length}`);
      if (frame.angles.length !== ANGLE_NAMES.length) violations.push(`帧${f} 角度数 ${frame.angles.length}`);
      frame.points.forEach((point, index) => {
        const [x, y, z, c] = point;
        if (![x, y, z].every(Number.isFinite)) violations.push(`帧${f} 点${index} 非有限`);
        if (!(c === 0 || c === 0.5 || c === 1)) violations.push(`帧${f} 点${index} 置信度 ${c}`);
        // 骨盆归零后投影到图像空间，两个方向都必须落在 [0,1]
        if (Math.abs(x) > 1.0001 || Math.abs(y) > 1.0001) violations.push(`帧${f} 点${index} 越界 ${x},${y}`);
        const set = confidence.get(index) ?? new Set<number>();
        set.add(c);
        confidence.set(index, set);
      });
      for (const angle of frame.angles) {
        // 关节角按契约存 cos 值，必须有限且落在 [-1,1]
        if (!Number.isFinite(angle) || Math.abs(angle) > 1.0001) violations.push(`帧${f} 角度 ${angle}`);
      }
    }
    expect(violations.slice(0, 5)).toEqual([]);

    // 直接绑定的索引必须**只能**是置信度 1.0，绝不退化成推算值
    for (const index of DIRECT) expect([...confidence.get(index)!]).toEqual([1]);
    // 其余 18 个点全部由层级推算，且没有任何点彻底缺失
    const derived = [...confidence.keys()].filter((i) => !DIRECT.includes(i));
    expect(derived).toHaveLength(KEYPOINT_COUNT - DIRECT.length);
    for (const index of derived) expect([...confidence.get(index)!]).toEqual([0.5]);
  }, 300_000);

  it('内部标准序列化：NDJSON 往返一致、缩略图有效、动作幅度非退化', () => {
    const motion = vmd();
    const { frames } = vmdToWorldFrames(motion, { fps: VMD_FPS });
    const keypoints = worldFramesToKeypointFrames(frames, VMD_FPS);

    const ndjson = serializeFramesNdjson(keypoints);
    expect(ndjson.endsWith('\n')).toBe(true);
    const roundTrip = parseFramesNdjson(ndjson);
    expect(roundTrip).toHaveLength(keypoints.length);
    expect(roundTrip[0].points).toHaveLength(KEYPOINT_COUNT);
    expect(roundTrip.every((f, i) => f.frame === keypoints[i].frame)).toBe(true);
    expect(KEYPOINT_FORMAT).toBe('mediapipe33@0.1.0');

    const thumbnail = renderMotionThumbnail(keypoints, { size: 256 });
    expect(thumbnail).not.toBeNull();
    expect(thumbnail!.subarray(0, 8).toString('hex')).toBe('89504e470d0a1a0a');

    // 动作幅度：手腕必须真的在动，鼻子相对稳定
    const spread = (index: number) => {
      const xs = keypoints.map((f) => f.points[index][0]);
      return Math.max(...xs) - Math.min(...xs);
    };
    expect(spread(MP.LEFT_WRIST)).toBeGreaterThan(0.1);
    expect(spread(MP.RIGHT_WRIST)).toBeGreaterThan(0.1);
    expect(spread(MP.LEFT_ANKLE)).toBeGreaterThan(0.05);
    expect(spread(MP.NOSE)).toBeLessThan(spread(MP.LEFT_WRIST));
  }, 120_000);
});