zijunchiang/opendance-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 17 Wiki Activity

fix(thumbnail): 缩略图采样 baseColorTexture，修复 MMD 模型显示为白模

Browse Source 
现象（用户反馈）：导入 PMX 转换出的 GLB 后，列表页/审核面板里的模型是白模，
脸部颜色信息全部丢失。

定位过程（本地装真 Assimp 6.0.4 + 该模型完整复现）：
1. 转换本身正常：23.7MB GLB、643 骨骼、93,350 面；
2. 18 张贴图逐字节与压缩包原图一致；用一个权威 PMX 解析器比对后确认
   39 个材质 -> 贴图的对应关系与 PMX 完全一致（不是映射错）；
3. UV 采样也正确：脸部材质取到的就是皮肤色 (246,221,216)；
4. 真正的问题在**我们自己的缩略图渲染器**：它刻意不解码 baseColorTexture，
   只用 baseColorFactor 上色（见文件头取舍说明）；而 MMD 模型的
   baseColorFactor 基本都是白色 -> 缩略图必然全白。管理端 3D 预览有贴图，
   只有列表/审核的缩略图是白的。

修复：
1. model-thumbnail.ts 内置极简 PNG 解码器（zlib inflate + 逐行反滤波，
   8 位非隔行，colorType 0/2/3/4/6），按 baseColorFactor x 贴图逐像素采样，
   透视校正 UV；解码失败/非 PNG/隔行扫描回退纯色，绝不抛错；
2. 按 alphaMode 处理透明度：OPAQUE 忽略贴图 alpha（MMD 语义）、
   MASK 按 alphaCutoff 丢弃、BLEND 做 source-over；
3. 取景收紧 FRAMING 1.6 -> 1.3，40px 列表缩略图主体更清楚；
4. 新增 test/model-thumbnail-texture.spec.ts 6 项（含白模回归：
   baseColorFactor 全白 + 红贴图必须输出偏红，以及 MASK/OPAQUE/坏图回退）；
5. 版本 0.2.4，README(.md/.txt)/DSH.md 补修订说明与 FAQ（含重新生成缩略图
   的 reanalyze-models.mjs --force 命令）。
This commit is contained in:
opendance
2026-09-16 04:02:35 +08:00
parent 8e8cc0475b
commit ae8a68e98f
11 changed files with 605 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web-server/src/models/model-thumbnail.ts
+321 -9
View File
@@ -1,4 +1,5 @@
import { readFile } from 'node:fs/promises';
import { inflateSync } from 'node:zlib';
import { dirname, resolve as resolvePath } from 'node:path';
import { deflateSync } from 'node:zlib';
@@ -22,9 +23,12 @@ import { deflateSync } from 'node:zlib';
* **退化为不蒙皮**，按节点世界矩阵画绑定姿态（bind pose）——多数 VRM 的绑定姿态
* 就是可看的 T/A 姿态，比错误的蒙皮矩阵导致的"爆炸形状"安全得多。
* - 按 glTF 规范，蒙皮网格节点自身的变换被忽略（顶点直接由关节矩阵驱动）。
* 2. **不采样贴图**：`pbrMetallicRoughness.baseColorFactor` 作为纯色基色，
* 即使存在 `baseColorTexture` 也不解码（避免再引入 PNG/JPEG 解码器），
* 因此带贴图的模型缩略图会是纯色块，但轮廓与光影正确。
* 2. **采样 baseColorTexture**：内置极简 PNG 解码器（zlib + 反滤波，8 位、非隔行、
* colorType 0/2/3/4/6），按 `baseColorFactor x 贴图` 逐像素取色，并按
* `alphaMode`（OPAQUE / MASK / BLEND）处理透明度。
* 这一版之前**不采样贴图**：MMD 模型（PMX 转换而来）的 `baseColorFactor` 几乎
* 都是白色，于是缩略图全是"白模"——用户报的正是这个。
* 解码失败/非 PNG/隔行扫描时回退为 `baseColorFactor` 纯色，绝不抛错。
* 3. **稀疏 accessor（sparse）不支持**：遇到就直接跳过该 primitive。
* 4. **只画三角形（mode=4）**，其余图元（点/线/条带/扇形）跳过。
* 5. 大模型（>17M 索引）不会出现，索引统一按 uint32 读取。
@@ -70,7 +74,8 @@ const DEFAULT_BACKGROUND: [number, number, number] = [24, 26, 32];
/** 垂直视场角（度） */
const FOV_Y_DEG = 35;
/** 相机距离系数：让模型占画面约 1/1.6，保证不裁剪 */
const FRAMING = 1.6;
// 1.6 -> 1.3：列表页只有 40px，主体太小就只剩一团像素
const FRAMING = 1.3;
/** 主光方向（世界空间，未归一化，用前会归一化） */
const LIGHT_DIR: [number, number, number] = normalize3(0.4, 0.8, 0.6);
/** 环境光强度 */
@@ -244,8 +249,27 @@ interface GltfSkin {
interface GltfMaterial {
pbrMetallicRoughness?: {
baseColorFactor?: number[];
baseColorTexture?: unknown;
baseColorTexture?: { index?: number; texCoord?: number };
};
alphaMode?: string;
alphaCutoff?: number;
doubleSided?: boolean;
}
interface GltfTexture {
source?: number;
sampler?: number;
}
interface GltfImage {
bufferView?: number;
mimeType?: string;
uri?: string;
}
interface GltfSampler {
wrapS?: number;
wrapT?: number;
}
interface GltfScene {
@@ -262,6 +286,9 @@ interface GltfDocument {
bufferViews?: GltfBufferView[];
buffers?: GltfBufferRef[];
materials?: GltfMaterial[];
textures?: GltfTexture[];
images?: GltfImage[];
samplers?: GltfSampler[];
skins?: GltfSkin[];
}
@@ -278,9 +305,26 @@ interface AccessorData {
}
/** 收集好的三角形：positions 每 9 个数一个三角形，colors 每 3 个数一个三角形 */
interface DecodedImage {
width: number;
height: number;
/** RGBA，行优先，长度 = width*height*4 */
rgba: Uint8Array;
}
interface SceneGeometry {
positions: Float32Array;
colors: Float32Array;
/** 每顶点 UV（2 个/顶点）；没有 TEXCOORD_0 时为 null */
uvs: Float32Array | null;
/** 每顶点贴图槽位（索引进 textures）；-1 表示该顶点无贴图 */
texSlots: Int32Array | null;
/** 每顶点 alpha 模式：0=OPAQUE 1=MASK 2=BLEND */
alphaModes: Uint8Array | null;
/** 每顶点 MASK 阈值 */
cutoffs: Float32Array | null;
/** 解码后的贴图（RGBA） */
textures: DecodedImage[];
triangleCount: number;
}
@@ -667,6 +711,36 @@ function materialBaseColor(doc: GltfDocument, materialIndex: number | undefined)
return [0.8, 0.8, 0.8];
}
/** 取 bufferView 的原始字节（贴图解压用）；越界返回 null */
function readBufferViewBytes(
doc: GltfDocument,
buffers: Array<Buffer | null>,
bufferViewIndex: number,
): Buffer | null {
const view = doc.bufferViews?.[bufferViewIndex];
if (!view || typeof view.buffer !== 'number') return null;
const buffer = buffers[view.buffer];
if (!buffer) return null;
const offset = view.byteOffset ?? 0;
const length = view.byteLength ?? 0;
if (offset < 0 || length <= 0 || offset + length > buffer.length) return null;
return buffer.subarray(offset, offset + length);
}
/** 解码 GLB 内嵌的 PNG 贴图；外部 uri / 非 PNG / 解码失败都返回 null（回退纯色） */
function decodeEmbeddedImage(
doc: GltfDocument,
buffers: Array<Buffer | null>,
imageIndex: number,
): DecodedImage | null {
const image = doc.images?.[imageIndex];
if (!image || typeof image.bufferView !== 'number') return null;
if (typeof image.mimeType === 'string' && !image.mimeType.toLowerCase().includes('png')) return null;
const bytes = readBufferViewBytes(doc, buffers, image.bufferView);
if (!bytes) return null;
return decodePng(bytes);
}
/** 遍历场景图收集世界空间三角形 */
function collectTriangles(container: GltfContainer): SceneGeometry | null {
const { doc, buffers } = container;
@@ -676,8 +750,49 @@ function collectTriangles(container: GltfContainer): SceneGeometry | null {
const positions: number[] = [];
const colors: number[] = [];
const uvs: number[] = [];
const texSlots: number[] = [];
const alphaModes: number[] = [];
const cutoffs: number[] = [];
const textures: DecodedImage[] = [];
const materialTextureCache = new Map<number, { slot: number; alphaMode: number; cutoff: number }>();
const imageCache = new Map<number, DecodedImage | null>();
const skinScratch = new Float64Array(16);
/**
* 解析材质的 baseColorTexture：解码一次并登记槽位。
* MMD 模型的 baseColorFactor 基本都是白色，颜色全在贴图里 —— 不解码就是"白模"。
*/
const resolveMaterialTexture = (
materialIndex: number | undefined,
): { slot: number; alphaMode: number; cutoff: number } => {
const key = typeof materialIndex === 'number' ? materialIndex : -1;
const cached = materialTextureCache.get(key);
if (cached) return cached;
const material = typeof materialIndex === 'number' ? doc.materials?.[materialIndex] : undefined;
const alphaMode = material?.alphaMode === 'MASK' ? 1 : material?.alphaMode === 'BLEND' ? 2 : 0;
const rawCutoff = typeof material?.alphaCutoff === 'number' ? material.alphaCutoff : 0.5;
const cutoff = clamp(rawCutoff, 0, 1);
let slot = -1;
const textureIndex = material?.pbrMetallicRoughness?.baseColorTexture?.index;
if (typeof textureIndex === 'number') {
const imageIndex = doc.textures?.[textureIndex]?.source;
if (typeof imageIndex === 'number') {
if (!imageCache.has(imageIndex)) {
imageCache.set(imageIndex, decodeEmbeddedImage(doc, buffers, imageIndex));
}
const image = imageCache.get(imageIndex) ?? null;
if (image) {
slot = textures.length;
textures.push(image);
}
}
}
const resolved = { slot, alphaMode, cutoff };
materialTextureCache.set(key, resolved);
return resolved;
};
const appendMesh = (nodeIndex: number): void => {
const node = nodes[nodeIndex];
if (!node || typeof node.mesh !== 'number') return;
@@ -710,6 +825,10 @@ function collectTriangles(container: GltfContainer): SceneGeometry | null {
const vertexCount = positionData.count;
const [cr, cg, cb] = materialBaseColor(doc, primitive.material);
const materialTexture = resolveMaterialTexture(primitive.material);
const uvData =
materialTexture.slot >= 0 ? readAccessorFloats(doc, buffers, attributes.TEXCOORD_0, 2) : null;
const usableUv = uvData && uvData.count === vertexCount ? uvData : null;
// 顶点只变换一次：蒙皮或节点世界矩阵
const px = new Float64Array(vertexCount);
@@ -758,6 +877,18 @@ function collectTriangles(container: GltfContainer): SceneGeometry | null {
}
positions.push(x0, y0, z0, x1, y1, z1, x2, y2, z2);
colors.push(cr, cg, cb);
if (usableUv) {
uvs.push(
usableUv.values[i0 * 2], usableUv.values[i0 * 2 + 1],
usableUv.values[i1 * 2], usableUv.values[i1 * 2 + 1],
usableUv.values[i2 * 2], usableUv.values[i2 * 2 + 1],
);
} else {
uvs.push(0, 0, 0, 0, 0, 0);
}
texSlots.push(materialTexture.slot, materialTexture.slot, materialTexture.slot);
alphaModes.push(materialTexture.alphaMode, materialTexture.alphaMode, materialTexture.alphaMode);
cutoffs.push(materialTexture.cutoff, materialTexture.cutoff, materialTexture.cutoff);
}
}
};
@@ -784,6 +915,11 @@ function collectTriangles(container: GltfContainer): SceneGeometry | null {
return {
positions: Float32Array.from(positions),
colors: Float32Array.from(colors),
uvs: Float32Array.from(uvs),
texSlots: Int32Array.from(texSlots),
alphaModes: Uint8Array.from(alphaModes),
cutoffs: Float32Array.from(cutoffs),
textures,
triangleCount,
};
}
@@ -831,13 +967,25 @@ function edge(ax: number, ay: number, bx: number, by: number, px: number, py: nu
* 屏幕空间有符号面积为负的是正面（NDC 里 CCW 正面经 y 翻转后变成负），
* 因此 cullBackFace 时丢弃 area >= 0 的三角形。
*/
/** 采样贴图（REPEAT 环绕；glTF/PNG 的 UV 原点在左上，v 不翻转） */
function sampleTexture(image: DecodedImage, u: number, v: number, out: Float32Array): void {
const wrap = (value: number): number => value - Math.floor(value);
const x = Math.min(image.width - 1, Math.floor(wrap(u) * image.width));
const y = Math.min(image.height - 1, Math.floor(wrap(v) * image.height));
const o = (y * image.width + x) * 4;
out[0] = image.rgba[o] / 255;
out[1] = image.rgba[o + 1] / 255;
out[2] = image.rgba[o + 2] / 255;
out[3] = image.rgba[o + 3] / 255;
}
function rasterizeTriangles(
geometry: SceneGeometry,
viewProj: Mat4,
options: RasterOptions,
): RasterOutput {
const { size, background, cullBackFace } = options;
const { positions, colors, triangleCount } = geometry;
const { positions, colors, uvs, texSlots, alphaModes, cutoffs, textures, triangleCount } = geometry;
const pixels = Buffer.alloc(size * size * 3);
for (let i = 0; i < size * size; i += 1) {
@@ -909,9 +1057,23 @@ function rasterizeTriangles(
const baseR = colors[t * 3];
const baseG = colors[t * 3 + 1];
const baseB = colors[t * 3 + 2];
const r = clamp(Math.round(baseR * intensity * 255), 0, 255);
const g = clamp(Math.round(baseG * intensity * 255), 0, 255);
const b = clamp(Math.round(baseB * intensity * 255), 0, 255);
// 贴图与透明度（每三角形常量；顶点数组按 3 顶点/三角形展开）
const slot = texSlots ? texSlots[t * 3] : -1;
const image = slot >= 0 ? textures[slot] ?? null : null;
const alphaMode = alphaModes ? alphaModes[t * 3] : 0;
const cutoff = cutoffs ? cutoffs[t * 3] : 0.5;
const u0 = uvs ? uvs[t * 6] : 0;
const v0 = uvs ? uvs[t * 6 + 1] : 0;
const u1 = uvs ? uvs[t * 6 + 2] : 0;
const v1 = uvs ? uvs[t * 6 + 3] : 0;
const u2 = uvs ? uvs[t * 6 + 4] : 0;
const v2 = uvs ? uvs[t * 6 + 5] : 0;
const texel = new Float32Array(4);
// 透视校正用：1/w 在三个顶点上的值
const iw0 = 1 / p0[3];
const iw1 = 1 / p1[3];
const iw2 = 1 / p2[3];
const invArea = 1 / area;
for (let y = minY; y <= maxY; y += 1) {
@@ -926,6 +1088,27 @@ function rasterizeTriangles(
const d = w0 * p0[2] + w1 * p1[2] + w2 * p2[2];
const index = rowBase + x;
if (d < depth[index]) {
let r = clamp(Math.round(baseR * intensity * 255), 0, 255);
let g = clamp(Math.round(baseG * intensity * 255), 0, 255);
let b = clamp(Math.round(baseB * intensity * 255), 0, 255);
if (image) {
const iw = w0 * iw0 + w1 * iw1 + w2 * iw2;
const u = (w0 * iw0 * u0 + w1 * iw1 * u1 + w2 * iw2 * u2) / (iw || 1);
const v = (w0 * iw0 * v0 + w1 * iw1 * v1 + w2 * iw2 * v2) / (iw || 1);
sampleTexture(image, u, v, texel);
if (alphaMode === 1 && texel[3] < cutoff) continue; // MASK：整像素丢弃
const a = alphaMode === 0 ? 1 : texel[3];
r = clamp(Math.round(baseR * texel[0] * intensity * 255), 0, 255);
g = clamp(Math.round(baseG * texel[1] * intensity * 255), 0, 255);
b = clamp(Math.round(baseB * texel[2] * intensity * 255), 0, 255);
if (alphaMode === 2 && a < 1) {
// BLEND：与背景/已画像素做 source-over（缩略图不做排序，够看即可）
const q = index * 3;
r = clamp(Math.round(r * a + pixels[q] * (1 - a)), 0, 255);
g = clamp(Math.round(g * a + pixels[q + 1] * (1 - a)), 0, 255);
b = clamp(Math.round(b * a + pixels[q + 2] * (1 - a)), 0, 255);
}
}
depth[index] = d;
const p = index * 3;
pixels[p] = r;
@@ -1045,6 +1228,133 @@ function buildViewProjection(geometry: SceneGeometry, size: number, yawDeg: numb
return mat4Multiply(proj, view);
}
// ---------------------------------------------------------------------------
// PNG 解码（缩略图要贴图才不是"白模"；只做 8 位非隔行，够用且零原生依赖）
// ---------------------------------------------------------------------------
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
/**
* 极简 PNG 解码器：zlib 解压 + 逐行反滤波，输出 RGBA。
* 不支持 16 位 / 隔行扫描 / 未知 colorType —— 返回 null 由调用方回退纯色。
*/
function decodePng(buffer: Buffer): DecodedImage | null {
if (buffer.length < 8 + 25 || !buffer.subarray(0, 8).equals(PNG_SIGNATURE)) return null;
let offset = 8;
let width = 0;
let height = 0;
let bitDepth = 0;
let colorType = 0;
let interlace = 0;
let palette: Uint8Array | null = null;
let transparency: Uint8Array | null = null;
const idat: Buffer[] = [];
while (offset + 8 <= buffer.length) {
const length = buffer.readUInt32BE(offset);
const type = buffer.toString('ascii', offset + 4, offset + 8);
const dataStart = offset + 8;
if (dataStart + length + 4 > buffer.length) return null;
const data = buffer.subarray(dataStart, dataStart + length);
if (type === 'IHDR') {
if (length < 13) return null;
width = data.readUInt32BE(0);
height = data.readUInt32BE(4);
bitDepth = data[8];
colorType = data[9];
interlace = data[12];
} else if (type === 'PLTE') {
palette = new Uint8Array(data);
} else if (type === 'tRNS') {
transparency = new Uint8Array(data);
} else if (type === 'IDAT') {
idat.push(Buffer.from(data));
} else if (type === 'IEND') {
break;
}
offset = dataStart + length + 4;
}
if (bitDepth !== 8 || interlace !== 0) return null;
if (!(width > 0) || !(height > 0) || width * height > 64 * 1024 * 1024) return null;
if (idat.length === 0) return null;
const channels =
colorType === 0 ? 1 : colorType === 2 ? 3 : colorType === 3 ? 1 : colorType === 4 ? 2 : colorType === 6 ? 4 : 0;
if (channels === 0) return null;
if (colorType === 3 && !palette) return null;
let raw: Buffer;
try {
raw = inflateSync(Buffer.concat(idat));
} catch {
return null;
}
const stride = width * channels;
if (raw.length < (stride + 1) * height) return null;
const out = new Uint8Array(width * height * 4);
const line = new Uint8Array(stride);
const prev = new Uint8Array(stride);
let rawPos = 0;
for (let y = 0; y < height; y += 1) {
const filter = raw[rawPos];
rawPos += 1;
for (let i = 0; i < stride; i += 1) {
const value = raw[rawPos + i];
const left = i >= channels ? line[i - channels] : 0;
const up = prev[i];
const upLeft = i >= channels ? prev[i - channels] : 0;
let restored: number;
switch (filter) {
case 0: restored = value; break;
case 1: restored = value + left; break;
case 2: restored = value + up; break;
case 3: restored = value + ((left + up) >> 1); break;
case 4: {
const p = left + up - upLeft;
const pa = Math.abs(p - left);
const pb = Math.abs(p - up);
const pc = Math.abs(p - upLeft);
restored = value + (pa <= pb && pa <= pc ? left : pb <= pc ? up : upLeft);
break;
}
default: return null;
}
line[i] = restored & 0xff;
}
rawPos += stride;
for (let x = 0; x < width; x += 1) {
const o = (y * width + x) * 4;
if (colorType === 6) {
out[o] = line[x * 4];
out[o + 1] = line[x * 4 + 1];
out[o + 2] = line[x * 4 + 2];
out[o + 3] = line[x * 4 + 3];
} else if (colorType === 2) {
out[o] = line[x * 3];
out[o + 1] = line[x * 3 + 1];
out[o + 2] = line[x * 3 + 2];
out[o + 3] = 255;
} else if (colorType === 0) {
const g = line[x];
out[o] = g; out[o + 1] = g; out[o + 2] = g; out[o + 3] = 255;
} else if (colorType === 4) {
const g = line[x * 2];
out[o] = g; out[o + 1] = g; out[o + 2] = g; out[o + 3] = line[x * 2 + 1];
} else {
const index = line[x];
out[o] = palette![index * 3] ?? 0;
out[o + 1] = palette![index * 3 + 1] ?? 0;
out[o + 2] = palette![index * 3 + 2] ?? 0;
out[o + 3] = transparency && index < transparency.length ? transparency[index] : 255;
}
}
prev.set(line);
}
return { width, height, rgba: out };
}
// ---------------------------------------------------------------------------
// PNG 编码（8 位 RGB、无隔行、filter=0；不引第三方库）
// ---------------------------------------------------------------------------
@@ -1133,6 +1443,8 @@ export async function renderModelThumbnail(
) {
throw new Error('缩略图背景色非法：需要 [r, g, b] 三个有限数字');
}
// 默认看**正面**：eye = 中心 + (sin yaw, .., cos yaw)*d，yaw=0 时相机在 +Z；
// MMD（PMX）与 VRM 1.0 都是 +Z 朝前，因此 -30 是"正面偏 30 度"的常规 3/4 视角
const yawDeg = options?.yawDeg ?? -30;
const rawPitchDeg = options?.pitchDeg ?? 12;
if (!Number.isFinite(yawDeg) || !Number.isFinite(rawPitchDeg)) {