fix(models): 修复上传 PMX 返回 500（models.stats 超 TEXT 上限）与 GBK 丢贴图
现象（真实故障）：导入示例文件/【奥黛塔】_by_原神.zip 返回 500 Internal server error。 根因一（500）： 该 PMX 643 根骨骼、93,350 面，转换本身成功（GLB 23.7MB），但分析结果里的 骨骼树按骨骼逐条展开，JSON 实测 70,980 字节，而 models.stats 是 TEXT （上限 65,535）。MySQL/MariaDB 严格模式直接拒绝写入（ER_DATA_TOO_LONG / 1406），而这一步没有兜底 -> 未捕获 -> 500。 根因二（丢贴图）： tex/髮.png 的 GBK 字节是 F3 8C。文件名解码先试 Shift-JIS，而 WHATWG 的 shift_jis 把未定义码位映射成私用区不可见字符（不是 U+FFFD），于是被判为 「解码成功」，永不回退 GBK；贴图名对不上，该模型 18 张贴图只内嵌了 17 张。 修复： 1. models.stats -> MEDIUMTEXT（实体 mediumtext + transformer，保留 JSON 语义； 不用 simple-json 以免元数据与库列型不一致），新增迁移 AddModelStatsMediumText1789550000000； 2. 新增 shrinkStatsForStorage() 兜底：即使库未迁移，也只截断骨骼树 （父在前，保留祖先链），面数/骨骼数/humanoid 覆盖/可驱动性/warnings 全部保留； 服务层 saveWithStatsFallback() 仅在 1406/DATA_TOO_LONG 时裁剪重试一次， 其它数据库错误照旧抛出； 3. 压缩包文件名解码把「私用区字符」也算作解错信号，回退 GBK；实测该模型 贴图由 17/18 变为 18/18 全部内嵌； 4. 新增 12 项测试（test/model-stats-limit.spec.ts 9 项 + 压缩包编码/目录条目 3 项）； 5. 版本 0.2.3，README(.md/.txt)/DSH.md 补修订说明、500 的 FAQ 与 stats 容量约束。
This commit is contained in:
opendance
@@ -22,6 +22,11 @@ interface FakeEntry {
|
||||
encrypted?: boolean;
|
||||
/** 未知压缩方式 */
|
||||
badMethod?: number;
|
||||
/**
|
||||
* 直接指定文件名的原始字节，且**不置 UTF-8 标志位**（模拟 GBK / Shift-JIS 压缩包）。
|
||||
* 中文 MMD 压缩包很常见：有的条目是 UTF-8、有的是 GBK，且都没打标志位。
|
||||
*/
|
||||
rawName?: Buffer;
|
||||
}
|
||||
|
||||
/** 手写一个 ZIP（测试用；不依赖任何第三方库） */
|
||||
@@ -31,7 +36,9 @@ function buildZip(entries: FakeEntry[]): Buffer {
|
||||
let offset = 0;
|
||||
|
||||
for (const entry of entries) {
|
||||
const name = Buffer.from(entry.path, 'utf8');
|
||||
const name = entry.rawName ?? Buffer.from(entry.path, 'utf8');
|
||||
// 给了 rawName 就当作「未声明 UTF-8」的老压缩包
|
||||
const nameFlags = entry.rawName ? (entry.encrypted ? 0x0001 : 0) : (entry.encrypted ? 0x0001 : 0) | 0x0800;
|
||||
const method = entry.badMethod ?? entry.method ?? 8;
|
||||
const raw = entry.content;
|
||||
const data = method === 8 ? deflateRawSync(raw) : raw;
|
||||
@@ -40,7 +47,7 @@ function buildZip(entries: FakeEntry[]): Buffer {
|
||||
const local = Buffer.alloc(30 + name.length);
|
||||
local.writeUInt32LE(0x04034b50, 0);
|
||||
local.writeUInt16LE(20, 4);
|
||||
local.writeUInt16LE((entry.encrypted ? 0x0001 : 0) | 0x0800, 6);
|
||||
local.writeUInt16LE(nameFlags, 6);
|
||||
local.writeUInt16LE(method, 8);
|
||||
local.writeUInt32LE(checksum, 14);
|
||||
local.writeUInt32LE(data.length, 18);
|
||||
@@ -53,7 +60,7 @@ function buildZip(entries: FakeEntry[]): Buffer {
|
||||
central.writeUInt32LE(0x02014b50, 0);
|
||||
central.writeUInt16LE(20, 4);
|
||||
central.writeUInt16LE(20, 6);
|
||||
central.writeUInt16LE((entry.encrypted ? 0x0001 : 0) | 0x0800, 8);
|
||||
central.writeUInt16LE(nameFlags, 8);
|
||||
central.writeUInt16LE(method, 10);
|
||||
central.writeUInt32LE(checksum, 16);
|
||||
central.writeUInt32LE(data.length, 20);
|
||||
@@ -220,6 +227,35 @@ describe('ZIP 读取器', () => {
|
||||
expect(findZipEntry(entries, 'HATSUNE MIKU.PMX')?.path).toBe('Hatsune Miku.pmx');
|
||||
});
|
||||
|
||||
it('GBK 压缩包：Shift-JIS 解出私用区乱码时回退 GBK（发/髮 这类字不再丢字符）', () => {
|
||||
// b'tex/\xf3\x8c.png' 是 GBK 的「tex/髮.png」；Shift-JIS 会把它解成一个
|
||||
// 私用区不可见字符（不是 U+FFFD），旧实现会直接采信，导致贴图名对不上而丢图。
|
||||
const zip = buildZip([
|
||||
{ path: 'raw', content: Buffer.from('PMX '.repeat(50)), rawName: Buffer.from([0x74, 0x65, 0x78, 0x2f, 0xf3, 0x8c, 0x2e, 0x70, 0x6e, 0x67]) },
|
||||
]);
|
||||
const entries = readZipEntries(zip);
|
||||
expect(entries.map((e) => e.path)).toEqual(['tex/髮.png']);
|
||||
});
|
||||
|
||||
it('Shift-JIS 名字仍然优先按 Shift-JIS 解（日文 MMD 压缩包）', () => {
|
||||
// b'\x83\x41\x83\x43.pmx' 是 Shift-JIS 的「アイ.pmx」（半角片假名判断的姊妹用例）
|
||||
const zip = buildZip([
|
||||
{ path: 'raw', content: Buffer.from('PMX '.repeat(50)), rawName: Buffer.from([0x83, 0x41, 0x83, 0x43, 0x2e, 0x70, 0x6d, 0x78]) },
|
||||
]);
|
||||
const entries = readZipEntries(zip);
|
||||
expect(entries[0].path).toBe('アイ.pmx');
|
||||
});
|
||||
|
||||
it('目录条目不会当成文件（避免解压时把目录当文件写）', () => {
|
||||
const zip = buildZip([
|
||||
{ path: 'spa/', content: Buffer.alloc(0), method: 0 },
|
||||
{ path: 'spa/a.png', content: Buffer.from('PNG') },
|
||||
{ path: '奥黛塔.pmx', content: Buffer.from('PMX '.repeat(50)) },
|
||||
]);
|
||||
const entries = readZipEntries(zip);
|
||||
expect(entries.map((e) => e.path)).toEqual(['spa/a.png', '奥黛塔.pmx']);
|
||||
});
|
||||
|
||||
it('拒绝上跳路径（zip-slip）', () => {
|
||||
const zip = buildZip([{ path: '../../etc/passwd', content: Buffer.from('x') }]);
|
||||
expect(() => readZipEntries(zip)).toThrow(/上跳路径/);
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
import { readFileSync, readdirSync } from 'node:fs';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { getMetadataArgsStorage } from 'typeorm';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { CharacterModel } from '../src/entities/model.entity';
|
||||
import {
|
||||
MYSQL_TEXT_LIMIT,
|
||||
type BoneNode,
|
||||
type ModelStats,
|
||||
shrinkStatsForStorage,
|
||||
} from '../src/models/model-inspector';
|
||||
|
||||
/**
|
||||
* 骨骼树超出数据库字段上限导致的 500。
|
||||
*
|
||||
* 起因（真实故障）：上传含 PMX 的压缩包（【奥黛塔】_by_原神.zip，643 根骨骼）
|
||||
* 返回 500。转换本身成功，但分析结果里骨骼树按骨骼逐条展开，JSON 实测
|
||||
* **70,980 字节** > `models.stats` 的 TEXT 上限 65,535，MySQL / MariaDB 严格模式
|
||||
* 直接拒绝写入（ER_DATA_TOO_LONG / 1406），而写入没有兜底。
|
||||
*
|
||||
* 这里锁三件事：
|
||||
* 1. 列必须是 MEDIUMTEXT（实体 + 迁移一致）；
|
||||
* 2. 服务层对「数据超长」有裁剪重试兜底；
|
||||
* 3. 裁剪函数本身永不抛、且保留审核必需的汇总字段。
|
||||
*/
|
||||
|
||||
/** 造一个规模接近真实的骨骼树（【奥黛塔】实测 643 根、每根约 110 字节） */
|
||||
function buildBoneTree(count: number): BoneNode[] {
|
||||
return Array.from({ length: count }, (_, index) => ({
|
||||
index,
|
||||
name: `Bone_${String(index).padStart(4, '0')}_髪_腕捩_ねじ`,
|
||||
depth: Math.min(index % 40, 35),
|
||||
parentName: index === 0 ? null : `Bone_${String(Math.max(0, index - 1)).padStart(4, '0')}_髪_腕捩_ねじ`,
|
||||
childCount: index % 3,
|
||||
inSkin: index % 7 !== 0,
|
||||
humanoid: index < 7 ? ['hips', 'spine', 'chest', 'neck', 'head', 'leftShoulder', 'leftUpperArm'][index] : null,
|
||||
}));
|
||||
}
|
||||
|
||||
function buildStats(bones: number): ModelStats {
|
||||
return {
|
||||
container: 'glb',
|
||||
vrmVersion: null,
|
||||
fileSize: 23_700_000,
|
||||
meshes: 12,
|
||||
primitives: 18,
|
||||
vertices: 120_000,
|
||||
triangles: 93_350,
|
||||
materials: 9,
|
||||
textures: 17,
|
||||
images: 17,
|
||||
animations: 0,
|
||||
morphTargets: 24,
|
||||
nodes: 700,
|
||||
bones,
|
||||
skins: 1,
|
||||
humanoidMapped: 0,
|
||||
humanoidTotal: 22,
|
||||
missingRequiredBones: ['hips', 'spine', 'chest', 'neck', 'head', 'leftUpperArm'],
|
||||
drivable: false,
|
||||
maxBoneDepth: 35,
|
||||
boneTree: buildBoneTree(bones),
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
|
||||
function findMigrationsDir(): string {
|
||||
const candidates = [
|
||||
resolve(process.cwd(), 'src', 'migrations'),
|
||||
resolve(process.cwd(), '..', 'src', 'migrations'),
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
const files = readdirSync(candidate).filter((f) => f.endsWith('.ts'));
|
||||
if (files.length > 0) return candidate;
|
||||
}
|
||||
throw new Error('找不到 src/migrations');
|
||||
}
|
||||
|
||||
describe('models.stats 字段容量', () => {
|
||||
it('643 根骨骼的分析结果确实超过 TEXT 上限（这就是 500 的原因）', () => {
|
||||
const json = JSON.stringify(buildStats(643));
|
||||
expect(json.length).toBeGreaterThan(MYSQL_TEXT_LIMIT);
|
||||
// 与真实故障量级一致（实测 70,980 字节）
|
||||
expect(json.length).toBeGreaterThan(65_000);
|
||||
expect(json.length).toBeLessThan(120_000);
|
||||
});
|
||||
|
||||
it('实体列型是 mediumtext，且能 JSON 往返（不再被 simple-json 钉死在 text）', () => {
|
||||
const column = getMetadataArgsStorage().columns.find(
|
||||
(item) => item.target === CharacterModel && item.propertyName === 'stats',
|
||||
);
|
||||
expect(column?.options.type).toBe('mediumtext');
|
||||
const transformer = column?.options.transformer as
|
||||
| { to?: (v: unknown) => unknown; from?: (v: unknown) => unknown }
|
||||
| undefined;
|
||||
expect(typeof transformer?.to).toBe('function');
|
||||
expect(typeof transformer?.from).toBe('function');
|
||||
const stats = buildStats(643);
|
||||
const stored = transformer!.to!(stats) as string;
|
||||
expect(typeof stored).toBe('string');
|
||||
expect(JSON.parse(stored).bones).toBe(643);
|
||||
expect((transformer!.from!(stored) as ModelStats).bones).toBe(643);
|
||||
// 脏数据不能让接口 500
|
||||
expect(transformer!.from!('{ 坏 JSON')).toEqual({ parseError: true, raw: '{ 坏 JSON' });
|
||||
});
|
||||
|
||||
it('存在把 stats 扩到 MEDIUMTEXT 的迁移（install.sh 会自动执行）', () => {
|
||||
const dir = findMigrationsDir();
|
||||
const file = readdirSync(dir).find((f) => /AddModelStatsMediumText.*\.ts$/.test(f));
|
||||
expect(file, '缺少 AddModelStatsMediumText 迁移').toBeTruthy();
|
||||
const sql = readFileSync(join(dir, file as string), 'utf8');
|
||||
// TS 源码里反引号是转义的，这里只断言关键字与方向
|
||||
expect(sql).toMatch(/MODIFY/i);
|
||||
expect(sql).toMatch(/mediumtext/i);
|
||||
expect(sql).toMatch(/ALTER TABLE/i);
|
||||
expect(sql).toMatch(/implements MigrationInterface/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shrinkStatsForStorage：写入前的兜底裁剪', () => {
|
||||
it('没超限时原样返回（不做无谓修改）', () => {
|
||||
const stats = buildStats(20);
|
||||
expect(shrinkStatsForStorage(stats)).toBe(stats);
|
||||
});
|
||||
|
||||
it('超限时裁剪骨骼树到能放下，并追加一条说明用的 warning', () => {
|
||||
const stats = buildStats(643);
|
||||
const shrunk = shrinkStatsForStorage(stats);
|
||||
expect(JSON.stringify(shrunk).length).toBeLessThanOrEqual(MYSQL_TEXT_LIMIT);
|
||||
expect(shrunk.boneTree.length).toBeLessThan(stats.boneTree.length);
|
||||
// 审核必需的汇总字段一个都不能少
|
||||
expect(shrunk.triangles).toBe(93_350);
|
||||
expect(shrunk.bones).toBe(643);
|
||||
expect(shrunk.humanoidMapped).toBe(0);
|
||||
expect(shrunk.humanoidTotal).toBe(22);
|
||||
expect(shrunk.drivable).toBe(false);
|
||||
expect(shrunk.missingRequiredBones).toEqual(stats.missingRequiredBones);
|
||||
expect(shrunk.warnings.join()).toContain('已截断');
|
||||
});
|
||||
|
||||
it('保留的骨骼仍是父在前的祖先链（审核界面按 depth 缩进不会错乱）', () => {
|
||||
const stats = buildStats(643);
|
||||
const shrunk = shrinkStatsForStorage(stats);
|
||||
const kept = shrunk.boneTree;
|
||||
expect(kept.length).toBeGreaterThan(0);
|
||||
for (let i = 1; i < kept.length; i += 1) {
|
||||
expect(kept[i].depth).toBeLessThanOrEqual(kept[i - 1].depth + 1);
|
||||
}
|
||||
});
|
||||
|
||||
it('极端情况（连最小骨骼树都放不下）也不会抛，仍返回可存储的结果', () => {
|
||||
const stats = buildStats(400);
|
||||
const shrunk = shrinkStatsForStorage(stats, 512);
|
||||
expect(JSON.stringify(shrunk).length).toBeLessThanOrEqual(1024);
|
||||
expect(shrunk.triangles).toBe(93_350);
|
||||
});
|
||||
});
|
||||
|
||||
describe('服务层兜底：数据超长时裁剪重试而不是 500', () => {
|
||||
const source = readFileSync(
|
||||
[
|
||||
resolve(process.cwd(), 'src', 'models', 'models.service.ts'),
|
||||
resolve(process.cwd(), '..', 'src', 'models', 'models.service.ts'),
|
||||
].find((p) => {
|
||||
try {
|
||||
readFileSync(p);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}) as string,
|
||||
'utf8',
|
||||
);
|
||||
|
||||
it('create 与 reanalyze 都走 saveWithStatsFallback', () => {
|
||||
const calls = source.match(/saveWithStatsFallback\(model\)/g) ?? [];
|
||||
expect(calls.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('兜底只用 1406 / DATA_TOO_LONG 触发，其它数据库错误照旧抛出', () => {
|
||||
expect(source).toContain('isDataTooLongError');
|
||||
expect(source).toMatch(/errno === 1406/);
|
||||
expect(source).toMatch(/DATA_TOO_LONG/);
|
||||
expect(source).toContain('if (!isDataTooLongError(error)) throw error;');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user