合并自建歌曲源支线（selfhost-source）+ 版本号按约定修正为 1.6.0

版本号： - 1.5.11 越界（规则 3 规定 patch ≤ 10）→ 修正为 1.6.0（进位到 minor、patch 归 0） - versionCode 43 不变：43 从未对外发布，本次修正只是把命名拉回约定合并支线（原在工作区根目录、App 仓库之外，DESIGN.md 自称"支线"）： - selfhost-source/server/ 自建歌曲源服务器（纯 Python 标准库、零 pip 依赖） - selfhost-source/deploy/、DESIGN.md、LIBRARY.md、build_inventory.py - selfhost-source/inventory/ 小清单（playlists.csv、song-stats.md 等） - ⚠️ inventory 生成物（songs-*.csv / singers-top.csv，共 84MB）不入库，已写入 .gitignore；需要时用 build_inventory.py 依据 muse.db 重新生成同时入库的 App 侧支线改动（此前为工作树未提交状态）： - Whoami.kt（whoami 设备心跳）、DeviceIdentity.kt（设备身份伪装） - PushDownload.kt / PushDownloadClient.kt（点歌推送） - AdVideoFilter 带 query/fragment 的地址闸门；KTV_BRIDGE_API 11 → 13 - 修 ktv_api.js 的 global.KtvBridgeApiVersion（12 → 13，与注释及 Kotlin 两处校验一致） - 对应单测：WhoamiTest / DeviceIdentityTest / PushDownloadTest / AdVideoFilterTest（共 39 例）文档： - DESIGN.md 的"在 App 仓库之外"约定改为"已并入仓库统一版本化" - AGENTS.md 版本号行更新为 1.6.0/43，并新增支线并入说明验证：assembleRelease + testReleaseUnitTest 全过（39 例）
2026-10-14 02:06:28 +08:00
parent e060ae5ebe
commit 87aa979abd
68 changed files with 21305 additions and 105 deletions
@@ -0,0 +1,325 @@
+"""曲库同步（复刻 App 的 MuseDbSync）：login → sqlite/sync → 下载 gz → md5 校验 → 解压 → 导入清单。
+
+要点：
+  * **纯标准库实现 RSA/PKCS#1 v1.5**（Debian 上不需要装任何 pip 包，也不需要 openssl 命令行）；
+  * 同步前必须重新 login 拿 `muse-portal-connector` cookie——没有它 sync 直接 401；
+  * `mconn` 与 `mws` 两个主机都能返回文件（同一后台），任选其一；
+  * 官方库**没有** `cloud_url` 列（29 列版本），导入时只取存在的列。
+"""
+from __future__ import annotations
+
+import base64
+import gzip
+import hashlib
+import json
+import os
+import sqlite3
+import threading
+import time
+import urllib.request
+from typing import Callable
+
+# App 里使用的 RSA 公钥（与 MuseDbSync.RSA_PUBLIC_KEY 一致）
+RSA_PUBLIC_KEY_B64 = (
+    "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAucL0oFErd7REM6TSNa3EZdN1YaOs4J1eCybLPyoQ9ru3q1HU67agC9FzhrCG/RvAQUya5iPmQ8Caed05vqcCVcyJChmkSOGQ7DVShe2rGuTMNlpoRV6UzfcraaVS++7m2K/+kSZJ8OAhhhVuqPruMjsFYpdtstAwvyZT28b+eENwzpp9UHqsooZc7FZ0H8kTbs6XMkw4nIWo+4HoPAhNLEY+xdHvwY6drF/3WDTvsaoMrs73TVQCEEHzZNIz2H/is9VLMnIyOfnfcJi9br78Fj2xHzxu3sAySBOTVLmUMxqYh/g1ox5OXGcW93HJkQLkBi42tFAEkWYlYyl93+jbbQIDAQAB"
+)
+CANON_LANGS = ["国语", "粤语", "闽南语", "英语", "日语", "韩语", "其它"]
+LANG_ALIAS = {"闽南": "闽南语", "台语": "闽南语", "H国语": "国语", "其他": "其它", "西语": "其它",
+              "西班牙语": "其它", "法语": "其它", "俄语": "其它", "泰语": "其它", "印度": "其它",
+              "潮汕语": "其它", "消音": "其它", "原版": "其它"}
+
+
+def norm_lang(v: str) -> str:
+    k = (v or "").strip()
+    if k in CANON_LANGS:
+        return k
+    if k in LANG_ALIAS:
+        return LANG_ALIAS[k]
+    return "其它" if k else "未标注"
+
+
+# ───────────────────────── 极简 DER / RSA ─────────────────────────
+def _der_tlv(buf: bytes, pos: int):
+    tag = buf[pos]
+    pos += 1
+    ln = buf[pos]
+    pos += 1
+    if ln & 0x80:
+        n = ln & 0x7F
+        ln = int.from_bytes(buf[pos:pos + n], "big")
+        pos += n
+    start = pos
+    return tag, start, start + ln
+
+
+def _der_walk(buf: bytes, start: int, end: int):
+    """产出 (tag, start, end) 的扁平列表（含容器内部）。"""
+    out = []
+    p = start
+    while p < end:
+        tag, s, e = _der_tlv(buf, p)
+        out.append((tag, s, e))
+        if tag & 0x20:                      # constructed → 递归
+            out.extend(_der_walk(buf, s, e))
+        elif tag == 0x03:                   # BIT STRING：跳过"未使用位数"字节后继续
+            #     SubjectPublicKeyInfo 里的 RSAPublicKey 就包在 BIT STRING 内层。
+            out.extend(_der_walk(buf, s + 1, e))
+        p = e
+    return out
+
+
+def rsa_public_numbers(b64: str = RSA_PUBLIC_KEY_B64) -> tuple[int, int]:
+    """从 SubjectPublicKeyInfo(DER) 里取出 (n, e)。"""
+    der = base64.b64decode(b64)
+    ints = []
+    for tag, s, e in _der_walk(der, 0, len(der)):
+        if tag == 0x02:                     # INTEGER
+            ints.append(int.from_bytes(der[s:e], "big"))
+    if len(ints) < 2:
+        raise ValueError("无法解析 RSA 公钥")
+    return ints[-2], ints[-1]               # modulus, exponent
+
+
+def rsa_pkcs1v15_encrypt(plaintext: bytes, n: int, e: int) -> bytes:
+    """PKCS#1 v1.5 加密（EM = 0x00 0x02 PS 0x00 M）。"""
+    k = (n.bit_length() + 7) // 8
+    if len(plaintext) > k - 11:
+        raise ValueError("明文过长")
+    ps_len = k - 3 - len(plaintext)
+    ps = bytearray()
+    while len(ps) < ps_len:
+        b = os.urandom(1)[0]
+        if b:                               # PS 必须非零
+            ps.append(b)
+    em = b"\x00\x02" + bytes(ps) + b"\x00" + plaintext
+    return pow(int.from_bytes(em, "big"), e, n).to_bytes(k, "big")
+
+
+def encrypt_device_id(device_id: str) -> str:
+    n, e = rsa_public_numbers()
+    return base64.b64encode(rsa_pkcs1v15_encrypt(json.dumps({"device_id": device_id}).encode(), n, e)).decode()
+
+
+# ───────────────────────── HTTP ─────────────────────────
+def _post_json(url: str, payload: dict, cookie: str = "", timeout: int = 60, ua: str = "MaidongKTV/1.2"):
+    body = json.dumps(payload, separators=(",", ":")).encode()
+    req = urllib.request.Request(url, data=body, method="POST")
+    req.add_header("Content-Type", "application/json; charset=UTF-8")
+    req.add_header("Accept", "*/*")
+    req.add_header("User-Agent", ua)
+    if cookie:
+        req.add_header("Cookie", cookie)
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        raw = resp.read()
+        set_cookie = resp.headers.get("Set-Cookie", "") or ""
+        code = resp.status
+    return code, raw, set_cookie
+
+
+def login(cfg) -> str:
+    """返回 `muse-portal-connector=…` cookie 字符串；失败抛异常。"""
+    payload = {
+        "encrypted_data": encrypt_device_id(str(cfg.get("muse_device_id"))),
+        "ip": "172.16.32.15", "dns": "", "router": "",
+        "subnet_mask": "255.255.255.0", "channel": "common", "mode": "ott",
+    }
+    code, raw, set_cookie = _post_json(str(cfg.get("muse_login_url")), payload,
+                                       timeout=int(cfg.get("timeout_sec")), ua=str(cfg.get("user_agent")))
+    cookie = ""
+    for part in set_cookie.split(","):
+        for seg in part.split(";"):
+            seg = seg.strip()
+            if seg.startswith("muse-portal-connector="):
+                cookie = seg
+    if code != 200 or not cookie:
+        raise RuntimeError(f"曲库登录失败：HTTP {code}，cookie={'有' if cookie else '无'}")
+    return cookie
+
+
+def remote_file(cfg) -> dict:
+    """返回 {url, md5, day, db_vc, date, key, size?}。"""
+    cookie = login(cfg)
+    payload = {"last_sync_at": 0, "music_file_type": int(cfg.get("muse_music_file_type"))}
+    code, raw, _ = _post_json(str(cfg.get("muse_sync_url")), payload, cookie,
+                              timeout=int(cfg.get("timeout_sec")), ua=str(cfg.get("user_agent")))
+    if code != 200:
+        raise RuntimeError(f"曲库同步 HTTP {code}")
+    data = json.loads(raw.decode("utf-8", "replace"))
+    files = data.get("files") or []
+    if not files:
+        raise RuntimeError("曲库同步返回空 files（服务端未授权或该设备无可用曲库）")
+    f = files[0]
+    return {"url": f.get("url", ""), "md5": (f.get("md5") or "").lower(), "day": f.get("day", ""),
+            "db_vc": f.get("db_vc", ""), "date": f.get("date", 0), "key": f.get("key", "")}
+
+
+def version_of(info: dict) -> str:
+    return str(info.get("date") or info.get("day") or info.get("db_vc") or "0")
+
+
+def download(url: str, dest: str, on_progress: Callable[[int, int], None] | None = None,
+             timeout: int = 120):
+    req = urllib.request.Request(url, headers={"User-Agent": "MaidongKTV/1.2"})
+    total = 0
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        total = int(resp.headers.get("Content-Length") or 0)
+        have = 0
+        with open(dest, "wb") as fh:
+            while True:
+                chunk = resp.read(1 << 20)
+                if not chunk:
+                    break
+                fh.write(chunk)
+                have += len(chunk)
+                if on_progress:
+                    on_progress(have, total)
+    return total
+
+
+def md5_file(path: str) -> str:
+    h = hashlib.md5()
+    with open(path, "rb") as fh:
+        for chunk in iter(lambda: fh.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+# ───────────────────────── 导入清单 ─────────────────────────
+def import_from_muse_db(muse_path: str) -> list[tuple]:
+    """把官方 muse.db 的歌曲清单（含歌手、语言）读成 store.import_library 需要的行。
+
+    关系表 `song_singer_relations.song_id` 是 **songs.id (uuid)**，不是 tid，
+    所以先建 uuid→tid 映射再拼歌手名。
+    """
+    conn = sqlite3.connect(f"file:{muse_path}?mode=ro", uri=True)
+    conn.row_factory = sqlite3.Row
+    try:
+        cols = {r[1] for r in conn.execute("PRAGMA table_info(songs)")}
+        has_rel = bool(conn.execute(
+            "SELECT 1 FROM sqlite_master WHERE type='table' AND name='song_singer_relations'").fetchone())
+        has_singers = bool(conn.execute(
+            "SELECT 1 FROM sqlite_master WHERE type='table' AND name='singers'").fetchone())
+
+        # 歌手名：uuid(song_id) → tid
+        by_tid: dict[int, list[str]] = {}
+        if has_rel and has_singers and "id" in cols:
+            for sid, tid, nm in conn.execute(
+                    """SELECT r.song_id, s.tid, sg.name
+                       FROM song_singer_relations r
+                       JOIN songs s ON s.id = r.song_id
+                       JOIN singers sg ON sg.id = r.singer_id
+                       WHERE sg.name IS NOT NULL AND sg.name != ''"""):
+                if tid:
+                    by_tid.setdefault(int(tid), []).append(nm)
+
+        album_col = "coalesce(s.album_name,'')" if "album_name" in cols else "''"
+        hot_col = "coalesce(s.hot_score,0)" if "hot_score" in cols else "0"
+        tags_col = "coalesce(s.tags_str,'')" if "tags_str" in cols else "''"
+        rows = conn.execute(
+            f"""SELECT s.tid, s.name, s.lang, {album_col} album, s.filename, {hot_col} hot, {tags_col} tags
+                FROM songs s WHERE s.tid IS NOT NULL AND s.tid > 0""").fetchall()
+        out = []
+        for r in rows:
+            tid = int(r["tid"])
+            singers = " / ".join(by_tid.get(tid, [])[:6])
+            out.append((tid, r["name"] or "", norm_lang(r["lang"] or ""), r["album"] or "", singers,
+                        r["filename"] or f"{tid}.ts", r["hot"], r["tags"]))
+        return out
+    finally:
+        conn.close()
+
+
+def sync_library(cfg, store, on_progress: Callable[[str, int], None] | None = None,
+                 cancel: "threading.Event | None" = None) -> dict:
+    """完整同步一遍：查询版本 → 需要则下载安装 → **与既有清单比对** → 导入。
+
+    整个过程挂在一条 `sync_tasks` 记录上（谁在跑、跑完新增/删除/变更各多少），
+    支持中途取消（`cancel` 置位后在下一个阶段边界退出）。
+    """
+    def prog(text: str, pct: int = 0):
+        if on_progress:
+            on_progress(text, pct)
+        store.event("INFO", f"曲库同步：{text}")
+
+    def cancelled() -> bool:
+        return bool(cancel is not None and cancel.is_set())
+
+    cur = store.get_meta("library_version")
+    task_id = store.sync_task_start(cur)
+    try:
+        info = remote_file(cfg)
+        version = version_of(info)
+        prog(f"远端版本 {version}（day={info.get('day')} db_vc={info.get('db_vc')}），本地版本 {cur or '无'}", 3)
+        if cancelled():
+            store.sync_task_finish(task_id, "cancelled", version, error="用户取消")
+            return {"updated": False, "cancelled": True, "version": version}
+        if version and version == cur and store.library_meta()["songs"] > 0:
+            prog("已是最新，跳过下载", 100)
+            store.sync_task_finish(task_id, "done", version, {"total": store.library_meta()["songs"]})
+            return {"updated": False, "version": version, "songs": store.library_meta()["songs"]}
+
+    except Exception as exc:
+        store.sync_task_finish(task_id, "failed", "", error=str(exc))
+        raise
+    data_dir = str(cfg.get("data_dir"))
+    gz = os.path.join(data_dir, "muse.db.gz.download")
+    muse_tmp = os.path.join(data_dir, "muse.db.download")
+    os.makedirs(data_dir, exist_ok=True)
+
+    if cancelled():
+        store.sync_task_finish(task_id, "cancelled", version, error="用户取消")
+        return {"updated": False, "cancelled": True, "version": version}
+    prog("开始下载曲库压缩包", 5)
+    last = [0]
+
+    def on_dl(have: int, total: int):
+        pct = 5 + int(have * 70 / total) if total else 5
+        if pct - last[0] >= 2:
+            last[0] = pct
+            prog(f"下载中 {have // 1048576}MB/{total // 1048576}MB", pct)
+
+    download(info["url"], gz, on_dl, timeout=600)
+    prog("校验 md5", 78)
+    got = md5_file(gz)
+    if info["md5"] and got != info["md5"]:
+        os.remove(gz)
+        raise RuntimeError(f"曲库 md5 不一致 got={got} want={info['md5']}")
+    prog("解压曲库", 82)
+    written = 0
+    with gzip.open(gz, "rb") as src, open(muse_tmp, "wb") as dst:
+        while True:
+            chunk = src.read(1 << 20)
+            if not chunk:
+                break
+            dst.write(chunk)
+            written += len(chunk)
+    os.remove(gz)
+    if cancelled():
+        try:
+            os.remove(muse_tmp)
+        except Exception:
+            pass
+        store.sync_task_finish(task_id, "cancelled", version, error="用户取消")
+        return {"updated": False, "cancelled": True, "version": version}
+    prog(f"解压完成 {written // 1048576}MB，开始比对清单并导入", 90)
+    rows = import_from_muse_db(muse_tmp)
+    diff = store.diff_and_import(rows, version)          # ★ 需求①：更新时比对清单
+    prog(f"清单比对完成：总 {diff['total']}，新增 {diff['added']}，"
+         f"上游删除 {diff['removed']}，变更 {diff['changed']}", 99)
+    # 原始库留档，便于排错/二次导入
+    try:
+        keep = os.path.join(data_dir, f"muse-{info.get('day') or version}.db")
+        if os.path.exists(keep):
+            os.remove(keep)
+        os.replace(muse_tmp, keep)
+        store.set_meta("library_file", keep)
+    except Exception as exc:                      # pragma: no cover
+        store.event("WARN", f"保留原始曲库失败：{exc}")
+        try:
+            os.remove(muse_tmp)
+        except Exception:
+            pass
+    result = {"updated": True, "version": version, "songs": diff["total"], "day": info.get("day"),
+              "md5": info["md5"], "diff": diff, "task_id": task_id}
+    store.sync_task_finish(task_id, "done", version, diff)
+    return result