zijunchiang
87aa979abd
版本号：
- 1.5.11 越界（规则 3 规定 patch ≤ 10）→ 修正为 1.6.0（进位到 minor、patch 归 0）
- versionCode 43 不变：43 从未对外发布，本次修正只是把命名拉回约定
合并支线（原在工作区根目录、App 仓库之外，DESIGN.md 自称"支线"）：
- selfhost-source/server/ 自建歌曲源服务器（纯 Python 标准库、零 pip 依赖）
- selfhost-source/deploy/、DESIGN.md、LIBRARY.md、build_inventory.py
- selfhost-source/inventory/ 小清单（playlists.csv、song-stats.md 等）
- ⚠️ inventory 生成物（songs-*.csv / singers-top.csv，共 84MB）不入库，
已写入 .gitignore；需要时用 build_inventory.py 依据 muse.db 重新生成
同时入库的 App 侧支线改动（此前为工作树未提交状态）：
- Whoami.kt（whoami 设备心跳）、DeviceIdentity.kt（设备身份伪装）
- PushDownload.kt / PushDownloadClient.kt（点歌推送）
- AdVideoFilter 带 query/fragment 的地址闸门；KTV_BRIDGE_API 11 → 13
- 修 ktv_api.js 的 global.KtvBridgeApiVersion（12 → 13，与注释及 Kotlin 两处校验一致）
- 对应单测：WhoamiTest / DeviceIdentityTest / PushDownloadTest / AdVideoFilterTest（共 39 例）
文档：
- DESIGN.md 的"在 App 仓库之外"约定改为"已并入仓库统一版本化"
- AGENTS.md 版本号行更新为 1.6.0/43，并新增支线并入说明
验证：assembleRelease + testReleaseUnitTest 全过（39 例）
326 lines
14 KiB
Python
326 lines
14 KiB
Python
"""曲库同步（复刻 App 的 MuseDbSync）：login → sqlite/sync → 下载 gz → md5 校验 → 解压 → 导入清单。
|
||
|
||
要点：
|
||
* **纯标准库实现 RSA/PKCS#1 v1.5**（Debian 上不需要装任何 pip 包，也不需要 openssl 命令行）；
|
||
* 同步前必须重新 login 拿 `muse-portal-connector` cookie——没有它 sync 直接 401；
|
||
* `mconn` 与 `mws` 两个主机都能返回文件（同一后台），任选其一；
|
||
* 官方库**没有** `cloud_url` 列（29 列版本），导入时只取存在的列。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import base64
|
||
import gzip
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import threading
|
||
import time
|
||
import urllib.request
|
||
from typing import Callable
|
||
|
||
# App 里使用的 RSA 公钥（与 MuseDbSync.RSA_PUBLIC_KEY 一致）
|
||
RSA_PUBLIC_KEY_B64 = (
|
||
"MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAucL0oFErd7REM6TSNa3EZdN1YaOs4J1eCybLPyoQ9ru3q1HU67agC9FzhrCG/RvAQUya5iPmQ8Caed05vqcCVcyJChmkSOGQ7DVShe2rGuTMNlpoRV6UzfcraaVS++7m2K/+kSZJ8OAhhhVuqPruMjsFYpdtstAwvyZT28b+eENwzpp9UHqsooZc7FZ0H8kTbs6XMkw4nIWo+4HoPAhNLEY+xdHvwY6drF/3WDTvsaoMrs73TVQCEEHzZNIz2H/is9VLMnIyOfnfcJi9br78Fj2xHzxu3sAySBOTVLmUMxqYh/g1ox5OXGcW93HJkQLkBi42tFAEkWYlYyl93+jbbQIDAQAB"
|
||
)
|
||
CANON_LANGS = ["国语", "粤语", "闽南语", "英语", "日语", "韩语", "其它"]
|
||
LANG_ALIAS = {"闽南": "闽南语", "台语": "闽南语", "H国语": "国语", "其他": "其它", "西语": "其它",
|
||
"西班牙语": "其它", "法语": "其它", "俄语": "其它", "泰语": "其它", "印度": "其它",
|
||
"潮汕语": "其它", "消音": "其它", "原版": "其它"}
|
||
|
||
|
||
def norm_lang(v: str) -> str:
|
||
k = (v or "").strip()
|
||
if k in CANON_LANGS:
|
||
return k
|
||
if k in LANG_ALIAS:
|
||
return LANG_ALIAS[k]
|
||
return "其它" if k else "未标注"
|
||
|
||
|
||
# ───────────────────────── 极简 DER / RSA ─────────────────────────
|
||
def _der_tlv(buf: bytes, pos: int):
|
||
tag = buf[pos]
|
||
pos += 1
|
||
ln = buf[pos]
|
||
pos += 1
|
||
if ln & 0x80:
|
||
n = ln & 0x7F
|
||
ln = int.from_bytes(buf[pos:pos + n], "big")
|
||
pos += n
|
||
start = pos
|
||
return tag, start, start + ln
|
||
|
||
|
||
def _der_walk(buf: bytes, start: int, end: int):
|
||
"""产出 (tag, start, end) 的扁平列表（含容器内部）。"""
|
||
out = []
|
||
p = start
|
||
while p < end:
|
||
tag, s, e = _der_tlv(buf, p)
|
||
out.append((tag, s, e))
|
||
if tag & 0x20: # constructed → 递归
|
||
out.extend(_der_walk(buf, s, e))
|
||
elif tag == 0x03: # BIT STRING：跳过"未使用位数"字节后继续
|
||
# SubjectPublicKeyInfo 里的 RSAPublicKey 就包在 BIT STRING 内层。
|
||
out.extend(_der_walk(buf, s + 1, e))
|
||
p = e
|
||
return out
|
||
|
||
|
||
def rsa_public_numbers(b64: str = RSA_PUBLIC_KEY_B64) -> tuple[int, int]:
|
||
"""从 SubjectPublicKeyInfo(DER) 里取出 (n, e)。"""
|
||
der = base64.b64decode(b64)
|
||
ints = []
|
||
for tag, s, e in _der_walk(der, 0, len(der)):
|
||
if tag == 0x02: # INTEGER
|
||
ints.append(int.from_bytes(der[s:e], "big"))
|
||
if len(ints) < 2:
|
||
raise ValueError("无法解析 RSA 公钥")
|
||
return ints[-2], ints[-1] # modulus, exponent
|
||
|
||
|
||
def rsa_pkcs1v15_encrypt(plaintext: bytes, n: int, e: int) -> bytes:
|
||
"""PKCS#1 v1.5 加密（EM = 0x00 0x02 PS 0x00 M）。"""
|
||
k = (n.bit_length() + 7) // 8
|
||
if len(plaintext) > k - 11:
|
||
raise ValueError("明文过长")
|
||
ps_len = k - 3 - len(plaintext)
|
||
ps = bytearray()
|
||
while len(ps) < ps_len:
|
||
b = os.urandom(1)[0]
|
||
if b: # PS 必须非零
|
||
ps.append(b)
|
||
em = b"\x00\x02" + bytes(ps) + b"\x00" + plaintext
|
||
return pow(int.from_bytes(em, "big"), e, n).to_bytes(k, "big")
|
||
|
||
|
||
def encrypt_device_id(device_id: str) -> str:
|
||
n, e = rsa_public_numbers()
|
||
return base64.b64encode(rsa_pkcs1v15_encrypt(json.dumps({"device_id": device_id}).encode(), n, e)).decode()
|
||
|
||
|
||
# ───────────────────────── HTTP ─────────────────────────
|
||
def _post_json(url: str, payload: dict, cookie: str = "", timeout: int = 60, ua: str = "MaidongKTV/1.2"):
|
||
body = json.dumps(payload, separators=(",", ":")).encode()
|
||
req = urllib.request.Request(url, data=body, method="POST")
|
||
req.add_header("Content-Type", "application/json; charset=UTF-8")
|
||
req.add_header("Accept", "*/*")
|
||
req.add_header("User-Agent", ua)
|
||
if cookie:
|
||
req.add_header("Cookie", cookie)
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
raw = resp.read()
|
||
set_cookie = resp.headers.get("Set-Cookie", "") or ""
|
||
code = resp.status
|
||
return code, raw, set_cookie
|
||
|
||
|
||
def login(cfg) -> str:
|
||
"""返回 `muse-portal-connector=…` cookie 字符串；失败抛异常。"""
|
||
payload = {
|
||
"encrypted_data": encrypt_device_id(str(cfg.get("muse_device_id"))),
|
||
"ip": "172.16.32.15", "dns": "", "router": "",
|
||
"subnet_mask": "255.255.255.0", "channel": "common", "mode": "ott",
|
||
}
|
||
code, raw, set_cookie = _post_json(str(cfg.get("muse_login_url")), payload,
|
||
timeout=int(cfg.get("timeout_sec")), ua=str(cfg.get("user_agent")))
|
||
cookie = ""
|
||
for part in set_cookie.split(","):
|
||
for seg in part.split(";"):
|
||
seg = seg.strip()
|
||
if seg.startswith("muse-portal-connector="):
|
||
cookie = seg
|
||
if code != 200 or not cookie:
|
||
raise RuntimeError(f"曲库登录失败：HTTP {code}，cookie={'有' if cookie else '无'}")
|
||
return cookie
|
||
|
||
|
||
def remote_file(cfg) -> dict:
|
||
"""返回 {url, md5, day, db_vc, date, key, size?}。"""
|
||
cookie = login(cfg)
|
||
payload = {"last_sync_at": 0, "music_file_type": int(cfg.get("muse_music_file_type"))}
|
||
code, raw, _ = _post_json(str(cfg.get("muse_sync_url")), payload, cookie,
|
||
timeout=int(cfg.get("timeout_sec")), ua=str(cfg.get("user_agent")))
|
||
if code != 200:
|
||
raise RuntimeError(f"曲库同步 HTTP {code}")
|
||
data = json.loads(raw.decode("utf-8", "replace"))
|
||
files = data.get("files") or []
|
||
if not files:
|
||
raise RuntimeError("曲库同步返回空 files（服务端未授权或该设备无可用曲库）")
|
||
f = files[0]
|
||
return {"url": f.get("url", ""), "md5": (f.get("md5") or "").lower(), "day": f.get("day", ""),
|
||
"db_vc": f.get("db_vc", ""), "date": f.get("date", 0), "key": f.get("key", "")}
|
||
|
||
|
||
def version_of(info: dict) -> str:
|
||
return str(info.get("date") or info.get("day") or info.get("db_vc") or "0")
|
||
|
||
|
||
def download(url: str, dest: str, on_progress: Callable[[int, int], None] | None = None,
|
||
timeout: int = 120):
|
||
req = urllib.request.Request(url, headers={"User-Agent": "MaidongKTV/1.2"})
|
||
total = 0
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
total = int(resp.headers.get("Content-Length") or 0)
|
||
have = 0
|
||
with open(dest, "wb") as fh:
|
||
while True:
|
||
chunk = resp.read(1 << 20)
|
||
if not chunk:
|
||
break
|
||
fh.write(chunk)
|
||
have += len(chunk)
|
||
if on_progress:
|
||
on_progress(have, total)
|
||
return total
|
||
|
||
|
||
def md5_file(path: str) -> str:
|
||
h = hashlib.md5()
|
||
with open(path, "rb") as fh:
|
||
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
|
||
# ───────────────────────── 导入清单 ─────────────────────────
|
||
def import_from_muse_db(muse_path: str) -> list[tuple]:
|
||
"""把官方 muse.db 的歌曲清单（含歌手、语言）读成 store.import_library 需要的行。
|
||
|
||
关系表 `song_singer_relations.song_id` 是 **songs.id (uuid)**，不是 tid，
|
||
所以先建 uuid→tid 映射再拼歌手名。
|
||
"""
|
||
conn = sqlite3.connect(f"file:{muse_path}?mode=ro", uri=True)
|
||
conn.row_factory = sqlite3.Row
|
||
try:
|
||
cols = {r[1] for r in conn.execute("PRAGMA table_info(songs)")}
|
||
has_rel = bool(conn.execute(
|
||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='song_singer_relations'").fetchone())
|
||
has_singers = bool(conn.execute(
|
||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='singers'").fetchone())
|
||
|
||
# 歌手名：uuid(song_id) → tid
|
||
by_tid: dict[int, list[str]] = {}
|
||
if has_rel and has_singers and "id" in cols:
|
||
for sid, tid, nm in conn.execute(
|
||
"""SELECT r.song_id, s.tid, sg.name
|
||
FROM song_singer_relations r
|
||
JOIN songs s ON s.id = r.song_id
|
||
JOIN singers sg ON sg.id = r.singer_id
|
||
WHERE sg.name IS NOT NULL AND sg.name != ''"""):
|
||
if tid:
|
||
by_tid.setdefault(int(tid), []).append(nm)
|
||
|
||
album_col = "coalesce(s.album_name,'')" if "album_name" in cols else "''"
|
||
hot_col = "coalesce(s.hot_score,0)" if "hot_score" in cols else "0"
|
||
tags_col = "coalesce(s.tags_str,'')" if "tags_str" in cols else "''"
|
||
rows = conn.execute(
|
||
f"""SELECT s.tid, s.name, s.lang, {album_col} album, s.filename, {hot_col} hot, {tags_col} tags
|
||
FROM songs s WHERE s.tid IS NOT NULL AND s.tid > 0""").fetchall()
|
||
out = []
|
||
for r in rows:
|
||
tid = int(r["tid"])
|
||
singers = " / ".join(by_tid.get(tid, [])[:6])
|
||
out.append((tid, r["name"] or "", norm_lang(r["lang"] or ""), r["album"] or "", singers,
|
||
r["filename"] or f"{tid}.ts", r["hot"], r["tags"]))
|
||
return out
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def sync_library(cfg, store, on_progress: Callable[[str, int], None] | None = None,
|
||
cancel: "threading.Event | None" = None) -> dict:
|
||
"""完整同步一遍：查询版本 → 需要则下载安装 → **与既有清单比对** → 导入。
|
||
|
||
整个过程挂在一条 `sync_tasks` 记录上（谁在跑、跑完新增/删除/变更各多少），
|
||
支持中途取消（`cancel` 置位后在下一个阶段边界退出）。
|
||
"""
|
||
def prog(text: str, pct: int = 0):
|
||
if on_progress:
|
||
on_progress(text, pct)
|
||
store.event("INFO", f"曲库同步：{text}")
|
||
|
||
def cancelled() -> bool:
|
||
return bool(cancel is not None and cancel.is_set())
|
||
|
||
cur = store.get_meta("library_version")
|
||
task_id = store.sync_task_start(cur)
|
||
try:
|
||
info = remote_file(cfg)
|
||
version = version_of(info)
|
||
prog(f"远端版本 {version}（day={info.get('day')} db_vc={info.get('db_vc')}），本地版本 {cur or '无'}", 3)
|
||
if cancelled():
|
||
store.sync_task_finish(task_id, "cancelled", version, error="用户取消")
|
||
return {"updated": False, "cancelled": True, "version": version}
|
||
if version and version == cur and store.library_meta()["songs"] > 0:
|
||
prog("已是最新，跳过下载", 100)
|
||
store.sync_task_finish(task_id, "done", version, {"total": store.library_meta()["songs"]})
|
||
return {"updated": False, "version": version, "songs": store.library_meta()["songs"]}
|
||
|
||
except Exception as exc:
|
||
store.sync_task_finish(task_id, "failed", "", error=str(exc))
|
||
raise
|
||
data_dir = str(cfg.get("data_dir"))
|
||
gz = os.path.join(data_dir, "muse.db.gz.download")
|
||
muse_tmp = os.path.join(data_dir, "muse.db.download")
|
||
os.makedirs(data_dir, exist_ok=True)
|
||
|
||
if cancelled():
|
||
store.sync_task_finish(task_id, "cancelled", version, error="用户取消")
|
||
return {"updated": False, "cancelled": True, "version": version}
|
||
prog("开始下载曲库压缩包", 5)
|
||
last = [0]
|
||
|
||
def on_dl(have: int, total: int):
|
||
pct = 5 + int(have * 70 / total) if total else 5
|
||
if pct - last[0] >= 2:
|
||
last[0] = pct
|
||
prog(f"下载中 {have // 1048576}MB/{total // 1048576}MB", pct)
|
||
|
||
download(info["url"], gz, on_dl, timeout=600)
|
||
prog("校验 md5", 78)
|
||
got = md5_file(gz)
|
||
if info["md5"] and got != info["md5"]:
|
||
os.remove(gz)
|
||
raise RuntimeError(f"曲库 md5 不一致 got={got} want={info['md5']}")
|
||
prog("解压曲库", 82)
|
||
written = 0
|
||
with gzip.open(gz, "rb") as src, open(muse_tmp, "wb") as dst:
|
||
while True:
|
||
chunk = src.read(1 << 20)
|
||
if not chunk:
|
||
break
|
||
dst.write(chunk)
|
||
written += len(chunk)
|
||
os.remove(gz)
|
||
if cancelled():
|
||
try:
|
||
os.remove(muse_tmp)
|
||
except Exception:
|
||
pass
|
||
store.sync_task_finish(task_id, "cancelled", version, error="用户取消")
|
||
return {"updated": False, "cancelled": True, "version": version}
|
||
prog(f"解压完成 {written // 1048576}MB，开始比对清单并导入", 90)
|
||
rows = import_from_muse_db(muse_tmp)
|
||
diff = store.diff_and_import(rows, version) # ★ 需求①：更新时比对清单
|
||
prog(f"清单比对完成：总 {diff['total']}，新增 {diff['added']}，"
|
||
f"上游删除 {diff['removed']}，变更 {diff['changed']}", 99)
|
||
# 原始库留档，便于排错/二次导入
|
||
try:
|
||
keep = os.path.join(data_dir, f"muse-{info.get('day') or version}.db")
|
||
if os.path.exists(keep):
|
||
os.remove(keep)
|
||
os.replace(muse_tmp, keep)
|
||
store.set_meta("library_file", keep)
|
||
except Exception as exc: # pragma: no cover
|
||
store.event("WARN", f"保留原始曲库失败：{exc}")
|
||
try:
|
||
os.remove(muse_tmp)
|
||
except Exception:
|
||
pass
|
||
result = {"updated": True, "version": version, "songs": diff["total"], "day": info.get("day"),
|
||
"md5": info["md5"], "diff": diff, "task_id": task_id}
|
||
store.sync_task_finish(task_id, "done", version, diff)
|
||
return result
|