From 4d5041ba349e741f190c6c0337ce252edffba841 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 10:44:15 +0800 Subject: [PATCH 001/187] feat: add isolated TikTok proxy account pool --- .dockerignore | 1 + Dockerfile | 15 +- docker-compose.yml | 26 +- scripts/proxy_pool.py | 1736 +++++++++++++++++++++++++++++++++++++ scripts/static/proxy.html | 91 ++ scripts/web_app.py | 76 ++ 6 files changed, 1942 insertions(+), 3 deletions(-) create mode 100644 scripts/proxy_pool.py create mode 100644 scripts/static/proxy.html diff --git a/.dockerignore b/.dockerignore index c9cb2c0..6192c99 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ .env.* backups/ data/ +data-dev/ videos/ output/ __pycache__/ diff --git a/Dockerfile b/Dockerfile index af4b8d9..ef8a109 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ ARG all_proxy= ARG NO_PROXY= ARG no_proxy= -ENV PYTHONDONTWRITEBYTECODE=1 \ +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 @@ -35,8 +36,20 @@ RUN python -c "import video_analyzer, os; p=os.path.join(os.path.dirname(video_a RUN pip install yt-dlp playwright httpx "scrapling[ai]" \ && python -m playwright install --with-deps chromium \ + && python -m playwright install chrome \ && scrapling install +RUN apt-get -o Acquire::ForceIPv4=true -o Acquire::http::Timeout=20 update \ + && apt-get -o Acquire::ForceIPv4=true -o Acquire::http::Timeout=20 install -y --no-install-recommends \ + novnc \ + websockify \ + x11vnc \ + xvfb \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd --system --gid 10001 tikbrowser \ + && useradd --system --uid 10001 --gid tikbrowser --create-home --home-dir /home/tikbrowser tikbrowser + RUN pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn "curl_cffi>=0.15,<0.16" RUN apt-get update \ diff --git a/docker-compose.yml b/docker-compose.yml index ec31b6d..073e5b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,13 @@ services: no_proxy: ${no_proxy:-} image: short-video-analyzer:latest environment: + TZ: ${TZ:-America/Los_Angeles} + TIKTOK_BROWSER_MAX_SLOTS: ${TIKTOK_BROWSER_MAX_SLOTS:-2} + TIKTOK_PENDING_LOGIN_TTL_SECONDS: ${TIKTOK_PENDING_LOGIN_TTL_SECONDS:-900} + TIKTOK_BROWSER_LOCALE: ${TIKTOK_BROWSER_LOCALE:-en-US} + TIKTOK_BROWSER_ACCEPT_LANGUAGE: ${TIKTOK_BROWSER_ACCEPT_LANGUAGE:-en-US,en} + NOVNC_PORT: ${NOVNC_PORT:-6080} + NOVNC_MANUAL_PORTS: ${NOVNC_MANUAL_PORTS:-1} VISION_API_KEY: ${VISION_API_KEY:-} VISION_API_URL: ${VISION_API_URL:-https://dashscope.aliyuncs.com/compatible-mode/v1} VISION_MODEL: ${VISION_MODEL:-qwen3-vl-flash} @@ -72,8 +79,10 @@ services: volumes: - ./videos:/workspace/videos - ./output:/workspace/output - - ./data:/workspace/data + - ${DATA_DIR:-./data}:/workspace/data - ./scripts:/workspace/scripts:ro + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro networks: - analyzer @@ -90,8 +99,19 @@ services: NO_PROXY: ${NO_PROXY:-} no_proxy: ${no_proxy:-} image: short-video-analyzer:latest + init: true + shm_size: 1gb + cap_add: + - SYS_ADMIN command: python scripts/web_app.py environment: + TZ: ${TZ:-America/Los_Angeles} + TIKTOK_BROWSER_MAX_SLOTS: ${TIKTOK_BROWSER_MAX_SLOTS:-2} + TIKTOK_PENDING_LOGIN_TTL_SECONDS: ${TIKTOK_PENDING_LOGIN_TTL_SECONDS:-900} + TIKTOK_BROWSER_LOCALE: ${TIKTOK_BROWSER_LOCALE:-en-US} + TIKTOK_BROWSER_ACCEPT_LANGUAGE: ${TIKTOK_BROWSER_ACCEPT_LANGUAGE:-en-US,en} + NOVNC_PORT: ${NOVNC_PORT:-6080} + NOVNC_MANUAL_PORTS: ${NOVNC_MANUAL_PORTS:-1} VISION_API_KEY: ${VISION_API_KEY:-} VISION_API_URL: ${VISION_API_URL:-https://dashscope.aliyuncs.com/compatible-mode/v1} VISION_MODEL: ${VISION_MODEL:-qwen3-vl-flash} @@ -154,8 +174,10 @@ services: volumes: - ./videos:/workspace/videos - ./output:/workspace/output - - ./data:/workspace/data + - ${DATA_DIR:-./data}:/workspace/data - ./scripts:/workspace/scripts:ro + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro - ${OCR_SHARED_DIR:-/home/openclaw/ocr-shared}:${OCR_SERVER_SHARED_DIR:-/home/openclaw/ocr-shared} - /var/run/docker.sock:/var/run/docker.sock network_mode: host diff --git a/scripts/proxy_pool.py b/scripts/proxy_pool.py new file mode 100644 index 0000000..343aac6 --- /dev/null +++ b/scripts/proxy_pool.py @@ -0,0 +1,1736 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import base64 +import binascii +import calendar +import http.client +import hashlib +import json +import os +import signal +import shutil +import socket +import sqlite3 +import subprocess +import time +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, unquote, urlparse +from urllib.error import HTTPError, URLError +from urllib.request import ProxyHandler, Request, build_opener + + +ROOT = Path.cwd() +DATA_DIR = ROOT / "data" +DB_PATH = DATA_DIR / "proxy_pool.sqlite" +DEFAULT_NOVNC_PUBLIC_URL = os.getenv("NOVNC_PUBLIC_URL", "http://192.168.1.254:6080/vnc.html?autoconnect=1&resize=scale") +DEFAULT_MIHOMO_API = os.getenv("MIHOMO_API_URL", "http://127.0.0.1:9090") +PROXY_PORT_START = int(os.getenv("PROXY_POOL_PORT_START", "18900") or "18900") +PROXY_PORT_END = int(os.getenv("PROXY_POOL_PORT_END", "18999") or "18999") +NOVNC_PORT = int(os.getenv("NOVNC_PORT", "6080") or "6080") +NOVNC_MANUAL_PORTS = int(os.getenv("NOVNC_MANUAL_PORTS", "1") or "1") +VNC_PORT = int(os.getenv("VNC_PORT", "5900") or "5900") +XVFB_DISPLAY_BASE = int(os.getenv("TIKTOK_XVFB_DISPLAY_BASE", "90") or "90") +TIKTOK_BROWSER_UID = int(os.getenv("TIKTOK_BROWSER_UID", "10001") or "10001") +TIKTOK_BROWSER_GID = int(os.getenv("TIKTOK_BROWSER_GID", "10001") or "10001") +TIKTOK_BROWSER_LOCALE = os.getenv("TIKTOK_BROWSER_LOCALE", "en-US").strip() or "en-US" +TIKTOK_BROWSER_ACCEPT_LANGUAGE = os.getenv("TIKTOK_BROWSER_ACCEPT_LANGUAGE", "en-US,en").strip() or "en-US,en" +STATUS_ACTIVE = "启用" +STATUS_PAUSED = "禁用" +STATUS_ERROR = "不可用" +STATUS_MAP = { + "active": STATUS_ACTIVE, + "enabled": STATUS_ACTIVE, + "ok": STATUS_ACTIVE, + "bound": STATUS_ACTIVE, + "可用": STATUS_ACTIVE, + "已绑定": STATUS_ACTIVE, + "未绑定": STATUS_ACTIVE, + "启用": STATUS_ACTIVE, + "paused": STATUS_PAUSED, + "disabled": STATUS_PAUSED, + "disable": STATUS_PAUSED, + "暂停": STATUS_PAUSED, + "禁用": STATUS_PAUSED, + "blocked": STATUS_ERROR, + "error": STATUS_ERROR, + "异常": STATUS_ERROR, + "不可用": STATUS_ERROR, +} +ACCOUNT_STATUS_ACTIVE = "可用" +ACCOUNT_STATUS_PAUSED = "暂停" +ACCOUNT_STATUS_ERROR = "异常" +ACCOUNT_STATUS_MAP = { + "active": ACCOUNT_STATUS_ACTIVE, + "enabled": ACCOUNT_STATUS_ACTIVE, + "ok": ACCOUNT_STATUS_ACTIVE, + "可用": ACCOUNT_STATUS_ACTIVE, + "paused": ACCOUNT_STATUS_PAUSED, + "disabled": ACCOUNT_STATUS_PAUSED, + "暂停": ACCOUNT_STATUS_PAUSED, + "blocked": ACCOUNT_STATUS_ERROR, + "error": ACCOUNT_STATUS_ERROR, + "异常": ACCOUNT_STATUS_ERROR, +} + + +def browser_max_slots() -> int: + return max(1, int(os.getenv("TIKTOK_BROWSER_MAX_SLOTS", "2") or "2")) + + +def pending_login_ttl_seconds() -> int: + return max(60, int(os.getenv("TIKTOK_PENDING_LOGIN_TTL_SECONDS", "900") or "900")) + + +def novnc_port_plan() -> dict[str, Any]: + max_slots = browser_max_slots() + manual_ports = max(1, NOVNC_MANUAL_PORTS) + total_ports = max_slots + manual_ports + end_port = NOVNC_PORT + total_ports - 1 + return { + "base_port": NOVNC_PORT, + "manual_ports": manual_ports, + "max_slots": max_slots, + "total_ports": total_ports, + "allowed_ports": list(range(NOVNC_PORT, end_port + 1)), + "allowed_range": f"{NOVNC_PORT}-{end_port}" if end_port != NOVNC_PORT else str(NOVNC_PORT), + } + + +def now_iso() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def connect() -> sqlite3.Connection: + DATA_DIR.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(DB_PATH) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + init_db(conn) + return conn + + +def init_db(conn: sqlite3.Connection) -> None: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS proxy_profiles ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + source_type TEXT NOT NULL DEFAULT 'vless', + source_uri TEXT NOT NULL DEFAULT '', + expected_exit_ip TEXT NOT NULL DEFAULT '', + region TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'active', + notes TEXT NOT NULL DEFAULT '', + parse_status TEXT NOT NULL DEFAULT 'manual', + parse_error TEXT NOT NULL DEFAULT '', + mihomo_name TEXT NOT NULL DEFAULT '', + parsed_json TEXT NOT NULL DEFAULT '{}', + mihomo_proxy_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + local_port INTEGER NOT NULL DEFAULT 0, + detected_exit_ip TEXT NOT NULL DEFAULT '', + detected_country TEXT NOT NULL DEFAULT '', + detected_region TEXT NOT NULL DEFAULT '', + detected_city TEXT NOT NULL DEFAULT '', + detected_address TEXT NOT NULL DEFAULT '', + detected_at TEXT NOT NULL DEFAULT '' + ); + CREATE TABLE IF NOT EXISTS tiktok_accounts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL DEFAULT '', + proxy_profile_id INTEGER NOT NULL REFERENCES proxy_profiles(id) ON DELETE RESTRICT, + status TEXT NOT NULL DEFAULT 'active', + profile_json TEXT NOT NULL DEFAULT '{}', + notes TEXT NOT NULL DEFAULT '', + last_checked_ip TEXT NOT NULL DEFAULT '', + last_check_status TEXT NOT NULL DEFAULT '', + last_check_at TEXT NOT NULL DEFAULT '', + last_login_at TEXT NOT NULL DEFAULT '', + last_collect_at TEXT NOT NULL DEFAULT '', + last_error TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_tiktok_accounts_proxy ON tiktok_accounts(proxy_profile_id); + CREATE INDEX IF NOT EXISTS idx_tiktok_accounts_status ON tiktok_accounts(status); + CREATE TABLE IF NOT EXISTS browser_sessions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + slot INTEGER NOT NULL, + proxy_profile_id INTEGER NOT NULL REFERENCES proxy_profiles(id) ON DELETE RESTRICT, + account_id INTEGER REFERENCES tiktok_accounts(id) ON DELETE SET NULL, + username TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'starting', + channel_url TEXT NOT NULL DEFAULT '', + pid INTEGER NOT NULL DEFAULT 0, + xvfb_pid INTEGER NOT NULL DEFAULT 0, + x11vnc_pid INTEGER NOT NULL DEFAULT 0, + websockify_pid INTEGER NOT NULL DEFAULT 0, + display TEXT NOT NULL DEFAULT '', + vnc_port INTEGER NOT NULL DEFAULT 0, + novnc_port INTEGER NOT NULL DEFAULT 0, + profile_key TEXT NOT NULL DEFAULT '', + user_data_dir TEXT NOT NULL DEFAULT '', + last_error TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_browser_sessions_status ON browser_sessions(status); + CREATE INDEX IF NOT EXISTS idx_browser_sessions_proxy ON browser_sessions(proxy_profile_id); + """ + ) + for name, definition in { + "local_port": "INTEGER NOT NULL DEFAULT 0", + "detected_exit_ip": "TEXT NOT NULL DEFAULT ''", + "detected_country": "TEXT NOT NULL DEFAULT ''", + "detected_region": "TEXT NOT NULL DEFAULT ''", + "detected_city": "TEXT NOT NULL DEFAULT ''", + "detected_address": "TEXT NOT NULL DEFAULT ''", + "detected_at": "TEXT NOT NULL DEFAULT ''", + }.items(): + existing = {row[1] for row in conn.execute("PRAGMA table_info(proxy_profiles)")} + if name not in existing: + conn.execute(f"ALTER TABLE proxy_profiles ADD COLUMN {name} {definition}") + existing_session_cols = {row[1] for row in conn.execute("PRAGMA table_info(browser_sessions)")} + for name, definition in { + "xvfb_pid": "INTEGER NOT NULL DEFAULT 0", + "x11vnc_pid": "INTEGER NOT NULL DEFAULT 0", + "websockify_pid": "INTEGER NOT NULL DEFAULT 0", + "display": "TEXT NOT NULL DEFAULT ''", + "vnc_port": "INTEGER NOT NULL DEFAULT 0", + "novnc_port": "INTEGER NOT NULL DEFAULT 0", + }.items(): + if name not in existing_session_cols: + conn.execute(f"ALTER TABLE browser_sessions ADD COLUMN {name} {definition}") + conn.commit() + + +def _json_loads(value: str, fallback: Any) -> Any: + try: + return json.loads(value or "") + except Exception: + return fallback + + +def _clean_text(value: Any, max_len: int = 1000) -> str: + return str(value or "").strip()[:max_len] + + +def _clean_status(value: Any, default: str = STATUS_ACTIVE) -> str: + raw = _clean_text(value, 40) + if not raw: + return default + return STATUS_MAP.get(raw.lower(), STATUS_MAP.get(raw, raw if raw in {STATUS_ACTIVE, STATUS_PAUSED, STATUS_ERROR} else default)) + + +def _clean_account_status(value: Any, default: str = ACCOUNT_STATUS_ACTIVE) -> str: + raw = _clean_text(value, 40) + if not raw: + return default + return ACCOUNT_STATUS_MAP.get(raw.lower(), ACCOUNT_STATUS_MAP.get(raw, raw if raw in {ACCOUNT_STATUS_ACTIVE, ACCOUNT_STATUS_PAUSED, ACCOUNT_STATUS_ERROR} else default)) + + +def _allocate_port(conn: sqlite3.Connection, current_id: int = 0) -> int: + if current_id: + row = conn.execute("SELECT local_port FROM proxy_profiles WHERE id = ?", (current_id,)).fetchone() + if row and int(row["local_port"] or 0): + return int(row["local_port"]) + used = {int(row["local_port"]) for row in conn.execute("SELECT local_port FROM proxy_profiles WHERE local_port > 0")} + for port in range(PROXY_PORT_START, PROXY_PORT_END + 1): + if port not in used: + return port + raise ValueError(f"proxy port range exhausted: {PROXY_PORT_START}-{PROXY_PORT_END}") + + +def _normal_username(value: Any) -> str: + username = _clean_text(value, 120).lstrip("@") + if not username: + raise ValueError("username is required") + return username + + +def _safe_profile_key(username: str) -> str: + safe = "".join(ch if ch.isalnum() or ch in {"-", "_", "."} else "_" for ch in username.lower()) + return safe.strip("._-") or "account" + + +def _deep_merge(base: dict[str, Any], overlay: dict[str, Any]) -> dict[str, Any]: + merged = dict(base) + for key, value in overlay.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _deep_merge(merged[key], value) + else: + merged[key] = value + return merged + + +def _isolation_profile(username: str, proxy_profile_id: int, pool: sqlite3.Row | None) -> dict[str, Any]: + key = _safe_profile_key(username) + profile_root = os.getenv("TIKTOK_BROWSER_PROFILE_ROOT", "data/tiktok_browser_profiles").rstrip("/") + session_root = os.getenv("TIKTOK_SESSION_ROOT", "data/tiktok_sessions").rstrip("/") + max_slots = browser_max_slots() + return { + "manual_login": { + "surface": "novnc", + "official_site": "https://www.tiktok.com/", + "updated_at": now_iso(), + }, + "proxy_binding": { + "proxy_profile_id": proxy_profile_id, + "proxy_name": str(pool["name"] or "") if pool else "", + "local_port": int(pool["local_port"] or 0) if pool else 0, + "expected_exit_ip": str(pool["expected_exit_ip"] or "") if pool else "", + "detected_address": str(pool["detected_address"] or pool["region"] or "") if pool else "", + }, + "isolation": { + "browser_profile_key": f"tiktok-{key}", + "user_data_dir": f"{profile_root}/{key}/user-data", + "cookie_store_dir": f"{profile_root}/{key}/cookies", + "session_dir": f"{session_root}/{key}", + "cache_dir": f"{profile_root}/{key}/cache", + "download_dir": f"{profile_root}/{key}/downloads", + "browser_context": "per_account_required", + "storage_state": "per_account_required", + "cookie_jar": "per_account_required", + "local_storage": "per_account_required", + "indexed_db": "per_account_required", + "service_workers": "per_account_required", + "web_rtc_policy": "disable_non_proxied_udp_required", + "runner_must_preflight_proxy_ip": True, + }, + "browser_settings": { + "locale": TIKTOK_BROWSER_LOCALE, + "timezone": os.getenv("TZ", "America/Los_Angeles"), + "accept_language": TIKTOK_BROWSER_ACCEPT_LANGUAGE, + "geolocation": "deny_or_match_proxy_region", + "permissions": "per_account_profile_only", + "proxy_server": f"127.0.0.1:{int(pool['local_port'] or 0) if pool else 0}", + "disable_background_networking": True, + "disable_default_apps": True, + "disable_sync": True, + "disable_translate": True, + "disable_non_proxied_udp": True, + }, + "system_settings": { + "notes": "Use a per-account browser process/profile. OS-level global settings are shared unless the runner starts isolated containers or desktops.", + "preferred_desktop_mode": "per_slot_novnc_or_container", + "clipboard_isolation": "avoid_cross_account_copy_paste", + "downloads_isolation": "per_account_download_dir", + }, + "worker": { + "mode": "slot_pool", + "max_parallel_slots": max_slots, + "one_account_per_browser_context": True, + "one_account_per_browser_process": True, + "novnc_observation": "per_slot_or_selected_account", + }, + } + + +def _row_to_pool(row: sqlite3.Row, account_count: int = 0, account_names: list[str] | None = None) -> dict[str, Any]: + return { + "id": row["id"], + "name": row["name"], + "source_type": row["source_type"], + "source_uri": row["source_uri"], + "expected_exit_ip": row["expected_exit_ip"], + "region": row["region"], + "local_port": row["local_port"], + "detected_exit_ip": row["detected_exit_ip"], + "detected_country": row["detected_country"], + "detected_region": row["detected_region"], + "detected_city": row["detected_city"], + "detected_address": row["detected_address"], + "detected_at": row["detected_at"], + "status": _clean_status(row["status"]), + "notes": row["notes"], + "parse_status": row["parse_status"], + "parse_error": row["parse_error"], + "mihomo_name": row["mihomo_name"], + "parsed": _json_loads(row["parsed_json"], {}), + "mihomo_proxy": _json_loads(row["mihomo_proxy_json"], {}), + "account_count": account_count, + "account_names": account_names or [], + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + +def _row_to_account(row: sqlite3.Row) -> dict[str, Any]: + return { + "id": row["id"], + "username": row["username"], + "display_name": row["display_name"], + "proxy_profile_id": row["proxy_profile_id"], + "status": _clean_account_status(row["status"]), + "profile": _json_loads(row["profile_json"], {}), + "notes": row["notes"], + "last_checked_ip": row["last_checked_ip"], + "last_check_status": row["last_check_status"], + "last_check_at": row["last_check_at"], + "last_login_at": row["last_login_at"], + "last_collect_at": row["last_collect_at"], + "last_error": row["last_error"], + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + +def _row_to_session(row: sqlite3.Row) -> dict[str, Any]: + result = { + "id": row["id"], + "slot": row["slot"], + "proxy_profile_id": row["proxy_profile_id"], + "account_id": row["account_id"], + "username": row["username"], + "status": row["status"], + "channel_url": row["channel_url"], + "pid": row["pid"], + "xvfb_pid": row["xvfb_pid"], + "x11vnc_pid": row["x11vnc_pid"], + "websockify_pid": row["websockify_pid"], + "display": row["display"], + "vnc_port": row["vnc_port"], + "novnc_port": row["novnc_port"], + "profile_key": row["profile_key"], + "user_data_dir": row["user_data_dir"], + "last_error": row["last_error"], + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + if not row["account_id"] and row["status"] in {"starting", "running", "observing"}: + created_at = _iso_epoch(str(row["created_at"] or "")) + expires_at = created_at + pending_login_ttl_seconds() + result["expires_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(expires_at)) + result["expires_in_seconds"] = max(0, int(expires_at - time.time())) + return result + + +def _pid_alive(pid: int) -> bool: + if pid <= 0: + return False + stat_path = Path(f"/proc/{pid}/stat") + if stat_path.exists(): + try: + parts = stat_path.read_text(encoding="utf-8", errors="ignore").split() + if len(parts) > 2 and parts[2] == "Z": + return False + except OSError: + pass + try: + os.kill(pid, 0) + return True + except OSError: + return False + + +def _terminate_pid(pid: int) -> None: + if pid <= 0: + return + for sig in (signal.SIGTERM, signal.SIGKILL): + if not _pid_alive(pid): + break + try: + os.killpg(pid, sig) + except OSError: + try: + os.kill(pid, sig) + except OSError: + pass + time.sleep(0.3) + for _ in range(10): + try: + waited, _status = os.waitpid(pid, os.WNOHANG) + if waited: + return + except (ChildProcessError, OSError): + return + time.sleep(0.1) + + +def _terminate_session_processes(row: sqlite3.Row | dict[str, Any]) -> None: + for key in ("pid", "websockify_pid", "x11vnc_pid", "xvfb_pid"): + try: + pid = int(row[key] or 0) + except Exception: + pid = 0 + if pid: + _terminate_pid(pid) + + +def _remove_unbound_session_profile(row: sqlite3.Row | dict[str, Any]) -> None: + try: + if int(row["account_id"] or 0): + return + user_data_value = str(row["user_data_dir"] or "") + except Exception: + return + if not user_data_value: + return + profiles_root = (DATA_DIR / "tiktok_browser_profiles").resolve() + profile_root = Path(user_data_value).resolve().parent + if profile_root == profiles_root or profiles_root not in profile_root.parents: + return + shutil.rmtree(profile_root, ignore_errors=True) + + +def _iso_epoch(value: str) -> float: + try: + return float(calendar.timegm(time.strptime(value, "%Y-%m-%dT%H:%M:%SZ"))) + except (TypeError, ValueError): + return 0.0 + + +def _active_sessions(conn: sqlite3.Connection) -> list[sqlite3.Row]: + rows = conn.execute("SELECT * FROM browser_sessions WHERE status IN ('starting','running','observing') ORDER BY updated_at DESC").fetchall() + active = [] + now = now_iso() + for row in rows: + pid = int(row["pid"] or 0) + pending_expired = not row["account_id"] and _iso_epoch(str(row["created_at"] or "")) + pending_login_ttl_seconds() <= time.time() + if pending_expired: + _terminate_session_processes(row) + _remove_unbound_session_profile(row) + conn.execute("UPDATE browser_sessions SET status = 'stopped', last_error = ?, updated_at = ? WHERE id = ?", ("未完成账号登记，临时登录通道超时自动释放", now, row["id"])) + elif pid and not _pid_alive(pid): + _terminate_session_processes(row) + _remove_unbound_session_profile(row) + conn.execute("UPDATE browser_sessions SET status = 'stopped', last_error = COALESCE(NULLIF(last_error, ''), 'browser process exited'), updated_at = ? WHERE id = ?", (now, row["id"])) + else: + active.append(row) + conn.commit() + return active + + +def cleanup_expired_sessions() -> int: + with connect() as conn: + before = conn.execute("SELECT COUNT(*) AS count FROM browser_sessions WHERE status IN ('starting','running','observing')").fetchone()["count"] + active = _active_sessions(conn) + return max(0, int(before) - len(active)) + + +def _allocate_session_slot(conn: sqlite3.Connection) -> int: + max_slots = browser_max_slots() + used = {int(row["slot"] or 0) for row in _active_sessions(conn)} + for slot in range(1, max_slots + 1): + if slot not in used: + return slot + raise ValueError(f"浏览器观测槽位已满，当前最多同时运行 {max_slots} 个") + + +def _browser_binary() -> str: + configured = os.getenv("TIKTOK_BROWSER_BIN", "").strip() + candidates = [configured] if configured else [] + candidates.extend(["google-chrome-stable", "google-chrome", "chromium-browser", "chromium"]) + for item in candidates: + if not item: + continue + found = shutil.which(item) + if found: + return found + if Path(item).exists(): + return item + for root in ( + Path.home() / ".cache" / "ms-playwright", + Path("/root/.cache/ms-playwright"), + Path("/ms-playwright"), + ): + if root.exists(): + for pattern in ("chromium-*/chrome-linux64/chrome", "chromium-*/chrome-linux/chrome"): + for chrome in root.glob(pattern): + return str(chrome) + raise ValueError("服务器未找到 Chromium/Chrome；请配置 TIKTOK_BROWSER_BIN 或安装浏览器") + + +def _required_binary(name: str) -> str: + found = shutil.which(name) + if found: + return found + raise ValueError(f"服务器未找到 {name}；请重建镜像或安装 noVNC 隔离依赖") + + +def _novnc_web_dir() -> str: + configured = os.getenv("NOVNC_WEB_DIR", "").strip() + candidates = [configured] if configured else [] + candidates.extend(["/usr/share/novnc", "/usr/local/share/novnc"]) + for item in candidates: + if item and Path(item).exists(): + return item + raise ValueError("服务器未找到 noVNC Web 目录；请安装 novnc") + + +def _slot_ports(slot: int) -> dict[str, Any]: + manual_ports = max(1, NOVNC_MANUAL_PORTS) + return { + "display": f":{XVFB_DISPLAY_BASE + slot}", + "vnc_port": VNC_PORT + manual_ports + slot - 1, + "novnc_port": NOVNC_PORT + manual_ports + slot - 1, + } + + +def _public_novnc_url(port: int) -> str: + parsed = urlparse(DEFAULT_NOVNC_PUBLIC_URL) + host = parsed.hostname or "192.168.1.254" + scheme = parsed.scheme or "http" + return f"{scheme}://{host}:{port}/vnc.html?autoconnect=1&resize=scale" + + +def _wait_for_port(port: int, label: str, timeout: float = 8.0) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + if _port_open("127.0.0.1", port, timeout=0.4): + return + time.sleep(0.2) + raise ValueError(f"{label} 端口 {port} 未在 {timeout:.0f}s 内启动") + + +def _open_process( + log_dir: Path, + name: str, + args: list[str], + env: dict[str, str] | None = None, + user: int | None = None, + group: int | None = None, +) -> subprocess.Popen: + stdout = open(log_dir / f"{name}.log", "ab") + stderr = open(log_dir / f"{name}.err.log", "ab") + return subprocess.Popen( + args, + cwd=str(ROOT), + env=env or os.environ.copy(), + stdout=stdout, + stderr=stderr, + close_fds=True, + start_new_session=True, + user=user, + group=group, + ) + + +def _launch_observation_channel(slot: int, session_id: int, log_dir: Path) -> dict[str, Any]: + ports = _slot_ports(slot) + display = str(ports["display"]) + vnc_port = int(ports["vnc_port"]) + novnc_port = int(ports["novnc_port"]) + xvfb = _required_binary("Xvfb") + x11vnc = _required_binary("x11vnc") + websockify = _required_binary("websockify") + novnc_web = _novnc_web_dir() + + xvfb_proc = _open_process(log_dir, "xvfb", [xvfb, display, "-screen", "0", "1280x900x24", "-nolisten", "tcp"]) + time.sleep(0.8) + if not _pid_alive(int(xvfb_proc.pid)): + raise ValueError("独立 Xvfb 显示通道启动失败") + + x11vnc_proc = _open_process( + log_dir, + "x11vnc", + [x11vnc, "-display", display, "-rfbport", str(vnc_port), "-localhost", "-forever", "-shared", "-nopw", "-quiet"], + ) + try: + _wait_for_port(vnc_port, "VNC") + websockify_proc = _open_process( + log_dir, + "websockify", + [websockify, "--web", novnc_web, str(novnc_port), f"127.0.0.1:{vnc_port}"], + ) + _wait_for_port(novnc_port, "noVNC") + except Exception: + _terminate_pid(int(x11vnc_proc.pid)) + _terminate_pid(int(xvfb_proc.pid)) + raise + + return { + "display": display, + "vnc_port": vnc_port, + "novnc_port": novnc_port, + "channel_url": _public_novnc_url(novnc_port), + "xvfb_pid": int(xvfb_proc.pid), + "x11vnc_pid": int(x11vnc_proc.pid), + "websockify_pid": int(websockify_proc.pid), + } + + +def _abs_workspace_path(value: str) -> Path: + path = Path(value) + if not path.is_absolute(): + path = ROOT / path + path.mkdir(parents=True, exist_ok=True) + return path + + +def _prepare_browser_profile_dir(user_data_dir: Path) -> dict[str, Path]: + profiles_root = (DATA_DIR / "tiktok_browser_profiles").resolve() + profile_root = user_data_dir.parent.resolve() + if profile_root != profiles_root and profiles_root not in profile_root.parents: + raise ValueError("浏览器 profile 必须位于 data/tiktok_browser_profiles 目录") + + paths = { + "home": profile_root / "home", + "config": profile_root / "config", + "cache": profile_root / "cache", + "downloads": profile_root / "downloads", + "runtime": profile_root / "runtime", + "user_data": user_data_dir, + } + for path in paths.values(): + path.mkdir(parents=True, exist_ok=True) + def chown_path(path: Path | str) -> None: + try: + os.chown(path, TIKTOK_BROWSER_UID, TIKTOK_BROWSER_GID, follow_symlinks=False) + except FileNotFoundError: + pass + + for current_root, dirs, files in os.walk(profile_root, followlinks=False): + chown_path(current_root) + for name in dirs: + chown_path(Path(current_root) / name) + for name in files: + chown_path(Path(current_root) / name) + paths["runtime"].chmod(0o700) + return paths + + +def _configure_browser_preferences(user_data_dir: Path) -> None: + preferences_path = user_data_dir / "Default" / "Preferences" + preferences_path.parent.mkdir(parents=True, exist_ok=True) + preferences = _json_loads(preferences_path.read_text(encoding="utf-8") if preferences_path.is_file() else "", {}) + if not isinstance(preferences, dict): + preferences = {} + if not isinstance(preferences.get("intl"), dict): + preferences["intl"] = {} + if not isinstance(preferences.get("translate"), dict): + preferences["translate"] = {} + preferences["intl"]["accept_languages"] = TIKTOK_BROWSER_ACCEPT_LANGUAGE + preferences["translate"]["enabled"] = False + preferences_path.write_text(json.dumps(preferences, ensure_ascii=False, separators=(",", ":")), encoding="utf-8") + + +def _decrypt_chrome_cookie(host: str, value: str, encrypted_value: bytes) -> str: + if value: + return value + encrypted = bytes(encrypted_value or b"") + if not encrypted.startswith((b"v10", b"v11")): + return "" + try: + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + key = hashlib.pbkdf2_hmac("sha1", b"peanuts", b"saltysalt", 1, 16) + decryptor = Cipher(algorithms.AES(key), modes.CBC(b" " * 16)).decryptor() + plain = decryptor.update(encrypted[3:]) + decryptor.finalize() + padding = plain[-1] + if not 1 <= padding <= 16: + return "" + plain = plain[:-padding] + host_digest = hashlib.sha256(host.encode("utf-8")).digest() + if plain.startswith(host_digest): + plain = plain[len(host_digest):] + return plain.decode("utf-8", errors="strict") + except Exception: + return "" + + +def _tiktok_profile_cookies(user_data_dir: str) -> dict[str, str]: + root = Path(user_data_dir) + candidates = (root / "Default" / "Cookies", root / "Default" / "Network" / "Cookies") + for cookie_path in candidates: + if not cookie_path.is_file(): + continue + try: + cookie_conn = sqlite3.connect(f"file:{cookie_path}?mode=ro", uri=True, timeout=1) + try: + rows = cookie_conn.execute( + "SELECT host_key, name, value, encrypted_value FROM cookies WHERE host_key LIKE '%tiktok%'" + ).fetchall() + finally: + cookie_conn.close() + except sqlite3.Error: + continue + cookies: dict[str, str] = {} + for host, name, value, encrypted_value in rows: + decoded = _decrypt_chrome_cookie(str(host), str(value or ""), encrypted_value) + if decoded: + cookies[str(name)] = decoded + return cookies + return {} + + +def _proxy_json_with_cookies(url: str, proxy_port: int, cookies: dict[str, str], timeout: float = 10.0) -> tuple[bool, Any, str]: + proxy_url = f"http://127.0.0.1:{proxy_port}" + opener = build_opener(ProxyHandler({"http": proxy_url, "https": proxy_url})) + request = Request( + url, + headers={ + "Accept": "application/json, text/plain, */*", + "Accept-Language": TIKTOK_BROWSER_ACCEPT_LANGUAGE, + "Cookie": "; ".join(f"{name}={value}" for name, value in cookies.items()), + "Referer": "https://www.tiktok.com/", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36", + }, + ) + try: + with opener.open(request, timeout=timeout) as response: + raw = response.read(1024 * 1024).decode("utf-8", errors="replace") + return True, json.loads(raw), "" + except HTTPError as exc: + return False, None, f"HTTP {exc.code}" + except (URLError, TimeoutError, json.JSONDecodeError, OSError) as exc: + return False, None, str(exc) + + +def _tiktok_identity(body: Any) -> dict[str, str]: + if not isinstance(body, dict): + return {} + candidates: list[dict[str, Any]] = [] + for item in (body.get("data"), body.get("user"), body): + if isinstance(item, dict): + candidates.append(item) + if isinstance(item.get("user"), dict): + candidates.append(item["user"]) + for item in candidates: + username = str(item.get("username") or item.get("unique_id") or item.get("uniqueId") or "").strip().lstrip("@") + user_id = str(item.get("user_id") or item.get("user_id_str") or item.get("uid") or "").strip() + display_name = str(item.get("screen_name") or item.get("nickname") or item.get("display_name") or "").strip() + if username or user_id: + return { + "username": username or f"uid_{user_id}", + "display_name": display_name, + "user_id": user_id, + } + return {} + + +def _launch_browser_for_session(profile: dict[str, Any], pool: sqlite3.Row, session_id: int, display: str) -> tuple[int, str]: + isolation = profile.get("isolation") if isinstance(profile.get("isolation"), dict) else {} + user_data_dir = _abs_workspace_path(str(isolation.get("user_data_dir") or f"data/tiktok_browser_profiles/session-{session_id}/user-data")) + _configure_browser_preferences(user_data_dir) + profile_paths = _prepare_browser_profile_dir(user_data_dir) + log_dir = _abs_workspace_path(f"data/tiktok_browser_sessions/{session_id}") + browser = _browser_binary() + proxy_port = int(pool["local_port"] or 0) + if not proxy_port: + raise ValueError("代理没有专用本地端口，不能启动独立浏览器") + args = [ + browser, + f"--user-data-dir={user_data_dir}", + f"--proxy-server=http://127.0.0.1:{proxy_port}", + f"--lang={TIKTOK_BROWSER_LOCALE}", + "--no-first-run", + "--no-default-browser-check", + "--disable-sync", + "--disable-translate", + "--disable-background-networking", + "--disable-default-apps", + "--force-webrtc-ip-handling-policy=disable_non_proxied_udp", + "--disable-session-crashed-bubble", + "--window-size=1280,900", + "--new-window", + "https://www.tiktok.com/login", + ] + env = os.environ.copy() + env["DISPLAY"] = display + env["HOME"] = str(profile_paths["home"]) + env["XDG_CONFIG_HOME"] = str(profile_paths["config"]) + env["XDG_CACHE_HOME"] = str(profile_paths["cache"]) + env["XDG_RUNTIME_DIR"] = str(profile_paths["runtime"]) + env["LANGUAGE"] = "en_US:en" + env["LANG"] = "C.UTF-8" + env["LC_ALL"] = "C.UTF-8" + proc = _open_process( + log_dir, + "browser", + args, + env=env, + user=TIKTOK_BROWSER_UID, + group=TIKTOK_BROWSER_GID, + ) + return int(proc.pid), str(user_data_dir) + + +def parse_vless_uri(uri: str, fallback_name: str = "") -> dict[str, Any]: + uri = _clean_text(uri, 10000) + if not uri: + return {"parse_status": "manual", "parsed": {}, "mihomo_proxy": {}, "mihomo_name": fallback_name} + if not uri.startswith("vless://"): + raise ValueError("Only vless:// URI is supported") + + parsed = urlparse(uri) + query = {key: values[-1] for key, values in parse_qs(parsed.query).items() if values} + if not parsed.username and not parsed.port: + encoded = (parsed.netloc + parsed.path).strip("/") + try: + padded = encoded + "=" * (-len(encoded) % 4) + decoded = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8", errors="replace") + if "@" in decoded: + userinfo, hostinfo = decoded.rsplit("@", 1) + uuid_part = userinfo.split(":", 1)[-1] + rebuilt = f"vless://{uuid_part}@{hostinfo}" + parsed = urlparse(rebuilt) + except (binascii.Error, UnicodeError, ValueError): + pass + uuid = unquote(parsed.username or "") + server = parsed.hostname or "" + port = parsed.port + name = unquote(parsed.fragment or "") or unquote(query.get("remarks") or query.get("remark") or "") or fallback_name or server or "vless-node" + if not uuid or not server or not port: + raise ValueError("VLESS URI must include uuid, server and port") + + network = query.get("type") or query.get("network") or "tcp" + security = query.get("security", "") + tls_enabled = security in {"tls", "reality"} or query.get("tls") in {"1", "true", "tls"} + reality_enabled = security == "reality" or bool(query.get("pbk")) + mihomo: dict[str, Any] = { + "name": name, + "type": "vless", + "server": server, + "port": int(port), + "uuid": uuid, + "network": network, + "udp": True, + } + if query.get("flow"): + mihomo["flow"] = query["flow"] + if tls_enabled or reality_enabled: + mihomo["tls"] = True + if reality_enabled: + mihomo["reality-opts"] = {} + if query.get("pbk"): + mihomo["reality-opts"]["public-key"] = query["pbk"] + if query.get("sid"): + mihomo["reality-opts"]["short-id"] = query["sid"] + if query.get("sni") or query.get("peer"): + mihomo["servername"] = query.get("sni") or query.get("peer") + if query.get("fp"): + mihomo["client-fingerprint"] = query["fp"] + elif reality_enabled: + mihomo["client-fingerprint"] = "chrome" + if network == "ws": + ws_opts: dict[str, Any] = {} + if query.get("path"): + ws_opts["path"] = query["path"] + if query.get("host"): + ws_opts["headers"] = {"Host": query["host"]} + if ws_opts: + mihomo["ws-opts"] = ws_opts + if network == "grpc" and query.get("serviceName"): + mihomo["grpc-opts"] = {"grpc-service-name": query["serviceName"]} + + return { + "parse_status": "ok", + "mihomo_name": name, + "parsed": { + "uuid": uuid, + "server": server, + "port": int(port), + "network": network, + "security": security, + "query": query, + "name": name, + }, + "mihomo_proxy": mihomo, + } + + +def list_state() -> dict[str, Any]: + with connect() as conn: + _active_sessions(conn) + counts = { + int(row["proxy_profile_id"]): int(row["count"]) + for row in conn.execute("SELECT proxy_profile_id, COUNT(*) AS count FROM tiktok_accounts GROUP BY proxy_profile_id") + } + names: dict[int, list[str]] = {} + for row in conn.execute("SELECT proxy_profile_id, username FROM tiktok_accounts ORDER BY username"): + names.setdefault(int(row["proxy_profile_id"]), []).append(str(row["username"])) + pools = [_row_to_pool(row, counts.get(int(row["id"]), 0), names.get(int(row["id"]), [])) for row in conn.execute("SELECT * FROM proxy_profiles ORDER BY updated_at DESC, id DESC")] + accounts = [_row_to_account(row) for row in conn.execute("SELECT * FROM tiktok_accounts ORDER BY updated_at DESC, id DESC")] + sessions = [_row_to_session(row) for row in conn.execute("SELECT * FROM browser_sessions ORDER BY updated_at DESC, id DESC LIMIT 20")] + return { + "pools": pools, + "accounts": accounts, + "sessions": sessions, + "stats": { + "pool_count": len(pools), + "account_count": len(accounts), + "blocked_accounts": sum(1 for item in accounts if item["last_check_status"] in {"阻断", "blocked"}), + }, + } + + +def upsert_pool(payload: dict[str, Any]) -> dict[str, Any]: + pool_id = int(payload.get("id") or 0) + name = _clean_text(payload.get("name"), 160) + source_uri = _clean_text(payload.get("source_uri"), 10000) + expected_exit_ip = _clean_text(payload.get("expected_exit_ip"), 80) + + parse_status = "manual" + parse_error = "" + parsed: dict[str, Any] = {} + mihomo_proxy: dict[str, Any] = {} + mihomo_name = name + if source_uri: + try: + parsed_result = parse_vless_uri(source_uri, fallback_name=name) + parse_status = str(parsed_result["parse_status"]) + parsed = parsed_result["parsed"] + mihomo_proxy = parsed_result["mihomo_proxy"] + mihomo_name = str(parsed_result.get("mihomo_name") or name) + except Exception as exc: + parse_status = "error" + parse_error = str(exc) + if not name: + name = mihomo_name or expected_exit_ip + if mihomo_proxy and not mihomo_proxy.get("name"): + mihomo_proxy["name"] = name + + now = now_iso() + values = { + "name": name, + "source_type": _clean_text(payload.get("source_type"), 40) or "vless", + "source_uri": source_uri, + "expected_exit_ip": expected_exit_ip, + "region": _clean_text(payload.get("region"), 80), + "status": _clean_status(payload.get("status")), + "notes": _clean_text(payload.get("notes"), 2000), + "parse_status": parse_status, + "parse_error": parse_error, + "mihomo_name": mihomo_name or name, + "parsed_json": json.dumps(parsed, ensure_ascii=False, separators=(",", ":")), + "mihomo_proxy_json": json.dumps(mihomo_proxy, ensure_ascii=False, separators=(",", ":")), + "updated_at": now, + } + with connect() as conn: + if pool_id: + exists = conn.execute("SELECT id FROM proxy_profiles WHERE id = ?", (pool_id,)).fetchone() + if not exists: + raise ValueError("proxy profile not found") + values["local_port"] = _allocate_port(conn, pool_id) + conn.execute( + """ + UPDATE proxy_profiles + SET name=:name, source_type=:source_type, source_uri=:source_uri, + expected_exit_ip=:expected_exit_ip, region=:region, status=:status, local_port=:local_port, + notes=:notes, parse_status=:parse_status, parse_error=:parse_error, + mihomo_name=:mihomo_name, parsed_json=:parsed_json, + mihomo_proxy_json=:mihomo_proxy_json, updated_at=:updated_at + WHERE id=:id + """, + {**values, "id": pool_id}, + ) + else: + values["local_port"] = _allocate_port(conn, 0) + cur = conn.execute( + """ + INSERT INTO proxy_profiles ( + name, source_type, source_uri, expected_exit_ip, region, status, notes, local_port, + parse_status, parse_error, mihomo_name, parsed_json, mihomo_proxy_json, + created_at, updated_at + ) VALUES ( + :name, :source_type, :source_uri, :expected_exit_ip, :region, :status, :notes, :local_port, + :parse_status, :parse_error, :mihomo_name, :parsed_json, :mihomo_proxy_json, + :created_at, :updated_at + ) + """, + {**values, "created_at": now}, + ) + pool_id = int(cur.lastrowid) + conn.commit() + return {"pool": get_pool(pool_id), **list_state()} + + +def get_pool(pool_id: int) -> dict[str, Any]: + with connect() as conn: + row = conn.execute("SELECT * FROM proxy_profiles WHERE id = ?", (pool_id,)).fetchone() + if not row: + raise ValueError("proxy profile not found") + count = conn.execute("SELECT COUNT(*) AS count FROM tiktok_accounts WHERE proxy_profile_id = ?", (pool_id,)).fetchone()["count"] + names = [str(item["username"]) for item in conn.execute("SELECT username FROM tiktok_accounts WHERE proxy_profile_id = ? ORDER BY username", (pool_id,))] + return _row_to_pool(row, int(count), names) + + +def delete_pool(pool_id: int) -> dict[str, Any]: + with connect() as conn: + count = conn.execute("SELECT COUNT(*) AS count FROM tiktok_accounts WHERE proxy_profile_id = ?", (pool_id,)).fetchone()["count"] + if int(count): + raise ValueError("Cannot delete a proxy profile with bound accounts") + conn.execute("DELETE FROM proxy_profiles WHERE id = ?", (pool_id,)) + conn.commit() + return list_state() + + +def upsert_account(payload: dict[str, Any]) -> dict[str, Any]: + account_id = int(payload.get("id") or 0) + username = _normal_username(payload.get("username")) + proxy_profile_id = int(payload.get("proxy_profile_id") or 0) + if not proxy_profile_id: + raise ValueError("proxy_profile_id is required") + profile_provided = "profile" in payload + profile = payload.get("profile", {}) + if isinstance(profile, str): + profile = _json_loads(profile, {}) + if not isinstance(profile, dict): + raise ValueError("profile must be a JSON object") + + session_id = int(payload.get("session_id") or 0) + now = now_iso() + values = { + "username": username, + "display_name": _clean_text(payload.get("display_name"), 160), + "proxy_profile_id": proxy_profile_id, + "status": _clean_account_status(payload.get("status")), + "profile_json": "{}", + "notes": _clean_text(payload.get("notes"), 2000), + "updated_at": now, + } + with connect() as conn: + pool_row = conn.execute("SELECT * FROM proxy_profiles WHERE id = ?", (proxy_profile_id,)).fetchone() + if not pool_row: + raise ValueError("proxy profile not found") + profile = _deep_merge(_isolation_profile(username, proxy_profile_id, pool_row), profile) + if session_id: + session_row = _session_by_id(conn, session_id) + if int(session_row["proxy_profile_id"] or 0) != proxy_profile_id: + raise ValueError("登录会话与账号绑定代理不一致") + if session_row["status"] not in {"starting", "running", "observing"}: + raise ValueError("登录会话已经结束，不能保存为账号 profile") + session_user_data_value = str(session_row["user_data_dir"] or "") + if not session_user_data_value: + raise ValueError("登录会话没有浏览器 profile 路径") + session_user_data = Path(session_user_data_value) + session_root = session_user_data.parent + profile["isolation"] = { + **(profile.get("isolation") if isinstance(profile.get("isolation"), dict) else {}), + "browser_profile_key": str(session_row["profile_key"] or f"tiktok-{_safe_profile_key(username)}"), + "user_data_dir": str(session_user_data), + "cookie_store_dir": str(session_root / "cookies"), + "session_dir": str(session_root / "session"), + "cache_dir": str(session_root / "cache"), + "download_dir": str(session_root / "downloads"), + } + profile["observation_session"] = {"session_id": session_id, "bound_at": now, "persisted": True} + values["profile_json"] = json.dumps(profile, ensure_ascii=False, separators=(",", ":")) + if account_id: + existing_account = conn.execute("SELECT profile_json FROM tiktok_accounts WHERE id = ?", (account_id,)).fetchone() + if not existing_account: + raise ValueError("account not found") + existing_profile = _json_loads(existing_account["profile_json"], {}) + if not profile_provided: + values["profile_json"] = json.dumps(_deep_merge(_isolation_profile(username, proxy_profile_id, pool_row), existing_profile), ensure_ascii=False, separators=(",", ":")) + elif not session_id and isinstance(existing_profile.get("isolation"), dict): + profile["isolation"] = existing_profile["isolation"] + values["profile_json"] = json.dumps(profile, ensure_ascii=False, separators=(",", ":")) + conn.execute( + """ + UPDATE tiktok_accounts + SET username=:username, display_name=:display_name, proxy_profile_id=:proxy_profile_id, + status=:status, profile_json=:profile_json, notes=:notes, updated_at=:updated_at + WHERE id=:id + """, + {**values, "id": account_id}, + ) + else: + cur = conn.execute( + """ + INSERT INTO tiktok_accounts ( + username, display_name, proxy_profile_id, status, profile_json, notes, + created_at, updated_at + ) VALUES ( + :username, :display_name, :proxy_profile_id, :status, :profile_json, :notes, + :created_at, :updated_at + ) + """, + {**values, "created_at": now}, + ) + account_id = int(cur.lastrowid) + if session_id: + conn.execute("UPDATE browser_sessions SET account_id = ?, username = ?, updated_at = ? WHERE id = ?", (account_id, username, now, session_id)) + conn.commit() + return {"account": get_account(account_id), **list_state()} + + +def get_account(account_id: int) -> dict[str, Any]: + with connect() as conn: + row = conn.execute("SELECT * FROM tiktok_accounts WHERE id = ?", (account_id,)).fetchone() + if not row: + raise ValueError("account not found") + return _row_to_account(row) + + +def delete_account(account_id: int) -> dict[str, Any]: + with connect() as conn: + conn.execute("DELETE FROM tiktok_accounts WHERE id = ?", (account_id,)) + conn.commit() + return list_state() + + +def _pool_for_check(conn: sqlite3.Connection, payload: dict[str, Any]) -> tuple[sqlite3.Row | None, sqlite3.Row | None]: + account = None + if payload.get("account_id"): + account = conn.execute("SELECT * FROM tiktok_accounts WHERE id = ?", (int(payload["account_id"]),)).fetchone() + if not account: + raise ValueError("account not found") + pool_id = int(account["proxy_profile_id"]) + elif payload.get("username"): + account = conn.execute("SELECT * FROM tiktok_accounts WHERE username = ?", (_normal_username(payload["username"]),)).fetchone() + if not account: + raise ValueError("account not found") + pool_id = int(account["proxy_profile_id"]) + else: + pool_id = int(payload.get("proxy_profile_id") or payload.get("pool_id") or 0) + if not pool_id: + raise ValueError("proxy profile is required") + pool = conn.execute("SELECT * FROM proxy_profiles WHERE id = ?", (pool_id,)).fetchone() + if not pool: + raise ValueError("proxy profile not found") + return pool, account + + + +def _mihomo_headers() -> dict[str, str]: + secret = os.getenv("MIHOMO_SECRET", "").strip() + return {"Authorization": f"Bearer {secret}"} if secret else {} + + +def _mihomo_request(method: str, path: str, body: dict[str, Any] | None = None, timeout: float = 5.0) -> tuple[bool, Any, str]: + parsed = urlparse(DEFAULT_MIHOMO_API.rstrip("/")) + conn = http.client.HTTPConnection(parsed.hostname or "127.0.0.1", parsed.port or 9090, timeout=timeout) + headers = _mihomo_headers() + payload = None + if body is not None: + headers["Content-Type"] = "application/json" + payload = json.dumps(body, ensure_ascii=False).encode("utf-8") + try: + conn.request(method, path, body=payload, headers=headers) + response = conn.getresponse() + text = response.read(1024 * 1024).decode("utf-8", errors="replace") + if not (200 <= response.status < 300): + return False, None, f"HTTP {response.status}: {text[:300]}" + return True, json.loads(text) if text else {}, "" + except Exception as exc: + return False, None, str(exc) + finally: + conn.close() + + +def _switch_mihomo_node(node_name: str) -> dict[str, Any]: + ok, body, error = _mihomo_request("GET", "/proxies") + if not ok or not isinstance(body, dict): + raise ValueError(f"无法读取服务器 mihomo 节点：{error}") + proxies = body.get("proxies") if isinstance(body.get("proxies"), dict) else {} + if node_name not in proxies: + raise ValueError(f"节点 {node_name} 没有加载到服务器 mihomo；请先把导出的配置导入 mihomo 并重载") + preferred = ["GLOBAL", "Proxy", "代理", "CoffeeCloud", "自动选择"] + candidates = [] + for name, item in proxies.items(): + all_nodes = item.get("all") if isinstance(item, dict) else None + if isinstance(all_nodes, list) and node_name in all_nodes: + candidates.append(str(name)) + candidates.sort(key=lambda item: (0 if item in preferred else 1, preferred.index(item) if item in preferred else 999, item)) + switched = [] + for group in candidates[:3]: + ok, _body, error = _mihomo_request("PUT", f"/proxies/{quote_path(group)}", {"name": node_name}) + if ok: + switched.append(group) + if candidates and not switched: + raise ValueError(f"mihomo 找到节点 {node_name}，但切换策略组失败") + return {"node": node_name, "groups": switched, "loaded": True} + + +def quote_path(value: str) -> str: + from urllib.parse import quote + return quote(value, safe="") + + +def _proxy_get_json(url: str, proxy_port: int, timeout: float = 10.0) -> tuple[bool, Any, str]: + parsed = urlparse(url) + conn = http.client.HTTPConnection("127.0.0.1", proxy_port, timeout=timeout) + headers = {"Host": parsed.netloc, "User-Agent": "ShortVideoAnalyzer/1.0"} + try: + conn.request("GET", url, headers=headers) + response = conn.getresponse() + text = response.read(1024 * 1024).decode("utf-8", errors="replace") + if not (200 <= response.status < 300): + return False, None, f"HTTP {response.status}: {text[:300]}" + return True, json.loads(text), "" + except Exception as exc: + return False, None, str(exc) + finally: + conn.close() + + +def detect_exit_ip_for_pool(pool: sqlite3.Row) -> dict[str, Any]: + node_name = str(pool["mihomo_name"] or pool["name"] or "").strip() + if not node_name: + raise ValueError("代理没有 mihomo 节点名") + local_port = int(pool["local_port"] or 0) + if local_port and _port_open("127.0.0.1", local_port, timeout=1.0): + proxy_port = local_port + switch = {"node": node_name, "groups": [], "loaded": True, "listener_port": local_port} + else: + switch = _switch_mihomo_node(node_name) + proxy_port = int(os.getenv("MIHOMO_PROXY_PORT", "7890") or "7890") + target = os.getenv("PROXY_IP_CHECK_URL", "http://ip-api.com/json/?fields=status,country,regionName,city,query") + ok, body, error = _proxy_get_json(target, proxy_port) + if not ok or not isinstance(body, dict): + raise ValueError(f"通过服务器 mihomo 查询出口 IP 失败：{error}") + ip = str(body.get("query") or body.get("ip") or "").strip() + if not ip: + raise ValueError(f"IP 查询接口没有返回出口 IP：{body}") + country = str(body.get("country") or "") + region = str(body.get("regionName") or body.get("region") or "") + city = str(body.get("city") or "") + address = " / ".join(item for item in (country, region, city) if item) + return {"ip": ip, "geo": {"country": country, "region": region, "city": city, "address": address}, "mihomo": switch, "raw": body} + +def check_binding(payload: dict[str, Any], require_account: bool = False) -> dict[str, Any]: + observed_ip = _clean_text(payload.get("observed_ip") or payload.get("current_ip"), 80) + detected: dict[str, Any] = {} + now = now_iso() + with connect() as conn: + pool, account = _pool_for_check(conn, payload) + if require_account and account is None: + raise ValueError("account_id or username is required") + if not observed_ip: + detected = detect_exit_ip_for_pool(pool) + observed_ip = str(detected.get("ip") or "") + if not observed_ip: + raise ValueError("服务器未能自动查询到出口 IP") + expected_ip = str(pool["expected_exit_ip"] or "").strip() + should_bind = str(payload.get("bind") or "").lower() in {"1", "true", "yes", "on"} + if not expected_ip and should_bind: + expected_ip = observed_ip + pool_status = _clean_status(pool["status"]) + next_pool_status = STATUS_ACTIVE if should_bind and expected_ip and pool_status != STATUS_PAUSED else pool_status + allowed = bool(expected_ip and observed_ip == expected_ip and next_pool_status == STATUS_ACTIVE) + reason = "" + if not expected_ip: + reason = "代理还没有绑定出口 IP" + elif observed_ip != expected_ip: + reason = f"当前出口 IP {observed_ip} 与绑定 IP {expected_ip} 不一致" + elif next_pool_status != STATUS_ACTIVE: + reason = f"代理状态为 {next_pool_status}" + else: + reason = "通过" + geo = detected.get("geo") or lookup_ip_geo(observed_ip) + conn.execute(""" + UPDATE proxy_profiles + SET expected_exit_ip = ?, detected_exit_ip = ?, detected_country = ?, + detected_region = ?, detected_city = ?, detected_address = ?, detected_at = ?, + status = ?, region = COALESCE(NULLIF(?, ''), region), updated_at = ? + WHERE id = ? + """, (expected_ip, observed_ip, geo.get("country", ""), geo.get("region", ""), geo.get("city", ""), geo.get("address", ""), now, next_pool_status, geo.get("region", ""), now, pool["id"])) + if account is not None: + conn.execute( + """ + UPDATE tiktok_accounts + SET last_checked_ip = ?, last_check_status = ?, last_check_at = ?, + last_error = ?, updated_at = ? + WHERE id = ? + """, + (observed_ip, "通过" if allowed else "阻断", now, "" if allowed else reason, now, account["id"]), + ) + conn.commit() + pool = conn.execute("SELECT * FROM proxy_profiles WHERE id = ?", (pool["id"],)).fetchone() + if account is not None: + account = conn.execute("SELECT * FROM tiktok_accounts WHERE id = ?", (account["id"],)).fetchone() + return { + "allowed": allowed, + "reason": reason, + "observed_ip": observed_ip, + "expected_exit_ip": expected_ip, + "checked_at": now, + "pool": _row_to_pool(pool), + "account": _row_to_account(account) if account is not None else None, + "detected": detected, + } + + +def _session_by_id(conn: sqlite3.Connection, session_id: int) -> sqlite3.Row: + row = conn.execute("SELECT * FROM browser_sessions WHERE id = ?", (session_id,)).fetchone() + if not row: + raise ValueError("browser session not found") + return row + + +def start_login_session(payload: dict[str, Any]) -> dict[str, Any]: + account_id = int(payload.get("account_id") or 0) + proxy_profile_id = int(payload.get("proxy_profile_id") or payload.get("pool_id") or 0) + saved_profile: dict[str, Any] = {} + if account_id: + with connect() as conn: + account_row = conn.execute("SELECT * FROM tiktok_accounts WHERE id = ?", (account_id,)).fetchone() + if not account_row: + raise ValueError("account not found") + bound_proxy_id = int(account_row["proxy_profile_id"] or 0) + if proxy_profile_id and proxy_profile_id != bound_proxy_id: + raise ValueError("账号与请求代理不一致") + proxy_profile_id = bound_proxy_id + username = str(account_row["username"] or "") + saved_profile = _json_loads(account_row["profile_json"], {}) + else: + username = _clean_text(payload.get("username"), 120).lstrip("@") + if not proxy_profile_id: + raise ValueError("proxy_profile_id is required") + try: + preflight_payload = {"account_id": account_id} if account_id else {"proxy_profile_id": proxy_profile_id, "bind": True} + preflight = check_binding(preflight_payload, require_account=bool(account_id)) + except Exception as exc: + with connect() as conn: + conn.execute("UPDATE proxy_profiles SET status = ?, parse_error = COALESCE(NULLIF(parse_error, ''), ?), updated_at = ? WHERE id = ?", (STATUS_ERROR, str(exc), now_iso(), proxy_profile_id)) + conn.commit() + raise + if not preflight.get("allowed"): + reason = str(preflight.get("reason") or "代理 IP 校验未通过") + with connect() as conn: + conn.execute("UPDATE proxy_profiles SET status = ?, parse_error = ?, updated_at = ? WHERE id = ?", (STATUS_ERROR, reason, now_iso(), proxy_profile_id)) + conn.commit() + raise ValueError(reason) + now = now_iso() + with connect() as conn: + pool = conn.execute("SELECT * FROM proxy_profiles WHERE id = ?", (proxy_profile_id,)).fetchone() + if not pool: + raise ValueError("proxy profile not found") + if _clean_status(pool["status"]) != STATUS_ACTIVE: + raise ValueError(f"代理状态为 {_clean_status(pool['status'])}") + if account_id: + for active_row in _active_sessions(conn): + if int(active_row["account_id"] or 0) == account_id: + raise ValueError("账号已经处于唤醒状态") + slot = _allocate_session_slot(conn) + pending_name = f"pending-{proxy_profile_id}-{slot}-{int(time.time())}" if not username else username + profile = _deep_merge(_isolation_profile(pending_name, proxy_profile_id, pool), saved_profile) if account_id else _isolation_profile(pending_name, proxy_profile_id, pool) + profile_key = str((profile.get("isolation") or {}).get("browser_profile_key") or "") + slot_ports = _slot_ports(slot) + channel_url = _public_novnc_url(int(slot_ports["novnc_port"])) + cur = conn.execute( + """ + INSERT INTO browser_sessions ( + slot, proxy_profile_id, account_id, username, status, channel_url, + pid, xvfb_pid, x11vnc_pid, websockify_pid, display, vnc_port, novnc_port, + profile_key, user_data_dir, last_error, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, 0, ?, ?, ?, ?, ?, '', ?, ?) + """, + ( + slot, + proxy_profile_id, + account_id or None, + username, + "starting", + channel_url, + str(slot_ports["display"]), + int(slot_ports["vnc_port"]), + int(slot_ports["novnc_port"]), + profile_key, + str((profile.get("isolation") or {}).get("user_data_dir") or ""), + now, + now, + ), + ) + session_id = int(cur.lastrowid) + conn.commit() + try: + log_dir = _abs_workspace_path(f"data/tiktok_browser_sessions/{session_id}") + channel = _launch_observation_channel(slot, session_id, log_dir) + pid, user_data_dir = _launch_browser_for_session(profile, pool, session_id, str(channel["display"])) + time.sleep(2.0) + if not _pid_alive(pid): + raise ValueError("Chrome 启动后立即退出，请检查 browser.err.log") + conn.execute( + """ + UPDATE browser_sessions + SET status = 'observing', + channel_url = ?, + pid = ?, + xvfb_pid = ?, + x11vnc_pid = ?, + websockify_pid = ?, + display = ?, + vnc_port = ?, + novnc_port = ?, + user_data_dir = ?, + updated_at = ? + WHERE id = ? + """, + ( + str(channel["channel_url"]), + pid, + int(channel["xvfb_pid"]), + int(channel["x11vnc_pid"]), + int(channel["websockify_pid"]), + str(channel["display"]), + int(channel["vnc_port"]), + int(channel["novnc_port"]), + user_data_dir, + now_iso(), + session_id, + ), + ) + conn.commit() + except Exception as exc: + err = str(exc) + row = conn.execute("SELECT * FROM browser_sessions WHERE id = ?", (session_id,)).fetchone() + if row: + _terminate_session_processes(row) + _remove_unbound_session_profile(row) + conn.execute("UPDATE browser_sessions SET status = 'failed', last_error = ?, updated_at = ? WHERE id = ?", (err, now_iso(), session_id)) + if account_id: + conn.execute("UPDATE tiktok_accounts SET status = ?, last_error = ?, updated_at = ? WHERE id = ?", (ACCOUNT_STATUS_ERROR, err, now_iso(), account_id)) + conn.commit() + raise ValueError(f"观测浏览器启动失败：{err}") + session = _session_by_id(conn, session_id) + return {"session": _row_to_session(session), "preflight": preflight, **list_state()} + + +def stop_login_session(payload: dict[str, Any]) -> dict[str, Any]: + session_id = int(payload.get("session_id") or payload.get("id") or 0) + mark_failed = bool(payload.get("failed") or payload.get("login_failed")) + reason = _clean_text(payload.get("reason") or ("登录失败" if mark_failed else "手动关闭观测通道"), 1000) + if not session_id: + raise ValueError("session_id is required") + now = now_iso() + with connect() as conn: + row = _session_by_id(conn, session_id) + _terminate_session_processes(row) + _remove_unbound_session_profile(row) + status = "failed" if mark_failed else "stopped" + conn.execute("UPDATE browser_sessions SET status = ?, last_error = ?, updated_at = ? WHERE id = ?", (status, reason, now, session_id)) + account_id = int(row["account_id"] or 0) + if account_id and mark_failed: + conn.execute("UPDATE tiktok_accounts SET status = ?, last_error = ?, updated_at = ? WHERE id = ?", (ACCOUNT_STATUS_ERROR, reason, now, account_id)) + conn.commit() + return list_state() + + +def inspect_login_session(payload: dict[str, Any]) -> dict[str, Any]: + session_id = int(payload.get("session_id") or payload.get("id") or 0) + if not session_id: + raise ValueError("session_id is required") + with connect() as conn: + row = _session_by_id(conn, session_id) + if row["status"] not in {"starting", "running", "observing"}: + return {"active": False, "bound": False, "status": row["status"], "reason": str(row["last_error"] or "登录通道已结束")} + if row["account_id"]: + account = conn.execute("SELECT * FROM tiktok_accounts WHERE id = ?", (row["account_id"],)).fetchone() + return {"active": True, "bound": True, "status": "bound", "account": _row_to_account(account) if account else None, **list_state()} + pool = conn.execute("SELECT * FROM proxy_profiles WHERE id = ?", (row["proxy_profile_id"],)).fetchone() + user_data_dir = str(row["user_data_dir"] or "") + if not pool or not user_data_dir: + return {"active": True, "bound": False, "status": "waiting", "reason": "浏览器 profile 尚未就绪"} + + cookies = _tiktok_profile_cookies(user_data_dir) + login_cookie_names = {"sessionid", "sessionid_ss", "sid_tt", "sid_guard"} + if not any(cookies.get(name) for name in login_cookie_names): + return {"active": True, "bound": False, "status": "waiting_login"} + + account_info_url = os.getenv( + "TIKTOK_ACCOUNT_INFO_URL", + "https://www.tiktok.com/passport/web/account/info/?aid=1459&app_language=en&device_platform=web_pc", + ) + ok, body, error = _proxy_json_with_cookies(account_info_url, int(pool["local_port"] or 0), cookies) + identity = _tiktok_identity(body) if ok else {} + if not identity: + return { + "active": True, + "bound": False, + "status": "login_detected", + "reason": error or "已检测到 TikTok 登录 Cookie，正在读取账号身份", + } + + username = _normal_username(identity["username"]) + with connect() as conn: + existing = conn.execute("SELECT * FROM tiktok_accounts WHERE username = ?", (username,)).fetchone() + if existing: + return { + "active": True, + "bound": False, + "status": "duplicate_account", + "reason": f"@{username} 已在账号池中，请关闭此次通道并从账号列表唤醒", + } + result = upsert_account( + { + "username": username, + "display_name": identity.get("display_name", ""), + "proxy_profile_id": int(pool["id"]), + "status": ACCOUNT_STATUS_ACTIVE, + "session_id": session_id, + "notes": "TikTok 登录成功后自动绑定", + } + ) + with connect() as conn: + conn.execute( + "UPDATE tiktok_accounts SET last_login_at = ?, last_error = '', updated_at = ? WHERE id = ?", + (now_iso(), now_iso(), result["account"]["id"]), + ) + conn.commit() + return {"active": True, "bound": True, "status": "bound", **result} + + +def update_account_status(payload: dict[str, Any]) -> dict[str, Any]: + account_id = int(payload.get("account_id") or payload.get("id") or 0) + if not account_id: + raise ValueError("account_id is required") + now = now_iso() + with connect() as conn: + if not conn.execute("SELECT id FROM tiktok_accounts WHERE id = ?", (account_id,)).fetchone(): + raise ValueError("account not found") + conn.execute( + """ + UPDATE tiktok_accounts + SET status = COALESCE(NULLIF(?, ''), status), + last_login_at = COALESCE(NULLIF(?, ''), last_login_at), + last_collect_at = COALESCE(NULLIF(?, ''), last_collect_at), + last_error = COALESCE(NULLIF(?, ''), last_error), + updated_at = ? + WHERE id = ? + """, + ( + _clean_account_status(payload.get("status"), ""), + _clean_text(payload.get("last_login_at"), 80), + _clean_text(payload.get("last_collect_at"), 80), + _clean_text(payload.get("last_error"), 1000), + now, + account_id, + ), + ) + conn.commit() + return {"account": get_account(account_id), **list_state()} + + +def _yaml_scalar(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + text = str(value) + if not text or any(ch in text for ch in ":#{}[],-&*?!|>'\"%@`") or text.strip() != text: + return json.dumps(text, ensure_ascii=False) + return text + + +def _yaml_lines(value: Any, indent: int = 0) -> list[str]: + space = " " * indent + if isinstance(value, dict): + lines: list[str] = [] + for key, item in value.items(): + if isinstance(item, (dict, list)): + lines.append(f"{space}{key}:") + lines.extend(_yaml_lines(item, indent + 2)) + else: + lines.append(f"{space}{key}: {_yaml_scalar(item)}") + return lines + if isinstance(value, list): + lines = [] + for item in value: + if isinstance(item, dict): + lines.append(f"{space}-") + lines.extend(_yaml_lines(item, indent + 2)) + else: + lines.append(f"{space}- {_yaml_scalar(item)}") + return lines + return [f"{space}{_yaml_scalar(value)}"] + + +def mihomo_export() -> dict[str, Any]: + with connect() as conn: + rows = conn.execute("SELECT * FROM proxy_profiles WHERE status IN (?, 'active', '可用', '已绑定', '未绑定') ORDER BY id", (STATUS_ACTIVE,)).fetchall() + proxies = [] + skipped = [] + for row in rows: + proxy = _json_loads(row["mihomo_proxy_json"], {}) + if proxy: + proxies.append(proxy) + else: + skipped.append({"id": row["id"], "name": row["name"], "reason": row["parse_error"] or "no parsed mihomo proxy"}) + yaml = "proxies:\n" + for proxy in proxies: + lines = _yaml_lines(proxy, 4) + if lines: + first = lines[0].lstrip() + yaml += f" - {first}\n" + yaml += "\n".join(lines[1:]) + ("\n" if len(lines) > 1 else "") + listeners = [ + {"name": f"tiktok-{row['name']}", "type": "mixed", "port": int(row["local_port"] or 0), "proxy": row["mihomo_name"] or row["name"]} + for row in rows + if int(row["local_port"] or 0) + ] + if listeners: + yaml += "listeners:\n" + for listener in listeners: + lines = _yaml_lines(listener, 4) + first = lines[0].lstrip() + yaml += f" - {first}\n" + yaml += "\n".join(lines[1:]) + ("\n" if len(lines) > 1 else "") + return {"proxies": proxies, "listeners": listeners, "skipped": skipped, "yaml": yaml, "port_range": f"{PROXY_PORT_START}-{PROXY_PORT_END}", "generated_at": now_iso()} + + +def lookup_ip_geo(ip: str) -> dict[str, str]: + ok, body, _error = _http_get_json(f"http://ip-api.com/json/{ip}?fields=status,country,regionName,city,query", timeout=4) + if not ok or not isinstance(body, dict) or body.get("status") != "success": + return {"country": "", "region": "", "city": "", "address": ""} + country = str(body.get("country") or "") + region = str(body.get("regionName") or "") + city = str(body.get("city") or "") + address = " / ".join(item for item in (country, region, city) if item) + return {"country": country, "region": region, "city": city, "address": address} + + +def _port_open(host: str, port: int, timeout: float = 1.5) -> bool: + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except Exception: + return False + + +def _http_get_json(url: str, timeout: float = 3.0) -> tuple[bool, Any, str]: + parsed = urlparse(url) + conn = http.client.HTTPConnection(parsed.hostname or "127.0.0.1", parsed.port or 80, timeout=timeout) + path = parsed.path or "/" + if parsed.query: + path += "?" + parsed.query + headers = {} + secret = os.getenv("MIHOMO_SECRET", "").strip() + if secret: + headers["Authorization"] = f"Bearer {secret}" + try: + conn.request("GET", path, headers=headers) + response = conn.getresponse() + body = response.read(8192).decode("utf-8", errors="replace") + if not (200 <= response.status < 300): + return False, None, f"HTTP {response.status}: {body[:200]}" + return True, json.loads(body), "" + except Exception as exc: + return False, None, str(exc) + finally: + conn.close() + + +def runtime_status() -> dict[str, Any]: + mihomo_api = DEFAULT_MIHOMO_API.rstrip("/") + mihomo_ok, mihomo_body, mihomo_error = _http_get_json(mihomo_api + "/version") + novnc_ports = novnc_port_plan() + novnc_checks = {str(port): _port_open("127.0.0.1", port) for port in novnc_ports["allowed_ports"]} + return { + "checked_at": now_iso(), + "novnc_url": DEFAULT_NOVNC_PUBLIC_URL, + "novnc_local_port": NOVNC_PORT, + "novnc_ports": novnc_ports, + "vnc_port": int(os.getenv("VNC_PORT", "5900")), + "mihomo_proxy_port": int(os.getenv("MIHOMO_PROXY_PORT", "7890")), + "mihomo_api_url": mihomo_api, + "checks": { + "novnc_local": novnc_checks.get(str(NOVNC_PORT), False), + "novnc_ports": novnc_checks, + "vnc_local": _port_open("127.0.0.1", int(os.getenv("VNC_PORT", "5900"))), + "mihomo_proxy_local": _port_open("127.0.0.1", int(os.getenv("MIHOMO_PROXY_PORT", "7890"))), + "mihomo_api_local": mihomo_ok, + }, + "mihomo_version": (mihomo_body or {}).get("version") if isinstance(mihomo_body, dict) else "", + "mihomo_error": mihomo_error, + "port_range": f"{PROXY_PORT_START}-{PROXY_PORT_END}", + "pending_login_ttl_seconds": pending_login_ttl_seconds(), + "browser_locale": TIKTOK_BROWSER_LOCALE, + "browser_notice": f"noVNC 放行端口按账号并发 {novnc_ports['max_slots']} + 手动 {novnc_ports['manual_ports']} 计算：{novnc_ports['allowed_range']}；服务器本机检测为准。", + } diff --git a/scripts/static/proxy.html b/scripts/static/proxy.html new file mode 100644 index 0000000..90b5a6b --- /dev/null +++ b/scripts/static/proxy.html @@ -0,0 +1,91 @@ + + + + + +账号 IP 池 + + + +

账号 IP 池

+
+
+
+ + +

账号管理

账号绑定代理状态最近校验操作
+
+
+ + + + + diff --git a/scripts/web_app.py b/scripts/web_app.py index 3ba617f..25bf2e8 100644 --- a/scripts/web_app.py +++ b/scripts/web_app.py @@ -8,6 +8,7 @@ import os import re import shutil +import sqlite3 import subprocess import tempfile import threading @@ -131,6 +132,7 @@ set_hidden_from_analyzer, ) from proxy_state import ensure_us_proxy +import proxy_pool MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024 SAFE_CHARS = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-") AUDIO_ONLY_SUFFIXES = {".aac", ".flac", ".m4a", ".mp3", ".ogg", ".opus", ".wav"} @@ -352,6 +354,7 @@ class AmazonJob: {"key": "amazon", "href": "/amazon", "label": "Amazon", "title": "Amazon", "icon": ''}, {"key": "fastmoss", "href": "/fastmoss", "label": "FastMoss", "title": "FastMoss", "icon": ''}, {"key": "shop", "href": "/shop", "label": "Shop", "title": "Shop", "icon": ''}, + {"key": "proxy", "href": "/proxy", "label": "Proxy", "title": "账号 IP 池", "icon": ''}, {"key": "metrics", "href": "/metrics", "label": "\u6570\u636e", "title": "\u6570\u636e", "icon": ''}, {"key": "extract", "href": "/extract", "label": "\u5206\u6790", "title": "\u89c6\u9891\u5206\u6790", "icon": ''}, ] @@ -5182,6 +5185,10 @@ def do_GET(self) -> None: return text_response(self, HTTPStatus.OK, inject_unified_nav(SHOP_HTML, parsed.path), "text/html; charset=utf-8") if parsed.path == "/metrics": return text_response(self, HTTPStatus.OK, inject_unified_nav(METRICS_HTML, parsed.path), "text/html; charset=utf-8") + if parsed.path == "/proxy": + return text_response(self, HTTPStatus.OK, inject_unified_nav(PROXY_HTML, parsed.path), "text/html; charset=utf-8") + if parsed.path.startswith("/api/proxy/"): + return self.handle_proxy_api_get(parsed.path) if parsed.path.startswith("/assets/"): return self.serve_static_asset(parsed.path.removeprefix("/assets/")) if parsed.path == "/api/prompt": @@ -5755,6 +5762,8 @@ def do_POST(self) -> None: return proxy_mcp_chat(self, "sellersprite") if parsed.path.startswith("/fastmoss/"): return proxy_mcp_chat(self, "fastmoss") + if parsed.path.startswith("/api/proxy/"): + return self.handle_proxy_api_post(parsed.path) if parsed.path == "/api/upload": return self.handle_upload() if parsed.path == "/api/download": @@ -5798,6 +5807,59 @@ def do_POST(self) -> None: return self.handle_delete() return json_response(self, HTTPStatus.NOT_FOUND, {"error": "Not found"}) + def read_json_body(self) -> dict[str, Any]: + content_length = int(self.headers.get("Content-Length", "0") or "0") + raw = self.rfile.read(content_length) if content_length else b"{}" + if not raw: + return {} + data = json.loads(raw.decode("utf-8")) + if not isinstance(data, dict): + raise ValueError("JSON body must be an object") + return data + + def handle_proxy_api_get(self, path: str) -> None: + try: + if path == "/api/proxy/pools": + return json_response(self, HTTPStatus.OK, proxy_pool.list_state()) + if path == "/api/proxy/mihomo-export": + return json_response(self, HTTPStatus.OK, proxy_pool.mihomo_export()) + if path == "/api/proxy/runtime": + return json_response(self, HTTPStatus.OK, proxy_pool.runtime_status()) + return json_response(self, HTTPStatus.NOT_FOUND, {"error": "Not found"}) + except Exception as exc: + return json_response(self, HTTPStatus.INTERNAL_SERVER_ERROR, {"error": str(exc)}) + + def handle_proxy_api_post(self, path: str) -> None: + try: + payload = self.read_json_body() + if path == "/api/proxy/pools": + return json_response(self, HTTPStatus.OK, proxy_pool.upsert_pool(payload)) + if path == "/api/proxy/pools/delete": + return json_response(self, HTTPStatus.OK, proxy_pool.delete_pool(int(payload.get("id") or payload.get("proxy_profile_id") or 0))) + if path == "/api/proxy/accounts": + return json_response(self, HTTPStatus.OK, proxy_pool.upsert_account(payload)) + if path == "/api/proxy/accounts/delete": + return json_response(self, HTTPStatus.OK, proxy_pool.delete_account(int(payload.get("id") or payload.get("account_id") or 0))) + if path == "/api/proxy/check": + return json_response(self, HTTPStatus.OK, proxy_pool.check_binding(payload, require_account=False)) + if path == "/api/proxy/accounts/preflight": + return json_response(self, HTTPStatus.OK, proxy_pool.check_binding(payload, require_account=True)) + if path == "/api/proxy/accounts/status": + return json_response(self, HTTPStatus.OK, proxy_pool.update_account_status(payload)) + if path == "/api/proxy/login-session/start": + return json_response(self, HTTPStatus.OK, proxy_pool.start_login_session(payload)) + if path == "/api/proxy/login-session/stop": + return json_response(self, HTTPStatus.OK, proxy_pool.stop_login_session(payload)) + if path == "/api/proxy/login-session/status": + return json_response(self, HTTPStatus.OK, proxy_pool.inspect_login_session(payload)) + return json_response(self, HTTPStatus.NOT_FOUND, {"error": "Not found"}) + except ValueError as exc: + return json_response(self, HTTPStatus.BAD_REQUEST, {"error": str(exc)}) + except sqlite3.IntegrityError as exc: + return json_response(self, HTTPStatus.BAD_REQUEST, {"error": str(exc)}) + except Exception as exc: + return json_response(self, HTTPStatus.INTERNAL_SERVER_ERROR, {"error": str(exc)}) + def do_DELETE(self) -> None: parsed = urlparse(self.path) if parsed.path.startswith("/amazon/"): @@ -6600,6 +6662,19 @@ def do_DELETE(self) -> None: SHOP_HTML_PATH = SCRIPTS_DIR / "static" / "shop.html" SHOP_HTML = SHOP_HTML_PATH.read_text(encoding="utf-8") if SHOP_HTML_PATH.is_file() else "" +PROXY_HTML_PATH = SCRIPTS_DIR / "static" / "proxy.html" +PROXY_HTML = PROXY_HTML_PATH.read_text(encoding="utf-8") if PROXY_HTML_PATH.is_file() else "" + + +def proxy_session_janitor() -> None: + while True: + try: + released = proxy_pool.cleanup_expired_sessions() + if released: + print(f"Released {released} expired proxy login session(s)", flush=True) + except Exception as exc: + print(f"Proxy session cleanup failed: {exc}", flush=True) + time.sleep(15) def main() -> int: @@ -6609,6 +6684,7 @@ def main() -> int: for store in chat_provider_stores.values(): load_sessions_from_disk(store) mark_interrupted_chat_messages() + threading.Thread(target=proxy_session_janitor, daemon=True).start() normalize_stored_chat_tool_results() video_queue.start(execute_queue_job) report_scheduler_enabled = os.getenv("HOT_VIDEO_REPORT_SCHEDULER_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"} From b1ea82ffcd88cbcfdef98a9a036b0b0fc4058c5b Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 10:53:08 +0800 Subject: [PATCH 002/187] Fix failed report insight fallback --- scripts/hot_video_report.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/scripts/hot_video_report.py b/scripts/hot_video_report.py index 5e9ed3b..dcc9472 100644 --- a/scripts/hot_video_report.py +++ b/scripts/hot_video_report.py @@ -2502,6 +2502,8 @@ def _compact_extraction(analysis: Any) -> dict[str, Any]: def _compact_summary_video(video: dict[str, Any]) -> dict[str, Any]: + insight = video.get("insight") + valid_insight = insight if _is_valid_video_insight(insight) else {} return { "rank": video.get("report_rank"), "title": video.get("title"), @@ -2509,8 +2511,8 @@ def _compact_summary_video(video: dict[str, Any]) -> dict[str, Any]: "metrics": video.get("metrics"), "hot_score": video.get("hot_score"), "source_label": video.get("source_label"), - "insight": video.get("insight") or {}, - "extraction_fallback": _compact_extraction(video.get("analysis")) if not video.get("insight") else {}, + "insight": valid_insight, + "extraction_fallback": _compact_extraction(video.get("analysis")) if not valid_insight else {}, } @@ -2551,14 +2553,28 @@ def _generate_video_insight(video: dict[str, Any], social_context: dict[str, Any api_key = os.getenv("DEEPSEEK_API_KEY", "").strip() if not api_key: raise RuntimeError("Missing required environment variable: DEEPSEEK_API_KEY") + max_tokens = _to_int(os.getenv("REPORT_VIDEO_INSIGHT_MAX_TOKENS", "2200")) response = call_deepseek( api_key=api_key, prompt=_video_insight_prompt(video, social_context), api_url=os.getenv("DEEPSEEK_API_URL", DEFAULT_API_URL), model=os.getenv("DEEPSEEK_MODEL", DEFAULT_MODEL), - max_tokens=_to_int(os.getenv("REPORT_VIDEO_INSIGHT_MAX_TOKENS", "2200")), + max_tokens=max_tokens, ) - content = extract_content(response) + try: + content = extract_content(response) + except ValueError as exc: + retry_max_tokens = _to_int(os.getenv("REPORT_VIDEO_INSIGHT_RETRY_MAX_TOKENS", "4000")) + if "truncated" not in str(exc) or retry_max_tokens <= max_tokens: + raise + response = call_deepseek( + api_key=api_key, + prompt=_video_insight_prompt(video, social_context), + api_url=os.getenv("DEEPSEEK_API_URL", DEFAULT_API_URL), + model=os.getenv("DEEPSEEK_MODEL", DEFAULT_MODEL), + max_tokens=retry_max_tokens, + ) + content = extract_content(response) try: return parse_json_content(content) except Exception: @@ -2705,6 +2721,7 @@ def _compact_summary_video_for_retry(video: dict[str, Any]) -> dict[str, Any]: compact_insight["raw_result"] = _trim_text(insight.get("raw_result"), 700) elif insight: compact_insight["raw"] = _trim_text(insight, 700) + fallback = video.get("extraction_fallback") if not compact_insight else {} return { "rank": video.get("rank"), "title": _trim_text(video.get("title"), 140), @@ -2712,6 +2729,7 @@ def _compact_summary_video_for_retry(video: dict[str, Any]) -> dict[str, Any]: "hot_score": video.get("hot_score"), "source_label": video.get("source_label"), "insight": compact_insight, + "extraction_fallback": _trim_json_payload(fallback, 1600) if fallback else {}, } From 3cedc3f066510f880fb5a2088cf27c5dec4a9cad Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 10:59:52 +0800 Subject: [PATCH 003/187] Guard report deep dives after insight failure --- scripts/hot_video_report.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/scripts/hot_video_report.py b/scripts/hot_video_report.py index dcc9472..aa03a48 100644 --- a/scripts/hot_video_report.py +++ b/scripts/hot_video_report.py @@ -2733,6 +2733,49 @@ def _compact_summary_video_for_retry(video: dict[str, Any]) -> dict[str, Any]: } +def _extraction_fallback_deep_dive(video: dict[str, Any]) -> dict[str, Any]: + extraction = _compact_extraction(video.get("analysis")) + summary = _trim_text(extraction.get("summary"), 360) or "当前视频解析摘要不足。" + timeline = extraction.get("timeline") if isinstance(extraction.get("timeline"), list) else [] + first_event = timeline[0] if timeline else {} + hook = _trim_text(first_event.get("visual") if isinstance(first_event, dict) else first_event, 220) + metrics = video.get("metrics") if isinstance(video.get("metrics"), dict) else {} + return { + "rank": _to_int(video.get("report_rank")) or 0, + "title": str(video.get("title") or "").strip(), + "boom_reason": f"单条深度拆解生成失败；当前仅依据该视频解析摘要：{summary}", + "hook": hook or f"标题/入口：{_trim_text(video.get('title'), 180)}", + "structure": summary, + "audience_trigger": "单条洞察证据不足，需结合当前视频标题、画面解析和评论样本进一步判断。", + "engagement_driver": ( + f"分享 {_to_int(metrics.get('share_count'))}、评论 {_to_int(metrics.get('comment_count'))}、" + f"点赞 {_to_int(metrics.get('like_count'))}。" + ), + "replicable_formula": "证据不足，不从失败的单条洞察中推导复用公式。", + "risk": "本条使用当前视频解析的确定性回退，未采用汇总模型生成的画面描述。", + } + + +def _replace_invalid_insight_deep_dives(report: dict[str, Any], videos: list[dict[str, Any]]) -> None: + fallback_by_rank = { + _to_int(video.get("report_rank")): _extraction_fallback_deep_dive(video) + for video in videos + if not _is_valid_video_insight(video.get("insight")) + } + if not fallback_by_rank: + return + deep_dives = report.get("video_deep_dives") + if not isinstance(deep_dives, list): + deep_dives = [] + by_rank = { + _to_int(item.get("rank")): item + for item in deep_dives + if isinstance(item, dict) and _to_int(item.get("rank")) + } + by_rank.update(fallback_by_rank) + report["video_deep_dives"] = [by_rank[rank] for rank in sorted(by_rank)] + + def _chunk_summary_prompt_v2(report_date: str, chunk_index: int, video_items: list[dict[str, Any]], compact: bool = False) -> str: payload_items = [_compact_summary_video_for_retry(video) for video in video_items] if compact else video_items payload = {"report_date": report_date, "chunk_index": chunk_index, "video_insights": payload_items} @@ -2832,6 +2875,7 @@ def _generate_daily_summary(report_date: str, success_videos: list[dict[str, Any model=os.getenv("DEEPSEEK_MODEL", DEFAULT_MODEL), ) if isinstance(report, dict): + _replace_invalid_insight_deep_dives(report, normalized_videos) markdown = _markdown_from_report(report) report = _normalize_report_for_display(report) return report, markdown From 5d0b861a434b58265f6455413fc7e8e52d958e10 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 11:36:52 +0800 Subject: [PATCH 004/187] Gate proxy pool behind development flag --- docker-compose.yml | 2 ++ scripts/web_app.py | 22 ++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 073e5b4..21958d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,7 @@ services: TIKTOK_PENDING_LOGIN_TTL_SECONDS: ${TIKTOK_PENDING_LOGIN_TTL_SECONDS:-900} TIKTOK_BROWSER_LOCALE: ${TIKTOK_BROWSER_LOCALE:-en-US} TIKTOK_BROWSER_ACCEPT_LANGUAGE: ${TIKTOK_BROWSER_ACCEPT_LANGUAGE:-en-US,en} + PROXY_POOL_ENABLED: ${PROXY_POOL_ENABLED:-0} NOVNC_PORT: ${NOVNC_PORT:-6080} NOVNC_MANUAL_PORTS: ${NOVNC_MANUAL_PORTS:-1} VISION_API_KEY: ${VISION_API_KEY:-} @@ -110,6 +111,7 @@ services: TIKTOK_PENDING_LOGIN_TTL_SECONDS: ${TIKTOK_PENDING_LOGIN_TTL_SECONDS:-900} TIKTOK_BROWSER_LOCALE: ${TIKTOK_BROWSER_LOCALE:-en-US} TIKTOK_BROWSER_ACCEPT_LANGUAGE: ${TIKTOK_BROWSER_ACCEPT_LANGUAGE:-en-US,en} + PROXY_POOL_ENABLED: ${PROXY_POOL_ENABLED:-0} NOVNC_PORT: ${NOVNC_PORT:-6080} NOVNC_MANUAL_PORTS: ${NOVNC_MANUAL_PORTS:-1} VISION_API_KEY: ${VISION_API_KEY:-} diff --git a/scripts/web_app.py b/scripts/web_app.py index 25bf2e8..cdba9c7 100644 --- a/scripts/web_app.py +++ b/scripts/web_app.py @@ -347,8 +347,9 @@ class AmazonJob: "fastmoss": {"system", "fastmoss"}, } FORCED_MCP_CHAT_PROVIDERS = {"amazon", "fastmoss"} -MCP_TOOL_CACHE: dict[str, dict[str, Any]] = {} -NAV_ITEMS = [ +MCP_TOOL_CACHE: dict[str, dict[str, Any]] = {} +PROXY_POOL_ENABLED = os.getenv("PROXY_POOL_ENABLED", "0").strip().lower() in {"1", "true", "yes", "on"} +NAV_ITEMS = [ {"key": "home", "href": "/", "label": "\u9996\u9875", "title": "AI \u804a\u5929", "icon": ''}, {"key": "report", "href": "/report", "label": "\u65e5\u62a5", "title": "\u6bcf\u65e5\u62a5\u544a", "icon": ''}, {"key": "amazon", "href": "/amazon", "label": "Amazon", "title": "Amazon", "icon": ''}, @@ -356,9 +357,11 @@ class AmazonJob: {"key": "shop", "href": "/shop", "label": "Shop", "title": "Shop", "icon": ''}, {"key": "proxy", "href": "/proxy", "label": "Proxy", "title": "账号 IP 池", "icon": ''}, {"key": "metrics", "href": "/metrics", "label": "\u6570\u636e", "title": "\u6570\u636e", "icon": ''}, - {"key": "extract", "href": "/extract", "label": "\u5206\u6790", "title": "\u89c6\u9891\u5206\u6790", "icon": ''}, -] -APP_NAV_CSS = """ + {"key": "extract", "href": "/extract", "label": "\u5206\u6790", "title": "\u89c6\u9891\u5206\u6790", "icon": ''}, +] +if not PROXY_POOL_ENABLED: + NAV_ITEMS = [item for item in NAV_ITEMS if item["key"] != "proxy"] +APP_NAV_CSS = """ @@ -5186,8 +5189,12 @@ def do_GET(self) -> None: if parsed.path == "/metrics": return text_response(self, HTTPStatus.OK, inject_unified_nav(METRICS_HTML, parsed.path), "text/html; charset=utf-8") if parsed.path == "/proxy": + if not PROXY_POOL_ENABLED: + return text_response(self, HTTPStatus.NOT_FOUND, "Not found") return text_response(self, HTTPStatus.OK, inject_unified_nav(PROXY_HTML, parsed.path), "text/html; charset=utf-8") if parsed.path.startswith("/api/proxy/"): + if not PROXY_POOL_ENABLED: + return json_response(self, HTTPStatus.NOT_FOUND, {"error": "Not found"}) return self.handle_proxy_api_get(parsed.path) if parsed.path.startswith("/assets/"): return self.serve_static_asset(parsed.path.removeprefix("/assets/")) @@ -5763,6 +5770,8 @@ def do_POST(self) -> None: if parsed.path.startswith("/fastmoss/"): return proxy_mcp_chat(self, "fastmoss") if parsed.path.startswith("/api/proxy/"): + if not PROXY_POOL_ENABLED: + return json_response(self, HTTPStatus.NOT_FOUND, {"error": "Not found"}) return self.handle_proxy_api_post(parsed.path) if parsed.path == "/api/upload": return self.handle_upload() @@ -6684,7 +6693,8 @@ def main() -> int: for store in chat_provider_stores.values(): load_sessions_from_disk(store) mark_interrupted_chat_messages() - threading.Thread(target=proxy_session_janitor, daemon=True).start() + if PROXY_POOL_ENABLED: + threading.Thread(target=proxy_session_janitor, daemon=True).start() normalize_stored_chat_tool_results() video_queue.start(execute_queue_job) report_scheduler_enabled = os.getenv("HOT_VIDEO_REPORT_SCHEDULER_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"} From 82a5f14c56a12b65f6520ba5b50fff7b777178c2 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 11:39:27 +0800 Subject: [PATCH 005/187] Isolate proxy development runtime --- .dockerignore | 2 ++ docker-compose.dev.yml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 6192c99..c9178e0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,7 +5,9 @@ backups/ data/ data-dev/ videos/ +videos-dev/ output/ +output-dev/ __pycache__/ *.py[cod] .pytest_cache/ diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 66452e8..7935e08 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -11,6 +11,7 @@ services: web: environment: WEB_PORT: ${WEB_PORT:-4003} + PROXY_POOL_ENABLED: "1" LAN_HOST: ${LAN_HOST:-192.168.1.254} REPORT_LAN_HOST: ${REPORT_LAN_HOST:-192.168.1.254} REPORT_LAN_PORT: ${REPORT_LAN_PORT:-4003} @@ -26,4 +27,4 @@ services: sellersprite-redirect: profiles: - - stable-redirect \ No newline at end of file + - stable-redirect From 80bb816a3538021f25f9d82e6a2262c2200b99a6 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 13:29:36 +0800 Subject: [PATCH 006/187] Avoid conflicting timezone bind mounts --- docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 21958d9..97c49c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,8 +82,6 @@ services: - ./output:/workspace/output - ${DATA_DIR:-./data}:/workspace/data - ./scripts:/workspace/scripts:ro - - /etc/localtime:/etc/localtime:ro - - /etc/timezone:/etc/timezone:ro networks: - analyzer @@ -178,8 +176,6 @@ services: - ./output:/workspace/output - ${DATA_DIR:-./data}:/workspace/data - ./scripts:/workspace/scripts:ro - - /etc/localtime:/etc/localtime:ro - - /etc/timezone:/etc/timezone:ro - ${OCR_SHARED_DIR:-/home/openclaw/ocr-shared}:${OCR_SERVER_SHARED_DIR:-/home/openclaw/ocr-shared} - /var/run/docker.sock:/var/run/docker.sock network_mode: host From 8886ad2d28ad9620976ce4bbf75f4cc7944222cf Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 13:41:43 +0800 Subject: [PATCH 007/187] Clean terminated unbound browser profiles --- scripts/proxy_pool.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/scripts/proxy_pool.py b/scripts/proxy_pool.py index 343aac6..0f861d2 100644 --- a/scripts/proxy_pool.py +++ b/scripts/proxy_pool.py @@ -505,11 +505,29 @@ def _active_sessions(conn: sqlite3.Connection) -> list[sqlite3.Row]: return active +def _cleanup_terminated_unbound_profiles(conn: sqlite3.Connection) -> int: + rows = conn.execute( + "SELECT * FROM browser_sessions " + "WHERE account_id IS NULL AND status IN ('stopped','failed') AND user_data_dir <> ''" + ).fetchall() + cleaned = 0 + for row in rows: + user_data_value = str(row["user_data_dir"] or "") + profile_root = Path(user_data_value).resolve().parent + _remove_unbound_session_profile(row) + if not profile_root.exists(): + conn.execute("UPDATE browser_sessions SET user_data_dir = '' WHERE id = ?", (row["id"],)) + cleaned += 1 + conn.commit() + return cleaned + + def cleanup_expired_sessions() -> int: with connect() as conn: before = conn.execute("SELECT COUNT(*) AS count FROM browser_sessions WHERE status IN ('starting','running','observing')").fetchone()["count"] active = _active_sessions(conn) - return max(0, int(before) - len(active)) + cleaned_profiles = _cleanup_terminated_unbound_profiles(conn) + return max(0, int(before) - len(active)) + cleaned_profiles def _allocate_session_slot(conn: sqlite3.Connection) -> int: From 452bd9df2d8a029a6c7e48000cc71bcdf9f210c4 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 18:05:56 +0800 Subject: [PATCH 008/187] feat: add TikTok Studio publish queue --- .env.dev.example | 5 +- docker-compose.yml | 12 + scripts/proxy_pool.py | 149 +++++++- scripts/static/proxy.html | 56 ++- scripts/tiktok_studio_publish.py | 630 +++++++++++++++++++++++++++++++ scripts/web_app.py | 32 +- 6 files changed, 854 insertions(+), 30 deletions(-) create mode 100644 scripts/tiktok_studio_publish.py diff --git a/.env.dev.example b/.env.dev.example index 32d416e..432d137 100644 --- a/.env.dev.example +++ b/.env.dev.example @@ -11,7 +11,10 @@ REPORT_LAN_HOST=192.168.1.254 REPORT_LAN_PORT=4003 SELLERSPRITE_REDIRECT_PORT=0 HOT_VIDEO_REPORT_SCHEDULER_ENABLED=0 +TIKTOK_PUBLISH_DRY_RUN=1 +TIKTOK_NATIVE_SCHEDULE_LEAD_SECONDS=1800 +TIKTOK_NATIVE_SCHEDULE_MIN_MINUTES=20 API_CACHE_ENABLED=1 API_CACHE_TTL_SECONDS=604800 -HF_ENDPOINT=https://hf-mirror.com \ No newline at end of file +HF_ENDPOINT=https://hf-mirror.com diff --git a/docker-compose.yml b/docker-compose.yml index 97c49c7..07c67a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,12 @@ services: TIKTOK_PENDING_LOGIN_TTL_SECONDS: ${TIKTOK_PENDING_LOGIN_TTL_SECONDS:-900} TIKTOK_BROWSER_LOCALE: ${TIKTOK_BROWSER_LOCALE:-en-US} TIKTOK_BROWSER_ACCEPT_LANGUAGE: ${TIKTOK_BROWSER_ACCEPT_LANGUAGE:-en-US,en} + TIKTOK_CDP_PORT_START: ${TIKTOK_CDP_PORT_START:-19220} + TIKTOK_PUBLISH_MAX_BYTES: ${TIKTOK_PUBLISH_MAX_BYTES:-2147483648} + TIKTOK_NATIVE_SCHEDULE_LEAD_SECONDS: ${TIKTOK_NATIVE_SCHEDULE_LEAD_SECONDS:-1800} + TIKTOK_NATIVE_SCHEDULE_MIN_MINUTES: ${TIKTOK_NATIVE_SCHEDULE_MIN_MINUTES:-20} + TIKTOK_PUBLISH_DRY_RUN: ${TIKTOK_PUBLISH_DRY_RUN:-1} + TIKTOK_PUBLISH_UPLOAD_TIMEOUT_SECONDS: ${TIKTOK_PUBLISH_UPLOAD_TIMEOUT_SECONDS:-1800} PROXY_POOL_ENABLED: ${PROXY_POOL_ENABLED:-0} NOVNC_PORT: ${NOVNC_PORT:-6080} NOVNC_MANUAL_PORTS: ${NOVNC_MANUAL_PORTS:-1} @@ -109,6 +115,12 @@ services: TIKTOK_PENDING_LOGIN_TTL_SECONDS: ${TIKTOK_PENDING_LOGIN_TTL_SECONDS:-900} TIKTOK_BROWSER_LOCALE: ${TIKTOK_BROWSER_LOCALE:-en-US} TIKTOK_BROWSER_ACCEPT_LANGUAGE: ${TIKTOK_BROWSER_ACCEPT_LANGUAGE:-en-US,en} + TIKTOK_CDP_PORT_START: ${TIKTOK_CDP_PORT_START:-19220} + TIKTOK_PUBLISH_MAX_BYTES: ${TIKTOK_PUBLISH_MAX_BYTES:-2147483648} + TIKTOK_NATIVE_SCHEDULE_LEAD_SECONDS: ${TIKTOK_NATIVE_SCHEDULE_LEAD_SECONDS:-1800} + TIKTOK_NATIVE_SCHEDULE_MIN_MINUTES: ${TIKTOK_NATIVE_SCHEDULE_MIN_MINUTES:-20} + TIKTOK_PUBLISH_DRY_RUN: ${TIKTOK_PUBLISH_DRY_RUN:-1} + TIKTOK_PUBLISH_UPLOAD_TIMEOUT_SECONDS: ${TIKTOK_PUBLISH_UPLOAD_TIMEOUT_SECONDS:-1800} PROXY_POOL_ENABLED: ${PROXY_POOL_ENABLED:-0} NOVNC_PORT: ${NOVNC_PORT:-6080} NOVNC_MANUAL_PORTS: ${NOVNC_MANUAL_PORTS:-1} diff --git a/scripts/proxy_pool.py b/scripts/proxy_pool.py index 0f861d2..0732cd1 100644 --- a/scripts/proxy_pool.py +++ b/scripts/proxy_pool.py @@ -31,6 +31,7 @@ NOVNC_PORT = int(os.getenv("NOVNC_PORT", "6080") or "6080") NOVNC_MANUAL_PORTS = int(os.getenv("NOVNC_MANUAL_PORTS", "1") or "1") VNC_PORT = int(os.getenv("VNC_PORT", "5900") or "5900") +CDP_PORT = int(os.getenv("TIKTOK_CDP_PORT_START", "19220") or "19220") XVFB_DISPLAY_BASE = int(os.getenv("TIKTOK_XVFB_DISPLAY_BASE", "90") or "90") TIKTOK_BROWSER_UID = int(os.getenv("TIKTOK_BROWSER_UID", "10001") or "10001") TIKTOK_BROWSER_GID = int(os.getenv("TIKTOK_BROWSER_GID", "10001") or "10001") @@ -180,6 +181,39 @@ def init_db(conn: sqlite3.Connection) -> None: ); CREATE INDEX IF NOT EXISTS idx_browser_sessions_status ON browser_sessions(status); CREATE INDEX IF NOT EXISTS idx_browser_sessions_proxy ON browser_sessions(proxy_profile_id); + CREATE TABLE IF NOT EXISTS publish_assets ( + id TEXT PRIMARY KEY, + account_id INTEGER NOT NULL REFERENCES tiktok_accounts(id) ON DELETE RESTRICT, + original_name TEXT NOT NULL, + stored_path TEXT NOT NULL, + content_type TEXT NOT NULL DEFAULT 'video/mp4', + size_bytes INTEGER NOT NULL DEFAULT 0, + sha256 TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS publish_jobs ( + id TEXT PRIMARY KEY, + account_id INTEGER NOT NULL REFERENCES tiktok_accounts(id) ON DELETE RESTRICT, + proxy_profile_id INTEGER NOT NULL REFERENCES proxy_profiles(id) ON DELETE RESTRICT, + asset_id TEXT NOT NULL REFERENCES publish_assets(id) ON DELETE RESTRICT, + description TEXT NOT NULL DEFAULT '', + ai_generated INTEGER NOT NULL DEFAULT 0, + schedule_mode TEXT NOT NULL DEFAULT 'server', + scheduled_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'draft', + stage TEXT NOT NULL DEFAULT '', + attempt_count INTEGER NOT NULL DEFAULT 0, + next_attempt_at TEXT NOT NULL DEFAULT '', + session_id INTEGER REFERENCES browser_sessions(id) ON DELETE SET NULL, + final_click_at TEXT NOT NULL DEFAULT '', + actual_publish_at TEXT NOT NULL DEFAULT '', + result_url TEXT NOT NULL DEFAULT '', + last_error TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_publish_jobs_account ON publish_jobs(account_id, created_at DESC); + CREATE INDEX IF NOT EXISTS idx_publish_jobs_due ON publish_jobs(status, scheduled_at); """ ) for name, definition in { @@ -194,6 +228,11 @@ def init_db(conn: sqlite3.Connection) -> None: existing = {row[1] for row in conn.execute("PRAGMA table_info(proxy_profiles)")} if name not in existing: conn.execute(f"ALTER TABLE proxy_profiles ADD COLUMN {name} {definition}") + existing_account_cols = {row[1] for row in conn.execute("PRAGMA table_info(tiktok_accounts)")} + if "deleted_at" not in existing_account_cols: + conn.execute("ALTER TABLE tiktok_accounts ADD COLUMN deleted_at TEXT NOT NULL DEFAULT ''") + if "last_publish_at" not in existing_account_cols: + conn.execute("ALTER TABLE tiktok_accounts ADD COLUMN last_publish_at TEXT NOT NULL DEFAULT ''") existing_session_cols = {row[1] for row in conn.execute("PRAGMA table_info(browser_sessions)")} for name, definition in { "xvfb_pid": "INTEGER NOT NULL DEFAULT 0", @@ -202,9 +241,15 @@ def init_db(conn: sqlite3.Connection) -> None: "display": "TEXT NOT NULL DEFAULT ''", "vnc_port": "INTEGER NOT NULL DEFAULT 0", "novnc_port": "INTEGER NOT NULL DEFAULT 0", + "debug_port": "INTEGER NOT NULL DEFAULT 0", + "owner": "TEXT NOT NULL DEFAULT 'manual'", + "current_job_id": "TEXT NOT NULL DEFAULT ''", }.items(): if name not in existing_session_cols: conn.execute(f"ALTER TABLE browser_sessions ADD COLUMN {name} {definition}") + existing_publish_cols = {row[1] for row in conn.execute("PRAGMA table_info(publish_jobs)")} + if "next_attempt_at" not in existing_publish_cols: + conn.execute("ALTER TABLE publish_jobs ADD COLUMN next_attempt_at TEXT NOT NULL DEFAULT ''") conn.commit() @@ -373,6 +418,7 @@ def _row_to_account(row: sqlite3.Row) -> dict[str, Any]: "last_check_at": row["last_check_at"], "last_login_at": row["last_login_at"], "last_collect_at": row["last_collect_at"], + "last_publish_at": row["last_publish_at"], "last_error": row["last_error"], "created_at": row["created_at"], "updated_at": row["updated_at"], @@ -395,6 +441,9 @@ def _row_to_session(row: sqlite3.Row) -> dict[str, Any]: "display": row["display"], "vnc_port": row["vnc_port"], "novnc_port": row["novnc_port"], + "debug_port": row["debug_port"], + "owner": row["owner"], + "current_job_id": row["current_job_id"], "profile_key": row["profile_key"], "user_data_dir": row["user_data_dir"], "last_error": row["last_error"], @@ -539,6 +588,12 @@ def _allocate_session_slot(conn: sqlite3.Connection) -> int: raise ValueError(f"浏览器观测槽位已满，当前最多同时运行 {max_slots} 个") +def _allocate_manual_slot(conn: sqlite3.Connection) -> int: + if any(int(row["slot"] or 0) == 0 for row in _active_sessions(conn)): + raise ValueError("手动登录观测通道正在使用，请先完成或关闭当前登录") + return 0 + + def _browser_binary() -> str: configured = os.getenv("TIKTOK_BROWSER_BIN", "").strip() candidates = [configured] if configured else [] @@ -586,6 +641,7 @@ def _slot_ports(slot: int) -> dict[str, Any]: "display": f":{XVFB_DISPLAY_BASE + slot}", "vnc_port": VNC_PORT + manual_ports + slot - 1, "novnc_port": NOVNC_PORT + manual_ports + slot - 1, + "debug_port": CDP_PORT + slot, } @@ -821,7 +877,7 @@ def _tiktok_identity(body: Any) -> dict[str, str]: return {} -def _launch_browser_for_session(profile: dict[str, Any], pool: sqlite3.Row, session_id: int, display: str) -> tuple[int, str]: +def _launch_browser_for_session(profile: dict[str, Any], pool: sqlite3.Row, session_id: int, display: str, debug_port: int, start_url: str) -> tuple[int, str]: isolation = profile.get("isolation") if isinstance(profile.get("isolation"), dict) else {} user_data_dir = _abs_workspace_path(str(isolation.get("user_data_dir") or f"data/tiktok_browser_profiles/session-{session_id}/user-data")) _configure_browser_preferences(user_data_dir) @@ -836,6 +892,8 @@ def _launch_browser_for_session(profile: dict[str, Any], pool: sqlite3.Row, sess f"--user-data-dir={user_data_dir}", f"--proxy-server=http://127.0.0.1:{proxy_port}", f"--lang={TIKTOK_BROWSER_LOCALE}", + "--remote-debugging-address=127.0.0.1", + f"--remote-debugging-port={debug_port}", "--no-first-run", "--no-default-browser-check", "--disable-sync", @@ -846,7 +904,7 @@ def _launch_browser_for_session(profile: dict[str, Any], pool: sqlite3.Row, sess "--disable-session-crashed-bubble", "--window-size=1280,900", "--new-window", - "https://www.tiktok.com/login", + start_url, ] env = os.environ.copy() env["DISPLAY"] = display @@ -957,13 +1015,13 @@ def list_state() -> dict[str, Any]: _active_sessions(conn) counts = { int(row["proxy_profile_id"]): int(row["count"]) - for row in conn.execute("SELECT proxy_profile_id, COUNT(*) AS count FROM tiktok_accounts GROUP BY proxy_profile_id") + for row in conn.execute("SELECT proxy_profile_id, COUNT(*) AS count FROM tiktok_accounts WHERE deleted_at = '' GROUP BY proxy_profile_id") } names: dict[int, list[str]] = {} - for row in conn.execute("SELECT proxy_profile_id, username FROM tiktok_accounts ORDER BY username"): + for row in conn.execute("SELECT proxy_profile_id, username FROM tiktok_accounts WHERE deleted_at = '' ORDER BY username"): names.setdefault(int(row["proxy_profile_id"]), []).append(str(row["username"])) pools = [_row_to_pool(row, counts.get(int(row["id"]), 0), names.get(int(row["id"]), [])) for row in conn.execute("SELECT * FROM proxy_profiles ORDER BY updated_at DESC, id DESC")] - accounts = [_row_to_account(row) for row in conn.execute("SELECT * FROM tiktok_accounts ORDER BY updated_at DESC, id DESC")] + accounts = [_row_to_account(row) for row in conn.execute("SELECT * FROM tiktok_accounts WHERE deleted_at = '' ORDER BY updated_at DESC, id DESC")] sessions = [_row_to_session(row) for row in conn.execute("SELECT * FROM browser_sessions ORDER BY updated_at DESC, id DESC LIMIT 20")] return { "pools": pools, @@ -1178,7 +1236,22 @@ def get_account(account_id: int) -> dict[str, Any]: def delete_account(account_id: int) -> dict[str, Any]: with connect() as conn: - conn.execute("DELETE FROM tiktok_accounts WHERE id = ?", (account_id,)) + if any(int(row["account_id"] or 0) == account_id for row in _active_sessions(conn)): + raise ValueError("账号仍处于唤醒或运行状态，请先休眠账号") + active_job = conn.execute( + "SELECT id FROM publish_jobs WHERE account_id = ? AND status NOT IN ('published','failed','cancelled','scheduled_on_tiktok','dry_run') LIMIT 1", + (account_id,), + ).fetchone() + if active_job: + raise ValueError("账号仍有草稿、待发布或运行中的发布任务，请先处理任务") + account = conn.execute("SELECT username FROM tiktok_accounts WHERE id = ? AND deleted_at = ''", (account_id,)).fetchone() + if not account: + raise ValueError("account not found") + now = now_iso() + conn.execute( + "UPDATE tiktok_accounts SET username = ?, status = ?, deleted_at = ?, updated_at = ? WHERE id = ?", + (f"{account['username']}__deleted_{account_id}", ACCOUNT_STATUS_PAUSED, now, now, account_id), + ) conn.commit() return list_state() @@ -1379,14 +1452,16 @@ def start_login_session(payload: dict[str, Any]) -> dict[str, Any]: if account_id: with connect() as conn: account_row = conn.execute("SELECT * FROM tiktok_accounts WHERE id = ?", (account_id,)).fetchone() - if not account_row: - raise ValueError("account not found") - bound_proxy_id = int(account_row["proxy_profile_id"] or 0) - if proxy_profile_id and proxy_profile_id != bound_proxy_id: - raise ValueError("账号与请求代理不一致") - proxy_profile_id = bound_proxy_id - username = str(account_row["username"] or "") - saved_profile = _json_loads(account_row["profile_json"], {}) + if not account_row: + raise ValueError("account not found") + if "deleted_at" in account_row.keys() and account_row["deleted_at"]: + raise ValueError("account has been deleted") + bound_proxy_id = int(account_row["proxy_profile_id"] or 0) + if proxy_profile_id and proxy_profile_id != bound_proxy_id: + raise ValueError("账号与请求代理不一致") + proxy_profile_id = bound_proxy_id + username = str(account_row["username"] or "") + saved_profile = _json_loads(account_row["profile_json"], {}) else: username = _clean_text(payload.get("username"), 120).lstrip("@") if not proxy_profile_id: @@ -1416,7 +1491,9 @@ def start_login_session(payload: dict[str, Any]) -> dict[str, Any]: for active_row in _active_sessions(conn): if int(active_row["account_id"] or 0) == account_id: raise ValueError("账号已经处于唤醒状态") - slot = _allocate_session_slot(conn) + owner = "automation" if payload.get("_automation") else "manual" + current_job_id = _clean_text(payload.get("_current_job_id"), 80) if owner == "automation" else "" + slot = _allocate_session_slot(conn) if account_id else _allocate_manual_slot(conn) pending_name = f"pending-{proxy_profile_id}-{slot}-{int(time.time())}" if not username else username profile = _deep_merge(_isolation_profile(pending_name, proxy_profile_id, pool), saved_profile) if account_id else _isolation_profile(pending_name, proxy_profile_id, pool) profile_key = str((profile.get("isolation") or {}).get("browser_profile_key") or "") @@ -1427,8 +1504,8 @@ def start_login_session(payload: dict[str, Any]) -> dict[str, Any]: INSERT INTO browser_sessions ( slot, proxy_profile_id, account_id, username, status, channel_url, pid, xvfb_pid, x11vnc_pid, websockify_pid, display, vnc_port, novnc_port, - profile_key, user_data_dir, last_error, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, 0, ?, ?, ?, ?, ?, '', ?, ?) + debug_port, owner, current_job_id, profile_key, user_data_dir, last_error, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, 0, ?, ?, ?, ?, ?, ?, ?, ?, '', ?, ?) """, ( slot, @@ -1440,6 +1517,9 @@ def start_login_session(payload: dict[str, Any]) -> dict[str, Any]: str(slot_ports["display"]), int(slot_ports["vnc_port"]), int(slot_ports["novnc_port"]), + int(slot_ports["debug_port"]), + owner, + current_job_id, profile_key, str((profile.get("isolation") or {}).get("user_data_dir") or ""), now, @@ -1451,10 +1531,19 @@ def start_login_session(payload: dict[str, Any]) -> dict[str, Any]: try: log_dir = _abs_workspace_path(f"data/tiktok_browser_sessions/{session_id}") channel = _launch_observation_channel(slot, session_id, log_dir) - pid, user_data_dir = _launch_browser_for_session(profile, pool, session_id, str(channel["display"])) + start_url = "https://www.tiktok.com/" if account_id else "https://www.tiktok.com/login" + pid, user_data_dir = _launch_browser_for_session( + profile, + pool, + session_id, + str(channel["display"]), + int(slot_ports["debug_port"]), + start_url, + ) time.sleep(2.0) if not _pid_alive(pid): raise ValueError("Chrome 启动后立即退出，请检查 browser.err.log") + _wait_for_port(int(slot_ports["debug_port"]), "Chrome CDP", timeout=10.0) conn.execute( """ UPDATE browser_sessions @@ -1467,6 +1556,7 @@ def start_login_session(payload: dict[str, Any]) -> dict[str, Any]: display = ?, vnc_port = ?, novnc_port = ?, + debug_port = ?, user_data_dir = ?, updated_at = ? WHERE id = ? @@ -1480,6 +1570,7 @@ def start_login_session(payload: dict[str, Any]) -> dict[str, Any]: str(channel["display"]), int(channel["vnc_port"]), int(channel["novnc_port"]), + int(slot_ports["debug_port"]), user_data_dir, now_iso(), session_id, @@ -1510,6 +1601,8 @@ def stop_login_session(payload: dict[str, Any]) -> dict[str, Any]: now = now_iso() with connect() as conn: row = _session_by_id(conn, session_id) + if row["current_job_id"] and not payload.get("force"): + raise ValueError("账号正在发布，确认终止任务后才能休眠") _terminate_session_processes(row) _remove_unbound_session_profile(row) status = "failed" if mark_failed else "stopped" @@ -1521,6 +1614,26 @@ def stop_login_session(payload: dict[str, Any]) -> dict[str, Any]: return list_state() +def start_automation_session(account_id: int, job_id: str) -> dict[str, Any]: + return start_login_session({"account_id": account_id, "_automation": True, "_current_job_id": job_id}) + + +def finish_automation_session(session_id: int, reason: str = "自动发布任务结束") -> dict[str, Any]: + return stop_login_session({"session_id": session_id, "force": True, "reason": reason}) + + +def handoff_automation_session(session_id: int, reason: str) -> dict[str, Any]: + now = now_iso() + with connect() as conn: + row = _session_by_id(conn, session_id) + conn.execute( + "UPDATE browser_sessions SET owner = 'manual_review', current_job_id = '', last_error = ?, updated_at = ? WHERE id = ?", + (_clean_text(reason, 1000), now, session_id), + ) + conn.commit() + return _row_to_session(_session_by_id(conn, session_id)) + + def inspect_login_session(payload: dict[str, Any]) -> dict[str, Any]: session_id = int(payload.get("session_id") or payload.get("id") or 0) if not session_id: diff --git a/scripts/static/proxy.html b/scripts/static/proxy.html index 90b5a6b..cf7ed0c 100644 --- a/scripts/static/proxy.html +++ b/scripts/static/proxy.html @@ -8,6 +8,9 @@ :root{--bg:#f6f8fb;--surface:#fff;--line:#e2e8f0;--line2:#f1f5f9;--text:#0f172a;--muted:#64748b;--brand:#2563eb;--brand2:#1d4ed8;--ok:#059669;--bad:#dc2626;--warn:#d97706;--soft:#eff6ff;--shadow:0 1px 2px rgba(15,23,42,.05)} *{box-sizing:border-box;margin:0}html,body{height:100%}body{background:var(--bg);color:var(--text);font-family:"Segoe UI","Microsoft YaHei",system-ui,sans-serif;line-height:1.5;overflow:hidden}button,input,select,textarea{font:inherit;color:inherit}button{min-height:34px;border:1px solid var(--line);border-radius:8px;background:#fff;padding:7px 12px;font-size:13px;font-weight:700;cursor:pointer}button.primary{background:var(--brand);border-color:var(--brand);color:#fff}button.primary:hover{background:var(--brand2)}button.danger{border-color:#fecaca;color:var(--bad)}button.danger.solid{background:var(--bad);border-color:var(--bad);color:#fff}button.ghost{border-color:transparent;background:transparent;color:var(--muted)}input,select,textarea{width:100%;border:1px solid var(--line);border-radius:8px;background:#fff;padding:8px 10px;font-size:13px;outline:none}input:focus,select:focus,textarea:focus{border-color:var(--brand);box-shadow:0 0 0 3px rgba(37,99,235,.1)}input[readonly]{background:#f8fafc;color:#475569}textarea{min-height:96px;resize:vertical;font:12px/1.55 ui-monospace,Consolas,monospace}header{height:60px;display:flex;align-items:center;justify-content:space-between;padding:0 22px;background:rgba(255,255,255,.92);border-bottom:1px solid var(--line)}header h1{font-size:18px}main{height:calc(100vh - 60px);display:grid;grid-template-rows:auto minmax(0,1fr);gap:12px;padding:14px}.runtime{display:grid;grid-template-columns:repeat(6,minmax(0,1fr));gap:10px}.tile{border:1px solid var(--line);border-radius:8px;background:#fff;padding:10px 12px;box-shadow:var(--shadow);min-width:0}.tile span{display:block;color:var(--muted);font-size:12px;font-weight:700}.tile b{display:block;margin-top:4px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.tile.ok b{color:var(--ok)}.tile.bad b{color:var(--bad)}.work{display:grid;grid-template-columns:310px 400px minmax(0,1fr);gap:12px;min-height:0}.panel{border:1px solid var(--line);border-radius:8px;background:#fff;min-height:0;overflow:hidden;box-shadow:var(--shadow);display:grid;grid-template-rows:auto minmax(0,1fr)}.head{display:flex;align-items:center;justify-content:space-between;gap:10px;padding:10px 12px;border-bottom:1px solid var(--line);background:var(--line2)}.head h2{font-size:14px}.body{padding:12px;overflow:auto}.stack{display:grid;gap:10px}.row{display:flex;gap:8px;align-items:center;flex-wrap:wrap}.split{display:grid;grid-template-columns:1fr 1fr;gap:8px}.tool-row{display:flex;gap:8px;padding-bottom:12px;margin-bottom:12px;border-bottom:1px solid var(--line)}.icon-btn{width:36px;height:36px;min-height:36px;padding:0;display:inline-grid;place-items:center;border-radius:8px}.icon-btn svg{width:17px;height:17px;stroke:currentColor;fill:none;stroke-width:2;stroke-linecap:round;stroke-linejoin:round}.pool-list{align-content:start}.pool{height:92px;display:grid;grid-template-rows:22px 22px 22px;gap:5px;border:1px solid var(--line);border-radius:8px;padding:10px;background:#fff;cursor:pointer;transition:border-color .15s,background .15s,box-shadow .15s}.pool:hover{border-color:#bfdbfe;box-shadow:0 2px 8px rgba(15,23,42,.06)}.pool.active{border-color:var(--brand);background:var(--soft)}.pool-line{display:flex;align-items:center;justify-content:space-between;gap:10px;min-width:0}.pool-main,.pool-sub{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.pool-sub{color:var(--muted);font-size:12px}.empty-state{height:100%;min-height:220px;display:grid;place-items:center;text-align:center;color:var(--muted);border:1px dashed var(--line);border-radius:8px;background:#f8fafc;padding:20px}.empty-state b{display:block;color:var(--text);margin-bottom:6px}.muted{color:var(--muted);font-size:12px}.badge{display:inline-flex;align-items:center;border-radius:999px;padding:2px 7px;font-size:12px;font-weight:700;background:#f1f5f9;color:var(--muted)}.badge.ok{background:#ecfdf5;color:var(--ok)}.badge.bad{background:#fef2f2;color:var(--bad)}.badge.warn{background:#fffbeb;color:var(--warn)}table{width:100%;border-collapse:collapse;font-size:13px}th,td{padding:9px 8px;border-bottom:1px solid var(--line);text-align:left;vertical-align:top}th{position:sticky;top:0;background:#fff;color:var(--muted);font-size:12px;z-index:1}tr.selected{background:var(--soft)}.actions{display:flex;gap:6px;flex-wrap:wrap}.status{min-height:36px;border:1px solid var(--line);border-radius:8px;background:var(--line2);padding:8px 10px;color:var(--muted);font-size:12px;overflow-wrap:anywhere}.status.ok{background:#ecfdf5;color:var(--ok);border-color:#a7f3d0}.status.bad{background:#fef2f2;color:var(--bad);border-color:#fecaca}pre{margin:0;white-space:pre-wrap;word-break:break-word;font:12px/1.55 ui-monospace,Consolas,monospace;background:#0f172a;color:#e2e8f0;border-radius:8px;padding:10px;max-height:220px;overflow:auto}.modal{position:fixed;inset:0;display:none;align-items:center;justify-content:center;background:rgba(15,23,42,.42);z-index:20;padding:22px}.modal.open{display:flex}.dialog{width:min(620px,96vw);max-height:92vh;display:grid;grid-template-rows:auto minmax(0,1fr);border:1px solid var(--line);border-radius:8px;background:#fff;box-shadow:0 18px 48px rgba(15,23,42,.2);overflow:hidden}.dialog-head{display:flex;justify-content:space-between;align-items:center;padding:12px 14px;border-bottom:1px solid var(--line);background:var(--line2)}.dialog-body{padding:14px;overflow:auto}.progress{height:8px;border-radius:999px;background:#e2e8f0;overflow:hidden}.progress i{display:block;width:0;height:100%;background:var(--brand);transition:width .25s}.link-btn{display:none;align-items:center;min-height:34px;border-radius:8px;padding:7px 12px;text-decoration:none;font-size:13px;font-weight:700;background:#fff;border:1px solid var(--line);color:var(--brand)}.link-btn.show{display:inline-flex}@media(max-width:1100px){body{overflow:auto}.runtime{grid-template-columns:1fr 1fr}.work{grid-template-columns:1fr;height:auto}.panel{min-height:360px}main{height:auto}} +

账号 IP 池

@@ -16,7 +19,7 @@
-

账号管理

账号绑定代理状态最近校验操作
+

账号管理

账号绑定代理状态最近校验操作
+ + + diff --git a/scripts/tiktok_studio_publish.py b/scripts/tiktok_studio_publish.py new file mode 100644 index 0000000..abbc616 --- /dev/null +++ b/scripts/tiktok_studio_publish.py @@ -0,0 +1,630 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import hashlib +import mimetypes +import os +import re +import threading +import time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any +from zoneinfo import ZoneInfo + +import proxy_pool + + +ROOT = Path.cwd() +PUBLISH_ROOT = ROOT / "videos" / "tiktok_publish" +LOG_ROOT = ROOT / "data" / "tiktok_publish_jobs" +MAX_UPLOAD_BYTES = int(os.getenv("TIKTOK_PUBLISH_MAX_BYTES", str(2 * 1024 * 1024 * 1024))) +TIMEZONE_NAME = os.getenv("TZ", "America/Los_Angeles") or "America/Los_Angeles" +NATIVE_LEAD_SECONDS = max(300, int(os.getenv("TIKTOK_NATIVE_SCHEDULE_LEAD_SECONDS", "1800") or "1800")) +NATIVE_MIN_MINUTES = max(15, int(os.getenv("TIKTOK_NATIVE_SCHEDULE_MIN_MINUTES", "20") or "20")) +DRY_RUN = os.getenv("TIKTOK_PUBLISH_DRY_RUN", "1").strip().lower() in {"1", "true", "yes", "on"} +UPLOAD_TIMEOUT_SECONDS = max(60, int(os.getenv("TIKTOK_PUBLISH_UPLOAD_TIMEOUT_SECONDS", "1800") or "1800")) +ALLOWED_SUFFIXES = {".mp4", ".mov", ".m4v", ".webm"} +EDITABLE_STATUSES = {"draft", "queued", "delayed"} +STATUS_LABELS = { + "draft": "草稿箱", + "queued": "待发布", + "delayed": "延迟等待", + "preparing": "准备中", + "uploading": "上传中", + "publishing": "发布中", + "published": "已发布", + "failed": "发布失败", + "result_uncertain": "结果待确认", + "cancelled": "已取消", + "scheduled_on_tiktok": "TikTok已排程", + "dry_run": "演练完成", +} + +_worker_started = False +_worker_lock = threading.Lock() +_active_jobs: set[str] = set() + + +class ManualReviewRequired(RuntimeError): + pass + + +class ResultUncertain(RuntimeError): + pass + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _iso(value: datetime | None = None) -> str: + return (value or _utc_now()).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _parse_schedule(value: Any) -> datetime: + raw = str(value or "").strip() + if not raw: + return _utc_now() + try: + parsed = datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError("发布时间格式无效") from exc + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=ZoneInfo(TIMEZONE_NAME)) + return parsed.astimezone(timezone.utc) + + +def _clean_text(value: Any, limit: int) -> str: + return str(value or "").strip()[:limit] + + +def _row_to_job(row: Any) -> dict[str, Any]: + return { + "id": row["id"], + "account_id": row["account_id"], + "proxy_profile_id": row["proxy_profile_id"], + "asset_id": row["asset_id"], + "original_name": row["original_name"], + "size_bytes": row["size_bytes"], + "content_type": row["content_type"], + "description": row["description"], + "ai_generated": bool(row["ai_generated"]), + "schedule_mode": row["schedule_mode"], + "scheduled_at": row["scheduled_at"], + "status": row["status"], + "status_label": STATUS_LABELS.get(row["status"], row["status"]), + "stage": row["stage"], + "attempt_count": row["attempt_count"], + "session_id": row["session_id"], + "actual_publish_at": row["actual_publish_at"], + "result_url": row["result_url"], + "last_error": row["last_error"], + "preview_url": f"/api/proxy/publish/videos/{row['asset_id']}", + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + +def _job_query(where: str = "", params: tuple[Any, ...] = ()) -> list[dict[str, Any]]: + sql = ( + "SELECT j.*, a.original_name, a.size_bytes, a.content_type, a.stored_path " + "FROM publish_jobs j JOIN publish_assets a ON a.id = j.asset_id " + ) + if where: + sql += "WHERE " + where + " " + sql += "ORDER BY j.created_at DESC" + with proxy_pool.connect() as conn: + return [_row_to_job(row) for row in conn.execute(sql, params).fetchall()] + + +def list_jobs(account_id: int) -> dict[str, Any]: + if not account_id: + raise ValueError("account_id is required") + return {"jobs": _job_query("j.account_id = ?", (account_id,)), "timezone": TIMEZONE_NAME, "dry_run": DRY_RUN} + + +def _validate_schedule(mode: str, scheduled_at: datetime, queued: bool) -> None: + if mode not in {"server", "tiktok"}: + raise ValueError("定时方式必须为 server 或 tiktok") + if queued and mode == "tiktok" and scheduled_at < _utc_now() + timedelta(minutes=NATIVE_MIN_MINUTES): + raise ValueError(f"TikTok 定时发布至少需要提前 {NATIVE_MIN_MINUTES} 分钟") + + +def create_job(form: Any) -> dict[str, Any]: + account_id = int(form.getfirst("account_id") or 0) + if not account_id: + raise ValueError("account_id is required") + action = _clean_text(form.getfirst("action"), 20) or "queue" + queued = action != "draft" + schedule_mode = _clean_text(form.getfirst("schedule_mode"), 20) or "server" + scheduled_at = _parse_schedule(form.getfirst("scheduled_at")) + _validate_schedule(schedule_mode, scheduled_at, queued) + try: + file_item = form["video"] + except KeyError as exc: + raise ValueError("请选择视频文件") from exc + if isinstance(file_item, list): + file_item = file_item[0] + original_name = Path(str(getattr(file_item, "filename", "") or "")).name + suffix = Path(original_name).suffix.lower() + if not original_name or suffix not in ALLOWED_SUFFIXES: + raise ValueError("仅支持 MP4、MOV、M4V 或 WebM 视频") + + with proxy_pool.connect() as conn: + account = conn.execute("SELECT * FROM tiktok_accounts WHERE id = ? AND deleted_at = ''", (account_id,)).fetchone() + if not account: + raise ValueError("account not found") + proxy_profile_id = int(account["proxy_profile_id"]) + + asset_id = uuid.uuid4().hex + job_id = uuid.uuid4().hex + target_dir = PUBLISH_ROOT / str(account_id) + target_dir.mkdir(parents=True, exist_ok=True) + target = target_dir / f"{asset_id}{suffix}" + digest = hashlib.sha256() + size = 0 + try: + with target.open("wb") as output: + while True: + chunk = file_item.file.read(1024 * 1024) + if not chunk: + break + size += len(chunk) + if size > MAX_UPLOAD_BYTES: + raise ValueError("视频超过 2GB 上传限制") + digest.update(chunk) + output.write(chunk) + if size <= 0: + raise ValueError("视频文件为空") + now = _iso() + stored_path = target.relative_to(ROOT).as_posix() + content_type = _clean_text(getattr(file_item, "type", ""), 120) or mimetypes.guess_type(original_name)[0] or "video/mp4" + with proxy_pool.connect() as conn: + conn.execute( + "INSERT INTO publish_assets (id, account_id, original_name, stored_path, content_type, size_bytes, sha256, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (asset_id, account_id, original_name, stored_path, content_type, size, digest.hexdigest(), now), + ) + conn.execute( + """ + INSERT INTO publish_jobs ( + id, account_id, proxy_profile_id, asset_id, description, ai_generated, + schedule_mode, scheduled_at, status, stage, attempt_count, next_attempt_at, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, '', 0, '', ?, ?) + """, + ( + job_id, + account_id, + proxy_profile_id, + asset_id, + _clean_text(form.getfirst("description"), 2200), + 1 if str(form.getfirst("ai_generated") or "").lower() in {"1", "true", "yes", "on"} else 0, + schedule_mode, + _iso(scheduled_at), + "queued" if queued else "draft", + now, + now, + ), + ) + conn.commit() + except Exception: + target.unlink(missing_ok=True) + raise + return {"job": _job_query("j.id = ?", (job_id,))[0], **list_jobs(account_id)} + + +def update_job(payload: dict[str, Any]) -> dict[str, Any]: + job_id = _clean_text(payload.get("id") or payload.get("job_id"), 80) + if not job_id: + raise ValueError("job_id is required") + with proxy_pool.connect() as conn: + row = conn.execute("SELECT * FROM publish_jobs WHERE id = ?", (job_id,)).fetchone() + if not row: + raise ValueError("publish job not found") + if row["status"] not in EDITABLE_STATUSES: + raise ValueError("只有草稿、待发布或延迟任务可以编辑") + mode = _clean_text(payload.get("schedule_mode"), 20) or row["schedule_mode"] + scheduled = _parse_schedule(payload.get("scheduled_at") or row["scheduled_at"]) + queue = bool(payload.get("queue")) or row["status"] != "draft" + _validate_schedule(mode, scheduled, queue) + status = "queued" if payload.get("queue") else row["status"] + conn.execute( + "UPDATE publish_jobs SET description = ?, ai_generated = ?, schedule_mode = ?, scheduled_at = ?, status = ?, next_attempt_at = '', last_error = '', updated_at = ? WHERE id = ?", + ( + _clean_text(payload.get("description"), 2200), + 1 if payload.get("ai_generated") else 0, + mode, + _iso(scheduled), + status, + _iso(), + job_id, + ), + ) + conn.commit() + account_id = int(row["account_id"]) + return {"job": _job_query("j.id = ?", (job_id,))[0], **list_jobs(account_id)} + + +def cancel_job(payload: dict[str, Any]) -> dict[str, Any]: + job_id = _clean_text(payload.get("id") or payload.get("job_id"), 80) + with proxy_pool.connect() as conn: + row = conn.execute("SELECT * FROM publish_jobs WHERE id = ?", (job_id,)).fetchone() + if not row: + raise ValueError("publish job not found") + if row["status"] not in EDITABLE_STATUSES: + raise ValueError("只能取消尚未开始的发布任务") + conn.execute("UPDATE publish_jobs SET status = 'cancelled', stage = '', updated_at = ? WHERE id = ?", (_iso(), job_id)) + conn.commit() + account_id = int(row["account_id"]) + return list_jobs(account_id) + + +def video_path(asset_id: str) -> Path: + with proxy_pool.connect() as conn: + row = conn.execute("SELECT stored_path FROM publish_assets WHERE id = ?", (_clean_text(asset_id, 80),)).fetchone() + if not row: + raise ValueError("publish video not found") + path = (ROOT / str(row["stored_path"])).resolve() + root = PUBLISH_ROOT.resolve() + if root != path.parent and root not in path.parents: + raise ValueError("invalid publish video path") + if not path.is_file(): + raise ValueError("publish video file is missing") + return path + + +def runtime_status() -> dict[str, Any]: + with proxy_pool.connect() as conn: + counts = {row["status"]: int(row["count"]) for row in conn.execute("SELECT status, COUNT(*) AS count FROM publish_jobs GROUP BY status")} + with _worker_lock: + active = sorted(_active_jobs) + return { + "worker_started": _worker_started, + "dry_run": DRY_RUN, + "timezone": TIMEZONE_NAME, + "max_automatic_slots": proxy_pool.browser_max_slots(), + "active_jobs": active, + "counts": counts, + "native_schedule_lead_seconds": NATIVE_LEAD_SECONDS, + } + + +def _set_job(job_id: str, status: str, stage: str = "", error: str = "", **values: Any) -> None: + fields = ["status = ?", "stage = ?", "last_error = ?", "updated_at = ?"] + params: list[Any] = [status, stage, _clean_text(error, 2000), _iso()] + allowed = {"session_id", "final_click_at", "actual_publish_at", "result_url", "next_attempt_at"} + for key, value in values.items(): + if key in allowed: + fields.append(f"{key} = ?") + params.append(value) + params.append(job_id) + with proxy_pool.connect() as conn: + conn.execute(f"UPDATE publish_jobs SET {', '.join(fields)} WHERE id = ?", params) + conn.commit() + + +def _update_account(account_id: int, error: str = "", published_at: str = "") -> None: + with proxy_pool.connect() as conn: + conn.execute( + "UPDATE tiktok_accounts SET last_error = ?, last_publish_at = COALESCE(NULLIF(?, ''), last_publish_at), updated_at = ? WHERE id = ?", + (_clean_text(error, 2000), published_at, _iso(), account_id), + ) + conn.commit() + + +def _first_visible(locators: list[Any]) -> Any | None: + for locator in locators: + try: + count = min(locator.count(), 8) + for index in range(count): + item = locator.nth(index) + if item.is_visible(): + return item + except Exception: + continue + return None + + +def _skip_onboarding(page: Any) -> None: + pattern = re.compile(r"^(skip|skip for now|not now|got it|later|跳过|暂不|稍后|知道了)$", re.I) + for _ in range(4): + button = _first_visible([page.get_by_role("button", name=pattern), page.get_by_text(pattern, exact=True)]) + if not button: + return + button.click(timeout=3000) + page.wait_for_timeout(500) + + +def _assert_account_ready(page: Any) -> None: + url = page.url.lower() + if "/login" in url: + raise ManualReviewRequired("TikTok 登录已失效，请从观测通道重新登录") + challenge = _first_visible([ + page.get_by_text(re.compile(r"captcha|verify to continue|security verification|验证码|安全验证", re.I)), + page.locator("iframe[src*='captcha']"), + ]) + if challenge: + raise ManualReviewRequired("TikTok 要求验证码或安全验证，请从观测通道人工处理") + + +def _set_description(page: Any, description: str) -> None: + if not description: + return + target = _first_visible([ + page.locator("[data-e2e*='caption'] [contenteditable='true']"), + page.locator("[contenteditable='true'][role='textbox']"), + page.get_by_label(re.compile(r"description|caption|说明|描述", re.I)), + page.locator("textarea[placeholder*='caption' i], textarea[placeholder*='description' i]"), + ]) + if not target: + raise RuntimeError("未找到 TikTok Studio 的 Description 输入框") + target.click() + try: + target.fill(description) + except Exception: + target.press("Control+A") + target.type(description) + + +def _set_ai_generated(page: Any, enabled: bool) -> None: + if not enabled: + return + show_more = _first_visible([ + page.get_by_role("button", name=re.compile(r"show more|更多|展开", re.I)), + page.get_by_text(re.compile(r"^show more$|^显示更多$|^更多设置$", re.I), exact=True), + ]) + if show_more: + show_more.click() + page.wait_for_timeout(500) + label_pattern = re.compile(r"AI.generated content|AI 生成|人工智能生成", re.I) + checkbox = _first_visible([page.get_by_role("checkbox", name=label_pattern), page.get_by_label(label_pattern)]) + if checkbox: + if not checkbox.is_checked(): + checkbox.check() + return + label = _first_visible([page.get_by_text(label_pattern)]) + if not label: + raise RuntimeError("未找到 AI-generated content 设置") + label.click() + + +def _set_schedule(page: Any, mode: str, scheduled_at: str) -> None: + if mode == "server": + now_option = _first_visible([ + page.get_by_role("radio", name=re.compile(r"^now$|立即|现在", re.I)), + page.get_by_text(re.compile(r"^now$|^立即发布$|^现在$", re.I), exact=True), + ]) + if now_option: + now_option.click() + return + + schedule_option = _first_visible([ + page.get_by_role("radio", name=re.compile(r"schedule|定时发布", re.I)), + page.get_by_text(re.compile(r"^schedule$|^定时发布$", re.I), exact=True), + ]) + if not schedule_option: + raise RuntimeError("当前 TikTok Studio 页面不支持定时发布") + schedule_option.click() + local = _parse_schedule(scheduled_at).astimezone(ZoneInfo(TIMEZONE_NAME)) + date_value = local.strftime("%Y-%m-%d") + time_value = local.strftime("%H:%M") + date_input = _first_visible([page.locator("input[type='date']"), page.get_by_label(re.compile(r"date|日期", re.I))]) + time_input = _first_visible([page.locator("input[type='time']"), page.get_by_label(re.compile(r"time|时间", re.I))]) + if not date_input or not time_input: + raise RuntimeError("未找到 TikTok 定时发布的日期或时间输入框") + date_input.fill(date_value) + time_input.fill(time_value) + + +def _execute_browser(job: dict[str, Any], session: dict[str, Any]) -> tuple[str, str]: + from playwright.sync_api import sync_playwright + + log_dir = LOG_ROOT / job["id"] + log_dir.mkdir(parents=True, exist_ok=True) + video = video_path(job["asset_id"]) + final_clicked = False + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp(f"http://127.0.0.1:{session['debug_port']}") + try: + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() + try: + page.wait_for_load_state("domcontentloaded", timeout=20000) + except Exception: + pass + page.wait_for_timeout(2000) + _assert_account_ready(page) + page.goto("https://www.tiktok.com/tiktokstudio?lang=en", wait_until="domcontentloaded", timeout=60000) + page.wait_for_timeout(2500) + _assert_account_ready(page) + _skip_onboarding(page) + upload_link = _first_visible([ + page.locator("a[href*='/tiktokstudio/upload']"), + page.get_by_role("link", name=re.compile(r"upload|create|上传|发布", re.I)), + page.get_by_role("button", name=re.compile(r"upload|create|上传|发布", re.I)), + ]) + if upload_link: + upload_link.click() + page.wait_for_timeout(1500) + if "/tiktokstudio/upload" not in page.url: + page.goto("https://www.tiktok.com/tiktokstudio/upload?from=creator_center&tab=video", wait_until="domcontentloaded", timeout=60000) + _skip_onboarding(page) + _assert_account_ready(page) + file_inputs = page.locator("input[type='file'][accept*='video']") + if not file_inputs.count(): + file_inputs = page.locator("input[type='file']") + if not file_inputs.count(): + raise RuntimeError("未找到 TikTok Studio 视频选择控件") + _set_job(job["id"], "uploading", "uploading", session_id=session["id"]) + file_inputs.first.set_input_files(str(video)) + page.wait_for_timeout(3000) + _set_description(page, job["description"]) + _set_ai_generated(page, bool(job["ai_generated"])) + _set_schedule(page, job["schedule_mode"], job["scheduled_at"]) + post_button = _first_visible([ + page.get_by_role("button", name=re.compile(r"^post$|^publish$|^发布$", re.I)), + page.locator("button[data-e2e*='post']"), + ]) + if not post_button: + raise RuntimeError("未找到 TikTok Studio 最终发布按钮") + deadline = time.time() + UPLOAD_TIMEOUT_SECONDS + while time.time() < deadline: + _assert_account_ready(page) + upload_error = _first_visible([ + page.get_by_text(re.compile(r"upload failed|couldn't upload|上传失败|处理失败", re.I)), + ]) + if upload_error: + raise RuntimeError("TikTok Studio 报告视频上传或处理失败") + try: + if post_button.is_enabled(): + break + except Exception: + pass + page.wait_for_timeout(1000) + else: + raise RuntimeError(f"等待视频处理完成超过 {UPLOAD_TIMEOUT_SECONDS} 秒") + page.screenshot(path=str(log_dir / "ready-to-publish.png"), full_page=True) + if DRY_RUN: + return "dry_run", page.url + _set_job(job["id"], "publishing", "final_click", session_id=session["id"], final_click_at=_iso()) + post_button.click() + final_clicked = True + try: + page.wait_for_url(re.compile(r"tiktokstudio(?!/upload)|manage|content"), timeout=45000) + except Exception: + success = _first_visible([ + page.get_by_text(re.compile(r"uploaded|published|scheduled|上传成功|发布成功|已定时", re.I)), + ]) + if not success: + raise ResultUncertain("已点击发布，但未收到明确成功信号，请人工确认，系统不会自动重试") + return ("scheduled_on_tiktok" if job["schedule_mode"] == "tiktok" else "published"), page.url + except (ManualReviewRequired, ResultUncertain): + raise + except Exception as exc: + if final_clicked: + raise ResultUncertain(f"最终发布后页面异常：{exc}") from exc + raise + finally: + try: + current_page = context.pages[0] if context.pages else None + if current_page: + current_page.screenshot(path=str(log_dir / "last-state.png"), full_page=True) + except Exception: + pass + # Leaving the CDP browser alive here lets proxy_pool own process cleanup + # and preserves the same noVNC channel for manual review when required. + + +def _run_job(job_id: str) -> None: + session_id = 0 + keep_for_review = False + try: + jobs = _job_query("j.id = ?", (job_id,)) + if not jobs: + return + job = jobs[0] + session_result = proxy_pool.start_automation_session(int(job["account_id"]), job_id) + session = session_result["session"] + session_id = int(session["id"]) + _set_job(job_id, "preparing", "browser_ready", session_id=session_id) + status, result_url = _execute_browser(job, session) + actual = "" if status == "dry_run" else (job["scheduled_at"] if status == "scheduled_on_tiktok" else _iso()) + _set_job(job_id, status, "complete", result_url=result_url, actual_publish_at=actual) + _update_account(int(job["account_id"]), published_at=actual if status != "dry_run" else "") + except ManualReviewRequired as exc: + keep_for_review = True + _set_job(job_id, "failed", "manual_review", str(exc), session_id=session_id or None) + _update_account(int(job["account_id"]), error=str(exc)) + if session_id: + proxy_pool.handoff_automation_session(session_id, str(exc)) + except ResultUncertain as exc: + _set_job(job_id, "result_uncertain", "confirm_required", str(exc), session_id=session_id or None) + _update_account(int(job["account_id"]), error=str(exc)) + except Exception as exc: + message = str(exc) + if "槽位已满" in message or "已经处于唤醒状态" in message: + _set_job(job_id, "delayed", "waiting_slot", message, next_attempt_at=_iso(_utc_now() + timedelta(seconds=30))) + else: + _set_job(job_id, "failed", "failed", message, session_id=session_id or None) + if 'job' in locals(): + _update_account(int(job["account_id"]), error=message) + finally: + if session_id and not keep_for_review: + try: + proxy_pool.finish_automation_session(session_id) + except Exception as exc: + print(f"Publish session cleanup failed for {job_id}: {exc}", flush=True) + with _worker_lock: + _active_jobs.discard(job_id) + + +def _claim_due_jobs() -> list[str]: + now = _utc_now() + native_due = now + timedelta(seconds=NATIVE_LEAD_SECONDS) + with _worker_lock: + capacity = max(0, proxy_pool.browser_max_slots() - len(_active_jobs)) + if capacity <= 0: + return [] + claimed: list[str] = [] + with proxy_pool.connect() as conn: + conn.execute("BEGIN IMMEDIATE") + rows = conn.execute( + """ + SELECT id FROM publish_jobs + WHERE status IN ('queued','delayed') + AND (next_attempt_at = '' OR next_attempt_at <= ?) + AND ((schedule_mode = 'server' AND scheduled_at <= ?) + OR (schedule_mode = 'tiktok' AND scheduled_at <= ?)) + ORDER BY scheduled_at ASC LIMIT ? + """, + (_iso(now), _iso(now), _iso(native_due), capacity), + ).fetchall() + for row in rows: + job_id = str(row["id"]) + conn.execute( + "UPDATE publish_jobs SET status = 'preparing', stage = 'claimed', attempt_count = attempt_count + 1, next_attempt_at = '', updated_at = ? WHERE id = ? AND status IN ('queued','delayed')", + (_iso(), job_id), + ) + claimed.append(job_id) + conn.commit() + return claimed + + +def _recover_interrupted() -> None: + now = _iso() + with proxy_pool.connect() as conn: + conn.execute( + "UPDATE publish_jobs SET status = CASE WHEN final_click_at <> '' THEN 'result_uncertain' ELSE 'queued' END, stage = 'recovered', last_error = '服务器重启后恢复任务', session_id = NULL, next_attempt_at = '', updated_at = ? WHERE status IN ('preparing','uploading','publishing')", + (now,), + ) + conn.commit() + + +def _worker_loop() -> None: + while True: + try: + for job_id in _claim_due_jobs(): + with _worker_lock: + if job_id in _active_jobs: + continue + _active_jobs.add(job_id) + threading.Thread(target=_run_job, args=(job_id,), daemon=True, name=f"tiktok-publish-{job_id[:8]}").start() + except Exception as exc: + print(f"TikTok publish scheduler failed: {exc}", flush=True) + time.sleep(5) + + +def start_worker() -> None: + global _worker_started + with _worker_lock: + if _worker_started: + return + _worker_started = True + PUBLISH_ROOT.mkdir(parents=True, exist_ok=True) + LOG_ROOT.mkdir(parents=True, exist_ok=True) + with proxy_pool.connect(): + pass + _recover_interrupted() + threading.Thread(target=_worker_loop, daemon=True, name="tiktok-publish-scheduler").start() diff --git a/scripts/web_app.py b/scripts/web_app.py index cdba9c7..189eda8 100644 --- a/scripts/web_app.py +++ b/scripts/web_app.py @@ -133,6 +133,7 @@ ) from proxy_state import ensure_us_proxy import proxy_pool +import tiktok_studio_publish MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024 SAFE_CHARS = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-") AUDIO_ONLY_SUFFIXES = {".aac", ".flac", ".m4a", ".mp3", ".ogg", ".opus", ".wav"} @@ -5195,7 +5196,7 @@ def do_GET(self) -> None: if parsed.path.startswith("/api/proxy/"): if not PROXY_POOL_ENABLED: return json_response(self, HTTPStatus.NOT_FOUND, {"error": "Not found"}) - return self.handle_proxy_api_get(parsed.path) + return self.handle_proxy_api_get(parsed.path, parsed.query) if parsed.path.startswith("/assets/"): return self.serve_static_asset(parsed.path.removeprefix("/assets/")) if parsed.path == "/api/prompt": @@ -5826,7 +5827,7 @@ def read_json_body(self) -> dict[str, Any]: raise ValueError("JSON body must be an object") return data - def handle_proxy_api_get(self, path: str) -> None: + def handle_proxy_api_get(self, path: str, query: str = "") -> None: try: if path == "/api/proxy/pools": return json_response(self, HTTPStatus.OK, proxy_pool.list_state()) @@ -5834,12 +5835,34 @@ def handle_proxy_api_get(self, path: str) -> None: return json_response(self, HTTPStatus.OK, proxy_pool.mihomo_export()) if path == "/api/proxy/runtime": return json_response(self, HTTPStatus.OK, proxy_pool.runtime_status()) + if path == "/api/proxy/publish/jobs": + account_id = int(parse_qs(query).get("account_id", ["0"])[0] or 0) + return json_response(self, HTTPStatus.OK, tiktok_studio_publish.list_jobs(account_id)) + if path == "/api/proxy/publish/runtime": + return json_response(self, HTTPStatus.OK, tiktok_studio_publish.runtime_status()) + if path.startswith("/api/proxy/publish/videos/"): + asset_id = unquote(path.removeprefix("/api/proxy/publish/videos/")) + return self.serve_video(tiktok_studio_publish.video_path(asset_id)) return json_response(self, HTTPStatus.NOT_FOUND, {"error": "Not found"}) except Exception as exc: return json_response(self, HTTPStatus.INTERNAL_SERVER_ERROR, {"error": str(exc)}) def handle_proxy_api_post(self, path: str) -> None: try: + if path == "/api/proxy/publish/jobs": + content_length = int(self.headers.get("Content-Length", "0") or "0") + if content_length <= 0 or content_length > tiktok_studio_publish.MAX_UPLOAD_BYTES + 2 * 1024 * 1024: + raise ValueError("上传内容为空或超过 2GB 限制") + form = cgi.FieldStorage( + fp=self.rfile, + headers=self.headers, + environ={ + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": self.headers.get("Content-Type", ""), + "CONTENT_LENGTH": str(content_length), + }, + ) + return json_response(self, HTTPStatus.ACCEPTED, tiktok_studio_publish.create_job(form)) payload = self.read_json_body() if path == "/api/proxy/pools": return json_response(self, HTTPStatus.OK, proxy_pool.upsert_pool(payload)) @@ -5861,6 +5884,10 @@ def handle_proxy_api_post(self, path: str) -> None: return json_response(self, HTTPStatus.OK, proxy_pool.stop_login_session(payload)) if path == "/api/proxy/login-session/status": return json_response(self, HTTPStatus.OK, proxy_pool.inspect_login_session(payload)) + if path == "/api/proxy/publish/jobs/update": + return json_response(self, HTTPStatus.OK, tiktok_studio_publish.update_job(payload)) + if path == "/api/proxy/publish/jobs/cancel": + return json_response(self, HTTPStatus.OK, tiktok_studio_publish.cancel_job(payload)) return json_response(self, HTTPStatus.NOT_FOUND, {"error": "Not found"}) except ValueError as exc: return json_response(self, HTTPStatus.BAD_REQUEST, {"error": str(exc)}) @@ -6695,6 +6722,7 @@ def main() -> int: mark_interrupted_chat_messages() if PROXY_POOL_ENABLED: threading.Thread(target=proxy_session_janitor, daemon=True).start() + tiktok_studio_publish.start_worker() normalize_stored_chat_tool_results() video_queue.start(execute_queue_job) report_scheduler_enabled = os.getenv("HOT_VIDEO_REPORT_SCHEDULER_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"} From b3374657e6127afb1da1cfff0f02e385f5fbf759 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 18:10:23 +0800 Subject: [PATCH 009/187] chore: isolate development Docker image --- docker-compose.dev.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 7935e08..8596d9e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,5 +1,6 @@ services: analyzer: + image: short-video-analyzer-dev:latest volumes: - ./videos-dev:/workspace/videos - ./output-dev:/workspace/output @@ -9,6 +10,7 @@ services: HOT_VIDEO_REPORT_SCHEDULER_ENABLED: "0" web: + image: short-video-analyzer-dev:latest environment: WEB_PORT: ${WEB_PORT:-4003} PROXY_POOL_ENABLED: "1" From bedf08fa2ecb5fa45f868a25bb9196a179bd8d56 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 22:17:13 +0800 Subject: [PATCH 010/187] ui: remove proxy runtime overview --- scripts/static/proxy.html | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/static/proxy.html b/scripts/static/proxy.html index cf7ed0c..d14e1b3 100644 --- a/scripts/static/proxy.html +++ b/scripts/static/proxy.html @@ -6,7 +6,7 @@ 账号 IP 池 @@ -41,17 +42,20 @@

发布队列

正在加载...
-

视频详情

留空则不添加链接；填写内容先作为商品待绑定信息保存，后续从账号实时商品列表选择。
选择视频并设置发布时间。
+

视频详情

未添加商品。
选择视频并设置发布时间。
+ + + diff --git a/scripts/web_app.py b/scripts/web_app.py index 66cd670..66c72ec 100644 --- a/scripts/web_app.py +++ b/scripts/web_app.py @@ -96,7 +96,8 @@ import sys sys.path.insert(0, str(SCRIPTS_DIR)) -from chat_session import ChatStore, Message, Session, load_sessions_from_disk +from chat_session import ChatStore, Message, Session, load_sessions_from_disk +from lan_chat import LanChatError, LanChatStore from sociavault_usage import read_sociavault_usage from sociavault_tiktok import call_api as call_sociavault_tiktok_api from tools import TOOLS, execute_tool, get_tools_for_model, list_tools @@ -333,7 +334,8 @@ class AmazonJob: social_jobs_running: set[str] = set() # Chat system -chat_store = ChatStore(DATA_DIR / "sessions.json") +chat_store = ChatStore(DATA_DIR / "sessions.json") +lan_chat_store = LanChatStore(DATA_DIR / "lan_chat.sqlite") chat_provider_stores = { "home": chat_store, "amazon": ChatStore(SELLERSPRITE_CHAT_DATA_DIR / "chat_sessions.json"), @@ -351,7 +353,8 @@ class AmazonJob: MCP_TOOL_CACHE: dict[str, dict[str, Any]] = {} PROXY_POOL_ENABLED = os.getenv("PROXY_POOL_ENABLED", "0").strip().lower() in {"1", "true", "yes", "on"} NAV_ITEMS = [ - {"key": "home", "href": "/", "label": "\u9996\u9875", "title": "AI \u804a\u5929", "icon": ''}, + {"key": "home", "href": "/", "label": "\u9996\u9875", "title": "AI \u804a\u5929", "icon": ''}, + {"key": "lan-chat", "href": "/lan-chat", "label": "\u90bb\u804a", "title": "\u5c40\u57df\u7f51\u804a\u5929", "icon": ''}, {"key": "report", "href": "/report", "label": "\u65e5\u62a5", "title": "\u6bcf\u65e5\u62a5\u544a", "icon": ''}, {"key": "amazon", "href": "/amazon", "label": "Amazon", "title": "Amazon", "icon": ''}, {"key": "fastmoss", "href": "/fastmoss", "label": "FastMoss", "title": "FastMoss", "icon": ''}, @@ -5154,6 +5157,109 @@ def proxy_mcp_chat(handler: BaseHTTPRequestHandler, chat_type: str) -> None: def proxy_sellersprite_chat(handler: BaseHTTPRequestHandler) -> None: return proxy_mcp_chat(handler, "sellersprite") + +def _lan_chat_token(handler: BaseHTTPRequestHandler) -> str: + return handler.headers.get("X-Lan-Chat-Token", "").strip() + + +def _lan_chat_request_json(handler: BaseHTTPRequestHandler) -> dict[str, Any]: + try: + length = int(handler.headers.get("Content-Length", "0") or "0") + except ValueError as exc: + raise LanChatError("请求长度无效") from exc + if length < 0 or length > 65536: + raise LanChatError("请求内容过大", 413) + try: + payload = json.loads(handler.rfile.read(length).decode("utf-8")) if length else {} + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise LanChatError("请求 JSON 无效") from exc + if not isinstance(payload, dict): + raise LanChatError("请求内容必须是对象") + return payload + + +def handle_lan_chat_get(handler: BaseHTTPRequestHandler, parsed) -> bool: + path = parsed.path + try: + if path == "/api/lan-chat/bootstrap": + json_response(handler, HTTPStatus.OK, lan_chat_store.bootstrap(_lan_chat_token(handler))) + return True + avatar_match = re.fullmatch(r"/api/lan-chat/avatars/([0-9a-f]{16})", path) + if avatar_match: + body, content_type = lan_chat_store.avatar_bytes(avatar_match.group(1)) + binary_response(handler, HTTPStatus.OK, body, content_type) + return True + message_match = re.fullmatch(r"/api/lan-chat/rooms/([^/]+)/messages", path) + if message_match: + query = parse_qs(parsed.query) + try: + after_id = int(query.get("after", ["0"])[0]) + limit = int(query.get("limit", ["100"])[0]) + except ValueError as exc: + raise LanChatError("分页参数无效") from exc + payload = lan_chat_store.list_messages( + _lan_chat_token(handler), unquote(message_match.group(1)), after_id, limit + ) + json_response(handler, HTTPStatus.OK, payload) + return True + except LanChatError as exc: + json_response(handler, exc.status, {"error": str(exc)}) + return True + return False + + +def handle_lan_chat_post(handler: BaseHTTPRequestHandler, parsed) -> bool: + path = parsed.path + if not path.startswith("/api/lan-chat/"): + return False + try: + payload = _lan_chat_request_json(handler) + if path == "/api/lan-chat/register": + user, created = lan_chat_store.register( + str(payload.get("deviceToken") or ""), str(payload.get("nickname") or "") + ) + json_response(handler, HTTPStatus.CREATED if created else HTTPStatus.OK, { + "user": user, + "created": created, + }) + return True + if path == "/api/lan-chat/profile": + user = lan_chat_store.update_profile( + _lan_chat_token(handler), str(payload.get("nickname") or "") + ) + json_response(handler, HTTPStatus.OK, {"user": user}) + return True + if path == "/api/lan-chat/direct": + room = lan_chat_store.open_direct( + _lan_chat_token(handler), str(payload.get("targetUserId") or "") + ) + json_response(handler, HTTPStatus.OK, {"room": room}) + return True + if path == "/api/lan-chat/rooms": + member_ids = payload.get("memberIds") + if member_ids is not None and not isinstance(member_ids, list): + raise LanChatError("memberIds 必须是数组") + room = lan_chat_store.create_group( + _lan_chat_token(handler), str(payload.get("name") or ""), member_ids + ) + json_response(handler, HTTPStatus.CREATED, {"room": room}) + return True + message_match = re.fullmatch(r"/api/lan-chat/rooms/([^/]+)/messages", path) + if message_match: + message = lan_chat_store.send_message( + _lan_chat_token(handler), + unquote(message_match.group(1)), + str(payload.get("content") or ""), + ) + json_response(handler, HTTPStatus.CREATED, {"message": message}) + return True + json_response(handler, HTTPStatus.NOT_FOUND, {"error": "LAN chat API not found"}) + return True + except LanChatError as exc: + json_response(handler, exc.status, {"error": str(exc)}) + return True + + class Handler(BaseHTTPRequestHandler): server_version = "ShortVideoAnalyzer/1.0" @@ -5172,6 +5278,9 @@ def do_GET(self) -> None: return proxy_mcp_chat(self, "fastmoss") if parsed.path == "/" or parsed.path == "/chat": return serve_chat_template(self, "home", parsed.path) + if parsed.path == "/lan-chat": + lan_chat_html = (SCRIPTS_DIR / "static" / "lan_chat.html").read_text(encoding="utf-8") + return text_response(self, HTTPStatus.OK, inject_unified_nav(lan_chat_html, parsed.path), "text/html; charset=utf-8") if parsed.path == "/report": report_html = (SCRIPTS_DIR / "static" / "report.html").read_text(encoding="utf-8") return text_response(self, HTTPStatus.OK, inject_unified_nav(report_html, parsed.path), "text/html; charset=utf-8") @@ -5199,6 +5308,8 @@ def do_GET(self) -> None: return self.handle_proxy_api_get(parsed.path, parsed.query) if parsed.path.startswith("/assets/"): return self.serve_static_asset(parsed.path.removeprefix("/assets/")) + if parsed.path.startswith("/api/lan-chat/") and handle_lan_chat_get(self, parsed): + return if parsed.path == "/api/prompt": return json_response(self, HTTPStatus.OK, {"prompt": load_prompt(), "feedback_prompt": load_feedback_prompt()}) if parsed.path == "/api/chat/sessions": @@ -5762,6 +5873,8 @@ def serve_video(self, path: Path) -> None: def do_POST(self) -> None: parsed = urlparse(self.path) + if parsed.path.startswith("/api/lan-chat/") and handle_lan_chat_post(self, parsed): + return if parsed.path == "/amazon/api/chat/export-pdf": return self.handle_mcp_chat_export_pdf("sellersprite") if parsed.path == "/fastmoss/api/chat/export-pdf": @@ -6720,9 +6833,10 @@ def proxy_session_janitor() -> None: def main() -> int: - load_env_file() - VIDEOS_DIR.mkdir(parents=True, exist_ok=True) - OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + load_env_file() + VIDEOS_DIR.mkdir(parents=True, exist_ok=True) + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + lan_chat_store.initialize() for store in chat_provider_stores.values(): load_sessions_from_disk(store) mark_interrupted_chat_messages() From 4be95a4a128699ad85c6281bc781043e56ae0e18 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 10:53:52 +0800 Subject: [PATCH 042/187] fix: preserve LAN chat nickname while editing --- scripts/static/lan_chat.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/static/lan_chat.html b/scripts/static/lan_chat.html index 11b8d3e..51992f9 100644 --- a/scripts/static/lan_chat.html +++ b/scripts/static/lan_chat.html @@ -86,7 +86,7 @@ async function register(){state.token=token();const response=await fetch("/api/lan-chat/register",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({deviceToken:state.token})});const payload=await response.json();if(!response.ok)throw new Error(payload.error||"设备注册失败");state.currentUser=payload.user;if(payload.created)openProfile(true)} async function bootstrap(render=true){const payload=await request("/api/lan-chat/bootstrap");state.currentUser=payload.currentUser;state.users=payload.users||[];state.rooms=payload.rooms||[];if(!state.rooms.some(room=>room.id===state.activeRoomId))state.activeRoomId=payload.publicRoomId||"public";if(render)renderAll();else{renderProfile();renderRooms();renderMembers();renderHeader()}return payload} function activeRoom(){return state.rooms.find(room=>room.id===state.activeRoomId)||state.rooms[0]} -function renderProfile(){const user=state.currentUser;if(!user)return;$("profileAvatar").src=user.avatarUrl;$("profileModalAvatar").src=user.avatarUrl;$("profileName").textContent=user.nickname;$("profileModalName").textContent=user.nickname;$("nicknameInput").value=user.nickname;$("avatarStatusText").textContent=user.avatarStatus==="ready"?"AI 默认头像已生成":user.avatarStatus==="pending"?"AI 默认头像正在后台生成":"本地默认头像已启用"} +function renderProfile(){const user=state.currentUser;if(!user)return;$("profileAvatar").src=user.avatarUrl;$("profileModalAvatar").src=user.avatarUrl;$("profileName").textContent=user.nickname;$("profileModalName").textContent=user.nickname;if(!$("profileModal").classList.contains("show"))$("nicknameInput").value=user.nickname;$("avatarStatusText").textContent=user.avatarStatus==="ready"?"AI 默认头像已生成":user.avatarStatus==="pending"?"AI 默认头像正在后台生成":"本地默认头像已启用"} function renderRoomList(kind,elementId){const list=$(elementId),rooms=state.rooms.filter(room=>room.kind===kind);list.innerHTML=rooms.map(room=>{const unread=(room.latestMessage?.createdAt||0)>(state.readRooms[room.id]||0)&&room.id!==state.activeRoomId;return ``}).join("")||`
${kind==="direct"?"点击右侧成员发起私信":"暂时没有群组"}
`;list.querySelectorAll("[data-room-id]").forEach(button=>button.onclick=()=>selectRoom(button.dataset.roomId))} function renderRooms(){renderRoomList("public","publicRooms");renderRoomList("direct","directRooms");renderRoomList("group","groupRooms")} function renderMembers(){const query=$("memberSearch").value.trim().toLowerCase(),users=state.users.filter(user=>!query||user.nickname.toLowerCase().includes(query));$("memberCount").textContent=`${state.users.length} 人`;$("memberList").innerHTML=users.map(user=>``).join("");$("memberList").querySelectorAll("[data-user-id]:not([disabled])").forEach(button=>button.onclick=()=>startDirect(button.dataset.userId))} From 19d0a4e8f93507360df6a1dfbb58a353aa996c99 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 10:46:38 +0800 Subject: [PATCH 043/187] feat: add TikTok Studio analytics collection v2 --- scripts/proxy_pool.py | 65 ++ scripts/static/proxy.html | 32 +- scripts/tiktok_studio_collect.py | 1013 ++++++++++++++++++++++++++++++ scripts/tiktok_studio_publish.py | 18 + scripts/web_app.py | 15 + 5 files changed, 1134 insertions(+), 9 deletions(-) create mode 100644 scripts/tiktok_studio_collect.py diff --git a/scripts/proxy_pool.py b/scripts/proxy_pool.py index 5712ec8..06755bd 100644 --- a/scripts/proxy_pool.py +++ b/scripts/proxy_pool.py @@ -237,6 +237,65 @@ def init_db(conn: sqlite3.Connection) -> None: ); CREATE INDEX IF NOT EXISTS idx_publish_jobs_account ON publish_jobs(account_id, created_at DESC); CREATE INDEX IF NOT EXISTS idx_publish_jobs_due ON publish_jobs(status, scheduled_at); + CREATE TABLE IF NOT EXISTS collect_settings ( + account_id INTEGER PRIMARY KEY REFERENCES tiktok_accounts(id) ON DELETE CASCADE, + enabled INTEGER NOT NULL DEFAULT 0, + daily_time TEXT NOT NULL DEFAULT '03:00', + max_videos INTEGER NOT NULL DEFAULT 20, + last_scheduled_date TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS collect_jobs ( + id TEXT PRIMARY KEY, + account_id INTEGER NOT NULL REFERENCES tiktok_accounts(id) ON DELETE RESTRICT, + proxy_profile_id INTEGER NOT NULL REFERENCES proxy_profiles(id) ON DELETE RESTRICT, + trigger_type TEXT NOT NULL DEFAULT 'manual', + schedule_date TEXT NOT NULL DEFAULT '', + max_videos INTEGER NOT NULL DEFAULT 20, + status TEXT NOT NULL DEFAULT 'queued', + stage TEXT NOT NULL DEFAULT '', + attempt_count INTEGER NOT NULL DEFAULT 0, + next_attempt_at TEXT NOT NULL DEFAULT '', + session_id INTEGER REFERENCES browser_sessions(id) ON DELETE SET NULL, + total_videos INTEGER NOT NULL DEFAULT 0, + completed_videos INTEGER NOT NULL DEFAULT 0, + failed_videos INTEGER NOT NULL DEFAULT 0, + current_video_id TEXT NOT NULL DEFAULT '', + started_at TEXT NOT NULL DEFAULT '', + completed_at TEXT NOT NULL DEFAULT '', + last_error TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_collect_jobs_account ON collect_jobs(account_id, created_at DESC); + CREATE INDEX IF NOT EXISTS idx_collect_jobs_due ON collect_jobs(status, next_attempt_at, created_at); + CREATE TABLE IF NOT EXISTS collect_results ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL REFERENCES collect_jobs(id) ON DELETE RESTRICT, + account_id INTEGER NOT NULL REFERENCES tiktok_accounts(id) ON DELETE RESTRICT, + video_id TEXT NOT NULL, + video_url TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + published_at TEXT NOT NULL DEFAULT '', + collected_at TEXT NOT NULL, + retention_complete INTEGER NOT NULL DEFAULT 0, + payload_json TEXT NOT NULL DEFAULT '{}', + UNIQUE(job_id, video_id) + ); + CREATE INDEX IF NOT EXISTS idx_collect_results_account ON collect_results(account_id, collected_at DESC); + CREATE INDEX IF NOT EXISTS idx_collect_results_video ON collect_results(account_id, video_id, collected_at DESC); + CREATE TABLE IF NOT EXISTS collect_errors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL REFERENCES collect_jobs(id) ON DELETE RESTRICT, + account_id INTEGER NOT NULL REFERENCES tiktok_accounts(id) ON DELETE RESTRICT, + video_id TEXT NOT NULL DEFAULT '', + video_url TEXT NOT NULL DEFAULT '', + stage TEXT NOT NULL DEFAULT '', + message TEXT NOT NULL, + created_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_collect_errors_job ON collect_errors(job_id, created_at DESC); """ ) for name, definition in { @@ -1516,6 +1575,12 @@ def delete_account(account_id: int) -> dict[str, Any]: ).fetchone() if active_job: raise ValueError("账号仍有草稿、待发布或运行中的发布任务，请先处理任务") + active_collect = conn.execute( + "SELECT id FROM collect_jobs WHERE account_id = ? AND status IN ('queued','delayed','preparing','collecting') LIMIT 1", + (account_id,), + ).fetchone() + if active_collect: + raise ValueError("账号仍有待执行或运行中的统计采集任务，请先处理任务") account = conn.execute("SELECT username FROM tiktok_accounts WHERE id = ? AND deleted_at = ''", (account_id,)).fetchone() if not account: raise ValueError("account not found") diff --git a/scripts/static/proxy.html b/scripts/static/proxy.html index 1dd0205..93baa43 100644 --- a/scripts/static/proxy.html +++ b/scripts/static/proxy.html @@ -11,6 +11,7 @@ @@ -51,11 +52,15 @@

添加商品链接

商品名称商品 ID价格库存状态
diff --git a/scripts/tiktok_studio_collect.py b/scripts/tiktok_studio_collect.py new file mode 100644 index 0000000..ef570b5 --- /dev/null +++ b/scripts/tiktok_studio_collect.py @@ -0,0 +1,1013 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import os +import re +import threading +import time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urljoin +from zoneinfo import ZoneInfo + +import proxy_pool + + +ROOT = Path.cwd() +LOG_ROOT = ROOT / "data" / "tiktok_collect_jobs" +TIMEZONE_NAME = os.getenv("TZ", "America/Los_Angeles") or "America/Los_Angeles" +DEFAULT_DAILY_TIME = os.getenv("TIKTOK_COLLECT_DAILY_TIME", "03:00").strip() or "03:00" +DEFAULT_MAX_VIDEOS = max(1, min(50, int(os.getenv("TIKTOK_COLLECT_MAX_VIDEOS", "20") or "20"))) +RETENTION_MAX_SECONDS = max(10, int(os.getenv("TIKTOK_COLLECT_RETENTION_MAX_SECONDS", "300") or "300")) +WORKER_INTERVAL_SECONDS = max(3, int(os.getenv("TIKTOK_COLLECT_WORKER_INTERVAL_SECONDS", "10") or "10")) +JOB_ACTIVE_STATUSES = {"queued", "delayed", "preparing", "collecting"} +JOB_RETRYABLE_STATUSES = {"failed", "partial", "cancelled"} +STATUS_LABELS = { + "queued": "待采集", + "delayed": "等待槽位", + "preparing": "准备中", + "collecting": "采集中", + "complete": "采集完成", + "partial": "部分失败", + "failed": "采集失败", + "cancelled": "已取消", +} + +_worker_started = False +_worker_lock = threading.Lock() +_active_jobs: set[str] = set() + + +class AccountReviewRequired(RuntimeError): + pass + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _iso(value: datetime | None = None) -> str: + return (value or _utc_now()).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _clean_text(value: Any, limit: int = 2000) -> str: + return str(value or "").strip()[:limit] + + +def _json_loads(value: Any, fallback: Any) -> Any: + try: + return json.loads(str(value or "")) + except Exception: + return fallback + + +def _validate_daily_time(value: Any) -> str: + raw = _clean_text(value, 5) + if not re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", raw): + raise ValueError("每日采集时间必须为 HH:MM") + return raw + + +def _max_videos(value: Any) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + parsed = DEFAULT_MAX_VIDEOS + return max(1, min(50, parsed)) + + +def _setting_row(row: Any | None, account_id: int) -> dict[str, Any]: + return { + "account_id": account_id, + "enabled": bool(row["enabled"]) if row else False, + "daily_time": str(row["daily_time"]) if row else DEFAULT_DAILY_TIME, + "max_videos": int(row["max_videos"]) if row else DEFAULT_MAX_VIDEOS, + "last_scheduled_date": str(row["last_scheduled_date"]) if row else "", + "timezone": TIMEZONE_NAME, + "updated_at": str(row["updated_at"]) if row else "", + } + + +def _job_row(row: Any) -> dict[str, Any]: + return { + "id": str(row["id"]), + "account_id": int(row["account_id"]), + "proxy_profile_id": int(row["proxy_profile_id"]), + "trigger_type": str(row["trigger_type"]), + "schedule_date": str(row["schedule_date"]), + "max_videos": int(row["max_videos"]), + "status": str(row["status"]), + "status_label": STATUS_LABELS.get(str(row["status"]), str(row["status"])), + "stage": str(row["stage"]), + "attempt_count": int(row["attempt_count"]), + "session_id": int(row["session_id"] or 0), + "total_videos": int(row["total_videos"]), + "completed_videos": int(row["completed_videos"]), + "failed_videos": int(row["failed_videos"]), + "current_video_id": str(row["current_video_id"]), + "started_at": str(row["started_at"]), + "completed_at": str(row["completed_at"]), + "last_error": str(row["last_error"]), + "created_at": str(row["created_at"]), + "updated_at": str(row["updated_at"]), + } + + +def _result_row(row: Any) -> dict[str, Any]: + payload = _json_loads(row["payload_json"], {}) + return { + "id": int(row["id"]), + "job_id": str(row["job_id"]), + "account_id": int(row["account_id"]), + "video_id": str(row["video_id"]), + "video_url": str(row["video_url"]), + "title": str(row["title"]), + "published_at": str(row["published_at"]), + "collected_at": str(row["collected_at"]), + "retention_complete": bool(row["retention_complete"]), + "payload": payload, + } + + +def _account(conn: Any, account_id: int) -> Any: + row = conn.execute( + "SELECT * FROM tiktok_accounts WHERE id = ? AND deleted_at = ''", + (account_id,), + ).fetchone() + if not row: + raise ValueError("account not found") + return row + + +def dashboard(account_id: int) -> dict[str, Any]: + if not account_id: + raise ValueError("account_id is required") + with proxy_pool.connect() as conn: + account = _account(conn, account_id) + setting = conn.execute("SELECT * FROM collect_settings WHERE account_id = ?", (account_id,)).fetchone() + jobs = [ + _job_row(row) + for row in conn.execute( + "SELECT * FROM collect_jobs WHERE account_id = ? ORDER BY created_at DESC LIMIT 40", + (account_id,), + ).fetchall() + ] + results = [ + _result_row(row) + for row in conn.execute( + "SELECT * FROM collect_results WHERE account_id = ? ORDER BY collected_at DESC, id DESC LIMIT 100", + (account_id,), + ).fetchall() + ] + errors = [ + dict(row) + for row in conn.execute( + "SELECT * FROM collect_errors WHERE account_id = ? ORDER BY created_at DESC, id DESC LIMIT 30", + (account_id,), + ).fetchall() + ] + return { + "account": {"id": int(account["id"]), "username": str(account["username"])}, + "setting": _setting_row(setting, account_id), + "jobs": jobs, + "results": results, + "errors": errors, + "worker": runtime_status(), + } + + +def save_settings(payload: dict[str, Any]) -> dict[str, Any]: + account_id = int(payload.get("account_id") or 0) + if not account_id: + raise ValueError("account_id is required") + enabled = 1 if payload.get("enabled") else 0 + daily_time = _validate_daily_time(payload.get("daily_time") or DEFAULT_DAILY_TIME) + max_videos = _max_videos(payload.get("max_videos")) + now = _iso() + with proxy_pool.connect() as conn: + _account(conn, account_id) + conn.execute( + """ + INSERT INTO collect_settings (account_id, enabled, daily_time, max_videos, last_scheduled_date, created_at, updated_at) + VALUES (?, ?, ?, ?, '', ?, ?) + ON CONFLICT(account_id) DO UPDATE SET + enabled = excluded.enabled, + daily_time = excluded.daily_time, + max_videos = excluded.max_videos, + updated_at = excluded.updated_at + """, + (account_id, enabled, daily_time, max_videos, now, now), + ) + conn.commit() + return dashboard(account_id) + + +def _insert_job( + conn: Any, + account: Any, + trigger_type: str, + max_videos: int, + schedule_date: str = "", + session_id: int = 0, +) -> str: + active = conn.execute( + "SELECT id FROM collect_jobs WHERE account_id = ? AND status IN ('queued','delayed','preparing','collecting') LIMIT 1", + (int(account["id"]),), + ).fetchone() + if active: + raise ValueError("该账号已有待执行或运行中的采集任务") + active_publish = conn.execute( + """ + SELECT id FROM publish_jobs + WHERE account_id = ? AND deleted_at = '' + AND status IN ('queued','delayed','preparing','uploading','publishing') + LIMIT 1 + """, + (int(account["id"]),), + ).fetchone() + if active_publish: + raise ValueError("该账号已有待执行或运行中的发布任务") + job_id = f"collect_{uuid.uuid4().hex}" + now = _iso() + conn.execute( + """ + INSERT INTO collect_jobs ( + id, account_id, proxy_profile_id, trigger_type, schedule_date, max_videos, + status, stage, attempt_count, next_attempt_at, session_id, + total_videos, completed_videos, failed_videos, current_video_id, + started_at, completed_at, last_error, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, 'queued', '', 0, '', ?, 0, 0, 0, '', '', '', '', ?, ?) + """, + ( + job_id, + int(account["id"]), + int(account["proxy_profile_id"]), + trigger_type, + schedule_date, + max_videos, + session_id or None, + now, + now, + ), + ) + return job_id + + +def create_job(payload: dict[str, Any]) -> dict[str, Any]: + account_id = int(payload.get("account_id") or 0) + if not account_id: + raise ValueError("account_id is required") + with proxy_pool.connect() as conn: + account = _account(conn, account_id) + setting = conn.execute("SELECT * FROM collect_settings WHERE account_id = ?", (account_id,)).fetchone() + max_videos = _max_videos(payload.get("max_videos") or (setting["max_videos"] if setting else DEFAULT_MAX_VIDEOS)) + job_id = _insert_job( + conn, + account, + "manual", + max_videos, + session_id=int(payload.get("observation_session_id") or 0), + ) + conn.commit() + data = dashboard(account_id) + data["job"] = next(job for job in data["jobs"] if job["id"] == job_id) + return data + + +def retry_job(payload: dict[str, Any]) -> dict[str, Any]: + job_id = _clean_text(payload.get("job_id") or payload.get("id"), 80) + if not job_id: + raise ValueError("job_id is required") + with proxy_pool.connect() as conn: + row = conn.execute("SELECT * FROM collect_jobs WHERE id = ?", (job_id,)).fetchone() + if not row: + raise ValueError("collect job not found") + if str(row["status"]) not in JOB_RETRYABLE_STATUSES: + raise ValueError("只有失败、部分失败或已取消的采集任务可以重试") + now = _iso() + conn.execute( + """ + UPDATE collect_jobs + SET status = 'queued', stage = 'retry_queued', attempt_count = 0, + next_attempt_at = '', session_id = ?, current_video_id = '', + completed_at = '', last_error = '', updated_at = ? + WHERE id = ? + """, + (int(payload.get("observation_session_id") or 0) or None, now, job_id), + ) + conn.commit() + account_id = int(row["account_id"]) + return dashboard(account_id) + + +def cancel_job(payload: dict[str, Any]) -> dict[str, Any]: + job_id = _clean_text(payload.get("job_id") or payload.get("id"), 80) + if not job_id: + raise ValueError("job_id is required") + with proxy_pool.connect() as conn: + row = conn.execute("SELECT * FROM collect_jobs WHERE id = ?", (job_id,)).fetchone() + if not row: + raise ValueError("collect job not found") + if str(row["status"]) not in {"queued", "delayed"}: + raise ValueError("只能取消尚未开始的采集任务") + conn.execute( + "UPDATE collect_jobs SET status = 'cancelled', stage = '', completed_at = ?, updated_at = ? WHERE id = ?", + (_iso(), _iso(), job_id), + ) + conn.commit() + account_id = int(row["account_id"]) + return dashboard(account_id) + + +def runtime_status() -> dict[str, Any]: + with proxy_pool.connect() as conn: + counts = { + str(row["status"]): int(row["count"]) + for row in conn.execute("SELECT status, COUNT(*) AS count FROM collect_jobs GROUP BY status").fetchall() + } + with _worker_lock: + active = sorted(_active_jobs) + return { + "worker_started": _worker_started, + "timezone": TIMEZONE_NAME, + "active_jobs": active, + "counts": counts, + "max_automatic_slots": proxy_pool.browser_max_slots(), + "retention_max_seconds": RETENTION_MAX_SECONDS, + } + + +def _set_job(job_id: str, status: str, stage: str = "", error: str = "", **values: Any) -> None: + fields = ["status = ?", "stage = ?", "last_error = ?", "updated_at = ?"] + params: list[Any] = [status, stage, _clean_text(error), _iso()] + allowed = { + "session_id", "total_videos", "completed_videos", "failed_videos", + "current_video_id", "started_at", "completed_at", "next_attempt_at", + } + for key, value in values.items(): + if key in allowed: + fields.append(f"{key} = ?") + params.append(value) + params.append(job_id) + with proxy_pool.connect() as conn: + conn.execute(f"UPDATE collect_jobs SET {', '.join(fields)} WHERE id = ?", params) + conn.commit() + + +def _record_error(job: dict[str, Any], video_id: str, video_url: str, stage: str, error: Exception | str) -> None: + with proxy_pool.connect() as conn: + conn.execute( + "INSERT INTO collect_errors (job_id, account_id, video_id, video_url, stage, message, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + job["id"], + job["account_id"], + _clean_text(video_id, 120), + _clean_text(video_url, 1000), + _clean_text(stage, 120), + _clean_text(error), + _iso(), + ), + ) + conn.commit() + + +def _save_result(job: dict[str, Any], payload: dict[str, Any]) -> None: + video = payload.get("video") or {} + with proxy_pool.connect() as conn: + conn.execute( + """ + INSERT INTO collect_results ( + job_id, account_id, video_id, video_url, title, published_at, + collected_at, retention_complete, payload_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(job_id, video_id) DO UPDATE SET + video_url = excluded.video_url, + title = excluded.title, + published_at = excluded.published_at, + collected_at = excluded.collected_at, + retention_complete = excluded.retention_complete, + payload_json = excluded.payload_json + """, + ( + job["id"], + job["account_id"], + _clean_text(video.get("id"), 120), + _clean_text(video.get("url"), 1000), + _clean_text(video.get("title"), 2000), + _clean_text(video.get("published_at"), 120), + payload["collected_at"], + 1 if payload.get("retention_complete") else 0, + json.dumps(payload, ensure_ascii=False, separators=(",", ":")), + ), + ) + conn.commit() + + +def _first_visible(locators: list[Any]) -> Any | None: + for locator in locators: + try: + for index in range(min(locator.count(), 10)): + item = locator.nth(index) + if item.is_visible(): + return item + except Exception: + continue + return None + + +def _assert_account_ready(page: Any) -> None: + if "/login" in page.url.lower(): + raise AccountReviewRequired("TikTok 登录已失效，请先从观测通道重新登录") + challenge = _first_visible([ + page.get_by_text(re.compile(r"captcha|verify to continue|security verification|验证码|安全验证", re.I)), + page.locator("iframe[src*='captcha']"), + ]) + if challenge: + raise AccountReviewRequired("TikTok 要求验证码或安全验证，请从观测通道人工处理") + + +def _skip_onboarding(page: Any) -> None: + pattern = re.compile(r"^(skip|skip for now|not now|got it|later|跳过|暂不|稍后|知道了)$", re.I) + for _ in range(4): + button = _first_visible([page.get_by_role("button", name=pattern), page.get_by_text(pattern, exact=True)]) + if not button: + return + button.click(timeout=3000) + page.wait_for_timeout(400) + + +def _video_id(url: str) -> str: + match = re.search(r"/analytics/(\d+)", url) + return match.group(1) if match else "" + + +def _discover_links_on_page(page: Any) -> list[dict[str, str]]: + rows: list[dict[str, str]] = [] + links = page.locator("a[href*='/tiktokstudio/analytics/']") + for index in range(min(links.count(), 200)): + link = links.nth(index) + try: + href = str(link.get_attribute("href") or "") + if not href: + continue + href = urljoin(page.url, href) + video_id = _video_id(href) + if not video_id: + continue + text = _clean_text(link.inner_text(), 2000) + rows.append({"id": video_id, "url": href, "title_hint": text}) + except Exception: + continue + return rows + + +def _discover_video_links(page: Any, max_videos: int) -> list[dict[str, str]]: + found: dict[str, dict[str, str]] = {} + + def collect() -> None: + for row in _discover_links_on_page(page): + found.setdefault(row["id"], row) + + collect() + for _ in range(5): + if len(found) >= max_videos: + break + page.mouse.wheel(0, 900) + page.wait_for_timeout(700) + collect() + + if len(found) < max_videos: + content_link = _first_visible([ + page.locator("a[href*='/tiktokstudio/content']"), + page.locator("a[href*='/tiktokstudio/manage']"), + page.get_by_role("link", name=re.compile(r"content|posts|manage|内容|作品", re.I)), + ]) + if content_link: + href = str(content_link.get_attribute("href") or "") + if href: + page.goto(urljoin(page.url, href), wait_until="domcontentloaded", timeout=60000) + else: + content_link.click(timeout=5000) + page.wait_for_timeout(1800) + _assert_account_ready(page) + for _ in range(8): + collect() + if len(found) >= max_videos: + break + page.mouse.wheel(0, 1000) + page.wait_for_timeout(700) + + return list(found.values())[:max_videos] + + +def _lines(text: str) -> list[str]: + return [re.sub(r"\s+", " ", line).strip() for line in str(text or "").splitlines() if line.strip()] + + +def _value_after_label(lines: list[str], labels: list[str]) -> str: + label_pattern = re.compile(r"^(?:" + "|".join(re.escape(label) for label in labels) + r")$", re.I) + inline_pattern = re.compile(r"^(?:" + "|".join(re.escape(label) for label in labels) + r")\s*[:：]?\s+(.+)$", re.I) + for index, line in enumerate(lines): + inline = inline_pattern.match(line) + if inline: + return inline.group(1).strip() + if label_pattern.match(line): + for candidate in lines[index + 1:index + 5]: + if not label_pattern.match(candidate): + return candidate + return "" + + +def _percent_section(lines: list[str], headings: list[str], stop_headings: list[str]) -> dict[str, str]: + start = -1 + heading_pattern = re.compile(r"^(?:" + "|".join(re.escape(item) for item in headings) + r")$", re.I) + stop_pattern = re.compile(r"^(?:" + "|".join(re.escape(item) for item in stop_headings) + r")$", re.I) if stop_headings else None + for index, line in enumerate(lines): + if heading_pattern.match(line): + start = index + 1 + break + if start < 0: + return {} + section: list[str] = [] + for line in lines[start:start + 80]: + if stop_pattern and stop_pattern.match(line): + break + section.append(line) + values: dict[str, str] = {} + percent_pattern = re.compile(r"^ str: + pattern = re.compile("|".join(re.escape(name) for name in names), re.I) + target = _first_visible([ + page.get_by_text(pattern, exact=False), + page.locator("[aria-label]").filter(has_text=pattern), + ]) + if not target: + return "" + for ancestor in [target, target.locator("xpath=.."), target.locator("xpath=../..")]: + try: + text = _clean_text(ancestor.inner_text(), 300) + numbers = re.findall(r"(?:\d[\d,.]*[KMB]?|<\d+(?:\.\d+)?%)", text, re.I) + if numbers: + return numbers[-1] + except Exception: + continue + return "" + + +def _overview(lines: list[str]) -> dict[str, str]: + return { + "play_count": _value_after_label(lines, ["Video views", "Views", "播放量"]), + "total_play_time": _value_after_label(lines, ["Total play time", "总播放时间"]), + "average_watch_time": _value_after_label(lines, ["Average watch time", "平均观看时间"]), + "completion_rate": _value_after_label(lines, ["Watched full video", "Full video watched", "已观看完整视频", "完播率"]), + "new_followers": _value_after_label(lines, ["New followers", "新增粉丝"]), + } + + +def _engagement(page: Any, overview: dict[str, str]) -> dict[str, str]: + return { + "play": overview.get("play_count", ""), + "likes": _locator_metric(page, ["Likes", "Like", "点赞"]), + "comments": _locator_metric(page, ["Comments", "Comment", "评论"]), + "shares": _locator_metric(page, ["Shares", "Share", "分享"]), + "favorites": _locator_metric(page, ["Favorites", "Favorite", "Saves", "收藏"]), + } + + +def _duration_seconds(text: str) -> int: + values: list[int] = [] + for minutes, seconds in re.findall(r"\b(\d+):(\d{2})\b", text): + values.append(int(minutes) * 60 + int(seconds)) + return max(values) if values else 0 + + +def _tooltip_value(text: str) -> tuple[int, str] | None: + match = re.search(r"(\d+):(\d{2})\s*(\d+(?:\.\d+)?%)", str(text or "")) + if not match: + return None + return int(match.group(1)) * 60 + int(match.group(2)), match.group(3) + + +def _retention_chart(page: Any) -> tuple[Any | None, int, str]: + charts = page.locator(".echarts-for-react") + fallback: tuple[Any | None, int, str] = (None, 0, "未找到留存率图表") + for index in range(min(charts.count(), 12)): + chart = charts.nth(index) + try: + if not chart.is_visible(): + continue + chart.scroll_into_view_if_needed(timeout=3000) + page.wait_for_timeout(250) + box = chart.bounding_box() + if not box or box["width"] < 280 or box["height"] > 120: + continue + context_text = "" + for levels in ("..", "../..", "../../..", "../../../.."): + try: + context_text = _clean_text(chart.locator(f"xpath={levels}").inner_text(), 5000) + if re.search(r"retention rate|观众留存|留存率", context_text, re.I): + break + except Exception: + continue + duration = _duration_seconds(context_text) + candidate = (chart, duration, "") + if re.search(r"retention rate|观众留存|留存率", context_text, re.I): + return candidate + fallback = candidate + except Exception: + continue + return fallback + + +def _sample_retention(page: Any) -> tuple[dict[str, str], bool, list[str], str]: + chart, duration, reason = _retention_chart(page) + if not chart: + return {}, False, [], reason + if duration <= 0: + return {}, False, [], "留存率图表没有可识别的视频时长" + if duration > RETENTION_MAX_SECONDS: + return {}, False, [], f"视频时长 {duration} 秒超过逐秒采集上限 {RETENTION_MAX_SECONDS} 秒" + box = chart.bounding_box() + if not box: + return {}, False, [], "留存率图表不可见" + plot_left = box["x"] + 10 + plot_width = max(1.0, box["width"] - 80) + y = box["y"] + box["height"] / 2 + rows: dict[int, str] = {} + targets = range(duration + 1) + + def read() -> tuple[int, str] | None: + try: + return _tooltip_value(chart.text_content() or "") + except Exception: + return None + + for second in targets: + x = plot_left + plot_width * second / max(1, duration) + page.mouse.move(x, y) + page.wait_for_timeout(85) + parsed = read() + if parsed and parsed[0] == second: + rows[second] = parsed[1] + + missing = [second for second in targets if second not in rows] + for second in missing: + target_x = plot_left + plot_width * second / max(1, duration) + offsets = list(range(-36, 37, 6)) + if second in {0, duration}: + offsets += list(range(-60, 61, 4)) + seen: set[int] = set() + for offset in offsets: + if offset in seen: + continue + seen.add(offset) + page.mouse.move(target_x + offset, y) + page.wait_for_timeout(60) + parsed = read() + if parsed and parsed[0] == second: + rows[second] = parsed[1] + break + + missing_labels = [f"{second // 60}:{second % 60:02d}" for second in targets if second not in rows] + output = {f"{second // 60}:{second % 60:02d}": rows[second] for second in sorted(rows)} + return output, not missing_labels, missing_labels, "" if not missing_labels else "部分秒点未命中 ECharts tooltip" + + +def _title_and_date(lines: list[str], hint: str) -> tuple[str, str]: + cleaned_hint = _lines(hint) + title = cleaned_hint[0] if cleaned_hint else "" + date_pattern = re.compile(r"(?:[A-Z][a-z]{2}\s+\d{1,2},\s+\d{4}|\d{4}[/-]\d{1,2}[/-]\d{1,2})") + published = "" + for line in cleaned_hint + lines[:80]: + match = date_pattern.search(line) + if match: + published = match.group(0) + break + if not title: + for line in lines[:50]: + if len(line) >= 8 and not re.match(r"^(TikTok Studio|Video analytics|Analytics)$", line, re.I): + title = line + break + return title, published + + +def _collect_video(page: Any, job: dict[str, Any], source: dict[str, str], log_dir: Path) -> dict[str, Any]: + page.goto(source["url"], wait_until="domcontentloaded", timeout=60000) + page.wait_for_timeout(2200) + _assert_account_ready(page) + body = page.locator("body").inner_text(timeout=15000) + lines = _lines(body) + overview = _overview(lines) + if not any(overview.values()): + page.screenshot(path=str(log_dir / f"{source['id']}-missing-overview.png"), full_page=True) + raise RuntimeError("视频分析页没有识别到概览指标") + title, published_at = _title_and_date(lines, source.get("title_hint", "")) + retention, retention_complete, missing, retention_reason = _sample_retention(page) + payload = { + "account": { + "id": job["account_id"], + "username": job["username"], + "proxy_profile_id": job["proxy_profile_id"], + "observed_ip": job["observed_ip"], + "browser_session_id": job["session_id"], + }, + "collection_job": {"job_id": job["id"], "trigger_type": job["trigger_type"]}, + "video": {"id": source["id"], "title": title, "published_at": published_at, "url": page.url}, + "time_filter": {"requested": None, "applied": None, "scope": "video_lifetime", "applied_successfully": False}, + "overview": overview, + "engagement": _engagement(page, overview), + "retention": retention, + "retention_complete": retention_complete, + "missing_retention_seconds": missing, + "retention_reason": retention_reason, + "traffic_sources": _percent_section(lines, ["Traffic source", "Traffic sources", "流量来源"], ["Search queries", "搜索查询"]), + "search_queries": _percent_section(lines, ["Search queries", "搜索查询"], ["Viewer types", "Audience", "观众"]), + "updated_at": _value_after_label(lines, ["Updated", "Last updated", "更新时间"]), + "collected_at": _iso(), + } + page.screenshot(path=str(log_dir / f"{source['id']}-collected.png"), full_page=True) + return payload + + +def _load_job(job_id: str) -> dict[str, Any] | None: + with proxy_pool.connect() as conn: + row = conn.execute( + """ + SELECT j.*, a.username, a.last_checked_ip + FROM collect_jobs j JOIN tiktok_accounts a ON a.id = j.account_id + WHERE j.id = ? + """, + (job_id,), + ).fetchone() + if not row: + return None + job = _job_row(row) + job["username"] = str(row["username"]) + job["observed_ip"] = str(row["last_checked_ip"]) + return job + + +def _completed_video_ids(job_id: str) -> set[str]: + with proxy_pool.connect() as conn: + return { + str(row["video_id"]) + for row in conn.execute("SELECT video_id FROM collect_results WHERE job_id = ?", (job_id,)).fetchall() + } + + +def _execute_browser(job: dict[str, Any], session: dict[str, Any]) -> tuple[int, int, int]: + from playwright.sync_api import sync_playwright + + log_dir = LOG_ROOT / job["id"] + log_dir.mkdir(parents=True, exist_ok=True) + completed_ids = _completed_video_ids(job["id"]) + completed = len(completed_ids) + failed = 0 + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp(f"http://127.0.0.1:{session['debug_port']}") + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() + page.goto("https://www.tiktok.com/tiktokstudio?lang=en", wait_until="domcontentloaded", timeout=60000) + page.wait_for_timeout(2200) + _assert_account_ready(page) + _skip_onboarding(page) + links = _discover_video_links(page, int(job["max_videos"])) + if not links: + page.screenshot(path=str(log_dir / "no-video-links.png"), full_page=True) + raise RuntimeError("TikTok Studio 没有发现可采集的视频分析入口") + _set_job( + job["id"], + "collecting", + "video_list_ready", + session_id=session["id"], + total_videos=len(links), + completed_videos=completed, + failed_videos=0, + ) + for source in links: + with proxy_pool.connect() as conn: + current = conn.execute("SELECT status FROM collect_jobs WHERE id = ?", (job["id"],)).fetchone() + if current and str(current["status"]) == "cancelled": + break + if source["id"] in completed_ids: + continue + _set_job( + job["id"], + "collecting", + "collect_video", + session_id=session["id"], + total_videos=len(links), + completed_videos=completed, + failed_videos=failed, + current_video_id=source["id"], + ) + job["session_id"] = int(session["id"]) + try: + payload = _collect_video(page, job, source, log_dir) + _save_result(job, payload) + completed += 1 + completed_ids.add(source["id"]) + except AccountReviewRequired: + raise + except Exception as exc: + failed += 1 + _record_error(job, source["id"], source["url"], "collect_video", exc) + _set_job( + job["id"], + "collecting", + "collect_video", + session_id=session["id"], + total_videos=len(links), + completed_videos=completed, + failed_videos=failed, + current_video_id="", + ) + return len(links), completed, failed + + +def _update_account(account_id: int, collected_at: str = "", error: str = "") -> None: + with proxy_pool.connect() as conn: + conn.execute( + """ + UPDATE tiktok_accounts + SET last_collect_at = COALESCE(NULLIF(?, ''), last_collect_at), + last_error = ?, updated_at = ? + WHERE id = ? + """, + (collected_at, _clean_text(error), _iso(), account_id), + ) + conn.commit() + + +def _run_job(job_id: str) -> None: + session_id = 0 + reused_observation = False + try: + job = _load_job(job_id) + if not job: + return + requested_session_id = int(job.get("session_id") or 0) + session = proxy_pool.claim_observation_session_for_job(job["account_id"], requested_session_id, job_id) + if session is not None: + reused_observation = True + else: + session = proxy_pool.start_automation_session(job["account_id"], job_id)["session"] + session_id = int(session["id"]) + _set_job(job_id, "preparing", "browser_ready", session_id=session_id, started_at=_iso()) + total, completed, failed = _execute_browser(job, session) + status = "complete" if failed == 0 else ("partial" if completed else "failed") + message = "" if failed == 0 else f"{failed} 个视频采集失败" + _set_job( + job_id, + status, + "complete", + message, + session_id=session_id, + total_videos=total, + completed_videos=completed, + failed_videos=failed, + current_video_id="", + completed_at=_iso(), + ) + _update_account(job["account_id"], collected_at=_iso() if completed else "", error=message) + except Exception as exc: + message = str(exc) + if "槽位已满" in message or "已经处于唤醒状态" in message: + _set_job(job_id, "delayed", "waiting_slot", message, next_attempt_at=_iso(_utc_now() + timedelta(seconds=30))) + else: + _set_job(job_id, "failed", "failed", message, session_id=session_id or None, completed_at=_iso()) + job = _load_job(job_id) + if job: + _record_error(job, "", "", "job", message) + _update_account(job["account_id"], error=message) + finally: + if session_id and reused_observation: + try: + proxy_pool.release_observation_session_job(session_id, job_id) + except Exception as exc: + print(f"Collect observation session release failed for {job_id}: {exc}", flush=True) + elif session_id: + try: + proxy_pool.finish_automation_session(session_id, "自动采集任务结束") + except Exception as exc: + print(f"Collect session cleanup failed for {job_id}: {exc}", flush=True) + with _worker_lock: + _active_jobs.discard(job_id) + + +def _schedule_daily_jobs() -> None: + local_now = _utc_now().astimezone(ZoneInfo(TIMEZONE_NAME)) + local_date = local_now.date().isoformat() + local_time = local_now.strftime("%H:%M") + with proxy_pool.connect() as conn: + conn.execute("BEGIN IMMEDIATE") + settings = conn.execute( + """ + SELECT s.*, a.proxy_profile_id, a.deleted_at + FROM collect_settings s JOIN tiktok_accounts a ON a.id = s.account_id + WHERE s.enabled = 1 + """ + ).fetchall() + for setting in settings: + if setting["deleted_at"] or str(setting["last_scheduled_date"]) == local_date: + continue + if local_time < str(setting["daily_time"]): + continue + account = _account(conn, int(setting["account_id"])) + try: + _insert_job(conn, account, "daily", int(setting["max_videos"]), schedule_date=local_date) + except ValueError: + # Keep trying after the account becomes idle; recording the date + # here would silently drop today's scheduled collection. + continue + conn.execute( + "UPDATE collect_settings SET last_scheduled_date = ?, updated_at = ? WHERE account_id = ?", + (local_date, _iso(), int(setting["account_id"])), + ) + conn.commit() + + +def _claim_due_jobs() -> list[str]: + with _worker_lock: + capacity = max(0, proxy_pool.browser_max_slots() - len(_active_jobs)) + if capacity <= 0: + return [] + claimed: list[str] = [] + with proxy_pool.connect() as conn: + conn.execute("BEGIN IMMEDIATE") + rows = conn.execute( + """ + SELECT id FROM collect_jobs + WHERE status IN ('queued','delayed') + AND (next_attempt_at = '' OR next_attempt_at <= ?) + ORDER BY created_at ASC LIMIT ? + """, + (_iso(), capacity), + ).fetchall() + for row in rows: + job_id = str(row["id"]) + changed = conn.execute( + """ + UPDATE collect_jobs + SET status = 'preparing', stage = 'claimed', attempt_count = attempt_count + 1, + next_attempt_at = '', updated_at = ? + WHERE id = ? AND status IN ('queued','delayed') + """, + (_iso(), job_id), + ).rowcount + if changed: + claimed.append(job_id) + conn.commit() + return claimed + + +def _recover_interrupted() -> None: + with proxy_pool.connect() as conn: + conn.execute( + """ + UPDATE collect_jobs + SET status = 'queued', stage = 'recovered', session_id = NULL, + next_attempt_at = '', last_error = '服务器重启后恢复采集任务', updated_at = ? + WHERE status IN ('preparing','collecting') + """, + (_iso(),), + ) + conn.commit() + + +def _worker_loop() -> None: + while True: + try: + _schedule_daily_jobs() + for job_id in _claim_due_jobs(): + with _worker_lock: + if job_id in _active_jobs: + continue + _active_jobs.add(job_id) + threading.Thread(target=_run_job, args=(job_id,), daemon=True, name=f"tiktok-collect-{job_id[-8:]}").start() + except Exception as exc: + print(f"TikTok collect scheduler failed: {exc}", flush=True) + time.sleep(WORKER_INTERVAL_SECONDS) + + +def start_worker() -> None: + global _worker_started + with _worker_lock: + if _worker_started: + return + _worker_started = True + LOG_ROOT.mkdir(parents=True, exist_ok=True) + _recover_interrupted() + threading.Thread(target=_worker_loop, daemon=True, name="tiktok-collect-worker").start() diff --git a/scripts/tiktok_studio_publish.py b/scripts/tiktok_studio_publish.py index e3ecd9f..f12f2f4 100644 --- a/scripts/tiktok_studio_publish.py +++ b/scripts/tiktok_studio_publish.py @@ -185,6 +185,19 @@ def _resolve_schedule_mode(mode: str, scheduled_at: datetime, queued: bool) -> s return mode +def _ensure_no_active_collection(conn: Any, account_id: int) -> None: + active = conn.execute( + """ + SELECT id FROM collect_jobs + WHERE account_id = ? AND status IN ('queued','delayed','preparing','collecting') + LIMIT 1 + """, + (account_id,), + ).fetchone() + if active: + raise ValueError("该账号已有待执行或运行中的统计采集任务") + + def create_job(form: Any) -> dict[str, Any]: account_id = int(form.getfirst("account_id") or 0) if not account_id: @@ -218,6 +231,8 @@ def create_job(form: Any) -> dict[str, Any]: account = conn.execute("SELECT * FROM tiktok_accounts WHERE id = ? AND deleted_at = ''", (account_id,)).fetchone() if not account: raise ValueError("account not found") + if queued: + _ensure_no_active_collection(conn, account_id) proxy_profile_id = int(account["proxy_profile_id"]) asset_id = uuid.uuid4().hex @@ -302,6 +317,8 @@ def update_job(payload: dict[str, Any]) -> dict[str, Any]: else row["session_id"] ) queue = bool(payload.get("queue")) + if queue: + _ensure_no_active_collection(conn, int(row["account_id"])) mode = "server" if keep_observing else "tiktok" mode = _resolve_schedule_mode(mode, scheduled, queue) if manual_publish: @@ -358,6 +375,7 @@ def retry_job(payload: dict[str, Any]) -> dict[str, Any]: if row["status"] not in RETRYABLE_STATUSES: raise ValueError("只有发布失败的任务可以重试") account_id = int(row["account_id"]) + _ensure_no_active_collection(conn, account_id) scheduled = _parse_schedule(row["scheduled_at"]) now = _utc_now() mode = "tiktok" diff --git a/scripts/web_app.py b/scripts/web_app.py index 66c72ec..f70b8b7 100644 --- a/scripts/web_app.py +++ b/scripts/web_app.py @@ -135,6 +135,7 @@ from proxy_state import ensure_us_proxy import proxy_pool import tiktok_studio_publish +import tiktok_studio_collect MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024 SAFE_CHARS = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-") AUDIO_ONLY_SUFFIXES = {".aac", ".flac", ".m4a", ".mp3", ".ogg", ".opus", ".wav"} @@ -5955,6 +5956,11 @@ def handle_proxy_api_get(self, path: str, query: str = "") -> None: return json_response(self, HTTPStatus.OK, proxy_pool.list_products()) if path == "/api/proxy/publish/runtime": return json_response(self, HTTPStatus.OK, tiktok_studio_publish.runtime_status()) + if path == "/api/proxy/collect/dashboard": + account_id = int(parse_qs(query).get("account_id", ["0"])[0] or 0) + return json_response(self, HTTPStatus.OK, tiktok_studio_collect.dashboard(account_id)) + if path == "/api/proxy/collect/runtime": + return json_response(self, HTTPStatus.OK, tiktok_studio_collect.runtime_status()) if path.startswith("/api/proxy/publish/videos/"): asset_id = unquote(path.removeprefix("/api/proxy/publish/videos/")) return self.serve_video(tiktok_studio_publish.video_path(asset_id)) @@ -6007,6 +6013,14 @@ def handle_proxy_api_post(self, path: str) -> None: return json_response(self, HTTPStatus.OK, tiktok_studio_publish.retry_job(payload)) if path == "/api/proxy/publish/jobs/delete": return json_response(self, HTTPStatus.OK, tiktok_studio_publish.delete_job(payload)) + if path == "/api/proxy/collect/settings": + return json_response(self, HTTPStatus.OK, tiktok_studio_collect.save_settings(payload)) + if path == "/api/proxy/collect/jobs": + return json_response(self, HTTPStatus.ACCEPTED, tiktok_studio_collect.create_job(payload)) + if path == "/api/proxy/collect/jobs/retry": + return json_response(self, HTTPStatus.OK, tiktok_studio_collect.retry_job(payload)) + if path == "/api/proxy/collect/jobs/cancel": + return json_response(self, HTTPStatus.OK, tiktok_studio_collect.cancel_job(payload)) return json_response(self, HTTPStatus.NOT_FOUND, {"error": "Not found"}) except ValueError as exc: return json_response(self, HTTPStatus.BAD_REQUEST, {"error": str(exc)}) @@ -6843,6 +6857,7 @@ def main() -> int: if PROXY_POOL_ENABLED: threading.Thread(target=proxy_session_janitor, daemon=True).start() tiktok_studio_publish.start_worker() + tiktok_studio_collect.start_worker() normalize_stored_chat_tool_results() video_queue.start(execute_queue_job) report_scheduler_enabled = os.getenv("HOT_VIDEO_REPORT_SCHEDULER_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"} From bafed3eb93371421271e15b977dc342031bfecb7 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 11:24:07 +0800 Subject: [PATCH 044/187] ops: add LAN-only local domain setup --- scripts/setup_lan_domain.sh | 94 +++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100755 scripts/setup_lan_domain.sh diff --git a/scripts/setup_lan_domain.sh b/scripts/setup_lan_domain.sh new file mode 100755 index 0000000..e06fb71 --- /dev/null +++ b/scripts/setup_lan_domain.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -euo pipefail + +lan_domain="${LAN_DOMAIN:-video-analyzer.local}" +lan_ip="${LAN_IP:-192.168.1.254}" +lan_cidr="${LAN_CIDR:-192.168.0.0/23}" +http_port="${LAN_HTTP_PORT:-80}" +upstream_port="${LAN_UPSTREAM_PORT:-4003}" + +if [ "${EUID}" -ne 0 ]; then + exec sudo --preserve-env=LAN_DOMAIN,LAN_IP,LAN_CIDR,LAN_HTTP_PORT,LAN_UPSTREAM_PORT "$0" "$@" +fi + +if [[ ! "${lan_domain}" =~ ^[a-z0-9][a-z0-9.-]*\.local$ ]]; then + echo "LAN_DOMAIN must be a lowercase .local name: ${lan_domain}" >&2 + exit 2 +fi +if [[ ! "${lan_ip}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "LAN_IP must be an IPv4 address: ${lan_ip}" >&2 + exit 2 +fi +if [[ ! "${http_port}" =~ ^[0-9]+$ || ! "${upstream_port}" =~ ^[0-9]+$ ]]; then + echo "LAN_HTTP_PORT and LAN_UPSTREAM_PORT must be numeric." >&2 + exit 2 +fi + +export DEBIAN_FRONTEND=noninteractive +apt-get install -y avahi-daemon avahi-utils + +systemctl stop video-analyzer-mdns-test.service 2>/dev/null || true +systemctl reset-failed video-analyzer-mdns-test.service 2>/dev/null || true + +cat >/etc/systemd/system/video-analyzer-mdns.service </etc/systemd/system/video-analyzer-lan-proxy.socket </etc/systemd/system/video-analyzer-lan-proxy.service </dev/null 2>&1 && ufw status | grep -q '^Status: active'; then + ufw allow from "${lan_cidr}" to "${lan_ip}" port "${http_port}" proto tcp comment 'Video Analyzer LAN HTTP' + ufw allow from "${lan_cidr}" to any port 5353 proto udp comment 'Video Analyzer mDNS' +fi + +systemctl daemon-reload +systemctl enable --now avahi-daemon.service +systemctl enable video-analyzer-mdns.service +systemctl restart video-analyzer-mdns.service +systemctl enable video-analyzer-lan-proxy.socket +systemctl stop video-analyzer-lan-proxy.service 2>/dev/null || true +systemctl restart video-analyzer-lan-proxy.socket + +systemctl is-active --quiet avahi-daemon.service +systemctl is-active --quiet video-analyzer-mdns.service +systemctl is-active --quiet video-analyzer-lan-proxy.socket +curl --fail --silent --show-error --output /dev/null "http://${lan_ip}:${http_port}/lan-chat" + +echo "LAN URL ready: http://${lan_domain}/lan-chat" From 1e01ce4e17066ed1dbe511101325d46df941cc37 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 11:25:24 +0800 Subject: [PATCH 045/187] fix: collect TikTok Studio post analytics --- scripts/tiktok_studio_collect.py | 92 +++++++++++++++++++++++++++----- 1 file changed, 79 insertions(+), 13 deletions(-) diff --git a/scripts/tiktok_studio_collect.py b/scripts/tiktok_studio_collect.py index ef570b5..3e06322 100644 --- a/scripts/tiktok_studio_collect.py +++ b/scripts/tiktok_studio_collect.py @@ -440,25 +440,45 @@ def _skip_onboarding(page: Any) -> None: def _video_id(url: str) -> str: - match = re.search(r"/analytics/(\d+)", url) + match = re.search(r"/(?:analytics|video)/(\d+)", url) return match.group(1) if match else "" def _discover_links_on_page(page: Any) -> list[dict[str, str]]: rows: list[dict[str, str]] = [] - links = page.locator("a[href*='/tiktokstudio/analytics/']") + links = page.locator("a[href*='/tiktokstudio/analytics/'], a[href*='/video/']") for index in range(min(links.count(), 200)): link = links.nth(index) try: href = str(link.get_attribute("href") or "") if not href: continue - href = urljoin(page.url, href) - video_id = _video_id(href) + absolute_href = urljoin(page.url, href) + video_id = _video_id(absolute_href) if not video_id: continue text = _clean_text(link.inner_text(), 2000) - rows.append({"id": video_id, "url": href, "title_hint": text}) + if "/video/" in absolute_href: + for levels in ("..", "../..", "../../..", "../../../..", "../../../../..", "../../../../../.."): + try: + candidate = _clean_text(link.locator(f"xpath={levels}").inner_text(), 2000) + if len(candidate) <= 1000 and re.search(r"(?m)^\d{1,2}:\d{2}$", candidate): + text = candidate + break + except Exception: + continue + elif not text: + for levels in ("..", "../..", "../../.."): + try: + text = _clean_text(link.locator(f"xpath={levels}").inner_text(), 2000) + if text: + break + except Exception: + continue + analytics_url = absolute_href if "/tiktokstudio/analytics/" in absolute_href else urljoin( + page.url, f"/tiktokstudio/analytics/{video_id}" + ) + rows.append({"id": video_id, "url": analytics_url, "title_hint": text}) except Exception: continue return rows @@ -484,6 +504,7 @@ def collect() -> None: page.locator("a[href*='/tiktokstudio/content']"), page.locator("a[href*='/tiktokstudio/manage']"), page.get_by_role("link", name=re.compile(r"content|posts|manage|内容|作品", re.I)), + page.get_by_role("button", name=re.compile(r"^recent posts$|^posts$|最近作品|近期作品", re.I)), ]) if content_link: href = str(content_link.get_attribute("href") or "") @@ -576,7 +597,21 @@ def _overview(lines: list[str]) -> dict[str, str]: } -def _engagement(page: Any, overview: dict[str, str]) -> dict[str, str]: +def _engagement(page: Any, lines: list[str], overview: dict[str, str]) -> dict[str, str]: + values: list[str] = [] + for index, line in enumerate(lines): + if not re.match(r"^(?:Posted on|发布于)", line, re.I): + continue + for candidate in lines[index + 1:index + 12]: + if re.match(r"^(?:Video views|播放量)$", candidate, re.I): + break + if re.fullmatch(r"\d[\d,.]*[KMB]?", candidate, re.I): + values.append(candidate) + if len(values) >= 5: + break + break + if len(values) >= 5: + return dict(zip(("play", "likes", "comments", "shares", "favorites"), values[:5])) return { "play": overview.get("play_count", ""), "likes": _locator_metric(page, ["Likes", "Like", "点赞"]), @@ -588,7 +623,7 @@ def _engagement(page: Any, overview: dict[str, str]) -> dict[str, str]: def _duration_seconds(text: str) -> int: values: list[int] = [] - for minutes, seconds in re.findall(r"\b(\d+):(\d{2})\b", text): + for minutes, seconds in re.findall(r"\b(\d+):(\d{2})\b(?!\s*(?:AM|PM)\b)", text, re.I): values.append(int(minutes) * 60 + int(seconds)) return max(values) if values else 0 @@ -631,10 +666,11 @@ def _retention_chart(page: Any) -> tuple[Any | None, int, str]: return fallback -def _sample_retention(page: Any) -> tuple[dict[str, str], bool, list[str], str]: +def _sample_retention(page: Any, duration_hint: int = 0) -> tuple[dict[str, str], bool, list[str], str]: chart, duration, reason = _retention_chart(page) if not chart: return {}, False, [], reason + duration = max(duration, duration_hint) if duration <= 0: return {}, False, [], "留存率图表没有可识别的视频时长" if duration > RETENTION_MAX_SECONDS: @@ -680,6 +716,20 @@ def read() -> tuple[int, str] | None: rows[second] = parsed[1] break + unresolved = {second for second in targets if second not in rows} + if unresolved: + scan_left = int(box["x"]) + scan_right = int(box["x"] + box["width"]) + for x in range(scan_left, scan_right + 1, 3): + page.mouse.move(x, y) + page.wait_for_timeout(55) + parsed = read() + if parsed and parsed[0] in unresolved: + rows[parsed[0]] = parsed[1] + unresolved.discard(parsed[0]) + if not unresolved: + break + missing_labels = [f"{second // 60}:{second % 60:02d}" for second in targets if second not in rows] output = {f"{second // 60}:{second % 60:02d}": rows[second] for second in sorted(rows)} return output, not missing_labels, missing_labels, "" if not missing_labels else "部分秒点未命中 ECharts tooltip" @@ -687,8 +737,15 @@ def read() -> tuple[int, str] | None: def _title_and_date(lines: list[str], hint: str) -> tuple[str, str]: cleaned_hint = _lines(hint) - title = cleaned_hint[0] if cleaned_hint else "" - date_pattern = re.compile(r"(?:[A-Z][a-z]{2}\s+\d{1,2},\s+\d{4}|\d{4}[/-]\d{1,2}[/-]\d{1,2})") + title = "" + date_pattern = re.compile(r"(?:[A-Z][a-z]{2}\s+\d{1,2},\s+\d{4}|\d{4}[/-]\d{1,2}[/-]\d{1,2}|\d{1,2}/\d{1,2}/\d{4})") + for candidate in cleaned_hint: + if re.fullmatch(r"\d{1,2}:\d{2}", candidate) or date_pattern.search(candidate): + continue + if candidate.lower() in {"everyone", "friends", "only you"} or re.fullmatch(r"[\d,.]+", candidate): + continue + title = candidate + break published = "" for line in cleaned_hint + lines[:80]: match = date_pattern.search(line) @@ -705,7 +762,14 @@ def _title_and_date(lines: list[str], hint: str) -> tuple[str, str]: def _collect_video(page: Any, job: dict[str, Any], source: dict[str, str], log_dir: Path) -> dict[str, Any]: page.goto(source["url"], wait_until="domcontentloaded", timeout=60000) - page.wait_for_timeout(2200) + _assert_account_ready(page) + try: + page.get_by_text(re.compile(r"^(?:Video views|播放量)$", re.I)).first.wait_for( + state="visible", timeout=15000 + ) + page.wait_for_timeout(1200) + except Exception: + page.wait_for_timeout(2200) _assert_account_ready(page) body = page.locator("body").inner_text(timeout=15000) lines = _lines(body) @@ -714,7 +778,9 @@ def _collect_video(page: Any, job: dict[str, Any], source: dict[str, str], log_d page.screenshot(path=str(log_dir / f"{source['id']}-missing-overview.png"), full_page=True) raise RuntimeError("视频分析页没有识别到概览指标") title, published_at = _title_and_date(lines, source.get("title_hint", "")) - retention, retention_complete, missing, retention_reason = _sample_retention(page) + retention, retention_complete, missing, retention_reason = _sample_retention( + page, _duration_seconds(source.get("title_hint", "")) + ) payload = { "account": { "id": job["account_id"], @@ -727,7 +793,7 @@ def _collect_video(page: Any, job: dict[str, Any], source: dict[str, str], log_d "video": {"id": source["id"], "title": title, "published_at": published_at, "url": page.url}, "time_filter": {"requested": None, "applied": None, "scope": "video_lifetime", "applied_successfully": False}, "overview": overview, - "engagement": _engagement(page, overview), + "engagement": _engagement(page, lines, overview), "retention": retention, "retention_complete": retention_complete, "missing_retention_seconds": missing, From 983f3da2f9ee246042bf2fd54c83de7dfe769174 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 11:42:31 +0800 Subject: [PATCH 046/187] fix(proxy): allow deleting unbound archived proxies --- scripts/proxy_pool.py | 54 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/scripts/proxy_pool.py b/scripts/proxy_pool.py index 06755bd..0e8ddc4 100644 --- a/scripts/proxy_pool.py +++ b/scripts/proxy_pool.py @@ -1452,16 +1452,62 @@ def get_pool(pool_id: int) -> dict[str, Any]: row = conn.execute("SELECT * FROM proxy_profiles WHERE id = ?", (pool_id,)).fetchone() if not row: raise ValueError("proxy profile not found") - count = conn.execute("SELECT COUNT(*) AS count FROM tiktok_accounts WHERE proxy_profile_id = ?", (pool_id,)).fetchone()["count"] - names = [str(item["username"]) for item in conn.execute("SELECT username FROM tiktok_accounts WHERE proxy_profile_id = ? ORDER BY username", (pool_id,))] + count = conn.execute( + "SELECT COUNT(*) AS count FROM tiktok_accounts WHERE proxy_profile_id = ? AND deleted_at = ''", + (pool_id,), + ).fetchone()["count"] + names = [ + str(item["username"]) + for item in conn.execute( + "SELECT username FROM tiktok_accounts WHERE proxy_profile_id = ? AND deleted_at = '' ORDER BY username", + (pool_id,), + ) + ] return _row_to_pool(row, int(count), names) def delete_pool(pool_id: int) -> dict[str, Any]: with connect() as conn: - count = conn.execute("SELECT COUNT(*) AS count FROM tiktok_accounts WHERE proxy_profile_id = ?", (pool_id,)).fetchone()["count"] + pool = conn.execute("SELECT id FROM proxy_profiles WHERE id = ?", (pool_id,)).fetchone() + if not pool: + raise ValueError("proxy profile not found") + + _active_sessions(conn) + active_session = conn.execute( + "SELECT id FROM browser_sessions WHERE proxy_profile_id = ? AND status IN ('starting','running','observing') LIMIT 1", + (pool_id,), + ).fetchone() + if active_session: + raise ValueError("代理仍有运行中的浏览器或观测通道，请先释放") + + count = conn.execute( + "SELECT COUNT(*) AS count FROM tiktok_accounts WHERE proxy_profile_id = ? AND deleted_at = ''", + (pool_id,), + ).fetchone()["count"] if int(count): - raise ValueError("Cannot delete a proxy profile with bound accounts") + raise ValueError("代理仍绑定账号，请先删除或迁移账号") + + archived_account = conn.execute( + """ + SELECT a.id + FROM tiktok_accounts AS a + WHERE a.proxy_profile_id = ? AND a.deleted_at <> '' + AND ( + EXISTS (SELECT 1 FROM publish_assets WHERE account_id = a.id) + OR EXISTS (SELECT 1 FROM publish_jobs WHERE account_id = a.id) + OR EXISTS (SELECT 1 FROM collect_jobs WHERE account_id = a.id) + OR EXISTS (SELECT 1 FROM collect_results WHERE account_id = a.id) + OR EXISTS (SELECT 1 FROM collect_errors WHERE account_id = a.id) + ) + LIMIT 1 + """, + (pool_id,), + ).fetchone() + if archived_account: + raise ValueError("代理关联的已删除账号仍有发布或采集记录，不能删除") + + conn.execute("DELETE FROM browser_sessions WHERE proxy_profile_id = ?", (pool_id,)) + conn.execute("DELETE FROM tiktok_accounts WHERE proxy_profile_id = ? AND deleted_at <> ''", (pool_id,)) conn.execute("DELETE FROM proxy_profiles WHERE id = ?", (pool_id,)) conn.commit() return list_state() From 936f60abbd546713a361882ee1792f6dfed9b277 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 12:00:44 +0800 Subject: [PATCH 047/187] feat: support LAN chat image messages --- scripts/lan_chat.py | 134 ++++++++++++++++++++++++++++++----- scripts/static/lan_chat.html | 55 ++++++++++++-- scripts/web_app.py | 21 +++++- 3 files changed, 182 insertions(+), 28 deletions(-) diff --git a/scripts/lan_chat.py b/scripts/lan_chat.py index b2fd1f0..e78a863 100644 --- a/scripts/lan_chat.py +++ b/scripts/lan_chat.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 +import binascii import hashlib import html import json @@ -30,6 +31,13 @@ "#D9825B", "#5A7D9A", ) +MESSAGE_IMAGE_MAX_BYTES = 5 * 1024 * 1024 +MESSAGE_IMAGE_TYPES = { + "jpg": "image/jpeg", + "png": "image/png", + "gif": "image/gif", + "webp": "image/webp", +} class LanChatError(Exception): @@ -39,15 +47,22 @@ def __init__(self, message: str, status: int = 400): class LanChatStore: - def __init__(self, db_path: Path, avatar_dir: Path | None = None): + def __init__( + self, + db_path: Path, + avatar_dir: Path | None = None, + media_dir: Path | None = None, + ): self.db_path = Path(db_path) self.avatar_dir = Path(avatar_dir or self.db_path.parent / "lan_chat_avatars") + self.media_dir = Path(media_dir or self.db_path.parent / "lan_chat_media") self._avatar_lock = threading.Lock() self._avatar_jobs: set[str] = set() def initialize(self) -> None: self.db_path.parent.mkdir(parents=True, exist_ok=True) self.avatar_dir.mkdir(parents=True, exist_ok=True) + self.media_dir.mkdir(parents=True, exist_ok=True) with self._connect() as conn: conn.execute("PRAGMA journal_mode = WAL") conn.executescript( @@ -85,6 +100,8 @@ def initialize(self) -> None: room_id TEXT NOT NULL, sender_id TEXT NOT NULL, content TEXT NOT NULL, + image_filename TEXT, + image_mime_type TEXT, created_at REAL NOT NULL, FOREIGN KEY (room_id) REFERENCES rooms(id) ON DELETE CASCADE, FOREIGN KEY (sender_id) REFERENCES users(id) ON DELETE CASCADE @@ -95,6 +112,13 @@ def initialize(self) -> None: ON room_members(user_id, room_id); """ ) + message_columns = { + str(row["name"]) for row in conn.execute("PRAGMA table_info(messages)").fetchall() + } + if "image_filename" not in message_columns: + conn.execute("ALTER TABLE messages ADD COLUMN image_filename TEXT") + if "image_mime_type" not in message_columns: + conn.execute("ALTER TABLE messages ADD COLUMN image_mime_type TEXT") now = time.time() conn.execute( """INSERT OR IGNORE INTO rooms @@ -278,29 +302,71 @@ def list_messages( messages = [self._message_payload(row, current["id"]) for row in rows] return {"messages": messages, "lastId": messages[-1]["id"] if messages else after_id} - def send_message(self, device_token: str, room_id: str, content: str) -> dict[str, Any]: + def send_message( + self, + device_token: str, + room_id: str, + content: str, + image_data: str = "", + ) -> dict[str, Any]: current = self.authenticate(device_token) clean_content = str(content or "").strip() - if not clean_content: - raise LanChatError("消息不能为空") if len(clean_content) > 4000: raise LanChatError("消息不能超过 4000 个字符") + image = self._decode_message_image(image_data) + if not clean_content and image is None: + raise LanChatError("消息或图片不能为空") now = time.time() - with self._connect() as conn: - self._require_room_access(conn, room_id, current["id"]) - cursor = conn.execute( - "INSERT INTO messages(room_id, sender_id, content, created_at) VALUES (?, ?, ?, ?)", - (room_id, current["id"], clean_content, now), - ) - conn.execute("UPDATE rooms SET updated_at = ? WHERE id = ?", (now, room_id)) - row = conn.execute( - """SELECT m.*, u.nickname, u.avatar_color, u.avatar_status - FROM messages m JOIN users u ON u.id = m.sender_id - WHERE m.id = ?""", - (cursor.lastrowid,), - ).fetchone() + image_filename = "" + image_mime_type = "" + if image is not None: + image_bytes, image_mime_type, extension = image + image_filename = f"{uuid.uuid4().hex}.{extension}" + (self.media_dir / image_filename).write_bytes(image_bytes) + try: + with self._connect() as conn: + self._require_room_access(conn, room_id, current["id"]) + cursor = conn.execute( + """INSERT INTO messages + (room_id, sender_id, content, image_filename, image_mime_type, created_at) + VALUES (?, ?, ?, ?, ?, ?)""", + ( + room_id, + current["id"], + clean_content, + image_filename or None, + image_mime_type or None, + now, + ), + ) + conn.execute("UPDATE rooms SET updated_at = ? WHERE id = ?", (now, room_id)) + row = conn.execute( + """SELECT m.*, u.nickname, u.avatar_color, u.avatar_status + FROM messages m JOIN users u ON u.id = m.sender_id + WHERE m.id = ?""", + (cursor.lastrowid,), + ).fetchone() + except Exception: + if image_filename: + (self.media_dir / image_filename).unlink(missing_ok=True) + raise return self._message_payload(row, current["id"]) + def message_image_bytes(self, filename: str) -> tuple[bytes, str]: + clean_name = str(filename or "").strip().lower() + stem, separator, extension = clean_name.rpartition(".") + if ( + not separator + or len(stem) != 32 + or extension not in MESSAGE_IMAGE_TYPES + or any(char not in "0123456789abcdef" for char in stem) + ): + raise LanChatError("图片不存在", 404) + path = (self.media_dir / clean_name).resolve() + if path.parent != self.media_dir.resolve() or not path.is_file(): + raise LanChatError("图片不存在", 404) + return path.read_bytes(), MESSAGE_IMAGE_TYPES[extension] + def avatar_bytes(self, user_id: str) -> tuple[bytes, str]: with self._connect() as conn: row = conn.execute("SELECT * FROM users WHERE id = ?", (user_id,)).fetchone() @@ -367,7 +433,7 @@ def _room_payload( other = next((item for item in members if item["id"] != current_user_id), None) name = other["nickname"] if other is not None else "私信" latest = conn.execute( - """SELECT m.content, m.created_at, u.nickname + """SELECT m.content, m.image_filename, m.created_at, u.nickname FROM messages m JOIN users u ON u.id = m.sender_id WHERE m.room_id = ? ORDER BY m.id DESC LIMIT 1""", (room["id"],), @@ -383,6 +449,7 @@ def _room_payload( "latestMessage": ( { "content": latest["content"], + "hasImage": bool(latest["image_filename"]), "nickname": latest["nickname"], "createdAt": float(latest["created_at"]), } @@ -398,6 +465,7 @@ def _user_count(conn: sqlite3.Connection) -> int: @staticmethod def _message_payload(row: sqlite3.Row, current_user_id: str) -> dict[str, Any]: + image_filename = str(row["image_filename"] or "") return { "id": int(row["id"]), "roomId": row["room_id"], @@ -405,10 +473,40 @@ def _message_payload(row: sqlite3.Row, current_user_id: str) -> dict[str, Any]: "senderName": row["nickname"], "senderAvatarUrl": f"/api/lan-chat/avatars/{row['sender_id']}", "content": row["content"], + "imageUrl": f"/api/lan-chat/media/{image_filename}" if image_filename else "", "createdAt": float(row["created_at"]), "isMine": row["sender_id"] == current_user_id, } + @staticmethod + def _decode_message_image(image_data: str) -> tuple[bytes, str, str] | None: + value = str(image_data or "").strip() + if not value: + return None + if "," in value: + header, value = value.split(",", 1) + if not header.lower().startswith("data:image/"): + raise LanChatError("图片数据无效") + if len(value) > (MESSAGE_IMAGE_MAX_BYTES * 4 // 3) + 8: + raise LanChatError("图片不能超过 5MB", 413) + try: + payload = base64.b64decode(value, validate=True) + except (binascii.Error, ValueError, TypeError) as exc: + raise LanChatError("图片数据无效") from exc + if not payload or len(payload) > MESSAGE_IMAGE_MAX_BYTES: + raise LanChatError("图片不能超过 5MB", 413) + if payload.startswith(b"\xff\xd8\xff"): + extension = "jpg" + elif payload.startswith(b"\x89PNG\r\n\x1a\n"): + extension = "png" + elif payload.startswith((b"GIF87a", b"GIF89a")): + extension = "gif" + elif len(payload) >= 12 and payload[:4] == b"RIFF" and payload[8:12] == b"WEBP": + extension = "webp" + else: + raise LanChatError("仅支持 JPG、PNG、GIF 或 WebP 图片") + return payload, MESSAGE_IMAGE_TYPES[extension], extension + @staticmethod def _require_room_access( conn: sqlite3.Connection, room_id: str, user_id: str diff --git a/scripts/static/lan_chat.html b/scripts/static/lan_chat.html index 51992f9..5bda20a 100644 --- a/scripts/static/lan_chat.html +++ b/scripts/static/lan_chat.html @@ -2,7 +2,7 @@ - + 邻聊 · 局域网聊天 +
@@ -34,7 +68,7 @@
邻

公共频道

这里的消息对局域网内所有已注册设备可见。打个招呼吧。

-
+