AI 聊天助手输入问题后将按当前入口和工具选择调用能力
引用内容
__CHAT_PROVIDER_LABEL__ 对话
__CHAT_PROVIDER_LABEL__ 对话
${esc(domain.label)}
`; (domain.categories||[]).forEach(cat=>{const catId=`${domain.id}/${cat.id}`;html+=`'});html+='
${esc(cat.label)}
`; (cat.tools||[]).forEach(t=>{const checked=selected.has(t.id);html+=``});html+='
邻聊
同一网络，随时开聊
公共频道所有注册成员都在这里
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+进入邻聊
先选择飞书用户，再选择这台物理机使用的账户。
+
+
+
+
+
+
+
+
+
+ 第一步 · 选择飞书用户
+ 正在读取账户…
新设备或新浏览器
+
+
+
+
+
+
+
+
+
+
+
+
+
+管理群组
管理员可以修改群名称和移出成员。
+
+
+
+
+
管理员主动退出后，管理员身份会按进群时间移交给下一位成员。
+ 账号 IP 池
账号管理
| 账号 | 绑定代理 | 状态 | 操作 |
|---|
+
+
+
+
+
+
+新增账号
+
+
+
+
+自动发布视频
+
+
+
+
+
+ 发布队列
正在加载...
视频详情
未添加商品。
加入发布队列后会立即唤醒账号，在 TikTok Studio 设置此发布时间。
选择视频并设置发布时间。
+
+
+商品管理
搜索 Shop 商品并维护通用商品库
支持关键词和 TikTok Shop 链接；暂不支持直接输入纯数字商品 ID。
从通用商品中选择，或先搜索添加。
+
+
+编辑通用商品
商品 ID 用于唯一性判断，保存时不可修改。
+
+
+视频统计采集
+
+
自动采集：未启用
正在加载采集设置...
+ 采集任务
已采集视频
| 采集时间 | 视频 | 播放量 | 完播率 | 留存 | 飞书 |
|---|
+
+
+
+
+
+
diff --git a/scripts/test_chat_tool_normalization.py b/scripts/test_chat_tool_normalization.py
index e5e0b23..5e86da3 100644
--- a/scripts/test_chat_tool_normalization.py
+++ b/scripts/test_chat_tool_normalization.py
@@ -3,13 +3,16 @@
from __future__ import annotations
import json
+import os
import sys
from pathlib import Path
+from types import SimpleNamespace
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
-from web_app import build_prefixed_model_tools, chat_markdown_to_html, chat_request_needs_tools, filter_locked_provider_tool_ids, normalize_tool_result, provider_default_enabled_tool_ids, provider_forces_mcp_tools, route_chat_intent # noqa: E402
+import web_app # noqa: E402
+from web_app import build_chat_history_context, build_deepseek_tool_assistant_message, build_prefixed_model_tools, build_tool_limit_final_context, chat_markdown_to_html, chat_request_needs_tools, chat_routing_text, compact_chat_tool_evidence, deepseek_tool_protocol_present, estimate_chat_context_tokens, fastmoss_analysis_evidence_gaps, fastmoss_availability_search_arguments, fastmoss_defaults_to_us, fastmoss_empty_availability_answer, fastmoss_playbook_instruction, fastmoss_playbook_intent, fastmoss_product_evidence_required, fastmoss_required_capability_gaps, filter_locked_provider_tool_ids, forced_provider_domain_tool_available, is_chat_retry_request, manage_chat_context, normalize_mcp_tool_arguments, normalize_prefixed_tool_result, normalize_tool_result, parse_chat_intent_decision, provider_default_enabled_tool_ids, provider_forces_mcp_tools, resolve_chat_intent, route_chat_intent # noqa: E402
from tools import _filter_relevant_search_results, execute_tool, get_tools_for_model, list_tools, parse_bing_html, parse_duckduckgo_html # noqa: E402
@@ -150,6 +153,144 @@ def test_amazon_url_query_api_fragment_does_not_disable_tools() -> None:
assert force_mcp_tools is True
+def test_ocr_metadata_does_not_change_chat_route() -> None:
+ enriched = (
+ "User question:\n这些产品的销量你分析了吗"
+ "\n\nImage OCR result:\nllmStatus: skipped_api_key_missing\nAPI fallback"
+ )
+ routing_text = chat_routing_text(enriched)
+ assert routing_text == "这些产品的销量你分析了吗"
+ assert route_chat_intent(routing_text)["intent"] != "mcp_interface"
+
+
+def test_fastmoss_defaults_to_us_unless_another_region_is_named() -> None:
+ assert fastmoss_defaults_to_us("分析 Hidden Camera Detector 在 TK 的销售数据") is True
+ assert fastmoss_defaults_to_us("分析美区 Hidden Camera Detector") is True
+ assert fastmoss_defaults_to_us("分析 product id 1732424427368190285") is True
+ assert fastmoss_defaults_to_us("分析日本和墨西哥的 Hidden Camera Detector") is False
+ assert fastmoss_defaults_to_us("Compare US and JP markets") is False
+
+
+def test_fastmoss_playbook_intent_routes_official_workflows() -> None:
+ cases = {
+ "帮我做防偷拍探测器选品，并给出定价建议": "product",
+ "拆解这个竞品店铺的打法": "competitor",
+ "给这个店铺做一次店铺诊断": "shop",
+ "拆解这条爆款视频为什么能卖": "content_dissect",
+ "为我的产品制定内容策略和拍摄 brief": "content_strategy",
+ "用 28 天数据做价格带和月度 GMV 测算": "pricing",
+ "按带货力帮我找达人并写建联文案": "creator",
+ }
+ for text, expected in cases.items():
+ assert fastmoss_playbook_intent(text) == expected
+ route = route_chat_intent(text, "fastmoss")
+ assert route["intent"] == f"fastmoss_{expected}"
+ assert route["playbook"] == expected
+
+ assert "playbook" not in route_chat_intent("给这个商品定价", "home")
+
+
+def test_fastmoss_selection_playbook_includes_pricing_model() -> None:
+ instruction = fastmoss_playbook_instruction("product")
+ assert "建议上市价" in instruction
+ assert "月度销量=月流量×转化率" in instruction
+ assert "缺少输入时只列公式和待补参数" in instruction
+ assert "不得自行设定库存、预算、达人数量、周期或经营目标" in instruction
+ assert "保守/基准/激进三套" not in instruction
+ assert "不得把 GMV 当利润" in instruction
+
+
+def test_fastmoss_product_evidence_is_scoped_by_playbook() -> None:
+ assert fastmoss_product_evidence_required("帮我做防偷拍探测器选品") is True
+ assert fastmoss_product_evidence_required("给这个品类做价格测算") is True
+ assert fastmoss_product_evidence_required("拆解这个竞品商品") is True
+ assert fastmoss_product_evidence_required("拆解这个竞品店铺") is False
+ assert fastmoss_product_evidence_required("给这个店铺做店铺诊断") is False
+ assert fastmoss_product_evidence_required("为我的产品制定内容策略") is False
+ empty_message = SimpleNamespace(tool_calls=[], tool_results=[])
+ assert fastmoss_analysis_evidence_gaps("给这个店铺做店铺诊断", empty_message) == []
+ assert fastmoss_analysis_evidence_gaps("给这个品类做价格测算", empty_message) == [
+ "category_lookup", "market_ranking", "us_region", "product_reviews"
+ ]
+
+
+def _model_tool(name: str) -> dict:
+ return {"type": "function", "function": {"name": name, "parameters": {"type": "object"}}}
+
+
+def test_fastmoss_analysis_requires_domain_and_evidence_capabilities() -> None:
+ query = "酒店防偷拍探测器在 TK 的销售怎样？"
+ complete_tools = [
+ _model_tool("system__current_time"),
+ _model_tool("fastmoss__search_category_by_words"),
+ _model_tool("fastmoss__product_rank_top_selling"),
+ _model_tool("fastmoss__product_review_list"),
+ ]
+ assert forced_provider_domain_tool_available("fastmoss", complete_tools) is True
+ assert forced_provider_domain_tool_available("fastmoss", [_model_tool("system__current_time")]) is False
+ assert fastmoss_required_capability_gaps(query, complete_tools) == []
+ assert fastmoss_required_capability_gaps(query, complete_tools[:-1]) == ["product_reviews"]
+
+
+def test_fastmoss_analysis_requires_us_ranking_and_reviews() -> None:
+ query = "分析 Hidden Camera Detector 在 TK 的销售数据"
+ empty_message = SimpleNamespace(tool_calls=[], tool_results=[])
+ assert fastmoss_analysis_evidence_gaps(query, empty_message) == [
+ "category_lookup", "market_ranking", "us_region", "product_reviews"
+ ]
+
+ message = SimpleNamespace(
+ tool_calls=[
+ {"function": {"name": "fastmoss__search_category_by_words", "arguments": '{"keywords":"camera detector"}'}},
+ {"function": {"name": "fastmoss__product_rank_top_selling", "arguments": '{"region":"US","category_id":"911752"}'}},
+ {"function": {"name": "fastmoss__product_review_list", "arguments": '{"product_id":"1732424427368190285","region":"US"}'}},
+ ],
+ tool_results=[
+ {"tool_name": "fastmoss__search_category_by_words", "result": {"ok": True, "enough_data": True}},
+ {"tool_name": "fastmoss__product_rank_top_selling", "result": {"ok": True, "enough_data": True}},
+ {"tool_name": "fastmoss__product_review_list", "result": {"ok": True, "enough_data": False}},
+ ],
+ )
+ assert fastmoss_analysis_evidence_gaps(query, message) == []
+
+ message.tool_calls.insert(
+ 2,
+ {"function": {"name": "fastmoss__product_search", "arguments": '{"keywords":"camera detector"}'}},
+ )
+ message.tool_results.insert(
+ 2,
+ {"tool_name": "fastmoss__product_search", "result": {"ok": True, "enough_data": True}},
+ )
+ message.tool_calls.append(
+ {"function": {"name": "fastmoss__shop_search", "arguments": '{"seller_id":"7495582349874924375"}'}},
+ )
+ message.tool_results.append(
+ {"tool_name": "fastmoss__shop_search", "result": {"ok": True, "enough_data": True}},
+ )
+ assert fastmoss_analysis_evidence_gaps(query, message) == []
+
+ message.tool_results[1]["result"]["enough_data"] = False
+ message.tool_results[1]["result"].update({"data_state": "empty", "evidence_observed": True})
+ assert fastmoss_analysis_evidence_gaps(query, message) == []
+
+
+def test_fastmoss_explicit_other_region_does_not_require_us() -> None:
+ query = "分析日本 Hidden Camera Detector 在 TK 的销售数据"
+ message = SimpleNamespace(
+ tool_calls=[
+ {"function": {"name": "fastmoss__search_category_by_words", "arguments": '{"keywords":"camera detector"}'}},
+ {"function": {"name": "fastmoss__market_category_ranking", "arguments": '{"region":"JP","category_id":"911752"}'}},
+ {"function": {"name": "fastmoss__product_review_list", "arguments": '{"product_id":"1732424427368190285","region":"JP"}'}},
+ ],
+ tool_results=[
+ {"tool_name": "fastmoss__search_category_by_words", "result": {"ok": True, "enough_data": True}},
+ {"tool_name": "fastmoss__market_category_ranking", "result": {"ok": True, "enough_data": True}},
+ {"tool_name": "fastmoss__product_review_list", "result": {"ok": True, "enough_data": True}},
+ ],
+ )
+ assert fastmoss_analysis_evidence_gaps(query, message) == []
+
+
def test_short_cjk_web_search_filters_irrelevant_results() -> None:
results = [
{"title": "知乎 - 有问题，就会有答案", "snippet": "中文问答社区", "url": "https://www.zhihu.com/"},
@@ -213,6 +354,2674 @@ def test_web_search_tool_is_registered_and_normalized() -> None:
categories = {item["category"]: item["tools"] for item in list_tools()}
assert any(tool["name"] == "web_search" for tools in categories.values() for tool in tools)
+
+def _chat_message(
+ message_id: str,
+ role: str,
+ content: str,
+ *,
+ status: str = "done",
+ tool_calls: list[dict] | None = None,
+ tool_results: list[dict] | None = None,
+) -> SimpleNamespace:
+ return SimpleNamespace(
+ id=message_id,
+ role=role,
+ content=content,
+ status=status,
+ tool_calls=tool_calls or [],
+ tool_results=tool_results or [],
+ attachments=[],
+ )
+
+
+def test_chat_history_archives_done_tools_and_recovers_failed_results() -> None:
+ done_call = {
+ "id": "call_done",
+ "function": {"name": "sellersprite__keyword_research", "arguments": '{"keyword":"camera"}'},
+ }
+ error_calls = [
+ {"id": "call_1", "function": {"name": "sellersprite__keyword_research", "arguments": '{"keyword":"detector"}'}},
+ {"id": "call_2", "function": {"name": "sellersprite__product_search", "arguments": '{"keyword":"detector"}'}},
+ ]
+ messages = [
+ _chat_message("u1", "user", "分析 camera"),
+ _chat_message(
+ "a1",
+ "assistant",
+ "历史报告",
+ tool_calls=[done_call],
+ tool_results=[{
+ "tool_name": "sellersprite__keyword_research",
+ "result": {"ok": True, "mcp_text_preview": "ARCHIVED_RAW_PAYLOAD" * 1000},
+ }],
+ ),
+ _chat_message("u2", "user", "继续分析 detector"),
+ _chat_message(
+ "a2",
+ "assistant",
+ "Request failed: 402 Payment Required",
+ status="error",
+ tool_calls=error_calls,
+ tool_results=[
+ {"tool_name": "sellersprite__keyword_research", "result": {"ok": True, "mcp_data": {"search_volume": 12000}}},
+ {"tool_name": "sellersprite__product_search", "result": {"ok": True, "mcp_data": {"products_total": 4321}}},
+ ],
+ ),
+ _chat_message("u3", "user", "继续"),
+ _chat_message("a3", "assistant", "", status="pending"),
+ ]
+
+ history, recovery = build_chat_history_context(messages, "a3")
+ encoded = json.dumps(history, ensure_ascii=False)
+ assert "Historical tool evidence archived" in encoded
+ assert "ARCHIVED_RAW_PAYLOAD" not in encoded
+ assert "previous_tool_collection" in encoded
+ assert "402 Payment Required" in encoded
+ assert "sellersprite__product_search" in encoded
+ assert recovery == {"complete": True, "tool_count": 2, "message_id": "a2"}
+ assert is_chat_retry_request("继续") is True
+ assert is_chat_retry_request("请重新分析这个新产品并给出完整报告") is False
+
+
+def test_tool_evidence_is_compact_but_keeps_business_fields() -> None:
+ evidence = compact_chat_tool_evidence(
+ "sellersprite__keyword_research",
+ {
+ "ok": True,
+ "mcp_text_preview": "RAW_SHOULD_BE_DROPPED" * 1000,
+ "mcp_data": {
+ "keyword": "camera detector",
+ "search_volume": 12000,
+ "items": [{"title": f"Product {index}", "description": "x" * 2000} for index in range(30)],
+ },
+ },
+ max_chars=1200,
+ )
+ assert len(evidence) <= 1200
+ assert "camera detector" in evidence
+ assert "12000" in evidence
+ assert "RAW_SHOULD_BE_DROPPED" not in evidence
+
+
+def test_current_tool_evidence_is_lossless_until_budget_pressure() -> None:
+ marker = "CURRENT_EVIDENCE_AFTER_12000_CHARS"
+ payload = {
+ "code": "OK",
+ "message": "成功",
+ "data": {
+ "total": 30,
+ "items": [
+ {
+ "keyword": f"keyword-{index}",
+ "searches": index * 100,
+ "description": ("x" * 700) + (marker if index == 20 else ""),
+ }
+ for index in range(30)
+ ],
+ },
+ }
+ raw_result = {
+ "ok": True,
+ "data": {"content": [{"type": "text", "text": json.dumps(payload, ensure_ascii=False)}]},
+ }
+ normalized = normalize_prefixed_tool_result(
+ "sellersprite__keyword_research_trends",
+ raw_result,
+ )
+ assert isinstance(normalized["mcp_data"], dict)
+ assert marker in json.dumps(normalized["mcp_data"], ensure_ascii=False)
+ assert len(normalized["mcp_text_preview"]) == 4000
+ assert marker not in normalized["mcp_text_preview"]
+
+ evidence = web_app.current_chat_tool_evidence(
+ "sellersprite__keyword_research_trends",
+ normalized,
+ {"marketplace": "US", "keyword": "air pump"},
+ raw_result,
+ )
+ assert marker in evidence
+ assert '"marketplace":"US"' in evidence
+ assert "mcp_text_preview" not in evidence
+
+ messages = [
+ {
+ "role": "assistant",
+ "content": "old-" + ("h" * 30000),
+ "_context_scope": "history",
+ "_context_priority": "normal",
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_current",
+ "content": evidence,
+ "_context_scope": "current",
+ },
+ ]
+ request_messages, _, stats = manage_chat_context(messages, [], max_tokens=12000)
+ retained = next(message["content"] for message in request_messages if message.get("role") == "tool")
+ assert retained == evidence
+ assert stats["current_evidence_compressed"] == 0
+ assert stats["compressed"] is True
+ history_content = next(message["content"] for message in request_messages if message.get("role") == "assistant")
+ assert len(history_content) <= 3000
+
+
+def test_product_availability_is_a_shallow_lookup() -> None:
+ cases = (
+ "贪吃蛇小车这款玩具在TK上有销售吗",
+ "这款产品TK是否有销售？",
+ "Hidden Camera Detector 在 TikTok Shop 有没有卖？",
+ "这个同款是否上架？",
+ )
+ empty_message = SimpleNamespace(tool_calls=[], tool_results=[])
+ for text in cases:
+ route = route_chat_intent(text, "fastmoss")
+ assert route["intent"] == "product_availability"
+ assert route["task_depth"] == "lookup"
+ assert route["max_rounds"] == 2
+ assert fastmoss_product_evidence_required(text, route) is False
+ assert fastmoss_analysis_evidence_gaps(text, empty_message, route) == []
+
+ analysis_text = "分析这款产品在TK的销量、市场和竞争机会"
+ analysis_route = route_chat_intent(analysis_text, "fastmoss")
+ assert analysis_route["intent"] == "product_research"
+ assert fastmoss_product_evidence_required(analysis_text, analysis_route) is True
+
+
+def test_intent_decision_validation_and_fallback() -> None:
+ fallback = route_chat_intent("帮我看看这个产品", "fastmoss")
+ valid = parse_chat_intent_decision(
+ {
+ "intent": "product_availability",
+ "task_depth": "lookup",
+ "entity": "磁力贪吃蛇小车",
+ "region": "CN",
+ "confidence": 0.94,
+ },
+ fallback,
+ "fastmoss",
+ "这款产品TK是否有销售？",
+ )
+ assert valid["intent"] == "product_availability"
+ assert valid["route_source"] == "llm"
+ assert valid["entity"] == "磁力贪吃蛇小车"
+ assert valid["region"] == "US"
+ assert fastmoss_availability_search_arguments(valid, "这款产品TK是否有销售？") == {
+ "keywords": "磁力贪吃蛇小车",
+ "region": "US",
+ "pagesize": 10,
+ }
+ empty_answer = fastmoss_empty_availability_answer({"keywords": "磁力贪吃蛇小车", "region": "US"})
+ assert "本次" in empty_answer
+ assert "未检索到" in empty_answer
+ assert "不表示平台上绝对没有销售" in empty_answer
+
+ low_confidence = parse_chat_intent_decision(
+ {"intent": "product_availability", "task_depth": "lookup", "confidence": 0.4},
+ fallback,
+ "fastmoss",
+ "帮我看看这个产品",
+ )
+ assert low_confidence["intent"] == "product_research"
+ assert low_confidence["route_source"] == "rules"
+ assert parse_chat_intent_decision({"intent": "unknown", "task_depth": "lookup", "confidence": 1}, fallback, "fastmoss", "x")["intent"] == "product_research"
+ assert parse_chat_intent_decision(None, fallback, "fastmoss", "x")["intent"] == "product_research"
+
+
+def test_intent_router_uses_recent_context_and_falls_back_on_failure() -> None:
+ class FakeResponse:
+ def raise_for_status(self) -> None:
+ return None
+
+ def json(self) -> dict:
+ return {
+ "choices": [{"message": {"content": json.dumps({
+ "intent": "product_availability",
+ "task_depth": "lookup",
+ "entity": "磁力贪吃蛇小车",
+ "region": "US",
+ "confidence": 0.96,
+ }, ensure_ascii=False)}}],
+ }
+
+ class FakeRequests:
+ def __init__(self, fail: bool = False) -> None:
+ self.fail = fail
+ self.payload = None
+
+ def post(self, _url: str, **kwargs):
+ self.payload = kwargs.get("json")
+ if self.fail:
+ raise TimeoutError("router timeout")
+ return FakeResponse()
+
+ previous_enabled = os.environ.get("CHAT_INTENT_ROUTER_ENABLED")
+ os.environ["CHAT_INTENT_ROUTER_ENABLED"] = "1"
+ try:
+ messages = [
+ SimpleNamespace(role="user", content="贪吃蛇小车这款玩具在TK上有销售吗"),
+ SimpleNamespace(role="assistant", content="旧回答失败"),
+ SimpleNamespace(role="user", content="这款产品TK是否有销售？"),
+ ]
+ fake = FakeRequests()
+ route = resolve_chat_intent(messages, "这款产品TK是否有销售？", "fastmoss", "key", "https://example.test/v1", "model", fake)
+ assert route["intent"] == "product_availability"
+ encoded_payload = json.dumps(fake.payload, ensure_ascii=False)
+ assert "贪吃蛇小车" in encoded_payload
+ assert "这款产品TK是否有销售" in encoded_payload
+
+ fallback = resolve_chat_intent(messages, "帮我看看这个产品", "fastmoss", "key", "https://example.test/v1", "model", FakeRequests(fail=True))
+ assert fallback["intent"] == "product_research"
+ assert fallback["route_source"] == "rules_fallback"
+ finally:
+ if previous_enabled is None:
+ os.environ.pop("CHAT_INTENT_ROUTER_ENABLED", None)
+ else:
+ os.environ["CHAT_INTENT_ROUTER_ENABLED"] = previous_enabled
+
+
+def test_empty_mcp_collections_are_not_enough_data() -> None:
+ def result(payload: dict) -> dict:
+ return {
+ "ok": True,
+ "data": {
+ "content": [{"type": "text", "text": json.dumps(payload, ensure_ascii=False)}],
+ },
+ }
+
+ empty = normalize_prefixed_tool_result(
+ "fastmoss__product_search",
+ result({"code": 0, "message": "success", "data": {"list": [], "total": 0}}),
+ )
+ assert empty["enough_data"] is False
+ assert empty["data_state"] == "empty"
+ assert empty["evidence_observed"] is True
+ assert empty["suggested_next_action"] == "answer_with_limitation"
+
+ populated = normalize_prefixed_tool_result(
+ "fastmoss__product_search",
+ result({"code": 0, "data": {"list": [{"product_id": "123", "title": "Magnetic snake toy"}], "total": 1}}),
+ )
+ assert populated["enough_data"] is True
+ assert populated["data_state"] == "data"
+ assert populated["evidence_observed"] is True
+ assert populated["suggested_next_action"] == "answer_from_results"
+
+ for payload in (
+ {"reviews": [], "total_review_count": 0},
+ {"ranked_categories": [], "total": 0},
+ {"category": {"category_id": 0, "name": "", "region": ""}, "top_products_summary": []},
+ ):
+ normalized = normalize_prefixed_tool_result("fastmoss__market_category_analysis", result(payload))
+ assert normalized["enough_data"] is False
+ assert normalized["data_state"] == "empty"
+
+ for key in ("videos", "shops", "creators", "skus", "rows"):
+ normalized = normalize_prefixed_tool_result("fastmoss__product_search", result({key: [], "total": 0}))
+ assert normalized["data_state"] == "empty"
+ assert normalized["evidence_observed"] is True
+
+ partial = normalize_prefixed_tool_result(
+ "fastmoss__market_category_analysis",
+ result({"category": {"category_id": 935176, "name": "Food Processors"}, "gmv": 12345, "top_products_summary": []}),
+ )
+ assert partial["data_state"] == "data"
+ assert partial["enough_data"] is True
+
+ evidence = compact_chat_tool_evidence("fastmoss__product_review_list", empty)
+ assert "接口调用成功但本轮返回空结果" in evidence
+ assert "不得推断为平台绝对不存在" in evidence
+
+ sellersprite_empty = normalize_prefixed_tool_result(
+ "sellersprite__market_research",
+ result({
+ "code": "OK",
+ "message": "成功",
+ "data": {
+ "pages": 1,
+ "page": 1,
+ "size": 5,
+ "total": 0,
+ "order": {"field": "", "desc": True},
+ "items": [],
+ "guestVisited": False,
+ },
+ }),
+ )
+ assert sellersprite_empty["data_state"] == "empty"
+ assert sellersprite_empty["enough_data"] is False
+
+
+def test_mcp_content_error_rules_are_provider_specific() -> None:
+ def result(payload: dict) -> dict:
+ return {
+ "ok": True,
+ "data": {
+ "content": [{"type": "text", "text": json.dumps(payload, ensure_ascii=False)}],
+ "isError": False,
+ },
+ }
+
+ sellersprite = normalize_prefixed_tool_result(
+ "sellersprite__asin_detail",
+ result({
+ "code": "OK",
+ "message": "成功",
+ "data": {"asin": "B0FBVL7SG7", "price": 26.99, "rating": 4.3, "ratings": 344},
+ }),
+ )
+ assert sellersprite["ok"] is True
+ assert sellersprite["enough_data"] is True
+ assert sellersprite["data_state"] == "data"
+ assert sellersprite["mcp_data"]["data"]["asin"] == "B0FBVL7SG7"
+ assert "error" not in sellersprite
+
+ fastmoss = normalize_prefixed_tool_result(
+ "fastmoss__product_search",
+ result({"code": "ERROR_QUERY", "message": "query failed", "data": None}),
+ )
+ assert fastmoss["ok"] is False
+ assert fastmoss["error"] == "query failed"
+ assert fastmoss["data_state"] == "error"
+
+
+def test_fastmoss_zero_analysis_metadata_is_empty_without_affecting_sellersprite() -> None:
+ payload = {
+ "code": 0,
+ "message": "success",
+ "data": {
+ "analysis_type": "basic_metrics",
+ "category_id": 935176,
+ "category_name": "Food Processors",
+ "region": "US",
+ "stat_date": "2026-W28",
+ "currency": {"code": "USD", "symbol": "$"},
+ "product_count": 0,
+ "gmv": 0,
+ "units_sold": 0,
+ "creator_count": 0,
+ "video_count": 0,
+ "trend_series": [],
+ },
+ }
+ raw = {"ok": True, "data": {"content": [{"type": "text", "text": json.dumps(payload)}]}}
+ fastmoss = normalize_prefixed_tool_result("fastmoss__market_category_analysis", raw)
+ sellersprite = normalize_prefixed_tool_result("sellersprite__market_research", raw)
+ assert fastmoss["data_state"] == "empty"
+ assert fastmoss["evidence_observed"] is True
+ assert sellersprite["data_state"] == "data"
+
+
+def test_mcp_sql_error_text_is_not_evidence() -> None:
+ raw = {
+ "ok": True,
+ "data": {
+ "content": [{"type": "text", "text": "SQLSTATE[42S22]: Unknown column 'format_price' in 'field list'"}],
+ },
+ }
+ normalized = normalize_prefixed_tool_result("fastmoss__product_detail_info", raw)
+ assert normalized["ok"] is False
+ assert normalized["enough_data"] is False
+ assert normalized["data_state"] == "error"
+ assert normalized["evidence_observed"] is False
+ assert normalized["suggested_next_action"] == "answer_with_limitation"
+
+def test_deepseek_tool_turn_preserves_reasoning_content() -> None:
+ tool_calls = [{"id": "call_1", "function": {"name": "fastmoss__product_search", "arguments": "{}"}}]
+ turn = build_deepseek_tool_assistant_message(
+ {"content": "", "reasoning_content": "internal reasoning", "tool_calls": tool_calls},
+ tool_calls,
+ True,
+ )
+ assert turn["role"] == "assistant"
+ assert turn["tool_calls"] == tool_calls
+ assert turn["reasoning_content"] == "internal reasoning"
+
+
+def test_dynamic_chat_context_compresses_to_budget() -> None:
+ messages = [{"role": "system", "content": "system rules", "_context_scope": "system"}]
+ messages.extend(
+ {
+ "role": "user" if index % 2 == 0 else "assistant",
+ "content": f"history-{index}-" + ("x" * 12000),
+ "_context_scope": "history",
+ "_context_priority": "normal",
+ }
+ for index in range(8)
+ )
+ messages.append({
+ "role": "user",
+ "content": "请根据已有数据生成报告",
+ "_context_scope": "history",
+ "_context_priority": "keep",
+ })
+ messages.append({
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [{"id": "call_1", "function": {"name": "tool__one", "arguments": "{}"}}],
+ "_context_scope": "current",
+ })
+ messages.append({
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": "evidence-" + ("y" * 20000),
+ "_context_scope": "current",
+ })
+ tools = [{
+ "type": "function",
+ "function": {"name": "tool__one", "description": "z" * 8000, "parameters": {"type": "object"}},
+ }]
+
+ request_messages, request_tools, stats = manage_chat_context(messages, tools, max_tokens=3000)
+ assert stats["initial_tokens"] > stats["max_tokens"]
+ assert stats["final_tokens"] <= stats["max_tokens"]
+ assert stats["compressed"] is True
+ assert stats["dropped_history"] > 0
+ assert stats["current_evidence_compressed"] == 1
+ assert stats["current_evidence_chars_after"] < stats["current_evidence_chars_before"]
+ assert estimate_chat_context_tokens(request_messages, request_tools) <= 3000
+ assert any(message.get("role") == "tool" for message in request_messages)
+ assert all(not any(key.startswith("_context_") for key in message) for message in request_messages)
+
+
+def test_tool_limit_final_context_removes_protocol_and_detects_dsml() -> None:
+ dsml = (
+ '<｜｜DSML｜｜tool_calls><｜｜DSML｜｜invoke name="sellersprite__google_trend">'
+ '<｜｜DSML｜｜parameter name="request">{}'
+ ''
+ )
+ assert deepseek_tool_protocol_present({"content": dsml}) is True
+ assert deepseek_tool_protocol_present({"content": "这是最终的中文分析报告。"}) is False
+ assert deepseek_tool_protocol_present({"tool_calls": [{"id": "call_1"}]}) is True
+
+ messages = [
+ {"role": "user", "content": "分析产品", "_context_scope": "history"},
+ {"role": "assistant", "content": "感谢认可，继续做1688比价", "_context_scope": "current"},
+ {
+ "role": "assistant",
+ "content": dsml,
+ "tool_calls": [{"id": "call_1", "function": {"name": "sellersprite__google_trend"}}],
+ "_context_scope": "current",
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": json.dumps({
+ "trend": "up",
+ "series": ["x" * 3000, "DEEP_EVIDENCE_MARKER"],
+ }),
+ "_context_scope": "current",
+ },
+ ]
+ final_context = build_tool_limit_final_context(messages, "分析产品")
+ assert all(message.get("role") != "tool" for message in final_context)
+ assert all(not message.get("tool_calls") for message in final_context)
+ assert all("DSML" not in str(message.get("content") or "") for message in final_context)
+ assert all("感谢认可" not in str(message.get("content") or "") for message in final_context)
+ assert any("completed_tool_collection" in str(message.get("content") or "") for message in final_context)
+ assert any("completed_tool_evidence" in str(message.get("content") or "") for message in final_context)
+ assert any("original_user_request" in str(message.get("content") or "") for message in final_context)
+ assert any("DEEP_EVIDENCE_MARKER" in str(message.get("content") or "") for message in final_context)
+ request_messages, _, stats = manage_chat_context(final_context, [], max_tokens=100000)
+ assert stats["current_evidence_compressed"] == 0
+ assert any("DEEP_EVIDENCE_MARKER" in str(message.get("content") or "") for message in request_messages)
+
+
+def test_tool_limit_keeps_large_current_collection_when_capacity_allows() -> None:
+ messages = [{"role": "user", "content": "分析 Air Pump", "_context_scope": "history"}]
+ markers = []
+ for index in range(21):
+ marker = f"CURRENT_TOOL_MARKER_{index:02d}"
+ markers.append(marker)
+ messages.append({
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [{
+ "id": f"call_{index}",
+ "function": {"name": "sellersprite__keyword_research_trends", "arguments": "{}"},
+ }],
+ "_context_scope": "current",
+ })
+ messages.append({
+ "role": "tool",
+ "tool_call_id": f"call_{index}",
+ "content": web_app.current_chat_tool_evidence(
+ "sellersprite__keyword_research_trends",
+ {"ok": True, "mcp_data": {"series": ["x" * 8200, marker]}},
+ {"marketplace": "US", "keyword": f"air pump {index}"},
+ ),
+ "_context_scope": "current",
+ })
+
+ final_context = build_tool_limit_final_context(messages, "分析 Air Pump")
+ request_messages, request_tools, stats = manage_chat_context(final_context, [], max_tokens=120000)
+ encoded = json.dumps(request_messages, ensure_ascii=False)
+ assert request_tools == []
+ assert stats["over_budget"] is False
+ assert stats["current_evidence_compressed"] == 0
+ assert stats["current_evidence_chars_before"] > 170000
+ assert stats["current_evidence_chars_after"] == stats["current_evidence_chars_before"]
+ assert all(marker in encoded for marker in markers)
+
+
+def test_sellersprite_schema_argument_normalization() -> None:
+ schemas = [
+ {
+ "name": "keyword_research_trends",
+ "inputSchema": {
+ "type": "object",
+ "properties": {
+ "marketplace": {"type": "string"},
+ "keyword": {"type": "string"},
+ "month": {"type": "string"},
+ },
+ "required": ["marketplace", "keyword"],
+ "additionalProperties": False,
+ },
+ },
+ {
+ "name": "product_research",
+ "inputSchema": {
+ "type": "object",
+ "properties": {
+ "request": {
+ "type": "object",
+ "properties": {"marketplace": {"type": "string"}, "keyword": {"type": "string"}},
+ "required": ["marketplace"],
+ },
+ },
+ "required": ["request"],
+ "additionalProperties": False,
+ },
+ },
+ ]
+ original = web_app.list_mcp_bridge_tools
+ web_app.list_mcp_bridge_tools = lambda chat_type: schemas
+ try:
+ unwrapped, action = normalize_mcp_tool_arguments(
+ "sellersprite",
+ "keyword_research_trends",
+ {"request": {"marketplace": "US", "keyword": "electric food chopper", "month": "202606"}},
+ )
+ assert unwrapped == {"marketplace": "US", "keyword": "electric food chopper", "month": "202606"}
+ assert action and action.startswith("unwrapped")
+
+ wrapped, action = normalize_mcp_tool_arguments(
+ "sellersprite", "product_research", {"marketplace": "US", "keyword": "mini chopper"}
+ )
+ assert wrapped == {"request": {"marketplace": "US", "keyword": "mini chopper"}}
+ assert action and action.startswith("wrapped")
+
+ try:
+ normalize_mcp_tool_arguments("sellersprite", "keyword_research_trends", {"month": "202606"})
+ except ValueError as exc:
+ assert "marketplace" in str(exc) and "keyword" in str(exc)
+ else:
+ raise AssertionError("missing required fields should fail before the MCP call")
+ finally:
+ web_app.list_mcp_bridge_tools = original
+
+
+def test_llm_router_can_select_fastmoss_playbook() -> None:
+ fallback = route_chat_intent("给我一份厨房切碎机的完整调研报告", "fastmoss")
+ route = parse_chat_intent_decision(
+ {
+ "intent": "product_research",
+ "task_depth": "analysis",
+ "playbook": "product",
+ "entity": "electric food shredder",
+ "region": "US",
+ "confidence": 0.96,
+ },
+ fallback,
+ "fastmoss",
+ "给我一份厨房切碎机的完整调研报告",
+ )
+ assert route["intent"] == "fastmoss_product"
+ assert route["task_depth"] == "workflow"
+ assert route["playbook"] == "product"
+ assert route["max_rounds"] == web_app.FASTMOSS_PLAYBOOKS["product"]["max_rounds"]
+
+
+def test_fastmoss_workflow_phases_accept_empty_and_error_attempts() -> None:
+ available = {tool_id for phases in web_app.FASTMOSS_WORKFLOW_PHASES.values() for _, tools in phases for tool_id in tools}
+ for playbook_id, phases in web_app.FASTMOSS_WORKFLOW_PHASES.items():
+ message = SimpleNamespace(tool_calls=[], tool_results=[])
+ phase = web_app.fastmoss_workflow_phase(playbook_id, message, available)
+ assert phase is not None
+ assert phase[0] == phases[0][0]
+ assert phase[1] == set(phases[0][1])
+
+ message = SimpleNamespace(tool_calls=[], tool_results=[])
+ first = web_app.fastmoss_workflow_phase("product", message, available)
+ assert first and first[1] == {"fastmoss__search_category_by_words"}
+ message.tool_results.append({
+ "tool_name": "fastmoss__search_category_by_words",
+ "result": {"ok": True, "enough_data": False, "data_state": "empty", "evidence_observed": True},
+ })
+ second = web_app.fastmoss_workflow_phase("product", message, available)
+ assert second and second[0].startswith("获取类目规模与趋势")
+ message.tool_calls.append({
+ "function": {"name": "fastmoss__market_category_analysis", "arguments": json.dumps({"analysis_type": "basic_metrics"})},
+ })
+ message.tool_results.append({
+ "tool_name": "fastmoss__market_category_analysis",
+ "result": {"ok": False, "data_state": "error", "evidence_observed": False},
+ })
+ alternative = web_app.fastmoss_workflow_phase("product", message, available)
+ assert alternative and alternative[0].startswith("获取类目规模与趋势")
+ assert alternative[1] == {"fastmoss__market_category_analysis"}
+ assert "sales_trends" in alternative[0]
+ for analysis_type in ("sales_trends", "price_distribution"):
+ message.tool_calls.append({
+ "function": {"name": "fastmoss__market_category_analysis", "arguments": json.dumps({"analysis_type": analysis_type})},
+ })
+ message.tool_results.append({
+ "tool_name": "fastmoss__market_category_analysis",
+ "result": {"ok": False, "data_state": "error", "evidence_observed": False},
+ })
+ ranking = web_app.fastmoss_workflow_phase("product", message, available)
+ assert ranking and ranking[1] == {"fastmoss__market_category_ranking"}
+ message.tool_results.append({
+ "tool_name": "fastmoss__market_category_ranking",
+ "result": {"ok": False, "data_state": "error", "evidence_observed": False},
+ })
+ third = web_app.fastmoss_workflow_phase("product", message, available)
+ assert third and third[0] == "获取热销样本"
+ assert third[1] == {"fastmoss__product_rank_top_selling"}
+
+
+def test_fastmoss_product_phase_requires_complete_sample_coverage() -> None:
+ available = {tool_id for phases in web_app.FASTMOSS_WORKFLOW_PHASES.values() for _, tools in phases for tool_id in tools}
+ observed_tools = (
+ "fastmoss__search_category_by_words",
+ "fastmoss__market_category_analysis",
+ "fastmoss__market_category_ranking",
+ "fastmoss__product_search",
+ )
+ message = SimpleNamespace(
+ tool_calls=[{
+ "function": {
+ "name": "fastmoss__market_category_analysis",
+ "arguments": json.dumps({"analysis_type": analysis_type}),
+ },
+ } for analysis_type in web_app.FASTMOSS_PRODUCT_MARKET_ANALYSIS_TYPES],
+ tool_results=[{
+ "tool_name": tool_name,
+ "result": {"ok": True, "data_state": "data", "evidence_observed": True},
+ } for tool_name in observed_tools],
+ )
+ phase = web_app.fastmoss_workflow_phase("product", message, available)
+ assert phase and phase[1] == {"fastmoss__product_rank_top_selling"}
+ message.tool_results.append({
+ "tool_name": "fastmoss__product_rank_top_selling",
+ "result": {"ok": True, "data_state": "empty", "evidence_observed": True},
+ })
+ phase = web_app.fastmoss_workflow_phase("product", message, available)
+ assert phase and phase[1] == {"fastmoss__product_rank_new_listed"}
+ instruction = web_app.fastmoss_workflow_instruction(phase)
+ assert "新品成功率" in instruction
+ assert "统一截止日" in instruction
+
+
+def test_fastmoss_business_defaults_use_verified_category_levels() -> None:
+ message = SimpleNamespace(tool_calls=[], tool_results=[{
+ "tool_name": "fastmoss__search_category_by_words",
+ "result": {
+ "ok": True,
+ "mcp_data": {"data": {"list": [{
+ "category_id_level1": 13,
+ "category_id_level2": 844168,
+ "category_id_level3": 935176,
+ }]}},
+ },
+ }])
+ fixed_today = __import__("datetime").date(2026, 7, 16)
+ assert web_app.fastmoss_current_category_path(message) == {
+ "level1": 13, "level2": 844168, "level3": 935176,
+ }
+ market = web_app.apply_fastmoss_business_defaults(
+ "market_category_analysis", {"analysis_type": "sales_trends", "filter": {"category_id": 935176}}, message, fixed_today
+ )
+ assert market["filter"]["category_id"] == 844168
+ assert market["filter"]["date_value"] == "2026-W28"
+ ranking = web_app.apply_fastmoss_business_defaults("market_category_ranking", {}, message, fixed_today)
+ assert ranking["filter"]["category_id"] == 13
+ top = web_app.apply_fastmoss_business_defaults("product_rank_top_selling", {}, message, fixed_today)
+ assert top["filter"]["category_id"] == 844168
+ new = web_app.apply_fastmoss_business_defaults("product_rank_new_listed", {}, message, fixed_today)
+ assert new["filter"]["category_l1_id"] == 13
+ assert new["filter"]["category_l3_id"] == 935176
+ assert new["filter"]["listing_start_date"] == "2026-06-13"
+ assert new["filter"]["listing_end_date"] == "2026-07-12"
+ assert "lang" not in new
+ assert top["orderby"] == [{"field": "period_units_sold", "order": "desc"}]
+ assert new["orderby"] == [{"field": "day3_units_sold", "order": "desc"}]
+ search = web_app.apply_fastmoss_business_defaults("product_search", {"keywords": "mini chopper"}, message, fixed_today)
+ assert search["filter"]["category_path"] == [13, 844168, 935176]
+ planned = web_app.fastmoss_planned_product_search_arguments(
+ message,
+ "Electric Food Shredder, Mini Meat Grinder 调研报告",
+ {"playbook": "product", "entity": "Electric Food Shredder, Mini Meat Grinder"},
+ "US",
+ )
+ assert planned == {
+ "filter": {"category_path": [13, 844168, 935176], "region": "US"},
+ "page": 1,
+ "pagesize": 10,
+ "orderby": [{"field": "day28_units_sold", "order": "desc"}],
+ }
+
+ named_category_message = SimpleNamespace(tool_calls=[], tool_results=[{
+ "tool_name": "fastmoss__search_category_by_words",
+ "result": {"ok": True, "mcp_data": {"result": {"categories": [
+ {
+ "category_id_level1": 13, "category_id_level2": 844168, "category_id_level3": 934792,
+ "cn_name": "搅拌机", "score": 0.6977,
+ },
+ {
+ "category_id_level1": 13, "category_id_level2": 844168, "category_id_level3": 935176,
+ "cn_name": "料理机", "score": 0.5023,
+ },
+ ]}}},
+ }])
+ assert web_app.fastmoss_current_category_path(named_category_message) == {
+ "level1": 13, "level2": 844168, "level3": 934792,
+ }
+ assert web_app.fastmoss_current_category_path(named_category_message, "第二个，料理机") == {
+ "level1": 13, "level2": 844168, "level3": 935176,
+ }
+ named_search = web_app.apply_fastmoss_business_defaults(
+ "product_search", {}, named_category_message, fixed_today,
+ user_text="目标类目明确选择料理机", route={"playbook": "product", "entity": "料理机"},
+ )
+ assert named_search["filter"]["category_path"] == [13, 844168, 935176]
+ category_args = web_app.apply_fastmoss_business_defaults(
+ "search_category_by_words", {"query": ["food processor"], "desc": "瑜伽裤", "unexpected": 1},
+ named_category_message, fixed_today, user_text="料理机",
+ )
+ assert category_args == {"query": ["料理机"]}
+
+ original_query_args = web_app.apply_fastmoss_business_defaults(
+ "search_category_by_words",
+ {"query": ["Electric Food Shredder", "Mini Meat Grinder", "Food Processor"], "top_k": 8},
+ SimpleNamespace(tool_calls=[], tool_results=[]),
+ fixed_today,
+ user_text="给我一份 Electric Food Shredder, Mini Meat Grinder 这类产品的调研报告",
+ route={"playbook": "product", "entity": "Electric Food Shredder, Mini Meat Grinder, Food Processor"},
+ )
+ assert original_query_args == {
+ "query": ["Electric Food Shredder", "Mini Meat Grinder"], "top_k": 8,
+ }
+
+
+def test_fastmoss_product_workflow_keeps_its_round_budget_isolated() -> None:
+ assert web_app.chat_max_tool_rounds("fastmoss", {"playbook": "product"}, 2) == 24
+ assert web_app.chat_max_tool_rounds("fastmoss", {"playbook": "product", "max_rounds": 14}, 2) == 24
+ assert web_app.chat_max_tool_rounds(
+ "fastmoss", {"playbook": "product", "max_rounds": 14, "full_ranking": True}, 2
+ ) == 27
+ assert web_app.chat_max_tool_rounds("amazon", {"max_rounds": 14}, 20) == 10
+
+
+def test_fastmoss_research_report_uses_product_playbook_on_rule_fallback() -> None:
+ text = "给我一份 Electric Food Shredder, Mini Meat Grinder 这类产品的调研报告"
+ assert web_app.fastmoss_playbook_intent(text) == "product"
+ assert web_app.fastmoss_segment_keywords(text, {"playbook": "product"}) == [
+ "Electric Food Shredder", "Mini Meat Grinder",
+ ]
+ explicit = "目标类目明确选择料理机。请给我一份 Electric Food Shredder 和 Mini Meat Grinder 这类产品的完整调研报告。"
+ assert web_app.fastmoss_segment_keywords(explicit, {"entity": explicit}) == [
+ "Electric Food Shredder", "Mini Meat Grinder",
+ ]
+ route = web_app.route_chat_intent(text, "fastmoss")
+ assert route["intent"] == "fastmoss_product"
+ assert route["playbook"] == "product"
+ messages = [
+ SimpleNamespace(role="user", content=text),
+ SimpleNamespace(role="assistant", content="请选择第二个类目"),
+ SimpleNamespace(role="user", content="第二个，料理机。请继续生成完整调研报告。"),
+ ]
+ inherited = web_app.fastmoss_inherited_segment_keywords(messages, messages[-1].content)
+ assert inherited == ["Electric Food Shredder", "Mini Meat Grinder"]
+ assert web_app.fastmoss_segment_keywords(messages[-1].content, {"segment_keywords": inherited}) == inherited
+ exact_confirmation = [
+ SimpleNamespace(role="user", content=text),
+ SimpleNamespace(
+ role="assistant",
+ content="FastMoss 对这个关键词的类目匹配很接近，请直接回复要研究的类目名称。",
+ ),
+ SimpleNamespace(role="user", content="料理机"),
+ ]
+ exact_inherited = web_app.fastmoss_inherited_segment_keywords(
+ exact_confirmation, exact_confirmation[-1].content
+ )
+ assert exact_inherited == ["Electric Food Shredder", "Mini Meat Grinder"]
+ later_research = exact_confirmation + [
+ SimpleNamespace(role="assistant", content="料理机调研报告已完成。"),
+ SimpleNamespace(role="user", content="重新调研 air fryer"),
+ ]
+ assert web_app.fastmoss_inherited_segment_keywords(later_research, later_research[-1].content) == []
+ assert web_app.fastmoss_inherited_segment_keywords(messages, "重新调研 air fryer") == []
+
+
+def test_fastmoss_clarification_is_targeted_and_provider_isolated() -> None:
+ route = {"intent": "fastmoss_product", "task_depth": "workflow", "playbook": "product", "entity": ""}
+ question = web_app.fastmoss_clarifying_question("fastmoss", route, "帮我做一份选品报告")
+ assert question and "具体商品或品类关键词" in question
+ assert "美国区" in question and "最近已完成周期" in question
+ assert web_app.fastmoss_clarifying_question("fastmoss", route, "给我一份 electric food shredder 调研报告") is None
+ assert web_app.fastmoss_clarifying_question("fastmoss", route, "继续分析这款产品") is None
+ assert web_app.fastmoss_clarifying_question("amazon", route, "帮我做一份选品报告") is None
+
+
+def test_fastmoss_close_cross_category_matches_request_confirmation() -> None:
+ result = {"mcp_data": {"result": {"categories": [
+ {
+ "category_id_level1": 11, "category_id_level2": 858632, "category_id_level3": 861576,
+ "cn_full_name": "厨房用品-刀具-厨房剪刀", "score": 0.52,
+ },
+ {
+ "category_id_level1": 13, "category_id_level2": 844168, "category_id_level3": 935176,
+ "cn_full_name": "家电-厨房家电-料理机", "score": 0.502,
+ },
+ ]}}}
+ question = web_app.fastmoss_category_ambiguity_question("Electric Food Shredder, Mini Meat Grinder 调研", result)
+ assert question and "厨房剪刀" in question and "料理机" in question
+ assert "不会继续消耗" in question
+ result["mcp_data"]["result"]["categories"][0]["category_id_level2"] = 844168
+ assert web_app.fastmoss_category_ambiguity_question("Electric Food Shredder 调研", result)
+
+ latest_result = {"mcp_data": {"result": {"categories": [
+ {
+ "category_id_level1": 13, "category_id_level2": 844168, "category_id_level3": 934664,
+ "cn_name": "面包机", "cn_full_name": "家电-厨房家电-面包机", "score": 0.5152,
+ "matched_query": "Food Processor",
+ },
+ {
+ "category_id_level1": 13, "category_id_level2": 844168, "category_id_level3": 935176,
+ "cn_name": "料理机", "cn_full_name": "家电-厨房家电-料理机", "score": 0.5023,
+ "matched_query": "Electric Food Shredder",
+ },
+ {
+ "category_id_level1": 13, "category_id_level2": 844168, "category_id_level3": 983944,
+ "cn_name": "垃圾处理器", "cn_full_name": "家电-厨房家电-垃圾处理器", "score": 0.4891,
+ "matched_query": "Electric Food Shredder",
+ },
+ ]}}}
+ latest_question = web_app.fastmoss_category_ambiguity_question(
+ "Electric Food Shredder, Mini Meat Grinder 调研", latest_result
+ )
+ assert latest_question and "料理机" in latest_question and "垃圾处理器" in latest_question
+ assert "面包机" not in latest_question
+ assert web_app.fastmoss_category_ambiguity_question("目标类目明确选择料理机", latest_result) is None
+
+
+def test_provider_profiles_use_aggregated_sellersprite_and_staged_fastmoss_tools() -> None:
+ selected = {
+ "system__current_time",
+ "sellersprite__keyword_research",
+ "sellersprite__market_research",
+ "sellersprite__product_research",
+ "sellersprite__asin_detail",
+ "sellersprite__asin_sales_trend",
+ "sellersprite__review",
+ }
+ message = SimpleNamespace(tool_calls=[], tool_results=[])
+ research = web_app.provider_profile_tool_ids("amazon", {"intent": "product_research"}, "electric chopper", selected, message)
+ assert "sellersprite__keyword_research" in research
+ assert "sellersprite__market_research" in research
+ assert "sellersprite__asin_detail" not in research
+ asin = web_app.provider_profile_tool_ids("amazon", {"intent": "product_research"}, "分析 B0ABCDEFGH", selected, message)
+ assert "sellersprite__asin_detail" in asin
+ assert "sellersprite__keyword_research" not in asin
+
+ fastmoss_selected = {"system__current_time", "fastmoss__search_category_by_words", "fastmoss__market_category_analysis", "fastmoss__product_detail_info"}
+ staged = web_app.provider_profile_tool_ids("fastmoss", {"playbook": "product"}, "electric chopper", fastmoss_selected, message)
+ assert staged == {"system__current_time", "fastmoss__search_category_by_words"}
+
+
+def test_region_default_only_applies_when_schema_supports_it() -> None:
+ schemas = [
+ {
+ "name": "market_category_analysis",
+ "inputSchema": {
+ "type": "object",
+ "properties": {"filter": {"type": "object", "properties": {"category_id": {"type": "integer"}, "region": {"type": "string"}}}},
+ },
+ },
+ {
+ "name": "product_detail_info",
+ "inputSchema": {
+ "type": "object",
+ "properties": {"filter": {"type": "object", "properties": {"product_id": {"type": "string"}}}},
+ },
+ },
+ ]
+ original = web_app.list_mcp_bridge_tools
+ web_app.list_mcp_bridge_tools = lambda _chat_type: schemas
+ try:
+ regional = web_app.apply_mcp_region_default("fastmoss", "market_category_analysis", {"filter": {"category_id": 935176}}, "US")
+ assert regional == {"filter": {"category_id": 935176, "region": "US"}}
+ no_region = web_app.apply_mcp_region_default("fastmoss", "product_detail_info", {"filter": {"product_id": "1732183167826498507"}}, "US")
+ assert no_region == {"filter": {"product_id": "1732183167826498507"}}
+ finally:
+ web_app.list_mcp_bridge_tools = original
+
+
+def test_fastmoss_deep_dive_ids_must_come_from_current_task() -> None:
+ message = SimpleNamespace(
+ tool_calls=[],
+ tool_results=[{
+ "tool_name": "fastmoss__product_search",
+ "result": {"ok": True, "mcp_data": {"products": [{"product_id": "1732183167826498507"}]}},
+ }, {
+ "tool_name": "fastmoss__search_category_by_words",
+ "result": {"ok": True, "mcp_data": {"items": [{"category_id": 935176, "category_id_level2": 844168}]}},
+ }],
+ )
+ assert web_app.fastmoss_deep_dive_call_error(
+ "fastmoss__product_detail_info", {"filter": {"product_id": "1732183167826498507"}}, "electric chopper", message
+ ) is None
+ assert "未经当前任务" in web_app.fastmoss_deep_dive_call_error(
+ "fastmoss__product_detail_info", {"filter": {"product_id": "1730819059386716431"}}, "electric chopper", message
+ )
+ assert web_app.fastmoss_deep_dive_call_error(
+ "fastmoss__market_category_analysis", {"filter": {"category_id": 935176}}, "electric chopper", message
+ ) is None
+ assert web_app.fastmoss_deep_dive_call_error(
+ "fastmoss__market_category_analysis", {"filter": {"category_id": 844168}}, "electric chopper", message
+ ) is None
+ assert "类目 ID" in web_app.fastmoss_deep_dive_call_error(
+ "fastmoss__market_category_analysis", {"filter": {"category_id": 855944}}, "electric chopper", message
+ )
+
+
+def test_tool_call_signature_deduplicates_argument_order() -> None:
+ left = web_app.tool_call_signature("fastmoss__product_search", {"keywords": "chopper", "page": 1})
+ right = web_app.tool_call_signature("fastmoss__product_search", {"page": 1, "keywords": "chopper"})
+ assert left == right
+
+
+def _fastmoss_search_message(calls: list[dict], results: list[dict] | None = None) -> SimpleNamespace:
+ return SimpleNamespace(
+ tool_calls=[{
+ "id": f"call_{index}",
+ "function": {"name": "fastmoss__product_search", "arguments": json.dumps(args)},
+ } for index, args in enumerate(calls)],
+ tool_results=results or [],
+ )
+
+
+def test_fastmoss_dual_ranking_plan_uses_three_sorted_category_pages_then_segments() -> None:
+ route = {"playbook": "product", "entity": "Electric Food Shredder / Mini Meat Grinder"}
+ message = _fastmoss_search_message([])
+ plan = web_app.fastmoss_product_search_plan(message, "给我一份调研报告", route)
+ assert plan["category_pages"] == 3
+ assert plan["segment_keywords"] == ["Electric Food Shredder", "Mini Meat Grinder"]
+ assert plan["next_call"] == {"scope": "category_head", "page": 1, "pagesize": 10}
+
+ message = _fastmoss_search_message([
+ {"page": 1, "pagesize": 10},
+ {"page": 2, "pagesize": 10},
+ {"page": 3, "pagesize": 10},
+ ])
+ plan = web_app.fastmoss_product_search_plan(message, "给我一份调研报告", route)
+ assert plan["next_call"]["scope"] == "segment_head"
+ assert plan["next_call"]["keywords"] == "Electric Food Shredder"
+
+ message.tool_calls.extend([
+ {"id": "segment_1", "function": {"name": "fastmoss__product_search", "arguments": json.dumps({"keywords": "Electric Food Shredder", "page": 1})}},
+ {"id": "segment_2", "function": {"name": "fastmoss__product_search", "arguments": json.dumps({"keywords": "Mini Meat Grinder", "page": 1})}},
+ ])
+ assert web_app.fastmoss_product_search_plan(message, "给我一份调研报告", route)["complete"] is True
+ assert web_app.fastmoss_product_search_plan(message, "给我完整类目榜单", route)["category_pages"] == 6
+
+
+def test_fastmoss_product_phase_waits_for_all_category_and_segment_searches() -> None:
+ available = {tool_id for phases in web_app.FASTMOSS_WORKFLOW_PHASES.values() for _, tools in phases for tool_id in tools}
+ completed_before_search = (
+ "fastmoss__search_category_by_words",
+ "fastmoss__market_category_analysis",
+ "fastmoss__market_category_ranking",
+ "fastmoss__product_rank_top_selling",
+ "fastmoss__product_rank_new_listed",
+ )
+ message = _fastmoss_search_message([{"page": 1}, {"page": 2}], [
+ *[{
+ "tool_name": tool_name,
+ "result": {"ok": True, "data_state": "data", "evidence_observed": True},
+ } for tool_name in completed_before_search],
+ {"tool_name": "fastmoss__product_search", "result": {"ok": True, "data_state": "data", "evidence_observed": True}},
+ {"tool_name": "fastmoss__product_search", "result": {"ok": True, "data_state": "empty", "evidence_observed": True}},
+ ])
+ message.tool_calls.extend({
+ "function": {
+ "name": "fastmoss__market_category_analysis",
+ "arguments": json.dumps({"analysis_type": analysis_type}),
+ },
+ } for analysis_type in web_app.FASTMOSS_PRODUCT_MARKET_ANALYSIS_TYPES)
+ route = {"playbook": "product", "entity": "Electric Food Shredder / Mini Meat Grinder"}
+ phase = web_app.fastmoss_workflow_phase("product", message, available, "调研", route)
+ assert phase and phase[0] == "获取类目销量头部（第 3/3 页）"
+ message.tool_calls.append({
+ "function": {"name": "fastmoss__product_search", "arguments": json.dumps({"page": 3})},
+ })
+ phase = web_app.fastmoss_workflow_phase("product", message, available, "调研", route)
+ assert phase and phase[0] == "补充细分匹配样本"
+ assert phase[1] == {"fastmoss__product_search"}
+
+
+def test_fastmoss_product_workflow_deterministically_advances_and_binds_two_targets() -> None:
+ available = provider_default_enabled_tool_ids("fastmoss")
+ route = {"playbook": "product", "entity": "Electric Food Shredder / Mini Meat Grinder"}
+ category_result = {
+ "tool_name": "fastmoss__search_category_by_words",
+ "result": {
+ "ok": True,
+ "data_state": "data",
+ "evidence_observed": True,
+ "mcp_data": {"items": [{
+ "category_id_level1": 13,
+ "category_id_level2": 844168,
+ "category_id_level3": 935176,
+ }]},
+ },
+ }
+ message = SimpleNamespace(tool_calls=[], tool_results=[category_result])
+ first = web_app.fastmoss_planned_product_workflow_call(message, "调研", route, available, "US")
+ assert first and first[0] == "fastmoss__market_category_analysis"
+ assert first[1]["analysis_type"] == "basic_metrics"
+ assert first[1]["filter"]["region"] == "US"
+ message.tool_calls.append({
+ "function": {"name": first[0], "arguments": json.dumps(first[1])},
+ })
+ message.tool_results.append({
+ "tool_name": first[0],
+ "result": {"ok": True, "data_state": "empty", "evidence_observed": False},
+ })
+ second = web_app.fastmoss_planned_product_workflow_call(message, "调研", route, available, "US")
+ assert second and second[0] == "fastmoss__market_category_analysis"
+ assert second[1]["analysis_type"] == "sales_trends"
+
+ message.tool_calls.extend({
+ "function": {
+ "name": "fastmoss__market_category_analysis",
+ "arguments": json.dumps({"analysis_type": analysis_type}),
+ },
+ } for analysis_type in ("sales_trends", "price_distribution"))
+ for tool_name in (
+ "fastmoss__market_category_analysis",
+ "fastmoss__market_category_analysis",
+ "fastmoss__market_category_ranking",
+ "fastmoss__product_rank_top_selling",
+ "fastmoss__product_rank_new_listed",
+ ):
+ message.tool_results.append({
+ "tool_name": tool_name,
+ "result": {"ok": True, "data_state": "data", "evidence_observed": True},
+ })
+ message.tool_calls.extend([
+ {"function": {"name": "fastmoss__market_category_ranking", "arguments": "{}"}},
+ {"function": {"name": "fastmoss__product_rank_top_selling", "arguments": "{}"}},
+ {"function": {"name": "fastmoss__product_rank_new_listed", "arguments": "{}"}},
+ ])
+ search_calls = [
+ {"page": 1}, {"page": 2}, {"page": 3},
+ {"keywords": "Electric Food Shredder", "page": 1},
+ {"keywords": "Mini Meat Grinder", "page": 1},
+ ]
+ message.tool_calls.extend({
+ "function": {"name": "fastmoss__product_search", "arguments": json.dumps(arguments)},
+ } for arguments in search_calls)
+ for query, product in (
+ ("Electric Food Shredder", {
+ "product_id": "172900000000000001", "day28_units_sold": 100,
+ "day28_gmv": 1000, "price_min": 10, "price_max": 10,
+ }),
+ ("Mini Meat Grinder", {
+ "product_id": "172900000000000002", "day28_units_sold": 80,
+ "day28_gmv": 1600, "price_min": 20, "price_max": 20,
+ }),
+ ):
+ message.tool_results.append({
+ "tool_name": "fastmoss__product_search",
+ "result": {
+ "ok": True,
+ "data_state": "data",
+ "evidence_observed": True,
+ "evidence_metadata": {"scope": "segment_head", "query": query},
+ "evidence_product_records": [product],
+ },
+ })
+ # The three category pages need stored results as well, even though target
+ # selection comes from the two segment calls above.
+ message.tool_results.extend({
+ "tool_name": "fastmoss__product_search",
+ "result": {
+ "ok": True, "data_state": "empty", "evidence_observed": False,
+ "evidence_metadata": {"scope": "category_head", "page": page},
+ "evidence_product_records": [],
+ },
+ } for page in (1, 2, 3))
+
+ overview = web_app.fastmoss_planned_product_workflow_call(message, "调研", route, available, "US")
+ assert overview and overview[0] == "fastmoss__product_overview"
+ assert overview[1]["filter"]["product_id"] == "172900000000000001"
+ for product_id in ("172900000000000001", "172900000000000002"):
+ message.tool_calls.append({
+ "function": {
+ "name": "fastmoss__product_overview",
+ "arguments": json.dumps({"filter": {"product_id": product_id, "time_range_days": 28}}),
+ },
+ })
+ trend = web_app.fastmoss_planned_product_workflow_call(message, "调研", route, available, "US")
+ assert trend and trend[0] == "fastmoss__product_sales_trend"
+ assert trend[1]["filter"] == {"product_id": "172900000000000001", "time_range_days": 90}
+
+
+def test_fastmoss_product_search_defaults_force_category_pages_and_short_segment_queries() -> None:
+ category_result = {
+ "tool_name": "fastmoss__search_category_by_words",
+ "result": {"ok": True, "mcp_data": {"items": [{
+ "category_id_level1": 13, "category_id_level2": 844168, "category_id_level3": 935176,
+ }]}},
+ }
+ route = {"playbook": "product", "entity": "Electric Food Shredder / Mini Meat Grinder"}
+ message = SimpleNamespace(tool_calls=[], tool_results=[category_result])
+ first = web_app.apply_fastmoss_business_defaults(
+ "product_search", {
+ "keywords": "wrong long keyword", "page": 99,
+ "filter": {"day28_units_sold_range": {"min": 0}, "region": "US"},
+ }, message,
+ user_text="调研", route=route,
+ )
+ assert "keywords" not in first
+ assert first["page"] == 1 and first["pagesize"] == 10
+ assert first["orderby"] == [{"field": "day28_units_sold", "order": "desc"}]
+ assert first["filter"]["category_path"] == [13, 844168, 935176]
+ assert first["filter"] == {"category_path": [13, 844168, 935176], "region": "US"}
+
+ message.tool_calls = [
+ {"function": {"name": "fastmoss__product_search", "arguments": json.dumps({"page": page})}}
+ for page in (1, 2, 3)
+ ]
+ segment = web_app.apply_fastmoss_business_defaults(
+ "product_search", {"keywords": "ignored"}, message,
+ user_text="调研", route=route,
+ )
+ assert segment["keywords"] == "Electric Food Shredder"
+ assert segment["page"] == 1
+
+
+def test_fastmoss_evidence_manifest_separates_total_fetched_overlap_and_conflicts() -> None:
+ def result(scope: str, page: int, query: str, total: int, records: list[dict]) -> dict:
+ return {
+ "tool_name": "fastmoss__product_search",
+ "result": {
+ "ok": True,
+ "data_state": "data",
+ "evidence_observed": True,
+ "evidence_metadata": {
+ "scope": scope, "page": page, "query": query or None,
+ "reported_total": total, "fetched_records": len(records), "sort_verified": True,
+ "category_level": "L3",
+ },
+ "evidence_product_records": records,
+ },
+ }
+ category_records = [
+ {"product_id": "1732183167826498507", "title": "Head", "day7_units_sold": 120, "day28_units_sold": 100, "day28_gmv": 1000, "price_min": 9, "price_max": 11},
+ {"product_id": "1732183167826498508", "title": "Second", "day28_units_sold": 80},
+ ]
+ segment_records = [dict(category_records[0])]
+ tool_results = [
+ result("category_head", 1, "", 60, category_records),
+ result("category_head", 2, "", 60, []),
+ result("category_head", 3, "", 60, []),
+ result("segment_head", 1, "mini grinder", 12, segment_records),
+ ]
+ calls = [{"page": 1}, {"page": 2}, {"page": 3}, {"page": 1, "keywords": "mini grinder"}]
+ message = _fastmoss_search_message(calls, tool_results)
+ manifest = web_app.fastmoss_evidence_manifest(message, "普通调研", {"playbook": "product", "entity": "mini grinder"})
+ assert manifest["category_head"]["reported_total"] == 60
+ assert manifest["category_head"]["fetched_unique"] == 2
+ assert manifest["category_head"]["completed_pages"] == [1, 2, 3]
+ assert manifest["overlap_product_ids"] == ["1732183167826498507"]
+ assert any("近7天销量高于近28天销量" in item["issue"] for item in manifest["conflicts"])
+ assert sum("近7天销量高于近28天销量" in item["issue"] for item in manifest["conflicts"]) == 1
+ assert any("接口报告匹配总数 60" in item for item in manifest["limitations"])
+ signals = manifest["derived_signals"]
+ assert signals["category_sample_units_total"] == 180
+ assert signals["category_top1_share"] == 100 / 180
+ assert signals["overlap_rate_of_segment_sample"] == 1
+ assert signals["price_midpoint_median"] == 10
+ assert signals["segment_queries"][0]["sample_units_total"] == 100
+
+
+def test_fastmoss_deterministic_fallback_contains_analysis_and_prioritized_advice() -> None:
+ category_products = [
+ {"product_id": "1", "title": "Leader", "day28_units_sold": 400, "price_min": 30, "price_max": 40},
+ {"product_id": "2", "title": "Second", "day28_units_sold": 300, "price_min": 35, "price_max": 45},
+ {"product_id": "3", "title": "Third", "day28_units_sold": 200, "price_min": 40, "price_max": 50},
+ {"product_id": "4", "title": "Tail", "day28_units_sold": 100, "price_min": 45, "price_max": 55},
+ ]
+ segment_products = [
+ {"product_id": "1", "title": "Mini grinder leader", "query": "Mini Meat Grinder", "day28_units_sold": 300},
+ {"product_id": "5", "title": "Mini grinder tail", "query": "Mini Meat Grinder", "day28_units_sold": 100},
+ {"product_id": "6", "title": "Shredder", "query": "Electric Food Shredder", "day28_units_sold": 20},
+ ]
+ manifest = {
+ "category_head": {
+ "products": category_products, "fetched_unique": 4, "target_pages": 3,
+ "completed_pages": [1, 2, 3], "reported_total": 20,
+ },
+ "segment_head": {"products": segment_products, "fetched_unique": 3},
+ "overlap_product_ids": ["1"],
+ "target_category_path": {"level1": 13, "level2": 844168, "level3": 935176},
+ "derived_signals": {
+ "category_top3_share": 0.9,
+ "overlap_rate_of_segment_sample": 1 / 3,
+ "price_midpoint_q1": 38.75,
+ "price_midpoint_median": 42.5,
+ "price_midpoint_q3": 46.25,
+ "segment_queries": [
+ {"query": "Mini Meat Grinder", "fetched_unique": 2, "sample_units_total": 400, "top_product_units": 300},
+ {"query": "Electric Food Shredder", "fetched_unique": 2, "sample_units_total": 20, "top_product_units": 20},
+ ],
+ },
+ "limitations": [],
+ "conflicts": [],
+ }
+ report = web_app.fastmoss_deterministic_quality_fallback(manifest)
+ assert "样本销量明显向少数商品集中" in report
+ assert "Mini Meat Grinder 的样本销量合计为 400" in report
+ assert "把 Mini Meat Grinder 放到更高的验证优先级" in report
+ assert "中间 50%" in report and "38.75–46.25" in report and "中位数约 42.5" in report
+ assert "**先定方向：** 优先围绕 Mini Meat Grinder" in report
+ assert "不能把配件、不同规格和不同使用场景的商品混成一个价格带" in report
+
+
+def test_fastmoss_l2_market_metrics_are_only_upstream_category_reference() -> None:
+ message = SimpleNamespace(tool_calls=[], tool_results=[{
+ "tool_name": "fastmoss__search_category_by_words",
+ "result": {"ok": True, "data_state": "data", "evidence_observed": True, "mcp_data": {"items": [{
+ "category_id_level1": 13, "category_id_level2": 844168, "category_id_level3": 935176,
+ }]}},
+ }, {
+ "tool_name": "fastmoss__market_category_analysis",
+ "result": {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_metadata": {"source_tool": "fastmoss__market_category_analysis", "category_level": "L2", "scope": "supporting"},
+ },
+ }])
+ manifest = web_app.fastmoss_evidence_manifest(message, "调研", {"playbook": "product"})
+ assert manifest["target_category_path"] == {"level1": 13, "level2": 844168, "level3": 935176}
+ assert manifest["market_category_levels"] == ["L2"]
+ assert any("不能直接作为目标 L3 类目规模" in item for item in manifest["limitations"])
+
+
+def test_fastmoss_failed_category_page_is_not_counted_as_coverage() -> None:
+ message = _fastmoss_search_message([{"page": 1}, {"page": 2}, {"page": 3}], [{
+ "tool_name": "fastmoss__product_search",
+ "result": {
+ "ok": False, "data_state": "error", "evidence_observed": False,
+ "evidence_metadata": {"scope": "category_head", "page": 1, "data_state": "error"},
+ },
+ }, *[{
+ "tool_name": "fastmoss__product_search",
+ "result": {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_metadata": {"scope": "category_head", "page": page, "data_state": "data"},
+ },
+ } for page in (2, 3)]])
+ manifest = web_app.fastmoss_evidence_manifest(message, "调研", {"playbook": "product"})
+ assert manifest["category_head"]["attempted_pages"] == [1, 2, 3]
+ assert manifest["category_head"]["completed_pages"] == [2, 3]
+ assert manifest["category_head"]["coverage_complete"] is False
+ assert any("页码 [1] 调用失败" in item for item in manifest["limitations"])
+
+
+def test_fastmoss_metadata_detects_unsorted_page_and_string_product_ids() -> None:
+ payload = {"total": 60, "list": [
+ {"product": {"product_id": "1732183167826498507", "title": "Low", "floor_price": 9, "ceiling_price": 12}, "sales_summary": {"last_28d_units_sold": 10, "last_28d_gmv": 100}},
+ {"product": {"product_id": "1732183167826498508", "title": "High", "floor_price": 10, "ceiling_price": 15}, "sales_summary": {"last_28d_units_sold": 20, "last_28d_gmv": 250}},
+ ]}
+ raw = {"data": {"content": [{"text": json.dumps(payload)}]}}
+ normalized = {"ok": True, "data_state": "data", "evidence_observed": True}
+ annotated = web_app.annotate_fastmoss_tool_result(
+ "fastmoss__product_search",
+ {"page": 1, "pagesize": 10, "orderby": [{"field": "day28_units_sold", "order": "desc"}]},
+ normalized,
+ raw,
+ )
+ assert annotated["evidence_metadata"]["reported_total"] == 60
+ assert annotated["evidence_metadata"]["sort_verified"] is False
+ assert annotated["evidence_product_records"][0]["day28_units_sold"] == 10
+ assert annotated["evidence_product_records"][0]["price_max"] == 12
+ assert web_app._collect_named_ids(json.dumps(payload), {"productid", "goodsid", "itemid"}) == {
+ "1732183167826498507", "1732183167826498508",
+ }
+
+ dated_payload = {"trend_series": [{"date": "2026-01-16", "period_units_sold": 10}]}
+ dated_raw = {"data": {"content": [{"text": json.dumps(dated_payload)}]}}
+ dated = web_app.annotate_fastmoss_tool_result(
+ "fastmoss__market_category_analysis",
+ {"filter": {"date_type": "week", "date_value": "2026-W28"}},
+ {"ok": True, "data_state": "data", "evidence_observed": True},
+ dated_raw,
+ )
+ dated_message = SimpleNamespace(tool_calls=[], tool_results=[{
+ "tool_name": "fastmoss__market_category_analysis", "result": dated,
+ }])
+ dated_manifest = web_app.fastmoss_evidence_manifest(dated_message, "调研", {"playbook": "product"})
+ assert any("两者不重叠" in item["issue"] for item in dated_manifest["conflicts"])
+
+
+def test_fastmoss_evidence_ledger_keeps_native_dimensions_and_late_trends() -> None:
+ def annotate(name: str, arguments: dict, payload: dict) -> dict:
+ return web_app.annotate_fastmoss_tool_result(
+ f"fastmoss__{name}",
+ arguments,
+ {"ok": True, "data_state": "data", "evidence_observed": True},
+ {"data": {"content": [{"text": json.dumps(payload)}]}},
+ )
+
+ category_trend = annotate("market_category_analysis", {
+ "filter": {"category_id": 844168, "region": "US", "date_type": "week", "date_value": "2026-W28"},
+ }, {
+ "analysis_type": "sales_trends",
+ "category": {"category_id": 844168, "category_name": "厨房家电", "region": "US"},
+ "summary_metrics": {"category_units_sold_total": 473150, "selling_video_count_total": 70612},
+ "trend_series": [{"period_label": "2026-06-01 ~ 2026-06-07", "category_units_sold": 71374}],
+ })
+ ranking = annotate("market_category_ranking", {"filter": {"category_id": 13, "region": "US"}}, {
+ "ranking_scope": {"parent_category_id": 13, "ranked_category_level": 2, "region": "US"},
+ "ranked_categories": [{
+ "category_id": 844168, "category_name": "厨房家电", "category_units_sold": 425507,
+ "channel_gmv_share": {"video_gmv_share_percent": 57.15, "live_gmv_share_percent": 11.87},
+ }],
+ })
+ new_products = annotate("product_rank_new_listed", {"filter": {"category_id": 935176, "region": "US"}}, {
+ "list": [{
+ "product_id": "1732461324214113003", "title": "Mini Chopper", "current_price": 13.57,
+ "commission_rate_percent": 14.5, "first_3d_units_sold": 18, "first_3d_gmv": 210.42,
+ }],
+ })
+ product_sample = annotate("product_search", {"keywords": "Mini Meat Grinder", "page": 1}, {
+ "list": [{
+ "distribution_summary": {"linked_creator_count": 406, "linked_video_count": 119},
+ "product": {"product_id": "1730898744848192092", "title": "Mini Meat Grinder", "floor_price": 36.05, "ceiling_price": 53},
+ "sales_summary": {"last_28d_units_sold": 424, "last_28d_gmv": 29727},
+ }],
+ })
+ overview = annotate("product_overview", {"filter": {"product_id": "1730898744848192092"}}, {
+ "ads_distribution": {"breakdown": [{"traffic_source": "ad_traffic", "gmv_share_percent": 59}]},
+ "channel_distribution": {"breakdown": [{"sales_channel": "affiliate", "units_sold_share_percent": 67}]},
+ "content_distribution": {"breakdown": [{"content_type": "video", "gmv_share_percent": 66}]},
+ "daily_trend": [{"date": "2026-06-18", "daily_units_sold": 4, "daily_gmv": 304}],
+ })
+ trend_17 = annotate("product_sales_trend", {"filter": {"product_id": "1730898744848192092"}}, {
+ "daily_trend": [
+ {"date": f"2026-04-{day:02d}", "daily_units_sold": day, "daily_gmv": day * 10}
+ for day in range(1, 31)
+ ] + [{"date": "2026-05-01", "daily_units_sold": 1, "daily_gmv": 10}],
+ })
+ trend_18 = annotate("product_sales_trend", {"filter": {"product_id": "1731973841209955212"}}, {
+ "daily_trend": [
+ {"date": f"2026-05-{day:02d}", "daily_units_sold": 1 if day <= 10 else 0, "daily_gmv": 10 if day <= 10 else 0}
+ for day in range(1, 31)
+ ],
+ })
+ reviews = annotate("product_review_list", {"filter": {"product_id": "1730898744848192092"}}, {
+ "reviews": [], "total_review_count": 0,
+ })
+
+ tool_results = [{"tool_name": name, "result": result} for name, result in (
+ ("fastmoss__market_category_analysis", category_trend),
+ ("fastmoss__market_category_ranking", ranking),
+ ("fastmoss__product_rank_new_listed", new_products),
+ ("fastmoss__product_search", product_sample),
+ ("fastmoss__product_overview", overview),
+ ("fastmoss__product_sales_trend", trend_17),
+ ("fastmoss__product_sales_trend", trend_18),
+ ("fastmoss__product_review_list", reviews),
+ )]
+ manifest = web_app.fastmoss_evidence_manifest(
+ SimpleNamespace(tool_calls=[], tool_results=tool_results),
+ "调研",
+ {"playbook": "product", "entity": "Mini Meat Grinder"},
+ )
+ facts = manifest["evidence_facts"]
+ assert manifest["evidence_fact_count"] == 8
+ assert "evidence_excerpts" not in manifest
+ assert {fact["dimension"] for fact in facts} == {
+ "category_trend", "category_channel_ranking", "new_products", "product_sample",
+ "product_overview", "product_90d_trend", "review_status",
+ }
+ assert next(fact for fact in facts if fact["dimension"] == "category_trend")["summary_metrics"]["category_units_sold_total"] == 473150
+ assert next(fact for fact in facts if fact["dimension"] == "category_channel_ranking")["categories"][0]["channel_gmv_share"]["video_gmv_share_percent"] == 57.15
+ assert next(fact for fact in facts if fact["dimension"] == "new_products")["products"][0]["first_3d_units_sold"] == 18
+ assert next(fact for fact in facts if fact["dimension"] == "product_sample")["products"][0]["linked_creator_count"] == 406
+ assert next(fact for fact in facts if fact["dimension"] == "product_overview")["ads_distribution"]["breakdown"][0]["gmv_share_percent"] == 59
+ late_trends = [fact for fact in facts if fact["dimension"] == "product_90d_trend"]
+ assert [fact["product_id"] for fact in late_trends] == ["1730898744848192092", "1731973841209955212"]
+ assert late_trends[0]["trend_summary"]["first_30d_units_sold"] == 465
+ assert late_trends[0]["trend_summary"]["last_30d_units_sold"] == 465
+ assert next(fact for fact in facts if fact["dimension"] == "review_status")["state"] == "empty"
+
+
+def test_fastmoss_answer_verifier_applies_local_edits_and_keeps_draft_on_failure() -> None:
+ message = SimpleNamespace(tool_calls=[], tool_results=[{
+ "tool_name": "fastmoss__product_overview",
+ "result": {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_metadata": {"source_tool": "fastmoss__product_overview", "scope": "supporting"},
+ "evidence_facts": [{
+ "source_tool": "fastmoss__product_overview", "dimension": "product_overview",
+ "product_id": "1730898744848192092",
+ "ads_distribution": {"breakdown": [{"traffic_source": "ad_traffic", "gmv_share_percent": 59}]},
+ }, {
+ "source_tool": "fastmoss__product_overview", "dimension": "product_sample",
+ "scope": "segment_head", "products": [{
+ "product_id": "1730898744848192092", "day28_units_sold": 424,
+ }],
+ }],
+ },
+ }])
+ route = {"playbook": "product", "task_depth": "workflow", "entity": "mini grinder"}
+ style = web_app.fastmoss_report_style_instruction(route)
+ assert "标题、章节名称和顺序完全由你" in style
+ assert "完整调研报告" in style and "可以自由提出执行建议" in style
+ assert web_app.fastmoss_report_style_instruction({"task_depth": "lookup"}) == ""
+
+ native_report = (
+ "## 核心判断\n类目渠道结构显示视频成交更重要。\n"
+ "## 研究口径\n这里说明上级类目与样本边界。\n"
+ "## 平台成交结构\n广告和联盟数据用于核对代表商品。\n"
+ "## 验证建议\n优先验证同形态商品。"
+ )
+ native_manifest = {
+ "evidence_facts": [{"dimension": "category_channel_ranking"}, {"dimension": "product_overview"}],
+ }
+ assert web_app.fastmoss_rewrite_preserves_report_detail("", native_report, native_manifest, route)
+
+ class FakeResponse:
+ finish_reason = "stop"
+
+ def __init__(self, decision=None):
+ self.decision = decision or {"approved": True, "risk_level": "low", "edits": []}
+
+ def raise_for_status(self):
+ return None
+
+ def json(self):
+ return {"choices": [{"message": {"content": json.dumps(self.decision)}, "finish_reason": self.finish_reason}]}
+
+ class FakeRequests:
+ @staticmethod
+ def post(*_args, **kwargs):
+ payload = json.loads(kwargs["data"].decode("utf-8"))
+ claims = json.loads(payload["messages"][1]["content"])["candidate_claims"]
+ edits = []
+ for claim in claims:
+ if "低竞争市场" in claim["text"]:
+ edits.append({
+ "claim_id": claim["claim_id"],
+ "replacement": "本轮样本显示该细分仍需补充竞争强度证据。",
+ "reason": "当前样本不能证明低竞争",
+ "evidence_refs": ["call:1"],
+ })
+ elif "首批500-1000件" in claim["text"]:
+ edits.append({
+ "claim_id": claim["claim_id"],
+ "replacement": "具体库存、预算、达人合作和经营目标需结合实际成本与测试数据确定。",
+ "reason": "执行数字没有工具证据或用户输入",
+ "evidence_refs": [],
+ })
+ return FakeResponse({
+ "approved": not edits, "risk_level": "high" if edits else "low", "edits": edits,
+ })
+
+ original_record = web_app.record_api_call
+ web_app.record_api_call = lambda *_args, **_kwargs: None
+ try:
+ draft = (
+ "## 核心判断\n这个细分已经被证明是低竞争市场。\n\n"
+ "## 代表商品\n| 商品 | 近28天销量 | 观测价格 |\n|---|---:|---:|\n"
+ "| Mini Grinder | 424 | $36.05–$53 |\n\n"
+ "## 渠道证据\n该商品广告GMV占比为59%，关联达人406名。\n\n"
+ "## 执行建议\n首批500-1000件，测试预算$2000，配合3-5个达人，观察2周，"
+ "目标ROI达到2.5，MOQ建议不超过500，建议定价$29.99-$39.99。"
+ "建议售价 **$13–$16**；以$14.99在TikTok Shop上架；邀请3–5个达人；"
+ "预计1–2个月做到月销1,000件；如果按$5–$8广告成本计算ROI。"
+ )
+ edited = web_app.verify_fastmoss_final_answer(
+ draft, message, "调研", route, FakeRequests, "key", "https://example.test/v1", "model"
+ )
+ assert "本轮样本显示该细分仍需补充竞争强度证据" in edited
+ assert "## 核心判断" in edited and "## 代表商品" in edited and "## 渠道证据" in edited
+ assert "| Mini Grinder | 424 | $36.05–$53 |" in edited
+ assert "广告GMV占比为59%" in edited and "关联达人406名" in edited
+ # Execution recommendations are writing judgments, not observed facts.
+ # A verifier edit without an exact evidence reference must not erase them.
+ for recommendation in (
+ "500-1000", "$2000", "3-5", "3–5", "2周", "2.5", "MOQ建议不超过500",
+ "$29.99", "$39.99", "$13–$16", "$14.99", "1–2个月", "月销1,000", "$5–$8",
+ ):
+ assert recommendation in edited, (recommendation, edited)
+
+ downgraded = web_app.polish_fastmoss_report_tone(
+ "该细分不存在独立市场，但它是真实细分机会。"
+ )
+ assert "尚未显示出足以确认独立市场的强信号" in downgraded
+ assert "是否构成可进入机会仍需验证" in downgraded
+
+ softened = web_app.polish_fastmoss_report_tone(
+ "首批500-1000件，配合3-5个达人，筛选10k-50k粉的创作者，观察2周，建议定价$29.99-$39.99。"
+ )
+ assert "500" not in softened and "3-5" not in softened
+ assert "10k" not in softened and "2周" not in softened
+ assert "$29.99" not in softened and "$39.99" not in softened
+ assert "小批量" in softened and "少量匹配达人" in softened
+ assert "受众匹配" in softened and "完整测试周期" in softened
+
+ class ApprovedResponse(FakeResponse):
+ def json(self):
+ return {"choices": [{"message": {"content": json.dumps({
+ "approved": True, "risk_level": "low", "edits": [],
+ })}, "finish_reason": "stop"}]}
+
+ class ApprovedRequests:
+ @staticmethod
+ def post(*_args, **_kwargs):
+ return ApprovedResponse()
+
+ approved = web_app.verify_fastmoss_final_answer(
+ native_report, message, "调研", route, ApprovedRequests, "key", "https://example.test/v1", "model"
+ )
+ assert approved == native_report
+
+ detailed_report = (
+ "# TikTok Shop 调研报告\n\n"
+ "**结论先行：** 当前证据支持继续比较。\n\n"
+ "## 4. 代表商品与趋势\n\n"
+ "### 商品A\n\n- **定价**：$80.23–$85.58\n\n"
+ "### 商品B\n\n- **定价**：$34.99\n\n"
+ "## 7. 风险与验证顺序\n\n保留样本边界。"
+ )
+ preserved = web_app.verify_fastmoss_final_answer(
+ detailed_report, message, "调研", route,
+ ApprovedRequests, "key", "https://example.test/v1", "model",
+ )
+ assert preserved == detailed_report
+
+ class InvalidJsonResponse(FakeResponse):
+ def json(self):
+ return {"choices": [{"message": {"content": "{invalid"}, "finish_reason": "stop"}]}
+
+ class InvalidJsonRequests:
+ @staticmethod
+ def post(*_args, **_kwargs):
+ return InvalidJsonResponse()
+
+ class LengthResponse(FakeResponse):
+ finish_reason = "length"
+
+ class LengthRequests:
+ @staticmethod
+ def post(*_args, **_kwargs):
+ return LengthResponse()
+
+ class MissingTargetResponse(FakeResponse):
+ def json(self):
+ return {"choices": [{"message": {"content": json.dumps({
+ "approved": False, "risk_level": "high", "edits": [{
+ "original": "草稿里不存在的句子", "replacement": "", "reason": "风险",
+ "evidence_refs": ["fastmoss__product_overview"],
+ }],
+ })}, "finish_reason": "stop"}]}
+
+ class MissingTargetRequests:
+ @staticmethod
+ def post(*_args, **_kwargs):
+ return MissingTargetResponse()
+
+ class TimeoutRequests:
+ @staticmethod
+ def post(*_args, **_kwargs):
+ raise TimeoutError("verifier timeout")
+
+ for failing_requests in (InvalidJsonRequests, LengthRequests, MissingTargetRequests, TimeoutRequests):
+ kept = web_app.verify_fastmoss_final_answer(
+ draft, message, "调研", route, failing_requests, "key", "https://example.test/v1", "model"
+ )
+ assert "## 核心判断" in kept and "## 代表商品" in kept and "## 渠道证据" in kept
+ assert "| Mini Grinder | 424 | $36.05–$53 |" in kept
+ assert "这个细分已经被证明是低竞争市场" in kept
+ assert "## 先说结论" not in kept
+ assert "500-1000" in kept and "$2000" in kept and "$29.99" in kept
+ assert "$13–$16" in kept and "$14.99" in kept and "3–5" in kept
+ assert "1–2个月" in kept and "$5–$8" in kept
+
+ system_only = SimpleNamespace(tool_calls=[], tool_results=[{
+ "tool_name": "system__current_time",
+ "result": {"ok": True, "data_state": "data", "evidence_observed": True, "summary": "2026-07-17"},
+ }])
+
+ class MustNotCallRequests:
+ @staticmethod
+ def post(*_args, **_kwargs):
+ raise AssertionError("verifier must not run without FastMoss evidence")
+
+ no_evidence = web_app.verify_fastmoss_final_answer(
+ native_report, system_only, "调研", route, MustNotCallRequests, "key", "https://example.test/v1", "model"
+ )
+ assert "## 先说结论" in no_evidence
+ system_manifest = web_app.fastmoss_evidence_manifest(system_only, "调研", route)
+ assert system_manifest["quality_states"]["data"] == []
+ assert system_manifest["evidence_fact_count"] == 0
+ finally:
+ web_app.record_api_call = original_record
+
+
+def test_fastmoss_all_workflow_tools_emit_supported_envelopes() -> None:
+ generic_payload = {
+ "summary_metrics": {"gmv": 100, "units_sold": 5},
+ "list": [{"entity_id": "sample", "gmv": 100}],
+ "total": 1,
+ }
+ for tool_name in sorted(web_app.FASTMOSS_SUPPORTED_EVIDENCE_TOOLS):
+ arguments = {
+ "filter": {
+ "category_id": 935176,
+ "product_id": "1730898744848192092",
+ "shop_id": "7496115528088652380",
+ "creator_uid": "7025465585010885637",
+ "video_id": "7661483798802009357",
+ }
+ }
+ payload = generic_payload
+ if tool_name == "search_category_by_words":
+ payload = {"categories": [{"category_id_level3": 935176, "cn_name": "料理机"}]}
+ elif tool_name == "product_creator_analysis":
+ payload = {
+ "creator_summary": {"follower_tier_distribution": [{"follower_tier": "10k-50k", "creator_count": 7}]},
+ "linked_creators": {"list": [], "total": 7},
+ }
+ elif tool_name == "product_video_list":
+ payload = {"product_id": "1730898744848192092", "time_range_days": 28, "videos": [], "total": 16}
+ normalized = {
+ "ok": True, "enough_data": True, "data_state": "data", "mcp_data": payload,
+ }
+ annotated = web_app.annotate_fastmoss_tool_result(
+ f"fastmoss__{tool_name}", arguments, normalized
+ )
+ envelope = annotated["evidence_envelope"]
+ assert envelope["parser_status"] == "supported", tool_name
+ assert envelope["tool_family"] in web_app.FASTMOSS_EVIDENCE_TOOL_FAMILIES, tool_name
+ assert annotated.get("evidence_facts"), tool_name
+
+ empty = web_app.annotate_fastmoss_tool_result(
+ "fastmoss__product_search", {"keywords": "empty"},
+ {"ok": True, "enough_data": False, "data_state": "empty", "mcp_data": {"list": [], "total": 0}},
+ )
+ assert empty["evidence_envelope"]["data_state"] == "empty"
+ assert "evidence_facts" not in empty
+
+ unknown = web_app.annotate_fastmoss_tool_result(
+ "fastmoss__future_metric", {},
+ {"ok": True, "enough_data": True, "data_state": "data", "mcp_data": {"value": 1}},
+ )
+ assert unknown["evidence_envelope"]["parser_status"] == "unsupported_parser"
+ assert "evidence_facts" not in unknown
+
+
+def test_fastmoss_verifier_batches_claims_and_isolates_failures() -> None:
+ message = SimpleNamespace(tool_calls=[], tool_results=[{
+ "tool_name": "fastmoss__market_category_ranking",
+ "result": {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_envelope": {
+ "source_tool": "fastmoss__market_category_ranking", "metric_grain": "market_category_ranking",
+ "tool_family": "category", "parser_status": "supported", "data_state": "data",
+ "entity_refs": [{"type": "category", "id": "844168"}],
+ },
+ "evidence_facts": [{
+ "source_tool": "fastmoss__market_category_ranking", "data_state": "data",
+ "dimension": "category_channel_ranking", "categories": [{
+ "category_id": 844168, "category_name": "厨房家电", "rank": 2,
+ "category_units_sold": 82995,
+ }],
+ }],
+ },
+ }])
+ draft = "# 判断\n" + "\n".join(
+ f"判断{index}：该市场已经进入成长期，是低竞争蓝海。" for index in range(1, 12)
+ )
+
+ class Response:
+ def __init__(self, body: dict):
+ self.body = body
+
+ def raise_for_status(self) -> None:
+ return None
+
+ def json(self) -> dict:
+ return self.body
+
+ class Requests:
+ def __init__(self) -> None:
+ self.payloads: list[dict] = []
+
+ def post(self, _url: str, **kwargs):
+ payload = json.loads(kwargs["data"].decode("utf-8"))
+ self.payloads.append(payload)
+ batch = json.loads(payload["messages"][1]["content"])["candidate_claims"]
+ if len(self.payloads) == 1:
+ decision = {
+ "approved": False, "risk_level": "high", "edits": [{
+ "claim_id": batch[0]["claim_id"], "replacement": "判断1：竞争强度仍需验证。",
+ "reason": "样本不能证明低竞争", "evidence_refs": ["fm-c1-f1"],
+ }],
+ }
+ return Response({"choices": [{"finish_reason": "stop", "message": {"content": json.dumps(decision)}}]})
+ if len(self.payloads) == 2:
+ return Response({"choices": [{"finish_reason": "length", "message": {"content": ""}}]})
+ return Response({"choices": [{"finish_reason": "stop", "message": {"content": "{invalid"}}]})
+
+ requests = Requests()
+ original_record = web_app.record_api_call
+ web_app.record_api_call = lambda *_args, **_kwargs: None
+ try:
+ result = web_app.verify_fastmoss_final_answer(
+ draft, message, "调研厨房家电", {"playbook": "product", "task_depth": "workflow"},
+ requests, "key", "https://example.test/v1", "model",
+ )
+ finally:
+ web_app.record_api_call = original_record
+ assert len(requests.payloads) == 3
+ assert all(len(json.loads(item["messages"][1]["content"])["candidate_claims"]) <= 5 for item in requests.payloads)
+ assert all(item["max_tokens"] == 4800 for item in requests.payloads)
+ assert all(len(json.dumps(item, ensure_ascii=False)) < 20000 for item in requests.payloads)
+ assert "判断1：竞争强度仍需验证" in result
+ for index in range(2, 12):
+ assert f"判断{index}：" in result
+
+ unchanged, count = web_app.sanitize_fastmoss_state_contradictions(
+ "直播数据为0，但这是观测值。",
+ {"evidence_envelopes": [{"data_state": "empty", "entity_refs": [{"type": "live", "id": "0"}]}]},
+ )
+ assert unchanged == "直播数据为0，但这是观测值。" and count == 0
+
+
+def test_fastmoss_creator_video_facts_and_historical_rebuild() -> None:
+ creator_payload = {
+ "creator_summary": {
+ "follower_tier_distribution": [
+ {"follower_tier": "1k-5k", "creator_count": 3},
+ {"follower_tier": "10k-50k", "creator_count": 7},
+ ],
+ },
+ "linked_creators": {
+ "total": 10,
+ "list": [{
+ "creator": {"creator_uid": "7025465585010885637", "creator_name": "A", "follower_count": 12000},
+ "product_contribution": {"product_gmv": 500, "product_units_sold": 10},
+ }],
+ },
+ }
+ video_payload = {
+ "product_id": "1730898744848192092", "time_range_days": 28, "total": 16,
+ "videos": [{
+ "video_id": "7661483798802009357",
+ "creator": {"creator_uid": "7025465585010885637"},
+ "engagement_metrics": {"play_count": 1124},
+ "product_contribution": {"window_gmv": 304, "window_units_sold": 4},
+ "traffic_flags": {"is_ad": True},
+ }],
+ }
+ calls = [
+ {"function": {"name": "fastmoss__product_creator_analysis", "arguments": json.dumps({"filter": {"product_id": "1730898744848192092"}})}},
+ {"function": {"name": "fastmoss__product_video_list", "arguments": json.dumps({"filter": {"product_id": "1730898744848192092", "time_range_days": 28}})}},
+ ]
+ results = [
+ {"tool_name": "fastmoss__product_creator_analysis", "result": {"ok": True, "enough_data": True, "data_state": "data", "mcp_data": creator_payload}},
+ {"tool_name": "fastmoss__product_video_list", "result": {"ok": True, "enough_data": True, "data_state": "data", "mcp_data": video_payload}},
+ ]
+ manifest = web_app.fastmoss_evidence_manifest(
+ SimpleNamespace(tool_calls=calls, tool_results=results), "调研", {"playbook": "product"}
+ )
+ assert manifest["evidence_envelope_count"] == 2
+ assert manifest["unsupported_parser_count"] == 0
+ by_dimension = {fact["dimension"]: fact for fact in manifest["evidence_facts"]}
+ assert by_dimension["product_creator_analysis"]["reported_creator_total"] == 10
+ assert "not_gmv_contribution" in by_dimension["product_creator_analysis"]["metric_semantics"]["follower_tier_distribution"]
+ assert by_dimension["product_videos"]["reported_video_total"] == 16
+ assert "not_category_content_preference" in by_dimension["product_videos"]["metric_semantics"]
+ product_bundle = next(
+ item for item in manifest["entity_bundles"]
+ if item["entity_type"] == "product" and item["entity_id"] == "1730898744848192092"
+ )
+ assert {"product_creator_analysis", "product_videos"}.issubset(set(product_bundle["dimensions"]))
+
+
+def test_fastmoss_report_packets_are_workflow_native_and_numeric_policy_is_strict() -> None:
+ manifest = {
+ "quality_states": {"data": ["fastmoss__product_search"], "empty": [], "error": []},
+ "evidence_envelope_count": 1,
+ "evidence_fact_count": 2,
+ "unsupported_parser_count": 0,
+ "category_head": {"target_pages": 3, "completed_pages": [1], "reported_total": 20, "fetched_unique": 10, "coverage_complete": False},
+ "segment_head": {"queries": {}, "fetched_unique": 0},
+ "entity_bundles": [],
+ "evidence_facts": [{
+ "dimension": "category_trend", "data_state": "data",
+ "source_tool": "fastmoss__market_category_analysis",
+ }, {
+ "fact_id": "fm-c2-f1", "dimension": "shop_data_trends", "data_state": "data",
+ "source_call_index": 2, "source_tool": "fastmoss__shop_data_trends",
+ "entity_type": "shop", "entity_id": "7495123456789012345",
+ "payload": {"summary": {"period_gmv": 1200}},
+ }],
+ "derived_facts": [], "conflicts": [], "limitations": [],
+ }
+ packets = {
+ playbook: web_app.fastmoss_report_packet(manifest, {"playbook": playbook})
+ for playbook in ("product", "pricing", "competitor", "shop", "creator", "content_dissect", "content_strategy")
+ }
+ assert all(
+ packet["available_dimensions"] == ["category_trend", "shop_data_trends"]
+ for packet in packets.values()
+ )
+ assert all(
+ any(item.get("dimension") == "shop_data_trends" for item in packet["supporting_evidence"])
+ for packet in packets.values()
+ )
+ assert all("suggested_modules" not in packet and "claim_boundaries" not in packet for packet in packets.values())
+ assert all(packet["numeric_policy"] == "only_tool_evidence_user_input_or_explicit_calculation" for packet in packets.values())
+ integrity = web_app.fastmoss_report_integrity_stats("店铺趋势和类目市场均已覆盖。", manifest)
+ assert integrity["used_dimensions"] == ["category_trend", "shop_data_trends"]
+
+
+def test_fastmoss_list_truncation_is_explicit_and_invalid_entities_are_filtered() -> None:
+ products = [{
+ "product": {"product_id": str(1000 + index), "title": f"Product {index}"},
+ "sales_summary": {"last_28d_units_sold": index},
+ } for index in range(1, 11)]
+ normalized = web_app.annotate_fastmoss_tool_result(
+ "fastmoss__product_search",
+ {
+ "filter": {"category_id_level2": 844168, "category_id_level3": 935176},
+ "page": 2, "pagesize": 10,
+ },
+ {"ok": True, "data_state": "data", "evidence_observed": True},
+ {"data": {"content": [{"text": json.dumps({"total": 2000, "list": products})}]}},
+ )
+ fact = normalized["evidence_facts"][0]
+ assert fact["returned_count"] == 10
+ assert fact["included_count"] == 10
+ assert fact["omitted_count"] == 0 and fact["truncated"] is False
+ assert fact["returned_day28_units_sold_sum"] == 55
+
+ envelope = dict(normalized["evidence_envelope"])
+ envelope["source_call_index"] = 2
+ envelope["scope"] = "category_head"
+ manifest = {
+ "quality_states": {"data": ["fastmoss__product_search"], "empty": [], "error": []},
+ "evidence_envelope_count": 1, "evidence_fact_count": 1, "unsupported_parser_count": 0,
+ "category_head": {"target_pages": 3, "completed_pages": [2], "reported_total": 2000, "fetched_unique": 10},
+ "segment_head": {"queries": {}, "fetched_unique": 0},
+ "entity_bundles": [], "analysis_targets": [], "evidence_envelopes": [envelope],
+ "evidence_facts": [{**fact, "scope": "category_head", "page": 2}],
+ "derived_facts": [], "conflicts": [], "limitations": [],
+ }
+ packet = web_app.fastmoss_report_packet(manifest, {"playbook": "product"})
+ compact = packet["supporting_evidence"][0]
+ assert compact["returned_count"] == 10
+ assert compact["included_count"] == 2
+ assert compact["omitted_count"] == 8 and compact["truncated"] is True
+ assert [item["product_id"] for item in compact["products"]] == ["1001", "1010"]
+ assert packet["source_catalog"][0]["arguments"]["page"] == 2
+ assert packet["source_catalog"][0]["arguments"]["filter"] == {
+ "category_id_level2": 844168, "category_id_level3": 935176,
+ }
+
+ segment_manifest = dict(manifest)
+ segment_manifest["evidence_facts"] = [{
+ **fact, "scope": "segment_head", "page": 1, "query": "Mini Meat Grinder",
+ }]
+ segment_packet = web_app.fastmoss_report_packet(segment_manifest, {"playbook": "product"})
+ segment_compact = segment_packet["supporting_evidence"][0]
+ assert segment_compact["included_count"] == 10
+ assert segment_compact["omitted_count"] == 0 and segment_compact["truncated"] is False
+
+ refs = web_app._fastmoss_entity_refs(
+ {"filter": {"category_id_level2": 844168, "category_id_level3": 0, "product_id": "0"}},
+ {"category_id": "0", "product_id": "1730898744848192092"},
+ )
+ assert {tuple(sorted(item.items())) for item in refs} == {
+ tuple(sorted({"type": "category", "id": "844168"}.items())),
+ tuple(sorted({"type": "product", "id": "1730898744848192092"}.items())),
+ }
+
+
+def test_fastmoss_22_call_semantic_registry_fixture() -> None:
+ """Compact regression fixture derived from the 22-call product workflow."""
+ calls: list[dict] = []
+ results: list[dict] = []
+
+ def add(name: str, arguments: dict, result: dict) -> None:
+ calls.append({"function": {"name": f"fastmoss__{name}", "arguments": json.dumps(arguments)}})
+ results.append({"tool_name": f"fastmoss__{name}", "result": result})
+
+ def envelope(name: str, state: str, arguments: dict, returned: int | None = None, total: int | None = None) -> dict:
+ item = {
+ "source_tool": f"fastmoss__{name}", "metric_grain": name,
+ "tool_family": web_app._fastmoss_tool_family(name), "parser_status": "supported",
+ "data_state": state, "arguments": arguments,
+ "entity_refs": web_app._fastmoss_entity_refs(arguments, None),
+ }
+ if returned is not None:
+ item["returned_count"] = returned
+ if total is not None:
+ item["reported_total"] = total
+ return item
+
+ def product(product_id: str, units: int, title: str = "Sample") -> dict:
+ return {
+ "product_id": product_id, "title": title,
+ "day28_units_sold": units, "day28_gmv": units * 20,
+ "price_min": 18, "price_max": 22,
+ }
+
+ category_ids = [str(1730000000000000000 + index) for index in range(30)]
+ segment_ids = category_ids[:2] + [str(1740000000000000000 + index) for index in range(18)]
+ top_ids = category_ids[:1] + [str(1750000000000000000 + index) for index in range(9)]
+ new_ids = [str(1760000000000000000 + index) for index in range(10)]
+
+ arguments = {"keywords": "料理机"}
+ add("search_category_by_words", arguments, {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_envelope": envelope("search_category_by_words", "data", arguments, 2),
+ "evidence_facts": [{
+ "source_tool": "fastmoss__search_category_by_words", "data_state": "data",
+ "dimension": "category_candidates", "categories": [
+ {"category_id_level3": 935176, "cn_name": "料理机", "score": 0.5023},
+ {"category_id_level3": 934920, "cn_name": "垃圾处理器", "score": 0.4891},
+ ],
+ }],
+ })
+ for analysis_type in ("basic_metrics", "sales_trends", "price_distribution"):
+ arguments = {"filter": {"category_id": 844168}, "analysis_type": analysis_type}
+ if analysis_type != "sales_trends":
+ add("market_category_analysis", arguments, {
+ "ok": True, "data_state": "empty", "evidence_observed": False,
+ "evidence_metadata": {"scope": "supporting", "category_level": "L2", "category_id": 844168},
+ "evidence_envelope": {
+ **envelope("market_category_analysis", "empty", arguments, 0),
+ "entity_refs": [{"type": "category", "id": "844168"}],
+ },
+ })
+ else:
+ add("market_category_analysis", arguments, {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_envelope": envelope("market_category_analysis", "data", arguments, 16),
+ "evidence_facts": [{
+ "source_tool": "fastmoss__market_category_analysis", "data_state": "data",
+ "dimension": "category_trend", "category_id": 844168, "entity_type": "category", "entity_id": "844168",
+ "summary_metrics": {"category_units_sold_total": 437785, "selling_video_count_total": 63822},
+ }],
+ })
+ arguments = {"filter": {"category_id": 13}, "page": 1}
+ add("market_category_ranking", arguments, {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_envelope": envelope("market_category_ranking", "data", arguments, 4),
+ "evidence_facts": [{
+ "source_tool": "fastmoss__market_category_ranking", "data_state": "data",
+ "dimension": "category_channel_ranking", "categories": [{
+ "category_id": 844168, "category_name": "厨房家电", "rank": 2,
+ "category_units_sold": 82995, "category_units_sold_yoy_percent": -15.57,
+ "category_gmv_yoy_percent": -25.66,
+ "channel_gmv_share": {"video_gmv_share_percent": 51.46},
+ }],
+ }],
+ })
+
+ for name, ids in (("product_rank_top_selling", top_ids), ("product_rank_new_listed", new_ids)):
+ arguments = {"filter": {"category_id": 935176}, "page": 1, "pagesize": 10}
+ dimension = "top_products" if name.endswith("top_selling") else "new_products"
+ products = [product(product_id, 100 - index) for index, product_id in enumerate(ids)]
+ add(name, arguments, {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_envelope": envelope(name, "data", arguments, 10),
+ "evidence_facts": [{"source_tool": f"fastmoss__{name}", "data_state": "data", "dimension": dimension, "products": products, "returned_count": 10}],
+ "evidence_product_records": products,
+ })
+
+ search_groups = [category_ids[:10], category_ids[10:20], category_ids[20:30], segment_ids[:10], segment_ids[10:20]]
+ search_queries = ["", "", "", "Electric Food Shredder", "Mini Meat Grinder"]
+ for page_index, (ids, query) in enumerate(zip(search_groups, search_queries), start=1):
+ page = page_index if not query else 1
+ arguments = {"filter": {"category_path": [13, 844168, 935176]}, "page": page, "pagesize": 10}
+ if query:
+ arguments["keywords"] = query
+ products = [product(product_id, 30 - index, query or "Category product") for index, product_id in enumerate(ids)]
+ scope = "segment_head" if query else "category_head"
+ add("product_search", arguments, {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_metadata": {"scope": scope, "page": page, "query": query or None, "reported_total": 2000 if not query else 20, "fetched_records": 10, "data_state": "data"},
+ "evidence_envelope": {**envelope("product_search", "data", arguments, 10, 2000 if not query else 20), "scope": scope},
+ "evidence_facts": [{"source_tool": "fastmoss__product_search", "data_state": "data", "dimension": "product_sample", "scope": scope, "page": page, "query": query or None, "products": products, "returned_count": 10}],
+ "evidence_product_records": products,
+ })
+
+ representatives = [(segment_ids[2], "Mini Meat Grinder"), (segment_ids[3], "Electric Food Shredder")]
+ for index, (product_id, title) in enumerate(representatives):
+ arguments = {"filter": {"product_id": product_id}}
+ card_gmv, card_units = ((34, 33) if index == 0 else (91, 92))
+ add("product_overview", arguments, {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_envelope": envelope("product_overview", "data", arguments, 28),
+ "evidence_facts": [{
+ "source_tool": "fastmoss__product_overview", "data_state": "data", "dimension": "product_overview",
+ "product_id": product_id, "entity_type": "product", "entity_id": product_id,
+ "channel_distribution": {"breakdown": [{
+ "sales_channel": "product_card", "gmv_share_percent": card_gmv,
+ "units_sold_share_percent": card_units,
+ }]},
+ }],
+ })
+ for product_id, _title in representatives:
+ arguments = {"filter": {"product_id": product_id, "time_range_days": 90}}
+ add("product_sales_trend", arguments, {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_envelope": envelope("product_sales_trend", "data", arguments, 90),
+ "evidence_facts": [{
+ "source_tool": "fastmoss__product_sales_trend", "data_state": "data", "dimension": "product_90d_trend",
+ "product_id": product_id, "entity_type": "product", "entity_id": product_id,
+ "trend_summary": {"days_returned": 90, "total_units_sold": 1716 if product_id == representatives[0][0] else 31},
+ }],
+ })
+ for product_id, _title in representatives:
+ arguments = {"filter": {"product_id": product_id}}
+ add("product_review_list", arguments, {
+ "ok": True, "data_state": "empty", "evidence_observed": False,
+ "evidence_envelope": envelope("product_review_list", "empty", arguments, 0),
+ })
+ for index, (product_id, _title) in enumerate(representatives):
+ arguments = {"filter": {"product_id": product_id}, "page": 1}
+ creators = [{
+ "creator_uid": str(1770000000000000000 + index * 10), "creator_name": "Apex",
+ "creator_category": {"name": "Shopping & Retail"},
+ "product_contribution": {"product_linked_video_count": 670},
+ }, {
+ "creator_uid": str(1770000000000000001 + index * 10), "creator_name": "T ao",
+ "creator_category": {"name": "Other"},
+ "product_contribution": {"product_linked_video_count": 14},
+ }]
+ add("product_creator_analysis", arguments, {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_envelope": envelope("product_creator_analysis", "data", arguments, 2, 3),
+ "evidence_facts": [{
+ "source_tool": "fastmoss__product_creator_analysis", "data_state": "data", "dimension": "product_creator_analysis",
+ "product_id": product_id, "entity_type": "product", "entity_id": product_id,
+ "reported_creator_total": 3, "returned_creators": 2,
+ "creator_summary": {"creator_category_distribution": [{"creator_category": "Shopping & Retail", "creator_share_percent": 100}]},
+ "top_creators": creators,
+ }],
+ })
+ for product_id, _title in representatives:
+ arguments = {"filter": {"product_id": product_id, "time_range_days": 90}, "page": 1}
+ add("product_video_list", arguments, {
+ "ok": True, "data_state": "data", "evidence_observed": True,
+ "evidence_envelope": envelope("product_video_list", "data", arguments, 10, 146),
+ "evidence_facts": [{
+ "source_tool": "fastmoss__product_video_list", "data_state": "data", "dimension": "product_videos",
+ "product_id": product_id, "entity_type": "product", "entity_id": product_id,
+ "reported_video_total": 146, "returned_videos": 10, "time_range_days": 90,
+ "top_videos": [],
+ }],
+ })
+
+ assert len(calls) == len(results) == 22
+ manifest = web_app.fastmoss_evidence_manifest(
+ SimpleNamespace(tool_calls=calls, tool_results=results),
+ "Electric Food Shredder 和 Mini Meat Grinder 调研",
+ {"playbook": "product", "entity": "Electric Food Shredder / Mini Meat Grinder"},
+ )
+ coverage = manifest["coverage_summary"]
+ registry = manifest["metric_registry"]
+ assert manifest["evidence_envelope_count"] == 22
+ assert manifest["evidence_fact_count"] == 18
+ assert manifest["unsupported_parser_count"] == 0
+ assert coverage["category_search"]["returned_rows"] == 30
+ assert coverage["category_search"]["unique_products"] == 30
+ assert coverage["all_product_search_calls"] == {"returned_rows": 50, "unique_products": 48}
+ assert coverage["all_product_list_calls"] == {"returned_rows": 70, "unique_products": 67}
+ assert all(item["entity_refs"] == [{"type": "category", "id": "844168"}] for item in coverage["exact_empty_results"][:2])
+
+ def metric(metric_name: str, entity_id: str) -> float:
+ return next(item["value"] for item in registry if item["metric"] == metric_name and item["entity_id"] == entity_id)
+
+ assert metric("category_units_sold_yoy_percent", "844168") == -15.57
+ assert metric("category_gmv_yoy_percent", "844168") == -25.66
+ assert metric("channel_distribution.product_card.gmv_share_percent", representatives[1][0]) == 91
+ assert metric("channel_distribution.product_card.units_sold_share_percent", representatives[1][0]) == 92
+ assert metric("score", "935176") == 0.5023
+ assert metric("score", "934920") == 0.4891
+ assert metric("product_contribution.product_linked_video_count", "1770000000000000000") == 670
+ assert metric("product_contribution.product_linked_video_count", "1770000000000000001") == 14
+ assert any(item.get("conflict_type") == "summary_vs_returned_top_rows" for item in manifest["conflicts"])
+ assert not any("11.7" in json.dumps(item, ensure_ascii=False) for item in manifest["derived_facts"])
+
+ packet = web_app.fastmoss_report_packet(manifest, {"playbook": "product"})
+ packet_facts = [
+ item for item in packet["supporting_evidence"] if isinstance(item, dict)
+ ] + [
+ item for target in packet["target_evidence"] if isinstance(target, dict)
+ for item in (target.get("facts") or []) if isinstance(item, dict)
+ ]
+ packet_dimensions = {item.get("dimension") for item in packet_facts}
+ assert {
+ "category_trend", "category_channel_ranking", "product_overview", "product_90d_trend",
+ }.issubset(packet_dimensions)
+ segment_included_by_fact: dict[str, int] = {}
+ for item in packet_facts:
+ if item.get("dimension") != "product_sample" or item.get("scope") != "segment_head":
+ continue
+ fact_id = str(item.get("fact_id") or "")
+ segment_included_by_fact[fact_id] = (
+ segment_included_by_fact.get(fact_id, 0) + int(item.get("included_count") or 0)
+ )
+ assert segment_included_by_fact and all(
+ included == 10 for included in segment_included_by_fact.values()
+ ), segment_included_by_fact
+ target_ids = {
+ target["entity_id"] for target in packet["target_evidence"] if isinstance(target, dict)
+ }
+ assert target_ids
+ for target in packet["target_evidence"]:
+ target_id = target["entity_id"]
+ assert target["facts"], target
+ for fact in target["facts"]:
+ listed_ids = {
+ str(product.get("product_id") or "")
+ for product in (fact.get("products") or []) if isinstance(product, dict)
+ }
+ assert not listed_ids or listed_ids == {target_id}, (target_id, fact)
+ if fact.get("product_id"):
+ assert str(fact["product_id"]) == target_id, (target_id, fact)
+ assert target_id in str(fact.get("entity_fact_ref") or "")
+ assert all(
+ not {
+ str(product.get("product_id") or "")
+ for product in (fact.get("products") or []) if isinstance(product, dict)
+ }.intersection(target_ids)
+ for fact in packet["supporting_evidence"] if isinstance(fact, dict)
+ )
+
+ # A category claim and a product claim may share one verifier HTTP batch,
+ # but their evidence must remain isolated by claim_id. The old batch-wide
+ # entity union removed call 5 from the category claim and caused valid L2
+ # metrics to be rewritten as "not returned".
+ category_claim = {
+ "claim_id": "line-category", "entity_ids": [],
+ "reasons": ["metric_period", "cross_period_ratio"],
+ "text": "上级类目厨房家电GMV同比下降25.66%，销量同比下降15.57%，视频GMV占比51.46%。",
+ }
+ product_claim = {
+ "claim_id": "line-product", "entity_ids": [representatives[0][0]],
+ "reasons": ["lifecycle"],
+ "text": f"{representatives[0][0]} 已进入生命周期衰退期。",
+ }
+ isolated = web_app.fastmoss_candidate_evidence(
+ manifest, [category_claim, product_claim], {"playbook": "product"}
+ )
+ slices = {item["claim_id"]: item for item in isolated["claim_evidence"]}
+ category_slice = slices["line-category"]
+ product_slice = slices["line-product"]
+ assert any(
+ item.get("dimension") == "category_channel_ranking"
+ and int(item.get("source_call_index") or 0) == 5
+ for item in category_slice["facts"]
+ )
+ assert not any(item.get("dimension") == "product_90d_trend" for item in category_slice["facts"])
+ assert any(
+ item.get("dimension") == "product_90d_trend"
+ and item.get("product_id") == representatives[0][0]
+ for item in product_slice["facts"]
+ )
+ original_category_text = category_claim["text"]
+ guarded, applied = web_app.apply_fastmoss_verifier_edits(
+ original_category_text,
+ [{
+ "claim_id": "line-category",
+ "replacement": "厨房家电相关指标在证据中未返回。",
+ "reason": "误判为缺少证据",
+ "evidence_refs": ["call:5"],
+ }],
+ [category_claim, product_claim],
+ isolated,
+ )
+ assert guarded == original_category_text and applied == 0
+
+ lifecycle_claims = web_app.fastmoss_high_risk_claims(
+ f"{representatives[1][0]} 已进入生命周期衰退期。"
+ )
+ lifecycle_evidence = web_app.fastmoss_candidate_evidence(
+ manifest, lifecycle_claims, {"playbook": "product"}
+ )
+ assert any(
+ item.get("dimension") == "product_90d_trend"
+ and item.get("product_id") == representatives[1][0]
+ for item in lifecycle_evidence["facts"]
+ )
+ assert any(
+ str(item.get("metric") or "").startswith("trend_summary.")
+ and item.get("entity_id") == representatives[1][0]
+ for item in lifecycle_evidence["metric_registry"]
+ )
+ content_claims = web_app.fastmoss_high_risk_claims(
+ f"{representatives[0][0]} 的达人视频是销量的核心驱动力，"
+ "返回样本播放248次、26次点赞。"
+ )
+ assert content_claims and "content_causality" in content_claims[0]["reasons"]
+ content_evidence = web_app.fastmoss_candidate_evidence(
+ manifest, content_claims, {"playbook": "product"}
+ )
+ assert any(item.get("dimension") == "product_videos" for item in content_evidence["facts"])
+ assert len(json.dumps(content_evidence, ensure_ascii=False)) < 40000
+
+ draft = (
+ "厨房家电销量同比 -15.57%，GMV同比 -15.57%。\n"
+ f"{representatives[1][0]} 商品卡GMV占比94%，商品卡销量占比94%。\n"
+ "Apex有670条广告视频，T ao有14条广告视频。\n"
+ "1716件 ÷ 146条视频 = 平均11.7件。"
+ )
+ corrected, edits, _bound, _unbound = web_app.validate_fastmoss_numeric_claims(draft, manifest)
+ assert edits == 6, (edits, corrected)
+ assert "GMV同比 -25.66%" in corrected
+ assert "商品卡GMV占比91%" in corrected and "商品卡销量占比92%" in corrected
+ assert "670条关联视频" in corrected and "14条关联视频" in corrected
+ assert "11.7" not in corrected and "不能直接相除" in corrected
+
+
+def test_fastmoss_dossier_synthesis_preserves_complete_tool_evidence() -> None:
+ class Response:
+ def raise_for_status(self) -> None:
+ return None
+
+ def json(self) -> dict:
+ return {
+ "choices": [{
+ "finish_reason": "stop",
+ "message": {"content": "# 结论\n\n证据包已独立合成。"},
+ }],
+ }
+
+ class Requests:
+ def __init__(self) -> None:
+ self.payloads: list[dict] = []
+
+ def post(self, _url: str, **kwargs):
+ self.payloads.append(json.loads(kwargs["data"].decode("utf-8")))
+ return Response()
+
+ requests = Requests()
+ trend_rows = [
+ {"date": f"2026-06-{day:02d}", "units_sold": day}
+ for day in range(1, 31)
+ ] + [{"date": "2026-07-01", "units_sold": 31}]
+ products = [{
+ "product_id": str(1730000000000000000 + index),
+ "title": f"Product {index}",
+ "category": {"l3": {"id": 935176 if index < 9 else 934920, "name": "Food Processors" if index < 9 else "Other"}},
+ "current_price": 10 + index,
+ } for index in range(10)]
+ calls = [
+ {"id": "c1", "function": {"name": "fastmoss__product_sales_trend", "arguments": json.dumps({"filter": {"product_id": "1730000000000000001", "time_range_days": 90}})}},
+ {"id": "c2", "function": {"name": "fastmoss__product_rank_new_listed", "arguments": json.dumps({"filter": {"category_l1_id": 13, "category_l2_id": 844168, "category_l3_id": 935176}, "page": 1, "pagesize": 10})}},
+ {"id": "c3", "function": {"name": "fastmoss__product_review_list", "arguments": json.dumps({"filter": {"product_id": "1730000000000000001"}, "page": 1})}},
+ ]
+ results = [
+ {"tool_name": "fastmoss__product_sales_trend", "result": {"ok": True, "data_state": "data", "mcp_data": {"trend_series": trend_rows}}},
+ {"tool_name": "fastmoss__product_rank_new_listed", "result": {"ok": True, "data_state": "data", "mcp_data": {"list": products, "total": 10}}},
+ {"tool_name": "fastmoss__product_review_list", "result": {"ok": True, "data_state": "empty", "mcp_data": {"list": [], "total": 0}}},
+ ]
+ original_verify = web_app.verify_fastmoss_final_answer
+ web_app.verify_fastmoss_final_answer = lambda draft, *_args, **_kwargs: draft
+ try:
+ result = web_app.synthesize_fastmoss_report_from_packet(
+ SimpleNamespace(tool_calls=calls, tool_results=results),
+ "做一份完整产品调研",
+ {"playbook": "product", "task_depth": "workflow"},
+ requests,
+ "test-key",
+ "https://example.invalid/v1",
+ "test-model",
+ )
+ finally:
+ web_app.verify_fastmoss_final_answer = original_verify
+ assert result.startswith("# 结论")
+ assert len(requests.payloads) == 1
+ payload = requests.payloads[0]
+ assert "tools" not in payload
+ assert len(payload["messages"]) == 2
+ system_content = payload["messages"][0]["content"]
+ assert "evidence_dossier" in system_content
+ assert "report_packet" not in system_content
+ assert system_content.count("fastmoss__product_sales_trend") == 1
+ assert "2026-06-01" in system_content and "2026-07-01" in system_content
+ assert all(product["product_id"] in system_content for product in products)
+ assert "returned_product_outside_requested_l3" in system_content
+ assert "关键词返回量不是市场容量" in system_content
+ assert "不得写成流量来源、因果、效率或生命周期结论" in system_content
+ assert '"data_state":"empty"' in system_content
+ assert '"product_id":"1730000000000000001"' in system_content
+ assert '"source_ref":"call:3"' in system_content
+ assert '"report_date":' in system_content
+ assert '"hard_fact_boundaries":' in system_content
+ assert system_content.rfind("hard_fact_boundaries") > system_content.rfind("tool_evidence")
+ assert system_content.endswith("不得把样本占比写成市场份额，也不得从渠道占比推导自然流量、广告花费、ROI、因果或生命周期。")
+ assert "omitted_items" not in system_content
+
+
+def test_fastmoss_claim_ids_and_extended_mechanical_cleanup() -> None:
+ draft = (
+ "## 建议\n"
+ "建议月度广告预算 $3K–$5K，首批联系20位达人，每周发布5–7条视频，"
+ "CPO控制在$15–$18，测试周期2周，建议功率300–500W，售价定在$39.99。\n"
+ "计划与 1–2 位 5k–50k 粉丝的西语美食达人合作，测试 $12–$15 价位，头部商品毛利率潜力较高。\n"
+ "其余1970个商品合计不足25%，所以广告ROI稳定。"
+ )
+ cleaned, count = web_app.sanitize_fastmoss_unsupported_recommendations(draft)
+ cleaned = web_app.downgrade_fastmoss_absolute_market_claims(cleaned)
+ assert count >= 6
+ for unsupported in (
+ "$3K", "$5K", "20位", "5–7", "$15", "$18", "2周", "300–500W", "$39.99",
+ "1–2 位", "5k–50k", "毛利率潜力", "1970", "25%", "ROI稳定",
+ ):
+ assert unsupported not in cleaned
+
+ downgraded = web_app.downgrade_fastmoss_absolute_market_claims(
+ "Electric Food Shredder 市场极窄，内容效率为零，广告回报极低。"
+ )
+ assert "市场极窄" not in downgraded
+ assert "效率为零" not in downgraded
+ assert "回报极低" not in downgraded
+
+ corrected, corrected_count = web_app.sanitize_fastmoss_state_contradictions(
+ "Electric Food Shredder 细分几乎无活跃商品，说明头部集中度高。",
+ {
+ "evidence_envelopes": [],
+ "derived_signals": {
+ "category_top3_share": 0.43,
+ "segment_queries": [{
+ "query": "Electric Food Shredder",
+ "fetched_unique": 10,
+ "products_with_units": 6,
+ }],
+ },
+ },
+ )
+ assert corrected_count == 2
+ assert "10 款样本" in corrected and "6 款有可核对销量" in corrected
+ assert "并非高度集中" in corrected
+
+ claims = web_app.fastmoss_high_risk_claims("结论\n这个市场已经是低竞争蓝海。")
+ assert claims and claims[0]["claim_id"] == "line-2"
+ edited, applied = web_app.apply_fastmoss_verifier_edits(
+ "结论\n这个市场已经是低竞争蓝海。",
+ [{"claim_id": "line-2", "replacement": "竞争强度仍需验证。", "reason": "样本不足", "evidence_refs": []}],
+ claims,
+ )
+ assert applied == 0 and "低竞争蓝海" in edited
+ referenced_edit, referenced_count = web_app.apply_fastmoss_verifier_edits(
+ "结论\n这个市场已经是低竞争蓝海。",
+ [{"claim_id": "line-2", "replacement": "竞争强度仍需验证。", "reason": "样本不足", "evidence_refs": ["call:1"]}],
+ claims,
+ {"source_catalog": [{"source_ref": "call:1", "source_call_index": 1}]},
+ )
+ assert referenced_count == 1 and "竞争强度仍需验证" in referenced_edit
+ malformed_ref_edit, malformed_ref_count = web_app.apply_fastmoss_verifier_edits(
+ "结论\n这个市场已经是低竞争蓝海。",
+ [{"claim_id": "line-2", "replacement": "竞争强度仍需验证。", "reason": "样本不足", "evidence_refs": [{"source_ref": "call:1"}]}],
+ claims,
+ {"source_catalog": [{"source_ref": "call:1", "source_call_index": 1}]},
+ )
+ assert malformed_ref_count == 0 and "低竞争蓝海" in malformed_ref_edit
+
+ entity_claims = [{
+ "claim_id": "line-1",
+ "text": "代表商品ID 1732363961421369948。",
+ "reasons": ["multiple_entity_ids"],
+ "entity_ids": ["1732363961421369948"],
+ }]
+ unchanged_entity, entity_edits = web_app.apply_fastmoss_verifier_edits(
+ "代表商品ID 1732363961421369948。",
+ [{
+ "claim_id": "line-1",
+ "replacement": "代表商品ID 1730898744848192092。",
+ "reason": "改用销量更高商品",
+ "evidence_refs": ["fm-c12-f1"],
+ }],
+ entity_claims,
+ )
+ assert entity_edits == 0 and "1732363961421369948" in unchanged_entity
+
+ restored_id, restored_count = web_app.normalize_fastmoss_entity_id_abbreviations(
+ "代表商品ID 1732363961。",
+ {
+ "entity_bundles": [{"entity_type": "product", "entity_id": "1732363961421369948"}],
+ "analysis_targets": [],
+ },
+ )
+ assert restored_id == "代表商品ID 1732363961421369948。" and restored_count == 1
+
+ table_claims = [{
+ "claim_id": "line-1", "text": "| 达人链接数 | 382 | 766 | 853 |",
+ "reasons": ["operational_number"], "entity_ids": [],
+ }]
+ removed_table, table_edits = web_app.apply_fastmoss_verifier_edits(
+ table_claims[0]["text"],
+ [{"claim_id": "line-1", "replacement": "", "reason": "证据不足", "evidence_refs": []}],
+ table_claims,
+ )
+ assert table_edits == 0 and removed_table == table_claims[0]["text"]
+ kept_numbers, number_edits = web_app.apply_fastmoss_verifier_edits(
+ "本轮返回10件商品。",
+ [{"original": "本轮返回10件商品。", "replacement": "本轮返回20件商品。", "reason": "修正", "evidence_refs": []}],
+ )
+ assert number_edits == 0 and "10件" in kept_numbers
+ corrected_numbers, corrected_number_edits = web_app.apply_fastmoss_verifier_edits(
+ "本轮返回10件商品。",
+ [{"original": "本轮返回10件商品。", "replacement": "本轮返回20件商品。", "reason": "修正", "evidence_refs": ["f1"]}],
+ evidence_slice={"facts": [{"fact_id": "f1", "returned_count": 20}]},
+ )
+ assert corrected_number_edits == 1 and "20件" in corrected_numbers
+ unbound_numbers, unbound_number_edits = web_app.apply_fastmoss_verifier_edits(
+ "本轮返回10件商品。",
+ [{"original": "本轮返回10件商品。", "replacement": "本轮返回20件商品。", "reason": "修正", "evidence_refs": []}],
+ evidence_slice={"facts": [{"fact_id": "f1", "returned_count": 20}]},
+ )
+ assert unbound_number_edits == 0 and "10件" in unbound_numbers
+
+ cleaned_markdown = web_app.cleanup_fastmoss_markdown_structure(
+ "| 指标 | A |\n|---|---|\n\n| 销量 | 10 |\n\n## 空章节\n\n## 下一节\n\n有内容。"
+ )
+ assert "|---|---|\n| 销量" in cleaned_markdown
+ assert "空章节" not in cleaned_markdown and "## 下一节" in cleaned_markdown
+
+ observed_price = "当前观测定价 $24.30，GMV推导单价为$21.21。"
+ kept_price, price_edits = web_app.sanitize_fastmoss_unsupported_recommendations(observed_price)
+ assert kept_price == observed_price and price_edits == 0
+
+ future_threshold = (
+ "如果其达人链接数在接下来2周内从10人翻倍至20人以上且出现首单，则测试$10-$15的 USB-C 产品。"
+ )
+ cleaned_threshold, threshold_edits = web_app.sanitize_fastmoss_unsupported_recommendations(future_threshold)
+ assert threshold_edits == 2
+ assert "2周" not in cleaned_threshold and "10人" not in cleaned_threshold and "20人" not in cleaned_threshold
+ assert "$10" not in cleaned_threshold and "$15" not in cleaned_threshold
+
+ semantic_claims = web_app.fastmoss_high_risk_claims(
+ "关键词返回量证明该细分市场容量很小，商业化程度低，不具备规模投入条件。\n"
+ "该商品依靠广告投放实现爆发，说明产品生命周期已自然衰退。"
+ )
+ semantic_reasons = {reason for claim in semantic_claims for reason in claim["reasons"]}
+ assert {"market_scale", "channel_causality", "lifecycle"}.issubset(semantic_reasons)
+ fenced_claims = web_app.fastmoss_high_risk_claims(
+ "TikTok Shop尚无直接对应品类，也没有找到精确匹配的活跃商品。\n"
+ "返回的987条视频几乎全是广告内容，且已有近百万美金广告投入。"
+ )
+ fenced_reasons = {reason for claim in fenced_claims for reason in claim["reasons"]}
+ assert {"state_claim", "sample_extrapolation", "channel_causality"}.issubset(fenced_reasons)
+ metric_claims = web_app.fastmoss_high_risk_claims(
+ "## 4. 代表商品与趋势\n| 维度 | 商品A |\n| 30天销量 | 5710 |\n"
+ "商品A近30天销量为5710件。\nMae贡献了该商品90%+的GMV。"
+ )
+ assert all(not claim["text"].startswith("##") for claim in metric_claims)
+ metric_reasons = {reason for claim in metric_claims for reason in claim["reasons"]}
+ assert {"metric_period", "cross_period_ratio"}.issubset(metric_reasons)
+
+ causal_cleaned = web_app.downgrade_fastmoss_absolute_market_claims(
+ "周均仍有250件新品入场，表明供给侧仍在积极测试，竞争加剧。\n"
+ "视频份额下降，说明消费者对内容的耐受度在降低，决策路径转向搜索。\n"
+ "样本10件销量28件，说明消费者认知尚未打开，多数购买者通过别的关键词进入。\n"
+ "该商品0条视频，说明主要依靠商品卡自然流量或付费广告，而非内容驱动。"
+ )
+ for unsupported in ("竞争加剧", "耐受度在降低", "消费者认知尚未打开", "主要依靠商品卡"):
+ assert unsupported not in causal_cleaned
+ assert "新品供给仍活跃" in causal_cleaned
+ assert "消费者认知和搜索路径仍需验证" in causal_cleaned
+ assert "渠道归因数据验证" in causal_cleaned
+
+ expanded_causal_cleaned = web_app.downgrade_fastmoss_absolute_market_claims(
+ "消费者搜了再买，商品卡自然搜索流量说明这个产品更成熟、更活跃、销售潜力更大。\n"
+ "另一个细分仍处于萌芽或需求低迷，前者由达人积极带动。\n"
+ "类目销量同比下降说明整体大盘萎缩、进入存量竞争阶段，但已经形成内容护城河和用户心智。\n"
+ "两款产品过度依赖“商品卡”自然搜索流量，这意味着它们缺乏可持续的达人内容驱动力；"
+ "一旦竞争品进入，销量将锐减。前者至少有达人愿意带，其增长有持续动力，后者依赖搜索截流。\n"
+ "视频份额变化表明单纯的视频种草转化难度在增加，Top5 功能进一步验证了它是类目的核心驱动力。"
+ )
+ for unsupported in (
+ "搜了再买", "自然搜索流量", "更成熟", "更活跃", "销售潜力更大",
+ "萌芽", "需求低迷", "达人积极带动", "大盘萎缩", "存量竞争", "内容护城河", "用户心智",
+ "缺乏可持续", "销量将锐减", "达人愿意带", "持续动力", "搜索截流", "转化难度在增加", "核心驱动力",
+ ):
+ assert unsupported not in expanded_causal_cleaned
+ assert "搜索与购买路径未被本轮数据观测" in expanded_causal_cleaned
+ assert "长期市场与竞争阶段仍待验证" in expanded_causal_cleaned
+
+ cross_entity_claims = web_app.fastmoss_high_risk_claims(
+ "两款商品都来自同一品牌 SPZTJK。"
+ )
+ assert cross_entity_claims and "cross_entity_attribute" in cross_entity_claims[0]["reasons"]
+
+ evidence_manifest = {
+ "quality_states": {"data": ["fastmoss__product_search"], "empty": [], "error": []},
+ "evidence_envelope_count": 1, "evidence_fact_count": 1, "unsupported_parser_count": 0,
+ "category_head": {"target_pages": 3, "completed_pages": [1], "reported_total": 30, "fetched_unique": 10},
+ "segment_head": {"queries": {}, "fetched_unique": 0},
+ "entity_bundles": [], "analysis_targets": [], "evidence_envelopes": [],
+ "evidence_facts": [{
+ "source_tool": "fastmoss__product_search", "source_call_index": 1,
+ "data_state": "data", "dimension": "product_sample", "scope": "category_head", "page": 1,
+ "returned_count": 10,
+ "products": [
+ {"product_id": str(1730000000000000000 + index), "title": f"Product {index}"}
+ for index in range(10)
+ ],
+ }],
+ "metric_registry": [], "derived_facts": [], "conflicts": [], "limitations": [],
+ }
+ operational_evidence = web_app.fastmoss_candidate_evidence(
+ evidence_manifest,
+ [{"claim_id": "line-1", "text": "达人链接数为382。", "reasons": ["operational_number"], "entity_ids": []}],
+ {"playbook": "product"},
+ )
+ category_fact = next(item for item in operational_evidence["facts"] if item.get("scope") == "category_head")
+ assert len(category_fact["products"]) == 3 and category_fact["page"] == 1
+
+ creator_cleaned, creator_count = web_app.sanitize_fastmoss_unsupported_recommendations(
+ "建议找1–2位达人先做验证。"
+ )
+ assert creator_count == 1 and "1–2" not in creator_cleaned
+
+
+def test_analytical_routes_use_report_model_and_fastmoss_preserves_pro_draft() -> None:
+ assert web_app.chat_route_uses_report_model(
+ "home", {"intent": "product_research", "task_depth": "analysis"}
+ )
+ assert web_app.chat_route_uses_report_model(
+ "amazon", {"intent": "product_research", "task_depth": "analysis"}
+ )
+ assert web_app.chat_route_uses_report_model(
+ "fastmoss", {"intent": "fastmoss_product", "task_depth": "workflow", "playbook": "product"}
+ )
+ assert web_app.chat_route_uses_report_model(
+ "home", {"intent": "video_analysis", "task_depth": "lookup"}
+ )
+ assert not web_app.chat_route_uses_report_model(
+ "fastmoss", {"intent": "product_availability", "task_depth": "lookup"}
+ )
+ assert not web_app.chat_route_uses_report_model(
+ "home", {"intent": "help", "task_depth": "direct"}
+ )
+
+ old_report_model = os.environ.get("DEEPSEEK_REPORT_MODEL")
+ old_verifier = os.environ.get("FASTMOSS_LLM_VERIFIER_ENABLED")
+ original_manifest = web_app.fastmoss_evidence_manifest
+ calls = []
+
+ class Requests:
+ def post(self, *_args, **_kwargs):
+ calls.append(True)
+ raise AssertionError("disabled verifier must not make another LLM request")
+
+ web_app.fastmoss_evidence_manifest = lambda *_args, **_kwargs: {
+ "quality_states": {"data": ["fastmoss__product_search"], "empty": [], "error": []},
+ "evidence_fact_count": 1,
+ "category_head": {},
+ "segment_head": {},
+ "entity_bundles": [],
+ "evidence_envelopes": [],
+ "evidence_facts": [{"fact_id": "fm-c1-f1", "data_state": "data"}],
+ "derived_facts": [],
+ "conflicts": [],
+ "limitations": [],
+ }
+ os.environ["DEEPSEEK_REPORT_MODEL"] = "deepseek-v4-pro-test"
+ os.environ["FASTMOSS_LLM_VERIFIER_ENABLED"] = "0"
+ draft = "# 完整调研报告\n\n## 市场判断\n\n保留原始结构、表格和分析。"
+ try:
+ assert web_app.chat_report_model() == "deepseek-v4-pro-test"
+ result = web_app.finalize_fastmoss_answer(
+ draft,
+ SimpleNamespace(tool_calls=[], tool_results=[]),
+ "调研这个类目",
+ {"task_depth": "workflow", "playbook": "product"},
+ Requests(),
+ "key",
+ "https://example.test/v1",
+ "deepseek-v4-pro-test",
+ )
+ finally:
+ web_app.fastmoss_evidence_manifest = original_manifest
+ if old_report_model is None:
+ os.environ.pop("DEEPSEEK_REPORT_MODEL", None)
+ else:
+ os.environ["DEEPSEEK_REPORT_MODEL"] = old_report_model
+ if old_verifier is None:
+ os.environ.pop("FASTMOSS_LLM_VERIFIER_ENABLED", None)
+ else:
+ os.environ["FASTMOSS_LLM_VERIFIER_ENABLED"] = old_verifier
+ assert result == draft
+ assert calls == []
+
+
if __name__ == "__main__":
test_tiktok_search_keeps_analysis_fields()
test_amazon_keeps_product_fields()
@@ -221,8 +3030,61 @@ def test_web_search_tool_is_registered_and_normalized() -> None:
test_locked_amazon_provider_filters_system_web_search()
test_locked_amazon_product_route_keeps_sellersprite_tools()
test_amazon_url_query_api_fragment_does_not_disable_tools()
+ test_ocr_metadata_does_not_change_chat_route()
+ test_fastmoss_defaults_to_us_unless_another_region_is_named()
+ test_fastmoss_playbook_intent_routes_official_workflows()
+ test_fastmoss_selection_playbook_includes_pricing_model()
+ test_fastmoss_product_evidence_is_scoped_by_playbook()
+ test_fastmoss_analysis_requires_domain_and_evidence_capabilities()
+ test_fastmoss_analysis_requires_us_ranking_and_reviews()
+ test_fastmoss_explicit_other_region_does_not_require_us()
test_short_cjk_web_search_filters_irrelevant_results()
test_pdf_markdown_export_matches_frontend_quote_heading()
test_web_search_tool_is_registered_and_normalized()
+ test_chat_history_archives_done_tools_and_recovers_failed_results()
+ test_tool_evidence_is_compact_but_keeps_business_fields()
+ test_current_tool_evidence_is_lossless_until_budget_pressure()
+ test_product_availability_is_a_shallow_lookup()
+ test_intent_decision_validation_and_fallback()
+ test_intent_router_uses_recent_context_and_falls_back_on_failure()
+ test_empty_mcp_collections_are_not_enough_data()
+ test_mcp_content_error_rules_are_provider_specific()
+ test_fastmoss_zero_analysis_metadata_is_empty_without_affecting_sellersprite()
+ test_mcp_sql_error_text_is_not_evidence()
+ test_deepseek_tool_turn_preserves_reasoning_content()
+ test_dynamic_chat_context_compresses_to_budget()
+ test_tool_limit_final_context_removes_protocol_and_detects_dsml()
+ test_tool_limit_keeps_large_current_collection_when_capacity_allows()
+ test_sellersprite_schema_argument_normalization()
+ test_llm_router_can_select_fastmoss_playbook()
+ test_fastmoss_workflow_phases_accept_empty_and_error_attempts()
+ test_fastmoss_product_phase_requires_complete_sample_coverage()
+ test_fastmoss_business_defaults_use_verified_category_levels()
+ test_fastmoss_product_workflow_keeps_its_round_budget_isolated()
+ test_fastmoss_clarification_is_targeted_and_provider_isolated()
+ test_fastmoss_close_cross_category_matches_request_confirmation()
+ test_provider_profiles_use_aggregated_sellersprite_and_staged_fastmoss_tools()
+ test_region_default_only_applies_when_schema_supports_it()
+ test_fastmoss_deep_dive_ids_must_come_from_current_task()
+ test_tool_call_signature_deduplicates_argument_order()
+ test_fastmoss_dual_ranking_plan_uses_three_sorted_category_pages_then_segments()
+ test_fastmoss_product_phase_waits_for_all_category_and_segment_searches()
+ test_fastmoss_product_workflow_deterministically_advances_and_binds_two_targets()
+ test_fastmoss_product_search_defaults_force_category_pages_and_short_segment_queries()
+ test_fastmoss_evidence_manifest_separates_total_fetched_overlap_and_conflicts()
+ test_fastmoss_deterministic_fallback_contains_analysis_and_prioritized_advice()
+ test_fastmoss_l2_market_metrics_are_only_upstream_category_reference()
+ test_fastmoss_failed_category_page_is_not_counted_as_coverage()
+ test_fastmoss_metadata_detects_unsorted_page_and_string_product_ids()
+ test_fastmoss_evidence_ledger_keeps_native_dimensions_and_late_trends()
+ test_fastmoss_answer_verifier_applies_local_edits_and_keeps_draft_on_failure()
+ test_fastmoss_all_workflow_tools_emit_supported_envelopes()
+ test_fastmoss_verifier_batches_claims_and_isolates_failures()
+ test_fastmoss_creator_video_facts_and_historical_rebuild()
+ test_fastmoss_report_packets_are_workflow_native_and_numeric_policy_is_strict()
+ test_fastmoss_list_truncation_is_explicit_and_invalid_entities_are_filtered()
+ test_fastmoss_22_call_semantic_registry_fixture()
+ test_fastmoss_dossier_synthesis_preserves_complete_tool_evidence()
+ test_fastmoss_claim_ids_and_extended_mechanical_cleanup()
+ test_analytical_routes_use_report_model_and_fastmoss_preserves_pro_draft()
print("chat tool normalization tests passed")
-
diff --git a/scripts/test_json_to_markdown.py b/scripts/test_json_to_markdown.py
new file mode 100644
index 0000000..8fb883f
--- /dev/null
+++ b/scripts/test_json_to_markdown.py
@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""Ad-hoc tests for the lossless JSON-to-Markdown renderer."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "scripts"))
+
+from json_to_markdown import json_text_to_markdown, json_to_markdown # noqa: E402
+
+
+def fastmoss_fixture() -> dict:
+ return {
+ "code": 0,
+ "message": "",
+ "timestamp": 1775714099,
+ "request_id": "req-demo",
+ "data": {
+ "total": 2,
+ "date_type": "day",
+ "date_value": "2026-04-06",
+ "list": [
+ {
+ "product_id": "1729739021314527787",
+ "title": "Food Processor | 400W",
+ "sales": 0,
+ "is_ad": False,
+ "growth_rate": None,
+ },
+ {
+ "product_id": "1729844364239993614",
+ "title": "Mini Meat\nGrinder",
+ "sales": 2493,
+ "is_ad": True,
+ "growth_rate": "-15.57%",
+ },
+ ],
+ },
+ }
+
+
+def test_fastmoss_envelope_and_flat_list_table() -> None:
+ markdown = json_to_markdown(fastmoss_fixture(), title="FastMoss 响应")
+ assert markdown.startswith("# FastMoss 响应\n")
+ assert "## 响应元数据" in markdown
+ assert "| code | 0 | $.code |" in markdown
+ assert "| message | \"\" | $.message |" in markdown
+ assert "## data (`$.data`)" in markdown
+ assert "数组，共 2 项；以下完整展示全部 2 项。" in markdown
+ assert "1729739021314527787" in markdown
+ assert "Food Processor \\| 400W" in markdown
+ assert "Mini Meat自动采集设置
+
+
默认采集执行日的前一天，日期会随每天执行自动变化。
+
+ Grinder" in markdown + assert "| 0 | 1729739021314527787" in markdown + assert "| false | null | $.data.list[0] |" in markdown + + +def test_nested_values_and_empty_states_are_explicit() -> None: + source = { + "data": { + "shop": { + "shop_id": "7494049503488000000", + "metrics": {"gmv": 0, "enabled": False, "note": None}, + }, + "videos": [], + "facets": {}, + "groups": [[1, 2], [3]], + }, + "code": 0, + "msg": "success", + } + markdown = json_to_markdown(source) + assert "7494049503488000000" in markdown + assert "空数组[]（`$.data.videos`）" in markdown + assert "空对象{}（`$.data.facets`）" in markdown + assert "### 项目 1 (`$.data.groups[0]`)" in markdown + assert "| enabled | false | $.data.shop.metrics.enabled |" in markdown + assert "| note | null | $.data.shop.metrics.note |" in markdown + + +def test_missing_field_is_distinct_from_null() -> None: + markdown = json_to_markdown([{"id": "1", "score": None}, {"id": "2"}]) + assert "| 0 | 1 | null | $[0] |" in markdown + assert "| 1 | 2 | （字段缺失） | $[1] |" in markdown + + +def test_optional_row_limit_is_never_silent() -> None: + markdown = json_to_markdown( + {"items": [{"id": str(index)} for index in range(5)]}, + max_table_rows=2, + ) + assert "共 5 项，本次展示 2 项，省略 3 项" in markdown + assert "数组，共 5 项；本次展示前 2 项。" in markdown + assert "\n| 2 | 2 |" not in markdown + + +def test_default_rendering_is_complete_and_deterministic() -> None: + source = {"items": [{"id": str(index)} for index in range(20)]} + first = json_to_markdown(source) + second = json_to_markdown(source) + assert first == second + assert "显式裁剪" not in first + assert "$.items" not in first # Guard against invisible path characters. + for index in range(20): + assert f"| {index} | {index} | $.items[{index}] |" in first + + +def test_json_text_and_invalid_input() -> None: + text = json.dumps(fastmoss_fixture(), ensure_ascii=False) + assert json_text_to_markdown(text) == json_to_markdown(fastmoss_fixture()) + try: + json_text_to_markdown('{"data":') + except ValueError as exc: + assert "line 1" in str(exc) and "column 9" in str(exc) + else: + raise AssertionError("invalid JSON should fail") + + +def test_rejects_non_json_python_values() -> None: + try: + json_to_markdown({"ids": {"1", "2"}}) # type: ignore[dict-item] + except ValueError as exc: + assert "non-JSON value" in str(exc) + else: + raise AssertionError("sets are not JSON-compatible") + + +def test_cli_reads_file_and_writes_stdout() -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = Path(temp_dir) / "response.json" + source.write_text(json.dumps(fastmoss_fixture()), encoding="utf-8") + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "json_to_markdown.py"), str(source)], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert "## 响应元数据" in result.stdout + assert "1729739021314527787" in result.stdout + + +if __name__ == "__main__": + test_fastmoss_envelope_and_flat_list_table() + test_nested_values_and_empty_states_are_explicit() + test_missing_field_is_distinct_from_null() + test_optional_row_limit_is_never_silent() + test_default_rendering_is_complete_and_deterministic() + test_json_text_and_invalid_input() + test_rejects_non_json_python_values() + test_cli_reads_file_and_writes_stdout() + print("json-to-markdown tests passed") diff --git a/scripts/test_lan_chat_file_transfer.py b/scripts/test_lan_chat_file_transfer.py new file mode 100644 index 0000000..0aaa178 --- /dev/null +++ b/scripts/test_lan_chat_file_transfer.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""Ad-hoc verification for LAN chat media and temporary file transfers.""" + +from __future__ import annotations + +import base64 +import sqlite3 +import tempfile +import time +import unittest +from io import BytesIO +from pathlib import Path + +from lan_chat import ( + DEFAULT_FEISHU_USER_ID, + FILE_TRANSFER_MAX_BYTES, + FILE_TRANSFER_RETENTION_SECONDS, + MESSAGE_MEDIA_MAX_BYTES, + MESSAGE_MEDIA_RETENTION_SECONDS, + LanChatError, + LanChatStore, +) + + +class LanChatFileTransferTest(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + root = Path(self.temp_dir.name) + self.store = LanChatStore(root / "lan_chat.sqlite") + self.store.initialize() + self.sender = self.store.create_account(DEFAULT_FEISHU_USER_ID, "发送者") + self.receiver = self.store.create_account(DEFAULT_FEISHU_USER_ID, "接收者") + + def tearDown(self) -> None: + self.temp_dir.cleanup() + + def test_initialize_migrates_legacy_inline_media_expiry(self) -> None: + with tempfile.TemporaryDirectory() as directory: + db_path = Path(directory) / "legacy.sqlite" + created_at = time.time() + with sqlite3.connect(db_path) as conn: + conn.execute( + """CREATE TABLE messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + room_id TEXT NOT NULL, + sender_id TEXT NOT NULL, + content TEXT NOT NULL, + image_filename TEXT, + image_mime_type TEXT, + file_id TEXT, + created_at REAL NOT NULL + )""" + ) + conn.execute( + """INSERT INTO messages + (room_id, sender_id, content, image_filename, + image_mime_type, file_id, created_at) + VALUES ('public', 'legacy', '', ?, 'image/jpeg', NULL, ?)""", + ("0" * 32 + ".jpg", created_at), + ) + + store = LanChatStore(db_path) + store.initialize() + with sqlite3.connect(db_path) as conn: + columns = { + row[1] for row in conn.execute("PRAGMA table_info(messages)") + } + expires_at, deleted_at = conn.execute( + "SELECT media_expires_at, media_deleted_at FROM messages" + ).fetchone() + self.assertIn("media_expires_at", columns) + self.assertIn("media_deleted_at", columns) + self.assertIn("client_upload_id", columns) + self.assertEqual(expires_at, created_at + MESSAGE_MEDIA_RETENTION_SECONDS) + self.assertIsNone(deleted_at) + + def test_group_file_is_immediately_available_and_expires(self) -> None: + room = self.store.create_group( + self.sender["sessionToken"], + "文件群", + [self.receiver["user"]["id"]], + ) + message, created = self.store.send_file( + self.sender["sessionToken"], + room["id"], + "报表 2026.xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + BytesIO(b"group-file"), + "请查收", + ) + self.assertTrue(created) + + attachment = message["file"] + self.assertEqual(attachment["receiptStatus"], "available") + self.assertTrue(attachment["downloadAllowed"]) + received = self.store.list_messages( + self.receiver["sessionToken"], room["id"] + )["messages"][0]["file"] + self.assertFalse(received["requiresAcceptance"]) + self.assertTrue(received["downloadAllowed"]) + + path, name, content_type, size = self.store.file_download_info( + self.receiver["sessionToken"], attachment["id"] + ) + self.assertEqual(path.read_bytes(), b"group-file") + self.assertEqual(name, "报表 2026.xlsx") + self.assertEqual(size, len(b"group-file")) + self.assertTrue(content_type.startswith("application/")) + + cleaned = self.store.cleanup_expired_files( + message["createdAt"] + FILE_TRANSFER_RETENTION_SECONDS + 1 + ) + self.assertEqual(cleaned, 1) + self.assertFalse(path.exists()) + expired = self.store.list_messages( + self.receiver["sessionToken"], room["id"] + )["messages"][0]["file"] + self.assertTrue(expired["expired"]) + self.assertFalse(expired["downloadAllowed"]) + with self.assertRaises(LanChatError) as context: + self.store.file_download_info( + self.receiver["sessionToken"], attachment["id"] + ) + self.assertEqual(context.exception.status, 410) + + def test_direct_file_requires_receiver_acceptance(self) -> None: + room = self.store.open_direct( + self.sender["sessionToken"], self.receiver["user"]["id"] + ) + message, created = self.store.send_file( + self.sender["sessionToken"], + room["id"], + "方案.pdf", + "application/pdf", + BytesIO(b"direct-file"), + ) + self.assertTrue(created) + file_id = message["file"]["id"] + self.assertEqual(message["file"]["receiptStatus"], "pending") + self.assertTrue(message["file"]["downloadAllowed"]) + + receiver_message = self.store.list_messages( + self.receiver["sessionToken"], room["id"] + )["messages"][0] + self.assertTrue(receiver_message["file"]["requiresAcceptance"]) + self.assertFalse(receiver_message["file"]["downloadAllowed"]) + with self.assertRaises(LanChatError) as context: + self.store.file_download_info(self.receiver["sessionToken"], file_id) + self.assertEqual(context.exception.status, 403) + + accepted = self.store.accept_file(self.receiver["sessionToken"], file_id) + self.assertEqual(accepted["file"]["receiptStatus"], "accepted") + self.assertTrue(accepted["file"]["downloadAllowed"]) + path, _, _, _ = self.store.file_download_info( + self.receiver["sessionToken"], file_id + ) + self.assertEqual(path.read_bytes(), b"direct-file") + + def test_small_mp4_remains_inline_media(self) -> None: + bootstrap = self.store.bootstrap(self.sender["sessionToken"]) + self.assertEqual(bootstrap["inlineMediaMaxBytes"], 100 * 1024 * 1024) + self.assertEqual(MESSAGE_MEDIA_MAX_BYTES, bootstrap["inlineMediaMaxBytes"]) + self.assertEqual( + MESSAGE_MEDIA_RETENTION_SECONDS, + bootstrap["inlineMediaRetentionSeconds"], + ) + self.assertEqual(bootstrap["messagePollIntervalMs"], 3000) + self.assertEqual(bootstrap["bootstrapPollIntervalMs"], 10000) + self.assertEqual(bootstrap["fileMaxBytes"], 10 * 1024 * 1024 * 1024) + self.assertEqual(FILE_TRANSFER_MAX_BYTES, bootstrap["fileMaxBytes"]) + payload = b"\x00\x00\x00\x18ftypisom" + b"small-video" + media_data = "data:video/mp4;base64," + base64.b64encode(payload).decode("ascii") + message, created = self.store.send_message( + self.sender["sessionToken"], "public", "", media_data + ) + self.assertTrue(created) + self.assertEqual(message["mediaKind"], "video") + self.assertFalse(message["mediaExpired"]) + self.assertEqual( + message["mediaExpiresAt"], + message["createdAt"] + MESSAGE_MEDIA_RETENTION_SECONDS, + ) + self.assertTrue(message["mediaUrl"].endswith(".mp4")) + self.assertEqual(message["mediaPosterUrl"], f"{message['mediaUrl']}/poster") + self.assertEqual(message["mediaDownloadUrl"], f"{message['mediaUrl']}/download") + filename = message["mediaUrl"].rsplit("/", 1)[-1] + path, stored_name, stored_type, stored_size = self.store.message_media_info(filename) + self.assertEqual(stored_name, filename) + self.assertEqual(stored_type, "video/mp4") + self.assertEqual(stored_size, len(payload)) + self.assertEqual(path.read_bytes(), payload) + body, content_type = self.store.message_media_bytes(filename) + self.assertEqual(body, payload) + self.assertEqual(content_type, "video/mp4") + + poster_path = path.with_name(f"{path.stem}.poster.jpg") + poster_path.write_bytes(b"poster") + cleaned = self.store.cleanup_expired_media( + message["createdAt"] + MESSAGE_MEDIA_RETENTION_SECONDS + 1 + ) + self.assertEqual(cleaned, 1) + self.assertFalse(path.exists()) + self.assertFalse(poster_path.exists()) + expired = self.store.list_messages( + self.sender["sessionToken"], "public" + )["messages"][0] + self.assertTrue(expired["mediaExpired"]) + self.assertEqual(expired["mediaKind"], "video") + self.assertEqual(expired["mediaUrl"], "") + self.assertEqual(expired["mediaPosterUrl"], "") + self.assertEqual(expired["mediaDownloadUrl"], "") + with self.assertRaises(LanChatError) as context: + self.store.message_media_info(filename) + self.assertEqual(context.exception.status, 410) + + def test_inline_media_client_upload_id_is_idempotent(self) -> None: + upload_id = "inline_upload_1234567890" + media_data = "data:image/png;base64," + base64.b64encode( + b"\x89PNG\r\n\x1a\ninline" + ).decode("ascii") + first, first_created = self.store.send_message( + self.sender["sessionToken"], + "public", + "图片", + media_data, + upload_id, + ) + second, second_created = self.store.send_message( + self.sender["sessionToken"], + "public", + "不会覆盖", + media_data, + upload_id, + ) + + self.assertTrue(first_created) + self.assertFalse(second_created) + self.assertEqual(first["id"], second["id"]) + self.assertEqual(first["clientUploadId"], upload_id) + self.assertEqual(len(list(self.store.media_dir.iterdir())), 1) + with sqlite3.connect(self.store.db_path) as conn: + self.assertEqual(conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0], 1) + + def test_direct_file_client_upload_id_does_not_duplicate_receipt(self) -> None: + room = self.store.open_direct( + self.sender["sessionToken"], self.receiver["user"]["id"] + ) + upload_id = "direct_file_1234567890" + first, first_created = self.store.send_file( + self.sender["sessionToken"], + room["id"], + "原始.pdf", + "application/pdf", + BytesIO(b"first-file"), + "正文", + upload_id, + ) + second, second_created = self.store.send_file( + self.sender["sessionToken"], + room["id"], + "重复.pdf", + "application/pdf", + BytesIO(b"second-file"), + "不同正文", + upload_id, + ) + + self.assertTrue(first_created) + self.assertFalse(second_created) + self.assertEqual(first["id"], second["id"]) + self.assertEqual(first["file"]["id"], second["file"]["id"]) + with sqlite3.connect(self.store.db_path) as conn: + self.assertEqual(conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0], 1) + self.assertEqual(conn.execute("SELECT COUNT(*) FROM file_attachments").fetchone()[0], 1) + self.assertEqual(conn.execute("SELECT COUNT(*) FROM file_receipts").fetchone()[0], 1) + + def test_client_upload_id_rejects_cross_room_reuse(self) -> None: + room = self.store.create_group( + self.sender["sessionToken"], + "另一个房间", + [self.receiver["user"]["id"]], + ) + upload_id = "cross_room_1234567890" + self.store.send_message( + self.sender["sessionToken"], "public", "第一条", "", upload_id + ) + with self.assertRaises(LanChatError) as context: + self.store.send_message( + self.sender["sessionToken"], room["id"], "第二条", "", upload_id + ) + self.assertEqual(context.exception.status, 409) + + def test_legacy_messages_without_client_upload_id_are_not_deduplicated(self) -> None: + first, first_created = self.store.send_message( + self.sender["sessionToken"], "public", "旧客户端" + ) + second, second_created = self.store.send_message( + self.sender["sessionToken"], "public", "旧客户端" + ) + self.assertTrue(first_created) + self.assertTrue(second_created) + self.assertNotEqual(first["id"], second["id"]) + self.assertEqual(first["clientUploadId"], "") + self.assertEqual(second["clientUploadId"], "") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_lan_chat_group_management.py b/scripts/test_lan_chat_group_management.py new file mode 100644 index 0000000..a852f7a --- /dev/null +++ b/scripts/test_lan_chat_group_management.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Ad-hoc verification for LAN chat default groups and group governance.""" + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from lan_chat import DEFAULT_FEISHU_USER_ID, LanChatError, LanChatStore + + +class LanChatGroupManagementTest(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.store = LanChatStore(Path(self.temp_dir.name) / "lan_chat.sqlite") + self.store.initialize() + self.owner = self.store.create_account(DEFAULT_FEISHU_USER_ID, "组织者") + self.second = self.store.create_account(DEFAULT_FEISHU_USER_ID, "第二位") + self.third = self.store.create_account(DEFAULT_FEISHU_USER_ID, "第三位") + + def tearDown(self) -> None: + self.temp_dir.cleanup() + + def assert_forbidden(self, action) -> None: + with self.assertRaises(LanChatError) as context: + action() + self.assertEqual(context.exception.status, 403) + + def test_every_account_gets_two_immutable_default_groups(self) -> None: + owner_rooms = self.store.list_rooms(self.owner["user"]["id"]) + second_rooms = self.store.list_rooms(self.second["user"]["id"]) + defaults = [room for room in owner_rooms if room["isDefault"]] + self.assertEqual([room["systemKind"] for room in defaults], ["public", "feishu"]) + + feishu_room = next(room for room in defaults if room["systemKind"] == "feishu") + second_feishu_room = next( + room for room in second_rooms if room["systemKind"] == "feishu" + ) + self.assertEqual(feishu_room["id"], second_feishu_room["id"]) + self.assertEqual(feishu_room["memberCount"], 3) + self.assertFalse(feishu_room["canLeave"]) + self.assertFalse(feishu_room["canDissolve"]) + + token = self.owner["sessionToken"] + room_id = feishu_room["id"] + self.assert_forbidden(lambda: self.store.rename_group(token, room_id, "不能改名")) + self.assert_forbidden(lambda: self.store.leave_group(token, room_id)) + self.assert_forbidden(lambda: self.store.dissolve_group(token, room_id)) + self.assert_forbidden( + lambda: self.store.remove_group_member( + token, room_id, self.second["user"]["id"] + ) + ) + + def test_admin_transfers_by_join_order_and_successor_can_govern(self) -> None: + room = self.store.create_group( + self.owner["sessionToken"], + "项目群", + [self.second["user"]["id"], self.third["user"]["id"]], + ) + self.assertEqual(room["adminUserId"], self.owner["user"]["id"]) + self.assertTrue(room["currentUserIsAdmin"]) + self.assertTrue(room["canRename"]) + self.assertTrue(room["canDissolve"]) + + second_token = self.second["sessionToken"] + self.assert_forbidden( + lambda: self.store.rename_group(second_token, room["id"], "越权改名") + ) + self.assert_forbidden(lambda: self.store.dissolve_group(second_token, room["id"])) + self.assert_forbidden( + lambda: self.store.remove_group_member( + second_token, room["id"], self.third["user"]["id"] + ) + ) + + ordered_remaining = [ + member["id"] + for member in room["members"] + if member["id"] != self.owner["user"]["id"] + ] + result = self.store.leave_group(self.owner["sessionToken"], room["id"]) + self.assertEqual(result["newAdminUserId"], ordered_remaining[0]) + + accounts = { + self.second["user"]["id"]: self.second, + self.third["user"]["id"]: self.third, + } + successor = accounts[ordered_remaining[0]] + other_id = ordered_remaining[1] + renamed = self.store.rename_group( + successor["sessionToken"], room["id"], "继任管理员群" + ) + self.assertEqual(renamed["name"], "继任管理员群") + self.assertEqual(renamed["adminUserId"], successor["user"]["id"]) + + after_removal = self.store.remove_group_member( + successor["sessionToken"], room["id"], other_id + ) + self.assertEqual(after_removal["memberCount"], 1) + dissolved = self.store.dissolve_group(successor["sessionToken"], room["id"]) + self.assertTrue(dissolved["dissolved"]) + self.assertFalse( + any( + item["id"] == room["id"] + for item in self.store.list_rooms(successor["user"]["id"]) + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_lan_chat_upload_queue_playwright.py b/scripts/test_lan_chat_upload_queue_playwright.py new file mode 100644 index 0000000..2536c13 --- /dev/null +++ b/scripts/test_lan_chat_upload_queue_playwright.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +"""Playwright regression checks for the LAN chat multi-file upload queue.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import re +import time +from pathlib import Path + +from playwright.async_api import Route, async_playwright, expect + + +AVATAR = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg'/%3E" + + +async def wait_until(predicate, timeout: float = 8.0) -> None: + deadline = asyncio.get_running_loop().time() + timeout + while not predicate(): + if asyncio.get_running_loop().time() >= deadline: + raise AssertionError("condition did not become true before timeout") + await asyncio.sleep(0.05) + + +class MockLanChatApi: + def __init__(self, *, fail_once: set[str] | None = None, delay: float = 0.0): + self.fail_once = set(fail_once or set()) + self.failed: set[str] = set() + self.delay = delay + self.messages: dict[str, list[dict]] = {"public": [], "group-1": []} + self.upload_names: list[str] = [] + self.upload_bodies: list[str] = [] + self.active_uploads = 0 + self.max_active_uploads = 0 + self.next_message_id = 1 + + @staticmethod + def bootstrap_payload() -> dict: + user = { + "id": "user-1", + "nickname": "测试手机", + "avatarUrl": AVATAR, + "avatarStatus": "ready", + "isCurrent": True, + "online": True, + "lastSeen": time.time(), + } + return { + "currentUser": user, + "users": [user], + "rooms": [ + { + "id": "public", + "kind": "public", + "systemKind": "public", + "name": "公共频道", + "memberCount": 1, + "members": [user], + "unreadCount": 0, + "latestMessage": None, + }, + { + "id": "group-1", + "kind": "group", + "systemKind": "custom", + "name": "测试群", + "memberCount": 1, + "members": [user], + "unreadCount": 0, + "latestMessage": None, + "currentUserIsAdmin": True, + "canRename": True, + "canRemoveMembers": True, + "canLeave": True, + "canDissolve": True, + }, + ], + "publicRoomId": "public", + "inlineMediaMaxBytes": 100 * 1024 * 1024, + "inlineMediaRetentionSeconds": 7 * 86400, + "fileMaxBytes": 10 * 1024 * 1024 * 1024, + "fileRetentionSeconds": 7 * 86400, + "messagePollIntervalMs": 3000, + "bootstrapPollIntervalMs": 10000, + } + + async def handle(self, route: Route) -> None: + request = route.request + path = request.url.split("?", 1)[0] + if path.endswith("/api/lan-chat/bootstrap"): + await route.fulfill(json=self.bootstrap_payload()) + return + message_match = re.search(r"/api/lan-chat/rooms/([^/]+)/messages$", path) + if message_match and request.method == "GET": + room_id = message_match.group(1) + messages = self.messages.get(room_id, []) + await route.fulfill( + json={"messages": messages, "lastId": messages[-1]["id"] if messages else 0} + ) + return + file_match = re.search(r"/api/lan-chat/rooms/([^/]+)/files$", path) + if file_match and request.method == "POST": + await self._handle_upload(route, file_match.group(1)) + return + await route.fulfill(status=404, json={"error": "mock route not found"}) + + async def _handle_upload(self, route: Route, room_id: str) -> None: + body = (route.request.post_data_buffer or b"").decode("utf-8", "replace") + filename_match = re.search(r'filename="([^"]+)"', body) + upload_id_match = re.search( + r'name="clientUploadId"\r\n\r\n([^\r\n]+)', body + ) + filename = filename_match.group(1) if filename_match else "unknown.bin" + client_upload_id = upload_id_match.group(1) if upload_id_match else "" + self.upload_names.append(filename) + self.upload_bodies.append(body) + self.active_uploads += 1 + self.max_active_uploads = max(self.max_active_uploads, self.active_uploads) + try: + if self.delay: + await asyncio.sleep(self.delay) + if filename in self.fail_once and filename not in self.failed: + self.failed.add(filename) + await route.fulfill(status=400, json={"error": "模拟的 4xx 失败"}) + return + message = { + "id": self.next_message_id, + "roomId": room_id, + "clientUploadId": client_upload_id, + "senderId": "user-1", + "senderName": "测试手机", + "senderAvatarUrl": AVATAR, + "content": "批次说明" if "批次说明" in body else "", + "imageUrl": "", + "mediaUrl": "", + "mediaPosterUrl": "", + "mediaDownloadUrl": "", + "mediaMimeType": "", + "mediaKind": "", + "mediaExpiresAt": None, + "mediaExpired": False, + "file": { + "id": f"file-{self.next_message_id}", + "name": filename, + "mimeType": "text/plain", + "size": 4, + "expiresAt": time.time() + 7 * 86400, + "expired": False, + "requiresAcceptance": False, + "receiptStatus": "available", + "downloadAllowed": True, + "downloadUrl": f"/mock/{self.next_message_id}", + }, + "createdAt": time.time(), + "isMine": True, + } + self.next_message_id += 1 + self.messages.setdefault(room_id, []).append(message) + await route.fulfill(status=201, json={"message": message, "created": True}) + finally: + self.active_uploads -= 1 + + +async def open_chat(browser, base_url: str, viewport: dict, api: MockLanChatApi): + context = await browser.new_context(viewport=viewport) + await context.add_init_script( + "localStorage.setItem('videoAnalyzer.lanChat.sessionToken.v2','test-token')" + ) + page = await context.new_page() + console_errors: list[str] = [] + page.on( + "console", + lambda message: console_errors.append(message.text) + if message.type == "error" + else None, + ) + await page.route("**/api/lan-chat/**", api.handle) + await page.goto(f"{base_url}/lan-chat", wait_until="domcontentloaded") + await expect(page.locator("#loginModal")).not_to_have_class(re.compile("show")) + return context, page, console_errors + + +async def desktop_scenario(browser, base_url: str, screenshot_dir: Path) -> None: + api = MockLanChatApi(delay=0.15) + context, page, console_errors = await open_chat( + browser, base_url, {"width": 1280, "height": 820}, api + ) + try: + file_input = page.locator("#imageInput") + assert await file_input.get_attribute("multiple") is not None + + await file_input.set_input_files( + [{"name": "single.txt", "mimeType": "text/plain", "buffer": b"one"}] + ) + await expect(page.locator("#imageDraft")).to_be_visible() + assert not api.upload_names + await page.locator("#removeImage").click() + + await page.locator("#messageInput").fill("批次说明") + await file_input.set_input_files( + [ + {"name": "a.txt", "mimeType": "text/plain", "buffer": b"a"}, + {"name": "b.txt", "mimeType": "text/plain", "buffer": b"b"}, + {"name": "c.txt", "mimeType": "text/plain", "buffer": b"c"}, + ] + ) + await expect(page.locator("#uploadQueue")).to_be_visible() + await expect(page.locator(".queue-placeholder")).to_have_count(3) + await expect(page.locator("#queueSummaryStatus")).to_contain_text("上传中") + await expect(page.locator(".queue-progress").first).to_be_visible() + await page.locator('[data-room-id="group-1"]').click() + await expect(page.locator("#queueSummaryText")).to_contain_text("公共频道") + await expect(page.locator(".queue-placeholder")).to_have_count(0) + await wait_until(lambda: len(api.upload_names) == 3) + assert api.upload_names == ["a.txt", "b.txt", "c.txt"] + assert api.max_active_uploads == 1 + assert sum("批次说明" in body for body in api.upload_bodies) == 1 + await page.locator('[data-room-id="public"]').click() + await expect(page.locator("[data-message-id]")).to_have_count(3) + await page.evaluate("mergeServerMessage(state.messages[0]); renderMessages()") + await expect(page.locator("[data-message-id]")).to_have_count(3) + await page.screenshot(path=str(screenshot_dir / "lan-chat-upload-queue-desktop.png")) + assert not console_errors, console_errors + finally: + await context.close() + + +async def mobile_network_retry_scenario( + browser, base_url: str, screenshot_dir: Path +) -> None: + api = MockLanChatApi(fail_once={"retry.txt"}) + context, page, console_errors = await open_chat( + browser, base_url, {"width": 390, "height": 844}, api + ) + try: + await context.set_offline(True) + await page.locator("#messageInput").fill("批次说明") + await page.locator("#imageInput").set_input_files( + [ + {"name": "cancel.txt", "mimeType": "text/plain", "buffer": b"x"}, + {"name": "retry.txt", "mimeType": "text/plain", "buffer": b"y"}, + {"name": "ok.txt", "mimeType": "text/plain", "buffer": b"z"}, + ] + ) + await expect(page.locator("#queueSummaryStatus")).to_have_text("等待网络") + await expect(page.locator(".queue-placeholder-status").first).to_contain_text( + "等待网络" + ) + await page.screenshot(path=str(screenshot_dir / "lan-chat-upload-queue-mobile.png")) + await page.locator("[data-queue-cancel]").first.click() + await context.set_offline(False) + await expect(page.locator("[data-queue-retry]").first).to_be_visible(timeout=8000) + await page.locator("[data-queue-retry]").first.click() + await wait_until(lambda: len(api.messages["public"]) == 2) + await expect(page.locator("[data-message-id]")).to_have_count(2) + assert api.upload_names == ["retry.txt", "ok.txt", "retry.txt"] + assert sum("批次说明" in body for body in api.upload_bodies) == 2 + assert api.max_active_uploads == 1 + unexpected_errors = [ + error for error in console_errors if "400 (Bad Request)" not in error + ] + assert not unexpected_errors, unexpected_errors + finally: + await context.close() + + +async def main(base_url: str, screenshot_dir: Path) -> None: + screenshot_dir.mkdir(parents=True, exist_ok=True) + async with async_playwright() as playwright: + browser = await playwright.chromium.launch(headless=True) + try: + await desktop_scenario(browser, base_url, screenshot_dir) + await mobile_network_retry_scenario(browser, base_url, screenshot_dir) + finally: + await browser.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://127.0.0.1:4003") + parser.add_argument("--screenshot-dir", type=Path, default=Path("/tmp")) + arguments = parser.parse_args() + asyncio.run(main(arguments.base_url, arguments.screenshot_dir)) diff --git a/scripts/test_mcp_bridge_cache.js b/scripts/test_mcp_bridge_cache.js new file mode 100644 index 0000000..81cad05 --- /dev/null +++ b/scripts/test_mcp_bridge_cache.js @@ -0,0 +1,12 @@ +#!/usr/bin/env node +"use strict"; + +const assert = require("node:assert/strict"); +const { mcpToolResponseIsError } = require("../sellersprite_mcp_chat/server.js"); + +assert.equal(mcpToolResponseIsError({ isError: true, content: [{ type: "text", text: "keyword is required" }] }), true); +assert.equal(mcpToolResponseIsError({ error: { message: "failed" } }), true); +assert.equal(mcpToolResponseIsError({ isError: false, content: [{ type: "text", text: "ok" }] }), false); +assert.equal(mcpToolResponseIsError(null), false); + +console.log("MCP bridge cache tests passed"); diff --git a/scripts/tiktok_studio_collect.py b/scripts/tiktok_studio_collect.py new file mode 100644 index 0000000..2127ed6 --- /dev/null +++ b/scripts/tiktok_studio_collect.py @@ -0,0 +1,1630 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import os +import re +import threading +import time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urljoin, urlparse +from zoneinfo import ZoneInfo + +import proxy_pool +from browser_page_state import ( + BrowserPageBlocked, + BrowserPageLoadError, + BrowserPageTimeout, + navigate_with_retries, + wait_for_page_state, +) +from feishu_capabilities import FeishuCapabilityClient, FeishuCapabilityError + + +ROOT = Path.cwd() +LOG_ROOT = ROOT / "data" / "tiktok_collect_jobs" +TIMEZONE_NAME = os.getenv("TZ", "America/Los_Angeles") or "America/Los_Angeles" +DEFAULT_DAILY_TIME = os.getenv("TIKTOK_COLLECT_DAILY_TIME", "03:00").strip() or "03:00" +DEFAULT_DATE_RULE = "previous_day" +DATE_RULES = {"previous_day", "same_day"} +RETENTION_MAX_SECONDS = max(10, int(os.getenv("TIKTOK_COLLECT_RETENTION_MAX_SECONDS", "300") or "300")) +WORKER_INTERVAL_SECONDS = max(3, int(os.getenv("TIKTOK_COLLECT_WORKER_INTERVAL_SECONDS", "10") or "10")) +JOB_ACTIVE_STATUSES = {"queued", "delayed", "preparing", "collecting"} +JOB_RETRYABLE_STATUSES = {"failed", "partial", "cancelled"} +STATUS_LABELS = { + "queued": "待采集", + "delayed": "延迟等待", + "preparing": "准备中", + "collecting": "采集中", + "complete": "采集完成", + "partial": "部分失败", + "failed": "采集失败", + "cancelled": "已取消", +} + +_worker_started = False +_worker_lock = threading.Lock() +_active_jobs: set[str] = set() +_feishu_client = FeishuCapabilityClient(timeout=30) + + +class AccountReviewRequired(RuntimeError): + pass + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _iso(value: datetime | None = None) -> str: + return (value or _utc_now()).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _clean_text(value: Any, limit: int = 2000) -> str: + return str(value or "").strip()[:limit] + + +def _json_loads(value: Any, fallback: Any) -> Any: + try: + return json.loads(str(value or "")) + except Exception: + return fallback + + +def _validate_daily_time(value: Any) -> str: + raw = _clean_text(value, 5) + if not re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", raw): + raise ValueError("每日采集时间必须为 HH:MM") + return raw + + +def _local_today() -> str: + return _utc_now().astimezone(ZoneInfo(TIMEZONE_NAME)).date().isoformat() + + +def _validate_publish_range(start_value: Any, end_value: Any) -> tuple[str, str]: + start = _clean_text(start_value, 10) or _local_today() + end = _clean_text(end_value, 10) or start + for label, value in (("开始", start), ("结束", end)): + try: + datetime.strptime(value, "%Y-%m-%d") + except ValueError as exc: + raise ValueError(f"发布{label}日期必须为 YYYY-MM-DD") from exc + if end < start: + raise ValueError("发布结束日期不能早于开始日期") + return start, end + + +def _validate_date_rule(value: Any) -> str: + rule = _clean_text(value, 30) or DEFAULT_DATE_RULE + if rule not in DATE_RULES: + raise ValueError("自动采集日期规则无效") + return rule + + +def _automatic_publish_range(rule: str, local_date: Any) -> tuple[str, str]: + target_date = local_date - timedelta(days=1) if rule == "previous_day" else local_date + value = target_date.isoformat() + return value, value + + +def _setting_row(row: Any | None, account_id: int) -> dict[str, Any]: + return { + "account_id": account_id, + "enabled": bool(row["enabled"]) if row else False, + "daily_time": str(row["daily_time"]) if row else DEFAULT_DAILY_TIME, + "date_rule": _validate_date_rule(row["date_rule"] if row else DEFAULT_DATE_RULE), + "feishu_target": _json_loads(row["feishu_target_json"], {}) if row else {}, + "last_scheduled_date": str(row["last_scheduled_date"]) if row else "", + "timezone": TIMEZONE_NAME, + "updated_at": str(row["updated_at"]) if row else "", + } + + +def _job_row(row: Any) -> dict[str, Any]: + columns = set(row.keys()) if hasattr(row, "keys") else set() + return { + "id": str(row["id"]), + "account_id": int(row["account_id"]), + "proxy_profile_id": int(row["proxy_profile_id"]), + "trigger_type": str(row["trigger_type"]), + "schedule_date": str(row["schedule_date"]), + "max_videos": int(row["max_videos"]), + "publish_date_start": str(row["publish_date_start"]), + "publish_date_end": str(row["publish_date_end"]), + "feishu_target": _json_loads(row["feishu_target_json"], {}), + "status": str(row["status"]), + "status_label": STATUS_LABELS.get(str(row["status"]), str(row["status"])), + "stage": str(row["stage"]), + "status_detail": str(row["status_detail"]), + "attempt_count": int(row["attempt_count"]), + "session_id": int(row["session_id"] or 0), + "total_videos": int(row["total_videos"]), + "completed_videos": int(row["completed_videos"]), + "failed_videos": int(row["failed_videos"]), + "current_video_id": str(row["current_video_id"]), + "started_at": str(row["started_at"]), + "completed_at": str(row["completed_at"]), + "last_error": str(row["last_error"]), + "feishu_failed_results": int(row["feishu_failed_results"] or 0) if "feishu_failed_results" in columns else 0, + "created_at": str(row["created_at"]), + "updated_at": str(row["updated_at"]), + } + + +def _result_row(row: Any) -> dict[str, Any]: + payload = _json_loads(row["payload_json"], {}) + return { + "id": int(row["id"]), + "job_id": str(row["job_id"]), + "account_id": int(row["account_id"]), + "video_id": str(row["video_id"]), + "video_url": str(row["video_url"]), + "title": str(row["title"]), + "published_at": str(row["published_at"]), + "collected_at": str(row["collected_at"]), + "retention_complete": bool(row["retention_complete"]), + "feishu_target": _json_loads(row["feishu_target_json"], {}), + "feishu_record_id": str(row["feishu_record_id"]), + "feishu_sync_status": str(row["feishu_sync_status"]), + "feishu_sync_error": str(row["feishu_sync_error"]), + "feishu_synced_at": str(row["feishu_synced_at"]), + "payload": payload, + } + + +def list_feishu_targets() -> dict[str, Any]: + return _feishu_client.list_bitable_targets() + + +def _target_key(target: dict[str, Any]) -> str: + app = _clean_text(target.get("appToken") or target.get("wikiToken"), 200) + table = _clean_text(target.get("tableId"), 200) + return f"{app}:{table}" if app and table else "" + + +def _job_feishu_target(job: dict[str, Any]) -> dict[str, Any]: + target = job.get("feishu_target") + if isinstance(target, str): + target = _json_loads(target, {}) + if isinstance(target, dict) and _target_key(target): + return target + target = _json_loads(job.get("feishu_target_json"), {}) + return target if isinstance(target, dict) else {} + + +def _validate_feishu_target(value: Any) -> dict[str, Any]: + requested = value if isinstance(value, dict) else _json_loads(value, {}) + key = _target_key(requested) + if not key: + raise ValueError("请选择采集结果要写入的飞书多维表格") + targets = list_feishu_targets().get("targets") or [] + matched = next((item for item in targets if isinstance(item, dict) and _target_key(item) == key), None) + if not matched: + raise ValueError("选择的多维表格不在当前写入白名单中") + return { + "appToken": _clean_text(matched.get("appToken"), 200), + "appName": _clean_text(matched.get("appName"), 200), + "wikiToken": _clean_text(matched.get("wikiToken"), 200), + "tableId": _clean_text(matched.get("tableId"), 200), + "tableName": _clean_text(matched.get("tableName"), 200), + "url": _clean_text(matched.get("url"), 1000), + "fields": [ + { + "id": _clean_text(field.get("id"), 200), + "name": _clean_text(field.get("name"), 200), + "type": int(field.get("type") or 0), + "uiType": _clean_text(field.get("uiType"), 100), + "isPrimary": bool(field.get("isPrimary")), + } + for field in matched.get("fields") or [] + if isinstance(field, dict) and _clean_text(field.get("name"), 200) + ], + } + + +def _account(conn: Any, account_id: int) -> Any: + row = conn.execute( + "SELECT * FROM tiktok_accounts WHERE id = ? AND deleted_at = ''", + (account_id,), + ).fetchone() + if not row: + raise ValueError("account not found") + return row + + +def dashboard(account_id: int) -> dict[str, Any]: + if not account_id: + raise ValueError("account_id is required") + with proxy_pool.connect() as conn: + account = _account(conn, account_id) + setting = conn.execute("SELECT * FROM collect_settings WHERE account_id = ?", (account_id,)).fetchone() + jobs = [ + _job_row(row) + for row in conn.execute( + """ + SELECT j.*, + (SELECT COUNT(*) FROM collect_results r + WHERE r.job_id = j.id AND r.feishu_sync_status = 'failed') AS feishu_failed_results + FROM collect_jobs j + WHERE j.account_id = ? + ORDER BY j.created_at DESC + LIMIT 40 + """, + (account_id,), + ).fetchall() + ] + results = [ + _result_row(row) + for row in conn.execute( + "SELECT * FROM collect_results WHERE account_id = ? ORDER BY collected_at DESC, id DESC LIMIT 100", + (account_id,), + ).fetchall() + ] + errors = [ + dict(row) + for row in conn.execute( + "SELECT * FROM collect_errors WHERE account_id = ? ORDER BY created_at DESC, id DESC LIMIT 30", + (account_id,), + ).fetchall() + ] + setting_data = _setting_row(setting, account_id) + if not _target_key(setting_data.get("feishu_target") or {}): + last_target = next( + (job.get("feishu_target") for job in jobs if _target_key(job.get("feishu_target") or {})), + {}, + ) + setting_data["feishu_target"] = last_target + return { + "account": {"id": int(account["id"]), "username": str(account["username"])}, + "setting": setting_data, + "jobs": jobs, + "results": results, + "errors": errors, + "worker": runtime_status(), + } + + +def save_settings(payload: dict[str, Any]) -> dict[str, Any]: + account_id = int(payload.get("account_id") or 0) + if not account_id: + raise ValueError("account_id is required") + enabled = 1 if payload.get("enabled") else 0 + daily_time = _validate_daily_time(payload.get("daily_time") or DEFAULT_DAILY_TIME) + date_rule = _validate_date_rule(payload.get("date_rule")) + feishu_target = _validate_feishu_target(payload.get("feishu_target")) + feishu_target_json = json.dumps(feishu_target, ensure_ascii=False, separators=(",", ":")) + now = _iso() + with proxy_pool.connect() as conn: + account = _account(conn, account_id) + proxy_pool.require_account_proxy_bound(account) + conn.execute( + """ + INSERT INTO collect_settings ( + account_id, enabled, daily_time, max_videos, date_rule, publish_date_start, publish_date_end, + feishu_target_json, last_scheduled_date, created_at, updated_at + ) VALUES (?, ?, ?, 0, ?, '', '', ?, '', ?, ?) + ON CONFLICT(account_id) DO UPDATE SET + enabled = excluded.enabled, + daily_time = excluded.daily_time, + max_videos = 0, + date_rule = excluded.date_rule, + feishu_target_json = excluded.feishu_target_json, + updated_at = excluded.updated_at + """, + ( + account_id, enabled, daily_time, date_rule, feishu_target_json, now, now, + ), + ) + conn.commit() + return dashboard(account_id) + + +def _insert_job( + conn: Any, + account: Any, + trigger_type: str, + publish_date_start: str, + publish_date_end: str, + feishu_target_json: str, + schedule_date: str = "", + session_id: int = 0, +) -> str: + job_id = f"collect_{uuid.uuid4().hex}" + now = _iso() + conn.execute( + """ + INSERT INTO collect_jobs ( + id, account_id, proxy_profile_id, trigger_type, schedule_date, max_videos, + publish_date_start, publish_date_end, feishu_target_json, + status, stage, attempt_count, next_attempt_at, session_id, + total_videos, completed_videos, failed_videos, current_video_id, + started_at, completed_at, last_error, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', '', 0, '', ?, 0, 0, 0, '', '', '', '', ?, ?) + """, + ( + job_id, + int(account["id"]), + int(account["proxy_profile_id"]), + trigger_type, + schedule_date, + 0, + publish_date_start, + publish_date_end, + feishu_target_json, + session_id or None, + now, + now, + ), + ) + return job_id + + +def create_job(payload: dict[str, Any]) -> dict[str, Any]: + account_id = int(payload.get("account_id") or 0) + if not account_id: + raise ValueError("account_id is required") + with proxy_pool.connect() as conn: + account = _account(conn, account_id) + proxy_pool.require_account_proxy_bound(account) + publish_date_start, publish_date_end = _validate_publish_range( + payload.get("publish_date_start"), payload.get("publish_date_end") + ) + feishu_target = _validate_feishu_target(payload.get("feishu_target")) + feishu_target_json = json.dumps(feishu_target, ensure_ascii=False, separators=(",", ":")) + job_id = _insert_job( + conn, + account, + "manual", + publish_date_start, + publish_date_end, + feishu_target_json, + session_id=int(payload.get("observation_session_id") or 0), + ) + now = _iso() + conn.execute( + """ + INSERT INTO collect_settings (account_id, feishu_target_json, created_at, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(account_id) DO UPDATE SET + feishu_target_json = excluded.feishu_target_json, + updated_at = excluded.updated_at + """, + (account_id, feishu_target_json, now, now), + ) + conn.commit() + data = dashboard(account_id) + data["job"] = next(job for job in data["jobs"] if job["id"] == job_id) + return data + + +def retry_job(payload: dict[str, Any]) -> dict[str, Any]: + job_id = _clean_text(payload.get("job_id") or payload.get("id"), 80) + if not job_id: + raise ValueError("job_id is required") + with proxy_pool.connect() as conn: + row = conn.execute("SELECT * FROM collect_jobs WHERE id = ?", (job_id,)).fetchone() + if not row: + raise ValueError("collect job not found") + account = _account(conn, int(row["account_id"])) + proxy_pool.require_account_proxy_bound(account) + if str(row["status"]) not in JOB_RETRYABLE_STATUSES: + raise ValueError("只有失败、部分失败或已取消的采集任务可以重试") + now = _iso() + conn.execute( + """ + UPDATE collect_jobs + SET status = 'queued', stage = 'retry_queued', attempt_count = 0, + status_detail = '', next_attempt_at = '', session_id = ?, current_video_id = '', + completed_at = '', last_error = '', updated_at = ? + WHERE id = ? + """, + (int(payload.get("observation_session_id") or 0) or None, now, job_id), + ) + conn.commit() + account_id = int(row["account_id"]) + return dashboard(account_id) + + +def cancel_job(payload: dict[str, Any]) -> dict[str, Any]: + job_id = _clean_text(payload.get("job_id") or payload.get("id"), 80) + if not job_id: + raise ValueError("job_id is required") + with proxy_pool.connect() as conn: + row = conn.execute("SELECT * FROM collect_jobs WHERE id = ?", (job_id,)).fetchone() + if not row: + raise ValueError("collect job not found") + if str(row["status"]) not in {"queued", "delayed"}: + raise ValueError("只能取消尚未开始的采集任务") + conn.execute( + "UPDATE collect_jobs SET status = 'cancelled', stage = '', status_detail = '', completed_at = ?, updated_at = ? WHERE id = ?", + (_iso(), _iso(), job_id), + ) + conn.commit() + account_id = int(row["account_id"]) + return dashboard(account_id) + + +def runtime_status() -> dict[str, Any]: + with proxy_pool.connect() as conn: + counts = { + str(row["status"]): int(row["count"]) + for row in conn.execute("SELECT status, COUNT(*) AS count FROM collect_jobs GROUP BY status").fetchall() + } + with _worker_lock: + active = sorted(_active_jobs) + return { + "worker_started": _worker_started, + "timezone": TIMEZONE_NAME, + "active_jobs": active, + "counts": counts, + "max_automatic_slots": proxy_pool.browser_max_slots(), + "retention_max_seconds": RETENTION_MAX_SECONDS, + } + + +def _set_job(job_id: str, status: str, stage: str = "", error: str = "", **values: Any) -> None: + status_detail = _clean_text(values.pop("status_detail", ""), 500) + fields = ["status = ?", "stage = ?", "status_detail = ?", "last_error = ?", "updated_at = ?"] + params: list[Any] = [status, stage, status_detail, _clean_text(error), _iso()] + allowed = { + "session_id", "total_videos", "completed_videos", "failed_videos", + "current_video_id", "started_at", "completed_at", "next_attempt_at", + } + for key, value in values.items(): + if key in allowed: + fields.append(f"{key} = ?") + params.append(value) + params.append(job_id) + with proxy_pool.connect() as conn: + conn.execute(f"UPDATE collect_jobs SET {', '.join(fields)} WHERE id = ?", params) + conn.commit() + + +def _record_error(job: dict[str, Any], video_id: str, video_url: str, stage: str, error: Exception | str) -> None: + with proxy_pool.connect() as conn: + conn.execute( + "INSERT INTO collect_errors (job_id, account_id, video_id, video_url, stage, message, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + job["id"], + job["account_id"], + _clean_text(video_id, 120), + _clean_text(video_url, 1000), + _clean_text(stage, 120), + _clean_text(error), + _iso(), + ), + ) + conn.commit() + + +def _metric_number(value: Any) -> int | float | None: + raw = _clean_text(value, 80).replace(",", "") + match = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)([KMB]?)", raw, re.I) + if not match: + return None + number = float(match.group(1)) + number *= {"": 1, "K": 1_000, "M": 1_000_000, "B": 1_000_000_000}[match.group(2).upper()] + return int(number) if number.is_integer() else number + + +def _datetime_millis(value: Any) -> int | None: + raw = _clean_text(value, 120) + if not raw: + return None + if raw.isdigit() and len(raw) >= 10: + number = int(raw) + return number if len(raw) >= 13 else number * 1000 + parsed: datetime | None = None + try: + parsed = datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError: + for pattern in ("%m/%d/%Y", "%m/%d/%y", "%Y-%m-%d"): + try: + parsed = datetime.strptime(raw, pattern) + break + except ValueError: + continue + if not parsed: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=ZoneInfo(TIMEZONE_NAME)) + return int(parsed.timestamp() * 1000) + + +def _feishu_fields(target: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]: + account = payload.get("account") or {} + video = payload.get("video") or {} + overview = payload.get("overview") or {} + engagement = payload.get("engagement") or {} + retention = payload.get("retention") or {} + available = {str(field.get("name") or ""): field for field in target.get("fields") or []} + values = { + "账号名称": account.get("username") or "", + "发布时间": video.get("published_at") or "", + "24小时播放量": _metric_number(overview.get("play_count")), + "视频完播率": overview.get("completion_rate") or "", + "头3秒播放率": retention.get("0:03") or "", + "头6秒播放率": retention.get("0:06") or "", + "点赞": _metric_number(engagement.get("likes")), + "评论": _metric_number(engagement.get("comments")), + "收藏": _metric_number(engagement.get("favorites")), + "解析状态": "已解析" if payload.get("retention_complete") else "需人工确认", + } + mapped: dict[str, Any] = {} + for name, value in values.items(): + definition = available.get(name) + if not definition or value is None or value == "": + continue + field_type = int(definition.get("type") or 0) + if field_type == 5: + value = _datetime_millis(value) + elif field_type == 1: + value = str(value) + if value is not None and value != "": + mapped[name] = value + if not mapped: + raise ValueError("目标多维表格没有可映射的视频统计字段") + return mapped + + +def _set_result_sync(result_id: int, status: str, record_id: str = "", error: str = "") -> None: + with proxy_pool.connect() as conn: + conn.execute( + """ + UPDATE collect_results + SET feishu_sync_status = ?, feishu_record_id = COALESCE(NULLIF(?, ''), feishu_record_id), + feishu_sync_error = ?, feishu_synced_at = ? + WHERE id = ? + """, + (status, record_id, _clean_text(error), _iso() if status == "synced" else "", result_id), + ) + conn.commit() + + +def _sync_result_to_feishu(result_id: int, job: dict[str, Any], payload: dict[str, Any]) -> None: + target = _job_feishu_target(job) + try: + fields = _feishu_fields(target, payload) + request = { + "appToken": target.get("appToken"), + "wikiToken": target.get("wikiToken"), + "tableId": target.get("tableId"), + "fields": fields, + } + with proxy_pool.connect() as conn: + current = conn.execute( + "SELECT feishu_record_id FROM collect_results WHERE id = ?", + (result_id,), + ).fetchone() + previous = conn.execute( + """ + SELECT feishu_record_id, feishu_target_json + FROM collect_results + WHERE account_id = ? AND video_id = ? AND id <> ? AND feishu_record_id <> '' + ORDER BY collected_at DESC, id DESC + """, + (int(job["account_id"]), _clean_text((payload.get("video") or {}).get("id"), 120), result_id), + ).fetchall() + record_id = _clean_text(current["feishu_record_id"], 200) if current else "" + record_id = record_id or next( + ( + str(row["feishu_record_id"]) + for row in previous + if _target_key(_json_loads(row["feishu_target_json"], {})) == _target_key(target) + ), + "", + ) + _set_result_sync(result_id, "syncing", record_id) + if record_id: + request["recordId"] = record_id + result = _feishu_client.update_bitable_record(request) + else: + result = _feishu_client.create_bitable_record(request) + record_id = _clean_text(result.get("recordId"), 200) + _set_result_sync(result_id, "synced", record_id) + except (FeishuCapabilityError, ValueError, TypeError) as exc: + _set_result_sync(result_id, "failed", error=str(exc)) + + +def _save_result(job: dict[str, Any], payload: dict[str, Any]) -> None: + video = payload.get("video") or {} + target_json = json.dumps( + _job_feishu_target(job), ensure_ascii=False, separators=(",", ":") + ) + with proxy_pool.connect() as conn: + conn.execute( + """ + INSERT INTO collect_results ( + job_id, account_id, video_id, video_url, title, published_at, + collected_at, retention_complete, payload_json, feishu_target_json, + feishu_sync_status, feishu_sync_error, feishu_synced_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', '', '') + ON CONFLICT(job_id, video_id) DO UPDATE SET + video_url = excluded.video_url, + title = excluded.title, + published_at = excluded.published_at, + collected_at = excluded.collected_at, + retention_complete = excluded.retention_complete, + payload_json = excluded.payload_json, + feishu_target_json = excluded.feishu_target_json, + feishu_sync_status = 'pending', + feishu_sync_error = '', + feishu_synced_at = '' + """, + ( + job["id"], + job["account_id"], + _clean_text(video.get("id"), 120), + _clean_text(video.get("url"), 1000), + _clean_text(video.get("title"), 2000), + _clean_text(video.get("published_at"), 120), + payload["collected_at"], + 1 if payload.get("retention_complete") else 0, + json.dumps(payload, ensure_ascii=False, separators=(",", ":")), + target_json, + ), + ) + result = conn.execute( + "SELECT id FROM collect_results WHERE job_id = ? AND video_id = ?", + (job["id"], _clean_text(video.get("id"), 120)), + ).fetchone() + conn.commit() + if result: + _sync_result_to_feishu(int(result["id"]), job, payload) + + +def retry_failed_feishu_sync(payload: dict[str, Any]) -> dict[str, Any]: + job_id = _clean_text(payload.get("job_id"), 80) + account_id = int(payload.get("account_id") or 0) + raw_result_ids = payload.get("result_ids") or [] + if not isinstance(raw_result_ids, list): + raw_result_ids = [raw_result_ids] + if payload.get("result_id"): + raw_result_ids = [payload.get("result_id"), *raw_result_ids] + try: + result_ids = sorted({int(value) for value in raw_result_ids if int(value) > 0}) + except (TypeError, ValueError) as exc: + raise ValueError("result_id must be a positive integer") from exc + if not job_id and not account_id and not result_ids: + raise ValueError("job_id, account_id or result_id is required") + clauses = ["r.feishu_sync_status = 'failed'"] + params: list[Any] = [] + if job_id: + clauses.append("r.job_id = ?") + params.append(job_id) + if account_id: + clauses.append("r.account_id = ?") + params.append(account_id) + if result_ids: + clauses.append(f"r.id IN ({','.join('?' for _ in result_ids)})") + params.extend(result_ids) + with proxy_pool.connect() as conn: + rows = conn.execute( + f""" + SELECT r.id, r.account_id, r.payload_json, r.feishu_target_json, + j.feishu_target_json AS job_target_json + FROM collect_results r + JOIN collect_jobs j ON j.id = r.job_id + WHERE {' AND '.join(clauses)} + ORDER BY r.id + LIMIT 200 + """, + params, + ).fetchall() + outcomes: list[dict[str, Any]] = [] + for row in rows: + target = _json_loads(row["job_target_json"], {}) + if not isinstance(target, dict) or not _target_key(target): + target = _json_loads(row["feishu_target_json"], {}) + if not isinstance(target, dict): + target = {} + result_id = int(row["id"]) + with proxy_pool.connect() as conn: + conn.execute( + """ + UPDATE collect_results + SET feishu_target_json = ?, feishu_sync_status = 'pending', + feishu_sync_error = '', feishu_synced_at = '' + WHERE id = ? + """, + (json.dumps(target, ensure_ascii=False, separators=(",", ":")), result_id), + ) + conn.commit() + _sync_result_to_feishu( + result_id, + {"account_id": int(row["account_id"]), "feishu_target": target}, + _json_loads(row["payload_json"], {}), + ) + with proxy_pool.connect() as conn: + updated = conn.execute( + """ + SELECT id, feishu_sync_status, feishu_record_id, + feishu_sync_error, feishu_synced_at + FROM collect_results WHERE id = ? + """, + (result_id,), + ).fetchone() + outcomes.append(dict(updated)) + return { + "attempted": len(outcomes), + "synced": sum(1 for item in outcomes if item["feishu_sync_status"] == "synced"), + "failed": sum(1 for item in outcomes if item["feishu_sync_status"] == "failed"), + "results": outcomes, + } + + +def _first_visible(locators: list[Any]) -> Any | None: + for locator in locators: + try: + for index in range(min(locator.count(), 10)): + item = locator.nth(index) + if item.is_visible(): + return item + except Exception: + continue + return None + + +def _assert_account_ready(page: Any) -> None: + if "/login" in page.url.lower(): + raise AccountReviewRequired("TikTok 登录已失效，请先从观测通道重新登录") + challenge = _first_visible([ + page.get_by_text(re.compile(r"captcha|verify to continue|security verification|验证码|安全验证", re.I)), + page.locator("iframe[src*='captcha']"), + ]) + if challenge: + raise AccountReviewRequired("TikTok 要求验证码或安全验证，请从观测通道人工处理") + + +def _skip_onboarding(page: Any) -> None: + pattern = re.compile(r"^(skip|skip for now|not now|got it|later|跳过|暂不|稍后|知道了)$", re.I) + for _ in range(4): + button = _first_visible([page.get_by_role("button", name=pattern), page.get_by_text(pattern, exact=True)]) + if not button: + return + button.click(timeout=3000) + page.wait_for_timeout(400) + + +def _video_id(url: str) -> str: + match = re.search(r"/(?:analytics|video)/(\d+)", url) + return match.group(1) if match else "" + + +_MONTH_NUMBERS = { + "jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6, + "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12, +} + + +def _source_published_date(hint: str, range_start: str, range_end: str) -> str: + text = " ".join(_lines(hint)) + start = datetime.strptime(range_start, "%Y-%m-%d").date() + end = datetime.strptime(range_end, "%Y-%m-%d").date() + + match = re.search(r"\b(20\d{2})[/-](\d{1,2})[/-](\d{1,2})\b", text) + if match: + try: + return datetime(int(match.group(1)), int(match.group(2)), int(match.group(3))).date().isoformat() + except ValueError: + return "" + + numeric = re.search(r"\b(\d{1,2})/(\d{1,2})(?:/(20\d{2}))?\b", text) + month_name = re.search( + r"\b(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|" + r"Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|" + r"Dec(?:ember)?)\s+(\d{1,2})(?:,\s*(20\d{2}))?\b", + text, + re.I, + ) + if numeric: + month, day = int(numeric.group(1)), int(numeric.group(2)) + explicit_year = int(numeric.group(3)) if numeric.group(3) else 0 + elif month_name: + month = _MONTH_NUMBERS[month_name.group(1)[:3].lower()] + day = int(month_name.group(2)) + explicit_year = int(month_name.group(3)) if month_name.group(3) else 0 + else: + return "" + + years = [explicit_year] if explicit_year else sorted({start.year, end.year}) + candidates = [] + for year in years: + try: + candidates.append(datetime(year, month, day).date()) + except ValueError: + continue + if not candidates: + return "" + chosen = min( + candidates, + key=lambda value: 0 if start <= value <= end else min(abs((value - start).days), abs((value - end).days)), + ) + return chosen.isoformat() + + +def _discover_links_on_page(page: Any) -> list[dict[str, str]]: + rows: list[dict[str, str]] = [] + links = page.locator("a[href*='/tiktokstudio/analytics/'], a[href*='/video/']") + for index in range(min(links.count(), 200)): + link = links.nth(index) + try: + href = str(link.get_attribute("href") or "") + if not href: + continue + absolute_href = urljoin(page.url, href) + video_id = _video_id(absolute_href) + if not video_id: + continue + text = _clean_text(link.inner_text(), 2000) + if "/video/" in absolute_href: + for levels in ("..", "../..", "../../..", "../../../..", "../../../../..", "../../../../../.."): + try: + candidate = _clean_text(link.locator(f"xpath={levels}").inner_text(), 2000) + if len(candidate) <= 1000 and re.search(r"(?m)^\d{1,2}:\d{2}$", candidate): + text = candidate + break + except Exception: + continue + elif not text: + for levels in ("..", "../..", "../../.."): + try: + text = _clean_text(link.locator(f"xpath={levels}").inner_text(), 2000) + if text: + break + except Exception: + continue + analytics_url = absolute_href if "/tiktokstudio/analytics/" in absolute_href else urljoin( + page.url, f"/tiktokstudio/analytics/{video_id}" + ) + rows.append({"id": video_id, "url": analytics_url, "title_hint": text}) + except Exception: + continue + return rows + + +def _discover_video_links( + page: Any, publish_date_start: str, publish_date_end: str +) -> list[dict[str, str]]: + found: dict[str, dict[str, str]] = {} + + def collect() -> None: + for row in _discover_links_on_page(page): + row["published_date"] = _source_published_date( + row.get("title_hint", ""), publish_date_start, publish_date_end + ) + existing = found.get(row["id"]) + if not existing or (not existing.get("published_date") and row["published_date"]): + found[row["id"]] = row + + def matching() -> list[dict[str, str]]: + return [ + row for row in found.values() + if publish_date_start <= row.get("published_date", "") <= publish_date_end + ] + + current_path = urlparse(page.url).path + if not (current_path.startswith("/tiktokstudio/content") or current_path.startswith("/tiktokstudio/manage")): + content_link = _first_visible([ + page.locator("a[href*='/tiktokstudio/content']"), + page.locator("a[href*='/tiktokstudio/manage']"), + page.get_by_role("link", name=re.compile(r"content|posts|manage|内容|作品", re.I)), + page.get_by_role("button", name=re.compile(r"^recent posts$|^posts$|最近作品|近期作品", re.I)), + ]) + if content_link: + href = str(content_link.get_attribute("href") or "") + if href: + page.goto(urljoin(page.url, href), wait_until="domcontentloaded", timeout=60000) + else: + content_link.click(timeout=5000) + page.wait_for_timeout(1800) + _assert_account_ready(page) + + unchanged_rounds = 0 + for _ in range(100): + before = len(found) + collect() + dated_rows = [row["published_date"] for row in found.values() if row.get("published_date")] + if dated_rows and min(dated_rows) < publish_date_start: + break + unchanged_rounds = unchanged_rounds + 1 if len(found) == before else 0 + if unchanged_rounds >= 3: + break + page.mouse.wheel(0, 1000) + page.wait_for_timeout(700) + + return matching() + + +def _lines(text: str) -> list[str]: + return [re.sub(r"\s+", " ", line).strip() for line in str(text or "").splitlines() if line.strip()] + + +def _value_after_label(lines: list[str], labels: list[str]) -> str: + label_pattern = re.compile(r"^(?:" + "|".join(re.escape(label) for label in labels) + r")$", re.I) + inline_pattern = re.compile(r"^(?:" + "|".join(re.escape(label) for label in labels) + r")\s*[:：]?\s+(.+)$", re.I) + for index, line in enumerate(lines): + inline = inline_pattern.match(line) + if inline: + return inline.group(1).strip() + if label_pattern.match(line): + for candidate in lines[index + 1:index + 5]: + if not label_pattern.match(candidate): + return candidate + return "" + + +def _percent_section(lines: list[str], headings: list[str], stop_headings: list[str]) -> dict[str, str]: + start = -1 + heading_pattern = re.compile(r"^(?:" + "|".join(re.escape(item) for item in headings) + r")$", re.I) + stop_pattern = re.compile(r"^(?:" + "|".join(re.escape(item) for item in stop_headings) + r")$", re.I) if stop_headings else None + for index, line in enumerate(lines): + if heading_pattern.match(line): + start = index + 1 + break + if start < 0: + return {} + section: list[str] = [] + for line in lines[start:start + 80]: + if stop_pattern and stop_pattern.match(line): + break + section.append(line) + values: dict[str, str] = {} + percent_pattern = re.compile(r"^ str: + pattern = re.compile("|".join(re.escape(name) for name in names), re.I) + target = _first_visible([ + page.get_by_text(pattern, exact=False), + page.locator("[aria-label]").filter(has_text=pattern), + ]) + if not target: + return "" + for ancestor in [target, target.locator("xpath=.."), target.locator("xpath=../..")]: + try: + text = _clean_text(ancestor.inner_text(), 300) + numbers = re.findall(r"(?:\d[\d,.]*[KMB]?|<\d+(?:\.\d+)?%)", text, re.I) + if numbers: + return numbers[-1] + except Exception: + continue + return "" + + +def _overview(lines: list[str]) -> dict[str, str]: + return { + "play_count": _value_after_label(lines, ["Video views", "Views", "播放量"]), + "total_play_time": _value_after_label(lines, ["Total play time", "总播放时间"]), + "average_watch_time": _value_after_label(lines, ["Average watch time", "平均观看时间"]), + "completion_rate": _value_after_label(lines, ["Watched full video", "Full video watched", "已观看完整视频", "完播率"]), + "new_followers": _value_after_label(lines, ["New followers", "新增粉丝"]), + } + + +def _engagement(page: Any, lines: list[str], overview: dict[str, str]) -> dict[str, str]: + values: list[str] = [] + for index, line in enumerate(lines): + if not re.match(r"^(?:Posted on|发布于)", line, re.I): + continue + for candidate in lines[index + 1:index + 12]: + if re.match(r"^(?:Video views|播放量)$", candidate, re.I): + break + if re.fullmatch(r"\d[\d,.]*[KMB]?", candidate, re.I): + values.append(candidate) + if len(values) >= 5: + break + break + if len(values) >= 5: + return dict(zip(("play", "likes", "comments", "shares", "favorites"), values[:5])) + return { + "play": overview.get("play_count", ""), + "likes": _locator_metric(page, ["Likes", "Like", "点赞"]), + "comments": _locator_metric(page, ["Comments", "Comment", "评论"]), + "shares": _locator_metric(page, ["Shares", "Share", "分享"]), + "favorites": _locator_metric(page, ["Favorites", "Favorite", "Saves", "收藏"]), + } + + +def _duration_seconds(text: str) -> int: + values: list[int] = [] + for minutes, seconds in re.findall(r"\b(\d+):(\d{2})\b(?!\s*(?:AM|PM)\b)", text, re.I): + values.append(int(minutes) * 60 + int(seconds)) + return max(values) if values else 0 + + +def _tooltip_value(text: str) -> tuple[int, str] | None: + match = re.search(r"(\d+):(\d{2})\s*(\d+(?:\.\d+)?%)", str(text or "")) + if not match: + return None + return int(match.group(1)) * 60 + int(match.group(2)), match.group(3) + + +def _retention_chart(page: Any) -> tuple[Any | None, int, str]: + charts = page.locator(".echarts-for-react") + fallback: tuple[Any | None, int, str] = (None, 0, "未找到留存率图表") + for index in range(min(charts.count(), 12)): + chart = charts.nth(index) + try: + if not chart.is_visible(): + continue + chart.scroll_into_view_if_needed(timeout=3000) + page.wait_for_timeout(250) + box = chart.bounding_box() + if not box or box["width"] < 280 or box["height"] > 120: + continue + context_text = "" + for levels in ("..", "../..", "../../..", "../../../.."): + try: + context_text = _clean_text(chart.locator(f"xpath={levels}").inner_text(), 5000) + if re.search(r"retention rate|观众留存|留存率", context_text, re.I): + break + except Exception: + continue + duration = _duration_seconds(context_text) + candidate = (chart, duration, "") + if re.search(r"retention rate|观众留存|留存率", context_text, re.I): + return candidate + fallback = candidate + except Exception: + continue + return fallback + + +def _sample_retention(page: Any, duration_hint: int = 0) -> tuple[dict[str, str], bool, list[str], str]: + chart, duration, reason = _retention_chart(page) + if not chart: + return {}, False, [], reason + duration = max(duration, duration_hint) + if duration <= 0: + return {}, False, [], "留存率图表没有可识别的视频时长" + if duration > RETENTION_MAX_SECONDS: + return {}, False, [], f"视频时长 {duration} 秒超过逐秒采集上限 {RETENTION_MAX_SECONDS} 秒" + box = chart.bounding_box() + if not box: + return {}, False, [], "留存率图表不可见" + plot_left = box["x"] + 10 + plot_width = max(1.0, box["width"] - 80) + y = box["y"] + box["height"] / 2 + rows: dict[int, str] = {} + targets = range(duration + 1) + + def read() -> tuple[int, str] | None: + try: + return _tooltip_value(chart.text_content() or "") + except Exception: + return None + + for second in targets: + x = plot_left + plot_width * second / max(1, duration) + page.mouse.move(x, y) + page.wait_for_timeout(85) + parsed = read() + if parsed and parsed[0] == second: + rows[second] = parsed[1] + + missing = [second for second in targets if second not in rows] + for second in missing: + target_x = plot_left + plot_width * second / max(1, duration) + offsets = list(range(-36, 37, 6)) + if second in {0, duration}: + offsets += list(range(-60, 61, 4)) + seen: set[int] = set() + for offset in offsets: + if offset in seen: + continue + seen.add(offset) + page.mouse.move(target_x + offset, y) + page.wait_for_timeout(60) + parsed = read() + if parsed and parsed[0] == second: + rows[second] = parsed[1] + break + + unresolved = {second for second in targets if second not in rows} + if unresolved: + scan_left = int(box["x"]) + scan_right = int(box["x"] + box["width"]) + for x in range(scan_left, scan_right + 1, 3): + page.mouse.move(x, y) + page.wait_for_timeout(55) + parsed = read() + if parsed and parsed[0] in unresolved: + rows[parsed[0]] = parsed[1] + unresolved.discard(parsed[0]) + if not unresolved: + break + + missing_labels = [f"{second // 60}:{second % 60:02d}" for second in targets if second not in rows] + output = {f"{second // 60}:{second % 60:02d}": rows[second] for second in sorted(rows)} + return output, not missing_labels, missing_labels, "" if not missing_labels else "部分秒点未命中 ECharts tooltip" + + +def _title_and_date(lines: list[str], hint: str) -> tuple[str, str]: + cleaned_hint = _lines(hint) + title = "" + date_pattern = re.compile(r"(?:[A-Z][a-z]{2}\s+\d{1,2},\s+\d{4}|\d{4}[/-]\d{1,2}[/-]\d{1,2}|\d{1,2}/\d{1,2}/\d{4})") + for candidate in cleaned_hint: + if re.fullmatch(r"\d{1,2}:\d{2}", candidate) or date_pattern.search(candidate): + continue + if candidate.lower() in {"everyone", "friends", "only you"} or re.fullmatch(r"[\d,.]+", candidate): + continue + title = candidate + break + published = "" + for line in cleaned_hint + lines[:80]: + match = date_pattern.search(line) + if match: + published = match.group(0) + break + if not title: + for line in lines[:50]: + if len(line) >= 8 and not re.match(r"^(TikTok Studio|Video analytics|Analytics)$", line, re.I): + title = line + break + return title, published + + +def _collect_video(page: Any, job: dict[str, Any], source: dict[str, str], log_dir: Path) -> dict[str, Any]: + page.goto(source["url"], wait_until="domcontentloaded", timeout=60000) + _assert_account_ready(page) + try: + page.get_by_text(re.compile(r"^(?:Video views|播放量)$", re.I)).first.wait_for( + state="visible", timeout=15000 + ) + page.wait_for_timeout(1200) + except Exception: + page.wait_for_timeout(2200) + _assert_account_ready(page) + body = page.locator("body").inner_text(timeout=15000) + lines = _lines(body) + overview = _overview(lines) + if not any(overview.values()): + page.screenshot(path=str(log_dir / f"{source['id']}-missing-overview.png"), full_page=True) + raise RuntimeError("视频分析页没有识别到概览指标") + title, published_at = _title_and_date(lines, source.get("title_hint", "")) + published_at = published_at or source.get("published_date", "") + retention, retention_complete, missing, retention_reason = _sample_retention( + page, _duration_seconds(source.get("title_hint", "")) + ) + payload = { + "account": { + "id": job["account_id"], + "username": job["username"], + "proxy_profile_id": job["proxy_profile_id"], + "observed_ip": job["observed_ip"], + "browser_session_id": job["session_id"], + }, + "collection_job": {"job_id": job["id"], "trigger_type": job["trigger_type"]}, + "video": {"id": source["id"], "title": title, "published_at": published_at, "url": page.url}, + "time_filter": { + "requested": { + "publish_date_start": job["publish_date_start"], + "publish_date_end": job["publish_date_end"], + "timezone": TIMEZONE_NAME, + }, + "applied": source.get("published_date", ""), + "scope": "video_publish_date", + "applied_successfully": True, + }, + "overview": overview, + "engagement": _engagement(page, lines, overview), + "retention": retention, + "retention_complete": retention_complete, + "missing_retention_seconds": missing, + "retention_reason": retention_reason, + "traffic_sources": _percent_section(lines, ["Traffic source", "Traffic sources", "流量来源"], ["Search queries", "搜索查询"]), + "search_queries": _percent_section(lines, ["Search queries", "搜索查询"], ["Viewer types", "Audience", "观众"]), + "updated_at": _value_after_label(lines, ["Updated", "Last updated", "更新时间"]), + "collected_at": _iso(), + } + page.screenshot(path=str(log_dir / f"{source['id']}-collected.png"), full_page=True) + return payload + + +def _load_job(job_id: str) -> dict[str, Any] | None: + with proxy_pool.connect() as conn: + row = conn.execute( + """ + SELECT j.*, a.username, a.last_checked_ip + FROM collect_jobs j JOIN tiktok_accounts a ON a.id = j.account_id + WHERE j.id = ? + """, + (job_id,), + ).fetchone() + if not row: + return None + job = _job_row(row) + job["username"] = str(row["username"]) + job["observed_ip"] = str(row["last_checked_ip"]) + return job + + +def _completed_video_ids(job_id: str) -> set[str]: + with proxy_pool.connect() as conn: + return { + str(row["video_id"]) + for row in conn.execute("SELECT video_id FROM collect_results WHERE job_id = ?", (job_id,)).fetchall() + } + + +def _execute_browser(job: dict[str, Any], session: dict[str, Any]) -> tuple[int, int, int]: + from playwright.sync_api import sync_playwright + + log_dir = LOG_ROOT / job["id"] + log_dir.mkdir(parents=True, exist_ok=True) + completed_ids = _completed_video_ids(job["id"]) + completed = len(completed_ids) + failed = 0 + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp(f"http://127.0.0.1:{session['debug_port']}") + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() + def page_status(event: dict[str, Any]) -> None: + _set_job( + job["id"], + "preparing", + "loading_video_list", + session_id=session["id"], + status_detail=event.get("message", ""), + ) + + target = "https://www.tiktok.com/tiktokstudio/content?lang=en" + navigate_with_retries(page, target, label="TikTok 视频列表", on_status=page_status) + _skip_onboarding(page) + try: + list_state = wait_for_page_state( + page, + label="TikTok 视频列表", + ready=lambda: bool(_discover_links_on_page(page)), + empty=lambda: bool(_first_visible([ + page.get_by_text(re.compile(r"no posts|no videos|haven't posted|暂无(?:作品|视频)|没有(?:作品|视频)", re.I)), + ])), + allow_ocr_empty=True, + on_status=page_status, + timeout_seconds=75, + reload_attempts=1, + retry_action=lambda: navigate_with_retries( + page, target, label="TikTok 视频列表", on_status=page_status + ), + diagnostic_dir=log_dir, + diagnostic_step="video-list", + ) + except BrowserPageBlocked as exc: + raise AccountReviewRequired(str(exc)) from exc + _assert_account_ready(page) + if list_state.state == "empty": + raise RuntimeError("TikTok Studio 视频列表已加载，但账号当前没有视频") + links = _discover_video_links( + page, + job["publish_date_start"], + job["publish_date_end"], + ) + if not links: + page.screenshot(path=str(log_dir / "no-video-links.png"), full_page=True) + raise RuntimeError( + "TikTok Studio 没有发现发布日期位于 " + f"{job['publish_date_start']} 至 {job['publish_date_end']} 的视频" + ) + _set_job( + job["id"], + "collecting", + "video_list_ready", + session_id=session["id"], + total_videos=len(links), + completed_videos=completed, + failed_videos=0, + ) + for source in links: + with proxy_pool.connect() as conn: + current = conn.execute("SELECT status FROM collect_jobs WHERE id = ?", (job["id"],)).fetchone() + if current and str(current["status"]) == "cancelled": + break + if source["id"] in completed_ids: + continue + _set_job( + job["id"], + "collecting", + "collect_video", + session_id=session["id"], + total_videos=len(links), + completed_videos=completed, + failed_videos=failed, + current_video_id=source["id"], + ) + job["session_id"] = int(session["id"]) + try: + payload = _collect_video(page, job, source, log_dir) + _save_result(job, payload) + completed += 1 + completed_ids.add(source["id"]) + except AccountReviewRequired: + raise + except Exception as exc: + failed += 1 + _record_error(job, source["id"], source["url"], "collect_video", exc) + _set_job( + job["id"], + "collecting", + "collect_video", + session_id=session["id"], + total_videos=len(links), + completed_videos=completed, + failed_videos=failed, + current_video_id="", + ) + return len(links), completed, failed + + +def _update_account(account_id: int, collected_at: str = "", error: str = "") -> None: + with proxy_pool.connect() as conn: + conn.execute( + """ + UPDATE tiktok_accounts + SET last_collect_at = COALESCE(NULLIF(?, ''), last_collect_at), + last_error = ?, updated_at = ? + WHERE id = ? + """, + (collected_at, _clean_text(error), _iso(), account_id), + ) + conn.commit() + + +def _delay_for_proxy(job_id: str, job: dict[str, Any], error: Exception | str) -> None: + message = str(error) + next_attempt_at = _iso(_utc_now() + timedelta(seconds=proxy_pool.PROXY_QUEUE_RECHECK_SECONDS)) + _set_job( + job_id, + "delayed", + "waiting_proxy", + message, + session_id=None, + completed_at="", + next_attempt_at=next_attempt_at, + ) + _update_account(int(job["account_id"]), error=message) + proxy_pool.schedule_proxy_recheck_for_pending_job(int(job["proxy_profile_id"]), message) + + +def _delay_for_page(job_id: str, job: dict[str, Any], error: Exception | str) -> None: + message = str(error) + _set_job( + job_id, + "delayed", + "waiting_page", + message, + session_id=None, + completed_at="", + next_attempt_at=_iso(_utc_now() + timedelta(seconds=60)), + status_detail="页面加载超时，60 秒后自动重试", + ) + _update_account(int(job["account_id"]), error=message) + + +def _run_job(job_id: str) -> None: + session_id = 0 + reused_observation = False + try: + job = _load_job(job_id) + if not job: + return + requested_session_id = int(job.get("session_id") or 0) + session = proxy_pool.claim_observation_session_for_job(job["account_id"], requested_session_id, job_id) + if session is not None: + reused_observation = True + else: + session = proxy_pool.start_automation_session(job["account_id"], job_id)["session"] + session_id = int(session["id"]) + _set_job(job_id, "preparing", "browser_ready", session_id=session_id, started_at=_iso()) + total, completed, failed = _execute_browser(job, session) + status = "complete" if failed == 0 else ("partial" if completed else "failed") + message = "" if failed == 0 else f"{failed} 个视频采集失败" + _set_job( + job_id, + status, + "complete", + message, + session_id=session_id, + total_videos=total, + completed_videos=completed, + failed_videos=failed, + current_video_id="", + completed_at=_iso(), + ) + _update_account(job["account_id"], collected_at=_iso() if completed else "", error=message) + except Exception as exc: + message = str(exc) + if "槽位已满" in message or "已经处于唤醒状态" in message: + _set_job(job_id, "delayed", "waiting_slot", message, next_attempt_at=_iso(_utc_now() + timedelta(seconds=30))) + elif 'job' in locals() and proxy_pool.is_retryable_proxy_error(message): + _delay_for_proxy(job_id, job, message) + elif 'job' in locals() and isinstance(exc, (BrowserPageTimeout, BrowserPageLoadError)) and int(job.get("attempt_count") or 0) < 3: + _delay_for_page(job_id, job, message) + else: + _set_job(job_id, "failed", "failed", message, session_id=session_id or None, completed_at=_iso()) + job = _load_job(job_id) + if job: + _record_error(job, "", "", "job", message) + _update_account(job["account_id"], error=message) + finally: + if session_id and reused_observation: + try: + proxy_pool.release_observation_session_job(session_id, job_id) + except Exception as exc: + print(f"Collect observation session release failed for {job_id}: {exc}", flush=True) + elif session_id: + try: + proxy_pool.finish_automation_session(session_id, "自动采集任务结束") + except Exception as exc: + print(f"Collect session cleanup failed for {job_id}: {exc}", flush=True) + with _worker_lock: + _active_jobs.discard(job_id) + + +def _schedule_daily_jobs() -> None: + local_now = _utc_now().astimezone(ZoneInfo(TIMEZONE_NAME)) + local_day = local_now.date() + local_date = local_day.isoformat() + local_time = local_now.strftime("%H:%M") + with proxy_pool.connect() as conn: + conn.execute("BEGIN IMMEDIATE") + settings = conn.execute( + """ + SELECT s.*, a.proxy_profile_id, a.deleted_at + FROM collect_settings s JOIN tiktok_accounts a ON a.id = s.account_id + WHERE s.enabled = 1 AND a.proxy_bound = 1 + """ + ).fetchall() + for setting in settings: + if setting["deleted_at"] or str(setting["last_scheduled_date"]) == local_date: + continue + if local_time < str(setting["daily_time"]): + continue + account = _account(conn, int(setting["account_id"])) + try: + target_json = str(setting["feishu_target_json"] or "{}") + if not _target_key(_json_loads(target_json, {})): + continue + publish_date_start, publish_date_end = _automatic_publish_range( + _validate_date_rule(setting["date_rule"]), local_day + ) + _insert_job( + conn, + account, + "daily", + publish_date_start, + publish_date_end, + target_json, + schedule_date=local_date, + ) + except ValueError: + # Keep trying after the account becomes idle; recording the date + # here would silently drop today's scheduled collection. + continue + conn.execute( + "UPDATE collect_settings SET last_scheduled_date = ?, updated_at = ? WHERE account_id = ?", + (local_date, _iso(), int(setting["account_id"])), + ) + conn.commit() + + +def _claim_due_jobs() -> list[str]: + with _worker_lock: + capacity = max(0, proxy_pool.browser_max_slots() - len(_active_jobs)) + if capacity <= 0: + return [] + claimed: list[str] = [] + with proxy_pool.connect() as conn: + conn.execute("BEGIN IMMEDIATE") + rows = conn.execute( + """ + SELECT c.id, c.account_id FROM collect_jobs c + WHERE c.status IN ('queued','delayed') + AND EXISTS ( + SELECT 1 FROM tiktok_accounts a + WHERE a.id = c.account_id AND a.deleted_at = '' AND a.proxy_bound = 1 + ) + AND (c.next_attempt_at = '' OR c.next_attempt_at <= ?) + AND NOT EXISTS ( + SELECT 1 FROM collect_jobs running + WHERE running.account_id = c.account_id + AND running.status IN ('preparing','collecting') + ) + AND NOT EXISTS ( + SELECT 1 FROM publish_jobs p + WHERE p.account_id = c.account_id AND p.deleted_at = '' + AND p.status IN ('preparing','uploading','publishing') + ) + ORDER BY c.created_at ASC LIMIT ? + """, + (_iso(), capacity * 4), + ).fetchall() + claimed_accounts: set[int] = set() + for row in rows: + account_id = int(row["account_id"]) + if account_id in claimed_accounts or len(claimed) >= capacity: + continue + job_id = str(row["id"]) + changed = conn.execute( + """ + UPDATE collect_jobs + SET status = 'preparing', stage = 'claimed', attempt_count = attempt_count + 1, + status_detail = '', next_attempt_at = '', updated_at = ? + WHERE id = ? AND status IN ('queued','delayed') + """, + (_iso(), job_id), + ).rowcount + if changed: + claimed.append(job_id) + claimed_accounts.add(account_id) + conn.commit() + return claimed + + +def _recover_interrupted() -> None: + proxy_failures: list[tuple[int, str]] = [] + with proxy_pool.connect() as conn: + conn.execute( + """ + UPDATE collect_jobs + SET status = 'queued', stage = 'recovered', session_id = NULL, + status_detail = '服务器重启，采集任务已重新排队', next_attempt_at = '', + last_error = '服务器重启后恢复采集任务', updated_at = ? + WHERE status IN ('preparing','collecting') + """, + (_iso(),), + ) + now = _iso() + retry_at = _iso(_utc_now() + timedelta(seconds=proxy_pool.PROXY_QUEUE_RECHECK_SECONDS)) + rows = conn.execute( + "SELECT id, proxy_profile_id, last_error FROM collect_jobs WHERE status = 'failed' AND completed_videos = 0" + ).fetchall() + for row in rows: + message = str(row["last_error"] or "") + if not proxy_pool.is_retryable_proxy_error(message): + continue + conn.execute( + "UPDATE collect_jobs SET status = 'delayed', stage = 'waiting_proxy', status_detail = '', session_id = NULL, completed_at = '', next_attempt_at = ?, updated_at = ? WHERE id = ?", + (retry_at, now, row["id"]), + ) + proxy_failures.append((int(row["proxy_profile_id"]), message)) + conn.commit() + for pool_id, message in proxy_failures: + proxy_pool.schedule_proxy_recheck_for_pending_job(pool_id, message) + + +def _worker_loop() -> None: + while True: + try: + _schedule_daily_jobs() + for job_id in _claim_due_jobs(): + with _worker_lock: + if job_id in _active_jobs: + continue + _active_jobs.add(job_id) + threading.Thread(target=_run_job, args=(job_id,), daemon=True, name=f"tiktok-collect-{job_id[-8:]}").start() + except Exception as exc: + print(f"TikTok collect scheduler failed: {exc}", flush=True) + time.sleep(WORKER_INTERVAL_SECONDS) + + +def start_worker() -> None: + global _worker_started + with _worker_lock: + if _worker_started: + return + _worker_started = True + LOG_ROOT.mkdir(parents=True, exist_ok=True) + _recover_interrupted() + threading.Thread(target=_worker_loop, daemon=True, name="tiktok-collect-worker").start() diff --git a/scripts/tiktok_studio_publish.py b/scripts/tiktok_studio_publish.py new file mode 100644 index 0000000..1fa6b9f --- /dev/null +++ b/scripts/tiktok_studio_publish.py @@ -0,0 +1,1869 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import base64 +import hashlib +import json +import mimetypes +import os +import re +import threading +import time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlparse +from urllib.request import Request, urlopen +from zoneinfo import ZoneInfo + +import proxy_pool +from browser_page_state import ( + BrowserPageBlocked, + BrowserPageTimeout, + wait_for_page_state, +) + + +ROOT = Path.cwd() +PUBLISH_ROOT = ROOT / "videos" / "tiktok_publish" +LOG_ROOT = ROOT / "data" / "tiktok_publish_jobs" +MAX_UPLOAD_BYTES = int(os.getenv("TIKTOK_PUBLISH_MAX_BYTES", str(2 * 1024 * 1024 * 1024))) +TIMEZONE_NAME = os.getenv("TZ", "America/Los_Angeles") or "America/Los_Angeles" +NATIVE_MIN_MINUTES = max(15, int(os.getenv("TIKTOK_NATIVE_SCHEDULE_MIN_MINUTES", "20") or "20")) +DRY_RUN = os.getenv("TIKTOK_PUBLISH_DRY_RUN", "1").strip().lower() in {"1", "true", "yes", "on"} +FINAL_CLICK_ENABLED = os.getenv("TIKTOK_PUBLISH_FINAL_CLICK_ENABLED", "1").strip().lower() in {"1", "true", "yes", "on"} +UPLOAD_TIMEOUT_SECONDS = max(60, int(os.getenv("TIKTOK_PUBLISH_UPLOAD_TIMEOUT_SECONDS", "1800") or "1800")) +ALLOWED_SUFFIXES = {".mp4", ".mov", ".m4v", ".webm"} +EDITABLE_STATUSES = {"draft", "queued", "delayed", "failed", "cancelled", "dry_run", "manual_ready"} +RETRYABLE_STATUSES = {"failed", "product_link_failed", "product_link_review"} +DELETE_BLOCKED_STATUSES = {"preparing", "uploading", "publishing", "scheduled_on_tiktok"} +STATUS_LABELS = { + "draft": "草稿箱", + "queued": "待发布", + "delayed": "延迟等待", + "preparing": "准备中", + "uploading": "上传中", + "publishing": "发布中", + "published": "已发布", + "failed": "发布失败", + "result_uncertain": "结果待确认", + "product_link_review": "商品绑定待确认", + "product_link_failed": "商品未绑定成功", + "cancelled": "已取消", + "scheduled_on_tiktok": "TikTok已排程", + "dry_run": "演练完成", + "manual_ready": "等待手动发布", +} + +SAFE_POPUP_BUTTONS = { + "allow": "allow", + "cancel": "cancel", + "close": "close", + "got it": "got it", + "later": "later", + "maybe later": "maybe later", + "not now": "not now", + "skip": "skip", +} + +SAFE_PAGE_RECOVERY_BUTTONS = { + **SAFE_POPUP_BUTTONS, + "continue": "continue", + "discard": "discard", + "reload": "reload", + "retry": "retry", +} + +_worker_started = False +_worker_lock = threading.Lock() +_active_jobs: set[str] = set() + + +class ManualReviewRequired(RuntimeError): + pass + + +class ProductLinkReviewRequired(RuntimeError): + pass + + +class ProductLinkUnavailable(RuntimeError): + pass + + +class ManualPublishReady(RuntimeError): + pass + + +class ResultUncertain(RuntimeError): + pass + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _iso(value: datetime | None = None) -> str: + return (value or _utc_now()).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _parse_schedule(value: Any) -> datetime: + raw = str(value or "").strip() + if not raw: + return _utc_now() + try: + parsed = datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError("发布时间格式无效") from exc + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=ZoneInfo(TIMEZONE_NAME)) + return parsed.astimezone(timezone.utc) + + +def _normalize_native_schedule(value: datetime) -> datetime: + normalized = value.astimezone(timezone.utc).replace(second=0, microsecond=0) + remainder = normalized.minute % 5 + if remainder: + normalized += timedelta(minutes=5 - remainder) + return normalized + + +def _clean_text(value: Any, limit: int) -> str: + return str(value or "").strip()[:limit] + + +def _clean_product_link(value: Any) -> str: + return _clean_text(value, 2000) + + +def _row_to_job(row: Any) -> dict[str, Any]: + return { + "id": row["id"], + "account_id": row["account_id"], + "proxy_profile_id": row["proxy_profile_id"], + "asset_id": row["asset_id"], + "original_name": row["original_name"], + "size_bytes": row["size_bytes"], + "content_type": row["content_type"], + "description": row["description"], + "ai_generated": bool(row["ai_generated"]), + "product_link": row["product_link"], + "product_link_status": "待绑定" if row["product_link"] else "不添加", + "keep_observing": bool(row["keep_observing"]), + "manual_publish": bool(row["manual_publish"]), + "schedule_mode": row["schedule_mode"], + "scheduled_at": row["scheduled_at"], + "status": row["status"], + "status_label": STATUS_LABELS.get(row["status"], row["status"]), + "stage": row["stage"], + "status_detail": row["status_detail"], + "attempt_count": row["attempt_count"], + "session_id": row["session_id"], + "actual_publish_at": row["actual_publish_at"], + "result_url": row["result_url"], + "last_error": row["last_error"], + "preview_url": f"/api/proxy/publish/videos/{row['asset_id']}", + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + +def _job_query(where: str = "", params: tuple[Any, ...] = ()) -> list[dict[str, Any]]: + sql = ( + "SELECT j.*, a.original_name, a.size_bytes, a.content_type, a.stored_path " + "FROM publish_jobs j JOIN publish_assets a ON a.id = j.asset_id " + "WHERE j.deleted_at = '' " + ) + if where: + sql += "AND (" + where + ") " + sql += "ORDER BY j.created_at DESC" + with proxy_pool.connect() as conn: + return [_row_to_job(row) for row in conn.execute(sql, params).fetchall()] + + +def list_jobs(account_id: int) -> dict[str, Any]: + if not account_id: + raise ValueError("account_id is required") + return {"jobs": _job_query("j.account_id = ?", (account_id,)), "timezone": TIMEZONE_NAME, "dry_run": DRY_RUN} + + +def _validate_schedule(mode: str, scheduled_at: datetime, queued: bool) -> None: + if mode not in {"server", "tiktok"}: + raise ValueError("发布模式配置无效") + if queued and mode == "tiktok" and scheduled_at < _utc_now() + timedelta(minutes=NATIVE_MIN_MINUTES): + raise ValueError(f"TikTok 定时发布至少需要提前 {NATIVE_MIN_MINUTES} 分钟") + + +def _resolve_schedule_mode(mode: str, scheduled_at: datetime, queued: bool) -> str: + if queued and mode == "tiktok" and scheduled_at <= _utc_now(): + return "server" + return mode + + +def create_job(form: Any) -> dict[str, Any]: + account_id = int(form.getfirst("account_id") or 0) + if not account_id: + raise ValueError("account_id is required") + action = _clean_text(form.getfirst("action"), 20) or "queue" + if action not in {"draft", "queue", "immediate", "manual"}: + raise ValueError("发布操作无效") + queued = action != "draft" + manual_publish = action == "manual" + keep_observing = action in {"immediate", "manual"} + requested_session_id = int(form.getfirst("observation_session_id") or 0) + schedule_mode = "server" if action == "immediate" else "tiktok" + scheduled_at = _parse_schedule(form.getfirst("scheduled_at")) + schedule_mode = _resolve_schedule_mode(schedule_mode, scheduled_at, queued) + if not manual_publish: + if queued and schedule_mode == "tiktok": + scheduled_at = _normalize_native_schedule(scheduled_at) + _validate_schedule(schedule_mode, scheduled_at, queued) + try: + file_item = form["video"] + except KeyError as exc: + raise ValueError("请选择视频文件") from exc + if isinstance(file_item, list): + file_item = file_item[0] + original_name = Path(str(getattr(file_item, "filename", "") or "")).name + suffix = Path(original_name).suffix.lower() + if not original_name or suffix not in ALLOWED_SUFFIXES: + raise ValueError("仅支持 MP4、MOV、M4V 或 WebM 视频") + + with proxy_pool.connect() as conn: + account = conn.execute("SELECT * FROM tiktok_accounts WHERE id = ? AND deleted_at = ''", (account_id,)).fetchone() + if not account: + raise ValueError("account not found") + proxy_pool.require_account_proxy_bound(account) + proxy_profile_id = int(account["proxy_profile_id"]) + + asset_id = uuid.uuid4().hex + job_id = uuid.uuid4().hex + target_dir = PUBLISH_ROOT / str(account_id) + target_dir.mkdir(parents=True, exist_ok=True) + target = target_dir / f"{asset_id}{suffix}" + digest = hashlib.sha256() + size = 0 + try: + with target.open("wb") as output: + while True: + chunk = file_item.file.read(1024 * 1024) + if not chunk: + break + size += len(chunk) + if size > MAX_UPLOAD_BYTES: + raise ValueError("视频超过 2GB 上传限制") + digest.update(chunk) + output.write(chunk) + if size <= 0: + raise ValueError("视频文件为空") + now = _iso() + stored_path = target.relative_to(ROOT).as_posix() + content_type = _clean_text(getattr(file_item, "type", ""), 120) or mimetypes.guess_type(original_name)[0] or "video/mp4" + with proxy_pool.connect() as conn: + conn.execute( + "INSERT INTO publish_assets (id, account_id, original_name, stored_path, content_type, size_bytes, sha256, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (asset_id, account_id, original_name, stored_path, content_type, size, digest.hexdigest(), now), + ) + conn.execute( + """ + INSERT INTO publish_jobs ( + id, account_id, proxy_profile_id, asset_id, description, ai_generated, + product_link, keep_observing, manual_publish, schedule_mode, scheduled_at, status, stage, attempt_count, next_attempt_at, + session_id, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '', 0, '', ?, ?, ?) + """, + ( + job_id, + account_id, + proxy_profile_id, + asset_id, + _clean_text(form.getfirst("description"), 2200), + 1 if str(form.getfirst("ai_generated") or "").lower() in {"1", "true", "yes", "on"} else 0, + _clean_product_link(form.getfirst("product_link")), + 1 if keep_observing else 0, + 1 if manual_publish else 0, + schedule_mode, + _iso(scheduled_at), + "queued" if queued else "draft", + requested_session_id or None, + now, + now, + ), + ) + conn.commit() + except Exception: + target.unlink(missing_ok=True) + raise + return {"job": _job_query("j.id = ?", (job_id,))[0], **list_jobs(account_id)} + + +def update_job(payload: dict[str, Any]) -> dict[str, Any]: + job_id = _clean_text(payload.get("id") or payload.get("job_id"), 80) + if not job_id: + raise ValueError("job_id is required") + with proxy_pool.connect() as conn: + row = conn.execute("SELECT * FROM publish_jobs WHERE id = ? AND deleted_at = ''", (job_id,)).fetchone() + if not row: + raise ValueError("publish job not found") + account = conn.execute( + "SELECT * FROM tiktok_accounts WHERE id = ? AND deleted_at = ''", + (int(row["account_id"]),), + ).fetchone() + if not account: + raise ValueError("account not found") + proxy_pool.require_account_proxy_bound(account) + if row["status"] not in EDITABLE_STATUSES: + raise ValueError("当前任务状态不能编辑") + scheduled = _parse_schedule(payload.get("scheduled_at") or row["scheduled_at"]) + product_link = _clean_product_link(payload["product_link"]) if "product_link" in payload else str(row["product_link"] or "") + manual_publish = bool(payload.get("manual_publish")) + keep_observing = bool(payload["keep_observing"]) if "keep_observing" in payload else bool(row["keep_observing"]) + requested_session_id = ( + int(payload.get("observation_session_id") or 0) or None + if "observation_session_id" in payload + else row["session_id"] + ) + queue = bool(payload.get("queue")) + mode = "server" if keep_observing else "tiktok" + mode = _resolve_schedule_mode(mode, scheduled, queue) + if manual_publish: + keep_observing = True + mode = "tiktok" + else: + if queue and mode == "tiktok": + scheduled = _normalize_native_schedule(scheduled) + _validate_schedule(mode, scheduled, queue) + status = "queued" if queue else "draft" + conn.execute( + "UPDATE publish_jobs SET description = ?, ai_generated = ?, product_link = ?, keep_observing = ?, manual_publish = ?, schedule_mode = ?, scheduled_at = ?, status = ?, stage = '', status_detail = '', attempt_count = 0, next_attempt_at = '', session_id = ?, final_click_at = '', actual_publish_at = '', result_url = '', last_error = '', updated_at = ? WHERE id = ?", + ( + _clean_text(payload.get("description"), 2200), + 1 if payload.get("ai_generated") else 0, + product_link, + 1 if keep_observing else 0, + 1 if manual_publish else 0, + mode, + _iso(scheduled), + status, + requested_session_id, + _iso(), + job_id, + ), + ) + conn.commit() + account_id = int(row["account_id"]) + return {"job": _job_query("j.id = ?", (job_id,))[0], **list_jobs(account_id)} + + +def cancel_job(payload: dict[str, Any]) -> dict[str, Any]: + job_id = _clean_text(payload.get("id") or payload.get("job_id"), 80) + with proxy_pool.connect() as conn: + row = conn.execute("SELECT * FROM publish_jobs WHERE id = ? AND deleted_at = ''", (job_id,)).fetchone() + if not row: + raise ValueError("publish job not found") + if row["status"] not in EDITABLE_STATUSES: + raise ValueError("只能取消尚未开始的发布任务") + conn.execute("UPDATE publish_jobs SET status = 'cancelled', stage = '', status_detail = '', updated_at = ? WHERE id = ?", (_iso(), job_id)) + conn.commit() + account_id = int(row["account_id"]) + return list_jobs(account_id) + + +def retry_job(payload: dict[str, Any]) -> dict[str, Any]: + job_id = _clean_text(payload.get("id") or payload.get("job_id"), 80) + if not job_id: + raise ValueError("job_id is required") + with proxy_pool.connect() as conn: + row = conn.execute("SELECT * FROM publish_jobs WHERE id = ? AND deleted_at = ''", (job_id,)).fetchone() + if not row: + raise ValueError("publish job not found") + account = conn.execute( + "SELECT * FROM tiktok_accounts WHERE id = ? AND deleted_at = ''", + (int(row["account_id"]),), + ).fetchone() + if not account: + raise ValueError("account not found") + proxy_pool.require_account_proxy_bound(account) + if row["status"] not in RETRYABLE_STATUSES: + raise ValueError("只有发布失败的任务可以重试") + video_path(str(row["asset_id"])) + account_id = int(row["account_id"]) + scheduled = _parse_schedule(row["scheduled_at"]) + now = _utc_now() + mode = "tiktok" + if scheduled <= now: + mode = "server" + scheduled = now + else: + scheduled = _normalize_native_schedule(scheduled) + _validate_schedule(mode, scheduled, True) + requested_session_id = int(payload.get("observation_session_id") or 0) or None + conn.execute( + """ + UPDATE publish_jobs + SET keep_observing = ?, manual_publish = 0, schedule_mode = ?, scheduled_at = ?, status = 'queued', + stage = 'retry_queued', status_detail = '', next_attempt_at = '', session_id = ?, final_click_at = '', + actual_publish_at = '', result_url = '', last_error = '', updated_at = ? + WHERE id = ? + """, + (1 if requested_session_id else 0, mode, _iso(scheduled), requested_session_id, _iso(now), job_id), + ) + conn.execute( + "UPDATE tiktok_accounts SET last_error = '', updated_at = ? WHERE id = ?", + (_iso(now), account_id), + ) + conn.commit() + return {"job": _job_query("j.id = ?", (job_id,))[0], **list_jobs(account_id)} + + +def delete_job(payload: dict[str, Any]) -> dict[str, Any]: + job_id = _clean_text(payload.get("id") or payload.get("job_id"), 80) + if not job_id: + raise ValueError("job_id is required") + with proxy_pool.connect() as conn: + row = conn.execute("SELECT * FROM publish_jobs WHERE id = ? AND deleted_at = ''", (job_id,)).fetchone() + if not row: + raise ValueError("publish job not found") + if row["status"] in DELETE_BLOCKED_STATUSES: + raise ValueError("运行中或 TikTok 已排程的任务不能删除") + now = _iso() + status = "cancelled" if row["status"] in {"draft", "queued", "delayed"} else row["status"] + conn.execute( + "UPDATE publish_jobs SET status = ?, stage = '', status_detail = '', deleted_at = ?, updated_at = ? WHERE id = ?", + (status, now, now, job_id), + ) + conn.commit() + account_id = int(row["account_id"]) + return list_jobs(account_id) + + +def video_path(asset_id: str) -> Path: + with proxy_pool.connect() as conn: + row = conn.execute("SELECT stored_path FROM publish_assets WHERE id = ?", (_clean_text(asset_id, 80),)).fetchone() + if not row: + raise ValueError("publish video not found") + path = (ROOT / str(row["stored_path"])).resolve() + root = PUBLISH_ROOT.resolve() + if root != path.parent and root not in path.parents: + raise ValueError("invalid publish video path") + if not path.is_file(): + raise ValueError("发布视频文件不存在，请重新上传后再重试") + return path + + +def runtime_status() -> dict[str, Any]: + with proxy_pool.connect() as conn: + counts = {row["status"]: int(row["count"]) for row in conn.execute("SELECT status, COUNT(*) AS count FROM publish_jobs WHERE deleted_at = '' GROUP BY status")} + with _worker_lock: + active = sorted(_active_jobs) + return { + "worker_started": _worker_started, + "dry_run": DRY_RUN, + "final_click_enabled": FINAL_CLICK_ENABLED, + "timezone": TIMEZONE_NAME, + "max_automatic_slots": proxy_pool.browser_max_slots(), + "active_jobs": active, + "counts": counts, + "native_schedule_lead_seconds": 0, + "native_schedule_starts_immediately": True, + } + + +def _set_job(job_id: str, status: str, stage: str = "", error: str = "", **values: Any) -> None: + status_detail = _clean_text(values.pop("status_detail", ""), 500) + fields = ["status = ?", "stage = ?", "status_detail = ?", "last_error = ?", "updated_at = ?"] + params: list[Any] = [status, stage, status_detail, _clean_text(error, 2000), _iso()] + allowed = {"session_id", "final_click_at", "actual_publish_at", "result_url", "next_attempt_at"} + for key, value in values.items(): + if key in allowed: + fields.append(f"{key} = ?") + params.append(value) + params.append(job_id) + with proxy_pool.connect() as conn: + conn.execute(f"UPDATE publish_jobs SET {', '.join(fields)} WHERE id = ?", params) + conn.commit() + + +def _update_account(account_id: int, error: str = "", published_at: str = "") -> None: + with proxy_pool.connect() as conn: + conn.execute( + "UPDATE tiktok_accounts SET last_error = ?, last_publish_at = COALESCE(NULLIF(?, ''), last_publish_at), updated_at = ? WHERE id = ?", + (_clean_text(error, 2000), published_at, _iso(), account_id), + ) + conn.commit() + + +def _delay_for_proxy(job_id: str, job: dict[str, Any], error: Exception | str) -> None: + message = str(error) + next_attempt_at = _iso(_utc_now() + timedelta(seconds=proxy_pool.PROXY_QUEUE_RECHECK_SECONDS)) + _set_job(job_id, "delayed", "waiting_proxy", message, session_id=None, next_attempt_at=next_attempt_at) + _update_account(int(job["account_id"]), error=message) + proxy_pool.schedule_proxy_recheck_for_pending_job(int(job["proxy_profile_id"]), message) + + +def _delay_for_page(job_id: str, job: dict[str, Any], error: Exception | str) -> None: + message = str(error) + _set_job( + job_id, + "delayed", + "waiting_page", + message, + session_id=None, + next_attempt_at=_iso(_utc_now() + timedelta(seconds=60)), + status_detail="页面加载超时，60 秒后自动重试", + ) + _update_account(int(job["account_id"]), error=message) + + +def _first_visible(locators: list[Any]) -> Any | None: + for locator in locators: + try: + count = min(locator.count(), 8) + for index in range(count): + item = locator.nth(index) + if item.is_visible(): + return item + except Exception: + continue + return None + + +def _skip_onboarding(page: Any) -> None: + pattern = re.compile(r"^(skip|skip for now|not now|got it|later|跳过|暂不|稍后|知道了)$", re.I) + for _ in range(4): + button = _first_visible([page.get_by_role("button", name=pattern), page.get_by_text(pattern, exact=True)]) + if not button: + return + button.click(timeout=3000) + page.wait_for_timeout(500) + + +def _dismiss_upload_prompts(page: Any) -> None: + for _ in range(6): + got_it = _first_visible([ + page.get_by_role("button", name=re.compile(r"^got it$|^知道了$", re.I)), + ]) + if not got_it: + break + got_it.click(timeout=3000, force=True) + page.wait_for_timeout(500) + + automatic_checks = _first_visible([ + page.get_by_text(re.compile(r"turn on automatic content checks|开启自动内容检查", re.I)), + ]) + if automatic_checks: + cancel = _first_visible([ + page.get_by_role("button", name=re.compile(r"^cancel$|^取消$", re.I)), + ]) + if cancel: + cancel.click(timeout=3000) + page.wait_for_timeout(500) + + +def _assert_account_ready(page: Any) -> None: + url = page.url.lower() + if "/login" in url: + raise ManualReviewRequired("TikTok 登录已失效，请从观测通道重新登录") + challenge = _first_visible([ + page.get_by_text(re.compile(r"captcha|verify to continue|security verification|验证码|安全验证", re.I)), + page.locator("iframe[src*='captcha']"), + ]) + if challenge: + raise ManualReviewRequired("TikTok 要求验证码或安全验证，请从观测通道人工处理") + + +def _set_description(page: Any, description: str) -> None: + if not description: + return + target = _first_visible([ + page.locator("[data-e2e*='caption'] [contenteditable='true']"), + page.locator(".public-DraftEditor-content[contenteditable='true']"), + page.locator("[contenteditable='true'][role='combobox']"), + page.locator("[contenteditable='true'][role='textbox']"), + page.get_by_label(re.compile(r"description|caption|说明|描述", re.I)), + page.locator("textarea[placeholder*='caption' i], textarea[placeholder*='description' i]"), + ]) + if not target: + raise RuntimeError("未找到 TikTok Studio 的 Description 输入框") + target.click() + try: + target.fill(description) + except Exception: + target.press("Control+A") + target.type(description) + + +def _set_ai_generated(page: Any, enabled: bool) -> None: + if not enabled: + return + show_more = _first_visible([ + page.get_by_role("button", name=re.compile(r"show more|更多|展开", re.I)), + page.get_by_text(re.compile(r"^show more$|^显示更多$|^更多设置$", re.I), exact=True), + ]) + if show_more: + show_more.click() + page.wait_for_timeout(500) + label_pattern = re.compile(r"AI.generated content|AI 生成|人工智能生成", re.I) + checkbox = _first_visible([ + page.locator("[data-e2e='aigc_container'] input[role='switch']"), + page.get_by_role("checkbox", name=label_pattern), + page.get_by_label(label_pattern), + ]) + if checkbox: + confirmation = _first_visible([ + page.get_by_text(re.compile(r"labeling AI.generated content|标记 AI 生成内容", re.I)), + ]) + if not checkbox.is_checked() and not confirmation: + switch = _first_visible([ + page.locator("[data-e2e='aigc_container'] .Switch__content"), + page.locator("[data-e2e='aigc_container'] [data-layout='switch-root']"), + ]) + if switch: + switch.click(timeout=3000) + else: + checkbox.click(timeout=3000, force=True) + page.wait_for_timeout(300) + confirmation = _first_visible([ + page.get_by_text(re.compile(r"labeling AI.generated content|标记 AI 生成内容", re.I)), + ]) + if confirmation: + turn_on = _first_visible([ + page.get_by_role("button", name=re.compile(r"^turn on$|^开启$", re.I)), + ]) + if not turn_on: + raise RuntimeError("未找到 AI-generated content 确认按钮") + turn_on.click(timeout=3000) + page.wait_for_timeout(300) + if not checkbox.is_checked() and checkbox.get_attribute("aria-checked") != "true": + raise RuntimeError("无法启用 AI-generated content 设置") + return + label = _first_visible([page.get_by_text(label_pattern)]) + if not label: + raise RuntimeError("未找到 AI-generated content 设置") + label.click() + + +def _radio_selected(locator: Any) -> bool: + try: + if not locator.count(): + return False + item = locator.first + try: + if item.is_checked(): + return True + except Exception: + pass + if str(item.get_attribute("aria-checked") or "").lower() == "true": + return True + classes = str(item.get_attribute("class") or "") + return bool(re.search(r"(?:^|[-_\s])(checked|selected|active)(?:$|[-_\s])", classes, re.I)) + except Exception: + return False + + +def _popup_snapshot(page: Any, log_dir: Path, step: str) -> Path: + target = log_dir / f"parameter-{step}-{int(time.time())}.png" + page.screenshot(path=str(target), full_page=False) + return target + + +def _visible_dialog(page: Any) -> Any | None: + return _first_visible([ + page.get_by_role("dialog"), + page.locator("[role='dialog']"), + page.locator("[aria-modal='true']"), + ]) + + +def _dialog_details(dialog: Any) -> tuple[str, list[str]]: + try: + text = _clean_text(dialog.inner_text(), 2000) + except Exception: + text = "" + buttons: list[str] = [] + try: + locator = dialog.get_by_role("button") + for index in range(min(locator.count(), 12)): + label = _clean_text(locator.nth(index).inner_text(), 80) + if label: + buttons.append(label) + except Exception: + pass + return text, buttons + + +def _vision_popup_decision(snapshot: Path, step: str, dialog_text: str, buttons: list[str]) -> dict[str, Any]: + api_key = os.getenv("VISION_API_KEY", "").strip() + api_url = os.getenv("VISION_API_URL", "").strip().rstrip("/") + model = os.getenv("VISION_MODEL", "qwen3-vl-flash").strip() or "qwen3-vl-flash" + if not api_key or not api_url: + return {"action": "manual_review", "reason": "视觉模型未配置"} + if not api_url.endswith("/chat/completions"): + api_url += "/chat/completions" + encoded = base64.b64encode(snapshot.read_bytes()).decode("ascii") + prompt = ( + "分析 TikTok Studio 参数页的弹窗。只能返回 JSON：" + '{"action":"click|manual_review","button_text":"","x":0,"y":0,"reason":""}。' + "仅当弹窗是说明、提示或可安全处理的参数授权时才选择 click；" + "button_text 必须原样来自候选按钮，x/y 是截图中的按钮中心坐标。" + "只有弹窗明确询问 scheduled posting storage 时才可选择 Allow。" + "禁止选择 Post、Publish、Next、Confirm、Delete。" + f"\n当前步骤：{step}\n弹窗文字：{dialog_text}\n候选按钮：{buttons}" + ) + payload = { + "model": model, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{encoded}"}}, + ], + }], + "temperature": 0, + "max_tokens": 240, + } + request = Request( + api_url, + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=45) as response: + response_body = json.loads(response.read().decode("utf-8")) + content = str(response_body["choices"][0]["message"]["content"]) + match = re.search(r"\{.*\}", content, re.S) + if not match: + return {"action": "manual_review", "reason": "视觉模型未返回 JSON"} + parsed = json.loads(match.group(0)) + return { + "action": _clean_text(parsed.get("action"), 40), + "button_text": _clean_text(parsed.get("button_text"), 80), + "x": parsed.get("x"), + "y": parsed.get("y"), + "reason": _clean_text(parsed.get("reason"), 300), + } + + +def _click_popup_coordinate(page: Any, dialog: Any, button_text: str, x: Any, y: Any) -> bool: + button = _first_visible([ + dialog.get_by_role("button", name=re.compile(rf"^{re.escape(button_text)}$", re.I)), + ]) + if not button: + return False + box = button.bounding_box() + dialog_box = dialog.bounding_box() + if not box or not dialog_box: + return False + try: + click_x = float(x) + click_y = float(y) + except (TypeError, ValueError): + click_x = box["x"] + box["width"] / 2 + click_y = box["y"] + box["height"] / 2 + inside_button = box["x"] <= click_x <= box["x"] + box["width"] and box["y"] <= click_y <= box["y"] + box["height"] + inside_dialog = dialog_box["x"] <= click_x <= dialog_box["x"] + dialog_box["width"] and dialog_box["y"] <= click_y <= dialog_box["y"] + dialog_box["height"] + if not inside_button or not inside_dialog: + click_x = box["x"] + box["width"] / 2 + click_y = box["y"] + box["height"] / 2 + page.mouse.click(click_x, click_y) + page.wait_for_timeout(500) + return True + + +def _handle_parameter_popup(page: Any, log_dir: Path, step: str) -> bool: + dialog = _visible_dialog(page) + if not dialog: + return False + dialog_text, buttons = _dialog_details(dialog) + snapshot = _popup_snapshot(page, log_dir, step) + normalized = dialog_text.lower() + try: + decision = _vision_popup_decision(snapshot, step, dialog_text, buttons) + except Exception as exc: + decision = {"action": "manual_review", "reason": f"视觉模型调用失败：{exc}"} + known_schedule_permission = "allow your video to be saved for scheduled posting" in normalized + if known_schedule_permission and not ( + decision.get("action") == "click" and str(decision.get("button_text") or "").strip().lower() == "allow" + ): + decision = { + "source": "known_popup_fallback", + "action": "click", + "button_text": "Allow", + "x": None, + "y": None, + "reason": "已确认的 TikTok 定时发布存储授权弹窗", + } + (log_dir / f"parameter-{step}-decision.json").write_text( + json.dumps(decision, ensure_ascii=False, indent=2), encoding="utf-8" + ) + button_text = str(decision.get("button_text") or "").strip() + safe_name = SAFE_POPUP_BUTTONS.get(button_text.lower()) + allow_allowed = button_text.lower() != "allow" or known_schedule_permission + if decision.get("action") == "click" and safe_name and allow_allowed and button_text in buttons: + if _click_popup_coordinate(page, dialog, button_text, decision.get("x"), decision.get("y")): + return True + reason = str(decision.get("reason") or "弹窗不属于可自动处理的安全类型") + raise ManualReviewRequired(f"参数页出现未识别弹窗，已保留观测通道：{reason}") + + +def _page_details(page: Any) -> tuple[str, list[str]]: + try: + text = _clean_text(page.locator("body").inner_text(timeout=3000), 3000) + except Exception: + text = "" + buttons: list[str] = [] + try: + locator = page.get_by_role("button") + for index in range(min(locator.count(), 30)): + item = locator.nth(index) + try: + label = _clean_text(item.inner_text(), 80) if item.is_visible() else "" + except Exception: + label = "" + if label and label not in buttons: + buttons.append(label) + except Exception: + pass + return text, buttons + + +def _page_is_blank(page: Any) -> bool: + text, _buttons = _page_details(page) + if len(text) >= 40: + return False + try: + controls = page.locator("input, textarea, button, [role='button'], [role='dialog']") + for index in range(min(controls.count(), 30)): + if controls.nth(index).is_visible(): + return False + except Exception: + pass + return True + + +def _vision_page_recovery_decision( + snapshot: Path, + step: str, + page_url: str, + page_text: str, + buttons: list[str], + error: str, +) -> dict[str, Any]: + api_key = os.getenv("VISION_API_KEY", "").strip() + api_url = os.getenv("VISION_API_URL", "").strip().rstrip("/") + model = os.getenv("VISION_MODEL", "qwen3-vl-flash").strip() or "qwen3-vl-flash" + if not api_key or not api_url: + return {"action": "manual_review", "reason": "视觉模型未配置"} + if not api_url.endswith("/chat/completions"): + api_url += "/chat/completions" + encoded = base64.b64encode(snapshot.read_bytes()).decode("ascii") + prompt = ( + "分析 TikTok Studio 自动发布过程中遇到的异常页面，只返回 JSON：" + '{"action":"reload|click|manual_review","button_text":"","x":0,"y":0,"reason":""}。' + "页面空白、主体未加载或网络加载明显失败时可以选择 reload；" + "click 只能选择候选按钮中的 Continue、Discard、Reload、Retry、Cancel、Close、Got it、Later、Not now、Skip；" + "禁止选择 Post、Publish、Next、Add、Confirm、Delete、Schedule、Now，也禁止提交或发布内容。" + "不能确定时必须选择 manual_review。" + f"\n步骤：{step}\nURL：{page_url}\n错误：{error}\n页面文字：{page_text}\n候选按钮：{buttons}" + ) + payload = { + "model": model, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{encoded}"}}, + ], + }], + "temperature": 0, + "max_tokens": 260, + } + request = Request( + api_url, + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=45) as response: + response_body = json.loads(response.read().decode("utf-8")) + content = str(response_body["choices"][0]["message"]["content"]) + match = re.search(r"\{.*\}", content, re.S) + if not match: + return {"action": "manual_review", "reason": "视觉模型未返回 JSON"} + parsed = json.loads(match.group(0)) + return { + "action": _clean_text(parsed.get("action"), 40), + "button_text": _clean_text(parsed.get("button_text"), 80), + "x": parsed.get("x"), + "y": parsed.get("y"), + "reason": _clean_text(parsed.get("reason"), 300), + } + + +def _click_safe_page_button(page: Any, button_text: str, x: Any, y: Any) -> bool: + button = _first_visible([ + page.get_by_role("button", name=re.compile(rf"^{re.escape(button_text)}$", re.I)), + ]) + if not button: + return False + box = button.bounding_box() + if not box: + return False + try: + click_x = float(x) + click_y = float(y) + except (TypeError, ValueError): + click_x = box["x"] + box["width"] / 2 + click_y = box["y"] + box["height"] / 2 + if not (box["x"] <= click_x <= box["x"] + box["width"] and box["y"] <= click_y <= box["y"] + box["height"]): + click_x = box["x"] + box["width"] / 2 + click_y = box["y"] + box["height"] / 2 + page.mouse.click(click_x, click_y) + page.wait_for_timeout(1200) + return True + + +def _recover_unexpected_page(page: Any, log_dir: Path, step: str, error: Exception | str) -> bool: + snapshot = _popup_snapshot(page, log_dir, f"page-{step}") + page_text, buttons = _page_details(page) + blank = _page_is_blank(page) + try: + decision = _vision_page_recovery_decision( + snapshot, + step, + page.url, + page_text, + buttons, + _clean_text(error, 500), + ) + except Exception as exc: + decision = {"action": "manual_review", "reason": f"视觉模型调用失败：{exc}"} + if blank and decision.get("action") == "manual_review": + decision = { + "source": "blank_page_safe_fallback", + "action": "reload", + "button_text": "", + "reason": "截图和 DOM 均显示 TikTok Studio 主体为空，执行一次安全刷新", + } + (log_dir / f"page-{step}-decision.json").write_text( + json.dumps(decision, ensure_ascii=False, indent=2), encoding="utf-8" + ) + action = str(decision.get("action") or "").strip().lower() + host = (urlparse(page.url).hostname or "").lower() + if action == "reload" and (host == "tiktok.com" or host.endswith(".tiktok.com")): + page.reload(wait_until="domcontentloaded", timeout=60000) + page.wait_for_timeout(2500) + return True + button_text = str(decision.get("button_text") or "").strip() + if ( + action == "click" + and button_text.lower() in SAFE_PAGE_RECOVERY_BUTTONS + and button_text in buttons + and _click_safe_page_button(page, button_text, decision.get("x"), decision.get("y")) + ): + return True + reason = str(decision.get("reason") or "页面不属于可自动恢复的安全类型") + raise ManualReviewRequired(f"页面异常，已保留观测通道：{reason}") + + +def _discard_stale_edit(page: Any, log_dir: Path) -> bool: + notice = _first_visible([ + page.get_by_text(re.compile(r"video you were editing.*(?:wasn't|was not) saved|continue editing", re.I)), + ]) + if not notice: + return False + discard = _first_visible([ + page.get_by_role("button", name=re.compile(r"^discard$|^放弃$|^丢弃$", re.I)), + ]) + if not discard: + raise ManualReviewRequired("检测到未保存的旧视频提示，但没有找到 Discard 按钮") + page.screenshot(path=str(log_dir / "stale-edit-before-discard.png"), full_page=False) + discard.click(timeout=5000) + page.wait_for_timeout(1500) + page.screenshot(path=str(log_dir / "stale-edit-discarded.png"), full_page=False) + return True + + +def _goto_with_proxy_retries(page: Any, target: str) -> Any: + retryable_markers = ( + "err_connection_closed", + "err_tunnel_connection_failed", + "err_proxy_connection_failed", + "unexpected eof", + "connection reset", + "remote end closed", + ) + attempts = max(1, int(os.getenv("TIKTOK_PROXY_NAVIGATION_ATTEMPTS", "10") or "10")) + for attempt in range(attempts): + try: + return page.goto(target, wait_until="domcontentloaded", timeout=60000) + except Exception as exc: + retryable = any(marker in str(exc).lower() for marker in retryable_markers) + if not retryable or attempt + 1 >= attempts: + raise + page.wait_for_timeout(min(2000, 500 * (attempt + 1))) + raise RuntimeError(f"TikTok 页面无法加载：{target}") + + +def _ensure_studio_page(page: Any, log_dir: Path, on_status: Any = None) -> None: + parsed = urlparse(page.url) + already_in_studio = (parsed.hostname or "").endswith("tiktok.com") and parsed.path.startswith("/tiktokstudio") + if already_in_studio and not _page_is_blank(page): + return + if already_in_studio and _recover_unexpected_page(page, log_dir, "studio-blank", "TikTok Studio 页面主体为空"): + if not _page_is_blank(page): + return + target = "https://www.tiktok.com/tiktokstudio?lang=en" + for attempt in range(2): + try: + _goto_with_proxy_retries(page, target) + try: + wait_for_page_state( + page, + label="TikTok Studio", + ready=lambda: not _page_is_blank(page), + on_status=on_status, + timeout_seconds=60, + reload_attempts=1, + retry_action=lambda: _goto_with_proxy_retries(page, target), + diagnostic_dir=log_dir, + diagnostic_step="studio", + ) + return + except BrowserPageBlocked as exc: + raise ManualReviewRequired(str(exc)) from exc + except Exception as exc: + if isinstance(exc, BrowserPageTimeout): + raise + if attempt == 0 and _recover_unexpected_page(page, log_dir, "studio-navigation", exc): + if not _page_is_blank(page): + return + continue + raise ManualReviewRequired(f"TikTok Studio 无法加载，已保留观测通道：{exc}") from exc + + +def _upload_page_ready(page: Any) -> bool: + return bool(_first_visible([ + page.locator("input[type='file'][accept*='video']"), + page.locator("input[type='file']"), + page.locator("button[data-e2e='select_video_button']"), + page.get_by_role("button", name=re.compile(r"^select video$|^选择视频$|^上传视频$", re.I)), + page.get_by_text(re.compile(r"drag and drop|select video to upload|选择视频上传|拖放", re.I)), + ])) + + +def _ensure_upload_page(page: Any, log_dir: Path, on_status: Any = None) -> None: + if "/tiktokstudio/upload" in page.url and _upload_page_ready(page): + return + upload_link = _first_visible([ + page.locator("a[href*='/tiktokstudio/upload']"), + page.get_by_role("link", name=re.compile(r"upload|create|上传|发布", re.I)), + page.get_by_role("button", name=re.compile(r"upload|create|上传|发布", re.I)), + ]) + if upload_link: + upload_link.click(timeout=5000) + page.wait_for_timeout(500) + target = "https://www.tiktok.com/tiktokstudio/upload?from=creator_center&tab=video" + for attempt in range(2): + try: + _goto_with_proxy_retries(page, target) + try: + wait_for_page_state( + page, + label="TikTok 上传页", + ready=lambda: "/tiktokstudio/upload" in page.url and _upload_page_ready(page), + on_status=on_status, + timeout_seconds=75, + reload_attempts=1, + retry_action=lambda: _goto_with_proxy_retries(page, target), + diagnostic_dir=log_dir, + diagnostic_step="upload-page", + ) + return + except BrowserPageBlocked as exc: + raise ManualReviewRequired(str(exc)) from exc + except Exception as exc: + if isinstance(exc, BrowserPageTimeout): + raise + if attempt == 0 and _recover_unexpected_page(page, log_dir, "upload-navigation", exc): + if not _page_is_blank(page): + return + continue + raise ManualReviewRequired(f"TikTok Studio 上传页无法加载，已保留观测通道：{exc}") from exc + + +def _run_parameter_step(page: Any, log_dir: Path, step: str, action: Any) -> None: + try: + action() + return + except ManualReviewRequired: + raise + except Exception as first_error: + _popup_snapshot(page, log_dir, f"{step}-error") + if _handle_parameter_popup(page, log_dir, step): + try: + action() + return + except Exception as retry_error: + raise ManualReviewRequired( + f"{step} 参数设置失败，已保留观测通道：{retry_error}" + ) from retry_error + if _recover_unexpected_page(page, log_dir, step, first_error): + try: + action() + return + except Exception as retry_error: + raise ManualReviewRequired( + f"{step} 页面恢复后参数设置仍失败，已保留观测通道：{retry_error}" + ) from retry_error + raise ManualReviewRequired(f"{step} 参数设置失败，已保留观测通道：{first_error}") from first_error + + +def _select_schedule_radio(page: Any, value: str, label_pattern: re.Pattern[str], log_dir: Path) -> None: + selector = f"input[name='postSchedule'][value='{value}']" + input_locator = page.locator(selector) + role_locator = page.get_by_role("radio", name=label_pattern) + label_locator = page.locator(f"label:has({selector})") + state_locators = [input_locator, role_locator, label_locator] + if any(_radio_selected(locator) for locator in state_locators): + return + candidates = [label_locator, role_locator, page.get_by_text(label_pattern, exact=True)] + if input_locator.count(): + item = input_locator.first + candidates.extend([ + item.locator("xpath=ancestor::label[1]"), + item.locator("xpath=.."), + item.locator("xpath=../.."), + input_locator, + ]) + for locator in candidates: + option = _first_visible([locator]) + if not option: + continue + try: + option.click(timeout=3000) + except Exception: + continue + _handle_parameter_popup(page, log_dir, "schedule") + for _ in range(10): + if any(_radio_selected(state) for state in state_locators): + return + page.wait_for_timeout(150) + label = "定时发布" if value == "schedule" else "立即发布" + raise RuntimeError(f"无法选择 TikTok Studio 的{label}选项") + + +def _custom_schedule_fields(page: Any) -> tuple[Any | None, Any | None]: + container = page.locator("[data-e2e='schedule_container']") + deadline = time.time() + 5 + while time.time() < deadline: + time_field = None + date_field = None + fields = container.locator("input.TUXTextInputCore-input[readonly]") + for index in range(fields.count()): + field = fields.nth(index) + try: + if not field.is_visible(): + continue + value = field.input_value() + except Exception: + continue + if re.fullmatch(r"\d{2}:\d{2}", value): + time_field = field + elif re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): + date_field = field + if time_field and date_field: + return time_field, date_field + page.wait_for_timeout(200) + return None, None + + +def _select_custom_date(page: Any, date_field: Any, target: datetime) -> None: + expected = target.strftime("%Y-%m-%d") + if date_field.input_value() == expected: + return + date_field.click(timeout=5000) + calendar = _first_visible([page.locator(".calendar-wrapper")]) + if not calendar: + raise RuntimeError("TikTok 日期选择器未打开") + target_month = (target.year, target.month) + for _ in range(24): + month_text = calendar.locator(".month-title").inner_text().strip() + year_text = calendar.locator(".year-title").inner_text().strip() + try: + current = datetime.strptime(f"{month_text} {year_text}", "%B %Y") + except ValueError as exc: + raise RuntimeError(f"无法识别 TikTok 日历月份：{month_text} {year_text}") from exc + current_month = (current.year, current.month) + if current_month == target_month: + break + arrows = calendar.locator(".month-header-wrapper .arrow") + if arrows.count() < 2: + raise RuntimeError("TikTok 日期选择器缺少月份切换按钮") + arrows.first.click() if current_month > target_month else arrows.last.click() + page.wait_for_timeout(200) + else: + raise RuntimeError("目标日期超出 TikTok 日期选择范围") + day = calendar.locator("span.day.valid").filter(has_text=re.compile(rf"^{target.day}$")) + if not day.count(): + raise RuntimeError(f"TikTok 日期选择器中没有可选日期 {expected}") + day.first.click(timeout=5000) + page.wait_for_timeout(200) + if date_field.input_value() != expected: + raise RuntimeError(f"TikTok 日期设置未生效：期望 {expected}，当前 {date_field.input_value()}") + + +def _select_custom_time(page: Any, time_field: Any, target: datetime) -> None: + expected = target.strftime("%H:%M") + picker_locator = page.locator(".tiktok-timepicker-time-picker-container") + + def open_picker() -> Any | None: + for index in range(picker_locator.count()): + candidate = picker_locator.nth(index) + try: + classes = str(candidate.get_attribute("class") or "") + box = candidate.bounding_box() + if "tiktok-timepicker-invisible" not in classes and box and box["height"] > 20: + return candidate + except Exception: + continue + return None + + picker = open_picker() + if time_field.input_value() == expected: + if picker: + time_field.locator("xpath=..").click(timeout=5000) + page.wait_for_timeout(150) + return + if not picker: + time_field.locator("xpath=..").click(timeout=5000) + deadline = time.time() + 3 + while time.time() < deadline and not picker: + page.wait_for_timeout(100) + picker = open_picker() + if not picker: + raise RuntimeError("TikTok 时间选择器未打开") + option_lists = picker.locator(".tiktok-timepicker-option-list") + if option_lists.count() < 2: + raise RuntimeError("TikTok 时间选择器结构异常") + values = (target.strftime("%H"), target.strftime("%M")) + for index, value in enumerate(values): + option = option_lists.nth(index).locator(".tiktok-timepicker-option-text").filter( + has_text=re.compile(rf"^{re.escape(value)}$") + ) + if not option.count(): + raise RuntimeError(f"TikTok 时间选择器中没有可选值 {value}") + option.first.scroll_into_view_if_needed() + option.first.click(timeout=5000) + page.wait_for_timeout(150) + if open_picker(): + time_field.locator("xpath=..").click(timeout=5000) + page.wait_for_timeout(150) + if time_field.input_value() != expected: + raise RuntimeError(f"TikTok 时间设置未生效：期望 {expected}，当前 {time_field.input_value()}") + + +def _set_schedule(page: Any, mode: str, scheduled_at: str, log_dir: Path) -> None: + if mode == "server": + _select_schedule_radio(page, "post_now", re.compile(r"^now$|立即|现在", re.I), log_dir) + return + + _select_schedule_radio(page, "schedule", re.compile(r"^schedule$|定时发布", re.I), log_dir) + local = _normalize_native_schedule(_parse_schedule(scheduled_at)).astimezone(ZoneInfo(TIMEZONE_NAME)) + date_value = local.strftime("%Y-%m-%d") + time_value = local.strftime("%H:%M") + custom_time, custom_date = _custom_schedule_fields(page) + if custom_time and custom_date: + _select_custom_date(page, custom_date, local) + _select_custom_time(page, custom_time, local) + return + date_input = None + time_input = None + deadline = time.time() + 5 + while time.time() < deadline and (not date_input or not time_input): + date_input = date_input or _first_visible([ + page.locator("input[type='date']"), + page.locator("input[placeholder*='MM/DD'], input[placeholder*='YYYY']"), + page.get_by_label(re.compile(r"date|日期", re.I)), + ]) + time_input = time_input or _first_visible([ + page.locator("input[type='time']"), + page.locator("input[placeholder*='HH'], input[placeholder*='hh']"), + page.get_by_label(re.compile(r"time|时间", re.I)), + ]) + if not date_input or not time_input: + page.wait_for_timeout(200) + if not date_input or not time_input: + raise RuntimeError("未找到 TikTok 定时发布的日期或时间输入框") + placeholder = str(date_input.get_attribute("placeholder") or "") + if date_input.get_attribute("type") != "date" and "/" in placeholder: + date_value = local.strftime("%m/%d/%Y") + date_input.fill(date_value) + time_input.fill(time_value) + + +def _set_video_file(page: Any, video: Path) -> None: + deadline = time.time() + 30 + while time.time() < deadline: + file_inputs = page.locator("input[type='file'][accept*='video']") + if not file_inputs.count(): + file_inputs = page.locator("input[type='file']") + if file_inputs.count(): + file_inputs.first.set_input_files(str(video)) + return + select_button = _first_visible([ + page.locator("button[data-e2e='select_video_button']"), + page.get_by_role("button", name=re.compile(r"^select video$|^选择视频$", re.I)), + ]) + if select_button: + try: + with page.expect_file_chooser(timeout=5000) as chooser_info: + select_button.click() + chooser_info.value.set_files(str(video)) + return + except Exception: + pass + page.wait_for_timeout(500) + raise RuntimeError("未找到 TikTok Studio 视频选择控件") + + +def _selected_product(product_id: str) -> dict[str, Any]: + conn = proxy_pool.connect() + try: + row = conn.execute( + "SELECT product_id, product_name FROM tiktok_products WHERE product_id = ?", + (_clean_text(product_id, 120),), + ).fetchone() + finally: + conn.close() + if not row: + raise ProductLinkUnavailable(f"公共商品库中未找到商品 ID：{product_id}") + return {"product_id": str(row["product_id"]), "product_name": str(row["product_name"])} + + +def _cancel_product_link(page: Any) -> None: + cancel = _first_visible([ + page.get_by_role("button", name=re.compile(r"^cancel$|^取消$", re.I)), + ]) + if cancel: + cancel.click(timeout=5000) + page.wait_for_timeout(400) + + +def _find_product_row(page: Any, product_id: str) -> Any | None: + dialog = _visible_dialog(page) + root = dialog if dialog else page + for selector in ("tr", "[role='row']"): + rows = root.locator(selector) + for index in range(min(rows.count(), 80)): + row = rows.nth(index) + try: + if row.is_visible() and product_id in row.inner_text(): + return row + except Exception: + continue + return None + + +def _wait_for_product_row(page: Any, product_id: str, timeout_ms: int = 8000) -> Any | None: + deadline = time.time() + timeout_ms / 1000 + while time.time() < deadline: + row = _find_product_row(page, product_id) + if row: + return row + page.wait_for_timeout(300) + return None + + +def _trigger_product_search(page: Any, search: Any) -> None: + wrapper = search.locator("xpath=..") + search_button = _first_visible([ + wrapper.get_by_role("button", name=re.compile(r"search|搜索", re.I)), + wrapper.locator("button"), + wrapper.locator("[role='button']"), + search.locator("xpath=following-sibling::button[1]"), + search.locator("xpath=following-sibling::*[@role='button'][1]"), + ]) + if search_button: + search_button.click(timeout=5000) + else: + search.press("Enter") + + +def _search_product(page: Any, search: Any, query: str, product_id: str, log_dir: Path, label: str) -> Any | None: + search.fill(query) + _trigger_product_search(page, search) + page.wait_for_timeout(1200) + page.screenshot(path=str(log_dir / f"product-search-{label}.png"), full_page=True) + return _wait_for_product_row(page, product_id) + + +def _scan_product_pages(page: Any, product_id: str, log_dir: Path) -> Any | None: + row = _find_product_row(page, product_id) + if row: + return row + dialog = _visible_dialog(page) + root = dialog if dialog else page + page_buttons = root.get_by_role("button", name=re.compile(r"^\d+$")) + page_numbers: list[int] = [] + for index in range(min(page_buttons.count(), 30)): + button = page_buttons.nth(index) + try: + label = button.inner_text().strip() + number = int(label) + if button.is_visible() and 1 <= number <= 30 and number not in page_numbers: + page_numbers.append(number) + except Exception: + continue + for number in sorted(page_numbers): + button = _first_visible([ + root.get_by_role("button", name=re.compile(rf"^{number}$")), + ]) + if not button: + continue + try: + if button.get_attribute("aria-current") == "page": + continue + except Exception: + pass + button.click(timeout=5000) + page.wait_for_timeout(1200) + page.screenshot(path=str(log_dir / f"product-page-{number}.png"), full_page=True) + row = _wait_for_product_row(page, product_id, 3000) + if row: + return row + return None + + +def _add_product_link(page: Any, product_id: str, log_dir: Path) -> None: + product = _selected_product(product_id) + add_link = _first_visible([ + page.locator("button[data-e2e*='add-link' i]"), + page.get_by_role("button", name=re.compile(r"^add$|^add link$|^添加$", re.I)), + page.locator("button:has-text('Add')"), + ]) + if not add_link: + raise ManualReviewRequired("已填写商品信息，但未找到 TikTok Studio 的 Add link 控件") + add_link.scroll_into_view_if_needed(timeout=3000) + add_link.click(timeout=5000) + page.wait_for_timeout(800) + dialog = _first_visible([ + page.get_by_role("dialog"), + page.get_by_text(re.compile(r"^add link$|^添加链接$", re.I), exact=True), + ]) + if not dialog: + raise ManualReviewRequired("已点击 Add link，但未检测到商品绑定弹窗") + page.screenshot(path=str(log_dir / "product-link-dialog.png"), full_page=True) + product_list_title = _first_visible([ + page.get_by_text(re.compile(r"^add product links$|^添加商品链接$", re.I), exact=True), + ]) + if not product_list_title: + next_button = _first_visible([ + page.get_by_role("button", name=re.compile(r"^next$|^下一步$", re.I)), + ]) + if not next_button or not next_button.is_enabled(): + _cancel_product_link(page) + raise ProductLinkUnavailable("Add link 的 Products 步骤未就绪，已取消商品绑定") + next_button.click(timeout=5000) + page.wait_for_timeout(900) + + page.screenshot(path=str(log_dir / "product-list-initial.png"), full_page=True) + row = _find_product_row(page, product["product_id"]) + if not row: + search = _first_visible([ + page.get_by_placeholder(re.compile(r"search products|搜索商品", re.I)), + page.locator("input[placeholder*='search products' i]"), + ]) + if not search: + row = _scan_product_pages(page, product["product_id"], log_dir) + if not row: + _cancel_product_link(page) + raise ProductLinkUnavailable( + f"当前账号没有商品 ID {product['product_id']}，且页面没有搜索框，已遍历分页并取消 Add link" + ) + else: + row = _search_product( + page, search, product["product_id"], product["product_id"], log_dir, "id" + ) + if not row: + row = _scan_product_pages(page, product["product_id"], log_dir) + if not row and product["product_name"]: + row = _search_product( + page, search, product["product_name"], product["product_id"], log_dir, "name" + ) + if not row: + search.fill("") + _trigger_product_search(page, search) + page.wait_for_timeout(1200) + row = _scan_product_pages(page, product["product_id"], log_dir) + if not row: + page.screenshot(path=str(log_dir / "product-not-found.png"), full_page=True) + _cancel_product_link(page) + raise ProductLinkUnavailable( + f"当前账号搜索及分页均未找到商品 ID {product['product_id']}，已取消 Add link" + ) + + row.scroll_into_view_if_needed(timeout=3000) + selector = _first_visible([row.locator("input[type='radio']")]) + if selector: + selector.check(timeout=5000) + else: + row.click(timeout=5000) + page.wait_for_timeout(500) + next_button = _first_visible([ + page.get_by_role("button", name=re.compile(r"^next$|^下一步$", re.I)), + ]) + if not next_button or not next_button.is_enabled(): + _cancel_product_link(page) + raise ProductLinkUnavailable(f"商品 ID {product['product_id']} 未能选中，已取消 Add link") + page.screenshot(path=str(log_dir / "product-selected.png"), full_page=True) + next_button.click(timeout=5000) + product_name_hint = page.get_by_text( + re.compile(r"product name will appear on your video|商品名称.*视频", re.I) + ) + try: + product_name_hint.wait_for(state="visible", timeout=15000) + except Exception as exc: + page.screenshot(path=str(log_dir / "product-next-failed.png"), full_page=True) + if _handle_parameter_popup(page, log_dir, "product-next"): + try: + product_name_hint.wait_for(state="visible", timeout=10000) + except Exception as retry_exc: + raise ManualReviewRequired("LLM 已处理 Next 后的提示，但仍未进入商品名称确认页") from retry_exc + else: + raise ManualReviewRequired("商品已选中，但点击 Next 后未进入商品名称确认页") from exc + + detail_dialog = _visible_dialog(page) + if not detail_dialog: + raise ManualReviewRequired("商品名称确认页已出现，但未检测到 Add product links 弹窗") + product_name_input = _first_visible([ + detail_dialog.get_by_role("textbox"), + detail_dialog.locator("input:not([type='hidden'])"), + ]) + if not product_name_input or not str(product_name_input.input_value() or "").strip(): + raise ManualReviewRequired("商品名称确认页没有可用的默认商品名称") + add_button = _first_visible([ + detail_dialog.get_by_role("button", name=re.compile(r"^add$|^添加$", re.I)), + ]) + if not add_button or not add_button.is_enabled(): + raise ManualReviewRequired("商品名称确认页的 Add 按钮不可用") + page.screenshot(path=str(log_dir / "product-name-ready.png"), full_page=True) + add_button.click(timeout=5000) + try: + detail_dialog.wait_for(state="hidden", timeout=15000) + except Exception as exc: + page.screenshot(path=str(log_dir / "product-add-failed.png"), full_page=True) + raise ManualReviewRequired("点击 Add 后商品绑定弹窗未关闭") from exc + product_name = _first_visible([ + page.get_by_text(re.compile(re.escape(product["product_name"][:24]), re.I)), + ]) + if not product_name: + page.screenshot(path=str(log_dir / "product-link-unconfirmed.png"), full_page=True) + raise ManualReviewRequired("商品弹窗已关闭，但页面未显示所选商品，无法确认绑定成功") + page.screenshot(path=str(log_dir / "product-linked.png"), full_page=True) + + +def _execute_browser(job: dict[str, Any], session: dict[str, Any]) -> tuple[str, str]: + from playwright.sync_api import sync_playwright + + log_dir = LOG_ROOT / job["id"] + log_dir.mkdir(parents=True, exist_ok=True) + video = video_path(job["asset_id"]) + final_clicked = False + with sync_playwright() as playwright: + browser = playwright.chromium.connect_over_cdp(f"http://127.0.0.1:{session['debug_port']}") + try: + context = browser.contexts[0] + page = context.pages[0] if context.pages else context.new_page() + try: + page.wait_for_load_state("domcontentloaded", timeout=20000) + except Exception: + pass + page.wait_for_timeout(2000) + def page_status(status: str, stage: str) -> Any: + def update(event: dict[str, Any]) -> None: + _set_job( + job["id"], + status, + stage, + session_id=session["id"], + status_detail=event.get("message", ""), + ) + return update + + _ensure_studio_page(page, log_dir, page_status("preparing", "loading_studio")) + _assert_account_ready(page) + _skip_onboarding(page) + _ensure_upload_page(page, log_dir, page_status("preparing", "loading_upload_page")) + _skip_onboarding(page) + _assert_account_ready(page) + _discard_stale_edit(page, log_dir) + _set_job(job["id"], "uploading", "uploading", session_id=session["id"]) + _set_video_file(page, video) + page.wait_for_timeout(3000) + _dismiss_upload_prompts(page) + if job["manual_publish"]: + page.screenshot(path=str(log_dir / "manual-ready.png"), full_page=True) + raise ManualPublishReady("视频已上传，等待在 noVNC 中手动填写参数并发布") + _run_parameter_step(page, log_dir, "description", lambda: _set_description(page, job["description"])) + _run_parameter_step(page, log_dir, "ai-generated", lambda: _set_ai_generated(page, bool(job["ai_generated"]))) + _run_parameter_step( + page, + log_dir, + "schedule", + lambda: _set_schedule(page, job["schedule_mode"], job["scheduled_at"], log_dir), + ) + post_button = _first_visible([ + page.get_by_role("button", name=re.compile(r"^post$|^publish$|^发布$", re.I)), + page.locator("button[data-e2e*='post']"), + ]) + if not post_button: + raise RuntimeError("未找到 TikTok Studio 最终发布按钮") + def upload_failure() -> str: + upload_error = _first_visible([ + page.get_by_text(re.compile(r"upload failed|couldn't upload|上传失败|处理失败", re.I)), + ]) + return "TikTok Studio 报告视频上传或处理失败" if upload_error else "" + + try: + wait_for_page_state( + page, + label="视频上传处理", + ready=lambda: bool(post_button.is_enabled()), + failure=upload_failure, + on_status=page_status("uploading", "processing_video"), + timeout_seconds=UPLOAD_TIMEOUT_SECONDS, + reload_attempts=0, + diagnostic_dir=log_dir, + diagnostic_step="video-processing", + ) + except BrowserPageBlocked as exc: + raise ManualReviewRequired(str(exc)) from exc + page.screenshot(path=str(log_dir / "ready-to-publish.png"), full_page=True) + if job["product_link"]: + _add_product_link(page, str(job["product_link"]), log_dir) + if DRY_RUN or not FINAL_CLICK_ENABLED: + return "dry_run", page.url + _set_job(job["id"], "publishing", "final_click", session_id=session["id"], final_click_at=_iso()) + post_button.click() + final_clicked = True + try: + page.wait_for_url(re.compile(r"tiktokstudio(?!/upload)|manage|content"), timeout=45000) + except Exception: + success = _first_visible([ + page.get_by_text(re.compile(r"uploaded|published|scheduled|上传成功|发布成功|已定时", re.I)), + ]) + if not success: + raise ResultUncertain("已点击发布，但未收到明确成功信号，请人工确认，系统不会自动重试") + return ("scheduled_on_tiktok" if job["schedule_mode"] == "tiktok" else "published"), page.url + except (ManualReviewRequired, ManualPublishReady, ProductLinkReviewRequired, ProductLinkUnavailable, ResultUncertain): + raise + except Exception as exc: + if final_clicked: + raise ResultUncertain(f"最终发布后页面异常：{exc}") from exc + raise + finally: + try: + current_page = context.pages[0] if context.pages else None + if current_page: + current_page.screenshot(path=str(log_dir / "last-state.png"), full_page=True) + except Exception: + pass + # Leaving the CDP browser alive here lets proxy_pool own process cleanup + # and preserves the same noVNC channel for manual review when required. + + +def _run_job(job_id: str) -> None: + session_id = 0 + keep_for_review = False + reused_observation = False + keep_observing = False + try: + jobs = _job_query("j.id = ?", (job_id,)) + if not jobs: + return + job = jobs[0] + keep_observing = bool(job.get("keep_observing")) + requested_session_id = int(job.get("session_id") or 0) + session = proxy_pool.claim_observation_session_for_job(int(job["account_id"]), requested_session_id, job_id) + if session is not None: + reused_observation = True + else: + session = proxy_pool.start_automation_session(int(job["account_id"]), job_id)["session"] + session_id = int(session["id"]) + _set_job(job_id, "preparing", "browser_ready", session_id=session_id) + status, result_url = _execute_browser(job, session) + actual = job["scheduled_at"] if status == "scheduled_on_tiktok" else (_iso() if status == "published" else "") + _set_job(job_id, status, "complete", result_url=result_url, actual_publish_at=actual) + if status in {"published", "scheduled_on_tiktok"}: + _update_account(int(job["account_id"]), published_at=actual) + if status == "dry_run" and session_id: + keep_for_review = True + proxy_pool.handoff_automation_session(session_id, "演练已到达最终发布前，保留观测通道") + elif keep_observing and session_id: + keep_for_review = True + proxy_pool.handoff_automation_session(session_id, "立即发布完成，保留观测通道") + except ProductLinkReviewRequired as exc: + keep_for_review = True + _set_job(job_id, "product_link_review", "product_link_review", str(exc), session_id=session_id or None) + if session_id: + proxy_pool.handoff_automation_session(session_id, str(exc)) + except ManualPublishReady as exc: + keep_for_review = True + _set_job(job_id, "manual_ready", "manual_ready", str(exc), session_id=session_id or None) + if session_id: + proxy_pool.handoff_automation_session(session_id, str(exc)) + except ProductLinkUnavailable as exc: + keep_for_review = True + _set_job(job_id, "product_link_failed", "product_link_failed", str(exc), session_id=session_id or None) + _update_account(int(job["account_id"]), error=str(exc)) + if session_id: + proxy_pool.handoff_automation_session(session_id, str(exc)) + except ManualReviewRequired as exc: + keep_for_review = True + _set_job(job_id, "failed", "manual_review", str(exc), session_id=session_id or None) + _update_account(int(job["account_id"]), error=str(exc)) + if session_id: + proxy_pool.handoff_automation_session(session_id, str(exc)) + except ResultUncertain as exc: + keep_for_review = True + _set_job(job_id, "result_uncertain", "confirm_required", str(exc), session_id=session_id or None) + _update_account(int(job["account_id"]), error=str(exc)) + if session_id: + proxy_pool.handoff_automation_session(session_id, str(exc)) + except Exception as exc: + message = str(exc) + if "槽位已满" in message or "已经处于唤醒状态" in message: + _set_job(job_id, "delayed", "waiting_slot", message, next_attempt_at=_iso(_utc_now() + timedelta(seconds=30))) + elif 'job' in locals() and proxy_pool.is_retryable_proxy_error(message): + _delay_for_proxy(job_id, job, message) + elif 'job' in locals() and isinstance(exc, BrowserPageTimeout) and int(job.get("attempt_count") or 0) < 3: + _delay_for_page(job_id, job, message) + else: + _set_job(job_id, "failed", "failed", message, session_id=session_id or None) + if 'job' in locals(): + _update_account(int(job["account_id"]), error=message) + if session_id and (DRY_RUN or reused_observation or keep_observing): + keep_for_review = True + proxy_pool.handoff_automation_session(session_id, f"发布失败，保留观测通道：{message}") + finally: + if session_id and reused_observation and not keep_for_review: + try: + proxy_pool.release_observation_session_job(session_id, job_id) + except Exception as exc: + print(f"Publish observation session release failed for {job_id}: {exc}", flush=True) + elif session_id and not keep_for_review: + try: + proxy_pool.finish_automation_session(session_id) + except Exception as exc: + print(f"Publish session cleanup failed for {job_id}: {exc}", flush=True) + with _worker_lock: + _active_jobs.discard(job_id) + + +def _claim_due_jobs() -> list[str]: + now = _utc_now() + with _worker_lock: + capacity = max(0, proxy_pool.browser_max_slots() - len(_active_jobs)) + if capacity <= 0: + return [] + claimed: list[str] = [] + with proxy_pool.connect() as conn: + conn.execute("BEGIN IMMEDIATE") + rows = conn.execute( + """ + SELECT p.id, p.account_id FROM publish_jobs p + WHERE p.status IN ('queued','delayed') + AND EXISTS ( + SELECT 1 FROM tiktok_accounts a + WHERE a.id = p.account_id AND a.deleted_at = '' AND a.proxy_bound = 1 + ) + AND (p.next_attempt_at = '' OR p.next_attempt_at <= ?) + AND (p.manual_publish = 1 + OR p.schedule_mode = 'tiktok' + OR (p.schedule_mode = 'server' AND p.scheduled_at <= ?)) + AND NOT EXISTS ( + SELECT 1 FROM collect_jobs c + WHERE c.account_id = p.account_id + AND c.status IN ('preparing','collecting') + ) + AND NOT EXISTS ( + SELECT 1 FROM publish_jobs running + WHERE running.account_id = p.account_id AND running.deleted_at = '' + AND running.status IN ('preparing','uploading','publishing') + ) + ORDER BY p.scheduled_at ASC LIMIT ? + """, + (_iso(now), _iso(now), capacity * 4), + ).fetchall() + claimed_accounts: set[int] = set() + for row in rows: + account_id = int(row["account_id"]) + if account_id in claimed_accounts or len(claimed) >= capacity: + continue + job_id = str(row["id"]) + changed = conn.execute( + "UPDATE publish_jobs SET status = 'preparing', stage = 'claimed', status_detail = '', attempt_count = attempt_count + 1, next_attempt_at = '', updated_at = ? WHERE id = ? AND status IN ('queued','delayed')", + (_iso(), job_id), + ).rowcount + if changed: + claimed.append(job_id) + claimed_accounts.add(account_id) + conn.commit() + return claimed + + +def _recover_interrupted() -> None: + now = _iso() + proxy_failures: list[tuple[int, str]] = [] + with proxy_pool.connect() as conn: + conn.execute( + "UPDATE publish_jobs SET status = CASE WHEN final_click_at <> '' THEN 'result_uncertain' ELSE 'queued' END, stage = 'recovered', status_detail = '服务器重启，任务状态已恢复', last_error = '服务器重启后恢复任务', session_id = NULL, next_attempt_at = '', updated_at = ? WHERE status IN ('preparing','uploading','publishing')", + (now,), + ) + retry_at = _iso(_utc_now() + timedelta(seconds=proxy_pool.PROXY_QUEUE_RECHECK_SECONDS)) + rows = conn.execute( + "SELECT id, proxy_profile_id, last_error FROM publish_jobs WHERE status = 'failed' AND final_click_at = '' AND deleted_at = ''" + ).fetchall() + for row in rows: + message = str(row["last_error"] or "") + if not proxy_pool.is_retryable_proxy_error(message): + continue + conn.execute( + "UPDATE publish_jobs SET status = 'delayed', stage = 'waiting_proxy', status_detail = '', session_id = NULL, next_attempt_at = ?, updated_at = ? WHERE id = ?", + (retry_at, now, row["id"]), + ) + proxy_failures.append((int(row["proxy_profile_id"]), message)) + conn.commit() + for pool_id, message in proxy_failures: + proxy_pool.schedule_proxy_recheck_for_pending_job(pool_id, message) + + +def _worker_loop() -> None: + while True: + try: + for job_id in _claim_due_jobs(): + with _worker_lock: + if job_id in _active_jobs: + continue + _active_jobs.add(job_id) + threading.Thread(target=_run_job, args=(job_id,), daemon=True, name=f"tiktok-publish-{job_id[:8]}").start() + except Exception as exc: + print(f"TikTok publish scheduler failed: {exc}", flush=True) + time.sleep(5) + + +def start_worker() -> None: + global _worker_started + with _worker_lock: + if _worker_started: + return + _worker_started = True + PUBLISH_ROOT.mkdir(parents=True, exist_ok=True) + LOG_ROOT.mkdir(parents=True, exist_ok=True) + with proxy_pool.connect(): + pass + _recover_interrupted() + threading.Thread(target=_worker_loop, daemon=True, name="tiktok-publish-scheduler").start() diff --git a/scripts/web_app.py b/scripts/web_app.py index 3ba617f..704caef 100644 --- a/scripts/web_app.py +++ b/scripts/web_app.py @@ -4,23 +4,26 @@ import binascii import hmac import http.client +import math import mimetypes import os import re import shutil +import sqlite3 import subprocess import tempfile import threading import time import uuid from dataclasses import dataclass, field -from datetime import datetime +from datetime import datetime, timedelta from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any -from urllib.parse import parse_qs, quote_plus, urlparse +from urllib.parse import parse_qs, quote, quote_plus, urlparse from urllib.parse import unquote +from zoneinfo import ZoneInfo import cgi from html import escape as html_escape from html import unescape as html_unescape @@ -95,9 +98,17 @@ import sys sys.path.insert(0, str(SCRIPTS_DIR)) -from chat_session import ChatStore, Message, Session, load_sessions_from_disk +from chat_session import ChatStore, Message, Session, load_sessions_from_disk +from feishu_capabilities import FeishuCapabilityClient, FeishuCapabilityError +from lan_chat import ( + FILE_TRANSFER_MAX_BYTES, + MESSAGE_MEDIA_MAX_BYTES, + LanChatError, + LanChatStore, +) from sociavault_usage import read_sociavault_usage from sociavault_tiktok import call_api as call_sociavault_tiktok_api +import sociavault_tiktok_shop from tools import TOOLS, execute_tool, get_tools_for_model, list_tools from video_queue import video_queue, STATUS_META from api_cache import get_cached_or_call, record_api_call @@ -131,6 +142,9 @@ set_hidden_from_analyzer, ) from proxy_state import ensure_us_proxy +import proxy_pool +import tiktok_studio_publish +import tiktok_studio_collect MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024 SAFE_CHARS = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-") AUDIO_ONLY_SUFFIXES = {".aac", ".flac", ".m4a", ".mp3", ".ogg", ".opus", ".wav"} @@ -330,7 +344,9 @@ class AmazonJob: social_jobs_running: set[str] = set() # Chat system -chat_store = ChatStore(DATA_DIR / "sessions.json") +chat_store = ChatStore(DATA_DIR / "sessions.json") +lan_chat_store = LanChatStore(DATA_DIR / "lan_chat.sqlite") +feishu_capability_client = FeishuCapabilityClient() chat_provider_stores = { "home": chat_store, "amazon": ChatStore(SELLERSPRITE_CHAT_DATA_DIR / "chat_sessions.json"), @@ -345,17 +361,22 @@ class AmazonJob: "fastmoss": {"system", "fastmoss"}, } FORCED_MCP_CHAT_PROVIDERS = {"amazon", "fastmoss"} -MCP_TOOL_CACHE: dict[str, dict[str, Any]] = {} -NAV_ITEMS = [ - {"key": "home", "href": "/", "label": "\u9996\u9875", "title": "AI \u804a\u5929", "icon": '