refactor: 教学版移除全平台用户个人信息采集与持久化

- 用户 ID 转为匿名 creator_hash，昵称中间脱敏，IP/头像/主页/签名/性别不再采集 - 覆盖 xhs/weibo/bilibili/douyin/kuaishou/tieba/zhihu 7 个平台 - 删除 7 张 creator 档案 ORM 表，15 张内容/评论表新增 creator_hash 列 - B 站禁用粉丝/关注/联系人列表抓取 - 新增 tools/user_hash.py 与 4 个平台的 mock+SQLite 端到端测试测试: pytest tests/test_no_user_info.py tests/test_weibo_no_user_info.py tests/test_douyin_no_user_info.py tests/test_kuaishou_no_user_info.py (21 passed)
2026-10-07 21:48:11 +08:00 · 2026-07-01 13:09:55 +08:00
parent 8b4d8fcffa
commit 9f4f8bf768
26 changed files with 1426 additions and 822 deletions
@@ -0,0 +1,284 @@
+# -*- coding: utf-8 -*-
+"""
+教学版回归测试：快手(kuaishou)存储链路不再持久化可定位真人的用户个人信息。
+
+覆盖：
+1. update_kuaishou_video —— mock video_item(含 author.id/name/headerUrl + photo 内容字段 + type)
+   经 FakeStore 捕获，断言捕获 dict 不含禁用键(user_id/avatar/signature/ip_location/gender)、
+   含 creator_hash(≠原 user_id)、nickname 脱敏(≠原文)。
+2. update_ks_video_comment —— V2(snake_case) 与旧 GraphQL(camelCase) 两种 comment_item 格式各测一次。
+3. test_kuaishou_store_end_to_end_sqlite —— 端到端：FakeStore 捕获真实 dict ->
+   KuaishouVideo(**captured) 触发 ORM 列校验 -> 写入内存 SQLite -> 查询回读校验属性。
+
+约束：只新建本测试文件，只读源码不改源码；不依赖网络/登录。
+"""
+import asyncio
+import contextlib
+import types
+
+import pytest
+
+import store.kuaishou as ks
+from store.kuaishou import update_kuaishou_video, update_ks_video_comment
+from tools.user_hash import anonymize_user_id, mask_nickname
+
+# 教学版禁用字段(键)：一律不得出现在存储 dict 中。
+# 昵称字段 nickname 允许保留，但值须脱敏。
+FORBIDDEN_KEYS = {"user_id", "avatar", "signature", "ip_location", "gender"}
+
+# ----------------------------- mock 数据 -----------------------------
+
+MOCK_VIDEO_ID = "3xf8e9kq2b7w4"
+MOCK_AUTHOR_ID = "ks_author_001"
+MOCK_AUTHOR_NAME = "快手达人"
+MOCK_CAPTION = "这是一条测试视频，教学版脱敏回归 #测试"
+
+# 评论作者(两种格式用不同 id/昵称，便于分别断言)
+MOCK_COMMENT_V2_AUTHOR_ID = "ks_user_888"
+MOCK_COMMENT_V2_AUTHOR_NAME = "快手老铁"
+MOCK_COMMENT_LEGACY_AUTHOR_ID = "ks_user_777"
+MOCK_COMMENT_LEGACY_AUTHOR_NAME = "快乐源泉"
+
+
+def make_mock_video() -> dict:
+    """贴近真实快手结构的 video_item：author 在顶层，photo 含内容字段。
+    author.headerUrl 与各禁用字段一样不应进入存储 dict。"""
+    return {
+        "type": 1,
+        "photo": {
+            "id": MOCK_VIDEO_ID,
+            "caption": MOCK_CAPTION,
+            "timestamp": 1700000000000,
+            "coverUrl": "https://p.kuaishou.com/cover/abc.jpg",
+            "photoUrl": "https://v.kuaishou.com/play/abc.mp4",
+            "realLikeCount": 12345,
+            "viewCount": 67890,
+        },
+        "author": {
+            "id": MOCK_AUTHOR_ID,
+            "name": MOCK_AUTHOR_NAME,
+            "headerUrl": "https://p.kuaishou.com/header/u001.jpg",
+        },
+    }
+
+
+def make_mock_comment_v2() -> dict:
+    """V2 API 格式：snake_case 字段名，comment_id 为 int。"""
+    return {
+        "comment_id": 9001,
+        "timestamp": 1700000001234,
+        "content": "太搞笑了哈哈哈",
+        "author_id": MOCK_COMMENT_V2_AUTHOR_ID,
+        "author_name": MOCK_COMMENT_V2_AUTHOR_NAME,
+        "headurl": "https://p.kuaishou.com/header/u888.jpg",
+        "commentCount": 7,
+    }
+
+
+def make_mock_comment_legacy() -> dict:
+    """旧 GraphQL API 格式：camelCase 字段名。"""
+    return {
+        "commentId": 8001,
+        "timestamp": 1700000005678,
+        "content": "这条评论来自旧 GraphQL 接口",
+        "authorId": MOCK_COMMENT_LEGACY_AUTHOR_ID,
+        "authorName": MOCK_COMMENT_LEGACY_AUTHOR_NAME,
+        "headurl": "https://p.kuaishou.com/header/u777.jpg",
+        "subCommentCount": 3,
+    }
+
+
+# ----------------------------- FakeStore 捕获 -----------------------------
+
+
+@contextlib.contextmanager
+def _patch_create_store():
+    """把 KuaishouStoreFactory.create_store 替换为返回捕获用 FakeStore 的 staticmethod。
+    FakeStore 把 store_content/store_comment 收到的 dict 原样写入 holder.content / holder.comment。
+
+    保存/还原走类 __dict__ 中的 staticmethod 描述符本身，避免把 staticmethod 退化为普通方法
+    而污染后续测试。"""
+    holder = types.SimpleNamespace(content={}, comment={})
+
+    class FakeStore:
+        async def store_content(self, content_item):
+            holder.content.clear()
+            holder.content.update(content_item)
+
+        async def store_comment(self, comment_item):
+            holder.comment.clear()
+            holder.comment.update(comment_item)
+
+        async def store_creator(self, creator):
+            pass
+
+    orig = ks.KuaishouStoreFactory.__dict__["create_store"]
+    ks.KuaishouStoreFactory.create_store = staticmethod(lambda: FakeStore())
+    try:
+        yield holder
+    finally:
+        ks.KuaishouStoreFactory.create_store = orig
+
+
+def _assert_no_forbidden_keys(d: dict, label: str):
+    hit = set(d.keys()) & FORBIDDEN_KEYS
+    assert not hit, f"[{label}] 输出仍含禁用字段键: {hit}"
+
+
+# ----------------------------- 测试 -----------------------------
+
+
+def test_kuaishou_video_masks_user_info():
+    """video 链路：原始 user_id 转 creator_hash、昵称脱敏、headerUrl/禁用键不落库。"""
+    with _patch_create_store() as holder:
+        asyncio.run(update_kuaishou_video(make_mock_video()))
+    captured = holder.content
+
+    assert captured, "FakeStore 未捕获到 content_item"
+    _assert_no_forbidden_keys(captured, "kuaishou_video")
+    # 头像字段不应进入存储 dict(author.headerUrl 已被丢弃)
+    assert "headerUrl" not in captured and "avatar" not in captured
+
+    # creator_hash 存在且不等于原始 user_id
+    assert captured.get("creator_hash") == anonymize_user_id(MOCK_AUTHOR_ID)
+    assert captured["creator_hash"] != MOCK_AUTHOR_ID
+    assert captured["creator_hash"]  # 非空
+
+    # 昵称已脱敏：等于 mask_nickname(原文) 且不等于原文
+    assert captured.get("nickname") == mask_nickname(MOCK_AUTHOR_NAME)
+    assert captured["nickname"] != MOCK_AUTHOR_NAME
+    assert "*" in captured["nickname"]
+
+    # 内容字段保留(video_id / desc / title)
+    assert captured["video_id"] == MOCK_VIDEO_ID
+    assert captured["desc"] == MOCK_CAPTION
+    assert captured["title"] == MOCK_CAPTION
+    # video_type 来自 video_item.type，被 str() 化
+    assert captured["video_type"] == "1"
+    # 计数字段被 str() 化
+    assert captured["liked_count"] == "12345"
+    assert captured["viewd_count"] == "67890"
+
+
+def test_kuaishou_comment_v2_masks_user_info():
+    """评论 V2(snake_case)格式：author_id/author_name 经匿名+脱敏，headurl 不落库。"""
+    with _patch_create_store() as holder:
+        asyncio.run(update_ks_video_comment(MOCK_VIDEO_ID, make_mock_comment_v2()))
+    captured = holder.comment
+
+    assert captured, "FakeStore 未捕获到 comment_item"
+    _assert_no_forbidden_keys(captured, "kuaishou_comment_v2")
+    assert "headurl" not in captured and "avatar" not in captured
+
+    # comment_id 由 int 转为 str
+    assert captured["comment_id"] == "9001"
+    assert captured["video_id"] == MOCK_VIDEO_ID
+    assert captured["content"] == "太搞笑了哈哈哈"
+    # V2 用 commentCount
+    assert captured["sub_comment_count"] == "7"
+
+    # creator_hash / 昵称
+    assert captured["creator_hash"] == anonymize_user_id(MOCK_COMMENT_V2_AUTHOR_ID)
+    assert captured["creator_hash"] != MOCK_COMMENT_V2_AUTHOR_ID
+    assert captured["nickname"] == mask_nickname(MOCK_COMMENT_V2_AUTHOR_NAME)
+    assert captured["nickname"] != MOCK_COMMENT_V2_AUTHOR_NAME
+    assert "*" in captured["nickname"]
+
+
+def test_kuaishou_comment_legacy_masks_user_info():
+    """评论旧 GraphQL(camelCase)格式：authorId/authorName 经匿名+脱敏，headurl 不落库。"""
+    with _patch_create_store() as holder:
+        asyncio.run(update_ks_video_comment(MOCK_VIDEO_ID, make_mock_comment_legacy()))
+    captured = holder.comment
+
+    assert captured, "FakeStore 未捕获到 comment_item"
+    _assert_no_forbidden_keys(captured, "kuaishou_comment_legacy")
+    assert "headurl" not in captured and "avatar" not in captured
+
+    # commentId 由 int 转为 str
+    assert captured["comment_id"] == "8001"
+    assert captured["video_id"] == MOCK_VIDEO_ID
+    assert captured["content"] == "这条评论来自旧 GraphQL 接口"
+    # 旧格式用 subCommentCount
+    assert captured["sub_comment_count"] == "3"
+
+    # creator_hash / 昵称
+    assert captured["creator_hash"] == anonymize_user_id(MOCK_COMMENT_LEGACY_AUTHOR_ID)
+    assert captured["creator_hash"] != MOCK_COMMENT_LEGACY_AUTHOR_ID
+    assert captured["nickname"] == mask_nickname(MOCK_COMMENT_LEGACY_AUTHOR_NAME)
+    assert captured["nickname"] != MOCK_COMMENT_LEGACY_AUTHOR_NAME
+    assert "*" in captured["nickname"]
+
+
+def test_kuaishou_store_end_to_end_sqlite():
+    """端到端：FakeStore 捕获真实 dict -> KuaishouVideo(**captured) ORM 列校验
+    -> 写入内存 SQLite -> 查询回读，验证属性正确且无禁用列。
+
+    全程在同一个事件循环内完成(aiosqlite 连接绑定事件循环，跨 loop 会报错)，
+    不 patch db_session.get_session，而是直接用自建内存 engine 走 ORM 写入/查询。"""
+    from sqlalchemy import select
+    from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
+    from sqlalchemy.orm import sessionmaker
+    from sqlalchemy.pool import StaticPool
+
+    from database.models import Base, KuaishouVideo, KuaishouVideoComment
+
+    async def run():
+        # 内存 SQLite + StaticPool：单连接共享，保证 create_all 与后续读写同一库
+        engine = create_async_engine("sqlite+aiosqlite://", poolclass=StaticPool)
+        SessionFactory = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+        async with engine.begin() as conn:
+            await conn.run_sync(
+                Base.metadata.create_all,
+                tables=[KuaishouVideo.__table__, KuaishouVideoComment.__table__],
+            )
+
+        # 1) 经 update_kuaishou_video 产生真实存储 dict(FakeStore 捕获)
+        with _patch_create_store() as holder:
+            await update_kuaishou_video(make_mock_video())
+        captured = holder.content
+        assert captured, "FakeStore 未捕获到 content_item"
+
+        # 2) ORM 列校验：captured 的所有 key 必须是 KuaishouVideo 的合法列，
+        #    否则 KuaishouVideo(**captured) 抛 TypeError(若有禁用/多余键即暴露源码 bug)
+        valid_cols = {c.name for c in KuaishouVideo.__table__.columns}
+        assert set(captured.keys()) <= valid_cols, (
+            f"captured 含非合法列: {set(captured.keys()) - valid_cols}"
+        )
+        obj = KuaishouVideo(**captured)  # 不抛异常即通过列校验
+        assert obj.video_id == MOCK_VIDEO_ID
+        assert obj.creator_hash == anonymize_user_id(MOCK_AUTHOR_ID)
+        assert obj.creator_hash != MOCK_AUTHOR_ID
+        assert obj.nickname == mask_nickname(MOCK_AUTHOR_NAME)
+        assert obj.nickname != MOCK_AUTHOR_NAME
+        assert obj.desc == MOCK_CAPTION  # 内容保留
+
+        # 3) 真实写库 + 查询回读
+        async with SessionFactory() as session:
+            session.add(obj)
+            await session.commit()
+
+            res = await session.execute(
+                select(KuaishouVideo).where(KuaishouVideo.video_id == MOCK_VIDEO_ID)
+            )
+            row = res.scalar_one()
+            assert row is not None
+            assert row.video_id == MOCK_VIDEO_ID
+            assert row.creator_hash == anonymize_user_id(MOCK_AUTHOR_ID)
+            assert row.creator_hash != MOCK_AUTHOR_ID
+            assert row.nickname == mask_nickname(MOCK_AUTHOR_NAME)
+            assert row.nickname != MOCK_AUTHOR_NAME
+            # 内容字段保留
+            assert row.desc == MOCK_CAPTION
+            assert row.title == MOCK_CAPTION
+            # ORM 行对象上不应存在任何禁用列属性
+            for bad in FORBIDDEN_KEYS:
+                assert not hasattr(row, bad), f"KuaishouVideo 行对象仍含禁用属性: {bad}"
+
+        await engine.dispose()
+
+    asyncio.run(run())
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])