refactor: 教学版移除全平台用户个人信息采集与持久化

- 用户 ID 转为匿名 creator_hash,昵称中间脱敏,IP/头像/主页/签名/性别不再采集
- 覆盖 xhs/weibo/bilibili/douyin/kuaishou/tieba/zhihu 7 个平台
- 删除 7 张 creator 档案 ORM 表,15 张内容/评论表新增 creator_hash 列
- B 站禁用粉丝/关注/联系人列表抓取
- 新增 tools/user_hash.py 与 4 个平台的 mock+SQLite 端到端测试

测试: pytest tests/test_no_user_info.py tests/test_weibo_no_user_info.py tests/test_douyin_no_user_info.py tests/test_kuaishou_no_user_info.py (21 passed)
This commit is contained in:
程序员阿江(Relakkes)
2026-07-01 13:09:55 +08:00
parent 8b4d8fcffa
commit 9f4f8bf768
26 changed files with 1426 additions and 822 deletions
+10 -10
View File
@@ -459,11 +459,11 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
}
return await self.get(uri, params)
async def get_all_anwser_by_creator(self, creator: ZhihuCreator, crawl_interval: float = 1.0, callback: Optional[Callable] = None) -> List[ZhihuContent]:
async def get_all_anwser_by_creator(self, url_token: str, crawl_interval: float = 1.0, callback: Optional[Callable] = None) -> List[ZhihuContent]:
"""
Get all answers by creator
Args:
creator: Creator information
url_token: Creator url token (in-memory only, not persisted)
crawl_interval: Crawl delay interval in seconds
callback: Callback after completing one crawl
@@ -475,10 +475,10 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
offset: int = 0
limit: int = 20
while not is_end:
res = await self.get_creator_answers(creator.url_token, offset, limit)
res = await self.get_creator_answers(url_token, offset, limit)
if not res:
break
utils.logger.info(f"[ZhiHuClient.get_all_anwser_by_creator] Get creator {creator.url_token} answers: {res}")
utils.logger.info(f"[ZhiHuClient.get_all_anwser_by_creator] Get creator {url_token} answers: {res}")
paging_info = res.get("paging", {})
is_end = paging_info.get("is_end")
contents = self._extractor.extract_content_list_from_creator(res.get("data"))
@@ -491,14 +491,14 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_all_articles_by_creator(
self,
creator: ZhihuCreator,
url_token: str,
crawl_interval: float = 1.0,
callback: Optional[Callable] = None,
) -> List[ZhihuContent]:
"""
Get all articles by creator
Args:
creator:
url_token: Creator url token (in-memory only, not persisted)
crawl_interval:
callback:
@@ -510,7 +510,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
offset: int = 0
limit: int = 20
while not is_end:
res = await self.get_creator_articles(creator.url_token, offset, limit)
res = await self.get_creator_articles(url_token, offset, limit)
if not res:
break
paging_info = res.get("paging", {})
@@ -525,14 +525,14 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_all_videos_by_creator(
self,
creator: ZhihuCreator,
url_token: str,
crawl_interval: float = 1.0,
callback: Optional[Callable] = None,
) -> List[ZhihuContent]:
"""
Get all videos by creator
Args:
creator:
url_token: Creator url token (in-memory only, not persisted)
crawl_interval:
callback:
@@ -544,7 +544,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
offset: int = 0
limit: int = 20
while not is_end:
res = await self.get_creator_videos(creator.url_token, offset, limit)
res = await self.get_creator_videos(url_token, offset, limit)
if not res:
break
paging_info = res.get("paging", {})