mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-10-03 23:48:12 +08:00
refactor: 教学版移除全平台用户个人信息采集与持久化
- 用户 ID 转为匿名 creator_hash,昵称中间脱敏,IP/头像/主页/签名/性别不再采集 - 覆盖 xhs/weibo/bilibili/douyin/kuaishou/tieba/zhihu 7 个平台 - 删除 7 张 creator 档案 ORM 表,15 张内容/评论表新增 creator_hash 列 - B 站禁用粉丝/关注/联系人列表抓取 - 新增 tools/user_hash.py 与 4 个平台的 mock+SQLite 端到端测试 测试: pytest tests/test_no_user_info.py tests/test_weibo_no_user_info.py tests/test_douyin_no_user_info.py tests/test_kuaishou_no_user_info.py (21 passed)
This commit is contained in:
+7
-45
@@ -25,6 +25,7 @@ from typing import List
|
||||
|
||||
import config
|
||||
from var import source_keyword_var
|
||||
from tools.user_hash import anonymize_user_id, mask_nickname
|
||||
|
||||
from .xhs_store_media import *
|
||||
from ._store_impl import *
|
||||
@@ -113,14 +114,12 @@ async def update_xhs_note(note_item: Dict):
|
||||
"video_url": video_url, # Note video url
|
||||
"time": note_item.get("time"), # Note publish time
|
||||
"last_update_time": note_item.get("last_update_time", 0), # Note last update time
|
||||
"user_id": user_info.get("user_id"), # User ID
|
||||
"nickname": user_info.get("nickname"), # User nickname
|
||||
"avatar": user_info.get("avatar"), # User avatar
|
||||
"creator_hash": anonymize_user_id(user_info.get("user_id")), # 创作者匿名哈希(不存原始 user_id)
|
||||
"nickname": mask_nickname(user_info.get("nickname")), # 用户昵称(已脱敏)
|
||||
"liked_count": interact_info.get("liked_count"), # Like count
|
||||
"collected_count": interact_info.get("collected_count"), # Collection count
|
||||
"comment_count": interact_info.get("comment_count"), # Comment count
|
||||
"share_count": interact_info.get("share_count"), # Share count
|
||||
"ip_location": note_item.get("ip_location", ""), # IP location
|
||||
"image_list": ','.join([img.get('url', '') for img in image_list]), # Image URLs
|
||||
"tag_list": ','.join([tag.get('name', '') for tag in tag_list if tag.get('type') == 'topic']), # Tags
|
||||
"last_modify_ts": utils.get_current_timestamp(), # Last modification timestamp (Generated by MediaCrawler, mainly used to record the latest update time of a record in DB storage)
|
||||
@@ -165,12 +164,10 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict):
|
||||
local_db_item = {
|
||||
"comment_id": comment_id, # Comment ID
|
||||
"create_time": comment_item.get("create_time"), # Comment time
|
||||
"ip_location": comment_item.get("ip_location"), # IP location
|
||||
"note_id": note_id, # Note ID
|
||||
"content": comment_item.get("content"), # Comment content
|
||||
"user_id": user_info.get("user_id"), # User ID
|
||||
"nickname": user_info.get("nickname"), # User nickname
|
||||
"avatar": user_info.get("image"), # User avatar
|
||||
"creator_hash": anonymize_user_id(user_info.get("user_id")), # 创作者匿名哈希(不存原始 user_id)
|
||||
"nickname": mask_nickname(user_info.get("nickname")), # 用户昵称(已脱敏)
|
||||
"sub_comment_count": comment_item.get("sub_comment_count", 0), # Sub-comment count
|
||||
"pictures": ",".join(comment_pictures), # Comment pictures
|
||||
"parent_comment_id": target_comment.get("id", 0), # Parent comment ID
|
||||
@@ -191,43 +188,8 @@ async def save_creator(user_id: str, creator: Dict):
|
||||
Returns:
|
||||
|
||||
"""
|
||||
user_info = creator.get('basicInfo', {})
|
||||
|
||||
follows = 0
|
||||
fans = 0
|
||||
interaction = 0
|
||||
for i in creator.get('interactions'):
|
||||
if i.get('type') == 'follows':
|
||||
follows = i.get('count')
|
||||
elif i.get('type') == 'fans':
|
||||
fans = i.get('count')
|
||||
elif i.get('type') == 'interaction':
|
||||
interaction = i.get('count')
|
||||
|
||||
def get_gender(gender):
|
||||
if gender == 1:
|
||||
return 'Female'
|
||||
elif gender == 0:
|
||||
return 'Male'
|
||||
else:
|
||||
return None
|
||||
|
||||
local_db_item = {
|
||||
'user_id': user_id, # User ID
|
||||
'nickname': user_info.get('nickname'), # Nickname
|
||||
'gender': get_gender(user_info.get('gender')), # Gender
|
||||
'avatar': user_info.get('images'), # Avatar
|
||||
'desc': user_info.get('desc'), # Personal description
|
||||
'ip_location': user_info.get('ipLocation'), # IP location
|
||||
'follows': follows, # Following count
|
||||
'fans': fans, # Fans count
|
||||
'interaction': interaction, # Interaction count
|
||||
'tag_list': json.dumps({tag.get('tagType'): tag.get('name')
|
||||
for tag in creator.get('tags')}, ensure_ascii=False), # Tags
|
||||
"last_modify_ts": utils.get_current_timestamp(), # Last modification timestamp (Generated by MediaCrawler, mainly used to record the latest update time of a record in DB storage)
|
||||
}
|
||||
utils.logger.info(f"[store.xhs.save_creator] creator:{local_db_item}")
|
||||
await XhsStoreFactory.create_store().store_creator(local_db_item)
|
||||
# 教学版:创作者个人资料(昵称/性别/头像/IP/粉丝数等)不再落库,防骚扰。
|
||||
return
|
||||
|
||||
|
||||
async def update_xhs_note_image(note_id, pic_content, extension_file_name):
|
||||
|
||||
Reference in New Issue
Block a user