mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-10-03 18:18:12 +08:00
refactor: 教学版移除全平台用户个人信息采集与持久化
- 用户 ID 转为匿名 creator_hash,昵称中间脱敏,IP/头像/主页/签名/性别不再采集 - 覆盖 xhs/weibo/bilibili/douyin/kuaishou/tieba/zhihu 7 个平台 - 删除 7 张 creator 档案 ORM 表,15 张内容/评论表新增 creator_hash 列 - B 站禁用粉丝/关注/联系人列表抓取 - 新增 tools/user_hash.py 与 4 个平台的 mock+SQLite 端到端测试 测试: pytest tests/test_no_user_info.py tests/test_weibo_no_user_info.py tests/test_douyin_no_user_info.py tests/test_kuaishou_no_user_info.py (21 passed)
This commit is contained in:
@@ -30,7 +30,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from base.base_crawler import AbstractStore
|
||||
from database.db_session import get_session
|
||||
from database.models import XhsNote, XhsNoteComment, XhsCreator
|
||||
from database.models import XhsNote, XhsNoteComment
|
||||
|
||||
from tools.async_file_writer import AsyncFileWriter
|
||||
from tools.time_util import get_current_timestamp
|
||||
@@ -137,10 +137,8 @@ class XhsDbStoreImplement(AbstractStore):
|
||||
add_ts = int(get_current_timestamp())
|
||||
last_modify_ts = int(get_current_timestamp())
|
||||
note = XhsNote(
|
||||
user_id=content_item.get("user_id"),
|
||||
creator_hash=content_item.get("creator_hash"),
|
||||
nickname=content_item.get("nickname"),
|
||||
avatar=content_item.get("avatar"),
|
||||
ip_location=content_item.get("ip_location"),
|
||||
add_ts=add_ts,
|
||||
last_modify_ts=last_modify_ts,
|
||||
note_id=content_item.get("note_id"),
|
||||
@@ -197,10 +195,8 @@ class XhsDbStoreImplement(AbstractStore):
|
||||
add_ts = int(get_current_timestamp())
|
||||
last_modify_ts = int(get_current_timestamp())
|
||||
comment = XhsNoteComment(
|
||||
user_id=comment_item.get("user_id"),
|
||||
creator_hash=comment_item.get("creator_hash"),
|
||||
nickname=comment_item.get("nickname"),
|
||||
avatar=comment_item.get("avatar"),
|
||||
ip_location=comment_item.get("ip_location"),
|
||||
add_ts=add_ts,
|
||||
last_modify_ts=last_modify_ts,
|
||||
comment_id=comment_item.get("comment_id"),
|
||||
@@ -231,54 +227,8 @@ class XhsDbStoreImplement(AbstractStore):
|
||||
return result.first() is not None
|
||||
|
||||
async def store_creator(self, creator_item: Dict):
|
||||
user_id = creator_item.get("user_id")
|
||||
if not user_id:
|
||||
return
|
||||
async with get_session() as session:
|
||||
if await self.creator_is_exist(session, user_id):
|
||||
await self.update_creator(session, creator_item)
|
||||
else:
|
||||
await self.add_creator(session, creator_item)
|
||||
|
||||
async def add_creator(self, session: AsyncSession, creator_item: Dict):
|
||||
add_ts = int(get_current_timestamp())
|
||||
last_modify_ts = int(get_current_timestamp())
|
||||
creator = XhsCreator(
|
||||
user_id=creator_item.get("user_id"),
|
||||
nickname=creator_item.get("nickname"),
|
||||
avatar=creator_item.get("avatar"),
|
||||
ip_location=creator_item.get("ip_location"),
|
||||
add_ts=add_ts,
|
||||
last_modify_ts=last_modify_ts,
|
||||
desc=creator_item.get("desc"),
|
||||
gender=creator_item.get("gender"),
|
||||
follows=str(creator_item.get("follows")),
|
||||
fans=str(creator_item.get("fans")),
|
||||
interaction=str(creator_item.get("interaction")),
|
||||
tag_list=json.dumps(creator_item.get("tag_list"))
|
||||
)
|
||||
session.add(creator)
|
||||
|
||||
async def update_creator(self, session: AsyncSession, creator_item: Dict):
|
||||
user_id = creator_item.get("user_id")
|
||||
last_modify_ts = int(get_current_timestamp())
|
||||
update_data = {
|
||||
"last_modify_ts": last_modify_ts,
|
||||
"nickname": creator_item.get("nickname"),
|
||||
"avatar": creator_item.get("avatar"),
|
||||
"desc": creator_item.get("desc"),
|
||||
"follows": str(creator_item.get("follows")),
|
||||
"fans": str(creator_item.get("fans")),
|
||||
"interaction": str(creator_item.get("interaction")),
|
||||
"tag_list": json.dumps(creator_item.get("tag_list"))
|
||||
}
|
||||
stmt = update(XhsCreator).where(XhsCreator.user_id == user_id).values(**update_data)
|
||||
await session.execute(stmt)
|
||||
|
||||
async def creator_is_exist(self, session: AsyncSession, user_id: str) -> bool:
|
||||
stmt = select(XhsCreator).where(XhsCreator.user_id == user_id)
|
||||
result = await session.execute(stmt)
|
||||
return result.first() is not None
|
||||
# 教学版:创作者个人资料不再落库
|
||||
pass
|
||||
|
||||
async def get_all_content(self) -> List[Dict]:
|
||||
async with get_session() as session:
|
||||
@@ -345,16 +295,8 @@ class XhsMongoStoreImplement(AbstractStore):
|
||||
Args:
|
||||
creator_item: Creator data
|
||||
"""
|
||||
user_id = creator_item.get("user_id")
|
||||
if not user_id:
|
||||
return
|
||||
|
||||
await self.mongo_store.save_or_update(
|
||||
collection_suffix="creators",
|
||||
query={"user_id": user_id},
|
||||
data=creator_item
|
||||
)
|
||||
utils.logger.info(f"[XhsMongoStoreImplement.store_creator] Saved creator {user_id} to MongoDB")
|
||||
# 教学版:创作者个人资料不再落库
|
||||
pass
|
||||
|
||||
|
||||
class XhsExcelStoreImplement:
|
||||
|
||||
Reference in New Issue
Block a user