fork/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-10-09 09:38:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 支持爬取小红书二级评论

Browse Source
This commit is contained in:
leantli
2024-04-11 17:16:13 +08:00
parent bba9841c26
commit 81a9946afd
8 changed files with 232 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
store/xhs/xhs_store_impl.py
+42
View File
@@ -81,6 +81,17 @@ class XhsCsvStoreImplement(AbstractStore):
"""
await self.save_data_to_csv(save_item=creator, store_type="creator")
async def store_sub_comment(self, sub_comment_items: Dict):
"""
Xiaohongshu sub_comments CSV storage implementation
Args:
sub_comment_items: sub_comments item dict
Returns:
"""
await self.save_data_to_csv(save_item=sub_comment_items, store_type="sub_comments")
class XhsDbStoreImplement(AbstractStore):
@@ -142,6 +153,26 @@ class XhsDbStoreImplement(AbstractStore):
await add_new_creator(creator)
else:
await update_creator_by_user_id(user_id, creator)
async def store_sub_comment(self, sub_comment_items: Dict):
"""
Xiaohongshu content DB storage implementation
Args:
sub_comment_items: comment item dict
Returns:
"""
from .xhs_store_sql import (add_new_sub_comment,
query_sub_comment_by_comment_id,
update_sub_comment_by_comment_id)
comment_id = sub_comment_items.get("comment_id")
comment_detail: Dict = await query_sub_comment_by_comment_id(comment_id=comment_id)
if not comment_detail:
sub_comment_items["add_ts"] = utils.get_current_timestamp()
await add_new_sub_comment(sub_comment_items)
else:
await update_sub_comment_by_comment_id(comment_id, comment_item=sub_comment_items)
class XhsJsonStoreImplement(AbstractStore):
@@ -214,3 +245,14 @@ class XhsJsonStoreImplement(AbstractStore):
"""
await self.save_data_to_json(creator, "creator")
async def store_sub_comment(self, sub_comment_items: Dict):
"""
sub_comment JSON storage implementatio
Args:
sub_comment_items:
Returns:
"""
await self.save_data_to_json(sub_comment_items, "sub_comments")