fork/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-10-10 11:00:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 轻量化支持爬取小红书二级评论

Browse Source
This commit is contained in:
leantli
2024-04-12 17:32:20 +08:00
parent 81a9946afd
commit ad01dfba95
8 changed files with 52 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files
store/xhs/xhs_store_impl.py
+1 -42
View File
@@ -81,17 +81,6 @@ class XhsCsvStoreImplement(AbstractStore):
"""
await self.save_data_to_csv(save_item=creator, store_type="creator")
async def store_sub_comment(self, sub_comment_items: Dict):
"""
Xiaohongshu sub_comments CSV storage implementation
Args:
sub_comment_items: sub_comments item dict
Returns:
"""
await self.save_data_to_csv(save_item=sub_comment_items, store_type="sub_comments")
class XhsDbStoreImplement(AbstractStore):
@@ -154,26 +143,6 @@ class XhsDbStoreImplement(AbstractStore):
else:
await update_creator_by_user_id(user_id, creator)
async def store_sub_comment(self, sub_comment_items: Dict):
"""
Xiaohongshu content DB storage implementation
Args:
sub_comment_items: comment item dict
Returns:
"""
from .xhs_store_sql import (add_new_sub_comment,
query_sub_comment_by_comment_id,
update_sub_comment_by_comment_id)
comment_id = sub_comment_items.get("comment_id")
comment_detail: Dict = await query_sub_comment_by_comment_id(comment_id=comment_id)
if not comment_detail:
sub_comment_items["add_ts"] = utils.get_current_timestamp()
await add_new_sub_comment(sub_comment_items)
else:
await update_sub_comment_by_comment_id(comment_id, comment_item=sub_comment_items)
class XhsJsonStoreImplement(AbstractStore):
json_store_path: str = "data/xhs"
@@ -245,14 +214,4 @@ class XhsJsonStoreImplement(AbstractStore):
"""
await self.save_data_to_json(creator, "creator")
async def store_sub_comment(self, sub_comment_items: Dict):
"""
sub_comment JSON storage implementatio
Args:
sub_comment_items:
Returns:
"""
await self.save_data_to_json(sub_comment_items, "sub_comments")