fork/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-10-09 09:28:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 轻量化支持爬取小红书二级评论

Browse Source
This commit is contained in:
leantli
2024-04-12 17:32:20 +08:00
parent 81a9946afd
commit ad01dfba95
8 changed files with 52 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media_platform/xhs/core.py
+7 -51
View File
@@ -123,9 +123,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
note_id_list.append(note_detail.get("note_id"))
page += 1
utils.logger.info(f"[XiaoHongShuCrawler.search] Note details: {note_details}")
note_comments = await self.batch_get_note_comments(note_id_list)
await self.batch_get_sub_comments(note_comments)
await self.batch_get_note_comments(note_id_list)
async def get_creators_and_notes(self) -> None:
"""Get creator's notes and retrieve their comment information."""
@@ -185,11 +183,11 @@ class XiaoHongShuCrawler(AbstractCrawler):
f"[XiaoHongShuCrawler.get_note_detail] have not fund note detail note_id:{note_id}, err: {ex}")
return None
async def batch_get_note_comments(self, note_list: List[str]) -> List[Dict]:
async def batch_get_note_comments(self, note_list: List[str]):
"""Batch get note comments"""
if not config.ENABLE_GET_COMMENTS:
utils.logger.info(f"[XiaoHongShuCrawler.batch_get_note_comments] Crawling comment mode is not enabled")
return None
return
utils.logger.info(
f"[XiaoHongShuCrawler.batch_get_note_comments] Begin batch get note comments, note list: {note_list}")
@@ -198,59 +196,17 @@ class XiaoHongShuCrawler(AbstractCrawler):
for note_id in note_list:
task = asyncio.create_task(self.get_comments(note_id, semaphore), name=note_id)
task_list.append(task)
note_comments = await asyncio.gather(*task_list)
# 这里是每个note_id的评论列表，可以将其合并成一个列表, 方便后续操作
note_comments = [comment for comments in note_comments for comment in comments]
return note_comments
await asyncio.gather(*task_list)
async def get_comments(self, note_id: str, semaphore: asyncio.Semaphore) -> List[Dict]:
async def get_comments(self, note_id: str, semaphore: asyncio.Semaphore):
"""Get note comments with keyword filtering and quantity limitation"""
async with semaphore:
utils.logger.info(f"[XiaoHongShuCrawler.get_comments] Begin get note id comments {note_id}")
return await self.xhs_client.get_note_all_comments(
await self.xhs_client.get_note_all_comments(
note_id=note_id,
crawl_interval=random.random(),
callback=xhs_store.batch_update_xhs_note_comments
)
async def batch_get_sub_comments(self, note_comments: List[Dict]):
"""Batch get note sub_comments"""
if not config.ENABLE_GET_SUB_COMMENTS:
utils.logger.info(f"[XiaoHongShuCrawler.batch_get_sub_comments] Crawling sub_comment mode is not enabled")
return
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list: List[Task] = []
for note_comment in note_comments:
note_id = note_comment.get("note_id")
sub_comments = note_comment.get("sub_comments")
if sub_comments:
await xhs_store.batch_update_xhs_note_sub_comments(note_id, sub_comments)
sub_comment_has_more = note_comment.get("sub_comment_has_more")
if not sub_comment_has_more:
continue
root_comment_id = note_comment.get("id")
sub_comment_cursor = note_comment.get("sub_comment_cursor")
task = asyncio.create_task(self.get_sub_comments(note_id, root_comment_id,
sub_comment_cursor, sub_comments, semaphore), name=note_id)
task_list.append(task)
await asyncio.gather(*task_list)
async def get_sub_comments(self, note_id: str, root_comment_id: str, sub_comment_cursor: str,
sub_comments: List[Dict], semaphore: asyncio.Semaphore) -> List[Dict]:
"""Get note sub_comments with keyword filtering and quantity limitation"""
async with semaphore:
utils.logger.info(f"[XiaoHongShuCrawler.get_sub_comments] Begin get note id sub_comments {note_id}")
await self.xhs_client.get_comment_all_sub_comments(
note_id=note_id,
root_comment_id=root_comment_id,
sub_comment_cursor=sub_comment_cursor,
sub_comments=sub_comments,
crawl_interval=random.random(),
callback=xhs_store.batch_update_xhs_note_sub_comments
)
@staticmethod
def format_proxy_info(ip_proxy_info: IpInfoModel) -> Tuple[Optional[Dict], Optional[Dict]]:
@@ -317,4 +273,4 @@ class XiaoHongShuCrawler(AbstractCrawler):
async def close(self):
"""Close browser context"""
await self.browser_context.close()
utils.logger.info("[XiaoHongShuCrawler.close] Browser context closed ...")
utils.logger.info("[XiaoHongShuCrawler.close] Browser context closed ...")