mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-10-05 19:58:12 +08:00
fix(xhs,bilibili): 修复访问受限异常击穿与评论采集边界问题
xhs: PR #958 把 IPBlockError / PlatformAccessError 加入 request() 的 retry_if_not_exception_type 后,tenacity 会直接重抛原异常而不再包装成 RetryError,core 层的 except 分支接不住,单条笔记被限流会让整批 asyncio.gather 抛出,同批已抓取但未入库的数据全部丢失。 - get_note_detail_async_task / get_creators_and_notes 捕获访问受限异常, 记录明确日志后跳过当前条目,恢复原有的"跳过并继续"语义 - 移除 request() 中已不可达的 IP_ERROR_CODE 分支 bilibili: 修复 get_video_all_comments 的两处翻页边界问题 - is_first_page 改为独立标志,接口返回 next=0 且 is_end=False 时 不再把后续页误判为首页而重复注入置顶评论 - result 无条件累加,否则开启楼中楼抓取时循环守卫永不推进, max_count 完全失效并可能死循环;截断提前到抓取楼中楼之前, 避免为已被丢弃的评论抓子评论
This commit is contained in:
@@ -289,8 +289,8 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
is_end = False
|
||||
next_page = 0
|
||||
max_retries = 3
|
||||
is_first_page = True
|
||||
while not is_end and len(result) < max_count:
|
||||
is_first_page = next_page == 0
|
||||
comments_res = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
@@ -332,6 +332,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
for comment in comment_list
|
||||
if str(comment.get("rpid")) not in pinned_ids
|
||||
]
|
||||
is_first_page = False
|
||||
|
||||
# Check if is_end and next exist
|
||||
if "is_end" not in cursor_info or "next" not in cursor_info:
|
||||
@@ -344,6 +345,10 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
if not isinstance(is_end, bool):
|
||||
utils.logger.warning(f"[BilibiliClient.get_video_all_comments] 'is_end' is not a boolean for video_id: {video_id}. Assuming end of comments.")
|
||||
is_end = True
|
||||
# Truncate before fetching sub-comments, otherwise max_count neither caps
|
||||
# the stored comments nor stops us from crawling sub-comments we discard.
|
||||
if len(result) + len(comment_list) > max_count:
|
||||
comment_list = comment_list[:max_count - len(result)]
|
||||
if is_fetch_sub_comments:
|
||||
for comment in comment_list:
|
||||
comment_id = comment['rpid']
|
||||
@@ -356,14 +361,12 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
crawl_interval,
|
||||
callback,
|
||||
)
|
||||
if len(result) + len(comment_list) > max_count:
|
||||
comment_list = comment_list[:max_count - len(result)]
|
||||
if callback: # If there is a callback function, execute it
|
||||
await callback(video_id, comment_list)
|
||||
await asyncio.sleep(crawl_interval)
|
||||
if not is_fetch_sub_comments:
|
||||
result.extend(comment_list)
|
||||
continue
|
||||
# Always accumulate, otherwise the `len(result) < max_count` loop guard
|
||||
# never advances when sub-comment crawling is enabled.
|
||||
result.extend(comment_list)
|
||||
return result
|
||||
|
||||
async def get_video_all_level_two_comments(
|
||||
|
||||
Reference in New Issue
Block a user