mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-10-03 02:28:12 +08:00
fix(xhs,bilibili): 修复访问受限异常击穿与评论采集边界问题
xhs: PR #958 把 IPBlockError / PlatformAccessError 加入 request() 的 retry_if_not_exception_type 后,tenacity 会直接重抛原异常而不再包装成 RetryError,core 层的 except 分支接不住,单条笔记被限流会让整批 asyncio.gather 抛出,同批已抓取但未入库的数据全部丢失。 - get_note_detail_async_task / get_creators_and_notes 捕获访问受限异常, 记录明确日志后跳过当前条目,恢复原有的"跳过并继续"语义 - 移除 request() 中已不可达的 IP_ERROR_CODE 分支 bilibili: 修复 get_video_all_comments 的两处翻页边界问题 - is_first_page 改为独立标志,接口返回 next=0 且 is_end=False 时 不再把后续页误判为首页而重复注入置顶评论 - result 无条件累加,否则开启楼中楼抓取时循环守卫永不推进, max_count 完全失效并可能死循环;截断提前到抓取楼中楼之前, 避免为已被丢弃的评论抓子评论
This commit is contained in:
@@ -73,3 +73,79 @@ async def test_video_comments_accept_direct_top_comment():
|
||||
)
|
||||
|
||||
assert [comment["rpid"] for comment in callbacks] == [1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pinned_comment_not_reinjected_when_cursor_next_stays_zero():
|
||||
"""cursor.next 仍为 0 时继续翻页,置顶评论不能被重复注入。"""
|
||||
client = object.__new__(BilibiliClient)
|
||||
collected = []
|
||||
page_calls = 0
|
||||
|
||||
async def get_video_comments(video_id, order_mode, next_page):
|
||||
nonlocal page_calls
|
||||
page_calls += 1
|
||||
# 异常/兜底场景:接口一直回 next=0 且 is_end=False
|
||||
return {
|
||||
"cursor": {"is_end": False, "next": 0},
|
||||
"replies": [{"rpid": 2, "rcount": 0}],
|
||||
"top": {"upper": {"rpid": 1, "rcount": 0}},
|
||||
}
|
||||
|
||||
async def callback(video_id, comments):
|
||||
collected.extend(comment["rpid"] for comment in comments)
|
||||
|
||||
client.get_video_comments = get_video_comments
|
||||
|
||||
await client.get_video_all_comments(
|
||||
video_id="323173868", crawl_interval=0, callback=callback, max_count=5
|
||||
)
|
||||
|
||||
assert page_calls > 1, "该用例需要真正翻过页才有意义"
|
||||
assert collected.count(1) == 1, f"置顶评论被重复采集: {collected}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_count_applies_when_fetching_sub_comments():
|
||||
"""开启楼中楼抓取时 max_count 依然生效,且不为被截断的评论抓子评论。"""
|
||||
client = object.__new__(BilibiliClient)
|
||||
collected = []
|
||||
sub_comment_calls = []
|
||||
page_calls = 0
|
||||
|
||||
async def get_video_comments(video_id, order_mode, next_page):
|
||||
nonlocal page_calls
|
||||
page_calls += 1
|
||||
base = page_calls * 10
|
||||
return {
|
||||
"cursor": {"is_end": False, "next": page_calls},
|
||||
"replies": [
|
||||
{"rpid": base + 1, "rcount": 1},
|
||||
{"rpid": base + 2, "rcount": 1},
|
||||
],
|
||||
}
|
||||
|
||||
async def get_video_all_level_two_comments(
|
||||
video_id, comment_id, order_mode, ps, crawl_interval, callback
|
||||
):
|
||||
sub_comment_calls.append(comment_id)
|
||||
|
||||
async def callback(video_id, comments):
|
||||
collected.extend(comment["rpid"] for comment in comments)
|
||||
|
||||
client.get_video_comments = get_video_comments
|
||||
client.get_video_all_level_two_comments = get_video_all_level_two_comments
|
||||
|
||||
result = await client.get_video_all_comments(
|
||||
video_id="323173868",
|
||||
crawl_interval=0,
|
||||
is_fetch_sub_comments=True,
|
||||
callback=callback,
|
||||
max_count=3,
|
||||
)
|
||||
|
||||
# is_end 永远是 False,只能靠 max_count 收敛
|
||||
assert len(result) == 3
|
||||
assert collected == [11, 12, 21]
|
||||
# 第 2 页的 rpid=22 被截断,不应该再去抓它的楼中楼
|
||||
assert sub_comment_calls == [11, 12, 21]
|
||||
|
||||
Reference in New Issue
Block a user