diff --git a/media_platform/bilibili/client.py b/media_platform/bilibili/client.py index a9c3262..7f46096 100644 --- a/media_platform/bilibili/client.py +++ b/media_platform/bilibili/client.py @@ -44,6 +44,16 @@ from .field import CommentOrderType, SearchOrderType from .help import BilibiliSign +def _extract_pinned_comments(value: Any) -> List[Dict]: + if isinstance(value, dict): + if value.get("rpid") is not None: + return [value] + return [comment for child in value.values() for comment in _extract_pinned_comments(child)] + if isinstance(value, (list, tuple)): + return [comment for child in value for comment in _extract_pinned_comments(child)] + return [] + + class BilibiliClient(AbstractApiClient, ProxyRefreshMixin): def __init__( @@ -280,6 +290,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin): next_page = 0 max_retries = 3 while not is_end and len(result) < max_count: + is_first_page = next_page == 0 comments_res = None for attempt in range(max_retries): try: @@ -302,7 +313,25 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin): utils.logger.warning(f"[BilibiliClient.get_video_all_comments] Could not find 'cursor' in response for video_id: {video_id}. Skipping.") break - comment_list: List[Dict] = comments_res.get("replies", []) + comment_list: List[Dict] = comments_res.get("replies") or [] + + # The first page carries pinned comments separately from replies. + if is_first_page: + pinned_comments = _extract_pinned_comments( + (comments_res.get("top"), comments_res.get("top_replies")) + ) + pinned_ids = set() + unique_pinned_comments: List[Dict] = [] + for comment in pinned_comments: + comment_id = str(comment["rpid"]) + if comment_id not in pinned_ids: + pinned_ids.add(comment_id) + unique_pinned_comments.append(comment) + comment_list = unique_pinned_comments + [ + comment + for comment in comment_list + if str(comment.get("rpid")) not in pinned_ids + ] # Check if is_end and next exist if "is_end" not in cursor_info or "next" not in cursor_info: @@ -319,7 +348,14 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin): for comment in comment_list: comment_id = comment['rpid'] if (comment.get("rcount", 0) > 0): - {await self.get_video_all_level_two_comments(video_id, comment_id, CommentOrderType.DEFAULT, 10, crawl_interval, callback)} + await self.get_video_all_level_two_comments( + video_id, + comment_id, + CommentOrderType.DEFAULT, + 10, + crawl_interval, + callback, + ) if len(result) + len(comment_list) > max_count: comment_list = comment_list[:max_count - len(result)] if callback: # If there is a callback function, execute it diff --git a/tests/test_bilibili_client_comments.py b/tests/test_bilibili_client_comments.py new file mode 100644 index 0000000..48bb66f --- /dev/null +++ b/tests/test_bilibili_client_comments.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +教学版回归测试(B站 bilibili):确保视频评论首页的置顶评论及楼中楼不会遗漏。 + +覆盖: +1. top/top_replies 与 replies 重复时,置顶评论只回调一次并触发楼中楼抓取。 +2. 兼容 top 直接返回评论对象的接口形态。 +""" + +import pytest + +from media_platform.bilibili.client import BilibiliClient + + +@pytest.mark.asyncio +async def test_video_comments_include_pinned_comment_once_and_fetch_replies(): + client = object.__new__(BilibiliClient) + pinned = {"rpid": 193769108192, "rcount": 6} + regular = {"rpid": 193434771680, "rcount": 0} + callbacks = [] + sub_comment_calls = [] + + async def get_video_comments(video_id, order_mode, next_page): + return { + "cursor": {"is_end": True, "next": 0}, + "replies": [pinned.copy(), regular], + "top": {"upper": pinned}, + "top_replies": [pinned], + } + + async def get_video_all_level_two_comments( + video_id, comment_id, order_mode, ps, crawl_interval, callback + ): + sub_comment_calls.append(comment_id) + + async def callback(video_id, comments): + callbacks.append([comment["rpid"] for comment in comments]) + + client.get_video_comments = get_video_comments + client.get_video_all_level_two_comments = get_video_all_level_two_comments + + await client.get_video_all_comments( + video_id="323173868", + crawl_interval=0, + is_fetch_sub_comments=True, + callback=callback, + max_count=10, + ) + + assert callbacks == [[pinned["rpid"], regular["rpid"]]] + assert sub_comment_calls == [pinned["rpid"]] + + +@pytest.mark.asyncio +async def test_video_comments_accept_direct_top_comment(): + client = object.__new__(BilibiliClient) + callbacks = [] + + async def get_video_comments(video_id, order_mode, next_page): + return { + "cursor": {"is_end": True, "next": 0}, + "replies": [], + "top": {"rpid": 1, "rcount": 0}, + } + + async def callback(video_id, comments): + callbacks.extend(comments) + + client.get_video_comments = get_video_comments + + await client.get_video_all_comments( + video_id="323173868", crawl_interval=0, callback=callback, max_count=10 + ) + + assert [comment["rpid"] for comment in callbacks] == [1]