Merge pull request #957 from muzimu/fix_bilibili_top

fix(bilibili): 修复缺失置顶评论

media_platform/bilibili/client.py

@@ -44,6 +44,16 @@ from .field import CommentOrderType, SearchOrderType
 from .help import BilibiliSign
 
 
+def _extract_pinned_comments(value: Any) -> List[Dict]:
+    if isinstance(value, dict):
+        if value.get("rpid") is not None:
+            return [value]
+        return [comment for child in value.values() for comment in _extract_pinned_comments(child)]
+    if isinstance(value, (list, tuple)):
+        return [comment for child in value for comment in _extract_pinned_comments(child)]
+    return []
+
+
 class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
 
     def __init__(
@@ -280,6 +290,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
         next_page = 0
         max_retries = 3
         while not is_end and len(result) < max_count:
+            is_first_page = next_page == 0
             comments_res = None
             for attempt in range(max_retries):
                 try:
@@ -302,7 +313,25 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
                 utils.logger.warning(f"[BilibiliClient.get_video_all_comments] Could not find 'cursor' in response for video_id: {video_id}. Skipping.")
                 break
 
-            comment_list: List[Dict] = comments_res.get("replies", [])
+            comment_list: List[Dict] = comments_res.get("replies") or []
+
+            # The first page carries pinned comments separately from replies.
+            if is_first_page:
+                pinned_comments = _extract_pinned_comments(
+                    (comments_res.get("top"), comments_res.get("top_replies"))
+                )
+                pinned_ids = set()
+                unique_pinned_comments: List[Dict] = []
+                for comment in pinned_comments:
+                    comment_id = str(comment["rpid"])
+                    if comment_id not in pinned_ids:
+                        pinned_ids.add(comment_id)
+                        unique_pinned_comments.append(comment)
+                comment_list = unique_pinned_comments + [
+                    comment
+                    for comment in comment_list
+                    if str(comment.get("rpid")) not in pinned_ids
+                ]
 
             # Check if is_end and next exist
             if "is_end" not in cursor_info or "next" not in cursor_info:
@@ -319,7 +348,14 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
                 for comment in comment_list:
                     comment_id = comment['rpid']
                     if (comment.get("rcount", 0) > 0):
-                        {await self.get_video_all_level_two_comments(video_id, comment_id, CommentOrderType.DEFAULT, 10, crawl_interval, callback)}
+                        await self.get_video_all_level_two_comments(
+                            video_id,
+                            comment_id,
+                            CommentOrderType.DEFAULT,
+                            10,
+                            crawl_interval,
+                            callback,
+                        )
             if len(result) + len(comment_list) > max_count:
                 comment_list = comment_list[:max_count - len(result)]
             if callback:  # If there is a callback function, execute it

tests/test_bilibili_client_comments.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""
+教学版回归测试(B站 bilibili):确保视频评论首页的置顶评论及楼中楼不会遗漏。
+
+覆盖:
+1. top/top_replies 与 replies 重复时,置顶评论只回调一次并触发楼中楼抓取。
+2. 兼容 top 直接返回评论对象的接口形态。
+"""
+
+import pytest
+
+from media_platform.bilibili.client import BilibiliClient
+
+
+@pytest.mark.asyncio
+async def test_video_comments_include_pinned_comment_once_and_fetch_replies():
+    client = object.__new__(BilibiliClient)
+    pinned = {"rpid": 193769108192, "rcount": 6}
+    regular = {"rpid": 193434771680, "rcount": 0}
+    callbacks = []
+    sub_comment_calls = []
+
+    async def get_video_comments(video_id, order_mode, next_page):
+        return {
+            "cursor": {"is_end": True, "next": 0},
+            "replies": [pinned.copy(), regular],
+            "top": {"upper": pinned},
+            "top_replies": [pinned],
+        }
+
+    async def get_video_all_level_two_comments(
+        video_id, comment_id, order_mode, ps, crawl_interval, callback
+    ):
+        sub_comment_calls.append(comment_id)
+
+    async def callback(video_id, comments):
+        callbacks.append([comment["rpid"] for comment in comments])
+
+    client.get_video_comments = get_video_comments
+    client.get_video_all_level_two_comments = get_video_all_level_two_comments
+
+    await client.get_video_all_comments(
+        video_id="323173868",
+        crawl_interval=0,
+        is_fetch_sub_comments=True,
+        callback=callback,
+        max_count=10,
+    )
+
+    assert callbacks == [[pinned["rpid"], regular["rpid"]]]
+    assert sub_comment_calls == [pinned["rpid"]]
+
+
+@pytest.mark.asyncio
+async def test_video_comments_accept_direct_top_comment():
+    client = object.__new__(BilibiliClient)
+    callbacks = []
+
+    async def get_video_comments(video_id, order_mode, next_page):
+        return {
+            "cursor": {"is_end": True, "next": 0},
+            "replies": [],
+            "top": {"rpid": 1, "rcount": 0},
+        }
+
+    async def callback(video_id, comments):
+        callbacks.extend(comments)
+
+    client.get_video_comments = get_video_comments
+
+    await client.get_video_all_comments(
+        video_id="323173868", crawl_interval=0, callback=callback, max_count=10
+    )
+
+    assert [comment["rpid"] for comment in callbacks] == [1]