fork/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-10-02 21:48:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(bilibili): 修复缺失置顶评论

Browse Source
This commit is contained in:
muzimu
2026-08-08 18:01:26 +08:00
parent 071c8c0aca
commit 17c6d386b8
2 changed files with 113 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media_platform/bilibili/client.py
+38 -2
View File
@@ -44,6 +44,16 @@ from .field import CommentOrderType, SearchOrderType
from .help import BilibiliSign
def _extract_pinned_comments(value: Any) -> List[Dict]:
if isinstance(value, dict):
if value.get("rpid") is not None:
return [value]
return [comment for child in value.values() for comment in _extract_pinned_comments(child)]
if isinstance(value, (list, tuple)):
return [comment for child in value for comment in _extract_pinned_comments(child)]
return []
class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
def __init__(
@@ -280,6 +290,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
next_page = 0
max_retries = 3
while not is_end and len(result) < max_count:
is_first_page = next_page == 0
comments_res = None
for attempt in range(max_retries):
try:
@@ -302,7 +313,25 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
utils.logger.warning(f"[BilibiliClient.get_video_all_comments] Could not find 'cursor' in response for video_id: {video_id}. Skipping.")
break
comment_list: List[Dict] = comments_res.get("replies", [])
comment_list: List[Dict] = comments_res.get("replies") or []
# The first page carries pinned comments separately from replies.
if is_first_page:
pinned_comments = _extract_pinned_comments(
(comments_res.get("top"), comments_res.get("top_replies"))
)
pinned_ids = set()
unique_pinned_comments: List[Dict] = []
for comment in pinned_comments:
comment_id = str(comment["rpid"])
if comment_id not in pinned_ids:
pinned_ids.add(comment_id)
unique_pinned_comments.append(comment)
comment_list = unique_pinned_comments + [
comment
for comment in comment_list
if str(comment.get("rpid")) not in pinned_ids
]
# Check if is_end and next exist
if "is_end" not in cursor_info or "next" not in cursor_info:
@@ -319,7 +348,14 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
for comment in comment_list:
comment_id = comment['rpid']
if (comment.get("rcount", 0) > 0):
{await self.get_video_all_level_two_comments(video_id, comment_id, CommentOrderType.DEFAULT, 10, crawl_interval, callback)}
await self.get_video_all_level_two_comments(
video_id,
comment_id,
CommentOrderType.DEFAULT,
10,
crawl_interval,
callback,
)
if len(result) + len(comment_list) > max_count:
comment_list = comment_list[:max_count - len(result)]
if callback: # If there is a callback function, execute it