fix(xhs,bilibili): 修复访问受限异常击穿与评论采集边界问题

xhs: PR #958 把 IPBlockError / PlatformAccessError 加入 request() 的
retry_if_not_exception_type 后,tenacity 会直接重抛原异常而不再包装成
RetryError,core 层的 except 分支接不住,单条笔记被限流会让整批
asyncio.gather 抛出,同批已抓取但未入库的数据全部丢失。

- get_note_detail_async_task / get_creators_and_notes 捕获访问受限异常,
  记录明确日志后跳过当前条目,恢复原有的"跳过并继续"语义
- 移除 request() 中已不可达的 IP_ERROR_CODE 分支

bilibili: 修复 get_video_all_comments 的两处翻页边界问题

- is_first_page 改为独立标志,接口返回 next=0 且 is_end=False 时
  不再把后续页误判为首页而重复注入置顶评论
- result 无条件累加,否则开启楼中楼抓取时循环守卫永不推进,
  max_count 完全失效并可能死循环;截断提前到抓取楼中楼之前,
  避免为已被丢弃的评论抓子评论
This commit is contained in:
程序员阿江(Relakkes)
2026-08-11 18:10:16 +08:00
parent 3c25521bbb
commit d594c20c13
5 changed files with 231 additions and 9 deletions
+1 -2
View File
@@ -188,8 +188,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
data: Dict = response_data if response_data is not None else response.json()
if data["success"]:
return data.get("data", data.get("success", {}))
elif data["code"] == self.IP_ERROR_CODE:
raise IPBlockError(self.IP_ERROR_STR)
# IP_ERROR_CODE / SECURITY_LIMIT_CODE are already handled above, before return_response.
elif data["code"] in (self.NOTE_NOT_FOUND_CODE, self.NOTE_ABNORMAL_CODE):
raise NoteNotFoundError(f"Note not found or abnormal, code: {data['code']}")
else:
+21 -1
View File
@@ -42,7 +42,12 @@ from tools.cdp_browser import CDPBrowserManager
from var import crawler_type_var, source_keyword_var
from .client import XiaoHongShuClient
from .exception import DataFetchError, NoteNotFoundError
from .exception import (
DataFetchError,
IPBlockError,
NoteNotFoundError,
PlatformAccessError,
)
from .field import SearchSortType
from .help import parse_note_info_from_note_url, parse_creator_info_from_url, get_search_id
from .login import XiaoHongShuLogin
@@ -206,6 +211,13 @@ class XiaoHongShuCrawler(AbstractCrawler):
except ValueError as e:
utils.logger.error(f"[XiaoHongShuCrawler.get_creators_and_notes] Failed to parse creator URL: {e}")
continue
except (IPBlockError, PlatformAccessError) as e:
# Access restricted on the creator homepage, skip this creator instead of crashing the run.
utils.logger.error(
f"[XiaoHongShuCrawler.get_creators_and_notes] Access restricted for creator {creator_url}: {e}. "
f"建议降低采集频率、更换 IP 或检查账号状态"
)
continue
# Use fixed crawling interval
crawl_interval = config.CRAWLER_MAX_SLEEP_SEC
@@ -316,6 +328,14 @@ class XiaoHongShuCrawler(AbstractCrawler):
except NoteNotFoundError as ex:
utils.logger.warning(f"[XiaoHongShuCrawler.get_note_detail_async_task] Note not found: {note_id}, {ex}")
return None
except (IPBlockError, PlatformAccessError) as ex:
# Access restricted (IP block / rate limit / account security).
# Skip this note instead of aborting the whole asyncio.gather batch.
utils.logger.error(
f"[XiaoHongShuCrawler.get_note_detail_async_task] Access restricted while getting note {note_id}: {ex}. "
f"建议降低采集频率、更换 IP 或检查账号状态"
)
return None
except DataFetchError as ex:
utils.logger.error(f"[XiaoHongShuCrawler.get_note_detail_async_task] Get note detail error: {ex}")
return None