mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-10-04 06:08:12 +08:00
fix(xhs,bilibili): 修复访问受限异常击穿与评论采集边界问题
xhs: PR #958 把 IPBlockError / PlatformAccessError 加入 request() 的 retry_if_not_exception_type 后,tenacity 会直接重抛原异常而不再包装成 RetryError,core 层的 except 分支接不住,单条笔记被限流会让整批 asyncio.gather 抛出,同批已抓取但未入库的数据全部丢失。 - get_note_detail_async_task / get_creators_and_notes 捕获访问受限异常, 记录明确日志后跳过当前条目,恢复原有的"跳过并继续"语义 - 移除 request() 中已不可达的 IP_ERROR_CODE 分支 bilibili: 修复 get_video_all_comments 的两处翻页边界问题 - is_first_page 改为独立标志,接口返回 next=0 且 is_end=False 时 不再把后续页误判为首页而重复注入置顶评论 - result 无条件累加,否则开启楼中楼抓取时循环守卫永不推进, max_count 完全失效并可能死循环;截断提前到抓取楼中楼之前, 避免为已被丢弃的评论抓子评论
This commit is contained in:
@@ -188,8 +188,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
data: Dict = response_data if response_data is not None else response.json()
|
||||
if data["success"]:
|
||||
return data.get("data", data.get("success", {}))
|
||||
elif data["code"] == self.IP_ERROR_CODE:
|
||||
raise IPBlockError(self.IP_ERROR_STR)
|
||||
# IP_ERROR_CODE / SECURITY_LIMIT_CODE are already handled above, before return_response.
|
||||
elif data["code"] in (self.NOTE_NOT_FOUND_CODE, self.NOTE_ABNORMAL_CODE):
|
||||
raise NoteNotFoundError(f"Note not found or abnormal, code: {data['code']}")
|
||||
else:
|
||||
|
||||
@@ -42,7 +42,12 @@ from tools.cdp_browser import CDPBrowserManager
|
||||
from var import crawler_type_var, source_keyword_var
|
||||
|
||||
from .client import XiaoHongShuClient
|
||||
from .exception import DataFetchError, NoteNotFoundError
|
||||
from .exception import (
|
||||
DataFetchError,
|
||||
IPBlockError,
|
||||
NoteNotFoundError,
|
||||
PlatformAccessError,
|
||||
)
|
||||
from .field import SearchSortType
|
||||
from .help import parse_note_info_from_note_url, parse_creator_info_from_url, get_search_id
|
||||
from .login import XiaoHongShuLogin
|
||||
@@ -206,6 +211,13 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
except ValueError as e:
|
||||
utils.logger.error(f"[XiaoHongShuCrawler.get_creators_and_notes] Failed to parse creator URL: {e}")
|
||||
continue
|
||||
except (IPBlockError, PlatformAccessError) as e:
|
||||
# Access restricted on the creator homepage, skip this creator instead of crashing the run.
|
||||
utils.logger.error(
|
||||
f"[XiaoHongShuCrawler.get_creators_and_notes] Access restricted for creator {creator_url}: {e}. "
|
||||
f"建议降低采集频率、更换 IP 或检查账号状态"
|
||||
)
|
||||
continue
|
||||
|
||||
# Use fixed crawling interval
|
||||
crawl_interval = config.CRAWLER_MAX_SLEEP_SEC
|
||||
@@ -316,6 +328,14 @@ class XiaoHongShuCrawler(AbstractCrawler):
|
||||
except NoteNotFoundError as ex:
|
||||
utils.logger.warning(f"[XiaoHongShuCrawler.get_note_detail_async_task] Note not found: {note_id}, {ex}")
|
||||
return None
|
||||
except (IPBlockError, PlatformAccessError) as ex:
|
||||
# Access restricted (IP block / rate limit / account security).
|
||||
# Skip this note instead of aborting the whole asyncio.gather batch.
|
||||
utils.logger.error(
|
||||
f"[XiaoHongShuCrawler.get_note_detail_async_task] Access restricted while getting note {note_id}: {ex}. "
|
||||
f"建议降低采集频率、更换 IP 或检查账号状态"
|
||||
)
|
||||
return None
|
||||
except DataFetchError as ex:
|
||||
utils.logger.error(f"[XiaoHongShuCrawler.get_note_detail_async_task] Get note detail error: {ex}")
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user