From 6937b1273853526c0b12531f7e534a00628d76eb Mon Sep 17 00:00:00 2001 From: ottercoconut Date: Tue, 11 Aug 2026 16:45:49 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B0=8F=E7=BA=A2=E4=B9=A6?= =?UTF-8?q?=E5=8E=9F=E5=A7=8B=E5=93=8D=E5=BA=94=E9=94=99=E8=AF=AF=E5=88=86?= =?UTF-8?q?=E7=B1=BB=E4=B8=8E=E9=87=8D=E5=A4=8D=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- media_platform/xhs/client.py | 59 ++++++++++- media_platform/xhs/exception.py | 4 + tests/test_xhs_raw_response_errors.py | 136 ++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 tests/test_xhs_raw_response_errors.py diff --git a/media_platform/xhs/client.py b/media_platform/xhs/client.py index 1e07b08..39bb663 100644 --- a/media_platform/xhs/client.py +++ b/media_platform/xhs/client.py @@ -24,7 +24,13 @@ from urllib.parse import quote, urlencode import httpx from playwright.async_api import BrowserContext, Page -from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_not_exception_type +from tenacity import ( + RetryError, + retry, + retry_if_not_exception_type, + stop_after_attempt, + wait_fixed, +) from tools.httpx_util import make_async_client import config @@ -35,7 +41,12 @@ from tools import utils if TYPE_CHECKING: from proxy.proxy_ip_pool import ProxyIpPool -from .exception import DataFetchError, IPBlockError, NoteNotFoundError +from .exception import ( + DataFetchError, + IPBlockError, + NoteNotFoundError, + PlatformAccessError, +) from .field import SearchNoteType, SearchSortType from .help import get_search_id from .extractor import XiaoHongShuExtractor @@ -66,6 +77,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): self.cookie_urls = [self._domain] self.IP_ERROR_STR = "Network connection error, please check network settings or restart" self.IP_ERROR_CODE = 300012 + self.SECURITY_LIMIT_CODE = 300011 self.NOTE_NOT_FOUND_CODE = -510000 self.NOTE_ABNORMAL_STR = "Note status abnormal, please check later" self.NOTE_ABNORMAL_CODE = -510001 @@ -112,7 +124,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): self.headers.update(headers) return self.headers - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1), retry=retry_if_not_exception_type(NoteNotFoundError)) + @retry( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + retry=retry_if_not_exception_type( + (NoteNotFoundError, IPBlockError, PlatformAccessError) + ), + ) async def request(self, method, url, **kwargs) -> Union[str, Any]: """ Wrapper for httpx common request method, processes request response @@ -132,6 +150,11 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): async with make_async_client(proxy=self.proxy) as client: response = await client.request(method, url, timeout=self.timeout, **kwargs) + if response.status_code in {401, 403, 429}: + raise PlatformAccessError( + f"XHS request blocked with HTTP {response.status_code}" + ) + if response.status_code == 471 or response.status_code == 461: # someday someone maybe will bypass captcha verify_type = response.headers["Verifytype"] @@ -140,9 +163,29 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): utils.logger.error(msg) raise Exception(msg) + response_data: Optional[Dict] = None + try: + candidate_data = response.json() + if isinstance(candidate_data, dict): + response_data = candidate_data + except (TypeError, ValueError): + pass + + response_code = ( + str(response_data.get("code")) + if response_data is not None and response_data.get("code") is not None + else "" + ) + if response_code == str(self.IP_ERROR_CODE): + raise IPBlockError(self.IP_ERROR_STR) + if response_code == str(self.SECURITY_LIMIT_CODE): + raise PlatformAccessError( + f"XHS account security restriction, code: {self.SECURITY_LIMIT_CODE}" + ) + if return_response: return response.text - data: Dict = response.json() + data: Dict = response_data if response_data is not None else response.json() if data["success"]: return data.get("data", data.get("success", {})) elif data["code"] == self.IP_ERROR_CODE: @@ -667,7 +710,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): data = {"original_url": f"{self._domain}/discovery/item/{note_id}"} return await self.post(uri, data=data, return_response=True) - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) + @retry( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + retry=retry_if_not_exception_type( + (RetryError, IPBlockError, PlatformAccessError) + ), + ) async def get_note_by_id_from_html( self, note_id: str, diff --git a/media_platform/xhs/exception.py b/media_platform/xhs/exception.py index a956d93..8d9bfd3 100644 --- a/media_platform/xhs/exception.py +++ b/media_platform/xhs/exception.py @@ -29,5 +29,9 @@ class IPBlockError(RequestError): """fetch so fast that the server block us ip""" +class PlatformAccessError(RequestError): + """authentication, rate-limit, or account-security restriction""" + + class NoteNotFoundError(RequestError): """Note does not exist or is abnormal""" diff --git a/tests/test_xhs_raw_response_errors.py b/tests/test_xhs_raw_response_errors.py new file mode 100644 index 0000000..9e76a4f --- /dev/null +++ b/tests/test_xhs_raw_response_errors.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- + +from unittest.mock import AsyncMock + +import httpx +import pytest +from tenacity import RetryError + +from media_platform.xhs.client import XiaoHongShuClient +from media_platform.xhs.exception import IPBlockError, PlatformAccessError + + +class FakeAsyncClient: + def __init__(self, request_impl): + self.request_impl = request_impl + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback): + return False + + async def request(self, *args, **kwargs): + return await self.request_impl(*args, **kwargs) + + +def make_client(): + client = XiaoHongShuClient( + headers={"Cookie": "web_session=test"}, + playwright_page=object(), + cookie_dict={}, + ) + client._refresh_proxy_if_expired = AsyncMock() + return client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code", [401, 403, 429]) +async def test_raw_response_rejects_access_http_status(monkeypatch, status_code): + calls = 0 + + async def request_impl(method, url, **kwargs): + nonlocal calls + calls += 1 + return httpx.Response( + status_code, + text="blocked", + request=httpx.Request(method, url), + ) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + with pytest.raises(PlatformAccessError): + await make_client().request( + "GET", "https://www.xiaohongshu.com/user/profile/test", return_response=True + ) + + assert calls == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("code", "expected_exception"), + [(300011, PlatformAccessError), ("300012", IPBlockError)], +) +async def test_raw_response_rejects_known_business_block( + monkeypatch, code, expected_exception +): + calls = 0 + + async def request_impl(method, url, **kwargs): + nonlocal calls + calls += 1 + return httpx.Response( + 200, + json={"success": False, "code": code, "msg": "blocked"}, + request=httpx.Request(method, url), + ) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + with pytest.raises(expected_exception): + await make_client().request( + "GET", "https://www.xiaohongshu.com/explore/test", return_response=True + ) + + assert calls == 1 + + +@pytest.mark.asyncio +async def test_raw_response_keeps_successful_html(monkeypatch): + async def request_impl(method, url, **kwargs): + return httpx.Response( + 200, + text="ok", + request=httpx.Request(method, url), + ) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + result = await make_client().request( + "GET", "https://www.xiaohongshu.com/explore/test", return_response=True + ) + + assert result == "ok" + + +@pytest.mark.asyncio +async def test_html_detail_does_not_multiply_transport_retries(monkeypatch): + calls = 0 + + async def request_impl(method, url, **kwargs): + nonlocal calls + calls += 1 + raise httpx.ReadTimeout("timed out", request=httpx.Request(method, url)) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + with pytest.raises(RetryError): + await make_client().get_note_by_id_from_html( + "test", xsec_source="pc_search", xsec_token="token" + ) + + assert calls == 3