From 6937b1273853526c0b12531f7e534a00628d76eb Mon Sep 17 00:00:00 2001 From: ottercoconut Date: Tue, 11 Aug 2026 16:45:49 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B0=8F=E7=BA=A2?= =?UTF-8?q?=E4=B9=A6=E5=8E=9F=E5=A7=8B=E5=93=8D=E5=BA=94=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E5=88=86=E7=B1=BB=E4=B8=8E=E9=87=8D=E5=A4=8D=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- media_platform/xhs/client.py | 59 ++++++++++- media_platform/xhs/exception.py | 4 + tests/test_xhs_raw_response_errors.py | 136 ++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 tests/test_xhs_raw_response_errors.py diff --git a/media_platform/xhs/client.py b/media_platform/xhs/client.py index 1e07b08..39bb663 100644 --- a/media_platform/xhs/client.py +++ b/media_platform/xhs/client.py @@ -24,7 +24,13 @@ from urllib.parse import quote, urlencode import httpx from playwright.async_api import BrowserContext, Page -from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_not_exception_type +from tenacity import ( + RetryError, + retry, + retry_if_not_exception_type, + stop_after_attempt, + wait_fixed, +) from tools.httpx_util import make_async_client import config @@ -35,7 +41,12 @@ from tools import utils if TYPE_CHECKING: from proxy.proxy_ip_pool import ProxyIpPool -from .exception import DataFetchError, IPBlockError, NoteNotFoundError +from .exception import ( + DataFetchError, + IPBlockError, + NoteNotFoundError, + PlatformAccessError, +) from .field import SearchNoteType, SearchSortType from .help import get_search_id from .extractor import XiaoHongShuExtractor @@ -66,6 +77,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): self.cookie_urls = [self._domain] self.IP_ERROR_STR = "Network connection error, please check network settings or restart" self.IP_ERROR_CODE = 300012 + self.SECURITY_LIMIT_CODE = 300011 self.NOTE_NOT_FOUND_CODE = -510000 self.NOTE_ABNORMAL_STR = "Note status abnormal, please check later" self.NOTE_ABNORMAL_CODE = -510001 @@ -112,7 +124,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): self.headers.update(headers) return self.headers - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1), retry=retry_if_not_exception_type(NoteNotFoundError)) + @retry( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + retry=retry_if_not_exception_type( + (NoteNotFoundError, IPBlockError, PlatformAccessError) + ), + ) async def request(self, method, url, **kwargs) -> Union[str, Any]: """ Wrapper for httpx common request method, processes request response @@ -132,6 +150,11 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): async with make_async_client(proxy=self.proxy) as client: response = await client.request(method, url, timeout=self.timeout, **kwargs) + if response.status_code in {401, 403, 429}: + raise PlatformAccessError( + f"XHS request blocked with HTTP {response.status_code}" + ) + if response.status_code == 471 or response.status_code == 461: # someday someone maybe will bypass captcha verify_type = response.headers["Verifytype"] @@ -140,9 +163,29 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): utils.logger.error(msg) raise Exception(msg) + response_data: Optional[Dict] = None + try: + candidate_data = response.json() + if isinstance(candidate_data, dict): + response_data = candidate_data + except (TypeError, ValueError): + pass + + response_code = ( + str(response_data.get("code")) + if response_data is not None and response_data.get("code") is not None + else "" + ) + if response_code == str(self.IP_ERROR_CODE): + raise IPBlockError(self.IP_ERROR_STR) + if response_code == str(self.SECURITY_LIMIT_CODE): + raise PlatformAccessError( + f"XHS account security restriction, code: {self.SECURITY_LIMIT_CODE}" + ) + if return_response: return response.text - data: Dict = response.json() + data: Dict = response_data if response_data is not None else response.json() if data["success"]: return data.get("data", data.get("success", {})) elif data["code"] == self.IP_ERROR_CODE: @@ -667,7 +710,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): data = {"original_url": f"{self._domain}/discovery/item/{note_id}"} return await self.post(uri, data=data, return_response=True) - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) + @retry( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + retry=retry_if_not_exception_type( + (RetryError, IPBlockError, PlatformAccessError) + ), + ) async def get_note_by_id_from_html( self, note_id: str, diff --git a/media_platform/xhs/exception.py b/media_platform/xhs/exception.py index a956d93..8d9bfd3 100644 --- a/media_platform/xhs/exception.py +++ b/media_platform/xhs/exception.py @@ -29,5 +29,9 @@ class IPBlockError(RequestError): """fetch so fast that the server block us ip""" +class PlatformAccessError(RequestError): + """authentication, rate-limit, or account-security restriction""" + + class NoteNotFoundError(RequestError): """Note does not exist or is abnormal""" diff --git a/tests/test_xhs_raw_response_errors.py b/tests/test_xhs_raw_response_errors.py new file mode 100644 index 0000000..9e76a4f --- /dev/null +++ b/tests/test_xhs_raw_response_errors.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- + +from unittest.mock import AsyncMock + +import httpx +import pytest +from tenacity import RetryError + +from media_platform.xhs.client import XiaoHongShuClient +from media_platform.xhs.exception import IPBlockError, PlatformAccessError + + +class FakeAsyncClient: + def __init__(self, request_impl): + self.request_impl = request_impl + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback): + return False + + async def request(self, *args, **kwargs): + return await self.request_impl(*args, **kwargs) + + +def make_client(): + client = XiaoHongShuClient( + headers={"Cookie": "web_session=test"}, + playwright_page=object(), + cookie_dict={}, + ) + client._refresh_proxy_if_expired = AsyncMock() + return client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code", [401, 403, 429]) +async def test_raw_response_rejects_access_http_status(monkeypatch, status_code): + calls = 0 + + async def request_impl(method, url, **kwargs): + nonlocal calls + calls += 1 + return httpx.Response( + status_code, + text="blocked", + request=httpx.Request(method, url), + ) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + with pytest.raises(PlatformAccessError): + await make_client().request( + "GET", "https://www.xiaohongshu.com/user/profile/test", return_response=True + ) + + assert calls == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("code", "expected_exception"), + [(300011, PlatformAccessError), ("300012", IPBlockError)], +) +async def test_raw_response_rejects_known_business_block( + monkeypatch, code, expected_exception +): + calls = 0 + + async def request_impl(method, url, **kwargs): + nonlocal calls + calls += 1 + return httpx.Response( + 200, + json={"success": False, "code": code, "msg": "blocked"}, + request=httpx.Request(method, url), + ) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + with pytest.raises(expected_exception): + await make_client().request( + "GET", "https://www.xiaohongshu.com/explore/test", return_response=True + ) + + assert calls == 1 + + +@pytest.mark.asyncio +async def test_raw_response_keeps_successful_html(monkeypatch): + async def request_impl(method, url, **kwargs): + return httpx.Response( + 200, + text="ok", + request=httpx.Request(method, url), + ) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + result = await make_client().request( + "GET", "https://www.xiaohongshu.com/explore/test", return_response=True + ) + + assert result == "ok" + + +@pytest.mark.asyncio +async def test_html_detail_does_not_multiply_transport_retries(monkeypatch): + calls = 0 + + async def request_impl(method, url, **kwargs): + nonlocal calls + calls += 1 + raise httpx.ReadTimeout("timed out", request=httpx.Request(method, url)) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + with pytest.raises(RetryError): + await make_client().get_note_by_id_from_html( + "test", xsec_source="pc_search", xsec_token="token" + ) + + assert calls == 3 From 508abc504f0a933cdc94af8a527bb59356326f22 Mon Sep 17 00:00:00 2001 From: ottercoconut Date: Tue, 11 Aug 2026 17:52:54 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E8=A1=A5=E5=85=A8=E5=B0=8F=E7=BA=A2?= =?UTF-8?q?=E4=B9=A6=E5=93=8D=E5=BA=94=E4=B8=8E=E9=87=8D=E8=AF=95=E5=9B=9E?= =?UTF-8?q?=E5=BD=92=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_xhs_raw_response_errors.py | 66 +++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/tests/test_xhs_raw_response_errors.py b/tests/test_xhs_raw_response_errors.py index 9e76a4f..89c77a9 100644 --- a/tests/test_xhs_raw_response_errors.py +++ b/tests/test_xhs_raw_response_errors.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -from unittest.mock import AsyncMock +import json +from unittest.mock import AsyncMock, Mock import httpx import pytest @@ -63,11 +64,16 @@ async def test_raw_response_rejects_access_http_status(monkeypatch, status_code) @pytest.mark.asyncio @pytest.mark.parametrize( - ("code", "expected_exception"), - [(300011, PlatformAccessError), ("300012", IPBlockError)], + ("code", "expected_exception", "return_response"), + [ + (300011, PlatformAccessError, True), + ("300011", PlatformAccessError, False), + (300012, IPBlockError, True), + ("300012", IPBlockError, False), + ], ) async def test_raw_response_rejects_known_business_block( - monkeypatch, code, expected_exception + monkeypatch, code, expected_exception, return_response ): calls = 0 @@ -87,7 +93,9 @@ async def test_raw_response_rejects_known_business_block( with pytest.raises(expected_exception): await make_client().request( - "GET", "https://www.xiaohongshu.com/explore/test", return_response=True + "GET", + "https://www.xiaohongshu.com/explore/test", + return_response=return_response, ) assert calls == 1 @@ -114,6 +122,37 @@ async def test_raw_response_keeps_successful_html(monkeypatch): assert result == "ok" +@pytest.mark.asyncio +async def test_successful_json_keeps_raw_and_parsed_return_modes(monkeypatch): + calls = 0 + + async def request_impl(method, url, **kwargs): + nonlocal calls + calls += 1 + return httpx.Response( + 200, + json={"success": True, "data": {"id": "test"}}, + request=httpx.Request(method, url), + ) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + client = make_client() + + raw_result = await client.request( + "GET", "https://www.xiaohongshu.com/explore/test", return_response=True + ) + parsed_result = await client.request( + "GET", "https://edith.xiaohongshu.com/api/test" + ) + + assert json.loads(raw_result) == {"success": True, "data": {"id": "test"}} + assert parsed_result == {"id": "test"} + assert calls == 2 + + @pytest.mark.asyncio async def test_html_detail_does_not_multiply_transport_retries(monkeypatch): calls = 0 @@ -134,3 +173,20 @@ async def test_html_detail_does_not_multiply_transport_retries(monkeypatch): ) assert calls == 3 + + +@pytest.mark.asyncio +async def test_html_detail_still_retries_parse_failures(): + client = make_client() + client.request = AsyncMock(return_value="incomplete") + client._extractor.extract_note_detail_from_html = Mock( + side_effect=ValueError("incomplete initial state") + ) + + with pytest.raises(RetryError): + await client.get_note_by_id_from_html( + "test", xsec_source="pc_search", xsec_token="token" + ) + + assert client.request.await_count == 3 + assert client._extractor.extract_note_detail_from_html.call_count == 3