diff --git a/media_platform/xhs/client.py b/media_platform/xhs/client.py index 1e07b08..39bb663 100644 --- a/media_platform/xhs/client.py +++ b/media_platform/xhs/client.py @@ -24,7 +24,13 @@ from urllib.parse import quote, urlencode import httpx from playwright.async_api import BrowserContext, Page -from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_not_exception_type +from tenacity import ( + RetryError, + retry, + retry_if_not_exception_type, + stop_after_attempt, + wait_fixed, +) from tools.httpx_util import make_async_client import config @@ -35,7 +41,12 @@ from tools import utils if TYPE_CHECKING: from proxy.proxy_ip_pool import ProxyIpPool -from .exception import DataFetchError, IPBlockError, NoteNotFoundError +from .exception import ( + DataFetchError, + IPBlockError, + NoteNotFoundError, + PlatformAccessError, +) from .field import SearchNoteType, SearchSortType from .help import get_search_id from .extractor import XiaoHongShuExtractor @@ -66,6 +77,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): self.cookie_urls = [self._domain] self.IP_ERROR_STR = "Network connection error, please check network settings or restart" self.IP_ERROR_CODE = 300012 + self.SECURITY_LIMIT_CODE = 300011 self.NOTE_NOT_FOUND_CODE = -510000 self.NOTE_ABNORMAL_STR = "Note status abnormal, please check later" self.NOTE_ABNORMAL_CODE = -510001 @@ -112,7 +124,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): self.headers.update(headers) return self.headers - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1), retry=retry_if_not_exception_type(NoteNotFoundError)) + @retry( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + retry=retry_if_not_exception_type( + (NoteNotFoundError, IPBlockError, PlatformAccessError) + ), + ) async def request(self, method, url, **kwargs) -> Union[str, Any]: """ Wrapper for httpx common request method, processes request response @@ -132,6 +150,11 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): async with make_async_client(proxy=self.proxy) as client: response = await client.request(method, url, timeout=self.timeout, **kwargs) + if response.status_code in {401, 403, 429}: + raise PlatformAccessError( + f"XHS request blocked with HTTP {response.status_code}" + ) + if response.status_code == 471 or response.status_code == 461: # someday someone maybe will bypass captcha verify_type = response.headers["Verifytype"] @@ -140,9 +163,29 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): utils.logger.error(msg) raise Exception(msg) + response_data: Optional[Dict] = None + try: + candidate_data = response.json() + if isinstance(candidate_data, dict): + response_data = candidate_data + except (TypeError, ValueError): + pass + + response_code = ( + str(response_data.get("code")) + if response_data is not None and response_data.get("code") is not None + else "" + ) + if response_code == str(self.IP_ERROR_CODE): + raise IPBlockError(self.IP_ERROR_STR) + if response_code == str(self.SECURITY_LIMIT_CODE): + raise PlatformAccessError( + f"XHS account security restriction, code: {self.SECURITY_LIMIT_CODE}" + ) + if return_response: return response.text - data: Dict = response.json() + data: Dict = response_data if response_data is not None else response.json() if data["success"]: return data.get("data", data.get("success", {})) elif data["code"] == self.IP_ERROR_CODE: @@ -667,7 +710,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): data = {"original_url": f"{self._domain}/discovery/item/{note_id}"} return await self.post(uri, data=data, return_response=True) - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) + @retry( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + retry=retry_if_not_exception_type( + (RetryError, IPBlockError, PlatformAccessError) + ), + ) async def get_note_by_id_from_html( self, note_id: str, diff --git a/media_platform/xhs/exception.py b/media_platform/xhs/exception.py index a956d93..8d9bfd3 100644 --- a/media_platform/xhs/exception.py +++ b/media_platform/xhs/exception.py @@ -29,5 +29,9 @@ class IPBlockError(RequestError): """fetch so fast that the server block us ip""" +class PlatformAccessError(RequestError): + """authentication, rate-limit, or account-security restriction""" + + class NoteNotFoundError(RequestError): """Note does not exist or is abnormal""" diff --git a/tests/test_xhs_raw_response_errors.py b/tests/test_xhs_raw_response_errors.py new file mode 100644 index 0000000..89c77a9 --- /dev/null +++ b/tests/test_xhs_raw_response_errors.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- + +import json +from unittest.mock import AsyncMock, Mock + +import httpx +import pytest +from tenacity import RetryError + +from media_platform.xhs.client import XiaoHongShuClient +from media_platform.xhs.exception import IPBlockError, PlatformAccessError + + +class FakeAsyncClient: + def __init__(self, request_impl): + self.request_impl = request_impl + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback): + return False + + async def request(self, *args, **kwargs): + return await self.request_impl(*args, **kwargs) + + +def make_client(): + client = XiaoHongShuClient( + headers={"Cookie": "web_session=test"}, + playwright_page=object(), + cookie_dict={}, + ) + client._refresh_proxy_if_expired = AsyncMock() + return client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code", [401, 403, 429]) +async def test_raw_response_rejects_access_http_status(monkeypatch, status_code): + calls = 0 + + async def request_impl(method, url, **kwargs): + nonlocal calls + calls += 1 + return httpx.Response( + status_code, + text="blocked", + request=httpx.Request(method, url), + ) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + with pytest.raises(PlatformAccessError): + await make_client().request( + "GET", "https://www.xiaohongshu.com/user/profile/test", return_response=True + ) + + assert calls == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("code", "expected_exception", "return_response"), + [ + (300011, PlatformAccessError, True), + ("300011", PlatformAccessError, False), + (300012, IPBlockError, True), + ("300012", IPBlockError, False), + ], +) +async def test_raw_response_rejects_known_business_block( + monkeypatch, code, expected_exception, return_response +): + calls = 0 + + async def request_impl(method, url, **kwargs): + nonlocal calls + calls += 1 + return httpx.Response( + 200, + json={"success": False, "code": code, "msg": "blocked"}, + request=httpx.Request(method, url), + ) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + with pytest.raises(expected_exception): + await make_client().request( + "GET", + "https://www.xiaohongshu.com/explore/test", + return_response=return_response, + ) + + assert calls == 1 + + +@pytest.mark.asyncio +async def test_raw_response_keeps_successful_html(monkeypatch): + async def request_impl(method, url, **kwargs): + return httpx.Response( + 200, + text="ok", + request=httpx.Request(method, url), + ) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + result = await make_client().request( + "GET", "https://www.xiaohongshu.com/explore/test", return_response=True + ) + + assert result == "ok" + + +@pytest.mark.asyncio +async def test_successful_json_keeps_raw_and_parsed_return_modes(monkeypatch): + calls = 0 + + async def request_impl(method, url, **kwargs): + nonlocal calls + calls += 1 + return httpx.Response( + 200, + json={"success": True, "data": {"id": "test"}}, + request=httpx.Request(method, url), + ) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + client = make_client() + + raw_result = await client.request( + "GET", "https://www.xiaohongshu.com/explore/test", return_response=True + ) + parsed_result = await client.request( + "GET", "https://edith.xiaohongshu.com/api/test" + ) + + assert json.loads(raw_result) == {"success": True, "data": {"id": "test"}} + assert parsed_result == {"id": "test"} + assert calls == 2 + + +@pytest.mark.asyncio +async def test_html_detail_does_not_multiply_transport_retries(monkeypatch): + calls = 0 + + async def request_impl(method, url, **kwargs): + nonlocal calls + calls += 1 + raise httpx.ReadTimeout("timed out", request=httpx.Request(method, url)) + + monkeypatch.setattr( + "media_platform.xhs.client.make_async_client", + lambda **kwargs: FakeAsyncClient(request_impl), + ) + + with pytest.raises(RetryError): + await make_client().get_note_by_id_from_html( + "test", xsec_source="pc_search", xsec_token="token" + ) + + assert calls == 3 + + +@pytest.mark.asyncio +async def test_html_detail_still_retries_parse_failures(): + client = make_client() + client.request = AsyncMock(return_value="incomplete") + client._extractor.extract_note_detail_from_html = Mock( + side_effect=ValueError("incomplete initial state") + ) + + with pytest.raises(RetryError): + await client.get_note_by_id_from_html( + "test", xsec_source="pc_search", xsec_token="token" + ) + + assert client.request.await_count == 3 + assert client._extractor.extract_note_detail_from_html.call_count == 3