Merge pull request #958 from ottercoconut/agent/fix-xhs-raw-response-errors

fix(xhs): 在返回原始 HTML 前识别访问限制

media_platform/xhs/client.py

@@ -24,7 +24,13 @@ from urllib.parse import quote, urlencode
 
 import httpx
 from playwright.async_api import BrowserContext, Page
-from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_not_exception_type
+from tenacity import (
+    RetryError,
+    retry,
+    retry_if_not_exception_type,
+    stop_after_attempt,
+    wait_fixed,
+)
 from tools.httpx_util import make_async_client
 
 import config
@@ -35,7 +41,12 @@ from tools import utils
 if TYPE_CHECKING:
     from proxy.proxy_ip_pool import ProxyIpPool
 
-from .exception import DataFetchError, IPBlockError, NoteNotFoundError
+from .exception import (
+    DataFetchError,
+    IPBlockError,
+    NoteNotFoundError,
+    PlatformAccessError,
+)
 from .field import SearchNoteType, SearchSortType
 from .help import get_search_id
 from .extractor import XiaoHongShuExtractor
@@ -66,6 +77,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
         self.cookie_urls = [self._domain]
         self.IP_ERROR_STR = "Network connection error, please check network settings or restart"
         self.IP_ERROR_CODE = 300012
+        self.SECURITY_LIMIT_CODE = 300011
         self.NOTE_NOT_FOUND_CODE = -510000
         self.NOTE_ABNORMAL_STR = "Note status abnormal, please check later"
         self.NOTE_ABNORMAL_CODE = -510001
@@ -112,7 +124,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
         self.headers.update(headers)
         return self.headers
 
-    @retry(stop=stop_after_attempt(3), wait=wait_fixed(1), retry=retry_if_not_exception_type(NoteNotFoundError))
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_fixed(1),
+        retry=retry_if_not_exception_type(
+            (NoteNotFoundError, IPBlockError, PlatformAccessError)
+        ),
+    )
     async def request(self, method, url, **kwargs) -> Union[str, Any]:
         """
         Wrapper for httpx common request method, processes request response
@@ -132,6 +150,11 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
         async with make_async_client(proxy=self.proxy) as client:
             response = await client.request(method, url, timeout=self.timeout, **kwargs)
 
+        if response.status_code in {401, 403, 429}:
+            raise PlatformAccessError(
+                f"XHS request blocked with HTTP {response.status_code}"
+            )
+
         if response.status_code == 471 or response.status_code == 461:
             # someday someone maybe will bypass captcha
             verify_type = response.headers["Verifytype"]
@@ -140,9 +163,29 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
             utils.logger.error(msg)
             raise Exception(msg)
 
+        response_data: Optional[Dict] = None
+        try:
+            candidate_data = response.json()
+            if isinstance(candidate_data, dict):
+                response_data = candidate_data
+        except (TypeError, ValueError):
+            pass
+
+        response_code = (
+            str(response_data.get("code"))
+            if response_data is not None and response_data.get("code") is not None
+            else ""
+        )
+        if response_code == str(self.IP_ERROR_CODE):
+            raise IPBlockError(self.IP_ERROR_STR)
+        if response_code == str(self.SECURITY_LIMIT_CODE):
+            raise PlatformAccessError(
+                f"XHS account security restriction, code: {self.SECURITY_LIMIT_CODE}"
+            )
+
         if return_response:
             return response.text
-        data: Dict = response.json()
+        data: Dict = response_data if response_data is not None else response.json()
         if data["success"]:
             return data.get("data", data.get("success", {}))
         elif data["code"] == self.IP_ERROR_CODE:
@@ -667,7 +710,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
         data = {"original_url": f"{self._domain}/discovery/item/{note_id}"}
         return await self.post(uri, data=data, return_response=True)
 
-    @retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_fixed(1),
+        retry=retry_if_not_exception_type(
+            (RetryError, IPBlockError, PlatformAccessError)
+        ),
+    )
     async def get_note_by_id_from_html(
         self,
         note_id: str,

media_platform/xhs/exception.py

@@ -29,5 +29,9 @@ class IPBlockError(RequestError):
     """fetch so fast that the server block us ip"""
 
 
+class PlatformAccessError(RequestError):
+    """authentication, rate-limit, or account-security restriction"""
+
+
 class NoteNotFoundError(RequestError):
     """Note does not exist or is abnormal"""

tests/test_xhs_raw_response_errors.py

@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+
+import json
+from unittest.mock import AsyncMock, Mock
+
+import httpx
+import pytest
+from tenacity import RetryError
+
+from media_platform.xhs.client import XiaoHongShuClient
+from media_platform.xhs.exception import IPBlockError, PlatformAccessError
+
+
+class FakeAsyncClient:
+    def __init__(self, request_impl):
+        self.request_impl = request_impl
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc, traceback):
+        return False
+
+    async def request(self, *args, **kwargs):
+        return await self.request_impl(*args, **kwargs)
+
+
+def make_client():
+    client = XiaoHongShuClient(
+        headers={"Cookie": "web_session=test"},
+        playwright_page=object(),
+        cookie_dict={},
+    )
+    client._refresh_proxy_if_expired = AsyncMock()
+    return client
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("status_code", [401, 403, 429])
+async def test_raw_response_rejects_access_http_status(monkeypatch, status_code):
+    calls = 0
+
+    async def request_impl(method, url, **kwargs):
+        nonlocal calls
+        calls += 1
+        return httpx.Response(
+            status_code,
+            text="blocked",
+            request=httpx.Request(method, url),
+        )
+
+    monkeypatch.setattr(
+        "media_platform.xhs.client.make_async_client",
+        lambda **kwargs: FakeAsyncClient(request_impl),
+    )
+
+    with pytest.raises(PlatformAccessError):
+        await make_client().request(
+            "GET", "https://www.xiaohongshu.com/user/profile/test", return_response=True
+        )
+
+    assert calls == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("code", "expected_exception", "return_response"),
+    [
+        (300011, PlatformAccessError, True),
+        ("300011", PlatformAccessError, False),
+        (300012, IPBlockError, True),
+        ("300012", IPBlockError, False),
+    ],
+)
+async def test_raw_response_rejects_known_business_block(
+    monkeypatch, code, expected_exception, return_response
+):
+    calls = 0
+
+    async def request_impl(method, url, **kwargs):
+        nonlocal calls
+        calls += 1
+        return httpx.Response(
+            200,
+            json={"success": False, "code": code, "msg": "blocked"},
+            request=httpx.Request(method, url),
+        )
+
+    monkeypatch.setattr(
+        "media_platform.xhs.client.make_async_client",
+        lambda **kwargs: FakeAsyncClient(request_impl),
+    )
+
+    with pytest.raises(expected_exception):
+        await make_client().request(
+            "GET",
+            "https://www.xiaohongshu.com/explore/test",
+            return_response=return_response,
+        )
+
+    assert calls == 1
+
+
+@pytest.mark.asyncio
+async def test_raw_response_keeps_successful_html(monkeypatch):
+    async def request_impl(method, url, **kwargs):
+        return httpx.Response(
+            200,
+            text="<html>ok</html>",
+            request=httpx.Request(method, url),
+        )
+
+    monkeypatch.setattr(
+        "media_platform.xhs.client.make_async_client",
+        lambda **kwargs: FakeAsyncClient(request_impl),
+    )
+
+    result = await make_client().request(
+        "GET", "https://www.xiaohongshu.com/explore/test", return_response=True
+    )
+
+    assert result == "<html>ok</html>"
+
+
+@pytest.mark.asyncio
+async def test_successful_json_keeps_raw_and_parsed_return_modes(monkeypatch):
+    calls = 0
+
+    async def request_impl(method, url, **kwargs):
+        nonlocal calls
+        calls += 1
+        return httpx.Response(
+            200,
+            json={"success": True, "data": {"id": "test"}},
+            request=httpx.Request(method, url),
+        )
+
+    monkeypatch.setattr(
+        "media_platform.xhs.client.make_async_client",
+        lambda **kwargs: FakeAsyncClient(request_impl),
+    )
+    client = make_client()
+
+    raw_result = await client.request(
+        "GET", "https://www.xiaohongshu.com/explore/test", return_response=True
+    )
+    parsed_result = await client.request(
+        "GET", "https://edith.xiaohongshu.com/api/test"
+    )
+
+    assert json.loads(raw_result) == {"success": True, "data": {"id": "test"}}
+    assert parsed_result == {"id": "test"}
+    assert calls == 2
+
+
+@pytest.mark.asyncio
+async def test_html_detail_does_not_multiply_transport_retries(monkeypatch):
+    calls = 0
+
+    async def request_impl(method, url, **kwargs):
+        nonlocal calls
+        calls += 1
+        raise httpx.ReadTimeout("timed out", request=httpx.Request(method, url))
+
+    monkeypatch.setattr(
+        "media_platform.xhs.client.make_async_client",
+        lambda **kwargs: FakeAsyncClient(request_impl),
+    )
+
+    with pytest.raises(RetryError):
+        await make_client().get_note_by_id_from_html(
+            "test", xsec_source="pc_search", xsec_token="token"
+        )
+
+    assert calls == 3
+
+
+@pytest.mark.asyncio
+async def test_html_detail_still_retries_parse_failures():
+    client = make_client()
+    client.request = AsyncMock(return_value="<html>incomplete</html>")
+    client._extractor.extract_note_detail_from_html = Mock(
+        side_effect=ValueError("incomplete initial state")
+    )
+
+    with pytest.raises(RetryError):
+        await client.get_note_by_id_from_html(
+            "test", xsec_source="pc_search", xsec_token="token"
+        )
+
+    assert client.request.await_count == 3
+    assert client._extractor.extract_note_detail_from_html.call_count == 3