From 2e558f1352646251525c3f99d552a13002603718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Tue, 4 Aug 2026 01:14:39 +0800 Subject: [PATCH] =?UTF-8?q?fix(kuaishou):=20=E6=94=AF=E6=8C=81=E7=BD=91?= =?UTF-8?q?=E9=A1=B5=E7=AB=AF=20REST=20=E6=8E=A5=E5=8F=A3=E7=AD=BE?= =?UTF-8?q?=E5=90=8D=E8=AF=B7=E6=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增页面签名环境捕获脚本，通过 caver 属性赋值轨迹获取签名调用入口 - 作品列表/关键词搜索接口迁移到带签名的 REST v2 请求，并显式校验 result 状态码 - 修复分页停止条件，空列表时结束翻页 - 更新默认创作者主页 ID --- config/ks_config.py | 2 +- media_platform/kuaishou/client.py | 82 +++++++++++++++++++++++++++++-- media_platform/kuaishou/core.py | 17 ++++--- media_platform/kuaishou/help.py | 62 +++++++++++++++++++++++ 4 files changed, 152 insertions(+), 11 deletions(-) diff --git a/config/ks_config.py b/config/ks_config.py index 0f83478..6d9f491 100644 --- a/config/ks_config.py +++ b/config/ks_config.py @@ -34,7 +34,7 @@ KS_SPECIFIED_ID_LIST = [ # 1. Creator homepage URL: "https://www.kuaishou.com/profile/3x84qugg4ch9zhs" # 2. Pure user_id: "3x4sm73aye7jq7i" KS_CREATOR_ID_LIST = [ - "https://www.kuaishou.com/profile/3x84qugg4ch9zhs", + "https://www.kuaishou.com/profile/3xf79edg9msa85c", "3x4sm73aye7jq7i", # ........................ ] diff --git a/media_platform/kuaishou/client.py b/media_platform/kuaishou/client.py index 2de2405..fc6ab62 100644 --- a/media_platform/kuaishou/client.py +++ b/media_platform/kuaishou/client.py @@ -38,6 +38,7 @@ if TYPE_CHECKING: from .exception import DataFetchError from .graphql import KuaiShouGraphQL +from .help import get_ks_sign_from_playwright class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin): @@ -112,6 +113,67 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin): raise DataFetchError(f"REST API V2 error: {result}") return result + async def request_rest_v2_signed(self, uri: str, data: dict) -> Dict: + """ + 带 __NS_hxfalcon 签名的 REST API V2 请求 + 快手网页端批量列表接口（作品列表/搜索）需要签名，未签名请求会返回 result:50 + :param uri: API endpoint path + :param data: request body + :return: response data + """ + await self._refresh_proxy_if_expired() + + sign = await get_ks_sign_from_playwright( + self.playwright_page, + uri, + {"caver": 2}, + data, + ) + + json_str = json.dumps(data, separators=(",", ":"), ensure_ascii=False) + async with make_async_client(proxy=self.proxy) as client: + response = await client.request( + method="POST", + url=f"{self._rest_host}{uri}?__NS_hxfalcon={sign}&caver=2", + data=json_str, + timeout=self.timeout, + headers=self.headers, + ) + result: Dict = response.json() + if result.get("result") != 1: + raise DataFetchError(f"REST API V2 error: {result}") + return result + + async def get_video_by_creater_v2(self, userId: str, pcursor: str = "") -> Dict: + """ + 获取用户作品列表 - REST 签名版 + 网页端已将作品列表迁移到 /rest/v/profile/feed 并需要签名 + :param userId: 用户ID + :param pcursor: 分页游标 + :return: 顶层结构 {result, pcursor, feeds, ...} + """ + post_data = {"user_id": userId, "pcursor": pcursor, "page": "profile"} + return await self.request_rest_v2_signed("/rest/v/profile/feed", post_data) + + async def search_info_by_keyword_v2( + self, keyword: str, pcursor: str, search_session_id: str = "" + ) -> Dict: + """ + 关键词搜索 - REST 签名版 + 网页端已将搜索迁移到 /rest/v/search/feed 并需要签名 + :param keyword: 搜索关键词 + :param pcursor: 分页游标（数字页码字符串） + :param search_session_id: 搜索会话ID + :return: 顶层结构 {result, pcursor, feeds, searchSessionId, ...} + """ + post_data = { + "keyword": keyword, + "pcursor": pcursor, + "page": "search", + "searchSessionId": search_session_id, + } + return await self.request_rest_v2_signed("/rest/v/search/feed", post_data) + async def pong(self) -> bool: """get a note to check if login state is ok""" utils.logger.info("[KuaiShouClient.pong] Begin pong kuaishou...") @@ -336,20 +398,32 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin): pcursor = "" while pcursor != "no_more": - videos_res = await self.get_video_by_creater(user_id, pcursor) + videos_res = await self.get_video_by_creater_v2(user_id, pcursor) if not videos_res: utils.logger.error( f"[KuaiShouClient.get_all_videos_by_creator] The current creator may have been banned by ks, so they cannot access the data." ) break - vision_profile_photo_list = videos_res.get("visionProfilePhotoList", {}) - pcursor = vision_profile_photo_list.get("pcursor", "") + # REST 接口用 result 字段表示业务状态，必须显式校验， + # 否则接口被拒(result:50)会被静默当成"没有更多视频" + result_code = videos_res.get("result") + if result_code != 1: + utils.logger.error( + f"[KuaiShouClient.get_all_videos_by_creator] ks api returned business error " + f"(result: {result_code}), stop pagination for user_id: {user_id}" + ) + break - videos = vision_profile_photo_list.get("feeds", []) + pcursor = videos_res.get("pcursor", "") + + videos = videos_res.get("feeds", []) utils.logger.info( f"[KuaiShouClient.get_all_videos_by_creator] got user_id:{user_id} videos len : {len(videos)}" ) + if not videos: + pcursor = "no_more" + break if callback: await callback(videos) diff --git a/media_platform/kuaishou/core.py b/media_platform/kuaishou/core.py index 79773cb..7675286 100644 --- a/media_platform/kuaishou/core.py +++ b/media_platform/kuaishou/core.py @@ -44,7 +44,11 @@ from var import comment_tasks_var, crawler_type_var, source_keyword_var from .client import KuaiShouClient from .exception import DataFetchError -from .help import parse_video_info_from_url, parse_creator_info_from_url +from .help import ( + KS_SIGN_CAPTURE_SCRIPT, + parse_video_info_from_url, + parse_creator_info_from_url, +) from .login import KuaishouLogin @@ -94,6 +98,8 @@ class KuaishouCrawler(AbstractCrawler): self.context_page = await self.browser_context.new_page() + # 注入快手签名环境捕获脚本，页面加载后即可通过 __ks_realm 生成 __NS_hxfalcon 签名 + await self.context_page.add_init_script(KS_SIGN_CAPTURE_SCRIPT) await self.context_page.goto(f"{self.index_url}?isHome=1") # Create a client to interact with the kuaishou website. @@ -151,7 +157,7 @@ class KuaishouCrawler(AbstractCrawler): f"[KuaishouCrawler.search] search kuaishou keyword: {keyword}, page: {page}" ) video_id_list: List[str] = [] - videos_res = await self.ks_client.search_info_by_keyword( + videos_res = await self.ks_client.search_info_by_keyword_v2( keyword=keyword, pcursor=str(page), search_session_id=search_session_id, @@ -162,14 +168,13 @@ class KuaishouCrawler(AbstractCrawler): ) break - vision_search_photo: Dict = videos_res.get("visionSearchPhoto") - if vision_search_photo.get("result") != 1: + if videos_res.get("result") != 1: utils.logger.error( f"[KuaishouCrawler.search] search info by keyword:{keyword} not found data " ) break - search_session_id = vision_search_photo.get("searchSessionId", "") - for video_detail in vision_search_photo.get("feeds"): + search_session_id = videos_res.get("searchSessionId", "") + for video_detail in videos_res.get("feeds", []): video_id_list.append(video_detail.get("photo", {}).get("id")) await kuaishou_store.update_kuaishou_video(video_item=video_detail) diff --git a/media_platform/kuaishou/help.py b/media_platform/kuaishou/help.py index ea4ebc8..a7bca7c 100644 --- a/media_platform/kuaishou/help.py +++ b/media_platform/kuaishou/help.py @@ -21,8 +21,70 @@ # -*- coding: utf-8 -*- import re + +from playwright.async_api import Page + from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo +# 快手网页端签名（__NS_hxfalcon）支持。 +# 快手网页端已将批量列表接口迁移到带签名的 REST 端点， +# 通过页面加载时注入的捕获脚本获取页面内置签名环境的调用入口， +# 再调用 $encode 生成签名。仅复用页面自身已加载的 JS 环境， +# 不引入额外的签名代码文件。 + +KS_SIGN_CAPTURE_SCRIPT = """ +// 捕获快手页面内置签名环境的调用入口（学习用途） +(() => { + if (window.__ks_realm) return; + let done = false; + const setter = function (v) { + if (!done && this && typeof this === "object" && this !== window && + typeof this.$encode === "function" && + typeof this.$getCatVersion === "function") { + done = true; + window.__ks_realm = this; + // 捕获成功后移除钩子，避免影响页面其他行为 + try { delete Object.prototype.caver; } catch (e) {} + } + Object.defineProperty(this, "caver", { + value: v, writable: true, enumerable: true, configurable: true, + }); + }; + try { + Object.defineProperty(Object.prototype, "caver", { set: setter, configurable: true }); + } catch (e) {} +})(); +""" + + +async def get_ks_sign_from_playwright(page: Page, url: str, query: dict, body: dict) -> str: + """ + 通过浏览器页面生成快手 __NS_hxfalcon 签名 + Args: + page: 已加载快手页面的 playwright page（需先注入 KS_SIGN_CAPTURE_SCRIPT） + url: 请求路径，如 /rest/v/profile/feed + query: 请求 query 参数，如 {"caver": 2} + body: 请求 body（JSON 对象） + Returns: + 签名串 + """ + try: + await page.wait_for_function("() => !!window.__ks_realm", timeout=15000) + except Exception: + # 页面可能在 cookie 注入前就已加载（未登录态），此时签名环境未初始化， + # 重载页面让其在登录态下加载并触发签名请求，捕获脚本将随新 document 生效 + await page.reload(wait_until="domcontentloaded") + await page.wait_for_function("() => !!window.__ks_realm", timeout=20000) + return await page.evaluate( + """([u, q, b]) => new Promise((resolve, reject) => { + window.__ks_realm.call('$encode', [ + { url: u, query: q, form: {}, requestBody: b }, + { suc: s => resolve(s), err: e => reject(new Error(String(e))) } + ]); + })""", + [url, query, body], + ) + def parse_video_info_from_url(url: str) -> VideoUrlInfo: """