fix(kuaishou): 支持网页端 REST 接口签名请求

- 新增页面签名环境捕获脚本,通过 caver 属性赋值轨迹获取签名调用入口
- 作品列表/关键词搜索接口迁移到带签名的 REST v2 请求,并显式校验 result 状态码
- 修复分页停止条件,空列表时结束翻页
- 更新默认创作者主页 ID
This commit is contained in:
程序员阿江(Relakkes)
2026-08-04 01:14:39 +08:00
parent 1779dde972
commit 2e558f1352
4 changed files with 152 additions and 11 deletions

View File

@@ -34,7 +34,7 @@ KS_SPECIFIED_ID_LIST = [
# 1. Creator homepage URL: "https://www.kuaishou.com/profile/3x84qugg4ch9zhs"
# 2. Pure user_id: "3x4sm73aye7jq7i"
KS_CREATOR_ID_LIST = [
"https://www.kuaishou.com/profile/3x84qugg4ch9zhs",
"https://www.kuaishou.com/profile/3xf79edg9msa85c",
"3x4sm73aye7jq7i",
# ........................
]

View File

@@ -38,6 +38,7 @@ if TYPE_CHECKING:
from .exception import DataFetchError
from .graphql import KuaiShouGraphQL
from .help import get_ks_sign_from_playwright
class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
@@ -112,6 +113,67 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
raise DataFetchError(f"REST API V2 error: {result}")
return result
async def request_rest_v2_signed(self, uri: str, data: dict) -> Dict:
"""
带 __NS_hxfalcon 签名的 REST API V2 请求
快手网页端批量列表接口(作品列表/搜索)需要签名,未签名请求会返回 result:50
:param uri: API endpoint path
:param data: request body
:return: response data
"""
await self._refresh_proxy_if_expired()
sign = await get_ks_sign_from_playwright(
self.playwright_page,
uri,
{"caver": 2},
data,
)
json_str = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
async with make_async_client(proxy=self.proxy) as client:
response = await client.request(
method="POST",
url=f"{self._rest_host}{uri}?__NS_hxfalcon={sign}&caver=2",
data=json_str,
timeout=self.timeout,
headers=self.headers,
)
result: Dict = response.json()
if result.get("result") != 1:
raise DataFetchError(f"REST API V2 error: {result}")
return result
async def get_video_by_creater_v2(self, userId: str, pcursor: str = "") -> Dict:
"""
获取用户作品列表 - REST 签名版
网页端已将作品列表迁移到 /rest/v/profile/feed 并需要签名
:param userId: 用户ID
:param pcursor: 分页游标
:return: 顶层结构 {result, pcursor, feeds, ...}
"""
post_data = {"user_id": userId, "pcursor": pcursor, "page": "profile"}
return await self.request_rest_v2_signed("/rest/v/profile/feed", post_data)
async def search_info_by_keyword_v2(
self, keyword: str, pcursor: str, search_session_id: str = ""
) -> Dict:
"""
关键词搜索 - REST 签名版
网页端已将搜索迁移到 /rest/v/search/feed 并需要签名
:param keyword: 搜索关键词
:param pcursor: 分页游标(数字页码字符串)
:param search_session_id: 搜索会话ID
:return: 顶层结构 {result, pcursor, feeds, searchSessionId, ...}
"""
post_data = {
"keyword": keyword,
"pcursor": pcursor,
"page": "search",
"searchSessionId": search_session_id,
}
return await self.request_rest_v2_signed("/rest/v/search/feed", post_data)
async def pong(self) -> bool:
"""get a note to check if login state is ok"""
utils.logger.info("[KuaiShouClient.pong] Begin pong kuaishou...")
@@ -336,20 +398,32 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
pcursor = ""
while pcursor != "no_more":
videos_res = await self.get_video_by_creater(user_id, pcursor)
videos_res = await self.get_video_by_creater_v2(user_id, pcursor)
if not videos_res:
utils.logger.error(
f"[KuaiShouClient.get_all_videos_by_creator] The current creator may have been banned by ks, so they cannot access the data."
)
break
vision_profile_photo_list = videos_res.get("visionProfilePhotoList", {})
pcursor = vision_profile_photo_list.get("pcursor", "")
# REST 接口用 result 字段表示业务状态,必须显式校验,
# 否则接口被拒(result:50)会被静默当成"没有更多视频"
result_code = videos_res.get("result")
if result_code != 1:
utils.logger.error(
f"[KuaiShouClient.get_all_videos_by_creator] ks api returned business error "
f"(result: {result_code}), stop pagination for user_id: {user_id}"
)
break
videos = vision_profile_photo_list.get("feeds", [])
pcursor = videos_res.get("pcursor", "")
videos = videos_res.get("feeds", [])
utils.logger.info(
f"[KuaiShouClient.get_all_videos_by_creator] got user_id:{user_id} videos len : {len(videos)}"
)
if not videos:
pcursor = "no_more"
break
if callback:
await callback(videos)

View File

@@ -44,7 +44,11 @@ from var import comment_tasks_var, crawler_type_var, source_keyword_var
from .client import KuaiShouClient
from .exception import DataFetchError
from .help import parse_video_info_from_url, parse_creator_info_from_url
from .help import (
KS_SIGN_CAPTURE_SCRIPT,
parse_video_info_from_url,
parse_creator_info_from_url,
)
from .login import KuaishouLogin
@@ -94,6 +98,8 @@ class KuaishouCrawler(AbstractCrawler):
self.context_page = await self.browser_context.new_page()
# 注入快手签名环境捕获脚本,页面加载后即可通过 __ks_realm 生成 __NS_hxfalcon 签名
await self.context_page.add_init_script(KS_SIGN_CAPTURE_SCRIPT)
await self.context_page.goto(f"{self.index_url}?isHome=1")
# Create a client to interact with the kuaishou website.
@@ -151,7 +157,7 @@ class KuaishouCrawler(AbstractCrawler):
f"[KuaishouCrawler.search] search kuaishou keyword: {keyword}, page: {page}"
)
video_id_list: List[str] = []
videos_res = await self.ks_client.search_info_by_keyword(
videos_res = await self.ks_client.search_info_by_keyword_v2(
keyword=keyword,
pcursor=str(page),
search_session_id=search_session_id,
@@ -162,14 +168,13 @@ class KuaishouCrawler(AbstractCrawler):
)
break
vision_search_photo: Dict = videos_res.get("visionSearchPhoto")
if vision_search_photo.get("result") != 1:
if videos_res.get("result") != 1:
utils.logger.error(
f"[KuaishouCrawler.search] search info by keyword:{keyword} not found data "
)
break
search_session_id = vision_search_photo.get("searchSessionId", "")
for video_detail in vision_search_photo.get("feeds"):
search_session_id = videos_res.get("searchSessionId", "")
for video_detail in videos_res.get("feeds", []):
video_id_list.append(video_detail.get("photo", {}).get("id"))
await kuaishou_store.update_kuaishou_video(video_item=video_detail)

View File

@@ -21,8 +21,70 @@
# -*- coding: utf-8 -*-
import re
from playwright.async_api import Page
from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo
# 快手网页端签名__NS_hxfalcon支持。
# 快手网页端已将批量列表接口迁移到带签名的 REST 端点,
# 通过页面加载时注入的捕获脚本获取页面内置签名环境的调用入口,
# 再调用 $encode 生成签名。仅复用页面自身已加载的 JS 环境,
# 不引入额外的签名代码文件。
KS_SIGN_CAPTURE_SCRIPT = """
// 捕获快手页面内置签名环境的调用入口(学习用途)
(() => {
if (window.__ks_realm) return;
let done = false;
const setter = function (v) {
if (!done && this && typeof this === "object" && this !== window &&
typeof this.$encode === "function" &&
typeof this.$getCatVersion === "function") {
done = true;
window.__ks_realm = this;
// 捕获成功后移除钩子,避免影响页面其他行为
try { delete Object.prototype.caver; } catch (e) {}
}
Object.defineProperty(this, "caver", {
value: v, writable: true, enumerable: true, configurable: true,
});
};
try {
Object.defineProperty(Object.prototype, "caver", { set: setter, configurable: true });
} catch (e) {}
})();
"""
async def get_ks_sign_from_playwright(page: Page, url: str, query: dict, body: dict) -> str:
"""
通过浏览器页面生成快手 __NS_hxfalcon 签名
Args:
page: 已加载快手页面的 playwright page需先注入 KS_SIGN_CAPTURE_SCRIPT
url: 请求路径,如 /rest/v/profile/feed
query: 请求 query 参数,如 {"caver": 2}
body: 请求 bodyJSON 对象)
Returns:
签名串
"""
try:
await page.wait_for_function("() => !!window.__ks_realm", timeout=15000)
except Exception:
# 页面可能在 cookie 注入前就已加载(未登录态),此时签名环境未初始化,
# 重载页面让其在登录态下加载并触发签名请求,捕获脚本将随新 document 生效
await page.reload(wait_until="domcontentloaded")
await page.wait_for_function("() => !!window.__ks_realm", timeout=20000)
return await page.evaluate(
"""([u, q, b]) => new Promise((resolve, reject) => {
window.__ks_realm.call('$encode', [
{ url: u, query: q, form: {}, requestBody: b },
{ suc: s => resolve(s), err: e => reject(new Error(String(e))) }
]);
})""",
[url, query, body],
)
def parse_video_info_from_url(url: str) -> VideoUrlInfo:
"""