mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-09-15 14:17:54 +08:00
fix(kuaishou): 支持网页端 REST 接口签名请求
- 新增页面签名环境捕获脚本,通过 caver 属性赋值轨迹获取签名调用入口 - 作品列表/关键词搜索接口迁移到带签名的 REST v2 请求,并显式校验 result 状态码 - 修复分页停止条件,空列表时结束翻页 - 更新默认创作者主页 ID
This commit is contained in:
@@ -34,7 +34,7 @@ KS_SPECIFIED_ID_LIST = [
|
||||
# 1. Creator homepage URL: "https://www.kuaishou.com/profile/3x84qugg4ch9zhs"
|
||||
# 2. Pure user_id: "3x4sm73aye7jq7i"
|
||||
KS_CREATOR_ID_LIST = [
|
||||
"https://www.kuaishou.com/profile/3x84qugg4ch9zhs",
|
||||
"https://www.kuaishou.com/profile/3xf79edg9msa85c",
|
||||
"3x4sm73aye7jq7i",
|
||||
# ........................
|
||||
]
|
||||
|
||||
@@ -38,6 +38,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from .exception import DataFetchError
|
||||
from .graphql import KuaiShouGraphQL
|
||||
from .help import get_ks_sign_from_playwright
|
||||
|
||||
|
||||
class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
@@ -112,6 +113,67 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
raise DataFetchError(f"REST API V2 error: {result}")
|
||||
return result
|
||||
|
||||
async def request_rest_v2_signed(self, uri: str, data: dict) -> Dict:
|
||||
"""
|
||||
带 __NS_hxfalcon 签名的 REST API V2 请求
|
||||
快手网页端批量列表接口(作品列表/搜索)需要签名,未签名请求会返回 result:50
|
||||
:param uri: API endpoint path
|
||||
:param data: request body
|
||||
:return: response data
|
||||
"""
|
||||
await self._refresh_proxy_if_expired()
|
||||
|
||||
sign = await get_ks_sign_from_playwright(
|
||||
self.playwright_page,
|
||||
uri,
|
||||
{"caver": 2},
|
||||
data,
|
||||
)
|
||||
|
||||
json_str = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
|
||||
async with make_async_client(proxy=self.proxy) as client:
|
||||
response = await client.request(
|
||||
method="POST",
|
||||
url=f"{self._rest_host}{uri}?__NS_hxfalcon={sign}&caver=2",
|
||||
data=json_str,
|
||||
timeout=self.timeout,
|
||||
headers=self.headers,
|
||||
)
|
||||
result: Dict = response.json()
|
||||
if result.get("result") != 1:
|
||||
raise DataFetchError(f"REST API V2 error: {result}")
|
||||
return result
|
||||
|
||||
async def get_video_by_creater_v2(self, userId: str, pcursor: str = "") -> Dict:
|
||||
"""
|
||||
获取用户作品列表 - REST 签名版
|
||||
网页端已将作品列表迁移到 /rest/v/profile/feed 并需要签名
|
||||
:param userId: 用户ID
|
||||
:param pcursor: 分页游标
|
||||
:return: 顶层结构 {result, pcursor, feeds, ...}
|
||||
"""
|
||||
post_data = {"user_id": userId, "pcursor": pcursor, "page": "profile"}
|
||||
return await self.request_rest_v2_signed("/rest/v/profile/feed", post_data)
|
||||
|
||||
async def search_info_by_keyword_v2(
|
||||
self, keyword: str, pcursor: str, search_session_id: str = ""
|
||||
) -> Dict:
|
||||
"""
|
||||
关键词搜索 - REST 签名版
|
||||
网页端已将搜索迁移到 /rest/v/search/feed 并需要签名
|
||||
:param keyword: 搜索关键词
|
||||
:param pcursor: 分页游标(数字页码字符串)
|
||||
:param search_session_id: 搜索会话ID
|
||||
:return: 顶层结构 {result, pcursor, feeds, searchSessionId, ...}
|
||||
"""
|
||||
post_data = {
|
||||
"keyword": keyword,
|
||||
"pcursor": pcursor,
|
||||
"page": "search",
|
||||
"searchSessionId": search_session_id,
|
||||
}
|
||||
return await self.request_rest_v2_signed("/rest/v/search/feed", post_data)
|
||||
|
||||
async def pong(self) -> bool:
|
||||
"""get a note to check if login state is ok"""
|
||||
utils.logger.info("[KuaiShouClient.pong] Begin pong kuaishou...")
|
||||
@@ -336,20 +398,32 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
pcursor = ""
|
||||
|
||||
while pcursor != "no_more":
|
||||
videos_res = await self.get_video_by_creater(user_id, pcursor)
|
||||
videos_res = await self.get_video_by_creater_v2(user_id, pcursor)
|
||||
if not videos_res:
|
||||
utils.logger.error(
|
||||
f"[KuaiShouClient.get_all_videos_by_creator] The current creator may have been banned by ks, so they cannot access the data."
|
||||
)
|
||||
break
|
||||
|
||||
vision_profile_photo_list = videos_res.get("visionProfilePhotoList", {})
|
||||
pcursor = vision_profile_photo_list.get("pcursor", "")
|
||||
# REST 接口用 result 字段表示业务状态,必须显式校验,
|
||||
# 否则接口被拒(result:50)会被静默当成"没有更多视频"
|
||||
result_code = videos_res.get("result")
|
||||
if result_code != 1:
|
||||
utils.logger.error(
|
||||
f"[KuaiShouClient.get_all_videos_by_creator] ks api returned business error "
|
||||
f"(result: {result_code}), stop pagination for user_id: {user_id}"
|
||||
)
|
||||
break
|
||||
|
||||
videos = vision_profile_photo_list.get("feeds", [])
|
||||
pcursor = videos_res.get("pcursor", "")
|
||||
|
||||
videos = videos_res.get("feeds", [])
|
||||
utils.logger.info(
|
||||
f"[KuaiShouClient.get_all_videos_by_creator] got user_id:{user_id} videos len : {len(videos)}"
|
||||
)
|
||||
if not videos:
|
||||
pcursor = "no_more"
|
||||
break
|
||||
|
||||
if callback:
|
||||
await callback(videos)
|
||||
|
||||
@@ -44,7 +44,11 @@ from var import comment_tasks_var, crawler_type_var, source_keyword_var
|
||||
|
||||
from .client import KuaiShouClient
|
||||
from .exception import DataFetchError
|
||||
from .help import parse_video_info_from_url, parse_creator_info_from_url
|
||||
from .help import (
|
||||
KS_SIGN_CAPTURE_SCRIPT,
|
||||
parse_video_info_from_url,
|
||||
parse_creator_info_from_url,
|
||||
)
|
||||
from .login import KuaishouLogin
|
||||
|
||||
|
||||
@@ -94,6 +98,8 @@ class KuaishouCrawler(AbstractCrawler):
|
||||
|
||||
|
||||
self.context_page = await self.browser_context.new_page()
|
||||
# 注入快手签名环境捕获脚本,页面加载后即可通过 __ks_realm 生成 __NS_hxfalcon 签名
|
||||
await self.context_page.add_init_script(KS_SIGN_CAPTURE_SCRIPT)
|
||||
await self.context_page.goto(f"{self.index_url}?isHome=1")
|
||||
|
||||
# Create a client to interact with the kuaishou website.
|
||||
@@ -151,7 +157,7 @@ class KuaishouCrawler(AbstractCrawler):
|
||||
f"[KuaishouCrawler.search] search kuaishou keyword: {keyword}, page: {page}"
|
||||
)
|
||||
video_id_list: List[str] = []
|
||||
videos_res = await self.ks_client.search_info_by_keyword(
|
||||
videos_res = await self.ks_client.search_info_by_keyword_v2(
|
||||
keyword=keyword,
|
||||
pcursor=str(page),
|
||||
search_session_id=search_session_id,
|
||||
@@ -162,14 +168,13 @@ class KuaishouCrawler(AbstractCrawler):
|
||||
)
|
||||
break
|
||||
|
||||
vision_search_photo: Dict = videos_res.get("visionSearchPhoto")
|
||||
if vision_search_photo.get("result") != 1:
|
||||
if videos_res.get("result") != 1:
|
||||
utils.logger.error(
|
||||
f"[KuaishouCrawler.search] search info by keyword:{keyword} not found data "
|
||||
)
|
||||
break
|
||||
search_session_id = vision_search_photo.get("searchSessionId", "")
|
||||
for video_detail in vision_search_photo.get("feeds"):
|
||||
search_session_id = videos_res.get("searchSessionId", "")
|
||||
for video_detail in videos_res.get("feeds", []):
|
||||
video_id_list.append(video_detail.get("photo", {}).get("id"))
|
||||
await kuaishou_store.update_kuaishou_video(video_item=video_detail)
|
||||
|
||||
|
||||
@@ -21,8 +21,70 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import re
|
||||
|
||||
from playwright.async_api import Page
|
||||
|
||||
from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo
|
||||
|
||||
# 快手网页端签名(__NS_hxfalcon)支持。
|
||||
# 快手网页端已将批量列表接口迁移到带签名的 REST 端点,
|
||||
# 通过页面加载时注入的捕获脚本获取页面内置签名环境的调用入口,
|
||||
# 再调用 $encode 生成签名。仅复用页面自身已加载的 JS 环境,
|
||||
# 不引入额外的签名代码文件。
|
||||
|
||||
KS_SIGN_CAPTURE_SCRIPT = """
|
||||
// 捕获快手页面内置签名环境的调用入口(学习用途)
|
||||
(() => {
|
||||
if (window.__ks_realm) return;
|
||||
let done = false;
|
||||
const setter = function (v) {
|
||||
if (!done && this && typeof this === "object" && this !== window &&
|
||||
typeof this.$encode === "function" &&
|
||||
typeof this.$getCatVersion === "function") {
|
||||
done = true;
|
||||
window.__ks_realm = this;
|
||||
// 捕获成功后移除钩子,避免影响页面其他行为
|
||||
try { delete Object.prototype.caver; } catch (e) {}
|
||||
}
|
||||
Object.defineProperty(this, "caver", {
|
||||
value: v, writable: true, enumerable: true, configurable: true,
|
||||
});
|
||||
};
|
||||
try {
|
||||
Object.defineProperty(Object.prototype, "caver", { set: setter, configurable: true });
|
||||
} catch (e) {}
|
||||
})();
|
||||
"""
|
||||
|
||||
|
||||
async def get_ks_sign_from_playwright(page: Page, url: str, query: dict, body: dict) -> str:
|
||||
"""
|
||||
通过浏览器页面生成快手 __NS_hxfalcon 签名
|
||||
Args:
|
||||
page: 已加载快手页面的 playwright page(需先注入 KS_SIGN_CAPTURE_SCRIPT)
|
||||
url: 请求路径,如 /rest/v/profile/feed
|
||||
query: 请求 query 参数,如 {"caver": 2}
|
||||
body: 请求 body(JSON 对象)
|
||||
Returns:
|
||||
签名串
|
||||
"""
|
||||
try:
|
||||
await page.wait_for_function("() => !!window.__ks_realm", timeout=15000)
|
||||
except Exception:
|
||||
# 页面可能在 cookie 注入前就已加载(未登录态),此时签名环境未初始化,
|
||||
# 重载页面让其在登录态下加载并触发签名请求,捕获脚本将随新 document 生效
|
||||
await page.reload(wait_until="domcontentloaded")
|
||||
await page.wait_for_function("() => !!window.__ks_realm", timeout=20000)
|
||||
return await page.evaluate(
|
||||
"""([u, q, b]) => new Promise((resolve, reject) => {
|
||||
window.__ks_realm.call('$encode', [
|
||||
{ url: u, query: q, form: {}, requestBody: b },
|
||||
{ suc: s => resolve(s), err: e => reject(new Error(String(e))) }
|
||||
]);
|
||||
})""",
|
||||
[url, query, body],
|
||||
)
|
||||
|
||||
|
||||
def parse_video_info_from_url(url: str) -> VideoUrlInfo:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user