mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-09-15 14:17:54 +08:00
fix(kuaishou): 支持网页端 REST 接口签名请求
- 新增页面签名环境捕获脚本,通过 caver 属性赋值轨迹获取签名调用入口 - 作品列表/关键词搜索接口迁移到带签名的 REST v2 请求,并显式校验 result 状态码 - 修复分页停止条件,空列表时结束翻页 - 更新默认创作者主页 ID
This commit is contained in:
@@ -34,7 +34,7 @@ KS_SPECIFIED_ID_LIST = [
|
|||||||
# 1. Creator homepage URL: "https://www.kuaishou.com/profile/3x84qugg4ch9zhs"
|
# 1. Creator homepage URL: "https://www.kuaishou.com/profile/3x84qugg4ch9zhs"
|
||||||
# 2. Pure user_id: "3x4sm73aye7jq7i"
|
# 2. Pure user_id: "3x4sm73aye7jq7i"
|
||||||
KS_CREATOR_ID_LIST = [
|
KS_CREATOR_ID_LIST = [
|
||||||
"https://www.kuaishou.com/profile/3x84qugg4ch9zhs",
|
"https://www.kuaishou.com/profile/3xf79edg9msa85c",
|
||||||
"3x4sm73aye7jq7i",
|
"3x4sm73aye7jq7i",
|
||||||
# ........................
|
# ........................
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
from .exception import DataFetchError
|
from .exception import DataFetchError
|
||||||
from .graphql import KuaiShouGraphQL
|
from .graphql import KuaiShouGraphQL
|
||||||
|
from .help import get_ks_sign_from_playwright
|
||||||
|
|
||||||
|
|
||||||
class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
|
class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
|
||||||
@@ -112,6 +113,67 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
|
|||||||
raise DataFetchError(f"REST API V2 error: {result}")
|
raise DataFetchError(f"REST API V2 error: {result}")
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
async def request_rest_v2_signed(self, uri: str, data: dict) -> Dict:
|
||||||
|
"""
|
||||||
|
带 __NS_hxfalcon 签名的 REST API V2 请求
|
||||||
|
快手网页端批量列表接口(作品列表/搜索)需要签名,未签名请求会返回 result:50
|
||||||
|
:param uri: API endpoint path
|
||||||
|
:param data: request body
|
||||||
|
:return: response data
|
||||||
|
"""
|
||||||
|
await self._refresh_proxy_if_expired()
|
||||||
|
|
||||||
|
sign = await get_ks_sign_from_playwright(
|
||||||
|
self.playwright_page,
|
||||||
|
uri,
|
||||||
|
{"caver": 2},
|
||||||
|
data,
|
||||||
|
)
|
||||||
|
|
||||||
|
json_str = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
|
||||||
|
async with make_async_client(proxy=self.proxy) as client:
|
||||||
|
response = await client.request(
|
||||||
|
method="POST",
|
||||||
|
url=f"{self._rest_host}{uri}?__NS_hxfalcon={sign}&caver=2",
|
||||||
|
data=json_str,
|
||||||
|
timeout=self.timeout,
|
||||||
|
headers=self.headers,
|
||||||
|
)
|
||||||
|
result: Dict = response.json()
|
||||||
|
if result.get("result") != 1:
|
||||||
|
raise DataFetchError(f"REST API V2 error: {result}")
|
||||||
|
return result
|
||||||
|
|
||||||
|
async def get_video_by_creater_v2(self, userId: str, pcursor: str = "") -> Dict:
|
||||||
|
"""
|
||||||
|
获取用户作品列表 - REST 签名版
|
||||||
|
网页端已将作品列表迁移到 /rest/v/profile/feed 并需要签名
|
||||||
|
:param userId: 用户ID
|
||||||
|
:param pcursor: 分页游标
|
||||||
|
:return: 顶层结构 {result, pcursor, feeds, ...}
|
||||||
|
"""
|
||||||
|
post_data = {"user_id": userId, "pcursor": pcursor, "page": "profile"}
|
||||||
|
return await self.request_rest_v2_signed("/rest/v/profile/feed", post_data)
|
||||||
|
|
||||||
|
async def search_info_by_keyword_v2(
|
||||||
|
self, keyword: str, pcursor: str, search_session_id: str = ""
|
||||||
|
) -> Dict:
|
||||||
|
"""
|
||||||
|
关键词搜索 - REST 签名版
|
||||||
|
网页端已将搜索迁移到 /rest/v/search/feed 并需要签名
|
||||||
|
:param keyword: 搜索关键词
|
||||||
|
:param pcursor: 分页游标(数字页码字符串)
|
||||||
|
:param search_session_id: 搜索会话ID
|
||||||
|
:return: 顶层结构 {result, pcursor, feeds, searchSessionId, ...}
|
||||||
|
"""
|
||||||
|
post_data = {
|
||||||
|
"keyword": keyword,
|
||||||
|
"pcursor": pcursor,
|
||||||
|
"page": "search",
|
||||||
|
"searchSessionId": search_session_id,
|
||||||
|
}
|
||||||
|
return await self.request_rest_v2_signed("/rest/v/search/feed", post_data)
|
||||||
|
|
||||||
async def pong(self) -> bool:
|
async def pong(self) -> bool:
|
||||||
"""get a note to check if login state is ok"""
|
"""get a note to check if login state is ok"""
|
||||||
utils.logger.info("[KuaiShouClient.pong] Begin pong kuaishou...")
|
utils.logger.info("[KuaiShouClient.pong] Begin pong kuaishou...")
|
||||||
@@ -336,20 +398,32 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
|
|||||||
pcursor = ""
|
pcursor = ""
|
||||||
|
|
||||||
while pcursor != "no_more":
|
while pcursor != "no_more":
|
||||||
videos_res = await self.get_video_by_creater(user_id, pcursor)
|
videos_res = await self.get_video_by_creater_v2(user_id, pcursor)
|
||||||
if not videos_res:
|
if not videos_res:
|
||||||
utils.logger.error(
|
utils.logger.error(
|
||||||
f"[KuaiShouClient.get_all_videos_by_creator] The current creator may have been banned by ks, so they cannot access the data."
|
f"[KuaiShouClient.get_all_videos_by_creator] The current creator may have been banned by ks, so they cannot access the data."
|
||||||
)
|
)
|
||||||
break
|
break
|
||||||
|
|
||||||
vision_profile_photo_list = videos_res.get("visionProfilePhotoList", {})
|
# REST 接口用 result 字段表示业务状态,必须显式校验,
|
||||||
pcursor = vision_profile_photo_list.get("pcursor", "")
|
# 否则接口被拒(result:50)会被静默当成"没有更多视频"
|
||||||
|
result_code = videos_res.get("result")
|
||||||
|
if result_code != 1:
|
||||||
|
utils.logger.error(
|
||||||
|
f"[KuaiShouClient.get_all_videos_by_creator] ks api returned business error "
|
||||||
|
f"(result: {result_code}), stop pagination for user_id: {user_id}"
|
||||||
|
)
|
||||||
|
break
|
||||||
|
|
||||||
videos = vision_profile_photo_list.get("feeds", [])
|
pcursor = videos_res.get("pcursor", "")
|
||||||
|
|
||||||
|
videos = videos_res.get("feeds", [])
|
||||||
utils.logger.info(
|
utils.logger.info(
|
||||||
f"[KuaiShouClient.get_all_videos_by_creator] got user_id:{user_id} videos len : {len(videos)}"
|
f"[KuaiShouClient.get_all_videos_by_creator] got user_id:{user_id} videos len : {len(videos)}"
|
||||||
)
|
)
|
||||||
|
if not videos:
|
||||||
|
pcursor = "no_more"
|
||||||
|
break
|
||||||
|
|
||||||
if callback:
|
if callback:
|
||||||
await callback(videos)
|
await callback(videos)
|
||||||
|
|||||||
@@ -44,7 +44,11 @@ from var import comment_tasks_var, crawler_type_var, source_keyword_var
|
|||||||
|
|
||||||
from .client import KuaiShouClient
|
from .client import KuaiShouClient
|
||||||
from .exception import DataFetchError
|
from .exception import DataFetchError
|
||||||
from .help import parse_video_info_from_url, parse_creator_info_from_url
|
from .help import (
|
||||||
|
KS_SIGN_CAPTURE_SCRIPT,
|
||||||
|
parse_video_info_from_url,
|
||||||
|
parse_creator_info_from_url,
|
||||||
|
)
|
||||||
from .login import KuaishouLogin
|
from .login import KuaishouLogin
|
||||||
|
|
||||||
|
|
||||||
@@ -94,6 +98,8 @@ class KuaishouCrawler(AbstractCrawler):
|
|||||||
|
|
||||||
|
|
||||||
self.context_page = await self.browser_context.new_page()
|
self.context_page = await self.browser_context.new_page()
|
||||||
|
# 注入快手签名环境捕获脚本,页面加载后即可通过 __ks_realm 生成 __NS_hxfalcon 签名
|
||||||
|
await self.context_page.add_init_script(KS_SIGN_CAPTURE_SCRIPT)
|
||||||
await self.context_page.goto(f"{self.index_url}?isHome=1")
|
await self.context_page.goto(f"{self.index_url}?isHome=1")
|
||||||
|
|
||||||
# Create a client to interact with the kuaishou website.
|
# Create a client to interact with the kuaishou website.
|
||||||
@@ -151,7 +157,7 @@ class KuaishouCrawler(AbstractCrawler):
|
|||||||
f"[KuaishouCrawler.search] search kuaishou keyword: {keyword}, page: {page}"
|
f"[KuaishouCrawler.search] search kuaishou keyword: {keyword}, page: {page}"
|
||||||
)
|
)
|
||||||
video_id_list: List[str] = []
|
video_id_list: List[str] = []
|
||||||
videos_res = await self.ks_client.search_info_by_keyword(
|
videos_res = await self.ks_client.search_info_by_keyword_v2(
|
||||||
keyword=keyword,
|
keyword=keyword,
|
||||||
pcursor=str(page),
|
pcursor=str(page),
|
||||||
search_session_id=search_session_id,
|
search_session_id=search_session_id,
|
||||||
@@ -162,14 +168,13 @@ class KuaishouCrawler(AbstractCrawler):
|
|||||||
)
|
)
|
||||||
break
|
break
|
||||||
|
|
||||||
vision_search_photo: Dict = videos_res.get("visionSearchPhoto")
|
if videos_res.get("result") != 1:
|
||||||
if vision_search_photo.get("result") != 1:
|
|
||||||
utils.logger.error(
|
utils.logger.error(
|
||||||
f"[KuaishouCrawler.search] search info by keyword:{keyword} not found data "
|
f"[KuaishouCrawler.search] search info by keyword:{keyword} not found data "
|
||||||
)
|
)
|
||||||
break
|
break
|
||||||
search_session_id = vision_search_photo.get("searchSessionId", "")
|
search_session_id = videos_res.get("searchSessionId", "")
|
||||||
for video_detail in vision_search_photo.get("feeds"):
|
for video_detail in videos_res.get("feeds", []):
|
||||||
video_id_list.append(video_detail.get("photo", {}).get("id"))
|
video_id_list.append(video_detail.get("photo", {}).get("id"))
|
||||||
await kuaishou_store.update_kuaishou_video(video_item=video_detail)
|
await kuaishou_store.update_kuaishou_video(video_item=video_detail)
|
||||||
|
|
||||||
|
|||||||
@@ -21,8 +21,70 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
from playwright.async_api import Page
|
||||||
|
|
||||||
from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo
|
from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo
|
||||||
|
|
||||||
|
# 快手网页端签名(__NS_hxfalcon)支持。
|
||||||
|
# 快手网页端已将批量列表接口迁移到带签名的 REST 端点,
|
||||||
|
# 通过页面加载时注入的捕获脚本获取页面内置签名环境的调用入口,
|
||||||
|
# 再调用 $encode 生成签名。仅复用页面自身已加载的 JS 环境,
|
||||||
|
# 不引入额外的签名代码文件。
|
||||||
|
|
||||||
|
KS_SIGN_CAPTURE_SCRIPT = """
|
||||||
|
// 捕获快手页面内置签名环境的调用入口(学习用途)
|
||||||
|
(() => {
|
||||||
|
if (window.__ks_realm) return;
|
||||||
|
let done = false;
|
||||||
|
const setter = function (v) {
|
||||||
|
if (!done && this && typeof this === "object" && this !== window &&
|
||||||
|
typeof this.$encode === "function" &&
|
||||||
|
typeof this.$getCatVersion === "function") {
|
||||||
|
done = true;
|
||||||
|
window.__ks_realm = this;
|
||||||
|
// 捕获成功后移除钩子,避免影响页面其他行为
|
||||||
|
try { delete Object.prototype.caver; } catch (e) {}
|
||||||
|
}
|
||||||
|
Object.defineProperty(this, "caver", {
|
||||||
|
value: v, writable: true, enumerable: true, configurable: true,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
Object.defineProperty(Object.prototype, "caver", { set: setter, configurable: true });
|
||||||
|
} catch (e) {}
|
||||||
|
})();
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
async def get_ks_sign_from_playwright(page: Page, url: str, query: dict, body: dict) -> str:
|
||||||
|
"""
|
||||||
|
通过浏览器页面生成快手 __NS_hxfalcon 签名
|
||||||
|
Args:
|
||||||
|
page: 已加载快手页面的 playwright page(需先注入 KS_SIGN_CAPTURE_SCRIPT)
|
||||||
|
url: 请求路径,如 /rest/v/profile/feed
|
||||||
|
query: 请求 query 参数,如 {"caver": 2}
|
||||||
|
body: 请求 body(JSON 对象)
|
||||||
|
Returns:
|
||||||
|
签名串
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
await page.wait_for_function("() => !!window.__ks_realm", timeout=15000)
|
||||||
|
except Exception:
|
||||||
|
# 页面可能在 cookie 注入前就已加载(未登录态),此时签名环境未初始化,
|
||||||
|
# 重载页面让其在登录态下加载并触发签名请求,捕获脚本将随新 document 生效
|
||||||
|
await page.reload(wait_until="domcontentloaded")
|
||||||
|
await page.wait_for_function("() => !!window.__ks_realm", timeout=20000)
|
||||||
|
return await page.evaluate(
|
||||||
|
"""([u, q, b]) => new Promise((resolve, reject) => {
|
||||||
|
window.__ks_realm.call('$encode', [
|
||||||
|
{ url: u, query: q, form: {}, requestBody: b },
|
||||||
|
{ suc: s => resolve(s), err: e => reject(new Error(String(e))) }
|
||||||
|
]);
|
||||||
|
})""",
|
||||||
|
[url, query, body],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def parse_video_info_from_url(url: str) -> VideoUrlInfo:
|
def parse_video_info_from_url(url: str) -> VideoUrlInfo:
|
||||||
"""
|
"""
|
||||||
|
|||||||
Reference in New Issue
Block a user