fork/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-09-15 14:17:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(kuaishou): 降低服务端限流概率并支持限流退避重试

Browse Source 
- 请求延时在固定基础上加 1-3 秒随机抖动，降低被限流概率
- 签名 REST 请求遇到限流(result:2)时指数退避重试 3 次，不再直接中断
- 重试时重新生成签名（签名绑定请求内容和时间窗口）
This commit is contained in:
程序员阿江(Relakkes)
2026-08-05 17:39:33 +08:00
parent 2a8063a9f2
commit 071c8c0aca
2 changed files with 52 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
media_platform/kuaishou/client.py
View File

@@ -21,6 +21,7 @@
# -*- coding: utf-8 -*-
import asyncio
import json
import random
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
from urllib.parse import urlencode
@@ -117,32 +118,45 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
"""
带 __NS_hxfalcon 签名的 REST API V2 请求
快手网页端批量列表接口（作品列表/搜索）需要签名，未签名请求会返回 result:50
服务端限流（result:2）时指数退避重试
:param uri: API endpoint path
:param data: request body
:return: response data
"""
await self._refresh_proxy_if_expired()
max_retry = 3
for attempt in range(max_retry):
await self._refresh_proxy_if_expired()
sign = await get_ks_sign_from_playwright(
self.playwright_page,
uri,
{"caver": 2},
data,
)
json_str = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
async with make_async_client(proxy=self.proxy) as client:
response = await client.request(
method="POST",
url=f"{self._rest_host}{uri}?__NS_hxfalcon={sign}&caver=2",
data=json_str,
timeout=self.timeout,
headers=self.headers,
# 签名绑定请求内容和时间窗口，重试时必须重新生成
sign = await get_ks_sign_from_playwright(
self.playwright_page,
uri,
{"caver": 2},
data,
)
result: Dict = response.json()
if result.get("result") != 1:
json_str = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
async with make_async_client(proxy=self.proxy) as client:
response = await client.request(
method="POST",
url=f"{self._rest_host}{uri}?__NS_hxfalcon={sign}&caver=2",
data=json_str,
timeout=self.timeout,
headers=self.headers,
)
result: Dict = response.json()
if result.get("result") == 1:
return result
if result.get("result") == 2:
delay = 5 * (2**attempt) + random.uniform(0, 2)
utils.logger.warning(
f"[KuaiShouClient.request_rest_v2_signed] rate limited (result:2) on {uri}, "
f"retry in {delay:.1f}s, attempt {attempt + 1}/{max_retry}"
)
await asyncio.sleep(delay)
continue
raise DataFetchError(f"REST API V2 error: {result}")
return result
raise DataFetchError(f"REST API V2 error: {result}")
async def get_video_by_creater_v2(self, userId: str, pcursor: str = "") -> Dict:
"""
@@ -312,7 +326,8 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
if callback: # If there is a callback function, execute the callback function
await callback(photo_id, comments)
result.extend(comments)
await asyncio.sleep(crawl_interval)
# 固定延时基础上加 1-3 秒随机抖动，降低服务端限流概率
await asyncio.sleep(crawl_interval + random.uniform(1, 3))
sub_comments = await self.get_comments_all_sub_comments(
comments, photo_id, crawl_interval, callback
)
@@ -366,7 +381,8 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
if callback and sub_comments:
await callback(photo_id, sub_comments)
await asyncio.sleep(crawl_interval)
# 固定延时基础上加 1-3 秒随机抖动，降低服务端限流概率
await asyncio.sleep(crawl_interval + random.uniform(1, 3))
result.extend(sub_comments)
return result
@@ -427,6 +443,7 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
if callback:
await callback(videos)
await asyncio.sleep(crawl_interval)
# 固定延时基础上加 1-3 秒随机抖动，降低服务端限流概率
await asyncio.sleep(crawl_interval + random.uniform(1, 3))
result.extend(videos)
return result

20
media_platform/kuaishou/core.py
View File

@@ -20,7 +20,7 @@
import asyncio
import os
# import random # Removed as we now use fixed config.CRAWLER_MAX_SLEEP_SEC intervals
import random
import time
from asyncio import Task
from typing import Dict, List, Optional, Tuple
@@ -186,8 +186,10 @@ class KuaishouCrawler(AbstractCrawler):
page += 1
# Sleep after page navigation
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
utils.logger.info(f"[KuaishouCrawler.search] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {page-1}")
# 固定延时基础上加 1-3 秒随机抖动，降低服务端限流概率
sleep_sec = config.CRAWLER_MAX_SLEEP_SEC + random.uniform(1, 3)
await asyncio.sleep(sleep_sec)
utils.logger.info(f"[KuaishouCrawler.search] Sleeping for {sleep_sec:.1f} seconds after page {page-1}")
await self.batch_get_video_comments(video_id_list)
@@ -224,8 +226,10 @@ class KuaishouCrawler(AbstractCrawler):
result = await self.ks_client.get_video_info(video_id)
# Sleep after fetching video details
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
utils.logger.info(f"[KuaishouCrawler.get_video_info_task] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching video details {video_id}")
# 固定延时基础上加 1-3 秒随机抖动，降低服务端限流概率
sleep_sec = config.CRAWLER_MAX_SLEEP_SEC + random.uniform(1, 3)
await asyncio.sleep(sleep_sec)
utils.logger.info(f"[KuaishouCrawler.get_video_info_task] Sleeping for {sleep_sec:.1f} seconds after fetching video details {video_id}")
detail = result.get("visionVideoDetail")
if detail:
@@ -289,8 +293,10 @@ class KuaishouCrawler(AbstractCrawler):
)
# Sleep before fetching comments
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
utils.logger.info(f"[KuaishouCrawler.get_comments] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds before fetching comments for video {video_id}")
# 固定延时基础上加 1-3 秒随机抖动，降低服务端限流概率
sleep_sec = config.CRAWLER_MAX_SLEEP_SEC + random.uniform(1, 3)
await asyncio.sleep(sleep_sec)
utils.logger.info(f"[KuaishouCrawler.get_comments] Sleeping for {sleep_sec:.1f} seconds before fetching comments for video {video_id}")
await self.ks_client.get_video_all_comments(
photo_id=video_id,