feat(media): 重构媒体下载,支持 xhs/dy/ks/bili/wb 五平台

旧实现只覆盖 4 个平台,且把整个文件读进内存、无重试与完整性校验,
代码按平台复制粘贴了 4 份。本次用统一下载器替换:

- 新增 media_downloader/:流式写入、Range 续传、指数退避重试、大小校验、
  路径穿越防护;B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py:从平台原始响应提取媒体地址,
  与下载器解耦;快手首次接入下载能力
- 开关:config.ENABLE_GET_MEDIA 与 --get_media,并打通 API/WebUI;
  同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合:{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质,否则降级 mp4 直链(产物 video-durl.mp4,
  避免低清文件阻塞后续的高清路径)
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法

媒体下载失败只记录日志,不中断爬取主流程。
This commit is contained in:
程序员阿江(Relakkes)
2026-09-17 22:54:22 +08:00
parent 60e66f2a92
commit 0ca7b29cf0
47 changed files with 5187 additions and 907 deletions
+48
View File
@@ -35,6 +35,7 @@ from playwright.async_api import (
import config
from base.base_crawler import AbstractCrawler
from media_downloader import MediaDownloader
from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import kuaishou as kuaishou_store
@@ -42,6 +43,7 @@ from tools import utils
from tools.cdp_browser import CDPBrowserManager
from var import comment_tasks_var, crawler_type_var, source_keyword_var
from . import media as kuaishou_media
from .client import KuaiShouClient
from .exception import DataFetchError
from .help import (
@@ -64,6 +66,7 @@ class KuaishouCrawler(AbstractCrawler):
self.user_agent = utils.get_user_agent()
self.cdp_manager = None
self.ip_proxy_pool = None # Proxy IP pool, used for automatic proxy refresh
self._media_downloader: Optional[MediaDownloader] = None
async def start(self):
playwright_proxy_format, httpx_proxy_format = None, None
@@ -177,6 +180,7 @@ class KuaishouCrawler(AbstractCrawler):
for video_detail in videos_res.get("feeds", []):
video_id_list.append(video_detail.get("photo", {}).get("id"))
await kuaishou_store.update_kuaishou_video(video_item=video_detail)
await self.download_media(video_item=video_detail)
utils.logger.info(
f"[KuaishouCrawler.search] keyword: {keyword}, page: {page}, got {len(video_id_list)} videos"
@@ -215,6 +219,7 @@ class KuaishouCrawler(AbstractCrawler):
for video_detail in video_details:
if video_detail is not None:
await kuaishou_store.update_kuaishou_video(video_detail)
await self.download_media(video_item=video_detail)
await self.batch_get_video_comments(video_ids)
async def get_video_info_task(
@@ -464,6 +469,49 @@ class KuaishouCrawler(AbstractCrawler):
for video_detail in video_details:
if video_detail is not None:
await kuaishou_store.update_kuaishou_video(video_detail)
await self.download_media(video_item=video_detail)
async def download_media(self, video_item: Dict):
"""下载快手作品的媒体资源(封面 + 视频)
Args:
video_item (Dict): 快手作品详情(结构为 {"photo": {...}, "author": {...}})
"""
if not config.ENABLE_GET_MEDIA:
return
try:
items = kuaishou_media.build_media_items(video_item)
if items:
await self._get_media_downloader().download_all(items)
except Exception as exc:
# 媒体下载是旁路能力,解析异常/网络异常都不能中断爬取主流程
utils.logger.error(f"[KuaishouCrawler.download_media] 媒体下载异常: {exc}")
def _get_media_downloader(self) -> MediaDownloader:
"""惰性创建媒体下载器,并同步最新的代理与 UA(代理池是就地刷新的)"""
if self._media_downloader is None:
self._media_downloader = MediaDownloader(
platform="ks",
proxy=getattr(self.ks_client, "proxy", None),
extra_headers=self._media_headers(),
)
else:
self._media_downloader.update_credentials(
proxy=getattr(self.ks_client, "proxy", None),
extra_headers=self._media_headers(),
)
return self._media_downloader
def _media_headers(self) -> Dict:
"""媒体请求头:平台 Referer(防盗链)+ UA。
client.headers 里含 Cookie,不能整体透传到 CDN 请求上,因此只取 UA。
"""
headers = {"Referer": "https://www.kuaishou.com/"}
client_headers = getattr(self.ks_client, "headers", {}) or {}
if client_headers.get("User-Agent"):
headers["User-Agent"] = client_headers["User-Agent"]
return headers
async def close(self):
"""Close browser context"""