feat(media): 重构媒体下载，支持 xhs/dy/ks/bili/wb 五平台

旧实现只覆盖 4 个平台，且把整个文件读进内存、无重试与完整性校验，代码按平台复制粘贴了 4 份。本次用统一下载器替换： - 新增 media_downloader/：流式写入、Range 续传、指数退避重试、大小校验、路径穿越防护；B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流 - 新增 media_platform/<平台>/media.py：从平台原始响应提取媒体地址，与下载器解耦；快手首次接入下载能力 - 开关：config.ENABLE_GET_MEDIA 与 --get_media，并打通 API/WebUI；同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写 - 落盘按帖子聚合：{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/ - B 站装好 ffmpeg 时走 DASH 最高画质，否则降级 mp4 直链（产物 video-durl.mp4，避免低清文件阻塞后续的高清路径） - 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法媒体下载失败只记录日志，不中断爬取主流程。
2026-10-04 13:58:11 +08:00 · 2026-09-17 22:54:22 +08:00
parent 60e66f2a92
commit 0ca7b29cf0
47 changed files with 5187 additions and 907 deletions
@@ -35,6 +35,7 @@ from playwright.async_api import (

 import config
 from base.base_crawler import AbstractCrawler
+from media_downloader import MediaDownloader
 from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo
 from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
 from store import kuaishou as kuaishou_store
@@ -42,6 +43,7 @@ from tools import utils
 from tools.cdp_browser import CDPBrowserManager
 from var import comment_tasks_var, crawler_type_var, source_keyword_var

+from . import media as kuaishou_media
 from .client import KuaiShouClient
 from .exception import DataFetchError
 from .help import (
@@ -64,6 +66,7 @@ class KuaishouCrawler(AbstractCrawler):
        self.user_agent = utils.get_user_agent()
        self.cdp_manager = None
        self.ip_proxy_pool = None  # Proxy IP pool, used for automatic proxy refresh
+        self._media_downloader: Optional[MediaDownloader] = None

    async def start(self):
        playwright_proxy_format, httpx_proxy_format = None, None
@@ -177,6 +180,7 @@ class KuaishouCrawler(AbstractCrawler):
                for video_detail in videos_res.get("feeds", []):
                    video_id_list.append(video_detail.get("photo", {}).get("id"))
                    await kuaishou_store.update_kuaishou_video(video_item=video_detail)
+                    await self.download_media(video_item=video_detail)

                utils.logger.info(
                    f"[KuaishouCrawler.search] keyword: {keyword}, page: {page}, got {len(video_id_list)} videos"
@@ -215,6 +219,7 @@ class KuaishouCrawler(AbstractCrawler):
        for video_detail in video_details:
            if video_detail is not None:
                await kuaishou_store.update_kuaishou_video(video_detail)
+                await self.download_media(video_item=video_detail)
        await self.batch_get_video_comments(video_ids)

    async def get_video_info_task(
@@ -464,6 +469,49 @@ class KuaishouCrawler(AbstractCrawler):
        for video_detail in video_details:
            if video_detail is not None:
                await kuaishou_store.update_kuaishou_video(video_detail)
+                await self.download_media(video_item=video_detail)
+
+    async def download_media(self, video_item: Dict):
+        """下载快手作品的媒体资源（封面 + 视频）
+
+        Args:
+            video_item (Dict): 快手作品详情（结构为 {"photo": {...}, "author": {...}}）
+        """
+        if not config.ENABLE_GET_MEDIA:
+            return
+        try:
+            items = kuaishou_media.build_media_items(video_item)
+            if items:
+                await self._get_media_downloader().download_all(items)
+        except Exception as exc:
+            # 媒体下载是旁路能力，解析异常/网络异常都不能中断爬取主流程
+            utils.logger.error(f"[KuaishouCrawler.download_media] 媒体下载异常: {exc}")
+
+    def _get_media_downloader(self) -> MediaDownloader:
+        """惰性创建媒体下载器，并同步最新的代理与 UA（代理池是就地刷新的）"""
+        if self._media_downloader is None:
+            self._media_downloader = MediaDownloader(
+                platform="ks",
+                proxy=getattr(self.ks_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        else:
+            self._media_downloader.update_credentials(
+                proxy=getattr(self.ks_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        return self._media_downloader
+
+    def _media_headers(self) -> Dict:
+        """媒体请求头：平台 Referer（防盗链）+ UA。
+
+        client.headers 里含 Cookie，不能整体透传到 CDN 请求上，因此只取 UA。
+        """
+        headers = {"Referer": "https://www.kuaishou.com/"}
+        client_headers = getattr(self.ks_client, "headers", {}) or {}
+        if client_headers.get("User-Agent"):
+            headers["User-Agent"] = client_headers["User-Agent"]
+        return headers

    async def close(self):
        """Close browser context"""
@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 relakkes@gmail.com
+#
+# This file is part of MediaCrawler project.
+# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/kuaishou/media.py
+# GitHub: https://github.com/NanmiCoder
+# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
+#
+
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于任何非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+"""快手媒体地址提取。
+
+输入是快手作品详情（``{"photo": {...}, "author": {...}}``），
+输出是可下载的 ``MediaItem`` 列表。无 IO（少数平台会读取 config 决定策略）。
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List
+
+from media_downloader import MediaItem, MediaType
+
+
+def _first_url(urls) -> str:
+    """从 url 列表里取第一个非空字符串"""
+    if isinstance(urls, str):
+        return urls
+    for url in urls or []:
+        if isinstance(url, str) and url:
+            return url
+    return ""
+
+
+def _photo_of(video_item: Dict) -> Dict:
+    """接口响应里 photo 字段可能是字符串或 None，统一收敛为 dict"""
+    photo = video_item.get("photo")
+    return photo if isinstance(photo, dict) else {}
+
+
+def _extract_representation_url(codec_resource) -> str:
+    """从 videoResource.{hevc,h264} 的 adaptationSet 里取第一个播放地址"""
+    if not isinstance(codec_resource, dict):
+        return ""
+    for adaptation in codec_resource.get("adaptationSet") or []:
+        if not isinstance(adaptation, dict):
+            continue
+        for representation in adaptation.get("representation") or []:
+            if isinstance(representation, dict) and representation.get("url"):
+                return representation["url"]
+    return ""
+
+
+def extract_video_urls(video_item: Dict) -> List[str]:
+    """提取视频地址候选列表（H265 优先，画质与体积更优）"""
+    photo = _photo_of(video_item)
+    if not photo:
+        return []
+
+    video_resource = photo.get("videoResource") if isinstance(photo.get("videoResource"), dict) else {}
+    candidates = [
+        photo.get("photoH265Url"),
+        photo.get("photoUrl"),
+        _extract_representation_url(video_resource.get("hevc")),
+        _extract_representation_url(video_resource.get("h264")),
+        # 详情接口另有 manifest.adaptationSet[].representation[].url 一份等价描述，
+        # search 接口的 photo 对象字段更少时靠它兜底
+        _extract_representation_url(photo.get("manifest")),
+    ]
+
+    urls: List[str] = []
+    for candidate in candidates:
+        if candidate and isinstance(candidate, str) and candidate not in urls:
+            urls.append(candidate)
+    return urls
+
+
+def extract_cover_url(video_item: Dict) -> str:
+    """提取视频封面地址"""
+    photo = _photo_of(video_item)
+    if not photo:
+        return ""
+
+    for candidate in (
+        photo.get("coverUrl"),
+        _first_url([entry.get("url") for entry in photo.get("coverUrls") or [] if isinstance(entry, dict)]),
+        photo.get("animatedCoverUrl"),
+    ):
+        if isinstance(candidate, str) and candidate:
+            return candidate
+    return ""
+
+
+def build_media_items(video_item: Dict) -> List[MediaItem]:
+    """把一个快手作品转换成待下载的媒体任务列表：封面 + 视频"""
+    if not isinstance(video_item, dict):
+        return []
+
+    content_id = _photo_of(video_item).get("id")
+    if not content_id:
+        return []
+
+    content_id = str(content_id)
+    items: List[MediaItem] = []
+
+    cover_url = extract_cover_url(video_item)
+    if cover_url:
+        items.append(
+            MediaItem(
+                url=cover_url,
+                media_type=MediaType.IMAGE,
+                content_id=content_id,
+                stem="cover",
+            )
+        )
+
+    video_urls = extract_video_urls(video_item)
+    if video_urls:
+        items.append(
+            MediaItem(
+                url=video_urls[0],
+                backup_urls=tuple(video_urls[1:]),
+                media_type=MediaType.VIDEO,
+                content_id=content_id,
+                stem="video",
+            )
+        )
+
+    return items