feat(media): 重构媒体下载，支持 xhs/dy/ks/bili/wb 五平台

旧实现只覆盖 4 个平台，且把整个文件读进内存、无重试与完整性校验，
代码按平台复制粘贴了 4 份。本次用统一下载器替换：

- 新增 media_downloader/：流式写入、Range 续传、指数退避重试、大小校验、
  路径穿越防护；B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py：从平台原始响应提取媒体地址，
  与下载器解耦；快手首次接入下载能力
- 开关：config.ENABLE_GET_MEDIA 与 --get_media，并打通 API/WebUI；
  同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合：{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质，否则降级 mp4 直链（产物 video-durl.mp4，
  避免低清文件阻塞后续的高清路径）
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法

媒体下载失败只记录日志，不中断爬取主流程。

media_platform/douyin/client.py

@@ -344,20 +344,6 @@ class DouYinClient(AbstractApiClient, ProxyRefreshMixin):
             result.extend(aweme_list)
         return result
 
-    async def get_aweme_media(self, url: str) -> Union[bytes, None]:
-        async with make_async_client(proxy=self.proxy) as client:
-            try:
-                response = await client.request("GET", url, timeout=self.timeout, follow_redirects=True)
-                response.raise_for_status()
-                if not response.reason_phrase == "OK":
-                    utils.logger.error(f"[DouYinClient.get_aweme_media] request {url} err, res:{response.text}")
-                    return None
-                else:
-                    return response.content
-            except httpx.HTTPError as exc:  # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
-                utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}")  # Keep the original exception type name for developers to debug
-                return None
-
     async def resolve_short_url(self, short_url: str) -> str:
         """
         解析抖音短链接,获取重定向后的真实URL

media_platform/douyin/core.py

@@ -33,12 +33,14 @@ from playwright.async_api import (
 
 import config
 from base.base_crawler import AbstractCrawler
+from media_downloader import MediaDownloader
 from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
 from store import douyin as douyin_store
 from tools import utils
 from tools.cdp_browser import CDPBrowserManager
 from var import crawler_type_var, source_keyword_var
 
+from . import media as douyin_media
 from .client import DouYinClient
 from .exception import DataFetchError
 from .field import PublishTimeType
@@ -63,6 +65,7 @@ class DouYinCrawler(AbstractCrawler):
         ]
         self.cdp_manager = None
         self.ip_proxy_pool = None  # Proxy IP pool for automatic proxy refresh
+        self._media_downloader: Optional[MediaDownloader] = None
 
     async def start(self) -> None:
         playwright_proxy_format, httpx_proxy_format = None, None
@@ -170,7 +173,7 @@ class DouYinCrawler(AbstractCrawler):
                     aweme_list.append(aweme_info.get("aweme_id", ""))
                     page_aweme_list.append(aweme_info.get("aweme_id", ""))
                     await douyin_store.update_douyin_aweme(aweme_item=aweme_info)
-                    await self.get_aweme_media(aweme_item=aweme_info)
+                    await self.download_media(aweme_item=aweme_info)
                 
                 # Batch get note comments for the current page
                 await self.batch_get_note_comments(page_aweme_list)
@@ -212,7 +215,7 @@ class DouYinCrawler(AbstractCrawler):
         for aweme_detail in aweme_details:
             if aweme_detail is not None:
                 await douyin_store.update_douyin_aweme(aweme_item=aweme_detail)
-                await self.get_aweme_media(aweme_item=aweme_detail)
+                await self.download_media(aweme_item=aweme_detail)
         await self.batch_get_note_comments(aweme_id_list)
 
     async def get_aweme_detail(self, aweme_id: str, semaphore: asyncio.Semaphore) -> Any:
@@ -304,7 +307,7 @@ class DouYinCrawler(AbstractCrawler):
         for aweme_item in note_details:
             if aweme_item is not None:
                 await douyin_store.update_douyin_aweme(aweme_item=aweme_item)
-                await self.get_aweme_media(aweme_item=aweme_item)
+                await self.download_media(aweme_item=aweme_item)
 
     async def create_douyin_client(self, httpx_proxy: Optional[str]) -> DouYinClient:
         """Create douyin client"""
@@ -399,72 +402,44 @@ class DouYinCrawler(AbstractCrawler):
             await self.browser_context.close()
         utils.logger.info("[DouYinCrawler.close] Browser context closed ...")
 
-    async def get_aweme_media(self, aweme_item: Dict):
-        """
-        获取抖音媒体，自动判断媒体类型是短视频还是帖子图片并下载
+    async def download_media(self, aweme_item: Dict) -> None:
+        """下载抖音作品的媒体资源（图集图片，或视频 + 封面）
 
         Args:
             aweme_item (Dict): 抖音作品详情
         """
-        if not config.ENABLE_GET_MEIDAS:
-            utils.logger.info(f"[DouYinCrawler.get_aweme_media] Crawling image mode is not enabled")
+        if not config.ENABLE_GET_MEDIA:
             return
-        # List of note urls. If it is a short video type, an empty list will be returned.
-        note_download_url: List[str] = douyin_store._extract_note_image_list(aweme_item)
-        # The video URL will always exist, but when it is a short video type, the file is actually an audio file.
-        video_download_url: str = douyin_store._extract_video_download_url(aweme_item)
-        # TODO: Douyin does not adopt the audio and video separation strategy, so the audio can be separated from the original video and will not be extracted for the time being.
-        if note_download_url:
-            await self.get_aweme_images(aweme_item)
+        try:
+            items = douyin_media.build_media_items(aweme_item)
+            if items:
+                await self._get_media_downloader().download_all(items)
+        except Exception as exc:
+            # 媒体下载是旁路能力，解析异常/网络异常都不能中断爬取主流程
+            utils.logger.error(f"[DouYinCrawler.download_media] 媒体下载异常: {exc}")
+
+    def _get_media_downloader(self) -> MediaDownloader:
+        """惰性创建媒体下载器，并同步最新的代理与 UA（代理池是就地刷新的）"""
+        if self._media_downloader is None:
+            self._media_downloader = MediaDownloader(
+                platform="dy",
+                proxy=getattr(self.dy_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
         else:
-            await self.get_aweme_video(aweme_item)
+            self._media_downloader.update_credentials(
+                proxy=getattr(self.dy_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        return self._media_downloader
 
-    async def get_aweme_images(self, aweme_item: Dict):
+    def _media_headers(self) -> Dict:
+        """媒体请求头：平台 Referer（防盗链）+ UA。
+
+        client.headers 里含 Cookie，不能整体透传到 CDN 请求上，因此只取 UA。
         """
-        get aweme images. please use get_aweme_media
-
-        Args:
-            aweme_item (Dict): 抖音作品详情
-        """
-        if not config.ENABLE_GET_MEIDAS:
-            return
-        aweme_id = aweme_item.get("aweme_id")
-        # List of note urls. If it is a short video type, an empty list will be returned.
-        note_download_url: List[str] = douyin_store._extract_note_image_list(aweme_item)
-
-        if not note_download_url:
-            return
-        picNum = 0
-        for url in note_download_url:
-            if not url:
-                continue
-            content = await self.dy_client.get_aweme_media(url)
-            await asyncio.sleep(random.random())
-            if content is None:
-                continue
-            extension_file_name = f"{picNum:>03d}.jpeg"
-            picNum += 1
-            await douyin_store.update_dy_aweme_image(aweme_id, content, extension_file_name)
-
-    async def get_aweme_video(self, aweme_item: Dict):
-        """
-        get aweme videos. please use get_aweme_media
-
-        Args:
-            aweme_item (Dict): 抖音作品详情
-        """
-        if not config.ENABLE_GET_MEIDAS:
-            return
-        aweme_id = aweme_item.get("aweme_id")
-
-        # The video URL will always exist, but when it is a short video type, the file is actually an audio file.
-        video_download_url: str = douyin_store._extract_video_download_url(aweme_item)
-
-        if not video_download_url:
-            return
-        content = await self.dy_client.get_aweme_media(video_download_url)
-        await asyncio.sleep(random.random())
-        if content is None:
-            return
-        extension_file_name = f"video.mp4"
-        await douyin_store.update_dy_aweme_video(aweme_id, content, extension_file_name)
+        headers = {"Referer": "https://www.douyin.com/"}
+        client_headers = getattr(self.dy_client, "headers", {}) or {}
+        if client_headers.get("User-Agent"):
+            headers["User-Agent"] = client_headers["User-Agent"]
+        return headers

media_platform/douyin/media.py

@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 relakkes@gmail.com
+#
+# This file is part of MediaCrawler project.
+# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/douyin/media.py
+# GitHub: https://github.com/NanmiCoder
+# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
+#
+
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于任何非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+"""抖音媒体地址提取。
+
+输入是抖音作品详情（``/aweme/v1/web/aweme/detail/`` 的响应），
+输出是可下载的 ``MediaItem`` 列表。无 IO（少数平台会读取 config 决定策略）。
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List
+
+from media_downloader import MediaItem, MediaType
+
+# 视频清晰度字段，按优先级排列（h264 与 256 通常为无水印源）
+_VIDEO_ADDR_KEYS = ("play_addr_h264", "play_addr_256", "play_addr")
+
+# 封面字段，按优先级排列
+_COVER_KEYS = ("raw_cover", "origin_cover", "cover", "dynamic_cover")
+
+
+def _url_list_of(container) -> List[str]:
+    """接口响应里同名字段可能是字符串或 None，统一收敛为 url 列表"""
+    if not isinstance(container, dict):
+        return []
+    return [url for url in (container.get("url_list") or []) if url and isinstance(url, str)]
+
+
+def extract_image_urls(aweme_detail: Dict) -> List[str]:
+    """提取图集作品的图片地址（url_list 的最后一个通常是无水印原图）"""
+    urls: List[str] = []
+    for image in aweme_detail.get("images") or []:
+        candidates = _url_list_of(image)
+        if candidates:
+            urls.append(candidates[-1])
+    return urls
+
+
+def extract_cover_url(aweme_detail: Dict) -> str:
+    """提取视频封面地址"""
+    video_item = aweme_detail.get("video")
+    if not isinstance(video_item, dict):
+        return ""
+    for key in _COVER_KEYS:
+        candidates = _url_list_of(video_item.get(key))
+        if candidates:
+            return candidates[-1]
+    return ""
+
+
+def extract_video_urls(aweme_detail: Dict) -> List[str]:
+    """提取视频地址候选列表。
+
+    同一档清晰度会返回多个 CDN 地址，优先取 url_list 最后一个（通常无水印），
+    其余地址作为备用；主地址全部不可用时依次降级到更低清晰度档位。
+    """
+    video_item = aweme_detail.get("video")
+    if not isinstance(video_item, dict):
+        return []
+
+    for key in _VIDEO_ADDR_KEYS:
+        candidates = _url_list_of(video_item.get(key))
+        if candidates:
+            return list(reversed(candidates))
+
+    # 兜底：bit_rate 列表里码率最高的那一档
+    bit_rates = [entry for entry in (video_item.get("bit_rate") or []) if isinstance(entry, dict)]
+    for entry in sorted(bit_rates, key=lambda item: item.get("bit_rate", 0), reverse=True):
+        candidates = _url_list_of(entry.get("play_addr"))
+        if candidates:
+            return list(reversed(candidates))
+
+    return []
+
+
+def build_media_items(aweme_item: Dict) -> List[MediaItem]:
+    """把一个抖音作品转换成待下载的媒体任务列表。
+
+    - 图集作品：各张图片（第一张即封面，不再单独下载一份 cover）
+    - 视频作品：封面 + 视频
+    """
+    if not isinstance(aweme_item, dict):
+        return []
+
+    content_id = aweme_item.get("aweme_id") or ""
+    if not content_id:
+        return []
+
+    items: List[MediaItem] = []
+
+    image_urls = extract_image_urls(aweme_item)
+    if image_urls:
+        for index, image_url in enumerate(image_urls, start=1):
+            items.append(
+                MediaItem(
+                    url=image_url,
+                    media_type=MediaType.IMAGE,
+                    content_id=content_id,
+                    stem=f"{index:03d}",
+                )
+            )
+        return items
+
+    cover_url = extract_cover_url(aweme_item)
+    if cover_url:
+        items.append(
+            MediaItem(
+                url=cover_url,
+                media_type=MediaType.IMAGE,
+                content_id=content_id,
+                stem="cover",
+            )
+        )
+
+    video_urls = extract_video_urls(aweme_item)
+    if video_urls:
+        items.append(
+            MediaItem(
+                url=video_urls[0],
+                backup_urls=tuple(video_urls[1:]),
+                media_type=MediaType.VIDEO,
+                content_id=content_id,
+                stem="video",
+            )
+        )
+
+    return items