feat(media): 重构媒体下载，支持 xhs/dy/ks/bili/wb 五平台

旧实现只覆盖 4 个平台，且把整个文件读进内存、无重试与完整性校验，
代码按平台复制粘贴了 4 份。本次用统一下载器替换：

- 新增 media_downloader/：流式写入、Range 续传、指数退避重试、大小校验、
  路径穿越防护；B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py：从平台原始响应提取媒体地址，
  与下载器解耦；快手首次接入下载能力
- 开关：config.ENABLE_GET_MEDIA 与 --get_media，并打通 API/WebUI；
  同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合：{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质，否则降级 mp4 直链（产物 video-durl.mp4，
  避免低清文件阻塞后续的高清路径）
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法

媒体下载失败只记录日志，不中断爬取主流程。

media_platform/weibo/client.py

@@ -65,7 +65,6 @@ class WeiboClient(ProxyRefreshMixin):
         self.cookie_urls = [self._host]
         self.playwright_page = playwright_page
         self.cookie_dict = cookie_dict
-        self._image_agent_host = "https://i1.wp.com/"
         # Initialize proxy pool (from ProxyRefreshMixin)
         self.init_proxy_pool(proxy_ip_pool)
 
@@ -277,33 +276,6 @@ class WeiboClient(ProxyRefreshMixin):
                 utils.logger.info(f"[WeiboClient.get_note_info_by_id] $render_data value not found")
                 return dict()
 
-    async def get_note_image(self, image_url: str) -> bytes:
-        image_url = image_url[8:]  # Remove https://
-        sub_url = image_url.split("/")
-        image_url = ""
-        for i in range(len(sub_url)):
-            if i == 1:
-                image_url += "large/"  # Get high-resolution images
-            elif i == len(sub_url) - 1:
-                image_url += sub_url[i]
-            else:
-                image_url += sub_url[i] + "/"
-        # Weibo image hosting has anti-hotlinking, so proxy access is needed
-        # Since Weibo images are accessed through i1.wp.com, we need to concatenate the URL
-        final_uri = (f"{self._image_agent_host}"
-                     f"{image_url}")
-        async with make_async_client(proxy=self.proxy) as client:
-            try:
-                response = await client.request("GET", final_uri, timeout=self.timeout)
-                response.raise_for_status()
-                if not response.reason_phrase == "OK":
-                    utils.logger.error(f"[WeiboClient.get_note_image] request {final_uri} err, res:{response.text}")
-                    return None
-                else:
-                    return response.content
-            except httpx.HTTPError as exc:  # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
-                utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}")    # Keep original exception type name for developer debugging
-                return None
 
     async def get_creator_container_info(self, creator_id: str) -> Dict:
         """

media_platform/weibo/core.py

@@ -38,12 +38,14 @@ from playwright.async_api import (
 
 import config
 from base.base_crawler import AbstractCrawler
+from media_downloader import MediaDownloader
 from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
 from store import weibo as weibo_store
 from tools import utils
 from tools.cdp_browser import CDPBrowserManager
 from var import crawler_type_var, source_keyword_var
 
+from . import media as weibo_media
 from .client import WeiboClient
 from .exception import DataFetchError
 from .field import SearchType
@@ -65,6 +67,7 @@ class WeiboCrawler(AbstractCrawler):
         self.mobile_user_agent = utils.get_mobile_user_agent()
         self.cdp_manager = None
         self.ip_proxy_pool = None  # Proxy IP pool for automatic proxy refresh
+        self._media_downloader: Optional[MediaDownloader] = None
 
     async def start(self):
         playwright_proxy_format, httpx_proxy_format = None, None
@@ -179,7 +182,7 @@ class WeiboCrawler(AbstractCrawler):
                         if mblog:
                             note_id_list.append(mblog.get("id"))
                             await weibo_store.update_weibo_note(note_item)
-                            await self.get_note_images(mblog)
+                            await self.download_media(mblog)
 
                 page += 1
 
@@ -200,6 +203,9 @@ class WeiboCrawler(AbstractCrawler):
         for note_item in video_details:
             if note_item:
                 await weibo_store.update_weibo_note(note_item)
+                mblog = note_item.get("mblog")
+                if mblog:
+                    await self.download_media(mblog)
         await self.batch_get_notes_comments(config.WEIBO_SPECIFIED_ID_LIST)
 
     async def get_note_info_task(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[Dict]:
@@ -269,36 +275,46 @@ class WeiboCrawler(AbstractCrawler):
             except Exception as e:
                 utils.logger.error(f"[WeiboCrawler.get_note_comments] may be been blocked, err:{e}")
 
-    async def get_note_images(self, mblog: Dict):
-        """
-        get note images
-        :param mblog:
-        :return:
-        """
-        if not config.ENABLE_GET_MEIDAS:
-            utils.logger.info(f"[WeiboCrawler.get_note_images] Crawling image mode is not enabled")
-            return
+    async def download_media(self, mblog: Dict) -> None:
+        """下载微博的媒体资源（配图，或视频 + 封面）
 
-        pics: List = mblog.get("pics")
-        if not pics:
+        :param mblog: 微博正文数据
+        """
+        if not config.ENABLE_GET_MEDIA:
             return
-        for pic in pics:
-            if isinstance(pic, str):
-                url = pic
-                pid = url.split("/")[-1].split(".")[0]
-            elif isinstance(pic, dict):
-                url = pic.get("url")
-                pid = pic.get("pid", "")
-            else:
-                continue
-            if not url:
-                continue
-            content = await self.wb_client.get_note_image(url)
-            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
-            utils.logger.info(f"[WeiboCrawler.get_note_images] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching image")
-            if content != None:
-                extension_file_name = url.split(".")[-1]
-                await weibo_store.update_weibo_note_image(pid, content, extension_file_name)
+        try:
+            items = weibo_media.build_media_items(mblog)
+            if items:
+                await self._get_media_downloader().download_all(items)
+        except Exception as exc:
+            # 媒体下载是旁路能力，解析异常/网络异常都不能中断爬取主流程
+            utils.logger.error(f"[WeiboCrawler.download_media] 媒体下载异常: {exc}")
+
+    def _get_media_downloader(self) -> MediaDownloader:
+        """惰性创建媒体下载器，并同步最新的代理与 UA（代理池是就地刷新的）"""
+        if self._media_downloader is None:
+            self._media_downloader = MediaDownloader(
+                platform="wb",
+                proxy=getattr(self.wb_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        else:
+            self._media_downloader.update_credentials(
+                proxy=getattr(self.wb_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        return self._media_downloader
+
+    def _media_headers(self) -> Dict:
+        """媒体请求头：平台 Referer（防盗链）+ UA。
+
+        client.headers 里含 Cookie，不能整体透传到 CDN 请求上，因此只取 UA。
+        """
+        headers = {"Referer": "https://weibo.com/"}
+        client_headers = getattr(self.wb_client, "headers", {}) or {}
+        if client_headers.get("User-Agent"):
+            headers["User-Agent"] = client_headers["User-Agent"]
+        return headers
 
     async def get_creators_and_notes(self) -> None:
         """
@@ -321,6 +337,10 @@ class WeiboCrawler(AbstractCrawler):
                     # If full text fetching is enabled, batch get full text first
                     updated_note_list = await self.batch_get_notes_full_text(note_list)
                     await weibo_store.batch_update_weibo_notes(updated_note_list)
+                    for note_item in updated_note_list:
+                        mblog = (note_item or {}).get("mblog")
+                        if mblog:
+                            await self.download_media(mblog)
 
                 # Get all note information of the creator
                 all_notes_list = await self.wb_client.get_all_notes_by_creator_id(

media_platform/weibo/media.py

@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 relakkes@gmail.com
+#
+# This file is part of MediaCrawler project.
+# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/weibo/media.py
+# GitHub: https://github.com/NanmiCoder
+# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
+#
+
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于任何非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+"""微博媒体地址提取。
+
+输入是微博 ``mblog``，输出是可下载的 ``MediaItem`` 列表。无 IO（少数平台会读取 config 决定策略）。
+
+图片需要特殊处理：微博图床有防盗链，且缩略图路径里带尺寸段，
+统一改写为 ``large`` 清晰度后经 i1.wp.com 代理访问。
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List
+from urllib.parse import urlsplit
+
+from media_downloader import MediaItem, MediaType
+
+# 微博图床存在防盗链，统一走该图片代理访问
+WEIBO_IMAGE_AGENT_HOST = "https://i1.wp.com/"
+
+# 视频地址字段，按清晰度从高到低尝试
+_VIDEO_KEYS = (
+    "mp4_1080p_mp4",
+    "mp4_hd_mp4",
+    "mp4_720p_mp4",
+    "hevc_mp4_hd",
+    "mp4_ld_mp4",
+    "stream_url",
+)
+
+
+def rewrite_image_url(raw_url: str) -> str:
+    """把微博图床地址改写为 large 清晰度并加图片代理前缀。
+
+    ``https://wx1.sinaimg.cn/orj360/abc.jpg``
+    -> ``https://i1.wp.com/wx1.sinaimg.cn/large/abc.jpg``
+    """
+    parts = urlsplit(raw_url)
+    if not parts.scheme or not parts.netloc:
+        return raw_url
+
+    segments = [segment for segment in parts.path.split("/") if segment]
+    if segments:
+        segments[0] = "large"  # 原始路径首段是尺寸标识（orj360 / thumbnail 等）
+
+    path = "/" + "/".join(segments) if segments else parts.path
+    return f"{WEIBO_IMAGE_AGENT_HOST}{parts.netloc}{path}"
+
+
+def extract_image_urls(mblog: Dict) -> List[str]:
+    """提取微博配图地址（已改写为可访问地址）"""
+    urls: List[str] = []
+    for pic in mblog.get("pics") or []:
+        if isinstance(pic, str):
+            raw_url = pic
+        elif isinstance(pic, dict):
+            raw_url = pic.get("url") or ""
+        else:
+            continue
+        if isinstance(raw_url, str) and raw_url:
+            urls.append(rewrite_image_url(raw_url))
+    return urls
+
+
+def extract_video_urls(mblog: Dict) -> List[str]:
+    """提取微博视频地址候选列表（按清晰度从高到低）"""
+    page_info = mblog.get("page_info") or {}
+    if not isinstance(page_info, dict):
+        return []
+    if page_info.get("type") != "video" and not page_info.get("media_info"):
+        return []
+
+    urls: List[str] = []
+    for container_key in ("media_info", "urls"):
+        container = page_info.get(container_key) or {}
+        if not isinstance(container, dict):
+            continue
+        for key in _VIDEO_KEYS:
+            url = container.get(key)
+            if isinstance(url, str) and url and url not in urls:
+                urls.append(url)
+    return urls
+
+
+def extract_cover_url(mblog: Dict) -> str:
+    """提取视频封面地址"""
+    page_info = mblog.get("page_info") or {}
+    if not isinstance(page_info, dict):
+        return ""
+    page_pic = page_info.get("page_pic") or {}
+    if isinstance(page_pic, dict):
+        url = page_pic.get("url")
+        return url if isinstance(url, str) else ""
+    return ""
+
+
+def media_source_mblog(mblog: Dict) -> Dict:
+    """返回真正承载媒体内容的那一层微博。
+
+    纯转发的 ``pics`` / ``page_info`` 位于 ``retweeted_status`` 里，顶层两者皆无；
+    不处理的话转发帖会静默地一条媒体都下载不到。
+    若转发时自己带了图（顶层有媒体），则以顶层为准。
+    """
+    if mblog.get("pics") or mblog.get("page_info"):
+        return mblog
+
+    retweeted = mblog.get("retweeted_status")
+    if isinstance(retweeted, dict):
+        return retweeted
+    return mblog
+
+
+def build_media_items(mblog: Dict) -> List[MediaItem]:
+    """把一条微博转换成待下载的媒体任务列表。
+
+    - 视频微博：封面 + 视频（配图通常就是封面，不重复下载）
+    - 普通微博：各张配图
+    - 转发微博：取原博的媒体，目录仍按转发帖自身的 id 组织
+    """
+    if not isinstance(mblog, dict):
+        return []
+
+    content_id = mblog.get("id") or mblog.get("mid") or ""
+    if not content_id:
+        return []
+    content_id = str(content_id)
+
+    source = media_source_mblog(mblog)
+    items: List[MediaItem] = []
+
+    video_urls = extract_video_urls(source)
+    if video_urls:
+        cover_url = extract_cover_url(source)
+        if cover_url:
+            items.append(
+                MediaItem(
+                    url=cover_url,
+                    media_type=MediaType.IMAGE,
+                    content_id=content_id,
+                    stem="cover",
+                )
+            )
+        items.append(
+            MediaItem(
+                url=video_urls[0],
+                backup_urls=tuple(video_urls[1:]),
+                media_type=MediaType.VIDEO,
+                content_id=content_id,
+                stem="video",
+            )
+        )
+        return items
+
+    for index, image_url in enumerate(extract_image_urls(source), start=1):
+        items.append(
+            MediaItem(
+                url=image_url,
+                media_type=MediaType.IMAGE,
+                content_id=content_id,
+                stem=f"{index:03d}",
+            )
+        )
+
+    return items