feat(media): 重构媒体下载，支持 xhs/dy/ks/bili/wb 五平台

旧实现只覆盖 4 个平台，且把整个文件读进内存、无重试与完整性校验，
代码按平台复制粘贴了 4 份。本次用统一下载器替换：

- 新增 media_downloader/：流式写入、Range 续传、指数退避重试、大小校验、
  路径穿越防护；B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py：从平台原始响应提取媒体地址，
  与下载器解耦；快手首次接入下载能力
- 开关：config.ENABLE_GET_MEDIA 与 --get_media，并打通 API/WebUI；
  同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合：{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质，否则降级 mp4 直链（产物 video-durl.mp4，
  避免低清文件阻塞后续的高清路径）
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法

媒体下载失败只记录日志，不中断爬取主流程。

media_platform/bilibili/client.py

@@ -42,6 +42,7 @@ if TYPE_CHECKING:
 from .exception import DataFetchError
 from .field import CommentOrderType, SearchOrderType
 from .help import BilibiliSign
+from .media import DASH_FNVAL
 
 
 def _extract_pinned_comments(value: Any) -> List[Dict]:
@@ -213,11 +214,13 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
             params.update({"bvid": bvid})
         return await self.get(uri, params, enable_params_sign=False)
 
-    async def get_video_play_url(self, aid: int, cid: int) -> Dict:
+    async def get_video_play_url(self, aid: int, cid: int, fnval: int = DASH_FNVAL) -> Dict:
         """
         Bilibli web video play url api
         :param aid: Video aid
         :param cid: cid
+        :param fnval: 请求的媒体格式位掩码，默认请求 DASH（含 4K/8K/HDR/AV1）；
+                      传 1 则请求音视频合一的 mp4 直链
         :return:
         """
         if not aid or not cid or aid <= 0 or cid <= 0:
@@ -229,28 +232,12 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
             "cid": cid,
             "qn": qn_value,
             "fourk": 1,
-            "fnval": 1,
+            "fnval": fnval,
             "platform": "pc",
         }
 
         return await self.get(uri, params, enable_params_sign=True)
 
-    async def get_video_media(self, url: str) -> Union[bytes, None]:
-        # Follow CDN 302 redirects and treat any 2xx as success (some endpoints return 206)
-        async with make_async_client(proxy=self.proxy, follow_redirects=True) as client:
-            try:
-                response = await client.request("GET", url, timeout=self.timeout, headers=self.headers)
-                response.raise_for_status()
-                if 200 <= response.status_code < 300:
-                    return response.content
-                utils.logger.error(
-                    f"[BilibiliClient.get_video_media] Unexpected status {response.status_code} for {url}"
-                )
-                return None
-            except httpx.HTTPError as exc:  # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
-                utils.logger.error(f"[BilibiliClient.get_video_media] {exc.__class__.__name__} for {exc.request.url} - {exc}")  # Keep original exception type name for developer debugging
-                return None
-
     async def get_video_comments(
         self,
         video_id: str,

media_platform/bilibili/core.py

@@ -41,12 +41,14 @@ from playwright._impl._errors import TargetClosedError
 
 import config
 from base.base_crawler import AbstractCrawler
+from media_downloader import MediaDownloader, is_ffmpeg_available
 from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
 from store import bilibili as bilibili_store
 from tools import utils
 from tools.cdp_browser import CDPBrowserManager
 from var import crawler_type_var, source_keyword_var
 
+from . import media as bili_media
 from .client import BilibiliClient
 from .exception import DataFetchError
 from .field import SearchOrderType
@@ -66,6 +68,7 @@ class BilibiliCrawler(AbstractCrawler):
         self.user_agent = utils.get_user_agent()
         self.cdp_manager = None
         self.ip_proxy_pool = None  # Proxy IP pool for automatic proxy refresh
+        self._media_downloader: Optional[MediaDownloader] = None
 
     async def start(self):
         playwright_proxy_format, httpx_proxy_format = None, None
@@ -228,7 +231,7 @@ class BilibiliCrawler(AbstractCrawler):
                         video_id_list.append(video_item.get("View").get("aid"))
                         await bilibili_store.update_bilibili_video(video_item)
                         await bilibili_store.update_up_info(video_item)
-                        await self.get_bilibili_video(video_item, semaphore)
+                        await self.download_media(video_item, semaphore)
                 page += 1
 
                 # Sleep after page navigation
@@ -308,7 +311,7 @@ class BilibiliCrawler(AbstractCrawler):
                                 video_id_list.append(video_item.get("View").get("aid"))
                                 await bilibili_store.update_bilibili_video(video_item)
                                 await bilibili_store.update_up_info(video_item)
-                                await self.get_bilibili_video(video_item, semaphore)
+                                await self.download_media(video_item, semaphore)
 
                         page += 1
 
@@ -413,7 +416,7 @@ class BilibiliCrawler(AbstractCrawler):
                     video_aids_list.append(video_aid)
                 await bilibili_store.update_bilibili_video(video_detail)
                 await bilibili_store.update_up_info(video_detail)
-                await self.get_bilibili_video(video_detail, semaphore)
+                await self.download_media(video_detail, semaphore)
         await self.batch_get_video_comments(video_aids_list)
 
     async def get_video_info_task(self, aid: int, bvid: str, semaphore: asyncio.Semaphore) -> Optional[Dict]:
@@ -440,17 +443,24 @@ class BilibiliCrawler(AbstractCrawler):
                 utils.logger.error(f"[BilibiliCrawler.get_video_info_task] have not fund note detail video_id:{bvid}, err: {ex}")
                 return None
 
-    async def get_video_play_url_task(self, aid: int, cid: int, semaphore: asyncio.Semaphore) -> Union[Dict, None]:
+    async def get_video_play_url_task(
+        self,
+        aid: int,
+        cid: int,
+        semaphore: asyncio.Semaphore,
+        fnval: int = bili_media.DASH_FNVAL,
+    ) -> Union[Dict, None]:
         """
         Get video play url
         :param aid:
         :param cid:
         :param semaphore:
+        :param fnval: 媒体格式位掩码，默认请求 DASH 分轨流
         :return:
         """
         async with semaphore:
             try:
-                result = await self.bili_client.get_video_play_url(aid=aid, cid=cid)
+                result = await self.bili_client.get_video_play_url(aid=aid, cid=cid, fnval=fnval)
                 return result
             except DataFetchError as ex:
                 utils.logger.error(f"[BilibiliCrawler.get_video_play_url_task] Get video play url error: {ex}")
@@ -570,42 +580,128 @@ class BilibiliCrawler(AbstractCrawler):
         except Exception as e:
             utils.logger.error(f"[BilibiliCrawler.close] An error occurred during close: {e}")
 
-    async def get_bilibili_video(self, video_item: Dict, semaphore: asyncio.Semaphore):
+    async def download_media(self, video_item: Dict, semaphore: asyncio.Semaphore):
+        """下载 B 站视频与封面。
+
+        优先走 DASH（音视频分轨 + ffmpeg 合流，画质最好）；本机没有 ffmpeg
+        或 DASH 流不可用时，降级为 mp4 直链（音视频合一，清晰度受接口限制）。
+
+        :param video_item: 视频详情，需包含 View 字段（含 aid/cid/pic/bvid）
+        :param semaphore: 用于限制 playurl 接口请求并发
         """
-        download bilibili video
-        :param video_item:
-        :param semaphore:
-        :return:
-        """
-        if not config.ENABLE_GET_MEIDAS:
-            utils.logger.info(f"[BilibiliCrawler.get_bilibili_video] Crawling image mode is not enabled")
+        if not config.ENABLE_GET_MEDIA:
             return
-        video_item_view: Dict = video_item.get("View")
+        try:
+            await self._download_media(video_item, semaphore)
+        except Exception as exc:
+            # 媒体下载是旁路能力：playurl 是额外的接口调用，网络抖动不能中断爬取主流程
+            utils.logger.error(f"[BilibiliCrawler.download_media] 媒体下载异常: {exc}")
+
+    async def _download_media(self, video_item: Dict, semaphore: asyncio.Semaphore) -> None:
+        video_item_view: Dict = video_item.get("View") or {}
         aid = video_item_view.get("aid")
         cid = video_item_view.get("cid")
-        result = await self.get_video_play_url_task(aid, cid, semaphore)
-        if result is None:
-            utils.logger.info("[BilibiliCrawler.get_bilibili_video] get video play url failed")
-            return
-        durl_list = result.get("durl")
-        max_size = -1
-        video_url = ""
-        for durl in durl_list:
-            size = durl.get("size")
-            if size > max_size:
-                max_size = size
-                video_url = durl.get("url")
-        if video_url == "":
-            utils.logger.info("[BilibiliCrawler.get_bilibili_video] get video url failed")
+        content_id = video_item_view.get("bvid") or (str(aid) if aid else "")
+        if not content_id:
+            utils.logger.warning("[BilibiliCrawler.download_media] 缺少 aid/bvid，跳过媒体下载")
             return
 
-        content = await self.bili_client.get_video_media(video_url)
-        await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
-        utils.logger.info(f"[BilibiliCrawler.get_bilibili_video] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching video {aid}")
-        if content is None:
+        downloader = self._get_media_downloader()
+
+        cover_item = bili_media.build_cover_item(video_item_view, content_id)
+        if cover_item is not None:
+            await downloader.download(cover_item)
+
+        if not aid or not cid:
+            utils.logger.warning("[BilibiliCrawler.download_media] 缺少 aid/cid，无法获取播放地址")
             return
-        extension_file_name = f"video.mp4"
-        await bilibili_store.store_video(aid, content, extension_file_name)
+
+        # ffmpeg 是否可用在请求之前就是确定的（进程内缓存），据此一次性决定 fnval：
+        # 没装 ffmpeg 却先按 DASH 请求一次，既多打一次风控最敏感的 playurl 接口，
+        # 也拿不到可用的 durl
+        dash_supported = is_ffmpeg_available()
+        play_fnval = bili_media.DASH_FNVAL if dash_supported else bili_media.MP4_FNVAL
+        play_info = await self.get_video_play_url_task(aid, cid, semaphore, fnval=play_fnval)
+        await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+        if play_info is None:
+            utils.logger.error("[BilibiliCrawler.download_media] 获取播放地址失败")
+            return
+
+        if dash_supported:
+            # 逐档降级：未登录或权限不足时 B 站会返回高清晰度 URL，但实际取流被 CDN 403，
+            # 此时退到更低的可用档位往往能成功，比直接放弃（或退回更低质的直链）更好
+            quality: Optional[int] = getattr(config, "BILI_QN", 80)
+            while quality is not None:
+                dash_item = bili_media.build_dash_item(play_info, content_id, preferred_quality=quality)
+                if dash_item is None:
+                    break
+                if await downloader.download(dash_item) is not None:
+                    return
+                lower_quality = bili_media.next_lower_quality(play_info, quality)
+                if lower_quality is None:
+                    break
+                utils.logger.warning(
+                    f"[BilibiliCrawler.download_media] 清晰度 {quality} 取流失败，降级到 {lower_quality}"
+                )
+                quality = lower_quality
+
+            utils.logger.warning("[BilibiliCrawler.download_media] DASH 路径失败，降级为 mp4 直链")
+        else:
+            utils.logger.info("[BilibiliCrawler.download_media] 未检测到 ffmpeg，使用 mp4 直链下载（低清晰度）")
+
+        durl_play_info = play_info
+        durl_item = bili_media.build_durl_item(durl_play_info, content_id)
+        if durl_item is None:
+            # DASH 请求不会返回 durl，需要再按 mp4 格式请求一次
+            durl_play_info = await self.get_video_play_url_task(
+                aid, cid, semaphore, fnval=bili_media.MP4_FNVAL
+            ) or {}
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+            durl_item = bili_media.build_durl_item(durl_play_info, content_id)
+
+        if durl_item is None:
+            utils.logger.error(
+                f"[BilibiliCrawler.download_media] 未能获取到视频直链 aid={aid} cid={cid}"
+            )
+            return
+
+        segment_count = bili_media.count_durl_segments(durl_play_info)
+        if segment_count > 1:
+            utils.logger.warning(
+                f"[BilibiliCrawler.download_media] 该视频直链被切成 {segment_count} 段，"
+                f"当前仅下载体积最大的一段，文件可能不完整；"
+                f"确认本机 ffmpeg 可用后走 DASH 路径可获取完整视频"
+            )
+
+        await downloader.download(durl_item)
+
+    def _get_media_downloader(self) -> MediaDownloader:
+        """惰性创建媒体下载器，并同步最新的代理与 Cookie（两者都会就地刷新）"""
+        if self._media_downloader is None:
+            self._media_downloader = MediaDownloader(
+                platform="bili",
+                proxy=getattr(self.bili_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        else:
+            self._media_downloader.update_credentials(
+                proxy=getattr(self.bili_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        return self._media_downloader
+
+    def _media_headers(self) -> Dict:
+        """媒体请求头：平台 Referer（防盗链）+ UA。
+
+        B 站清晰度由 playurl 接口依据 Cookie 决定，返回的直链里已固化流版本，
+        且带 deadline/upsig 签名，下载本身不需要 Cookie（已实测无 Cookie 可直接取流），
+        因此不透传 Cookie，避免把账号凭证扩散到 CDN 域名。
+        """
+        headers = {"Referer": "https://www.bilibili.com/"}
+        client_headers = getattr(self.bili_client, "headers", {}) or {}
+        if client_headers.get("User-Agent"):
+            headers["User-Agent"] = client_headers["User-Agent"]
+        return headers
 
     async def get_all_creator_details(self, creator_url_list: List[str]):
         """

media_platform/bilibili/media.py

@@ -0,0 +1,202 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 relakkes@gmail.com
+#
+# This file is part of MediaCrawler project.
+# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/bilibili/media.py
+# GitHub: https://github.com/NanmiCoder
+# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
+#
+
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于任何非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+"""B 站媒体地址提取与流选择。
+
+B 站的播放地址需要先调 playurl 接口获取：
+- ``fnval=DASH_FNVAL`` 返回音视频分轨的 DASH 流，画质最好，但需要 ffmpeg 合流
+- ``fnval=MP4_FNVAL`` 返回音视频合一的 mp4 直链，无需外部依赖，但清晰度受限
+
+本模块只做纯计算，接口调用与降级编排由 core 负责。
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+import config
+from media_downloader import MediaItem, MediaType
+
+# fnval 位掩码：16(DASH) | 64(HDR) | 128(4K) | 256(Dolby Audio) | 512(Dolby Vision) | 1024(8K) | 2048(AV1)
+DASH_FNVAL = 4048
+# 请求音视频合一的 mp4 直链
+MP4_FNVAL = 1
+
+# B 站 codecid：7=AVC(H.264) 12=HEVC(H.265) 13=AV1 14=AV2
+# 同一清晰度下优先 AVC：下载后的文件要进剪辑软件/播放器，H.264 的兼容性最好
+_CODEC_PRIORITY = {7: 0, 13: 1, 12: 2}
+_UNKNOWN_CODEC_PRIORITY = 3
+
+
+def _as_dict(value) -> Dict:
+    """接口响应里同名字段可能是字符串或 None，统一收敛为 dict"""
+    return value if isinstance(value, dict) else {}
+
+
+def _stream_url(stream) -> str:
+    if not isinstance(stream, dict):
+        return ""
+    return stream.get("base_url") or stream.get("baseUrl") or ""
+
+
+def _to_int(value, default: int = 0) -> int:
+    """接口偶尔把 id/bandwidth 返回成字符串，排序与比较前统一转 int"""
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _stream_sort_key(stream: Dict) -> tuple:
+    """同清晰度下把兼容性最好的编码排在后面（取 [-1] 即选中它）"""
+    codec_rank = _CODEC_PRIORITY.get(_to_int(stream.get("codecid"), -1), _UNKNOWN_CODEC_PRIORITY)
+    return (_to_int(stream.get("id")), -codec_rank, _to_int(stream.get("bandwidth")))
+
+
+def _stream_backup_urls(stream) -> tuple:
+    if not isinstance(stream, dict):
+        return ()
+    backups = stream.get("backup_url") or stream.get("backupUrl") or []
+    return tuple(url for url in backups if isinstance(url, str) and url)
+
+
+def build_cover_item(view: Dict, content_id: str) -> Optional[MediaItem]:
+    """封面地址来自稿件详情里的 pic 字段"""
+    cover_url = _as_dict(view).get("pic")
+    if not isinstance(cover_url, str) or not cover_url:
+        return None
+    return MediaItem(
+        url=cover_url,
+        media_type=MediaType.IMAGE,
+        content_id=content_id,
+        stem="cover",
+    )
+
+
+def pick_video_stream(play_info: Dict, preferred_quality: Optional[int] = None) -> Optional[Dict]:
+    """挑选视频流：优先不超过用户清晰度设置的最高档，都超出时取最低档。
+
+    同一清晰度可能同时提供 AVC/HEVC/AV1 多种编码，按 ``_CODEC_PRIORITY``
+    取兼容性最好的那一种（排序 key 让优先级高的排在末尾）。
+    """
+    if preferred_quality is None:
+        preferred_quality = getattr(config, "BILI_QN", 80)
+
+    dash = _as_dict(play_info).get("dash")
+    if not isinstance(dash, dict):
+        return None
+    streams = [stream for stream in (dash.get("video") or []) if _stream_url(stream)]
+    if not streams:
+        return None
+
+    streams.sort(key=_stream_sort_key)
+    within_quality = [stream for stream in streams if _to_int(stream.get("id")) <= preferred_quality]
+    if within_quality:
+        return within_quality[-1]
+
+    # 所有档位都超过用户设置时取最低清晰度，但编码优先级仍要生效：
+    # 排序把每个 id 档内兼容性最好的编码放在该档最后，所以要在同 id 组里取最后一个
+    lowest_id = _to_int(streams[0].get("id"))
+    same_quality = [stream for stream in streams if _to_int(stream.get("id")) == lowest_id]
+    return same_quality[-1]
+
+
+def pick_audio_stream(play_info: Dict) -> Optional[Dict]:
+    """音频流取码率最高的一档（id 越大码率越高）"""
+    dash = _as_dict(play_info).get("dash")
+    if not isinstance(dash, dict):
+        return None
+    streams = [stream for stream in (dash.get("audio") or []) if _stream_url(stream)]
+    if not streams:
+        return None
+    return max(streams, key=lambda stream: (_to_int(stream.get("id")), _to_int(stream.get("bandwidth"))))
+
+
+def build_dash_item(play_info: Dict, content_id: str, preferred_quality: Optional[int] = None) -> Optional[MediaItem]:
+    """构建 DASH 下载任务（视频轨 + 音频轨，交由下载器用 ffmpeg 合流）"""
+    video_stream = pick_video_stream(play_info, preferred_quality)
+    audio_stream = pick_audio_stream(play_info)
+    if video_stream is None or audio_stream is None:
+        return None
+
+    return MediaItem(
+        url=_stream_url(video_stream),
+        backup_urls=_stream_backup_urls(video_stream),
+        audio_url=_stream_url(audio_stream),
+        audio_backup_urls=_stream_backup_urls(audio_stream),
+        media_type=MediaType.VIDEO,
+        content_id=content_id,
+        stem="video",
+    )
+
+
+def available_qualities(play_info: Dict) -> List[int]:
+    """响应中实际可用的视频清晰度档位（从高到低）"""
+    dash = _as_dict(play_info).get("dash")
+    if not isinstance(dash, dict):
+        return []
+    qualities = {
+        _to_int(stream.get("id"))
+        for stream in (dash.get("video") or [])
+        if _stream_url(stream)
+    }
+    return sorted(qualities, reverse=True)
+
+
+def next_lower_quality(play_info: Dict, current_quality: int) -> Optional[int]:
+    """比 ``current_quality`` 更低的下一个可用档位，没有则返回 None。
+
+    用于"playurl 给了高清地址但 CDN 拒绝下载"时逐档降级
+    （未登录或权限不足时 B 站会返回高清晰度 URL，但实际取流会被 403）。
+    """
+    lower = [quality for quality in available_qualities(play_info) if quality < current_quality]
+    return lower[0] if lower else None
+
+
+def count_durl_segments(play_info: Dict) -> int:
+    """durl 的分段数量。大于 1 表示整片被切分，只下最大一段会得到不完整的文件。"""
+    return len(
+        [
+            entry
+            for entry in _as_dict(play_info).get("durl") or []
+            if isinstance(entry, dict) and entry.get("url")
+        ]
+    )
+
+
+def build_durl_item(play_info: Dict, content_id: str) -> Optional[MediaItem]:
+    """构建 mp4 直链下载任务（取体积最大的一段）。
+
+    stem 刻意与 DASH 产物（``video``）区分：直链画质受接口限制，是降级产物。
+    两者同名会让先下到的低清文件被"已存在即跳过"永久信任，
+    之后即使装好 ffmpeg 也再拿不到高清。
+    """
+    durl_list: List[Dict] = _as_dict(play_info).get("durl") or []
+    candidates = [entry for entry in durl_list if isinstance(entry, dict) and entry.get("url")]
+    if not candidates:
+        return None
+
+    best = max(candidates, key=lambda entry: _to_int(entry.get("size")))
+    return MediaItem(
+        url=best["url"],
+        backup_urls=tuple(best.get("backup_url") or ()),
+        media_type=MediaType.VIDEO,
+        content_id=content_id,
+        stem="video-durl",
+    )

media_platform/douyin/client.py

@@ -344,20 +344,6 @@ class DouYinClient(AbstractApiClient, ProxyRefreshMixin):
             result.extend(aweme_list)
         return result
 
-    async def get_aweme_media(self, url: str) -> Union[bytes, None]:
-        async with make_async_client(proxy=self.proxy) as client:
-            try:
-                response = await client.request("GET", url, timeout=self.timeout, follow_redirects=True)
-                response.raise_for_status()
-                if not response.reason_phrase == "OK":
-                    utils.logger.error(f"[DouYinClient.get_aweme_media] request {url} err, res:{response.text}")
-                    return None
-                else:
-                    return response.content
-            except httpx.HTTPError as exc:  # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
-                utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}")  # Keep the original exception type name for developers to debug
-                return None
-
     async def resolve_short_url(self, short_url: str) -> str:
         """
         解析抖音短链接,获取重定向后的真实URL

media_platform/douyin/core.py

@@ -33,12 +33,14 @@ from playwright.async_api import (
 
 import config
 from base.base_crawler import AbstractCrawler
+from media_downloader import MediaDownloader
 from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
 from store import douyin as douyin_store
 from tools import utils
 from tools.cdp_browser import CDPBrowserManager
 from var import crawler_type_var, source_keyword_var
 
+from . import media as douyin_media
 from .client import DouYinClient
 from .exception import DataFetchError
 from .field import PublishTimeType
@@ -63,6 +65,7 @@ class DouYinCrawler(AbstractCrawler):
         ]
         self.cdp_manager = None
         self.ip_proxy_pool = None  # Proxy IP pool for automatic proxy refresh
+        self._media_downloader: Optional[MediaDownloader] = None
 
     async def start(self) -> None:
         playwright_proxy_format, httpx_proxy_format = None, None
@@ -170,7 +173,7 @@ class DouYinCrawler(AbstractCrawler):
                     aweme_list.append(aweme_info.get("aweme_id", ""))
                     page_aweme_list.append(aweme_info.get("aweme_id", ""))
                     await douyin_store.update_douyin_aweme(aweme_item=aweme_info)
-                    await self.get_aweme_media(aweme_item=aweme_info)
+                    await self.download_media(aweme_item=aweme_info)
                 
                 # Batch get note comments for the current page
                 await self.batch_get_note_comments(page_aweme_list)
@@ -212,7 +215,7 @@ class DouYinCrawler(AbstractCrawler):
         for aweme_detail in aweme_details:
             if aweme_detail is not None:
                 await douyin_store.update_douyin_aweme(aweme_item=aweme_detail)
-                await self.get_aweme_media(aweme_item=aweme_detail)
+                await self.download_media(aweme_item=aweme_detail)
         await self.batch_get_note_comments(aweme_id_list)
 
     async def get_aweme_detail(self, aweme_id: str, semaphore: asyncio.Semaphore) -> Any:
@@ -304,7 +307,7 @@ class DouYinCrawler(AbstractCrawler):
         for aweme_item in note_details:
             if aweme_item is not None:
                 await douyin_store.update_douyin_aweme(aweme_item=aweme_item)
-                await self.get_aweme_media(aweme_item=aweme_item)
+                await self.download_media(aweme_item=aweme_item)
 
     async def create_douyin_client(self, httpx_proxy: Optional[str]) -> DouYinClient:
         """Create douyin client"""
@@ -399,72 +402,44 @@ class DouYinCrawler(AbstractCrawler):
             await self.browser_context.close()
         utils.logger.info("[DouYinCrawler.close] Browser context closed ...")
 
-    async def get_aweme_media(self, aweme_item: Dict):
-        """
-        获取抖音媒体，自动判断媒体类型是短视频还是帖子图片并下载
+    async def download_media(self, aweme_item: Dict) -> None:
+        """下载抖音作品的媒体资源（图集图片，或视频 + 封面）
 
         Args:
             aweme_item (Dict): 抖音作品详情
         """
-        if not config.ENABLE_GET_MEIDAS:
-            utils.logger.info(f"[DouYinCrawler.get_aweme_media] Crawling image mode is not enabled")
+        if not config.ENABLE_GET_MEDIA:
             return
-        # List of note urls. If it is a short video type, an empty list will be returned.
-        note_download_url: List[str] = douyin_store._extract_note_image_list(aweme_item)
-        # The video URL will always exist, but when it is a short video type, the file is actually an audio file.
-        video_download_url: str = douyin_store._extract_video_download_url(aweme_item)
-        # TODO: Douyin does not adopt the audio and video separation strategy, so the audio can be separated from the original video and will not be extracted for the time being.
-        if note_download_url:
-            await self.get_aweme_images(aweme_item)
+        try:
+            items = douyin_media.build_media_items(aweme_item)
+            if items:
+                await self._get_media_downloader().download_all(items)
+        except Exception as exc:
+            # 媒体下载是旁路能力，解析异常/网络异常都不能中断爬取主流程
+            utils.logger.error(f"[DouYinCrawler.download_media] 媒体下载异常: {exc}")
+
+    def _get_media_downloader(self) -> MediaDownloader:
+        """惰性创建媒体下载器，并同步最新的代理与 UA（代理池是就地刷新的）"""
+        if self._media_downloader is None:
+            self._media_downloader = MediaDownloader(
+                platform="dy",
+                proxy=getattr(self.dy_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
         else:
-            await self.get_aweme_video(aweme_item)
+            self._media_downloader.update_credentials(
+                proxy=getattr(self.dy_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        return self._media_downloader
 
-    async def get_aweme_images(self, aweme_item: Dict):
+    def _media_headers(self) -> Dict:
+        """媒体请求头：平台 Referer（防盗链）+ UA。
+
+        client.headers 里含 Cookie，不能整体透传到 CDN 请求上，因此只取 UA。
         """
-        get aweme images. please use get_aweme_media
-
-        Args:
-            aweme_item (Dict): 抖音作品详情
-        """
-        if not config.ENABLE_GET_MEIDAS:
-            return
-        aweme_id = aweme_item.get("aweme_id")
-        # List of note urls. If it is a short video type, an empty list will be returned.
-        note_download_url: List[str] = douyin_store._extract_note_image_list(aweme_item)
-
-        if not note_download_url:
-            return
-        picNum = 0
-        for url in note_download_url:
-            if not url:
-                continue
-            content = await self.dy_client.get_aweme_media(url)
-            await asyncio.sleep(random.random())
-            if content is None:
-                continue
-            extension_file_name = f"{picNum:>03d}.jpeg"
-            picNum += 1
-            await douyin_store.update_dy_aweme_image(aweme_id, content, extension_file_name)
-
-    async def get_aweme_video(self, aweme_item: Dict):
-        """
-        get aweme videos. please use get_aweme_media
-
-        Args:
-            aweme_item (Dict): 抖音作品详情
-        """
-        if not config.ENABLE_GET_MEIDAS:
-            return
-        aweme_id = aweme_item.get("aweme_id")
-
-        # The video URL will always exist, but when it is a short video type, the file is actually an audio file.
-        video_download_url: str = douyin_store._extract_video_download_url(aweme_item)
-
-        if not video_download_url:
-            return
-        content = await self.dy_client.get_aweme_media(video_download_url)
-        await asyncio.sleep(random.random())
-        if content is None:
-            return
-        extension_file_name = f"video.mp4"
-        await douyin_store.update_dy_aweme_video(aweme_id, content, extension_file_name)
+        headers = {"Referer": "https://www.douyin.com/"}
+        client_headers = getattr(self.dy_client, "headers", {}) or {}
+        if client_headers.get("User-Agent"):
+            headers["User-Agent"] = client_headers["User-Agent"]
+        return headers

media_platform/douyin/media.py

@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 relakkes@gmail.com
+#
+# This file is part of MediaCrawler project.
+# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/douyin/media.py
+# GitHub: https://github.com/NanmiCoder
+# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
+#
+
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于任何非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+"""抖音媒体地址提取。
+
+输入是抖音作品详情（``/aweme/v1/web/aweme/detail/`` 的响应），
+输出是可下载的 ``MediaItem`` 列表。无 IO（少数平台会读取 config 决定策略）。
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List
+
+from media_downloader import MediaItem, MediaType
+
+# 视频清晰度字段，按优先级排列（h264 与 256 通常为无水印源）
+_VIDEO_ADDR_KEYS = ("play_addr_h264", "play_addr_256", "play_addr")
+
+# 封面字段，按优先级排列
+_COVER_KEYS = ("raw_cover", "origin_cover", "cover", "dynamic_cover")
+
+
+def _url_list_of(container) -> List[str]:
+    """接口响应里同名字段可能是字符串或 None，统一收敛为 url 列表"""
+    if not isinstance(container, dict):
+        return []
+    return [url for url in (container.get("url_list") or []) if url and isinstance(url, str)]
+
+
+def extract_image_urls(aweme_detail: Dict) -> List[str]:
+    """提取图集作品的图片地址（url_list 的最后一个通常是无水印原图）"""
+    urls: List[str] = []
+    for image in aweme_detail.get("images") or []:
+        candidates = _url_list_of(image)
+        if candidates:
+            urls.append(candidates[-1])
+    return urls
+
+
+def extract_cover_url(aweme_detail: Dict) -> str:
+    """提取视频封面地址"""
+    video_item = aweme_detail.get("video")
+    if not isinstance(video_item, dict):
+        return ""
+    for key in _COVER_KEYS:
+        candidates = _url_list_of(video_item.get(key))
+        if candidates:
+            return candidates[-1]
+    return ""
+
+
+def extract_video_urls(aweme_detail: Dict) -> List[str]:
+    """提取视频地址候选列表。
+
+    同一档清晰度会返回多个 CDN 地址，优先取 url_list 最后一个（通常无水印），
+    其余地址作为备用；主地址全部不可用时依次降级到更低清晰度档位。
+    """
+    video_item = aweme_detail.get("video")
+    if not isinstance(video_item, dict):
+        return []
+
+    for key in _VIDEO_ADDR_KEYS:
+        candidates = _url_list_of(video_item.get(key))
+        if candidates:
+            return list(reversed(candidates))
+
+    # 兜底：bit_rate 列表里码率最高的那一档
+    bit_rates = [entry for entry in (video_item.get("bit_rate") or []) if isinstance(entry, dict)]
+    for entry in sorted(bit_rates, key=lambda item: item.get("bit_rate", 0), reverse=True):
+        candidates = _url_list_of(entry.get("play_addr"))
+        if candidates:
+            return list(reversed(candidates))
+
+    return []
+
+
+def build_media_items(aweme_item: Dict) -> List[MediaItem]:
+    """把一个抖音作品转换成待下载的媒体任务列表。
+
+    - 图集作品：各张图片（第一张即封面，不再单独下载一份 cover）
+    - 视频作品：封面 + 视频
+    """
+    if not isinstance(aweme_item, dict):
+        return []
+
+    content_id = aweme_item.get("aweme_id") or ""
+    if not content_id:
+        return []
+
+    items: List[MediaItem] = []
+
+    image_urls = extract_image_urls(aweme_item)
+    if image_urls:
+        for index, image_url in enumerate(image_urls, start=1):
+            items.append(
+                MediaItem(
+                    url=image_url,
+                    media_type=MediaType.IMAGE,
+                    content_id=content_id,
+                    stem=f"{index:03d}",
+                )
+            )
+        return items
+
+    cover_url = extract_cover_url(aweme_item)
+    if cover_url:
+        items.append(
+            MediaItem(
+                url=cover_url,
+                media_type=MediaType.IMAGE,
+                content_id=content_id,
+                stem="cover",
+            )
+        )
+
+    video_urls = extract_video_urls(aweme_item)
+    if video_urls:
+        items.append(
+            MediaItem(
+                url=video_urls[0],
+                backup_urls=tuple(video_urls[1:]),
+                media_type=MediaType.VIDEO,
+                content_id=content_id,
+                stem="video",
+            )
+        )
+
+    return items

media_platform/kuaishou/core.py

@@ -35,6 +35,7 @@ from playwright.async_api import (
 
 import config
 from base.base_crawler import AbstractCrawler
+from media_downloader import MediaDownloader
 from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo
 from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
 from store import kuaishou as kuaishou_store
@@ -42,6 +43,7 @@ from tools import utils
 from tools.cdp_browser import CDPBrowserManager
 from var import comment_tasks_var, crawler_type_var, source_keyword_var
 
+from . import media as kuaishou_media
 from .client import KuaiShouClient
 from .exception import DataFetchError
 from .help import (
@@ -64,6 +66,7 @@ class KuaishouCrawler(AbstractCrawler):
         self.user_agent = utils.get_user_agent()
         self.cdp_manager = None
         self.ip_proxy_pool = None  # Proxy IP pool, used for automatic proxy refresh
+        self._media_downloader: Optional[MediaDownloader] = None
 
     async def start(self):
         playwright_proxy_format, httpx_proxy_format = None, None
@@ -177,6 +180,7 @@ class KuaishouCrawler(AbstractCrawler):
                 for video_detail in videos_res.get("feeds", []):
                     video_id_list.append(video_detail.get("photo", {}).get("id"))
                     await kuaishou_store.update_kuaishou_video(video_item=video_detail)
+                    await self.download_media(video_item=video_detail)
 
                 utils.logger.info(
                     f"[KuaishouCrawler.search] keyword: {keyword}, page: {page}, got {len(video_id_list)} videos"
@@ -215,6 +219,7 @@ class KuaishouCrawler(AbstractCrawler):
         for video_detail in video_details:
             if video_detail is not None:
                 await kuaishou_store.update_kuaishou_video(video_detail)
+                await self.download_media(video_item=video_detail)
         await self.batch_get_video_comments(video_ids)
 
     async def get_video_info_task(
@@ -464,6 +469,49 @@ class KuaishouCrawler(AbstractCrawler):
         for video_detail in video_details:
             if video_detail is not None:
                 await kuaishou_store.update_kuaishou_video(video_detail)
+                await self.download_media(video_item=video_detail)
+
+    async def download_media(self, video_item: Dict):
+        """下载快手作品的媒体资源（封面 + 视频）
+
+        Args:
+            video_item (Dict): 快手作品详情（结构为 {"photo": {...}, "author": {...}}）
+        """
+        if not config.ENABLE_GET_MEDIA:
+            return
+        try:
+            items = kuaishou_media.build_media_items(video_item)
+            if items:
+                await self._get_media_downloader().download_all(items)
+        except Exception as exc:
+            # 媒体下载是旁路能力，解析异常/网络异常都不能中断爬取主流程
+            utils.logger.error(f"[KuaishouCrawler.download_media] 媒体下载异常: {exc}")
+
+    def _get_media_downloader(self) -> MediaDownloader:
+        """惰性创建媒体下载器，并同步最新的代理与 UA（代理池是就地刷新的）"""
+        if self._media_downloader is None:
+            self._media_downloader = MediaDownloader(
+                platform="ks",
+                proxy=getattr(self.ks_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        else:
+            self._media_downloader.update_credentials(
+                proxy=getattr(self.ks_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        return self._media_downloader
+
+    def _media_headers(self) -> Dict:
+        """媒体请求头：平台 Referer（防盗链）+ UA。
+
+        client.headers 里含 Cookie，不能整体透传到 CDN 请求上，因此只取 UA。
+        """
+        headers = {"Referer": "https://www.kuaishou.com/"}
+        client_headers = getattr(self.ks_client, "headers", {}) or {}
+        if client_headers.get("User-Agent"):
+            headers["User-Agent"] = client_headers["User-Agent"]
+        return headers
 
     async def close(self):
         """Close browser context"""

media_platform/kuaishou/media.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 relakkes@gmail.com
+#
+# This file is part of MediaCrawler project.
+# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/kuaishou/media.py
+# GitHub: https://github.com/NanmiCoder
+# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
+#
+
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于任何非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+"""快手媒体地址提取。
+
+输入是快手作品详情（``{"photo": {...}, "author": {...}}``），
+输出是可下载的 ``MediaItem`` 列表。无 IO（少数平台会读取 config 决定策略）。
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List
+
+from media_downloader import MediaItem, MediaType
+
+
+def _first_url(urls) -> str:
+    """从 url 列表里取第一个非空字符串"""
+    if isinstance(urls, str):
+        return urls
+    for url in urls or []:
+        if isinstance(url, str) and url:
+            return url
+    return ""
+
+
+def _photo_of(video_item: Dict) -> Dict:
+    """接口响应里 photo 字段可能是字符串或 None，统一收敛为 dict"""
+    photo = video_item.get("photo")
+    return photo if isinstance(photo, dict) else {}
+
+
+def _extract_representation_url(codec_resource) -> str:
+    """从 videoResource.{hevc,h264} 的 adaptationSet 里取第一个播放地址"""
+    if not isinstance(codec_resource, dict):
+        return ""
+    for adaptation in codec_resource.get("adaptationSet") or []:
+        if not isinstance(adaptation, dict):
+            continue
+        for representation in adaptation.get("representation") or []:
+            if isinstance(representation, dict) and representation.get("url"):
+                return representation["url"]
+    return ""
+
+
+def extract_video_urls(video_item: Dict) -> List[str]:
+    """提取视频地址候选列表（H265 优先，画质与体积更优）"""
+    photo = _photo_of(video_item)
+    if not photo:
+        return []
+
+    video_resource = photo.get("videoResource") if isinstance(photo.get("videoResource"), dict) else {}
+    candidates = [
+        photo.get("photoH265Url"),
+        photo.get("photoUrl"),
+        _extract_representation_url(video_resource.get("hevc")),
+        _extract_representation_url(video_resource.get("h264")),
+        # 详情接口另有 manifest.adaptationSet[].representation[].url 一份等价描述，
+        # search 接口的 photo 对象字段更少时靠它兜底
+        _extract_representation_url(photo.get("manifest")),
+    ]
+
+    urls: List[str] = []
+    for candidate in candidates:
+        if candidate and isinstance(candidate, str) and candidate not in urls:
+            urls.append(candidate)
+    return urls
+
+
+def extract_cover_url(video_item: Dict) -> str:
+    """提取视频封面地址"""
+    photo = _photo_of(video_item)
+    if not photo:
+        return ""
+
+    for candidate in (
+        photo.get("coverUrl"),
+        _first_url([entry.get("url") for entry in photo.get("coverUrls") or [] if isinstance(entry, dict)]),
+        photo.get("animatedCoverUrl"),
+    ):
+        if isinstance(candidate, str) and candidate:
+            return candidate
+    return ""
+
+
+def build_media_items(video_item: Dict) -> List[MediaItem]:
+    """把一个快手作品转换成待下载的媒体任务列表：封面 + 视频"""
+    if not isinstance(video_item, dict):
+        return []
+
+    content_id = _photo_of(video_item).get("id")
+    if not content_id:
+        return []
+
+    content_id = str(content_id)
+    items: List[MediaItem] = []
+
+    cover_url = extract_cover_url(video_item)
+    if cover_url:
+        items.append(
+            MediaItem(
+                url=cover_url,
+                media_type=MediaType.IMAGE,
+                content_id=content_id,
+                stem="cover",
+            )
+        )
+
+    video_urls = extract_video_urls(video_item)
+    if video_urls:
+        items.append(
+            MediaItem(
+                url=video_urls[0],
+                backup_urls=tuple(video_urls[1:]),
+                media_type=MediaType.VIDEO,
+                content_id=content_id,
+                stem="video",
+            )
+        )
+
+    return items

media_platform/weibo/client.py

@@ -65,7 +65,6 @@ class WeiboClient(ProxyRefreshMixin):
         self.cookie_urls = [self._host]
         self.playwright_page = playwright_page
         self.cookie_dict = cookie_dict
-        self._image_agent_host = "https://i1.wp.com/"
         # Initialize proxy pool (from ProxyRefreshMixin)
         self.init_proxy_pool(proxy_ip_pool)
 
@@ -277,33 +276,6 @@ class WeiboClient(ProxyRefreshMixin):
                 utils.logger.info(f"[WeiboClient.get_note_info_by_id] $render_data value not found")
                 return dict()
 
-    async def get_note_image(self, image_url: str) -> bytes:
-        image_url = image_url[8:]  # Remove https://
-        sub_url = image_url.split("/")
-        image_url = ""
-        for i in range(len(sub_url)):
-            if i == 1:
-                image_url += "large/"  # Get high-resolution images
-            elif i == len(sub_url) - 1:
-                image_url += sub_url[i]
-            else:
-                image_url += sub_url[i] + "/"
-        # Weibo image hosting has anti-hotlinking, so proxy access is needed
-        # Since Weibo images are accessed through i1.wp.com, we need to concatenate the URL
-        final_uri = (f"{self._image_agent_host}"
-                     f"{image_url}")
-        async with make_async_client(proxy=self.proxy) as client:
-            try:
-                response = await client.request("GET", final_uri, timeout=self.timeout)
-                response.raise_for_status()
-                if not response.reason_phrase == "OK":
-                    utils.logger.error(f"[WeiboClient.get_note_image] request {final_uri} err, res:{response.text}")
-                    return None
-                else:
-                    return response.content
-            except httpx.HTTPError as exc:  # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
-                utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}")    # Keep original exception type name for developer debugging
-                return None
 
     async def get_creator_container_info(self, creator_id: str) -> Dict:
         """

media_platform/weibo/core.py

@@ -38,12 +38,14 @@ from playwright.async_api import (
 
 import config
 from base.base_crawler import AbstractCrawler
+from media_downloader import MediaDownloader
 from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
 from store import weibo as weibo_store
 from tools import utils
 from tools.cdp_browser import CDPBrowserManager
 from var import crawler_type_var, source_keyword_var
 
+from . import media as weibo_media
 from .client import WeiboClient
 from .exception import DataFetchError
 from .field import SearchType
@@ -65,6 +67,7 @@ class WeiboCrawler(AbstractCrawler):
         self.mobile_user_agent = utils.get_mobile_user_agent()
         self.cdp_manager = None
         self.ip_proxy_pool = None  # Proxy IP pool for automatic proxy refresh
+        self._media_downloader: Optional[MediaDownloader] = None
 
     async def start(self):
         playwright_proxy_format, httpx_proxy_format = None, None
@@ -179,7 +182,7 @@ class WeiboCrawler(AbstractCrawler):
                         if mblog:
                             note_id_list.append(mblog.get("id"))
                             await weibo_store.update_weibo_note(note_item)
-                            await self.get_note_images(mblog)
+                            await self.download_media(mblog)
 
                 page += 1
 
@@ -200,6 +203,9 @@ class WeiboCrawler(AbstractCrawler):
         for note_item in video_details:
             if note_item:
                 await weibo_store.update_weibo_note(note_item)
+                mblog = note_item.get("mblog")
+                if mblog:
+                    await self.download_media(mblog)
         await self.batch_get_notes_comments(config.WEIBO_SPECIFIED_ID_LIST)
 
     async def get_note_info_task(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[Dict]:
@@ -269,36 +275,46 @@ class WeiboCrawler(AbstractCrawler):
             except Exception as e:
                 utils.logger.error(f"[WeiboCrawler.get_note_comments] may be been blocked, err:{e}")
 
-    async def get_note_images(self, mblog: Dict):
-        """
-        get note images
-        :param mblog:
-        :return:
-        """
-        if not config.ENABLE_GET_MEIDAS:
-            utils.logger.info(f"[WeiboCrawler.get_note_images] Crawling image mode is not enabled")
-            return
+    async def download_media(self, mblog: Dict) -> None:
+        """下载微博的媒体资源（配图，或视频 + 封面）
 
-        pics: List = mblog.get("pics")
-        if not pics:
+        :param mblog: 微博正文数据
+        """
+        if not config.ENABLE_GET_MEDIA:
             return
-        for pic in pics:
-            if isinstance(pic, str):
-                url = pic
-                pid = url.split("/")[-1].split(".")[0]
-            elif isinstance(pic, dict):
-                url = pic.get("url")
-                pid = pic.get("pid", "")
-            else:
-                continue
-            if not url:
-                continue
-            content = await self.wb_client.get_note_image(url)
-            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
-            utils.logger.info(f"[WeiboCrawler.get_note_images] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching image")
-            if content != None:
-                extension_file_name = url.split(".")[-1]
-                await weibo_store.update_weibo_note_image(pid, content, extension_file_name)
+        try:
+            items = weibo_media.build_media_items(mblog)
+            if items:
+                await self._get_media_downloader().download_all(items)
+        except Exception as exc:
+            # 媒体下载是旁路能力，解析异常/网络异常都不能中断爬取主流程
+            utils.logger.error(f"[WeiboCrawler.download_media] 媒体下载异常: {exc}")
+
+    def _get_media_downloader(self) -> MediaDownloader:
+        """惰性创建媒体下载器，并同步最新的代理与 UA（代理池是就地刷新的）"""
+        if self._media_downloader is None:
+            self._media_downloader = MediaDownloader(
+                platform="wb",
+                proxy=getattr(self.wb_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        else:
+            self._media_downloader.update_credentials(
+                proxy=getattr(self.wb_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        return self._media_downloader
+
+    def _media_headers(self) -> Dict:
+        """媒体请求头：平台 Referer（防盗链）+ UA。
+
+        client.headers 里含 Cookie，不能整体透传到 CDN 请求上，因此只取 UA。
+        """
+        headers = {"Referer": "https://weibo.com/"}
+        client_headers = getattr(self.wb_client, "headers", {}) or {}
+        if client_headers.get("User-Agent"):
+            headers["User-Agent"] = client_headers["User-Agent"]
+        return headers
 
     async def get_creators_and_notes(self) -> None:
         """
@@ -321,6 +337,10 @@ class WeiboCrawler(AbstractCrawler):
                     # If full text fetching is enabled, batch get full text first
                     updated_note_list = await self.batch_get_notes_full_text(note_list)
                     await weibo_store.batch_update_weibo_notes(updated_note_list)
+                    for note_item in updated_note_list:
+                        mblog = (note_item or {}).get("mblog")
+                        if mblog:
+                            await self.download_media(mblog)
 
                 # Get all note information of the creator
                 all_notes_list = await self.wb_client.get_all_notes_by_creator_id(

media_platform/weibo/media.py

@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 relakkes@gmail.com
+#
+# This file is part of MediaCrawler project.
+# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/weibo/media.py
+# GitHub: https://github.com/NanmiCoder
+# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
+#
+
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于任何非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+"""微博媒体地址提取。
+
+输入是微博 ``mblog``，输出是可下载的 ``MediaItem`` 列表。无 IO（少数平台会读取 config 决定策略）。
+
+图片需要特殊处理：微博图床有防盗链，且缩略图路径里带尺寸段，
+统一改写为 ``large`` 清晰度后经 i1.wp.com 代理访问。
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List
+from urllib.parse import urlsplit
+
+from media_downloader import MediaItem, MediaType
+
+# 微博图床存在防盗链，统一走该图片代理访问
+WEIBO_IMAGE_AGENT_HOST = "https://i1.wp.com/"
+
+# 视频地址字段，按清晰度从高到低尝试
+_VIDEO_KEYS = (
+    "mp4_1080p_mp4",
+    "mp4_hd_mp4",
+    "mp4_720p_mp4",
+    "hevc_mp4_hd",
+    "mp4_ld_mp4",
+    "stream_url",
+)
+
+
+def rewrite_image_url(raw_url: str) -> str:
+    """把微博图床地址改写为 large 清晰度并加图片代理前缀。
+
+    ``https://wx1.sinaimg.cn/orj360/abc.jpg``
+    -> ``https://i1.wp.com/wx1.sinaimg.cn/large/abc.jpg``
+    """
+    parts = urlsplit(raw_url)
+    if not parts.scheme or not parts.netloc:
+        return raw_url
+
+    segments = [segment for segment in parts.path.split("/") if segment]
+    if segments:
+        segments[0] = "large"  # 原始路径首段是尺寸标识（orj360 / thumbnail 等）
+
+    path = "/" + "/".join(segments) if segments else parts.path
+    return f"{WEIBO_IMAGE_AGENT_HOST}{parts.netloc}{path}"
+
+
+def extract_image_urls(mblog: Dict) -> List[str]:
+    """提取微博配图地址（已改写为可访问地址）"""
+    urls: List[str] = []
+    for pic in mblog.get("pics") or []:
+        if isinstance(pic, str):
+            raw_url = pic
+        elif isinstance(pic, dict):
+            raw_url = pic.get("url") or ""
+        else:
+            continue
+        if isinstance(raw_url, str) and raw_url:
+            urls.append(rewrite_image_url(raw_url))
+    return urls
+
+
+def extract_video_urls(mblog: Dict) -> List[str]:
+    """提取微博视频地址候选列表（按清晰度从高到低）"""
+    page_info = mblog.get("page_info") or {}
+    if not isinstance(page_info, dict):
+        return []
+    if page_info.get("type") != "video" and not page_info.get("media_info"):
+        return []
+
+    urls: List[str] = []
+    for container_key in ("media_info", "urls"):
+        container = page_info.get(container_key) or {}
+        if not isinstance(container, dict):
+            continue
+        for key in _VIDEO_KEYS:
+            url = container.get(key)
+            if isinstance(url, str) and url and url not in urls:
+                urls.append(url)
+    return urls
+
+
+def extract_cover_url(mblog: Dict) -> str:
+    """提取视频封面地址"""
+    page_info = mblog.get("page_info") or {}
+    if not isinstance(page_info, dict):
+        return ""
+    page_pic = page_info.get("page_pic") or {}
+    if isinstance(page_pic, dict):
+        url = page_pic.get("url")
+        return url if isinstance(url, str) else ""
+    return ""
+
+
+def media_source_mblog(mblog: Dict) -> Dict:
+    """返回真正承载媒体内容的那一层微博。
+
+    纯转发的 ``pics`` / ``page_info`` 位于 ``retweeted_status`` 里，顶层两者皆无；
+    不处理的话转发帖会静默地一条媒体都下载不到。
+    若转发时自己带了图（顶层有媒体），则以顶层为准。
+    """
+    if mblog.get("pics") or mblog.get("page_info"):
+        return mblog
+
+    retweeted = mblog.get("retweeted_status")
+    if isinstance(retweeted, dict):
+        return retweeted
+    return mblog
+
+
+def build_media_items(mblog: Dict) -> List[MediaItem]:
+    """把一条微博转换成待下载的媒体任务列表。
+
+    - 视频微博：封面 + 视频（配图通常就是封面，不重复下载）
+    - 普通微博：各张配图
+    - 转发微博：取原博的媒体，目录仍按转发帖自身的 id 组织
+    """
+    if not isinstance(mblog, dict):
+        return []
+
+    content_id = mblog.get("id") or mblog.get("mid") or ""
+    if not content_id:
+        return []
+    content_id = str(content_id)
+
+    source = media_source_mblog(mblog)
+    items: List[MediaItem] = []
+
+    video_urls = extract_video_urls(source)
+    if video_urls:
+        cover_url = extract_cover_url(source)
+        if cover_url:
+            items.append(
+                MediaItem(
+                    url=cover_url,
+                    media_type=MediaType.IMAGE,
+                    content_id=content_id,
+                    stem="cover",
+                )
+            )
+        items.append(
+            MediaItem(
+                url=video_urls[0],
+                backup_urls=tuple(video_urls[1:]),
+                media_type=MediaType.VIDEO,
+                content_id=content_id,
+                stem="video",
+            )
+        )
+        return items
+
+    for index, image_url in enumerate(extract_image_urls(source), start=1):
+        items.append(
+            MediaItem(
+                url=image_url,
+                media_type=MediaType.IMAGE,
+                content_id=content_id,
+                stem=f"{index:03d}",
+            )
+        )
+
+    return items

media_platform/xhs/client.py

@@ -246,29 +246,6 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
             **kwargs,
         )
 
-    async def get_note_media(self, url: str) -> Union[bytes, None]:
-        # Check if proxy is expired before request
-        await self._refresh_proxy_if_expired()
-
-        async with make_async_client(proxy=self.proxy) as client:
-            try:
-                response = await client.request("GET", url, timeout=self.timeout)
-                response.raise_for_status()
-                if not response.reason_phrase == "OK":
-                    utils.logger.error(
-                        f"[XiaoHongShuClient.get_note_media] request {url} err, res:{response.text}"
-                    )
-                    return None
-                else:
-                    return response.content
-            except (
-                httpx.HTTPError
-            ) as exc:  # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
-                utils.logger.error(
-                    f"[XiaoHongShuClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}"
-                )  # Keep original exception type name for developer debugging
-                return None
-
     async def query_self(self) -> Optional[Dict]:
         """
         Query self user info to check login state

media_platform/xhs/core.py

@@ -34,6 +34,7 @@ from tenacity import RetryError
 
 import config
 from base.base_crawler import AbstractCrawler
+from media_downloader import MediaDownloader
 from model.m_xiaohongshu import NoteUrlInfo, CreatorUrlInfo
 from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
 from store import xhs as xhs_store
@@ -41,6 +42,7 @@ from tools import utils
 from tools.cdp_browser import CDPBrowserManager
 from var import crawler_type_var, source_keyword_var
 
+from . import media as xhs_media
 from .client import XiaoHongShuClient
 from .exception import (
     DataFetchError,
@@ -66,6 +68,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
         self.user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
         self.cdp_manager = None
         self.ip_proxy_pool = None  # Proxy IP pool for automatic proxy refresh
+        self._media_downloader: Optional[MediaDownloader] = None
 
     async def start(self) -> None:
         playwright_proxy_format, httpx_proxy_format = None, None
@@ -176,7 +179,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
                     for note_detail in note_details:
                         if note_detail:
                             await xhs_store.update_xhs_note(note_detail)
-                            await self.get_notice_media(note_detail)
+                            await self.download_media(note_detail)
                             note_ids.append(note_detail.get("note_id"))
                             xsec_tokens.append(note_detail.get("xsec_token"))
                     page += 1
@@ -253,7 +256,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
         for note_detail in note_details:
             if note_detail:
                 await xhs_store.update_xhs_note(note_detail)
-                await self.get_notice_media(note_detail)
+                await self.download_media(note_detail)
 
     async def get_specified_notes(self):
         """Get the information and comments of the specified post
@@ -280,7 +283,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
                 need_get_comment_note_ids.append(note_detail.get("note_id", ""))
                 xsec_tokens.append(note_detail.get("xsec_token", ""))
                 await xhs_store.update_xhs_note(note_detail)
-                await self.get_notice_media(note_detail)
+                await self.download_media(note_detail)
         await self.batch_get_note_comments(need_get_comment_note_ids, xsec_tokens)
 
     async def get_note_detail_async_task(
@@ -480,63 +483,40 @@ class XiaoHongShuCrawler(AbstractCrawler):
             await self.browser_context.close()
         utils.logger.info("[XiaoHongShuCrawler.close] Browser context closed ...")
 
-    async def get_notice_media(self, note_detail: Dict):
-        if not config.ENABLE_GET_MEIDAS:
-            utils.logger.info(f"[XiaoHongShuCrawler.get_notice_media] Crawling image mode is not enabled")
-            return
-        await self.get_note_images(note_detail)
-        await self.get_notice_video(note_detail)
-
-    async def get_note_images(self, note_item: Dict):
-        """Get note images. Please use get_notice_media
+    async def download_media(self, note_detail: Dict) -> None:
+        """下载笔记的媒体资源（封面、视频或图文图片）
 
         Args:
-            note_item: Note item dictionary
+            note_detail: 笔记详情（原始 note_card 结构）
         """
-        if not config.ENABLE_GET_MEIDAS:
+        if not config.ENABLE_GET_MEDIA:
             return
-        note_id = note_item.get("note_id")
-        image_list: List[Dict] = note_item.get("image_list", [])
+        try:
+            items = xhs_media.build_media_items(note_detail)
+            if items:
+                await self._get_media_downloader().download_all(items)
+        except Exception as exc:
+            # 媒体下载是旁路能力，解析异常/网络异常都不能中断爬取主流程
+            utils.logger.error(f"[XiaoHongShuCrawler.download_media] 媒体下载异常: {exc}")
 
-        for img in image_list:
-            if img.get("url_default") != "":
-                img.update({"url": img.get("url_default")})
+    def _get_media_downloader(self) -> MediaDownloader:
+        """惰性创建媒体下载器，并同步最新的代理设置（代理池是就地刷新的）"""
+        if self._media_downloader is None:
+            self._media_downloader = MediaDownloader(
+                platform="xhs",
+                proxy=getattr(self.xhs_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        else:
+            self._media_downloader.update_credentials(
+                proxy=getattr(self.xhs_client, "proxy", None),
+                extra_headers=self._media_headers(),
+            )
+        return self._media_downloader
 
-        if not image_list:
-            return
-        picNum = 0
-        for pic in image_list:
-            url = pic.get("url")
-            if not url:
-                continue
-            content = await self.xhs_client.get_note_media(url)
-            await asyncio.sleep(random.random())
-            if content is None:
-                continue
-            extension_file_name = f"{picNum}.jpg"
-            picNum += 1
-            await xhs_store.update_xhs_note_image(note_id, content, extension_file_name)
-
-    async def get_notice_video(self, note_item: Dict):
-        """Get note videos. Please use get_notice_media
-
-        Args:
-            note_item: Note item dictionary
-        """
-        if not config.ENABLE_GET_MEIDAS:
-            return
-        note_id = note_item.get("note_id")
-
-        videos = xhs_store.get_video_url_arr(note_item)
-
-        if not videos:
-            return
-        videoNum = 0
-        for url in videos:
-            content = await self.xhs_client.get_note_media(url)
-            await asyncio.sleep(random.random())
-            if content is None:
-                continue
-            extension_file_name = f"{videoNum}.mp4"
-            videoNum += 1
-            await xhs_store.update_xhs_note_video(note_id, content, extension_file_name)
+    def _media_headers(self) -> Dict:
+        """媒体请求头：平台 Referer（防盗链）+ UA"""
+        return {
+            "Referer": "https://www.xiaohongshu.com/",
+            "User-Agent": self.user_agent,
+        }

media_platform/xhs/media.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 relakkes@gmail.com
+#
+# This file is part of MediaCrawler project.
+# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/xhs/media.py
+# GitHub: https://github.com/NanmiCoder
+# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
+#
+
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于任何非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+"""小红书媒体地址提取。
+
+输入是 ``/api/sns/web/v1/feed`` 返回的原始 ``note_card``（snake_case 字段），
+输出是下载器可直接消费的 ``MediaItem`` 列表。无 IO（会读取 config 决定国际版策略）。
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List
+
+import config
+from media_downloader import MediaItem, MediaType
+
+# 小红书视频 CDN：origin_video_key 拼在此域名后可拿到无水印源片
+XHS_VIDEO_CDN_HOST = "https://sns-video-bd.xhscdn.com"
+
+
+def _as_dict(value) -> Dict:
+    """接口响应里同名字段可能是字符串或 None，统一收敛为 dict"""
+    return value if isinstance(value, dict) else {}
+
+
+def _extract_origin_video_key(video_dict: Dict) -> str:
+    """从 video.consumer 中取无水印源片 key（兼容 snake_case 与 camelCase）"""
+    consumer = _as_dict(video_dict.get("consumer"))
+    return consumer.get("origin_video_key") or consumer.get("originVideoKey") or ""
+
+
+def extract_video_urls(note_item: Dict) -> List[str]:
+    """提取视频地址候选列表，按可用性排序。
+
+    优先无水印源片（origin_video_key），其余为带水印的 h264 master_url 备用。
+    国际版（rednote）的 CDN 域名与国内不同，不能拼接 xhscdn 域名。
+    """
+    if note_item.get("type") != "video":
+        return []
+
+    video_dict = _as_dict(note_item.get("video"))
+    if not video_dict:
+        return []
+
+    urls: List[str] = []
+    origin_video_key = _extract_origin_video_key(video_dict)
+    if origin_video_key and not getattr(config, "XHS_INTERNATIONAL", False):
+        urls.append(f"{XHS_VIDEO_CDN_HOST}/{origin_video_key}")
+
+    stream = _as_dict(_as_dict(video_dict.get("media")).get("stream"))
+    for item in stream.get("h264") or []:
+        master_url = item.get("master_url") if isinstance(item, dict) else None
+        if master_url and master_url not in urls:
+            urls.append(master_url)
+
+    return urls
+
+
+def extract_image_urls(note_item: Dict) -> List[str]:
+    """提取图文笔记的图片地址（按原始顺序）"""
+    urls: List[str] = []
+    for image_item in note_item.get("image_list") or []:
+        if not isinstance(image_item, dict):
+            continue
+        url = image_item.get("url_default") or image_item.get("url") or ""
+        if isinstance(url, str) and url:
+            urls.append(url)
+    return urls
+
+
+def extract_cover_url(note_item: Dict) -> str:
+    """提取封面地址。
+
+    小红书不同接口返回的封面字段位置不一致，这里按可靠性依次兜底；
+    视频笔记的 image_list 通常就是封面图，作为最后兜底。
+    """
+    video_dict = _as_dict(note_item.get("video"))
+    cover_dict = _as_dict(note_item.get("cover"))
+    video_cover = _as_dict(video_dict.get("cover"))
+
+    candidates = [
+        video_cover.get("url_default"),
+        video_cover.get("url"),
+        cover_dict.get("url_default"),
+        cover_dict.get("url"),
+    ]
+    for candidate in candidates:
+        if candidate:
+            return candidate
+
+    images = extract_image_urls(note_item)
+    return images[0] if images else ""
+
+
+def build_media_items(note_item: Dict) -> List[MediaItem]:
+    """把一个笔记转换成待下载的媒体任务列表。
+
+    - 视频笔记：封面 + 视频。其 image_list 通常只有封面一张，不再按图集重复下载
+    - 图文笔记：各张图片（第一张即封面，因此不再单独下载一份 cover）
+    """
+    if not isinstance(note_item, dict):
+        return []
+
+    content_id = note_item.get("note_id") or note_item.get("id") or ""
+    if not content_id:
+        return []
+
+    items: List[MediaItem] = []
+
+    if note_item.get("type") == "video":
+        cover_url = extract_cover_url(note_item)
+        if cover_url:
+            items.append(
+                MediaItem(
+                    url=cover_url,
+                    media_type=MediaType.IMAGE,
+                    content_id=content_id,
+                    stem="cover",
+                )
+            )
+
+        video_urls = extract_video_urls(note_item)
+        if video_urls:
+            items.append(
+                MediaItem(
+                    url=video_urls[0],
+                    backup_urls=tuple(video_urls[1:]),
+                    media_type=MediaType.VIDEO,
+                    content_id=content_id,
+                    stem="video",
+                )
+            )
+    else:
+        for index, image_url in enumerate(extract_image_urls(note_item), start=1):
+            items.append(
+                MediaItem(
+                    url=image_url,
+                    media_type=MediaType.IMAGE,
+                    content_id=content_id,
+                    stem=f"{index:03d}",
+                )
+            )
+
+    return items