fork/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-10-02 12:58:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(media): 重构媒体下载，支持 xhs/dy/ks/bili/wb 五平台

Browse Source 
旧实现只覆盖 4 个平台，且把整个文件读进内存、无重试与完整性校验，
代码按平台复制粘贴了 4 份。本次用统一下载器替换：

- 新增 media_downloader/：流式写入、Range 续传、指数退避重试、大小校验、
  路径穿越防护；B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py：从平台原始响应提取媒体地址，
  与下载器解耦；快手首次接入下载能力
- 开关：config.ENABLE_GET_MEDIA 与 --get_media，并打通 API/WebUI；
  同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合：{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质，否则降级 mp4 直链（产物 video-durl.mp4，
  避免低清文件阻塞后续的高清路径）
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法

媒体下载失败只记录日志，不中断爬取主流程。
This commit is contained in:
程序员阿江(Relakkes)
2026-09-17 22:54:22 +08:00
parent 60e66f2a92
commit 0ca7b29cf0
47 changed files with 5187 additions and 907 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media_platform/bilibili/core.py
+130 -34
View File
@@ -41,12 +41,14 @@ from playwright._impl._errors import TargetClosedError
import config
from base.base_crawler import AbstractCrawler
from media_downloader import MediaDownloader, is_ffmpeg_available
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from store import bilibili as bilibili_store
from tools import utils
from tools.cdp_browser import CDPBrowserManager
from var import crawler_type_var, source_keyword_var
from . import media as bili_media
from .client import BilibiliClient
from .exception import DataFetchError
from .field import SearchOrderType
@@ -66,6 +68,7 @@ class BilibiliCrawler(AbstractCrawler):
self.user_agent = utils.get_user_agent()
self.cdp_manager = None
self.ip_proxy_pool = None # Proxy IP pool for automatic proxy refresh
self._media_downloader: Optional[MediaDownloader] = None
async def start(self):
playwright_proxy_format, httpx_proxy_format = None, None
@@ -228,7 +231,7 @@ class BilibiliCrawler(AbstractCrawler):
video_id_list.append(video_item.get("View").get("aid"))
await bilibili_store.update_bilibili_video(video_item)
await bilibili_store.update_up_info(video_item)
await self.get_bilibili_video(video_item, semaphore)
await self.download_media(video_item, semaphore)
page += 1
# Sleep after page navigation
@@ -308,7 +311,7 @@ class BilibiliCrawler(AbstractCrawler):
video_id_list.append(video_item.get("View").get("aid"))
await bilibili_store.update_bilibili_video(video_item)
await bilibili_store.update_up_info(video_item)
await self.get_bilibili_video(video_item, semaphore)
await self.download_media(video_item, semaphore)
page += 1
@@ -413,7 +416,7 @@ class BilibiliCrawler(AbstractCrawler):
video_aids_list.append(video_aid)
await bilibili_store.update_bilibili_video(video_detail)
await bilibili_store.update_up_info(video_detail)
await self.get_bilibili_video(video_detail, semaphore)
await self.download_media(video_detail, semaphore)
await self.batch_get_video_comments(video_aids_list)
async def get_video_info_task(self, aid: int, bvid: str, semaphore: asyncio.Semaphore) -> Optional[Dict]:
@@ -440,17 +443,24 @@ class BilibiliCrawler(AbstractCrawler):
utils.logger.error(f"[BilibiliCrawler.get_video_info_task] have not fund note detail video_id:{bvid}, err: {ex}")
return None
async def get_video_play_url_task(self, aid: int, cid: int, semaphore: asyncio.Semaphore) -> Union[Dict, None]:
async def get_video_play_url_task(
self,
aid: int,
cid: int,
semaphore: asyncio.Semaphore,
fnval: int = bili_media.DASH_FNVAL,
) -> Union[Dict, None]:
"""
Get video play url
:param aid:
:param cid:
:param semaphore:
:param fnval: 媒体格式位掩码，默认请求 DASH 分轨流
:return:
"""
async with semaphore:
try:
result = await self.bili_client.get_video_play_url(aid=aid, cid=cid)
result = await self.bili_client.get_video_play_url(aid=aid, cid=cid, fnval=fnval)
return result
except DataFetchError as ex:
utils.logger.error(f"[BilibiliCrawler.get_video_play_url_task] Get video play url error: {ex}")
@@ -570,42 +580,128 @@ class BilibiliCrawler(AbstractCrawler):
except Exception as e:
utils.logger.error(f"[BilibiliCrawler.close] An error occurred during close: {e}")
async def get_bilibili_video(self, video_item: Dict, semaphore: asyncio.Semaphore):
async def download_media(self, video_item: Dict, semaphore: asyncio.Semaphore):
"""下载 B 站视频与封面。
优先走 DASH（音视频分轨 + ffmpeg 合流，画质最好）；本机没有 ffmpeg
或 DASH 流不可用时，降级为 mp4 直链（音视频合一，清晰度受接口限制）。
:param video_item: 视频详情，需包含 View 字段（含 aid/cid/pic/bvid）
:param semaphore: 用于限制 playurl 接口请求并发
"""
download bilibili video
:param video_item:
:param semaphore:
:return:
"""
if not config.ENABLE_GET_MEIDAS:
utils.logger.info(f"[BilibiliCrawler.get_bilibili_video] Crawling image mode is not enabled")
if not config.ENABLE_GET_MEDIA:
return
video_item_view: Dict = video_item.get("View")
try:
await self._download_media(video_item, semaphore)
except Exception as exc:
# 媒体下载是旁路能力：playurl 是额外的接口调用，网络抖动不能中断爬取主流程
utils.logger.error(f"[BilibiliCrawler.download_media] 媒体下载异常: {exc}")
async def _download_media(self, video_item: Dict, semaphore: asyncio.Semaphore) -> None:
video_item_view: Dict = video_item.get("View") or {}
aid = video_item_view.get("aid")
cid = video_item_view.get("cid")
result = await self.get_video_play_url_task(aid, cid, semaphore)
if result is None:
utils.logger.info("[BilibiliCrawler.get_bilibili_video] get video play url failed")
return
durl_list = result.get("durl")
max_size = -1
video_url = ""
for durl in durl_list:
size = durl.get("size")
if size > max_size:
max_size = size
video_url = durl.get("url")
if video_url == "":
utils.logger.info("[BilibiliCrawler.get_bilibili_video] get video url failed")
content_id = video_item_view.get("bvid") or (str(aid) if aid else "")
if not content_id:
utils.logger.warning("[BilibiliCrawler.download_media] 缺少 aid/bvid，跳过媒体下载")
return
content = await self.bili_client.get_video_media(video_url)
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
utils.logger.info(f"[BilibiliCrawler.get_bilibili_video] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching video {aid}")
if content is None:
downloader = self._get_media_downloader()
cover_item = bili_media.build_cover_item(video_item_view, content_id)
if cover_item is not None:
await downloader.download(cover_item)
if not aid or not cid:
utils.logger.warning("[BilibiliCrawler.download_media] 缺少 aid/cid，无法获取播放地址")
return
extension_file_name = f"video.mp4"
await bilibili_store.store_video(aid, content, extension_file_name)
# ffmpeg 是否可用在请求之前就是确定的（进程内缓存），据此一次性决定 fnval：
# 没装 ffmpeg 却先按 DASH 请求一次，既多打一次风控最敏感的 playurl 接口，
# 也拿不到可用的 durl
dash_supported = is_ffmpeg_available()
play_fnval = bili_media.DASH_FNVAL if dash_supported else bili_media.MP4_FNVAL
play_info = await self.get_video_play_url_task(aid, cid, semaphore, fnval=play_fnval)
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
if play_info is None:
utils.logger.error("[BilibiliCrawler.download_media] 获取播放地址失败")
return
if dash_supported:
# 逐档降级：未登录或权限不足时 B 站会返回高清晰度 URL，但实际取流被 CDN 403，
# 此时退到更低的可用档位往往能成功，比直接放弃（或退回更低质的直链）更好
quality: Optional[int] = getattr(config, "BILI_QN", 80)
while quality is not None:
dash_item = bili_media.build_dash_item(play_info, content_id, preferred_quality=quality)
if dash_item is None:
break
if await downloader.download(dash_item) is not None:
return
lower_quality = bili_media.next_lower_quality(play_info, quality)
if lower_quality is None:
break
utils.logger.warning(
f"[BilibiliCrawler.download_media] 清晰度 {quality} 取流失败，降级到 {lower_quality}"
)
quality = lower_quality
utils.logger.warning("[BilibiliCrawler.download_media] DASH 路径失败，降级为 mp4 直链")
else:
utils.logger.info("[BilibiliCrawler.download_media] 未检测到 ffmpeg，使用 mp4 直链下载（低清晰度）")
durl_play_info = play_info
durl_item = bili_media.build_durl_item(durl_play_info, content_id)
if durl_item is None:
# DASH 请求不会返回 durl，需要再按 mp4 格式请求一次
durl_play_info = await self.get_video_play_url_task(
aid, cid, semaphore, fnval=bili_media.MP4_FNVAL
) or {}
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
durl_item = bili_media.build_durl_item(durl_play_info, content_id)
if durl_item is None:
utils.logger.error(
f"[BilibiliCrawler.download_media] 未能获取到视频直链 aid={aid} cid={cid}"
)
return
segment_count = bili_media.count_durl_segments(durl_play_info)
if segment_count > 1:
utils.logger.warning(
f"[BilibiliCrawler.download_media] 该视频直链被切成 {segment_count} 段，"
f"当前仅下载体积最大的一段，文件可能不完整；"
f"确认本机 ffmpeg 可用后走 DASH 路径可获取完整视频"
)
await downloader.download(durl_item)
def _get_media_downloader(self) -> MediaDownloader:
"""惰性创建媒体下载器，并同步最新的代理与 Cookie（两者都会就地刷新）"""
if self._media_downloader is None:
self._media_downloader = MediaDownloader(
platform="bili",
proxy=getattr(self.bili_client, "proxy", None),
extra_headers=self._media_headers(),
)
else:
self._media_downloader.update_credentials(
proxy=getattr(self.bili_client, "proxy", None),
extra_headers=self._media_headers(),
)
return self._media_downloader
def _media_headers(self) -> Dict:
"""媒体请求头：平台 Referer（防盗链）+ UA。
B 站清晰度由 playurl 接口依据 Cookie 决定，返回的直链里已固化流版本，
且带 deadline/upsig 签名，下载本身不需要 Cookie（已实测无 Cookie 可直接取流），
因此不透传 Cookie，避免把账号凭证扩散到 CDN 域名。
"""
headers = {"Referer": "https://www.bilibili.com/"}
client_headers = getattr(self.bili_client, "headers", {}) or {}
if client_headers.get("User-Agent"):
headers["User-Agent"] = client_headers["User-Agent"]
return headers
async def get_all_creator_details(self, creator_url_list: List[str]):
"""