fork/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-09-25 08:58:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(media): 重构媒体下载，支持 xhs/dy/ks/bili/wb 五平台

Browse Source 
旧实现只覆盖 4 个平台，且把整个文件读进内存、无重试与完整性校验，
代码按平台复制粘贴了 4 份。本次用统一下载器替换：

- 新增 media_downloader/：流式写入、Range 续传、指数退避重试、大小校验、
  路径穿越防护；B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py：从平台原始响应提取媒体地址，
  与下载器解耦；快手首次接入下载能力
- 开关：config.ENABLE_GET_MEDIA 与 --get_media，并打通 API/WebUI；
  同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合：{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质，否则降级 mp4 直链（产物 video-durl.mp4，
  避免低清文件阻塞后续的高清路径）
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法

媒体下载失败只记录日志，不中断爬取主流程。
This commit is contained in:
程序员阿江(Relakkes)
2026-09-17 22:54:22 +08:00
parent 60e66f2a92
commit 0ca7b29cf0
47 changed files with 5187 additions and 907 deletions
Download Patch File Download Diff File Expand all files Collapse all files
store/bilibili/__init__.py
-16
View File
@@ -29,7 +29,6 @@ from var import source_keyword_var
from tools.user_hash import anonymize_user_id, mask_nickname
from ._store_impl import *
from .bilibilli_store_media import *
class BiliStoreFactory:
@@ -116,21 +115,6 @@ async def update_bilibili_video_comment(video_id: str, comment_item: Dict):
await BiliStoreFactory.create_store().store_comment(comment_item=save_comment_item)
async def store_video(aid, video_content, extension_file_name):
"""
video video storage implementation
Args:
aid:
video_content:
extension_file_name:
"""
await BilibiliVideo().store_video({
"aid": aid,
"video_content": video_content,
"extension_file_name": extension_file_name,
})
async def batch_update_bilibili_creator_fans(creator_info: Dict, fans_list: List[Dict]):
# 教学版：不再采集/存储粉丝列表(其他用户的个人信息)，防骚扰。
return
store/bilibili/bilibilli_store_media.py
-82
View File
@@ -1,82 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/bilibili/bilibilli_store_media.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
# -*- coding: utf-8 -*-
# @Author : helloteemo
# @Time : 2024/7/12 20:01
# @Desc : Bilibili media storage
import pathlib
from typing import Dict
import aiofiles
from base.base_crawler import AbstractStoreImage, AbstractStoreVideo
from tools import utils
import config
class BilibiliVideo(AbstractStoreVideo):
def __init__(self):
if config.SAVE_DATA_PATH:
self.video_store_path = f"{config.SAVE_DATA_PATH}/bili/videos"
else:
self.video_store_path = "data/bili/videos"
async def store_video(self, video_content_item: Dict):
"""
store content
Args:
video_content_item:
Returns:
"""
await self.save_video(video_content_item.get("aid"), video_content_item.get("video_content"), video_content_item.get("extension_file_name"))
def make_save_file_name(self, aid: str, extension_file_name: str) -> str:
"""
make save file name by store type
Args:
aid: aid
extension_file_name: video filename with extension
Returns:
"""
return f"{self.video_store_path}/{aid}/{extension_file_name}"
async def save_video(self, aid: int, video_content: str, extension_file_name="mp4"):
"""
save video to local
Args:
aid: aid
video_content: video content
extension_file_name: video filename with extension
Returns:
"""
pathlib.Path(self.video_store_path + "/" + str(aid)).mkdir(parents=True, exist_ok=True)
save_file_name = self.make_save_file_name(str(aid), extension_file_name)
async with aiofiles.open(save_file_name, 'wb') as f:
await f.write(video_content)
utils.logger.info(f"[BilibiliVideoImplement.save_video] save save_video {save_file_name} success ...")
store/douyin/__init__.py
+4 -98
View File
@@ -24,11 +24,11 @@
from typing import List
import config
from media_platform.douyin.media import extract_cover_url, extract_image_urls, extract_video_urls
from var import source_keyword_var
from tools.user_hash import anonymize_user_id, mask_nickname
from ._store_impl import *
from .douyin_store_media import *
class DouyinStoreFactory:
@@ -51,30 +51,6 @@ class DouyinStoreFactory:
return store_class()
def _extract_note_image_list(aweme_detail: Dict) -> List[str]:
"""
Extract note image list
Args:
aweme_detail (Dict): Douyin content details
Returns:
List[str]: Note image list
"""
images_res: List[str] = []
images: List[Dict] = aweme_detail.get("images", [])
if not images:
return []
for image in images:
image_url_list = image.get("url_list", []) # download_url_list has watermarked images, url_list has non-watermarked images
if image_url_list:
images_res.append(image_url_list[-1])
return images_res
def _extract_comment_image_list(comment_item: Dict) -> List[str]:
"""
Extract comment image list
@@ -99,46 +75,6 @@ def _extract_comment_image_list(comment_item: Dict) -> List[str]:
return images_res
def _extract_content_cover_url(aweme_detail: Dict) -> str:
"""
Extract video cover URL
Args:
aweme_detail (Dict): Douyin content details
Returns:
str: Video cover URL
"""
res_cover_url = ""
video_item = aweme_detail.get("video", {})
raw_cover_url_list = (video_item.get("raw_cover", {}) or video_item.get("origin_cover", {})).get("url_list", [])
if raw_cover_url_list and len(raw_cover_url_list) > 1:
res_cover_url = raw_cover_url_list[1]
return res_cover_url
def _extract_video_download_url(aweme_detail: Dict) -> str:
"""
Extract video download URL
Args:
aweme_detail (Dict): Douyin video
Returns:
str: Video download URL
"""
video_item = aweme_detail.get("video", {})
url_h264_list = video_item.get("play_addr_h264", {}).get("url_list", [])
url_256_list = video_item.get("play_addr_256", {}).get("url_list", [])
url_list = video_item.get("play_addr", {}).get("url_list", [])
actual_url_list = url_h264_list or url_256_list or url_list
if not actual_url_list or len(actual_url_list) < 2:
return ""
return actual_url_list[-1]
def _extract_music_download_url(aweme_detail: Dict) -> str:
"""
Extract music download URL
@@ -173,10 +109,10 @@ async def update_douyin_aweme(aweme_item: Dict):
"share_count": str(interact_info.get("share_count")),
"last_modify_ts": utils.get_current_timestamp(),
"aweme_url": f"https://www.douyin.com/video/{aweme_id}",
"cover_url": _extract_content_cover_url(aweme_item),
"video_download_url": _extract_video_download_url(aweme_item),
"cover_url": extract_cover_url(aweme_item),
"video_download_url": (extract_video_urls(aweme_item) or [""])[0],
"music_download_url": _extract_music_download_url(aweme_item),
"note_download_url": ",".join(_extract_note_image_list(aweme_item)),
"note_download_url": ",".join(extract_image_urls(aweme_item)),
"source_keyword": source_keyword_var.get(),
}
utils.logger.info(f"[store.douyin.update_douyin_aweme] douyin aweme id:{aweme_id}, title:{save_content_item.get('title')}")
@@ -219,33 +155,3 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict):
async def save_creator(user_id: str, creator: Dict):
# 教学版：创作者个人资料(昵称/性别/头像/签名/IP/粉丝数等)不再落库，防骚扰。
return
async def update_dy_aweme_image(aweme_id, pic_content, extension_file_name):
"""
Update Douyin note image
Args:
aweme_id:
pic_content:
extension_file_name:
Returns:
"""
await DouYinImage().store_image({"aweme_id": aweme_id, "pic_content": pic_content, "extension_file_name": extension_file_name})
async def update_dy_aweme_video(aweme_id, video_content, extension_file_name):
"""
Update Douyin short video
Args:
aweme_id:
video_content:
extension_file_name:
Returns:
"""
await DouYinVideo().store_video({"aweme_id": aweme_id, "video_content": video_content, "extension_file_name": extension_file_name})
store/douyin/douyin_store_media.py
-129
View File
@@ -1,129 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/douyin/douyin_store_media.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
import pathlib
from typing import Dict
import aiofiles
from base.base_crawler import AbstractStoreImage, AbstractStoreVideo
from tools import utils
import config
class DouYinImage(AbstractStoreImage):
def __init__(self):
if config.SAVE_DATA_PATH:
self.image_store_path = f"{config.SAVE_DATA_PATH}/douyin/images"
else:
self.image_store_path = "data/douyin/images"
async def store_image(self, image_content_item: Dict):
"""
store content
Args:
image_content_item:
Returns:
"""
await self.save_image(image_content_item.get("aweme_id"), image_content_item.get("pic_content"), image_content_item.get("extension_file_name"))
def make_save_file_name(self, aweme_id: str, extension_file_name: str) -> str:
"""
make save file name by store type
Args:
aweme_id: aweme id
extension_file_name: image filename with extension
Returns:
"""
return f"{self.image_store_path}/{aweme_id}/{extension_file_name}"
async def save_image(self, aweme_id: str, pic_content: str, extension_file_name):
"""
save image to local
Args:
aweme_id: aweme id
pic_content: image content
extension_file_name: image filename with extension
Returns:
"""
pathlib.Path(self.image_store_path + "/" + aweme_id).mkdir(parents=True, exist_ok=True)
save_file_name = self.make_save_file_name(aweme_id, extension_file_name)
async with aiofiles.open(save_file_name, 'wb') as f:
await f.write(pic_content)
utils.logger.info(f"[DouYinImageStoreImplement.save_image] save image {save_file_name} success ...")
class DouYinVideo(AbstractStoreVideo):
def __init__(self):
if config.SAVE_DATA_PATH:
self.video_store_path = f"{config.SAVE_DATA_PATH}/douyin/videos"
else:
self.video_store_path = "data/douyin/videos"
async def store_video(self, video_content_item: Dict):
"""
store content
Args:
video_content_item:
Returns:
"""
await self.save_video(video_content_item.get("aweme_id"), video_content_item.get("video_content"), video_content_item.get("extension_file_name"))
def make_save_file_name(self, aweme_id: str, extension_file_name: str) -> str:
"""
make save file name by store type
Args:
aweme_id: aweme id
extension_file_name: video filename with extension
Returns:
"""
return f"{self.video_store_path}/{aweme_id}/{extension_file_name}"
async def save_video(self, aweme_id: str, video_content: str, extension_file_name):
"""
save video to local
Args:
aweme_id: aweme id
video_content: video content
extension_file_name: video filename with extension
Returns:
"""
pathlib.Path(self.video_store_path + "/" + aweme_id).mkdir(parents=True, exist_ok=True)
save_file_name = self.make_save_file_name(aweme_id, extension_file_name)
async with aiofiles.open(save_file_name, 'wb') as f:
await f.write(video_content)
utils.logger.info(f"[DouYinVideoStoreImplement.save_video] save video {save_file_name} success ...")
store/weibo/__init__.py
-15
View File
@@ -28,7 +28,6 @@ from typing import List
from tools.user_hash import anonymize_user_id, mask_nickname
from var import source_keyword_var
from .weibo_store_media import *
from ._store_impl import *
@@ -160,20 +159,6 @@ async def update_weibo_note_comment(note_id: str, comment_item: Dict):
await WeibostoreFactory.create_store().store_comment(comment_item=save_comment_item)
async def update_weibo_note_image(picid: str, pic_content, extension_file_name):
"""
Save weibo note image to local
Args:
picid:
pic_content:
extension_file_name:
Returns:
"""
await WeiboStoreImage().store_image({"pic_id": picid, "pic_content": pic_content, "extension_file_name": extension_file_name})
async def save_creator(user_id: str, user_info: Dict):
"""
Save creator information to local
store/weibo/weibo_store_media.py
-82
View File
@@ -1,82 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/weibo/weibo_store_media.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
# -*- coding: utf-8 -*-
# @Author : Erm
# @Time : 2024/4/9 17:35
# @Desc : Weibo media storage
import pathlib
from typing import Dict
import aiofiles
from base.base_crawler import AbstractStoreImage, AbstractStoreVideo
from tools import utils
import config
class WeiboStoreImage(AbstractStoreImage):
def __init__(self):
if config.SAVE_DATA_PATH:
self.image_store_path = f"{config.SAVE_DATA_PATH}/weibo/images"
else:
self.image_store_path = "data/weibo/images"
async def store_image(self, image_content_item: Dict):
"""
store content
Args:
image_content_item:
Returns:
"""
await self.save_image(image_content_item.get("pic_id"), image_content_item.get("pic_content"), image_content_item.get("extension_file_name"))
def make_save_file_name(self, picid: str, extension_file_name: str) -> str:
"""
make save file name by store type
Args:
picid: image id
extension_file_name: video filename with extension
Returns:
"""
return f"{self.image_store_path}/{picid}.{extension_file_name}"
async def save_image(self, picid: str, pic_content: str, extension_file_name="jpg"):
"""
save image to local
Args:
picid: image id
pic_content: image content
extension_file_name: image filename with extension
Returns:
"""
pathlib.Path(self.image_store_path).mkdir(parents=True, exist_ok=True)
save_file_name = self.make_save_file_name(picid, extension_file_name)
async with aiofiles.open(save_file_name, 'wb') as f:
await f.write(pic_content)
utils.logger.info(f"[WeiboImageStoreImplement.save_image] save image {save_file_name} success ...")
store/xhs/__init__.py
+2 -66
View File
@@ -24,10 +24,10 @@
from typing import List
import config
from media_platform.xhs.media import extract_video_urls
from var import source_keyword_var
from tools.user_hash import anonymize_user_id, mask_nickname
from .xhs_store_media import *
from ._store_impl import *
@@ -51,40 +51,6 @@ class XhsStoreFactory:
return store_class()
def get_video_url_arr(note_item: Dict) -> List:
"""
Get video url array
Args:
note_item:
Returns:
"""
if note_item.get('type') != 'video':
return []
video_dict = note_item.get('video')
if not video_dict:
return []
videoArr = []
consumer = video_dict.get('consumer', {})
originVideoKey = consumer.get('origin_video_key', '')
if originVideoKey == '':
originVideoKey = consumer.get('originVideoKey', '')
# Fallback with watermark
if originVideoKey == '':
media = video_dict.get('media', {})
stream = media.get('stream', {})
videos = stream.get('h264')
if type(videos).__name__ == 'list':
videoArr = [v.get('master_url') for v in videos]
else:
videoArr = [f"http://sns-video-bd.xhscdn.com/{originVideoKey}"]
return videoArr
async def update_xhs_note(note_item: Dict):
"""
Update Xiaohongshu note
@@ -104,7 +70,7 @@ async def update_xhs_note(note_item: Dict):
if img.get('url_default') != '':
img.update({'url': img.get('url_default')})
video_url = ','.join(get_video_url_arr(note_item))
video_url = ','.join(extract_video_urls(note_item))
local_db_item = {
"note_id": note_item.get("note_id"), # Note ID
@@ -190,33 +156,3 @@ async def save_creator(user_id: str, creator: Dict):
"""
# 教学版：创作者个人资料(昵称/性别/头像/IP/粉丝数等)不再落库，防骚扰。
return
async def update_xhs_note_image(note_id, pic_content, extension_file_name):
"""
Update Xiaohongshu note image
Args:
note_id:
pic_content:
extension_file_name:
Returns:
"""
await XiaoHongShuImage().store_image({"notice_id": note_id, "pic_content": pic_content, "extension_file_name": extension_file_name})
async def update_xhs_note_video(note_id, video_content, extension_file_name):
"""
Update Xiaohongshu note video
Args:
note_id:
video_content:
extension_file_name:
Returns:
"""
await XiaoHongShuVideo().store_video({"notice_id": note_id, "video_content": video_content, "extension_file_name": extension_file_name})
store/xhs/xhs_store_media.py
-133
View File
@@ -1,133 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/xhs/xhs_store_media.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
# -*- coding: utf-8 -*-
# @Author : helloteemo
# @Time : 2024/7/11 22:35
# @Desc : Xiaohongshu media storage
import pathlib
from typing import Dict
import aiofiles
from base.base_crawler import AbstractStoreImage, AbstractStoreVideo
from tools import utils
import config
class XiaoHongShuImage(AbstractStoreImage):
def __init__(self):
if config.SAVE_DATA_PATH:
self.image_store_path = f"{config.SAVE_DATA_PATH}/xhs/images"
else:
self.image_store_path = "data/xhs/images"
async def store_image(self, image_content_item: Dict):
"""
store content
Args:
image_content_item:
Returns:
"""
await self.save_image(image_content_item.get("notice_id"), image_content_item.get("pic_content"), image_content_item.get("extension_file_name"))
def make_save_file_name(self, notice_id: str, extension_file_name: str) -> str:
"""
make save file name by store type
Args:
notice_id: notice id
extension_file_name: image filename with extension
Returns:
"""
return f"{self.image_store_path}/{notice_id}/{extension_file_name}"
async def save_image(self, notice_id: str, pic_content: str, extension_file_name):
"""
save image to local
Args:
notice_id: notice id
pic_content: image content
extension_file_name: image filename with extension
Returns:
"""
pathlib.Path(self.image_store_path + "/" + notice_id).mkdir(parents=True, exist_ok=True)
save_file_name = self.make_save_file_name(notice_id, extension_file_name)
async with aiofiles.open(save_file_name, 'wb') as f:
await f.write(pic_content)
utils.logger.info(f"[XiaoHongShuImageStoreImplement.save_image] save image {save_file_name} success ...")
class XiaoHongShuVideo(AbstractStoreVideo):
def __init__(self):
if config.SAVE_DATA_PATH:
self.video_store_path = f"{config.SAVE_DATA_PATH}/xhs/videos"
else:
self.video_store_path = "data/xhs/videos"
async def store_video(self, video_content_item: Dict):
"""
store content
Args:
video_content_item:
Returns:
"""
await self.save_video(video_content_item.get("notice_id"), video_content_item.get("video_content"), video_content_item.get("extension_file_name"))
def make_save_file_name(self, notice_id: str, extension_file_name: str) -> str:
"""
make save file name by store type
Args:
notice_id: notice id
extension_file_name: video filename with extension
Returns:
"""
return f"{self.video_store_path}/{notice_id}/{extension_file_name}"
async def save_video(self, notice_id: str, video_content: str, extension_file_name):
"""
save video to local
Args:
notice_id: notice id
video_content: video content
extension_file_name: video filename with extension
Returns:
"""
pathlib.Path(self.video_store_path + "/" + notice_id).mkdir(parents=True, exist_ok=True)
save_file_name = self.make_save_file_name(notice_id, extension_file_name)
async with aiofiles.open(save_file_name, 'wb') as f:
await f.write(video_content)
utils.logger.info(f"[XiaoHongShuVideoStoreImplement.save_video] save video {save_file_name} success ...")