mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-09-21 16:38:13 +08:00
feat(media): 重构媒体下载,支持 xhs/dy/ks/bili/wb 五平台
旧实现只覆盖 4 个平台,且把整个文件读进内存、无重试与完整性校验,
代码按平台复制粘贴了 4 份。本次用统一下载器替换:
- 新增 media_downloader/:流式写入、Range 续传、指数退避重试、大小校验、
路径穿越防护;B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py:从平台原始响应提取媒体地址,
与下载器解耦;快手首次接入下载能力
- 开关:config.ENABLE_GET_MEDIA 与 --get_media,并打通 API/WebUI;
同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合:{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质,否则降级 mp4 直链(产物 video-durl.mp4,
避免低清文件阻塞后续的高清路径)
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法
媒体下载失败只记录日志,不中断爬取主流程。
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
import type { ComponentType, ReactNode, KeyboardEvent } from 'react'
|
||||
import { useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { Database, Globe, KeyRound, MessageSquare, Play, Square, X } from 'lucide-react'
|
||||
import { Database, Globe, Image as ImageIcon, KeyRound, MessageSquare, Play, Square, X } from 'lucide-react'
|
||||
import { Label } from '@/components/ui/label'
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'
|
||||
import { Input } from '@/components/ui/input'
|
||||
@@ -371,6 +371,23 @@ export function CrawlerConfigPanel() {
|
||||
<p className="text-xs font-mono text-cyber-text-primary">{t('field.subComments')}</p>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-3 rounded-lg border border-cyber-border-subtle bg-cyber-bg-tertiary/30 p-2.5 hover:border-cyber-border-DEFAULT transition-colors">
|
||||
<Checkbox
|
||||
checked={config.enable_media}
|
||||
onCheckedChange={(checked) => updateConfig({ enable_media: checked === true })}
|
||||
disabled={isDisabled}
|
||||
/>
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<ImageIcon className="h-3.5 w-3.5 text-cyber-text-secondary" />
|
||||
<p className="text-xs font-mono text-cyber-text-primary">{t('field.downloadMedia')}</p>
|
||||
</div>
|
||||
<p className="text-[10px] text-cyber-text-muted leading-snug">
|
||||
{t('field.downloadMediaHint')}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-3 rounded-lg border border-cyber-border-subtle bg-cyber-bg-tertiary/30 p-2.5 hover:border-cyber-border-DEFAULT transition-colors">
|
||||
<Checkbox
|
||||
checked={config.headless}
|
||||
|
||||
@@ -55,6 +55,8 @@
|
||||
"saveFormatPlaceholder": "Select format",
|
||||
"commentExtraction": "Comment Extraction",
|
||||
"subComments": "Sub-comments",
|
||||
"downloadMedia": "DOWNLOAD_MEDIA",
|
||||
"downloadMediaHint": "Download cover and video (images for image posts). Supports xhs/dy/ks/bili/wb",
|
||||
"headlessMode": "HEADLESS_MODE",
|
||||
"headlessModeHint": "Run browser without GUI"
|
||||
},
|
||||
|
||||
@@ -55,6 +55,8 @@
|
||||
"saveFormatPlaceholder": "选择格式",
|
||||
"commentExtraction": "评论抓取",
|
||||
"subComments": "子评论",
|
||||
"downloadMedia": "下载媒体",
|
||||
"downloadMediaHint": "下载封面与视频(图文笔记下载图片),支持小红书/抖音/快手/B站/微博",
|
||||
"headlessMode": "无头模式",
|
||||
"headlessModeHint": "无 GUI 运行浏览器"
|
||||
},
|
||||
|
||||
@@ -17,6 +17,7 @@ export interface CrawlerConfig {
|
||||
start_page: number
|
||||
enable_comments: boolean
|
||||
enable_sub_comments: boolean
|
||||
enable_media: boolean
|
||||
save_option: string
|
||||
cookies: string
|
||||
headless: boolean
|
||||
|
||||
@@ -56,6 +56,7 @@ const defaultConfig: CrawlerConfig = {
|
||||
start_page: 1,
|
||||
enable_comments: true,
|
||||
enable_sub_comments: false,
|
||||
enable_media: false,
|
||||
save_option: 'json',
|
||||
cookies: '',
|
||||
headless: false,
|
||||
|
||||
@@ -8,6 +8,7 @@ export interface CrawlerConfig {
|
||||
start_page: number
|
||||
enable_comments: boolean
|
||||
enable_sub_comments: boolean
|
||||
enable_media: boolean // 是否下载媒体(封面/视频/图文图片),支持 xhs/dy/ks/bili/wb
|
||||
save_option: string
|
||||
cookies: string
|
||||
headless: boolean
|
||||
|
||||
Reference in New Issue
Block a user