fork/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-09-21 04:28:12 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
92c98e658e1f160941aaa78538a1b287f271bb3f
MediaCrawler/database/models.py
T
程序员阿江(Relakkes) a06273ea6c fix: 将平台业务ID统一改为String并自动建表 
- 抖音、B站、快手、微博的帖子/视频/评论ID从BigInteger改为String，
  避免PostgreSQL下字符串写入BIGINT报错及未来ID溢出风险
- B站dynamic_id改为String，修复API返回id_str被强转int导致的精度丢失
- 知乎提取器对content_id/question_id显式str()转换
- main.py启动数据库保存模式时自动建表，无需手动--init_db
- 同步更新相关老化测试
2026-07-01 23:03:43 +08:00

304 lines
17 KiB
Python
Raw Blame History

# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/database/models.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
#
# 教学版说明：为防止爬取到的用户个人信息被用于定位真人并私信骚扰，
# 本 ORM 不再持久化任何可识别用户的字段（用户 ID、IP 归属地、头像、
# 主页链接、签名、性别等一律不落库）。原始用户 ID 在提取层经
# tools.user_hash.anonymize_user_id 转为匿名 creator_hash 后写入，
# 仅用于"同一创作者"的内容分组；昵称保留但经 mask_nickname 中间脱敏。
# 创作者个人档案表（XhsCreator/DyCreator/WeiboCreator/TiebaCreator/
# ZhihuCreator/BilibiliUpInfo/BilibiliContactInfo）已整体移除。
from sqlalchemy import create_engine, Column, Integer, Text, String, BigInteger
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
Base = declarative_base()
class BilibiliVideo(Base):
__tablename__ = 'bilibili_video'
id = Column(Integer, primary_key=True, comment='主键ID')
video_id = Column(String(64), nullable=False, index=True, unique=True, comment='视频ID')
video_url = Column(Text, nullable=False, comment='视频URL')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
liked_count = Column(Integer, comment='点赞数')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
video_type = Column(Text, comment='视频类型')
title = Column(Text, comment='视频标题')
desc = Column(Text, comment='视频描述')
create_time = Column(BigInteger, index=True, comment='创建时间戳')
disliked_count = Column(Text, comment='点踩数')
video_play_count = Column(Text, comment='播放数')
video_favorite_count = Column(Text, comment='收藏数')
video_share_count = Column(Text, comment='分享数')
video_coin_count = Column(Text, comment='硬币数')
video_danmaku = Column(Text, comment='弹幕数')
video_comment = Column(Text, comment='评论数')
video_cover_url = Column(Text, comment='视频封面URL')
source_keyword = Column(Text, default='', comment='来源关键词')
class BilibiliVideoComment(Base):
__tablename__ = 'bilibili_video_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(String(128), index=True, comment='评论ID')
video_id = Column(String(64), index=True, comment='视频ID')
content = Column(Text, comment='评论内容')
create_time = Column(BigInteger, comment='创建时间戳')
sub_comment_count = Column(Text, comment='子评论数')
parent_comment_id = Column(String(255), comment='父评论ID')
like_count = Column(Text, default='0', comment='点赞数')
class BilibiliUpDynamic(Base):
__tablename__ = 'bilibili_up_dynamic'
id = Column(Integer, primary_key=True, comment='主键ID')
dynamic_id = Column(String(128), index=True, comment='动态ID')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
user_name = Column(Text, comment='用户名称(已脱敏)')
text = Column(Text, comment='动态内容')
type = Column(Text, comment='动态类型')
pub_ts = Column(BigInteger, comment='发布时间戳')
total_comments = Column(Integer, comment='总评论数')
total_forwards = Column(Integer, comment='总转发数')
total_liked = Column(Integer, comment='总点赞数')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
class DouyinAweme(Base):
__tablename__ = 'douyin_aweme'
id = Column(Integer, primary_key=True, comment='主键ID')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
aweme_id = Column(String(255), index=True, comment='作品ID')
aweme_type = Column(Text, comment='作品类型')
title = Column(Text, comment='作品标题')
desc = Column(Text, comment='作品描述')
create_time = Column(BigInteger, index=True, comment='创建时间戳')
liked_count = Column(Text, comment='点赞数')
comment_count = Column(Text, comment='评论数')
share_count = Column(Text, comment='分享数')
collected_count = Column(Text, comment='收藏数')
aweme_url = Column(Text, comment='作品URL')
cover_url = Column(Text, comment='封面URL')
video_download_url = Column(Text, comment='视频下载URL')
music_download_url = Column(Text, comment='音乐下载URL')
note_download_url = Column(Text, comment='笔记下载URL')
source_keyword = Column(Text, default='', comment='来源关键词')
class DouyinAwemeComment(Base):
__tablename__ = 'douyin_aweme_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(String(255), index=True, comment='评论ID')
aweme_id = Column(String(255), index=True, comment='作品ID')
content = Column(Text, comment='评论内容')
create_time = Column(BigInteger, comment='创建时间戳')
sub_comment_count = Column(Text, comment='子评论数')
parent_comment_id = Column(String(255), comment='父评论ID')
like_count = Column(Text, default='0', comment='点赞数')
pictures = Column(Text, default='', comment='图片')
class KuaishouVideo(Base):
__tablename__ = 'kuaishou_video'
id = Column(Integer, primary_key=True, comment='主键ID')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
video_id = Column(String(255), index=True, comment='视频ID')
video_type = Column(Text, comment='视频类型')
title = Column(Text, comment='视频标题')
desc = Column(Text, comment='视频描述')
create_time = Column(BigInteger, index=True, comment='创建时间戳')
liked_count = Column(Text, comment='点赞数')
viewd_count = Column(Text, comment='观看数')
video_url = Column(Text, comment='视频URL')
video_cover_url = Column(Text, comment='视频封面URL')
video_play_url = Column(Text, comment='视频播放URL')
source_keyword = Column(Text, default='', comment='来源关键词')
class KuaishouVideoComment(Base):
__tablename__ = 'kuaishou_video_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(String(255), index=True, comment='评论ID')
video_id = Column(String(255), index=True, comment='视频ID')
content = Column(Text, comment='评论内容')
create_time = Column(BigInteger, comment='创建时间戳')
sub_comment_count = Column(Text, comment='子评论数')
class WeiboNote(Base):
__tablename__ = 'weibo_note'
id = Column(Integer, primary_key=True, comment='主键ID')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
note_id = Column(String(64), index=True, comment='笔记ID')
content = Column(Text, comment='笔记内容')
create_time = Column(BigInteger, index=True, comment='创建时间戳')
create_date_time = Column(String(255), index=True, comment='创建日期时间')
liked_count = Column(Text, comment='点赞数')
comments_count = Column(Text, comment='评论数')
shared_count = Column(Text, comment='分享数')
note_url = Column(Text, comment='笔记URL')
source_keyword = Column(Text, default='', comment='来源关键词')
class WeiboNoteComment(Base):
__tablename__ = 'weibo_note_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(String(64), index=True, comment='评论ID')
note_id = Column(String(64), index=True, comment='笔记ID')
content = Column(Text, comment='评论内容')
create_time = Column(BigInteger, comment='创建时间戳')
create_date_time = Column(String(255), index=True, comment='创建日期时间')
comment_like_count = Column(Text, comment='评论点赞数')
sub_comment_count = Column(Text, comment='子评论数')
parent_comment_id = Column(String(255), comment='父评论ID')
class XhsNote(Base):
__tablename__ = 'xhs_note'
id = Column(Integer, primary_key=True, comment='主键ID')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
note_id = Column(String(255), index=True, comment='笔记ID')
type = Column(Text, comment='笔记类型')
title = Column(Text, comment='笔记标题')
desc = Column(Text, comment='笔记描述')
video_url = Column(Text, comment='视频URL')
time = Column(BigInteger, index=True, comment='时间戳')
last_update_time = Column(BigInteger, comment='最后更新时间戳')
liked_count = Column(Text, comment='点赞数')
collected_count = Column(Text, comment='收藏数')
comment_count = Column(Text, comment='评论数')
share_count = Column(Text, comment='分享数')
image_list = Column(Text, comment='图片列表')
tag_list = Column(Text, comment='标签列表')
note_url = Column(Text, comment='笔记URL')
source_keyword = Column(Text, default='', comment='来源关键词')
xsec_token = Column(Text, comment='Xsec Token')
class XhsNoteComment(Base):
__tablename__ = 'xhs_note_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
comment_id = Column(String(255), index=True, comment='评论ID')
create_time = Column(BigInteger, index=True, comment='创建时间戳')
note_id = Column(String(255), comment='笔记ID')
content = Column(Text, comment='评论内容')
sub_comment_count = Column(Integer, comment='子评论数')
pictures = Column(Text, comment='图片')
parent_comment_id = Column(String(255), comment='父评论ID')
like_count = Column(Text, comment='点赞数')
class TiebaNote(Base):
__tablename__ = 'tieba_note'
id = Column(Integer, primary_key=True, comment='主键ID')
note_id = Column(String(644), index=True, comment='笔记ID')
title = Column(Text, comment='笔记标题')
desc = Column(Text, comment='笔记描述')
note_url = Column(Text, comment='笔记URL')
publish_time = Column(String(255), index=True, comment='发布时间')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
user_nickname = Column(Text, default='', comment='用户昵称(已脱敏)')
tieba_id = Column(String(255), default='', comment='贴吧ID')
tieba_name = Column(Text, comment='贴吧名称')
tieba_link = Column(Text, comment='贴吧链接')
total_replay_num = Column(Integer, default=0, comment='总回复数')
total_replay_page = Column(Integer, default=0, comment='总回复页数')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
source_keyword = Column(Text, default='', comment='来源关键词')
class TiebaComment(Base):
__tablename__ = 'tieba_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
comment_id = Column(String(255), index=True, comment='评论ID')
parent_comment_id = Column(String(255), default='', comment='父评论ID')
content = Column(Text, comment='评论内容')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
user_nickname = Column(Text, default='', comment='用户昵称(已脱敏)')
tieba_id = Column(String(255), default='', comment='贴吧ID')
tieba_name = Column(Text, comment='贴吧名称')
tieba_link = Column(Text, comment='贴吧链接')
publish_time = Column(String(255), index=True, comment='发布时间')
sub_comment_count = Column(Integer, default=0, comment='子评论数')
note_id = Column(String(255), index=True, comment='笔记ID')
note_url = Column(Text, comment='笔记URL')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
class ZhihuContent(Base):
__tablename__ = 'zhihu_content'
id = Column(Integer, primary_key=True, comment='主键ID')
content_id = Column(String(64), index=True, comment='内容ID')
content_type = Column(Text, comment='内容类型')
content_text = Column(Text, comment='内容文本')
content_url = Column(Text, comment='内容URL')
question_id = Column(String(255), comment='问题ID')
title = Column(Text, comment='标题')
desc = Column(Text, comment='描述')
created_time = Column(String(32), index=True, comment='创建时间')
updated_time = Column(Text, comment='更新时间')
voteup_count = Column(Integer, default=0, comment='赞同数')
comment_count = Column(Integer, default=0, comment='评论数')
source_keyword = Column(Text, comment='来源关键词')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
user_nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
class ZhihuComment(Base):
__tablename__ = 'zhihu_comment'
id = Column(Integer, primary_key=True, comment='主键ID')
comment_id = Column(String(64), index=True, comment='评论ID')
parent_comment_id = Column(String(64), comment='父评论ID')
content = Column(Text, comment='评论内容')
publish_time = Column(String(32), index=True, comment='发布时间')
sub_comment_count = Column(Integer, default=0, comment='子评论数')
like_count = Column(Integer, default=0, comment='点赞数')
dislike_count = Column(Integer, default=0, comment='点踩数')
content_id = Column(String(64), index=True, comment='内容ID')
content_type = Column(Text, comment='内容类型')
creator_hash = Column(String(64), index=True, comment='创作者匿名哈希')
user_nickname = Column(Text, comment='用户昵称(已脱敏)')
add_ts = Column(BigInteger, comment='添加时间戳')
last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
Reference in New Issue View Git Blame Copy Permalink