fix: 移除orm的所有内容

store/xhs/__init__.py

@@ -7,7 +7,6 @@ from typing import List
 import config
 
 from . import xhs_store_impl
-from .xhs_store_db_types import *
 from .xhs_store_impl import *
 
 
@@ -26,7 +25,6 @@ class XhsStoreFactory:
         return store_class()
 
 
-
 async def update_xhs_note(note_item: Dict):
     note_id = note_item.get("note_id")
     user_info = note_item.get("user", {})
@@ -117,7 +115,8 @@ async def save_creator(user_id: str, creator: Dict):
         'follows': follows,
         'fans': fans,
         'interaction': interaction,
-        'tag_list': json.dumps({tag.get('tagType'): tag.get('name') for tag in creator.get('tags')}, ensure_ascii=False),
+        'tag_list': json.dumps({tag.get('tagType'): tag.get('name') for tag in creator.get('tags')},
+                               ensure_ascii=False),
         "last_modify_ts": utils.get_current_timestamp(),
     }
     utils.logger.info(f"[store.xhs.save_creator] creator:{local_db_item}")

store/xhs/xhs_store_db_types.py

@@ -1,76 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Author  : relakkes@gmail.com
-# @Time    : 2024/1/14 17:31
-# @Desc    : 小红书存储到DB的模型类集合
-
-from tortoise import fields
-from tortoise.models import Model
-
-
-class XhsBaseModel(Model):
-    id = fields.IntField(pk=True, autoincrement=True, description="自增ID")
-    user_id = fields.CharField(max_length=64, description="用户ID")
-    nickname = fields.CharField(null=True, max_length=64, description="用户昵称")
-    avatar = fields.CharField(null=True, max_length=255, description="用户头像地址")
-    ip_location = fields.CharField(null=True, max_length=255, description="评论时的IP地址")
-    add_ts = fields.BigIntField(description="记录添加时间戳")
-    last_modify_ts = fields.BigIntField(description="记录最后修改时间戳")
-
-    class Meta:
-        abstract = True
-
-
-class XHSNote(XhsBaseModel):
-    note_id = fields.CharField(max_length=64, index=True, description="笔记ID")
-    type = fields.CharField(null=True, max_length=16, description="笔记类型(normal | video)")
-    title = fields.CharField(null=True, max_length=255, description="笔记标题")
-    desc = fields.TextField(null=True, description="笔记描述")
-    video_url = fields.TextField(null=True, description="视频地址")
-    time = fields.BigIntField(description="笔记发布时间戳", index=True)
-    last_update_time = fields.BigIntField(description="笔记最后更新时间戳")
-    liked_count = fields.CharField(null=True, max_length=16, description="笔记点赞数")
-    collected_count = fields.CharField(null=True, max_length=16, description="笔记收藏数")
-    comment_count = fields.CharField(null=True, max_length=16, description="笔记评论数")
-    share_count = fields.CharField(null=True, max_length=16, description="笔记分享数")
-    image_list = fields.TextField(null=True, description="笔记封面图片列表")
-    tag_list = fields.TextField(null=True, description="标签列表")
-    note_url = fields.CharField(null=True, max_length=255, description="笔记详情页的URL")
-
-    class Meta:
-        table = "xhs_note"
-        table_description = "小红书笔记"
-
-    def __str__(self):
-        return f"{self.note_id} - {self.title}"
-
-
-class XHSNoteComment(XhsBaseModel):
-    comment_id = fields.CharField(max_length=64, index=True, description="评论ID")
-    create_time = fields.BigIntField(index=True, description="评论时间戳")
-    note_id = fields.CharField(max_length=64, description="笔记ID")
-    content = fields.TextField(description="评论内容")
-    sub_comment_count = fields.IntField(description="子评论数量")
-    pictures = fields.CharField(null=True, max_length=512, description="评论的图片集合")
-
-    class Meta:
-        table = "xhs_note_comment"
-        table_description = "小红书笔记评论"
-
-    def __str__(self):
-        return f"{self.comment_id} - {self.content}"
-
-
-class XhsCreator(XhsBaseModel):
-    desc = fields.TextField(null=True, description="用户描述")
-    gender = fields.CharField(null=True, max_length=1, description="性别")
-    follows = fields.CharField(null=True, max_length=16, description="关注数")
-    fans = fields.CharField(null=True, max_length=16, description="粉丝数")
-    interaction = fields.CharField(null=True, max_length=16, description="获赞和收藏数")
-    # follows = fields.IntField(description="关注数")
-    # fans = fields.IntField(description="粉丝数")
-    # interaction = fields.IntField(description="获赞和收藏数")
-    tag_list = fields.TextField(null=True, description="标签列表") # json字符串
-
-    class Meta:
-        table = "xhs_creator"
-        table_description = "小红书博主"

store/xhs/xhs_store_impl.py

@@ -10,12 +10,10 @@ import pathlib
 from typing import Dict
 
 import aiofiles
-from tortoise.contrib.pydantic import pydantic_model_creator
 
 from base.base_crawler import AbstractStore
-from db import AsyncMysqlDB
 from tools import utils
-from var import crawler_type_var, media_crawler_db_var
+from var import crawler_type_var
 
 
 class XhsCsvStoreImplement(AbstractStore):