feat: 支持数据保存到CSV中

media_platform/douyin/client.py

@@ -8,6 +8,7 @@ import httpx
 from playwright.async_api import BrowserContext, Page
 
 from tools import utils
+from var import request_keyword_var
 
 from .exception import *
 from .field import *
@@ -142,7 +143,7 @@ class DOUYINClient:
         del headers["Origin"]
         return await self.get("/aweme/v1/web/aweme/detail/", params, headers)
 
-    async def get_aweme_comments(self, aweme_id: str, cursor: int = 0, keywords: str = ""):
+    async def get_aweme_comments(self, aweme_id: str, cursor: int = 0):
         """get note comments
 
         """
@@ -153,6 +154,7 @@ class DOUYINClient:
             "count": 20,
             "item_type": 0
         }
+        keywords = request_keyword_var.get()
         referer_url = "https://www.douyin.com/search/" + keywords + '?aid=3a3cec5a-9e27-4040-b6aa-ef548c2c1138&publish_time=0&sort_type=0&source=search_history&type=general'
         headers = copy.copy(self.headers)
         headers["Referer"] = urllib.parse.quote(referer_url, safe=':/')
@@ -164,7 +166,6 @@ class DOUYINClient:
             crawl_interval: float = 1.0,
             is_fetch_sub_comments=False,
             callback: Optional[Callable] = None,
-            keywords: str = ""
     ):
         """
         get note all comments include sub comments
@@ -172,14 +173,13 @@ class DOUYINClient:
         :param crawl_interval:
         :param is_fetch_sub_comments:
         :param callback:
-        :param keywords:
         :return:
         """
         result = []
         comments_has_more = 1
         comments_cursor = 0
         while comments_has_more:
-            comments_res = await self.get_aweme_comments(aweme_id, comments_cursor, keywords)
+            comments_res = await self.get_aweme_comments(aweme_id, comments_cursor)
             comments_has_more = comments_res.get("has_more", 0)
             comments_cursor = comments_res.get("cursor", comments_cursor + 20)
             comments = comments_res.get("comments")

media_platform/douyin/core.py

@@ -11,6 +11,7 @@ from base.base_crawler import AbstractCrawler
 from base.proxy_account_pool import AccountPool
 from models import douyin
 from tools import utils
+from var import request_keyword_var
 
 from .client import DOUYINClient
 from .exception import DataFetchError
@@ -70,14 +71,15 @@ class DouYinCrawler(AbstractCrawler):
     async def search(self) -> None:
         utils.logger.info("Begin search douyin keywords")
         for keyword in config.KEYWORDS.split(","):
+            request_keyword_var.set(keyword)
             utils.logger.info(f"Current keyword: {keyword}")
             aweme_list: List[str] = []
-            dy_limite_count = 10  # douyin fixed limit page 10
+            dy_limit_count = 10  # douyin fixed limit page 10
             page = 0
-            while (page + 1) * dy_limite_count <= config.CRAWLER_MAX_NOTES_COUNT:
+            while (page + 1) * dy_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:
                 try:
                     posts_res = await self.dy_client.search_info_by_keyword(keyword=keyword,
-                                                                            offset=page * dy_limite_count)
+                                                                            offset=page * dy_limit_count)
                 except DataFetchError:
                     utils.logger.error(f"search douyin keyword: {keyword} failed")
                     break
@@ -91,23 +93,22 @@ class DouYinCrawler(AbstractCrawler):
                     aweme_list.append(aweme_info.get("aweme_id", ""))
                     await douyin.update_douyin_aweme(aweme_item=aweme_info)
             utils.logger.info(f"keyword:{keyword}, aweme_list:{aweme_list}")
-            await self.batch_get_note_comments(aweme_list, keyword)
+            await self.batch_get_note_comments(aweme_list)
 
-    async def batch_get_note_comments(self, aweme_list: List[str], keywords: str) -> None:
+    async def batch_get_note_comments(self, aweme_list: List[str]) -> None:
         task_list: List[Task] = []
         semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
         for aweme_id in aweme_list:
-            task = asyncio.create_task(self.get_comments(aweme_id, semaphore, keywords), name=aweme_id)
+            task = asyncio.create_task(self.get_comments(aweme_id, semaphore), name=aweme_id)
             task_list.append(task)
         await asyncio.wait(task_list)
 
-    async def get_comments(self, aweme_id: str, semaphore: asyncio.Semaphore, keywords: str) -> None:
+    async def get_comments(self, aweme_id: str, semaphore: asyncio.Semaphore) -> None:
         async with semaphore:
             try:
                 await self.dy_client.get_aweme_all_comments(
                     aweme_id=aweme_id,
                     callback=douyin.batch_update_dy_aweme_comments,
-                    keywords=keywords
                 )
                 utils.logger.info(f"aweme_id: {aweme_id} comments have all been obtained completed ...")
             except DataFetchError as e:

media_platform/xhs/core.py

@@ -12,6 +12,7 @@ from base.base_crawler import AbstractCrawler
 from base.proxy_account_pool import AccountPool
 from models import xiaohongshu as xhs_model
 from tools import utils
+from var import request_keyword_var
 
 from .client import XHSClient
 from .exception import DataFetchError
@@ -81,6 +82,8 @@ class XiaoHongShuCrawler(AbstractCrawler):
         utils.logger.info("Begin search xiaohongshu keywords")
         xhs_limit_count = 20  # xhs limit page fixed value
         for keyword in config.KEYWORDS.split(","):
+            # set keyword to context var
+            request_keyword_var.set(keyword)
             utils.logger.info(f"Current search keyword: {keyword}")
             page = 1
             while page * xhs_limit_count <= config.CRAWLER_MAX_NOTES_COUNT: