fork/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-10-07 21:48:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: xhs增加并发控制参数

Browse Source
This commit is contained in:
Relakkes
2023-07-15 22:25:56 +08:00
parent 2398a17e21
commit e5bdc63323
8 changed files with 56 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media_platform/douyin/core.py
+2 -3
View File
@@ -1,6 +1,5 @@
import os
import asyncio
import logging
from asyncio import Task
from argparse import Namespace
from typing import Optional, List, Dict, Tuple
@@ -78,7 +77,7 @@ class DouYinCrawler(AbstractCrawler):
try:
posts_res = await self.dy_client.search_info_by_keyword(keyword=keyword, offset=page * 10)
except DataFetchError:
logging.error(f"search douyin keyword: {keyword} failed")
utils.logger.error(f"search douyin keyword: {keyword} failed")
break
page += 1
max_note_len -= 10
@@ -108,7 +107,7 @@ class DouYinCrawler(AbstractCrawler):
)
utils.logger.info(f"aweme_id: {aweme_id} comments have all been obtained completed ...")
except DataFetchError as e:
logging.error(f"aweme_id: {aweme_id} get comments failed, error: {e}")
utils.logger.error(f"aweme_id: {aweme_id} get comments failed, error: {e}")
def create_proxy_info(self) -> Tuple[Optional[str], Optional[Dict], Optional[str]]:
"""Create proxy info for playwright and httpx"""
media_platform/douyin/login.py
+3 -4
View File
@@ -1,5 +1,4 @@
import sys
import logging
import asyncio
import functools
@@ -89,7 +88,7 @@ class DouYinLogin(AbstractLogin):
# check dialog box is auto popup and wait for 10 seconds
await self.context_page.wait_for_selector(dialog_selector, timeout=1000 * 10)
except Exception as e:
logging.error(f"login dialog box does not pop up automatically, error: {e}")
utils.logger.error(f"login dialog box does not pop up automatically, error: {e}")
utils.logger.info("login dialog box does not pop up automatically, we will manually click the login button")
login_button_ele = self.context_page.locator("xpath=//p[text() = '登录']")
await login_button_ele.click()
@@ -163,7 +162,7 @@ class DouYinLogin(AbstractLogin):
slider_verify_success = False
while not slider_verify_success:
if max_slider_try_times <= 0:
logging.error("slider verify failed ...")
utils.logger.error("slider verify failed ...")
sys.exit()
try:
await self.move_slider(back_selector, gap_selector, move_step, slider_level)
@@ -182,7 +181,7 @@ class DouYinLogin(AbstractLogin):
utils.logger.info("slider verify success ...")
slider_verify_success = True
except Exception as e:
logging.error(f"slider verify failed, error: {e}")
utils.logger.error(f"slider verify failed, error: {e}")
await asyncio.sleep(1)
max_slider_try_times -= 1
utils.logger.info(f"remaining slider try times: {max_slider_try_times}")