feat: 微博爬虫帖子搜索完成

This commit is contained in:
Relakkes
2023-12-24 17:57:48 +08:00
parent 9785abba5d
commit c5b64fdbf5
14 changed files with 671 additions and 19 deletions
+5 -5
View File
@@ -8,12 +8,10 @@ import functools
import sys
from typing import Optional
import redis
from playwright.async_api import BrowserContext, Page
from tenacity import (RetryError, retry, retry_if_result, stop_after_attempt,
wait_fixed)
import config
from base.base_crawler import AbstractLogin
from tools import utils
@@ -33,7 +31,7 @@ class BilibiliLogin(AbstractLogin):
self.cookie_str = cookie_str
async def begin(self):
"""Start login xiaohongshu"""
"""Start login bilibili"""
utils.logger.info("[BilibiliLogin.begin] Begin login Bilibili ...")
if self.login_type == "qrcode":
await self.login_by_qrcode()
@@ -42,7 +40,8 @@ class BilibiliLogin(AbstractLogin):
elif self.login_type == "cookie":
await self.login_by_cookies()
else:
raise ValueError("[BilibiliLogin.begin] Invalid Login Type Currently only supported qrcode or phone or cookie ...")
raise ValueError(
"[BilibiliLogin.begin] Invalid Login Type Currently only supported qrcode or phone or cookie ...")
@retry(stop=stop_after_attempt(20), wait=wait_fixed(1), retry=retry_if_result(lambda value: value is False))
async def check_login_state(self) -> bool:
@@ -89,7 +88,8 @@ class BilibiliLogin(AbstractLogin):
sys.exit()
wait_redirect_seconds = 5
utils.logger.info(f"[BilibiliLogin.login_by_qrcode] Login successful then wait for {wait_redirect_seconds} seconds redirect ...")
utils.logger.info(
f"[BilibiliLogin.login_by_qrcode] Login successful then wait for {wait_redirect_seconds} seconds redirect ...")
await asyncio.sleep(wait_redirect_seconds)
async def login_by_mobile(self):