337 lines
12 KiB
Python
337 lines
12 KiB
Python
|
"""
|
|||
|
|
浏览器管理工具
|
|||
|
|
基于Playwright的浏览器实例管理和反检测脚本注入。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import asyncio
|
|||
|
|
import os
|
|||
|
|
from pathlib import Path
|
|||
|
|
from typing import Optional, Dict, Any, List
|
|||
|
|
from playwright.async_api import Browser, BrowserContext, Page, Playwright, async_playwright
|
|||
|
|
|
|||
|
|
from ..config.settings import settings
|
|||
|
|
from .exceptions import BrowserError, TimeoutError
|
|||
|
|
from .logger import get_logger
|
|||
|
|
|
|||
|
|
logger = get_logger(__name__)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class BrowserManager:
|
|||
|
|
"""浏览器实例管理器"""
|
|||
|
|
|
|||
|
|
def __init__(self):
|
|||
|
|
self.playwright: Optional[Playwright] = None
|
|||
|
|
self.browsers: Dict[str, Browser] = {}
|
|||
|
|
self.contexts: Dict[str, BrowserContext] = {}
|
|||
|
|
self.stealth_script: Optional[str] = None
|
|||
|
|
self._lock = asyncio.Lock()
|
|||
|
|
|
|||
|
|
async def initialize(self):
|
|||
|
|
"""初始化Playwright"""
|
|||
|
|
async with self._lock:
|
|||
|
|
if self.playwright is None:
|
|||
|
|
self.playwright = await async_playwright().start()
|
|||
|
|
await self._load_stealth_script()
|
|||
|
|
logger.info("浏览器管理器初始化完成")
|
|||
|
|
|
|||
|
|
async def _load_stealth_script(self):
|
|||
|
|
"""加载反检测脚本"""
|
|||
|
|
try:
|
|||
|
|
script_path = Path(__file__).parent.parent / "assets" / "stealth.min.js"
|
|||
|
|
if script_path.exists():
|
|||
|
|
with open(script_path, 'r', encoding='utf-8') as f:
|
|||
|
|
self.stealth_script = f.read()
|
|||
|
|
logger.debug("反检测脚本加载成功")
|
|||
|
|
else:
|
|||
|
|
# 如果没有本地脚本，使用内置的基本脚本
|
|||
|
|
self.stealth_script = self._get_basic_stealth_script()
|
|||
|
|
logger.debug("使用内置反检测脚本")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.warning(f"反检测脚本加载失败: {e}")
|
|||
|
|
self.stealth_script = self._get_basic_stealth_script()
|
|||
|
|
|
|||
|
|
def _get_basic_stealth_script(self) -> str:
|
|||
|
|
"""获取基本的反检测脚本"""
|
|||
|
|
return """
|
|||
|
|
// 基本反检测脚本
|
|||
|
|
(() => {
|
|||
|
|
// 移除webdriver属性
|
|||
|
|
Object.defineProperty(navigator, 'webdriver', {
|
|||
|
|
get: () => undefined,
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// 修改plugins
|
|||
|
|
Object.defineProperty(navigator, 'plugins', {
|
|||
|
|
get: () => [1, 2, 3, 4, 5],
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// 修改languages
|
|||
|
|
Object.defineProperty(navigator, 'languages', {
|
|||
|
|
get: () => ['zh-CN', 'zh', 'en'],
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// 添加chrome属性
|
|||
|
|
window.chrome = {
|
|||
|
|
runtime: {},
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
// 重写permission query
|
|||
|
|
const originalQuery = window.navigator.permissions.query;
|
|||
|
|
window.navigator.permissions.query = (parameters) => (
|
|||
|
|
parameters.name === 'notifications' ?
|
|||
|
|
Promise.resolve({ state: Notification.permission }) :
|
|||
|
|
originalQuery(parameters)
|
|||
|
|
);
|
|||
|
|
})();
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
async def get_browser(
|
|||
|
|
self,
|
|||
|
|
headless: Optional[bool] = None,
|
|||
|
|
user_data_dir: Optional[str] = None,
|
|||
|
|
proxy: Optional[Dict[str, Any]] = None
|
|||
|
|
) -> Browser:
|
|||
|
|
"""获取或创建浏览器实例"""
|
|||
|
|
await self.initialize()
|
|||
|
|
|
|||
|
|
headless = headless if headless is not None else settings.browser.headless
|
|||
|
|
proxy = proxy or ({"server": settings.browser.proxy} if settings.browser.proxy else None)
|
|||
|
|
|
|||
|
|
# 生成浏览器标识
|
|||
|
|
browser_id = f"headless_{headless}_{user_data_dir or 'default'}"
|
|||
|
|
|
|||
|
|
async with self._lock:
|
|||
|
|
if browser_id not in self.browsers or self.browsers[browser_id].is_connected() is False:
|
|||
|
|
browser_args = self._get_browser_args()
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
browser = await self.playwright.chromium.launch(
|
|||
|
|
headless=headless,
|
|||
|
|
args=browser_args,
|
|||
|
|
proxy=proxy,
|
|||
|
|
executable_path=settings.browser.executable_path
|
|||
|
|
)
|
|||
|
|
self.browsers[browser_id] = browser
|
|||
|
|
logger.info(f"浏览器实例创建成功: {browser_id}")
|
|||
|
|
except Exception as e:
|
|||
|
|
raise BrowserError(f"浏览器启动失败: {e}")
|
|||
|
|
|
|||
|
|
return self.browsers[browser_id]
|
|||
|
|
|
|||
|
|
def _get_browser_args(self) -> List[str]:
|
|||
|
|
"""获取浏览器启动参数"""
|
|||
|
|
args = [
|
|||
|
|
'--no-sandbox',
|
|||
|
|
'--disable-setuid-sandbox',
|
|||
|
|
'--disable-infobars',
|
|||
|
|
'--window-position=0,0',
|
|||
|
|
'--ignore-certifcate-errors',
|
|||
|
|
'--ignore-certifcate-errors-spki-list',
|
|||
|
|
'--disable-blink-features=AutomationControlled',
|
|||
|
|
'--disable-features=VizDisplayCompositor',
|
|||
|
|
'--disable-extensions',
|
|||
|
|
'--disable-plugins',
|
|||
|
|
'--disable-images', # 可以根据需要开启
|
|||
|
|
'--disable-javascript', # 不能关闭，需要JS执行
|
|||
|
|
'--disable-default-apps',
|
|||
|
|
'--disable-background-timer-throttling',
|
|||
|
|
'--disable-backgrounding-occluded-windows',
|
|||
|
|
'--disable-renderer-backgrounding',
|
|||
|
|
'--disable-background-networking',
|
|||
|
|
'--disable-features=TranslateUI',
|
|||
|
|
'--disable-ipc-flooding-protection',
|
|||
|
|
'--enable-automation',
|
|||
|
|
'--password-store=basic',
|
|||
|
|
'--use-mock-keychain',
|
|||
|
|
'--no-first-run',
|
|||
|
|
'--no-default-browser-check',
|
|||
|
|
'--disable-fre',
|
|||
|
|
'--disable-features=VizDisplayCompositor'
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
if not settings.browser.headless:
|
|||
|
|
args.extend([
|
|||
|
|
'--start-maximized',
|
|||
|
|
'--disable-web-security',
|
|||
|
|
'--disable-features=VizDisplayCompositor'
|
|||
|
|
])
|
|||
|
|
|
|||
|
|
return args
|
|||
|
|
|
|||
|
|
async def get_context(
|
|||
|
|
self,
|
|||
|
|
browser: Optional[Browser] = None,
|
|||
|
|
user_data_dir: Optional[str] = None,
|
|||
|
|
viewport: Optional[Dict[str, int]] = None,
|
|||
|
|
user_agent: Optional[str] = None
|
|||
|
|
) -> BrowserContext:
|
|||
|
|
"""获取或创建浏览器上下文"""
|
|||
|
|
if browser is None:
|
|||
|
|
browser = await self.get_browser()
|
|||
|
|
|
|||
|
|
context_id = f"{browser}_{user_data_dir or 'default'}"
|
|||
|
|
|
|||
|
|
async with self._lock:
|
|||
|
|
if context_id not in self.contexts:
|
|||
|
|
context_options = {
|
|||
|
|
'viewport': viewport or {
|
|||
|
|
'width': settings.browser.viewport_width,
|
|||
|
|
'height': settings.browser.viewport_height
|
|||
|
|
},
|
|||
|
|
'locale': settings.browser.locale,
|
|||
|
|
'timezone_id': settings.browser.timezone,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if user_data_dir:
|
|||
|
|
context_options['user_data_dir'] = user_data_dir
|
|||
|
|
|
|||
|
|
if user_agent or settings.browser.user_agent:
|
|||
|
|
context_options['user_agent'] = user_agent or settings.browser.user_agent
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
context = await browser.new_context(**context_options)
|
|||
|
|
self.contexts[context_id] = context
|
|||
|
|
logger.debug(f"浏览器上下文创建成功: {context_id}")
|
|||
|
|
except Exception as e:
|
|||
|
|
raise BrowserError(f"浏览器上下文创建失败: {e}")
|
|||
|
|
|
|||
|
|
return self.contexts[context_id]
|
|||
|
|
|
|||
|
|
async def get_page(
|
|||
|
|
self,
|
|||
|
|
context: Optional[BrowserContext] = None,
|
|||
|
|
browser: Optional[Browser] = None
|
|||
|
|
) -> Page:
|
|||
|
|
"""创建新页面"""
|
|||
|
|
if context is None:
|
|||
|
|
context = await self.get_context(browser)
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
page = await context.new_page()
|
|||
|
|
|
|||
|
|
# 设置默认超时
|
|||
|
|
page.set_default_timeout(settings.browser.timeout)
|
|||
|
|
|
|||
|
|
# 注入反检测脚本
|
|||
|
|
if self.stealth_script:
|
|||
|
|
await page.add_init_script(self.stealth_script)
|
|||
|
|
|
|||
|
|
# 监听页面错误
|
|||
|
|
page.on("pageerror", self._handle_page_error)
|
|||
|
|
page.on("requestfailed", self._handle_request_failed)
|
|||
|
|
|
|||
|
|
logger.debug("新页面创建成功")
|
|||
|
|
return page
|
|||
|
|
except Exception as e:
|
|||
|
|
raise BrowserError(f"页面创建失败: {e}")
|
|||
|
|
|
|||
|
|
async def _handle_page_error(self, error):
|
|||
|
|
"""处理页面错误"""
|
|||
|
|
logger.warning(f"页面错误: {error}")
|
|||
|
|
|
|||
|
|
async def _handle_request_failed(self, request):
|
|||
|
|
"""处理请求失败"""
|
|||
|
|
url = request.url
|
|||
|
|
failure = request.failure
|
|||
|
|
logger.debug(f"请求失败: {url} - {failure}")
|
|||
|
|
|
|||
|
|
async def inject_stealth_script(self, page: Page):
|
|||
|
|
"""向页面注入反检测脚本"""
|
|||
|
|
if self.stealth_script:
|
|||
|
|
try:
|
|||
|
|
await page.evaluate(self.stealth_script)
|
|||
|
|
logger.debug("反检测脚本注入成功")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.warning(f"反检测脚本注入失败: {e}")
|
|||
|
|
|
|||
|
|
async def take_screenshot(self, page: Page, file_path: str, full_page: bool = True):
|
|||
|
|
"""截取页面截图"""
|
|||
|
|
try:
|
|||
|
|
await page.screenshot(path=file_path, full_page=full_page)
|
|||
|
|
logger.info(f"页面截图保存成功: {file_path}")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"页面截图失败: {e}")
|
|||
|
|
|
|||
|
|
async def wait_for_network_idle(self, page: Page, timeout: float = 5000):
|
|||
|
|
"""等待网络空闲"""
|
|||
|
|
try:
|
|||
|
|
await page.wait_for_load_state("networkidle", timeout=timeout)
|
|||
|
|
except Exception as e:
|
|||
|
|
raise TimeoutError(f"等待网络空闲超时: {timeout}ms", timeout=timeout)
|
|||
|
|
|
|||
|
|
async def close_page(self, page: Page):
|
|||
|
|
"""关闭页面"""
|
|||
|
|
try:
|
|||
|
|
await page.close()
|
|||
|
|
logger.debug("页面关闭成功")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.warning(f"页面关闭失败: {e}")
|
|||
|
|
|
|||
|
|
async def close_context(self, context: BrowserContext):
|
|||
|
|
"""关闭浏览器上下文"""
|
|||
|
|
try:
|
|||
|
|
# 从缓存中移除
|
|||
|
|
context_id = None
|
|||
|
|
for cid, ctx in self.contexts.items():
|
|||
|
|
if ctx == context:
|
|||
|
|
context_id = cid
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
if context_id:
|
|||
|
|
del self.contexts[context_id]
|
|||
|
|
|
|||
|
|
await context.close()
|
|||
|
|
logger.debug("浏览器上下文关闭成功")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.warning(f"浏览器上下文关闭失败: {e}")
|
|||
|
|
|
|||
|
|
async def close_browser(self, browser: Browser):
|
|||
|
|
"""关闭浏览器"""
|
|||
|
|
try:
|
|||
|
|
# 从缓存中移除
|
|||
|
|
browser_id = None
|
|||
|
|
for bid, br in self.browsers.items():
|
|||
|
|
if br == browser:
|
|||
|
|
browser_id = bid
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
if browser_id:
|
|||
|
|
del self.browsers[browser_id]
|
|||
|
|
|
|||
|
|
await browser.close()
|
|||
|
|
logger.debug("浏览器关闭成功")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.warning(f"浏览器关闭失败: {e}")
|
|||
|
|
|
|||
|
|
async def cleanup(self):
|
|||
|
|
"""清理所有资源"""
|
|||
|
|
async with self._lock:
|
|||
|
|
# 关闭所有上下文
|
|||
|
|
for context in list(self.contexts.values()):
|
|||
|
|
try:
|
|||
|
|
await context.close()
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
self.contexts.clear()
|
|||
|
|
|
|||
|
|
# 关闭所有浏览器
|
|||
|
|
for browser in list(self.browsers.values()):
|
|||
|
|
try:
|
|||
|
|
await browser.close()
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
self.browsers.clear()
|
|||
|
|
|
|||
|
|
# 关闭Playwright
|
|||
|
|
if self.playwright:
|
|||
|
|
try:
|
|||
|
|
await self.playwright.stop()
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
self.playwright = None
|
|||
|
|
|
|||
|
|
logger.info("浏览器管理器资源清理完成")
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 全局浏览器管理器实例
|
|||
|
|
browser_manager = BrowserManager()
|