244 lines
8.6 KiB
Python
244 lines
8.6 KiB
Python
|
#!/usr/bin/env python3
|
|||
|
|
# -*- coding: utf-8 -*-
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
XHS Spider Adapter
|
|||
|
|
小红书爬虫适配器
|
|||
|
|
|
|||
|
|
作为core模块与xhs_spider模块的桥梁，提供统一的接口
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import sys
|
|||
|
|
import os
|
|||
|
|
from typing import Dict, List, Optional, Any
|
|||
|
|
from pathlib import Path
|
|||
|
|
import logging
|
|||
|
|
|
|||
|
|
from .models import XHSNote, XHSSearchResult, SearchConfig
|
|||
|
|
|
|||
|
|
logger = logging.getLogger(__name__)
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
from .xhs_spider import Data_Spider, XHS_Apis
|
|||
|
|
XHS_AVAILABLE = True
|
|||
|
|
except ImportError as e:
|
|||
|
|
logger.error(f"XHS Spider模块导入失败: {e}")
|
|||
|
|
XHS_AVAILABLE = False
|
|||
|
|
|
|||
|
|
class XHSAdapter:
|
|||
|
|
"""小红书爬虫适配器"""
|
|||
|
|
|
|||
|
|
def __init__(self, cookies_str: Optional[str] = None):
|
|||
|
|
"""
|
|||
|
|
初始化适配器
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
cookies_str: Cookie字符串
|
|||
|
|
"""
|
|||
|
|
self.cookies_str = cookies_str or ""
|
|||
|
|
self.spider: Optional[Data_Spider] = None
|
|||
|
|
self.available = XHS_AVAILABLE
|
|||
|
|
|
|||
|
|
if not self.available:
|
|||
|
|
raise ImportError("XHS Spider模块不可用，请检查模块导入")
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
self.spider = Data_Spider()
|
|||
|
|
logger.info("XHS Spider适配器初始化成功")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"XHS Spider初始化失败: {e}")
|
|||
|
|
raise
|
|||
|
|
|
|||
|
|
def search_notes(self, config: SearchConfig) -> XHSSearchResult:
|
|||
|
|
"""
|
|||
|
|
搜索小红书笔记
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
config: 搜索配置
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
XHSSearchResult: 搜索结果
|
|||
|
|
"""
|
|||
|
|
if not self.cookies_str:
|
|||
|
|
logger.warning("未设置cookies，搜索可能失败")
|
|||
|
|
|
|||
|
|
if not self.spider:
|
|||
|
|
raise RuntimeError("Spider未初始化")
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
# 调用XHS_Apis的搜索方法
|
|||
|
|
success, msg, notes = self.spider.xhs_apis.search_some_note(
|
|||
|
|
query=config.keyword,
|
|||
|
|
require_num=config.max_notes,
|
|||
|
|
cookies_str=self.cookies_str,
|
|||
|
|
sort_type_choice=config.sort_type,
|
|||
|
|
note_type=config.note_type
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if not success:
|
|||
|
|
logger.error(f"搜索失败: {msg}")
|
|||
|
|
return XHSSearchResult(
|
|||
|
|
keyword=config.keyword,
|
|||
|
|
notes=[],
|
|||
|
|
total_count=0,
|
|||
|
|
success=False,
|
|||
|
|
error_message=str(msg)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# 转换为XHSNote对象
|
|||
|
|
xhs_notes = []
|
|||
|
|
for note_data in notes:
|
|||
|
|
if note_data.get('model_type') == 'note':
|
|||
|
|
note_card = note_data.get('note_card', {})
|
|||
|
|
user_info = note_card.get('user', {})
|
|||
|
|
interact_info = note_card.get('interact_info', {})
|
|||
|
|
|
|||
|
|
# 提取图片URL列表
|
|||
|
|
image_urls = []
|
|||
|
|
for img in note_card.get('image_list', []):
|
|||
|
|
for info in img.get('info_list', []):
|
|||
|
|
if info.get('image_scene') == 'WB_DFT':
|
|||
|
|
image_urls.append(info.get('url', ''))
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
# 处理点赞、评论、分享数（可能是字符串格式）
|
|||
|
|
def parse_count(count_str):
|
|||
|
|
if isinstance(count_str, str):
|
|||
|
|
try:
|
|||
|
|
return int(count_str)
|
|||
|
|
except ValueError:
|
|||
|
|
return 0
|
|||
|
|
return int(count_str) if count_str else 0
|
|||
|
|
|
|||
|
|
note = XHSNote(
|
|||
|
|
note_id=note_data.get('id', ''),
|
|||
|
|
title=note_card.get('display_title', ''),
|
|||
|
|
content=note_card.get('desc', ''), # 搜索结果中可能没有完整内容
|
|||
|
|
author=user_info.get('nickname', ''),
|
|||
|
|
author_id=user_info.get('user_id', ''),
|
|||
|
|
tags=note_card.get('tag_list', []),
|
|||
|
|
images=image_urls,
|
|||
|
|
videos=note_card.get('video', []),
|
|||
|
|
likes=parse_count(interact_info.get('liked_count', 0)),
|
|||
|
|
comments=parse_count(interact_info.get('comment_count', 0)),
|
|||
|
|
shares=parse_count(interact_info.get('shared_count', 0)),
|
|||
|
|
created_time=note_card.get('time', ''),
|
|||
|
|
note_url=f"https://www.xiaohongshu.com/explore/{note_data.get('id', '')}?xsec_token={note_data.get('xsec_token', '')}"
|
|||
|
|
)
|
|||
|
|
xhs_notes.append(note)
|
|||
|
|
|
|||
|
|
return XHSSearchResult(
|
|||
|
|
keyword=config.keyword,
|
|||
|
|
notes=xhs_notes,
|
|||
|
|
total_count=len(xhs_notes),
|
|||
|
|
success=True
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"搜索异常: {e}")
|
|||
|
|
return XHSSearchResult(
|
|||
|
|
keyword=config.keyword,
|
|||
|
|
notes=[],
|
|||
|
|
total_count=0,
|
|||
|
|
success=False,
|
|||
|
|
error_message=str(e)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def search_notes_with_content(self, config: SearchConfig, fetch_content: bool = True) -> XHSSearchResult:
|
|||
|
|
"""
|
|||
|
|
搜索小红书笔记并可选择获取详细内容
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
config: 搜索配置
|
|||
|
|
fetch_content: 是否获取详细内容
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
XHSSearchResult: 搜索结果
|
|||
|
|
"""
|
|||
|
|
# 首先进行基本搜索
|
|||
|
|
search_result = self.search_notes(config)
|
|||
|
|
|
|||
|
|
if not search_result.success or not fetch_content:
|
|||
|
|
return search_result
|
|||
|
|
|
|||
|
|
# 为每个笔记获取详细内容
|
|||
|
|
enhanced_notes = []
|
|||
|
|
for note in search_result.notes:
|
|||
|
|
try:
|
|||
|
|
detailed_note = self.get_note_info(note.note_url)
|
|||
|
|
if detailed_note:
|
|||
|
|
enhanced_notes.append(detailed_note)
|
|||
|
|
else:
|
|||
|
|
# 如果获取详情失败，使用原始笔记数据
|
|||
|
|
enhanced_notes.append(note)
|
|||
|
|
logger.warning(f"获取笔记详情失败，使用基本信息: {note.note_id}")
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"获取笔记详情异常: {e}")
|
|||
|
|
enhanced_notes.append(note)
|
|||
|
|
|
|||
|
|
return XHSSearchResult(
|
|||
|
|
keyword=config.keyword,
|
|||
|
|
notes=enhanced_notes,
|
|||
|
|
total_count=len(enhanced_notes),
|
|||
|
|
success=True
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def get_note_info(self, note_url: str) -> Optional[XHSNote]:
|
|||
|
|
"""
|
|||
|
|
获取笔记详细信息
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
note_url: 笔记URL
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
XHSNote: 笔记信息
|
|||
|
|
"""
|
|||
|
|
if not self.cookies_str:
|
|||
|
|
logger.warning("未设置cookies，获取笔记信息可能失败")
|
|||
|
|
|
|||
|
|
if not self.spider:
|
|||
|
|
raise RuntimeError("Spider未初始化")
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
success, msg, note_info = self.spider.spider_note(
|
|||
|
|
note_url=note_url,
|
|||
|
|
cookies_str=self.cookies_str
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if not success or not note_info:
|
|||
|
|
logger.error(f"获取笔记信息失败: {msg}")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
# 转换为XHSNote对象
|
|||
|
|
note = XHSNote(
|
|||
|
|
note_id=note_info.get('note_id', ''),
|
|||
|
|
title=note_info.get('title', ''),
|
|||
|
|
content=note_info.get('desc', ''),
|
|||
|
|
author=note_info.get('user', {}).get('nickname', ''),
|
|||
|
|
author_id=note_info.get('user', {}).get('user_id', ''),
|
|||
|
|
tags=note_info.get('tag_list', []),
|
|||
|
|
images=note_info.get('image_list', []),
|
|||
|
|
videos=note_info.get('video', []),
|
|||
|
|
likes=note_info.get('interact_info', {}).get('liked_count', 0),
|
|||
|
|
comments=note_info.get('interact_info', {}).get('comment_count', 0),
|
|||
|
|
shares=note_info.get('interact_info', {}).get('share_count', 0),
|
|||
|
|
created_time=note_info.get('time', ''),
|
|||
|
|
note_url=note_url
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return note
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"获取笔记信息异常: {e}")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
def is_available(self) -> bool:
|
|||
|
|
"""检查适配器是否可用"""
|
|||
|
|
return self.available and self.spider is not None
|
|||
|
|
|
|||
|
|
def set_cookies(self, cookies_str: str):
|
|||
|
|
"""设置cookies"""
|
|||
|
|
self.cookies_str = cookies_str
|
|||
|
|
logger.info("Cookies已更新")
|