#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ 内容整合服务 提供小红书内容和文档整合的API服务 """ import logging import uuid from typing import Dict, List, Optional, Any, Tuple from datetime import datetime from pathlib import Path from core.models import ( XHSNote, XHSSearchResult, DocumentContent, IntegratedContent, SearchConfig, IntegrationConfig, IntegrationResult, ProcessingStats ) from core.xhs_adapter import XHSAdapter from core.document_adapter import DocumentAdapter from core.cookie_manager import CookieManager from api.models.integration import ( XHSSearchRequest, IntegrationRequest, BatchIntegrationRequest, CookieManagementRequest, ExportRequest, XHSNoteResponse, XHSSearchResponse, DocumentContentResponse, IntegratedContentResponse, IntegrationResponse, CookieStatsResponse, ServiceStatusResponse, HealthCheckResponse, TaskSummaryResponse, ValidationResponse, ApiResponse, BatchSearchRequest, BatchSearchResponse, KeywordSearchResult, NoteInfo ) logger = logging.getLogger(__name__) class IntegrationService: """内容整合服务""" def __init__(self, config_manager=None, output_manager=None): """ 初始化整合服务 Args: config_manager: 配置管理器 output_manager: 输出管理器 """ self.config_manager = config_manager self.output_manager = output_manager # 初始化核心组件 self.cookie_manager = CookieManager() self.xhs_adapter = None self.document_adapter = DocumentAdapter() # 结果存储 self.integration_results: Dict[str, IntegrationResponse] = {} # 统计信息 self.stats = ProcessingStats() # 初始化XHS适配器 self._initialize_xhs_adapter() logger.info("整合服务初始化完成") def _initialize_xhs_adapter(self): """初始化XHS适配器""" try: # 获取可用的cookie cookie_string = self.cookie_manager.get_cookie_string() if cookie_string: self.xhs_adapter = XHSAdapter(cookie_string) logger.info("XHS适配器初始化成功") else: logger.warning("没有可用的Cookie，XHS适配器未初始化") except Exception as e: logger.error(f"XHS适配器初始化失败: {e}") self.xhs_adapter = None def search_xhs_notes(self, request: XHSSearchRequest) -> XHSSearchResponse: """搜索小红书笔记""" start_time = datetime.now() try: if not self.xhs_adapter: self._initialize_xhs_adapter() if not self.xhs_adapter: return XHSSearchResponse( keyword=request.keyword, notes=[], total_count=0, success=False, error_message="XHS适配器不可用，请检查Cookie配置", processing_time=0.0 ) # 创建搜索配置 config = SearchConfig( keyword=request.keyword, max_notes=request.max_notes, sort_type=request.sort_type, note_type=request.note_type, download_images=request.download_media, download_videos=request.download_media ) # 执行搜索 result = self.xhs_adapter.search_notes(config) # 转换为API响应格式 notes_response = [] for note in result.notes: note_response = XHSNoteResponse( note_id=note.note_id, title=note.title, content=note.content, author=note.author, author_id=note.author_id, tags=note.tags, images=note.images, videos=note.videos, likes=note.likes, comments=note.comments, shares=note.shares, created_time=note.created_time, note_url=note.note_url ) notes_response.append(note_response) processing_time = (datetime.now() - start_time).total_seconds() return XHSSearchResponse( keyword=result.keyword, notes=notes_response, total_count=result.total_count, success=result.success, error_message=result.error_message, processing_time=processing_time ) except Exception as e: logger.error(f"搜索小红书笔记失败: {e}") processing_time = (datetime.now() - start_time).total_seconds() return XHSSearchResponse( keyword=request.keyword, notes=[], total_count=0, success=False, error_message=str(e), processing_time=processing_time ) def integrate_content(self, request: IntegrationRequest) -> IntegrationResponse: """整合内容""" start_time = datetime.now() task_id = f"integration_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{str(uuid.uuid4())[:8]}" try: # 验证文档路径 validation_results = self.validate_documents(request.document_paths) invalid_docs = [path for path, valid in validation_results.items() if not valid] if invalid_docs: return IntegrationResponse( task_id=task_id, keyword=request.keyword, success=False, error_message=f"以下文档无效: {', '.join(invalid_docs)}", processing_time=0.0 ) # 搜索小红书内容 xhs_request = XHSSearchRequest( keyword=request.keyword, max_notes=request.max_notes, sort_type=request.sort_type, note_type=request.note_type, download_media=request.download_media ) xhs_result = self.search_xhs_notes(xhs_request) # 处理文档内容 document_result = self.document_adapter.integrate_documents(request.document_paths) # 转换文档内容为API响应格式 documents_response = [] for doc in document_result.documents: doc_response = DocumentContentResponse( file_path=doc.file_path, content=doc.content, file_type=doc.file_type, metadata=doc.metadata ) documents_response.append(doc_response) integrated_content_response = IntegratedContentResponse( documents=documents_response, integrated_text=document_result.integrated_text, summary=document_result.summary, key_topics=document_result.key_topics, total_length=document_result.total_length ) # 生成整合摘要 integrated_summary = self._generate_integrated_summary(xhs_result, document_result) processing_time = (datetime.now() - start_time).total_seconds() # 创建整合结果 integration_result = IntegrationResponse( task_id=task_id, keyword=request.keyword, success=True, xhs_content=xhs_result, document_content=integrated_content_response, integrated_summary=integrated_summary, processing_time=processing_time, created_time=datetime.now() ) # 存储结果 self.integration_results[task_id] = integration_result # 更新统计信息 self._update_stats(integration_result) return integration_result except Exception as e: logger.error(f"内容整合失败: {e}") processing_time = (datetime.now() - start_time).total_seconds() return IntegrationResponse( task_id=task_id, keyword=request.keyword, success=False, error_message=str(e), processing_time=processing_time, created_time=datetime.now() ) def batch_integrate_content(self, request: BatchIntegrationRequest) -> Dict[str, IntegrationResponse]: """批量整合内容""" logger.info(f"开始批量整合，共 {len(request.tasks)} 个任务") results = {} for i, task in enumerate(request.tasks): logger.info(f"处理任务 {i+1}/{len(request.tasks)}: {task.keyword}") result = self.integrate_content(task) results[task.keyword] = result success_count = sum(1 for r in results.values() if r.success) logger.info(f"批量整合完成，成功 {success_count}/{len(results)} 个任务") return results def get_integration_result(self, task_id: str) -> Optional[IntegrationResponse]: """获取整合结果""" return self.integration_results.get(task_id) def list_integration_results(self) -> List[TaskSummaryResponse]: """列出所有整合结果""" results = [] for task_id, result in self.integration_results.items(): summary = TaskSummaryResponse( task_id=task_id, keyword=result.keyword, success=result.success, processing_time=result.processing_time, created_time=result.created_time, notes_count=len(result.xhs_content.notes) if result.xhs_content else 0, documents_count=len(result.document_content.documents) if result.document_content else 0, content_preview=result.integrated_summary[:100] + "..." if len(result.integrated_summary) > 100 else result.integrated_summary ) results.append(summary) return sorted(results, key=lambda x: x.created_time, reverse=True) def export_result(self, request: ExportRequest) -> Optional[str]: """导出结果""" result = self.get_integration_result(request.task_id) if not result or not result.success: return None try: # 生成导出内容 export_content = self._generate_export_content(result, request.output_format) # 生成文件名 if request.filename: filename = request.filename else: keyword_safe = "".join(c for c in result.keyword if c.isalnum() or c in (' ', '-', '_')).strip() timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') filename = f"{keyword_safe}_{timestamp}.txt" # 保存文件 if self.output_manager: output_path = self.output_manager.save_content(export_content, filename) logger.info(f"导出成功: {output_path}") return str(output_path) else: # 默认保存到data/output目录 output_dir = Path("data/output") output_dir.mkdir(parents=True, exist_ok=True) output_path = output_dir / filename with open(output_path, 'w', encoding='utf-8') as f: f.write(export_content) logger.info(f"导出成功: {output_path}") return str(output_path) except Exception as e: logger.error(f"导出失败: {e}") return None def validate_documents(self, document_paths: List[str]) -> Dict[str, bool]: """验证文档""" validation_results = {} for doc_path in document_paths: try: path = Path(doc_path) if not path.exists(): validation_results[doc_path] = False continue # 检查文件类型 supported_formats = self.document_adapter.get_supported_formats() file_ext = path.suffix.lower() validation_results[doc_path] = file_ext in supported_formats except Exception as e: logger.error(f"验证文档失败 {doc_path}: {e}") validation_results[doc_path] = False return validation_results def add_cookie(self, request: CookieManagementRequest) -> bool: """添加Cookie""" try: success = self.cookie_manager.add_cookie( name=request.name, cookie_string=request.cookie_string, user_info=request.user_info ) if success: # 重新初始化XHS适配器 self._initialize_xhs_adapter() return success except Exception as e: logger.error(f"添加Cookie失败: {e}") return False def remove_cookie(self, cookie_name: str) -> bool: """删除Cookie""" try: success = self.cookie_manager.remove_cookie(cookie_name) if success: # 重新初始化XHS适配器 self._initialize_xhs_adapter() return success except Exception as e: logger.error(f"删除Cookie失败: {e}") return False def get_cookie_stats(self) -> CookieStatsResponse: """获取Cookie统计""" stats = self.cookie_manager.get_statistics() return CookieStatsResponse( total_cookies=stats.total_cookies, valid_cookies=stats.valid_cookies, invalid_cookies=stats.invalid_cookies, current_cookie=stats.current_cookie, cookie_details=stats.cookie_details ) def get_service_status(self) -> ServiceStatusResponse: """获取服务状态""" return ServiceStatusResponse( status="running", startup_time=datetime.now(), # 这里应该是实际的启动时间 output_path=str(Path("data/output").absolute()), xhs_service_available=self.xhs_adapter is not None and self.xhs_adapter.is_available(), document_service_available=self.document_adapter.is_available(), integration_stats={ "total_tasks": len(self.integration_results), "successful_tasks": sum(1 for r in self.integration_results.values() if r.success), "failed_tasks": sum(1 for r in self.integration_results.values() if not r.success) } ) def get_health_check(self) -> HealthCheckResponse: """获取健康检查""" services = { "xhs_service": self.xhs_adapter is not None and self.xhs_adapter.is_available(), "document_service": self.document_adapter.is_available(), "cookie_manager": len(self.cookie_manager.get_valid_cookies()) > 0 } all_healthy = all(services.values()) status = "healthy" if all_healthy else "degraded" return HealthCheckResponse( status=status, services=services ) def _generate_integrated_summary(self, xhs_result: XHSSearchResponse, document_result: IntegratedContent) -> str: """生成整合摘要""" summary_parts = [] # 小红书内容摘要 if xhs_result.success and xhs_result.notes: summary_parts.append(f"小红书搜索结果：找到 {len(xhs_result.notes)} 条相关笔记") # 统计点赞数最高的笔记 top_note = max(xhs_result.notes, key=lambda x: x.likes) summary_parts.append(f"最受欢迎的笔记：《{top_note.title}》(点赞 {top_note.likes})") # 文档内容摘要 if document_result.documents: summary_parts.append(f"文档处理结果：处理了 {len(document_result.documents)} 个文档") summary_parts.append(f"文档总长度：{document_result.total_length} 字符") if document_result.key_topics: summary_parts.append(f"关键主题：{', '.join(document_result.key_topics[:3])}") return "\n".join(summary_parts) def _generate_export_content(self, result: IntegrationResponse, output_format: str) -> str: """生成导出内容""" content_parts = [] # 添加标题 content_parts.append(f"# {result.keyword} - 内容整合报告") content_parts.append(f"生成时间: {result.created_time.strftime('%Y-%m-%d %H:%M:%S')}") content_parts.append(f"任务ID: {result.task_id}") content_parts.append("") # 添加摘要 content_parts.append("## 整合摘要") content_parts.append(result.integrated_summary) content_parts.append("") # 添加小红书内容 if result.xhs_content and result.xhs_content.notes: content_parts.append("## 小红书内容") content_parts.append(f"共找到 {len(result.xhs_content.notes)} 条笔记") content_parts.append("") for i, note in enumerate(result.xhs_content.notes[:5], 1): # 只显示前5条 content_parts.append(f"### {i}. {note.title}") content_parts.append(f"作者: {note.author}") content_parts.append(f"点赞: {note.likes} | 评论: {note.comments} | 分享: {note.shares}") content_parts.append(f"内容: {note.content[:200]}...") content_parts.append("") # 添加文档内容 if result.document_content: content_parts.append("## 文档内容") content_parts.append(f"处理了 {len(result.document_content.documents)} 个文档") content_parts.append("") content_parts.append("### 关键主题") content_parts.append(", ".join(result.document_content.key_topics)) content_parts.append("") content_parts.append("### 文档摘要") content_parts.append(result.document_content.summary) content_parts.append("") return "\n".join(content_parts) def _update_stats(self, result: IntegrationResponse): """更新统计信息""" self.stats.total_tasks += 1 self.stats.total_processing_time += result.processing_time if result.success: self.stats.successful_tasks += 1 if result.xhs_content: self.stats.total_notes_processed += len(result.xhs_content.notes) if result.document_content: self.stats.total_documents_processed += len(result.document_content.documents) else: self.stats.failed_tasks += 1 if self.stats.total_tasks > 0: self.stats.average_processing_time = self.stats.total_processing_time / self.stats.total_tasks self.stats.last_updated = datetime.now() def _parse_cookies(self, cookies_input) -> List[Dict[str, Any]]: """解析不同格式的Cookie输入""" import time from api.models.integration import CookieConfig parsed_cookies = [] if isinstance(cookies_input, str): # 方式3: 单一Cookie字符串 parsed_cookies.append({ "cookie_string": cookies_input, "name": f"cookie_{int(time.time())}", "priority": 1 }) elif isinstance(cookies_input, dict): # 方式2: 简单键值对，转换为cookie字符串 cookie_parts = [] for key, value in cookies_input.items(): cookie_parts.append(f"{key}={value}") cookie_string = "; ".join(cookie_parts) parsed_cookies.append({ "cookie_string": cookie_string, "name": f"dict_cookie_{int(time.time())}", "priority": 1 }) elif isinstance(cookies_input, list): # 方式1: 详细的Cookie配置列表 for cookie_config in cookies_input: if isinstance(cookie_config, CookieConfig): parsed_cookies.append({ "cookie_string": cookie_config.cookie_string, "name": cookie_config.name or f"cookie_{len(parsed_cookies)}", "priority": cookie_config.priority, "user_info": cookie_config.user_info }) elif isinstance(cookie_config, dict): parsed_cookies.append({ "cookie_string": cookie_config.get("cookie_string", ""), "name": cookie_config.get("name", f"cookie_{len(parsed_cookies)}"), "priority": cookie_config.get("priority", 1), "user_info": cookie_config.get("user_info") }) # 按优先级排序（高优先级在前） parsed_cookies.sort(key=lambda x: x.get("priority", 1), reverse=True) logger.info(f"解析Cookie配置: {len(parsed_cookies)} 个Cookie，优先级从高到低") for i, cookie in enumerate(parsed_cookies): logger.info(f" Cookie {i+1}: {cookie.get('name')} (优先级: {cookie.get('priority', 1)})") return parsed_cookies def _get_next_cookie(self, cookies_list: List[Dict[str, Any]], used_cookies: set, enable_rotation: bool) -> Optional[Dict[str, Any]]: """获取下一个可用的Cookie""" if not enable_rotation: # 不启用轮换，使用第一个Cookie return cookies_list[0] if cookies_list else None # 启用轮换，找到未使用的Cookie for cookie in cookies_list: cookie_id = cookie.get("name", "") if cookie_id not in used_cookies: used_cookies.add(cookie_id) return cookie # 如果所有Cookie都用过了，重置并使用第一个 if cookies_list: used_cookies.clear() first_cookie = cookies_list[0] used_cookies.add(first_cookie.get("name", "")) return first_cookie return None async def batch_search(self, request: 'BatchSearchRequest') -> 'BatchSearchResponse': """执行批量搜索""" import time import os import requests from datetime import datetime from api.models.integration import BatchSearchResponse, KeywordSearchResult, NoteInfo start_time = time.time() results = [] total_notes = 0 successful_searches = 0 failed_searches = 0 # 确保输出目录存在 os.makedirs(request.output_path, exist_ok=True) os.makedirs(request.image_storage_path, exist_ok=True) # 解析Cookie配置 cookies_list = self._parse_cookies(request.cookies) if not cookies_list: raise ValueError("未提供有效的Cookie配置") used_cookies = set() logger.info(f"开始批量搜索 {len(request.keywords)} 个关键词") logger.info(f"Cookie数量: {len(cookies_list)}") logger.info(f"Cookie轮换: {'启用' if request.cookie_rotation else '禁用'}") logger.info(f"最大重试次数: {request.max_retries}") for keyword in request.keywords: keyword_success = False keyword_error = None # 每个关键词重试逻辑 for attempt in range(request.max_retries): try: # 获取当前使用的Cookie current_cookie = self._get_next_cookie(cookies_list, used_cookies, request.cookie_rotation) if not current_cookie: raise ValueError("没有可用的Cookie") logger.info(f"搜索关键词: {keyword} (尝试 {attempt + 1}/{request.max_retries}, Cookie: {current_cookie.get('name')})") # 创建XHS适配器实例 xhs_adapter = XHSAdapter(cookies_str=current_cookie["cookie_string"]) # 配置搜索参数 search_config = SearchConfig( keyword=keyword, max_notes=request.max_notes_per_keyword, sort_type=request.sort_type, note_type=request.note_type ) # 搜索笔记 search_results = xhs_adapter.search_notes(search_config) # 检查搜索是否成功 if not search_results.success: raise Exception(f"搜索失败: {search_results.error_message}") notes = [] for note_data in search_results.notes: try: # 获取详细内容 note_content = note_data.content or "" # 先使用搜索结果中的content if request.get_detailed_content and note_data.note_url: try: detail_result = xhs_adapter.get_note_info(note_data.note_url) if detail_result and detail_result.content: note_content = detail_result.content except Exception as e: logger.warning(f"获取笔记详情失败 {note_data.note_url}: {e}") # 处理图片 saved_image_paths = [] if request.save_images and note_data.images: for i, img_url in enumerate(note_data.images): try: # 下载图片 response = requests.get(img_url, timeout=10) if response.status_code == 200: # 生成安全的文件名 safe_keyword = "".join(c for c in keyword if c.isalnum() or c in (' ', '-', '_')).rstrip()[:20] safe_title = "".join(c for c in (note_data.title or '')[:20] if c.isalnum() or c in (' ', '-', '_')).rstrip() filename = f"{safe_keyword}_{safe_title}_{note_data.note_id}_{i+1}.jpg" filepath = os.path.join(request.image_storage_path, filename) with open(filepath, 'wb') as f: f.write(response.content) saved_image_paths.append(filepath) logger.debug(f"保存图片: {filepath}") except Exception as e: logger.warning(f"下载图片失败 {img_url}: {e}") # 创建笔记信息 - 使用正确的字段映射 note_info = NoteInfo( note_id=note_data.note_id or '', title=note_data.title or '', author=note_data.author or '', content=note_content, like_count=note_data.likes or 0, comment_count=note_data.comments or 0, share_count=note_data.shares or 0, collect_count=getattr(note_data, 'collects', 0), note_url=note_data.note_url or '', images=note_data.images or [], saved_image_paths=saved_image_paths, publish_time=note_data.created_time, location=getattr(note_data, 'location', None), tags=note_data.tags or [] ) notes.append(note_info) except Exception as e: logger.error(f"处理笔记数据失败: {e}") continue # 成功获取数据 keyword_result = KeywordSearchResult( keyword=keyword, total_notes=len(notes), notes=notes, search_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), success=True, error_message=None ) results.append(keyword_result) total_notes += len(notes) successful_searches += 1 keyword_success = True logger.info(f"关键词 '{keyword}' 搜索成功，获得 {len(notes)} 条笔记") break # 成功则退出重试循环 except Exception as e: keyword_error = str(e) logger.warning(f"关键词 '{keyword}' 第 {attempt + 1} 次尝试失败: {e}") # 如果还有重试机会且启用了Cookie轮换，尝试下一个Cookie if attempt < request.max_retries - 1 and request.cookie_rotation: continue else: break # 如果所有重试都失败了 if not keyword_success: logger.error(f"关键词 '{keyword}' 搜索失败，已用尽所有重试机会") keyword_result = KeywordSearchResult( keyword=keyword, total_notes=0, notes=[], search_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), success=False, error_message=keyword_error or "未知错误" ) results.append(keyword_result) failed_searches += 1 # 计算处理时间 processing_time = time.time() - start_time # 生成摘要 top_authors = [] total_interactions = 0 saved_images = 0 for result in results: if result.success: for note in result.notes: if note.author and note.author not in top_authors: top_authors.append(note.author) total_interactions += note.like_count + note.comment_count + note.share_count + note.collect_count saved_images += len(note.saved_image_paths) summary = { "top_authors": top_authors[:10], "total_interactions": total_interactions, "saved_images": saved_images, "avg_notes_per_keyword": total_notes / len(request.keywords) if request.keywords else 0, "search_duration": f"{processing_time:.2f}秒", "cookies_used": len(cookies_list), "cookie_rotation_enabled": request.cookie_rotation } # 保存搜索结果到文件 output_file = os.path.join(request.output_path, f"batch_search_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json") try: import json with open(output_file, 'w', encoding='utf-8') as f: json.dump({ "total_keywords": len(request.keywords), "successful_searches": successful_searches, "failed_searches": failed_searches, "total_notes": total_notes, "results": [result.dict() for result in results], "summary": summary }, f, ensure_ascii=False, indent=2) logger.info(f"搜索结果已保存到: {output_file}") except Exception as e: logger.error(f"保存搜索结果失败: {e}") return BatchSearchResponse( total_keywords=len(request.keywords), successful_searches=successful_searches, failed_searches=failed_searches, total_notes=total_notes, results=results, image_storage_path=request.image_storage_path, output_path=request.output_path, processing_time=processing_time, summary=summary )