673 lines
53 KiB
JSON
673 lines
53 KiB
JSON
|
{
|
|||
|
|
"file_path": "document/text_extractor.py",
|
|||
|
|
"file_size": 12661,
|
|||
|
|
"line_count": 355,
|
|||
|
|
"functions": [
|
|||
|
|
{
|
|||
|
|
"name": "__post_init__",
|
|||
|
|
"line_start": 50,
|
|||
|
|
"line_end": 53,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": null,
|
|||
|
|
"docstring": "",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def __post_init__(self):\n # 确保content是字符串\n if not isinstance(self.content, str):\n self.content = str(self.content)",
|
|||
|
|
"code_hash": "e461fc43783cbe62b22d1a9349785191"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "__init__",
|
|||
|
|
"line_start": 58,
|
|||
|
|
"line_end": 69,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "config",
|
|||
|
|
"type_hint": "Optional[Dict[str, Any]]"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": null,
|
|||
|
|
"docstring": "",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def __init__(self, config: Optional[Dict[str, Any]] = None):\n self.config = config or {}\n self.supported_formats = {\n '.pdf': self._extract_pdf,\n '.docx': self._extract_docx,\n '.doc': self._extract_doc,\n '.txt': self._extract_txt,\n '.md': self._extract_txt,\n '.xlsx': self._extract_xlsx,\n '.xls': self._extract_xls,\n '.csv': self._extract_csv\n }",
|
|||
|
|
"code_hash": "4477c71180d28cd290e553df70152277"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "extract",
|
|||
|
|
"line_start": 71,
|
|||
|
|
"line_end": 102,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "str"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "ExtractedDocument",
|
|||
|
|
"docstring": "提取单个文件的文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def extract(self, file_path: str) -> ExtractedDocument:\n \"\"\"提取单个文件的文本内容\"\"\"\n path_obj = Path(file_path)\n \n if not path_obj.exists():\n raise FileNotFoundError(f\"文件不存在: {file_path}\")\n \n file_ext = path_obj.suffix.lower()\n if file_ext not in self.supported_formats:\n raise ValueError(f\"不支持的文件格式: {file_ext}\")\n \n try:\n # 获取文件信息\n file_size = path_obj.stat().st_size\n \n # 提取文本内容\n extractor = self.supported_formats[file_ext]\n content, metadata = extractor(path_obj)\n \n return ExtractedDocument(\n filename=path_obj.name,\n file_type=file_ext,\n content=content,\n metadata=metadata,\n extracted_at=datetime.now(),\n file_size=file_size,\n page_count=metadata.get('page_count')\n )\n \n except Exception as e:\n logger.error(f\"提取文件 {file_path} 时出错: {str(e)}\")\n raise",
|
|||
|
|
"code_hash": "5436f5f2cacc8d554f042f4bf89de554"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "extract_batch",
|
|||
|
|
"line_start": 104,
|
|||
|
|
"line_end": 126,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_paths",
|
|||
|
|
"type_hint": "List[str]"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "List[ExtractedDocument]",
|
|||
|
|
"docstring": "批量提取多个文件的文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def extract_batch(self, file_paths: List[str]) -> List[ExtractedDocument]:\n \"\"\"批量提取多个文件的文本内容\"\"\"\n results = []\n \n for file_path in file_paths:\n try:\n result = self.extract(file_path)\n results.append(result)\n logger.info(f\"成功提取文件: {file_path}\")\n except Exception as e:\n logger.error(f\"提取文件 {file_path} 失败: {str(e)}\")\n # 创建错误记录\n error_doc = ExtractedDocument(\n filename=Path(file_path).name,\n file_type=Path(file_path).suffix.lower(),\n content=f\"提取失败: {str(e)}\",\n metadata={\"error\": str(e)},\n extracted_at=datetime.now(),\n file_size=0\n )\n results.append(error_doc)\n \n return results",
|
|||
|
|
"code_hash": "f800214463cb7786ec16c690d19d451e"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_pdf",
|
|||
|
|
"line_start": 128,
|
|||
|
|
"line_end": 184,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取PDF文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_pdf(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取PDF文件的纯文本内容\"\"\"\n if not PDF_AVAILABLE:\n raise ImportError(\"需要安装 PyPDF2 和 pdfplumber: pip install PyPDF2 pdfplumber\")\n \n content_parts = []\n metadata = {}\n \n try:\n # 使用pdfplumber提取文本（更好的文本提取）\n with pdfplumber.open(file_path) as pdf:\n metadata['page_count'] = len(pdf.pages)\n \n for page_num, page in enumerate(pdf.pages, 1):\n page_text = page.extract_text()\n if page_text:\n content_parts.append(f\"=== 第 {page_num} 页 ===\\n{page_text}\\n\")\n \n # 获取文档元数据\n if pdf.metadata:\n metadata.update({\n 'title': pdf.metadata.get('Title', ''),\n 'author': pdf.metadata.get('Author', ''),\n 'subject': pdf.metadata.get('Subject', ''),\n 'creator': pdf.metadata.get('Creator', ''),\n 'producer': pdf.metadata.get('Producer', ''),\n 'creation_date': pdf.metadata.get('CreationDate', ''),\n 'modification_date': pdf.metadata.get('ModDate', '')\n })\n \n except Exception as e:\n logger.warning(f\"pdfplumber提取失败，尝试使用PyPDF2: {str(e)}\")\n \n # 备用方案：使用PyPDF2\n with open(file_path, 'rb') as file:\n pdf_reader = PyPDF2.PdfReader(file)\n metadata['page_count'] = len(pdf_reader.pages)\n \n for page_num, page in enumerate(pdf_reader.pages, 1):\n page_text = page.extract_text()\n if page_text:\n content_parts.append(f\"=== 第 {page_num} 页 ===\\n{page_text}\\n\")\n \n # 获取文档元数据\n if pdf_reader.metadata:\n metadata.update({\n 'title': pdf_reader.metadata.get('/Title', ''),\n 'author': pdf_reader.metadata.get('/Author', ''),\n 'subject': pdf_reader.metadata.get('/Subject', ''),\n 'creator': pdf_reader.metadata.get('/Creator', ''),\n 'producer': pdf_reader.metadata.get('/Producer', ''),\n 'creation_date': pdf_reader.metadata.get('/CreationDate', ''),\n 'modification_date': pdf_reader.metadata.get('/ModDate', '')\n })\n \n content = '\\n'.join(content_parts) if content_parts else \"\"\n return content, metadata",
|
|||
|
|
"code_hash": "92ff0fefec17a73a1c8224b15b5c70ea"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_docx",
|
|||
|
|
"line_start": 186,
|
|||
|
|
"line_end": 227,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取DOCX文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_docx(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取DOCX文件的纯文本内容\"\"\"\n if not DOCX_AVAILABLE:\n raise ImportError(\"需要安装 python-docx: pip install python-docx\")\n \n doc = Document(str(file_path))\n content_parts = []\n metadata = {}\n \n # 提取所有段落文本\n for paragraph in doc.paragraphs:\n if paragraph.text.strip():\n content_parts.append(paragraph.text)\n \n # 提取表格内容\n for table in doc.tables:\n table_content = []\n for row in table.rows:\n row_content = []\n for cell in row.cells:\n row_content.append(cell.text.strip())\n table_content.append('\\t'.join(row_content))\n if table_content:\n content_parts.append('\\n=== 表格 ===\\n' + '\\n'.join(table_content) + '\\n')\n \n # 获取文档属性\n core_props = doc.core_properties\n metadata.update({\n 'title': core_props.title or '',\n 'author': core_props.author or '',\n 'subject': core_props.subject or '',\n 'keywords': core_props.keywords or '',\n 'comments': core_props.comments or '',\n 'created': str(core_props.created) if core_props.created else '',\n 'modified': str(core_props.modified) if core_props.modified else '',\n 'last_modified_by': core_props.last_modified_by or '',\n 'paragraph_count': len(doc.paragraphs),\n 'table_count': len(doc.tables)\n })\n \n content = '\\n'.join(content_parts)\n return content, metadata",
|
|||
|
|
"code_hash": "c7b53be194f6c57266ac47d0f4191b08"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_doc",
|
|||
|
|
"line_start": 229,
|
|||
|
|
"line_end": 243,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取DOC文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_doc(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取DOC文件的纯文本内容\"\"\"\n # DOC格式较复杂，建议转换为DOCX或使用专门的库\n logger.warning(\"DOC格式支持有限，建议转换为DOCX格式\")\n \n # 尝试读取为文本文件\n try:\n with open(file_path, 'r', encoding='utf-8', errors='ignore') as file:\n content = file.read()\n except:\n with open(file_path, 'r', encoding='gbk', errors='ignore') as file:\n content = file.read()\n \n metadata = {'format': 'doc', 'encoding_note': '可能存在编码问题'}\n return content, metadata",
|
|||
|
|
"code_hash": "a9f6fc06bae7709495db7817d022c1ca"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_txt",
|
|||
|
|
"line_start": 245,
|
|||
|
|
"line_end": 272,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取TXT/MD文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_txt(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取TXT/MD文件的纯文本内容\"\"\"\n encodings = ['utf-8', 'gbk', 'gb2312', 'big5', 'utf-16']\n content = \"\"\n used_encoding = \"\"\n \n for encoding in encodings:\n try:\n with open(file_path, 'r', encoding=encoding) as file:\n content = file.read()\n used_encoding = encoding\n break\n except UnicodeDecodeError:\n continue\n \n if not content:\n # 最后尝试忽略错误\n with open(file_path, 'r', encoding='utf-8', errors='ignore') as file:\n content = file.read()\n used_encoding = 'utf-8 (with errors ignored)'\n \n metadata = {\n 'encoding': used_encoding,\n 'line_count': len(content.splitlines()),\n 'char_count': len(content)\n }\n \n return content, metadata",
|
|||
|
|
"code_hash": "a531b26825b63c4db57e234dc159a0d7"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_xlsx",
|
|||
|
|
"line_start": 274,
|
|||
|
|
"line_end": 301,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取XLSX文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_xlsx(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取XLSX文件的纯文本内容\"\"\"\n if not EXCEL_AVAILABLE:\n raise ImportError(\"需要安装 openpyxl: pip install openpyxl\")\n \n workbook = load_workbook(file_path, read_only=True)\n content_parts = []\n metadata = {\n 'sheet_count': len(workbook.sheetnames),\n 'sheet_names': workbook.sheetnames\n }\n \n for sheet_name in workbook.sheetnames:\n sheet = workbook[sheet_name]\n content_parts.append(f\"\\n=== 工作表: {sheet_name} ===\\n\")\n \n for row in sheet.iter_rows(values_only=True):\n row_content = []\n for cell in row:\n if cell is not None:\n row_content.append(str(cell))\n else:\n row_content.append(\"\")\n if any(cell.strip() for cell in row_content): # 跳过空行\n content_parts.append('\\t'.join(row_content))\n \n content = '\\n'.join(content_parts)\n return content, metadata",
|
|||
|
|
"code_hash": "4b90a3f26bbb6b4e5da3fbd78a7ea278"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_xls",
|
|||
|
|
"line_start": 303,
|
|||
|
|
"line_end": 315,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取XLS文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_xls(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取XLS文件的纯文本内容\"\"\"\n logger.warning(\"XLS格式支持有限，建议转换为XLSX格式\")\n \n # 简单的文本提取\n try:\n with open(file_path, 'rb') as file:\n content = file.read().decode('utf-8', errors='ignore')\n except:\n content = f\"无法读取XLS文件: {file_path}\"\n \n metadata = {'format': 'xls', 'note': '可能存在格式问题'}\n return content, metadata",
|
|||
|
|
"code_hash": "4a81d1dc3865abd2faa49705e5e8f2e9"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_csv",
|
|||
|
|
"line_start": 317,
|
|||
|
|
"line_end": 348,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取CSV文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_csv(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取CSV文件的纯文本内容\"\"\"\n encodings = ['utf-8', 'gbk', 'gb2312']\n content = \"\"\n used_encoding = \"\"\n \n for encoding in encodings:\n try:\n with open(file_path, 'r', encoding=encoding) as file:\n content = file.read()\n used_encoding = encoding\n break\n except UnicodeDecodeError:\n continue\n \n if not content:\n with open(file_path, 'r', encoding='utf-8', errors='ignore') as file:\n content = file.read()\n used_encoding = 'utf-8 (with errors ignored)'\n \n # 计算行数和列数\n lines = content.splitlines()\n row_count = len(lines)\n col_count = len(lines[0].split(',')) if lines else 0\n \n metadata = {\n 'encoding': used_encoding,\n 'row_count': row_count,\n 'estimated_col_count': col_count\n }\n \n return content, metadata",
|
|||
|
|
"code_hash": "d34952e59e05f67baf52a96ce6f99d9d"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "get_supported_formats",
|
|||
|
|
"line_start": 350,
|
|||
|
|
"line_end": 352,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "List[str]",
|
|||
|
|
"docstring": "获取支持的文件格式列表",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def get_supported_formats(self) -> List[str]:\n \"\"\"获取支持的文件格式列表\"\"\"\n return list(self.supported_formats.keys())",
|
|||
|
|
"code_hash": "657931bc9cac8245425c24c3fb872456"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "is_supported",
|
|||
|
|
"line_start": 354,
|
|||
|
|
"line_end": 356,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "str"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "bool",
|
|||
|
|
"docstring": "检查文件格式是否支持",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def is_supported(self, file_path: str) -> bool:\n \"\"\"检查文件格式是否支持\"\"\"\n return Path(file_path).suffix.lower() in self.supported_formats ",
|
|||
|
|
"code_hash": "5b7ac2ce0b4a7b9d115bef46ab04c2dd"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"classes": [
|
|||
|
|
{
|
|||
|
|
"name": "ExtractedDocument",
|
|||
|
|
"line_start": 40,
|
|||
|
|
"line_end": 53,
|
|||
|
|
"bases": [],
|
|||
|
|
"methods": [
|
|||
|
|
{
|
|||
|
|
"name": "__post_init__",
|
|||
|
|
"line_start": 50,
|
|||
|
|
"line_end": 53,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": null,
|
|||
|
|
"docstring": "",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def __post_init__(self):\n # 确保content是字符串\n if not isinstance(self.content, str):\n self.content = str(self.content)",
|
|||
|
|
"code_hash": "e461fc43783cbe62b22d1a9349785191"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"docstring": "提取的文档数据",
|
|||
|
|
"decorators": [
|
|||
|
|
"dataclass"
|
|||
|
|
],
|
|||
|
|
"code": "class ExtractedDocument:\n \"\"\"提取的文档数据\"\"\"\n filename: str\n file_type: str\n content: str # 纯文本内容\n metadata: Dict[str, Any] # 文档元数据\n extracted_at: datetime\n file_size: int\n page_count: Optional[int] = None\n \n def __post_init__(self):\n # 确保content是字符串\n if not isinstance(self.content, str):\n self.content = str(self.content)",
|
|||
|
|
"code_hash": "8b9d1fc812b3ea4aebc4e64cf53eaba6"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "TextExtractor",
|
|||
|
|
"line_start": 55,
|
|||
|
|
"line_end": 356,
|
|||
|
|
"bases": [],
|
|||
|
|
"methods": [
|
|||
|
|
{
|
|||
|
|
"name": "__init__",
|
|||
|
|
"line_start": 58,
|
|||
|
|
"line_end": 69,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "config",
|
|||
|
|
"type_hint": "Optional[Dict[str, Any]]"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": null,
|
|||
|
|
"docstring": "",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def __init__(self, config: Optional[Dict[str, Any]] = None):\n self.config = config or {}\n self.supported_formats = {\n '.pdf': self._extract_pdf,\n '.docx': self._extract_docx,\n '.doc': self._extract_doc,\n '.txt': self._extract_txt,\n '.md': self._extract_txt,\n '.xlsx': self._extract_xlsx,\n '.xls': self._extract_xls,\n '.csv': self._extract_csv\n }",
|
|||
|
|
"code_hash": "4477c71180d28cd290e553df70152277"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "extract",
|
|||
|
|
"line_start": 71,
|
|||
|
|
"line_end": 102,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "str"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "ExtractedDocument",
|
|||
|
|
"docstring": "提取单个文件的文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def extract(self, file_path: str) -> ExtractedDocument:\n \"\"\"提取单个文件的文本内容\"\"\"\n path_obj = Path(file_path)\n \n if not path_obj.exists():\n raise FileNotFoundError(f\"文件不存在: {file_path}\")\n \n file_ext = path_obj.suffix.lower()\n if file_ext not in self.supported_formats:\n raise ValueError(f\"不支持的文件格式: {file_ext}\")\n \n try:\n # 获取文件信息\n file_size = path_obj.stat().st_size\n \n # 提取文本内容\n extractor = self.supported_formats[file_ext]\n content, metadata = extractor(path_obj)\n \n return ExtractedDocument(\n filename=path_obj.name,\n file_type=file_ext,\n content=content,\n metadata=metadata,\n extracted_at=datetime.now(),\n file_size=file_size,\n page_count=metadata.get('page_count')\n )\n \n except Exception as e:\n logger.error(f\"提取文件 {file_path} 时出错: {str(e)}\")\n raise",
|
|||
|
|
"code_hash": "5436f5f2cacc8d554f042f4bf89de554"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "extract_batch",
|
|||
|
|
"line_start": 104,
|
|||
|
|
"line_end": 126,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_paths",
|
|||
|
|
"type_hint": "List[str]"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "List[ExtractedDocument]",
|
|||
|
|
"docstring": "批量提取多个文件的文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def extract_batch(self, file_paths: List[str]) -> List[ExtractedDocument]:\n \"\"\"批量提取多个文件的文本内容\"\"\"\n results = []\n \n for file_path in file_paths:\n try:\n result = self.extract(file_path)\n results.append(result)\n logger.info(f\"成功提取文件: {file_path}\")\n except Exception as e:\n logger.error(f\"提取文件 {file_path} 失败: {str(e)}\")\n # 创建错误记录\n error_doc = ExtractedDocument(\n filename=Path(file_path).name,\n file_type=Path(file_path).suffix.lower(),\n content=f\"提取失败: {str(e)}\",\n metadata={\"error\": str(e)},\n extracted_at=datetime.now(),\n file_size=0\n )\n results.append(error_doc)\n \n return results",
|
|||
|
|
"code_hash": "f800214463cb7786ec16c690d19d451e"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_pdf",
|
|||
|
|
"line_start": 128,
|
|||
|
|
"line_end": 184,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取PDF文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_pdf(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取PDF文件的纯文本内容\"\"\"\n if not PDF_AVAILABLE:\n raise ImportError(\"需要安装 PyPDF2 和 pdfplumber: pip install PyPDF2 pdfplumber\")\n \n content_parts = []\n metadata = {}\n \n try:\n # 使用pdfplumber提取文本（更好的文本提取）\n with pdfplumber.open(file_path) as pdf:\n metadata['page_count'] = len(pdf.pages)\n \n for page_num, page in enumerate(pdf.pages, 1):\n page_text = page.extract_text()\n if page_text:\n content_parts.append(f\"=== 第 {page_num} 页 ===\\n{page_text}\\n\")\n \n # 获取文档元数据\n if pdf.metadata:\n metadata.update({\n 'title': pdf.metadata.get('Title', ''),\n 'author': pdf.metadata.get('Author', ''),\n 'subject': pdf.metadata.get('Subject', ''),\n 'creator': pdf.metadata.get('Creator', ''),\n 'producer': pdf.metadata.get('Producer', ''),\n 'creation_date': pdf.metadata.get('CreationDate', ''),\n 'modification_date': pdf.metadata.get('ModDate', '')\n })\n \n except Exception as e:\n logger.warning(f\"pdfplumber提取失败，尝试使用PyPDF2: {str(e)}\")\n \n # 备用方案：使用PyPDF2\n with open(file_path, 'rb') as file:\n pdf_reader = PyPDF2.PdfReader(file)\n metadata['page_count'] = len(pdf_reader.pages)\n \n for page_num, page in enumerate(pdf_reader.pages, 1):\n page_text = page.extract_text()\n if page_text:\n content_parts.append(f\"=== 第 {page_num} 页 ===\\n{page_text}\\n\")\n \n # 获取文档元数据\n if pdf_reader.metadata:\n metadata.update({\n 'title': pdf_reader.metadata.get('/Title', ''),\n 'author': pdf_reader.metadata.get('/Author', ''),\n 'subject': pdf_reader.metadata.get('/Subject', ''),\n 'creator': pdf_reader.metadata.get('/Creator', ''),\n 'producer': pdf_reader.metadata.get('/Producer', ''),\n 'creation_date': pdf_reader.metadata.get('/CreationDate', ''),\n 'modification_date': pdf_reader.metadata.get('/ModDate', '')\n })\n \n content = '\\n'.join(content_parts) if content_parts else \"\"\n return content, metadata",
|
|||
|
|
"code_hash": "92ff0fefec17a73a1c8224b15b5c70ea"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_docx",
|
|||
|
|
"line_start": 186,
|
|||
|
|
"line_end": 227,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取DOCX文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_docx(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取DOCX文件的纯文本内容\"\"\"\n if not DOCX_AVAILABLE:\n raise ImportError(\"需要安装 python-docx: pip install python-docx\")\n \n doc = Document(str(file_path))\n content_parts = []\n metadata = {}\n \n # 提取所有段落文本\n for paragraph in doc.paragraphs:\n if paragraph.text.strip():\n content_parts.append(paragraph.text)\n \n # 提取表格内容\n for table in doc.tables:\n table_content = []\n for row in table.rows:\n row_content = []\n for cell in row.cells:\n row_content.append(cell.text.strip())\n table_content.append('\\t'.join(row_content))\n if table_content:\n content_parts.append('\\n=== 表格 ===\\n' + '\\n'.join(table_content) + '\\n')\n \n # 获取文档属性\n core_props = doc.core_properties\n metadata.update({\n 'title': core_props.title or '',\n 'author': core_props.author or '',\n 'subject': core_props.subject or '',\n 'keywords': core_props.keywords or '',\n 'comments': core_props.comments or '',\n 'created': str(core_props.created) if core_props.created else '',\n 'modified': str(core_props.modified) if core_props.modified else '',\n 'last_modified_by': core_props.last_modified_by or '',\n 'paragraph_count': len(doc.paragraphs),\n 'table_count': len(doc.tables)\n })\n \n content = '\\n'.join(content_parts)\n return content, metadata",
|
|||
|
|
"code_hash": "c7b53be194f6c57266ac47d0f4191b08"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_doc",
|
|||
|
|
"line_start": 229,
|
|||
|
|
"line_end": 243,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取DOC文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_doc(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取DOC文件的纯文本内容\"\"\"\n # DOC格式较复杂，建议转换为DOCX或使用专门的库\n logger.warning(\"DOC格式支持有限，建议转换为DOCX格式\")\n \n # 尝试读取为文本文件\n try:\n with open(file_path, 'r', encoding='utf-8', errors='ignore') as file:\n content = file.read()\n except:\n with open(file_path, 'r', encoding='gbk', errors='ignore') as file:\n content = file.read()\n \n metadata = {'format': 'doc', 'encoding_note': '可能存在编码问题'}\n return content, metadata",
|
|||
|
|
"code_hash": "a9f6fc06bae7709495db7817d022c1ca"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_txt",
|
|||
|
|
"line_start": 245,
|
|||
|
|
"line_end": 272,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取TXT/MD文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_txt(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取TXT/MD文件的纯文本内容\"\"\"\n encodings = ['utf-8', 'gbk', 'gb2312', 'big5', 'utf-16']\n content = \"\"\n used_encoding = \"\"\n \n for encoding in encodings:\n try:\n with open(file_path, 'r', encoding=encoding) as file:\n content = file.read()\n used_encoding = encoding\n break\n except UnicodeDecodeError:\n continue\n \n if not content:\n # 最后尝试忽略错误\n with open(file_path, 'r', encoding='utf-8', errors='ignore') as file:\n content = file.read()\n used_encoding = 'utf-8 (with errors ignored)'\n \n metadata = {\n 'encoding': used_encoding,\n 'line_count': len(content.splitlines()),\n 'char_count': len(content)\n }\n \n return content, metadata",
|
|||
|
|
"code_hash": "a531b26825b63c4db57e234dc159a0d7"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_xlsx",
|
|||
|
|
"line_start": 274,
|
|||
|
|
"line_end": 301,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取XLSX文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_xlsx(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取XLSX文件的纯文本内容\"\"\"\n if not EXCEL_AVAILABLE:\n raise ImportError(\"需要安装 openpyxl: pip install openpyxl\")\n \n workbook = load_workbook(file_path, read_only=True)\n content_parts = []\n metadata = {\n 'sheet_count': len(workbook.sheetnames),\n 'sheet_names': workbook.sheetnames\n }\n \n for sheet_name in workbook.sheetnames:\n sheet = workbook[sheet_name]\n content_parts.append(f\"\\n=== 工作表: {sheet_name} ===\\n\")\n \n for row in sheet.iter_rows(values_only=True):\n row_content = []\n for cell in row:\n if cell is not None:\n row_content.append(str(cell))\n else:\n row_content.append(\"\")\n if any(cell.strip() for cell in row_content): # 跳过空行\n content_parts.append('\\t'.join(row_content))\n \n content = '\\n'.join(content_parts)\n return content, metadata",
|
|||
|
|
"code_hash": "4b90a3f26bbb6b4e5da3fbd78a7ea278"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_xls",
|
|||
|
|
"line_start": 303,
|
|||
|
|
"line_end": 315,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取XLS文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_xls(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取XLS文件的纯文本内容\"\"\"\n logger.warning(\"XLS格式支持有限，建议转换为XLSX格式\")\n \n # 简单的文本提取\n try:\n with open(file_path, 'rb') as file:\n content = file.read().decode('utf-8', errors='ignore')\n except:\n content = f\"无法读取XLS文件: {file_path}\"\n \n metadata = {'format': 'xls', 'note': '可能存在格式问题'}\n return content, metadata",
|
|||
|
|
"code_hash": "4a81d1dc3865abd2faa49705e5e8f2e9"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "_extract_csv",
|
|||
|
|
"line_start": 317,
|
|||
|
|
"line_end": 348,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "Path"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "tuple[str, Dict[str, Any]]",
|
|||
|
|
"docstring": "提取CSV文件的纯文本内容",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def _extract_csv(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取CSV文件的纯文本内容\"\"\"\n encodings = ['utf-8', 'gbk', 'gb2312']\n content = \"\"\n used_encoding = \"\"\n \n for encoding in encodings:\n try:\n with open(file_path, 'r', encoding=encoding) as file:\n content = file.read()\n used_encoding = encoding\n break\n except UnicodeDecodeError:\n continue\n \n if not content:\n with open(file_path, 'r', encoding='utf-8', errors='ignore') as file:\n content = file.read()\n used_encoding = 'utf-8 (with errors ignored)'\n \n # 计算行数和列数\n lines = content.splitlines()\n row_count = len(lines)\n col_count = len(lines[0].split(',')) if lines else 0\n \n metadata = {\n 'encoding': used_encoding,\n 'row_count': row_count,\n 'estimated_col_count': col_count\n }\n \n return content, metadata",
|
|||
|
|
"code_hash": "d34952e59e05f67baf52a96ce6f99d9d"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "get_supported_formats",
|
|||
|
|
"line_start": 350,
|
|||
|
|
"line_end": 352,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "List[str]",
|
|||
|
|
"docstring": "获取支持的文件格式列表",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def get_supported_formats(self) -> List[str]:\n \"\"\"获取支持的文件格式列表\"\"\"\n return list(self.supported_formats.keys())",
|
|||
|
|
"code_hash": "657931bc9cac8245425c24c3fb872456"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "is_supported",
|
|||
|
|
"line_start": 354,
|
|||
|
|
"line_end": 356,
|
|||
|
|
"args": [
|
|||
|
|
{
|
|||
|
|
"name": "self"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "file_path",
|
|||
|
|
"type_hint": "str"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"return_type": "bool",
|
|||
|
|
"docstring": "检查文件格式是否支持",
|
|||
|
|
"is_async": false,
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": " def is_supported(self, file_path: str) -> bool:\n \"\"\"检查文件格式是否支持\"\"\"\n return Path(file_path).suffix.lower() in self.supported_formats ",
|
|||
|
|
"code_hash": "5b7ac2ce0b4a7b9d115bef46ab04c2dd"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"docstring": "文本提取器 - 只做纯文本提取，保留所有原始内容",
|
|||
|
|
"decorators": [],
|
|||
|
|
"code": "class TextExtractor:\n \"\"\"文本提取器 - 只做纯文本提取，保留所有原始内容\"\"\"\n \n def __init__(self, config: Optional[Dict[str, Any]] = None):\n self.config = config or {}\n self.supported_formats = {\n '.pdf': self._extract_pdf,\n '.docx': self._extract_docx,\n '.doc': self._extract_doc,\n '.txt': self._extract_txt,\n '.md': self._extract_txt,\n '.xlsx': self._extract_xlsx,\n '.xls': self._extract_xls,\n '.csv': self._extract_csv\n }\n \n def extract(self, file_path: str) -> ExtractedDocument:\n \"\"\"提取单个文件的文本内容\"\"\"\n path_obj = Path(file_path)\n \n if not path_obj.exists():\n raise FileNotFoundError(f\"文件不存在: {file_path}\")\n \n file_ext = path_obj.suffix.lower()\n if file_ext not in self.supported_formats:\n raise ValueError(f\"不支持的文件格式: {file_ext}\")\n \n try:\n # 获取文件信息\n file_size = path_obj.stat().st_size\n \n # 提取文本内容\n extractor = self.supported_formats[file_ext]\n content, metadata = extractor(path_obj)\n \n return ExtractedDocument(\n filename=path_obj.name,\n file_type=file_ext,\n content=content,\n metadata=metadata,\n extracted_at=datetime.now(),\n file_size=file_size,\n page_count=metadata.get('page_count')\n )\n \n except Exception as e:\n logger.error(f\"提取文件 {file_path} 时出错: {str(e)}\")\n raise\n \n def extract_batch(self, file_paths: List[str]) -> List[ExtractedDocument]:\n \"\"\"批量提取多个文件的文本内容\"\"\"\n results = []\n \n for file_path in file_paths:\n try:\n result = self.extract(file_path)\n results.append(result)\n logger.info(f\"成功提取文件: {file_path}\")\n except Exception as e:\n logger.error(f\"提取文件 {file_path} 失败: {str(e)}\")\n # 创建错误记录\n error_doc = ExtractedDocument(\n filename=Path(file_path).name,\n file_type=Path(file_path).suffix.lower(),\n content=f\"提取失败: {str(e)}\",\n metadata={\"error\": str(e)},\n extracted_at=datetime.now(),\n file_size=0\n )\n results.append(error_doc)\n \n return results\n \n def _extract_pdf(self, file_path: Path) -> tuple[str, Dict[str, Any]]:\n \"\"\"提取PDF文件的纯文本内容\"\"\"\n if not PDF_AVAILABLE:\n raise ImportError(\"需要安装 PyPDF2 和 pdfplumber: pip install PyPDF2 pdfplumber\")\n \n content_parts = []\n metadata = {}\n \n try:\n # 使用pdfplumber提取文本（更好的文本提取）\n with pdfplumber.open(file_path) as pdf:\n metadata['page_count'] = len(pdf.pages)\n \n for page_num, page in enumerate(pdf.pages, 1):\n page_text = page.extract_text()\n if page_text:\n content_parts.append(f\"=== 第 {page_num} 页 ===\\n{page_text}\\n\")\n \n # 获取文档元数据\n if pdf.metadata:\n metadata.update({\n 'title': pdf.metadata.get('Title', ''),\n 'author': pdf.metadata.get('Author', ''),\n 'subject': pdf.metadata.get('Subject', ''),\n 'creator': pdf.metadata.get('Creator', ''),\n 'producer': p
|
|||
|
|
"code_hash": "37d892d5ebf6d743d758c52421e08e2b"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"imports": [
|
|||
|
|
{
|
|||
|
|
"type": "import",
|
|||
|
|
"modules": [
|
|||
|
|
"os"
|
|||
|
|
],
|
|||
|
|
"aliases": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"type": "import",
|
|||
|
|
"modules": [
|
|||
|
|
"logging"
|
|||
|
|
],
|
|||
|
|
"aliases": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"type": "from_import",
|
|||
|
|
"module": "typing",
|
|||
|
|
"names": [
|
|||
|
|
"List",
|
|||
|
|
"Dict",
|
|||
|
|
"Any",
|
|||
|
|
"Optional"
|
|||
|
|
],
|
|||
|
|
"aliases": [],
|
|||
|
|
"level": 0
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"type": "from_import",
|
|||
|
|
"module": "pathlib",
|
|||
|
|
"names": [
|
|||
|
|
"Path"
|
|||
|
|
],
|
|||
|
|
"aliases": [],
|
|||
|
|
"level": 0
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"type": "from_import",
|
|||
|
|
"module": "dataclasses",
|
|||
|
|
"names": [
|
|||
|
|
"dataclass"
|
|||
|
|
],
|
|||
|
|
"aliases": [],
|
|||
|
|
"level": 0
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"type": "from_import",
|
|||
|
|
"module": "datetime",
|
|||
|
|
"names": [
|
|||
|
|
"datetime"
|
|||
|
|
],
|
|||
|
|
"aliases": [],
|
|||
|
|
"level": 0
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"type": "import",
|
|||
|
|
"modules": [
|
|||
|
|
"PyPDF2"
|
|||
|
|
],
|
|||
|
|
"aliases": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"type": "import",
|
|||
|
|
"modules": [
|
|||
|
|
"pdfplumber"
|
|||
|
|
],
|
|||
|
|
"aliases": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"type": "from_import",
|
|||
|
|
"module": "docx",
|
|||
|
|
"names": [
|
|||
|
|
"Document"
|
|||
|
|
],
|
|||
|
|
"aliases": [],
|
|||
|
|
"level": 0
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"type": "import",
|
|||
|
|
"modules": [
|
|||
|
|
"openpyxl"
|
|||
|
|
],
|
|||
|
|
"aliases": []
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"type": "from_import",
|
|||
|
|
"module": "openpyxl",
|
|||
|
|
"names": [
|
|||
|
|
"load_workbook"
|
|||
|
|
],
|
|||
|
|
"aliases": [],
|
|||
|
|
"level": 0
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"constants": [
|
|||
|
|
{
|
|||
|
|
"name": "PDF_AVAILABLE",
|
|||
|
|
"value": true,
|
|||
|
|
"type": "bool",
|
|||
|
|
"line": 20
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "DOCX_AVAILABLE",
|
|||
|
|
"value": true,
|
|||
|
|
"type": "bool",
|
|||
|
|
"line": 26
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "EXCEL_AVAILABLE",
|
|||
|
|
"value": true,
|
|||
|
|
"type": "bool",
|
|||
|
|
"line": 33
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "PDF_AVAILABLE",
|
|||
|
|
"value": false,
|
|||
|
|
"type": "bool",
|
|||
|
|
"line": 22
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "DOCX_AVAILABLE",
|
|||
|
|
"value": false,
|
|||
|
|
"type": "bool",
|
|||
|
|
"line": 28
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"name": "EXCEL_AVAILABLE",
|
|||
|
|
"value": false,
|
|||
|
|
"type": "bool",
|
|||
|
|
"line": 35
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"docstring": "文本提取器模块\n支持从PDF、Word、TXT等格式的文档中提取文本内容",
|
|||
|
|
"content_hash": "99faa0b695aa0a84886242bcb430ff10"
|
|||
|
|
}
|