| 1 | """Document text extraction utilities for nanobot.""" |
| 2 | |
| 3 | import mimetypes |
| 4 | from pathlib import Path |
| 5 | |
| 6 | from loguru import logger |
| 7 | |
| 8 | from nanobot.utils.helpers import detect_image_mime |
| 9 | |
| 10 | try: |
| 11 | from pypdf import PdfReader |
| 12 | except ImportError: |
| 13 | PdfReader = None # type: ignore |
| 14 | |
| 15 | try: |
| 16 | from docx import Document as DocxDocument |
| 17 | except ImportError: |
| 18 | DocxDocument = None # type: ignore |
| 19 | |
| 20 | try: |
| 21 | from openpyxl import load_workbook |
| 22 | except ImportError: |
| 23 | load_workbook = None # type: ignore |
| 24 | |
| 25 | try: |
| 26 | from pptx import Presentation as PptxPresentation |
| 27 | except ImportError: |
| 28 | PptxPresentation = None # type: ignore |
| 29 | |
| 30 | |
| 31 | # Supported file extensions for text extraction |
| 32 | SUPPORTED_EXTENSIONS: set[str] = { |
| 33 | # Document formats |
| 34 | ".pdf", |
| 35 | ".docx", |
| 36 | ".xlsx", |
| 37 | ".pptx", |
| 38 | # Text formats |
| 39 | ".txt", |
| 40 | ".md", |
| 41 | ".csv", |
| 42 | ".json", |
| 43 | ".xml", |
| 44 | ".html", |
| 45 | ".htm", |
| 46 | ".log", |
| 47 | ".yaml", |
| 48 | ".yml", |
| 49 | ".toml", |
| 50 | ".ini", |
| 51 | ".cfg", |
| 52 | # Image formats (for future OCR support) |
| 53 | ".png", |
| 54 | ".jpg", |
| 55 | ".jpeg", |
| 56 | ".gif", |
| 57 | ".webp", |
| 58 | } |
| 59 | |
| 60 | _MAX_TEXT_LENGTH = 200_000 |
| 61 | |
| 62 | |
| 63 | def extract_text(path: Path) -> str | None: |
| 64 | """Extract text from a file. |
| 65 | |
| 66 | Args: |
| 67 | path: Path to the file. |
| 68 | |
| 69 | Returns: |
| 70 | Extracted text as string, None for unsupported types, |
| 71 | or error string for failures. |
| 72 | """ |
| 73 | if not isinstance(path, Path): |
| 74 | path = Path(path) |
| 75 | |
| 76 | if not path.exists(): |
| 77 | return f"[error: file not found: {path}]" |
| 78 | |
| 79 | ext = path.suffix.lower() |
| 80 | |
| 81 | # Document formats |
| 82 | if ext == ".pdf": |
| 83 | if PdfReader is None: |
| 84 | return "[error: pypdf not installed]" |
| 85 | return _extract_pdf(path) |
| 86 | elif ext == ".docx": |
| 87 | if DocxDocument is None: |
| 88 | return "[error: python-docx not installed]" |
| 89 | return _extract_docx(path) |
| 90 | elif ext == ".xlsx": |
| 91 | if load_workbook is None: |
| 92 | return "[error: openpyxl not installed]" |
| 93 | return _extract_xlsx(path) |
| 94 | elif ext == ".pptx": |
| 95 | if PptxPresentation is None: |
| 96 | return "[error: python-pptx not installed]" |
| 97 | return _extract_pptx(path) |
| 98 | elif _is_text_extension(ext): |
| 99 | return _extract_text_file(path) |
| 100 | elif ext in {".png", ".jpg", ".jpeg", ".gif", ".webp"}: |
| 101 | # Image files - for future OCR support |
| 102 | return f"[image: {path.name}]" |
| 103 | else: |
| 104 | # Unsupported extension |
| 105 | return None |
| 106 | |
| 107 | |
| 108 | def _extract_pdf(path: Path) -> str: |
| 109 | """Extract text from PDF using pypdf.""" |
| 110 | try: |
| 111 | reader = PdfReader(path) |
| 112 | pages: list[str] = [] |
| 113 | for i, page in enumerate(reader.pages, 1): |
| 114 | text = page.extract_text() or "" |
| 115 | pages.append(f"--- Page {i} ---\n{text}") |
| 116 | return _truncate("\n\n".join(pages), _MAX_TEXT_LENGTH) |
| 117 | except Exception as e: |
| 118 | logger.error("Failed to extract PDF {}: {}", path, e) |
| 119 | return f"[error: failed to extract PDF: {e!s}]" |
| 120 | |
| 121 | |
| 122 | def _extract_docx(path: Path) -> str: |
| 123 | """Extract text from DOCX using python-docx.""" |
| 124 | try: |
| 125 | doc = DocxDocument(path) |
| 126 | paragraphs: list[str] = [p.text for p in doc.paragraphs if p.text.strip()] |
| 127 | return _truncate("\n\n".join(paragraphs), _MAX_TEXT_LENGTH) |
| 128 | except Exception as e: |
| 129 | logger.error("Failed to extract DOCX {}: {}", path, e) |
| 130 | return f"[error: failed to extract DOCX: {e!s}]" |
| 131 | |
| 132 | |
| 133 | def _extract_xlsx(path: Path) -> str: |
| 134 | """Extract text from XLSX using openpyxl.""" |
| 135 | try: |
| 136 | wb = load_workbook(path, read_only=True, data_only=True) |
| 137 | try: |
| 138 | sheets: list[str] = [] |
| 139 | for sheet_name in wb.sheetnames: |
| 140 | ws = wb[sheet_name] |
| 141 | rows: list[str] = [] |
| 142 | for row in ws.iter_rows(values_only=True): |
| 143 | row_text = "\t".join(str(cell) if cell is not None else "" for cell in row) |
| 144 | if row_text.strip(): |
| 145 | rows.append(row_text) |
| 146 | if rows: |
| 147 | sheets.append(f"--- Sheet: {sheet_name} ---\n" + "\n".join(rows)) |
| 148 | return _truncate("\n\n".join(sheets), _MAX_TEXT_LENGTH) |
| 149 | finally: |
| 150 | wb.close() |
| 151 | except Exception as e: |
| 152 | logger.error("Failed to extract XLSX {}: {}", path, e) |
| 153 | return f"[error: failed to extract XLSX: {e!s}]" |
| 154 | |
| 155 | |
| 156 | def _extract_pptx(path: Path) -> str: |
| 157 | """Extract text from PPTX using python-pptx.""" |
| 158 | try: |
| 159 | prs = PptxPresentation(path) |
| 160 | slides: list[str] = [] |
| 161 | for i, slide in enumerate(prs.slides, 1): |
| 162 | slide_text: list[str] = [] |
| 163 | for shape in slide.shapes: |
| 164 | _collect_pptx_shape_text(shape, slide_text) |
| 165 | if slide_text: |
| 166 | slides.append(f"--- Slide {i} ---\n" + "\n".join(slide_text)) |
| 167 | return _truncate("\n\n".join(slides), _MAX_TEXT_LENGTH) |
| 168 | except Exception as e: |
| 169 | logger.error("Failed to extract PPTX {}: {}", path, e) |
| 170 | return f"[error: failed to extract PPTX: {e!s}]" |
| 171 | |
| 172 | |
| 173 | def _collect_pptx_shape_text(shape, out: list[str]) -> None: |
| 174 | """Collect text from a PPTX shape, recursing into groups and tables. |
| 175 | |
| 176 | Groups have ``has_text_frame=False`` and must be walked via ``.shapes``; |
| 177 | tables are GraphicFrame objects whose cell text lives under ``.table``. |
| 178 | """ |
| 179 | sub_shapes = getattr(shape, "shapes", None) |
| 180 | if sub_shapes is not None: |
| 181 | for sub in sub_shapes: |
| 182 | _collect_pptx_shape_text(sub, out) |
| 183 | return |
| 184 | |
| 185 | if getattr(shape, "has_table", False): |
| 186 | for row in shape.table.rows: |
| 187 | cells = [cell.text.strip() for cell in row.cells] |
| 188 | line = "\t".join(cell for cell in cells if cell) |
| 189 | if line: |
| 190 | out.append(line) |
| 191 | return |
| 192 | |
| 193 | text = getattr(shape, "text", "") |
| 194 | if text: |
| 195 | out.append(text) |
| 196 | |
| 197 | |
| 198 | def _extract_text_file(path: Path) -> str: |
| 199 | """Extract text from a plain text file.""" |
| 200 | try: |
| 201 | # Try UTF-8 first, then latin-1 fallback |
| 202 | try: |
| 203 | content = path.read_text(encoding="utf-8") |
| 204 | except UnicodeDecodeError: |
| 205 | content = path.read_text(encoding="latin-1") |
| 206 | return _truncate(content, _MAX_TEXT_LENGTH) |
| 207 | except Exception as e: |
| 208 | logger.error("Failed to read text file {}: {}", path, e) |
| 209 | return f"[error: failed to read file: {e!s}]" |
| 210 | |
| 211 | |
| 212 | def _truncate(text: str, max_length: int) -> str: |
| 213 | """Truncate text with a suffix indicating truncation.""" |
| 214 | if len(text) <= max_length: |
| 215 | return text |
| 216 | return text[:max_length] + f"... (truncated, {len(text)} chars total)" |
| 217 | |
| 218 | |
| 219 | def _is_text_extension(ext: str) -> bool: |
| 220 | """Check if extension is a text format.""" |
| 221 | return ext in { |
| 222 | ".txt", |
| 223 | ".md", |
| 224 | ".csv", |
| 225 | ".json", |
| 226 | ".xml", |
| 227 | ".html", |
| 228 | ".htm", |
| 229 | ".log", |
| 230 | ".yaml", |
| 231 | ".yml", |
| 232 | ".toml", |
| 233 | ".ini", |
| 234 | ".cfg", |
| 235 | } |
| 236 | |
| 237 | |
| 238 | # --------------------------------------------------------------------------- |
| 239 | # High-level helper: split media into images + extracted document text |
| 240 | # --------------------------------------------------------------------------- |
| 241 | |
| 242 | _MAX_EXTRACT_FILE_SIZE = 50 * 1024 * 1024 # 50 MB |
| 243 | |
| 244 | |
| 245 | def extract_documents( |
| 246 | text: str, |
| 247 | media_paths: list[str], |
| 248 | *, |
| 249 | max_file_size: int = _MAX_EXTRACT_FILE_SIZE, |
| 250 | ) -> tuple[str, list[str]]: |
| 251 | """Separate images from documents in *media_paths*. |
| 252 | |
| 253 | Documents (PDF, DOCX, XLSX, PPTX, plain-text, …) have their text |
| 254 | extracted and appended to *text*. Only image paths are kept in the |
| 255 | returned list so that downstream layers only need to handle vision |
| 256 | blocks. |
| 257 | |
| 258 | Files larger than *max_file_size* bytes are skipped with a warning |
| 259 | to avoid unbounded memory / CPU usage. |
| 260 | """ |
| 261 | image_paths: list[str] = [] |
| 262 | doc_texts: list[str] = [] |
| 263 | |
| 264 | for path_str in media_paths: |
| 265 | p = Path(path_str) |
| 266 | if not p.is_file(): |
| 267 | continue |
| 268 | |
| 269 | try: |
| 270 | size = p.stat().st_size |
| 271 | except OSError: |
| 272 | continue |
| 273 | if size > max_file_size: |
| 274 | logger.warning( |
| 275 | "Skipping oversized file for extraction: {} ({:.1f} MB > {} MB limit)", |
| 276 | p.name, size / (1024 * 1024), max_file_size // (1024 * 1024), |
| 277 | ) |
| 278 | continue |
| 279 | |
| 280 | with open(p, "rb") as f: |
| 281 | header = f.read(16) |
| 282 | mime = detect_image_mime(header) or mimetypes.guess_type(path_str)[0] |
| 283 | if mime and mime.startswith("image/"): |
| 284 | image_paths.append(path_str) |
| 285 | else: |
| 286 | extracted = extract_text(p) |
| 287 | if extracted and not extracted.startswith("[error:"): |
| 288 | doc_texts.append(f"[File: {p.name}]\n{extracted}") |
| 289 | |
| 290 | if doc_texts: |
| 291 | text = text + "\n\n" + "\n\n".join(doc_texts) |
| 292 | |
| 293 | return text, image_paths |
| 294 |