pdf_to_md.py - ppt-master

pdf_to_md.py

1	#!/usr/bin/env python3
2	"""
3	PDF to Markdown Converter
4	Uses PyMuPDF to extract PDF text content and convert to Markdown format.
5	Supports heading levels, bold, italic, and list detection.
6	"""
7
8	import argparse
9	import hashlib
10	import json
11	import os
12	import re
13	import sys
14	from pathlib import Path
15	from collections import Counter
16
17	_SCRIPTS_DIR = Path(__file__).resolve().parents[1]
18	if str(_SCRIPTS_DIR) not in sys.path:
19	sys.path.insert(0, str(_SCRIPTS_DIR))
20
21	from console_encoding import configure_utf8_stdio # noqa: E402
22	from _batch import run_path_batch # noqa: E402
23	from _conversion_profile import write_conversion_profile_best_effort # noqa: E402
24
25	configure_utf8_stdio()
26
27	try:
28	import fitz # PyMuPDF
29	except ImportError:
30	print("[ERROR] PyMuPDF not installed. Run: pip install PyMuPDF", file=sys.stderr)
31	sys.exit(1)
32
33	FONT_BODY_SIZE = 12
34	FONT_H1_SIZE = 24
35	FONT_H2_SIZE = 18
36	FONT_H3_SIZE = 14
37	HEADER_FOOTER_SAMPLE_LIMIT = 40
38	HEADER_FOOTER_EDGE_SAMPLE_SIZE = 20
39	CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f]')
40
41
42	def analyze_font_sizes(doc: fitz.Document) -> dict[str, float]:
43	"""Analyze font size distribution to infer heading levels.
44
45	Args:
46	doc: Open PDF document.
47
48	Returns:
49	A size mapping containing body and inferred heading sizes.
50	"""
51	size_counter = Counter()
52
53	for page in doc:
54	blocks = page.get_text("dict")["blocks"]
55	for block in blocks:
56	if block["type"] == 0:
57	for line in block["lines"]:
58	for span in line["spans"]:
59	size = round(span["size"], 1)
60	text = span["text"].strip()
61	if text:
62	size_counter[size] += len(text)
63
64	if not size_counter:
65	return {
66	"body": FONT_BODY_SIZE,
67	"h1": FONT_H1_SIZE,
68	"h2": FONT_H2_SIZE,
69	"h3": FONT_H3_SIZE,
70	}
71
72	sorted_sizes = sorted(size_counter.items(), key=lambda x: x[1], reverse=True)
73	body_size = sorted_sizes[0][0]
74
75	all_sizes = sorted(size_counter.keys(), reverse=True)
76	larger_sizes = [s for s in all_sizes if s > body_size + 1]
77
78	size_map = {"body": body_size}
79	if len(larger_sizes) >= 1:
80	size_map["h1"] = larger_sizes[0]
81	if len(larger_sizes) >= 2:
82	size_map["h2"] = larger_sizes[1]
83	if len(larger_sizes) >= 3:
84	size_map["h3"] = larger_sizes[2]
85
86	return size_map
87
88
89	def get_heading_level(size: float, size_map: dict, text: str = "",
90	flags: int = 0, strict: bool = True) -> int:
91	"""
92	Determine heading level using multiple heuristics.
93
94	Args:
95	size: Font size
96	size_map: Font size mapping
97	text: Text content (used for additional heuristics)
98	flags: Font flags (bit 4 = bold)
99	strict: Strict mode, requires more conditions to be met
100
101	Returns:
102	Heading level (0 = body text, 1-3 = H1-H3)
103	"""
104	# Initial determination based on font size
105	level = 0
106	if "h1" in size_map and size >= size_map["h1"] - 0.5:
107	level = 1
108	elif "h2" in size_map and size >= size_map["h2"] - 0.5:
109	level = 2
110	elif "h3" in size_map and size >= size_map["h3"] - 0.5:
111	level = 3
112
113	if level == 0:
114	return 0
115
116	# Non-strict mode returns directly (backward compatible)
117	if not strict or not text:
118	return level
119
120	# Strict mode: additional validation conditions
121	text = text.strip()
122
123	# Exclusion: text too long is unlikely to be a heading
124	if len(text) > 80:
125	return 0
126
127	# Exclusion: complete sentences ending with punctuation
128	sentence_endings = '.。!！?？'
129	if text and text[-1] in sentence_endings:
130	# But keep numbered headings like "1. Overview" or "Chapter 1."
131	if not re.match(r'^[\d第]+[.、章节]', text):
132	return 0
133
134	# Bonus: bold text is more likely to be a heading
135	is_bold = flags & 16
136	if not is_bold and level >= 2:
137	# Non-bold subheadings require a larger font size difference
138	body_size = size_map.get("body", 12)
139	if size < body_size + 2:
140	return 0
141
142	return level
143
144	def is_monospace_font(font_name: str) -> bool:
145	"""
146	Determine if the font is monospace (typically used for code).
147	"""
148	if not font_name:
149	return False
150	font_lower = font_name.lower()
151	mono_fonts = [
152	'courier', 'consolas', 'monaco', 'menlo', 'monospace',
153	'source code', 'fira code', 'jetbrains', 'inconsolata',
154	'dejavu sans mono', 'liberation mono', 'ubuntu mono',
155	'roboto mono', 'robotomono', 'sf mono', 'cascadia', 'hack'
156	]
157	return any(f in font_lower for f in mono_fonts)
158
159
160	def format_span_text(text: str, flags: int) -> str:
161	"""Format text based on font flags (bold, italic)."""
162	text = CONTROL_CHARS_RE.sub('', text)
163	text = text.strip()
164	if not text:
165	return ""
166
167	is_bold = flags & 16
168	is_italic = flags & 2
169
170	if is_bold and is_italic:
171	return f"*{text}*"
172	elif is_bold:
173	return f"{text}"
174	elif is_italic:
175	return f"{text}"
176	return text
177
178
179	def detect_list_item(text: str) -> tuple:
180	"""Detect if the text is a list item. Returns (is_list, list_type, content)."""
181	text = text.strip()
182
183	ul_patterns = [
184	(r'^[•●○◦▪▸►]\s*', '-'),
185	(r'^[-–—]\s+', '-'),
186	(r'^\*\s+', '-'),
187	]
188	for pattern, marker in ul_patterns:
189	match = re.match(pattern, text)
190	if match:
191	return (True, 'ul', marker + ' ' + text[match.end():])
192
193	ol_pattern = r'^(\d+)[.、)]\s*'
194	match = re.match(ol_pattern, text)
195	if match:
196	num = match.group(1)
197	return (True, 'ol', f"{num}. " + text[match.end():])
198
199	return (False, None, text)
200
201
202	def remove_page_footer(text: str) -> str:
203	"""
204	Remove page number patterns from footers, e.g. 'November 2025 8' or '2025年11月 8'.
205	"""
206	# English month + year + page number
207	months_en = r'(?:January\|February\|March\|April\|May\|June\|July\|August\|September\|October\|November\|December)'
208	pattern_en = rf'\s{months_en}\s+\d{{4}}\s+\d{{1,3}}\s$'
209	text = re.sub(pattern_en, '', text, flags=re.IGNORECASE)
210
211	# Chinese format: 2025年11月 8
212	pattern_cn = r'\s\d{4}年\d{1,2}月\s+\d{1,3}\s$'
213	text = re.sub(pattern_cn, '', text)
214
215	return text.rstrip()
216
217
218	def detect_headers_footers(doc: fitz.Document, threshold_ratio: float = 0.6) -> set[str]:
219	"""
220	Detect headers and footers statistically.
221
222	Principle: Headers and footers typically appear at fixed positions (top or bottom)
223	on each page with the same content. We collect top and bottom text from all pages,
224	and if certain text appears more frequently than the threshold, it is treated as noise.
225	"""
226	if len(doc) < 3:
227	return set()
228
229	headers = []
230	footers = []
231
232	# Sample first 20 and last 20 pages (avoid processing too slowly)
233	pages_to_scan = list(range(len(doc)))
234	if len(doc) > HEADER_FOOTER_SAMPLE_LIMIT:
235	pages_to_scan = (
236	pages_to_scan[:HEADER_FOOTER_EDGE_SAMPLE_SIZE]
237	+ pages_to_scan[-HEADER_FOOTER_EDGE_SAMPLE_SIZE:]
238	)
239
240	for i in pages_to_scan:
241	page = doc[i]
242	rect = page.rect
243	h = rect.height
244
245	# Define top and bottom regions (15% each)
246	top_rect = fitz.Rect(0, 0, rect.width, h * 0.15)
247	bottom_rect = fitz.Rect(0, h * 0.85, rect.width, h)
248
249	# Extract text blocks
250	blocks = page.get_text("blocks")
251	for b in blocks:
252	b_rect = fitz.Rect(b[:4])
253	text = b[4].strip()
254	if not text:
255	continue
256
257	# Simple spatial determination
258	if b_rect.intersects(top_rect):
259	headers.append(text)
260	elif b_rect.intersects(bottom_rect):
261	footers.append(text)
262
263	# Count frequencies
264	noise_texts = set()
265	total_scanned = len(pages_to_scan)
266
267	for collection in [headers, footers]:
268	counter = Counter(collection)
269	for text, count in counter.items():
270	# if text appears in > 60% of scanned pages, mark as noise
271	if count / total_scanned > threshold_ratio:
272	noise_texts.add(text)
273
274	return noise_texts
275
276
277	def merge_adjacent_headings(elements: list) -> list:
278	"""
279	Merge adjacent same-level short headings.
280	Example: '# Agent Tools &' + '# Interoperability' -> '# Agent Tools & Interoperability'
281	"""
282	if not elements:
283	return elements
284
285	merged = []
286	i = 0
287
288	while i < len(elements):
289	el = elements[i]
290
291	# Only process heading elements
292	if el.get("type") != 0 or not el.get("is_heading"):
293	merged.append(el)
294	i += 1
295	continue
296
297	content = el["content"]
298	# Extract heading level
299	match = re.match(r'^(#{1,6})\s+(.+)$', content)
300	if not match:
301	merged.append(el)
302	i += 1
303	continue
304
305	level = match.group(1)
306	title_text = match.group(2)
307
308	# If heading is short and the next one is also the same level, try to merge
309	j = i + 1
310	while j < len(elements) and len(title_text) < 60:
311	next_el = elements[j]
312	if next_el.get("type") != 0 or not next_el.get("is_heading"):
313	break
314
315	next_match = re.match(r'^(#{1,6})\s+(.+)$', next_el["content"])
316	if not next_match or next_match.group(1) != level:
317	break
318
319	next_text = next_match.group(2)
320	# Only merge short heading fragments
321	if len(next_text) > 40:
322	break
323
324	# Merge
325	title_text += " " + next_text
326	j += 1
327
328	# Create merged element
329	merged_el = el.copy()
330	merged_el["content"] = f"{level} {title_text}"
331	merged.append(merged_el)
332	i = j
333
334	return merged
335
336
337	# Image filtering thresholds
338	MIN_IMAGE_PIXELS = 100 # Minimum pixel dimension (width AND height)
339	MIN_IMAGE_AREA = 30000 # Minimum pixel area (e.g. 200x150)
340	MIN_IMAGE_BYTES = 2048 # Minimum image data size (2KB)
341	MIN_PAGE_RATIO = 0.05 # Minimum render size relative to page (5%)
342	MIN_VISIBLE_IMAGE_WIDTH = 40
343	MIN_VISIBLE_IMAGE_HEIGHT = 40
344	MIN_VISIBLE_IMAGE_AREA_RATIO = 0.01
345	MAX_ASPECT_RATIO = 12 # Maximum aspect ratio (filters decorative bars)
346	MAX_LOW_INFO_BPP = 0.08 # Bytes-per-pixel threshold for low-info images
347	MAX_LOW_INFO_AREA = 500000 # Area threshold: only apply bpp filter below this
348	MIN_VECTOR_FIGURE_WIDTH = 100
349	MIN_VECTOR_FIGURE_HEIGHT = 80
350	MIN_VECTOR_FIGURE_AREA = 30000
351	MAX_VECTOR_FIGURE_ASPECT_RATIO = 8
352	VECTOR_FIGURE_PADDING = 4
353	VECTOR_FIGURE_DPI = 180
354	VECTOR_CAPTION_SEARCH_HEIGHT = 380
355	VECTOR_CAPTION_HORIZONTAL_GAP = 90
356	MAX_VECTOR_BACKGROUND_AREA_RATIO = 1.9
357	# Caption delimiters: ``Figure 1:`` / ``Figure 1.`` (classic) and ``Figure 1 \|``
358	# (the DeepMind / Distill / Nature house style used by many ML papers, incl.
359	# full-width ``｜``). Without the pipe variants, captioned vector figures route
360	# to the generic per-drawing fallback, which discards fine-grained line plots
361	# (each plot is hundreds of tiny primitives, none large enough on its own).
362	FIGURE_CAPTION_RE = re.compile(r'^(?:Figure\|Fig\.?)\s\d+\s[:.\|｜]', re.IGNORECASE)
363
364
365	TABLE_CAPTION_RE = re.compile(
366	r'^\u8868\s\d+(?:\.\d+)\s+(?!(?:\u7684\|\u5217\u793a\|\u6240\u793a\|\u4e3a)).+'
367	)
368	TABLE_REFERENCE_PROSE_RE = re.compile(
369	r'^\u8868\s\d+(?:\.\d+)\s*(?:\u7684\|\u5217\u793a\|\u6240\u793a\|\u4e3a)'
370	)
371	SECTION_HEADING_RE = re.compile(r'^\d+(?:\.\d+){1,3}\s+\S')
372	NUMBER_RE = re.compile(r'[-+]?\d+(?:\.\d+)?')
373	MODEL_COLUMN_RE = re.compile(r'^[（(]\d+[）)]$')
374	TABLE_NOTE_PREFIX = '\u6ce8'
375	TABLE_CONTINUATION_Y_RATIO = 0.88
376	TABLE_SCAN_BOTTOM_RATIO = 0.92
377
378
379	def should_keep_image(
380	block: dict[str, object],
381	page_rect: fitz.Rect,
382	seen_hashes: set[str] \| None = None,
383	) -> bool:
384	"""Filter out small, decorative, or duplicate images.
385
386	Args:
387	block: Image block extracted from PyMuPDF.
388	page_rect: Current page rectangle.
389	seen_hashes: Optional set used to deduplicate image payloads.
390
391	Returns:
392	Whether the image should be kept in the Markdown output.
393	"""
394	w, h = block.get("width", 0), block.get("height", 0)
395	bbox = block.get("bbox", (0, 0, 0, 0))
396	render_w = bbox[2] - bbox[0]
397	render_h = bbox[3] - bbox[1]
398	page_area = page_rect.width * page_rect.height
399	render_area_ratio = (render_w * render_h) / page_area if page_area > 0 else 0
400	visibly_placed = (
401	render_w >= MIN_VISIBLE_IMAGE_WIDTH
402	and render_h >= MIN_VISIBLE_IMAGE_HEIGHT
403	and render_area_ratio >= MIN_VISIBLE_IMAGE_AREA_RATIO
404	)
405
406	# Pixel dimension filter
407	if not visibly_placed and (w < MIN_IMAGE_PIXELS or h < MIN_IMAGE_PIXELS):
408	return False
409
410	# Pixel area filter
411	area = w * h
412	if not visibly_placed and area < MIN_IMAGE_AREA:
413	return False
414
415	image_data = block.get("image", b"")
416	if not visibly_placed and len(image_data) < MIN_IMAGE_BYTES:
417	return False
418
419	# Deduplicate tiny repeats, but preserve visibly placed logos / charts on
420	# distinct pages. Academic PDFs often reuse a small logo on title pages.
421	if seen_hashes is not None:
422	img_hash = hashlib.md5(image_data).hexdigest()
423	if img_hash in seen_hashes and not visibly_placed:
424	return False
425	seen_hashes.add(img_hash)
426
427	# Check render size relative to page
428	page_w = page_rect.width
429	page_h = page_rect.height
430	if page_w > 0 and page_h > 0:
431	if render_w / page_w < MIN_PAGE_RATIO and render_h / page_h < MIN_PAGE_RATIO:
432	return False
433
434	# Filter extreme aspect ratios (decorative bars/separators)
435	aspect = max(w, h) / max(min(w, h), 1)
436	if aspect > MAX_ASPECT_RATIO:
437	return False
438
439	# Filter low-info images: solid color blocks / gradients have very high
440	# compression ratios (low bytes-per-pixel). Only apply to smaller images
441	# to avoid filtering large photos with dark/uniform backgrounds.
442	bpp = len(image_data) / area
443	if bpp < MAX_LOW_INFO_BPP and area < MAX_LOW_INFO_AREA and not visibly_placed:
444	return False
445
446	return True
447
448
449	def _clip_rect_to_page(rect: fitz.Rect, page_rect: fitz.Rect) -> fitz.Rect:
450	"""Clamp a rectangle to the current PDF page."""
451	return fitz.Rect(
452	max(page_rect.x0, rect.x0),
453	max(page_rect.y0, rect.y0),
454	min(page_rect.x1, rect.x1),
455	min(page_rect.y1, rect.y1),
456	)
457
458
459	def _is_rect_contained(inner: fitz.Rect, outer: fitz.Rect, threshold: float = 0.9) -> bool:
460	"""Return whether ``inner`` is mostly covered by ``outer``."""
461	inner_area = inner.get_area()
462	if inner_area <= 0:
463	return False
464	return ((inner & outer).get_area() / inner_area) >= threshold
465
466
467	def _overlaps_table(rect: fitz.Rect, tab_rects: list[fitz.Rect]) -> bool:
468	"""Skip vector regions that are already handled as extracted tables."""
469	rect_area = rect.get_area()
470	if rect_area <= 0:
471	return False
472	for tab_rect in tab_rects:
473	intersection = rect & tab_rect
474	if intersection.get_area() > 0.5 * min(rect_area, tab_rect.get_area()):
475	return True
476	return False
477
478
479	def _is_white(color: tuple[float, ...] \| None) -> bool:
480	"""Return whether a PDF drawing color is visually white."""
481	if color is None:
482	return False
483	return all(channel >= 0.98 for channel in color[:3])
484
485
486	def _is_background_drawing(drawing: dict[str, object]) -> bool:
487	"""Identify white background rectangles that should not drive crops."""
488	return _is_white(drawing.get("fill")) and drawing.get("color") is None
489
490
491	def find_figure_caption_rects(page: fitz.Page) -> list[fitz.Rect]:
492	"""Return text-line rectangles that look like figure captions."""
493	caption_rects = []
494	for block in page.get_text("dict")["blocks"]:
495	if block.get("type") != 0:
496	continue
497	for line in block["lines"]:
498	text = "".join(span["text"] for span in line["spans"]).strip()
499	if FIGURE_CAPTION_RE.match(text):
500	caption_rects.append(fitz.Rect(line["bbox"]))
501	return caption_rects
502
503
504	def _expand_rect(rect: fitz.Rect, padding: float, page_rect: fitz.Rect) -> fitz.Rect:
505	"""Pad a rectangle and clamp it to the current PDF page."""
506	return _clip_rect_to_page(
507	fitz.Rect(
508	rect.x0 - padding,
509	rect.y0 - padding,
510	rect.x1 + padding,
511	rect.y1 + padding,
512	),
513	page_rect,
514	)
515
516
517	def _union_rects(rects: list[fitz.Rect]) -> fitz.Rect:
518	"""Return the bounding union for one or more rectangles."""
519	result = fitz.Rect(rects[0])
520	for rect in rects[1:]:
521	result \|= rect
522	return result
523
524
525	def _find_captioned_vector_figures(
526	page: fitz.Page,
527	drawing_rects: list[fitz.Rect],
528	background_rects: list[fitz.Rect],
529	caption_rects: list[fitz.Rect],
530	) -> list[fitz.Rect]:
531	"""Build tight figure crops from non-background drawings above captions."""
532	figure_rects = []
533	page_rect = page.rect
534
535	for caption_rect in caption_rects:
536	related = []
537	for rect in drawing_rects:
538	horizontal_gap = max(caption_rect.x0 - rect.x1, rect.x0 - caption_rect.x1, 0)
539	if horizontal_gap > VECTOR_CAPTION_HORIZONTAL_GAP:
540	continue
541	if rect.y0 > caption_rect.y0:
542	continue
543	if caption_rect.y0 - rect.y1 > VECTOR_CAPTION_SEARCH_HEIGHT:
544	continue
545	related.append(rect)
546
547	if not related:
548	continue
549
550	content_rect = _union_rects(related)
551	rect = content_rect
552	for background_rect in background_rects:
553	background_rect = _clip_rect_to_page(background_rect, page_rect)
554	if not _is_rect_contained(content_rect, background_rect, threshold=0.95):
555	continue
556	if background_rect.get_area() > content_rect.get_area() * MAX_VECTOR_BACKGROUND_AREA_RATIO:
557	continue
558	rect = background_rect
559	break
560
561	rect = _expand_rect(rect, 10, page_rect)
562	rect.y1 = min(rect.y1, caption_rect.y0 - 2)
563	if rect.width >= MIN_VECTOR_FIGURE_WIDTH and rect.height >= MIN_VECTOR_FIGURE_HEIGHT:
564	figure_rects.append(rect)
565
566	return figure_rects
567
568
569	def detect_vector_figure_rects(page: fitz.Page, tab_rects: list[fitz.Rect]) -> list[fitz.Rect]:
570	"""Detect large vector drawing regions that should be rasterized as figures.
571
572	Some academic PDFs store charts and diagrams as vector drawing commands,
573	not image XObjects. ``page.get_text("dict")`` exposes only raster image
574	blocks, so those figures need a separate drawing-region fallback.
575	"""
576	candidates = []
577	page_rect = page.rect
578	caption_rects = find_figure_caption_rects(page)
579	drawing_rects = []
580	background_rects = []
581
582	for drawing in page.get_drawings():
583	rect = drawing.get("rect")
584	if not rect:
585	continue
586
587	rect = fitz.Rect(rect)
588	if rect.is_empty:
589	continue
590
591	if _is_background_drawing(drawing):
592	background_rects.append(rect)
593	continue
594
595	drawing_rects.append(rect)
596
597	if caption_rects:
598	return _find_captioned_vector_figures(page, drawing_rects, background_rects, caption_rects)
599
600	for rect in drawing_rects:
601	rect = _expand_rect(rect, VECTOR_FIGURE_PADDING, page_rect)
602	rect = _clip_rect_to_page(rect, page_rect)
603
604	width = rect.width
605	height = rect.height
606	if width < MIN_VECTOR_FIGURE_WIDTH or height < MIN_VECTOR_FIGURE_HEIGHT:
607	continue
608
609	area = rect.get_area()
610	if area < MIN_VECTOR_FIGURE_AREA:
611	continue
612
613	aspect = max(width, height) / max(min(width, height), 1)
614	if aspect > MAX_VECTOR_FIGURE_ASPECT_RATIO:
615	continue
616
617	if _overlaps_table(rect, tab_rects):
618	continue
619
620	candidates.append(rect)
621
622	candidates.sort(key=lambda r: r.get_area(), reverse=True)
623
624	kept = []
625	for rect in candidates:
626	if any(_is_rect_contained(rect, existing) for existing in kept):
627	continue
628	kept.append(rect)
629
630	return sorted(kept, key=lambda r: (r.y0, r.x0))
631
632
633	def clean_text(text: str) -> str:
634	"""Clean extracted text."""
635	lines = text.split('\n')
636	cleaned_lines = []
637	prev_empty = False
638
639	for line in lines:
640	line = line.rstrip()
641	is_empty = len(line.strip()) == 0
642
643	if is_empty:
644	if not prev_empty:
645	cleaned_lines.append('')
646	prev_empty = True
647	else:
648	cleaned_lines.append(line)
649	prev_empty = False
650
651	return '\n'.join(cleaned_lines)
652
653
654	def _extract_text_lines(page: fitz.Page) -> list[tuple[fitz.Rect, str]]:
655	"""Return text lines with their page rectangles in reading order."""
656	lines = []
657	for block in page.get_text("dict")["blocks"]:
658	if block.get("type") != 0:
659	continue
660	for line in block["lines"]:
661	text = "".join(span["text"] for span in line["spans"]).strip()
662	if text:
663	lines.append((fitz.Rect(line["bbox"]), text))
664	return sorted(lines, key=lambda item: (item[0].y0, item[0].x0))
665
666
667	def _is_table_caption(text: str) -> bool:
668	"""Return whether a line is a real table caption, not prose citing a table."""
669	return bool(TABLE_CAPTION_RE.match(text.strip()))
670
671
672	def _caption_table_start_y(
673	caption_rect: fitz.Rect,
674	lines: list[tuple[fitz.Rect, str]],
675	) -> float:
676	"""Start below a caption and its adjacent English translation line."""
677	start_y = caption_rect.y1 + 1
678	for rect, text in lines:
679	if rect.y0 < caption_rect.y1 - 1 or rect.y0 > caption_rect.y1 + 35:
680	continue
681	if text.startswith("Table"):
682	start_y = max(start_y, rect.y1 + 1)
683	return start_y
684
685
686	def _looks_like_numeric_table_line(text: str) -> bool:
687	"""Detect long data rows so they are not mistaken for prose boundaries."""
688	numbers = NUMBER_RE.findall(text)
689	if len(numbers) >= 3:
690	return True
691	tokens = [token for token in re.split(r'\s+', text.strip()) if token]
692	return len(tokens) >= 4 and len(numbers) >= 2
693
694
695	def _is_table_region_boundary(
696	rect: fitz.Rect,
697	text: str,
698	page: fitz.Page,
699	start_y: float,
700	) -> bool:
701	"""Return whether a line likely starts prose after a text-detected table."""
702	text = text.strip()
703	if rect.y0 < start_y + 45:
704	return False
705	if text.startswith(TABLE_NOTE_PREFIX):
706	return True
707	if SECTION_HEADING_RE.match(text):
708	return True
709	if TABLE_REFERENCE_PROSE_RE.match(text):
710	return True
711	if _looks_like_numeric_table_line(text):
712	return False
713
714	width_ratio = rect.width / page.rect.width if page.rect.width > 0 else 0
715	return len(text) >= 26 and width_ratio > 0.52 and rect.x0 < page.rect.width * 0.25
716
717
718	def _table_region_bottom(
719	page: fitz.Page,
720	lines: list[tuple[fitz.Rect, str]],
721	start_y: float,
722	) -> float:
723	"""Find a conservative bottom edge for a caption-guided text table scan."""
724	for rect, text in lines:
725	if rect.y0 <= start_y:
726	continue
727	if _is_table_region_boundary(rect, text, page, start_y):
728	return max(start_y + 20, rect.y0 - 2)
729	return page.rect.height * TABLE_SCAN_BOTTOM_RATIO
730
731
732	def _normalize_table_cell(value: object) -> str:
733	"""Normalize one extracted table cell for Markdown output."""
734	if value is None:
735	return ""
736	text = CONTROL_CHARS_RE.sub('', str(value))
737	text = re.sub(r'\s\n\s', '<br>', text.strip())
738	text = re.sub(r'[ \t]+', ' ', text)
739	return text.replace('\|', r'\\|')
740
741
742	def _clean_table_rows(rows: list[list[object]]) -> list[list[str]]:
743	"""Remove empty rows / columns from PyMuPDF table extraction output."""
744	normalized = [[_normalize_table_cell(cell) for cell in row] for row in rows]
745	normalized = [row for row in normalized if any(cell for cell in row)]
746	if not normalized:
747	return []
748
749	max_cols = max(len(row) for row in normalized)
750	padded = [row + [""] * (max_cols - len(row)) for row in normalized]
751	keep_cols = [
752	idx
753	for idx in range(max_cols)
754	if any(row[idx] for row in padded)
755	]
756	if len(keep_cols) < 2:
757	return []
758	return _postprocess_table_rows([[row[idx] for idx in keep_cols] for row in padded])
759
760
761	def _nonempty_cell_indexes(row: list[str]) -> list[int]:
762	"""Return indexes of non-empty cells in a row."""
763	return [idx for idx, cell in enumerate(row) if cell]
764
765
766	def _merge_label_underscore_rows(rows: list[list[str]]) -> list[list[str]]:
767	"""Join rows where PDF extraction split a leading underscore from a label."""
768	merged = []
769	for row in rows:
770	nonempty = _nonempty_cell_indexes(row)
771	if nonempty == [0] and row[0] == "_" and merged and merged[-1][0]:
772	merged[-1][0] = f"_{merged[-1][0]}"
773	continue
774	merged.append(row)
775	return merged
776
777
778	def _merge_single_cell_continuations(rows: list[list[str]]) -> list[list[str]]:
779	"""Merge wrapped single-cell table rows into the previous row."""
780	merged: list[list[str]] = []
781	for row in rows:
782	nonempty = _nonempty_cell_indexes(row)
783	if (
784	len(nonempty) == 1
785	and nonempty[0] > 0
786	and merged
787	and not MODEL_COLUMN_RE.match(row[nonempty[0]])
788	):
789	index = nonempty[0]
790	separator = "<br>" if merged[-1][index] else ""
791	merged[-1][index] = f"{merged[-1][index]}{separator}{row[index]}"
792	continue
793	merged.append(row)
794	return merged
795
796
797	def _looks_like_model_row(row: list[str]) -> bool:
798	"""Return whether a row contains model-number table headings."""
799	values = [cell for cell in row[1:] if cell]
800	return len(values) >= 2 and all(MODEL_COLUMN_RE.match(cell) for cell in values)
801
802
803	def _looks_like_outcome_row(row: list[str]) -> bool:
804	"""Return whether a row contains regression outcome labels."""
805	values = [cell for cell in row[1:] if cell]
806	if len(values) < 2:
807	return False
808	short_values = [cell for cell in values if len(cell) <= 12 and not NUMBER_RE.search(cell)]
809	return len(short_values) == len(values)
810
811
812	def _regression_group_labels(row: list[str], data_cols: int) -> list[str]:
813	"""Infer repeated group labels for common regression-table headings."""
814	compact = "".join(row)
815	if "总样本" in compact and "国有" in compact and "非国有" in compact and data_cols == 7:
816	return ["总样本"] * 3 + ["国有企业"] * 2 + ["非国有企业"] * 2
817	return [""] * data_cols
818
819
820	def _flatten_regression_header(rows: list[list[str]]) -> list[list[str]]:
821	"""Flatten multi-line regression headings into one Markdown header row."""
822	if len(rows) < 3:
823	return rows
824
825	if (
826	len(rows) >= 2
827	and rows[0][0]
828	and not any(rows[0][1:])
829	and _looks_like_model_row(rows[1])
830	):
831	outcome = rows[0][0]
832	header = ["变量"]
833	header.extend(
834	f"{model} {outcome}".strip()
835	for model in rows[1][1:]
836	)
837	return [header] + rows[2:]
838
839	header_offset = 0
840	groups = [""] * (len(rows[0]) - 1)
841	if not _looks_like_model_row(rows[0]) and _looks_like_model_row(rows[1]):
842	header_offset = 1
843	groups = _regression_group_labels(rows[0], len(rows[1]) - 1)
844
845	if not _looks_like_model_row(rows[header_offset]):
846	return rows
847	if len(rows) <= header_offset + 1 or not _looks_like_outcome_row(rows[header_offset + 1]):
848	return rows
849
850	model_row = rows[header_offset]
851	outcome_row = rows[header_offset + 1]
852	header = ["变量"]
853	for idx, model in enumerate(model_row[1:]):
854	pieces = []
855	if idx < len(groups) and groups[idx]:
856	pieces.append(groups[idx])
857	if model:
858	pieces.append(model)
859	if idx + 1 < len(outcome_row) and outcome_row[idx + 1]:
860	pieces.append(outcome_row[idx + 1])
861	header.append(" ".join(pieces).strip())
862	return [header] + rows[header_offset + 2:]
863
864
865	def _fix_paired_sample_t_table(rows: list[list[str]]) -> list[list[str]]:
866	"""Collapse multi-row paired-sample T-test headings into readable columns."""
867	if not rows or not any("成对差分" in cell for cell in rows[0]):
868	return rows
869	body = [row for row in rows if row and row[0].startswith("对")]
870	if len(body) < 1:
871	return rows
872	header = [
873	"配对",
874	"变量",
875	"均值",
876	"标准差",
877	"均值的标准误",
878	"差分95%置信区间下限",
879	"差分95%置信区间上限",
880	"t",
881	"Df",
882	"Sig.(双侧)",
883	]
884	fixed_rows = [header]
885	for row in body:
886	fixed_rows.append(row[:len(header)] + [""] * max(0, len(header) - len(row)))
887	return fixed_rows
888
889
890	def _fix_variable_definition_table(rows: list[list[str]]) -> list[list[str]]:
891	"""Repeat variable-category labels for common variable definition tables."""
892	if not rows or rows[0] != ["变量类型", "变量名称", "符号", "变量说明"]:
893	return rows
894
895	fixed = [rows[0]]
896	for row in rows[1:]:
897	if not any(row):
898	continue
899	name = row[1] if len(row) > 1 else ""
900	symbol = row[2] if len(row) > 2 else ""
901	description = row[3] if len(row) > 3 else ""
902	if not name or not symbol:
903	continue
904
905	if symbol in {"R＆D", "Fixed", "Hc"}:
906	category = "被解释变量"
907	elif symbol == "Vat":
908	category = "解释变量"
909	else:
910	category = "控制变量"
911	fixed.append([category, name, symbol, description])
912	return fixed
913
914
915	def _fix_correlation_triangle(rows: list[list[str]]) -> list[list[str]]:
916	"""Restore the missing last self-correlation column in triangular tables."""
917	if len(rows) < 4 or not rows[0] or rows[0][0] != "变量":
918	return rows
919	body_names = [row[0] for row in rows[1:] if row and row[0]]
920	header_names = rows[0][1:]
921	if len(body_names) != len(header_names) + 1:
922	return rows
923	missing_name = body_names[-1]
924	fixed = [rows[0] + [missing_name]]
925	for row in rows[1:-1]:
926	fixed.append(row + [""])
927	fixed.append(rows[-1] + ["1"])
928	return fixed
929
930
931	def _postprocess_table_rows(rows: list[list[str]]) -> list[list[str]]:
932	"""Apply Markdown-oriented cleanup to extracted table rows."""
933	rows = _merge_label_underscore_rows(rows)
934	rows = _merge_single_cell_continuations(rows)
935	rows = _fix_variable_definition_table(rows)
936	rows = _fix_paired_sample_t_table(rows)
937	rows = _flatten_regression_header(rows)
938	rows = _fix_correlation_triangle(rows)
939	return rows
940
941
942	def _rows_to_markdown(rows: list[list[str]]) -> str:
943	"""Convert cleaned table rows to GitHub-flavored Markdown."""
944	if len(rows) < 2:
945	return ""
946	col_count = max(len(row) for row in rows)
947	padded = [row + [""] * (col_count - len(row)) for row in rows]
948	header = padded[0]
949	body = padded[1:]
950	lines = [
951	"\|" + "\|".join(header) + "\|",
952	"\|" + "\|".join(["---"] * col_count) + "\|",
953	]
954	lines.extend("\|" + "\|".join(row) + "\|" for row in body)
955	return "\n".join(lines)
956
957
958	def _table_to_markdown(tab: object) -> str:
959	"""Convert a PyMuPDF table object to cleaned Markdown."""
960	try:
961	rows = tab.extract() or []
962	except Exception:
963	return ""
964	return _rows_to_markdown(_clean_table_rows(rows))
965
966
967	def _is_valid_table_markdown(markdown: str) -> bool:
968	"""Return whether generated Markdown contains a minimally useful table."""
969	return markdown.count("\n") >= 2 and markdown.startswith("\|")
970
971
972	def _markdown_col_count(markdown: str) -> int:
973	"""Return the column count implied by the first Markdown table row."""
974	first_line = markdown.splitlines()[0] if markdown else ""
975	return max(0, first_line.count("\|") - 1)
976
977
978	def _append_table_markdown_candidate(
979	candidates: list[dict[str, object]],
980	bbox: fitz.Rect,
981	markdown: str,
982	method: str,
983	replace_narrow: bool = False,
984	) -> None:
985	"""Append or replace a table candidate after overlap deduplication."""
986	if not _is_valid_table_markdown(markdown):
987	return
988
989	for index, candidate in enumerate(candidates):
990	existing = candidate["bbox"]
991	if not isinstance(existing, fitz.Rect):
992	continue
993	overlap = (bbox & existing).get_area()
994	if overlap <= 0.8 * min(bbox.get_area(), existing.get_area()):
995	continue
996
997	existing_markdown = str(candidate.get("content", ""))
998	can_replace = (
999	replace_narrow
1000	and bbox.width > existing.width * 1.4
1001	and _markdown_col_count(markdown) >= _markdown_col_count(existing_markdown)
1002	)
1003	if can_replace:
1004	candidates[index] = {
1005	"bbox": bbox,
1006	"content": markdown,
1007	"method": method,
1008	}
1009	return
1010
1011	candidates.append({
1012	"bbox": bbox,
1013	"content": markdown,
1014	"method": method,
1015	})
1016
1017
1018	def _add_table_candidate(
1019	candidates: list[dict[str, object]],
1020	tab: object,
1021	method: str,
1022	) -> None:
1023	"""Append a table candidate if it has useful Markdown and is not duplicate."""
1024	markdown = _table_to_markdown(tab)
1025	if not _is_valid_table_markdown(markdown):
1026	return
1027
1028	bbox = fitz.Rect(tab.bbox)
1029	_append_table_markdown_candidate(candidates, bbox, markdown, method)
1030
1031
1032	def _merge_word_runs(words: list[tuple]) -> list[dict[str, object]]:
1033	"""Group PyMuPDF words into row-level text runs."""
1034	rows: list[list[tuple]] = []
1035	for word in sorted(words, key=lambda item: (item[1], item[0])):
1036	if not rows or abs(rows[-1][0][1] - word[1]) > 4:
1037	rows.append([word])
1038	else:
1039	rows[-1].append(word)
1040
1041	runs = []
1042	for row in rows:
1043	row_runs = []
1044	for word in sorted(row, key=lambda item: item[0]):
1045	x0, y0, x1, y1, text = word[:5]
1046	if row_runs and x0 - row_runs[-1]["x1"] <= 8:
1047	row_runs[-1]["x1"] = x1
1048	row_runs[-1]["y0"] = min(row_runs[-1]["y0"], y0)
1049	row_runs[-1]["y1"] = max(row_runs[-1]["y1"], y1)
1050	row_runs[-1]["text"] = f"{row_runs[-1]['text']} {text}"
1051	else:
1052	row_runs.append({
1053	"x0": x0,
1054	"y0": y0,
1055	"x1": x1,
1056	"y1": y1,
1057	"text": text,
1058	})
1059	runs.extend(row_runs)
1060	return runs
1061
1062
1063	def _cluster_word_columns(runs: list[dict[str, object]]) -> list[float]:
1064	"""Infer stable table columns from word-run centers."""
1065	centers = sorted((float(run["x0"]) + float(run["x1"])) / 2 for run in runs)
1066	columns: list[float] = []
1067	for center in centers:
1068	if not columns or abs(center - columns[-1]) > 14:
1069	columns.append(center)
1070	else:
1071	columns[-1] = (columns[-1] + center) / 2
1072	return columns
1073
1074
1075	def _word_runs_to_rows(
1076	runs: list[dict[str, object]],
1077	columns: list[float],
1078	) -> list[list[str]]:
1079	"""Place word runs into inferred columns and return table rows."""
1080	row_groups: list[list[dict[str, object]]] = []
1081	for run in sorted(runs, key=lambda item: (float(item["y0"]), float(item["x0"]))):
1082	if not row_groups or abs(float(row_groups[-1][0]["y0"]) - float(run["y0"])) > 4:
1083	row_groups.append([run])
1084	else:
1085	row_groups[-1].append(run)
1086
1087	rows = []
1088	for group in row_groups:
1089	row = [""] * len(columns)
1090	for run in group:
1091	center = (float(run["x0"]) + float(run["x1"])) / 2
1092	col_index = min(range(len(columns)), key=lambda idx: abs(columns[idx] - center))
1093	text = _normalize_table_cell(run["text"])
1094	row[col_index] = f"{row[col_index]} {text}".strip() if row[col_index] else text
1095	rows.append(row)
1096	return rows
1097
1098
1099	def _words_to_markdown_table(
1100	page: fitz.Page,
1101	clip: fitz.Rect,
1102	) -> tuple[fitz.Rect, str] \| None:
1103	"""Build a simple table from word coordinates inside a clipped region."""
1104	words = page.get_text("words", clip=clip)
1105	if len(words) < 6:
1106	return None
1107
1108	runs = _merge_word_runs(words)
1109	if len(runs) < 6:
1110	return None
1111
1112	columns = _cluster_word_columns(runs)
1113	if len(columns) < 3:
1114	return None
1115
1116	rows = _word_runs_to_rows(runs, columns)
1117	cleaned_rows = _clean_table_rows(rows)
1118	markdown = _rows_to_markdown(cleaned_rows)
1119	if not _is_valid_table_markdown(markdown):
1120	return None
1121
1122	x0 = min(float(run["x0"]) for run in runs)
1123	y0 = min(float(run["y0"]) for run in runs)
1124	x1 = max(float(run["x1"]) for run in runs)
1125	y1 = max(float(run["y1"]) for run in runs)
1126	return fitz.Rect(x0, y0, x1, y1), markdown
1127
1128
1129	def _find_tables_in_clip(
1130	page: fitz.Page,
1131	clip: fitz.Rect,
1132	) -> list[object]:
1133	"""Find text-strategy tables inside a clipped page region."""
1134	if clip.height < 20 or clip.width < 80:
1135	return []
1136	try:
1137	return list(page.find_tables(strategy="text", clip=clip))
1138	except Exception:
1139	return []
1140
1141
1142	def find_page_tables(
1143	page: fitz.Page,
1144	include_top_continuation: bool = False,
1145	) -> tuple[list[dict[str, object]], bool]:
1146	"""Find line-detected tables plus caption-guided text tables on one page."""
1147	candidates: list[dict[str, object]] = []
1148	try:
1149	for tab in page.find_tables():
1150	_add_table_candidate(candidates, tab, "lines")
1151	except Exception:
1152	pass
1153
1154	lines = _extract_text_lines(page)
1155	for rect, text in lines:
1156	if not _is_table_caption(text):
1157	continue
1158	start_y = _caption_table_start_y(rect, lines)
1159	bottom_y = _table_region_bottom(page, lines, start_y)
1160	clip = fitz.Rect(0, start_y, page.rect.width, bottom_y)
1161	for tab in _find_tables_in_clip(page, clip):
1162	_add_table_candidate(candidates, tab, "caption-text")
1163	word_table = _words_to_markdown_table(page, clip)
1164	if word_table:
1165	bbox, markdown = word_table
1166	_append_table_markdown_candidate(
1167	candidates,
1168	bbox,
1169	markdown,
1170	"caption-words",
1171	replace_narrow=True,
1172	)
1173
1174	if include_top_continuation:
1175	start_y = page.rect.height * 0.08
1176	bottom_y = _table_region_bottom(page, lines, start_y)
1177	clip = fitz.Rect(0, start_y, page.rect.width, bottom_y)
1178	for tab in _find_tables_in_clip(page, clip):
1179	_add_table_candidate(candidates, tab, "continuation-text")
1180	word_table = _words_to_markdown_table(page, clip)
1181	if word_table:
1182	bbox, markdown = word_table
1183	_append_table_markdown_candidate(
1184	candidates,
1185	bbox,
1186	markdown,
1187	"continuation-words",
1188	replace_narrow=True,
1189	)
1190
1191	candidates.sort(key=lambda candidate: candidate["bbox"].y0)
1192	table_continues = any(
1193	isinstance(candidate["bbox"], fitz.Rect)
1194	and candidate["bbox"].y1 >= page.rect.height * TABLE_CONTINUATION_Y_RATIO
1195	for candidate in candidates
1196	)
1197	return candidates, table_continues
1198
1199
1200	def _is_markdown_table_line(line: str) -> bool:
1201	"""Return whether a Markdown line belongs to a pipe table."""
1202	return line.startswith("\|")
1203
1204
1205	def _is_markdown_separator_line(line: str) -> bool:
1206	"""Return whether a Markdown table line is the separator row."""
1207	cells = [cell.strip() for cell in line.strip("\|").split("\|")]
1208	return bool(cells) and all(cell and set(cell) <= {"-", ":"} for cell in cells)
1209
1210
1211	def _compatible_table_headers(first: list[str], second: list[str]) -> bool:
1212	"""Return whether two Markdown table blocks have the same flattened header."""
1213	if len(first) < 2 or len(second) < 2:
1214	return False
1215	if not _is_markdown_separator_line(first[1]) or not _is_markdown_separator_line(second[1]):
1216	return False
1217	return first[0] == second[0] and _markdown_col_count(first[0]) > 2
1218
1219
1220	def _is_table_continuation_noise(line: str) -> bool:
1221	"""Allow only page/header noise between split table parts."""
1222	text = line.strip()
1223	if not text:
1224	return True
1225	if text.startswith("<!-- Page ") and text.endswith("-->"):
1226	return True
1227	if re.fullmatch(r'\d+', text):
1228	return True
1229	if "重庆大学硕士学位论文" in text:
1230	return True
1231	continuation_labels = [
1232	"营改增",
1233	"深化增值税改革",
1234	"国有企业",
1235	"非国有企业",
1236	]
1237	return any(label in text for label in continuation_labels)
1238
1239
1240	def _read_markdown_table_block(lines: list[str], start: int) -> tuple[list[str], int]:
1241	"""Read a contiguous Markdown table block from ``start``."""
1242	end = start
1243	while end < len(lines) and _is_markdown_table_line(lines[end]):
1244	end += 1
1245	return lines[start:end], end
1246
1247
1248	def merge_markdown_continuation_tables(markdown: str) -> str:
1249	"""Merge split cross-page Markdown tables with repeated headers."""
1250	lines = markdown.splitlines()
1251	result = []
1252	index = 0
1253
1254	while index < len(lines):
1255	if not _is_markdown_table_line(lines[index]):
1256	result.append(lines[index])
1257	index += 1
1258	continue
1259
1260	table, table_end = _read_markdown_table_block(lines, index)
1261	search = table_end
1262	while True:
1263	between_start = search
1264	while search < len(lines) and not _is_markdown_table_line(lines[search]):
1265	if not _is_table_continuation_noise(lines[search]):
1266	break
1267	search += 1
1268
1269	if search >= len(lines) or not _is_markdown_table_line(lines[search]):
1270	break
1271	if any(
1272	not _is_table_continuation_noise(line)
1273	for line in lines[between_start:search]
1274	):
1275	break
1276
1277	next_table, next_end = _read_markdown_table_block(lines, search)
1278	if not _compatible_table_headers(table, next_table):
1279	break
1280
1281	table.extend(next_table[2:])
1282	search = next_end
1283
1284	result.extend(table)
1285	index = search if search != table_end else table_end
1286
1287	return "\n".join(result)
1288
1289
1290	def merge_adjacent_formatting(text: str) -> str:
1291	"""Merge adjacent same-style formatted spans split across PDF tokens.
1292
1293	PyMuPDF often emits a phrase as several spans, so per-span wrapping in
1294	``format_span_text`` produces ``XY`` (bold) or ``*X**Y*``
1295	(bold-italic) where one phrase is intended. Collapse the abutting markers
1296	so the run reads as a single phrase: ``X Y`` / ``*X Y*``.
1297
1298	Italic-italic adjacency (``XY``) is indistinguishable from a plain
1299	bold span's open/close pair and is left alone — merging it would corrupt
1300	every legitimate ``bold`` phrase on the page. The previous regexes
1301	``\\\\s\\`` and ``\\\\\\\\s\\\\\\`` did exactly that, deleting
1302	all bold formatting from the converted Markdown.
1303	"""
1304	# Bold-italic adjacency first so the inner ``******`` isn't half-eaten
1305	# by the bold pass.
1306	text = re.sub(r'\*{6}', ' ', text)
1307	text = re.sub(r'\*{4}', ' ', text)
1308	return text
1309
1310
1311	def is_sentence_end(text: str) -> bool:
1312	"""Check if the text ends with sentence-ending punctuation."""
1313	text = text.rstrip()
1314	if not text:
1315	return True
1316	end_puncts = '.。!！?？:：;；'
1317	return text[-1] in end_puncts
1318
1319
1320	def should_merge_lines(current: dict, next_line: dict) -> bool:
1321	"""Determine if two lines should be merged into the same paragraph."""
1322	if current.get("is_heading") or next_line.get("is_heading"):
1323	return False
1324	if current.get("is_list") or next_line.get("is_list"):
1325	return False
1326	if is_sentence_end(current.get("content", "")):
1327	return False
1328	return True
1329
1330
1331	def extract_pdf_to_markdown(
1332	pdf_path: str,
1333	output_path: str = None,
1334	images: str = "filtered",
1335	render_vector_figures: bool = False,
1336	vector_figure_dpi: int = VECTOR_FIGURE_DPI,
1337	) -> str:
1338	"""Extract text, images, and tables from a PDF and convert to Markdown.
1339
1340	Args:
1341	pdf_path: Path to the PDF file.
1342	output_path: Optional output path for the Markdown file.
1343	images: Image extraction mode.
1344	"filtered" = apply size/quality filters (default),
1345	"all" = extract all images without filtering,
1346	"none" = skip all images.
1347	render_vector_figures: Rasterize large vector drawing regions as PNGs.
1348	vector_figure_dpi: DPI used for rendered vector figure PNGs.
1349	"""
1350	try:
1351	doc = fitz.open(pdf_path)
1352	except Exception as e:
1353	print(f"[ERROR] Failed to open PDF file: {e}")
1354	return ""
1355
1356	if len(doc) >= 200:
1357	print(f"[HINT] {len(doc)} pages — for very large PDFs, consider splitting "
1358	f"the source by chapter beforehand (e.g. with pdftk / qpdf / PyPDF2) "
1359	f"and converting each part individually.")
1360
1361	filename = Path(pdf_path).stem
1362	title = re.sub(r'^\d+-', '', filename).strip()
1363
1364	print(f"[INFO] Analyzing document structure...")
1365	size_map = analyze_font_sizes(doc)
1366	print(f" Font size mapping: body={size_map.get('body', 'N/A')}, " +
1367	f"H1={size_map.get('h1', 'N/A')}, H2={size_map.get('h2', 'N/A')}, H3={size_map.get('h3', 'N/A')}")
1368
1369	print(f"[INFO] Detecting repeated headers/footers...")
1370	noise_texts = detect_headers_footers(doc)
1371	if noise_texts:
1372	print(f" Found {len(noise_texts)} repeated noise texts (will be removed):")
1373	for t in list(noise_texts)[:3]:
1374	print(f" - {t[:30]}...")
1375
1376	markdown_content = f"# {title}\n\n"
1377	seen_image_hashes = set() # Track seen image hashes for deduplication
1378
1379	img_dir = None
1380	rel_img_dir = None
1381	if output_path:
1382	output_path = Path(output_path)
1383	rel_img_dir = f"{output_path.stem}_files"
1384	img_dir = output_path.parent / rel_img_dir
1385
1386	img_count = 0
1387	image_manifest: list[dict[str, object]] = []
1388	previous_table_continues = False
1389
1390	for page_num, page in enumerate(doc, 1):
1391	if page_num > 1:
1392	# Add page break marker to help LLM understand context segmentation
1393	markdown_content += f"\n\n<!-- Page {page_num} -->\n\n"
1394
1395	table_candidates, previous_table_continues = find_page_tables(
1396	page,
1397	include_top_continuation=previous_table_continues,
1398	)
1399	tab_rects = [
1400	candidate["bbox"]
1401	for candidate in table_candidates
1402	if isinstance(candidate["bbox"], fitz.Rect)
1403	]
1404
1405	page_elements = []
1406
1407	for table in table_candidates:
1408	bbox = table["bbox"]
1409	if not isinstance(bbox, fitz.Rect):
1410	continue
1411	page_elements.append({
1412	"y0": bbox.y0,
1413	"type": 2,
1414	"content": table["content"]
1415	})
1416	print(f" [OK] Found table: P{page_num} ({table['method']})")
1417
1418	if render_vector_figures:
1419	for figure_rect in detect_vector_figure_rects(page, tab_rects):
1420	page_elements.append({
1421	"y0": figure_rect.y0,
1422	"type": 3,
1423	"content": figure_rect,
1424	})
1425	print(f" [OK] Found vector figure region: P{page_num} {tuple(round(v, 1) for v in figure_rect)}")
1426
1427	blocks = page.get_text("dict")["blocks"]
1428
1429	for block in blocks:
1430	block_rect = fitz.Rect(block["bbox"])
1431
1432	# Check if this is table content
1433	is_in_table = False
1434	for tab_rect in tab_rects:
1435	intersect = block_rect & tab_rect
1436	if intersect.get_area() > 0.6 * block_rect.get_area():
1437	is_in_table = True
1438	break
1439
1440	if is_in_table:
1441	continue
1442
1443	if block["type"] == 0:
1444	# Check if this is noise text to be filtered (whole block match)
1445	block_text_full = "".join([span["text"] for line in block["lines"] for span in line["spans"]]).strip()
1446	if block_text_full in noise_texts:
1447	continue
1448
1449	for line in block["lines"]:
1450	line_text = ""
1451	line_size = 0
1452	line_flags = 0
1453	span_count = 0
1454	is_code_line = False
1455
1456	formatted_spans = []
1457	for span in line["spans"]:
1458	span_text = CONTROL_CHARS_RE.sub('', span["text"])
1459	if not span_text.strip():
1460	if span_text:
1461	formatted_spans.append(span_text)
1462	continue
1463
1464	span_size = span["size"]
1465	span_flags = span["flags"]
1466
1467	line_size = max(line_size, span_size)
1468	line_flags \|= span_flags
1469	span_count += 1
1470
1471	heading_level = get_heading_level(span_size, size_map, span_text, span_flags)
1472
1473	# Detect code font
1474	font_name = span.get("font", "")
1475	if is_monospace_font(font_name):
1476	is_code_line = True
1477	formatted_spans.append(span_text) # No formatting for code
1478	elif heading_level > 0:
1479	formatted_spans.append(span_text.strip())
1480	else:
1481	formatted_spans.append(format_span_text(span_text, span_flags))
1482
1483	line_text = ''.join(formatted_spans).strip()
1484	if not line_text:
1485	continue
1486
1487	# Secondary check: line-level noise match (sometimes blocks are split)
1488	if line_text in noise_texts:
1489	continue
1490
1491	line_text = merge_adjacent_formatting(line_text)
1492
1493	heading_level = get_heading_level(line_size, size_map, line_text, line_flags)
1494
1495	is_list, list_type, list_content = detect_list_item(line_text)
1496
1497	if heading_level > 0:
1498	prefix = '#' * heading_level + ' '
1499	clean_line = re.sub(r'\+([^]+)\*+', r'\1', line_text)
1500	final_text = prefix + clean_line
1501	elif is_list:
1502	final_text = list_content
1503	else:
1504	final_text = line_text
1505
1506	page_elements.append({
1507	"y0": line["bbox"][1],
1508	"type": 0,
1509	"content": final_text,
1510	"is_heading": heading_level > 0,
1511	"is_list": is_list,
1512	"is_code": is_code_line
1513	})
1514
1515	elif block["type"] == 1:
1516	if images == "none":
1517	pass
1518	elif images == "all" or should_keep_image(block, page.rect, seen_image_hashes):
1519	page_elements.append({
1520	"y0": block["bbox"][1],
1521	"type": 1,
1522	"content": block
1523	})
1524	else:
1525	w, h = block.get("width", 0), block.get("height", 0)
1526	print(f" [SKIP] Filtered small/decorative image: {w}x{h}px, {len(block.get('image', b''))} bytes")
1527
1528	page_elements.sort(key=lambda x: x["y0"])
1529
1530	# Merge adjacent same-level short headings
1531	page_elements = merge_adjacent_headings(page_elements)
1532
1533	merged_elements = []
1534	i = 0
1535	while i < len(page_elements):
1536	el = page_elements[i]
1537	if el["type"] == 0 and not el.get("is_heading") and not el.get("is_list"):
1538	merged_content = el["content"]
1539	j = i + 1
1540	while j < len(page_elements):
1541	next_el = page_elements[j]
1542	if next_el["type"] != 0:
1543	break
1544	if not should_merge_lines({"content": merged_content, "is_heading": False, "is_list": False}, next_el):
1545	break
1546	merged_content += " " + next_el["content"]
1547	j += 1
1548	merged_elements.append({
1549	"type": 0,
1550	"content": remove_page_footer(merged_content),
1551	"is_heading": False,
1552	"is_list": False
1553	})
1554	i = j
1555	else:
1556	merged_elements.append(el)
1557	i += 1
1558
1559	prev_was_list = False
1560	prev_was_code = False
1561	code_block_lines = []
1562
1563	def flush_code_block():
1564	"""Flush accumulated code block."""
1565	nonlocal code_block_lines, markdown_content
1566	if code_block_lines:
1567	markdown_content += "```\n"
1568	markdown_content += "\n".join(code_block_lines) + "\n"
1569	markdown_content += "```\n\n"
1570	code_block_lines = []
1571
1572	for el in merged_elements:
1573	if el["type"] == 0:
1574	is_list = el.get("is_list", False)
1575	is_heading = el.get("is_heading", False)
1576	is_code = el.get("is_code", False)
1577
1578	if is_code:
1579	# Accumulate code lines
1580	if prev_was_list:
1581	markdown_content += "\n"
1582	prev_was_list = False
1583	code_block_lines.append(el["content"])
1584	prev_was_code = True
1585	else:
1586	# Non-code line, flush accumulated code block first
1587	if prev_was_code:
1588	flush_code_block()
1589	prev_was_code = False
1590
1591	if is_heading:
1592	if prev_was_list:
1593	markdown_content += "\n"
1594	markdown_content += el["content"] + "\n\n"
1595	prev_was_list = False
1596	elif is_list:
1597	markdown_content += el["content"] + "\n"
1598	prev_was_list = True
1599	else:
1600	if prev_was_list:
1601	markdown_content += "\n"
1602	markdown_content += el["content"] + "\n\n"
1603	prev_was_list = False
1604
1605	elif el["type"] == 2:
1606	if prev_was_code:
1607	flush_code_block()
1608	prev_was_code = False
1609	if prev_was_list:
1610	markdown_content += "\n"
1611	markdown_content += el["content"] + "\n\n"
1612	prev_was_list = False
1613
1614	elif el["type"] == 1:
1615	if prev_was_code:
1616	flush_code_block()
1617	prev_was_code = False
1618	if img_dir:
1619	block = el["content"]
1620	ext = block["ext"]
1621	image_data = block["image"]
1622	safe_filename = filename.replace(" ", "_")
1623	image_name = f"{safe_filename}_p{page_num}_{img_count}.{ext}"
1624	image_path = img_dir / image_name
1625
1626	try:
1627	img_dir.mkdir(parents=True, exist_ok=True)
1628	with open(image_path, "wb") as f:
1629	f.write(image_data)
1630
1631	if prev_was_list:
1632	markdown_content += "\n"
1633	markdown_content += f"![{image_name}]({rel_img_dir}/{image_name})\n\n"
1634	width = int(block.get("width", 0) or 0)
1635	height = int(block.get("height", 0) or 0)
1636	ratio = width / height if width > 0 and height > 0 else None
1637	image_manifest.append({
1638	"index": len(image_manifest) + 1,
1639	"filename": image_name,
1640	"original_filename": image_name,
1641	"asset_kind": "bitmap",
1642	"svg_renderable": True,
1643	"pptx_native_supported": True,
1644	"source_kind": "pdf_image",
1645	"source_ext": f".{ext}",
1646	"page_index": page_num,
1647	"occurrence_index": img_count + 1,
1648	"pixel_width": width or None,
1649	"pixel_height": height or None,
1650	"pixel_ratio": round(ratio, 6) if ratio else None,
1651	"display_ratio": round(ratio, 6) if ratio else None,
1652	"source_sha256": hashlib.sha256(image_data).hexdigest(),
1653	"bbox": list(block.get("bbox", [])),
1654	})
1655	img_count += 1
1656	prev_was_list = False
1657	print(f" [OK] Extracted image: {image_name}")
1658	except Exception as e:
1659	print(f" [WARN] Failed to save image: {e}")
1660
1661	elif el["type"] == 3:
1662	if prev_was_code:
1663	flush_code_block()
1664	prev_was_code = False
1665	if img_dir:
1666	figure_rect = el["content"]
1667	safe_filename = filename.replace(" ", "_")
1668	image_name = f"{safe_filename}_p{page_num}_figure_{img_count}.png"
1669	image_path = img_dir / image_name
1670
1671	try:
1672	img_dir.mkdir(parents=True, exist_ok=True)
1673	scale = vector_figure_dpi / 72
1674	pix = page.get_pixmap(
1675	matrix=fitz.Matrix(scale, scale),
1676	clip=figure_rect,
1677	alpha=False,
1678	)
1679	pix.save(str(image_path))
1680
1681	if prev_was_list:
1682	markdown_content += "\n"
1683	markdown_content += f"![{image_name}]({rel_img_dir}/{image_name})\n\n"
1684	ratio = pix.width / pix.height if pix.width > 0 and pix.height > 0 else None
1685	image_manifest.append({
1686	"index": len(image_manifest) + 1,
1687	"filename": image_name,
1688	"original_filename": image_name,
1689	"asset_kind": "bitmap",
1690	"svg_renderable": True,
1691	"pptx_native_supported": True,
1692	"source_kind": "pdf_vector_figure",
1693	"source_ext": ".png",
1694	"page_index": page_num,
1695	"occurrence_index": img_count + 1,
1696	"pixel_width": pix.width,
1697	"pixel_height": pix.height,
1698	"pixel_ratio": round(ratio, 6) if ratio else None,
1699	"display_ratio": round(ratio, 6) if ratio else None,
1700	"bbox": [
1701	figure_rect.x0,
1702	figure_rect.y0,
1703	figure_rect.x1,
1704	figure_rect.y1,
1705	],
1706	})
1707	img_count += 1
1708	prev_was_list = False
1709	print(f" [OK] Rendered vector figure: {image_name}")
1710	except Exception as e:
1711	print(f" [WARN] Failed to render vector figure: {e}")
1712
1713	# Flush code block at end of page
1714	if prev_was_code:
1715	flush_code_block()
1716
1717	doc.close()
1718
1719	markdown_content = merge_markdown_continuation_tables(markdown_content)
1720	markdown_content = CONTROL_CHARS_RE.sub('', markdown_content)
1721	markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content)
1722	markdown_content = markdown_content.strip() + "\n"
1723
1724	if output_path:
1725	os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
1726	with open(output_path, 'w', encoding='utf-8') as f:
1727	f.write(markdown_content)
1728	if img_dir and image_manifest:
1729	(img_dir / "image_manifest.json").write_text(
1730	json.dumps(image_manifest, ensure_ascii=False, indent=2) + "\n",
1731	encoding="utf-8",
1732	)
1733	profile_path = write_conversion_profile_best_effort(
1734	input_path=pdf_path,
1735	markdown_path=output_path,
1736	converter="pdf_to_md.py",
1737	conversion_type="pdf",
1738	asset_dir=img_dir,
1739	)
1740	print(f"[OK] Saved Markdown to: {output_path}")
1741	if profile_path:
1742	print(f" Wrote conversion profile -> {profile_path}")
1743
1744	return markdown_content
1745
1746
1747	def main() -> int:
1748	"""Run the CLI entry point."""
1749	parser = argparse.ArgumentParser(
1750	description='PDF to Markdown converter (with structure detection and LLM optimization)',
1751	formatter_class=argparse.RawDescriptionHelpFormatter,
1752	epilog='''
1753	Examples:
1754	python pdf_to_md.py book.pdf # Convert a single file
1755	python pdf_to_md.py book.pdf appendix.pdf # Convert multiple files
1756	python pdf_to_md.py ./pdfs -o ./markdown # Convert PDFs in a directory
1757	python pdf_to_md.py book.pdf -o output.md # Specify output file
1758	python pdf_to_md.py book.pdf --render-vector-figures
1759
1760	Structure detection features:
1761	- Auto-detect heading levels (based on font size)
1762	- Detect bold and italic text
1763	- Detect ordered and unordered lists
1764	- Extract tables and convert to Markdown format (with deduplication)
1765	- [New] Smart detection and removal of repeated page headers/footers
1766	- [New] Add <!-- Page N --> page break markers to help LLM understanding
1767	'''
1768	)
1769
1770	parser.add_argument('inputs', nargs='+', help='PDF file(s) or directories')
1771	parser.add_argument(
1772	'-o',
1773	'--output',
1774	help='Output Markdown file for one input, or output directory for multiple inputs/directories',
1775	)
1776	parser.add_argument(
1777	'--images',
1778	choices=['all', 'filtered', 'none'],
1779	default='filtered',
1780	help='Image extraction mode: filtered=apply size/quality filters (default), all=no filtering, none=skip images',
1781	)
1782	parser.add_argument(
1783	'--render-vector-figures',
1784	action='store_true',
1785	help='Render large vector drawing regions as PNG figure assets',
1786	)
1787	parser.add_argument(
1788	'--vector-figure-dpi',
1789	type=int,
1790	default=VECTOR_FIGURE_DPI,
1791	help=f'DPI for --render-vector-figures output (default: {VECTOR_FIGURE_DPI})',
1792	)
1793
1794	args = parser.parse_args()
1795
1796	return run_path_batch(
1797	args.inputs,
1798	{'.pdf'},
1799	args.output,
1800	lambda source, output: bool(
1801	extract_pdf_to_markdown(
1802	str(source),
1803	str(output),
1804	images=args.images,
1805	render_vector_figures=args.render_vector_figures,
1806	vector_figure_dpi=args.vector_figure_dpi,
1807	)
1808	),
1809	)
1810
1811
1812	if __name__ == '__main__':
1813	raise SystemExit(main())
1814

1814 lines PYTHON