✂️ The Garbage-In, Garbage-Out Dilemma: Why Document Parsing & Chunking Define RAG Quality
A Retrieval-Augmented Generation (RAG) system is only as effective as the quality of its underlying vector index. Naive text splitting—such as blindly slicing documents every 500 characters—frequently severs sentences mid-thought, separates table headers from numerical rows, and strips critical hierarchical context. Advanced Document Parsing and Chunking Strategies address these challenges by using semantic boundary detection, markdown heading preservation, layout-aware OCR, and parent-child hierarchical indexing to ensure that retrieved context windows contain complete, coherent thoughts.
from unstructured.partition.pdf import partition_pdf
from langchain_text_splitters import MarkdownHeaderTextSplitter
# 1. Layout-Aware PDF Parsing (Preserves Tables & Headings)
elements = partition_pdf(
filename="annual_report_2025.pdf",
strategy="hi_res",
infer_table_structure=True,
chunking_strategy="by_title",
max_characters=1500,
new_after_n_chars=1000,
combine_text_under_n_chars=200,
)
# 2. Extract structured Markdown elements
markdown_text = "\n\n".join([el.text for el in elements if el.text])
# 3. Split by Markdown Headers (# H1, ## H2) to respect document boundaries
headers_to_split_on = [
("#", "Header 1"),
("##", "Header 2"),
("###", "Header 3"),
]
markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
md_chunks = markdown_splitter.split_text(markdown_text)
print(f"[✔] Generated {len(md_chunks)} structure-preserving Markdown chunks.")
import numpy as np
from sentence_transformers import SentenceMetrics, SentenceTransformer
model = SentenceTransformer("all-MiniLM-L6-v2")
def semantic_chunking(sentences, threshold_percentile=85):
"""
Groups sentences into semantic chunks based on embedding dissimilarity spikes.
"""
embeddings = model.encode(sentences)
# Calculate cosine distances between adjacent sentences
distances = []
for i in range(len(embeddings) - 1):
sim = np.dot(embeddings[i], embeddings[i+1]) / (
np.linalg.norm(embeddings[i]) * np.linalg.norm(embeddings[i+1])
)
distances.append(1.0 - sim)
# Determine threshold via percentile cutoff
threshold = np.percentile(distances, threshold_percentile)
chunks, current_chunk = [], [sentences[0]]
for i, dist in enumerate(distances):
if dist > threshold:
chunks.append(" ".join(current_chunk))
current_chunk = [sentences[i+1]]
else:
current_chunk.append(sentences[i+1])
if current_chunk:
chunks.append(" ".join(current_chunk))
return chunks
from langchain.retrievers import ParentDocumentRetriever
from langchain.storage import InMemoryStore
from langchain_community.vectorstores import Chroma
from langchain_text_splitters import RecursiveCharacterTextSplitter
# Parent splitter for rich context generation
parent_splitter = RecursiveCharacterTextSplitter(chunk_size=1200)
# Child splitter for precise vector similarity
child_splitter = RecursiveCharacterTextSplitter(chunk_size=300)
vectorstore = Chroma(collection_name="split_parents")
store = InMemoryStore()
retriever = ParentDocumentRetriever(
vectorstore=vectorstore,
docstore=store,
child_splitter=child_splitter,
parent_splitter=parent_splitter,
)
# When queried, vector search matches precise child snippets,
# but the retriever injects the complete parent document into the prompt.
import fitz # PyMuPDF
def extract_multicolumn_text(pdf_path):
doc = fitz.open(pdf_path)
extracted_text = ""
for page in doc:
# Extract blocks sorted by vertical (y0) and horizontal (x0) coordinates
blocks = page.get_text("blocks", sort=True)
for b in blocks:
# b = (x0, y0, x1, y1, text, block_no, block_type)
extracted_text += b[4] + "\n"
return extracted_text
import ast
def chunk_python_code(file_path):
with open(file_path, "r") as f:
tree = ast.parse(f.read())
chunks = []
for node in ast.walk(tree):
if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
# Extract source code segment for function or class
segment = ast.unparse(node)
chunks.append({
"name": node.name,
"type": type(node).__name__,
"code": segment
})
return chunks
import re
def split_ehr_sections(clinical_note):
pattern = r"(CHIEF COMPLAINT:|HISTORY OF PRESENT ILLNESS:|MEDICATIONS:|ALLERGIES:)"
splits = re.split(pattern, clinical_note)
sections = {}
for i in range(1, len(splits), 2):
header = splits[i].strip()
content = splits[i+1].strip()
sections[header] = content
return sections
def parse_email_thread(raw_mbox):
# Splits email thread by "From: " headers and extracts sender, timestamp, and body
messages = raw_mbox.split("From: ")
parsed_chunks = []
for msg in messages:
if not msg.strip(): continue
lines = msg.split("\n")
sender = lines[0]
body = "\n".join(lines[3:])
parsed_chunks.append({
"content": f"Sender: {sender}\nMessage: {body}",
"metadata": {"sender": sender}
})
return parsed_chunks
from bs4 import BeautifulSoup
def parse_html_to_sections(html_content):
soup = BeautifulSoup(html_content, 'html.parser')
sections = []
# Find all h1 and h2 tags as section dividers
headings = soup.find_all(['h1', 'h2', 'h3'])
for h in headings:
section_title = h.get_text()
section_content = []
# Gather sibling elements until next heading
for sibling in h.find_next_siblings():
if sibling.name in ['h1', 'h2', 'h3']:
break
section_content.append(str(sibling))
sections.append({
"title": section_title,
"html": "".join(section_content)
})
return sections
import json
def chunk_stix_bundle(stix_json_str):
bundle = json.loads(stix_json_str)
chunks = []
for obj in bundle.get("objects", []):
# Create an independent chunk for each threat indicator or attack pattern object
chunk_text = f"Object Type: {obj.get('type')}\nID: {obj.get('id')}\nDescription: {obj.get('description', 'N/A')}"
chunks.append({
"content": chunk_text,
"metadata": {"object_id": obj.get("id"), "stix_type": obj.get("type")}
})
return chunks
import tiktoken
def token_sliding_window_chunker(text, max_tokens=500, overlap=50):
enc = tiktoken.encoding_for_model("gpt-4o")
tokens = enc.encode(text)
chunks = []
start = 0
while start < len(tokens):
end = start + max_tokens
chunk_tokens = tokens[start:end]
chunks.append(enc.decode(chunk_tokens))
start += (max_tokens - overlap)
return chunks