You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
21 lines
676 B
Python
21 lines
676 B
Python
# backend/app/rag/chunk.py
|
|
import re
|
|
|
|
|
|
def split_chunks(text: str, size: int = 400, overlap: int = 60) -> list[str]:
|
|
"""한국어 문단/문장 경계 우선 분할. 데모 시드 본문은 짧아 1청크."""
|
|
text = (text or "").strip()
|
|
if len(text) <= size:
|
|
return [text] if text else []
|
|
sentences = re.split(r"(?<=[.!?。])\s+|\n+", text)
|
|
chunks, cur = [], ""
|
|
for sent in sentences:
|
|
if len(cur) + len(sent) > size and cur:
|
|
chunks.append(cur.strip())
|
|
cur = cur[-overlap:] + sent
|
|
else:
|
|
cur = (cur + " " + sent).strip()
|
|
if cur.strip():
|
|
chunks.append(cur.strip())
|
|
return chunks
|