MABELab-RS/porting/tools/chunker.py

18 lines
487 B
Python

from typing import List
def chunk_text(s: str, max_chars: int) -> List[str]:
if len(s) <= max_chars:
return [s]
chunks = []
start = 0
while start < len(s):
end = min(len(s), start + max_chars)
# try to split on function boundary or newline
newline = s.rfind("\n", start, end)
if newline == -1 or newline <= start + 1000:
newline = end
chunks.append(s[start:newline])
start = newline
return chunks