Spaces:
Paused
Paused
| /** | |
| * SYNC 副本（demo 不抽共享包；改站内源请同步本文件）： | |
| * client/src/shared/cross/semanticUtils.ts → splitTextToChunks 及相关纯函数 | |
| */ | |
| globalThis.IL_splitTextToChunks = (function () { | |
| const encoder = new TextEncoder(); | |
| function getUtf8ByteLength(text, buf) { | |
| const { read, written } = encoder.encodeInto(text, buf); | |
| return read < text.length ? buf.length : written; | |
| } | |
| function nextParagraphEnd(text, start) { | |
| const nl = text.indexOf('\n\n', start); | |
| if (nl === -1) return text.length; | |
| let end = nl + 2; | |
| while (end < text.length && text[end] === '\n') end++; | |
| return end; | |
| } | |
| function nextLineEnd(text, start) { | |
| const nl = text.indexOf('\n', start); | |
| if (nl === -1) return text.length; | |
| let end = nl + 1; | |
| while (end < text.length && text[end] === '\n') end++; | |
| return end; | |
| } | |
| function charIndexForByteLimit(text, start, byteLimit) { | |
| const buf = new Uint8Array(4); | |
| let bytes = 0; | |
| let i = start; | |
| while (i < text.length) { | |
| const cp = text.codePointAt(i); | |
| const charLen = cp > 0xffff ? 2 : 1; | |
| const byteLen = encoder.encodeInto(text.slice(i, i + charLen), buf).written; | |
| if (bytes + byteLen > byteLimit) break; | |
| bytes += byteLen; | |
| i += charLen; | |
| } | |
| return i; | |
| } | |
| const SEPARATOR_GROUPS = [ | |
| ['。', '！', '？', '…'], | |
| ['；', '，'], | |
| ['.', '!', '?'], | |
| [';', ','], | |
| [' ', '\t'], | |
| ]; | |
| function findSplitPoint(text, start, maxEnd) { | |
| const window = text.slice(start, maxEnd); | |
| for (const group of SEPARATOR_GROUPS) { | |
| let bestEnd = -1; | |
| for (const sep of group) { | |
| const i = window.lastIndexOf(sep); | |
| if (i !== -1 && i + sep.length > bestEnd) bestEnd = i + sep.length; | |
| } | |
| if (bestEnd !== -1) return start + bestEnd; | |
| } | |
| return maxEnd; | |
| } | |
| /** @returns {{ text: string, startOffset: number }[]} */ | |
| function splitTextToChunks(text, bytesPerChunk) { | |
| if (bytesPerChunk <= 0) { | |
| throw new Error('bytesPerChunk must be > 0, got: ' + bytesPerChunk); | |
| } | |
| if (text.includes('\r')) { | |
| throw new Error('Text contains \\r (CR); only \\n (LF) is supported.'); | |
| } | |
| const chunks = []; | |
| let pos = 0; | |
| const encodeBuf = new Uint8Array(bytesPerChunk + 1); | |
| while (pos < text.length) { | |
| let chunkEnd = pos; | |
| let chunkBytes = 0; | |
| outer: while (chunkEnd < text.length) { | |
| const paragEnd = nextParagraphEnd(text, chunkEnd); | |
| const paragBytes = getUtf8ByteLength(text.slice(chunkEnd, paragEnd), encodeBuf); | |
| if (chunkBytes > 0 && chunkBytes + paragBytes > bytesPerChunk) break; | |
| if (chunkBytes === 0 && paragBytes > bytesPerChunk) { | |
| while (chunkEnd < paragEnd) { | |
| const lineEnd = nextLineEnd(text, chunkEnd); | |
| const lineBytes = getUtf8ByteLength(text.slice(chunkEnd, lineEnd), encodeBuf); | |
| if (lineBytes > bytesPerChunk) { | |
| const maxEnd = charIndexForByteLimit(text, chunkEnd, bytesPerChunk); | |
| chunkEnd = findSplitPoint(text, chunkEnd, maxEnd); | |
| break outer; | |
| } | |
| if (chunkBytes > 0 && chunkBytes + lineBytes > bytesPerChunk) break outer; | |
| chunkBytes += lineBytes; | |
| chunkEnd = lineEnd; | |
| } | |
| continue outer; | |
| } | |
| chunkBytes += paragBytes; | |
| chunkEnd = paragEnd; | |
| } | |
| chunks.push({ text: text.slice(pos, chunkEnd), startOffset: pos }); | |
| pos = chunkEnd; | |
| } | |
| return chunks; | |
| } | |
| return splitTextToChunks; | |
| })(); | |