Spaces:
Paused
Paused
File size: 3,565 Bytes
966d483 61195c6 966d483 61195c6 966d483 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | /**
* SYNC 副本（demo 不抽共享包；改站内源请同步本文件）：
* client/src/shared/cross/semanticUtils.ts → splitTextToChunks 及相关纯函数
*/
globalThis.IL_splitTextToChunks = (function () {
const encoder = new TextEncoder();
function getUtf8ByteLength(text, buf) {
const { read, written } = encoder.encodeInto(text, buf);
return read < text.length ? buf.length : written;
}
function nextParagraphEnd(text, start) {
const nl = text.indexOf('\n\n', start);
if (nl === -1) return text.length;
let end = nl + 2;
while (end < text.length && text[end] === '\n') end++;
return end;
}
function nextLineEnd(text, start) {
const nl = text.indexOf('\n', start);
if (nl === -1) return text.length;
let end = nl + 1;
while (end < text.length && text[end] === '\n') end++;
return end;
}
function charIndexForByteLimit(text, start, byteLimit) {
const buf = new Uint8Array(4);
let bytes = 0;
let i = start;
while (i < text.length) {
const cp = text.codePointAt(i);
const charLen = cp > 0xffff ? 2 : 1;
const byteLen = encoder.encodeInto(text.slice(i, i + charLen), buf).written;
if (bytes + byteLen > byteLimit) break;
bytes += byteLen;
i += charLen;
}
return i;
}
const SEPARATOR_GROUPS = [
['。', '！', '？', '…'],
['；', '，'],
['.', '!', '?'],
[';', ','],
[' ', '\t'],
];
function findSplitPoint(text, start, maxEnd) {
const window = text.slice(start, maxEnd);
for (const group of SEPARATOR_GROUPS) {
let bestEnd = -1;
for (const sep of group) {
const i = window.lastIndexOf(sep);
if (i !== -1 && i + sep.length > bestEnd) bestEnd = i + sep.length;
}
if (bestEnd !== -1) return start + bestEnd;
}
return maxEnd;
}
/** @returns {{ text: string, startOffset: number }[]} */
function splitTextToChunks(text, bytesPerChunk) {
if (bytesPerChunk <= 0) {
throw new Error('bytesPerChunk must be > 0, got: ' + bytesPerChunk);
}
if (text.includes('\r')) {
throw new Error('Text contains \\r (CR); only \\n (LF) is supported.');
}
const chunks = [];
let pos = 0;
const encodeBuf = new Uint8Array(bytesPerChunk + 1);
while (pos < text.length) {
let chunkEnd = pos;
let chunkBytes = 0;
outer: while (chunkEnd < text.length) {
const paragEnd = nextParagraphEnd(text, chunkEnd);
const paragBytes = getUtf8ByteLength(text.slice(chunkEnd, paragEnd), encodeBuf);
if (chunkBytes > 0 && chunkBytes + paragBytes > bytesPerChunk) break;
if (chunkBytes === 0 && paragBytes > bytesPerChunk) {
while (chunkEnd < paragEnd) {
const lineEnd = nextLineEnd(text, chunkEnd);
const lineBytes = getUtf8ByteLength(text.slice(chunkEnd, lineEnd), encodeBuf);
if (lineBytes > bytesPerChunk) {
const maxEnd = charIndexForByteLimit(text, chunkEnd, bytesPerChunk);
chunkEnd = findSplitPoint(text, chunkEnd, maxEnd);
break outer;
}
if (chunkBytes > 0 && chunkBytes + lineBytes > bytesPerChunk) break outer;
chunkBytes += lineBytes;
chunkEnd = lineEnd;
}
continue outer;
}
chunkBytes += paragBytes;
chunkEnd = paragEnd;
}
chunks.push({ text: text.slice(pos, chunkEnd), startOffset: pos });
pos = chunkEnd;
}
return chunks;
}
return splitTextToChunks;
})();
|