71 lines
2.1 KiB
JavaScript
71 lines
2.1 KiB
JavaScript
const REPEAT_COUNT = 7;
|
|
const MAX_PATTERN_TOKENS = 256;
|
|
const MAX_BUFFER_CHARS = 32768;
|
|
|
|
function tokenize(text) {
|
|
return String(text ?? "").match(/<[^>\s]+>|[\p{L}\p{M}\p{N}_]+|[^\s\p{L}\p{M}\p{N}_]+/gu) || [];
|
|
}
|
|
|
|
function sameBlock(tokens, aStart, bStart, length) {
|
|
for (let i = 0; i < length; i++) {
|
|
if (tokens[aStart + i] !== tokens[bStart + i]) return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
function findRepeatedTail(tokens, options = {}) {
|
|
const repeatCount = options.repeatCount ?? REPEAT_COUNT;
|
|
const maxPatternTokens = options.maxPatternTokens ?? MAX_PATTERN_TOKENS;
|
|
if (repeatCount < 2) throw new Error("repeatCount must be at least 2");
|
|
|
|
const maxPeriod = Math.min(maxPatternTokens, Math.floor(tokens.length / repeatCount));
|
|
for (let periodTokens = 1; periodTokens <= maxPeriod; periodTokens++) {
|
|
const tailStart = tokens.length - (periodTokens * repeatCount);
|
|
const referenceStart = tokens.length - periodTokens;
|
|
let repeated = true;
|
|
|
|
for (let repeat = 0; repeat < repeatCount - 1; repeat++) {
|
|
const candidateStart = tailStart + (repeat * periodTokens);
|
|
if (!sameBlock(tokens, candidateStart, referenceStart, periodTokens)) {
|
|
repeated = false;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!repeated) continue;
|
|
const pattern = tokens.slice(referenceStart);
|
|
return {
|
|
repeatCount,
|
|
periodTokens,
|
|
pattern,
|
|
sample: pattern.join(" ").slice(0, 240)
|
|
};
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function createDetector(options = {}) {
|
|
const maxBufferChars = options.maxBufferChars ?? MAX_BUFFER_CHARS;
|
|
let buffer = "";
|
|
|
|
return {
|
|
push(text) {
|
|
buffer += String(text ?? "");
|
|
if (buffer.length > maxBufferChars) buffer = buffer.slice(-maxBufferChars);
|
|
return findRepeatedTail(tokenize(buffer), options);
|
|
},
|
|
reset() {
|
|
buffer = "";
|
|
}
|
|
};
|
|
}
|
|
|
|
module.exports = {
|
|
REPEAT_COUNT,
|
|
MAX_PATTERN_TOKENS,
|
|
tokenize,
|
|
findRepeatedTail,
|
|
createDetector
|
|
};
|