first

repetition.js

@@ -0,0 +1,70 @@
+const REPEAT_COUNT = 7;
+const MAX_PATTERN_TOKENS = 256;
+const MAX_BUFFER_CHARS = 32768;
+
+function tokenize(text) {
+    return String(text ?? "").match(/<[^>\s]+>|[\p{L}\p{M}\p{N}_]+|[^\s\p{L}\p{M}\p{N}_]+/gu) || [];
+}
+
+function sameBlock(tokens, aStart, bStart, length) {
+    for (let i = 0; i < length; i++) {
+        if (tokens[aStart + i] !== tokens[bStart + i]) return false;
+    }
+    return true;
+}
+
+function findRepeatedTail(tokens, options = {}) {
+    const repeatCount = options.repeatCount ?? REPEAT_COUNT;
+    const maxPatternTokens = options.maxPatternTokens ?? MAX_PATTERN_TOKENS;
+    if (repeatCount < 2) throw new Error("repeatCount must be at least 2");
+
+    const maxPeriod = Math.min(maxPatternTokens, Math.floor(tokens.length / repeatCount));
+    for (let periodTokens = 1; periodTokens <= maxPeriod; periodTokens++) {
+        const tailStart = tokens.length - (periodTokens * repeatCount);
+        const referenceStart = tokens.length - periodTokens;
+        let repeated = true;
+
+        for (let repeat = 0; repeat < repeatCount - 1; repeat++) {
+            const candidateStart = tailStart + (repeat * periodTokens);
+            if (!sameBlock(tokens, candidateStart, referenceStart, periodTokens)) {
+                repeated = false;
+                break;
+            }
+        }
+
+        if (!repeated) continue;
+        const pattern = tokens.slice(referenceStart);
+        return {
+            repeatCount,
+            periodTokens,
+            pattern,
+            sample: pattern.join(" ").slice(0, 240)
+        };
+    }
+
+    return null;
+}
+
+function createDetector(options = {}) {
+    const maxBufferChars = options.maxBufferChars ?? MAX_BUFFER_CHARS;
+    let buffer = "";
+
+    return {
+        push(text) {
+            buffer += String(text ?? "");
+            if (buffer.length > maxBufferChars) buffer = buffer.slice(-maxBufferChars);
+            return findRepeatedTail(tokenize(buffer), options);
+        },
+        reset() {
+            buffer = "";
+        }
+    };
+}
+
+module.exports = {
+    REPEAT_COUNT,
+    MAX_PATTERN_TOKENS,
+    tokenize,
+    findRepeatedTail,
+    createDetector
+};