perry/lmstudio.js

// api-wrappers/lmstudio.js

const http = require("node:http");
const Repetition = require("./repetition");

const BASE_HOST = "localhost";
const BASE_PORT = 1234;
const MAX_INCOMPLETE_CONTINUATIONS = 10;
const INCOMPLETE_CONTINUE_PROMPT = [
    "Continue exactly where the previous response was cut off.",
    "Do not restart, summarize, or repeat material you already produced.",
    "Finish the original task and provide the remainder of the response."
].join("\n");

function instanceMatches(instance, configured) {
    if (!instance || typeof instance !== "object") return false;
    return [instance.id, instance.key, instance.identifier, instance.model, instance.model_key, instance.modelKey]
        .some(value => typeof value === "string" && value === configured);
}

async function warmUp(settings) {
    try {
        const listResponse = await fetch(`http://${BASE_HOST}:${BASE_PORT}/api/v1/models`);
        if (!listResponse.ok) throw new Error(`LM Studio model list returned ${listResponse.status}: ${await listResponse.text()}`);
        const list = await listResponse.json();
        const models = Array.isArray(list.models) ? list.models : [];
        const installed = models.find(item => item.key === settings.model);
        const loadedInstance = models.flatMap(item => item.loaded_instances || []).find(instance => instanceMatches(instance, settings.model));

        if (loadedInstance || installed?.loaded_instances?.length) return;

        if (!installed) {
            const openAiList = await fetch(`http://${BASE_HOST}:${BASE_PORT}/v1/models`);
            if (openAiList.ok) {
                const loaded = await openAiList.json();
                if (loaded.data?.some(item => item.id === settings.model)) return;
            }
        }

        const modelKey = installed?.key || settings.model;
        const response = await fetch(`http://${BASE_HOST}:${BASE_PORT}/api/v1/models/load`, {
            method: "POST",
            headers: { "Content-Type": "application/json" },
            body: JSON.stringify({ model: modelKey })
        });

        if (!response.ok) {
            console.error(`Model warmup failed: LM Studio returned ${response.status}: ${await response.text()}`);
            return;
        }

        await response.json();
    } catch (error) {
        console.error(`Model warmup failed: ${error.message}`);
    }
}

function inputFor(conversation) {
    return conversation.messages.map(message => ({
        role: message.role,
        content: message.content
    }));
}

function textFrom(content) {
    if (typeof content === "string") return content;
    if (!Array.isArray(content)) return "";
    return content.map(part => part?.text || "").join("");
}

function replyFromResponse(result) {
    const reply = { role: "assistant", content: "" };
    const reasoning = [];

    for (const item of result?.output || []) {
        if (item.type === "message") reply.content += textFrom(item.content);
        if (item.type === "reasoning") reasoning.push(textFrom(item.content));
    }

    if (reasoning.length > 0) reply.reasoning = reasoning.join("\n");
    return reply;
}

function deltaFromEvent(event) {
    if (!event || typeof event !== "object") return null;
    if (typeof event.delta !== "string" || !event.delta) return null;

    const type = String(event.type || "");
    if (type.includes("reasoning")) return { channel: "reasoning", text: event.delta };
    if (type.includes("output_text") || type.includes("text.delta")) return { channel: "content", text: event.delta };

    return null;
}

function pageDiagnostic(page) {
    return {
        responseId: page.responseId || null,
        status: page.status || null,
        incompleteDetails: page.incompleteDetails || null,
        usage: page.usage || null
    };
}

function sendPage(conversation, settings, options = {}) {
    const body = {
        model: settings.model,
        input: options.previousResponseId && options.continuationInput ? options.continuationInput : inputFor(conversation),
        stream: true
    };

    if (conversation.systemPrompt) body.instructions = conversation.systemPrompt;
    if (settings.reasoning && settings.reasoning !== "off") body.reasoning = { effort: settings.reasoning };
    if (options.previousResponseId) body.previous_response_id = options.previousResponseId;

    return new Promise((resolve, reject) => {
        const payload = JSON.stringify(body);
        const request = http.request({
            hostname: BASE_HOST,
            port: BASE_PORT,
            path: "/v1/responses",
            method: "POST",
            headers: {
                "Content-Type": "application/json",
                "Accept": "text/event-stream",
                "Content-Length": Buffer.byteLength(payload)
            }
        }, response => {
            if (response.statusCode < 200 || response.statusCode >= 300) {
                let errorText = "";
                response.setEncoding("utf8");
                response.on("data", chunk => errorText += chunk);
                response.on("end", () => {
                    const error = new Error(`LM Studio returned ${response.statusCode}: ${errorText}`);
                    error.retryable = response.statusCode >= 500;
                    reject(error);
                });
                return;
            }

            response.setEncoding("utf8");

            let buffer = "";
            let content = "";
            let reasoning = "";
            let finalResponse = null;
            let finalEventType = null;
            let responseId = null;
            let settled = false;
            const repetition = {
                content: Repetition.createDetector(),
                reasoning: Repetition.createDetector()
            };

            function fail(error) {
                if (settled) return;
                settled = true;
                response.destroy();
                request.destroy();
                reject(error);
            }

            function appendDelta(channel, text) {
                if (!text) return true;
                const loop = repetition[channel].push(text);
                if (loop) {
                    const error = new Error(
                        `Model repetition loop detected in ${channel}: ` +
                        `${loop.periodTokens}-token pattern repeated ${loop.repeatCount} times consecutively.`
                    );
                    error.code = "PERRY_REPETITION_LOOP";
                    error.retryable = true;
                    error.partial = Boolean(content || reasoning);
                    error.repetition = loop;
                    fail(error);
                    return false;
                }

                if (channel === "content") {
                    content += text;
                    if (typeof options.onContent === "function") options.onContent(text);
                } else {
                    reasoning += text;
                    if (typeof options.onReasoning === "function") options.onReasoning(text);
                }
                return true;
            }

            function consumeEvent(block) {
                const dataLines = block
                    .split(/\r?\n/)
                    .filter(line => line.startsWith("data:"))
                    .map(line => line.slice(5).trimStart());

                if (dataLines.length === 0) return;

                const data = dataLines.join("\n").trim();
                if (!data || data === "[DONE]") return;

                let event;
                try {
                    event = JSON.parse(data);
                } catch {
                    return;
                }

                if (event.response?.id) responseId = event.response.id;

                const delta = deltaFromEvent(event);
                if (delta && !appendDelta(delta.channel, delta.text)) return;

                if ((event.type === "response.completed" || event.type === "response.incomplete") && event.response) {
                    finalResponse = event.response;
                    finalEventType = event.type;
                }

                if (event.type === "response.failed" && event.response) {
                    const message = event.response.error?.message || "LM Studio response failed.";
                    const error = new Error(message);
                    error.retryable = true;
                    error.partial = Boolean(content || reasoning);
                    fail(error);
                }
            }

            function consumeBuffer(flush = false) {
                buffer = buffer.replace(/\r\n/g, "\n");

                let boundary;
                while ((boundary = buffer.indexOf("\n\n")) !== -1) {
                    const block = buffer.slice(0, boundary);
                    buffer = buffer.slice(boundary + 2);
                    consumeEvent(block);
                }

                if (flush && buffer.trim()) {
                    consumeEvent(buffer);
                    buffer = "";
                }
            }

            response.on("data", chunk => {
                if (settled) return;
                buffer += chunk;
                consumeBuffer();
            });

            response.on("end", () => {
                if (settled) return;
                consumeBuffer(true);

                if (!finalEventType) {
                    const error = new Error("LM Studio stream ended without a completed or incomplete terminal response event.");
                    error.retryable = true;
                    error.partial = Boolean(content || reasoning);
                    settled = true;
                    reject(error);
                    return;
                }

                const fallback = replyFromResponse(finalResponse);
                const reply = {
                    role: "assistant",
                    content: content || fallback.content,
                    status: finalEventType === "response.incomplete" ? "incomplete" : finalResponse?.status || "completed",
                    responseId: responseId || finalResponse?.id || null,
                    incompleteDetails: finalResponse?.incomplete_details || null,
                    usage: finalResponse?.usage || null
                };

                if (reasoning || fallback.reasoning) reply.reasoning = reasoning || fallback.reasoning;
                settled = true;
                resolve(reply);
            });

            response.on("error", error => {
                if (settled) return;
                error.retryable = true;
                error.partial = Boolean(content || reasoning);
                settled = true;
                reject(error);
            });
        });

        request.on("error", error => {
            error.retryable = true;
            reject(error);
        });

        request.write(payload);
        request.end();
    });
}

async function send(conversation, settings, options = {}) {
    const content = [];
    const reasoning = [];
    const pages = [];
    let previousResponseId = options.previousResponseId || null;
    let continuationInput = options.continuationInput || null;
    let lastReply = null;

    for (let continuation = 0; continuation <= MAX_INCOMPLETE_CONTINUATIONS; continuation++) {
        const page = await sendPage(conversation, settings, {
            ...options,
            previousResponseId,
            continuationInput
        });

        lastReply = page;
        pages.push(pageDiagnostic(page));
        if (page.reasoning) reasoning.push(page.reasoning);
        if (page.content) content.push(page.content);

        if (page.status !== "incomplete") {
            const reply = { role: "assistant", content: content.join("") };
            if (reasoning.length > 0) reply.reasoning = reasoning.join("");
            if (page.responseId) reply.responseId = page.responseId;
            reply.provider = {
                name: "lmstudio",
                continuationPages: pages.length - 1,
                pages
            };
            return reply;
        }

        if (!page.responseId) {
            const error = new Error("LM Studio returned an incomplete response without a response id; it cannot be continued safely.");
            error.retryable = false;
            error.partial = Boolean(content.length || reasoning.length);
            throw error;
        }

        if (continuation === MAX_INCOMPLETE_CONTINUATIONS) {
            const error = new Error(`LM Studio response remained incomplete after ${MAX_INCOMPLETE_CONTINUATIONS} automatic continuations.`);
            error.retryable = false;
            error.partial = Boolean(content.length || reasoning.length);
            error.provider = { name: "lmstudio", continuationPages: pages.length - 1, pages };
            throw error;
        }

        previousResponseId = page.responseId;
        continuationInput = INCOMPLETE_CONTINUE_PROMPT;
    }

    return lastReply;
}

module.exports = {
    warmUp,
    send
};