// api-wrappers/lmstudio.js const http = require("node:http"); const Repetition = require("./repetition"); const BASE_HOST = "localhost"; const BASE_PORT = 1234; const MAX_INCOMPLETE_CONTINUATIONS = 10; const INCOMPLETE_CONTINUE_PROMPT = [ "Continue exactly where the previous response was cut off.", "Do not restart, summarize, or repeat material you already produced.", "Finish the original task and provide the remainder of the response." ].join("\n"); function instanceMatches(instance, configured) { if (!instance || typeof instance !== "object") return false; return [instance.id, instance.key, instance.identifier, instance.model, instance.model_key, instance.modelKey] .some(value => typeof value === "string" && value === configured); } async function warmUp(settings) { try { const listResponse = await fetch(`http://${BASE_HOST}:${BASE_PORT}/api/v1/models`); if (!listResponse.ok) throw new Error(`LM Studio model list returned ${listResponse.status}: ${await listResponse.text()}`); const list = await listResponse.json(); const models = Array.isArray(list.models) ? list.models : []; const installed = models.find(item => item.key === settings.model); const loadedInstance = models.flatMap(item => item.loaded_instances || []).find(instance => instanceMatches(instance, settings.model)); if (loadedInstance || installed?.loaded_instances?.length) return; if (!installed) { const openAiList = await fetch(`http://${BASE_HOST}:${BASE_PORT}/v1/models`); if (openAiList.ok) { const loaded = await openAiList.json(); if (loaded.data?.some(item => item.id === settings.model)) return; } } const modelKey = installed?.key || settings.model; const response = await fetch(`http://${BASE_HOST}:${BASE_PORT}/api/v1/models/load`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: modelKey }) }); if (!response.ok) { console.error(`Model warmup failed: LM Studio returned ${response.status}: ${await response.text()}`); return; } await response.json(); } catch (error) { console.error(`Model warmup failed: ${error.message}`); } } function inputFor(conversation) { return conversation.messages.map(message => ({ role: message.role, content: message.content })); } function textFrom(content) { if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; return content.map(part => part?.text || "").join(""); } function replyFromResponse(result) { const reply = { role: "assistant", content: "" }; const reasoning = []; for (const item of result?.output || []) { if (item.type === "message") reply.content += textFrom(item.content); if (item.type === "reasoning") reasoning.push(textFrom(item.content)); } if (reasoning.length > 0) reply.reasoning = reasoning.join("\n"); return reply; } function deltaFromEvent(event) { if (!event || typeof event !== "object") return null; if (typeof event.delta !== "string" || !event.delta) return null; const type = String(event.type || ""); if (type.includes("reasoning")) return { channel: "reasoning", text: event.delta }; if (type.includes("output_text") || type.includes("text.delta")) return { channel: "content", text: event.delta }; return null; } function pageDiagnostic(page) { return { responseId: page.responseId || null, status: page.status || null, incompleteDetails: page.incompleteDetails || null, usage: page.usage || null }; } function sendPage(conversation, settings, options = {}) { const body = { model: settings.model, input: options.previousResponseId && options.continuationInput ? options.continuationInput : inputFor(conversation), stream: true }; if (conversation.systemPrompt) body.instructions = conversation.systemPrompt; if (settings.reasoning && settings.reasoning !== "off") body.reasoning = { effort: settings.reasoning }; if (options.previousResponseId) body.previous_response_id = options.previousResponseId; return new Promise((resolve, reject) => { const payload = JSON.stringify(body); const request = http.request({ hostname: BASE_HOST, port: BASE_PORT, path: "/v1/responses", method: "POST", headers: { "Content-Type": "application/json", "Accept": "text/event-stream", "Content-Length": Buffer.byteLength(payload) } }, response => { if (response.statusCode < 200 || response.statusCode >= 300) { let errorText = ""; response.setEncoding("utf8"); response.on("data", chunk => errorText += chunk); response.on("end", () => { const error = new Error(`LM Studio returned ${response.statusCode}: ${errorText}`); error.retryable = response.statusCode >= 500; reject(error); }); return; } response.setEncoding("utf8"); let buffer = ""; let content = ""; let reasoning = ""; let finalResponse = null; let finalEventType = null; let responseId = null; let settled = false; const repetition = { content: Repetition.createDetector(), reasoning: Repetition.createDetector() }; function fail(error) { if (settled) return; settled = true; response.destroy(); request.destroy(); reject(error); } function appendDelta(channel, text) { if (!text) return true; const loop = repetition[channel].push(text); if (loop) { const error = new Error( `Model repetition loop detected in ${channel}: ` + `${loop.periodTokens}-token pattern repeated ${loop.repeatCount} times consecutively.` ); error.code = "PERRY_REPETITION_LOOP"; error.retryable = true; error.partial = Boolean(content || reasoning); error.repetition = loop; fail(error); return false; } if (channel === "content") { content += text; if (typeof options.onContent === "function") options.onContent(text); } else { reasoning += text; if (typeof options.onReasoning === "function") options.onReasoning(text); } return true; } function consumeEvent(block) { const dataLines = block .split(/\r?\n/) .filter(line => line.startsWith("data:")) .map(line => line.slice(5).trimStart()); if (dataLines.length === 0) return; const data = dataLines.join("\n").trim(); if (!data || data === "[DONE]") return; let event; try { event = JSON.parse(data); } catch { return; } if (event.response?.id) responseId = event.response.id; const delta = deltaFromEvent(event); if (delta && !appendDelta(delta.channel, delta.text)) return; if ((event.type === "response.completed" || event.type === "response.incomplete") && event.response) { finalResponse = event.response; finalEventType = event.type; } if (event.type === "response.failed" && event.response) { const message = event.response.error?.message || "LM Studio response failed."; const error = new Error(message); error.retryable = true; error.partial = Boolean(content || reasoning); fail(error); } } function consumeBuffer(flush = false) { buffer = buffer.replace(/\r\n/g, "\n"); let boundary; while ((boundary = buffer.indexOf("\n\n")) !== -1) { const block = buffer.slice(0, boundary); buffer = buffer.slice(boundary + 2); consumeEvent(block); } if (flush && buffer.trim()) { consumeEvent(buffer); buffer = ""; } } response.on("data", chunk => { if (settled) return; buffer += chunk; consumeBuffer(); }); response.on("end", () => { if (settled) return; consumeBuffer(true); if (!finalEventType) { const error = new Error("LM Studio stream ended without a completed or incomplete terminal response event."); error.retryable = true; error.partial = Boolean(content || reasoning); settled = true; reject(error); return; } const fallback = replyFromResponse(finalResponse); const reply = { role: "assistant", content: content || fallback.content, status: finalEventType === "response.incomplete" ? "incomplete" : finalResponse?.status || "completed", responseId: responseId || finalResponse?.id || null, incompleteDetails: finalResponse?.incomplete_details || null, usage: finalResponse?.usage || null }; if (reasoning || fallback.reasoning) reply.reasoning = reasoning || fallback.reasoning; settled = true; resolve(reply); }); response.on("error", error => { if (settled) return; error.retryable = true; error.partial = Boolean(content || reasoning); settled = true; reject(error); }); }); request.on("error", error => { error.retryable = true; reject(error); }); request.write(payload); request.end(); }); } async function send(conversation, settings, options = {}) { const content = []; const reasoning = []; const pages = []; let previousResponseId = options.previousResponseId || null; let continuationInput = options.continuationInput || null; let lastReply = null; for (let continuation = 0; continuation <= MAX_INCOMPLETE_CONTINUATIONS; continuation++) { const page = await sendPage(conversation, settings, { ...options, previousResponseId, continuationInput }); lastReply = page; pages.push(pageDiagnostic(page)); if (page.reasoning) reasoning.push(page.reasoning); if (page.content) content.push(page.content); if (page.status !== "incomplete") { const reply = { role: "assistant", content: content.join("") }; if (reasoning.length > 0) reply.reasoning = reasoning.join(""); if (page.responseId) reply.responseId = page.responseId; reply.provider = { name: "lmstudio", continuationPages: pages.length - 1, pages }; return reply; } if (!page.responseId) { const error = new Error("LM Studio returned an incomplete response without a response id; it cannot be continued safely."); error.retryable = false; error.partial = Boolean(content.length || reasoning.length); throw error; } if (continuation === MAX_INCOMPLETE_CONTINUATIONS) { const error = new Error(`LM Studio response remained incomplete after ${MAX_INCOMPLETE_CONTINUATIONS} automatic continuations.`); error.retryable = false; error.partial = Boolean(content.length || reasoning.length); error.provider = { name: "lmstudio", continuationPages: pages.length - 1, pages }; throw error; } previousResponseId = page.responseId; continuationInput = INCOMPLETE_CONTINUE_PROMPT; } return lastReply; } module.exports = { warmUp, send };