diff --git a/README.md b/README.md index d19332c..e608b1c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,181 @@ # optimus -A pull based artifact pump that only rebuilds things that are stale \ No newline at end of file +A deliberately small project runtime for initialized modules, settings, named artifacts, and transformer-node helpers. + +## Project shape + +A host project provides JSON settings in `./settings/`: + +```text +settings/ + modules.json + artifacts.json + anything-else.json +``` + +`modules.json` maps semantic module names to initializer modules: + +```json +{ + "chatter": "./tree/chatter/index.js", + "markdown-builder": "./tree/markdown-builder/index.js" +} +``` + +Each module exports an initializer that receives the same shared `utils` object: + +```js +module.exports = utils => { + function doSomething() { + const chatter = utils.loadModule("chatter"); + const llm = utils.getSettings("llm"); + } + + return { doSomething }; +}; +``` + +## Utilities + +### `utils.loadModule(name)` + +Returns the initialized module registered under that semantic name. + +### `utils.getSettings(name)` + +Returns the parsed contents of `./settings/.json`. + +### `utils.hashPayload(value)` + +Returns a SHA-256 hex hash of `JSON.stringify(value)`. No key sorting or normalization is performed. + +### `utils.buildNode(payload, sources = {})` + +Builds a transformer-node packet from a payload and already-loaded source nodes: + +```js +const node = utils.buildNode(payload, { + hebrew: hebrewNode, + guidance: guidanceNode, +}); +``` + +Result: + +```json +{ + "sourceHash": { + "hebrew": "", + "guidance": "" + }, + "hash": "", + "payload": "" +} +``` + +Optimus does not persist or validate the node. The owning module decides where its artifacts live and when to rebuild them. + +### `utils.buildBinaryNode(filePath, sources = {})` + +Builds the binary equivalent of a transformer-node packet. The supplied path is kept as the usable binary payload locator, while `hash` is computed from the file bytes: + +```js +const node = utils.buildBinaryNode(resolvedCoverPath); +``` + +Result: + +```json +{ + "sourceHash": {}, + "hash": "", + "payloadBin": "" +} +``` + +Derived binary artifacts can name source nodes in the same way as `buildNode()`: + +```js +return utils.buildBinaryNode(pdfPath, { + volume: volumeNode, + style: styleNode, +}); +``` + +`buildBinaryNode()` does not copy the file, interpret the path, or know what kind of binary it represents. It hashes the bytes currently at `filePath` and returns that path unchanged as `payloadBin`. + +## Artifacts + +`settings/artifacts.json` explicitly names the module and public builder function for each top-level artifact: + +```json +{ + "job.en.md": { + "module": "unpointed-hebrew-markdown-builder", + "builder": "buildMarkdownOutput", + "params": ["job", "en"] + } +} +``` + +Build one artifact: + +```bash +npx optimus --artifact job.en.md +``` + +Build every declared artifact: + +```bash +npx optimus +``` + +Artifacts are logical requested outcomes. Their modules own filenames, directories, persistence, cache policy, and dependency behavior. + +A builder may call `utils.setArtifactOutcome("up to date")` (or another concise outcome) before returning. Optimus prints the requested artifact ID before invoking the builder and prints that outcome afterward; builders that do not report one receive the generic `completed — success` result. + +## Invocation setting overlays + +Optimus loads the project defaults from direct `settings/*.json` files. Each filename remains a free-form settings name available through `utils.getSettings(name)`. + +A process can layer one or more JSON overlays on top of those defaults: + +```bash +npx optimus --artifact psalms.en.md --setting settings/overlays/remote.json +``` + +Multiple overlays are applied left-to-right: + +```bash +npx optimus --artifact psalms.en.md \ + --setting settings/overlays/remote.json \ + --setting settings/overlays/experimental-model.json +``` + +An overlay is one object whose top-level keys are ordinary settings names: + +```json +{ + "llm": { + "host": "remote-box", + "model": "remote-model" + } +} +``` + +Overlay behavior is deliberately small: + +- object over object: recursively overlay properties; +- arrays: replace the previous array; +- scalars: replace the previous value; +- `null`: replace the previous value with `null`; +- omitted properties: inherit the value underneath; +- an overlay may introduce a new free-form settings name. + +`utils.getSettings("llm")` sees only the resulting effective settings. Modules do not know which values came from defaults versus invocation overlays. + +Overlay paths are resolved relative to the project root unless absolute. + +## Build-all memory behavior + +`buildAll()` awaits each declared artifact sequentially and does not retain prior artifact results. Artifact builders may still return either ordinary values or Promises; the runtime remains compatible with genuinely asynchronous work such as LLM requests. diff --git a/bin/optimus.js b/bin/optimus.js new file mode 100644 index 0000000..dfd7cd0 --- /dev/null +++ b/bin/optimus.js @@ -0,0 +1,52 @@ +#!/usr/bin/env node + +const { initialize } = require(".."); + +function getArtifactName(args) { + const inline = args.find(arg => arg.startsWith("--artifact=")); + if (inline) return inline.slice("--artifact=".length); + + const index = args.indexOf("--artifact"); + if (index === -1) return null; + if (!args[index + 1]) throw new Error("--artifact requires a name"); + return args[index + 1]; +} + +function getSettingOverlays(args) { + const overlays = []; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg.startsWith("--setting=")) { + const value = arg.slice("--setting=".length); + if (!value) throw new Error("--setting requires a file path"); + overlays.push(value); + continue; + } + if (arg !== "--setting") continue; + if (!args[i + 1]) throw new Error("--setting requires a file path"); + overlays.push(args[++i]); + } + return overlays; +} + +async function main() { + const args = process.argv.slice(2); + const optimus = initialize(process.cwd(), { settingOverlays: getSettingOverlays(args) }); + const artifactName = getArtifactName(args); + + if (artifactName) { + await optimus.buildArtifact(artifactName); + console.log(`Artifact complete: ${artifactName}`); + return; + } + + const artifacts = optimus.getSettings("artifacts"); + const names = Object.keys(artifacts); + await optimus.buildAll(); + console.log(names.length ? `Artifacts complete: ${names.join(", ")}` : "No artifacts configured."); +} + +main().catch(error => { + console.error(error.message || error); + process.exitCode = 1; +}); diff --git a/index.js b/index.js new file mode 100644 index 0000000..efd3015 --- /dev/null +++ b/index.js @@ -0,0 +1,191 @@ +const crypto = require("crypto"); +const fs = require("fs"); +const path = require("path"); + +function isPlainObject(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function overlaySettingsValue(base, overlay) { + if (!isPlainObject(base) || !isPlainObject(overlay)) return overlay; + + const result = { ...base }; + for (const [key, value] of Object.entries(overlay)) { + result[key] = key in result ? overlaySettingsValue(result[key], value) : value; + } + return result; +} + +function applySettingOverlay(settings, projectRoot, overlayPath) { + const resolvedPath = path.isAbsolute(overlayPath) ? overlayPath : path.resolve(projectRoot, overlayPath); + const overlay = JSON.parse(fs.readFileSync(resolvedPath, "utf8")); + if (!isPlainObject(overlay)) throw new Error(`Setting overlay must contain an object: ${overlayPath}`); + + for (const [name, value] of Object.entries(overlay)) { + settings.set(name, settings.has(name) ? overlaySettingsValue(settings.get(name), value) : value); + } +} + +function loadSettings(projectRoot, settingOverlays = []) { + const settingsRoot = path.join(projectRoot, "settings"); + if (!fs.existsSync(settingsRoot)) throw new Error(`No settings folder found in ${projectRoot}`); + + const settings = new Map(); + for (const fileName of fs.readdirSync(settingsRoot)) { + if (path.extname(fileName).toLowerCase() !== ".json") continue; + const name = path.basename(fileName, path.extname(fileName)); + const filePath = path.join(settingsRoot, fileName); + settings.set(name, JSON.parse(fs.readFileSync(filePath, "utf8"))); + } + for (const overlayPath of settingOverlays) applySettingOverlay(settings, projectRoot, overlayPath); + return settings; +} + +function hashPayload(value) { + const json = JSON.stringify(value); + return crypto.createHash("sha256").update(json === undefined ? "undefined" : json).digest("hex"); +} + +function getSourceHash(sources) { + const sourceHash = {}; + for (const [name, source] of Object.entries(sources)) { + if (!source || typeof source !== "object" || !("hash" in source)) { + throw new Error(`Source "${name}" must be a node with a hash`); + } + sourceHash[name] = source.hash; + } + return sourceHash; +} + +function buildNode(payload, sources = {}) { + return { + sourceHash: getSourceHash(sources), + hash: hashPayload(payload), + payload, + }; +} + +function buildBinaryNode(filePath, sources = {}) { + const bytes = fs.readFileSync(filePath); + return { + sourceHash: getSourceHash(sources), + hash: crypto.createHash("sha256").update(bytes).digest("hex"), + payloadBin: filePath, + }; +} + +function initialize(projectRoot = process.cwd(), options = {}) { + const settingOverlays = options.settingOverlays || []; + if (!Array.isArray(settingOverlays)) throw new Error("options.settingOverlays must be an array"); + const settings = loadSettings(projectRoot, settingOverlays); + const registry = new Map(); + let artifactOutcome; + + const utils = { + loadModule(name) { + const entry = registry.get(name); + if (!entry) throw new Error(`Unknown module: ${name}`); + if (entry.module) return entry.module; + if (entry.initializing) throw new Error(`Circular module initialization involving: ${name}`); + + entry.initializing = true; + try { + const initialized = entry.init(utils); + if (!initialized || (typeof initialized !== "object" && typeof initialized !== "function")) { + throw new Error(`Module "${name}" initializer must return a module object or function`); + } + entry.module = initialized; + return entry.module; + } finally { + entry.initializing = false; + } + }, + + getSettings(name) { + if (!settings.has(name)) throw new Error(`Unknown settings: ${name}`); + return settings.get(name); + }, + + hashPayload, + buildNode, + buildBinaryNode, + + setArtifactOutcome(value) { + artifactOutcome = `${value ?? ""}`.trim() || undefined; + }, + }; + + const modules = utils.getSettings("modules"); + if (!modules || typeof modules !== "object" || Array.isArray(modules)) throw new Error("settings/modules.json must contain an object"); + + for (const [name, modulePath] of Object.entries(modules)) { + const resolvedPath = path.resolve(projectRoot, modulePath); + const init = require(resolvedPath); + if (typeof init !== "function") throw new Error(`Module "${name}" must export an initializer function: ${modulePath}`); + registry.set(name, { init, module: null, initializing: false }); + } + + for (const name of registry.keys()) utils.loadModule(name); + + function buildArtifact(name) { + const artifacts = utils.getSettings("artifacts"); + const artifact = artifacts[name]; + if (!artifact) throw new Error(`Unknown artifact: ${name}`); + if (!artifact.module) throw new Error(`Artifact "${name}" is missing module`); + if (!artifact.builder) throw new Error(`Artifact "${name}" is missing builder`); + + const module = utils.loadModule(artifact.module); + const builder = module[artifact.builder]; + if (typeof builder !== "function") { + throw new Error(`Artifact "${name}" builder not found: ${artifact.module}.${artifact.builder}`); + } + + const params = artifact.params === undefined ? [] : artifact.params; + if (!Array.isArray(params)) throw new Error(`Artifact "${name}" params must be an array`); + + artifactOutcome = undefined; + console.log(`Ensuring artifact: ${name}`); + try { + const result = builder(...params); + if (result && typeof result.then === "function") { + return result.then(value => { + console.log(` ${artifactOutcome ?? "completed — success"}`); + artifactOutcome = undefined; + return value; + }, error => { + console.error(" failed"); + artifactOutcome = undefined; + throw error; + }); + } + + console.log(` ${artifactOutcome ?? "completed — success"}`); + artifactOutcome = undefined; + return result; + } + catch (error) { + console.error(" failed"); + artifactOutcome = undefined; + throw error; + } + } + + async function buildAll() { + const artifacts = utils.getSettings("artifacts"); + for (const name of Object.keys(artifacts)) await buildArtifact(name); + } + + return { + utils, + loadModule: utils.loadModule, + getSettings: utils.getSettings, + hashPayload, + buildNode, + buildBinaryNode, + buildArtifact, + buildAll, + modules: [...registry.keys()], + }; +} + +module.exports = { initialize, hashPayload, buildNode, buildBinaryNode }; diff --git a/optimus.zip b/optimus.zip new file mode 100644 index 0000000..e6e5d3b Binary files /dev/null and b/optimus.zip differ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..a982542 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,16 @@ +{ + "name": "optimus", + "version": "0.3.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "optimus", + "version": "0.3.0", + "license": "UNLICENSED", + "bin": { + "optimus": "bin/optimus.js" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..8a0e2da --- /dev/null +++ b/package.json @@ -0,0 +1,18 @@ +{ + "name": "optimus", + "version": "0.5.0", + "description": "Minimal project runtime for initialized modules, settings, artifacts, and transformer nodes. Optimal transformation pipeline foreman.", + "main": "index.js", + "bin": { + "optimus": "./bin/optimus.js" + }, + "files": [ + "bin", + "index.js", + "README.md" + ], + "scripts": { + "test": "node test/smoke.js" + }, + "license": "UNLICENSED" +} diff --git a/test/fixture/overlays/experiment.json b/test/fixture/overlays/experiment.json new file mode 100644 index 0000000..de4b4f7 --- /dev/null +++ b/test/fixture/overlays/experiment.json @@ -0,0 +1,13 @@ +{ + "llm": { + "model": "experimental-model", + "reasoning": null, + "nested": { + "extra": true + } + }, + "run": { + "label": "experiment", + "enabled": true + } +} diff --git a/test/fixture/overlays/remote.json b/test/fixture/overlays/remote.json new file mode 100644 index 0000000..db5524c --- /dev/null +++ b/test/fixture/overlays/remote.json @@ -0,0 +1,13 @@ +{ + "llm": { + "host": "remote-box", + "model": "remote-model", + "nested": { + "replace": "remote" + }, + "list": ["remote"] + }, + "run": { + "label": "remote" + } +} diff --git a/test/fixture/settings/artifacts.json b/test/fixture/settings/artifacts.json new file mode 100644 index 0000000..29f809e --- /dev/null +++ b/test/fixture/settings/artifacts.json @@ -0,0 +1,7 @@ +{ + "beta-demo": { + "module": "beta", + "builder": "buildBeta", + "params": ["demo"] + } +} diff --git a/test/fixture/settings/input.json b/test/fixture/settings/input.json new file mode 100644 index 0000000..a940c29 --- /dev/null +++ b/test/fixture/settings/input.json @@ -0,0 +1,3 @@ +{ + "prefix": "alpha" +} diff --git a/test/fixture/settings/llm.json b/test/fixture/settings/llm.json new file mode 100644 index 0000000..7b04957 --- /dev/null +++ b/test/fixture/settings/llm.json @@ -0,0 +1,12 @@ +{ + "client": "lm-studio", + "host": "localhost", + "port": 1234, + "model": "local-model", + "reasoning": true, + "nested": { + "keep": "default", + "replace": "default" + }, + "list": ["default"] +} diff --git a/test/fixture/settings/modules.json b/test/fixture/settings/modules.json new file mode 100644 index 0000000..4768676 --- /dev/null +++ b/test/fixture/settings/modules.json @@ -0,0 +1,4 @@ +{ + "alpha": "./tree/alpha/index.js", + "beta": "./tree/beta/index.js" +} diff --git a/test/fixture/tree/alpha/index.js b/test/fixture/tree/alpha/index.js new file mode 100644 index 0000000..c45b9ff --- /dev/null +++ b/test/fixture/tree/alpha/index.js @@ -0,0 +1,8 @@ +module.exports = utils => { + function loadAlpha(value) { + const prefix = utils.getSettings("input").prefix; + return utils.buildNode(`${prefix}:${value}`); + } + + return { loadAlpha }; +}; diff --git a/test/fixture/tree/beta/index.js b/test/fixture/tree/beta/index.js new file mode 100644 index 0000000..b3c720e --- /dev/null +++ b/test/fixture/tree/beta/index.js @@ -0,0 +1,9 @@ +module.exports = utils => { + function buildBeta(value) { + const alpha = utils.loadModule("alpha"); + const source = alpha.loadAlpha(value); + return utils.buildNode(`beta:${source.payload}`, { alpha: source }); + } + + return { buildBeta }; +}; diff --git a/test/smoke.js b/test/smoke.js new file mode 100644 index 0000000..c7f2a10 --- /dev/null +++ b/test/smoke.js @@ -0,0 +1,84 @@ +const assert = require("assert"); +const crypto = require("crypto"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { initialize, hashPayload, buildNode, buildBinaryNode } = require(".."); + +async function main() { + const projectRoot = path.join(__dirname, "fixture"); + const optimus = initialize(projectRoot); + + assert.deepStrictEqual(optimus.modules, ["alpha", "beta"]); + assert.strictEqual(optimus.getSettings("input").prefix, "alpha"); + assert.strictEqual(optimus.loadModule("alpha").loadAlpha("x").payload, "alpha:x"); + assert.throws(() => optimus.loadModule("missing"), /Unknown module/); + assert.throws(() => optimus.getSettings("missing"), /Unknown settings/); + + const overlaid = initialize(projectRoot, { + settingOverlays: ["overlays/remote.json", "overlays/experiment.json"], + }); + assert.deepStrictEqual(overlaid.getSettings("llm"), { + client: "lm-studio", + host: "remote-box", + port: 1234, + model: "experimental-model", + reasoning: null, + nested: { + keep: "default", + replace: "remote", + extra: true, + }, + list: ["remote"], + }); + assert.deepStrictEqual(overlaid.getSettings("run"), { label: "experiment", enabled: true }); + + const expectedHash = crypto.createHash("sha256").update(JSON.stringify({ a: 1 })).digest("hex"); + assert.strictEqual(hashPayload({ a: 1 }), expectedHash); + + const source = buildNode({ source: true }); + const node = buildNode({ result: true }, { source }); + assert.deepStrictEqual(node.sourceHash, { source: source.hash }); + assert.strictEqual(node.hash, hashPayload(node.payload)); + assert.deepStrictEqual(node.payload, { result: true }); + + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "optimus-binary-")); + const binaryPath = path.join(tempRoot, "sample.bin"); + const firstBytes = Buffer.from([0x00, 0x01, 0x02, 0xff]); + const secondBytes = Buffer.from([0x00, 0x01, 0x03, 0xff]); + + try { + fs.writeFileSync(binaryPath, firstBytes); + const binaryNode = buildBinaryNode(binaryPath, { source }); + const expectedBinaryHash = crypto.createHash("sha256").update(firstBytes).digest("hex"); + assert.deepStrictEqual(binaryNode.sourceHash, { source: source.hash }); + assert.strictEqual(binaryNode.hash, expectedBinaryHash); + assert.strictEqual(binaryNode.payloadBin, binaryPath); + assert.strictEqual("payload" in binaryNode, false); + + const binaryViaUtils = optimus.utils.buildBinaryNode(binaryPath); + assert.strictEqual(binaryViaUtils.hash, expectedBinaryHash); + + fs.writeFileSync(binaryPath, secondBytes); + const changedBinaryNode = buildBinaryNode(binaryPath); + assert.notStrictEqual(changedBinaryNode.hash, binaryNode.hash); + assert.strictEqual(changedBinaryNode.payloadBin, binaryPath); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } + + const artifact = optimus.buildArtifact("beta-demo"); + assert.strictEqual(typeof artifact?.then, "undefined"); + assert.deepStrictEqual(artifact.sourceHash, { alpha: optimus.loadModule("alpha").loadAlpha("demo").hash }); + assert.strictEqual(artifact.payload, "beta:alpha:demo"); + + const all = await optimus.buildAll(); + assert.strictEqual(all, undefined); + + console.log("smoke test passed"); +} + +main().catch(error => { + console.error(error); + process.exitCode = 1; +});