diff --git a/src/config/io.runtime-snapshot-write.test.ts b/src/config/io.runtime-snapshot-write.test.ts new file mode 100644 index 000000000..fff270ac8 --- /dev/null +++ b/src/config/io.runtime-snapshot-write.test.ts @@ -0,0 +1,63 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { withTempHome } from "./home-env.test-harness.js"; +import { + clearConfigCache, + clearRuntimeConfigSnapshot, + loadConfig, + setRuntimeConfigSnapshot, + writeConfigFile, +} from "./io.js"; +import type { OpenClawConfig } from "./types.js"; + +describe("runtime config snapshot writes", () => { + it("preserves source secret refs when writeConfigFile receives runtime-resolved config", async () => { + await withTempHome("openclaw-config-runtime-write-", async (home) => { + const configPath = path.join(home, ".openclaw", "openclaw.json"); + const sourceConfig: OpenClawConfig = { + models: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + apiKey: { source: "env", id: "OPENAI_API_KEY" }, + models: [], + }, + }, + }, + }; + const runtimeConfig: OpenClawConfig = { + models: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + apiKey: "sk-runtime-resolved", + models: [], + }, + }, + }, + }; + + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.writeFile(configPath, `${JSON.stringify(sourceConfig, null, 2)}\n`, "utf8"); + + try { + setRuntimeConfigSnapshot(runtimeConfig, sourceConfig); + expect(loadConfig().models?.providers?.openai?.apiKey).toBe("sk-runtime-resolved"); + + await writeConfigFile(loadConfig()); + + const persisted = JSON.parse(await fs.readFile(configPath, "utf8")) as { + models?: { providers?: { openai?: { apiKey?: unknown } } }; + }; + expect(persisted.models?.providers?.openai?.apiKey).toEqual({ + source: "env", + id: "OPENAI_API_KEY", + }); + } finally { + clearRuntimeConfigSnapshot(); + clearConfigCache(); + } + }); + }); +}); diff --git a/src/config/io.ts b/src/config/io.ts index 688031ea7..136ea5eae 100644 --- a/src/config/io.ts +++ b/src/config/io.ts @@ -1299,6 +1299,7 @@ let configCache: { config: OpenClawConfig; } | null = null; let runtimeConfigSnapshot: OpenClawConfig | null = null; +let runtimeConfigSourceSnapshot: OpenClawConfig | null = null; function resolveConfigCacheMs(env: NodeJS.ProcessEnv): number { const raw = env.OPENCLAW_CONFIG_CACHE_MS?.trim(); @@ -1326,13 +1327,18 @@ export function clearConfigCache(): void { configCache = null; } -export function setRuntimeConfigSnapshot(config: OpenClawConfig): void { +export function setRuntimeConfigSnapshot( + config: OpenClawConfig, + sourceConfig?: OpenClawConfig, +): void { runtimeConfigSnapshot = config; + runtimeConfigSourceSnapshot = sourceConfig ?? null; clearConfigCache(); } export function clearRuntimeConfigSnapshot(): void { runtimeConfigSnapshot = null; + runtimeConfigSourceSnapshot = null; clearConfigCache(); } @@ -1380,9 +1386,14 @@ export async function writeConfigFile( options: ConfigWriteOptions = {}, ): Promise { const io = createConfigIO(); + let nextCfg = cfg; + if (runtimeConfigSnapshot && runtimeConfigSourceSnapshot) { + const runtimePatch = createMergePatch(runtimeConfigSnapshot, cfg); + nextCfg = coerceConfig(applyMergePatch(runtimeConfigSourceSnapshot, runtimePatch)); + } const sameConfigPath = options.expectedConfigPath === undefined || options.expectedConfigPath === io.configPath; - await io.writeConfigFile(cfg, { + await io.writeConfigFile(nextCfg, { envSnapshotForRestore: sameConfigPath ? options.envSnapshotForRestore : undefined, unsetPaths: options.unsetPaths, }); diff --git a/src/config/types.secrets.ts b/src/config/types.secrets.ts index 49662d353..5eac0f558 100644 --- a/src/config/types.secrets.ts +++ b/src/config/types.secrets.ts @@ -13,6 +13,24 @@ export type SecretRef = { export type SecretInput = string | SecretRef; +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function isSecretRef(value: unknown): value is SecretRef { + if (!isRecord(value)) { + return false; + } + if (Object.keys(value).length !== 2) { + return false; + } + return ( + (value.source === "env" || value.source === "file") && + typeof value.id === "string" && + value.id.trim().length > 0 + ); +} + export type EnvSecretSourceConfig = { type?: "env"; }; diff --git a/src/secrets/json-pointer.ts b/src/secrets/json-pointer.ts new file mode 100644 index 000000000..c9761af61 --- /dev/null +++ b/src/secrets/json-pointer.ts @@ -0,0 +1,94 @@ +function failOrUndefined(params: { onMissing: "throw" | "undefined"; message: string }): undefined { + if (params.onMissing === "throw") { + throw new Error(params.message); + } + return undefined; +} + +export function decodeJsonPointerToken(token: string): string { + return token.replace(/~1/g, "/").replace(/~0/g, "~"); +} + +export function encodeJsonPointerToken(token: string): string { + return token.replace(/~/g, "~0").replace(/\//g, "~1"); +} + +export function readJsonPointer( + root: unknown, + pointer: string, + options: { onMissing?: "throw" | "undefined" } = {}, +): unknown { + const onMissing = options.onMissing ?? "throw"; + if (!pointer.startsWith("/")) { + return failOrUndefined({ + onMissing, + message: + 'File-backed secret ids must be absolute JSON pointers (for example: "/providers/openai/apiKey").', + }); + } + + const tokens = pointer + .slice(1) + .split("/") + .map((token) => decodeJsonPointerToken(token)); + + let current: unknown = root; + for (const token of tokens) { + if (Array.isArray(current)) { + const index = Number.parseInt(token, 10); + if (!Number.isFinite(index) || index < 0 || index >= current.length) { + return failOrUndefined({ + onMissing, + message: `JSON pointer segment "${token}" is out of bounds.`, + }); + } + current = current[index]; + continue; + } + if (typeof current !== "object" || current === null || Array.isArray(current)) { + return failOrUndefined({ + onMissing, + message: `JSON pointer segment "${token}" does not exist.`, + }); + } + const record = current as Record; + if (!Object.hasOwn(record, token)) { + return failOrUndefined({ + onMissing, + message: `JSON pointer segment "${token}" does not exist.`, + }); + } + current = record[token]; + } + return current; +} + +export function setJsonPointer( + root: Record, + pointer: string, + value: unknown, +): void { + if (!pointer.startsWith("/")) { + throw new Error(`Invalid JSON pointer "${pointer}".`); + } + + const tokens = pointer + .slice(1) + .split("/") + .map((token) => decodeJsonPointerToken(token)); + + let current: Record = root; + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]; + const isLast = index === tokens.length - 1; + if (isLast) { + current[token] = value; + return; + } + const child = current[token]; + if (typeof child !== "object" || child === null || Array.isArray(child)) { + current[token] = {}; + } + current = current[token] as Record; + } +} diff --git a/src/secrets/provider-env-vars.ts b/src/secrets/provider-env-vars.ts new file mode 100644 index 000000000..843acff4a --- /dev/null +++ b/src/secrets/provider-env-vars.ts @@ -0,0 +1,28 @@ +export const PROVIDER_ENV_VARS: Record = { + openai: ["OPENAI_API_KEY"], + anthropic: ["ANTHROPIC_API_KEY"], + google: ["GEMINI_API_KEY"], + minimax: ["MINIMAX_API_KEY"], + "minimax-cn": ["MINIMAX_API_KEY"], + moonshot: ["MOONSHOT_API_KEY"], + "kimi-coding": ["KIMI_API_KEY", "KIMICODE_API_KEY"], + synthetic: ["SYNTHETIC_API_KEY"], + venice: ["VENICE_API_KEY"], + zai: ["ZAI_API_KEY", "Z_AI_API_KEY"], + xiaomi: ["XIAOMI_API_KEY"], + openrouter: ["OPENROUTER_API_KEY"], + "cloudflare-ai-gateway": ["CLOUDFLARE_AI_GATEWAY_API_KEY"], + litellm: ["LITELLM_API_KEY"], + "vercel-ai-gateway": ["AI_GATEWAY_API_KEY"], + opencode: ["OPENCODE_API_KEY", "OPENCODE_ZEN_API_KEY"], + together: ["TOGETHER_API_KEY"], + huggingface: ["HUGGINGFACE_HUB_TOKEN", "HF_TOKEN"], + qianfan: ["QIANFAN_API_KEY"], + xai: ["XAI_API_KEY"], + volcengine: ["VOLCANO_ENGINE_API_KEY"], + byteplus: ["BYTEPLUS_API_KEY"], +}; + +export function listKnownSecretEnvVarNames(): string[] { + return [...new Set(Object.values(PROVIDER_ENV_VARS).flatMap((keys) => keys))]; +} diff --git a/src/secrets/runtime.ts b/src/secrets/runtime.ts index 086cffa38..7c7757c9d 100644 --- a/src/secrets/runtime.ts +++ b/src/secrets/runtime.ts @@ -10,13 +10,11 @@ import { clearRuntimeConfigSnapshot, setRuntimeConfigSnapshot, type OpenClawConfig, - type SecretRef, } from "../config/config.js"; -import { runExec } from "../process/exec.js"; +import { isSecretRef, type SecretRef } from "../config/types.secrets.js"; import { resolveUserPath } from "../utils.js"; - -const DEFAULT_SOPS_TIMEOUT_MS = 5_000; -const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024; +import { readJsonPointer } from "./json-pointer.js"; +import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js"; type SecretResolverWarningCode = "SECRETS_REF_OVERRIDES_PLAINTEXT"; @@ -77,61 +75,10 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -function isSecretRef(value: unknown): value is SecretRef { - if (!isRecord(value)) { - return false; - } - if (Object.keys(value).length !== 2) { - return false; - } - return ( - (value.source === "env" || value.source === "file") && - typeof value.id === "string" && - value.id.trim().length > 0 - ); -} - function isNonEmptyString(value: unknown): value is string { return typeof value === "string" && value.trim().length > 0; } -function decodeJsonPointerToken(token: string): string { - return token.replace(/~1/g, "/").replace(/~0/g, "~"); -} - -function readJsonPointer(root: unknown, pointer: string): unknown { - if (!pointer.startsWith("/")) { - throw new Error( - `File-backed secret ids must be absolute JSON pointers (for example: /providers/openai/apiKey).`, - ); - } - - const tokens = pointer - .slice(1) - .split("/") - .map((token) => decodeJsonPointerToken(token)); - - let current: unknown = root; - for (const token of tokens) { - if (Array.isArray(current)) { - const index = Number.parseInt(token, 10); - if (!Number.isFinite(index) || index < 0 || index >= current.length) { - throw new Error(`JSON pointer segment "${token}" is out of bounds.`); - } - current = current[index]; - continue; - } - if (!isRecord(current)) { - throw new Error(`JSON pointer segment "${token}" does not exist.`); - } - if (!Object.hasOwn(current, token)) { - throw new Error(`JSON pointer segment "${token}" does not exist.`); - } - current = current[token]; - } - return current; -} - async function decryptSopsFile(config: OpenClawConfig): Promise { const fileSource = config.secrets?.sources?.file; if (!fileSource) { @@ -148,30 +95,12 @@ async function decryptSopsFile(config: OpenClawConfig): Promise { typeof fileSource.timeoutMs === "number" && Number.isFinite(fileSource.timeoutMs) ? Math.max(1, Math.floor(fileSource.timeoutMs)) : DEFAULT_SOPS_TIMEOUT_MS; - - try { - const { stdout } = await runExec("sops", ["--decrypt", "--output-type", "json", resolvedPath], { - timeoutMs, - maxBuffer: MAX_SOPS_OUTPUT_BYTES, - }); - return JSON.parse(stdout) as unknown; - } catch (err) { - const error = err as NodeJS.ErrnoException & { message?: string }; - if (error.code === "ENOENT") { - throw new Error( - `sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.`, - { cause: err }, - ); - } - if (typeof error.message === "string" && error.message.toLowerCase().includes("timed out")) { - throw new Error(`sops decrypt timed out after ${timeoutMs}ms for ${resolvedPath}.`, { - cause: err, - }); - } - throw new Error(`sops decrypt failed for ${resolvedPath}: ${String(error.message ?? err)}`, { - cause: err, - }); - } + return await decryptSopsJsonFile({ + path: resolvedPath, + timeoutMs, + missingBinaryMessage: + "sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.", + }); } async function resolveSecretRefValue(ref: SecretRef, context: ResolverContext): Promise { @@ -191,7 +120,7 @@ async function resolveSecretRefValue(ref: SecretRef, context: ResolverContext): if (ref.source === "file") { context.fileSecretsPromise ??= decryptSopsFile(context.config); const payload = await context.fileSecretsPromise; - return readJsonPointer(payload, id); + return readJsonPointer(payload, id, { onMissing: "throw" }); } throw new Error(`Unsupported secret source "${String((ref as { source?: unknown }).source)}".`); @@ -369,7 +298,7 @@ export async function prepareSecretsRuntimeSnapshot(params: { export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): void { const next = cloneSnapshot(snapshot); - setRuntimeConfigSnapshot(next.config); + setRuntimeConfigSnapshot(next.config, next.sourceConfig); replaceRuntimeAuthProfileStoreSnapshots(next.authStores); activeSnapshot = next; } diff --git a/src/secrets/sops.ts b/src/secrets/sops.ts new file mode 100644 index 000000000..81dae502b --- /dev/null +++ b/src/secrets/sops.ts @@ -0,0 +1,120 @@ +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { runExec } from "../process/exec.js"; + +export const DEFAULT_SOPS_TIMEOUT_MS = 5_000; +const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024; + +function normalizeTimeoutMs(value: number | undefined): number { + if (typeof value === "number" && Number.isFinite(value)) { + return Math.max(1, Math.floor(value)); + } + return DEFAULT_SOPS_TIMEOUT_MS; +} + +function isTimeoutError(message: string | undefined): boolean { + return typeof message === "string" && message.toLowerCase().includes("timed out"); +} + +type SopsErrorContext = { + missingBinaryMessage: string; + operationLabel: string; +}; + +function toSopsError(err: unknown, params: SopsErrorContext): Error { + const error = err as NodeJS.ErrnoException & { message?: string }; + if (error.code === "ENOENT") { + return new Error(params.missingBinaryMessage, { cause: err }); + } + return new Error(`${params.operationLabel}: ${String(error.message ?? err)}`, { + cause: err, + }); +} + +function ensureDirForFile(filePath: string): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 }); +} + +export async function decryptSopsJsonFile(params: { + path: string; + timeoutMs?: number; + missingBinaryMessage: string; +}): Promise { + const timeoutMs = normalizeTimeoutMs(params.timeoutMs); + try { + const { stdout } = await runExec("sops", ["--decrypt", "--output-type", "json", params.path], { + timeoutMs, + maxBuffer: MAX_SOPS_OUTPUT_BYTES, + }); + return JSON.parse(stdout) as unknown; + } catch (err) { + const error = err as NodeJS.ErrnoException & { message?: string }; + if (isTimeoutError(error.message)) { + throw new Error(`sops decrypt timed out after ${timeoutMs}ms for ${params.path}.`, { + cause: err, + }); + } + throw toSopsError(err, { + missingBinaryMessage: params.missingBinaryMessage, + operationLabel: `sops decrypt failed for ${params.path}`, + }); + } +} + +export async function encryptSopsJsonFile(params: { + path: string; + payload: Record; + timeoutMs?: number; + missingBinaryMessage: string; +}): Promise { + ensureDirForFile(params.path); + const timeoutMs = normalizeTimeoutMs(params.timeoutMs); + const tmpPlain = path.join( + path.dirname(params.path), + `${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.plain.tmp`, + ); + const tmpEncrypted = path.join( + path.dirname(params.path), + `${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.enc.tmp`, + ); + + fs.writeFileSync(tmpPlain, `${JSON.stringify(params.payload, null, 2)}\n`, "utf8"); + fs.chmodSync(tmpPlain, 0o600); + + try { + await runExec( + "sops", + [ + "--encrypt", + "--input-type", + "json", + "--output-type", + "json", + "--output", + tmpEncrypted, + tmpPlain, + ], + { + timeoutMs, + maxBuffer: MAX_SOPS_OUTPUT_BYTES, + }, + ); + fs.renameSync(tmpEncrypted, params.path); + fs.chmodSync(params.path, 0o600); + } catch (err) { + const error = err as NodeJS.ErrnoException & { message?: string }; + if (isTimeoutError(error.message)) { + throw new Error(`sops encrypt timed out after ${timeoutMs}ms for ${params.path}.`, { + cause: err, + }); + } + throw toSopsError(err, { + missingBinaryMessage: params.missingBinaryMessage, + operationLabel: `sops encrypt failed for ${params.path}`, + }); + } finally { + fs.rmSync(tmpPlain, { force: true }); + fs.rmSync(tmpEncrypted, { force: true }); + } +}