admin/Moltbot
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

Secrets: preserve runtime snapshot source refs on write

Browse Source
This commit is contained in:
joshavant
2026-02-21 17:08:14 -08:00
committed by Peter Steinberger
parent b1533bc80c
commit e4915cb107
7 changed files with 347 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
src/config/io.runtime-snapshot-write.test.ts Normal file
View File

@@ -0,0 +1,63 @@
import fs from "node:fs/promises";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { withTempHome } from "./home-env.test-harness.js";
import {
clearConfigCache,
clearRuntimeConfigSnapshot,
loadConfig,
setRuntimeConfigSnapshot,
writeConfigFile,
} from "./io.js";
import type { OpenClawConfig } from "./types.js";
describe("runtime config snapshot writes", () => {
it("preserves source secret refs when writeConfigFile receives runtime-resolved config", async () => {
await withTempHome("openclaw-config-runtime-write-", async (home) => {
const configPath = path.join(home, ".openclaw", "openclaw.json");
const sourceConfig: OpenClawConfig = {
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: { source: "env", id: "OPENAI_API_KEY" },
models: [],
},
},
},
};
const runtimeConfig: OpenClawConfig = {
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: "sk-runtime-resolved",
models: [],
},
},
},
};
await fs.mkdir(path.dirname(configPath), { recursive: true });
await fs.writeFile(configPath, `${JSON.stringify(sourceConfig, null, 2)}\n`, "utf8");
try {
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
expect(loadConfig().models?.providers?.openai?.apiKey).toBe("sk-runtime-resolved");
await writeConfigFile(loadConfig());
const persisted = JSON.parse(await fs.readFile(configPath, "utf8")) as {
models?: { providers?: { openai?: { apiKey?: unknown } } };
};
expect(persisted.models?.providers?.openai?.apiKey).toEqual({
source: "env",
id: "OPENAI_API_KEY",
});
} finally {
clearRuntimeConfigSnapshot();
clearConfigCache();
}
});
});
});

15
src/config/io.ts
View File

@@ -1299,6 +1299,7 @@ let configCache: {
config: OpenClawConfig;
} | null = null;
let runtimeConfigSnapshot: OpenClawConfig | null = null;
let runtimeConfigSourceSnapshot: OpenClawConfig | null = null;
function resolveConfigCacheMs(env: NodeJS.ProcessEnv): number {
const raw = env.OPENCLAW_CONFIG_CACHE_MS?.trim();
@@ -1326,13 +1327,18 @@ export function clearConfigCache(): void {
configCache = null;
}
export function setRuntimeConfigSnapshot(config: OpenClawConfig): void {
export function setRuntimeConfigSnapshot(
config: OpenClawConfig,
sourceConfig?: OpenClawConfig,
): void {
runtimeConfigSnapshot = config;
runtimeConfigSourceSnapshot = sourceConfig ?? null;
clearConfigCache();
}
export function clearRuntimeConfigSnapshot(): void {
runtimeConfigSnapshot = null;
runtimeConfigSourceSnapshot = null;
clearConfigCache();
}
@@ -1380,9 +1386,14 @@ export async function writeConfigFile(
options: ConfigWriteOptions = {},
): Promise<void> {
const io = createConfigIO();
let nextCfg = cfg;
if (runtimeConfigSnapshot && runtimeConfigSourceSnapshot) {
const runtimePatch = createMergePatch(runtimeConfigSnapshot, cfg);
nextCfg = coerceConfig(applyMergePatch(runtimeConfigSourceSnapshot, runtimePatch));
}
const sameConfigPath =
options.expectedConfigPath === undefined || options.expectedConfigPath === io.configPath;
await io.writeConfigFile(cfg, {
await io.writeConfigFile(nextCfg, {
envSnapshotForRestore: sameConfigPath ? options.envSnapshotForRestore : undefined,
unsetPaths: options.unsetPaths,
});

18
src/config/types.secrets.ts
View File

@@ -13,6 +13,24 @@ export type SecretRef = {
export type SecretInput = string | SecretRef;
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
export function isSecretRef(value: unknown): value is SecretRef {
if (!isRecord(value)) {
return false;
}
if (Object.keys(value).length !== 2) {
return false;
}
return (
(value.source === "env" || value.source === "file") &&
typeof value.id === "string" &&
value.id.trim().length > 0
);
}
export type EnvSecretSourceConfig = {
type?: "env";
};

94
src/secrets/json-pointer.ts Normal file
View File

@@ -0,0 +1,94 @@
function failOrUndefined(params: { onMissing: "throw" | "undefined"; message: string }): undefined {
if (params.onMissing === "throw") {
throw new Error(params.message);
}
return undefined;
}
export function decodeJsonPointerToken(token: string): string {
return token.replace(/~1/g, "/").replace(/~0/g, "~");
}
export function encodeJsonPointerToken(token: string): string {
return token.replace(/~/g, "~0").replace(/\//g, "~1");
}
export function readJsonPointer(
root: unknown,
pointer: string,
options: { onMissing?: "throw" | "undefined" } = {},
): unknown {
const onMissing = options.onMissing ?? "throw";
if (!pointer.startsWith("/")) {
return failOrUndefined({
onMissing,
message:
'File-backed secret ids must be absolute JSON pointers (for example: "/providers/openai/apiKey").',
});
}
const tokens = pointer
.slice(1)
.split("/")
.map((token) => decodeJsonPointerToken(token));
let current: unknown = root;
for (const token of tokens) {
if (Array.isArray(current)) {
const index = Number.parseInt(token, 10);
if (!Number.isFinite(index) || index < 0 || index >= current.length) {
return failOrUndefined({
onMissing,
message: `JSON pointer segment "${token}" is out of bounds.`,
});
}
current = current[index];
continue;
}
if (typeof current !== "object" || current === null || Array.isArray(current)) {
return failOrUndefined({
onMissing,
message: `JSON pointer segment "${token}" does not exist.`,
});
}
const record = current as Record<string, unknown>;
if (!Object.hasOwn(record, token)) {
return failOrUndefined({
onMissing,
message: `JSON pointer segment "${token}" does not exist.`,
});
}
current = record[token];
}
return current;
}
export function setJsonPointer(
root: Record<string, unknown>,
pointer: string,
value: unknown,
): void {
if (!pointer.startsWith("/")) {
throw new Error(`Invalid JSON pointer "${pointer}".`);
}
const tokens = pointer
.slice(1)
.split("/")
.map((token) => decodeJsonPointerToken(token));
let current: Record<string, unknown> = root;
for (let index = 0; index < tokens.length; index += 1) {
const token = tokens[index];
const isLast = index === tokens.length - 1;
if (isLast) {
current[token] = value;
return;
}
const child = current[token];
if (typeof child !== "object" || child === null || Array.isArray(child)) {
current[token] = {};
}
current = current[token] as Record<string, unknown>;
}
}

28
src/secrets/provider-env-vars.ts Normal file
View File

@@ -0,0 +1,28 @@
export const PROVIDER_ENV_VARS: Record<string, readonly string[]> = {
openai: ["OPENAI_API_KEY"],
anthropic: ["ANTHROPIC_API_KEY"],
google: ["GEMINI_API_KEY"],
minimax: ["MINIMAX_API_KEY"],
"minimax-cn": ["MINIMAX_API_KEY"],
moonshot: ["MOONSHOT_API_KEY"],
"kimi-coding": ["KIMI_API_KEY", "KIMICODE_API_KEY"],
synthetic: ["SYNTHETIC_API_KEY"],
venice: ["VENICE_API_KEY"],
zai: ["ZAI_API_KEY", "Z_AI_API_KEY"],
xiaomi: ["XIAOMI_API_KEY"],
openrouter: ["OPENROUTER_API_KEY"],
"cloudflare-ai-gateway": ["CLOUDFLARE_AI_GATEWAY_API_KEY"],
litellm: ["LITELLM_API_KEY"],
"vercel-ai-gateway": ["AI_GATEWAY_API_KEY"],
opencode: ["OPENCODE_API_KEY", "OPENCODE_ZEN_API_KEY"],
together: ["TOGETHER_API_KEY"],
huggingface: ["HUGGINGFACE_HUB_TOKEN", "HF_TOKEN"],
qianfan: ["QIANFAN_API_KEY"],
xai: ["XAI_API_KEY"],
volcengine: ["VOLCANO_ENGINE_API_KEY"],
byteplus: ["BYTEPLUS_API_KEY"],
};
export function listKnownSecretEnvVarNames(): string[] {
return [...new Set(Object.values(PROVIDER_ENV_VARS).flatMap((keys) => keys))];
}

93
src/secrets/runtime.ts
View File

@@ -10,13 +10,11 @@ import {
clearRuntimeConfigSnapshot,
setRuntimeConfigSnapshot,
type OpenClawConfig,
type SecretRef,
} from "../config/config.js";
import { runExec } from "../process/exec.js";
import { isSecretRef, type SecretRef } from "../config/types.secrets.js";
import { resolveUserPath } from "../utils.js";
const DEFAULT_SOPS_TIMEOUT_MS = 5_000;
const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024;
import { readJsonPointer } from "./json-pointer.js";
import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js";
type SecretResolverWarningCode = "SECRETS_REF_OVERRIDES_PLAINTEXT";
@@ -77,61 +75,10 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function isSecretRef(value: unknown): value is SecretRef {
if (!isRecord(value)) {
return false;
}
if (Object.keys(value).length !== 2) {
return false;
}
return (
(value.source === "env" || value.source === "file") &&
typeof value.id === "string" &&
value.id.trim().length > 0
);
}
function isNonEmptyString(value: unknown): value is string {
return typeof value === "string" && value.trim().length > 0;
}
function decodeJsonPointerToken(token: string): string {
return token.replace(/~1/g, "/").replace(/~0/g, "~");
}
function readJsonPointer(root: unknown, pointer: string): unknown {
if (!pointer.startsWith("/")) {
throw new Error(
`File-backed secret ids must be absolute JSON pointers (for example: /providers/openai/apiKey).`,
);
}
const tokens = pointer
.slice(1)
.split("/")
.map((token) => decodeJsonPointerToken(token));
let current: unknown = root;
for (const token of tokens) {
if (Array.isArray(current)) {
const index = Number.parseInt(token, 10);
if (!Number.isFinite(index) || index < 0 || index >= current.length) {
throw new Error(`JSON pointer segment "${token}" is out of bounds.`);
}
current = current[index];
continue;
}
if (!isRecord(current)) {
throw new Error(`JSON pointer segment "${token}" does not exist.`);
}
if (!Object.hasOwn(current, token)) {
throw new Error(`JSON pointer segment "${token}" does not exist.`);
}
current = current[token];
}
return current;
}
async function decryptSopsFile(config: OpenClawConfig): Promise<unknown> {
const fileSource = config.secrets?.sources?.file;
if (!fileSource) {
@@ -148,30 +95,12 @@ async function decryptSopsFile(config: OpenClawConfig): Promise<unknown> {
typeof fileSource.timeoutMs === "number" && Number.isFinite(fileSource.timeoutMs)
? Math.max(1, Math.floor(fileSource.timeoutMs))
: DEFAULT_SOPS_TIMEOUT_MS;
try {
const { stdout } = await runExec("sops", ["--decrypt", "--output-type", "json", resolvedPath], {
timeoutMs,
maxBuffer: MAX_SOPS_OUTPUT_BYTES,
});
return JSON.parse(stdout) as unknown;
} catch (err) {
const error = err as NodeJS.ErrnoException & { message?: string };
if (error.code === "ENOENT") {
throw new Error(
`sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.`,
{ cause: err },
);
}
if (typeof error.message === "string" && error.message.toLowerCase().includes("timed out")) {
throw new Error(`sops decrypt timed out after ${timeoutMs}ms for ${resolvedPath}.`, {
cause: err,
});
}
throw new Error(`sops decrypt failed for ${resolvedPath}: ${String(error.message ?? err)}`, {
cause: err,
});
}
return await decryptSopsJsonFile({
path: resolvedPath,
timeoutMs,
missingBinaryMessage:
"sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.",
});
}
async function resolveSecretRefValue(ref: SecretRef, context: ResolverContext): Promise<unknown> {
@@ -191,7 +120,7 @@ async function resolveSecretRefValue(ref: SecretRef, context: ResolverContext):
if (ref.source === "file") {
context.fileSecretsPromise ??= decryptSopsFile(context.config);
const payload = await context.fileSecretsPromise;
return readJsonPointer(payload, id);
return readJsonPointer(payload, id, { onMissing: "throw" });
}
throw new Error(`Unsupported secret source "${String((ref as { source?: unknown }).source)}".`);
@@ -369,7 +298,7 @@ export async function prepareSecretsRuntimeSnapshot(params: {
export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): void {
const next = cloneSnapshot(snapshot);
setRuntimeConfigSnapshot(next.config);
setRuntimeConfigSnapshot(next.config, next.sourceConfig);
replaceRuntimeAuthProfileStoreSnapshots(next.authStores);
activeSnapshot = next;
}

120
src/secrets/sops.ts Normal file
View File

@@ -0,0 +1,120 @@
import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { runExec } from "../process/exec.js";
export const DEFAULT_SOPS_TIMEOUT_MS = 5_000;
const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024;
function normalizeTimeoutMs(value: number | undefined): number {
if (typeof value === "number" && Number.isFinite(value)) {
return Math.max(1, Math.floor(value));
}
return DEFAULT_SOPS_TIMEOUT_MS;
}
function isTimeoutError(message: string | undefined): boolean {
return typeof message === "string" && message.toLowerCase().includes("timed out");
}
type SopsErrorContext = {
missingBinaryMessage: string;
operationLabel: string;
};
function toSopsError(err: unknown, params: SopsErrorContext): Error {
const error = err as NodeJS.ErrnoException & { message?: string };
if (error.code === "ENOENT") {
return new Error(params.missingBinaryMessage, { cause: err });
}
return new Error(`${params.operationLabel}: ${String(error.message ?? err)}`, {
cause: err,
});
}
function ensureDirForFile(filePath: string): void {
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
}
export async function decryptSopsJsonFile(params: {
path: string;
timeoutMs?: number;
missingBinaryMessage: string;
}): Promise<unknown> {
const timeoutMs = normalizeTimeoutMs(params.timeoutMs);
try {
const { stdout } = await runExec("sops", ["--decrypt", "--output-type", "json", params.path], {
timeoutMs,
maxBuffer: MAX_SOPS_OUTPUT_BYTES,
});
return JSON.parse(stdout) as unknown;
} catch (err) {
const error = err as NodeJS.ErrnoException & { message?: string };
if (isTimeoutError(error.message)) {
throw new Error(`sops decrypt timed out after ${timeoutMs}ms for ${params.path}.`, {
cause: err,
});
}
throw toSopsError(err, {
missingBinaryMessage: params.missingBinaryMessage,
operationLabel: `sops decrypt failed for ${params.path}`,
});
}
}
export async function encryptSopsJsonFile(params: {
path: string;
payload: Record<string, unknown>;
timeoutMs?: number;
missingBinaryMessage: string;
}): Promise<void> {
ensureDirForFile(params.path);
const timeoutMs = normalizeTimeoutMs(params.timeoutMs);
const tmpPlain = path.join(
path.dirname(params.path),
`${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.plain.tmp`,
);
const tmpEncrypted = path.join(
path.dirname(params.path),
`${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.enc.tmp`,
);
fs.writeFileSync(tmpPlain, `${JSON.stringify(params.payload, null, 2)}\n`, "utf8");
fs.chmodSync(tmpPlain, 0o600);
try {
await runExec(
"sops",
[
"--encrypt",
"--input-type",
"json",
"--output-type",
"json",
"--output",
tmpEncrypted,
tmpPlain,
],
{
timeoutMs,
maxBuffer: MAX_SOPS_OUTPUT_BYTES,
},
);
fs.renameSync(tmpEncrypted, params.path);
fs.chmodSync(params.path, 0o600);
} catch (err) {
const error = err as NodeJS.ErrnoException & { message?: string };
if (isTimeoutError(error.message)) {
throw new Error(`sops encrypt timed out after ${timeoutMs}ms for ${params.path}.`, {
cause: err,
});
}
throw toSopsError(err, {
missingBinaryMessage: params.missingBinaryMessage,
operationLabel: `sops encrypt failed for ${params.path}`,
});
} finally {
fs.rmSync(tmpPlain, { force: true });
fs.rmSync(tmpEncrypted, { force: true });
}
}