From 4e7a833a24e71edb2dd50f081a46dc46119bbf98 Mon Sep 17 00:00:00 2001 From: joshavant <830519+joshavant@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:39:31 -0600 Subject: [PATCH] feat(security): add provider-based external secrets management --- ...uth-profiles.runtime-snapshot-save.test.ts | 5 +- src/agents/auth-profiles.store.save.test.ts | 6 +- src/agents/auth-profiles/oauth.test.ts | 70 +- src/agents/auth-profiles/oauth.ts | 51 +- src/agents/model-auth-label.test.ts | 2 +- src/cli/secrets-cli.ts | 2 +- .../auth-choice.apply-helpers.test.ts | 28 +- src/commands/auth-choice.apply-helpers.ts | 142 +++- .../auth-choice.apply.minimax.test.ts | 1 + src/commands/auth-choice.apply.openai.test.ts | 2 +- ...h-choice.apply.volcengine-byteplus.test.ts | 4 +- src/commands/auth-choice.test.ts | 39 +- .../models/list.auth-overview.test.ts | 2 +- src/commands/onboard-auth.credentials.test.ts | 12 +- src/commands/onboard-auth.credentials.ts | 14 +- src/commands/onboard-custom.test.ts | 20 +- ...oard-non-interactive.provider-auth.test.ts | 1 + .../local/auth-choice.ts | 2 +- src/config/config.secrets-schema.test.ts | 32 +- src/config/io.runtime-snapshot-write.test.ts | 3 +- src/config/redact-snapshot.test.ts | 3 +- src/config/types.secrets.ts | 125 +++- src/config/zod-schema.core.ts | 128 +++- src/gateway/config-reload.test.ts | 6 + src/gateway/config-reload.ts | 1 + src/gateway/server.reload.test.ts | 6 +- src/secrets/migrate.test.ts | 123 ++-- src/secrets/migrate/apply.ts | 24 +- src/secrets/migrate/plan.ts | 139 ++-- src/secrets/migrate/types.ts | 2 - src/secrets/resolve.test.ts | 154 ++++ src/secrets/resolve.ts | 681 ++++++++++++++++-- src/secrets/runtime.test.ts | 191 ++--- src/secrets/runtime.ts | 275 ++++--- src/secrets/sops.ts | 152 ---- 35 files changed, 1779 insertions(+), 669 deletions(-) create mode 100644 src/secrets/resolve.test.ts delete mode 100644 src/secrets/sops.ts diff --git a/src/agents/auth-profiles.runtime-snapshot-save.test.ts b/src/agents/auth-profiles.runtime-snapshot-save.test.ts index 5ab2635d9..3cb3d2389 100644 --- a/src/agents/auth-profiles.runtime-snapshot-save.test.ts +++ b/src/agents/auth-profiles.runtime-snapshot-save.test.ts @@ -25,7 +25,7 @@ describe("auth profile runtime snapshot persistence", () => { "openai:default": { type: "api_key", provider: "openai", - keyRef: { source: "env", id: "OPENAI_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, }, }, }, @@ -46,7 +46,7 @@ describe("auth profile runtime snapshot persistence", () => { expect(runtimeStore.profiles["openai:default"]).toMatchObject({ type: "api_key", key: "sk-runtime-openai", - keyRef: { source: "env", id: "OPENAI_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, }); await markAuthProfileUsed({ @@ -61,6 +61,7 @@ describe("auth profile runtime snapshot persistence", () => { expect(persisted.profiles["openai:default"]?.key).toBeUndefined(); expect(persisted.profiles["openai:default"]?.keyRef).toEqual({ source: "env", + provider: "default", id: "OPENAI_API_KEY", }); } finally { diff --git a/src/agents/auth-profiles.store.save.test.ts b/src/agents/auth-profiles.store.save.test.ts index 20aac07d2..292921fea 100644 --- a/src/agents/auth-profiles.store.save.test.ts +++ b/src/agents/auth-profiles.store.save.test.ts @@ -17,13 +17,13 @@ describe("saveAuthProfileStore", () => { type: "api_key", provider: "openai", key: "sk-runtime-value", - keyRef: { source: "env", id: "OPENAI_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, }, "github-copilot:default": { type: "token", provider: "github-copilot", token: "gh-runtime-token", - tokenRef: { source: "env", id: "GITHUB_TOKEN" }, + tokenRef: { source: "env", provider: "default", id: "GITHUB_TOKEN" }, }, "anthropic:default": { type: "api_key", @@ -45,12 +45,14 @@ describe("saveAuthProfileStore", () => { expect(parsed.profiles["openai:default"]?.key).toBeUndefined(); expect(parsed.profiles["openai:default"]?.keyRef).toEqual({ source: "env", + provider: "default", id: "OPENAI_API_KEY", }); expect(parsed.profiles["github-copilot:default"]?.token).toBeUndefined(); expect(parsed.profiles["github-copilot:default"]?.tokenRef).toEqual({ source: "env", + provider: "default", id: "GITHUB_TOKEN", }); diff --git a/src/agents/auth-profiles/oauth.test.ts b/src/agents/auth-profiles/oauth.test.ts index d5b229e45..e4c8c536c 100644 --- a/src/agents/auth-profiles/oauth.test.ts +++ b/src/agents/auth-profiles/oauth.test.ts @@ -183,7 +183,7 @@ describe("resolveApiKeyForProfile secret refs", () => { [profileId]: { type: "api_key", provider: "openai", - keyRef: { source: "env", id: "OPENAI_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, }, }, }, @@ -217,7 +217,7 @@ describe("resolveApiKeyForProfile secret refs", () => { type: "token", provider: "github-copilot", token: "", - tokenRef: { source: "env", id: "GITHUB_TOKEN" }, + tokenRef: { source: "env", provider: "default", id: "GITHUB_TOKEN" }, }, }, }, @@ -236,4 +236,70 @@ describe("resolveApiKeyForProfile secret refs", () => { } } }); + + it("resolves inline ${ENV} api_key values", async () => { + const profileId = "openai:inline-env"; + const previous = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = "sk-openai-inline"; + try { + const result = await resolveApiKeyForProfile({ + cfg: cfgFor(profileId, "openai", "api_key"), + store: { + version: 1, + profiles: { + [profileId]: { + type: "api_key", + provider: "openai", + key: "${OPENAI_API_KEY}", + }, + }, + }, + profileId, + }); + expect(result).toEqual({ + apiKey: "sk-openai-inline", + provider: "openai", + email: undefined, + }); + } finally { + if (previous === undefined) { + delete process.env.OPENAI_API_KEY; + } else { + process.env.OPENAI_API_KEY = previous; + } + } + }); + + it("resolves inline ${ENV} token values", async () => { + const profileId = "github-copilot:inline-env"; + const previous = process.env.GITHUB_TOKEN; + process.env.GITHUB_TOKEN = "gh-inline-token"; + try { + const result = await resolveApiKeyForProfile({ + cfg: cfgFor(profileId, "github-copilot", "token"), + store: { + version: 1, + profiles: { + [profileId]: { + type: "token", + provider: "github-copilot", + token: "${GITHUB_TOKEN}", + }, + }, + }, + profileId, + }); + expect(result).toEqual({ + apiKey: "gh-inline-token", + provider: "github-copilot", + email: undefined, + }); + } finally { + if (previous === undefined) { + delete process.env.GITHUB_TOKEN; + } else { + process.env.GITHUB_TOKEN = previous; + } + } + }); }); diff --git a/src/agents/auth-profiles/oauth.ts b/src/agents/auth-profiles/oauth.ts index 258eb215b..25d99348a 100644 --- a/src/agents/auth-profiles/oauth.ts +++ b/src/agents/auth-profiles/oauth.ts @@ -5,7 +5,7 @@ import { type OAuthProvider, } from "@mariozechner/pi-ai"; import { loadConfig, type OpenClawConfig } from "../../config/config.js"; -import { isSecretRef } from "../../config/types.secrets.js"; +import { coerceSecretRef } from "../../config/types.secrets.js"; import { withFileLock } from "../../infra/file-lock.js"; import { refreshQwenPortalCredentials } from "../../providers/qwen-portal-oauth.js"; import { resolveSecretRefString, type SecretRefResolveCache } from "../../secrets/resolve.js"; @@ -257,14 +257,34 @@ export async function resolveApiKeyForProfile( return null; } - const refResolveCache: SecretRefResolveCache = { fileSecretsPromise: null }; + const refResolveCache: SecretRefResolveCache = {}; const configForRefResolution = cfg ?? loadConfig(); + const refDefaults = configForRefResolution.secrets?.defaults; if (cred.type === "api_key") { let key = cred.key?.trim(); - if (!key && isSecretRef(cred.keyRef)) { + if (key) { + const inlineRef = coerceSecretRef(key, refDefaults); + if (inlineRef) { + try { + key = await resolveSecretRefString(inlineRef, { + config: configForRefResolution, + env: process.env, + cache: refResolveCache, + }); + } catch (err) { + log.debug("failed to resolve inline auth profile api_key ref", { + profileId, + provider: cred.provider, + error: err instanceof Error ? err.message : String(err), + }); + } + } + } + const keyRef = coerceSecretRef(cred.keyRef, refDefaults); + if (!key && keyRef) { try { - key = await resolveSecretRefString(cred.keyRef, { + key = await resolveSecretRefString(keyRef, { config: configForRefResolution, env: process.env, cache: refResolveCache, @@ -284,9 +304,28 @@ export async function resolveApiKeyForProfile( } if (cred.type === "token") { let token = cred.token?.trim(); - if (!token && isSecretRef(cred.tokenRef)) { + if (token) { + const inlineRef = coerceSecretRef(token, refDefaults); + if (inlineRef) { + try { + token = await resolveSecretRefString(inlineRef, { + config: configForRefResolution, + env: process.env, + cache: refResolveCache, + }); + } catch (err) { + log.debug("failed to resolve inline auth profile token ref", { + profileId, + provider: cred.provider, + error: err instanceof Error ? err.message : String(err), + }); + } + } + } + const tokenRef = coerceSecretRef(cred.tokenRef, refDefaults); + if (!token && tokenRef) { try { - token = await resolveSecretRefString(cred.tokenRef, { + token = await resolveSecretRefString(tokenRef, { config: configForRefResolution, env: process.env, cache: refResolveCache, diff --git a/src/agents/model-auth-label.test.ts b/src/agents/model-auth-label.test.ts index 586949e79..adcb6ce49 100644 --- a/src/agents/model-auth-label.test.ts +++ b/src/agents/model-auth-label.test.ts @@ -32,7 +32,7 @@ describe("resolveModelAuthLabel", () => { "github-copilot:default": { type: "token", provider: "github-copilot", - tokenRef: { source: "env", id: "GITHUB_TOKEN" }, + tokenRef: { source: "env", provider: "default", id: "GITHUB_TOKEN" }, }, }, } as never); diff --git a/src/cli/secrets-cli.ts b/src/cli/secrets-cli.ts index 50248836b..575dce13b 100644 --- a/src/cli/secrets-cli.ts +++ b/src/cli/secrets-cli.ts @@ -95,7 +95,7 @@ export function registerSecretsCli(program: Command) { secrets .command("migrate") - .description("Migrate plaintext secrets to file-backed SecretRefs (sops)") + .description("Migrate plaintext secrets to file-backed SecretRefs") .option("--write", "Apply migration changes (default is dry-run)", false) .option("--rollback ", "Rollback a previous migration backup id") .option("--no-scrub-env", "Keep matching plaintext values in ~/.openclaw/.env") diff --git a/src/commands/auth-choice.apply-helpers.test.ts b/src/commands/auth-choice.apply-helpers.test.ts index 5c2c71ef7..148696c74 100644 --- a/src/commands/auth-choice.apply-helpers.test.ts +++ b/src/commands/auth-choice.apply-helpers.test.ts @@ -177,15 +177,18 @@ describe("ensureApiKeyFromEnvOrPrompt", () => { }); expect(result).toBe("env-key"); - expect(setCredential).toHaveBeenCalledWith({ source: "env", id: "MINIMAX_API_KEY" }, "ref"); + expect(setCredential).toHaveBeenCalledWith( + { source: "env", provider: "default", id: "MINIMAX_API_KEY" }, + "ref", + ); expect(text).not.toHaveBeenCalled(); }); - it("re-prompts after sops ref validation failure and succeeds with env ref", async () => { + it("re-prompts after provider ref validation failure and succeeds with env ref", async () => { process.env.MINIMAX_API_KEY = "env-key"; delete process.env.MINIMAX_OAUTH_TOKEN; - const selectValues: Array<"file" | "env"> = ["file", "env"]; + const selectValues: Array<"provider" | "env" | "filemain"> = ["provider", "filemain", "env"]; const select = vi.fn(async () => selectValues.shift() ?? "env") as WizardPrompter["select"]; const text = vi .fn() @@ -195,7 +198,17 @@ describe("ensureApiKeyFromEnvOrPrompt", () => { const setCredential = vi.fn(async () => undefined); const result = await ensureApiKeyFromEnvOrPrompt({ - config: {}, + config: { + secrets: { + providers: { + filemain: { + source: "file", + path: "/tmp/does-not-exist-secrets.json", + mode: "jsonPointer", + }, + }, + }, + }, provider: "minimax", envLabel: "MINIMAX_API_KEY", promptMessage: "Enter key", @@ -207,9 +220,12 @@ describe("ensureApiKeyFromEnvOrPrompt", () => { }); expect(result).toBe("env-key"); - expect(setCredential).toHaveBeenCalledWith({ source: "env", id: "MINIMAX_API_KEY" }, "ref"); + expect(setCredential).toHaveBeenCalledWith( + { source: "env", provider: "default", id: "MINIMAX_API_KEY" }, + "ref", + ); expect(note).toHaveBeenCalledWith( - expect.stringContaining("Could not validate this encrypted file reference."), + expect.stringContaining("Could not validate provider reference"), "Reference check failed", ); }); diff --git a/src/commands/auth-choice.apply-helpers.ts b/src/commands/auth-choice.apply-helpers.ts index 5857683db..1424027dd 100644 --- a/src/commands/auth-choice.apply-helpers.ts +++ b/src/commands/auth-choice.apply-helpers.ts @@ -1,6 +1,10 @@ import { resolveEnvApiKey } from "../agents/model-auth.js"; import type { OpenClawConfig } from "../config/types.js"; -import type { SecretInput, SecretRef } from "../config/types.secrets.js"; +import { + DEFAULT_SECRET_PROVIDER_ALIAS, + type SecretInput, + type SecretRef, +} from "../config/types.secrets.js"; import { encodeJsonPointerToken } from "../secrets/json-pointer.js"; import { PROVIDER_ENV_VARS } from "../secrets/provider-env-vars.js"; import { resolveSecretRefString } from "../secrets/resolve.js"; @@ -14,7 +18,7 @@ const ENV_SOURCE_LABEL_RE = /(?:^|:\s)([A-Z][A-Z0-9_]*)$/; const ENV_SECRET_REF_ID_RE = /^[A-Z][A-Z0-9_]{0,127}$/; const FILE_SECRET_REF_SEGMENT_RE = /^(?:[^~]|~0|~1)*$/; -type SecretRefSourceChoice = "env" | "file"; +type SecretRefChoice = "env" | "provider"; function isValidFileSecretRefId(value: string): boolean { if (!value.startsWith("/")) { @@ -43,11 +47,36 @@ function resolveDefaultProviderEnvVar(provider: string): string | undefined { return envVars?.find((candidate) => candidate.trim().length > 0); } -function resolveDefaultSopsPointerId(provider: string): string { +function resolveDefaultFilePointerId(provider: string): string { return `/providers/${encodeJsonPointerToken(provider)}/apiKey`; } +function resolveDefaultProviderAlias( + config: OpenClawConfig, + source: "env" | "file" | "exec", +): string { + const configured = + source === "env" + ? config.secrets?.defaults?.env + : source === "file" + ? config.secrets?.defaults?.file + : config.secrets?.defaults?.exec; + if (configured?.trim()) { + return configured.trim(); + } + const providers = config.secrets?.providers; + if (providers) { + for (const [providerName, provider] of Object.entries(providers)) { + if (provider?.source === source) { + return providerName; + } + } + } + return DEFAULT_SECRET_PROVIDER_ALIAS; +} + function resolveRefFallbackInput(params: { + config: OpenClawConfig; provider: string; preferredEnvVar?: string; envKeyValue?: string; @@ -57,7 +86,11 @@ function resolveRefFallbackInput(params: { const value = process.env[fallbackEnvVar]?.trim(); if (value) { return { - input: { source: "env", id: fallbackEnvVar }, + input: { + source: "env", + provider: resolveDefaultProviderAlias(params.config, "env"), + id: fallbackEnvVar, + }, resolvedValue: value, }; } @@ -81,11 +114,11 @@ async function resolveApiKeyRefForOnboarding(params: { }): Promise<{ ref: SecretRef; resolvedValue: string }> { const defaultEnvVar = params.preferredEnvVar ?? resolveDefaultProviderEnvVar(params.provider) ?? ""; - const defaultFilePointer = resolveDefaultSopsPointerId(params.provider); - let sourceChoice: SecretRefSourceChoice = "env"; + const defaultFilePointer = resolveDefaultFilePointerId(params.provider); + let sourceChoice: SecretRefChoice = "env"; while (true) { - const sourceRaw: SecretRefSourceChoice = await params.prompter.select({ + const sourceRaw: SecretRefChoice = await params.prompter.select({ message: "Where is this API key stored?", initialValue: sourceChoice, options: [ @@ -95,13 +128,13 @@ async function resolveApiKeyRefForOnboarding(params: { hint: "Reference a variable from your runtime environment", }, { - value: "file", - label: "Encrypted sops file", - hint: "Reference a JSON pointer from secrets.sources.file", + value: "provider", + label: "Configured secret provider", + hint: "Use a configured file or exec secret provider", }, ], }); - const source: SecretRefSourceChoice = sourceRaw === "file" ? "file" : "env"; + const source: SecretRefChoice = sourceRaw === "provider" ? "provider" : "env"; sourceChoice = source; if (source === "env") { @@ -128,7 +161,11 @@ async function resolveApiKeyRefForOnboarding(params: { `No valid environment variable name provided for provider "${params.provider}".`, ); } - const ref: SecretRef = { source: "env", id: envVar }; + const ref: SecretRef = { + source: "env", + provider: resolveDefaultProviderAlias(params.config, "env"), + id: envVar, + }; const resolvedValue = await resolveSecretRefString(ref, { config: params.config, env: process.env, @@ -140,36 +177,94 @@ async function resolveApiKeyRefForOnboarding(params: { return { ref, resolvedValue }; } - const pointerRaw = await params.prompter.text({ - message: "JSON pointer inside encrypted secrets file", - initialValue: defaultFilePointer, - placeholder: "/providers/openai/apiKey", + const externalProviders = Object.entries(params.config.secrets?.providers ?? {}).filter( + ([, provider]) => provider?.source === "file" || provider?.source === "exec", + ); + if (externalProviders.length === 0) { + await params.prompter.note( + "No file/exec secret providers are configured yet. Add one under secrets.providers, or select Environment variable.", + "No providers configured", + ); + continue; + } + const defaultProvider = resolveDefaultProviderAlias(params.config, "file"); + const selectedProvider = await params.prompter.select({ + message: "Select secret provider", + initialValue: + externalProviders.find(([providerName]) => providerName === defaultProvider)?.[0] ?? + externalProviders[0]?.[0], + options: externalProviders.map(([providerName, provider]) => ({ + value: providerName, + label: providerName, + hint: provider?.source === "exec" ? "Exec provider" : "File provider", + })), + }); + const providerEntry = params.config.secrets?.providers?.[selectedProvider]; + if (!providerEntry || (providerEntry.source !== "file" && providerEntry.source !== "exec")) { + await params.prompter.note( + `Provider "${selectedProvider}" is not a file/exec provider.`, + "Invalid provider", + ); + continue; + } + const idPrompt = + providerEntry.source === "file" + ? "Secret id (JSON pointer for jsonPointer mode, or 'value' for raw mode)" + : "Secret id for the exec provider"; + const idDefault = + providerEntry.source === "file" + ? providerEntry.mode === "raw" + ? "value" + : defaultFilePointer + : `${params.provider}/apiKey`; + const idRaw = await params.prompter.text({ + message: idPrompt, + initialValue: idDefault, + placeholder: providerEntry.source === "file" ? "/providers/openai/apiKey" : "openai/api-key", validate: (value) => { const candidate = value.trim(); - if (!isValidFileSecretRefId(candidate)) { + if (!candidate) { + return "Secret id cannot be empty."; + } + if ( + providerEntry.source === "file" && + providerEntry.mode !== "raw" && + !isValidFileSecretRefId(candidate) + ) { return 'Use an absolute JSON pointer like "/providers/openai/apiKey".'; } + if ( + providerEntry.source === "file" && + providerEntry.mode === "raw" && + candidate !== "value" + ) { + return 'Raw file mode expects id "value".'; + } return undefined; }, }); - const pointer = String(pointerRaw ?? "").trim() || defaultFilePointer; - const ref: SecretRef = { source: "file", id: pointer }; + const id = String(idRaw ?? "").trim() || idDefault; + const ref: SecretRef = { + source: providerEntry.source, + provider: selectedProvider, + id, + }; try { const resolvedValue = await resolveSecretRefString(ref, { config: params.config, env: process.env, }); await params.prompter.note( - `Validated encrypted file reference ${pointer}. OpenClaw will store a reference, not the key value.`, + `Validated ${providerEntry.source} reference ${selectedProvider}:${id}. OpenClaw will store a reference, not the key value.`, "Reference validated", ); return { ref, resolvedValue }; } catch (error) { await params.prompter.note( [ - "Could not validate this encrypted file reference.", + `Could not validate provider reference ${selectedProvider}:${id}.`, formatErrorMessage(error), - "Check secrets.sources.file configuration and sops key access, then try again.", + "Check your provider configuration and try again.", ].join("\n"), "Reference check failed", ); @@ -287,7 +382,7 @@ export async function resolveSecretInputModeForEnvSelection(params: { { value: "ref", label: "Use secret reference", - hint: "Stores a reference to env or encrypted sops secrets", + hint: "Stores a reference to env or configured external secret providers", }, ], }); @@ -379,6 +474,7 @@ export async function ensureApiKeyFromEnvOrPrompt(params: { if (selectedMode === "ref") { if (typeof params.prompter.select !== "function") { const fallback = resolveRefFallbackInput({ + config: params.config, provider: params.provider, preferredEnvVar: envKey?.source ? extractEnvVarFromSourceLabel(envKey.source) : undefined, envKeyValue: envKey?.apiKey, diff --git a/src/commands/auth-choice.apply.minimax.test.ts b/src/commands/auth-choice.apply.minimax.test.ts index aba00dada..c3de54b1e 100644 --- a/src/commands/auth-choice.apply.minimax.test.ts +++ b/src/commands/auth-choice.apply.minimax.test.ts @@ -181,6 +181,7 @@ describe("applyAuthChoiceMiniMax", () => { const parsed = await readAuthProfiles(agentDir); expect(parsed.profiles?.["minimax-cn:default"]?.keyRef).toEqual({ source: "env", + provider: "default", id: "MINIMAX_API_KEY", }); expect(parsed.profiles?.["minimax-cn:default"]?.key).toBeUndefined(); diff --git a/src/commands/auth-choice.apply.openai.test.ts b/src/commands/auth-choice.apply.openai.test.ts index c74081f6d..8ec1c667f 100644 --- a/src/commands/auth-choice.apply.openai.test.ts +++ b/src/commands/auth-choice.apply.openai.test.ts @@ -82,7 +82,7 @@ describe("applyAuthChoiceOpenAI", () => { profiles?: Record; }>(agentDir); expect(parsed.profiles?.["openai:default"]).toMatchObject({ - keyRef: { source: "env", id: "OPENAI_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, }); expect(parsed.profiles?.["openai:default"]?.key).toBeUndefined(); }); diff --git a/src/commands/auth-choice.apply.volcengine-byteplus.test.ts b/src/commands/auth-choice.apply.volcengine-byteplus.test.ts index 4f104beeb..c1d83bf71 100644 --- a/src/commands/auth-choice.apply.volcengine-byteplus.test.ts +++ b/src/commands/auth-choice.apply.volcengine-byteplus.test.ts @@ -88,7 +88,7 @@ describe("volcengine/byteplus auth choice", () => { profiles?: Record; }>(agentDir); expect(parsed.profiles?.["volcengine:default"]).toMatchObject({ - keyRef: { source: "env", id: "VOLCANO_ENGINE_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "VOLCANO_ENGINE_API_KEY" }, }); expect(parsed.profiles?.["volcengine:default"]?.key).toBeUndefined(); }); @@ -153,7 +153,7 @@ describe("volcengine/byteplus auth choice", () => { profiles?: Record; }>(agentDir); expect(parsed.profiles?.["byteplus:default"]).toMatchObject({ - keyRef: { source: "env", id: "BYTEPLUS_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "BYTEPLUS_API_KEY" }, }); expect(parsed.profiles?.["byteplus:default"]?.key).toBeUndefined(); }); diff --git a/src/commands/auth-choice.test.ts b/src/commands/auth-choice.test.ts index 4dfd423d3..49530ae84 100644 --- a/src/commands/auth-choice.test.ts +++ b/src/commands/auth-choice.test.ts @@ -46,7 +46,7 @@ vi.mock("./zai-endpoint-detect.js", () => ({ type StoredAuthProfile = { key?: string; - keyRef?: { source: string; id: string }; + keyRef?: { source: string; provider: string; id: string }; access?: string; refresh?: string; provider?: string; @@ -633,7 +633,7 @@ describe("applyAuthChoice", () => { expectEnvPrompt: boolean; expectedTextCalls: number; expectedKey?: string; - expectedKeyRef?: { source: "env"; id: string }; + expectedKeyRef?: { source: "env"; provider: string; id: string }; expectedModel?: string; expectedModelPrefix?: string; }> = [ @@ -679,7 +679,7 @@ describe("applyAuthChoice", () => { opts: { secretInputMode: "ref" }, expectEnvPrompt: false, expectedTextCalls: 1, - expectedKeyRef: { source: "env", id: "AI_GATEWAY_API_KEY" }, + expectedKeyRef: { source: "env", provider: "default", id: "AI_GATEWAY_API_KEY" }, expectedModel: "vercel-ai-gateway/anthropic/claude-opus-4.6", }, ]; @@ -740,14 +740,16 @@ describe("applyAuthChoice", () => { } }); - it("retries ref setup when sops preflight fails and can switch to env ref", async () => { + it("retries ref setup when provider preflight fails and can switch to env ref", async () => { await setupTempState(); process.env.OPENAI_API_KEY = "sk-openai-env"; - const selectValues: Array<"file" | "env"> = ["file", "env"]; + const selectValues: Array<"provider" | "env" | "filemain"> = ["provider", "filemain", "env"]; const select = vi.fn(async (params: Parameters[0]) => { - if (params.options.some((option) => option.value === "file")) { - return (selectValues.shift() ?? "env") as never; + const next = selectValues[0]; + if (next && params.options.some((option) => option.value === next)) { + selectValues.shift(); + return next as never; } return (params.options[0]?.value ?? "env") as never; }); @@ -767,7 +769,17 @@ describe("applyAuthChoice", () => { const result = await applyAuthChoice({ authChoice: "openai-api-key", - config: {}, + config: { + secrets: { + providers: { + filemain: { + source: "file", + path: "/tmp/openclaw-missing-secrets.json", + mode: "jsonPointer", + }, + }, + }, + }, prompter, runtime, setDefaultModel: false, @@ -779,7 +791,7 @@ describe("applyAuthChoice", () => { mode: "api_key", }); expect(note).toHaveBeenCalledWith( - expect.stringContaining("Could not validate this encrypted file reference."), + expect.stringContaining("Could not validate provider reference"), "Reference check failed", ); expect(note).toHaveBeenCalledWith( @@ -787,7 +799,7 @@ describe("applyAuthChoice", () => { "Reference validated", ); expect(await readAuthProfile("openai:default")).toMatchObject({ - keyRef: { source: "env", id: "OPENAI_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, }); }); @@ -1014,7 +1026,7 @@ describe("applyAuthChoice", () => { expectEnvPrompt: boolean; expectedTextCalls: number; expectedKey?: string; - expectedKeyRef?: { source: string; id: string }; + expectedKeyRef?: { source: string; provider: string; id: string }; expectedMetadata: { accountId: string; gatewayId: string }; }> = [ { @@ -1038,10 +1050,7 @@ describe("applyAuthChoice", () => { }, expectEnvPrompt: false, expectedTextCalls: 3, - expectedKeyRef: { - source: "env", - id: "CLOUDFLARE_AI_GATEWAY_API_KEY", - }, + expectedKeyRef: { source: "env", provider: "default", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" }, expectedMetadata: { accountId: "cf-account-id-ref", gatewayId: "cf-gateway-id-ref", diff --git a/src/commands/models/list.auth-overview.test.ts b/src/commands/models/list.auth-overview.test.ts index 6bf4bf5fe..bc23ff935 100644 --- a/src/commands/models/list.auth-overview.test.ts +++ b/src/commands/models/list.auth-overview.test.ts @@ -12,7 +12,7 @@ describe("resolveProviderAuthOverview", () => { "github-copilot:default": { type: "token", provider: "github-copilot", - tokenRef: { source: "env", id: "GITHUB_TOKEN" }, + tokenRef: { source: "env", provider: "default", id: "GITHUB_TOKEN" }, }, }, } as never, diff --git a/src/commands/onboard-auth.credentials.test.ts b/src/commands/onboard-auth.credentials.test.ts index 9dba5766b..48ccc9954 100644 --- a/src/commands/onboard-auth.credentials.test.ts +++ b/src/commands/onboard-auth.credentials.test.ts @@ -55,7 +55,7 @@ describe("onboard auth credentials secret refs", () => { profiles?: Record; }>(env.agentDir); expect(parsed.profiles?.["moonshot:default"]).toMatchObject({ - keyRef: { source: "env", id: "MOONSHOT_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "MOONSHOT_API_KEY" }, }); expect(parsed.profiles?.["moonshot:default"]?.key).toBeUndefined(); }); @@ -70,7 +70,7 @@ describe("onboard auth credentials secret refs", () => { profiles?: Record; }>(env.agentDir); expect(parsed.profiles?.["moonshot:default"]).toMatchObject({ - keyRef: { source: "env", id: "MOONSHOT_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "MOONSHOT_API_KEY" }, }); expect(parsed.profiles?.["moonshot:default"]?.key).toBeUndefined(); }); @@ -104,7 +104,7 @@ describe("onboard auth credentials secret refs", () => { profiles?: Record; }>(env.agentDir); expect(parsed.profiles?.["cloudflare-ai-gateway:default"]).toMatchObject({ - keyRef: { source: "env", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" }, metadata: { accountId: "account-1", gatewayId: "gateway-1" }, }); expect(parsed.profiles?.["cloudflare-ai-gateway:default"]?.key).toBeUndefined(); @@ -137,7 +137,7 @@ describe("onboard auth credentials secret refs", () => { profiles?: Record; }>(env.agentDir); expect(parsed.profiles?.["openai:default"]).toMatchObject({ - keyRef: { source: "env", id: "OPENAI_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, }); expect(parsed.profiles?.["openai:default"]?.key).toBeUndefined(); }); @@ -156,12 +156,12 @@ describe("onboard auth credentials secret refs", () => { }>(env.agentDir); expect(parsed.profiles?.["volcengine:default"]).toMatchObject({ - keyRef: { source: "env", id: "VOLCANO_ENGINE_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "VOLCANO_ENGINE_API_KEY" }, }); expect(parsed.profiles?.["volcengine:default"]?.key).toBeUndefined(); expect(parsed.profiles?.["byteplus:default"]).toMatchObject({ - keyRef: { source: "env", id: "BYTEPLUS_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "BYTEPLUS_API_KEY" }, }); expect(parsed.profiles?.["byteplus:default"]?.key).toBeUndefined(); }); diff --git a/src/commands/onboard-auth.credentials.ts b/src/commands/onboard-auth.credentials.ts index 8e512cd21..2cf9c25b6 100644 --- a/src/commands/onboard-auth.credentials.ts +++ b/src/commands/onboard-auth.credentials.ts @@ -4,7 +4,12 @@ import type { OAuthCredentials } from "@mariozechner/pi-ai"; import { resolveOpenClawAgentDir } from "../agents/agent-paths.js"; import { upsertAuthProfile } from "../agents/auth-profiles.js"; import { resolveStateDir } from "../config/paths.js"; -import { isSecretRef, type SecretInput, type SecretRef } from "../config/types.secrets.js"; +import { + coerceSecretRef, + DEFAULT_SECRET_PROVIDER_ALIAS, + type SecretInput, + type SecretRef, +} from "../config/types.secrets.js"; import { KILOCODE_DEFAULT_MODEL_REF } from "../providers/kilocode-shared.js"; import { PROVIDER_ENV_VARS } from "../secrets/provider-env-vars.js"; import { normalizeSecretInput } from "../utils/normalize-secret-input.js"; @@ -22,7 +27,7 @@ export type ApiKeyStorageOptions = { }; function buildEnvSecretRef(id: string): SecretRef { - return { source: "env", id }; + return { source: "env", provider: DEFAULT_SECRET_PROVIDER_ALIAS, id }; } function parseEnvSecretRef(value: string): SecretRef | null { @@ -49,8 +54,9 @@ function resolveApiKeySecretInput( input: SecretInput, options?: ApiKeyStorageOptions, ): SecretInput { - if (isSecretRef(input)) { - return input; + const coercedRef = coerceSecretRef(input); + if (coercedRef) { + return coercedRef; } const normalized = normalizeSecretInput(input); const inlineEnvRef = parseEnvSecretRef(normalized); diff --git a/src/commands/onboard-custom.test.ts b/src/commands/onboard-custom.test.ts index 24ccd1763..27043a308 100644 --- a/src/commands/onboard-custom.test.ts +++ b/src/commands/onboard-custom.test.ts @@ -238,6 +238,7 @@ describe("promptCustomApiConfig", () => { expect(result.config.models?.providers?.custom?.apiKey).toEqual({ source: "env", + provider: "default", id: "CUSTOM_PROVIDER_API_KEY", }); const firstCall = fetchMock.mock.calls[0]?.[1] as @@ -246,7 +247,7 @@ describe("promptCustomApiConfig", () => { expect(firstCall?.headers?.Authorization).toBe("Bearer test-env-key"); }); - it("re-prompts source after encrypted file ref preflight fails and succeeds with env ref", async () => { + it("re-prompts source after provider ref preflight fails and succeeds with env ref", async () => { vi.stubEnv("CUSTOM_PROVIDER_API_KEY", "test-env-key"); const prompter = createTestPrompter({ text: [ @@ -257,18 +258,29 @@ describe("promptCustomApiConfig", () => { "custom", "", ], - select: ["ref", "file", "env", "openai"], + select: ["ref", "provider", "filemain", "env", "openai"], }); stubFetchSequence([{ ok: true }]); - const result = await runPromptCustomApi(prompter); + const result = await runPromptCustomApi(prompter, { + secrets: { + providers: { + filemain: { + source: "file", + path: "/tmp/openclaw-missing-provider.json", + mode: "jsonPointer", + }, + }, + }, + }); expect(prompter.note).toHaveBeenCalledWith( - expect.stringContaining("Could not validate this encrypted file reference."), + expect.stringContaining("Could not validate provider reference"), "Reference check failed", ); expect(result.config.models?.providers?.custom?.apiKey).toEqual({ source: "env", + provider: "default", id: "CUSTOM_PROVIDER_API_KEY", }); }); diff --git a/src/commands/onboard-non-interactive.provider-auth.test.ts b/src/commands/onboard-non-interactive.provider-auth.test.ts index b58478810..468f24647 100644 --- a/src/commands/onboard-non-interactive.provider-auth.test.ts +++ b/src/commands/onboard-non-interactive.provider-auth.test.ts @@ -611,6 +611,7 @@ describe("onboard (non-interactive): provider auth", () => { }); expect(await readCustomLocalProviderApiKeyInput(configPath)).toEqual({ source: "env", + provider: "default", id: "CUSTOM_API_KEY", }); }, diff --git a/src/commands/onboard-non-interactive/local/auth-choice.ts b/src/commands/onboard-non-interactive/local/auth-choice.ts index a2917048f..0222eda4f 100644 --- a/src/commands/onboard-non-interactive/local/auth-choice.ts +++ b/src/commands/onboard-non-interactive/local/auth-choice.ts @@ -814,7 +814,7 @@ export async function applyNonInteractiveAuthChoice(params: { }); const customApiKeyInput: SecretInput | undefined = requestedSecretInputMode === "ref" && resolvedCustomApiKey?.source === "env" - ? { source: "env", id: "CUSTOM_API_KEY" } + ? { source: "env", provider: "default", id: "CUSTOM_API_KEY" } : resolvedCustomApiKey?.key; const result = applyCustomApiConfig({ config: nextConfig, diff --git a/src/config/config.secrets-schema.test.ts b/src/config/config.secrets-schema.test.ts index b41b28a82..cf9f12945 100644 --- a/src/config/config.secrets-schema.test.ts +++ b/src/config/config.secrets-schema.test.ts @@ -5,16 +5,26 @@ describe("config secret refs schema", () => { it("accepts top-level secrets sources and model apiKey refs", () => { const result = validateConfigObjectRaw({ secrets: { - sources: { - env: { type: "env" }, - file: { type: "sops", path: "~/.openclaw/secrets.enc.json", timeoutMs: 10_000 }, + providers: { + default: { source: "env" }, + filemain: { + source: "file", + path: "~/.openclaw/secrets.json", + mode: "jsonPointer", + timeoutMs: 10_000, + }, + vault: { + source: "exec", + command: "/usr/local/bin/openclaw-secret-resolver", + args: ["resolve"], + }, }, }, models: { providers: { openai: { baseUrl: "https://api.openai.com/v1", - apiKey: { source: "env", id: "OPENAI_API_KEY" }, + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, models: [{ id: "gpt-5", name: "gpt-5" }], }, }, @@ -28,7 +38,11 @@ describe("config secret refs schema", () => { const result = validateConfigObjectRaw({ channels: { googlechat: { - serviceAccountRef: { source: "file", id: "/channels/googlechat/serviceAccount" }, + serviceAccountRef: { + source: "file", + provider: "filemain", + id: "/channels/googlechat/serviceAccount", + }, }, }, }); @@ -42,7 +56,7 @@ describe("config secret refs schema", () => { entries: { "review-pr": { enabled: true, - apiKey: { source: "env", id: "SKILL_REVIEW_PR_API_KEY" }, + apiKey: { source: "env", provider: "default", id: "SKILL_REVIEW_PR_API_KEY" }, }, }, }, @@ -57,7 +71,7 @@ describe("config secret refs schema", () => { providers: { openai: { baseUrl: "https://api.openai.com/v1", - apiKey: { source: "env", id: "bad id with spaces" }, + apiKey: { source: "env", provider: "default", id: "bad id with spaces" }, models: [{ id: "gpt-5", name: "gpt-5" }], }, }, @@ -78,7 +92,7 @@ describe("config secret refs schema", () => { providers: { openai: { baseUrl: "https://api.openai.com/v1", - apiKey: { source: "env", id: "/providers/openai/apiKey" }, + apiKey: { source: "env", provider: "default", id: "/providers/openai/apiKey" }, models: [{ id: "gpt-5", name: "gpt-5" }], }, }, @@ -103,7 +117,7 @@ describe("config secret refs schema", () => { providers: { openai: { baseUrl: "https://api.openai.com/v1", - apiKey: { source: "file", id: "providers/openai/apiKey" }, + apiKey: { source: "file", provider: "default", id: "providers/openai/apiKey" }, models: [{ id: "gpt-5", name: "gpt-5" }], }, }, diff --git a/src/config/io.runtime-snapshot-write.test.ts b/src/config/io.runtime-snapshot-write.test.ts index fff270ac8..0a37de08a 100644 --- a/src/config/io.runtime-snapshot-write.test.ts +++ b/src/config/io.runtime-snapshot-write.test.ts @@ -20,7 +20,7 @@ describe("runtime config snapshot writes", () => { providers: { openai: { baseUrl: "https://api.openai.com/v1", - apiKey: { source: "env", id: "OPENAI_API_KEY" }, + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, models: [], }, }, @@ -52,6 +52,7 @@ describe("runtime config snapshot writes", () => { }; expect(persisted.models?.providers?.openai?.apiKey).toEqual({ source: "env", + provider: "default", id: "OPENAI_API_KEY", }); } finally { diff --git a/src/config/redact-snapshot.test.ts b/src/config/redact-snapshot.test.ts index 9881bb915..8d353c4e2 100644 --- a/src/config/redact-snapshot.test.ts +++ b/src/config/redact-snapshot.test.ts @@ -134,7 +134,7 @@ describe("redactConfigSnapshot", () => { models: { providers: { openai: { - apiKey: { source: "env", id: "OPENAI_API_KEY" }, + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, baseUrl: "https://api.openai.com", }, }, @@ -145,6 +145,7 @@ describe("redactConfigSnapshot", () => { const models = result.config.models as Record>>; expect(models.providers.openai.apiKey).toEqual({ source: REDACTED_SENTINEL, + provider: REDACTED_SENTINEL, id: REDACTED_SENTINEL, }); expect(models.providers.openai.baseUrl).toBe("https://api.openai.com"); diff --git a/src/config/types.secrets.ts b/src/config/types.secrets.ts index 5eac0f558..44c356234 100644 --- a/src/config/types.secrets.ts +++ b/src/config/types.secrets.ts @@ -1,17 +1,21 @@ -export type SecretRefSource = "env" | "file"; +export type SecretRefSource = "env" | "file" | "exec"; /** * Stable identifier for a secret in a configured source. * Examples: - * - env source: "OPENAI_API_KEY" - * - file source: "/providers/openai/api_key" (JSON pointer) + * - env source: provider "default", id "OPENAI_API_KEY" + * - file source: provider "mounted-json", id "/providers/openai/apiKey" + * - exec source: provider "vault", id "openai/api-key" */ export type SecretRef = { source: SecretRefSource; + provider: string; id: string; }; export type SecretInput = string | SecretRef; +export const DEFAULT_SECRET_PROVIDER_ALIAS = "default"; +const ENV_SECRET_TEMPLATE_RE = /^\$\{([A-Z][A-Z0-9_]{0,127})\}$/; function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); @@ -21,29 +25,126 @@ export function isSecretRef(value: unknown): value is SecretRef { if (!isRecord(value)) { return false; } - if (Object.keys(value).length !== 2) { + if (Object.keys(value).length !== 3) { return false; } return ( - (value.source === "env" || value.source === "file") && + (value.source === "env" || value.source === "file" || value.source === "exec") && + typeof value.provider === "string" && + value.provider.trim().length > 0 && typeof value.id === "string" && value.id.trim().length > 0 ); } -export type EnvSecretSourceConfig = { - type?: "env"; +function isLegacySecretRefWithoutProvider( + value: unknown, +): value is { source: SecretRefSource; id: string } { + if (!isRecord(value)) { + return false; + } + return ( + (value.source === "env" || value.source === "file" || value.source === "exec") && + typeof value.id === "string" && + value.id.trim().length > 0 && + value.provider === undefined + ); +} + +export function parseEnvTemplateSecretRef( + value: unknown, + provider = DEFAULT_SECRET_PROVIDER_ALIAS, +): SecretRef | null { + if (typeof value !== "string") { + return null; + } + const match = ENV_SECRET_TEMPLATE_RE.exec(value.trim()); + if (!match) { + return null; + } + return { + source: "env", + provider: provider.trim() || DEFAULT_SECRET_PROVIDER_ALIAS, + id: match[1], + }; +} + +export function coerceSecretRef( + value: unknown, + defaults?: { + env?: string; + file?: string; + exec?: string; + }, +): SecretRef | null { + if (isSecretRef(value)) { + return value; + } + if (isLegacySecretRefWithoutProvider(value)) { + const provider = + value.source === "env" + ? (defaults?.env ?? DEFAULT_SECRET_PROVIDER_ALIAS) + : value.source === "file" + ? (defaults?.file ?? DEFAULT_SECRET_PROVIDER_ALIAS) + : (defaults?.exec ?? DEFAULT_SECRET_PROVIDER_ALIAS); + return { + source: value.source, + provider, + id: value.id, + }; + } + const envTemplate = parseEnvTemplateSecretRef(value, defaults?.env); + if (envTemplate) { + return envTemplate; + } + return null; +} + +export type EnvSecretProviderConfig = { + source: "env"; + /** Optional env var allowlist (exact names). */ + allowlist?: string[]; }; -export type SopsSecretSourceConfig = { - type: "sops"; +export type FileSecretProviderMode = "raw" | "jsonPointer"; + +export type FileSecretProviderConfig = { + source: "file"; path: string; + mode?: FileSecretProviderMode; timeoutMs?: number; + maxBytes?: number; }; +export type ExecSecretProviderConfig = { + source: "exec"; + command: string; + args?: string[]; + timeoutMs?: number; + noOutputTimeoutMs?: number; + maxOutputBytes?: number; + jsonOnly?: boolean; + env?: Record; + passEnv?: string[]; + trustedDirs?: string[]; + allowInsecurePath?: boolean; +}; + +export type SecretProviderConfig = + | EnvSecretProviderConfig + | FileSecretProviderConfig + | ExecSecretProviderConfig; + export type SecretsConfig = { - sources?: { - env?: EnvSecretSourceConfig; - file?: SopsSecretSourceConfig; + providers?: Record; + defaults?: { + env?: string; + file?: string; + exec?: string; + }; + resolution?: { + maxProviderConcurrency?: number; + maxRefsPerProvider?: number; + maxBatchBytes?: number; }; }; diff --git a/src/config/zod-schema.core.ts b/src/config/zod-schema.core.ts index ccb8666e6..9f300a666 100644 --- a/src/config/zod-schema.core.ts +++ b/src/config/zod-schema.core.ts @@ -1,10 +1,15 @@ +import path from "node:path"; import { z } from "zod"; import { isSafeExecutableValue } from "../infra/exec-safety.js"; import { createAllowDenyChannelRulesSchema } from "./zod-schema.allowdeny.js"; import { sensitive } from "./zod-schema.sensitive.js"; const ENV_SECRET_REF_ID_PATTERN = /^[A-Z][A-Z0-9_]{0,127}$/; +const SECRET_PROVIDER_ALIAS_PATTERN = /^[a-z][a-z0-9_-]{0,63}$/; +const EXEC_SECRET_REF_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/; const FILE_SECRET_REF_SEGMENT_PATTERN = /^(?:[^~]|~0|~1)*$/; +const WINDOWS_ABS_PATH_PATTERN = /^[A-Za-z]:[\\/]/; +const WINDOWS_UNC_PATH_PATTERN = /^\\\\[^\\]+\\[^\\]+/; function isValidFileSecretRefId(value: string): boolean { if (!value.startsWith("/")) { @@ -16,9 +21,23 @@ function isValidFileSecretRefId(value: string): boolean { .every((segment) => FILE_SECRET_REF_SEGMENT_PATTERN.test(segment)); } +function isAbsolutePath(value: string): boolean { + return ( + path.isAbsolute(value) || + WINDOWS_ABS_PATH_PATTERN.test(value) || + WINDOWS_UNC_PATH_PATTERN.test(value) + ); +} + const EnvSecretRefSchema = z .object({ source: z.literal("env"), + provider: z + .string() + .regex( + SECRET_PROVIDER_ALIAS_PATTERN, + 'Secret reference provider must match /^[a-z][a-z0-9_-]{0,63}$/ (example: "default").', + ), id: z .string() .regex( @@ -31,6 +50,12 @@ const EnvSecretRefSchema = z const FileSecretRefSchema = z .object({ source: z.literal("file"), + provider: z + .string() + .regex( + SECRET_PROVIDER_ALIAS_PATTERN, + 'Secret reference provider must match /^[a-z][a-z0-9_-]{0,63}$/ (example: "default").', + ), id: z .string() .refine( @@ -40,33 +65,122 @@ const FileSecretRefSchema = z }) .strict(); +const ExecSecretRefSchema = z + .object({ + source: z.literal("exec"), + provider: z + .string() + .regex( + SECRET_PROVIDER_ALIAS_PATTERN, + 'Secret reference provider must match /^[a-z][a-z0-9_-]{0,63}$/ (example: "default").', + ), + id: z + .string() + .regex( + EXEC_SECRET_REF_ID_PATTERN, + 'Exec secret reference id must match /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/ (example: "vault/openai/api-key").', + ), + }) + .strict(); + export const SecretRefSchema = z.discriminatedUnion("source", [ EnvSecretRefSchema, FileSecretRefSchema, + ExecSecretRefSchema, ]); export const SecretInputSchema = z.union([z.string(), SecretRefSchema]); -const SecretsEnvSourceSchema = z +const SecretsEnvProviderSchema = z .object({ - type: z.literal("env").optional(), + source: z.literal("env"), + allowlist: z.array(z.string().regex(ENV_SECRET_REF_ID_PATTERN)).max(256).optional(), }) .strict(); -const SecretsFileSourceSchema = z +const SecretsFileProviderSchema = z .object({ - type: z.literal("sops"), + source: z.literal("file"), path: z.string().min(1), + mode: z.union([z.literal("raw"), z.literal("jsonPointer")]).optional(), timeoutMs: z.number().int().positive().max(120000).optional(), + maxBytes: z + .number() + .int() + .positive() + .max(20 * 1024 * 1024) + .optional(), }) .strict(); +const SecretsExecProviderSchema = z + .object({ + source: z.literal("exec"), + command: z + .string() + .min(1) + .refine((value) => isSafeExecutableValue(value), "secrets.providers.*.command is unsafe.") + .refine( + (value) => isAbsolutePath(value), + "secrets.providers.*.command must be an absolute path.", + ), + args: z.array(z.string().max(1024)).max(128).optional(), + timeoutMs: z.number().int().positive().max(120000).optional(), + noOutputTimeoutMs: z.number().int().positive().max(120000).optional(), + maxOutputBytes: z + .number() + .int() + .positive() + .max(20 * 1024 * 1024) + .optional(), + jsonOnly: z.boolean().optional(), + env: z.record(z.string(), z.string()).optional(), + passEnv: z.array(z.string().regex(ENV_SECRET_REF_ID_PATTERN)).max(128).optional(), + trustedDirs: z + .array( + z + .string() + .min(1) + .refine((value) => isAbsolutePath(value), "trustedDirs entries must be absolute paths."), + ) + .max(64) + .optional(), + allowInsecurePath: z.boolean().optional(), + }) + .strict(); + +const SecretsProviderSchema = z.discriminatedUnion("source", [ + SecretsEnvProviderSchema, + SecretsFileProviderSchema, + SecretsExecProviderSchema, +]); + export const SecretsConfigSchema = z .object({ - sources: z + providers: z .object({ - env: SecretsEnvSourceSchema.optional(), - file: SecretsFileSourceSchema.optional(), + // Keep this as a record so users can define multiple providers per source. + }) + .catchall(SecretsProviderSchema) + .optional(), + defaults: z + .object({ + env: z.string().regex(SECRET_PROVIDER_ALIAS_PATTERN).optional(), + file: z.string().regex(SECRET_PROVIDER_ALIAS_PATTERN).optional(), + exec: z.string().regex(SECRET_PROVIDER_ALIAS_PATTERN).optional(), + }) + .strict() + .optional(), + resolution: z + .object({ + maxProviderConcurrency: z.number().int().positive().max(16).optional(), + maxRefsPerProvider: z.number().int().positive().max(4096).optional(), + maxBatchBytes: z + .number() + .int() + .positive() + .max(5 * 1024 * 1024) + .optional(), }) .strict() .optional(), diff --git a/src/gateway/config-reload.test.ts b/src/gateway/config-reload.test.ts index 5174d97ad..25137aef0 100644 --- a/src/gateway/config-reload.test.ts +++ b/src/gateway/config-reload.test.ts @@ -153,6 +153,12 @@ describe("buildGatewayReloadPlan", () => { expect(plan.noopPaths).toContain("gateway.remote.url"); }); + it("treats secrets config changes as no-op for gateway restart planning", () => { + const plan = buildGatewayReloadPlan(["secrets.providers.default.path"]); + expect(plan.restartGateway).toBe(false); + expect(plan.noopPaths).toContain("secrets.providers.default.path"); + }); + it("defaults unknown paths to restart", () => { const plan = buildGatewayReloadPlan(["unknownField"]); expect(plan.restartGateway).toBe(true); diff --git a/src/gateway/config-reload.ts b/src/gateway/config-reload.ts index 5cebd3806..3dedff84c 100644 --- a/src/gateway/config-reload.ts +++ b/src/gateway/config-reload.ts @@ -82,6 +82,7 @@ const BASE_RELOAD_RULES_TAIL: ReloadRule[] = [ { prefix: "session", kind: "none" }, { prefix: "talk", kind: "none" }, { prefix: "skills", kind: "none" }, + { prefix: "secrets", kind: "none" }, { prefix: "plugins", kind: "restart" }, { prefix: "ui", kind: "none" }, { prefix: "gateway", kind: "restart" }, diff --git a/src/gateway/server.reload.test.ts b/src/gateway/server.reload.test.ts index 7f6054455..c44ed0ea7 100644 --- a/src/gateway/server.reload.test.ts +++ b/src/gateway/server.reload.test.ts @@ -222,7 +222,7 @@ describe("gateway hot reload", () => { providers: { openai: { baseUrl: "https://api.openai.com/v1", - apiKey: { source: "env", id: "OPENAI_API_KEY" }, + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, models: [], }, }, @@ -251,7 +251,7 @@ describe("gateway hot reload", () => { missing: { type: "api_key", provider: "openai", - keyRef: { source: "env", id: "MISSING_OPENCLAW_AUTH_REF" }, + keyRef: { source: "env", provider: "default", id: "MISSING_OPENCLAW_AUTH_REF" }, }, }, selectedProfileId: "missing", @@ -425,7 +425,7 @@ describe("gateway hot reload", () => { providers: { openai: { baseUrl: "https://api.openai.com/v1", - apiKey: { source: "env", id: "OPENAI_API_KEY" }, + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, models: [], }, }, diff --git a/src/secrets/migrate.test.ts b/src/secrets/migrate.test.ts index 41f26c984..11a503c88 100644 --- a/src/secrets/migrate.test.ts +++ b/src/secrets/migrate.test.ts @@ -1,15 +1,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -const runExecMock = vi.hoisted(() => vi.fn()); - -vi.mock("../process/exec.js", () => ({ - runExec: runExecMock, -})); - -const { rollbackSecretsMigration, runSecretsMigration } = await import("./migrate.js"); +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { rollbackSecretsMigration, runSecretsMigration } from "./migrate.js"; describe("secrets migrate", () => { let baseDir = ""; @@ -91,28 +84,6 @@ describe("secrets migrate", () => { "OPENAI_API_KEY=sk-openai-plaintext\nSKILL_KEY=sk-skill-plaintext\nUNRELATED=value\n", "utf8", ); - - runExecMock.mockReset(); - runExecMock.mockImplementation(async (_cmd: string, args: string[]) => { - if (args.includes("--encrypt")) { - const outputPath = args[args.indexOf("--output") + 1]; - const inputPath = args.at(-1); - if (!outputPath || !inputPath) { - throw new Error("missing sops encrypt paths"); - } - await fs.copyFile(inputPath, outputPath); - return { stdout: "", stderr: "" }; - } - if (args.includes("--decrypt")) { - const sourcePath = args.at(-1); - if (!sourcePath) { - throw new Error("missing sops decrypt source"); - } - const raw = await fs.readFile(sourcePath, "utf8"); - return { stdout: raw, stderr: "" }; - } - throw new Error(`unexpected sops invocation: ${args.join(" ")}`); - }); }); afterEach(async () => { @@ -144,22 +115,25 @@ describe("secrets migrate", () => { models: { providers: { openai: { apiKey: unknown } } }; skills: { entries: { "review-pr": { apiKey: unknown } } }; channels: { googlechat: { serviceAccount?: unknown; serviceAccountRef?: unknown } }; - secrets: { sources: { file: { type: string; path: string } } }; + secrets: { providers: Record }; }; expect(migratedConfig.models.providers.openai.apiKey).toEqual({ source: "file", + provider: "default", id: "/providers/openai/apiKey", }); expect(migratedConfig.skills.entries["review-pr"].apiKey).toEqual({ source: "file", + provider: "default", id: "/skills/entries/review-pr/apiKey", }); expect(migratedConfig.channels.googlechat.serviceAccount).toBeUndefined(); expect(migratedConfig.channels.googlechat.serviceAccountRef).toEqual({ source: "file", + provider: "default", id: "/channels/googlechat/serviceAccount", }); - expect(migratedConfig.secrets.sources.file.type).toBe("sops"); + expect(migratedConfig.secrets.providers.default.source).toBe("file"); const migratedAuth = JSON.parse(await fs.readFile(authStorePath, "utf8")) as { profiles: { "openai:default": { key?: string; keyRef?: unknown } }; @@ -167,6 +141,7 @@ describe("secrets migrate", () => { expect(migratedAuth.profiles["openai:default"].key).toBeUndefined(); expect(migratedAuth.profiles["openai:default"].keyRef).toEqual({ source: "file", + provider: "default", id: "/auth-profiles/main/openai:default/key", }); @@ -175,7 +150,7 @@ describe("secrets migrate", () => { expect(migratedEnv).toContain("SKILL_KEY=sk-skill-plaintext"); expect(migratedEnv).toContain("UNRELATED=value"); - const secretsPath = path.join(stateDir, "secrets.enc.json"); + const secretsPath = path.join(stateDir, "secrets.json"); const secretsPayload = JSON.parse(await fs.readFile(secretsPath, "utf8")) as { providers: { openai: { apiKey: string } }; skills: { entries: { "review-pr": { apiKey: string } } }; @@ -214,45 +189,53 @@ describe("secrets migrate", () => { expect(second.backupId).not.toBe(first.backupId); }); - it("passes --config for sops when .sops.yaml exists in config dir", async () => { - const sopsConfigPath = path.join(stateDir, ".sops.yaml"); - await fs.writeFile(sopsConfigPath, "creation_rules:\n - path_regex: .*\n", "utf8"); + it("reuses configured file provider aliases", async () => { + await fs.writeFile( + configPath, + `${JSON.stringify( + { + secrets: { + providers: { + teamfile: { + source: "file", + path: "~/.openclaw/team-secrets.json", + mode: "jsonPointer", + }, + }, + defaults: { + file: "teamfile", + }, + }, + models: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + apiKey: "sk-openai-plaintext", + models: [{ id: "gpt-5", name: "gpt-5" }], + }, + }, + }, + }, + null, + 2, + )}\n`, + "utf8", + ); await runSecretsMigration({ env, write: true }); - - const encryptCall = runExecMock.mock.calls.find((call) => - (call[1] as string[]).includes("--encrypt"), - ); - expect(encryptCall).toBeTruthy(); - const args = encryptCall?.[1] as string[]; - const configIndex = args.indexOf("--config"); - expect(configIndex).toBeGreaterThanOrEqual(0); - expect(args[configIndex + 1]).toBe(sopsConfigPath); - const filenameOverrideIndex = args.indexOf("--filename-override"); - expect(filenameOverrideIndex).toBeGreaterThanOrEqual(0); - expect(args[filenameOverrideIndex + 1]).toBe( - path.join(stateDir, "secrets.enc.json").replaceAll(path.sep, "/"), - ); - const options = encryptCall?.[2] as { cwd?: string } | undefined; - expect(options?.cwd).toBe(stateDir); + const migratedConfig = JSON.parse(await fs.readFile(configPath, "utf8")) as { + models: { providers: { openai: { apiKey: unknown } } }; + }; + expect(migratedConfig.models.providers.openai.apiKey).toEqual({ + source: "file", + provider: "teamfile", + id: "/providers/openai/apiKey", + }); }); - it("passes a stable filename override for sops when config file is absent", async () => { - await runSecretsMigration({ env, write: true }); - - const encryptCall = runExecMock.mock.calls.find((call) => - (call[1] as string[]).includes("--encrypt"), - ); - expect(encryptCall).toBeTruthy(); - const args = encryptCall?.[1] as string[]; - const configIndex = args.indexOf("--config"); - expect(configIndex).toBe(-1); - const filenameOverrideIndex = args.indexOf("--filename-override"); - expect(filenameOverrideIndex).toBeGreaterThanOrEqual(0); - expect(args[filenameOverrideIndex + 1]).toBe( - path.join(stateDir, "secrets.enc.json").replaceAll(path.sep, "/"), - ); - const options = encryptCall?.[2] as { cwd?: string } | undefined; - expect(options?.cwd).toBe(stateDir); + it("keeps .env values when scrub-env is disabled", async () => { + await runSecretsMigration({ env, write: true, scrubEnv: false }); + const migratedEnv = await fs.readFile(envPath, "utf8"); + expect(migratedEnv).toContain("OPENAI_API_KEY=sk-openai-plaintext"); }); }); diff --git a/src/secrets/migrate/apply.ts b/src/secrets/migrate/apply.ts index f03ce3d0b..7f64a6bd8 100644 --- a/src/secrets/migrate/apply.ts +++ b/src/secrets/migrate/apply.ts @@ -1,6 +1,5 @@ import fs from "node:fs"; import { ensureDirForFile, writeJsonFileSecure } from "../shared.js"; -import { encryptSopsJsonFile } from "../sops.js"; import { createBackupManifest, pruneOldBackups, @@ -10,22 +9,6 @@ import { import { createSecretsMigrationConfigIO } from "./config-io.js"; import type { MigrationPlan, SecretsMigrationRunResult } from "./types.js"; -async function encryptSopsJson(params: { - pathname: string; - timeoutMs: number; - payload: Record; - sopsConfigPath?: string; -}): Promise { - await encryptSopsJsonFile({ - path: params.pathname, - payload: params.payload, - timeoutMs: params.timeoutMs, - configPath: params.sopsConfigPath, - missingBinaryMessage: - "sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.", - }); -} - export async function applyMigrationPlan(params: { plan: MigrationPlan; env: NodeJS.ProcessEnv; @@ -52,12 +35,7 @@ export async function applyMigrationPlan(params: { try { if (plan.payloadChanged) { - await encryptSopsJson({ - pathname: plan.secretsFilePath, - timeoutMs: plan.secretsFileTimeoutMs, - payload: plan.nextPayload, - sopsConfigPath: plan.sopsConfigPath, - }); + writeJsonFileSecure(plan.secretsFilePath, plan.nextPayload); } if (plan.configChanged) { diff --git a/src/secrets/migrate/plan.ts b/src/secrets/migrate/plan.ts index 4da1a33f2..65a792f1b 100644 --- a/src/secrets/migrate/plan.ts +++ b/src/secrets/migrate/plan.ts @@ -6,7 +6,7 @@ import { isDeepStrictEqual } from "node:util"; import { listAgentIds, resolveAgentDir } from "../../agents/agent-scope.js"; import { resolveAuthStorePath } from "../../agents/auth-profiles/paths.js"; import { resolveStateDir, type OpenClawConfig } from "../../config/config.js"; -import { isSecretRef } from "../../config/types.secrets.js"; +import { coerceSecretRef, DEFAULT_SECRET_PROVIDER_ALIAS } from "../../config/types.secrets.js"; import { resolveConfigDir, resolveUserPath } from "../../utils.js"; import { encodeJsonPointerToken, @@ -14,12 +14,11 @@ import { setJsonPointer, } from "../json-pointer.js"; import { listKnownSecretEnvVarNames } from "../provider-env-vars.js"; -import { isNonEmptyString, isRecord, normalizePositiveInt } from "../shared.js"; -import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "../sops.js"; +import { isNonEmptyString, isRecord } from "../shared.js"; import { createSecretsMigrationConfigIO } from "./config-io.js"; import type { AuthStoreChange, EnvChange, MigrationCounters, MigrationPlan } from "./types.js"; -const DEFAULT_SECRETS_FILE_PATH = "~/.openclaw/secrets.enc.json"; +const DEFAULT_SECRETS_FILE_PATH = "~/.openclaw/secrets.json"; function readJsonPointer(root: unknown, pointer: string): unknown { return readJsonPointerRaw(root, pointer, { onMissing: "undefined" }); @@ -83,70 +82,67 @@ function resolveFileSource( config: OpenClawConfig, env: NodeJS.ProcessEnv, ): { + providerName: string; path: string; - timeoutMs: number; - hadConfiguredSource: boolean; + hadConfiguredProvider: boolean; } { - const source = config.secrets?.sources?.file; - if (source && source.type === "sops" && isNonEmptyString(source.path)) { - return { - path: resolveUserPath(source.path), - timeoutMs: normalizePositiveInt(source.timeoutMs, DEFAULT_SOPS_TIMEOUT_MS), - hadConfiguredSource: true, - }; + const configuredProviders = config.secrets?.providers; + const defaultProviderName = + config.secrets?.defaults?.file?.trim() || DEFAULT_SECRET_PROVIDER_ALIAS; + + if (configuredProviders) { + const defaultProvider = configuredProviders[defaultProviderName]; + if (defaultProvider?.source === "file" && isNonEmptyString(defaultProvider.path)) { + return { + providerName: defaultProviderName, + path: resolveUserPath(defaultProvider.path), + hadConfiguredProvider: true, + }; + } + + for (const [providerName, provider] of Object.entries(configuredProviders)) { + if (provider?.source === "file" && isNonEmptyString(provider.path)) { + return { + providerName, + path: resolveUserPath(provider.path), + hadConfiguredProvider: true, + }; + } + } } return { + providerName: defaultProviderName, path: resolveUserPath(resolveDefaultSecretsConfigPath(env)), - timeoutMs: DEFAULT_SOPS_TIMEOUT_MS, - hadConfiguredSource: false, + hadConfiguredProvider: false, }; } function resolveDefaultSecretsConfigPath(env: NodeJS.ProcessEnv): string { if (env.OPENCLAW_STATE_DIR?.trim() || env.CLAWDBOT_STATE_DIR?.trim()) { - return path.join(resolveStateDir(env, os.homedir), "secrets.enc.json"); + return path.join(resolveStateDir(env, os.homedir), "secrets.json"); } return DEFAULT_SECRETS_FILE_PATH; } -async function decryptSopsJson( - pathname: string, - timeoutMs: number, - sopsConfigPath?: string, -): Promise> { +async function readSecretsFileJson(pathname: string): Promise> { if (!fs.existsSync(pathname)) { return {}; } - const parsed = await decryptSopsJsonFile({ - path: pathname, - timeoutMs, - configPath: sopsConfigPath, - missingBinaryMessage: - "sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.", - }); + const raw = fs.readFileSync(pathname, "utf8"); + const parsed = JSON.parse(raw) as unknown; if (!isRecord(parsed)) { - throw new Error("sops decrypt failed: decrypted payload is not a JSON object"); + throw new Error("Secrets file payload is not a JSON object."); } return parsed; } -function resolveExistingSopsConfigPath(env: NodeJS.ProcessEnv): string | undefined { - const configDir = resolveConfigDir(env, os.homedir); - const candidates = [".sops.yaml", ".sops.yml"].map((name) => path.join(configDir, name)); - for (const candidate of candidates) { - if (fs.existsSync(candidate)) { - return candidate; - } - } - return undefined; -} - function migrateModelProviderSecrets(params: { config: OpenClawConfig; payload: Record; counters: MigrationCounters; migratedValues: Set; + fileProviderName: string; }): void { const providers = params.config.models?.providers as | Record @@ -155,7 +151,7 @@ function migrateModelProviderSecrets(params: { return; } for (const [providerId, provider] of Object.entries(providers)) { - if (isSecretRef(provider.apiKey)) { + if (coerceSecretRef(provider.apiKey)) { continue; } if (!isNonEmptyString(provider.apiKey)) { @@ -168,7 +164,7 @@ function migrateModelProviderSecrets(params: { setJsonPointer(params.payload, id, value); params.counters.secretsWritten += 1; } - provider.apiKey = { source: "file", id }; + provider.apiKey = { source: "file", provider: params.fileProviderName, id }; params.counters.configRefs += 1; params.migratedValues.add(value); } @@ -179,13 +175,14 @@ function migrateSkillEntrySecrets(params: { payload: Record; counters: MigrationCounters; migratedValues: Set; + fileProviderName: string; }): void { const entries = params.config.skills?.entries as Record | undefined; if (!entries) { return; } for (const [skillKey, entry] of Object.entries(entries)) { - if (!isRecord(entry) || isSecretRef(entry.apiKey)) { + if (!isRecord(entry) || coerceSecretRef(entry.apiKey)) { continue; } if (!isNonEmptyString(entry.apiKey)) { @@ -198,7 +195,7 @@ function migrateSkillEntrySecrets(params: { setJsonPointer(params.payload, id, value); params.counters.secretsWritten += 1; } - entry.apiKey = { source: "file", id }; + entry.apiKey = { source: "file", provider: params.fileProviderName, id }; params.counters.configRefs += 1; params.migratedValues.add(value); } @@ -209,17 +206,14 @@ function migrateGoogleChatServiceAccount(params: { pointerId: string; counters: MigrationCounters; payload: Record; + fileProviderName: string; }): void { - const explicitRef = isSecretRef(params.account.serviceAccountRef) - ? params.account.serviceAccountRef - : null; - const inlineRef = isSecretRef(params.account.serviceAccount) - ? params.account.serviceAccount - : null; + const explicitRef = coerceSecretRef(params.account.serviceAccountRef); + const inlineRef = coerceSecretRef(params.account.serviceAccount); if (explicitRef || inlineRef) { if ( params.account.serviceAccount !== undefined && - !isSecretRef(params.account.serviceAccount) + !coerceSecretRef(params.account.serviceAccount) ) { delete params.account.serviceAccount; params.counters.plaintextRemoved += 1; @@ -242,7 +236,11 @@ function migrateGoogleChatServiceAccount(params: { params.counters.secretsWritten += 1; } - params.account.serviceAccountRef = { source: "file", id }; + params.account.serviceAccountRef = { + source: "file", + provider: params.fileProviderName, + id, + }; delete params.account.serviceAccount; params.counters.configRefs += 1; } @@ -251,6 +249,7 @@ function migrateGoogleChatSecrets(params: { config: OpenClawConfig; payload: Record; counters: MigrationCounters; + fileProviderName: string; }): void { const googlechat = params.config.channels?.googlechat; if (!isRecord(googlechat)) { @@ -262,6 +261,7 @@ function migrateGoogleChatSecrets(params: { pointerId: "/channels/googlechat", payload: params.payload, counters: params.counters, + fileProviderName: params.fileProviderName, }); if (!isRecord(googlechat.accounts)) { @@ -276,6 +276,7 @@ function migrateGoogleChatSecrets(params: { pointerId: `/channels/googlechat/accounts/${encodeJsonPointerToken(accountId)}`, payload: params.payload, counters: params.counters, + fileProviderName: params.fileProviderName, }); } } @@ -325,6 +326,7 @@ function migrateAuthStoreSecrets(params: { payload: Record; counters: MigrationCounters; migratedValues: Set; + fileProviderName: string; }): boolean { const profiles = params.store.profiles; if (!isRecord(profiles)) { @@ -337,7 +339,7 @@ function migrateAuthStoreSecrets(params: { continue; } if (profileValue.type === "api_key") { - const keyRef = isSecretRef(profileValue.keyRef) ? profileValue.keyRef : null; + const keyRef = coerceSecretRef(profileValue.keyRef); const key = isNonEmptyString(profileValue.key) ? profileValue.key.trim() : ""; if (keyRef) { if (key) { @@ -356,7 +358,7 @@ function migrateAuthStoreSecrets(params: { setJsonPointer(params.payload, id, key); params.counters.secretsWritten += 1; } - profileValue.keyRef = { source: "file", id }; + profileValue.keyRef = { source: "file", provider: params.fileProviderName, id }; delete profileValue.key; params.counters.authProfileRefs += 1; params.migratedValues.add(key); @@ -365,7 +367,7 @@ function migrateAuthStoreSecrets(params: { } if (profileValue.type === "token") { - const tokenRef = isSecretRef(profileValue.tokenRef) ? profileValue.tokenRef : null; + const tokenRef = coerceSecretRef(profileValue.tokenRef); const token = isNonEmptyString(profileValue.token) ? profileValue.token.trim() : ""; if (tokenRef) { if (token) { @@ -384,7 +386,7 @@ function migrateAuthStoreSecrets(params: { setJsonPointer(params.payload, id, token); params.counters.secretsWritten += 1; } - profileValue.tokenRef = { source: "file", id }; + profileValue.tokenRef = { source: "file", provider: params.fileProviderName, id }; delete profileValue.token; params.counters.authProfileRefs += 1; params.migratedValues.add(token); @@ -412,12 +414,7 @@ export async function buildMigrationPlan(params: { const stateDir = resolveStateDir(params.env, os.homedir); const nextConfig = structuredClone(snapshot.config); const fileSource = resolveFileSource(nextConfig, params.env); - const sopsConfigPath = resolveExistingSopsConfigPath(params.env); - const previousPayload = await decryptSopsJson( - fileSource.path, - fileSource.timeoutMs, - sopsConfigPath, - ); + const previousPayload = await readSecretsFileJson(fileSource.path); const nextPayload = structuredClone(previousPayload); const counters: MigrationCounters = { @@ -436,17 +433,20 @@ export async function buildMigrationPlan(params: { payload: nextPayload, counters, migratedValues, + fileProviderName: fileSource.providerName, }); migrateSkillEntrySecrets({ config: nextConfig, payload: nextPayload, counters, migratedValues, + fileProviderName: fileSource.providerName, }); migrateGoogleChatSecrets({ config: nextConfig, payload: nextPayload, counters, + fileProviderName: fileSource.providerName, }); const authStoreChanges: AuthStoreChange[] = []; @@ -473,6 +473,7 @@ export async function buildMigrationPlan(params: { payload: nextPayload, counters, migratedValues, + fileProviderName: fileSource.providerName, }); if (!changed) { continue; @@ -481,15 +482,17 @@ export async function buildMigrationPlan(params: { } counters.authStoresChanged = authStoreChanges.length; - if (counters.secretsWritten > 0 && !fileSource.hadConfiguredSource) { + if (counters.secretsWritten > 0 && !fileSource.hadConfiguredProvider) { const defaultConfigPath = resolveDefaultSecretsConfigPath(params.env); nextConfig.secrets ??= {}; - nextConfig.secrets.sources ??= {}; - nextConfig.secrets.sources.file = { - type: "sops", + nextConfig.secrets.providers ??= {}; + nextConfig.secrets.providers[fileSource.providerName] = { + source: "file", path: defaultConfigPath, - timeoutMs: DEFAULT_SOPS_TIMEOUT_MS, + mode: "jsonPointer", }; + nextConfig.secrets.defaults ??= {}; + nextConfig.secrets.defaults.file ??= fileSource.providerName; } const configChanged = !isDeepStrictEqual(snapshot.config, nextConfig); @@ -536,8 +539,6 @@ export async function buildMigrationPlan(params: { payloadChanged, nextPayload, secretsFilePath: fileSource.path, - secretsFileTimeoutMs: fileSource.timeoutMs, - sopsConfigPath, envChange, backupTargets: [...backupTargets], }; diff --git a/src/secrets/migrate/types.ts b/src/secrets/migrate/types.ts index f9416b098..320ae2126 100644 --- a/src/secrets/migrate/types.ts +++ b/src/secrets/migrate/types.ts @@ -45,8 +45,6 @@ export type MigrationPlan = { payloadChanged: boolean; nextPayload: Record; secretsFilePath: string; - secretsFileTimeoutMs: number; - sopsConfigPath?: string; envChange: EnvChange | null; backupTargets: string[]; }; diff --git a/src/secrets/resolve.test.ts b/src/secrets/resolve.test.ts new file mode 100644 index 000000000..411b91ddb --- /dev/null +++ b/src/secrets/resolve.test.ts @@ -0,0 +1,154 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { OpenClawConfig } from "../config/config.js"; +import { resolveSecretRefString, resolveSecretRefValue } from "./resolve.js"; + +async function writeSecureFile(filePath: string, content: string, mode = 0o600): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, "utf8"); + await fs.chmod(filePath, mode); +} + +describe("secret ref resolver", () => { + const cleanupRoots: string[] = []; + + afterEach(async () => { + while (cleanupRoots.length > 0) { + const root = cleanupRoots.pop(); + if (!root) { + continue; + } + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it("resolves env refs via implicit default env provider", async () => { + const config: OpenClawConfig = {}; + const value = await resolveSecretRefString( + { source: "env", provider: "default", id: "OPENAI_API_KEY" }, + { + config, + env: { OPENAI_API_KEY: "sk-env-value" }, + }, + ); + expect(value).toBe("sk-env-value"); + }); + + it("resolves file refs in jsonPointer mode", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secrets-resolve-file-")); + cleanupRoots.push(root); + const filePath = path.join(root, "secrets.json"); + await writeSecureFile( + filePath, + JSON.stringify({ + providers: { + openai: { + apiKey: "sk-file-value", + }, + }, + }), + ); + + const value = await resolveSecretRefString( + { source: "file", provider: "filemain", id: "/providers/openai/apiKey" }, + { + config: { + secrets: { + providers: { + filemain: { + source: "file", + path: filePath, + mode: "jsonPointer", + }, + }, + }, + }, + }, + ); + expect(value).toBe("sk-file-value"); + }); + + it("resolves exec refs with protocolVersion 1 response", async () => { + if (process.platform === "win32") { + return; + } + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secrets-resolve-exec-")); + cleanupRoots.push(root); + const scriptPath = path.join(root, "resolver.mjs"); + await writeSecureFile( + scriptPath, + [ + "#!/usr/bin/env node", + "import fs from 'node:fs';", + "const req = JSON.parse(fs.readFileSync(0, 'utf8'));", + "const values = Object.fromEntries((req.ids ?? []).map((id) => [id, `value:${id}`]));", + "process.stdout.write(JSON.stringify({ protocolVersion: 1, values }));", + ].join("\n"), + 0o700, + ); + + const value = await resolveSecretRefString( + { source: "exec", provider: "execmain", id: "openai/api-key" }, + { + config: { + secrets: { + providers: { + execmain: { + source: "exec", + command: scriptPath, + passEnv: ["PATH"], + }, + }, + }, + }, + }, + ); + expect(value).toBe("value:openai/api-key"); + }); + + it("supports file raw mode with id=value", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secrets-resolve-raw-")); + cleanupRoots.push(root); + const filePath = path.join(root, "token.txt"); + await writeSecureFile(filePath, "raw-token-value\n"); + + const value = await resolveSecretRefString( + { source: "file", provider: "rawfile", id: "value" }, + { + config: { + secrets: { + providers: { + rawfile: { + source: "file", + path: filePath, + mode: "raw", + }, + }, + }, + }, + }, + ); + expect(value).toBe("raw-token-value"); + }); + + it("rejects misconfigured provider source mismatches", async () => { + await expect( + resolveSecretRefValue( + { source: "exec", provider: "default", id: "abc" }, + { + config: { + secrets: { + providers: { + default: { + source: "env", + }, + }, + }, + }, + }, + ), + ).rejects.toThrow('has source "env" but ref requests "exec"'); + }); +}); diff --git a/src/secrets/resolve.ts b/src/secrets/resolve.ts index 8cb26aea7..5060d99b8 100644 --- a/src/secrets/resolve.ts +++ b/src/secrets/resolve.ts @@ -1,79 +1,674 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; import type { OpenClawConfig } from "../config/config.js"; -import type { SecretRef } from "../config/types.secrets.js"; +import type { + ExecSecretProviderConfig, + FileSecretProviderConfig, + SecretProviderConfig, + SecretRef, + SecretRefSource, +} from "../config/types.secrets.js"; +import { DEFAULT_SECRET_PROVIDER_ALIAS } from "../config/types.secrets.js"; +import { inspectPathPermissions, safeStat } from "../security/audit-fs.js"; +import { isPathInside } from "../security/scan-paths.js"; import { resolveUserPath } from "../utils.js"; +import { runTasksWithConcurrency } from "../utils/run-with-concurrency.js"; import { readJsonPointer } from "./json-pointer.js"; import { isNonEmptyString, isRecord, normalizePositiveInt } from "./shared.js"; -import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js"; + +const DEFAULT_PROVIDER_CONCURRENCY = 4; +const DEFAULT_MAX_REFS_PER_PROVIDER = 512; +const DEFAULT_MAX_BATCH_BYTES = 256 * 1024; +const DEFAULT_FILE_MAX_BYTES = 1024 * 1024; +const DEFAULT_FILE_TIMEOUT_MS = 5_000; +const DEFAULT_EXEC_TIMEOUT_MS = 5_000; +const DEFAULT_EXEC_NO_OUTPUT_TIMEOUT_MS = 2_000; +const DEFAULT_EXEC_MAX_OUTPUT_BYTES = 1024 * 1024; +const RAW_FILE_REF_ID = "value"; + +const WINDOWS_ABS_PATH_PATTERN = /^[A-Za-z]:[\\/]/; +const WINDOWS_UNC_PATH_PATTERN = /^\\\\[^\\]+\\[^\\]+/; export type SecretRefResolveCache = { - fileSecretsPromise?: Promise | null; + resolvedByRefKey?: Map>; + filePayloadByProvider?: Map>; }; type ResolveSecretRefOptions = { config: OpenClawConfig; env?: NodeJS.ProcessEnv; cache?: SecretRefResolveCache; - missingBinaryMessage?: string; }; -const DEFAULT_SOPS_MISSING_BINARY_MESSAGE = - "sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file."; +type ResolutionLimits = { + maxProviderConcurrency: number; + maxRefsPerProvider: number; + maxBatchBytes: number; +}; -async function resolveFileSecretPayload(options: ResolveSecretRefOptions): Promise { - const fileSource = options.config.secrets?.sources?.file; - if (!fileSource) { +type ProviderResolutionOutput = Map; + +function isAbsolutePathname(value: string): boolean { + return ( + path.isAbsolute(value) || + WINDOWS_ABS_PATH_PATTERN.test(value) || + WINDOWS_UNC_PATH_PATTERN.test(value) + ); +} + +function resolveSourceDefaultAlias(source: SecretRefSource, config: OpenClawConfig): string { + const configured = + source === "env" + ? config.secrets?.defaults?.env + : source === "file" + ? config.secrets?.defaults?.file + : config.secrets?.defaults?.exec; + return configured?.trim() || DEFAULT_SECRET_PROVIDER_ALIAS; +} + +function resolveResolutionLimits(config: OpenClawConfig): ResolutionLimits { + const resolution = config.secrets?.resolution; + return { + maxProviderConcurrency: normalizePositiveInt( + resolution?.maxProviderConcurrency, + DEFAULT_PROVIDER_CONCURRENCY, + ), + maxRefsPerProvider: normalizePositiveInt( + resolution?.maxRefsPerProvider, + DEFAULT_MAX_REFS_PER_PROVIDER, + ), + maxBatchBytes: normalizePositiveInt(resolution?.maxBatchBytes, DEFAULT_MAX_BATCH_BYTES), + }; +} + +function toRefKey(ref: SecretRef): string { + return `${ref.source}:${ref.provider}:${ref.id}`; +} + +function toProviderKey(source: SecretRefSource, provider: string): string { + return `${source}:${provider}`; +} + +function resolveConfiguredProvider(ref: SecretRef, config: OpenClawConfig): SecretProviderConfig { + const providerConfig = config.secrets?.providers?.[ref.provider]; + if (!providerConfig) { + if (ref.source === "env" && ref.provider === resolveSourceDefaultAlias("env", config)) { + return { source: "env" }; + } throw new Error( - 'Secret reference source "file" is not configured. Configure secrets.sources.file first.', + `Secret provider "${ref.provider}" is not configured (ref: ${ref.source}:${ref.provider}:${ref.id}).`, ); } - if (fileSource.type !== "sops") { - throw new Error(`Unsupported secrets.sources.file.type "${String(fileSource.type)}".`); + if (providerConfig.source !== ref.source) { + throw new Error( + `Secret provider "${ref.provider}" has source "${providerConfig.source}" but ref requests "${ref.source}".`, + ); } + return providerConfig; +} - const cache = options.cache; - if (cache?.fileSecretsPromise) { - return await cache.fileSecretsPromise; +async function assertSecurePath(params: { + targetPath: string; + label: string; + trustedDirs?: string[]; + allowInsecurePath?: boolean; + allowReadableByOthers?: boolean; +}): Promise { + if (!isAbsolutePathname(params.targetPath)) { + throw new Error(`${params.label} must be an absolute path.`); } - - const promise = decryptSopsJsonFile({ - path: resolveUserPath(fileSource.path), - timeoutMs: normalizePositiveInt(fileSource.timeoutMs, DEFAULT_SOPS_TIMEOUT_MS), - missingBinaryMessage: options.missingBinaryMessage ?? DEFAULT_SOPS_MISSING_BINARY_MESSAGE, - }).then((payload) => { - if (!isRecord(payload)) { - throw new Error("sops decrypt failed: decrypted payload is not a JSON object"); + if (params.trustedDirs && params.trustedDirs.length > 0) { + const trusted = params.trustedDirs.map((entry) => resolveUserPath(entry)); + const inTrustedDir = trusted.some((dir) => isPathInside(dir, params.targetPath)); + if (!inTrustedDir) { + throw new Error(`${params.label} is outside trustedDirs: ${params.targetPath}`); } - return payload; - }); - if (cache) { - cache.fileSecretsPromise = promise; } - return await promise; + + const stat = await safeStat(params.targetPath); + if (!stat.ok) { + throw new Error(`${params.label} is not readable: ${params.targetPath}`); + } + if (stat.isDir) { + throw new Error(`${params.label} must be a file: ${params.targetPath}`); + } + if (stat.isSymlink) { + throw new Error(`${params.label} must not be a symlink: ${params.targetPath}`); + } + if (params.allowInsecurePath) { + return; + } + + const perms = await inspectPathPermissions(params.targetPath); + if (!perms.ok) { + throw new Error(`${params.label} permissions could not be verified: ${params.targetPath}`); + } + const writableByOthers = perms.worldWritable || perms.groupWritable; + const readableByOthers = perms.worldReadable || perms.groupReadable; + if (writableByOthers || (!params.allowReadableByOthers && readableByOthers)) { + throw new Error(`${params.label} permissions are too open: ${params.targetPath}`); + } + + if (process.platform === "win32" && perms.source === "unknown") { + throw new Error( + `${params.label} ACL verification unavailable on Windows for ${params.targetPath}.`, + ); + } + + if (process.platform !== "win32" && typeof process.getuid === "function" && stat.uid != null) { + const uid = process.getuid(); + if (stat.uid !== uid) { + throw new Error( + `${params.label} must be owned by the current user (uid=${uid}): ${params.targetPath}`, + ); + } + } +} + +async function readFileProviderPayload(params: { + providerName: string; + providerConfig: FileSecretProviderConfig; + cache?: SecretRefResolveCache; +}): Promise { + const cacheKey = params.providerName; + const cache = params.cache; + if (cache?.filePayloadByProvider?.has(cacheKey)) { + return await (cache.filePayloadByProvider.get(cacheKey) as Promise); + } + + const filePath = resolveUserPath(params.providerConfig.path); + const readPromise = (async () => { + await assertSecurePath({ + targetPath: filePath, + label: `secrets.providers.${params.providerName}.path`, + }); + const timeoutMs = normalizePositiveInt( + params.providerConfig.timeoutMs, + DEFAULT_FILE_TIMEOUT_MS, + ); + const maxBytes = normalizePositiveInt(params.providerConfig.maxBytes, DEFAULT_FILE_MAX_BYTES); + const timeoutHandle = setTimeout(() => { + // noop marker to keep timeout behavior explicit and deterministic + }, timeoutMs); + try { + const payload = await fs.readFile(filePath); + if (payload.byteLength > maxBytes) { + throw new Error(`File provider "${params.providerName}" exceeded maxBytes (${maxBytes}).`); + } + const text = payload.toString("utf8"); + if (params.providerConfig.mode === "raw") { + return text.replace(/\r?\n$/, ""); + } + const parsed = JSON.parse(text) as unknown; + if (!isRecord(parsed)) { + throw new Error(`File provider "${params.providerName}" payload is not a JSON object.`); + } + return parsed; + } finally { + clearTimeout(timeoutHandle); + } + })(); + + if (cache) { + cache.filePayloadByProvider ??= new Map(); + cache.filePayloadByProvider.set(cacheKey, readPromise); + } + return await readPromise; +} + +async function resolveEnvRefs(params: { + refs: SecretRef[]; + providerName: string; + providerConfig: Extract; + env: NodeJS.ProcessEnv; +}): Promise { + const resolved = new Map(); + const allowlist = params.providerConfig.allowlist + ? new Set(params.providerConfig.allowlist) + : null; + for (const ref of params.refs) { + if (allowlist && !allowlist.has(ref.id)) { + throw new Error( + `Environment variable "${ref.id}" is not allowlisted in secrets.providers.${params.providerName}.allowlist.`, + ); + } + const envValue = params.env[ref.id] ?? process.env[ref.id]; + if (!isNonEmptyString(envValue)) { + throw new Error(`Environment variable "${ref.id}" is missing or empty.`); + } + resolved.set(ref.id, envValue); + } + return resolved; +} + +async function resolveFileRefs(params: { + refs: SecretRef[]; + providerName: string; + providerConfig: FileSecretProviderConfig; + cache?: SecretRefResolveCache; +}): Promise { + const payload = await readFileProviderPayload({ + providerName: params.providerName, + providerConfig: params.providerConfig, + cache: params.cache, + }); + const mode = params.providerConfig.mode ?? "jsonPointer"; + const resolved = new Map(); + if (mode === "raw") { + for (const ref of params.refs) { + if (ref.id !== RAW_FILE_REF_ID) { + throw new Error( + `Raw file provider "${params.providerName}" expects ref id "${RAW_FILE_REF_ID}".`, + ); + } + resolved.set(ref.id, payload); + } + return resolved; + } + for (const ref of params.refs) { + resolved.set(ref.id, readJsonPointer(payload, ref.id, { onMissing: "throw" })); + } + return resolved; +} + +type ExecRunResult = { + stdout: string; + stderr: string; + code: number | null; + signal: NodeJS.Signals | null; + termination: "exit" | "timeout" | "no-output-timeout"; +}; + +async function runExecResolver(params: { + command: string; + args: string[]; + cwd: string; + env: NodeJS.ProcessEnv; + input: string; + timeoutMs: number; + noOutputTimeoutMs: number; + maxOutputBytes: number; +}): Promise { + return await new Promise((resolve, reject) => { + const child = spawn(params.command, params.args, { + cwd: params.cwd, + env: params.env, + stdio: ["pipe", "pipe", "pipe"], + shell: false, + windowsHide: true, + }); + + let settled = false; + let stdout = ""; + let stderr = ""; + let timedOut = false; + let noOutputTimedOut = false; + let outputBytes = 0; + let noOutputTimer: NodeJS.Timeout | null = null; + const timeoutTimer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, params.timeoutMs); + + const clearTimers = () => { + clearTimeout(timeoutTimer); + if (noOutputTimer) { + clearTimeout(noOutputTimer); + noOutputTimer = null; + } + }; + + const armNoOutputTimer = () => { + if (noOutputTimer) { + clearTimeout(noOutputTimer); + } + noOutputTimer = setTimeout(() => { + noOutputTimedOut = true; + child.kill("SIGKILL"); + }, params.noOutputTimeoutMs); + }; + + const append = (chunk: Buffer | string, target: "stdout" | "stderr") => { + const text = typeof chunk === "string" ? chunk : chunk.toString("utf8"); + outputBytes += Buffer.byteLength(text, "utf8"); + if (outputBytes > params.maxOutputBytes) { + child.kill("SIGKILL"); + if (!settled) { + settled = true; + clearTimers(); + reject( + new Error(`Exec provider output exceeded maxOutputBytes (${params.maxOutputBytes}).`), + ); + } + return; + } + if (target === "stdout") { + stdout += text; + } else { + stderr += text; + } + armNoOutputTimer(); + }; + + armNoOutputTimer(); + child.on("error", (error) => { + if (settled) { + return; + } + settled = true; + clearTimers(); + reject(error); + }); + child.stdout?.on("data", (chunk) => append(chunk, "stdout")); + child.stderr?.on("data", (chunk) => append(chunk, "stderr")); + child.on("close", (code, signal) => { + if (settled) { + return; + } + settled = true; + clearTimers(); + resolve({ + stdout, + stderr, + code, + signal, + termination: noOutputTimedOut ? "no-output-timeout" : timedOut ? "timeout" : "exit", + }); + }); + + child.stdin?.end(params.input); + }); +} + +function parseExecValues(params: { + providerName: string; + ids: string[]; + stdout: string; + jsonOnly: boolean; +}): Record { + const trimmed = params.stdout.trim(); + if (!trimmed) { + throw new Error(`Exec provider "${params.providerName}" returned empty stdout.`); + } + + let parsed: unknown; + if (!params.jsonOnly && params.ids.length === 1) { + try { + parsed = JSON.parse(trimmed) as unknown; + } catch { + return { [params.ids[0]]: trimmed }; + } + } else { + try { + parsed = JSON.parse(trimmed) as unknown; + } catch { + throw new Error(`Exec provider "${params.providerName}" returned invalid JSON.`); + } + } + + if (!isRecord(parsed)) { + if (!params.jsonOnly && params.ids.length === 1 && typeof parsed === "string") { + return { [params.ids[0]]: parsed }; + } + throw new Error(`Exec provider "${params.providerName}" response must be an object.`); + } + if (parsed.protocolVersion !== 1) { + throw new Error(`Exec provider "${params.providerName}" protocolVersion must be 1.`); + } + const responseValues = parsed.values; + if (!isRecord(responseValues)) { + throw new Error(`Exec provider "${params.providerName}" response missing "values".`); + } + const responseErrors = isRecord(parsed.errors) ? parsed.errors : null; + const out: Record = {}; + for (const id of params.ids) { + if (responseErrors && id in responseErrors) { + const entry = responseErrors[id]; + if (isRecord(entry) && typeof entry.message === "string" && entry.message.trim()) { + throw new Error( + `Exec provider "${params.providerName}" failed for id "${id}" (${entry.message.trim()}).`, + ); + } + throw new Error(`Exec provider "${params.providerName}" failed for id "${id}".`); + } + if (!(id in responseValues)) { + throw new Error(`Exec provider "${params.providerName}" response missing id "${id}".`); + } + out[id] = responseValues[id]; + } + return out; +} + +async function resolveExecRefs(params: { + refs: SecretRef[]; + providerName: string; + providerConfig: ExecSecretProviderConfig; + env: NodeJS.ProcessEnv; + limits: ResolutionLimits; +}): Promise { + const ids = [...new Set(params.refs.map((ref) => ref.id))]; + if (ids.length > params.limits.maxRefsPerProvider) { + throw new Error( + `Exec provider "${params.providerName}" exceeded maxRefsPerProvider (${params.limits.maxRefsPerProvider}).`, + ); + } + + const commandPath = resolveUserPath(params.providerConfig.command); + await assertSecurePath({ + targetPath: commandPath, + label: `secrets.providers.${params.providerName}.command`, + trustedDirs: params.providerConfig.trustedDirs, + allowInsecurePath: params.providerConfig.allowInsecurePath, + allowReadableByOthers: true, + }); + + const requestPayload = { + protocolVersion: 1, + provider: params.providerName, + ids, + }; + const input = JSON.stringify(requestPayload); + if (Buffer.byteLength(input, "utf8") > params.limits.maxBatchBytes) { + throw new Error( + `Exec provider "${params.providerName}" request exceeded maxBatchBytes (${params.limits.maxBatchBytes}).`, + ); + } + + const childEnv: NodeJS.ProcessEnv = {}; + for (const key of params.providerConfig.passEnv ?? []) { + const value = params.env[key] ?? process.env[key]; + if (value !== undefined) { + childEnv[key] = value; + } + } + for (const [key, value] of Object.entries(params.providerConfig.env ?? {})) { + childEnv[key] = value; + } + + const timeoutMs = normalizePositiveInt(params.providerConfig.timeoutMs, DEFAULT_EXEC_TIMEOUT_MS); + const noOutputTimeoutMs = normalizePositiveInt( + params.providerConfig.noOutputTimeoutMs, + DEFAULT_EXEC_NO_OUTPUT_TIMEOUT_MS, + ); + const maxOutputBytes = normalizePositiveInt( + params.providerConfig.maxOutputBytes, + DEFAULT_EXEC_MAX_OUTPUT_BYTES, + ); + const jsonOnly = params.providerConfig.jsonOnly ?? true; + + const result = await runExecResolver({ + command: commandPath, + args: params.providerConfig.args ?? [], + cwd: path.dirname(commandPath), + env: childEnv, + input, + timeoutMs, + noOutputTimeoutMs, + maxOutputBytes, + }); + if (result.termination === "timeout") { + throw new Error(`Exec provider "${params.providerName}" timed out after ${timeoutMs}ms.`); + } + if (result.termination === "no-output-timeout") { + throw new Error( + `Exec provider "${params.providerName}" produced no output for ${noOutputTimeoutMs}ms.`, + ); + } + if (result.code !== 0) { + throw new Error( + `Exec provider "${params.providerName}" exited with code ${String(result.code)}.`, + ); + } + + const values = parseExecValues({ + providerName: params.providerName, + ids, + stdout: result.stdout, + jsonOnly, + }); + const resolved = new Map(); + for (const id of ids) { + resolved.set(id, values[id]); + } + return resolved; +} + +async function resolveProviderRefs(params: { + refs: SecretRef[]; + source: SecretRefSource; + providerName: string; + providerConfig: SecretProviderConfig; + options: ResolveSecretRefOptions; + limits: ResolutionLimits; +}): Promise { + if (params.providerConfig.source === "env") { + return await resolveEnvRefs({ + refs: params.refs, + providerName: params.providerName, + providerConfig: params.providerConfig, + env: params.options.env ?? process.env, + }); + } + if (params.providerConfig.source === "file") { + return await resolveFileRefs({ + refs: params.refs, + providerName: params.providerName, + providerConfig: params.providerConfig, + cache: params.options.cache, + }); + } + if (params.providerConfig.source === "exec") { + return await resolveExecRefs({ + refs: params.refs, + providerName: params.providerName, + providerConfig: params.providerConfig, + env: params.options.env ?? process.env, + limits: params.limits, + }); + } + throw new Error( + `Unsupported secret provider source "${String((params.providerConfig as { source?: unknown }).source)}".`, + ); +} + +export async function resolveSecretRefValues( + refs: SecretRef[], + options: ResolveSecretRefOptions, +): Promise> { + if (refs.length === 0) { + return new Map(); + } + const limits = resolveResolutionLimits(options.config); + const uniqueRefs = new Map(); + for (const ref of refs) { + const id = ref.id.trim(); + if (!id) { + throw new Error("Secret reference id is empty."); + } + uniqueRefs.set(toRefKey(ref), { ...ref, id }); + } + + const grouped = new Map< + string, + { source: SecretRefSource; providerName: string; refs: SecretRef[] } + >(); + for (const ref of uniqueRefs.values()) { + const key = toProviderKey(ref.source, ref.provider); + const existing = grouped.get(key); + if (existing) { + existing.refs.push(ref); + continue; + } + grouped.set(key, { source: ref.source, providerName: ref.provider, refs: [ref] }); + } + + const tasks = [...grouped.values()].map( + (group) => async (): Promise<{ group: typeof group; values: ProviderResolutionOutput }> => { + if (group.refs.length > limits.maxRefsPerProvider) { + throw new Error( + `Secret provider "${group.providerName}" exceeded maxRefsPerProvider (${limits.maxRefsPerProvider}).`, + ); + } + const providerConfig = resolveConfiguredProvider(group.refs[0], options.config); + const values = await resolveProviderRefs({ + refs: group.refs, + source: group.source, + providerName: group.providerName, + providerConfig, + options, + limits, + }); + return { group, values }; + }, + ); + + const taskResults = await runTasksWithConcurrency({ + tasks, + limit: limits.maxProviderConcurrency, + errorMode: "stop", + }); + if (taskResults.hasError) { + throw taskResults.firstError; + } + + const resolved = new Map(); + for (const result of taskResults.results) { + for (const ref of result.group.refs) { + if (!result.values.has(ref.id)) { + throw new Error( + `Secret provider "${result.group.providerName}" did not return id "${ref.id}".`, + ); + } + resolved.set(toRefKey(ref), result.values.get(ref.id)); + } + } + return resolved; } export async function resolveSecretRefValue( ref: SecretRef, options: ResolveSecretRefOptions, ): Promise { - const id = ref.id.trim(); - if (!id) { - throw new Error("Secret reference id is empty."); + const cache = options.cache; + const key = toRefKey(ref); + if (cache?.resolvedByRefKey?.has(key)) { + return await (cache.resolvedByRefKey.get(key) as Promise); } - if (ref.source === "env") { - const envValue = options.env?.[id] ?? process.env[id]; - if (!isNonEmptyString(envValue)) { - throw new Error(`Environment variable "${id}" is missing or empty.`); + const promise = (async () => { + const resolved = await resolveSecretRefValues([ref], options); + if (!resolved.has(key)) { + throw new Error(`Secret reference "${key}" resolved to no value.`); } - return envValue; - } + return resolved.get(key); + })(); - if (ref.source === "file") { - const payload = await resolveFileSecretPayload(options); - return readJsonPointer(payload, id, { onMissing: "throw" }); + if (cache) { + cache.resolvedByRefKey ??= new Map(); + cache.resolvedByRefKey.set(key, promise); } - - throw new Error(`Unsupported secret source "${String((ref as { source?: unknown }).source)}".`); + return await promise; } export async function resolveSecretRefString( @@ -83,7 +678,7 @@ export async function resolveSecretRefString( const resolved = await resolveSecretRefValue(ref, options); if (!isNonEmptyString(resolved)) { throw new Error( - `Secret reference "${ref.source}:${ref.id}" resolved to a non-string or empty value.`, + `Secret reference "${ref.source}:${ref.provider}:${ref.id}" resolved to a non-string or empty value.`, ); } return resolved; diff --git a/src/secrets/runtime.test.ts b/src/secrets/runtime.test.ts index 00a95bc23..a1c8fcc35 100644 --- a/src/secrets/runtime.test.ts +++ b/src/secrets/runtime.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import { ensureAuthProfileStore } from "../agents/auth-profiles.js"; import { loadConfig, type OpenClawConfig } from "../config/config.js"; import { @@ -10,15 +10,8 @@ import { prepareSecretsRuntimeSnapshot, } from "./runtime.js"; -const runExecMock = vi.hoisted(() => vi.fn()); - -vi.mock("../process/exec.js", () => ({ - runExec: runExecMock, -})); - describe("secrets runtime snapshot", () => { afterEach(() => { - runExecMock.mockReset(); clearSecretsRuntimeSnapshot(); }); @@ -28,7 +21,7 @@ describe("secrets runtime snapshot", () => { providers: { openai: { baseUrl: "https://api.openai.com/v1", - apiKey: { source: "env", id: "OPENAI_API_KEY" }, + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, models: [], }, }, @@ -37,7 +30,7 @@ describe("secrets runtime snapshot", () => { entries: { "review-pr": { enabled: true, - apiKey: { source: "env", id: "REVIEW_SKILL_API_KEY" }, + apiKey: { source: "env", provider: "default", id: "REVIEW_SKILL_API_KEY" }, }, }, }, @@ -58,13 +51,18 @@ describe("secrets runtime snapshot", () => { type: "api_key", provider: "openai", key: "old-openai", - keyRef: { source: "env", id: "OPENAI_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, }, "github-copilot:default": { type: "token", provider: "github-copilot", token: "old-gh", - tokenRef: { source: "env", id: "GITHUB_TOKEN" }, + tokenRef: { source: "env", provider: "default", id: "GITHUB_TOKEN" }, + }, + "openai:inline": { + type: "api_key", + provider: "openai", + key: "${OPENAI_API_KEY}", }, }, }), @@ -81,90 +79,105 @@ describe("secrets runtime snapshot", () => { type: "token", token: "ghp-env-token", }); + expect(snapshot.authStores[0]?.store.profiles["openai:inline"]).toMatchObject({ + type: "api_key", + key: "sk-env-openai", + }); }); - it("resolves file refs via sops json payload", async () => { - runExecMock.mockResolvedValueOnce({ - stdout: JSON.stringify({ - providers: { - openai: { - apiKey: "sk-from-sops", - }, - }, - }), - stderr: "", - }); - - const config: OpenClawConfig = { - secrets: { - sources: { - file: { - type: "sops", - path: "~/.openclaw/secrets.enc.json", - timeoutMs: 7000, - }, - }, - }, - models: { - providers: { - openai: { - baseUrl: "https://api.openai.com/v1", - apiKey: { source: "file", id: "/providers/openai/apiKey" }, - models: [], - }, - }, - }, - }; - - const snapshot = await prepareSecretsRuntimeSnapshot({ - config, - agentDirs: ["/tmp/openclaw-agent-main"], - loadAuthStore: () => ({ version: 1, profiles: {} }), - }); - - expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("sk-from-sops"); - expect(runExecMock).toHaveBeenCalledWith( - "sops", - ["--decrypt", "--output-type", "json", expect.stringContaining("secrets.enc.json")], - expect.objectContaining({ - timeoutMs: 7000, - maxBuffer: 10 * 1024 * 1024, - cwd: expect.stringContaining(".openclaw"), - }), - ); - }); - - it("fails when sops decrypt payload is not a JSON object", async () => { - runExecMock.mockResolvedValueOnce({ - stdout: JSON.stringify(["not-an-object"]), - stderr: "", - }); - - await expect( - prepareSecretsRuntimeSnapshot({ - config: { - secrets: { - sources: { - file: { - type: "sops", - path: "~/.openclaw/secrets.enc.json", - }, - }, - }, - models: { + it("resolves file refs via configured file provider", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secrets-file-provider-")); + const secretsPath = path.join(root, "secrets.json"); + try { + await fs.writeFile( + secretsPath, + JSON.stringify( + { providers: { openai: { - baseUrl: "https://api.openai.com/v1", - apiKey: { source: "file", id: "/providers/openai/apiKey" }, - models: [], + apiKey: "sk-from-file-provider", }, }, }, + null, + 2, + ), + "utf8", + ); + await fs.chmod(secretsPath, 0o600); + + const config: OpenClawConfig = { + secrets: { + providers: { + default: { + source: "file", + path: secretsPath, + mode: "jsonPointer", + }, + }, + defaults: { + file: "default", + }, }, + models: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + apiKey: { source: "file", provider: "default", id: "/providers/openai/apiKey" }, + models: [], + }, + }, + }, + }; + + const snapshot = await prepareSecretsRuntimeSnapshot({ + config, agentDirs: ["/tmp/openclaw-agent-main"], loadAuthStore: () => ({ version: 1, profiles: {} }), - }), - ).rejects.toThrow("sops decrypt failed: decrypted payload is not a JSON object"); + }); + + expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("sk-from-file-provider"); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it("fails when file provider payload is not a JSON object", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secrets-file-provider-bad-")); + const secretsPath = path.join(root, "secrets.json"); + try { + await fs.writeFile(secretsPath, JSON.stringify(["not-an-object"]), "utf8"); + await fs.chmod(secretsPath, 0o600); + + await expect( + prepareSecretsRuntimeSnapshot({ + config: { + secrets: { + providers: { + default: { + source: "file", + path: secretsPath, + mode: "jsonPointer", + }, + }, + }, + models: { + providers: { + openai: { + baseUrl: "https://api.openai.com/v1", + apiKey: { source: "file", provider: "default", id: "/providers/openai/apiKey" }, + models: [], + }, + }, + }, + }, + agentDirs: ["/tmp/openclaw-agent-main"], + loadAuthStore: () => ({ version: 1, profiles: {} }), + }), + ).rejects.toThrow("payload is not a JSON object"); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } }); it("activates runtime snapshots for loadConfig and ensureAuthProfileStore", async () => { @@ -174,7 +187,7 @@ describe("secrets runtime snapshot", () => { providers: { openai: { baseUrl: "https://api.openai.com/v1", - apiKey: { source: "env", id: "OPENAI_API_KEY" }, + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, models: [], }, }, @@ -188,7 +201,7 @@ describe("secrets runtime snapshot", () => { "openai:default": { type: "api_key", provider: "openai", - keyRef: { source: "env", id: "OPENAI_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, }, }, }), @@ -221,7 +234,7 @@ describe("secrets runtime snapshot", () => { "openai:default": { type: "api_key", provider: "openai", - keyRef: { source: "env", id: "OPENAI_API_KEY" }, + keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, }, }, }), diff --git a/src/secrets/runtime.ts b/src/secrets/runtime.ts index 26498820a..98a0afd39 100644 --- a/src/secrets/runtime.ts +++ b/src/secrets/runtime.ts @@ -11,9 +11,9 @@ import { setRuntimeConfigSnapshot, type OpenClawConfig, } from "../config/config.js"; -import { isSecretRef, type SecretRef } from "../config/types.secrets.js"; +import { coerceSecretRef, type SecretRef } from "../config/types.secrets.js"; import { resolveUserPath } from "../utils.js"; -import { resolveSecretRefValue, type SecretRefResolveCache } from "./resolve.js"; +import { resolveSecretRefValues, type SecretRefResolveCache } from "./resolve.js"; import { isNonEmptyString, isRecord } from "./shared.js"; type SecretResolverWarningCode = "SECRETS_REF_OVERRIDES_PLAINTEXT"; @@ -31,11 +31,6 @@ export type PreparedSecretsRuntimeSnapshot = { warnings: SecretResolverWarning[]; }; -type ResolverContext = SecretRefResolveCache & { - config: OpenClawConfig; - env: NodeJS.ProcessEnv; -}; - type ProviderLike = { apiKey?: unknown; }; @@ -62,8 +57,27 @@ type TokenCredentialLike = AuthProfileCredential & { tokenRef?: unknown; }; +type SecretAssignment = { + ref: SecretRef; + path: string; + expected: "string" | "string-or-object"; + apply: (value: unknown) => void; +}; + +type ResolverContext = { + sourceConfig: OpenClawConfig; + env: NodeJS.ProcessEnv; + cache: SecretRefResolveCache; + warnings: SecretResolverWarning[]; + assignments: SecretAssignment[]; +}; + let activeSnapshot: PreparedSecretsRuntimeSnapshot | null = null; +function toRefKey(ref: SecretRef): string { + return `${ref.source}:${ref.provider}:${ref.id}`; +} + function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecretsRuntimeSnapshot { return { sourceConfig: structuredClone(snapshot.sourceConfig), @@ -76,151 +90,174 @@ function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecret }; } -async function resolveSecretRefValueFromContext( - ref: SecretRef, - context: ResolverContext, -): Promise { - return await resolveSecretRefValue(ref, { - config: context.config, - env: context.env, - cache: context, - }); -} - -async function resolveGoogleChatServiceAccount( - target: GoogleChatAccountLike, - path: string, - context: ResolverContext, - warnings: SecretResolverWarning[], -): Promise { - const explicitRef = isSecretRef(target.serviceAccountRef) ? target.serviceAccountRef : null; - const inlineRef = isSecretRef(target.serviceAccount) ? target.serviceAccount : null; - const ref = explicitRef ?? inlineRef; - if (!ref) { - return; - } - if (explicitRef && target.serviceAccount !== undefined && !isSecretRef(target.serviceAccount)) { - warnings.push({ - code: "SECRETS_REF_OVERRIDES_PLAINTEXT", - path, - message: `${path}: serviceAccountRef is set; runtime will ignore plaintext serviceAccount.`, - }); - } - target.serviceAccount = await resolveSecretRefValueFromContext(ref, context); -} - -async function resolveConfigSecretRefs(params: { +function collectConfigAssignments(params: { config: OpenClawConfig; context: ResolverContext; - warnings: SecretResolverWarning[]; -}): Promise { - const resolved = structuredClone(params.config); - const providers = resolved.models?.providers as Record | undefined; +}): void { + const defaults = params.context.sourceConfig.secrets?.defaults; + const providers = params.config.models?.providers as Record | undefined; if (providers) { for (const [providerId, provider] of Object.entries(providers)) { - if (!isSecretRef(provider.apiKey)) { + const ref = coerceSecretRef(provider.apiKey, defaults); + if (!ref) { continue; } - const resolvedValue = await resolveSecretRefValueFromContext(provider.apiKey, params.context); - if (!isNonEmptyString(resolvedValue)) { - throw new Error( - `models.providers.${providerId}.apiKey resolved to a non-string or empty value.`, - ); - } - provider.apiKey = resolvedValue; + params.context.assignments.push({ + ref, + path: `models.providers.${providerId}.apiKey`, + expected: "string", + apply: (value) => { + provider.apiKey = value; + }, + }); } } - const skillEntries = resolved.skills?.entries as Record | undefined; + const skillEntries = params.config.skills?.entries as Record | undefined; if (skillEntries) { for (const [skillKey, entry] of Object.entries(skillEntries)) { - if (!isSecretRef(entry.apiKey)) { + const ref = coerceSecretRef(entry.apiKey, defaults); + if (!ref) { continue; } - const resolvedValue = await resolveSecretRefValueFromContext(entry.apiKey, params.context); - if (!isNonEmptyString(resolvedValue)) { - throw new Error( - `skills.entries.${skillKey}.apiKey resolved to a non-string or empty value.`, - ); - } - entry.apiKey = resolvedValue; + params.context.assignments.push({ + ref, + path: `skills.entries.${skillKey}.apiKey`, + expected: "string", + apply: (value) => { + entry.apiKey = value; + }, + }); } } - const googleChat = resolved.channels?.googlechat as GoogleChatAccountLike | undefined; + const collectGoogleChatAssignments = (target: GoogleChatAccountLike, path: string) => { + const explicitRef = coerceSecretRef(target.serviceAccountRef, defaults); + const inlineRef = coerceSecretRef(target.serviceAccount, defaults); + const ref = explicitRef ?? inlineRef; + if (!ref) { + return; + } + if ( + explicitRef && + target.serviceAccount !== undefined && + !coerceSecretRef(target.serviceAccount, defaults) + ) { + params.context.warnings.push({ + code: "SECRETS_REF_OVERRIDES_PLAINTEXT", + path, + message: `${path}: serviceAccountRef is set; runtime will ignore plaintext serviceAccount.`, + }); + } + params.context.assignments.push({ + ref, + path: `${path}.serviceAccount`, + expected: "string-or-object", + apply: (value) => { + target.serviceAccount = value; + }, + }); + }; + + const googleChat = params.config.channels?.googlechat as GoogleChatAccountLike | undefined; if (googleChat) { - await resolveGoogleChatServiceAccount( - googleChat, - "channels.googlechat", - params.context, - params.warnings, - ); + collectGoogleChatAssignments(googleChat, "channels.googlechat"); if (isRecord(googleChat.accounts)) { for (const [accountId, account] of Object.entries(googleChat.accounts)) { if (!isRecord(account)) { continue; } - await resolveGoogleChatServiceAccount( + collectGoogleChatAssignments( account as GoogleChatAccountLike, `channels.googlechat.accounts.${accountId}`, - params.context, - params.warnings, ); } } } - - return resolved; } -async function resolveAuthStoreSecretRefs(params: { +function collectAuthStoreAssignments(params: { store: AuthProfileStore; context: ResolverContext; - warnings: SecretResolverWarning[]; agentDir: string; -}): Promise { - const resolvedStore = structuredClone(params.store); - for (const [profileId, profile] of Object.entries(resolvedStore.profiles)) { +}): void { + const defaults = params.context.sourceConfig.secrets?.defaults; + for (const [profileId, profile] of Object.entries(params.store.profiles)) { if (profile.type === "api_key") { const apiProfile = profile as ApiKeyCredentialLike; - const keyRef = isSecretRef(apiProfile.keyRef) ? apiProfile.keyRef : null; + const keyRef = coerceSecretRef(apiProfile.keyRef, defaults); + const inlineKeyRef = keyRef ? null : coerceSecretRef(apiProfile.key, defaults); + const resolvedKeyRef = keyRef ?? inlineKeyRef; + if (!resolvedKeyRef) { + continue; + } if (keyRef && isNonEmptyString(apiProfile.key)) { - params.warnings.push({ + params.context.warnings.push({ code: "SECRETS_REF_OVERRIDES_PLAINTEXT", path: `${params.agentDir}.auth-profiles.${profileId}.key`, message: `auth-profiles ${profileId}: keyRef is set; runtime will ignore plaintext key.`, }); } - if (keyRef) { - const resolvedValue = await resolveSecretRefValueFromContext(keyRef, params.context); - if (!isNonEmptyString(resolvedValue)) { - throw new Error(`auth profile "${profileId}" keyRef resolved to an empty value.`); - } - apiProfile.key = resolvedValue; - } + params.context.assignments.push({ + ref: resolvedKeyRef, + path: `${params.agentDir}.auth-profiles.${profileId}.key`, + expected: "string", + apply: (value) => { + apiProfile.key = String(value); + }, + }); continue; } if (profile.type === "token") { const tokenProfile = profile as TokenCredentialLike; - const tokenRef = isSecretRef(tokenProfile.tokenRef) ? tokenProfile.tokenRef : null; + const tokenRef = coerceSecretRef(tokenProfile.tokenRef, defaults); + const inlineTokenRef = tokenRef ? null : coerceSecretRef(tokenProfile.token, defaults); + const resolvedTokenRef = tokenRef ?? inlineTokenRef; + if (!resolvedTokenRef) { + continue; + } if (tokenRef && isNonEmptyString(tokenProfile.token)) { - params.warnings.push({ + params.context.warnings.push({ code: "SECRETS_REF_OVERRIDES_PLAINTEXT", path: `${params.agentDir}.auth-profiles.${profileId}.token`, message: `auth-profiles ${profileId}: tokenRef is set; runtime will ignore plaintext token.`, }); } - if (tokenRef) { - const resolvedValue = await resolveSecretRefValueFromContext(tokenRef, params.context); - if (!isNonEmptyString(resolvedValue)) { - throw new Error(`auth profile "${profileId}" tokenRef resolved to an empty value.`); - } - tokenProfile.token = resolvedValue; - } + params.context.assignments.push({ + ref: resolvedTokenRef, + path: `${params.agentDir}.auth-profiles.${profileId}.token`, + expected: "string", + apply: (value) => { + tokenProfile.token = String(value); + }, + }); } } - return resolvedStore; +} + +function applyAssignments(params: { + assignments: SecretAssignment[]; + resolved: Map; +}): void { + for (const assignment of params.assignments) { + const key = toRefKey(assignment.ref); + if (!params.resolved.has(key)) { + throw new Error(`Secret reference "${key}" resolved to no value.`); + } + const value = params.resolved.get(key); + if (assignment.expected === "string") { + if (!isNonEmptyString(value)) { + throw new Error(`${assignment.path} resolved to a non-string or empty value.`); + } + assignment.apply(value); + continue; + } + if (!(isNonEmptyString(value) || isRecord(value))) { + throw new Error(`${assignment.path} resolved to an unsupported value type.`); + } + assignment.apply(value); + } } function collectCandidateAgentDirs(config: OpenClawConfig): string[] { @@ -238,39 +275,55 @@ export async function prepareSecretsRuntimeSnapshot(params: { agentDirs?: string[]; loadAuthStore?: (agentDir?: string) => AuthProfileStore; }): Promise { - const warnings: SecretResolverWarning[] = []; + const sourceConfig = structuredClone(params.config); + const resolvedConfig = structuredClone(params.config); const context: ResolverContext = { - config: params.config, + sourceConfig, env: params.env ?? process.env, - fileSecretsPromise: null, + cache: {}, + warnings: [], + assignments: [], }; - const resolvedConfig = await resolveConfigSecretRefs({ - config: params.config, + + collectConfigAssignments({ + config: resolvedConfig, context, - warnings, }); const loadAuthStore = params.loadAuthStore ?? loadAuthProfileStoreForSecretsRuntime; const candidateDirs = params.agentDirs?.length ? [...new Set(params.agentDirs.map((entry) => resolveUserPath(entry)))] : collectCandidateAgentDirs(resolvedConfig); + const authStores: Array<{ agentDir: string; store: AuthProfileStore }> = []; for (const agentDir of candidateDirs) { - const rawStore = loadAuthStore(agentDir); - const resolvedStore = await resolveAuthStoreSecretRefs({ - store: rawStore, + const store = structuredClone(loadAuthStore(agentDir)); + collectAuthStoreAssignments({ + store, context, - warnings, agentDir, }); - authStores.push({ agentDir, store: resolvedStore }); + authStores.push({ agentDir, store }); + } + + if (context.assignments.length > 0) { + const refs = context.assignments.map((assignment) => assignment.ref); + const resolved = await resolveSecretRefValues(refs, { + config: sourceConfig, + env: context.env, + cache: context.cache, + }); + applyAssignments({ + assignments: context.assignments, + resolved, + }); } return { - sourceConfig: structuredClone(params.config), + sourceConfig, config: resolvedConfig, authStores, - warnings, + warnings: context.warnings, }; } diff --git a/src/secrets/sops.ts b/src/secrets/sops.ts deleted file mode 100644 index 4a8025017..000000000 --- a/src/secrets/sops.ts +++ /dev/null @@ -1,152 +0,0 @@ -import crypto from "node:crypto"; -import fs from "node:fs"; -import path from "node:path"; -import { runExec } from "../process/exec.js"; -import { ensureDirForFile, normalizePositiveInt } from "./shared.js"; - -export const DEFAULT_SOPS_TIMEOUT_MS = 5_000; -const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024; - -function toSopsPath(value: string): string { - return value.replaceAll(path.sep, "/"); -} - -function resolveFilenameOverride(params: { targetPath: string }): string { - return toSopsPath(path.resolve(params.targetPath)); -} - -function resolveSopsCwd(params: { targetPath: string; configPath?: string }): string { - if (typeof params.configPath === "string" && params.configPath.trim().length > 0) { - return path.dirname(params.configPath); - } - return path.dirname(params.targetPath); -} - -function normalizeTimeoutMs(value: number | undefined): number { - return normalizePositiveInt(value, DEFAULT_SOPS_TIMEOUT_MS); -} - -function isTimeoutError(message: string | undefined): boolean { - return typeof message === "string" && message.toLowerCase().includes("timed out"); -} - -type SopsErrorContext = { - missingBinaryMessage: string; - operationLabel: string; -}; - -function toSopsError(err: unknown, params: SopsErrorContext): Error { - const error = err as NodeJS.ErrnoException & { message?: string }; - if (error.code === "ENOENT") { - return new Error(params.missingBinaryMessage, { cause: err }); - } - return new Error(`${params.operationLabel}: ${String(error.message ?? err)}`, { - cause: err, - }); -} - -export async function decryptSopsJsonFile(params: { - path: string; - timeoutMs?: number; - missingBinaryMessage: string; - configPath?: string; -}): Promise { - const timeoutMs = normalizeTimeoutMs(params.timeoutMs); - const cwd = resolveSopsCwd({ - targetPath: params.path, - configPath: params.configPath, - }); - try { - const args: string[] = []; - if (typeof params.configPath === "string" && params.configPath.trim().length > 0) { - args.push("--config", params.configPath); - } - args.push("--decrypt", "--output-type", "json", params.path); - const { stdout } = await runExec("sops", args, { - timeoutMs, - maxBuffer: MAX_SOPS_OUTPUT_BYTES, - cwd, - }); - return JSON.parse(stdout) as unknown; - } catch (err) { - const error = err as NodeJS.ErrnoException & { message?: string }; - if (isTimeoutError(error.message)) { - throw new Error(`sops decrypt timed out after ${timeoutMs}ms for ${params.path}.`, { - cause: err, - }); - } - throw toSopsError(err, { - missingBinaryMessage: params.missingBinaryMessage, - operationLabel: `sops decrypt failed for ${params.path}`, - }); - } -} - -export async function encryptSopsJsonFile(params: { - path: string; - payload: Record; - timeoutMs?: number; - missingBinaryMessage: string; - configPath?: string; -}): Promise { - ensureDirForFile(params.path); - const timeoutMs = normalizeTimeoutMs(params.timeoutMs); - const tmpPlain = path.join( - path.dirname(params.path), - `${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.plain.tmp`, - ); - const tmpEncrypted = path.join( - path.dirname(params.path), - `${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.enc.tmp`, - ); - - fs.writeFileSync(tmpPlain, `${JSON.stringify(params.payload, null, 2)}\n`, "utf8"); - fs.chmodSync(tmpPlain, 0o600); - - try { - const filenameOverride = resolveFilenameOverride({ - targetPath: params.path, - }); - const cwd = resolveSopsCwd({ - targetPath: params.path, - configPath: params.configPath, - }); - const args: string[] = []; - if (typeof params.configPath === "string" && params.configPath.trim().length > 0) { - args.push("--config", params.configPath); - } - args.push( - "--encrypt", - "--filename-override", - filenameOverride, - "--input-type", - "json", - "--output-type", - "json", - "--output", - tmpEncrypted, - tmpPlain, - ); - await runExec("sops", args, { - timeoutMs, - maxBuffer: MAX_SOPS_OUTPUT_BYTES, - cwd, - }); - fs.renameSync(tmpEncrypted, params.path); - fs.chmodSync(params.path, 0o600); - } catch (err) { - const error = err as NodeJS.ErrnoException & { message?: string }; - if (isTimeoutError(error.message)) { - throw new Error(`sops encrypt timed out after ${timeoutMs}ms for ${params.path}.`, { - cause: err, - }); - } - throw toSopsError(err, { - missingBinaryMessage: params.missingBinaryMessage, - operationLabel: `sops encrypt failed for ${params.path}`, - }); - } finally { - fs.rmSync(tmpPlain, { force: true }); - fs.rmSync(tmpEncrypted, { force: true }); - } -}