Secrets: keep read-only runtime sync in-memory

src/agents/auth-profiles.readonly-sync.test.ts

@@ -0,0 +1,67 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js";
+import type { AuthProfileStore } from "./auth-profiles/types.js";
+
+const mocks = vi.hoisted(() => ({
+  syncExternalCliCredentials: vi.fn((store: AuthProfileStore) => {
+    store.profiles["qwen-portal:default"] = {
+      type: "oauth",
+      provider: "qwen-portal",
+      access: "access-token",
+      refresh: "refresh-token",
+      expires: Date.now() + 60_000,
+    };
+    return true;
+  }),
+}));
+
+vi.mock("./auth-profiles/external-cli-sync.js", () => ({
+  syncExternalCliCredentials: mocks.syncExternalCliCredentials,
+}));
+
+const { loadAuthProfileStoreForRuntime } = await import("./auth-profiles.js");
+
+describe("auth profiles read-only external CLI sync", () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("syncs external CLI credentials in-memory without writing auth-profiles.json in read-only mode", () => {
+    const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-readonly-sync-"));
+    try {
+      const authPath = path.join(agentDir, "auth-profiles.json");
+      const baseline: AuthProfileStore = {
+        version: AUTH_STORE_VERSION,
+        profiles: {
+          "openai:default": {
+            type: "api_key",
+            provider: "openai",
+            key: "sk-test",
+          },
+        },
+      };
+      fs.writeFileSync(authPath, `${JSON.stringify(baseline, null, 2)}\n`, "utf8");
+
+      const loaded = loadAuthProfileStoreForRuntime(agentDir, { readOnly: true });
+
+      expect(mocks.syncExternalCliCredentials).toHaveBeenCalled();
+      expect(loaded.profiles["qwen-portal:default"]).toMatchObject({
+        type: "oauth",
+        provider: "qwen-portal",
+      });
+
+      const persisted = JSON.parse(fs.readFileSync(authPath, "utf8")) as AuthProfileStore;
+      expect(persisted.profiles["qwen-portal:default"]).toBeUndefined();
+      expect(persisted.profiles["openai:default"]).toMatchObject({
+        type: "api_key",
+        provider: "openai",
+        key: "sk-test",
+      });
+    } finally {
+      fs.rmSync(agentDir, { recursive: true, force: true });
+    }
+  });
+});

src/agents/auth-profiles/store.ts

@@ -383,13 +383,11 @@ function loadAuthProfileStoreForAgent(
   const authPath = resolveAuthStorePath(agentDir);
   const asStore = loadCoercedStoreWithExternalSync(authPath);
   if (asStore) {
-    // Runtime secret activation must remain read-only.
-    if (!readOnly) {
-      // Sync from external CLI tools on every load
-      const synced = syncExternalCliCredentials(asStore);
-      if (synced) {
-        saveJsonFile(authPath, asStore);
-      }
+    // Runtime secret activation must remain read-only:
+    // sync external CLI credentials in-memory, but never persist while readOnly.
+    const synced = syncExternalCliCredentials(asStore);
+    if (synced && !readOnly) {
+      saveJsonFile(authPath, asStore);
     }
     return asStore;
   }
@@ -418,7 +416,8 @@ function loadAuthProfileStoreForAgent(
   }
 
   const mergedOAuth = mergeOAuthFileIntoStore(store);
-  const syncedCli = readOnly ? false : syncExternalCliCredentials(store);
+  // Keep external CLI credentials visible in runtime even during read-only loads.
+  const syncedCli = syncExternalCliCredentials(store);
   const shouldWrite = !readOnly && (legacy !== null || mergedOAuth || syncedCli);
   if (shouldWrite) {
     saveJsonFile(authPath, store);

src/secrets/resolve.ts

@@ -0,0 +1,85 @@
+import type { OpenClawConfig } from "../config/config.js";
+import type { SecretRef } from "../config/types.secrets.js";
+import { resolveUserPath } from "../utils.js";
+import { readJsonPointer } from "./json-pointer.js";
+import { isNonEmptyString, normalizePositiveInt } from "./shared.js";
+import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js";
+
+export type SecretRefResolveCache = {
+  fileSecretsPromise?: Promise<unknown> | null;
+};
+
+type ResolveSecretRefOptions = {
+  config: OpenClawConfig;
+  env?: NodeJS.ProcessEnv;
+  cache?: SecretRefResolveCache;
+  missingBinaryMessage?: string;
+};
+
+const DEFAULT_SOPS_MISSING_BINARY_MESSAGE =
+  "sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.";
+
+async function resolveFileSecretPayload(options: ResolveSecretRefOptions): Promise<unknown> {
+  const fileSource = options.config.secrets?.sources?.file;
+  if (!fileSource) {
+    throw new Error(
+      'Secret reference source "file" is not configured. Configure secrets.sources.file first.',
+    );
+  }
+  if (fileSource.type !== "sops") {
+    throw new Error(`Unsupported secrets.sources.file.type "${String(fileSource.type)}".`);
+  }
+
+  const cache = options.cache;
+  if (cache?.fileSecretsPromise) {
+    return await cache.fileSecretsPromise;
+  }
+
+  const promise = decryptSopsJsonFile({
+    path: resolveUserPath(fileSource.path),
+    timeoutMs: normalizePositiveInt(fileSource.timeoutMs, DEFAULT_SOPS_TIMEOUT_MS),
+    missingBinaryMessage: options.missingBinaryMessage ?? DEFAULT_SOPS_MISSING_BINARY_MESSAGE,
+  });
+  if (cache) {
+    cache.fileSecretsPromise = promise;
+  }
+  return await promise;
+}
+
+export async function resolveSecretRefValue(
+  ref: SecretRef,
+  options: ResolveSecretRefOptions,
+): Promise<unknown> {
+  const id = ref.id.trim();
+  if (!id) {
+    throw new Error("Secret reference id is empty.");
+  }
+
+  if (ref.source === "env") {
+    const envValue = options.env?.[id] ?? process.env[id];
+    if (!isNonEmptyString(envValue)) {
+      throw new Error(`Environment variable "${id}" is missing or empty.`);
+    }
+    return envValue;
+  }
+
+  if (ref.source === "file") {
+    const payload = await resolveFileSecretPayload(options);
+    return readJsonPointer(payload, id, { onMissing: "throw" });
+  }
+
+  throw new Error(`Unsupported secret source "${String((ref as { source?: unknown }).source)}".`);
+}
+
+export async function resolveSecretRefString(
+  ref: SecretRef,
+  options: ResolveSecretRefOptions,
+): Promise<string> {
+  const resolved = await resolveSecretRefValue(ref, options);
+  if (!isNonEmptyString(resolved)) {
+    throw new Error(
+      `Secret reference "${ref.source}:${ref.id}" resolved to a non-string or empty value.`,
+    );
+  }
+  return resolved;
+}

src/secrets/runtime.ts

@@ -13,9 +13,8 @@ import {
 } from "../config/config.js";
 import { isSecretRef, type SecretRef } from "../config/types.secrets.js";
 import { resolveUserPath } from "../utils.js";
-import { readJsonPointer } from "./json-pointer.js";
-import { isNonEmptyString, isRecord, normalizePositiveInt } from "./shared.js";
-import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js";
+import { resolveSecretRefValue, type SecretRefResolveCache } from "./resolve.js";
+import { isNonEmptyString, isRecord } from "./shared.js";
 
 type SecretResolverWarningCode = "SECRETS_REF_OVERRIDES_PLAINTEXT";
 
@@ -32,10 +31,9 @@ export type PreparedSecretsRuntimeSnapshot = {
   warnings: SecretResolverWarning[];
 };
 
-type ResolverContext = {
+type ResolverContext = SecretRefResolveCache & {
   config: OpenClawConfig;
   env: NodeJS.ProcessEnv;
-  fileSecretsPromise: Promise<unknown> | null;
 };
 
 type ProviderLike = {
@@ -74,50 +72,17 @@ function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecret
   };
 }
 
-async function decryptSopsFile(config: OpenClawConfig): Promise<unknown> {
-  const fileSource = config.secrets?.sources?.file;
-  if (!fileSource) {
-    throw new Error(
-      `Secret reference source "file" is not configured. Configure secrets.sources.file first.`,
-    );
-  }
-  if (fileSource.type !== "sops") {
-    throw new Error(`Unsupported secrets.sources.file.type "${String(fileSource.type)}".`);
-  }
-
-  const resolvedPath = resolveUserPath(fileSource.path);
-  const timeoutMs = normalizePositiveInt(fileSource.timeoutMs, DEFAULT_SOPS_TIMEOUT_MS);
-  return await decryptSopsJsonFile({
-    path: resolvedPath,
-    timeoutMs,
-    missingBinaryMessage:
-      "sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.",
+async function resolveSecretRefValueFromContext(
+  ref: SecretRef,
+  context: ResolverContext,
+): Promise<unknown> {
+  return await resolveSecretRefValue(ref, {
+    config: context.config,
+    env: context.env,
+    cache: context,
   });
 }
 
-async function resolveSecretRefValue(ref: SecretRef, context: ResolverContext): Promise<unknown> {
-  const id = ref.id.trim();
-  if (!id) {
-    throw new Error(`Secret reference id is empty.`);
-  }
-
-  if (ref.source === "env") {
-    const envValue = context.env[id];
-    if (!isNonEmptyString(envValue)) {
-      throw new Error(`Environment variable "${id}" is missing or empty.`);
-    }
-    return envValue;
-  }
-
-  if (ref.source === "file") {
-    context.fileSecretsPromise ??= decryptSopsFile(context.config);
-    const payload = await context.fileSecretsPromise;
-    return readJsonPointer(payload, id, { onMissing: "throw" });
-  }
-
-  throw new Error(`Unsupported secret source "${String((ref as { source?: unknown }).source)}".`);
-}
-
 async function resolveGoogleChatServiceAccount(
   target: GoogleChatAccountLike,
   path: string,
@@ -137,7 +102,7 @@ async function resolveGoogleChatServiceAccount(
       message: `${path}: serviceAccountRef is set; runtime will ignore plaintext serviceAccount.`,
     });
   }
-  target.serviceAccount = await resolveSecretRefValue(ref, context);
+  target.serviceAccount = await resolveSecretRefValueFromContext(ref, context);
 }
 
 async function resolveConfigSecretRefs(params: {
@@ -152,7 +117,7 @@ async function resolveConfigSecretRefs(params: {
       if (!isSecretRef(provider.apiKey)) {
         continue;
       }
-      const resolvedValue = await resolveSecretRefValue(provider.apiKey, params.context);
+      const resolvedValue = await resolveSecretRefValueFromContext(provider.apiKey, params.context);
       if (!isNonEmptyString(resolvedValue)) {
         throw new Error(
           `models.providers.${providerId}.apiKey resolved to a non-string or empty value.`,
@@ -207,7 +172,7 @@ async function resolveAuthStoreSecretRefs(params: {
         });
       }
       if (keyRef) {
-        const resolvedValue = await resolveSecretRefValue(keyRef, params.context);
+        const resolvedValue = await resolveSecretRefValueFromContext(keyRef, params.context);
         if (!isNonEmptyString(resolvedValue)) {
           throw new Error(`auth profile "${profileId}" keyRef resolved to an empty value.`);
         }
@@ -227,7 +192,7 @@ async function resolveAuthStoreSecretRefs(params: {
         });
       }
       if (tokenRef) {
-        const resolvedValue = await resolveSecretRefValue(tokenRef, params.context);
+        const resolvedValue = await resolveSecretRefValueFromContext(tokenRef, params.context);
         if (!isNonEmptyString(resolvedValue)) {
           throw new Error(`auth profile "${profileId}" tokenRef resolved to an empty value.`);
         }