301 lines
9.0 KiB
TypeScript
301 lines
9.0 KiB
TypeScript
import type { ClawdbotConfig } from "../../config/config.js";
|
|
import { normalizeProviderId } from "../model-selection.js";
|
|
import { saveAuthProfileStore, updateAuthProfileStoreWithLock } from "./store.js";
|
|
import type { AuthProfileFailureReason, AuthProfileStore, ProfileUsageStats } from "./types.js";
|
|
|
|
function resolveProfileUnusableUntil(stats: ProfileUsageStats): number | null {
|
|
const values = [stats.cooldownUntil, stats.disabledUntil]
|
|
.filter((value): value is number => typeof value === "number")
|
|
.filter((value) => Number.isFinite(value) && value > 0);
|
|
if (values.length === 0) return null;
|
|
return Math.max(...values);
|
|
}
|
|
|
|
/**
|
|
* Check if a profile is currently in cooldown (due to rate limiting or errors).
|
|
*/
|
|
export function isProfileInCooldown(store: AuthProfileStore, profileId: string): boolean {
|
|
const stats = store.usageStats?.[profileId];
|
|
if (!stats) return false;
|
|
const unusableUntil = resolveProfileUnusableUntil(stats);
|
|
return unusableUntil ? Date.now() < unusableUntil : false;
|
|
}
|
|
|
|
/**
|
|
* Mark a profile as successfully used. Resets error count and updates lastUsed.
|
|
* Uses store lock to avoid overwriting concurrent usage updates.
|
|
*/
|
|
export async function markAuthProfileUsed(params: {
|
|
store: AuthProfileStore;
|
|
profileId: string;
|
|
agentDir?: string;
|
|
}): Promise<void> {
|
|
const { store, profileId, agentDir } = params;
|
|
const updated = await updateAuthProfileStoreWithLock({
|
|
agentDir,
|
|
updater: (freshStore) => {
|
|
if (!freshStore.profiles[profileId]) return false;
|
|
freshStore.usageStats = freshStore.usageStats ?? {};
|
|
freshStore.usageStats[profileId] = {
|
|
...freshStore.usageStats[profileId],
|
|
lastUsed: Date.now(),
|
|
errorCount: 0,
|
|
cooldownUntil: undefined,
|
|
disabledUntil: undefined,
|
|
disabledReason: undefined,
|
|
failureCounts: undefined,
|
|
};
|
|
return true;
|
|
},
|
|
});
|
|
if (updated) {
|
|
store.usageStats = updated.usageStats;
|
|
return;
|
|
}
|
|
if (!store.profiles[profileId]) return;
|
|
|
|
store.usageStats = store.usageStats ?? {};
|
|
store.usageStats[profileId] = {
|
|
...store.usageStats[profileId],
|
|
lastUsed: Date.now(),
|
|
errorCount: 0,
|
|
cooldownUntil: undefined,
|
|
disabledUntil: undefined,
|
|
disabledReason: undefined,
|
|
failureCounts: undefined,
|
|
};
|
|
saveAuthProfileStore(store, agentDir);
|
|
}
|
|
|
|
export function calculateAuthProfileCooldownMs(errorCount: number): number {
|
|
const normalized = Math.max(1, errorCount);
|
|
return Math.min(
|
|
60 * 60 * 1000, // 1 hour max
|
|
60 * 1000 * 5 ** Math.min(normalized - 1, 3),
|
|
);
|
|
}
|
|
|
|
type ResolvedAuthCooldownConfig = {
|
|
billingBackoffMs: number;
|
|
billingMaxMs: number;
|
|
failureWindowMs: number;
|
|
};
|
|
|
|
function resolveAuthCooldownConfig(params: {
|
|
cfg?: ClawdbotConfig;
|
|
providerId: string;
|
|
}): ResolvedAuthCooldownConfig {
|
|
const defaults = {
|
|
billingBackoffHours: 5,
|
|
billingMaxHours: 24,
|
|
failureWindowHours: 24,
|
|
} as const;
|
|
|
|
const resolveHours = (value: unknown, fallback: number) =>
|
|
typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallback;
|
|
|
|
const cooldowns = params.cfg?.auth?.cooldowns;
|
|
const billingOverride = (() => {
|
|
const map = cooldowns?.billingBackoffHoursByProvider;
|
|
if (!map) return undefined;
|
|
for (const [key, value] of Object.entries(map)) {
|
|
if (normalizeProviderId(key) === params.providerId) return value;
|
|
}
|
|
return undefined;
|
|
})();
|
|
|
|
const billingBackoffHours = resolveHours(
|
|
billingOverride ?? cooldowns?.billingBackoffHours,
|
|
defaults.billingBackoffHours,
|
|
);
|
|
const billingMaxHours = resolveHours(cooldowns?.billingMaxHours, defaults.billingMaxHours);
|
|
const failureWindowHours = resolveHours(
|
|
cooldowns?.failureWindowHours,
|
|
defaults.failureWindowHours,
|
|
);
|
|
|
|
return {
|
|
billingBackoffMs: billingBackoffHours * 60 * 60 * 1000,
|
|
billingMaxMs: billingMaxHours * 60 * 60 * 1000,
|
|
failureWindowMs: failureWindowHours * 60 * 60 * 1000,
|
|
};
|
|
}
|
|
|
|
function calculateAuthProfileBillingDisableMsWithConfig(params: {
|
|
errorCount: number;
|
|
baseMs: number;
|
|
maxMs: number;
|
|
}): number {
|
|
const normalized = Math.max(1, params.errorCount);
|
|
const baseMs = Math.max(60_000, params.baseMs);
|
|
const maxMs = Math.max(baseMs, params.maxMs);
|
|
const exponent = Math.min(normalized - 1, 10);
|
|
const raw = baseMs * 2 ** exponent;
|
|
return Math.min(maxMs, raw);
|
|
}
|
|
|
|
export function resolveProfileUnusableUntilForDisplay(
|
|
store: AuthProfileStore,
|
|
profileId: string,
|
|
): number | null {
|
|
const stats = store.usageStats?.[profileId];
|
|
if (!stats) return null;
|
|
return resolveProfileUnusableUntil(stats);
|
|
}
|
|
|
|
function computeNextProfileUsageStats(params: {
|
|
existing: ProfileUsageStats;
|
|
now: number;
|
|
reason: AuthProfileFailureReason;
|
|
cfgResolved: ResolvedAuthCooldownConfig;
|
|
}): ProfileUsageStats {
|
|
const windowMs = params.cfgResolved.failureWindowMs;
|
|
const windowExpired =
|
|
typeof params.existing.lastFailureAt === "number" &&
|
|
params.existing.lastFailureAt > 0 &&
|
|
params.now - params.existing.lastFailureAt > windowMs;
|
|
|
|
const baseErrorCount = windowExpired ? 0 : (params.existing.errorCount ?? 0);
|
|
const nextErrorCount = baseErrorCount + 1;
|
|
const failureCounts = windowExpired ? {} : { ...params.existing.failureCounts };
|
|
failureCounts[params.reason] = (failureCounts[params.reason] ?? 0) + 1;
|
|
|
|
const updatedStats: ProfileUsageStats = {
|
|
...params.existing,
|
|
errorCount: nextErrorCount,
|
|
failureCounts,
|
|
lastFailureAt: params.now,
|
|
};
|
|
|
|
if (params.reason === "billing") {
|
|
const billingCount = failureCounts.billing ?? 1;
|
|
const backoffMs = calculateAuthProfileBillingDisableMsWithConfig({
|
|
errorCount: billingCount,
|
|
baseMs: params.cfgResolved.billingBackoffMs,
|
|
maxMs: params.cfgResolved.billingMaxMs,
|
|
});
|
|
updatedStats.disabledUntil = params.now + backoffMs;
|
|
updatedStats.disabledReason = "billing";
|
|
} else {
|
|
const backoffMs = calculateAuthProfileCooldownMs(nextErrorCount);
|
|
updatedStats.cooldownUntil = params.now + backoffMs;
|
|
}
|
|
|
|
return updatedStats;
|
|
}
|
|
|
|
/**
|
|
* Mark a profile as failed for a specific reason. Billing failures are treated
|
|
* as "disabled" (longer backoff) vs the regular cooldown window.
|
|
*/
|
|
export async function markAuthProfileFailure(params: {
|
|
store: AuthProfileStore;
|
|
profileId: string;
|
|
reason: AuthProfileFailureReason;
|
|
cfg?: ClawdbotConfig;
|
|
agentDir?: string;
|
|
}): Promise<void> {
|
|
const { store, profileId, reason, agentDir, cfg } = params;
|
|
const updated = await updateAuthProfileStoreWithLock({
|
|
agentDir,
|
|
updater: (freshStore) => {
|
|
const profile = freshStore.profiles[profileId];
|
|
if (!profile) return false;
|
|
freshStore.usageStats = freshStore.usageStats ?? {};
|
|
const existing = freshStore.usageStats[profileId] ?? {};
|
|
|
|
const now = Date.now();
|
|
const providerKey = normalizeProviderId(profile.provider);
|
|
const cfgResolved = resolveAuthCooldownConfig({
|
|
cfg,
|
|
providerId: providerKey,
|
|
});
|
|
|
|
freshStore.usageStats[profileId] = computeNextProfileUsageStats({
|
|
existing,
|
|
now,
|
|
reason,
|
|
cfgResolved,
|
|
});
|
|
return true;
|
|
},
|
|
});
|
|
if (updated) {
|
|
store.usageStats = updated.usageStats;
|
|
return;
|
|
}
|
|
if (!store.profiles[profileId]) return;
|
|
|
|
store.usageStats = store.usageStats ?? {};
|
|
const existing = store.usageStats[profileId] ?? {};
|
|
const now = Date.now();
|
|
const providerKey = normalizeProviderId(store.profiles[profileId]?.provider ?? "");
|
|
const cfgResolved = resolveAuthCooldownConfig({
|
|
cfg,
|
|
providerId: providerKey,
|
|
});
|
|
|
|
store.usageStats[profileId] = computeNextProfileUsageStats({
|
|
existing,
|
|
now,
|
|
reason,
|
|
cfgResolved,
|
|
});
|
|
saveAuthProfileStore(store, agentDir);
|
|
}
|
|
|
|
/**
|
|
* Mark a profile as failed/rate-limited. Applies exponential backoff cooldown.
|
|
* Cooldown times: 1min, 5min, 25min, max 1 hour.
|
|
* Uses store lock to avoid overwriting concurrent usage updates.
|
|
*/
|
|
export async function markAuthProfileCooldown(params: {
|
|
store: AuthProfileStore;
|
|
profileId: string;
|
|
agentDir?: string;
|
|
}): Promise<void> {
|
|
await markAuthProfileFailure({
|
|
store: params.store,
|
|
profileId: params.profileId,
|
|
reason: "unknown",
|
|
agentDir: params.agentDir,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Clear cooldown for a profile (e.g., manual reset).
|
|
* Uses store lock to avoid overwriting concurrent usage updates.
|
|
*/
|
|
export async function clearAuthProfileCooldown(params: {
|
|
store: AuthProfileStore;
|
|
profileId: string;
|
|
agentDir?: string;
|
|
}): Promise<void> {
|
|
const { store, profileId, agentDir } = params;
|
|
const updated = await updateAuthProfileStoreWithLock({
|
|
agentDir,
|
|
updater: (freshStore) => {
|
|
if (!freshStore.usageStats?.[profileId]) return false;
|
|
|
|
freshStore.usageStats[profileId] = {
|
|
...freshStore.usageStats[profileId],
|
|
errorCount: 0,
|
|
cooldownUntil: undefined,
|
|
};
|
|
return true;
|
|
},
|
|
});
|
|
if (updated) {
|
|
store.usageStats = updated.usageStats;
|
|
return;
|
|
}
|
|
if (!store.usageStats?.[profileId]) return;
|
|
|
|
store.usageStats[profileId] = {
|
|
...store.usageStats[profileId],
|
|
errorCount: 0,
|
|
cooldownUntil: undefined,
|
|
};
|
|
saveAuthProfileStore(store, agentDir);
|
|
}
|