diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index 17ea903df..90463d005 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -162,7 +162,7 @@ out to QMD for retrieval. Key points: stable `name`). - `sessions`: opt into session JSONL indexing (`enabled`, `retentionDays`, `exportDir`). -- `update`: controls refresh cadence (`interval`, `debounceMs`, `onBoot`). +- `update`: controls refresh cadence (`interval`, `debounceMs`, `onBoot`, `embedInterval`). - `limits`: clamp recall payload (`maxResults`, `maxSnippetChars`, `maxInjectedChars`, `timeoutMs`). - `scope`: same schema as [`session.sendPolicy`](/reference/configuration#session-sendpolicy). diff --git a/src/agents/tools/memory-tool.ts b/src/agents/tools/memory-tool.ts index f6643bb67..1cf5b3356 100644 --- a/src/agents/tools/memory-tool.ts +++ b/src/agents/tools/memory-tool.ts @@ -51,7 +51,10 @@ export function createMemorySearchTool(options: { } try { const citationsMode = resolveMemoryCitationsMode(cfg); - const includeCitations = citationsMode !== "off"; + const includeCitations = shouldIncludeCitations({ + mode: citationsMode, + sessionKey: options.agentSessionKey, + }); const rawResults = await manager.search(query, { maxResults, minScore, @@ -141,3 +144,21 @@ function formatCitation(entry: MemorySearchResult): string { : `#L${entry.startLine}-L${entry.endLine}`; return `${entry.path}${lineRange}`; } + +function shouldIncludeCitations(params: { + mode: MemoryCitationsMode; + sessionKey?: string; +}): boolean { + if (params.mode === "on") return true; + if (params.mode === "off") return false; + // auto: show citations in direct chats; suppress in groups/channels by default. + const chatType = deriveChatTypeFromSessionKey(params.sessionKey); + return chatType === "direct"; +} + +function deriveChatTypeFromSessionKey(sessionKey?: string): "direct" | "group" | "channel" { + if (!sessionKey) return "direct"; + if (sessionKey.includes(":group:")) return "group"; + if (sessionKey.includes(":channel:")) return "channel"; + return "direct"; +} diff --git a/src/cli/memory-cli.test.ts b/src/cli/memory-cli.test.ts index 1eb77980f..4605a2a69 100644 --- a/src/cli/memory-cli.test.ts +++ b/src/cli/memory-cli.test.ts @@ -242,7 +242,7 @@ describe("memory cli", () => { await program.parseAsync(["memory", "status", "--index"], { from: "user" }); expect(sync).toHaveBeenCalledWith( - expect.objectContaining({ reason: "cli", force: true, progress: expect.any(Function) }), + expect.objectContaining({ reason: "cli", force: false, progress: expect.any(Function) }), ); expect(probeEmbeddingAvailability).toHaveBeenCalled(); expect(close).toHaveBeenCalled(); @@ -267,7 +267,7 @@ describe("memory cli", () => { await program.parseAsync(["memory", "index"], { from: "user" }); expect(sync).toHaveBeenCalledWith( - expect.objectContaining({ reason: "cli", force: true, progress: expect.any(Function) }), + expect.objectContaining({ reason: "cli", force: false, progress: expect.any(Function) }), ); expect(close).toHaveBeenCalled(); expect(log).toHaveBeenCalledWith("Memory index updated (main)."); @@ -294,7 +294,7 @@ describe("memory cli", () => { await program.parseAsync(["memory", "index"], { from: "user" }); expect(sync).toHaveBeenCalledWith( - expect.objectContaining({ reason: "cli", force: true, progress: expect.any(Function) }), + expect.objectContaining({ reason: "cli", force: false, progress: expect.any(Function) }), ); expect(close).toHaveBeenCalled(); expect(error).toHaveBeenCalledWith( diff --git a/src/cli/memory-cli.ts b/src/cli/memory-cli.ts index 16520a49d..59a6b9417 100644 --- a/src/cli/memory-cli.ts +++ b/src/cli/memory-cli.ts @@ -22,6 +22,7 @@ type MemoryCommandOptions = { json?: boolean; deep?: boolean; index?: boolean; + force?: boolean; verbose?: boolean; }; @@ -287,7 +288,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { try { await syncFn({ reason: "cli", - force: true, + force: Boolean(opts.force), progress: (syncUpdate) => { update({ completed: syncUpdate.completed, @@ -495,7 +496,7 @@ export function registerMemoryCli(program: Command) { .option("--deep", "Probe embedding provider availability") .option("--index", "Reindex if dirty (implies --deep)") .option("--verbose", "Verbose logging", false) - .action(async (opts: MemoryCommandOptions) => { + .action(async (opts: MemoryCommandOptions & { force?: boolean }) => { await runMemoryStatus(opts); }); @@ -503,6 +504,7 @@ export function registerMemoryCli(program: Command) { .command("index") .description("Reindex memory files") .option("--agent ", "Agent id (default: default agent)") + .option("--force", "Force full reindex", false) .option("--verbose", "Verbose logging", false) .action(async (opts: MemoryCommandOptions) => { setVerbose(Boolean(opts.verbose)); @@ -605,7 +607,7 @@ export function registerMemoryCli(program: Command) { try { await syncFn({ reason: "cli", - force: true, + force: Boolean(opts.force), progress: (syncUpdate) => { if (syncUpdate.label) { lastLabel = syncUpdate.label; diff --git a/src/config/schema.ts b/src/config/schema.ts index edc203d3d..61708c9d0 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -269,6 +269,7 @@ const FIELD_LABELS: Record = { "memory.qmd.update.interval": "QMD Update Interval", "memory.qmd.update.debounceMs": "QMD Update Debounce (ms)", "memory.qmd.update.onBoot": "QMD Update on Startup", + "memory.qmd.update.embedInterval": "QMD Embed Interval", "memory.qmd.limits.maxResults": "QMD Max Results", "memory.qmd.limits.maxSnippetChars": "QMD Max Snippet Chars", "memory.qmd.limits.maxInjectedChars": "QMD Max Injected Chars", @@ -591,6 +592,8 @@ const FIELD_HELP: Record = { "memory.qmd.update.debounceMs": "Minimum delay between successive QMD refresh runs (default: 15000).", "memory.qmd.update.onBoot": "Run QMD update once on gateway startup (default: true).", + "memory.qmd.update.embedInterval": + "How often QMD embeddings are refreshed (duration string, default: 60m). Set to 0 to disable periodic embed.", "memory.qmd.limits.maxResults": "Max QMD results returned to the agent loop (default: 6).", "memory.qmd.limits.maxSnippetChars": "Max characters per snippet pulled from QMD (default: 700).", "memory.qmd.limits.maxInjectedChars": "Max total characters injected from QMD hits per turn.", diff --git a/src/config/types.memory.ts b/src/config/types.memory.ts index dad2f9784..2fc185200 100644 --- a/src/config/types.memory.ts +++ b/src/config/types.memory.ts @@ -35,6 +35,7 @@ export type MemoryQmdUpdateConfig = { interval?: string; debounceMs?: number; onBoot?: boolean; + embedInterval?: string; }; export type MemoryQmdLimitsConfig = { diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index 1fb4c923c..af6668979 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -53,6 +53,7 @@ const MemoryQmdUpdateSchema = z interval: z.string().optional(), debounceMs: z.number().int().nonnegative().optional(), onBoot: z.boolean().optional(), + embedInterval: z.string().optional(), }) .strict(); diff --git a/src/memory/backend-config.ts b/src/memory/backend-config.ts index 15851ef29..2510d16c3 100644 --- a/src/memory/backend-config.ts +++ b/src/memory/backend-config.ts @@ -29,6 +29,7 @@ export type ResolvedQmdUpdateConfig = { intervalMs: number; debounceMs: number; onBoot: boolean; + embedIntervalMs: number; }; export type ResolvedQmdLimitsConfig = { @@ -59,6 +60,7 @@ const DEFAULT_CITATIONS: MemoryCitationsMode = "auto"; const DEFAULT_QMD_INTERVAL = "5m"; const DEFAULT_QMD_DEBOUNCE_MS = 15_000; const DEFAULT_QMD_TIMEOUT_MS = 4_000; +const DEFAULT_QMD_EMBED_INTERVAL = "60m"; const DEFAULT_QMD_LIMITS: ResolvedQmdLimitsConfig = { maxResults: 6, maxSnippetChars: 700, @@ -115,6 +117,16 @@ function resolveIntervalMs(raw: string | undefined): number { } } +function resolveEmbedIntervalMs(raw: string | undefined): number { + const value = raw?.trim(); + if (!value) return parseDurationMs(DEFAULT_QMD_EMBED_INTERVAL, { defaultUnit: "m" }); + try { + return parseDurationMs(value, { defaultUnit: "m" }); + } catch { + return parseDurationMs(DEFAULT_QMD_EMBED_INTERVAL, { defaultUnit: "m" }); + } +} + function resolveDebounceMs(raw: number | undefined): number { if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) { return Math.floor(raw); @@ -221,7 +233,7 @@ export function resolveMemoryBackendConfig(params: { ]; const resolved: ResolvedQmdConfig = { - command: qmdCfg?.command?.trim() || "qmd", + command: (qmdCfg?.command?.trim() || "qmd").split(/\s+/)[0] || "qmd", collections, includeDefaultMemory, sessions: resolveSessionConfig(qmdCfg?.sessions, workspaceDir), @@ -229,6 +241,7 @@ export function resolveMemoryBackendConfig(params: { intervalMs: resolveIntervalMs(qmdCfg?.update?.interval), debounceMs: resolveDebounceMs(qmdCfg?.update?.debounceMs), onBoot: qmdCfg?.update?.onBoot !== false, + embedIntervalMs: resolveEmbedIntervalMs(qmdCfg?.update?.embedInterval), }, limits: resolveLimits(qmdCfg?.limits), scope: qmdCfg?.scope ?? DEFAULT_QMD_SCOPE, diff --git a/src/memory/qmd-manager.test.ts b/src/memory/qmd-manager.test.ts index 4fcbb0c9f..c91a4dc4e 100644 --- a/src/memory/qmd-manager.test.ts +++ b/src/memory/qmd-manager.test.ts @@ -90,7 +90,8 @@ describe("QmdMemoryManager", () => { Date.now() - (resolved.qmd?.update.debounceMs ?? 0) - 10; await manager.sync({ reason: "after-wait" }); - expect(spawnMock.mock.calls.length).toBe(baselineCalls + 4); + // By default we refresh embeddings less frequently than index updates. + expect(spawnMock.mock.calls.length).toBe(baselineCalls + 3); await manager.close(); }); diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts index 003a6ca36..c984aa238 100644 --- a/src/memory/qmd-manager.ts +++ b/src/memory/qmd-manager.ts @@ -84,6 +84,7 @@ export class QmdMemoryManager implements MemorySearchManager { private closed = false; private db: SqliteDatabase | null = null; private lastUpdateAt: number | null = null; + private lastEmbedAt: number | null = null; private constructor(params: { cfg: MoltbotConfig; @@ -165,9 +166,27 @@ export class QmdMemoryManager implements MemorySearchManager { private async ensureCollections(): Promise { // QMD collections are persisted inside the index database and must be created - // via the CLI. The YAML file format is not supported by the QMD builds we - // target, so we ensure collections exist by running `qmd collection add`. + // via the CLI. Prefer listing existing collections when supported, otherwise + // fall back to best-effort idempotent `qmd collection add`. + const existing = new Set(); + try { + const result = await this.runQmd(["collection", "list", "--json"]); + const parsed = JSON.parse(result.stdout) as unknown; + if (Array.isArray(parsed)) { + for (const entry of parsed) { + if (typeof entry === "string") existing.add(entry); + else if (entry && typeof entry === "object") { + const name = (entry as { name?: unknown }).name; + if (typeof name === "string") existing.add(name); + } + } + } + } catch { + // ignore; older qmd versions might not support list --json. + } + for (const collection of this.qmd.collections) { + if (existing.has(collection.name)) continue; try { await this.runQmd([ "collection", @@ -181,7 +200,8 @@ export class QmdMemoryManager implements MemorySearchManager { } catch (err) { const message = err instanceof Error ? err.message : String(err); // Idempotency: qmd exits non-zero if the collection name already exists. - if (message.includes("already exists")) continue; + if (message.toLowerCase().includes("already exists")) continue; + if (message.toLowerCase().includes("exists")) continue; log.warn(`qmd collection add failed for ${collection.name}: ${message}`); } } @@ -335,10 +355,18 @@ export class QmdMemoryManager implements MemorySearchManager { await this.exportSessions(); } await this.runQmd(["update"], { timeoutMs: 120_000 }); - try { - await this.runQmd(["embed"], { timeoutMs: 120_000 }); - } catch (err) { - log.warn(`qmd embed failed (${reason}): ${String(err)}`); + const embedIntervalMs = this.qmd.update.embedIntervalMs; + const shouldEmbed = + Boolean(force) || + this.lastEmbedAt === null || + (embedIntervalMs > 0 && Date.now() - this.lastEmbedAt > embedIntervalMs); + if (shouldEmbed) { + try { + await this.runQmd(["embed"], { timeoutMs: 120_000 }); + this.lastEmbedAt = Date.now(); + } catch (err) { + log.warn(`qmd embed failed (${reason}): ${String(err)}`); + } } this.lastUpdateAt = Date.now(); this.docPathCache.clear(); diff --git a/src/memory/session-files.ts b/src/memory/session-files.ts index 1823e9669..195424a9e 100644 --- a/src/memory/session-files.ts +++ b/src/memory/session-files.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; +import { redactSensitiveText } from "../logging/redact.js"; import { hashText } from "./internal.js"; const log = createSubsystemLogger("memory"); @@ -104,8 +105,9 @@ export async function buildSessionEntry(absPath: string): Promise