feat(memory): add explicit paths config for memory search
Add a `paths` option to `memorySearch` config, allowing users to explicitly specify additional directories or files to include in memory search. Follow-up to #2961 as suggested by @gumadeiras — instead of auto-following symlinks (which has security implications), users can now explicitly declare additional search paths. - Add `memorySearch.paths` config option (array of strings) - Paths can be absolute or relative (resolved from workspace) - Directories are recursively scanned for `.md` files - Single `.md` files can also be specified - Paths from defaults and agent overrides are merged - Added 4 test cases for listMemoryFiles
This commit is contained in:
committed by
Gustavo Madeira Santana
parent
b717724275
commit
0fd9d3abd1
13237
package-lock.json
generated
Normal file
13237
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
@@ -9,6 +9,7 @@ import { resolveAgentConfig } from "./agent-scope.js";
|
||||
export type ResolvedMemorySearchConfig = {
|
||||
enabled: boolean;
|
||||
sources: Array<"memory" | "sessions">;
|
||||
paths: string[];
|
||||
provider: "openai" | "local" | "gemini" | "auto";
|
||||
remote?: {
|
||||
baseUrl?: string;
|
||||
@@ -162,6 +163,9 @@ function mergeConfig(
|
||||
modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir,
|
||||
};
|
||||
const sources = normalizeSources(overrides?.sources ?? defaults?.sources, sessionMemory);
|
||||
// Merge paths from defaults and overrides (both arrays combined, deduped)
|
||||
const pathsSet = new Set<string>([...(defaults?.paths ?? []), ...(overrides?.paths ?? [])]);
|
||||
const paths = Array.from(pathsSet);
|
||||
const vector = {
|
||||
enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true,
|
||||
extensionPath:
|
||||
@@ -236,6 +240,7 @@ function mergeConfig(
|
||||
return {
|
||||
enabled,
|
||||
sources,
|
||||
paths,
|
||||
provider,
|
||||
remote,
|
||||
experimental: {
|
||||
|
||||
@@ -222,6 +222,7 @@ const FIELD_LABELS: Record<string, string> = {
|
||||
"agents.defaults.memorySearch": "Memory Search",
|
||||
"agents.defaults.memorySearch.enabled": "Enable Memory Search",
|
||||
"agents.defaults.memorySearch.sources": "Memory Search Sources",
|
||||
"agents.defaults.memorySearch.paths": "Additional Memory Paths",
|
||||
"agents.defaults.memorySearch.experimental.sessionMemory":
|
||||
"Memory Search Session Index (Experimental)",
|
||||
"agents.defaults.memorySearch.provider": "Memory Search Provider",
|
||||
@@ -499,6 +500,8 @@ const FIELD_HELP: Record<string, string> = {
|
||||
"Vector search over MEMORY.md and memory/*.md (per-agent overrides supported).",
|
||||
"agents.defaults.memorySearch.sources":
|
||||
'Sources to index for memory search (default: ["memory"]; add "sessions" to include session transcripts).',
|
||||
"agents.defaults.memorySearch.paths":
|
||||
"Additional paths to include in memory search (directories or .md files; relative paths resolved from workspace).",
|
||||
"agents.defaults.memorySearch.experimental.sessionMemory":
|
||||
"Enable experimental session transcript indexing for memory search (default: false).",
|
||||
"agents.defaults.memorySearch.provider": 'Embedding provider ("openai", "gemini", or "local").',
|
||||
|
||||
@@ -226,6 +226,8 @@ export type MemorySearchConfig = {
|
||||
enabled?: boolean;
|
||||
/** Sources to index and search (default: ["memory"]). */
|
||||
sources?: Array<"memory" | "sessions">;
|
||||
/** Additional paths to include in memory search (directories or .md files). */
|
||||
paths?: string[];
|
||||
/** Experimental memory search settings. */
|
||||
experimental?: {
|
||||
/** Enable session transcript indexing (experimental, default: false). */
|
||||
|
||||
@@ -304,6 +304,7 @@ export const MemorySearchSchema = z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
sources: z.array(z.union([z.literal("memory"), z.literal("sessions")])).optional(),
|
||||
paths: z.array(z.string()).optional(),
|
||||
experimental: z
|
||||
.object({
|
||||
sessionMemory: z.boolean().optional(),
|
||||
|
||||
@@ -1,6 +1,70 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { chunkMarkdown } from "./internal.js";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { chunkMarkdown, listMemoryFiles } from "./internal.js";
|
||||
|
||||
describe("listMemoryFiles", () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-test-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("includes files from additional paths (directory)", async () => {
|
||||
// Create default memory file
|
||||
await fs.writeFile(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
|
||||
// Create additional directory with files
|
||||
const extraDir = path.join(tmpDir, "extra-notes");
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
await fs.writeFile(path.join(extraDir, "note1.md"), "# Note 1");
|
||||
await fs.writeFile(path.join(extraDir, "note2.md"), "# Note 2");
|
||||
await fs.writeFile(path.join(extraDir, "ignore.txt"), "Not a markdown file");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, [extraDir]);
|
||||
expect(files).toHaveLength(3); // MEMORY.md + 2 notes
|
||||
expect(files.some((f) => f.endsWith("MEMORY.md"))).toBe(true);
|
||||
expect(files.some((f) => f.endsWith("note1.md"))).toBe(true);
|
||||
expect(files.some((f) => f.endsWith("note2.md"))).toBe(true);
|
||||
expect(files.some((f) => f.endsWith("ignore.txt"))).toBe(false);
|
||||
});
|
||||
|
||||
it("includes files from additional paths (single file)", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const singleFile = path.join(tmpDir, "standalone.md");
|
||||
await fs.writeFile(singleFile, "# Standalone");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, [singleFile]);
|
||||
expect(files).toHaveLength(2);
|
||||
expect(files.some((f) => f.endsWith("standalone.md"))).toBe(true);
|
||||
});
|
||||
|
||||
it("handles relative paths in additional paths", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const extraDir = path.join(tmpDir, "subdir");
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
await fs.writeFile(path.join(extraDir, "nested.md"), "# Nested");
|
||||
|
||||
// Use relative path
|
||||
const files = await listMemoryFiles(tmpDir, ["subdir"]);
|
||||
expect(files).toHaveLength(2);
|
||||
expect(files.some((f) => f.endsWith("nested.md"))).toBe(true);
|
||||
});
|
||||
|
||||
it("ignores non-existent additional paths", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, ["/does/not/exist"]);
|
||||
expect(files).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("chunkMarkdown", () => {
|
||||
it("splits overly long lines into max-sized chunks", () => {
|
||||
|
||||
@@ -60,7 +60,10 @@ async function walkDir(dir: string, files: string[]) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function listMemoryFiles(workspaceDir: string): Promise<string[]> {
|
||||
export async function listMemoryFiles(
|
||||
workspaceDir: string,
|
||||
additionalPaths?: string[],
|
||||
): Promise<string[]> {
|
||||
const result: string[] = [];
|
||||
const memoryFile = path.join(workspaceDir, "MEMORY.md");
|
||||
const altMemoryFile = path.join(workspaceDir, "memory.md");
|
||||
@@ -70,6 +73,19 @@ export async function listMemoryFiles(workspaceDir: string): Promise<string[]> {
|
||||
if (await exists(memoryDir)) {
|
||||
await walkDir(memoryDir, result);
|
||||
}
|
||||
// Include files from additional explicit paths
|
||||
if (additionalPaths && additionalPaths.length > 0) {
|
||||
for (const p of additionalPaths) {
|
||||
const resolved = path.isAbsolute(p) ? p : path.resolve(workspaceDir, p);
|
||||
if (!(await exists(resolved))) continue;
|
||||
const stat = await fs.stat(resolved);
|
||||
if (stat.isDirectory()) {
|
||||
await walkDir(resolved, result);
|
||||
} else if (stat.isFile() && resolved.endsWith(".md")) {
|
||||
result.push(resolved);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (result.length <= 1) return result;
|
||||
const seen = new Set<string>();
|
||||
const deduped: string[] = [];
|
||||
|
||||
@@ -975,7 +975,7 @@ export class MemoryIndexManager {
|
||||
needsFullReindex: boolean;
|
||||
progress?: MemorySyncProgressState;
|
||||
}) {
|
||||
const files = await listMemoryFiles(this.workspaceDir);
|
||||
const files = await listMemoryFiles(this.workspaceDir, this.settings.paths);
|
||||
const fileEntries = await Promise.all(
|
||||
files.map(async (file) => buildFileEntry(file, this.workspaceDir)),
|
||||
);
|
||||
|
||||
@@ -14,6 +14,7 @@ type ProgressState = {
|
||||
|
||||
export async function syncMemoryFiles(params: {
|
||||
workspaceDir: string;
|
||||
additionalPaths?: string[];
|
||||
db: DatabaseSync;
|
||||
needsFullReindex: boolean;
|
||||
progress?: ProgressState;
|
||||
@@ -27,7 +28,7 @@ export async function syncMemoryFiles(params: {
|
||||
ftsAvailable: boolean;
|
||||
model: string;
|
||||
}) {
|
||||
const files = await listMemoryFiles(params.workspaceDir);
|
||||
const files = await listMemoryFiles(params.workspaceDir, params.additionalPaths);
|
||||
const fileEntries = await Promise.all(
|
||||
files.map(async (file) => buildFileEntry(file, params.workspaceDir)),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user