Fix build regressions after merge
This commit is contained in:
committed by
Vignesh
parent
465536e811
commit
23cfcd60df
@@ -87,7 +87,7 @@ const MemorySchema = z
|
||||
.strict()
|
||||
.optional();
|
||||
|
||||
export const MoltbotSchema = z
|
||||
export const OpenClawSchema = z
|
||||
.object({
|
||||
meta: z
|
||||
.object({
|
||||
|
||||
@@ -1,50 +1,11 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import chokidar, { type FSWatcher } from "chokidar";
|
||||
|
||||
import { resolveAgentDir, resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { ResolvedMemorySearchConfig } from "../agents/memory-search.js";
|
||||
import { resolveMemorySearchConfig } from "../agents/memory-search.js";
|
||||
import type { MoltbotConfig } from "../config/config.js";
|
||||
import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { onSessionTranscriptUpdate } from "../sessions/transcript-events.js";
|
||||
import { resolveUserPath } from "../utils.js";
|
||||
import {
|
||||
createEmbeddingProvider,
|
||||
type EmbeddingProvider,
|
||||
type EmbeddingProviderResult,
|
||||
type GeminiEmbeddingClient,
|
||||
type OpenAiEmbeddingClient,
|
||||
} from "./embeddings.js";
|
||||
import { DEFAULT_GEMINI_EMBEDDING_MODEL } from "./embeddings-gemini.js";
|
||||
import { DEFAULT_OPENAI_EMBEDDING_MODEL } from "./embeddings-openai.js";
|
||||
import {
|
||||
OPENAI_BATCH_ENDPOINT,
|
||||
type OpenAiBatchRequest,
|
||||
runOpenAiEmbeddingBatches,
|
||||
} from "./batch-openai.js";
|
||||
import { runGeminiEmbeddingBatches, type GeminiBatchRequest } from "./batch-gemini.js";
|
||||
import {
|
||||
buildFileEntry,
|
||||
chunkMarkdown,
|
||||
ensureDir,
|
||||
hashText,
|
||||
isMemoryPath,
|
||||
listMemoryFiles,
|
||||
type MemoryChunk,
|
||||
type MemoryFileEntry,
|
||||
normalizeRelPath,
|
||||
parseEmbedding,
|
||||
} from "./internal.js";
|
||||
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js";
|
||||
import { searchKeyword, searchVector } from "./manager-search.js";
|
||||
import { ensureMemoryIndexSchema } from "./memory-schema.js";
|
||||
import { requireNodeSqlite } from "./sqlite.js";
|
||||
import { loadSqliteVecExtension } from "./sqlite-vec.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import type {
|
||||
MemoryEmbeddingProbeResult,
|
||||
MemoryProviderStatus,
|
||||
@@ -53,6 +14,44 @@ import type {
|
||||
MemorySource,
|
||||
MemorySyncProgressUpdate,
|
||||
} from "./types.js";
|
||||
import { resolveAgentDir, resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
|
||||
import { resolveMemorySearchConfig } from "../agents/memory-search.js";
|
||||
import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { onSessionTranscriptUpdate } from "../sessions/transcript-events.js";
|
||||
import { resolveUserPath } from "../utils.js";
|
||||
import { runGeminiEmbeddingBatches, type GeminiBatchRequest } from "./batch-gemini.js";
|
||||
import {
|
||||
OPENAI_BATCH_ENDPOINT,
|
||||
type OpenAiBatchRequest,
|
||||
runOpenAiEmbeddingBatches,
|
||||
} from "./batch-openai.js";
|
||||
import { DEFAULT_GEMINI_EMBEDDING_MODEL } from "./embeddings-gemini.js";
|
||||
import { DEFAULT_OPENAI_EMBEDDING_MODEL } from "./embeddings-openai.js";
|
||||
import {
|
||||
createEmbeddingProvider,
|
||||
type EmbeddingProvider,
|
||||
type EmbeddingProviderResult,
|
||||
type GeminiEmbeddingClient,
|
||||
type OpenAiEmbeddingClient,
|
||||
} from "./embeddings.js";
|
||||
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js";
|
||||
import {
|
||||
buildFileEntry,
|
||||
chunkMarkdown,
|
||||
ensureDir,
|
||||
hashText,
|
||||
isMemoryPath,
|
||||
listMemoryFiles,
|
||||
normalizeExtraMemoryPaths,
|
||||
type MemoryChunk,
|
||||
type MemoryFileEntry,
|
||||
parseEmbedding,
|
||||
} from "./internal.js";
|
||||
import { searchKeyword, searchVector } from "./manager-search.js";
|
||||
import { ensureMemoryIndexSchema } from "./memory-schema.js";
|
||||
import { loadSqliteVecExtension } from "./sqlite-vec.js";
|
||||
import { requireNodeSqlite } from "./sqlite.js";
|
||||
|
||||
type MemoryIndexMeta = {
|
||||
model: string;
|
||||
@@ -108,7 +107,7 @@ const vectorToBlob = (embedding: number[]): Buffer =>
|
||||
|
||||
export class MemoryIndexManager implements MemorySearchManager {
|
||||
private readonly cacheKey: string;
|
||||
private readonly cfg: MoltbotConfig;
|
||||
private readonly cfg: OpenClawConfig;
|
||||
private readonly agentId: string;
|
||||
private readonly workspaceDir: string;
|
||||
private readonly settings: ResolvedMemorySearchConfig;
|
||||
@@ -164,16 +163,20 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
private syncing: Promise<void> | null = null;
|
||||
|
||||
static async get(params: {
|
||||
cfg: MoltbotConfig;
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
}): Promise<MemoryIndexManager | null> {
|
||||
const { cfg, agentId } = params;
|
||||
const settings = resolveMemorySearchConfig(cfg, agentId);
|
||||
if (!settings) return null;
|
||||
if (!settings) {
|
||||
return null;
|
||||
}
|
||||
const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId);
|
||||
const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}`;
|
||||
const existing = INDEX_CACHE.get(key);
|
||||
if (existing) return existing;
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const providerResult = await createEmbeddingProvider({
|
||||
config: cfg,
|
||||
agentDir: resolveAgentDir(cfg, agentId),
|
||||
@@ -197,7 +200,7 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
|
||||
private constructor(params: {
|
||||
cacheKey: string;
|
||||
cfg: MoltbotConfig;
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
workspaceDir: string;
|
||||
settings: ResolvedMemorySearchConfig;
|
||||
@@ -240,13 +243,19 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
async warmSession(sessionKey?: string): Promise<void> {
|
||||
if (!this.settings.sync.onSessionStart) return;
|
||||
if (!this.settings.sync.onSessionStart) {
|
||||
return;
|
||||
}
|
||||
const key = sessionKey?.trim() || "";
|
||||
if (key && this.sessionWarm.has(key)) return;
|
||||
if (key && this.sessionWarm.has(key)) {
|
||||
return;
|
||||
}
|
||||
void this.sync({ reason: "session-start" }).catch((err) => {
|
||||
log.warn(`memory sync failed (session-start): ${String(err)}`);
|
||||
});
|
||||
if (key) this.sessionWarm.add(key);
|
||||
if (key) {
|
||||
this.sessionWarm.add(key);
|
||||
}
|
||||
}
|
||||
|
||||
async search(
|
||||
@@ -264,7 +273,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
});
|
||||
}
|
||||
const cleaned = query.trim();
|
||||
if (!cleaned) return [];
|
||||
if (!cleaned) {
|
||||
return [];
|
||||
}
|
||||
const minScore = opts?.minScore ?? this.settings.query.minScore;
|
||||
const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
|
||||
const hybrid = this.settings.query.hybrid;
|
||||
@@ -323,7 +334,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
query: string,
|
||||
limit: number,
|
||||
): Promise<Array<MemorySearchResult & { id: string; textScore: number }>> {
|
||||
if (!this.fts.enabled || !this.fts.available) return [];
|
||||
if (!this.fts.enabled || !this.fts.available) {
|
||||
return [];
|
||||
}
|
||||
const sourceFilter = this.buildSourceFilter();
|
||||
const results = await searchKeyword({
|
||||
db: this.db,
|
||||
@@ -375,7 +388,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
force?: boolean;
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
}): Promise<void> {
|
||||
if (this.syncing) return this.syncing;
|
||||
if (this.syncing) {
|
||||
return this.syncing;
|
||||
}
|
||||
this.syncing = this.runSync(params).finally(() => {
|
||||
this.syncing = null;
|
||||
});
|
||||
@@ -387,13 +402,54 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
from?: number;
|
||||
lines?: number;
|
||||
}): Promise<{ text: string; path: string }> {
|
||||
const relPath = normalizeRelPath(params.relPath);
|
||||
if (!relPath || !isMemoryPath(relPath)) {
|
||||
const rawPath = params.relPath.trim();
|
||||
if (!rawPath) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const absPath = path.resolve(this.workspaceDir, relPath);
|
||||
if (!absPath.startsWith(this.workspaceDir)) {
|
||||
throw new Error("path escapes workspace");
|
||||
const absPath = path.isAbsolute(rawPath)
|
||||
? path.resolve(rawPath)
|
||||
: path.resolve(this.workspaceDir, rawPath);
|
||||
const relPath = path.relative(this.workspaceDir, absPath).replace(/\\/g, "/");
|
||||
const inWorkspace =
|
||||
relPath.length > 0 && !relPath.startsWith("..") && !path.isAbsolute(relPath);
|
||||
const allowedWorkspace = inWorkspace && isMemoryPath(relPath);
|
||||
let allowedAdditional = false;
|
||||
if (!allowedWorkspace && this.settings.extraPaths.length > 0) {
|
||||
const additionalPaths = normalizeExtraMemoryPaths(
|
||||
this.workspaceDir,
|
||||
this.settings.extraPaths,
|
||||
);
|
||||
for (const additionalPath of additionalPaths) {
|
||||
try {
|
||||
const stat = await fs.lstat(additionalPath);
|
||||
if (stat.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
if (stat.isDirectory()) {
|
||||
if (absPath === additionalPath || absPath.startsWith(`${additionalPath}${path.sep}`)) {
|
||||
allowedAdditional = true;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (stat.isFile()) {
|
||||
if (absPath === additionalPath && absPath.endsWith(".md")) {
|
||||
allowedAdditional = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
if (!allowedWorkspace && !allowedAdditional) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
if (!absPath.endsWith(".md")) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const stat = await fs.lstat(absPath);
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const content = await fs.readFile(absPath, "utf-8");
|
||||
if (!params.from && !params.lines) {
|
||||
@@ -420,7 +476,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
};
|
||||
const sourceCounts = (() => {
|
||||
const sources = Array.from(this.sources);
|
||||
if (sources.length === 0) return [];
|
||||
if (sources.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const bySource = new Map<MemorySource, { files: number; chunks: number }>();
|
||||
for (const source of sources) {
|
||||
bySource.set(source, { files: 0, chunks: 0 });
|
||||
@@ -445,7 +503,7 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
entry.chunks = row.c ?? 0;
|
||||
bySource.set(row.source, entry);
|
||||
}
|
||||
return sources.map((source) => ({ source, ...bySource.get(source)! }));
|
||||
return sources.map((source) => Object.assign({ source }, bySource.get(source)!));
|
||||
})();
|
||||
return {
|
||||
backend: "builtin",
|
||||
@@ -458,6 +516,7 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
model: this.provider.model,
|
||||
requestedProvider: this.requestedProvider,
|
||||
sources: Array.from(this.sources),
|
||||
extraPaths: this.settings.extraPaths,
|
||||
sourceCounts,
|
||||
cache: this.cache.enabled
|
||||
? {
|
||||
@@ -501,7 +560,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
async probeVectorAvailability(): Promise<boolean> {
|
||||
if (!this.vector.enabled) return false;
|
||||
if (!this.vector.enabled) {
|
||||
return false;
|
||||
}
|
||||
return this.ensureVectorReady();
|
||||
}
|
||||
|
||||
@@ -516,7 +577,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.closed) return;
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
this.closed = true;
|
||||
if (this.watchTimer) {
|
||||
clearTimeout(this.watchTimer);
|
||||
@@ -543,7 +606,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private async ensureVectorReady(dimensions?: number): Promise<boolean> {
|
||||
if (!this.vector.enabled) return false;
|
||||
if (!this.vector.enabled) {
|
||||
return false;
|
||||
}
|
||||
if (!this.vectorReady) {
|
||||
this.vectorReady = this.withTimeout(
|
||||
this.loadVectorExtension(),
|
||||
@@ -569,7 +634,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private async loadVectorExtension(): Promise<boolean> {
|
||||
if (this.vector.available !== null) return this.vector.available;
|
||||
if (this.vector.available !== null) {
|
||||
return this.vector.available;
|
||||
}
|
||||
if (!this.vector.enabled) {
|
||||
this.vector.available = false;
|
||||
return false;
|
||||
@@ -579,7 +646,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
? resolveUserPath(this.vector.extensionPath)
|
||||
: undefined;
|
||||
const loaded = await loadSqliteVecExtension({ db: this.db, extensionPath: resolvedPath });
|
||||
if (!loaded.ok) throw new Error(loaded.error ?? "unknown sqlite-vec load error");
|
||||
if (!loaded.ok) {
|
||||
throw new Error(loaded.error ?? "unknown sqlite-vec load error");
|
||||
}
|
||||
this.vector.extensionPath = loaded.extensionPath;
|
||||
this.vector.available = true;
|
||||
return true;
|
||||
@@ -593,7 +662,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private ensureVectorTable(dimensions: number): void {
|
||||
if (this.vector.dims === dimensions) return;
|
||||
if (this.vector.dims === dimensions) {
|
||||
return;
|
||||
}
|
||||
if (this.vector.dims && this.vector.dims !== dimensions) {
|
||||
this.dropVectorTable();
|
||||
}
|
||||
@@ -617,7 +688,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
|
||||
private buildSourceFilter(alias?: string): { sql: string; params: MemorySource[] } {
|
||||
const sources = Array.from(this.sources);
|
||||
if (sources.length === 0) return { sql: "", params: [] };
|
||||
if (sources.length === 0) {
|
||||
return { sql: "", params: [] };
|
||||
}
|
||||
const column = alias ? `${alias}.source` : "source";
|
||||
const placeholders = sources.map(() => "?").join(", ");
|
||||
return { sql: ` AND ${column} IN (${placeholders})`, params: sources };
|
||||
@@ -636,7 +709,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private seedEmbeddingCache(sourceDb: DatabaseSync): void {
|
||||
if (!this.cache.enabled) return;
|
||||
if (!this.cache.enabled) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const rows = sourceDb
|
||||
.prepare(
|
||||
@@ -651,7 +726,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
dims: number | null;
|
||||
updated_at: number;
|
||||
}>;
|
||||
if (!rows.length) return;
|
||||
if (!rows.length) {
|
||||
return;
|
||||
}
|
||||
const insert = this.db.prepare(
|
||||
`INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
@@ -728,12 +805,26 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private ensureWatcher() {
|
||||
if (!this.sources.has("memory") || !this.settings.sync.watch || this.watcher) return;
|
||||
const watchPaths = [
|
||||
if (!this.sources.has("memory") || !this.settings.sync.watch || this.watcher) {
|
||||
return;
|
||||
}
|
||||
const additionalPaths = normalizeExtraMemoryPaths(this.workspaceDir, this.settings.extraPaths)
|
||||
.map((entry) => {
|
||||
try {
|
||||
const stat = fsSync.lstatSync(entry);
|
||||
return stat.isSymbolicLink() ? null : entry;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((entry): entry is string => Boolean(entry));
|
||||
const watchPaths = new Set<string>([
|
||||
path.join(this.workspaceDir, "MEMORY.md"),
|
||||
path.join(this.workspaceDir, "memory.md"),
|
||||
path.join(this.workspaceDir, "memory"),
|
||||
];
|
||||
this.watcher = chokidar.watch(watchPaths, {
|
||||
...additionalPaths,
|
||||
]);
|
||||
this.watcher = chokidar.watch(Array.from(watchPaths), {
|
||||
ignoreInitial: true,
|
||||
awaitWriteFinish: {
|
||||
stabilityThreshold: this.settings.sync.watchDebounceMs,
|
||||
@@ -750,18 +841,26 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private ensureSessionListener() {
|
||||
if (!this.sources.has("sessions") || this.sessionUnsubscribe) return;
|
||||
if (!this.sources.has("sessions") || this.sessionUnsubscribe) {
|
||||
return;
|
||||
}
|
||||
this.sessionUnsubscribe = onSessionTranscriptUpdate((update) => {
|
||||
if (this.closed) return;
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
const sessionFile = update.sessionFile;
|
||||
if (!this.isSessionFileForAgent(sessionFile)) return;
|
||||
if (!this.isSessionFileForAgent(sessionFile)) {
|
||||
return;
|
||||
}
|
||||
this.scheduleSessionDirty(sessionFile);
|
||||
});
|
||||
}
|
||||
|
||||
private scheduleSessionDirty(sessionFile: string) {
|
||||
this.sessionPendingFiles.add(sessionFile);
|
||||
if (this.sessionWatchTimer) return;
|
||||
if (this.sessionWatchTimer) {
|
||||
return;
|
||||
}
|
||||
this.sessionWatchTimer = setTimeout(() => {
|
||||
this.sessionWatchTimer = null;
|
||||
void this.processSessionDeltaBatch().catch((err) => {
|
||||
@@ -771,13 +870,17 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private async processSessionDeltaBatch(): Promise<void> {
|
||||
if (this.sessionPendingFiles.size === 0) return;
|
||||
if (this.sessionPendingFiles.size === 0) {
|
||||
return;
|
||||
}
|
||||
const pending = Array.from(this.sessionPendingFiles);
|
||||
this.sessionPendingFiles.clear();
|
||||
let shouldSync = false;
|
||||
for (const sessionFile of pending) {
|
||||
const delta = await this.updateSessionDelta(sessionFile);
|
||||
if (!delta) continue;
|
||||
if (!delta) {
|
||||
continue;
|
||||
}
|
||||
const bytesThreshold = delta.deltaBytes;
|
||||
const messagesThreshold = delta.deltaMessages;
|
||||
const bytesHit =
|
||||
@@ -786,7 +889,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
messagesThreshold <= 0
|
||||
? delta.pendingMessages > 0
|
||||
: delta.pendingMessages >= messagesThreshold;
|
||||
if (!bytesHit && !messagesHit) continue;
|
||||
if (!bytesHit && !messagesHit) {
|
||||
continue;
|
||||
}
|
||||
this.sessionsDirtyFiles.add(sessionFile);
|
||||
this.sessionsDirty = true;
|
||||
delta.pendingBytes =
|
||||
@@ -809,7 +914,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
pendingMessages: number;
|
||||
} | null> {
|
||||
const thresholds = this.settings.sync.sessions;
|
||||
if (!thresholds) return null;
|
||||
if (!thresholds) {
|
||||
return null;
|
||||
}
|
||||
let stat: { size: number };
|
||||
try {
|
||||
stat = await fs.stat(sessionFile);
|
||||
@@ -860,7 +967,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private async countNewlines(absPath: string, start: number, end: number): Promise<number> {
|
||||
if (end <= start) return 0;
|
||||
if (end <= start) {
|
||||
return 0;
|
||||
}
|
||||
const handle = await fs.open(absPath, "r");
|
||||
try {
|
||||
let offset = start;
|
||||
@@ -869,9 +978,13 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
while (offset < end) {
|
||||
const toRead = Math.min(buffer.length, end - offset);
|
||||
const { bytesRead } = await handle.read(buffer, 0, toRead, offset);
|
||||
if (bytesRead <= 0) break;
|
||||
if (bytesRead <= 0) {
|
||||
break;
|
||||
}
|
||||
for (let i = 0; i < bytesRead; i += 1) {
|
||||
if (buffer[i] === 10) count += 1;
|
||||
if (buffer[i] === 10) {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
offset += bytesRead;
|
||||
}
|
||||
@@ -883,14 +996,18 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
|
||||
private resetSessionDelta(absPath: string, size: number): void {
|
||||
const state = this.sessionDeltas.get(absPath);
|
||||
if (!state) return;
|
||||
if (!state) {
|
||||
return;
|
||||
}
|
||||
state.lastSize = size;
|
||||
state.pendingBytes = 0;
|
||||
state.pendingMessages = 0;
|
||||
}
|
||||
|
||||
private isSessionFileForAgent(sessionFile: string): boolean {
|
||||
if (!sessionFile) return false;
|
||||
if (!sessionFile) {
|
||||
return false;
|
||||
}
|
||||
const sessionsDir = resolveSessionTranscriptsDirForAgent(this.agentId);
|
||||
const resolvedFile = path.resolve(sessionFile);
|
||||
const resolvedDir = path.resolve(sessionsDir);
|
||||
@@ -899,7 +1016,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
|
||||
private ensureIntervalSync() {
|
||||
const minutes = this.settings.sync.intervalMinutes;
|
||||
if (!minutes || minutes <= 0 || this.intervalTimer) return;
|
||||
if (!minutes || minutes <= 0 || this.intervalTimer) {
|
||||
return;
|
||||
}
|
||||
const ms = minutes * 60 * 1000;
|
||||
this.intervalTimer = setInterval(() => {
|
||||
void this.sync({ reason: "interval" }).catch((err) => {
|
||||
@@ -909,8 +1028,12 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private scheduleWatchSync() {
|
||||
if (!this.sources.has("memory") || !this.settings.sync.watch) return;
|
||||
if (this.watchTimer) clearTimeout(this.watchTimer);
|
||||
if (!this.sources.has("memory") || !this.settings.sync.watch) {
|
||||
return;
|
||||
}
|
||||
if (this.watchTimer) {
|
||||
clearTimeout(this.watchTimer);
|
||||
}
|
||||
this.watchTimer = setTimeout(() => {
|
||||
this.watchTimer = null;
|
||||
void this.sync({ reason: "watch" }).catch((err) => {
|
||||
@@ -923,11 +1046,19 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
params?: { reason?: string; force?: boolean },
|
||||
needsFullReindex = false,
|
||||
) {
|
||||
if (!this.sources.has("sessions")) return false;
|
||||
if (params?.force) return true;
|
||||
if (!this.sources.has("sessions")) {
|
||||
return false;
|
||||
}
|
||||
if (params?.force) {
|
||||
return true;
|
||||
}
|
||||
const reason = params?.reason;
|
||||
if (reason === "session-start" || reason === "watch") return false;
|
||||
if (needsFullReindex) return true;
|
||||
if (reason === "session-start" || reason === "watch") {
|
||||
return false;
|
||||
}
|
||||
if (needsFullReindex) {
|
||||
return true;
|
||||
}
|
||||
return this.sessionsDirty && this.sessionsDirtyFiles.size > 0;
|
||||
}
|
||||
|
||||
@@ -935,7 +1066,7 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
needsFullReindex: boolean;
|
||||
progress?: MemorySyncProgressState;
|
||||
}) {
|
||||
const files = await listMemoryFiles(this.workspaceDir);
|
||||
const files = await listMemoryFiles(this.workspaceDir, this.settings.extraPaths);
|
||||
const fileEntries = await Promise.all(
|
||||
files.map(async (file) => buildFileEntry(file, this.workspaceDir)),
|
||||
);
|
||||
@@ -984,7 +1115,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
.prepare(`SELECT path FROM files WHERE source = ?`)
|
||||
.all("memory") as Array<{ path: string }>;
|
||||
for (const stale of staleRows) {
|
||||
if (activePaths.has(stale.path)) continue;
|
||||
if (activePaths.has(stale.path)) {
|
||||
continue;
|
||||
}
|
||||
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory");
|
||||
try {
|
||||
this.db
|
||||
@@ -1079,7 +1212,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
.prepare(`SELECT path FROM files WHERE source = ?`)
|
||||
.all("sessions") as Array<{ path: string }>;
|
||||
for (const stale of staleRows) {
|
||||
if (activePaths.has(stale.path)) continue;
|
||||
if (activePaths.has(stale.path)) {
|
||||
continue;
|
||||
}
|
||||
this.db
|
||||
.prepare(`DELETE FROM files WHERE path = ? AND source = ?`)
|
||||
.run(stale.path, "sessions");
|
||||
@@ -1111,7 +1246,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
total: 0,
|
||||
label: undefined,
|
||||
report: (update) => {
|
||||
if (update.label) state.label = update.label;
|
||||
if (update.label) {
|
||||
state.label = update.label;
|
||||
}
|
||||
const label =
|
||||
update.total > 0 && state.label
|
||||
? `${state.label} ${update.completed}/${update.total}`
|
||||
@@ -1222,8 +1359,12 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
|
||||
private async activateFallbackProvider(reason: string): Promise<boolean> {
|
||||
const fallback = this.settings.fallback;
|
||||
if (!fallback || fallback === "none" || fallback === this.provider.id) return false;
|
||||
if (this.fallbackFrom) return false;
|
||||
if (!fallback || fallback === "none" || fallback === this.provider.id) {
|
||||
return false;
|
||||
}
|
||||
if (this.fallbackFrom) {
|
||||
return false;
|
||||
}
|
||||
const fallbackFrom = this.provider.id as "openai" | "gemini" | "local";
|
||||
|
||||
const fallbackModel =
|
||||
@@ -1375,7 +1516,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY) as
|
||||
| { value: string }
|
||||
| undefined;
|
||||
if (!row?.value) return null;
|
||||
if (!row?.value) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(row.value) as MemoryIndexMeta;
|
||||
} catch {
|
||||
@@ -1422,16 +1565,26 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
const normalized = this.normalizeSessionText(content);
|
||||
return normalized ? normalized : null;
|
||||
}
|
||||
if (!Array.isArray(content)) return null;
|
||||
if (!Array.isArray(content)) {
|
||||
return null;
|
||||
}
|
||||
const parts: string[] = [];
|
||||
for (const block of content) {
|
||||
if (!block || typeof block !== "object") continue;
|
||||
const record = block as { type?: unknown; text?: unknown };
|
||||
if (record.type !== "text" || typeof record.text !== "string") continue;
|
||||
const normalized = this.normalizeSessionText(record.text);
|
||||
if (normalized) parts.push(normalized);
|
||||
if (!block || typeof block !== "object") {
|
||||
continue;
|
||||
}
|
||||
const record = block as { type?: unknown; text?: unknown };
|
||||
if (record.type !== "text" || typeof record.text !== "string") {
|
||||
continue;
|
||||
}
|
||||
const normalized = this.normalizeSessionText(record.text);
|
||||
if (normalized) {
|
||||
parts.push(normalized);
|
||||
}
|
||||
}
|
||||
if (parts.length === 0) {
|
||||
return null;
|
||||
}
|
||||
if (parts.length === 0) return null;
|
||||
return parts.join(" ");
|
||||
}
|
||||
|
||||
@@ -1442,7 +1595,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
const lines = raw.split("\n");
|
||||
const collected: string[] = [];
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
if (!line.trim()) {
|
||||
continue;
|
||||
}
|
||||
let record: unknown;
|
||||
try {
|
||||
record = JSON.parse(line);
|
||||
@@ -1459,10 +1614,16 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
const message = (record as { message?: unknown }).message as
|
||||
| { role?: unknown; content?: unknown }
|
||||
| undefined;
|
||||
if (!message || typeof message.role !== "string") continue;
|
||||
if (message.role !== "user" && message.role !== "assistant") continue;
|
||||
if (!message || typeof message.role !== "string") {
|
||||
continue;
|
||||
}
|
||||
if (message.role !== "user" && message.role !== "assistant") {
|
||||
continue;
|
||||
}
|
||||
const text = this.extractSessionText(message.content);
|
||||
if (!text) continue;
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
const label = message.role === "user" ? "User" : "Assistant";
|
||||
collected.push(`${label}: ${text}`);
|
||||
}
|
||||
@@ -1482,7 +1643,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private estimateEmbeddingTokens(text: string): number {
|
||||
if (!text) return 0;
|
||||
if (!text) {
|
||||
return 0;
|
||||
}
|
||||
return Math.ceil(text.length / EMBEDDING_APPROX_CHARS_PER_TOKEN);
|
||||
}
|
||||
|
||||
@@ -1515,17 +1678,27 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private loadEmbeddingCache(hashes: string[]): Map<string, number[]> {
|
||||
if (!this.cache.enabled) return new Map();
|
||||
if (hashes.length === 0) return new Map();
|
||||
if (!this.cache.enabled) {
|
||||
return new Map();
|
||||
}
|
||||
if (hashes.length === 0) {
|
||||
return new Map();
|
||||
}
|
||||
const unique: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const hash of hashes) {
|
||||
if (!hash) continue;
|
||||
if (seen.has(hash)) continue;
|
||||
if (!hash) {
|
||||
continue;
|
||||
}
|
||||
if (seen.has(hash)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(hash);
|
||||
unique.push(hash);
|
||||
}
|
||||
if (unique.length === 0) return new Map();
|
||||
if (unique.length === 0) {
|
||||
return new Map();
|
||||
}
|
||||
|
||||
const out = new Map<string, number[]>();
|
||||
const baseParams = [this.provider.id, this.provider.model, this.providerKey];
|
||||
@@ -1547,8 +1720,12 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private upsertEmbeddingCache(entries: Array<{ hash: string; embedding: number[] }>): void {
|
||||
if (!this.cache.enabled) return;
|
||||
if (entries.length === 0) return;
|
||||
if (!this.cache.enabled) {
|
||||
return;
|
||||
}
|
||||
if (entries.length === 0) {
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
const stmt = this.db.prepare(
|
||||
`INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at)\n` +
|
||||
@@ -1573,14 +1750,20 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private pruneEmbeddingCacheIfNeeded(): void {
|
||||
if (!this.cache.enabled) return;
|
||||
if (!this.cache.enabled) {
|
||||
return;
|
||||
}
|
||||
const max = this.cache.maxEntries;
|
||||
if (!max || max <= 0) return;
|
||||
if (!max || max <= 0) {
|
||||
return;
|
||||
}
|
||||
const row = this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get() as
|
||||
| { c: number }
|
||||
| undefined;
|
||||
const count = row?.c ?? 0;
|
||||
if (count <= max) return;
|
||||
if (count <= max) {
|
||||
return;
|
||||
}
|
||||
const excess = count - max;
|
||||
this.db
|
||||
.prepare(
|
||||
@@ -1595,7 +1778,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private async embedChunksInBatches(chunks: MemoryChunk[]): Promise<number[][]> {
|
||||
if (chunks.length === 0) return [];
|
||||
if (chunks.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
|
||||
const embeddings: number[][] = Array.from({ length: chunks.length }, () => []);
|
||||
const missing: Array<{ index: number; chunk: MemoryChunk }> = [];
|
||||
@@ -1610,7 +1795,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
}
|
||||
|
||||
if (missing.length === 0) return embeddings;
|
||||
if (missing.length === 0) {
|
||||
return embeddings;
|
||||
}
|
||||
|
||||
const missingChunks = missing.map((m) => m.chunk);
|
||||
const batches = this.buildEmbeddingBatches(missingChunks);
|
||||
@@ -1636,7 +1823,7 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
if (this.provider.id === "openai" && this.openAi) {
|
||||
const entries = Object.entries(this.openAi.headers)
|
||||
.filter(([key]) => key.toLowerCase() !== "authorization")
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.toSorted(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, value]) => [key, value]);
|
||||
return hashText(
|
||||
JSON.stringify({
|
||||
@@ -1653,7 +1840,7 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
const lower = key.toLowerCase();
|
||||
return lower !== "authorization" && lower !== "x-goog-api-key";
|
||||
})
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.toSorted(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, value]) => [key, value]);
|
||||
return hashText(
|
||||
JSON.stringify({
|
||||
@@ -1690,7 +1877,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
if (!openAi) {
|
||||
return this.embedChunksInBatches(chunks);
|
||||
}
|
||||
if (chunks.length === 0) return [];
|
||||
if (chunks.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
|
||||
const embeddings: number[][] = Array.from({ length: chunks.length }, () => []);
|
||||
const missing: Array<{ index: number; chunk: MemoryChunk }> = [];
|
||||
@@ -1705,7 +1894,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
}
|
||||
|
||||
if (missing.length === 0) return embeddings;
|
||||
if (missing.length === 0) {
|
||||
return embeddings;
|
||||
}
|
||||
|
||||
const requests: OpenAiBatchRequest[] = [];
|
||||
const mapping = new Map<string, { index: number; hash: string }>();
|
||||
@@ -1740,13 +1931,17 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}),
|
||||
fallback: async () => await this.embedChunksInBatches(chunks),
|
||||
});
|
||||
if (Array.isArray(batchResult)) return batchResult;
|
||||
if (Array.isArray(batchResult)) {
|
||||
return batchResult;
|
||||
}
|
||||
const byCustomId = batchResult;
|
||||
|
||||
const toCache: Array<{ hash: string; embedding: number[] }> = [];
|
||||
for (const [customId, embedding] of byCustomId.entries()) {
|
||||
const mapped = mapping.get(customId);
|
||||
if (!mapped) continue;
|
||||
if (!mapped) {
|
||||
continue;
|
||||
}
|
||||
embeddings[mapped.index] = embedding;
|
||||
toCache.push({ hash: mapped.hash, embedding });
|
||||
}
|
||||
@@ -1763,7 +1958,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
if (!gemini) {
|
||||
return this.embedChunksInBatches(chunks);
|
||||
}
|
||||
if (chunks.length === 0) return [];
|
||||
if (chunks.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
|
||||
const embeddings: number[][] = Array.from({ length: chunks.length }, () => []);
|
||||
const missing: Array<{ index: number; chunk: MemoryChunk }> = [];
|
||||
@@ -1778,7 +1975,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
}
|
||||
|
||||
if (missing.length === 0) return embeddings;
|
||||
if (missing.length === 0) {
|
||||
return embeddings;
|
||||
}
|
||||
|
||||
const requests: GeminiBatchRequest[] = [];
|
||||
const mapping = new Map<string, { index: number; hash: string }>();
|
||||
@@ -1810,13 +2009,17 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}),
|
||||
fallback: async () => await this.embedChunksInBatches(chunks),
|
||||
});
|
||||
if (Array.isArray(batchResult)) return batchResult;
|
||||
if (Array.isArray(batchResult)) {
|
||||
return batchResult;
|
||||
}
|
||||
const byCustomId = batchResult;
|
||||
|
||||
const toCache: Array<{ hash: string; embedding: number[] }> = [];
|
||||
for (const [customId, embedding] of byCustomId.entries()) {
|
||||
const mapped = mapping.get(customId);
|
||||
if (!mapped) continue;
|
||||
if (!mapped) {
|
||||
continue;
|
||||
}
|
||||
embeddings[mapped.index] = embedding;
|
||||
toCache.push({ hash: mapped.hash, embedding });
|
||||
}
|
||||
@@ -1825,7 +2028,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private async embedBatchWithRetry(texts: string[]): Promise<number[][]> {
|
||||
if (texts.length === 0) return [];
|
||||
if (texts.length === 0) {
|
||||
return [];
|
||||
}
|
||||
let attempt = 0;
|
||||
let delayMs = EMBEDDING_RETRY_BASE_DELAY_MS;
|
||||
while (true) {
|
||||
@@ -1887,7 +2092,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
timeoutMs: number,
|
||||
message: string,
|
||||
): Promise<T> {
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return await promise;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
||||
return await promise;
|
||||
}
|
||||
let timer: NodeJS.Timeout | null = null;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
||||
@@ -1895,12 +2102,16 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
try {
|
||||
return (await Promise.race([promise, timeoutPromise])) as T;
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async runWithConcurrency<T>(tasks: Array<() => Promise<T>>, limit: number): Promise<T[]> {
|
||||
if (tasks.length === 0) return [];
|
||||
if (tasks.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
|
||||
const results: T[] = Array.from({ length: tasks.length });
|
||||
let next = 0;
|
||||
@@ -1908,10 +2119,14 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
|
||||
const workers = Array.from({ length: resolvedLimit }, async () => {
|
||||
while (true) {
|
||||
if (firstError) return;
|
||||
if (firstError) {
|
||||
return;
|
||||
}
|
||||
const index = next;
|
||||
next += 1;
|
||||
if (index >= tasks.length) return;
|
||||
if (index >= tasks.length) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
results[index] = await tasks[index]();
|
||||
} catch (err) {
|
||||
@@ -1922,7 +2137,9 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
});
|
||||
|
||||
await Promise.allSettled(workers);
|
||||
if (firstError) throw firstError;
|
||||
if (firstError) {
|
||||
throw firstError;
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ export type MemoryProviderStatus = {
|
||||
dirty?: boolean;
|
||||
workspaceDir?: string;
|
||||
dbPath?: string;
|
||||
extraPaths?: string[];
|
||||
sources?: MemorySource[];
|
||||
sourceCounts?: Array<{ source: MemorySource; files: number; chunks: number }>;
|
||||
cache?: { enabled: boolean; entries?: number; maxEntries?: number };
|
||||
|
||||
Reference in New Issue
Block a user