diff --git a/CHANGELOG.md b/CHANGELOG.md index bd28d771d..6e347b5c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Docs: https://docs.openclaw.ai - Config/web fetch: restore runtime validation for documented `tools.web.fetch.readability` and `tools.web.fetch.firecrawl` settings so valid web fetch configs no longer fail with unrecognized-key errors. (#42583) Thanks @stim64045-spec. - Signal/config validation: add `channels.signal.groups` schema support so per-group `requireMention`, `tools`, and `toolsBySender` overrides no longer get rejected during config validation. (#27199) Thanks @unisone. - Config/discovery: accept `discovery.wideArea.domain` in strict config validation so unicast DNS-SD gateway configs no longer fail with an unrecognized-key error. (#35615) Thanks @ingyukoh. +- Telegram/inbound media IPv4 fallback: retry SSRF-guarded Telegram file downloads once with the same IPv4 fallback policy as Bot API calls so fresh installs on IPv6-broken hosts no longer fail to download inbound images. - Security/exec approvals: unwrap more `pnpm` runtime forms during approval binding, including `pnpm --reporter ... exec` and direct `pnpm node` file runs, with matching regression coverage and docs updates. - Security/exec approvals: fail closed for Perl `-M` and `-I` approval flows so preload and load-path module resolution stays outside approval-backed runtime execution unless the operator uses a broader explicit trust path. - Security/exec approvals: recognize PowerShell `-File` and `-f` wrapper forms during inline-command extraction so approval and command-analysis paths treat file-based PowerShell launches like the existing `-Command` variants. diff --git a/src/media/fetch.telegram-network.test.ts b/src/media/fetch.telegram-network.test.ts index cb4cb1ab5..2fbb09e11 100644 --- a/src/media/fetch.telegram-network.test.ts +++ b/src/media/fetch.telegram-network.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { resolveTelegramTransport } from "../telegram/fetch.js"; +import { resolveTelegramTransport, shouldRetryTelegramIpv4Fallback } from "../telegram/fetch.js"; import { fetchRemoteMedia } from "./fetch.js"; const undiciMocks = vi.hoisted(() => { @@ -26,6 +26,12 @@ vi.mock("undici", () => ({ describe("fetchRemoteMedia telegram network policy", () => { type LookupFn = NonNullable[0]["lookupFn"]>; + function createTelegramFetchFailedError(code: string): Error { + return Object.assign(new TypeError("fetch failed"), { + cause: { code }, + }); + } + afterEach(() => { undiciMocks.fetch.mockReset(); undiciMocks.agentCtor.mockClear(); @@ -127,4 +133,116 @@ describe("fetchRemoteMedia telegram network policy", () => { expect(init?.dispatcher?.options?.uri).toBe("http://127.0.0.1:7890"); expect(undiciMocks.proxyAgentCtor).toHaveBeenCalled(); }); + + it("retries Telegram file downloads with IPv4 fallback when the first fetch fails", async () => { + const lookupFn = vi.fn(async () => [ + { address: "149.154.167.220", family: 4 }, + { address: "2001:67c:4e8:f004::9", family: 6 }, + ]) as unknown as LookupFn; + undiciFetch + .mockRejectedValueOnce(createTelegramFetchFailedError("EHOSTUNREACH")) + .mockResolvedValueOnce( + new Response(new Uint8Array([0xff, 0xd8, 0xff, 0x00]), { + status: 200, + headers: { "content-type": "image/jpeg" }, + }), + ); + + const telegramTransport = resolveTelegramTransport(undefined, { + network: { + autoSelectFamily: true, + dnsResultOrder: "ipv4first", + }, + }); + + await fetchRemoteMedia({ + url: "https://api.telegram.org/file/bottok/photos/2.jpg", + fetchImpl: telegramTransport.sourceFetch, + dispatcherPolicy: telegramTransport.pinnedDispatcherPolicy, + fallbackDispatcherPolicy: telegramTransport.fallbackPinnedDispatcherPolicy, + shouldRetryFetchError: shouldRetryTelegramIpv4Fallback, + lookupFn, + maxBytes: 1024, + ssrfPolicy: { + allowedHostnames: ["api.telegram.org"], + allowRfc2544BenchmarkRange: true, + }, + }); + + const firstInit = undiciFetch.mock.calls[0]?.[1] as + | (RequestInit & { + dispatcher?: { + options?: { + connect?: Record; + }; + }; + }) + | undefined; + const secondInit = undiciFetch.mock.calls[1]?.[1] as + | (RequestInit & { + dispatcher?: { + options?: { + connect?: Record; + }; + }; + }) + | undefined; + + expect(undiciFetch).toHaveBeenCalledTimes(2); + expect(firstInit?.dispatcher?.options?.connect).toEqual( + expect.objectContaining({ + autoSelectFamily: true, + autoSelectFamilyAttemptTimeout: 300, + lookup: expect.any(Function), + }), + ); + expect(secondInit?.dispatcher?.options?.connect).toEqual( + expect.objectContaining({ + family: 4, + autoSelectFamily: false, + lookup: expect.any(Function), + }), + ); + }); + + it("preserves both primary and fallback errors when Telegram media retry fails twice", async () => { + const lookupFn = vi.fn(async () => [ + { address: "149.154.167.220", family: 4 }, + { address: "2001:67c:4e8:f004::9", family: 6 }, + ]) as unknown as LookupFn; + const primaryError = createTelegramFetchFailedError("EHOSTUNREACH"); + const fallbackError = createTelegramFetchFailedError("ETIMEDOUT"); + undiciFetch.mockRejectedValueOnce(primaryError).mockRejectedValueOnce(fallbackError); + + const telegramTransport = resolveTelegramTransport(undefined, { + network: { + autoSelectFamily: true, + dnsResultOrder: "ipv4first", + }, + }); + + await expect( + fetchRemoteMedia({ + url: "https://api.telegram.org/file/bottok/photos/3.jpg", + fetchImpl: telegramTransport.sourceFetch, + dispatcherPolicy: telegramTransport.pinnedDispatcherPolicy, + fallbackDispatcherPolicy: telegramTransport.fallbackPinnedDispatcherPolicy, + shouldRetryFetchError: shouldRetryTelegramIpv4Fallback, + lookupFn, + maxBytes: 1024, + ssrfPolicy: { + allowedHostnames: ["api.telegram.org"], + allowRfc2544BenchmarkRange: true, + }, + }), + ).rejects.toMatchObject({ + name: "MediaFetchError", + code: "fetch_failed", + cause: expect.objectContaining({ + name: "Error", + cause: fallbackError, + primaryError, + }), + }); + }); }); diff --git a/src/media/fetch.test.ts b/src/media/fetch.test.ts index 00966e26a..4498ca4b5 100644 --- a/src/media/fetch.test.ts +++ b/src/media/fetch.test.ts @@ -25,13 +25,21 @@ function makeStallingFetch(firstChunk: Uint8Array) { }); } +function makeLookupFn() { + return vi.fn(async () => [{ address: "149.154.167.220", family: 4 }]) as unknown as NonNullable< + Parameters[0]["lookupFn"] + >; +} + describe("fetchRemoteMedia", () => { - type LookupFn = NonNullable[0]["lookupFn"]>; + const telegramToken = "123456789:ABCDEFGHIJKLMNOPQRSTUVWXYZabcd"; + const redactedTelegramToken = `${telegramToken.slice(0, 6)}…${telegramToken.slice(-4)}`; + const telegramFileUrl = `https://api.telegram.org/file/bot${telegramToken}/photos/1.jpg`; it("rejects when content-length exceeds maxBytes", async () => { const lookupFn = vi.fn(async () => [ { address: "93.184.216.34", family: 4 }, - ]) as unknown as LookupFn; + ]) as unknown as NonNullable[0]["lookupFn"]>; const fetchImpl = async () => new Response(makeStream([new Uint8Array([1, 2, 3, 4, 5])]), { status: 200, @@ -51,7 +59,7 @@ describe("fetchRemoteMedia", () => { it("rejects when streamed payload exceeds maxBytes", async () => { const lookupFn = vi.fn(async () => [ { address: "93.184.216.34", family: 4 }, - ]) as unknown as LookupFn; + ]) as unknown as NonNullable[0]["lookupFn"]>; const fetchImpl = async () => new Response(makeStream([new Uint8Array([1, 2, 3]), new Uint8Array([4, 5, 6])]), { status: 200, @@ -70,7 +78,7 @@ describe("fetchRemoteMedia", () => { it("aborts stalled body reads when idle timeout expires", async () => { const lookupFn = vi.fn(async () => [ { address: "93.184.216.34", family: 4 }, - ]) as unknown as LookupFn; + ]) as unknown as NonNullable[0]["lookupFn"]>; const fetchImpl = makeStallingFetch(new Uint8Array([1, 2])); await expect( @@ -87,6 +95,48 @@ describe("fetchRemoteMedia", () => { }); }, 5_000); + it("redacts Telegram bot tokens from fetch failure messages", async () => { + const fetchImpl = vi.fn(async () => { + throw new Error(`dial failed for ${telegramFileUrl}`); + }); + + const error = await fetchRemoteMedia({ + url: telegramFileUrl, + fetchImpl, + lookupFn: makeLookupFn(), + maxBytes: 1024, + ssrfPolicy: { + allowedHostnames: ["api.telegram.org"], + allowRfc2544BenchmarkRange: true, + }, + }).catch((err: unknown) => err as Error); + + expect(error).toBeInstanceOf(Error); + const errorText = error instanceof Error ? String(error) : ""; + expect(errorText).not.toContain(telegramToken); + expect(errorText).toContain(`bot${redactedTelegramToken}`); + }); + + it("redacts Telegram bot tokens from HTTP error messages", async () => { + const fetchImpl = vi.fn(async () => new Response("unauthorized", { status: 401 })); + + const error = await fetchRemoteMedia({ + url: telegramFileUrl, + fetchImpl, + lookupFn: makeLookupFn(), + maxBytes: 1024, + ssrfPolicy: { + allowedHostnames: ["api.telegram.org"], + allowRfc2544BenchmarkRange: true, + }, + }).catch((err: unknown) => err as Error); + + expect(error).toBeInstanceOf(Error); + const errorText = error instanceof Error ? String(error) : ""; + expect(errorText).not.toContain(telegramToken); + expect(errorText).toContain(`bot${redactedTelegramToken}`); + }); + it("blocks private IP literals before fetching", async () => { const fetchImpl = vi.fn(); await expect( diff --git a/src/media/fetch.ts b/src/media/fetch.ts index 40cd8b241..020ac8040 100644 --- a/src/media/fetch.ts +++ b/src/media/fetch.ts @@ -1,6 +1,8 @@ import path from "node:path"; +import { formatErrorMessage } from "../infra/errors.js"; import { fetchWithSsrFGuard, withStrictGuardedFetchMode } from "../infra/net/fetch-guard.js"; import type { LookupFn, PinnedDispatcherPolicy, SsrFPolicy } from "../infra/net/ssrf.js"; +import { redactSensitiveText } from "../logging/redact.js"; import { detectMime, extensionForMime } from "./mime.js"; import { readResponseWithLimit } from "./read-response-with-limit.js"; @@ -15,8 +17,8 @@ export type MediaFetchErrorCode = "max_bytes" | "http_error" | "fetch_failed"; export class MediaFetchError extends Error { readonly code: MediaFetchErrorCode; - constructor(code: MediaFetchErrorCode, message: string) { - super(message); + constructor(code: MediaFetchErrorCode, message: string, options?: { cause?: unknown }) { + super(message, options); this.code = code; this.name = "MediaFetchError"; } @@ -36,6 +38,8 @@ type FetchMediaOptions = { ssrfPolicy?: SsrFPolicy; lookupFn?: LookupFn; dispatcherPolicy?: PinnedDispatcherPolicy; + fallbackDispatcherPolicy?: PinnedDispatcherPolicy; + shouldRetryFetchError?: (error: unknown) => boolean; }; function stripQuotes(value: string): string { @@ -82,6 +86,10 @@ async function readErrorBodySnippet(res: Response, maxChars = 200): Promise { const { url, @@ -94,13 +102,16 @@ export async function fetchRemoteMedia(options: FetchMediaOptions): Promise Promise) | null = null; - try { - const result = await fetchWithSsrFGuard( + const runGuardedFetch = async (policy?: PinnedDispatcherPolicy) => + await fetchWithSsrFGuard( withStrictGuardedFetchMode({ url, fetchImpl, @@ -108,20 +119,50 @@ export async function fetchRemoteMedia(options: FetchMediaOptions): Promise maxBytes) { throw new MediaFetchError( "max_bytes", - `Failed to fetch media from ${url}: content length ${length} exceeds maxBytes ${maxBytes}`, + `Failed to fetch media from ${sourceUrl}: content length ${length} exceeds maxBytes ${maxBytes}`, ); } } @@ -155,7 +196,7 @@ export async function fetchRemoteMedia(options: FetchMediaOptions): Promise new MediaFetchError( "max_bytes", - `Failed to fetch media from ${res.url || url}: payload exceeds maxBytes ${maxBytes}`, + `Failed to fetch media from ${redactMediaUrl(res.url || url)}: payload exceeds maxBytes ${maxBytes}`, ), chunkTimeoutMs: readIdleTimeoutMs, }) @@ -166,7 +207,8 @@ export async function fetchRemoteMedia(options: FetchMediaOptions): Promise { if (!stickyIpv4Dispatcher) { - stickyIpv4Dispatcher = createTelegramDispatcher( - resolveTelegramDispatcherPolicy({ - autoSelectFamily: false, - dnsResultOrder: "ipv4first", - useEnvProxy: stickyShouldUseEnvProxy, - forceIpv4: true, - proxyUrl: explicitProxyUrl, - }).policy, - ).dispatcher; + if (!fallbackPinnedDispatcherPolicy) { + return defaultDispatcher.dispatcher; + } + stickyIpv4Dispatcher = createTelegramDispatcher(fallbackPinnedDispatcherPolicy).dispatcher; } return stickyIpv4Dispatcher; }; @@ -493,6 +502,7 @@ export function resolveTelegramTransport( fetch: resolvedFetch, sourceFetch, pinnedDispatcherPolicy: defaultDispatcher.effectivePolicy, + fallbackPinnedDispatcherPolicy, }; }