diff --git a/docs/development/CONFIG_FIELDS.md b/docs/development/CONFIG_FIELDS.md index 9a3ee4c..03f1a26 100644 --- a/docs/development/CONFIG_FIELDS.md +++ b/docs/development/CONFIG_FIELDS.md @@ -62,6 +62,7 @@ Used only for host plugin mode through the host runtime config file. | `retryAllAccountsRateLimited` | `true` | | `retryAllAccountsMaxWaitMs` | `0` | | `retryAllAccountsMaxRetries` | `Infinity` | +| `retryAllAccountsAbsoluteCeilingMs` | `0 ms (0–24h; 0 = unlimited)` | | `unsupportedCodexPolicy` | `strict` | | `fallbackOnUnsupportedCodexModel` | `false` | | `fallbackToGpt52OnUnsupportedGpt53` | `true` | @@ -193,6 +194,7 @@ Used only for host plugin mode through the host runtime config file. | `CODEX_TUI_V2` | Toggle TUI v2 | | `CODEX_TUI_COLOR_PROFILE` | TUI color profile | | `CODEX_TUI_GLYPHS` | TUI glyph mode | +| `CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS` | Absolute wait ceiling in ms for retry-all-on-rate-limit loop (`0–24h`, `0 = unlimited`) | | `CODEX_AUTH_FETCH_TIMEOUT_MS` | Request timeout override | | `CODEX_AUTH_STREAM_STALL_TIMEOUT_MS` | Stream stall timeout override | | `CODEX_MULTI_AUTH_SYNC_CODEX_CLI` | Toggle Codex CLI state sync | diff --git a/docs/reference/settings.md b/docs/reference/settings.md index 1466374..bb30b8d 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -86,6 +86,8 @@ Examples: - `retryAllAccountsRateLimited` - `retryAllAccountsMaxWaitMs` - `retryAllAccountsMaxRetries` +- `retryAllAccountsAbsoluteCeilingMs` + Unit: milliseconds. Bounds: `0` to `24h`. `0` means unlimited. ### Refresh and Recovery @@ -126,6 +128,8 @@ Common operator overrides: - `CODEX_TUI_V2` - `CODEX_TUI_COLOR_PROFILE` - `CODEX_TUI_GLYPHS` +- `CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS` + Rotation & Quota override for `retryAllAccountsAbsoluteCeilingMs` (ms, `0` to `24h`, `0` = unlimited). - `CODEX_AUTH_FETCH_TIMEOUT_MS` - `CODEX_AUTH_STREAM_STALL_TIMEOUT_MS` @@ -175,4 +179,4 @@ codex auth forecast --live - [commands.md](commands.md) - [storage-paths.md](storage-paths.md) -- [../configuration.md](../configuration.md) \ No newline at end of file +- [../configuration.md](../configuration.md) diff --git a/index.ts b/index.ts index 7db8808..f457df4 100644 --- a/index.ts +++ b/index.ts @@ -44,6 +44,7 @@ import { getFastSessionMaxInputItems, getRateLimitToastDebounceMs, getRetryAllAccountsMaxRetries, + getRetryAllAccountsAbsoluteCeilingMs, getRetryAllAccountsMaxWaitMs, getRetryAllAccountsRateLimited, getFallbackToGpt52OnUnsupportedGpt53, @@ -156,6 +157,10 @@ import { } from "./lib/request/rate-limit-backoff.js"; import { isEmptyResponse } from "./lib/request/response-handler.js"; import { addJitter } from "./lib/rotation.js"; +import { + decideRetryAllAccountsRateLimited, + type RetryAllAccountsRateLimitDecisionReason, +} from "./lib/request/retry-governor.js"; import { SessionAffinityStore } from "./lib/session-affinity.js"; import { LiveAccountSync } from "./lib/live-account-sync.js"; import { RefreshGuardian } from "./lib/refresh-guardian.js"; @@ -344,6 +349,9 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { streamFailoverAttempts: number; streamFailoverRecoveries: number; streamFailoverCrossAccountRecoveries: number; + retryGovernorStopsWaitExceedsMax: number; + retryGovernorStopsRetryLimitReached: number; + retryGovernorStopsAbsoluteCeilingExceeded: number; cumulativeLatencyMs: number; lastRequestAt: number | null; lastError: string | null; @@ -365,11 +373,32 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { streamFailoverAttempts: 0, streamFailoverRecoveries: 0, streamFailoverCrossAccountRecoveries: 0, + retryGovernorStopsWaitExceedsMax: 0, + retryGovernorStopsRetryLimitReached: 0, + retryGovernorStopsAbsoluteCeilingExceeded: 0, cumulativeLatencyMs: 0, lastRequestAt: null, lastError: null, }; + const recordRetryGovernorStopReason = ( + reason: RetryAllAccountsRateLimitDecisionReason, + ): void => { + switch (reason) { + case "wait-exceeds-max": + runtimeMetrics.retryGovernorStopsWaitExceedsMax += 1; + return; + case "retry-limit-reached": + runtimeMetrics.retryGovernorStopsRetryLimitReached += 1; + return; + case "absolute-ceiling-exceeded": + runtimeMetrics.retryGovernorStopsAbsoluteCeilingExceeded += 1; + return; + default: + return; + } + }; + type TokenSuccess = Extract; type TokenSuccessWithAccount = TokenSuccess & { accountIdOverride?: string; @@ -1124,6 +1153,8 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { const retryAllAccountsRateLimited = getRetryAllAccountsRateLimited(pluginConfig); const retryAllAccountsMaxWaitMs = getRetryAllAccountsMaxWaitMs(pluginConfig); const retryAllAccountsMaxRetries = getRetryAllAccountsMaxRetries(pluginConfig); + const retryAllAccountsAbsoluteCeilingMs = + getRetryAllAccountsAbsoluteCeilingMs(pluginConfig); const unsupportedCodexPolicy = getUnsupportedCodexPolicy(pluginConfig); const fallbackOnUnsupportedCodexModel = unsupportedCodexPolicy === "fallback"; const fallbackToGpt52OnUnsupportedGpt53 = @@ -1397,6 +1428,7 @@ export const OpenAIOAuthPlugin: Plugin = async ({ client }: PluginInput) => { }; let allRateLimitedRetries = 0; + let accumulatedAllRateLimitedWaitMs = 0; let emptyResponseRetries = 0; const attemptedUnsupportedFallbackModels = new Set(); if (model) { @@ -2368,20 +2400,53 @@ while (attempted.size < Math.max(1, accountCount)) { const waitMs = accountManager.getMinWaitTimeForFamily(modelFamily, model); const count = accountManager.getAccountCount(); + const jitteredWaitMs = waitMs > 0 ? addJitter(waitMs, 0.2) : 0; + const remainingCeilingMs = + retryAllAccountsAbsoluteCeilingMs > 0 + ? Math.max( + 0, + retryAllAccountsAbsoluteCeilingMs - accumulatedAllRateLimitedWaitMs, + ) + : Number.POSITIVE_INFINITY; + const plannedWaitMs = Math.min(jitteredWaitMs, remainingCeilingMs); + const retryDecision = decideRetryAllAccountsRateLimited({ + enabled: retryAllAccountsRateLimited, + accountCount: count, + waitMs, + plannedWaitMs, + maxWaitMs: retryAllAccountsMaxWaitMs, + currentRetryCount: allRateLimitedRetries, + maxRetries: retryAllAccountsMaxRetries, + accumulatedWaitMs: accumulatedAllRateLimitedWaitMs, + absoluteCeilingMs: retryAllAccountsAbsoluteCeilingMs, + }); - if ( - retryAllAccountsRateLimited && - count > 0 && - waitMs > 0 && - (retryAllAccountsMaxWaitMs === 0 || - waitMs <= retryAllAccountsMaxWaitMs) && - allRateLimitedRetries < retryAllAccountsMaxRetries - ) { + if (retryDecision.shouldRetry) { const countdownMessage = `All ${count} account(s) rate-limited. Waiting`; - await sleepWithCountdown(addJitter(waitMs, 0.2), countdownMessage); + await sleepWithCountdown(plannedWaitMs, countdownMessage); allRateLimitedRetries++; + accumulatedAllRateLimitedWaitMs += plannedWaitMs; continue; } + recordRetryGovernorStopReason(retryDecision.reason); + if ( + retryDecision.reason !== "disabled" && + retryDecision.reason !== "no-accounts" && + retryDecision.reason !== "no-wait" + ) { + logDebug("Retry governor blocked all-rate-limited retry", { + reason: retryDecision.reason, + accountCount: count, + waitMs, + jitteredWaitMs, + plannedWaitMs, + retryCount: allRateLimitedRetries, + accumulatedWaitMs: accumulatedAllRateLimitedWaitMs, + maxWaitMs: retryAllAccountsMaxWaitMs, + maxRetries: retryAllAccountsMaxRetries, + absoluteCeilingMs: retryAllAccountsAbsoluteCeilingMs, + }); + } const waitLabel = waitMs > 0 ? formatWaitTime(waitMs) : "a bit"; const message = @@ -3763,6 +3828,9 @@ while (attempted.size < Math.max(1, accountCount)) { `Stream failover attempts: ${runtimeMetrics.streamFailoverAttempts}`, `Stream failover recoveries: ${runtimeMetrics.streamFailoverRecoveries}`, `Stream failover cross-account recoveries: ${runtimeMetrics.streamFailoverCrossAccountRecoveries}`, + `Retry governor stops (wait>max): ${runtimeMetrics.retryGovernorStopsWaitExceedsMax}`, + `Retry governor stops (retry limit): ${runtimeMetrics.retryGovernorStopsRetryLimitReached}`, + `Retry governor stops (absolute ceiling): ${runtimeMetrics.retryGovernorStopsAbsoluteCeilingExceeded}`, `Empty-response retries: ${runtimeMetrics.emptyResponseRetries}`, `Session affinity entries: ${sessionAffinityEntries}`, `Live sync: ${liveSyncSnapshot?.running ? "on" : "off"} (${liveSyncSnapshot?.reloadCount ?? 0} reloads)`, @@ -3798,6 +3866,24 @@ while (attempted.size < Math.max(1, accountCount)) { String(runtimeMetrics.streamFailoverCrossAccountRecoveries), "accent", ), + formatUiKeyValue( + ui, + "Retry governor stops (wait>max)", + String(runtimeMetrics.retryGovernorStopsWaitExceedsMax), + "warning", + ), + formatUiKeyValue( + ui, + "Retry governor stops (retry limit)", + String(runtimeMetrics.retryGovernorStopsRetryLimitReached), + "warning", + ), + formatUiKeyValue( + ui, + "Retry governor stops (absolute ceiling)", + String(runtimeMetrics.retryGovernorStopsAbsoluteCeilingExceeded), + "warning", + ), formatUiKeyValue(ui, "Empty-response retries", String(runtimeMetrics.emptyResponseRetries), "warning"), formatUiKeyValue(ui, "Session affinity entries", String(sessionAffinityEntries), "muted"), formatUiKeyValue( diff --git a/lib/codex-manager/settings-hub.ts b/lib/codex-manager/settings-hub.ts index 99cbdae..77cb9f8 100644 --- a/lib/codex-manager/settings-hub.ts +++ b/lib/codex-manager/settings-hub.ts @@ -185,6 +185,7 @@ type BackendNumberSettingKey = | "proactiveRefreshBufferMs" | "parallelProbingMaxConcurrency" | "fastSessionMaxInputItems" + | "retryAllAccountsAbsoluteCeilingMs" | "networkErrorCooldownMs" | "serverErrorCooldownMs" | "fetchTimeoutMs" @@ -377,6 +378,15 @@ const BACKEND_NUMBER_OPTIONS: BackendNumberSettingOption[] = [ step: 2, unit: "count", }, + { + key: "retryAllAccountsAbsoluteCeilingMs", + label: "Retry-All Absolute Wait Ceiling", + description: "Total max wait for retry-all-on-rate-limit. Set 0 for unlimited.", + min: 0, + max: 24 * 60 * 60_000, + step: 30_000, + unit: "ms", + }, { key: "networkErrorCooldownMs", label: "Network Error Cooldown", @@ -486,6 +496,7 @@ const BACKEND_CATEGORY_OPTIONS: BackendCategoryOption[] = [ "preemptiveQuotaRemainingPercent5h", "preemptiveQuotaRemainingPercent7d", "preemptiveQuotaMaxDeferralMs", + "retryAllAccountsAbsoluteCeilingMs", ], }, { @@ -974,8 +985,20 @@ function buildBackendSettingsPreview( config.preemptiveQuotaRemainingPercent7d ?? BACKEND_DEFAULTS.preemptiveQuotaRemainingPercent7d ?? 5; + const retryAllAbsoluteCeilingMs = + config.retryAllAccountsAbsoluteCeilingMs ?? + BACKEND_DEFAULTS.retryAllAccountsAbsoluteCeilingMs ?? + 0; const fetchTimeout = config.fetchTimeoutMs ?? BACKEND_DEFAULTS.fetchTimeoutMs ?? 60_000; const stallTimeout = config.streamStallTimeoutMs ?? BACKEND_DEFAULTS.streamStallTimeoutMs ?? 45_000; + const retryAllAbsoluteCeilingOption = BACKEND_NUMBER_OPTION_BY_KEY.get( + "retryAllAccountsAbsoluteCeilingMs", + ); + const retryCeilingLabel = retryAllAbsoluteCeilingMs === 0 + ? "unlimited" + : retryAllAbsoluteCeilingOption + ? formatBackendNumberValue(retryAllAbsoluteCeilingOption, retryAllAbsoluteCeilingMs) + : `${retryAllAbsoluteCeilingMs}ms`; const fetchTimeoutOption = BACKEND_NUMBER_OPTION_BY_KEY.get("fetchTimeoutMs"); const stallTimeoutOption = BACKEND_NUMBER_OPTION_BY_KEY.get("streamStallTimeoutMs"); @@ -993,6 +1016,7 @@ function buildBackendSettingsPreview( const hint = [ `thresholds 5h<=${highlightIfFocused("preemptiveQuotaRemainingPercent5h", `${threshold5h}%`)}`, `7d<=${highlightIfFocused("preemptiveQuotaRemainingPercent7d", `${threshold7d}%`)}`, + `retry ceiling ${highlightIfFocused("retryAllAccountsAbsoluteCeilingMs", retryCeilingLabel)}`, `timeouts ${highlightIfFocused("fetchTimeoutMs", fetchTimeoutOption ? formatBackendNumberValue(fetchTimeoutOption, fetchTimeout) : `${fetchTimeout}ms`)}/${highlightIfFocused("streamStallTimeoutMs", stallTimeoutOption ? formatBackendNumberValue(stallTimeoutOption, stallTimeout) : `${stallTimeout}ms`)}`, ].join(" | "); @@ -1069,6 +1093,10 @@ function clampBackendNumberForTests(settingKey: string, value: number): number { return clampBackendNumber(option, value); } +function buildBackendSettingsPreviewForTests(config: PluginConfig): { label: string; hint: string } { + return buildBackendSettingsPreview(config, getUiRuntimeOptions()); +} + async function withQueuedRetryForTests( pathKey: string, task: () => Promise, @@ -1093,6 +1121,7 @@ async function persistBackendConfigSelectionForTests( const __testOnly = { clampBackendNumber: clampBackendNumberForTests, + buildBackendSettingsPreview: buildBackendSettingsPreviewForTests, formatMenuLayoutMode, cloneDashboardSettings, withQueuedRetry: withQueuedRetryForTests, diff --git a/lib/config.ts b/lib/config.ts index f9e7ecf..429e42b 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -125,6 +125,7 @@ export const DEFAULT_PLUGIN_CONFIG: PluginConfig = { retryAllAccountsRateLimited: true, retryAllAccountsMaxWaitMs: 0, retryAllAccountsMaxRetries: Infinity, + retryAllAccountsAbsoluteCeilingMs: 0, unsupportedCodexPolicy: "strict", fallbackOnUnsupportedCodexModel: false, fallbackToGpt52OnUnsupportedGpt53: true, @@ -591,6 +592,15 @@ export function getRetryAllAccountsMaxRetries(pluginConfig: PluginConfig): numbe ); } +export function getRetryAllAccountsAbsoluteCeilingMs(pluginConfig: PluginConfig): number { + return resolveNumberSetting( + "CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS", + pluginConfig.retryAllAccountsAbsoluteCeilingMs, + 0, + { min: 0, max: 24 * 60 * 60_000 }, + ); +} + export function getUnsupportedCodexPolicy( pluginConfig: PluginConfig, ): UnsupportedCodexPolicy { diff --git a/lib/request/retry-governor.ts b/lib/request/retry-governor.ts new file mode 100644 index 0000000..08e8856 --- /dev/null +++ b/lib/request/retry-governor.ts @@ -0,0 +1,78 @@ +export interface RetryAllAccountsRateLimitDecisionInput { + enabled: boolean; + accountCount: number; + waitMs: number; + plannedWaitMs?: number; + maxWaitMs: number; + currentRetryCount: number; + maxRetries: number; + accumulatedWaitMs: number; + absoluteCeilingMs: number; +} + +export type RetryAllAccountsRateLimitDecisionReason = + | "allowed" + | "disabled" + | "no-accounts" + | "no-wait" + | "wait-exceeds-max" + | "retry-limit-reached" + | "absolute-ceiling-exceeded"; + +export interface RetryAllAccountsRateLimitDecision { + shouldRetry: boolean; + reason: RetryAllAccountsRateLimitDecisionReason; +} + +function clampNonNegative(value: number): number { + if (!Number.isFinite(value)) return 0; + return Math.max(0, Math.floor(value)); +} + +function normalizeRetryLimit(value: number): number { + if (!Number.isFinite(value)) return Number.POSITIVE_INFINITY; + return clampNonNegative(value); +} + +/** + * Decide whether "retry all accounts when rate-limited" should run for the current loop. + * + * This helper is pure and deterministic so retry behavior can be tested without + * exercising the full request pipeline. + */ +export function decideRetryAllAccountsRateLimited( + input: RetryAllAccountsRateLimitDecisionInput, +): RetryAllAccountsRateLimitDecision { + const accountCount = clampNonNegative(input.accountCount); + const waitMs = clampNonNegative(input.waitMs); + const plannedWaitMs = clampNonNegative(input.plannedWaitMs ?? input.waitMs); + const maxWaitMs = clampNonNegative(input.maxWaitMs); + const currentRetryCount = clampNonNegative(input.currentRetryCount); + const maxRetries = normalizeRetryLimit(input.maxRetries); + const accumulatedWaitMs = clampNonNegative(input.accumulatedWaitMs); + const absoluteCeilingMs = clampNonNegative(input.absoluteCeilingMs); + const ceilingCheckWaitMs = accumulatedWaitMs === 0 ? waitMs : plannedWaitMs; + + if (!input.enabled) { + return { shouldRetry: false, reason: "disabled" }; + } + if (accountCount === 0) { + return { shouldRetry: false, reason: "no-accounts" }; + } + if (waitMs === 0) { + return { shouldRetry: false, reason: "no-wait" }; + } + if (maxWaitMs > 0 && waitMs > maxWaitMs) { + return { shouldRetry: false, reason: "wait-exceeds-max" }; + } + if (currentRetryCount >= maxRetries) { + return { shouldRetry: false, reason: "retry-limit-reached" }; + } + if (absoluteCeilingMs > 0 && accumulatedWaitMs >= absoluteCeilingMs) { + return { shouldRetry: false, reason: "absolute-ceiling-exceeded" }; + } + if (absoluteCeilingMs > 0 && accumulatedWaitMs + ceilingCheckWaitMs > absoluteCeilingMs) { + return { shouldRetry: false, reason: "absolute-ceiling-exceeded" }; + } + return { shouldRetry: true, reason: "allowed" }; +} diff --git a/lib/schemas.ts b/lib/schemas.ts index 55028b6..40e7bfa 100644 --- a/lib/schemas.ts +++ b/lib/schemas.ts @@ -21,6 +21,7 @@ export const PluginConfigSchema = z.object({ retryAllAccountsRateLimited: z.boolean().optional(), retryAllAccountsMaxWaitMs: z.number().min(0).optional(), retryAllAccountsMaxRetries: z.number().min(0).optional(), + retryAllAccountsAbsoluteCeilingMs: z.number().min(0).optional(), unsupportedCodexPolicy: z.enum(["strict", "fallback"]).optional(), fallbackOnUnsupportedCodexModel: z.boolean().optional(), fallbackToGpt52OnUnsupportedGpt53: z.boolean().optional(), diff --git a/test/codex-manager-cli.test.ts b/test/codex-manager-cli.test.ts index 27261cd..5762fc5 100644 --- a/test/codex-manager-cli.test.ts +++ b/test/codex-manager-cli.test.ts @@ -1358,6 +1358,7 @@ describe("codex manager cli commands", () => { { type: "open-category", key: "rotation-quota" }, { type: "toggle", key: "preemptiveQuotaEnabled" }, { type: "bump", key: "preemptiveQuotaRemainingPercent5h", direction: 1 }, + { type: "bump", key: "retryAllAccountsAbsoluteCeilingMs", direction: 1 }, { type: "back" }, { type: "save" }, { type: "back" }, @@ -1374,8 +1375,13 @@ describe("codex manager cli commands", () => { expect.objectContaining({ preemptiveQuotaEnabled: expect.any(Boolean), preemptiveQuotaRemainingPercent5h: expect.any(Number), + retryAllAccountsAbsoluteCeilingMs: 30_000, }), ); + const savedPluginConfig = savePluginConfigMock.mock.calls[0]?.[0] as + | { retryAllAccountsAbsoluteCeilingMs?: number } + | undefined; + expect(savedPluginConfig?.retryAllAccountsAbsoluteCeilingMs).toBe(30_000); }); it.each([ diff --git a/test/index-retry.test.ts b/test/index-retry.test.ts index 3813edc..7e05003 100644 --- a/test/index-retry.test.ts +++ b/test/index-retry.test.ts @@ -1,6 +1,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { stripVTControlCharacters } from "node:util"; process.env.CODEX_MULTI_AUTH_EXPOSE_ADMIN_TOOLS = "1"; +let mockInitialNullCalls = 1; vi.mock("@codex-ai/plugin/tool", () => { const makeSchema = () => ({ @@ -26,6 +28,7 @@ vi.mock("../lib/request/fetch-helpers.js", () => ({ refreshAndUpdateToken: async (auth: any) => auth, createCodexHeaders: () => new Headers(), handleErrorResponse: async (response: Response) => ({ response }), + getUnsupportedCodexModelInfo: () => ({ isUnsupported: false }), resolveUnsupportedCodexFallbackModel: () => undefined, shouldFallbackToGpt52OnUnsupportedGpt53: () => false, handleSuccessResponse: async (response: Response) => response, @@ -49,7 +52,7 @@ vi.mock("../lib/accounts.js", () => { getCurrentOrNextForFamily() { this.calls += 1; - if (this.calls === 1) return null; + if (this.calls <= mockInitialNullCalls) return null; return { index: 0, accountId: "account-1", email: "user@example.com" }; } @@ -146,6 +149,7 @@ describe("OpenAIAuthPlugin rate-limit retry", () => { "CODEX_AUTH_RETRY_ALL_RATE_LIMITED", "CODEX_AUTH_RETRY_ALL_MAX_WAIT_MS", "CODEX_AUTH_RETRY_ALL_MAX_RETRIES", + "CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS", "CODEX_AUTH_TOKEN_REFRESH_SKEW_MS", "CODEX_AUTH_RATE_LIMIT_TOAST_DEBOUNCE_MS", "CODEX_AUTH_PREWARM", @@ -160,9 +164,11 @@ describe("OpenAIAuthPlugin rate-limit retry", () => { process.env.CODEX_AUTH_RETRY_ALL_RATE_LIMITED = "1"; process.env.CODEX_AUTH_RETRY_ALL_MAX_WAIT_MS = "5000"; process.env.CODEX_AUTH_RETRY_ALL_MAX_RETRIES = "1"; + process.env.CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS = "0"; process.env.CODEX_AUTH_TOKEN_REFRESH_SKEW_MS = "0"; process.env.CODEX_AUTH_RATE_LIMIT_TOAST_DEBOUNCE_MS = "0"; process.env.CODEX_AUTH_PREWARM = "0"; + mockInitialNullCalls = 1; vi.useFakeTimers(); originalFetch = globalThis.fetch; @@ -213,5 +219,261 @@ describe("OpenAIAuthPlugin rate-limit retry", () => { expect(globalThis.fetch).toHaveBeenCalledTimes(1); expect(response.status).toBe(200); }); + + it("stops immediately when absolute ceiling is below the raw retry wait", async () => { + process.env.CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS = "500"; + const { OpenAIAuthPlugin } = await import("../index.js"); + const client = { + tui: { showToast: vi.fn() }, + auth: { set: vi.fn() }, + } as any; + + const plugin = await OpenAIAuthPlugin({ client }); + const getAuth = async () => ({ + type: "oauth" as const, + access: "a", + refresh: "r", + expires: Date.now() + 60_000, + multiAccount: true, + }); + + const sdk = (await plugin.auth.loader(getAuth, { options: {}, models: {} })) as any; + const response = await sdk.fetch("https://example.com", {}); + expect(response.status).toBe(429); + expect(globalThis.fetch).not.toHaveBeenCalled(); + + const metrics = await plugin.tool["codex-metrics"].execute(); + const plainMetrics = stripVTControlCharacters(String(metrics)); + expect(plainMetrics).toContain("Retry governor stops (absolute ceiling): 1"); + }); + + it("increments retry-limit stop metric when retries are disabled", async () => { + process.env.CODEX_AUTH_RETRY_ALL_MAX_RETRIES = "0"; + const { OpenAIAuthPlugin } = await import("../index.js"); + const client = { + tui: { showToast: vi.fn() }, + auth: { set: vi.fn() }, + } as any; + + const plugin = await OpenAIAuthPlugin({ client }); + const getAuth = async () => ({ + type: "oauth" as const, + access: "a", + refresh: "r", + expires: Date.now() + 60_000, + multiAccount: true, + }); + + const sdk = (await plugin.auth.loader(getAuth, { options: {}, models: {} })) as any; + const response = await sdk.fetch("https://example.com", {}); + expect(response.status).toBe(429); + expect(globalThis.fetch).not.toHaveBeenCalled(); + + const metrics = await plugin.tool["codex-metrics"].execute(); + const plainMetrics = stripVTControlCharacters(String(metrics)); + expect(plainMetrics).toContain("Retry governor stops (retry limit): 1"); + }); + + it("caps jittered retry waits at the configured absolute ceiling", async () => { + process.env.CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS = "1100"; + vi.spyOn(Math, "random").mockReturnValue(1); + const { OpenAIAuthPlugin } = await import("../index.js"); + const client = { + tui: { showToast: vi.fn() }, + auth: { set: vi.fn() }, + } as any; + + const plugin = await OpenAIAuthPlugin({ client }); + const getAuth = async () => ({ + type: "oauth" as const, + access: "a", + refresh: "r", + expires: Date.now() + 60_000, + multiAccount: true, + }); + + const sdk = (await plugin.auth.loader(getAuth, { options: {}, models: {} })) as any; + const fetchPromise = sdk.fetch("https://example.com", {}); + expect(globalThis.fetch).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(1150); + expect(globalThis.fetch).toHaveBeenCalledTimes(1); + + const response = await fetchPromise; + expect(response.status).toBe(200); + }); + + it("consumes remaining ceiling budget under -20% jitter without premature stop", async () => { + process.env.CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS = "1600"; + process.env.CODEX_AUTH_RETRY_ALL_MAX_RETRIES = "2"; + mockInitialNullCalls = 2; + vi.spyOn(Math, "random").mockReturnValue(0); + const { OpenAIAuthPlugin } = await import("../index.js"); + const client = { + tui: { showToast: vi.fn() }, + auth: { set: vi.fn() }, + } as any; + + const plugin = await OpenAIAuthPlugin({ client }); + const getAuth = async () => ({ + type: "oauth" as const, + access: "a", + refresh: "r", + expires: Date.now() + 60_000, + multiAccount: true, + }); + + const sdk = (await plugin.auth.loader(getAuth, { options: {}, models: {} })) as any; + const fetchPromise = sdk.fetch("https://example.com", {}); + expect(globalThis.fetch).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(1599); + expect(globalThis.fetch).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(globalThis.fetch).toHaveBeenCalledTimes(1); + + const response = await fetchPromise; + expect(response.status).toBe(200); + + const metrics = await plugin.tool["codex-metrics"].execute(); + const plainMetrics = stripVTControlCharacters(String(metrics)); + expect(plainMetrics).toContain("Retry governor stops (absolute ceiling): 0"); + }); + + it("stops once the absolute ceiling budget is exhausted instead of spinning zero-delay retries", async () => { + process.env.CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS = "1600"; + process.env.CODEX_AUTH_RETRY_ALL_MAX_RETRIES = "10"; + mockInitialNullCalls = 999; + vi.spyOn(Math, "random").mockReturnValue(0); + const { OpenAIAuthPlugin } = await import("../index.js"); + const client = { + tui: { showToast: vi.fn() }, + auth: { set: vi.fn() }, + } as any; + + const plugin = await OpenAIAuthPlugin({ client }); + const getAuth = async () => ({ + type: "oauth" as const, + access: "a", + refresh: "r", + expires: Date.now() + 60_000, + multiAccount: true, + }); + + const sdk = (await plugin.auth.loader(getAuth, { options: {}, models: {} })) as any; + const fetchPromise = sdk.fetch("https://example.com", {}); + expect(globalThis.fetch).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(1600); + await vi.advanceTimersByTimeAsync(0); + + const response = await fetchPromise; + expect(response.status).toBe(429); + expect(globalThis.fetch).not.toHaveBeenCalled(); + + const metrics = await plugin.tool["codex-metrics"].execute(); + const plainMetrics = stripVTControlCharacters(String(metrics)); + expect(plainMetrics).toContain("Retry governor stops (absolute ceiling): 1"); + expect(plainMetrics).toContain("Retry governor stops (retry limit): 0"); + }); + + it("keeps retry budgets isolated across overlapping requests", async () => { + process.env.CODEX_AUTH_RETRY_ALL_MAX_RETRIES = "1"; + mockInitialNullCalls = 2; + vi.spyOn(Math, "random").mockReturnValue(0.5); + const { OpenAIAuthPlugin } = await import("../index.js"); + const client = { + tui: { showToast: vi.fn() }, + auth: { set: vi.fn() }, + } as any; + + const plugin = await OpenAIAuthPlugin({ client }); + const getAuth = async () => ({ + type: "oauth" as const, + access: "a", + refresh: "r", + expires: Date.now() + 60_000, + multiAccount: true, + }); + + const sdk = (await plugin.auth.loader(getAuth, { options: {}, models: {} })) as any; + const requestA = sdk.fetch("https://example.com/a", {}); + const requestB = sdk.fetch("https://example.com/b", {}); + + await vi.advanceTimersByTimeAsync(1000); + const [responseA, responseB] = await Promise.all([requestA, requestB]); + + expect(responseA.status).toBe(200); + expect(responseB.status).toBe(200); + expect(globalThis.fetch).toHaveBeenCalledTimes(2); + + const metrics = await plugin.tool["codex-metrics"].execute(); + const plainMetrics = stripVTControlCharacters(String(metrics)); + expect(plainMetrics).toContain("Retry governor stops (retry limit): 0"); + }); + + it("keeps max wait checks deterministic at the raw wait threshold", async () => { + process.env.CODEX_AUTH_RETRY_ALL_MAX_WAIT_MS = "1000"; + process.env.CODEX_AUTH_RETRY_ALL_MAX_RETRIES = "2"; + vi.spyOn(Math, "random").mockReturnValue(1); + const { OpenAIAuthPlugin } = await import("../index.js"); + const client = { + tui: { showToast: vi.fn() }, + auth: { set: vi.fn() }, + } as any; + + const plugin = await OpenAIAuthPlugin({ client }); + const getAuth = async () => ({ + type: "oauth" as const, + access: "a", + refresh: "r", + expires: Date.now() + 60_000, + multiAccount: true, + }); + + const sdk = (await plugin.auth.loader(getAuth, { options: {}, models: {} })) as any; + const fetchPromise = sdk.fetch("https://example.com", {}); + expect(globalThis.fetch).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1199); + expect(globalThis.fetch).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(globalThis.fetch).toHaveBeenCalledTimes(1); + + const response = await fetchPromise; + expect(response.status).toBe(200); + + const metrics = await plugin.tool["codex-metrics"].execute(); + const plainMetrics = stripVTControlCharacters(String(metrics)); + expect(plainMetrics).toContain("Retry governor stops (wait>max): 0"); + }); + + it("blocks retries when raw wait exceeds max wait even under negative jitter", async () => { + process.env.CODEX_AUTH_RETRY_ALL_MAX_WAIT_MS = "999"; + process.env.CODEX_AUTH_RETRY_ALL_MAX_RETRIES = "2"; + vi.spyOn(Math, "random").mockReturnValue(0); + const { OpenAIAuthPlugin } = await import("../index.js"); + const client = { + tui: { showToast: vi.fn() }, + auth: { set: vi.fn() }, + } as any; + + const plugin = await OpenAIAuthPlugin({ client }); + const getAuth = async () => ({ + type: "oauth" as const, + access: "a", + refresh: "r", + expires: Date.now() + 60_000, + multiAccount: true, + }); + + const sdk = (await plugin.auth.loader(getAuth, { options: {}, models: {} })) as any; + const response = await sdk.fetch("https://example.com", {}); + expect(response.status).toBe(429); + expect(globalThis.fetch).not.toHaveBeenCalled(); + + const metrics = await plugin.tool["codex-metrics"].execute(); + const plainMetrics = stripVTControlCharacters(String(metrics)); + expect(plainMetrics).toContain("Retry governor stops (wait>max): 1"); + }); }); diff --git a/test/index.test.ts b/test/index.test.ts index d6d9549..c6976b9 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -76,6 +76,7 @@ vi.mock("../lib/config.js", () => ({ getFastSessionMaxInputItems: () => 30, getRateLimitToastDebounceMs: () => 5000, getRetryAllAccountsMaxRetries: () => 3, + getRetryAllAccountsAbsoluteCeilingMs: () => 0, getRetryAllAccountsMaxWaitMs: () => 30000, getRetryAllAccountsRateLimited: () => true, getUnsupportedCodexPolicy: vi.fn(() => "fallback"), diff --git a/test/plugin-config.test.ts b/test/plugin-config.test.ts index 9caebf9..eea4a2c 100644 --- a/test/plugin-config.test.ts +++ b/test/plugin-config.test.ts @@ -12,6 +12,7 @@ import { getUnsupportedCodexPolicy, getFallbackOnUnsupportedCodexModel, getTokenRefreshSkewMs, + getRetryAllAccountsAbsoluteCeilingMs, getRetryAllAccountsMaxRetries, getFallbackToGpt52OnUnsupportedGpt53, getUnsupportedCodexFallbackChain, @@ -63,6 +64,7 @@ describe('Plugin Configuration', () => { 'CODEX_AUTH_UNSUPPORTED_MODEL_POLICY', 'CODEX_AUTH_FALLBACK_UNSUPPORTED_MODEL', 'CODEX_AUTH_FALLBACK_GPT53_TO_GPT52', + 'CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS', 'CODEX_AUTH_PREEMPTIVE_QUOTA_ENABLED', 'CODEX_AUTH_PREEMPTIVE_QUOTA_5H_REMAINING_PCT', 'CODEX_AUTH_PREEMPTIVE_QUOTA_7D_REMAINING_PCT', @@ -106,6 +108,7 @@ describe('Plugin Configuration', () => { retryAllAccountsRateLimited: true, retryAllAccountsMaxWaitMs: 0, retryAllAccountsMaxRetries: Infinity, + retryAllAccountsAbsoluteCeilingMs: 0, unsupportedCodexPolicy: 'strict', fallbackOnUnsupportedCodexModel: false, fallbackToGpt52OnUnsupportedGpt53: true, @@ -164,6 +167,7 @@ describe('Plugin Configuration', () => { retryAllAccountsRateLimited: true, retryAllAccountsMaxWaitMs: 0, retryAllAccountsMaxRetries: Infinity, + retryAllAccountsAbsoluteCeilingMs: 0, unsupportedCodexPolicy: 'strict', fallbackOnUnsupportedCodexModel: false, fallbackToGpt52OnUnsupportedGpt53: true, @@ -419,6 +423,7 @@ describe('Plugin Configuration', () => { retryAllAccountsRateLimited: true, retryAllAccountsMaxWaitMs: 0, retryAllAccountsMaxRetries: Infinity, + retryAllAccountsAbsoluteCeilingMs: 0, unsupportedCodexPolicy: 'strict', fallbackOnUnsupportedCodexModel: false, fallbackToGpt52OnUnsupportedGpt53: true, @@ -483,6 +488,7 @@ describe('Plugin Configuration', () => { retryAllAccountsRateLimited: true, retryAllAccountsMaxWaitMs: 0, retryAllAccountsMaxRetries: Infinity, + retryAllAccountsAbsoluteCeilingMs: 0, unsupportedCodexPolicy: 'strict', fallbackOnUnsupportedCodexModel: false, fallbackToGpt52OnUnsupportedGpt53: true, @@ -541,6 +547,7 @@ describe('Plugin Configuration', () => { retryAllAccountsRateLimited: true, retryAllAccountsMaxWaitMs: 0, retryAllAccountsMaxRetries: Infinity, + retryAllAccountsAbsoluteCeilingMs: 0, unsupportedCodexPolicy: 'strict', fallbackOnUnsupportedCodexModel: false, fallbackToGpt52OnUnsupportedGpt53: true, @@ -939,6 +946,34 @@ describe('Plugin Configuration', () => { expect(result).toBe(5); }); + it('should default retry-all absolute ceiling to zero', () => { + delete process.env.CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS; + expect(getRetryAllAccountsAbsoluteCeilingMs({})).toBe(0); + }); + + it('should prioritize retry-all absolute ceiling env override', () => { + process.env.CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS = '12000'; + const config: PluginConfig = { retryAllAccountsAbsoluteCeilingMs: 5000 }; + expect(getRetryAllAccountsAbsoluteCeilingMs(config)).toBe(12000); + }); + + it('should clamp retry-all absolute ceiling to 24h upper bound', () => { + process.env.CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS = String(48 * 60 * 60_000); + expect(getRetryAllAccountsAbsoluteCeilingMs({ retryAllAccountsAbsoluteCeilingMs: 5000 })) + .toBe(24 * 60 * 60_000); + }); + + it('clamps negative retry-all absolute ceiling env override to zero', () => { + process.env.CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS = '-1'; + expect(getRetryAllAccountsAbsoluteCeilingMs({ retryAllAccountsAbsoluteCeilingMs: 5000 })).toBe(0); + }); + + it('falls back to config/default when retry-all absolute ceiling env override is invalid', () => { + process.env.CODEX_AUTH_RETRY_ALL_ABSOLUTE_CEILING_MS = 'not-a-number'; + expect(getRetryAllAccountsAbsoluteCeilingMs({ retryAllAccountsAbsoluteCeilingMs: 5000 })).toBe(5000); + expect(getRetryAllAccountsAbsoluteCeilingMs({})).toBe(0); + }); + it('should return env value without min constraint', () => { process.env.CODEX_AUTH_TOKEN_REFRESH_SKEW_MS = '30000'; const config: PluginConfig = { tokenRefreshSkewMs: 60000 }; diff --git a/test/retry-governor.test.ts b/test/retry-governor.test.ts new file mode 100644 index 0000000..7ac6ac7 --- /dev/null +++ b/test/retry-governor.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, it } from "vitest"; +import { decideRetryAllAccountsRateLimited } from "../lib/request/retry-governor.js"; + +describe("decideRetryAllAccountsRateLimited", () => { + it("allows retry when all limits permit it", () => { + const result = decideRetryAllAccountsRateLimited({ + enabled: true, + accountCount: 2, + waitMs: 1_000, + maxWaitMs: 2_000, + currentRetryCount: 1, + maxRetries: 3, + accumulatedWaitMs: 2_000, + absoluteCeilingMs: 10_000, + }); + + expect(result).toEqual({ shouldRetry: true, reason: "allowed" }); + }); + + it("rejects retry when disabled", () => { + const result = decideRetryAllAccountsRateLimited({ + enabled: false, + accountCount: 2, + waitMs: 1_000, + maxWaitMs: 0, + currentRetryCount: 0, + maxRetries: Infinity, + accumulatedWaitMs: 0, + absoluteCeilingMs: 0, + }); + + expect(result).toEqual({ shouldRetry: false, reason: "disabled" }); + }); + + it("rejects retry when there are no accounts", () => { + const result = decideRetryAllAccountsRateLimited({ + enabled: true, + accountCount: 0, + waitMs: 1_000, + maxWaitMs: 0, + currentRetryCount: 0, + maxRetries: Infinity, + accumulatedWaitMs: 0, + absoluteCeilingMs: 0, + }); + + expect(result).toEqual({ shouldRetry: false, reason: "no-accounts" }); + }); + + it("rejects retry when wait time is non-positive", () => { + const result = decideRetryAllAccountsRateLimited({ + enabled: true, + accountCount: 2, + waitMs: 0, + maxWaitMs: 0, + currentRetryCount: 0, + maxRetries: Infinity, + accumulatedWaitMs: 0, + absoluteCeilingMs: 0, + }); + + expect(result).toEqual({ shouldRetry: false, reason: "no-wait" }); + }); + + it("rejects retry when wait exceeds max wait", () => { + const result = decideRetryAllAccountsRateLimited({ + enabled: true, + accountCount: 2, + waitMs: 1_500, + maxWaitMs: 1_000, + currentRetryCount: 0, + maxRetries: Infinity, + accumulatedWaitMs: 0, + absoluteCeilingMs: 0, + }); + + expect(result).toEqual({ shouldRetry: false, reason: "wait-exceeds-max" }); + }); + + it("rejects retry when max retries reached", () => { + const result = decideRetryAllAccountsRateLimited({ + enabled: true, + accountCount: 2, + waitMs: 1_000, + maxWaitMs: 0, + currentRetryCount: 2, + maxRetries: 2, + accumulatedWaitMs: 0, + absoluteCeilingMs: 0, + }); + + expect(result).toEqual({ shouldRetry: false, reason: "retry-limit-reached" }); + }); + + it("rejects retry when absolute ceiling would be exceeded", () => { + const result = decideRetryAllAccountsRateLimited({ + enabled: true, + accountCount: 2, + waitMs: 1_001, + maxWaitMs: 0, + currentRetryCount: 0, + maxRetries: Infinity, + accumulatedWaitMs: 1_000, + absoluteCeilingMs: 2_000, + }); + + expect(result).toEqual({ + shouldRetry: false, + reason: "absolute-ceiling-exceeded", + }); + }); + + it("stops retry when accumulated wait exactly matches the absolute ceiling", () => { + const result = decideRetryAllAccountsRateLimited({ + enabled: true, + accountCount: 2, + waitMs: 1_000, + plannedWaitMs: 0, + maxWaitMs: 0, + currentRetryCount: 0, + maxRetries: Infinity, + accumulatedWaitMs: 1_000, + absoluteCeilingMs: 1_000, + }); + + expect(result).toEqual({ + shouldRetry: false, + reason: "absolute-ceiling-exceeded", + }); + }); + + it("allows first retry that exactly consumes the absolute ceiling budget", () => { + const result = decideRetryAllAccountsRateLimited({ + enabled: true, + accountCount: 2, + waitMs: 1_000, + maxWaitMs: 0, + currentRetryCount: 0, + maxRetries: Infinity, + accumulatedWaitMs: 0, + absoluteCeilingMs: 1_000, + }); + + expect(result).toEqual({ shouldRetry: true, reason: "allowed" }); + }); + + it("uses planned wait for absolute ceiling checks when provided", () => { + const result = decideRetryAllAccountsRateLimited({ + enabled: true, + accountCount: 2, + waitMs: 1_000, + plannedWaitMs: 800, + maxWaitMs: 0, + currentRetryCount: 0, + maxRetries: Infinity, + accumulatedWaitMs: 800, + absoluteCeilingMs: 1_600, + }); + + expect(result).toEqual({ shouldRetry: true, reason: "allowed" }); + }); + + it("treats zero absolute ceiling as unlimited", () => { + const result = decideRetryAllAccountsRateLimited({ + enabled: true, + accountCount: 2, + waitMs: 2_000, + maxWaitMs: 0, + currentRetryCount: 0, + maxRetries: Infinity, + accumulatedWaitMs: 100_000, + absoluteCeilingMs: 0, + }); + + expect(result).toEqual({ shouldRetry: true, reason: "allowed" }); + }); +}); diff --git a/test/schemas.test.ts b/test/schemas.test.ts index 16cd2f9..7d3ee8c 100644 --- a/test/schemas.test.ts +++ b/test/schemas.test.ts @@ -30,6 +30,7 @@ describe("PluginConfigSchema", () => { retryAllAccountsRateLimited: true, retryAllAccountsMaxWaitMs: 5000, retryAllAccountsMaxRetries: 3, + retryAllAccountsAbsoluteCeilingMs: 15000, unsupportedCodexPolicy: "strict", fallbackOnUnsupportedCodexModel: true, fallbackToGpt52OnUnsupportedGpt53: false, @@ -71,6 +72,7 @@ describe("PluginConfigSchema", () => { ["sessionAffinityMaxEntries", 7, 8], ["proactiveRefreshIntervalMs", 4999, 5000], ["proactiveRefreshBufferMs", 29_999, 30_000], + ["retryAllAccountsAbsoluteCeilingMs", -1, 0], ["preemptiveQuotaMaxDeferralMs", 999, 1000], ] as const)("enforces minimum for %s", (key, invalidValue, validValue) => { const invalidResult = PluginConfigSchema.safeParse({ [key]: invalidValue }); @@ -109,6 +111,7 @@ describe("PluginConfigSchema", () => { "sessionAffinityMaxEntries", "proactiveRefreshIntervalMs", "proactiveRefreshBufferMs", + "retryAllAccountsAbsoluteCeilingMs", "networkErrorCooldownMs", "serverErrorCooldownMs", "preemptiveQuotaRemainingPercent5h", diff --git a/test/settings-hub-utils.test.ts b/test/settings-hub-utils.test.ts index 24b48fb..8e90b23 100644 --- a/test/settings-hub-utils.test.ts +++ b/test/settings-hub-utils.test.ts @@ -7,6 +7,7 @@ import type { PluginConfig } from "../lib/types.js"; type SettingsHubTestApi = { clampBackendNumber: (settingKey: string, value: number) => number; + buildBackendSettingsPreview: (config: PluginConfig) => { label: string; hint: string }; formatMenuLayoutMode: (mode: "compact-details" | "expanded-rows") => string; cloneDashboardSettings: (settings: DashboardDisplaySettings) => DashboardDisplaySettings; withQueuedRetry: (pathKey: string, task: () => Promise) => Promise; @@ -64,11 +65,24 @@ describe("settings-hub utility coverage", () => { const api = await loadSettingsHubTestApi(); expect(api.clampBackendNumber("fetchTimeoutMs", 250)).toBe(1_000); expect(api.clampBackendNumber("fetchTimeoutMs", 999_999)).toBe(600_000); + expect(api.clampBackendNumber("retryAllAccountsAbsoluteCeilingMs", -1)).toBe(0); + expect(api.clampBackendNumber("retryAllAccountsAbsoluteCeilingMs", 999_999_999)).toBe( + 24 * 60 * 60_000, + ); expect(() => api.clampBackendNumber("unknown-setting", 5)).toThrow( "Unknown backend numeric setting key", ); }); + it("renders retry-all absolute ceiling 0 as unlimited in preview", async () => { + const api = await loadSettingsHubTestApi(); + const preview = api.buildBackendSettingsPreview({ + retryAllAccountsAbsoluteCeilingMs: 0, + }); + expect(preview.hint).toContain("retry ceiling unlimited"); + expect(preview.hint).not.toContain("retry ceiling 0ms"); + }); + it("formats layout mode labels", async () => { const api = await loadSettingsHubTestApi(); expect(api.formatMenuLayoutMode("expanded-rows")).toBe("Expanded Rows");