feat: configure queue retry delays with min/max bounds and update exponential backoff defaults

This commit is contained in:
2025-12-17 09:25:07 -05:00
parent 6f795bdde0
commit 6570c25617
2 changed files with 43 additions and 34 deletions

View File

@@ -123,9 +123,13 @@ export default {
}, },
} satisfies ExportedHandler<Env>; } satisfies ExportedHandler<Env>;
const retryDelayConfig: Record<QueueName, DurationLike> = { const retryDelayConfig: Partial<
ANILIST_UPDATES: Duration.fromObject({ minutes: 1 }), Record<QueueName, { min: DurationLike; max: DurationLike }>
NEW_EPISODE: Duration.fromObject({ hours: 1 }), > = {
NEW_EPISODE: {
min: Duration.fromObject({ hours: 1 }),
max: Duration.fromObject({ hours: 12 }),
},
}; };
function onMessageQueue<QN extends QueueName>( function onMessageQueue<QN extends QueueName>(
@@ -144,7 +148,8 @@ function onMessageQueue<QN extends QueueName>(
message.retry({ message.retry({
delaySeconds: calculateExponentialBackoff({ delaySeconds: calculateExponentialBackoff({
attempt: message.attempts, attempt: message.attempts,
baseMin: retryDelayConfig[messageBatch.queue as QN], baseMin: retryDelayConfig[messageBatch.queue as QN]?.min,
absCap: retryDelayConfig[messageBatch.queue as QN]?.max,
}), }),
}); });
} }

View File

@@ -1,49 +1,53 @@
import { Duration, type DurationLike } from "luxon"; import { Duration, type DurationLike } from "luxon";
interface CalculateExponentialBackoffOptions { interface CalculateExponentialBackoffOptions {
attempt: number; attempt: number;
baseMin?: DurationLike; baseMin?: DurationLike;
absCap?: DurationLike; absCap?: DurationLike;
fuzzFactor?: number; fuzzFactor?: number;
} }
/** /**
* Generates a backoff time where both the Minimum floor and Maximum ceiling * Generates a backoff time where both the Minimum floor and Maximum ceiling
* are "fuzzed" with jitter to prevent clustering at the edges. * are "fuzzed" with jitter to prevent clustering at the edges.
* *
* @param attempt - The current retry attempt (0-indexed). * @param attempt - The current retry attempt (0-indexed).
* @param baseMin - The nominal minimum wait time (default: 1s). * @param baseMin - The nominal minimum wait time (default: 1s).
* @param absCap - The absolute maximum wait time (default: 60s). * @param absCap - The absolute maximum wait time (default: 60s).
* @param fuzzFactor - How much to wobble the edges (0.1 = +/- 10%). * @param fuzzFactor - How much to wobble the edges (0.1 = +/- 10%).
* *
* @returns A random duration between the nominal minimum and maximum, in seconds. * @returns A random duration between the nominal minimum and maximum, in seconds.
*/ */
export function calculateExponentialBackoff( export function calculateExponentialBackoff({
{ attempt, baseMin: baseMinDuration = Duration.fromObject({ seconds: 1 }), absCap: absCapDuration = Duration.fromObject({ seconds: 60 }), fuzzFactor = 0.2 }: CalculateExponentialBackoffOptions attempt,
): number { baseMin: baseMinDuration = Duration.fromObject({ minutes: 1 }),
const baseMin = Duration.fromDurationLike(baseMinDuration).as('seconds'); absCap: absCapDuration = Duration.fromObject({ hours: 1 }),
const absCap = Duration.fromDurationLike(absCapDuration).as('seconds'); fuzzFactor = 0.2,
}: CalculateExponentialBackoffOptions): number {
const baseMin = Duration.fromDurationLike(baseMinDuration).as("seconds");
const absCap = Duration.fromDurationLike(absCapDuration).as("seconds");
// 1. Calculate nominal boundaries // 1. Calculate nominal boundaries
// Example: If baseMin is 1s, the nominal boundaries are 1s, 2s, 4s, 8s... (The 'ceiling' grows exponentially) // Example: If baseMin is 1s, the nominal boundaries are 1s, 2s, 4s, 8s... (The 'ceiling' grows exponentially)
const nominalMin = baseMin; const nominalMin = baseMin;
const nominalCeiling = Math.min(baseMin * Math.pow(2, attempt), absCap); const nominalCeiling = Math.min(baseMin * Math.pow(2, attempt), absCap);
// 2. Fuzz the Min (The Floor) // 2. Fuzz the Min (The Floor)
// Example: If min is 1s and fuzz is 0.2, the floor becomes random between 0.8s and 1.2s // Example: If min is 1s and fuzz is 0.2, the floor becomes random between 0.8s and 1.2s
const minFuzz = nominalMin * fuzzFactor; const minFuzz = nominalMin * fuzzFactor;
const fuzzedMin = nominalMin + (Math.random() * 2 * minFuzz - minFuzz); const fuzzedMin = nominalMin + (Math.random() * 2 * minFuzz - minFuzz);
// 3. Fuzz the Max (The Ceiling) // 3. Fuzz the Max (The Ceiling)
// Example: If ceiling is 4s (and fuzz is 0.2), it becomes random between 3.2s and 4.8s // Example: If ceiling is 4s (and fuzz is 0.2), it becomes random between 3.2s and 4.8s
const maxFuzz = nominalCeiling * fuzzFactor; const maxFuzz = nominalCeiling * fuzzFactor;
const fuzzedCeiling = nominalCeiling + (Math.random() * 2 * maxFuzz - maxFuzz); const fuzzedCeiling =
nominalCeiling + (Math.random() * 2 * maxFuzz - maxFuzz);
// Safety: Ensure we don't return a negative number or cross boundaries weirdly // Safety: Ensure we don't return a negative number or cross boundaries weirdly
// (e.g. if fuzz makes min > max, we swap or clamp) // (e.g. if fuzz makes min > max, we swap or clamp)
const safeMin = Math.max(0, fuzzedMin); const safeMin = Math.max(0, fuzzedMin);
const safeMax = Math.max(safeMin, fuzzedCeiling); const safeMax = Math.max(safeMin, fuzzedCeiling);
// 4. Return random value in the new fuzzy range // 4. Return random value in the new fuzzy range
return safeMin + Math.random() * (safeMax - safeMin); return safeMin + Math.random() * (safeMax - safeMin);
} }