diff --git a/service/src/egress-ledger-reconnect.test.ts b/service/src/egress-ledger-reconnect.test.ts index 0cf4bb78..6fc935c9 100644 --- a/service/src/egress-ledger-reconnect.test.ts +++ b/service/src/egress-ledger-reconnect.test.ts @@ -4,7 +4,7 @@ import IORedis from 'ioredis'; import { env } from './config'; import { startTestRedis } from './test/redis'; import { - EGRESS_LEDGER_REDIS_RETRY_OPTIONS, createEgressLedger, recordEgressRead, + EGRESS_LEDGER_REDIS_RETRY_OPTIONS, createEgressLedger, egressLedgerRedisOptions, recordEgressRead, assertEgressGrantActive, setEgressLedgerRedisForTest, } from './egress-ledger'; import type { EgressGrantClaims } from './egress-grant'; @@ -61,3 +61,10 @@ test('a lost Redis mutation reply rejects without replaying its applied counter await redis.closeTestServer(); } }, 10000); + +test('the ledger client keeps trying to reconnect using linear backoff with cap, however long Redis is unavailable', () => { + const { retryStrategy } = egressLedgerRedisOptions(); + for (const attempt of [1, 6, 100, 10_000]) { + expect(retryStrategy!(attempt)).toBeGreaterThan(0); + } +}); diff --git a/service/src/egress-ledger.ts b/service/src/egress-ledger.ts index c635dcc4..4e41038c 100644 --- a/service/src/egress-ledger.ts +++ b/service/src/egress-ledger.ts @@ -1,11 +1,11 @@ import IORedis from 'ioredis'; -import type { CommonRedisOptions } from 'ioredis'; +import type { RedisOptions } from 'ioredis'; import type * as tls from 'tls'; import { env } from './config'; import type { EgressGrantClaims } from './egress-grant'; import { EgressGrantError } from './egress-grant'; import logger from './logger'; -import { redisKeepAliveOptions } from './redis-options'; +import { redisKeepAliveOptions, redisReconnectDelay } from './redis-options'; import { EGRESS_LEDGER_SCRIPT } from './egress-ledger-script'; type LedgerStatus = 'active' | 'revoked'; @@ -53,18 +53,13 @@ function ttlSeconds(exp: number): number { return Math.max(1, exp - Math.floor(Date.now() / 1000) + env.EGRESS_LEDGER_TTL_GRACE_SECONDS); } -function redisConnection(): IORedis { - if (redis) return redis; - const retryStrategy: CommonRedisOptions['retryStrategy'] = times => { - if (times > 5) return null; - return 2000; - }; - redis = new IORedis({ +export function egressLedgerRedisOptions(): RedisOptions { + return { host: process.env.REDIS_HOST ?? 'redis', port: Number(process.env.REDIS_PORT) || 6379, password: process.env.REDIS_PASSWORD, ...EGRESS_LEDGER_REDIS_RETRY_OPTIONS, - retryStrategy, + retryStrategy: redisReconnectDelay, // Use linear backoff with a cap for reconnect attempts, otherwise Redis outages do not recover automatically enableReadyCheck: true, connectTimeout: 10000, ...redisKeepAliveOptions(), @@ -74,7 +69,12 @@ function redisConnection(): IORedis { ...(env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } : {}), - }); + }; +} + +function redisConnection(): IORedis { + if (redis) return redis; + redis = new IORedis(egressLedgerRedisOptions()); redis.on('error', error => logger.error('Egress ledger Redis error', { error })); return redis; }