Creepso/webstreamr-github
1
0
Fork 0
Code Issues 27 Pull requests 7 Projects Releases 240 Packages Wiki Activity Actions

chore(fetcher): wait out short rate limits and retry

Browse source
This commit is contained in:
WebStreamr 2025-09-05 08:52:36 +00:00
parent e846900384
commit 211babec37
No known key found for this signature in database
2 changed files with 41 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/utils/Fetcher.test.ts
View file

@ -190,14 +190,15 @@ describe('fetch', () => {
test('converts 429 to custom TooManyRequestsError and blocks consecutive requests', async () => {
const mockPool = mockAgent.get('https://some-rate-limited-retry-after-url.test');
mockPool.intercept({ path: '/' }).reply(429, undefined, { headers: { 'retry-after': '10' } });
mockPool.intercept({ path: '/' }).reply(429, undefined, { headers: { 'retry-after': '60' } });
mockPool.intercept({ path: '/' }).reply(200);
try {
await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-url.test/'));
fail();
} catch (error) {
expect(error).toBeInstanceOf(TooManyRequestsError);
expect(error).toMatchObject({ retryAfter: 10 });
expect(error).toMatchObject({ retryAfter: 60 });
}
try {
@ -208,6 +209,15 @@ describe('fetch', () => {
}
});
test('retries rate limits with short ttl', async () => {
const mockPool = mockAgent.get('https://some-rate-limited-retry-after-short-url.test');
mockPool.intercept({ path: '/' }).reply(429, undefined, { headers: { 'retry-after': '1' } });
mockPool.intercept({ path: '/' }).reply(200);
const responseText = await fetcher.text(ctx, new URL('https://some-rate-limited-retry-after-short-url.test/'));
expect(responseText).toBe('');
});
test('passes through other error as HttpError after retrying 3 times', async () => {
const mockPool = mockAgent.get('https://some-error-url.test');
mockPool.intercept({ path: '/' }).reply(500, undefined, { headers: { 'x-foo': 'bar' } }).times(4);

32
src/utils/Fetcher.ts
View file

@ -60,6 +60,7 @@ export class Fetcher {
private readonly DEFAULT_QUEUE_TIMEOUT = 5000;
private readonly DEFAULT_TIMEOUTS_COUNT_THROW = 30;
private readonly TIMEOUT_CACHE_TTL = 3600000; // 1h
private readonly MAX_WAIT_RETRY_AFTER = 10000;
private readonly logger: winston.Logger;
@ -262,7 +263,17 @@ export class Fetcher {
this.logger.info(`Fetch ${init?.method ?? 'GET'} ${url}`, ctx);
const isRateLimitedRaw = await this.rateLimitedCache.get<true>(url.host, { raw: true });
if (isRateLimitedRaw && isRateLimitedRaw.value) {
/* istanbul ignore if */
if (isRateLimitedRaw && isRateLimitedRaw.value && isRateLimitedRaw.expires) {
const ttl = isRateLimitedRaw.expires - Date.now();
if (ttl <= this.MAX_WAIT_RETRY_AFTER && tryCount < 1) {
this.logger.info('Wait out rate limit', ctx);
await this.sleep(ttl);
return await this.fetchWithTimeout(ctx, url, { ...init, queueLimit: 1 }, ++tryCount);
}
throw new TooManyRequestsError((isRateLimitedRaw.expires as number - Date.now()) / 1000);
}
@ -284,7 +295,7 @@ export class Fetcher {
if (error instanceof DOMException && error.name === 'TimeoutError') {
if (tryCount < 1) {
this.logger.warn(`Retrying fetch ${init?.method ?? 'GET'} ${url} because of timeout`, ctx);
await new Promise(sleep => setTimeout(sleep, 333));
await this.sleep(333);
return await this.fetchWithTimeout(ctx, url, init, ++tryCount);
}
@ -298,9 +309,20 @@ export class Fetcher {
await this.decreaseTimeoutsCount(url);
if (response.status === 429) {
const retryAfter = parseInt(`${response.headers.get('retry-after')}`) * 1000;
if (retryAfter <= this.MAX_WAIT_RETRY_AFTER && tryCount < 1) {
this.logger.info('Wait out rate limit', ctx);
await this.sleep(retryAfter);
return await this.fetchWithTimeout(ctx, url, { ...init, queueLimit: 1 }, ++tryCount);
}
}
if (response.status >= 500 && tryCount < 3) {
this.logger.warn(`Retrying fetch ${init?.method ?? 'GET'} ${url} because of error`, ctx);
await new Promise(sleep => setTimeout(sleep, 333));
await this.sleep(333);
return await this.fetchWithTimeout(ctx, url, init, ++tryCount);
}
@ -349,4 +371,8 @@ export class Fetcher {
release();
}
}
private sleep(ms: number): Promise<void> {
return new Promise(sleep => setTimeout(sleep, ms));
}
}