From 969ab8d0d77390ff3cd7757645978ac3355c92ae Mon Sep 17 00:00:00 2001
From: WebStreamr <210764791+webstreamr@users.noreply.github.com>
Date: Tue, 2 Sep 2025 14:41:29 +0000
Subject: [PATCH] chore(extractor): start refreshing results randomly after at
 least 2/3 of the TTL passed

---
 src/extractor/ExtractorRegistry.ts | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/extractor/ExtractorRegistry.ts b/src/extractor/ExtractorRegistry.ts
index fc477d3..6509900 100644
--- a/src/extractor/ExtractorRegistry.ts
+++ b/src/extractor/ExtractorRegistry.ts
@@ -28,8 +28,16 @@ export class ExtractorRegistry {
       : `${extractor.id}_${normalizedUrl}`;
 
     const storedDataRaw = await this.urlResultCache.get<UrlResult[]>(cacheKey, { raw: true });
-    if (storedDataRaw) {
-      return (storedDataRaw.value as UrlResult[]).map(urlResult => ({ ...urlResult, ttl: storedDataRaw.expires as number - Date.now(), url: new URL(urlResult.url) }));
+    const expires = storedDataRaw?.expires;
+    if (storedDataRaw && expires) {
+      // Ignore the cache randomly after at least 2/3 of the TTL passed to start refreshing results slowly
+      const refreshTimestamp = this.randomInteger(expires - extractor.ttl * (2 / 3), expires);
+      const now = Date.now();
+
+      /* istanbul ignore if */
+      if (refreshTimestamp > now) {
+        return (storedDataRaw.value as UrlResult[]).map(urlResult => ({ ...urlResult, ttl: expires - now, url: new URL(urlResult.url) }));
+      }
     }
 
     this.logger.info(`Extract stream URL using ${extractor.id} extractor from ${url}`, ctx);
@@ -42,4 +50,8 @@ export class ExtractorRegistry {
 
     return urlResults;
   };
+
+  private randomInteger(min: number, max: number) {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+  }
 }