From fa603ef5f29c113a61d0fcf6fd5860888f92e959 Mon Sep 17 00:00:00 2001
From: Emmanuel Raviart <emmanuel@raviart.com>
Date: Mon, 20 May 2024 14:27:05 +0200
Subject: [PATCH] Retry when page contains an error message

---
 src/scripts/download_dole_html.ts | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/scripts/download_dole_html.ts b/src/scripts/download_dole_html.ts
index 8c108c2..b18ca02 100644
--- a/src/scripts/download_dole_html.ts
+++ b/src/scripts/download_dole_html.ts
@@ -164,7 +164,21 @@ async function fetchHtmlPage(url: string): Promise<string> {
         throw new Error(`Retrieval of HTML page at <${url}> failed`)
       }
     }
-    return await response.text()
+    const html = await response.text()
+    if (html.includes("Request unsuccessful. Incapsula incident ID:")) {
+      if (retriesCount === 0) {
+        console.warn(response.status, response.statusText)
+        console.warn(JSON.stringify(response.headers, null, 2))
+        console.warn(html)
+      }
+      if (retriesCount < 10) {
+        await sleep(30)
+        console.info("Retrying…")
+      } else {
+        throw new Error(`Retrieval of HTML page at <${url}> failed`)
+      }
+    }
+    return html
   }
 }