diff --git a/src/scripts/download_dole_html.ts b/src/scripts/download_dole_html.ts
new file mode 100644
index 0000000..69957a3
--- /dev/null
+++ b/src/scripts/download_dole_html.ts
@@ -0,0 +1,189 @@
+import assert from "assert"
+import fs from "fs-extra"
+import { JSDOM } from "jsdom"
+import path from "path"
+import sade from "sade"
+import { $, cd } from "zx"
+
+import { parseDossierLegislatif } from "$lib/parsers"
+import { walkDir } from "$lib/server/file_systems"
+
+async function downloadDoleHtml(
+ dilaDir: string,
+ {
+ push,
+ resume,
+ silent,
+ verbose,
+ }: {
+ push?: boolean
+ resume?: string
+ silent?: boolean
+ verbose?: boolean
+ } = {},
+): Promise {
+ let changed = false
+ let skip = resume !== undefined
+
+ const dataDir = path.join(dilaDir, "dole")
+ assert(await fs.pathExists(dataDir))
+
+ const echeanciersHtmlDirName = "dole_echeanciers_html"
+ const echeanciersHtmlDir = path.join(dilaDir, echeanciersHtmlDirName)
+ assert(await fs.pathExists(echeanciersHtmlDir))
+
+ cd(echeanciersHtmlDir)
+ let latestCommitName: string | undefined = undefined
+ try {
+ latestCommitName = (await $`git log -1 --pretty=%B`).stdout.trim()
+ } catch (processOutput) {
+ // Git repository has no commit yet.
+ }
+ cd("..")
+
+ iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) {
+ const relativePath = path.join(...relativeSplitPath)
+ if (skip) {
+ if (relativePath.startsWith(resume as string)) {
+ skip = false
+ console.log(`Resuming at file ${relativePath}...`)
+ } else {
+ continue
+ }
+ }
+
+ const filePath = path.join(dataDir, relativePath)
+ if (!filePath.endsWith(".xml")) {
+ console.info(`Skipping non XML file at ${filePath}`)
+ continue
+ }
+
+ try {
+ const xmlString: string = await fs.readFile(filePath, {
+ encoding: "utf8",
+ })
+ const dossierLegislatif = parseDossierLegislatif(filePath, xmlString)
+ if (dossierLegislatif === undefined) {
+ break iterXmlFiles
+ }
+
+ if (dossierLegislatif.CONTENU.ECHEANCIER !== undefined) {
+ if (!silent && verbose) {
+ console.log(
+ `Retrieving échéancier of dossier législatif ${dossierLegislatif.META.META_COMMUN.ID}…`,
+ )
+ }
+ const dossierLegislatifUrl = `https://www.legifrance.gouv.fr/dossierlegislatif/${dossierLegislatif.META.META_COMMUN.ID}/`
+ const dossierLegislatifHtml = await fetchHtmlPage(dossierLegislatifUrl)
+ const { document: dossierLegislatifDocument } = new JSDOM(
+ dossierLegislatifHtml,
+ ).window
+ const echeancierA = [
+ ...dossierLegislatifDocument.querySelectorAll("a.marker").values(),
+ ].filter((aElement) => aElement.innerHTML.trim() === "Echeancier")[0]
+ if (echeancierA === undefined) {
+ if (!silent) {
+ console.warn(
+ `Dossier législatif ${dossierLegislatif.META.META_COMMUN.ID} has no échéancier.`,
+ )
+ }
+ continue
+ }
+ const echeancierRelativeUrl = echeancierA.getAttribute("href")
+ if (echeancierRelativeUrl === null) {
+ if (!silent) {
+ console.warn(
+ `Dossier législatif ${dossierLegislatif.META.META_COMMUN.ID} has no échéancier.`,
+ )
+ }
+ continue
+ }
+ const echeancierHtml = await fetchHtmlPage(
+ new URL(echeancierRelativeUrl, dossierLegislatifUrl).toString(),
+ )
+ const { document: echeancierDocument } = new JSDOM(echeancierHtml)
+ .window
+ const echeancierTable = echeancierDocument.querySelector(
+ "div.dossier-legislatif-detail-contenu > table",
+ )
+ if (echeancierTable === null) {
+ if (!silent) {
+ console.warn(
+ `Échéancier of dossier législatif ${dossierLegislatif.META.META_COMMUN.ID} has no table².`,
+ )
+ }
+ continue
+ }
+
+ await fs.writeFile(
+ path.join(
+ echeanciersHtmlDir,
+ `${dossierLegislatif.META.META_COMMUN.ID}_echeancier_table.html`,
+ ),
+ echeancierTable.outerHTML,
+ { encoding: "utf-8" },
+ )
+ }
+ } catch (e) {
+ console.error(
+ "An error occurred while handling dossier législatif XML file",
+ filePath,
+ )
+ throw e
+ }
+ }
+
+ cd(echeanciersHtmlDirName)
+ await $`git add .`
+ if ((await $`git diff --quiet --staged`.exitCode) !== 0) {
+ changed = true
+ const message = `Récupération des tables des échéanciers - ${new Date().toISOString()}`
+ await $`git commit -m ${message} --quiet`
+ if (push) {
+ await $`git push`
+ }
+ }
+ cd("..")
+
+ return changed ? 0 : 10 // No new commit has been added to git repository.
+}
+
+async function fetchHtmlPage(url: string): Promise {
+ for (let retriesCount = 0; ; retriesCount++) {
+ const response = await fetch(url)
+ if (!response.ok) {
+ if (retriesCount === 0) {
+ console.warn(response.status, response.statusText)
+ console.warn(JSON.stringify(response.headers, null, 2))
+ console.warn(await response.text())
+ }
+ if (retriesCount < 10) {
+ await sleep(30)
+ console.info("Retrying…")
+ } else {
+ throw new Error(`Retrieval of HTML page at <${url}> failed`)
+ }
+ }
+ return await response.text()
+ }
+}
+
+function sleep(s: number): Promise {
+ return new Promise((resolve) => setTimeout(resolve, s * 1000))
+}
+
+sade("download_dole_html ", true)
+ .describe(
+ "Download DOLE HTML to retrieve informations that lack in open data (links of decrees in échéanciers, etc)",
+ )
+ .option("-p, --push", "Push dataset repository")
+ .option("-r, --resume", "Resume import at given relative file path")
+ .option("-s, --silent", "Hide log messages")
+ .option("-v, --verbose", "Show all log messages")
+ .example(
+ "--resume dole/global/JORF/DOLE/00/00/36/07/36/JORFDOLE000036073697.xml ../dila-data/",
+ )
+ .action(async (dilaDir, options) => {
+ process.exit(await downloadDoleHtml(dilaDir, options))
+ })
+ .parse(process.argv)