diff --git a/src/scripts/download_dole_html.ts b/src/scripts/download_dole_html.ts new file mode 100644 index 0000000..69957a3 --- /dev/null +++ b/src/scripts/download_dole_html.ts @@ -0,0 +1,189 @@ +import assert from "assert" +import fs from "fs-extra" +import { JSDOM } from "jsdom" +import path from "path" +import sade from "sade" +import { $, cd } from "zx" + +import { parseDossierLegislatif } from "$lib/parsers" +import { walkDir } from "$lib/server/file_systems" + +async function downloadDoleHtml( + dilaDir: string, + { + push, + resume, + silent, + verbose, + }: { + push?: boolean + resume?: string + silent?: boolean + verbose?: boolean + } = {}, +): Promise { + let changed = false + let skip = resume !== undefined + + const dataDir = path.join(dilaDir, "dole") + assert(await fs.pathExists(dataDir)) + + const echeanciersHtmlDirName = "dole_echeanciers_html" + const echeanciersHtmlDir = path.join(dilaDir, echeanciersHtmlDirName) + assert(await fs.pathExists(echeanciersHtmlDir)) + + cd(echeanciersHtmlDir) + let latestCommitName: string | undefined = undefined + try { + latestCommitName = (await $`git log -1 --pretty=%B`).stdout.trim() + } catch (processOutput) { + // Git repository has no commit yet. + } + cd("..") + + iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) { + const relativePath = path.join(...relativeSplitPath) + if (skip) { + if (relativePath.startsWith(resume as string)) { + skip = false + console.log(`Resuming at file ${relativePath}...`) + } else { + continue + } + } + + const filePath = path.join(dataDir, relativePath) + if (!filePath.endsWith(".xml")) { + console.info(`Skipping non XML file at ${filePath}`) + continue + } + + try { + const xmlString: string = await fs.readFile(filePath, { + encoding: "utf8", + }) + const dossierLegislatif = parseDossierLegislatif(filePath, xmlString) + if (dossierLegislatif === undefined) { + break iterXmlFiles + } + + if (dossierLegislatif.CONTENU.ECHEANCIER !== undefined) { + if (!silent && verbose) { + console.log( + `Retrieving échéancier of dossier législatif ${dossierLegislatif.META.META_COMMUN.ID}…`, + ) + } + const dossierLegislatifUrl = `https://www.legifrance.gouv.fr/dossierlegislatif/${dossierLegislatif.META.META_COMMUN.ID}/` + const dossierLegislatifHtml = await fetchHtmlPage(dossierLegislatifUrl) + const { document: dossierLegislatifDocument } = new JSDOM( + dossierLegislatifHtml, + ).window + const echeancierA = [ + ...dossierLegislatifDocument.querySelectorAll("a.marker").values(), + ].filter((aElement) => aElement.innerHTML.trim() === "Echeancier")[0] + if (echeancierA === undefined) { + if (!silent) { + console.warn( + `Dossier législatif ${dossierLegislatif.META.META_COMMUN.ID} has no échéancier.`, + ) + } + continue + } + const echeancierRelativeUrl = echeancierA.getAttribute("href") + if (echeancierRelativeUrl === null) { + if (!silent) { + console.warn( + `Dossier législatif ${dossierLegislatif.META.META_COMMUN.ID} has no échéancier.`, + ) + } + continue + } + const echeancierHtml = await fetchHtmlPage( + new URL(echeancierRelativeUrl, dossierLegislatifUrl).toString(), + ) + const { document: echeancierDocument } = new JSDOM(echeancierHtml) + .window + const echeancierTable = echeancierDocument.querySelector( + "div.dossier-legislatif-detail-contenu > table", + ) + if (echeancierTable === null) { + if (!silent) { + console.warn( + `Échéancier of dossier législatif ${dossierLegislatif.META.META_COMMUN.ID} has no table².`, + ) + } + continue + } + + await fs.writeFile( + path.join( + echeanciersHtmlDir, + `${dossierLegislatif.META.META_COMMUN.ID}_echeancier_table.html`, + ), + echeancierTable.outerHTML, + { encoding: "utf-8" }, + ) + } + } catch (e) { + console.error( + "An error occurred while handling dossier législatif XML file", + filePath, + ) + throw e + } + } + + cd(echeanciersHtmlDirName) + await $`git add .` + if ((await $`git diff --quiet --staged`.exitCode) !== 0) { + changed = true + const message = `Récupération des tables des échéanciers - ${new Date().toISOString()}` + await $`git commit -m ${message} --quiet` + if (push) { + await $`git push` + } + } + cd("..") + + return changed ? 0 : 10 // No new commit has been added to git repository. +} + +async function fetchHtmlPage(url: string): Promise { + for (let retriesCount = 0; ; retriesCount++) { + const response = await fetch(url) + if (!response.ok) { + if (retriesCount === 0) { + console.warn(response.status, response.statusText) + console.warn(JSON.stringify(response.headers, null, 2)) + console.warn(await response.text()) + } + if (retriesCount < 10) { + await sleep(30) + console.info("Retrying…") + } else { + throw new Error(`Retrieval of HTML page at <${url}> failed`) + } + } + return await response.text() + } +} + +function sleep(s: number): Promise { + return new Promise((resolve) => setTimeout(resolve, s * 1000)) +} + +sade("download_dole_html ", true) + .describe( + "Download DOLE HTML to retrieve informations that lack in open data (links of decrees in échéanciers, etc)", + ) + .option("-p, --push", "Push dataset repository") + .option("-r, --resume", "Resume import at given relative file path") + .option("-s, --silent", "Hide log messages") + .option("-v, --verbose", "Show all log messages") + .example( + "--resume dole/global/JORF/DOLE/00/00/36/07/36/JORFDOLE000036073697.xml ../dila-data/", + ) + .action(async (dilaDir, options) => { + process.exit(await downloadDoleHtml(dilaDir, options)) + }) + .parse(process.argv)