Add script to download échéanciers tables from Légifrance HTML pages
This commit is contained in:
parent
2ea0a18380
commit
7a642b357d
1 changed files with 189 additions and 0 deletions
189
src/scripts/download_dole_html.ts
Normal file
189
src/scripts/download_dole_html.ts
Normal file
|
@ -0,0 +1,189 @@
|
||||||
|
import assert from "assert"
|
||||||
|
import fs from "fs-extra"
|
||||||
|
import { JSDOM } from "jsdom"
|
||||||
|
import path from "path"
|
||||||
|
import sade from "sade"
|
||||||
|
import { $, cd } from "zx"
|
||||||
|
|
||||||
|
import { parseDossierLegislatif } from "$lib/parsers"
|
||||||
|
import { walkDir } from "$lib/server/file_systems"
|
||||||
|
|
||||||
|
async function downloadDoleHtml(
|
||||||
|
dilaDir: string,
|
||||||
|
{
|
||||||
|
push,
|
||||||
|
resume,
|
||||||
|
silent,
|
||||||
|
verbose,
|
||||||
|
}: {
|
||||||
|
push?: boolean
|
||||||
|
resume?: string
|
||||||
|
silent?: boolean
|
||||||
|
verbose?: boolean
|
||||||
|
} = {},
|
||||||
|
): Promise<number> {
|
||||||
|
let changed = false
|
||||||
|
let skip = resume !== undefined
|
||||||
|
|
||||||
|
const dataDir = path.join(dilaDir, "dole")
|
||||||
|
assert(await fs.pathExists(dataDir))
|
||||||
|
|
||||||
|
const echeanciersHtmlDirName = "dole_echeanciers_html"
|
||||||
|
const echeanciersHtmlDir = path.join(dilaDir, echeanciersHtmlDirName)
|
||||||
|
assert(await fs.pathExists(echeanciersHtmlDir))
|
||||||
|
|
||||||
|
cd(echeanciersHtmlDir)
|
||||||
|
let latestCommitName: string | undefined = undefined
|
||||||
|
try {
|
||||||
|
latestCommitName = (await $`git log -1 --pretty=%B`).stdout.trim()
|
||||||
|
} catch (processOutput) {
|
||||||
|
// Git repository has no commit yet.
|
||||||
|
}
|
||||||
|
cd("..")
|
||||||
|
|
||||||
|
iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) {
|
||||||
|
const relativePath = path.join(...relativeSplitPath)
|
||||||
|
if (skip) {
|
||||||
|
if (relativePath.startsWith(resume as string)) {
|
||||||
|
skip = false
|
||||||
|
console.log(`Resuming at file ${relativePath}...`)
|
||||||
|
} else {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const filePath = path.join(dataDir, relativePath)
|
||||||
|
if (!filePath.endsWith(".xml")) {
|
||||||
|
console.info(`Skipping non XML file at ${filePath}`)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const xmlString: string = await fs.readFile(filePath, {
|
||||||
|
encoding: "utf8",
|
||||||
|
})
|
||||||
|
const dossierLegislatif = parseDossierLegislatif(filePath, xmlString)
|
||||||
|
if (dossierLegislatif === undefined) {
|
||||||
|
break iterXmlFiles
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dossierLegislatif.CONTENU.ECHEANCIER !== undefined) {
|
||||||
|
if (!silent && verbose) {
|
||||||
|
console.log(
|
||||||
|
`Retrieving échéancier of dossier législatif ${dossierLegislatif.META.META_COMMUN.ID}…`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const dossierLegislatifUrl = `https://www.legifrance.gouv.fr/dossierlegislatif/${dossierLegislatif.META.META_COMMUN.ID}/`
|
||||||
|
const dossierLegislatifHtml = await fetchHtmlPage(dossierLegislatifUrl)
|
||||||
|
const { document: dossierLegislatifDocument } = new JSDOM(
|
||||||
|
dossierLegislatifHtml,
|
||||||
|
).window
|
||||||
|
const echeancierA = [
|
||||||
|
...dossierLegislatifDocument.querySelectorAll("a.marker").values(),
|
||||||
|
].filter((aElement) => aElement.innerHTML.trim() === "Echeancier")[0]
|
||||||
|
if (echeancierA === undefined) {
|
||||||
|
if (!silent) {
|
||||||
|
console.warn(
|
||||||
|
`Dossier législatif ${dossierLegislatif.META.META_COMMUN.ID} has no échéancier.`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
const echeancierRelativeUrl = echeancierA.getAttribute("href")
|
||||||
|
if (echeancierRelativeUrl === null) {
|
||||||
|
if (!silent) {
|
||||||
|
console.warn(
|
||||||
|
`Dossier législatif ${dossierLegislatif.META.META_COMMUN.ID} has no échéancier.`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
const echeancierHtml = await fetchHtmlPage(
|
||||||
|
new URL(echeancierRelativeUrl, dossierLegislatifUrl).toString(),
|
||||||
|
)
|
||||||
|
const { document: echeancierDocument } = new JSDOM(echeancierHtml)
|
||||||
|
.window
|
||||||
|
const echeancierTable = echeancierDocument.querySelector(
|
||||||
|
"div.dossier-legislatif-detail-contenu > table",
|
||||||
|
)
|
||||||
|
if (echeancierTable === null) {
|
||||||
|
if (!silent) {
|
||||||
|
console.warn(
|
||||||
|
`Échéancier of dossier législatif ${dossierLegislatif.META.META_COMMUN.ID} has no table².`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
await fs.writeFile(
|
||||||
|
path.join(
|
||||||
|
echeanciersHtmlDir,
|
||||||
|
`${dossierLegislatif.META.META_COMMUN.ID}_echeancier_table.html`,
|
||||||
|
),
|
||||||
|
echeancierTable.outerHTML,
|
||||||
|
{ encoding: "utf-8" },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.error(
|
||||||
|
"An error occurred while handling dossier législatif XML file",
|
||||||
|
filePath,
|
||||||
|
)
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cd(echeanciersHtmlDirName)
|
||||||
|
await $`git add .`
|
||||||
|
if ((await $`git diff --quiet --staged`.exitCode) !== 0) {
|
||||||
|
changed = true
|
||||||
|
const message = `Récupération des tables des échéanciers - ${new Date().toISOString()}`
|
||||||
|
await $`git commit -m ${message} --quiet`
|
||||||
|
if (push) {
|
||||||
|
await $`git push`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cd("..")
|
||||||
|
|
||||||
|
return changed ? 0 : 10 // No new commit has been added to git repository.
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchHtmlPage(url: string): Promise<string> {
|
||||||
|
for (let retriesCount = 0; ; retriesCount++) {
|
||||||
|
const response = await fetch(url)
|
||||||
|
if (!response.ok) {
|
||||||
|
if (retriesCount === 0) {
|
||||||
|
console.warn(response.status, response.statusText)
|
||||||
|
console.warn(JSON.stringify(response.headers, null, 2))
|
||||||
|
console.warn(await response.text())
|
||||||
|
}
|
||||||
|
if (retriesCount < 10) {
|
||||||
|
await sleep(30)
|
||||||
|
console.info("Retrying…")
|
||||||
|
} else {
|
||||||
|
throw new Error(`Retrieval of HTML page at <${url}> failed`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return await response.text()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(s: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, s * 1000))
|
||||||
|
}
|
||||||
|
|
||||||
|
sade("download_dole_html <dilaDir>", true)
|
||||||
|
.describe(
|
||||||
|
"Download DOLE HTML to retrieve informations that lack in open data (links of decrees in échéanciers, etc)",
|
||||||
|
)
|
||||||
|
.option("-p, --push", "Push dataset repository")
|
||||||
|
.option("-r, --resume", "Resume import at given relative file path")
|
||||||
|
.option("-s, --silent", "Hide log messages")
|
||||||
|
.option("-v, --verbose", "Show all log messages")
|
||||||
|
.example(
|
||||||
|
"--resume dole/global/JORF/DOLE/00/00/36/07/36/JORFDOLE000036073697.xml ../dila-data/",
|
||||||
|
)
|
||||||
|
.action(async (dilaDir, options) => {
|
||||||
|
process.exit(await downloadDoleHtml(dilaDir, options))
|
||||||
|
})
|
||||||
|
.parse(process.argv)
|
Loading…
Reference in a new issue