Add script to download & update datasets

This commit is contained in:
Emmanuel 2022-11-15 14:35:57 +01:00
parent cd926a4491
commit f0363f698f
4 changed files with 987 additions and 292 deletions

View file

@ -47,6 +47,52 @@ Create a `.env` file to set configuration variables (you can use `example.env` a
npm run configure
```
## Datasets Initialization
```sh
mkdir -p ../dila-data/dole
cd ../dila-data/dole
git init
cd -
npx tsx src/scripts/download_dila_dataset.ts dole
mkdir -p ../dila-data/jorf
cd ../dila-data/jorf
git init
cd -
npx tsx src/scripts/download_dila_dataset.ts jorf
mkdir -p ../dila-data/kali
cd ../dila-data/kali
git init
cd -
npx tsx src/scripts/download_dila_dataset.ts kali
mkdir -p ../dila-data/legi
cd ../dila-data/legi
git init
cd -
npx tsx src/scripts/download_dila_dataset.ts legi
```
## Datasets Update
```sh
npx tsx src/scripts/download_dila_dataset.ts dole
npx tsx src/scripts/download_dila_dataset.ts jorf
npx tsx src/scripts/download_dila_dataset.ts kali
npx tsx src/scripts/download_dila_dataset.ts legi
```
## Database Update
```sh
npx tsx src/scripts/import_dole.ts
npx tsx src/scripts/import_jorf.ts
npx tsx src/scripts/import_kali.ts
npx tsx src/scripts/import_legi.ts
```
## Server Launch
In development mode:

1055
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -27,7 +27,7 @@
"@iconify/svelte": "^3.0.0",
"@playwright/test": "^1.22.2",
"@sveltejs/adapter-node": "^1.0.0-next.100",
"@sveltejs/kit": "^1.0.0-next.544",
"@sveltejs/kit": "^1.0.0-next.546",
"@sveltejs/package": "^1.0.0-next.5",
"@tailwindcss/typography": "^0.5.3",
"@tricoteuses/explorer-tools": "^0.2.0",
@ -62,6 +62,7 @@
"tslib": "^2.3.1",
"tsx": "^3.12.1",
"typescript": "^4.7.4",
"vite": "^3.0.0"
"vite": "^3.0.0",
"zx": "^7.1.1"
}
}

View file

@ -0,0 +1,173 @@
import assert from "assert"
import { XMLParser } from "fast-xml-parser"
import fs from "fs-extra"
import path from "path"
import sade from "sade"
import { $, cd } from "zx"
async function downloadDataset(
datasetName: string,
{ push }: { push?: boolean } = {},
): Promise<void> {
const datasetNameUpper = datasetName.toUpperCase()
const archivesUrl = `https://echanges.dila.gouv.fr/OPENDATA/${datasetNameUpper}/`
const fullArchiveNameRegExp = new RegExp(
`^Freemium_${datasetName}_global_(\\d{8}-\\d{6})\\.tar\\.gz$`,
)
const incrementalArchiveNameRegExp = new RegExp(
`^${datasetNameUpper}_(\\d{8}-\\d{6})\\.tar\\.gz$`,
)
const response = await fetch(archivesUrl)
assert(response.ok)
const html = await response.text()
const parsingOptions = {
attributeNamePrefix: "@",
htmlEntities: true,
ignoreAttributes: false,
// preserveOrder: true,
processEntities: true,
// In Dila server <pre> encapsulates the files list.
// => It must not be a stop node.
// stopNodes: ["*.pre", "*.script"],
stopNodes: ["*.script"],
// In Dila server <img> is not closed.
// unpairedTags: ["br", "hr", "link", "meta"],
unpairedTags: ["br", "hr", "img", "link", "meta"],
}
const parser = new XMLParser(parsingOptions)
const dom = parser.parse(html)
let archivesA = dom.html.body.pre.a
if (!Array.isArray(archivesA)) {
archivesA = [archivesA]
}
const archiveNameByDate: { [date: string]: string } = {}
let latestFullArchiveDate: string | undefined = undefined
for (const { "@href": filename } of archivesA) {
if (
filename.startsWith("?") ||
filename.endsWith(".pdf") ||
filename === "/OPENDATA/"
) {
continue
}
const fullArchiveMatch = filename.match(fullArchiveNameRegExp)
if (fullArchiveMatch !== null) {
const date = fullArchiveMatch[1]
archiveNameByDate[date] = filename
if (latestFullArchiveDate === undefined || date > latestFullArchiveDate) {
latestFullArchiveDate = date
}
continue
}
const incrementalArchiveMatch = filename.match(incrementalArchiveNameRegExp)
if (incrementalArchiveMatch !== null) {
archiveNameByDate[incrementalArchiveMatch[1]] = filename
continue
}
console.warn(`Unexpected file in Dila repository: ${filename}`)
}
assert.notStrictEqual(
latestFullArchiveDate,
undefined,
`Dila's ${datasetNameUpper} repository doesn't contain a full archive.`,
)
const dilaDir = path.join("..", "dila-data")
cd(dilaDir)
let latestArchiveName: string | undefined = undefined
if (await fs.pathExists(path.join(dilaDir, datasetName))) {
cd(datasetName)
try {
latestArchiveName = (await $`git log -1 --pretty=%B`).stdout.trim()
} catch (processOutput) {
// Git repository has no commit yet.
}
cd("..")
}
const latestArchiveDate = latestArchiveName?.match(/(\d{8}-\d{6})/)?.[1]
let changed = false
for (const [date, archiveName] of Object.entries(archiveNameByDate).sort(
([date1], [date2]) => date1.localeCompare(date2),
)) {
if (latestArchiveDate != null && date <= latestArchiveDate) {
continue
}
const archiveUrl = new URL(archiveName, archivesUrl).toString()
await $`curl --remote-name --show-error --silent ${archiveUrl}`
if (archiveName.match(fullArchiveNameRegExp) === null) {
// Incremental archive
await $`tar xzf ${archiveName}`
// Most of the times an incremental archive is untared in ${date} directory,
// but sometimes it is directly untared in ${datasetName} directory.
if (await fs.pathExists(date)) {
const archiveNodeNames = await fs.readdir(date)
assert(
archiveNodeNames.length <= 2,
`Unexpected files or directories in archive: ${archiveNodeNames.join(
", ",
)}`,
)
await $`cp -r ${date}/${datasetName}/* ${datasetName}/`
const removalListFilePath = path.join(
date,
`liste_suppression_${datasetName}.dat`,
)
if (await fs.pathExists(removalListFilePath)) {
const filesPathToRemove = (
await fs.readFile(removalListFilePath, "utf-8")
)
.split(/\r?\n/)
.map((filePath) => filePath.trim())
.filter((filePath) => filePath !== "")
for (const filePathToRemove of filesPathToRemove) {
await $`rm -f ${filePathToRemove}.xml`
}
}
await $`rm -R ${date}`
} else {
const removalListFilePath = `liste_suppression_${datasetName}.dat`
if (await fs.pathExists(removalListFilePath)) {
const filesPathToRemove = (
await fs.readFile(removalListFilePath, "utf-8")
)
.split(/\r?\n/)
.map((filePath) => filePath.trim())
.filter((filePath) => filePath !== "")
for (const filePathToRemove of filesPathToRemove) {
await $`rm -f ${filePathToRemove}.xml`
}
await $`rm ${removalListFilePath}`
}
}
} else {
// Full archive.
// Note: Remove every files except .git repository.
await $`rm -Rf ${datasetName}/*`
await $`tar xzf ${archiveName}`
}
cd(datasetName)
await $`git add .`
if ((await $`git diff --quiet --staged`.exitCode) !== 0) {
await $`git commit -m ${archiveName}`
changed = true
}
cd("..")
await $`rm ${archiveName}`
}
if (changed && push) {
await $`git push`
}
}
sade("download_dila_dataset <dataset>", true)
.describe("Download latest versions of a Dila dataset")
.example("dole")
.option("-p, --push", "Push dataset repository")
.action(async (dataset, options) => {
await downloadDataset(dataset, options)
process.exit(0)
})
.parse(process.argv)