Add script to download & update datasets
This commit is contained in:
parent
cd926a4491
commit
f0363f698f
4 changed files with 987 additions and 292 deletions
46
README.md
46
README.md
|
@ -47,6 +47,52 @@ Create a `.env` file to set configuration variables (you can use `example.env` a
|
|||
npm run configure
|
||||
```
|
||||
|
||||
## Datasets Initialization
|
||||
|
||||
```sh
|
||||
mkdir -p ../dila-data/dole
|
||||
cd ../dila-data/dole
|
||||
git init
|
||||
cd -
|
||||
npx tsx src/scripts/download_dila_dataset.ts dole
|
||||
|
||||
mkdir -p ../dila-data/jorf
|
||||
cd ../dila-data/jorf
|
||||
git init
|
||||
cd -
|
||||
npx tsx src/scripts/download_dila_dataset.ts jorf
|
||||
|
||||
mkdir -p ../dila-data/kali
|
||||
cd ../dila-data/kali
|
||||
git init
|
||||
cd -
|
||||
npx tsx src/scripts/download_dila_dataset.ts kali
|
||||
|
||||
mkdir -p ../dila-data/legi
|
||||
cd ../dila-data/legi
|
||||
git init
|
||||
cd -
|
||||
npx tsx src/scripts/download_dila_dataset.ts legi
|
||||
```
|
||||
|
||||
## Datasets Update
|
||||
|
||||
```sh
|
||||
npx tsx src/scripts/download_dila_dataset.ts dole
|
||||
npx tsx src/scripts/download_dila_dataset.ts jorf
|
||||
npx tsx src/scripts/download_dila_dataset.ts kali
|
||||
npx tsx src/scripts/download_dila_dataset.ts legi
|
||||
```
|
||||
|
||||
## Database Update
|
||||
|
||||
```sh
|
||||
npx tsx src/scripts/import_dole.ts
|
||||
npx tsx src/scripts/import_jorf.ts
|
||||
npx tsx src/scripts/import_kali.ts
|
||||
npx tsx src/scripts/import_legi.ts
|
||||
```
|
||||
|
||||
## Server Launch
|
||||
|
||||
In development mode:
|
||||
|
|
1055
package-lock.json
generated
1055
package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
@ -27,7 +27,7 @@
|
|||
"@iconify/svelte": "^3.0.0",
|
||||
"@playwright/test": "^1.22.2",
|
||||
"@sveltejs/adapter-node": "^1.0.0-next.100",
|
||||
"@sveltejs/kit": "^1.0.0-next.544",
|
||||
"@sveltejs/kit": "^1.0.0-next.546",
|
||||
"@sveltejs/package": "^1.0.0-next.5",
|
||||
"@tailwindcss/typography": "^0.5.3",
|
||||
"@tricoteuses/explorer-tools": "^0.2.0",
|
||||
|
@ -62,6 +62,7 @@
|
|||
"tslib": "^2.3.1",
|
||||
"tsx": "^3.12.1",
|
||||
"typescript": "^4.7.4",
|
||||
"vite": "^3.0.0"
|
||||
"vite": "^3.0.0",
|
||||
"zx": "^7.1.1"
|
||||
}
|
||||
}
|
||||
|
|
173
src/scripts/download_dila_dataset.ts
Normal file
173
src/scripts/download_dila_dataset.ts
Normal file
|
@ -0,0 +1,173 @@
|
|||
import assert from "assert"
|
||||
import { XMLParser } from "fast-xml-parser"
|
||||
import fs from "fs-extra"
|
||||
import path from "path"
|
||||
import sade from "sade"
|
||||
import { $, cd } from "zx"
|
||||
|
||||
async function downloadDataset(
|
||||
datasetName: string,
|
||||
{ push }: { push?: boolean } = {},
|
||||
): Promise<void> {
|
||||
const datasetNameUpper = datasetName.toUpperCase()
|
||||
const archivesUrl = `https://echanges.dila.gouv.fr/OPENDATA/${datasetNameUpper}/`
|
||||
const fullArchiveNameRegExp = new RegExp(
|
||||
`^Freemium_${datasetName}_global_(\\d{8}-\\d{6})\\.tar\\.gz$`,
|
||||
)
|
||||
const incrementalArchiveNameRegExp = new RegExp(
|
||||
`^${datasetNameUpper}_(\\d{8}-\\d{6})\\.tar\\.gz$`,
|
||||
)
|
||||
const response = await fetch(archivesUrl)
|
||||
assert(response.ok)
|
||||
const html = await response.text()
|
||||
const parsingOptions = {
|
||||
attributeNamePrefix: "@",
|
||||
htmlEntities: true,
|
||||
ignoreAttributes: false,
|
||||
// preserveOrder: true,
|
||||
processEntities: true,
|
||||
// In Dila server <pre> encapsulates the files list.
|
||||
// => It must not be a stop node.
|
||||
// stopNodes: ["*.pre", "*.script"],
|
||||
stopNodes: ["*.script"],
|
||||
// In Dila server <img> is not closed.
|
||||
// unpairedTags: ["br", "hr", "link", "meta"],
|
||||
unpairedTags: ["br", "hr", "img", "link", "meta"],
|
||||
}
|
||||
const parser = new XMLParser(parsingOptions)
|
||||
const dom = parser.parse(html)
|
||||
|
||||
let archivesA = dom.html.body.pre.a
|
||||
if (!Array.isArray(archivesA)) {
|
||||
archivesA = [archivesA]
|
||||
}
|
||||
const archiveNameByDate: { [date: string]: string } = {}
|
||||
let latestFullArchiveDate: string | undefined = undefined
|
||||
for (const { "@href": filename } of archivesA) {
|
||||
if (
|
||||
filename.startsWith("?") ||
|
||||
filename.endsWith(".pdf") ||
|
||||
filename === "/OPENDATA/"
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
const fullArchiveMatch = filename.match(fullArchiveNameRegExp)
|
||||
if (fullArchiveMatch !== null) {
|
||||
const date = fullArchiveMatch[1]
|
||||
archiveNameByDate[date] = filename
|
||||
if (latestFullArchiveDate === undefined || date > latestFullArchiveDate) {
|
||||
latestFullArchiveDate = date
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
const incrementalArchiveMatch = filename.match(incrementalArchiveNameRegExp)
|
||||
if (incrementalArchiveMatch !== null) {
|
||||
archiveNameByDate[incrementalArchiveMatch[1]] = filename
|
||||
continue
|
||||
}
|
||||
console.warn(`Unexpected file in Dila repository: ${filename}`)
|
||||
}
|
||||
assert.notStrictEqual(
|
||||
latestFullArchiveDate,
|
||||
undefined,
|
||||
`Dila's ${datasetNameUpper} repository doesn't contain a full archive.`,
|
||||
)
|
||||
|
||||
const dilaDir = path.join("..", "dila-data")
|
||||
cd(dilaDir)
|
||||
let latestArchiveName: string | undefined = undefined
|
||||
if (await fs.pathExists(path.join(dilaDir, datasetName))) {
|
||||
cd(datasetName)
|
||||
try {
|
||||
latestArchiveName = (await $`git log -1 --pretty=%B`).stdout.trim()
|
||||
} catch (processOutput) {
|
||||
// Git repository has no commit yet.
|
||||
}
|
||||
cd("..")
|
||||
}
|
||||
const latestArchiveDate = latestArchiveName?.match(/(\d{8}-\d{6})/)?.[1]
|
||||
let changed = false
|
||||
for (const [date, archiveName] of Object.entries(archiveNameByDate).sort(
|
||||
([date1], [date2]) => date1.localeCompare(date2),
|
||||
)) {
|
||||
if (latestArchiveDate != null && date <= latestArchiveDate) {
|
||||
continue
|
||||
}
|
||||
const archiveUrl = new URL(archiveName, archivesUrl).toString()
|
||||
await $`curl --remote-name --show-error --silent ${archiveUrl}`
|
||||
if (archiveName.match(fullArchiveNameRegExp) === null) {
|
||||
// Incremental archive
|
||||
await $`tar xzf ${archiveName}`
|
||||
// Most of the times an incremental archive is untared in ${date} directory,
|
||||
// but sometimes it is directly untared in ${datasetName} directory.
|
||||
if (await fs.pathExists(date)) {
|
||||
const archiveNodeNames = await fs.readdir(date)
|
||||
assert(
|
||||
archiveNodeNames.length <= 2,
|
||||
`Unexpected files or directories in archive: ${archiveNodeNames.join(
|
||||
", ",
|
||||
)}`,
|
||||
)
|
||||
await $`cp -r ${date}/${datasetName}/* ${datasetName}/`
|
||||
const removalListFilePath = path.join(
|
||||
date,
|
||||
`liste_suppression_${datasetName}.dat`,
|
||||
)
|
||||
if (await fs.pathExists(removalListFilePath)) {
|
||||
const filesPathToRemove = (
|
||||
await fs.readFile(removalListFilePath, "utf-8")
|
||||
)
|
||||
.split(/\r?\n/)
|
||||
.map((filePath) => filePath.trim())
|
||||
.filter((filePath) => filePath !== "")
|
||||
for (const filePathToRemove of filesPathToRemove) {
|
||||
await $`rm -f ${filePathToRemove}.xml`
|
||||
}
|
||||
}
|
||||
await $`rm -R ${date}`
|
||||
} else {
|
||||
const removalListFilePath = `liste_suppression_${datasetName}.dat`
|
||||
if (await fs.pathExists(removalListFilePath)) {
|
||||
const filesPathToRemove = (
|
||||
await fs.readFile(removalListFilePath, "utf-8")
|
||||
)
|
||||
.split(/\r?\n/)
|
||||
.map((filePath) => filePath.trim())
|
||||
.filter((filePath) => filePath !== "")
|
||||
for (const filePathToRemove of filesPathToRemove) {
|
||||
await $`rm -f ${filePathToRemove}.xml`
|
||||
}
|
||||
await $`rm ${removalListFilePath}`
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Full archive.
|
||||
// Note: Remove every files except .git repository.
|
||||
await $`rm -Rf ${datasetName}/*`
|
||||
await $`tar xzf ${archiveName}`
|
||||
}
|
||||
cd(datasetName)
|
||||
await $`git add .`
|
||||
if ((await $`git diff --quiet --staged`.exitCode) !== 0) {
|
||||
await $`git commit -m ${archiveName}`
|
||||
changed = true
|
||||
}
|
||||
cd("..")
|
||||
await $`rm ${archiveName}`
|
||||
}
|
||||
if (changed && push) {
|
||||
await $`git push`
|
||||
}
|
||||
}
|
||||
|
||||
sade("download_dila_dataset <dataset>", true)
|
||||
.describe("Download latest versions of a Dila dataset")
|
||||
.example("dole")
|
||||
.option("-p, --push", "Push dataset repository")
|
||||
.action(async (dataset, options) => {
|
||||
await downloadDataset(dataset, options)
|
||||
process.exit(0)
|
||||
})
|
||||
.parse(process.argv)
|
Loading…
Add table
Reference in a new issue