logiciels/tricoteuses-legifrance
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add dilaDar option to scripts

Browse source
This commit is contained in:
Emmanuel 2022-11-15 16:05:44 +01:00
parent 425da86cb9
commit 6006dd7a35
7 changed files with 53 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.gitlab-ci.yml
View file

@ -81,6 +81,7 @@ download:
script:
- rm -f .env
- ln -s example.env .env
- mkdir -p ../dila-data/
- git clone git@git.en-root.org:tricoteuses/data/dila/${DATASET}.git ../dila-data/${DATASET}
- npx tsx src/scripts/download_dila_dataset.ts ${DATASET} --push
- rm -Rf dila-data/
- mkdir dila-data/
- git clone git@git.en-root.org:tricoteuses/data/dila/${DATASET}.git dila-data/${DATASET}
- npx tsx src/scripts/download_dila_dataset.ts ${DATASET} dila-data/ --push

16
README.md
View file

@ -78,19 +78,19 @@ npx tsx src/scripts/download_dila_dataset.ts legi
## Datasets Update
```sh
npx tsx src/scripts/download_dila_dataset.ts dole
npx tsx src/scripts/download_dila_dataset.ts jorf
npx tsx src/scripts/download_dila_dataset.ts kali
npx tsx src/scripts/download_dila_dataset.ts legi
npx tsx src/scripts/download_dila_dataset.ts dole ../dila-data/
npx tsx src/scripts/download_dila_dataset.ts jorf ../dila-data/
npx tsx src/scripts/download_dila_dataset.ts kali ../dila-data/
npx tsx src/scripts/download_dila_dataset.ts legi ../dila-data/
```
## Database Update
```sh
npx tsx src/scripts/import_dole.ts
npx tsx src/scripts/import_jorf.ts
npx tsx src/scripts/import_kali.ts
npx tsx src/scripts/import_legi.ts
npx tsx src/scripts/import_dole.ts ../dila-data/
npx tsx src/scripts/import_jorf.ts ../dila-data/
npx tsx src/scripts/import_kali.ts ../dila-data/
npx tsx src/scripts/import_legi.ts ../dila-data/
```
## Server Launch

10
src/scripts/download_dila_dataset.ts
View file

@ -7,6 +7,7 @@ import { $, cd } from "zx"
async function downloadDataset(
datasetName: string,
dilaDir: string,
{ push }: { push?: boolean } = {},
): Promise<void> {
const datasetNameUpper = datasetName.toUpperCase()
@ -75,7 +76,6 @@ async function downloadDataset(
`Dila's ${datasetNameUpper} repository doesn't contain a full archive.`,
)
const dilaDir = path.join("..", "dila-data")
cd(dilaDir)
let latestArchiveName: string | undefined = undefined
if (await fs.pathExists(path.join(dilaDir, datasetName))) {
@ -164,12 +164,12 @@ async function downloadDataset(
}
}
sade("download_dila_dataset <dataset>", true)
sade("download_dila_dataset <dataset> <dilaDir>", true)
.describe("Download latest versions of a Dila dataset")
.example("dole")
.example("dole ../dila-data/s")
.option("-p, --push", "Push dataset repository")
.action(async (dataset, options) => {
await downloadDataset(dataset, options)
.action(async (dataset, dilaDir, options) => {
await downloadDataset(dataset, dilaDir, options)
process.exit(0)
})
.parse(process.argv)

15
src/scripts/import_dole.ts
View file

@ -21,7 +21,10 @@ const xmlParser = new XMLParser({
tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue),
})
async function importDole({ resume }: { resume?: string } = {}): Promise<void> {
async function importDole(
dilaDir: string,
{ resume }: { resume?: string } = {},
): Promise<void> {
let skip = resume !== undefined
const deleteRemainingIds = !skip
@ -34,7 +37,7 @@ async function importDole({ resume }: { resume?: string } = {}): Promise<void> {
).map(({ id }) => id),
)
const dataDir = path.join("..", "dila-data", "dole")
const dataDir = path.join(dilaDir, "dole")
assert(await fs.pathExists(dataDir))
iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) {
const relativePath = path.join(...relativeSplitPath)
@ -113,14 +116,14 @@ async function importDole({ resume }: { resume?: string } = {}): Promise<void> {
}
}
sade("import_dole", true)
sade("import_dole <dilaDir>", true)
.describe("Import Dila's DOLE database")
.option("-r, --resume", "Resume import at given relative file path")
.example(
"--resume dole/global/JORF/DOLE/00/00/36/07/36/JORFDOLE000036073697.xml",
"--resume dole/global/JORF/DOLE/00/00/36/07/36/JORFDOLE000036073697.xml ../dila-data/",
)
.action(async (options) => {
await importDole(options)
.action(async (dilaDir, options) => {
await importDole(dilaDir, options)
process.exit(0)
})
.parse(process.argv)

15
src/scripts/import_jorf.ts
View file

@ -39,7 +39,10 @@ const xmlParser = new XMLParser({
tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue),
})
async function importJorf({ resume }: { resume?: string } = {}): Promise<void> {
async function importJorf(
dilaDir: string,
{ resume }: { resume?: string } = {},
): Promise<void> {
let skip = resume !== undefined
const deleteRemainingIds = !skip
@ -104,7 +107,7 @@ async function importJorf({ resume }: { resume?: string } = {}): Promise<void> {
).map(({ eli }) => eli),
)
const dataDir = path.join("..", "dila-data", "jorf")
const dataDir = path.join(dilaDir, "dole")
assert(await fs.pathExists(dataDir))
iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) {
const relativePath = path.join(...relativeSplitPath)
@ -382,14 +385,14 @@ async function importJorf({ resume }: { resume?: string } = {}): Promise<void> {
}
}
sade("import_jorf", true)
sade("import_jorf <dilaDir>", true)
.describe("Import Dila's JORF database")
.option("-r, --resume", "Resume import at given relative file path")
.example(
"--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml",
"--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml ../dila-data/",
)
.action(async (options) => {
await importJorf(options)
.action(async (dilaDir, options) => {
await importJorf(dilaDir, options)
process.exit(0)
})
.parse(process.argv)

15
src/scripts/import_kali.ts
View file

@ -37,7 +37,10 @@ const xmlParser = new XMLParser({
tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue),
})
async function importKali({ resume }: { resume?: string } = {}): Promise<void> {
async function importKali(
dilaDir: string,
{ resume }: { resume?: string } = {},
): Promise<void> {
let skip = resume !== undefined
const deleteRemainingIds = !skip
@ -94,7 +97,7 @@ async function importKali({ resume }: { resume?: string } = {}): Promise<void> {
).map(({ id }) => id),
)
const dataDir = path.join("..", "dila-data", "kali")
const dataDir = path.join(dilaDir, "dole")
assert(await fs.pathExists(dataDir))
iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) {
const relativePath = path.join(...relativeSplitPath)
@ -313,14 +316,14 @@ async function importKali({ resume }: { resume?: string } = {}): Promise<void> {
}
}
sade("import_kali", true)
sade("import_kali <dilaDir>", true)
.describe("Import Dila's KALI database")
.option("-r, --resume", "Resume import at given relative file path")
.example(
"--resume global/conteneur/KALI/CONT/00/00/05/63/50/KALICONT000005635082.xml",
"--resume global/conteneur/KALI/CONT/00/00/05/63/50/KALICONT000005635082.xml ../dila-data/",
)
.action(async (options) => {
await importKali(options)
.action(async (dilaDir, options) => {
await importKali(dilaDir, options)
process.exit(0)
})
.parse(process.argv)

15
src/scripts/import_legi.ts
View file

@ -38,7 +38,10 @@ const xmlParser = new XMLParser({
tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue),
})
async function importLegi({ resume }: { resume?: string } = {}): Promise<void> {
async function importLegi(
dilaDir: string,
{ resume }: { resume?: string } = {},
): Promise<void> {
let skip = resume !== undefined
const deleteRemainingIds = !skip
@ -95,7 +98,7 @@ async function importLegi({ resume }: { resume?: string } = {}): Promise<void> {
).map(({ eli }) => eli),
)
const dataDir = path.join("..", "dila-data", "legi")
const dataDir = path.join(dilaDir, "dole")
assert(await fs.pathExists(dataDir))
iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) {
const relativePath = path.join(...relativeSplitPath)
@ -341,14 +344,14 @@ async function importLegi({ resume }: { resume?: string } = {}): Promise<void> {
}
}
sade("import_legi", true)
sade("import_legi <dilaDir>", true)
.describe("Import Dila's LEGI database")
.option("-r, --resume", "Resume import at given relative file path")
.example(
"--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml",
"--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml ../dila-data/",
)
.action(async (options) => {
await importLegi(options)
.action(async (dilaDir, options) => {
await importLegi(dilaDir, options)
process.exit(0)
})
.parse(process.argv)