From 6006dd7a3580a0527d44f42119cbd1414a7be7ec Mon Sep 17 00:00:00 2001 From: Emmanuel Raviart Date: Tue, 15 Nov 2022 16:05:44 +0100 Subject: [PATCH] Add dilaDar option to scripts --- .gitlab-ci.yml | 7 ++++--- README.md | 16 ++++++++-------- src/scripts/download_dila_dataset.ts | 10 +++++----- src/scripts/import_dole.ts | 15 +++++++++------ src/scripts/import_jorf.ts | 15 +++++++++------ src/scripts/import_kali.ts | 15 +++++++++------ src/scripts/import_legi.ts | 15 +++++++++------ 7 files changed, 53 insertions(+), 40 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11003a9..a0cab47 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -81,6 +81,7 @@ download: script: - rm -f .env - ln -s example.env .env - - mkdir -p ../dila-data/ - - git clone git@git.en-root.org:tricoteuses/data/dila/${DATASET}.git ../dila-data/${DATASET} - - npx tsx src/scripts/download_dila_dataset.ts ${DATASET} --push + - rm -Rf dila-data/ + - mkdir dila-data/ + - git clone git@git.en-root.org:tricoteuses/data/dila/${DATASET}.git dila-data/${DATASET} + - npx tsx src/scripts/download_dila_dataset.ts ${DATASET} dila-data/ --push diff --git a/README.md b/README.md index a3d5b8f..d7aa732 100644 --- a/README.md +++ b/README.md @@ -78,19 +78,19 @@ npx tsx src/scripts/download_dila_dataset.ts legi ## Datasets Update ```sh -npx tsx src/scripts/download_dila_dataset.ts dole -npx tsx src/scripts/download_dila_dataset.ts jorf -npx tsx src/scripts/download_dila_dataset.ts kali -npx tsx src/scripts/download_dila_dataset.ts legi +npx tsx src/scripts/download_dila_dataset.ts dole ../dila-data/ +npx tsx src/scripts/download_dila_dataset.ts jorf ../dila-data/ +npx tsx src/scripts/download_dila_dataset.ts kali ../dila-data/ +npx tsx src/scripts/download_dila_dataset.ts legi ../dila-data/ ``` ## Database Update ```sh -npx tsx src/scripts/import_dole.ts -npx tsx src/scripts/import_jorf.ts -npx tsx src/scripts/import_kali.ts -npx tsx src/scripts/import_legi.ts +npx tsx src/scripts/import_dole.ts ../dila-data/ +npx tsx src/scripts/import_jorf.ts ../dila-data/ +npx tsx src/scripts/import_kali.ts ../dila-data/ +npx tsx src/scripts/import_legi.ts ../dila-data/ ``` ## Server Launch diff --git a/src/scripts/download_dila_dataset.ts b/src/scripts/download_dila_dataset.ts index dd357e6..a9f3219 100644 --- a/src/scripts/download_dila_dataset.ts +++ b/src/scripts/download_dila_dataset.ts @@ -7,6 +7,7 @@ import { $, cd } from "zx" async function downloadDataset( datasetName: string, + dilaDir: string, { push }: { push?: boolean } = {}, ): Promise { const datasetNameUpper = datasetName.toUpperCase() @@ -75,7 +76,6 @@ async function downloadDataset( `Dila's ${datasetNameUpper} repository doesn't contain a full archive.`, ) - const dilaDir = path.join("..", "dila-data") cd(dilaDir) let latestArchiveName: string | undefined = undefined if (await fs.pathExists(path.join(dilaDir, datasetName))) { @@ -164,12 +164,12 @@ async function downloadDataset( } } -sade("download_dila_dataset ", true) +sade("download_dila_dataset ", true) .describe("Download latest versions of a Dila dataset") - .example("dole") + .example("dole ../dila-data/s") .option("-p, --push", "Push dataset repository") - .action(async (dataset, options) => { - await downloadDataset(dataset, options) + .action(async (dataset, dilaDir, options) => { + await downloadDataset(dataset, dilaDir, options) process.exit(0) }) .parse(process.argv) diff --git a/src/scripts/import_dole.ts b/src/scripts/import_dole.ts index 9315439..f2cf4da 100644 --- a/src/scripts/import_dole.ts +++ b/src/scripts/import_dole.ts @@ -21,7 +21,10 @@ const xmlParser = new XMLParser({ tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue), }) -async function importDole({ resume }: { resume?: string } = {}): Promise { +async function importDole( + dilaDir: string, + { resume }: { resume?: string } = {}, +): Promise { let skip = resume !== undefined const deleteRemainingIds = !skip @@ -34,7 +37,7 @@ async function importDole({ resume }: { resume?: string } = {}): Promise { ).map(({ id }) => id), ) - const dataDir = path.join("..", "dila-data", "dole") + const dataDir = path.join(dilaDir, "dole") assert(await fs.pathExists(dataDir)) iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) { const relativePath = path.join(...relativeSplitPath) @@ -113,14 +116,14 @@ async function importDole({ resume }: { resume?: string } = {}): Promise { } } -sade("import_dole", true) +sade("import_dole ", true) .describe("Import Dila's DOLE database") .option("-r, --resume", "Resume import at given relative file path") .example( - "--resume dole/global/JORF/DOLE/00/00/36/07/36/JORFDOLE000036073697.xml", + "--resume dole/global/JORF/DOLE/00/00/36/07/36/JORFDOLE000036073697.xml ../dila-data/", ) - .action(async (options) => { - await importDole(options) + .action(async (dilaDir, options) => { + await importDole(dilaDir, options) process.exit(0) }) .parse(process.argv) diff --git a/src/scripts/import_jorf.ts b/src/scripts/import_jorf.ts index afa99c9..b0711b7 100644 --- a/src/scripts/import_jorf.ts +++ b/src/scripts/import_jorf.ts @@ -39,7 +39,10 @@ const xmlParser = new XMLParser({ tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue), }) -async function importJorf({ resume }: { resume?: string } = {}): Promise { +async function importJorf( + dilaDir: string, + { resume }: { resume?: string } = {}, +): Promise { let skip = resume !== undefined const deleteRemainingIds = !skip @@ -104,7 +107,7 @@ async function importJorf({ resume }: { resume?: string } = {}): Promise { ).map(({ eli }) => eli), ) - const dataDir = path.join("..", "dila-data", "jorf") + const dataDir = path.join(dilaDir, "dole") assert(await fs.pathExists(dataDir)) iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) { const relativePath = path.join(...relativeSplitPath) @@ -382,14 +385,14 @@ async function importJorf({ resume }: { resume?: string } = {}): Promise { } } -sade("import_jorf", true) +sade("import_jorf ", true) .describe("Import Dila's JORF database") .option("-r, --resume", "Resume import at given relative file path") .example( - "--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml", + "--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml ../dila-data/", ) - .action(async (options) => { - await importJorf(options) + .action(async (dilaDir, options) => { + await importJorf(dilaDir, options) process.exit(0) }) .parse(process.argv) diff --git a/src/scripts/import_kali.ts b/src/scripts/import_kali.ts index 49aa713..bfb1f10 100644 --- a/src/scripts/import_kali.ts +++ b/src/scripts/import_kali.ts @@ -37,7 +37,10 @@ const xmlParser = new XMLParser({ tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue), }) -async function importKali({ resume }: { resume?: string } = {}): Promise { +async function importKali( + dilaDir: string, + { resume }: { resume?: string } = {}, +): Promise { let skip = resume !== undefined const deleteRemainingIds = !skip @@ -94,7 +97,7 @@ async function importKali({ resume }: { resume?: string } = {}): Promise { ).map(({ id }) => id), ) - const dataDir = path.join("..", "dila-data", "kali") + const dataDir = path.join(dilaDir, "dole") assert(await fs.pathExists(dataDir)) iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) { const relativePath = path.join(...relativeSplitPath) @@ -313,14 +316,14 @@ async function importKali({ resume }: { resume?: string } = {}): Promise { } } -sade("import_kali", true) +sade("import_kali ", true) .describe("Import Dila's KALI database") .option("-r, --resume", "Resume import at given relative file path") .example( - "--resume global/conteneur/KALI/CONT/00/00/05/63/50/KALICONT000005635082.xml", + "--resume global/conteneur/KALI/CONT/00/00/05/63/50/KALICONT000005635082.xml ../dila-data/", ) - .action(async (options) => { - await importKali(options) + .action(async (dilaDir, options) => { + await importKali(dilaDir, options) process.exit(0) }) .parse(process.argv) diff --git a/src/scripts/import_legi.ts b/src/scripts/import_legi.ts index e8c3e2e..4dc0cd2 100644 --- a/src/scripts/import_legi.ts +++ b/src/scripts/import_legi.ts @@ -38,7 +38,10 @@ const xmlParser = new XMLParser({ tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue), }) -async function importLegi({ resume }: { resume?: string } = {}): Promise { +async function importLegi( + dilaDir: string, + { resume }: { resume?: string } = {}, +): Promise { let skip = resume !== undefined const deleteRemainingIds = !skip @@ -95,7 +98,7 @@ async function importLegi({ resume }: { resume?: string } = {}): Promise { ).map(({ eli }) => eli), ) - const dataDir = path.join("..", "dila-data", "legi") + const dataDir = path.join(dilaDir, "dole") assert(await fs.pathExists(dataDir)) iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) { const relativePath = path.join(...relativeSplitPath) @@ -341,14 +344,14 @@ async function importLegi({ resume }: { resume?: string } = {}): Promise { } } -sade("import_legi", true) +sade("import_legi ", true) .describe("Import Dila's LEGI database") .option("-r, --resume", "Resume import at given relative file path") .example( - "--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml", + "--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml ../dila-data/", ) - .action(async (options) => { - await importLegi(options) + .action(async (dilaDir, options) => { + await importLegi(dilaDir, options) process.exit(0) }) .parse(process.argv)