Add dilaDar option to scripts

.gitlab-ci.yml

@@ -81,6 +81,7 @@ download:
   script:
     - rm -f .env
     - ln -s example.env .env
-    - mkdir -p ../dila-data/
+    - rm -Rf dila-data/
-    - git clone git@git.en-root.org:tricoteuses/data/dila/${DATASET}.git ../dila-data/${DATASET}
+    - mkdir dila-data/
-    - npx tsx src/scripts/download_dila_dataset.ts ${DATASET} --push
+    - git clone git@git.en-root.org:tricoteuses/data/dila/${DATASET}.git dila-data/${DATASET}
+    - npx tsx src/scripts/download_dila_dataset.ts ${DATASET} dila-data/ --push

README.md

@@ -78,19 +78,19 @@ npx tsx src/scripts/download_dila_dataset.ts legi
 ## Datasets Update
 
 ```sh
-npx tsx src/scripts/download_dila_dataset.ts dole
+npx tsx src/scripts/download_dila_dataset.ts dole ../dila-data/
-npx tsx src/scripts/download_dila_dataset.ts jorf
+npx tsx src/scripts/download_dila_dataset.ts jorf ../dila-data/
-npx tsx src/scripts/download_dila_dataset.ts kali
+npx tsx src/scripts/download_dila_dataset.ts kali ../dila-data/
-npx tsx src/scripts/download_dila_dataset.ts legi
+npx tsx src/scripts/download_dila_dataset.ts legi ../dila-data/
 ```
 
 ## Database Update
 
 ```sh
-npx tsx src/scripts/import_dole.ts
+npx tsx src/scripts/import_dole.ts ../dila-data/
-npx tsx src/scripts/import_jorf.ts
+npx tsx src/scripts/import_jorf.ts ../dila-data/
-npx tsx src/scripts/import_kali.ts
+npx tsx src/scripts/import_kali.ts ../dila-data/
-npx tsx src/scripts/import_legi.ts
+npx tsx src/scripts/import_legi.ts ../dila-data/
 ```
 
 ## Server Launch

src/scripts/download_dila_dataset.ts

@@ -7,6 +7,7 @@ import { $, cd } from "zx"
 
 async function downloadDataset(
   datasetName: string,
+  dilaDir: string,
   { push }: { push?: boolean } = {},
 ): Promise<void> {
   const datasetNameUpper = datasetName.toUpperCase()
@@ -75,7 +76,6 @@ async function downloadDataset(
     `Dila's ${datasetNameUpper} repository doesn't contain a full archive.`,
   )
 
-  const dilaDir = path.join("..", "dila-data")
   cd(dilaDir)
   let latestArchiveName: string | undefined = undefined
   if (await fs.pathExists(path.join(dilaDir, datasetName))) {
@@ -164,12 +164,12 @@ async function downloadDataset(
   }
 }
 
-sade("download_dila_dataset <dataset>", true)
+sade("download_dila_dataset <dataset> <dilaDir>", true)
   .describe("Download latest versions of a Dila dataset")
-  .example("dole")
+  .example("dole ../dila-data/s")
   .option("-p, --push", "Push dataset repository")
-  .action(async (dataset, options) => {
+  .action(async (dataset, dilaDir, options) => {
-    await downloadDataset(dataset, options)
+    await downloadDataset(dataset, dilaDir, options)
     process.exit(0)
   })
   .parse(process.argv)

src/scripts/import_dole.ts

@@ -21,7 +21,10 @@ const xmlParser = new XMLParser({
   tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue),
 })
 
-async function importDole({ resume }: { resume?: string } = {}): Promise<void> {
+async function importDole(
+  dilaDir: string,
+  { resume }: { resume?: string } = {},
+): Promise<void> {
   let skip = resume !== undefined
   const deleteRemainingIds = !skip
 
@@ -34,7 +37,7 @@ async function importDole({ resume }: { resume?: string } = {}): Promise<void> {
     ).map(({ id }) => id),
   )
 
-  const dataDir = path.join("..", "dila-data", "dole")
+  const dataDir = path.join(dilaDir, "dole")
   assert(await fs.pathExists(dataDir))
   iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) {
     const relativePath = path.join(...relativeSplitPath)
@@ -113,14 +116,14 @@ async function importDole({ resume }: { resume?: string } = {}): Promise<void> {
   }
 }
 
-sade("import_dole", true)
+sade("import_dole <dilaDir>", true)
   .describe("Import Dila's DOLE database")
   .option("-r, --resume", "Resume import at given relative file path")
   .example(
-    "--resume dole/global/JORF/DOLE/00/00/36/07/36/JORFDOLE000036073697.xml",
+    "--resume dole/global/JORF/DOLE/00/00/36/07/36/JORFDOLE000036073697.xml ../dila-data/",
   )
-  .action(async (options) => {
+  .action(async (dilaDir, options) => {
-    await importDole(options)
+    await importDole(dilaDir, options)
     process.exit(0)
   })
   .parse(process.argv)

src/scripts/import_jorf.ts

@@ -39,7 +39,10 @@ const xmlParser = new XMLParser({
   tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue),
 })
 
-async function importJorf({ resume }: { resume?: string } = {}): Promise<void> {
+async function importJorf(
+  dilaDir: string,
+  { resume }: { resume?: string } = {},
+): Promise<void> {
   let skip = resume !== undefined
   const deleteRemainingIds = !skip
 
@@ -104,7 +107,7 @@ async function importJorf({ resume }: { resume?: string } = {}): Promise<void> {
     ).map(({ eli }) => eli),
   )
 
-  const dataDir = path.join("..", "dila-data", "jorf")
+  const dataDir = path.join(dilaDir, "dole")
   assert(await fs.pathExists(dataDir))
   iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) {
     const relativePath = path.join(...relativeSplitPath)
@@ -382,14 +385,14 @@ async function importJorf({ resume }: { resume?: string } = {}): Promise<void> {
   }
 }
 
-sade("import_jorf", true)
+sade("import_jorf <dilaDir>", true)
   .describe("Import Dila's JORF database")
   .option("-r, --resume", "Resume import at given relative file path")
   .example(
-    "--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml",
+    "--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml ../dila-data/",
   )
-  .action(async (options) => {
+  .action(async (dilaDir, options) => {
-    await importJorf(options)
+    await importJorf(dilaDir, options)
     process.exit(0)
   })
   .parse(process.argv)

src/scripts/import_kali.ts

@@ -37,7 +37,10 @@ const xmlParser = new XMLParser({
   tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue),
 })
 
-async function importKali({ resume }: { resume?: string } = {}): Promise<void> {
+async function importKali(
+  dilaDir: string,
+  { resume }: { resume?: string } = {},
+): Promise<void> {
   let skip = resume !== undefined
   const deleteRemainingIds = !skip
 
@@ -94,7 +97,7 @@ async function importKali({ resume }: { resume?: string } = {}): Promise<void> {
     ).map(({ id }) => id),
   )
 
-  const dataDir = path.join("..", "dila-data", "kali")
+  const dataDir = path.join(dilaDir, "dole")
   assert(await fs.pathExists(dataDir))
   iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) {
     const relativePath = path.join(...relativeSplitPath)
@@ -313,14 +316,14 @@ async function importKali({ resume }: { resume?: string } = {}): Promise<void> {
   }
 }
 
-sade("import_kali", true)
+sade("import_kali <dilaDir>", true)
   .describe("Import Dila's KALI database")
   .option("-r, --resume", "Resume import at given relative file path")
   .example(
-    "--resume global/conteneur/KALI/CONT/00/00/05/63/50/KALICONT000005635082.xml",
+    "--resume global/conteneur/KALI/CONT/00/00/05/63/50/KALICONT000005635082.xml ../dila-data/",
   )
-  .action(async (options) => {
+  .action(async (dilaDir, options) => {
-    await importKali(options)
+    await importKali(dilaDir, options)
     process.exit(0)
   })
   .parse(process.argv)

src/scripts/import_legi.ts

@@ -38,7 +38,10 @@ const xmlParser = new XMLParser({
   tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue),
 })
 
-async function importLegi({ resume }: { resume?: string } = {}): Promise<void> {
+async function importLegi(
+  dilaDir: string,
+  { resume }: { resume?: string } = {},
+): Promise<void> {
   let skip = resume !== undefined
   const deleteRemainingIds = !skip
 
@@ -95,7 +98,7 @@ async function importLegi({ resume }: { resume?: string } = {}): Promise<void> {
     ).map(({ eli }) => eli),
   )
 
-  const dataDir = path.join("..", "dila-data", "legi")
+  const dataDir = path.join(dilaDir, "dole")
   assert(await fs.pathExists(dataDir))
   iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) {
     const relativePath = path.join(...relativeSplitPath)
@@ -341,14 +344,14 @@ async function importLegi({ resume }: { resume?: string } = {}): Promise<void> {
   }
 }
 
-sade("import_legi", true)
+sade("import_legi <dilaDir>", true)
   .describe("Import Dila's LEGI database")
   .option("-r, --resume", "Resume import at given relative file path")
   .example(
-    "--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml",
+    "--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml ../dila-data/",
   )
-  .action(async (options) => {
+  .action(async (dilaDir, options) => {
-    await importLegi(options)
+    await importLegi(dilaDir, options)
     process.exit(0)
   })
   .parse(process.argv)