Add JO validation and add category option to import scripts

This commit is contained in:
Emmanuel 2023-02-11 21:25:48 +01:00
parent 126aebd7b1
commit bf3cda1a09
8 changed files with 1010 additions and 449 deletions

View file

@ -23,6 +23,22 @@ import {
allJorfArticleTypesMutable, allJorfArticleTypesMutable,
} from "$lib/legal" } from "$lib/legal"
export const jorfArticleStats: {
// countByEtat: { [etat: string]: number }
// countByLienArtEtat: { [etat: string]: number }
// countByLienNature: { [nature: string]: number }
// countByLienType: { [type: string]: number }
// countByTexteNature: { [nature: string]: number }
// countByVersionEtat: { [etat: string]: number }
} = {
// countByEtat: {},
// countByLienArtEtat: {},
// countByLienNature: {},
// countByLienType: {},
// countByTexteNature: {},
// countByVersionEtat: {},
}
function auditBlocTextuel( function auditBlocTextuel(
audit: Audit, audit: Audit,
dataUnknown: unknown, dataUnknown: unknown,

View file

@ -1 +1,2 @@
export { auditJorfArticle } from "./articles" export { auditJo, joStats } from "./jo"
export { auditJorfArticle, jorfArticleStats } from "./articles"

376
src/lib/auditors/jorf/jo.ts Normal file
View file

@ -0,0 +1,376 @@
import {
type Audit,
auditRequire,
auditDateIso8601String,
auditTrimString,
auditInteger,
auditFunction,
auditEmptyToNull,
auditNullish,
auditSwitch,
auditNumber,
auditCleanArray,
auditOptions,
auditHttpUrl,
auditStringToNumber,
} from "@auditors/core"
import { allJoNaturesMutable, allJoOriginesMutable } from "$lib/legal"
export const joStats: {
countByNature: { [nature: string]: number }
countByOrigine: { [origine: string]: number }
} = {
countByNature: {},
countByOrigine: {},
}
export function auditJo(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"META",
true,
errors,
remainingKeys,
auditMeta,
auditRequire,
)
audit.attribute(
data,
"STRUCTURE_TXT",
true,
errors,
remainingKeys,
auditSwitch(
[auditTrimString, auditEmptyToNull, auditNullish],
auditStructureTxt,
),
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditLienTxt(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
for (const key of ["@idtxt", "@titretxt"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
}
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMeta(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"META_COMMUN",
true,
errors,
remainingKeys,
auditMetaCommun,
auditRequire,
)
audit.attribute(
data,
"META_SPEC",
true,
errors,
remainingKeys,
auditMetaSpec,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMetaCommun(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"ANCIEN_ID",
true,
errors,
remainingKeys,
auditSwitch(
[auditNumber, auditInteger, auditFunction((id) => id.toString())],
[auditTrimString, auditEmptyToNull],
),
auditNullish,
)
for (const key of ["ID", "URL"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
}
audit.attribute(
data,
"ID_ELI",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditHttpUrl,
)
audit.attribute(
data,
"NATURE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
// auditFunction((nature) => {
// joStats.countByNature[nature] = (joStats.countByNature[nature] ?? 0) + 1
// return nature
// }),
auditOptions(allJoNaturesMutable),
auditRequire,
)
audit.attribute(
data,
"ORIGINE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
// auditFunction((origine) => {
// joStats.countByOrigine[origine] =
// (joStats.countByOrigine[origine] ?? 0) + 1
// return origine
// }),
auditOptions(allJoOriginesMutable),
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMetaConteneur(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"DATE_PUBLI",
true,
errors,
remainingKeys,
auditDateIso8601String,
auditRequire,
)
audit.attribute(
data,
"NUM",
true,
errors,
remainingKeys,
auditSwitch(
[auditNumber, auditFunction((num) => num.toString())],
[auditTrimString, auditEmptyToNull],
),
)
audit.attribute(
data,
"TITRE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMetaSpec(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"META_CONTENEUR",
true,
errors,
remainingKeys,
auditMetaConteneur,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditStructureTxt(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"LIEN_TXT",
true,
errors,
remainingKeys,
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
auditCleanArray(auditLienTxt, auditRequire),
)
audit.attribute(
data,
"TM",
true,
errors,
remainingKeys,
auditFunction((tm) => (Array.isArray(tm) ? tm : [tm])),
auditCleanArray(auditTm, auditRequire),
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
/// Table des matières
function auditTm(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"@niv",
true,
errors,
remainingKeys,
auditStringToNumber,
auditNumber,
auditRequire,
)
audit.attribute(
data,
"LIEN_TXT",
true,
errors,
remainingKeys,
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
auditCleanArray(auditLienTxt, auditRequire),
)
audit.attribute(
data,
"TITRE_TM",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
audit.attribute(
data,
"TM",
true,
errors,
remainingKeys,
auditFunction((tm) => (Array.isArray(tm) ? tm : [tm])),
auditCleanArray(auditTm, auditRequire),
)
return audit.reduceRemaining(data, errors, remainingKeys)
}

View file

@ -20,6 +20,10 @@ export { default as TextelrView } from "./components/TextelrView.svelte"
export { default as TexteVersionView } from "./components/TexteVersionView.svelte" export { default as TexteVersionView } from "./components/TexteVersionView.svelte"
export { export {
allJoNatures,
allJoNaturesMutable,
allJoOrigines,
allJoOriginesMutable,
allJorfArticleEtats, allJorfArticleEtats,
allJorfArticleEtatsMutable, allJorfArticleEtatsMutable,
allJorfArticleLienArticleOrigines, allJorfArticleLienArticleOrigines,
@ -49,6 +53,8 @@ export {
allLegiArticleTypes, allLegiArticleTypes,
allLegiArticleTypesMutable, allLegiArticleTypesMutable,
type DossierLegislatif, type DossierLegislatif,
type JoNature,
type JoOrigine,
type JorfArticle, type JorfArticle,
type JorfArticleEtat, type JorfArticleEtat,
type JorfArticleLienArticleOrigine, type JorfArticleLienArticleOrigine,

View file

@ -4,6 +4,10 @@ import type { DossierLegislatif } from "./dole"
export type { DossierLegislatif } from "./dole" export type { DossierLegislatif } from "./dole"
export { export {
allJoNatures,
allJoNaturesMutable,
allJoOrigines,
allJoOriginesMutable,
allJorfArticleEtats, allJorfArticleEtats,
allJorfArticleEtatsMutable, allJorfArticleEtatsMutable,
allJorfArticleLienArticleOrigines, allJorfArticleLienArticleOrigines,
@ -16,6 +20,8 @@ export {
allJorfArticleTexteNaturesMutable, allJorfArticleTexteNaturesMutable,
allJorfArticleTypes, allJorfArticleTypes,
allJorfArticleTypesMutable, allJorfArticleTypesMutable,
type JoNature,
type JoOrigine,
type JorfArticle, type JorfArticle,
type JorfArticleEtat, type JorfArticleEtat,
type JorfArticleLienArticleOrigine, type JorfArticleLienArticleOrigine,

View file

@ -59,6 +59,10 @@ export interface JorfArticle {
} }
} }
export type JoNature = (typeof allJoNatures)[number]
export type JoOrigine = (typeof allJoOrigines)[number]
export type JorfArticleEtat = (typeof allJorfArticleEtats)[number] export type JorfArticleEtat = (typeof allJorfArticleEtats)[number]
export type JorfArticleLienArticleOrigine = export type JorfArticleLienArticleOrigine =
@ -83,6 +87,12 @@ export interface JorfArticleTm {
TM?: JorfArticleTm TM?: JorfArticleTm
} }
export const allJoNatures = ["JO"] as const
export const allJoNaturesMutable = [...allJoNatures]
export const allJoOrigines = ["JORF"] as const
export const allJoOriginesMutable = [...allJoOrigines]
export const allJorfArticleEtats = [ export const allJorfArticleEtats = [
"ABROGE", "ABROGE",
"ABROGE_DIFF", "ABROGE_DIFF",

View file

@ -1,4 +1,9 @@
import { auditChain, auditRequire, strictAudit } from "@auditors/core" import {
auditChain,
auditOptions,
auditRequire,
strictAudit,
} from "@auditors/core"
import assert from "assert" import assert from "assert"
import { XMLParser } from "fast-xml-parser" import { XMLParser } from "fast-xml-parser"
import fs from "fs-extra" import fs from "fs-extra"
@ -8,7 +13,11 @@ import type { JSONValue } from "postgres"
import sade from "sade" import sade from "sade"
import { auditId, auditVersions } from "$lib/auditors/legal" import { auditId, auditVersions } from "$lib/auditors/legal"
import { auditJorfArticle } from "$lib/auditors/jorf" import {
auditJo,
auditJorfArticle,
joStats,
} from "$lib/auditors/jorf"
import type { import type {
Jo, Jo,
JorfArticle, JorfArticle,
@ -21,6 +30,18 @@ import type {
import { db } from "$lib/server/database" import { db } from "$lib/server/database"
import { walkDir } from "$lib/server/file_systems" import { walkDir } from "$lib/server/file_systems"
type CategoryTag = (typeof allCategoriesCode)[number]
const allCategoriesCode = [
"ARTICLE",
"ID",
"JO",
"SECTION_TA",
"TEXTE_VERSION",
"TEXTELR",
"VERSIONS",
] as const
const xmlParser = new XMLParser({ const xmlParser = new XMLParser({
attributeNamePrefix: "@", attributeNamePrefix: "@",
ignoreAttributes: false, ignoreAttributes: false,
@ -42,12 +63,27 @@ const xmlParser = new XMLParser({
async function importJorf( async function importJorf(
dilaDir: string, dilaDir: string,
{ resume }: { resume?: string } = {}, { category, resume }: { category?: string; resume?: string } = {},
): Promise<void> { ): Promise<void> {
const [categoryTag, categoryError] = auditOptions([
...[...allCategoriesCode],
])(strictAudit, category) as [CategoryTag | undefined, unknown]
assert.strictEqual(
categoryError,
null,
`Error for category ${JSON.stringify(categoryTag)}:\n${JSON.stringify(
categoryError,
null,
2,
)}`,
)
let skip = resume !== undefined let skip = resume !== undefined
const deleteRemainingIds = !skip const deleteRemainingIds = !skip
const articleRemainingIds = new Set( const articleRemainingIds =
categoryTag === undefined || categoryTag === "ARTICLE"
? new Set(
( (
await db<{ id: string }[]>` await db<{ id: string }[]>`
SELECT id SELECT id
@ -56,7 +92,10 @@ async function importJorf(
` `
).map(({ id }) => id), ).map(({ id }) => id),
) )
const idRemainingElis = new Set( : new Set<string>()
const idRemainingElis =
categoryTag === undefined || categoryTag === "ID"
? new Set(
( (
await db<{ eli: string }[]>` await db<{ eli: string }[]>`
SELECT eli SELECT eli
@ -64,7 +103,10 @@ async function importJorf(
` `
).map(({ eli }) => eli), ).map(({ eli }) => eli),
) )
const joRemainingIds = new Set( : new Set<string>()
const joRemainingIds =
categoryTag === undefined || categoryTag === "JO"
? new Set(
( (
await db<{ id: string }[]>` await db<{ id: string }[]>`
SELECT id SELECT id
@ -72,7 +114,10 @@ async function importJorf(
` `
).map(({ id }) => id), ).map(({ id }) => id),
) )
const sectionTaRemainingIds = new Set( : new Set<string>()
const sectionTaRemainingIds =
categoryTag === undefined || categoryTag === "SECTION_TA"
? new Set(
( (
await db<{ id: string }[]>` await db<{ id: string }[]>`
SELECT id SELECT id
@ -81,7 +126,10 @@ async function importJorf(
` `
).map(({ id }) => id), ).map(({ id }) => id),
) )
const textelrRemainingIds = new Set( : new Set<string>()
const textelrRemainingIds =
categoryTag === undefined || categoryTag === "TEXTELR"
? new Set(
( (
await db<{ id: string }[]>` await db<{ id: string }[]>`
SELECT id SELECT id
@ -90,7 +138,10 @@ async function importJorf(
` `
).map(({ id }) => id), ).map(({ id }) => id),
) )
const texteVersionRemainingIds = new Set( : new Set<string>()
const texteVersionRemainingIds =
categoryTag === undefined || categoryTag === "TEXTE_VERSION"
? new Set(
( (
await db<{ id: string }[]>` await db<{ id: string }[]>`
SELECT id SELECT id
@ -99,7 +150,10 @@ async function importJorf(
` `
).map(({ id }) => id), ).map(({ id }) => id),
) )
const versionsRemainingElis = new Set( : new Set<string>()
const versionsRemainingElis =
categoryTag === undefined || categoryTag === "VERSIONS"
? new Set(
( (
await db<{ eli: string }[]>` await db<{ eli: string }[]>`
SELECT eli SELECT eli
@ -107,6 +161,7 @@ async function importJorf(
` `
).map(({ eli }) => eli), ).map(({ eli }) => eli),
) )
: new Set<string>()
const dataDir = path.join(dilaDir, "jorf") const dataDir = path.join(dilaDir, "jorf")
assert(await fs.pathExists(dataDir)) assert(await fs.pathExists(dataDir))
@ -132,30 +187,31 @@ async function importJorf(
encoding: "utf8", encoding: "utf8",
}) })
const xmlData = xmlParser.parse(xmlString) const xmlData = xmlParser.parse(xmlString)
for (const [key, element] of Object.entries(xmlData) as [ for (const [tag, element] of Object.entries(xmlData) as [
string, CategoryTag | "?xml",
( (
| JorfArticle
| Jo | Jo
| SectionTa | JorfArticle
| JorfSectionTa
| Textelr | Textelr
| TexteVersion | TexteVersion
| Versions | Versions
| XmlHeader | XmlHeader
), ),
][]) { ][]) {
switch (key) { switch (tag) {
case "?xml": { case "?xml": {
const xmlHeader = element as XmlHeader const xmlHeader = element as XmlHeader
assert.strictEqual(xmlHeader["@encoding"], "UTF-8", filePath) assert.strictEqual(xmlHeader["@encoding"], "UTF-8", filePath)
assert.strictEqual(xmlHeader["@version"], "1.0", filePath) assert.strictEqual(xmlHeader["@version"], "1.0", filePath)
break break
} }
case "ARTICLE": { case "ARTICLE":
const [article, error] = auditChain(auditJorfArticle, auditRequire)( if (categoryTag === undefined || categoryTag === tag) {
strictAudit, const [article, error] = auditChain(
element, auditJorfArticle,
) as [JorfArticle, unknown] auditRequire,
)(strictAudit, element) as [JorfArticle, unknown]
assert.strictEqual( assert.strictEqual(
error, error,
null, null,
@ -178,9 +234,10 @@ async function importJorf(
data = ${db.json(article as unknown as JSONValue)} data = ${db.json(article as unknown as JSONValue)}
` `
articleRemainingIds.delete(article.META.META_COMMUN.ID) articleRemainingIds.delete(article.META.META_COMMUN.ID)
break
} }
case "ID": { break
case "ID":
if (categoryTag === undefined || categoryTag === tag) {
assert.strictEqual(relativeSplitPath[0], "global") assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli") assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/") const eli = relativeSplitPath.slice(2, -1).join("/")
@ -210,10 +267,23 @@ async function importJorf(
id = ${id} id = ${id}
` `
idRemainingElis.delete(eli) idRemainingElis.delete(eli)
break
} }
case "JO": { break
const jo = element as Jo case "JO":
if (categoryTag === undefined || categoryTag === tag) {
const [jo, error] = auditChain(auditJo, auditRequire)(
strictAudit,
element,
) as [Jo, unknown]
assert.strictEqual(
error,
null,
`Unexpected format for JO:\n${JSON.stringify(
jo,
null,
2,
)}\nError:\n${JSON.stringify(error, null, 2)}`,
)
await db` await db`
INSERT INTO jo ( INSERT INTO jo (
id, id,
@ -227,10 +297,23 @@ async function importJorf(
data = ${db.json(jo as unknown as JSONValue)} data = ${db.json(jo as unknown as JSONValue)}
` `
joRemainingIds.delete(jo.META.META_COMMUN.ID) joRemainingIds.delete(jo.META.META_COMMUN.ID)
break
} }
case "SECTION_TA": { break
const section = element as SectionTa case "SECTION_TA":
if (categoryTag === undefined || categoryTag === tag) {
const [section, error] = auditChain(
auditJorfSectionTa,
auditRequire,
)(strictAudit, element) as [JorfSectionTa, unknown]
assert.strictEqual(
error,
null,
`Unexpected format for SECTION_TA:\n${JSON.stringify(
section,
null,
2,
)}\nError:\n${JSON.stringify(error, null, 2)}`,
)
await db` await db`
INSERT INTO section_ta ( INSERT INTO section_ta (
id, id,
@ -244,9 +327,10 @@ async function importJorf(
data = ${db.json(section as unknown as JSONValue)} data = ${db.json(section as unknown as JSONValue)}
` `
sectionTaRemainingIds.delete(section.ID) sectionTaRemainingIds.delete(section.ID)
break
} }
case "TEXTE_VERSION": { break
case "TEXTE_VERSION":
if (categoryTag === undefined || categoryTag === tag) {
const texteVersion = element as TexteVersion const texteVersion = element as TexteVersion
const textAFragments = [ const textAFragments = [
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE, texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE,
@ -275,9 +359,10 @@ async function importJorf(
)}), 'A') )}), 'A')
` `
texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID) texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID)
break
} }
case "TEXTELR": { break
case "TEXTELR":
if (categoryTag === undefined || categoryTag === tag) {
const textelr = element as Textelr const textelr = element as Textelr
await db` await db`
INSERT INTO textelr ( INSERT INTO textelr (
@ -292,9 +377,10 @@ async function importJorf(
data = ${db.json(textelr as unknown as JSONValue)} data = ${db.json(textelr as unknown as JSONValue)}
` `
textelrRemainingIds.delete(textelr.META.META_COMMUN.ID) textelrRemainingIds.delete(textelr.META.META_COMMUN.ID)
break
} }
case "VERSIONS": { break
case "VERSIONS":
if (categoryTag === undefined || categoryTag === tag) {
assert.strictEqual(relativeSplitPath[0], "global") assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli") assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/") const eli = relativeSplitPath.slice(2, -1).join("/")
@ -328,11 +414,11 @@ async function importJorf(
data = ${db.json(versions as unknown as JSONValue)} data = ${db.json(versions as unknown as JSONValue)}
` `
versionsRemainingElis.delete(id) versionsRemainingElis.delete(id)
break
} }
break
default: { default: {
console.warn( console.warn(
`Unexpected root element "${key}" in XML file: ${filePath}`, `Unexpected root element "${tag}" in XML file: ${filePath}`,
) )
break iterXmlFiles break iterXmlFiles
} }
@ -395,10 +481,17 @@ async function importJorf(
` `
} }
} }
// console.log(
// "JORF articles stats =",
// JSON.stringify(jorfArticleStats, null, 2),
// )
console.log("JO stats =", JSON.stringify(joStats, null, 2))
} }
sade("import_jorf <dilaDir>", true) sade("import_jorf <dilaDir>", true)
.describe("Import Dila's JORF database") .describe("Import Dila's JORF database")
.option("-k, --category", "Import only given type of data")
.option("-r, --resume", "Resume import at given relative file path") .option("-r, --resume", "Resume import at given relative file path")
.example( .example(
"--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml ../dila-data/", "--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml ../dila-data/",

View file

@ -1,4 +1,9 @@
import { auditChain, auditRequire, strictAudit } from "@auditors/core" import {
auditChain,
auditOptions,
auditRequire,
strictAudit,
} from "@auditors/core"
import assert from "assert" import assert from "assert"
import { XMLParser } from "fast-xml-parser" import { XMLParser } from "fast-xml-parser"
import fs from "fs-extra" import fs from "fs-extra"
@ -23,6 +28,17 @@ import type {
import { db } from "$lib/server/database" import { db } from "$lib/server/database"
import { walkDir } from "$lib/server/file_systems" import { walkDir } from "$lib/server/file_systems"
type CategoryTag = (typeof allCategoriesCode)[number]
const allCategoriesCode = [
"ARTICLE",
"ID",
"SECTION_TA",
"TEXTE_VERSION",
"TEXTELR",
"VERSIONS",
] as const
const xmlParser = new XMLParser({ const xmlParser = new XMLParser({
attributeNamePrefix: "@", attributeNamePrefix: "@",
ignoreAttributes: false, ignoreAttributes: false,
@ -44,12 +60,27 @@ const xmlParser = new XMLParser({
async function importLegi( async function importLegi(
dilaDir: string, dilaDir: string,
{ resume }: { resume?: string } = {}, { category, resume }: { category?: string; resume?: string } = {},
): Promise<void> { ): Promise<void> {
const [categoryTag, categoryError] = auditOptions([
...[...allCategoriesCode],
])(strictAudit, category) as [CategoryTag | undefined, unknown]
assert.strictEqual(
categoryError,
null,
`Error for category ${JSON.stringify(categoryTag)}:\n${JSON.stringify(
categoryError,
null,
2,
)}`,
)
let skip = resume !== undefined let skip = resume !== undefined
const deleteRemainingIds = !skip const deleteRemainingIds = !skip
const articleRemainingIds = new Set( const articleRemainingIds =
categoryTag === undefined || categoryTag === "ARTICLE"
? new Set(
( (
await db<{ id: string }[]>` await db<{ id: string }[]>`
SELECT id SELECT id
@ -58,7 +89,10 @@ async function importLegi(
` `
).map(({ id }) => id), ).map(({ id }) => id),
) )
const idRemainingElis = new Set( : new Set<string>()
const idRemainingElis =
categoryTag === undefined || categoryTag === "ID"
? new Set(
( (
await db<{ eli: string }[]>` await db<{ eli: string }[]>`
SELECT eli SELECT eli
@ -66,7 +100,10 @@ async function importLegi(
` `
).map(({ eli }) => eli), ).map(({ eli }) => eli),
) )
const sectionTaRemainingIds = new Set( : new Set<string>()
const sectionTaRemainingIds =
categoryTag === undefined || categoryTag === "SECTION_TA"
? new Set(
( (
await db<{ id: string }[]>` await db<{ id: string }[]>`
SELECT id SELECT id
@ -75,7 +112,10 @@ async function importLegi(
` `
).map(({ id }) => id), ).map(({ id }) => id),
) )
const textelrRemainingIds = new Set( : new Set<string>()
const textelrRemainingIds =
categoryTag === undefined || categoryTag === "TEXTELR"
? new Set(
( (
await db<{ id: string }[]>` await db<{ id: string }[]>`
SELECT id SELECT id
@ -84,7 +124,10 @@ async function importLegi(
` `
).map(({ id }) => id), ).map(({ id }) => id),
) )
const texteVersionRemainingIds = new Set( : new Set<string>()
const texteVersionRemainingIds =
categoryTag === undefined || categoryTag === "TEXTE_VERSION"
? new Set(
( (
await db<{ id: string }[]>` await db<{ id: string }[]>`
SELECT id SELECT id
@ -93,7 +136,10 @@ async function importLegi(
` `
).map(({ id }) => id), ).map(({ id }) => id),
) )
const versionsRemainingElis = new Set( : new Set<string>()
const versionsRemainingElis =
categoryTag === undefined || categoryTag === "VERSIONS"
? new Set(
( (
await db<{ eli: string }[]>` await db<{ eli: string }[]>`
SELECT eli SELECT eli
@ -101,6 +147,7 @@ async function importLegi(
` `
).map(({ eli }) => eli), ).map(({ eli }) => eli),
) )
: new Set<string>()
const dataDir = path.join(dilaDir, "legi") const dataDir = path.join(dilaDir, "legi")
assert(await fs.pathExists(dataDir)) assert(await fs.pathExists(dataDir))
@ -126,22 +173,23 @@ async function importLegi(
encoding: "utf8", encoding: "utf8",
}) })
const xmlData = xmlParser.parse(xmlString) const xmlData = xmlParser.parse(xmlString)
for (const [key, element] of Object.entries(xmlData) as [ for (const [tag, element] of Object.entries(xmlData) as [
string, CategoryTag | "?xml",
LegiArticle | SectionTa | Textelr | TexteVersion | Versions | XmlHeader, LegiArticle | SectionTa | Textelr | TexteVersion | Versions | XmlHeader,
][]) { ][]) {
switch (key) { switch (tag) {
case "?xml": { case "?xml": {
const xmlHeader = element as XmlHeader const xmlHeader = element as XmlHeader
assert.strictEqual(xmlHeader["@encoding"], "UTF-8", filePath) assert.strictEqual(xmlHeader["@encoding"], "UTF-8", filePath)
assert.strictEqual(xmlHeader["@version"], "1.0", filePath) assert.strictEqual(xmlHeader["@version"], "1.0", filePath)
break break
} }
case "ARTICLE": { case "ARTICLE":
const [article, error] = auditChain(auditLegiArticle, auditRequire)( if (categoryTag === undefined || categoryTag === tag) {
strictAudit, const [article, error] = auditChain(
element, auditLegiArticle,
) as [LegiArticle, unknown] auditRequire,
)(strictAudit, element) as [LegiArticle, unknown]
assert.strictEqual( assert.strictEqual(
error, error,
null, null,
@ -164,9 +212,10 @@ async function importLegi(
data = ${db.json(article as unknown as JSONValue)} data = ${db.json(article as unknown as JSONValue)}
` `
articleRemainingIds.delete(article.META.META_COMMUN.ID) articleRemainingIds.delete(article.META.META_COMMUN.ID)
break
} }
case "ID": { break
case "ID":
if (categoryTag === undefined || categoryTag === tag) {
assert.strictEqual(relativeSplitPath[0], "global") assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli") assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/") const eli = relativeSplitPath.slice(2, -1).join("/")
@ -197,9 +246,10 @@ async function importLegi(
id = ${id} id = ${id}
` `
idRemainingElis.delete(eli) idRemainingElis.delete(eli)
break
} }
case "SECTION_TA": { break
case "SECTION_TA":
if (categoryTag === undefined || categoryTag === tag) {
const section = element as SectionTa const section = element as SectionTa
await db` await db`
INSERT INTO section_ta ( INSERT INTO section_ta (
@ -214,9 +264,10 @@ async function importLegi(
data = ${db.json(section as unknown as JSONValue)} data = ${db.json(section as unknown as JSONValue)}
` `
sectionTaRemainingIds.delete(section.ID) sectionTaRemainingIds.delete(section.ID)
break
} }
case "TEXTE_VERSION": { break
case "TEXTE_VERSION":
if (categoryTag === undefined || categoryTag === tag) {
const texteVersion = element as TexteVersion const texteVersion = element as TexteVersion
const textAFragments = [ const textAFragments = [
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE, texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE,
@ -245,9 +296,10 @@ async function importLegi(
)}), 'A') )}), 'A')
` `
texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID) texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID)
break
} }
case "TEXTELR": { break
case "TEXTELR":
if (categoryTag === undefined || categoryTag === tag) {
const textelr = element as Textelr const textelr = element as Textelr
await db` await db`
INSERT INTO textelr ( INSERT INTO textelr (
@ -262,9 +314,10 @@ async function importLegi(
data = ${db.json(textelr as unknown as JSONValue)} data = ${db.json(textelr as unknown as JSONValue)}
` `
textelrRemainingIds.delete(textelr.META.META_COMMUN.ID) textelrRemainingIds.delete(textelr.META.META_COMMUN.ID)
break
} }
case "VERSIONS": { break
case "VERSIONS":
if (categoryTag === undefined || categoryTag === tag) {
assert.strictEqual(relativeSplitPath[0], "global") assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli") assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/") const eli = relativeSplitPath.slice(2, -1).join("/")
@ -298,11 +351,11 @@ async function importLegi(
data = ${db.json(versions as unknown as JSONValue)} data = ${db.json(versions as unknown as JSONValue)}
` `
versionsRemainingElis.delete(id) versionsRemainingElis.delete(id)
break
} }
break
default: { default: {
console.warn( console.warn(
`Unexpected root element "${key}" in XML file: ${filePath}`, `Unexpected root element "${tag}" in XML file: ${filePath}`,
) )
break iterXmlFiles break iterXmlFiles
} }