diff --git a/src/lib/auditors/jorf/articles.ts b/src/lib/auditors/jorf/articles.ts index 3d6db26..e619489 100644 --- a/src/lib/auditors/jorf/articles.ts +++ b/src/lib/auditors/jorf/articles.ts @@ -23,6 +23,22 @@ import { allJorfArticleTypesMutable, } from "$lib/legal" +export const jorfArticleStats: { + // countByEtat: { [etat: string]: number } + // countByLienArtEtat: { [etat: string]: number } + // countByLienNature: { [nature: string]: number } + // countByLienType: { [type: string]: number } + // countByTexteNature: { [nature: string]: number } + // countByVersionEtat: { [etat: string]: number } +} = { + // countByEtat: {}, + // countByLienArtEtat: {}, + // countByLienNature: {}, + // countByLienType: {}, + // countByTexteNature: {}, + // countByVersionEtat: {}, +} + function auditBlocTextuel( audit: Audit, dataUnknown: unknown, diff --git a/src/lib/auditors/jorf/index.ts b/src/lib/auditors/jorf/index.ts index b35bcbb..dfda818 100644 --- a/src/lib/auditors/jorf/index.ts +++ b/src/lib/auditors/jorf/index.ts @@ -1 +1,2 @@ -export { auditJorfArticle } from "./articles" +export { auditJo, joStats } from "./jo" +export { auditJorfArticle, jorfArticleStats } from "./articles" diff --git a/src/lib/auditors/jorf/jo.ts b/src/lib/auditors/jorf/jo.ts new file mode 100644 index 0000000..c16f184 --- /dev/null +++ b/src/lib/auditors/jorf/jo.ts @@ -0,0 +1,376 @@ +import { + type Audit, + auditRequire, + auditDateIso8601String, + auditTrimString, + auditInteger, + auditFunction, + auditEmptyToNull, + auditNullish, + auditSwitch, + auditNumber, + auditCleanArray, + auditOptions, + auditHttpUrl, + auditStringToNumber, +} from "@auditors/core" + +import { allJoNaturesMutable, allJoOriginesMutable } from "$lib/legal" + +export const joStats: { + countByNature: { [nature: string]: number } + countByOrigine: { [origine: string]: number } +} = { + countByNature: {}, + countByOrigine: {}, +} + +export function auditJo( + audit: Audit, + dataUnknown: unknown, +): [unknown, unknown] { + if (dataUnknown == null) { + return [dataUnknown, null] + } + if (typeof dataUnknown !== "object") { + return audit.unexpectedType(dataUnknown, "object") + } + + const data = { ...dataUnknown } + const errors: { [key: string]: unknown } = {} + const remainingKeys = new Set(Object.keys(data)) + audit.attribute( + data, + "META", + true, + errors, + remainingKeys, + auditMeta, + auditRequire, + ) + audit.attribute( + data, + "STRUCTURE_TXT", + true, + errors, + remainingKeys, + auditSwitch( + [auditTrimString, auditEmptyToNull, auditNullish], + auditStructureTxt, + ), + ) + + return audit.reduceRemaining(data, errors, remainingKeys) +} + +function auditLienTxt(audit: Audit, dataUnknown: unknown): [unknown, unknown] { + if (dataUnknown == null) { + return [dataUnknown, null] + } + if (typeof dataUnknown !== "object") { + return audit.unexpectedType(dataUnknown, "object") + } + + const data = { ...dataUnknown } + const errors: { [key: string]: unknown } = {} + const remainingKeys = new Set(Object.keys(data)) + + for (const key of ["@idtxt", "@titretxt"]) { + audit.attribute( + data, + key, + true, + errors, + remainingKeys, + auditTrimString, + auditEmptyToNull, + auditRequire, + ) + } + + return audit.reduceRemaining(data, errors, remainingKeys) +} + +function auditMeta(audit: Audit, dataUnknown: unknown): [unknown, unknown] { + if (dataUnknown == null) { + return [dataUnknown, null] + } + if (typeof dataUnknown !== "object") { + return audit.unexpectedType(dataUnknown, "object") + } + + const data = { ...dataUnknown } + const errors: { [key: string]: unknown } = {} + const remainingKeys = new Set(Object.keys(data)) + + audit.attribute( + data, + "META_COMMUN", + true, + errors, + remainingKeys, + auditMetaCommun, + auditRequire, + ) + audit.attribute( + data, + "META_SPEC", + true, + errors, + remainingKeys, + auditMetaSpec, + auditRequire, + ) + + return audit.reduceRemaining(data, errors, remainingKeys) +} + +function auditMetaCommun( + audit: Audit, + dataUnknown: unknown, +): [unknown, unknown] { + if (dataUnknown == null) { + return [dataUnknown, null] + } + if (typeof dataUnknown !== "object") { + return audit.unexpectedType(dataUnknown, "object") + } + + const data = { ...dataUnknown } + const errors: { [key: string]: unknown } = {} + const remainingKeys = new Set(Object.keys(data)) + + audit.attribute( + data, + "ANCIEN_ID", + true, + errors, + remainingKeys, + auditSwitch( + [auditNumber, auditInteger, auditFunction((id) => id.toString())], + [auditTrimString, auditEmptyToNull], + ), + auditNullish, + ) + for (const key of ["ID", "URL"]) { + audit.attribute( + data, + key, + true, + errors, + remainingKeys, + auditTrimString, + auditEmptyToNull, + auditRequire, + ) + } + audit.attribute( + data, + "ID_ELI", + true, + errors, + remainingKeys, + auditTrimString, + auditEmptyToNull, + auditHttpUrl, + ) + audit.attribute( + data, + "NATURE", + true, + errors, + remainingKeys, + auditTrimString, + auditEmptyToNull, + // auditFunction((nature) => { + // joStats.countByNature[nature] = (joStats.countByNature[nature] ?? 0) + 1 + // return nature + // }), + auditOptions(allJoNaturesMutable), + auditRequire, + ) + audit.attribute( + data, + "ORIGINE", + true, + errors, + remainingKeys, + auditTrimString, + auditEmptyToNull, + // auditFunction((origine) => { + // joStats.countByOrigine[origine] = + // (joStats.countByOrigine[origine] ?? 0) + 1 + // return origine + // }), + auditOptions(allJoOriginesMutable), + auditRequire, + ) + + return audit.reduceRemaining(data, errors, remainingKeys) +} + +function auditMetaConteneur( + audit: Audit, + dataUnknown: unknown, +): [unknown, unknown] { + if (dataUnknown == null) { + return [dataUnknown, null] + } + if (typeof dataUnknown !== "object") { + return audit.unexpectedType(dataUnknown, "object") + } + + const data = { ...dataUnknown } + const errors: { [key: string]: unknown } = {} + const remainingKeys = new Set(Object.keys(data)) + + audit.attribute( + data, + "DATE_PUBLI", + true, + errors, + remainingKeys, + auditDateIso8601String, + auditRequire, + ) + audit.attribute( + data, + "NUM", + true, + errors, + remainingKeys, + auditSwitch( + [auditNumber, auditFunction((num) => num.toString())], + [auditTrimString, auditEmptyToNull], + ), + ) + audit.attribute( + data, + "TITRE", + true, + errors, + remainingKeys, + auditTrimString, + auditEmptyToNull, + auditRequire, + ) + + return audit.reduceRemaining(data, errors, remainingKeys) +} + +function auditMetaSpec(audit: Audit, dataUnknown: unknown): [unknown, unknown] { + if (dataUnknown == null) { + return [dataUnknown, null] + } + if (typeof dataUnknown !== "object") { + return audit.unexpectedType(dataUnknown, "object") + } + + const data = { ...dataUnknown } + const errors: { [key: string]: unknown } = {} + const remainingKeys = new Set(Object.keys(data)) + + audit.attribute( + data, + "META_CONTENEUR", + true, + errors, + remainingKeys, + auditMetaConteneur, + auditRequire, + ) + + return audit.reduceRemaining(data, errors, remainingKeys) +} + +function auditStructureTxt( + audit: Audit, + dataUnknown: unknown, +): [unknown, unknown] { + if (dataUnknown == null) { + return [dataUnknown, null] + } + if (typeof dataUnknown !== "object") { + return audit.unexpectedType(dataUnknown, "object") + } + + const data = { ...dataUnknown } + const errors: { [key: string]: unknown } = {} + const remainingKeys = new Set(Object.keys(data)) + + audit.attribute( + data, + "LIEN_TXT", + true, + errors, + remainingKeys, + auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])), + auditCleanArray(auditLienTxt, auditRequire), + ) + audit.attribute( + data, + "TM", + true, + errors, + remainingKeys, + auditFunction((tm) => (Array.isArray(tm) ? tm : [tm])), + auditCleanArray(auditTm, auditRequire), + ) + + return audit.reduceRemaining(data, errors, remainingKeys) +} + +/// Table des matières +function auditTm(audit: Audit, dataUnknown: unknown): [unknown, unknown] { + if (dataUnknown == null) { + return [dataUnknown, null] + } + if (typeof dataUnknown !== "object") { + return audit.unexpectedType(dataUnknown, "object") + } + + const data = { ...dataUnknown } + const errors: { [key: string]: unknown } = {} + const remainingKeys = new Set(Object.keys(data)) + + audit.attribute( + data, + "@niv", + true, + errors, + remainingKeys, + auditStringToNumber, + auditNumber, + auditRequire, + ) + audit.attribute( + data, + "LIEN_TXT", + true, + errors, + remainingKeys, + auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])), + auditCleanArray(auditLienTxt, auditRequire), + ) + audit.attribute( + data, + "TITRE_TM", + true, + errors, + remainingKeys, + auditTrimString, + auditEmptyToNull, + auditRequire, + ) + audit.attribute( + data, + "TM", + true, + errors, + remainingKeys, + auditFunction((tm) => (Array.isArray(tm) ? tm : [tm])), + auditCleanArray(auditTm, auditRequire), + ) + + return audit.reduceRemaining(data, errors, remainingKeys) +} diff --git a/src/lib/index.ts b/src/lib/index.ts index 1a19f2f..172c7fb 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -20,6 +20,10 @@ export { default as TextelrView } from "./components/TextelrView.svelte" export { default as TexteVersionView } from "./components/TexteVersionView.svelte" export { + allJoNatures, + allJoNaturesMutable, + allJoOrigines, + allJoOriginesMutable, allJorfArticleEtats, allJorfArticleEtatsMutable, allJorfArticleLienArticleOrigines, @@ -49,6 +53,8 @@ export { allLegiArticleTypes, allLegiArticleTypesMutable, type DossierLegislatif, + type JoNature, + type JoOrigine, type JorfArticle, type JorfArticleEtat, type JorfArticleLienArticleOrigine, diff --git a/src/lib/legal/index.ts b/src/lib/legal/index.ts index ead56ad..4c9846d 100644 --- a/src/lib/legal/index.ts +++ b/src/lib/legal/index.ts @@ -4,6 +4,10 @@ import type { DossierLegislatif } from "./dole" export type { DossierLegislatif } from "./dole" export { + allJoNatures, + allJoNaturesMutable, + allJoOrigines, + allJoOriginesMutable, allJorfArticleEtats, allJorfArticleEtatsMutable, allJorfArticleLienArticleOrigines, @@ -16,6 +20,8 @@ export { allJorfArticleTexteNaturesMutable, allJorfArticleTypes, allJorfArticleTypesMutable, + type JoNature, + type JoOrigine, type JorfArticle, type JorfArticleEtat, type JorfArticleLienArticleOrigine, diff --git a/src/lib/legal/jorf.ts b/src/lib/legal/jorf.ts index 494b3a8..d4a13c6 100644 --- a/src/lib/legal/jorf.ts +++ b/src/lib/legal/jorf.ts @@ -59,6 +59,10 @@ export interface JorfArticle { } } +export type JoNature = (typeof allJoNatures)[number] + +export type JoOrigine = (typeof allJoOrigines)[number] + export type JorfArticleEtat = (typeof allJorfArticleEtats)[number] export type JorfArticleLienArticleOrigine = @@ -83,6 +87,12 @@ export interface JorfArticleTm { TM?: JorfArticleTm } +export const allJoNatures = ["JO"] as const +export const allJoNaturesMutable = [...allJoNatures] + +export const allJoOrigines = ["JORF"] as const +export const allJoOriginesMutable = [...allJoOrigines] + export const allJorfArticleEtats = [ "ABROGE", "ABROGE_DIFF", diff --git a/src/scripts/import_jorf.ts b/src/scripts/import_jorf.ts index cad46f8..7c7d86b 100644 --- a/src/scripts/import_jorf.ts +++ b/src/scripts/import_jorf.ts @@ -1,4 +1,9 @@ -import { auditChain, auditRequire, strictAudit } from "@auditors/core" +import { + auditChain, + auditOptions, + auditRequire, + strictAudit, +} from "@auditors/core" import assert from "assert" import { XMLParser } from "fast-xml-parser" import fs from "fs-extra" @@ -8,7 +13,11 @@ import type { JSONValue } from "postgres" import sade from "sade" import { auditId, auditVersions } from "$lib/auditors/legal" -import { auditJorfArticle } from "$lib/auditors/jorf" +import { + auditJo, + auditJorfArticle, + joStats, +} from "$lib/auditors/jorf" import type { Jo, JorfArticle, @@ -21,6 +30,18 @@ import type { import { db } from "$lib/server/database" import { walkDir } from "$lib/server/file_systems" +type CategoryTag = (typeof allCategoriesCode)[number] + +const allCategoriesCode = [ + "ARTICLE", + "ID", + "JO", + "SECTION_TA", + "TEXTE_VERSION", + "TEXTELR", + "VERSIONS", +] as const + const xmlParser = new XMLParser({ attributeNamePrefix: "@", ignoreAttributes: false, @@ -42,71 +63,105 @@ const xmlParser = new XMLParser({ async function importJorf( dilaDir: string, - { resume }: { resume?: string } = {}, + { category, resume }: { category?: string; resume?: string } = {}, ): Promise { + const [categoryTag, categoryError] = auditOptions([ + ...[...allCategoriesCode], + ])(strictAudit, category) as [CategoryTag | undefined, unknown] + assert.strictEqual( + categoryError, + null, + `Error for category ${JSON.stringify(categoryTag)}:\n${JSON.stringify( + categoryError, + null, + 2, + )}`, + ) let skip = resume !== undefined + const deleteRemainingIds = !skip - const articleRemainingIds = new Set( - ( - await db<{ id: string }[]>` - SELECT id - FROM article - WHERE id LIKE 'JORF%' - ` - ).map(({ id }) => id), - ) - const idRemainingElis = new Set( - ( - await db<{ eli: string }[]>` - SELECT eli - FROM id - ` - ).map(({ eli }) => eli), - ) - const joRemainingIds = new Set( - ( - await db<{ id: string }[]>` - SELECT id - FROM jo - ` - ).map(({ id }) => id), - ) - const sectionTaRemainingIds = new Set( - ( - await db<{ id: string }[]>` - SELECT id - FROM section_ta - WHERE id LIKE 'JORF%' - ` - ).map(({ id }) => id), - ) - const textelrRemainingIds = new Set( - ( - await db<{ id: string }[]>` - SELECT id - FROM textelr - WHERE id LIKE 'JORF%' - ` - ).map(({ id }) => id), - ) - const texteVersionRemainingIds = new Set( - ( - await db<{ id: string }[]>` - SELECT id - FROM texte_version - WHERE id LIKE 'JORF%' - ` - ).map(({ id }) => id), - ) - const versionsRemainingElis = new Set( - ( - await db<{ eli: string }[]>` - SELECT eli - FROM versions - ` - ).map(({ eli }) => eli), - ) + const articleRemainingIds = + categoryTag === undefined || categoryTag === "ARTICLE" + ? new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM article + WHERE id LIKE 'JORF%' + ` + ).map(({ id }) => id), + ) + : new Set() + const idRemainingElis = + categoryTag === undefined || categoryTag === "ID" + ? new Set( + ( + await db<{ eli: string }[]>` + SELECT eli + FROM id + ` + ).map(({ eli }) => eli), + ) + : new Set() + const joRemainingIds = + categoryTag === undefined || categoryTag === "JO" + ? new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM jo + ` + ).map(({ id }) => id), + ) + : new Set() + const sectionTaRemainingIds = + categoryTag === undefined || categoryTag === "SECTION_TA" + ? new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM section_ta + WHERE id LIKE 'JORF%' + ` + ).map(({ id }) => id), + ) + : new Set() + const textelrRemainingIds = + categoryTag === undefined || categoryTag === "TEXTELR" + ? new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM textelr + WHERE id LIKE 'JORF%' + ` + ).map(({ id }) => id), + ) + : new Set() + const texteVersionRemainingIds = + categoryTag === undefined || categoryTag === "TEXTE_VERSION" + ? new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM texte_version + WHERE id LIKE 'JORF%' + ` + ).map(({ id }) => id), + ) + : new Set() + const versionsRemainingElis = + categoryTag === undefined || categoryTag === "VERSIONS" + ? new Set( + ( + await db<{ eli: string }[]>` + SELECT eli + FROM versions + ` + ).map(({ eli }) => eli), + ) + : new Set() const dataDir = path.join(dilaDir, "jorf") assert(await fs.pathExists(dataDir)) @@ -132,207 +187,238 @@ async function importJorf( encoding: "utf8", }) const xmlData = xmlParser.parse(xmlString) - for (const [key, element] of Object.entries(xmlData) as [ - string, + for (const [tag, element] of Object.entries(xmlData) as [ + CategoryTag | "?xml", ( - | JorfArticle | Jo - | SectionTa + | JorfArticle + | JorfSectionTa | Textelr | TexteVersion | Versions | XmlHeader ), ][]) { - switch (key) { + switch (tag) { case "?xml": { const xmlHeader = element as XmlHeader assert.strictEqual(xmlHeader["@encoding"], "UTF-8", filePath) assert.strictEqual(xmlHeader["@version"], "1.0", filePath) break } - case "ARTICLE": { - const [article, error] = auditChain(auditJorfArticle, auditRequire)( - strictAudit, - element, - ) as [JorfArticle, unknown] - assert.strictEqual( - error, - null, - `Unexpected format for ARTICLE:\n${JSON.stringify( - article, + case "ARTICLE": + if (categoryTag === undefined || categoryTag === tag) { + const [article, error] = auditChain( + auditJorfArticle, + auditRequire, + )(strictAudit, element) as [JorfArticle, unknown] + assert.strictEqual( + error, null, - 2, - )}\nError:\n${JSON.stringify(error, null, 2)}`, - ) - await db` - INSERT INTO article ( - id, - data - ) VALUES ( - ${article.META.META_COMMUN.ID}, - ${db.json(article as unknown as JSONValue)} + `Unexpected format for ARTICLE:\n${JSON.stringify( + article, + null, + 2, + )}\nError:\n${JSON.stringify(error, null, 2)}`, ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(article as unknown as JSONValue)} - ` - articleRemainingIds.delete(article.META.META_COMMUN.ID) + await db` + INSERT INTO article ( + id, + data + ) VALUES ( + ${article.META.META_COMMUN.ID}, + ${db.json(article as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(article as unknown as JSONValue)} + ` + articleRemainingIds.delete(article.META.META_COMMUN.ID) + } break - } - case "ID": { - assert.strictEqual(relativeSplitPath[0], "global") - assert.strictEqual(relativeSplitPath[1], "eli") - const eli = relativeSplitPath.slice(2, -1).join("/") - const [id, idError] = auditChain(auditId, auditRequire)( - strictAudit, - element, - ) - assert.strictEqual( - idError, - null, - `Unexpected format for ID:\n${JSON.stringify( - id, + case "ID": + if (categoryTag === undefined || categoryTag === tag) { + assert.strictEqual(relativeSplitPath[0], "global") + assert.strictEqual(relativeSplitPath[1], "eli") + const eli = relativeSplitPath.slice(2, -1).join("/") + const [id, idError] = auditChain(auditId, auditRequire)( + strictAudit, + element, + ) + assert.strictEqual( + idError, null, - 2, - )}\nError:\n${JSON.stringify(idError, null, 2)}`, - ) - await db` - INSERT INTO id ( - eli, - id - ) VALUES ( - ${eli}, - ${id} + `Unexpected format for ID:\n${JSON.stringify( + id, + null, + 2, + )}\nError:\n${JSON.stringify(idError, null, 2)}`, ) - ON CONFLICT (eli) - DO UPDATE SET - id = ${id} - ` - idRemainingElis.delete(eli) + await db` + INSERT INTO id ( + eli, + id + ) VALUES ( + ${eli}, + ${id} + ) + ON CONFLICT (eli) + DO UPDATE SET + id = ${id} + ` + idRemainingElis.delete(eli) + } break - } - case "JO": { - const jo = element as Jo - await db` - INSERT INTO jo ( - id, - data - ) VALUES ( - ${jo.META.META_COMMUN.ID}, - ${db.json(jo as unknown as JSONValue)} - ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(jo as unknown as JSONValue)} - ` - joRemainingIds.delete(jo.META.META_COMMUN.ID) - break - } - case "SECTION_TA": { - const section = element as SectionTa - await db` - INSERT INTO section_ta ( - id, - data - ) VALUES ( - ${section.ID}, - ${db.json(section as unknown as JSONValue)} - ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(section as unknown as JSONValue)} - ` - sectionTaRemainingIds.delete(section.ID) - break - } - case "TEXTE_VERSION": { - const texteVersion = element as TexteVersion - const textAFragments = [ - texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE, - texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITREFULL, - ] - await db` - INSERT INTO texte_version ( - id, - data, - nature, - text_search - ) VALUES ( - ${texteVersion.META.META_COMMUN.ID}, - ${db.json(texteVersion as unknown as JSONValue)}, - ${texteVersion.META.META_COMMUN.NATURE}, - setweight(to_tsvector('french', ${textAFragments.join( - " ", - )}), 'A') - ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(texteVersion as unknown as JSONValue)}, - nature = ${texteVersion.META.META_COMMUN.NATURE}, - text_search = setweight(to_tsvector('french', ${textAFragments.join( - " ", - )}), 'A') - ` - texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID) - break - } - case "TEXTELR": { - const textelr = element as Textelr - await db` - INSERT INTO textelr ( - id, - data - ) VALUES ( - ${textelr.META.META_COMMUN.ID}, - ${db.json(textelr as unknown as JSONValue)} - ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(textelr as unknown as JSONValue)} - ` - textelrRemainingIds.delete(textelr.META.META_COMMUN.ID) - break - } - case "VERSIONS": { - assert.strictEqual(relativeSplitPath[0], "global") - assert.strictEqual(relativeSplitPath[1], "eli") - const eli = relativeSplitPath.slice(2, -1).join("/") - const [versions, versionsError] = auditChain( - auditVersions, - auditRequire, - )(strictAudit, element) - assert.strictEqual( - versionsError, - null, - `Unexpected format for VERSIONS:\n${JSON.stringify( - versions, + case "JO": + if (categoryTag === undefined || categoryTag === tag) { + const [jo, error] = auditChain(auditJo, auditRequire)( + strictAudit, + element, + ) as [Jo, unknown] + assert.strictEqual( + error, null, - 2, - )}\nError:\n${JSON.stringify(versionsError, null, 2)}`, - ) - const id = versions.VERSION["@id"] - await db` - INSERT INTO versions ( - eli, - id, - data - ) VALUES ( - ${eli}, - ${id}, - ${db.json(versions as unknown as JSONValue)} + `Unexpected format for JO:\n${JSON.stringify( + jo, + null, + 2, + )}\nError:\n${JSON.stringify(error, null, 2)}`, ) - ON CONFLICT (eli) - DO UPDATE SET - id = ${id}, - data = ${db.json(versions as unknown as JSONValue)} - ` - versionsRemainingElis.delete(id) + await db` + INSERT INTO jo ( + id, + data + ) VALUES ( + ${jo.META.META_COMMUN.ID}, + ${db.json(jo as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(jo as unknown as JSONValue)} + ` + joRemainingIds.delete(jo.META.META_COMMUN.ID) + } + break + case "SECTION_TA": + if (categoryTag === undefined || categoryTag === tag) { + const [section, error] = auditChain( + auditJorfSectionTa, + auditRequire, + )(strictAudit, element) as [JorfSectionTa, unknown] + assert.strictEqual( + error, + null, + `Unexpected format for SECTION_TA:\n${JSON.stringify( + section, + null, + 2, + )}\nError:\n${JSON.stringify(error, null, 2)}`, + ) + await db` + INSERT INTO section_ta ( + id, + data + ) VALUES ( + ${section.ID}, + ${db.json(section as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(section as unknown as JSONValue)} + ` + sectionTaRemainingIds.delete(section.ID) + } + break + case "TEXTE_VERSION": + if (categoryTag === undefined || categoryTag === tag) { + const texteVersion = element as TexteVersion + const textAFragments = [ + texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE, + texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITREFULL, + ] + await db` + INSERT INTO texte_version ( + id, + data, + nature, + text_search + ) VALUES ( + ${texteVersion.META.META_COMMUN.ID}, + ${db.json(texteVersion as unknown as JSONValue)}, + ${texteVersion.META.META_COMMUN.NATURE}, + setweight(to_tsvector('french', ${textAFragments.join( + " ", + )}), 'A') + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(texteVersion as unknown as JSONValue)}, + nature = ${texteVersion.META.META_COMMUN.NATURE}, + text_search = setweight(to_tsvector('french', ${textAFragments.join( + " ", + )}), 'A') + ` + texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID) + } + break + case "TEXTELR": + if (categoryTag === undefined || categoryTag === tag) { + const textelr = element as Textelr + await db` + INSERT INTO textelr ( + id, + data + ) VALUES ( + ${textelr.META.META_COMMUN.ID}, + ${db.json(textelr as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(textelr as unknown as JSONValue)} + ` + textelrRemainingIds.delete(textelr.META.META_COMMUN.ID) + } + break + case "VERSIONS": + if (categoryTag === undefined || categoryTag === tag) { + assert.strictEqual(relativeSplitPath[0], "global") + assert.strictEqual(relativeSplitPath[1], "eli") + const eli = relativeSplitPath.slice(2, -1).join("/") + const [versions, versionsError] = auditChain( + auditVersions, + auditRequire, + )(strictAudit, element) + assert.strictEqual( + versionsError, + null, + `Unexpected format for VERSIONS:\n${JSON.stringify( + versions, + null, + 2, + )}\nError:\n${JSON.stringify(versionsError, null, 2)}`, + ) + const id = versions.VERSION["@id"] + await db` + INSERT INTO versions ( + eli, + id, + data + ) VALUES ( + ${eli}, + ${id}, + ${db.json(versions as unknown as JSONValue)} + ) + ON CONFLICT (eli) + DO UPDATE SET + id = ${id}, + data = ${db.json(versions as unknown as JSONValue)} + ` + versionsRemainingElis.delete(id) + } break - } default: { console.warn( - `Unexpected root element "${key}" in XML file: ${filePath}`, + `Unexpected root element "${tag}" in XML file: ${filePath}`, ) break iterXmlFiles } @@ -395,10 +481,17 @@ async function importJorf( ` } } + + // console.log( + // "JORF articles stats =", + // JSON.stringify(jorfArticleStats, null, 2), + // ) + console.log("JO stats =", JSON.stringify(joStats, null, 2)) } sade("import_jorf ", true) .describe("Import Dila's JORF database") + .option("-k, --category", "Import only given type of data") .option("-r, --resume", "Resume import at given relative file path") .example( "--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml ../dila-data/", diff --git a/src/scripts/import_legi.ts b/src/scripts/import_legi.ts index d26965d..252eb32 100644 --- a/src/scripts/import_legi.ts +++ b/src/scripts/import_legi.ts @@ -1,4 +1,9 @@ -import { auditChain, auditRequire, strictAudit } from "@auditors/core" +import { + auditChain, + auditOptions, + auditRequire, + strictAudit, +} from "@auditors/core" import assert from "assert" import { XMLParser } from "fast-xml-parser" import fs from "fs-extra" @@ -23,6 +28,17 @@ import type { import { db } from "$lib/server/database" import { walkDir } from "$lib/server/file_systems" +type CategoryTag = (typeof allCategoriesCode)[number] + +const allCategoriesCode = [ + "ARTICLE", + "ID", + "SECTION_TA", + "TEXTE_VERSION", + "TEXTELR", + "VERSIONS", +] as const + const xmlParser = new XMLParser({ attributeNamePrefix: "@", ignoreAttributes: false, @@ -44,63 +60,94 @@ const xmlParser = new XMLParser({ async function importLegi( dilaDir: string, - { resume }: { resume?: string } = {}, + { category, resume }: { category?: string; resume?: string } = {}, ): Promise { + const [categoryTag, categoryError] = auditOptions([ + ...[...allCategoriesCode], + ])(strictAudit, category) as [CategoryTag | undefined, unknown] + assert.strictEqual( + categoryError, + null, + `Error for category ${JSON.stringify(categoryTag)}:\n${JSON.stringify( + categoryError, + null, + 2, + )}`, + ) let skip = resume !== undefined + const deleteRemainingIds = !skip - const articleRemainingIds = new Set( - ( - await db<{ id: string }[]>` - SELECT id - FROM article - WHERE id LIKE 'LEGI%' - ` - ).map(({ id }) => id), - ) - const idRemainingElis = new Set( - ( - await db<{ eli: string }[]>` - SELECT eli - FROM id - ` - ).map(({ eli }) => eli), - ) - const sectionTaRemainingIds = new Set( - ( - await db<{ id: string }[]>` - SELECT id - FROM section_ta - WHERE id LIKE 'LEGI%' - ` - ).map(({ id }) => id), - ) - const textelrRemainingIds = new Set( - ( - await db<{ id: string }[]>` - SELECT id - FROM textelr - WHERE id LIKE 'LEGI%' - ` - ).map(({ id }) => id), - ) - const texteVersionRemainingIds = new Set( - ( - await db<{ id: string }[]>` - SELECT id - FROM texte_version - WHERE id LIKE 'LEGI%' - ` - ).map(({ id }) => id), - ) - const versionsRemainingElis = new Set( - ( - await db<{ eli: string }[]>` - SELECT eli - FROM versions - ` - ).map(({ eli }) => eli), - ) + const articleRemainingIds = + categoryTag === undefined || categoryTag === "ARTICLE" + ? new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM article + WHERE id LIKE 'LEGI%' + ` + ).map(({ id }) => id), + ) + : new Set() + const idRemainingElis = + categoryTag === undefined || categoryTag === "ID" + ? new Set( + ( + await db<{ eli: string }[]>` + SELECT eli + FROM id + ` + ).map(({ eli }) => eli), + ) + : new Set() + const sectionTaRemainingIds = + categoryTag === undefined || categoryTag === "SECTION_TA" + ? new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM section_ta + WHERE id LIKE 'LEGI%' + ` + ).map(({ id }) => id), + ) + : new Set() + const textelrRemainingIds = + categoryTag === undefined || categoryTag === "TEXTELR" + ? new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM textelr + WHERE id LIKE 'LEGI%' + ` + ).map(({ id }) => id), + ) + : new Set() + const texteVersionRemainingIds = + categoryTag === undefined || categoryTag === "TEXTE_VERSION" + ? new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM texte_version + WHERE id LIKE 'LEGI%' + ` + ).map(({ id }) => id), + ) + : new Set() + const versionsRemainingElis = + categoryTag === undefined || categoryTag === "VERSIONS" + ? new Set( + ( + await db<{ eli: string }[]>` + SELECT eli + FROM versions + ` + ).map(({ eli }) => eli), + ) + : new Set() const dataDir = path.join(dilaDir, "legi") assert(await fs.pathExists(dataDir)) @@ -126,183 +173,189 @@ async function importLegi( encoding: "utf8", }) const xmlData = xmlParser.parse(xmlString) - for (const [key, element] of Object.entries(xmlData) as [ - string, + for (const [tag, element] of Object.entries(xmlData) as [ + CategoryTag | "?xml", LegiArticle | SectionTa | Textelr | TexteVersion | Versions | XmlHeader, ][]) { - switch (key) { + switch (tag) { case "?xml": { const xmlHeader = element as XmlHeader assert.strictEqual(xmlHeader["@encoding"], "UTF-8", filePath) assert.strictEqual(xmlHeader["@version"], "1.0", filePath) break } - case "ARTICLE": { - const [article, error] = auditChain(auditLegiArticle, auditRequire)( - strictAudit, - element, - ) as [LegiArticle, unknown] - assert.strictEqual( - error, - null, - `Unexpected format for ARTICLE:\n${JSON.stringify( - article, + case "ARTICLE": + if (categoryTag === undefined || categoryTag === tag) { + const [article, error] = auditChain( + auditLegiArticle, + auditRequire, + )(strictAudit, element) as [LegiArticle, unknown] + assert.strictEqual( + error, null, - 2, - )}\nError:\n${JSON.stringify(error, null, 2)}`, - ) - await db` - INSERT INTO article ( - id, - data - ) VALUES ( - ${article.META.META_COMMUN.ID}, - ${db.json(article as unknown as JSONValue)} + `Unexpected format for ARTICLE:\n${JSON.stringify( + article, + null, + 2, + )}\nError:\n${JSON.stringify(error, null, 2)}`, ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(article as unknown as JSONValue)} - ` - articleRemainingIds.delete(article.META.META_COMMUN.ID) + await db` + INSERT INTO article ( + id, + data + ) VALUES ( + ${article.META.META_COMMUN.ID}, + ${db.json(article as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(article as unknown as JSONValue)} + ` + articleRemainingIds.delete(article.META.META_COMMUN.ID) + } break - } - case "ID": { - assert.strictEqual(relativeSplitPath[0], "global") - assert.strictEqual(relativeSplitPath[1], "eli") - const eli = relativeSplitPath.slice(2, -1).join("/") - const [id, idError] = auditChain(auditId, auditRequire)( - strictAudit, - element, - ) - assert.strictEqual( - idError, - null, - `Unexpected format for ID:\n${JSON.stringify( - id, + case "ID": + if (categoryTag === undefined || categoryTag === tag) { + assert.strictEqual(relativeSplitPath[0], "global") + assert.strictEqual(relativeSplitPath[1], "eli") + const eli = relativeSplitPath.slice(2, -1).join("/") + const [id, idError] = auditChain(auditId, auditRequire)( + strictAudit, + element, + ) + assert.strictEqual( + idError, null, - 2, - )}\nError:\n${JSON.stringify(idError, null, 2)}`, - ) - assert - await db` - INSERT INTO id ( - eli, - id - ) VALUES ( - ${eli}, - ${id} + `Unexpected format for ID:\n${JSON.stringify( + id, + null, + 2, + )}\nError:\n${JSON.stringify(idError, null, 2)}`, ) - ON CONFLICT (eli) - DO UPDATE SET - id = ${id} - ` - idRemainingElis.delete(eli) + assert + await db` + INSERT INTO id ( + eli, + id + ) VALUES ( + ${eli}, + ${id} + ) + ON CONFLICT (eli) + DO UPDATE SET + id = ${id} + ` + idRemainingElis.delete(eli) + } break - } - case "SECTION_TA": { - const section = element as SectionTa - await db` - INSERT INTO section_ta ( - id, - data - ) VALUES ( - ${section.ID}, - ${db.json(section as unknown as JSONValue)} - ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(section as unknown as JSONValue)} - ` - sectionTaRemainingIds.delete(section.ID) + case "SECTION_TA": + if (categoryTag === undefined || categoryTag === tag) { + const section = element as SectionTa + await db` + INSERT INTO section_ta ( + id, + data + ) VALUES ( + ${section.ID}, + ${db.json(section as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(section as unknown as JSONValue)} + ` + sectionTaRemainingIds.delete(section.ID) + } break - } - case "TEXTE_VERSION": { - const texteVersion = element as TexteVersion - const textAFragments = [ - texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE, - texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITREFULL, - ] - await db` - INSERT INTO texte_version ( - id, - data, - nature, - text_search - ) VALUES ( - ${texteVersion.META.META_COMMUN.ID}, - ${db.json(texteVersion as unknown as JSONValue)}, - ${texteVersion.META.META_COMMUN.NATURE}, - setweight(to_tsvector('french', ${textAFragments.join( - " ", - )}), 'A') - ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(texteVersion as unknown as JSONValue)}, - nature = ${texteVersion.META.META_COMMUN.NATURE}, - text_search = setweight(to_tsvector('french', ${textAFragments.join( - " ", - )}), 'A') - ` - texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID) + case "TEXTE_VERSION": + if (categoryTag === undefined || categoryTag === tag) { + const texteVersion = element as TexteVersion + const textAFragments = [ + texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE, + texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITREFULL, + ] + await db` + INSERT INTO texte_version ( + id, + data, + nature, + text_search + ) VALUES ( + ${texteVersion.META.META_COMMUN.ID}, + ${db.json(texteVersion as unknown as JSONValue)}, + ${texteVersion.META.META_COMMUN.NATURE}, + setweight(to_tsvector('french', ${textAFragments.join( + " ", + )}), 'A') + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(texteVersion as unknown as JSONValue)}, + nature = ${texteVersion.META.META_COMMUN.NATURE}, + text_search = setweight(to_tsvector('french', ${textAFragments.join( + " ", + )}), 'A') + ` + texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID) + } break - } - case "TEXTELR": { - const textelr = element as Textelr - await db` - INSERT INTO textelr ( - id, - data - ) VALUES ( - ${textelr.META.META_COMMUN.ID}, - ${db.json(textelr as unknown as JSONValue)} - ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(textelr as unknown as JSONValue)} - ` - textelrRemainingIds.delete(textelr.META.META_COMMUN.ID) + case "TEXTELR": + if (categoryTag === undefined || categoryTag === tag) { + const textelr = element as Textelr + await db` + INSERT INTO textelr ( + id, + data + ) VALUES ( + ${textelr.META.META_COMMUN.ID}, + ${db.json(textelr as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(textelr as unknown as JSONValue)} + ` + textelrRemainingIds.delete(textelr.META.META_COMMUN.ID) + } break - } - case "VERSIONS": { - assert.strictEqual(relativeSplitPath[0], "global") - assert.strictEqual(relativeSplitPath[1], "eli") - const eli = relativeSplitPath.slice(2, -1).join("/") - const [versions, versionsError] = auditChain( - auditVersions, - auditRequire, - )(strictAudit, element) - assert.strictEqual( - versionsError, - null, - `Unexpected format for VERSIONS:\n${JSON.stringify( - versions, + case "VERSIONS": + if (categoryTag === undefined || categoryTag === tag) { + assert.strictEqual(relativeSplitPath[0], "global") + assert.strictEqual(relativeSplitPath[1], "eli") + const eli = relativeSplitPath.slice(2, -1).join("/") + const [versions, versionsError] = auditChain( + auditVersions, + auditRequire, + )(strictAudit, element) + assert.strictEqual( + versionsError, null, - 2, - )}\nError:\n${JSON.stringify(versionsError, null, 2)}`, - ) - const id = versions.VERSION["@id"] - await db` - INSERT INTO versions ( - eli, - id, - data - ) VALUES ( - ${eli}, - ${id}, - ${db.json(versions as unknown as JSONValue)} + `Unexpected format for VERSIONS:\n${JSON.stringify( + versions, + null, + 2, + )}\nError:\n${JSON.stringify(versionsError, null, 2)}`, ) - ON CONFLICT (eli) - DO UPDATE SET - id = ${id}, - data = ${db.json(versions as unknown as JSONValue)} - ` - versionsRemainingElis.delete(id) + const id = versions.VERSION["@id"] + await db` + INSERT INTO versions ( + eli, + id, + data + ) VALUES ( + ${eli}, + ${id}, + ${db.json(versions as unknown as JSONValue)} + ) + ON CONFLICT (eli) + DO UPDATE SET + id = ${id}, + data = ${db.json(versions as unknown as JSONValue)} + ` + versionsRemainingElis.delete(id) + } break - } default: { console.warn( - `Unexpected root element "${key}" in XML file: ${filePath}`, + `Unexpected root element "${tag}" in XML file: ${filePath}`, ) break iterXmlFiles }