diff --git a/src/lib/auditors/data.ts b/src/lib/auditors/data.ts new file mode 100644 index 0000000..34de0d0 --- /dev/null +++ b/src/lib/auditors/data.ts @@ -0,0 +1,99 @@ +import { + type Audit, + auditChain, + auditRequire, + auditString, + auditTest, + auditOptions, +} from "@auditors/core" + +export const auditId = auditChain( + auditString, + auditTest((id) => id.length === 20, "Invalid length for ID"), +) + +export function auditVersions( + audit: Audit, + dataUnknown: unknown, +): [unknown, unknown] { + if (dataUnknown == null) { + return [dataUnknown, null] + } + if (typeof dataUnknown !== "object") { + return audit.unexpectedType(dataUnknown, "object") + } + + const data = { ...dataUnknown } + const errors: { [key: string]: unknown } = {} + const remainingKeys = new Set(Object.keys(data)) + + audit.attribute( + data, + "VERSION", + true, + errors, + remainingKeys, + auditVersionsVersion, + auditRequire, + ) + + return audit.reduceRemaining(data, errors, remainingKeys) +} + +export function auditVersionsVersion( + audit: Audit, + dataUnknown: unknown, +): [unknown, unknown] { + if (dataUnknown == null) { + return [dataUnknown, null] + } + if (typeof dataUnknown !== "object") { + return audit.unexpectedType(dataUnknown, "object") + } + + const data = { ...dataUnknown } + const errors: { [key: string]: unknown } = {} + const remainingKeys = new Set(Object.keys(data)) + + audit.attribute( + data, + "@id", + true, + errors, + remainingKeys, + auditId, + auditRequire, + ) + audit.attribute( + data, + "@fin", + true, + errors, + remainingKeys, + auditString, + auditOptions(["2999-01-01"]), + auditRequire, + ) + audit.attribute( + data, + "@etat", + true, + errors, + remainingKeys, + auditString, + auditOptions([""]), + auditRequire, + ) + audit.attribute( + data, + "@debut", + true, + errors, + remainingKeys, + auditString, + auditOptions(["2999-01-01"]), + auditRequire, + ) + + return audit.reduceRemaining(data, errors, remainingKeys) +} diff --git a/src/lib/data.ts b/src/lib/data.ts index 5ea807b..fae6c82 100644 --- a/src/lib/data.ts +++ b/src/lib/data.ts @@ -11,22 +11,11 @@ export interface Article { } } } - CONTEXTE: { - TEXTE: { - TITRE_TXT: TitreTexte | TitreTexte[] - } - } + CONTEXTE: Contexte VERSIONS: { VERSION: Array<{ "@etat": Etat - LIEN_ART: { - "@id": string - "@fin": string - "@num": string - "@etat": Etat - "@debut": string - "@origine": "LEGI" - } + LIEN_ART: LienArt }> } BLOC_TEXTUEL: { @@ -34,36 +23,65 @@ export interface Article { } } -export type EliId = string - -export interface EliVersions {} +export interface Contexte { + TEXTE: { + TITRE_TXT: TitreTxt | TitreTxt[] + } +} export type Etat = "MODIFIE" | "VIGUEUR" -export type LegiObject = - | Article - | EliId - | EliVersions - | Section - | Struct - | TexteVersion -export type LegiObjectType = +export interface Jo { + META: { + META_SPEC: { + META_CONTENEUR: { + NUM: string + TITRE: string + DATE_PUBLI: string + } + } + META_COMMUN: MetaCommun + } +} + +export type LegalObject = + | Article + | Jo + | SectionTa + | Textelr + | TexteVersion + | Versions + +export type LegalObjectType = | "article" - | "eli_id" - | "eli_versions" - | "section" - | "struct" - | "texte" + | "id" + | "jo" + | "section_ta" + | "texte_version" + | "textelr" + | "versions" + +export interface LienArt { + "@id": string + "@fin": string + "@num": string + "@etat": Etat + "@debut": string + "@origine": "LEGI" +} export interface MetaCommun { ID: string } -export interface Section { +export interface SectionTa { ID: string + CONTEXTE: Contexte + TITRE_TA: string + STRUCTURE_TA: { LIEN_ART?: LienArt | LienArt[] } } -export interface Struct { +export interface Textelr { META: { META_COMMUN: MetaCommun } @@ -71,58 +89,102 @@ export interface Struct { export interface TexteVersion { META: { + META_SPEC: { + META_TEXTE_VERSION: { + TITRE: string + TITREFULL: string + } + } META_COMMUN: MetaCommun } } -export interface TitreTexte { +export interface TitreTxt { + "@fin": string + "@debut": string + "@id_txt": string + "@c_titre_court": string "#text": string } -export function assertNeverLegiObjectType(type: never): never { - throw `Unexpected type for LEGI object: ${type}` -} - -export function pathnameFromLegiObject( - type: LegiObjectType, - object: LegiObject, -): string { - switch (type) { - case "article": - return `/articles/${(object as Article).META.META_COMMUN.ID}` - case "eli_id": - return `/eli/ids/TODO` - case "eli_versions": - return `/eli/ids/TODO` - case "section": - return `/sections/${(object as Section).ID}` - case "struct": - return `/structs/${(object as Struct).META.META_COMMUN.ID}` - case "texte": - return `/textes/${(object as TexteVersion).META.META_COMMUN.ID}` - default: - assertNeverLegiObjectType(type) +export interface Versions { + VERSION: { + "@id": string + "@fin": "2999-01-01" + "@etat": "" + "@debut": "2999-01-01" } } -export function pathnameFromLegiObjectId( - type: LegiObjectType, +export interface XmlHeader { + "@encoding": "UTF-8" + "@version": "1.0" +} + +export function assertNeverLegalObjectType(type: never): never { + throw `Unexpected type for legal object: ${type}` +} + +export function bestItemForDate( + items: T | T[], + date: string, +): T | undefined { + if (!Array.isArray(items)) { + // Singleton + return items + } + for (const item of items) { + if (item["@debut"] <= date && date <= item["@fin"]) { + return item + } + } + return items[0] +} + +export function pathnameFromLegalObject( + type: LegalObjectType, + object: LegalObject, +): string { + switch (type) { + case "article": + return `/article/${(object as Article).META.META_COMMUN.ID}` + case "id": + return `/eli/ids/TODO` + case "jo": + return `/jo/${(object as Jo).META.META_COMMUN.ID}` + case "section_ta": + return `/section_ta/${(object as unknown as SectionTa).ID}` + case "texte_version": + return `/texte_version/${(object as TexteVersion).META.META_COMMUN.ID}` + case "textelr": + return `/textelr/${(object as Textelr).META.META_COMMUN.ID}` + case "versions": + return `/eli/versions/TODO` + default: + assertNeverLegalObjectType(type) + } +} + +export function pathnameFromLegalObjectId( + type: LegalObjectType, id: string, ): string { switch (type) { case "article": - return `/articles/${id}` - case "eli_id": + return `/article/${id}` + case "id": return `/eli/ids/TODO` - case "eli_versions": - return `/eli/ids/TODO` - case "section": - return `/sections/${id}` - case "struct": - return `/structs/${id}` - case "texte": - return `/textes/${id}` + case "jo": + return `/jo/${id}` + case "section_ta": + return `/section_ta/${id}` + case "texte_version": + return `/texte_version/${id}` + case "textelr": + return `/textelr/${id}` + case "versions": + return `/eli/versions/TODO` default: - assertNeverLegiObjectType(type) + assertNeverLegalObjectType(type) } } diff --git a/src/lib/scripts/import_jorf.ts b/src/lib/scripts/import_jorf.ts new file mode 100644 index 0000000..602ad82 --- /dev/null +++ b/src/lib/scripts/import_jorf.ts @@ -0,0 +1,369 @@ +import { auditChain, auditRequire, strictAudit } from "@auditors/core" +import assert from "assert" +import { XMLParser } from "fast-xml-parser" +import fs from "fs-extra" +import he from "he" +import path from "path" +import type { JSONValue } from "postgres" +import sade from "sade" + +import { auditId, auditVersions } from "$lib/auditors/data" +import type { + Article, + Jo, + SectionTa, + Textelr, + TexteVersion, + Versions, + XmlHeader, +} from "$lib/data" +import { db } from "$lib/server/database" +import { walkDir } from "$lib/server/file_systems" + +const xmlParser = new XMLParser({ + attributeNamePrefix: "@", + ignoreAttributes: false, + stopNodes: ["ARTICLE.BLOC_TEXTUEL.CONTENU", "ARTICLE.SM.CONTENU"], + tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue), +}) + +async function importJorf({ resume }: { resume?: string } = {}): Promise { + let skip = resume !== undefined + const deleteRemainingIds = !skip + + const articleRemainingIds = new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM article + WHERE id LIKE 'JORF%' + ` + ).map(({ id }) => id), + ) + const idRemainingElis = new Set( + ( + await db<{ eli: string }[]>` + SELECT eli + FROM id + ` + ).map(({ eli }) => eli), + ) + const joRemainingIds = new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM jo + ` + ).map(({ id }) => id), + ) + const sectionTaRemainingIds = new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM section_ta + WHERE id LIKE 'JORF%' + ` + ).map(({ id }) => id), + ) + const textelrRemainingIds = new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM textelr + WHERE id LIKE 'JORF%' + ` + ).map(({ id }) => id), + ) + const texteVersionRemainingIds = new Set( + ( + await db<{ id: string }[]>` + SELECT id + FROM texte_version + WHERE id LIKE 'JORF%' + ` + ).map(({ id }) => id), + ) + const versionsRemainingElis = new Set( + ( + await db<{ eli: string }[]>` + SELECT eli + FROM versions + ` + ).map(({ eli }) => eli), + ) + + const dataDir = path.join("..", "dila-data", "jorf") + assert(await fs.pathExists(dataDir)) + iterXmlFiles: for (const relativeSplitPath of walkDir(dataDir)) { + const relativePath = path.join(...relativeSplitPath) + if (skip) { + if (relativePath.startsWith(resume!)) { + skip = false + console.log(`Resuming at file ${relativePath}...`) + } else { + continue + } + } + + const filePath = path.join(dataDir, relativePath) + if (!filePath.endsWith(".xml")) { + console.info(`Skipping non XML file at ${filePath}`) + continue + } + + try { + const xmlString: string = await fs.readFile(filePath, { + encoding: "utf8", + }) + const xmlData = xmlParser.parse(xmlString) + for (const [key, element] of Object.entries(xmlData) as [ + string, + ( + | Article + | Jo + | SectionTa + | Textelr + | TexteVersion + | Versions + | XmlHeader + ), + ][]) { + switch (key) { + case "?xml": { + const xmlHeader = element as XmlHeader + assert.strictEqual(xmlHeader["@encoding"], "UTF-8", filePath) + assert.strictEqual(xmlHeader["@version"], "1.0", filePath) + break + } + case "ARTICLE": { + const article = element as Article + await db` + INSERT INTO article ( + id, + data + ) VALUES ( + ${article.META.META_COMMUN.ID}, + ${db.json(article as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(article as unknown as JSONValue)} + ` + articleRemainingIds.delete(article.META.META_COMMUN.ID) + break + } + case "ID": { + assert.strictEqual(relativeSplitPath[0], "global") + assert.strictEqual(relativeSplitPath[1], "eli") + const eli = relativeSplitPath.slice(2, -1).join("/") + const [id, idError] = auditChain(auditId, auditRequire)( + strictAudit, + element, + ) + assert.strictEqual( + idError, + null, + `Unexpected format for ID:\n${JSON.stringify( + id, + null, + 2, + )}\nError:\n${JSON.stringify(idError, null, 2)}`, + ) + assert + await db` + INSERT INTO id ( + eli, + id + ) VALUES ( + ${eli}, + ${id} + ) + ON CONFLICT (eli) + DO UPDATE SET + id = ${id} + ` + idRemainingElis.delete(eli) + break + } + case "JO": { + const jo = element as Jo + await db` + INSERT INTO jo ( + id, + data + ) VALUES ( + ${jo.META.META_COMMUN.ID}, + ${db.json(jo as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(jo as unknown as JSONValue)} + ` + joRemainingIds.delete(jo.META.META_COMMUN.ID) + break + } + case "SECTION_TA": { + const section = element as SectionTa + await db` + INSERT INTO section_ta ( + id, + data + ) VALUES ( + ${section.ID}, + ${db.json(section as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(section as unknown as JSONValue)} + ` + sectionTaRemainingIds.delete(section.ID) + break + } + case "TEXTE_VERSION": { + const version = element as TexteVersion + await db` + INSERT INTO texte_version ( + id, + data + ) VALUES ( + ${version.META.META_COMMUN.ID}, + ${db.json(version as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(version as unknown as JSONValue)} + ` + texteVersionRemainingIds.delete(version.META.META_COMMUN.ID) + break + } + case "TEXTELR": { + const textelr = element as Textelr + await db` + INSERT INTO textelr ( + id, + data + ) VALUES ( + ${textelr.META.META_COMMUN.ID}, + ${db.json(textelr as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(textelr as unknown as JSONValue)} + ` + textelrRemainingIds.delete(textelr.META.META_COMMUN.ID) + break + } + case "VERSIONS": { + assert.strictEqual(relativeSplitPath[0], "global") + assert.strictEqual(relativeSplitPath[1], "eli") + const eli = relativeSplitPath.slice(2, -1).join("/") + const [versions, versionsError] = auditChain( + auditVersions, + auditRequire, + )(strictAudit, element) + assert.strictEqual( + versionsError, + null, + `Unexpected format for VERSIONS:\n${JSON.stringify( + versions, + null, + 2, + )}\nError:\n${JSON.stringify(versionsError, null, 2)}`, + ) + const id = versions.VERSION["@id"] + await db` + INSERT INTO versions ( + eli, + id, + data + ) VALUES ( + ${eli}, + ${id}, + ${db.json(versions as unknown as JSONValue)} + ) + ON CONFLICT (eli) + DO UPDATE SET + id = ${id}, + data = ${db.json(versions as unknown as JSONValue)} + ` + versionsRemainingElis.delete(id) + break + } + default: { + console.warn( + `Unexpected root element "${key}" in XML file: ${filePath}`, + ) + break iterXmlFiles + } + } + } + } catch (e) { + console.error("An error occurred while parsing XML file", filePath) + throw e + } + } + + if (deleteRemainingIds) { + for (const id of articleRemainingIds) { + console.log(`Deleting ARTICLE ${id}…`) + await db` + DELETE FROM article + WHERE id = ${id} + ` + } + for (const eli of idRemainingElis) { + console.log(`Deleting ID ${eli}…`) + await db` + DELETE FROM id + WHERE eli = ${eli} + ` + } + for (const id of joRemainingIds) { + console.log(`Deleting JO ${id}…`) + await db` + DELETE FROM jo + WHERE id = ${id} + ` + } + for (const id of sectionTaRemainingIds) { + console.log(`Deleting SECTION_TA ${id}…`) + await db` + DELETE FROM section_ta + WHERE id = ${id} + ` + } + for (const id of textelrRemainingIds) { + console.log(`Deleting TEXTELR ${id}…`) + await db` + DELETE FROM textelr + WHERE id = ${id} + ` + } + for (const id of texteVersionRemainingIds) { + console.log(`Deleting TEXTE_VERSION ${id}…`) + await db` + DELETE FROM texte_version + WHERE id = ${id} + ` + } + for (const eli of versionsRemainingElis) { + console.log(`Deleting VERSIONS ${eli}…`) + await db` + DELETE FROM versions + WHERE eli = ${eli} + ` + } + } +} + +sade("import_jorf", true) + .describe("Import Dila's JORF database") + .option("-r", "--resume", "Resume import at given relative file path") + .example( + "--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml", + ) + .action(async (options) => { + await importJorf(options) + process.exit(0) + }) + .parse(process.argv) diff --git a/src/lib/scripts/import_legi.ts b/src/lib/scripts/import_legi.ts index 847e8dd..1c2107f 100644 --- a/src/lib/scripts/import_legi.ts +++ b/src/lib/scripts/import_legi.ts @@ -1,3 +1,4 @@ +import { auditChain, auditRequire, strictAudit } from "@auditors/core" import assert from "assert" import { XMLParser } from "fast-xml-parser" import fs from "fs-extra" @@ -6,22 +7,18 @@ import path from "path" import type { JSONValue } from "postgres" import sade from "sade" +import { auditId, auditVersions } from "$lib/auditors/data" import type { Article, - EliId, - EliVersions, - Section, - Struct, + SectionTa, + Textelr, TexteVersion, + Versions, + XmlHeader, } from "$lib/data" import { db } from "$lib/server/database" import { walkDir } from "$lib/server/file_systems" -interface XmlHeader { - "@encoding": "UTF-8" - "@version": "1.0" -} - const xmlParser = new XMLParser({ attributeNamePrefix: "@", ignoreAttributes: false, @@ -42,53 +39,57 @@ async function importLegi({ resume }: { resume?: string } = {}): Promise { let skip = resume !== undefined const deleteRemainingIds = !skip - const articlesRemainingIds = new Set( + const articleRemainingIds = new Set( ( await db<{ id: string }[]>` SELECT id - FROM articles + FROM article + WHERE id LIKE 'LEGI%' ` ).map(({ id }) => id), ) - const eliIdsRemainingIds = new Set( + const idRemainingElis = new Set( + ( + await db<{ eli: string }[]>` + SELECT eli + FROM id + ` + ).map(({ eli }) => eli), + ) + const sectionTaRemainingIds = new Set( ( await db<{ id: string }[]>` SELECT id - FROM eli_ids + FROM section_ta + WHERE id LIKE 'LEGI%' ` ).map(({ id }) => id), ) - const eliVersionsRemainingIds = new Set( + const textelrRemainingIds = new Set( ( await db<{ id: string }[]>` SELECT id - FROM eli_versions + FROM textelr + WHERE id LIKE 'LEGI%' ` ).map(({ id }) => id), ) - const sectionsRemainingIds = new Set( + const texteVersionRemainingElis = new Set( ( await db<{ id: string }[]>` SELECT id - FROM sections + FROM texte_version + WHERE id LIKE 'LEGI%' ` ).map(({ id }) => id), ) - const structsRemainingIds = new Set( + const versionsRemainingElis = new Set( ( - await db<{ id: string }[]>` - SELECT id - FROM structs + await db<{ eli: string }[]>` + SELECT eli + FROM versions ` - ).map(({ id }) => id), - ) - const textesRemainingIds = new Set( - ( - await db<{ id: string }[]>` - SELECT id - FROM textes_versions - ` - ).map(({ id }) => id), + ).map(({ eli }) => eli), ) const dataDir = path.join("..", "dila-data", "legi") @@ -109,190 +110,215 @@ async function importLegi({ resume }: { resume?: string } = {}): Promise { console.info(`Skipping non XML file at ${filePath}`) continue } - const xmlString: string = await fs.readFile(filePath, { - encoding: "utf8", - }) - const xmlData = xmlParser.parse(xmlString) - for (const [key, element] of Object.entries(xmlData) as [ - string, - ( - | Article - | EliId - | EliVersions - | Section - | Struct - | TexteVersion - | XmlHeader - ), - ][]) { - switch (key) { - case "?xml": - const xmlHeader = element as XmlHeader - assert.strictEqual(xmlHeader["@encoding"], "UTF-8", filePath) - assert.strictEqual(xmlHeader["@version"], "1.0", filePath) - break - case "ARTICLE": { - const article = element as Article - await db` - INSERT INTO articles ( - id, - data - ) VALUES ( - ${article.META.META_COMMUN.ID}, - ${db.json(article as unknown as JSONValue)} + + try { + const xmlString: string = await fs.readFile(filePath, { + encoding: "utf8", + }) + const xmlData = xmlParser.parse(xmlString) + for (const [key, element] of Object.entries(xmlData) as [ + string, + Article | SectionTa | Textelr | TexteVersion | Versions | XmlHeader, + ][]) { + switch (key) { + case "?xml": { + const xmlHeader = element as XmlHeader + assert.strictEqual(xmlHeader["@encoding"], "UTF-8", filePath) + assert.strictEqual(xmlHeader["@version"], "1.0", filePath) + break + } + case "ARTICLE": { + const article = element as Article + await db` + INSERT INTO article ( + id, + data + ) VALUES ( + ${article.META.META_COMMUN.ID}, + ${db.json(article as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(article as unknown as JSONValue)} + ` + articleRemainingIds.delete(article.META.META_COMMUN.ID) + break + } + case "ID": { + assert.strictEqual(relativeSplitPath[0], "global") + assert.strictEqual(relativeSplitPath[1], "eli") + const eli = relativeSplitPath.slice(2, -1).join("/") + const [id, idError] = auditChain(auditId, auditRequire)( + strictAudit, + element, ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(article as unknown as JSONValue)} - ` - articlesRemainingIds.delete(article.META.META_COMMUN.ID) - break - } - case "ID": { - assert.strictEqual(relativeSplitPath[0], "global") - assert.strictEqual(relativeSplitPath[1], "eli") - const id = relativeSplitPath.slice(2, -1).join("/") - const eliId = element as EliId - await db` - INSERT INTO eli_ids ( - id, - data - ) VALUES ( - ${id}, - ${db.json(eliId as unknown as JSONValue)} + assert.strictEqual( + idError, + null, + `Unexpected format for ID:\n${JSON.stringify( + id, + null, + 2, + )}\nError:\n${JSON.stringify(idError, null, 2)}`, ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(eliId as unknown as JSONValue)} - ` - eliIdsRemainingIds.delete(id) - break - } - case "SECTION_TA": { - const section = element as Section - await db` - INSERT INTO sections ( - id, - data - ) VALUES ( - ${section.ID}, - ${db.json(section as unknown as JSONValue)} + assert + await db` + INSERT INTO id ( + eli, + id + ) VALUES ( + ${eli}, + ${id} + ) + ON CONFLICT (eli) + DO UPDATE SET + id = ${id} + ` + idRemainingElis.delete(eli) + break + } + case "SECTION_TA": { + const section = element as SectionTa + await db` + INSERT INTO section_ta ( + id, + data + ) VALUES ( + ${section.ID}, + ${db.json(section as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(section as unknown as JSONValue)} + ` + sectionTaRemainingIds.delete(section.ID) + break + } + case "TEXTE_VERSION": { + const version = element as TexteVersion + await db` + INSERT INTO texte_version ( + id, + data + ) VALUES ( + ${version.META.META_COMMUN.ID}, + ${db.json(version as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(version as unknown as JSONValue)} + ` + texteVersionRemainingElis.delete(version.META.META_COMMUN.ID) + break + } + case "TEXTELR": { + const textelr = element as Textelr + await db` + INSERT INTO textelr ( + id, + data + ) VALUES ( + ${textelr.META.META_COMMUN.ID}, + ${db.json(textelr as unknown as JSONValue)} + ) + ON CONFLICT (id) + DO UPDATE SET + data = ${db.json(textelr as unknown as JSONValue)} + ` + textelrRemainingIds.delete(textelr.META.META_COMMUN.ID) + break + } + case "VERSIONS": { + assert.strictEqual(relativeSplitPath[0], "global") + assert.strictEqual(relativeSplitPath[1], "eli") + const eli = relativeSplitPath.slice(2, -1).join("/") + const [versions, versionsError] = auditChain( + auditVersions, + auditRequire, + )(strictAudit, element) + assert.strictEqual( + versionsError, + null, + `Unexpected format for VERSIONS:\n${JSON.stringify( + versions, + null, + 2, + )}\nError:\n${JSON.stringify(versionsError, null, 2)}`, ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(section as unknown as JSONValue)} - ` - sectionsRemainingIds.delete(section.ID) - break - } - case "TEXTELR": { - const struct = element as Struct - await db` - INSERT INTO structs ( - id, - data - ) VALUES ( - ${struct.META.META_COMMUN.ID}, - ${db.json(struct as unknown as JSONValue)} + const id = versions.VERSION["@id"] + await db` + INSERT INTO versions ( + eli, + id, + data + ) VALUES ( + ${eli}, + ${id}, + ${db.json(versions as unknown as JSONValue)} + ) + ON CONFLICT (eli) + DO UPDATE SET + id = ${id}, + data = ${db.json(versions as unknown as JSONValue)} + ` + versionsRemainingElis.delete(id) + break + } + default: { + console.warn( + `Unexpected root element "${key}" in XML file: ${filePath}`, ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(struct as unknown as JSONValue)} - ` - structsRemainingIds.delete(struct.META.META_COMMUN.ID) - break - } - case "TEXTE_VERSION": { - const version = element as TexteVersion - console.log("TEXTE_VERSION:", version.META.META_COMMUN.ID) - await db` - INSERT INTO textes_versions ( - id, - data - ) VALUES ( - ${version.META.META_COMMUN.ID}, - ${db.json(version as unknown as JSONValue)} - ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(version as unknown as JSONValue)} - ` - textesRemainingIds.delete(version.META.META_COMMUN.ID) - break - } - case "VERSIONS": { - assert.strictEqual(relativeSplitPath[0], "global") - assert.strictEqual(relativeSplitPath[1], "eli") - const id = relativeSplitPath.slice(2, -1).join("/") - const eliVersion = element as EliVersions - await db` - INSERT INTO eli_versions ( - id, - data - ) VALUES ( - ${id}, - ${db.json(eliVersion as unknown as JSONValue)} - ) - ON CONFLICT (id) - DO UPDATE SET - data = ${db.json(eliVersion as unknown as JSONValue)} - ` - eliVersionsRemainingIds.delete(id) - break - } - default: { - console.warn( - `Unexpected root element "${key}" in XML file: ${filePath}`, - ) - break iterXmlFiles + break iterXmlFiles + } } } + } catch (e) { + console.error("An error occurred while parsing XML file", filePath) + throw e } - // console.log(filePath) - // console.log(JSON.stringify(xmlData, null, 2)) } if (deleteRemainingIds) { - for (const id of articlesRemainingIds) { - console.log(`Deleting article ${id}…`) + for (const id of articleRemainingIds) { + console.log(`Deleting ARTICLE ${id}…`) await db` - DELETE FROM articles + DELETE FROM article WHERE id = ${id} ` } - for (const id of eliIdsRemainingIds) { - console.log(`Deleting ELI ID ${id}…`) + for (const eli of idRemainingElis) { + console.log(`Deleting ID ${eli}…`) await db` - DELETE FROM eli_ids + DELETE FROM id + WHERE eli = ${eli} + ` + } + for (const id of sectionTaRemainingIds) { + console.log(`Deleting SECTION_TA ${id}…`) + await db` + DELETE FROM section_ta WHERE id = ${id} ` } - for (const id of eliVersionsRemainingIds) { - console.log(`Deleting ELI versions ${id}…`) + for (const id of textelrRemainingIds) { + console.log(`Deleting TEXTELR ${id}…`) await db` - DELETE FROM eli_versions + DELETE FROM textelr WHERE id = ${id} ` } - for (const id of sectionsRemainingIds) { - console.log(`Deleting section ${id}…`) + for (const id of texteVersionRemainingElis) { + console.log(`Deleting TEXTE_VERSION ${id}…`) await db` - DELETE FROM sections + DELETE FROM texte_version WHERE id = ${id} ` } - for (const id of structsRemainingIds) { - console.log(`Deleting struct ${id}…`) + for (const eli of versionsRemainingElis) { + console.log(`Deleting VERSIONS ${eli}…`) await db` - DELETE FROM structs - WHERE id = ${id} - ` - } - for (const id of textesRemainingIds) { - console.log(`Deleting texte version ${id}…`) - await db` - DELETE FROM textes_versions - WHERE id = ${id} + DELETE FROM versions + WHERE eli = ${eli} ` } } diff --git a/src/lib/server/database/configuration.ts b/src/lib/server/database/configuration.ts index 674f154..d42bee9 100644 --- a/src/lib/server/database/configuration.ts +++ b/src/lib/server/database/configuration.ts @@ -1,5 +1,4 @@ import assert from "assert" -import dedent from "dedent-js" import { db, type Version, versionNumber } from "$lib/server/database" @@ -34,55 +33,72 @@ export async function configureDatabase() { // Apply patches that must be executed before every table is created. + if (version.number < 2) { + await db`ALTER TABLE IF EXISTS articles RENAME TO article` + await db`DROP TABLE IF EXISTS eli_ids` + await db`DROP TABLE IF EXISTS eli_versions` + await db`ALTER TABLE IF EXISTS sections RENAME TO section_ta` + await db`ALTER TABLE IF EXISTS structs RENAME TO textelr` + await db`ALTER TABLE IF EXISTS textes RENAME TO texte_version` + } + // Types // Tables - // Table: articles + // Table: article await db` - CREATE TABLE IF NOT EXISTS articles ( + CREATE TABLE IF NOT EXISTS article ( id char(20) PRIMARY KEY, data jsonb NOT NULL ) ` - // // Table: articles_autocompletions + // // Table: article_autocompletions // await ` - // CREATE TABLE IF NOT EXISTS articles_autocompletions ( + // CREATE TABLE IF NOT EXISTS article_autocompletions ( // autocompletion text NOT NULL, - // id bigint NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + // id bigint NOT NULL REFERENCES article(id) ON DELETE CASCADE, // weight int NOT NULL, // PRIMARY KEY (id, autocompletion) // ) // ` - // Table: eli_ids + // Table: id await db` - CREATE TABLE IF NOT EXISTS eli_ids ( - id text PRIMARY KEY, - data jsonb NOT NULL + CREATE TABLE IF NOT EXISTS id ( + eli text PRIMARY KEY, + id char(20) NOT NULL ) ` - // Table: eli_versions + // Table: jo await db` - CREATE TABLE IF NOT EXISTS eli_versions ( - id text PRIMARY KEY, - data jsonb NOT NULL - ) - ` - - // Table: sections - await db` - CREATE TABLE IF NOT EXISTS sections ( + CREATE TABLE IF NOT EXISTS jo ( id char(20) PRIMARY KEY, data jsonb NOT NULL ) ` - // Table: structs + // Table: section_ta await db` - CREATE TABLE IF NOT EXISTS structs ( + CREATE TABLE IF NOT EXISTS section_ta ( + id char(20) PRIMARY KEY, + data jsonb NOT NULL + ) + ` + + // Table: texte_version + await db` + CREATE TABLE IF NOT EXISTS texte_version ( + id char(20) PRIMARY KEY, + data jsonb NOT NULL + ) + ` + + // Table: textelr + await db` + CREATE TABLE IF NOT EXISTS textelr ( id char(20) PRIMARY KEY, data jsonb NOT NULL ) @@ -90,8 +106,9 @@ export async function configureDatabase() { // Table: versions await db` - CREATE TABLE IF NOT EXISTS textes_versions ( - id char(20) PRIMARY KEY, + CREATE TABLE IF NOT EXISTS versions ( + eli text PRIMARY KEY, + id char(20) NOT NULL, data jsonb NOT NULL ) ` @@ -101,8 +118,8 @@ export async function configureDatabase() { // Add indexes once every table and column exists. // await db` - // CREATE INDEX IF NOT EXISTS articles_autocompletions_trigrams_idx - // ON articles_autocompletions + // CREATE INDEX IF NOT EXISTS article_autocompletions_trigrams_idx + // ON article_autocompletions // USING GIST (autocompletion gist_trgm_ops) // ` diff --git a/src/lib/server/database/index.ts b/src/lib/server/database/index.ts index 73650f7..aa16dab 100644 --- a/src/lib/server/database/index.ts +++ b/src/lib/server/database/index.ts @@ -14,7 +14,7 @@ export const db = postgres({ port: config.db.port, user: config.db.user, }) -export const versionNumber = 1 +export const versionNumber = 2 /// Check that database exists and is up to date. export async function checkDb(): Promise { diff --git a/src/lib/summaries.ts b/src/lib/summaries.ts index 9dad72d..eb3eb0e 100644 --- a/src/lib/summaries.ts +++ b/src/lib/summaries.ts @@ -1,35 +1,46 @@ +import arrowRight from "@iconify-icons/codicon/arrow-small-right" import type { Access, Summarizer, Summary } from "augmented-data-viewer" import { type Article, - assertNeverLegiObjectType, - type LegiObject, - type LegiObjectType, - pathnameFromLegiObject, - pathnameFromLegiObjectId, - type Section, - type Struct, + assertNeverLegalObjectType, + bestItemForDate, + type Jo, + type LegalObject, + type LegalObjectType, + pathnameFromLegalObject, + pathnameFromLegalObjectId, + type SectionTa, + type Textelr, type TexteVersion, + type LienArt, } from "$lib/data" export const summarizeArticleProperties: Summarizer = (access, value) => { - if (access?.key === "article") { - return summarizeLegiObject(access, "article", value) + if (access?.key === "article" && typeof value !== "number") { + return summarizeLegalObject(access, "article", value) } - if (access?.access?.key === "articles") { - return summarizeLegiObjectToLink(access, "article", value) + if (typeof access?.key === "number" && access?.access?.key === "article") { + return summarizeLegalObjectToLink(access, "article", value) } - if (access?.key === "@id" && access?.access?.key === "LIEN_ART") { - return { - content: value as string, - href: pathnameFromLegiObjectId("article", value as string), - type: "link", - } + if ( + access?.key === "@id" && + (access?.access?.key === "LIEN_ART" || + (typeof access?.access?.key === "number" && + access?.access?.access?.key === "LIEN_ART")) + ) { + return summarizeLienArtId(access.access, access.parent) } if (access?.key === "CONTENU") { return { content: value as string, type: "html" } } + if (access?.key === "LIEN_ART" && !Array.isArray(value)) { + return summarizeLienArt(access, value) + } + if (typeof access?.key === "number" && access?.access?.key === "LIEN_ART") { + return summarizeLienArt(access, value) + } if (access?.access?.key === "VERSION") { const version = value as Article["VERSIONS"]["VERSION"][0] const lienArt = version.LIEN_ART @@ -60,16 +71,27 @@ export const summarizeArticleProperties: Summarizer = (access, value) => { ], type: "concatenation", }, - href: pathnameFromLegiObjectId("article", lienArt["@id"]), + href: pathnameFromLegalObjectId("article", lienArt["@id"]), type: "link", } } return undefined } -export function summarizeLegiObject( +export const summarizeJoProperties: Summarizer = (access, value) => { + if (access?.key === "jo" && typeof value !== "number") { + return summarizeLegalObject(access, "jo", value) + } + if (typeof access?.key === "number" && access?.access?.key === "jo") { + return summarizeLegalObjectToLink(access, "jo", value) + } + + return undefined +} + +export function summarizeLegalObject( access: Access | undefined, - type: LegiObjectType, + type: LegalObjectType, value: unknown, ): Summary | undefined { switch (type) { @@ -113,70 +135,127 @@ export function summarizeLegiObject( type: "concatenation", } } - case "eli_id": + case "id": return `/eli/ids/TODO` - case "eli_versions": - return `/eli/ids/TODO` - case "section": { - const section = value as Section | undefined - return section?.ID + case "jo": { + const jo = value as Jo | undefined + return jo?.META.META_SPEC.META_CONTENEUR.TITRE } - case "struct": { - const struct = value as Struct | undefined - return struct?.META.META_COMMUN.ID + case "section_ta": { + const sectionTa = value as SectionTa | undefined + if (sectionTa === undefined) { + return undefined + } + const today = new Date().toISOString().split("T")[0] + return `${sectionTa.TITRE_TA} — ${ + bestItemForDate(sectionTa.CONTEXTE.TEXTE.TITRE_TXT, today)?.["#text"] + }` } - case "texte": { - const texte = value as TexteVersion | undefined - return texte?.META.META_COMMUN.ID + case "texte_version": { + const texteVersion = value as TexteVersion | undefined + return texteVersion?.META.META_SPEC.META_TEXTE_VERSION.TITREFULL } + case "textelr": { + const textelr = value as Textelr | undefined + return textelr?.META.META_COMMUN.ID + } + case "versions": + return `/eli/versions/TODO` default: - assertNeverLegiObjectType(type) + assertNeverLegalObjectType(type) } } -export function summarizeLegiObjectToLink( +export function summarizeLegalObjectToLink( access: Access | undefined, - type: LegiObjectType, + type: LegalObjectType, value: unknown, ): Summary | undefined { - const objectSummary = summarizeLegiObject(access, type, value) + const objectSummary = summarizeLegalObject(access, type, value) return objectSummary === undefined ? undefined : { content: objectSummary, - href: pathnameFromLegiObject(type, value as LegiObject), + href: pathnameFromLegalObject(type, value as LegalObject), type: "link", } } -export const summarizeSectionProperties: Summarizer = (access, value) => { - if (access?.key === "section") { - return summarizeLegiObject(access, "section", value) +export const summarizeLienArt: Summarizer = (access, value) => { + const lienArt = value as LienArt | undefined + if (lienArt === undefined) { + return undefined } - if (access?.access?.key === "sections") { - return summarizeLegiObjectToLink(access, "section", value) + return { + content: `Article n° ${lienArt["@num"]}`, + href: pathnameFromLegalObjectId("article", lienArt["@id"]), + type: "link", + } +} + +export const summarizeLienArtId: Summarizer = (access, value) => { + const lienArt = value as LienArt | undefined + if (lienArt === undefined) { + return undefined + } + return { + items: [ + { + content: JSON.stringify(lienArt["@id"]), + type: "raw_data", + }, + { class: "mx-1", icon: arrowRight, inline: true, type: "icon" }, + summarizeLienArt(access, lienArt)!, + ], + type: "concatenation", + } +} + +export const summarizeSectionTaProperties: Summarizer = (access, value) => { + if (access?.key === "section_ta" && typeof value !== "number") { + return summarizeLegalObject(access, "section_ta", value) + } + if (typeof access?.key === "number" && access?.access?.key === "section_ta") { + return summarizeLegalObjectToLink(access, "section_ta", value) + } + + if ( + access?.key === "@id" && + (access?.access?.key === "LIEN_ART" || + (typeof access?.access?.key === "number" && + access?.access?.access?.key === "LIEN_ART")) + ) { + return summarizeLienArtId(access.access, access.parent) + } + if (access?.key === "LIEN_ART" && !Array.isArray(value)) { + return summarizeLienArt(access, value) + } + if (typeof access?.key === "number" && access?.access?.key === "LIEN_ART") { + return summarizeLienArt(access, value) + } + return undefined +} + +export const summarizeTextelrProperties: Summarizer = (access, value) => { + if (access?.key === "textelr" && typeof value !== "number") { + return summarizeLegalObject(access, "textelr", value) + } + if (typeof access?.key === "number" && access?.access?.key === "textelr") { + return summarizeLegalObjectToLink(access, "textelr", value) } return undefined } -export const summarizeStructProperties: Summarizer = (access, value) => { - if (access?.key === "struct") { - return summarizeLegiObject(access, "struct", value) +export const summarizeTexteVersionProperties: Summarizer = (access, value) => { + if (access?.key === "texte_version" && typeof value !== "number") { + return summarizeLegalObject(access, "texte_version", value) } - if (access?.access?.key === "structs") { - return summarizeLegiObjectToLink(access, "struct", value) - } - - return undefined -} - -export const summarizeTexteProperties: Summarizer = (access, value) => { - if (access?.key === "texte") { - return summarizeLegiObject(access, "texte", value) - } - if (access?.access?.key === "textes") { - return summarizeLegiObjectToLink(access, "texte", value) + if ( + typeof access?.key === "number" && + access?.access?.key === "texte_version" + ) { + return summarizeLegalObjectToLink(access, "texte_version", value) } return undefined diff --git a/src/routes/__layout.svelte b/src/routes/__layout.svelte index c6f637a..13ab40c 100644 --- a/src/routes/__layout.svelte +++ b/src/routes/__layout.svelte @@ -14,6 +14,7 @@ href?: string items?: MenuItemLink[] label: string + title?: string } interface MenuItemLink extends MenuItemBase { @@ -27,16 +28,36 @@ const menuItems: MenuItem[] = [ { href: "/recherche", label: "Recherche" }, { - label: "Données", items: [ - { href: "/articles", label: "Articles" }, - { href: "/eli/ids", label: "ELI ID" }, - { href: "/eli/versions", label: "ELI versions" }, - { href: "/sections", label: "Sections" }, - { href: "/structs", label: "Structures" }, - { href: "/textes", label: "Textes" }, + { href: "/article", label: "ARTICLE" }, + // { href: "/eli/ids", label: "ID" }, + { href: "/jo", label: "JO" }, + { href: "/section_ta", label: "SECTION_TA" }, + { href: "/texte_version", label: "TEXTE_VERSION" }, + { href: "/textelr", label: "TEXTELR" }, + // { href: "/eli/versions", label: "VERSIONS" }, ], + label: "Données", }, + // { + // items: [ + // { href: "/jo", label: "JO" }, + // ], + // label: "JORF", + // title: "Textes publiés au Journal officiel de la République française", + // }, + // { + // items: [ + // { href: "/article", label: "ARTICLE" }, + // // { href: "/eli/ids", label: "ID" }, + // { href: "/section_ta", label: "SECTION_TA" }, + // { href: "/texte_version", label: "TEXTE_VERSION" }, + // { href: "/textelr", label: "TEXTELR" }, + // // { href: "/eli/versions", label: "VERSIONS" }, + // ], + // label: "LEGI", + // title: "Codes, lois et règlements consolidés", + // }, ] const title = $session.title @@ -51,18 +72,18 @@ tabindex="0" class="dropdown-content menu rounded-box menu-compact mt-3 w-52 bg-neutral p-2 text-neutral-content shadow" > - {#each menuItems as { href, items, label }} + {#each menuItems as { href, items, label, title }} {#if href !== undefined} -
  • {label}
    • +
  • {label}
    • {:else if items !== undefined}
  • - + {label}
    • @@ -74,18 +95,18 @@