logiciels/tricoteuses-legifrance
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add JO validation and add category option to import scripts

Browse source
This commit is contained in:
Emmanuel 2023-02-11 21:25:48 +01:00
parent 126aebd7b1
commit bf3cda1a09
8 changed files with 1010 additions and 449 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
src/lib/auditors/jorf/articles.ts
View file

@ -23,6 +23,22 @@ import {
allJorfArticleTypesMutable,
} from "$lib/legal"
export const jorfArticleStats: {
// countByEtat: { [etat: string]: number }
// countByLienArtEtat: { [etat: string]: number }
// countByLienNature: { [nature: string]: number }
// countByLienType: { [type: string]: number }
// countByTexteNature: { [nature: string]: number }
// countByVersionEtat: { [etat: string]: number }
} = {
// countByEtat: {},
// countByLienArtEtat: {},
// countByLienNature: {},
// countByLienType: {},
// countByTexteNature: {},
// countByVersionEtat: {},
}
function auditBlocTextuel(
audit: Audit,
dataUnknown: unknown,

3
src/lib/auditors/jorf/index.ts
View file

@ -1 +1,2 @@
export { auditJorfArticle } from "./articles"
export { auditJo, joStats } from "./jo"
export { auditJorfArticle, jorfArticleStats } from "./articles"

376
src/lib/auditors/jorf/jo.ts Normal file
View file

@ -0,0 +1,376 @@
import {
type Audit,
auditRequire,
auditDateIso8601String,
auditTrimString,
auditInteger,
auditFunction,
auditEmptyToNull,
auditNullish,
auditSwitch,
auditNumber,
auditCleanArray,
auditOptions,
auditHttpUrl,
auditStringToNumber,
} from "@auditors/core"
import { allJoNaturesMutable, allJoOriginesMutable } from "$lib/legal"
export const joStats: {
countByNature: { [nature: string]: number }
countByOrigine: { [origine: string]: number }
} = {
countByNature: {},
countByOrigine: {},
}
export function auditJo(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"META",
true,
errors,
remainingKeys,
auditMeta,
auditRequire,
)
audit.attribute(
data,
"STRUCTURE_TXT",
true,
errors,
remainingKeys,
auditSwitch(
[auditTrimString, auditEmptyToNull, auditNullish],
auditStructureTxt,
),
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditLienTxt(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
for (const key of ["@idtxt", "@titretxt"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
}
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMeta(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"META_COMMUN",
true,
errors,
remainingKeys,
auditMetaCommun,
auditRequire,
)
audit.attribute(
data,
"META_SPEC",
true,
errors,
remainingKeys,
auditMetaSpec,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMetaCommun(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"ANCIEN_ID",
true,
errors,
remainingKeys,
auditSwitch(
[auditNumber, auditInteger, auditFunction((id) => id.toString())],
[auditTrimString, auditEmptyToNull],
),
auditNullish,
)
for (const key of ["ID", "URL"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
}
audit.attribute(
data,
"ID_ELI",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditHttpUrl,
)
audit.attribute(
data,
"NATURE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
// auditFunction((nature) => {
// joStats.countByNature[nature] = (joStats.countByNature[nature] ?? 0) + 1
// return nature
// }),
auditOptions(allJoNaturesMutable),
auditRequire,
)
audit.attribute(
data,
"ORIGINE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
// auditFunction((origine) => {
// joStats.countByOrigine[origine] =
// (joStats.countByOrigine[origine] ?? 0) + 1
// return origine
// }),
auditOptions(allJoOriginesMutable),
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMetaConteneur(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"DATE_PUBLI",
true,
errors,
remainingKeys,
auditDateIso8601String,
auditRequire,
)
audit.attribute(
data,
"NUM",
true,
errors,
remainingKeys,
auditSwitch(
[auditNumber, auditFunction((num) => num.toString())],
[auditTrimString, auditEmptyToNull],
),
)
audit.attribute(
data,
"TITRE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMetaSpec(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"META_CONTENEUR",
true,
errors,
remainingKeys,
auditMetaConteneur,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditStructureTxt(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"LIEN_TXT",
true,
errors,
remainingKeys,
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
auditCleanArray(auditLienTxt, auditRequire),
)
audit.attribute(
data,
"TM",
true,
errors,
remainingKeys,
auditFunction((tm) => (Array.isArray(tm) ? tm : [tm])),
auditCleanArray(auditTm, auditRequire),
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
/// Table des matières
function auditTm(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"@niv",
true,
errors,
remainingKeys,
auditStringToNumber,
auditNumber,
auditRequire,
)
audit.attribute(
data,
"LIEN_TXT",
true,
errors,
remainingKeys,
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
auditCleanArray(auditLienTxt, auditRequire),
)
audit.attribute(
data,
"TITRE_TM",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
audit.attribute(
data,
"TM",
true,
errors,
remainingKeys,
auditFunction((tm) => (Array.isArray(tm) ? tm : [tm])),
auditCleanArray(auditTm, auditRequire),
)
return audit.reduceRemaining(data, errors, remainingKeys)
}

6
src/lib/index.ts
View file

@ -20,6 +20,10 @@ export { default as TextelrView } from "./components/TextelrView.svelte"
export { default as TexteVersionView } from "./components/TexteVersionView.svelte"
export {
allJoNatures,
allJoNaturesMutable,
allJoOrigines,
allJoOriginesMutable,
allJorfArticleEtats,
allJorfArticleEtatsMutable,
allJorfArticleLienArticleOrigines,
@ -49,6 +53,8 @@ export {
allLegiArticleTypes,
allLegiArticleTypesMutable,
type DossierLegislatif,
type JoNature,
type JoOrigine,
type JorfArticle,
type JorfArticleEtat,
type JorfArticleLienArticleOrigine,

6
src/lib/legal/index.ts
View file

@ -4,6 +4,10 @@ import type { DossierLegislatif } from "./dole"
export type { DossierLegislatif } from "./dole"
export {
allJoNatures,
allJoNaturesMutable,
allJoOrigines,
allJoOriginesMutable,
allJorfArticleEtats,
allJorfArticleEtatsMutable,
allJorfArticleLienArticleOrigines,
@ -16,6 +20,8 @@ export {
allJorfArticleTexteNaturesMutable,
allJorfArticleTypes,
allJorfArticleTypesMutable,
type JoNature,
type JoOrigine,
type JorfArticle,
type JorfArticleEtat,
type JorfArticleLienArticleOrigine,

10
src/lib/legal/jorf.ts
View file

@ -59,6 +59,10 @@ export interface JorfArticle {
}
}
export type JoNature = (typeof allJoNatures)[number]
export type JoOrigine = (typeof allJoOrigines)[number]
export type JorfArticleEtat = (typeof allJorfArticleEtats)[number]
export type JorfArticleLienArticleOrigine =
@ -83,6 +87,12 @@ export interface JorfArticleTm {
TM?: JorfArticleTm
}
export const allJoNatures = ["JO"] as const
export const allJoNaturesMutable = [...allJoNatures]
export const allJoOrigines = ["JORF"] as const
export const allJoOriginesMutable = [...allJoOrigines]
export const allJorfArticleEtats = [
"ABROGE",
"ABROGE_DIFF",

571
src/scripts/import_jorf.ts
View file

@ -1,4 +1,9 @@
import { auditChain, auditRequire, strictAudit } from "@auditors/core"
import {
auditChain,
auditOptions,
auditRequire,
strictAudit,
} from "@auditors/core"
import assert from "assert"
import { XMLParser } from "fast-xml-parser"
import fs from "fs-extra"
@ -8,7 +13,11 @@ import type { JSONValue } from "postgres"
import sade from "sade"
import { auditId, auditVersions } from "$lib/auditors/legal"
import { auditJorfArticle } from "$lib/auditors/jorf"
import {
auditJo,
auditJorfArticle,
joStats,
} from "$lib/auditors/jorf"
import type {
Jo,
JorfArticle,
@ -21,6 +30,18 @@ import type {
import { db } from "$lib/server/database"
import { walkDir } from "$lib/server/file_systems"
type CategoryTag = (typeof allCategoriesCode)[number]
const allCategoriesCode = [
"ARTICLE",
"ID",
"JO",
"SECTION_TA",
"TEXTE_VERSION",
"TEXTELR",
"VERSIONS",
] as const
const xmlParser = new XMLParser({
attributeNamePrefix: "@",
ignoreAttributes: false,
@ -42,71 +63,105 @@ const xmlParser = new XMLParser({
async function importJorf(
dilaDir: string,
{ resume }: { resume?: string } = {},
{ category, resume }: { category?: string; resume?: string } = {},
): Promise<void> {
const [categoryTag, categoryError] = auditOptions([
...[...allCategoriesCode],
])(strictAudit, category) as [CategoryTag | undefined, unknown]
assert.strictEqual(
categoryError,
null,
`Error for category ${JSON.stringify(categoryTag)}:\n${JSON.stringify(
categoryError,
null,
2,
)}`,
)
let skip = resume !== undefined
const deleteRemainingIds = !skip
const articleRemainingIds = new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM article
WHERE id LIKE 'JORF%'
`
).map(({ id }) => id),
)
const idRemainingElis = new Set(
(
await db<{ eli: string }[]>`
SELECT eli
FROM id
`
).map(({ eli }) => eli),
)
const joRemainingIds = new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM jo
`
).map(({ id }) => id),
)
const sectionTaRemainingIds = new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM section_ta
WHERE id LIKE 'JORF%'
`
).map(({ id }) => id),
)
const textelrRemainingIds = new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM textelr
WHERE id LIKE 'JORF%'
`
).map(({ id }) => id),
)
const texteVersionRemainingIds = new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM texte_version
WHERE id LIKE 'JORF%'
`
).map(({ id }) => id),
)
const versionsRemainingElis = new Set(
(
await db<{ eli: string }[]>`
SELECT eli
FROM versions
`
).map(({ eli }) => eli),
)
const articleRemainingIds =
categoryTag === undefined || categoryTag === "ARTICLE"
? new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM article
WHERE id LIKE 'JORF%'
`
).map(({ id }) => id),
)
: new Set<string>()
const idRemainingElis =
categoryTag === undefined || categoryTag === "ID"
? new Set(
(
await db<{ eli: string }[]>`
SELECT eli
FROM id
`
).map(({ eli }) => eli),
)
: new Set<string>()
const joRemainingIds =
categoryTag === undefined || categoryTag === "JO"
? new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM jo
`
).map(({ id }) => id),
)
: new Set<string>()
const sectionTaRemainingIds =
categoryTag === undefined || categoryTag === "SECTION_TA"
? new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM section_ta
WHERE id LIKE 'JORF%'
`
).map(({ id }) => id),
)
: new Set<string>()
const textelrRemainingIds =
categoryTag === undefined || categoryTag === "TEXTELR"
? new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM textelr
WHERE id LIKE 'JORF%'
`
).map(({ id }) => id),
)
: new Set<string>()
const texteVersionRemainingIds =
categoryTag === undefined || categoryTag === "TEXTE_VERSION"
? new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM texte_version
WHERE id LIKE 'JORF%'
`
).map(({ id }) => id),
)
: new Set<string>()
const versionsRemainingElis =
categoryTag === undefined || categoryTag === "VERSIONS"
? new Set(
(
await db<{ eli: string }[]>`
SELECT eli
FROM versions
`
).map(({ eli }) => eli),
)
: new Set<string>()
const dataDir = path.join(dilaDir, "jorf")
assert(await fs.pathExists(dataDir))
@ -132,207 +187,238 @@ async function importJorf(
encoding: "utf8",
})
const xmlData = xmlParser.parse(xmlString)
for (const [key, element] of Object.entries(xmlData) as [
string,
for (const [tag, element] of Object.entries(xmlData) as [
CategoryTag | "?xml",
(
| JorfArticle
| Jo
| SectionTa
| JorfArticle
| JorfSectionTa
| Textelr
| TexteVersion
| Versions
| XmlHeader
),
][]) {
switch (key) {
switch (tag) {
case "?xml": {
const xmlHeader = element as XmlHeader
assert.strictEqual(xmlHeader["@encoding"], "UTF-8", filePath)
assert.strictEqual(xmlHeader["@version"], "1.0", filePath)
break
}
case "ARTICLE": {
const [article, error] = auditChain(auditJorfArticle, auditRequire)(
strictAudit,
element,
) as [JorfArticle, unknown]
assert.strictEqual(
error,
null,
`Unexpected format for ARTICLE:\n${JSON.stringify(
article,
case "ARTICLE":
if (categoryTag === undefined || categoryTag === tag) {
const [article, error] = auditChain(
auditJorfArticle,
auditRequire,
)(strictAudit, element) as [JorfArticle, unknown]
assert.strictEqual(
error,
null,
2,
)}\nError:\n${JSON.stringify(error, null, 2)}`,
)
await db`
INSERT INTO article (
id,
data
) VALUES (
${article.META.META_COMMUN.ID},
${db.json(article as unknown as JSONValue)}
`Unexpected format for ARTICLE:\n${JSON.stringify(
article,
null,
2,
)}\nError:\n${JSON.stringify(error, null, 2)}`,
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(article as unknown as JSONValue)}
`
articleRemainingIds.delete(article.META.META_COMMUN.ID)
await db`
INSERT INTO article (
id,
data
) VALUES (
${article.META.META_COMMUN.ID},
${db.json(article as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(article as unknown as JSONValue)}
`
articleRemainingIds.delete(article.META.META_COMMUN.ID)
}
break
}
case "ID": {
assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/")
const [id, idError] = auditChain(auditId, auditRequire)(
strictAudit,
element,
)
assert.strictEqual(
idError,
null,
`Unexpected format for ID:\n${JSON.stringify(
id,
case "ID":
if (categoryTag === undefined || categoryTag === tag) {
assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/")
const [id, idError] = auditChain(auditId, auditRequire)(
strictAudit,
element,
)
assert.strictEqual(
idError,
null,
2,
)}\nError:\n${JSON.stringify(idError, null, 2)}`,
)
await db`
INSERT INTO id (
eli,
id
) VALUES (
${eli},
${id}
`Unexpected format for ID:\n${JSON.stringify(
id,
null,
2,
)}\nError:\n${JSON.stringify(idError, null, 2)}`,
)
ON CONFLICT (eli)
DO UPDATE SET
id = ${id}
`
idRemainingElis.delete(eli)
await db`
INSERT INTO id (
eli,
id
) VALUES (
${eli},
${id}
)
ON CONFLICT (eli)
DO UPDATE SET
id = ${id}
`
idRemainingElis.delete(eli)
}
break
}
case "JO": {
const jo = element as Jo
await db`
INSERT INTO jo (
id,
data
) VALUES (
${jo.META.META_COMMUN.ID},
${db.json(jo as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(jo as unknown as JSONValue)}
`
joRemainingIds.delete(jo.META.META_COMMUN.ID)
break
}
case "SECTION_TA": {
const section = element as SectionTa
await db`
INSERT INTO section_ta (
id,
data
) VALUES (
${section.ID},
${db.json(section as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(section as unknown as JSONValue)}
`
sectionTaRemainingIds.delete(section.ID)
break
}
case "TEXTE_VERSION": {
const texteVersion = element as TexteVersion
const textAFragments = [
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE,
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITREFULL,
]
await db`
INSERT INTO texte_version (
id,
data,
nature,
text_search
) VALUES (
${texteVersion.META.META_COMMUN.ID},
${db.json(texteVersion as unknown as JSONValue)},
${texteVersion.META.META_COMMUN.NATURE},
setweight(to_tsvector('french', ${textAFragments.join(
" ",
)}), 'A')
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(texteVersion as unknown as JSONValue)},
nature = ${texteVersion.META.META_COMMUN.NATURE},
text_search = setweight(to_tsvector('french', ${textAFragments.join(
" ",
)}), 'A')
`
texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID)
break
}
case "TEXTELR": {
const textelr = element as Textelr
await db`
INSERT INTO textelr (
id,
data
) VALUES (
${textelr.META.META_COMMUN.ID},
${db.json(textelr as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(textelr as unknown as JSONValue)}
`
textelrRemainingIds.delete(textelr.META.META_COMMUN.ID)
break
}
case "VERSIONS": {
assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/")
const [versions, versionsError] = auditChain(
auditVersions,
auditRequire,
)(strictAudit, element)
assert.strictEqual(
versionsError,
null,
`Unexpected format for VERSIONS:\n${JSON.stringify(
versions,
case "JO":
if (categoryTag === undefined || categoryTag === tag) {
const [jo, error] = auditChain(auditJo, auditRequire)(
strictAudit,
element,
) as [Jo, unknown]
assert.strictEqual(
error,
null,
2,
)}\nError:\n${JSON.stringify(versionsError, null, 2)}`,
)
const id = versions.VERSION["@id"]
await db`
INSERT INTO versions (
eli,
id,
data
) VALUES (
${eli},
${id},
${db.json(versions as unknown as JSONValue)}
`Unexpected format for JO:\n${JSON.stringify(
jo,
null,
2,
)}\nError:\n${JSON.stringify(error, null, 2)}`,
)
ON CONFLICT (eli)
DO UPDATE SET
id = ${id},
data = ${db.json(versions as unknown as JSONValue)}
`
versionsRemainingElis.delete(id)
await db`
INSERT INTO jo (
id,
data
) VALUES (
${jo.META.META_COMMUN.ID},
${db.json(jo as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(jo as unknown as JSONValue)}
`
joRemainingIds.delete(jo.META.META_COMMUN.ID)
}
break
case "SECTION_TA":
if (categoryTag === undefined || categoryTag === tag) {
const [section, error] = auditChain(
auditJorfSectionTa,
auditRequire,
)(strictAudit, element) as [JorfSectionTa, unknown]
assert.strictEqual(
error,
null,
`Unexpected format for SECTION_TA:\n${JSON.stringify(
section,
null,
2,
)}\nError:\n${JSON.stringify(error, null, 2)}`,
)
await db`
INSERT INTO section_ta (
id,
data
) VALUES (
${section.ID},
${db.json(section as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(section as unknown as JSONValue)}
`
sectionTaRemainingIds.delete(section.ID)
}
break
case "TEXTE_VERSION":
if (categoryTag === undefined || categoryTag === tag) {
const texteVersion = element as TexteVersion
const textAFragments = [
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE,
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITREFULL,
]
await db`
INSERT INTO texte_version (
id,
data,
nature,
text_search
) VALUES (
${texteVersion.META.META_COMMUN.ID},
${db.json(texteVersion as unknown as JSONValue)},
${texteVersion.META.META_COMMUN.NATURE},
setweight(to_tsvector('french', ${textAFragments.join(
" ",
)}), 'A')
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(texteVersion as unknown as JSONValue)},
nature = ${texteVersion.META.META_COMMUN.NATURE},
text_search = setweight(to_tsvector('french', ${textAFragments.join(
" ",
)}), 'A')
`
texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID)
}
break
case "TEXTELR":
if (categoryTag === undefined || categoryTag === tag) {
const textelr = element as Textelr
await db`
INSERT INTO textelr (
id,
data
) VALUES (
${textelr.META.META_COMMUN.ID},
${db.json(textelr as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(textelr as unknown as JSONValue)}
`
textelrRemainingIds.delete(textelr.META.META_COMMUN.ID)
}
break
case "VERSIONS":
if (categoryTag === undefined || categoryTag === tag) {
assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/")
const [versions, versionsError] = auditChain(
auditVersions,
auditRequire,
)(strictAudit, element)
assert.strictEqual(
versionsError,
null,
`Unexpected format for VERSIONS:\n${JSON.stringify(
versions,
null,
2,
)}\nError:\n${JSON.stringify(versionsError, null, 2)}`,
)
const id = versions.VERSION["@id"]
await db`
INSERT INTO versions (
eli,
id,
data
) VALUES (
${eli},
${id},
${db.json(versions as unknown as JSONValue)}
)
ON CONFLICT (eli)
DO UPDATE SET
id = ${id},
data = ${db.json(versions as unknown as JSONValue)}
`
versionsRemainingElis.delete(id)
}
break
}
default: {
console.warn(
`Unexpected root element "${key}" in XML file: ${filePath}`,
`Unexpected root element "${tag}" in XML file: ${filePath}`,
)
break iterXmlFiles
}
@ -395,10 +481,17 @@ async function importJorf(
`
}
}
// console.log(
// "JORF articles stats =",
// JSON.stringify(jorfArticleStats, null, 2),
// )
console.log("JO stats =", JSON.stringify(joStats, null, 2))
}
sade("import_jorf <dilaDir>", true)
.describe("Import Dila's JORF database")
.option("-k, --category", "Import only given type of data")
.option("-r, --resume", "Resume import at given relative file path")
.example(
"--resume global/eli/accord/2002/5/5/MESS0221690X/jo/article_1/versions.xml ../dila-data/",

471
src/scripts/import_legi.ts
View file

@ -1,4 +1,9 @@
import { auditChain, auditRequire, strictAudit } from "@auditors/core"
import {
auditChain,
auditOptions,
auditRequire,
strictAudit,
} from "@auditors/core"
import assert from "assert"
import { XMLParser } from "fast-xml-parser"
import fs from "fs-extra"
@ -23,6 +28,17 @@ import type {
import { db } from "$lib/server/database"
import { walkDir } from "$lib/server/file_systems"
type CategoryTag = (typeof allCategoriesCode)[number]
const allCategoriesCode = [
"ARTICLE",
"ID",
"SECTION_TA",
"TEXTE_VERSION",
"TEXTELR",
"VERSIONS",
] as const
const xmlParser = new XMLParser({
attributeNamePrefix: "@",
ignoreAttributes: false,
@ -44,63 +60,94 @@ const xmlParser = new XMLParser({
async function importLegi(
dilaDir: string,
{ resume }: { resume?: string } = {},
{ category, resume }: { category?: string; resume?: string } = {},
): Promise<void> {
const [categoryTag, categoryError] = auditOptions([
...[...allCategoriesCode],
])(strictAudit, category) as [CategoryTag | undefined, unknown]
assert.strictEqual(
categoryError,
null,
`Error for category ${JSON.stringify(categoryTag)}:\n${JSON.stringify(
categoryError,
null,
2,
)}`,
)
let skip = resume !== undefined
const deleteRemainingIds = !skip
const articleRemainingIds = new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM article
WHERE id LIKE 'LEGI%'
`
).map(({ id }) => id),
)
const idRemainingElis = new Set(
(
await db<{ eli: string }[]>`
SELECT eli
FROM id
`
).map(({ eli }) => eli),
)
const sectionTaRemainingIds = new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM section_ta
WHERE id LIKE 'LEGI%'
`
).map(({ id }) => id),
)
const textelrRemainingIds = new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM textelr
WHERE id LIKE 'LEGI%'
`
).map(({ id }) => id),
)
const texteVersionRemainingIds = new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM texte_version
WHERE id LIKE 'LEGI%'
`
).map(({ id }) => id),
)
const versionsRemainingElis = new Set(
(
await db<{ eli: string }[]>`
SELECT eli
FROM versions
`
).map(({ eli }) => eli),
)
const articleRemainingIds =
categoryTag === undefined || categoryTag === "ARTICLE"
? new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM article
WHERE id LIKE 'LEGI%'
`
).map(({ id }) => id),
)
: new Set<string>()
const idRemainingElis =
categoryTag === undefined || categoryTag === "ID"
? new Set(
(
await db<{ eli: string }[]>`
SELECT eli
FROM id
`
).map(({ eli }) => eli),
)
: new Set<string>()
const sectionTaRemainingIds =
categoryTag === undefined || categoryTag === "SECTION_TA"
? new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM section_ta
WHERE id LIKE 'LEGI%'
`
).map(({ id }) => id),
)
: new Set<string>()
const textelrRemainingIds =
categoryTag === undefined || categoryTag === "TEXTELR"
? new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM textelr
WHERE id LIKE 'LEGI%'
`
).map(({ id }) => id),
)
: new Set<string>()
const texteVersionRemainingIds =
categoryTag === undefined || categoryTag === "TEXTE_VERSION"
? new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM texte_version
WHERE id LIKE 'LEGI%'
`
).map(({ id }) => id),
)
: new Set<string>()
const versionsRemainingElis =
categoryTag === undefined || categoryTag === "VERSIONS"
? new Set(
(
await db<{ eli: string }[]>`
SELECT eli
FROM versions
`
).map(({ eli }) => eli),
)
: new Set<string>()
const dataDir = path.join(dilaDir, "legi")
assert(await fs.pathExists(dataDir))
@ -126,183 +173,189 @@ async function importLegi(
encoding: "utf8",
})
const xmlData = xmlParser.parse(xmlString)
for (const [key, element] of Object.entries(xmlData) as [
string,
for (const [tag, element] of Object.entries(xmlData) as [
CategoryTag | "?xml",
LegiArticle | SectionTa | Textelr | TexteVersion | Versions | XmlHeader,
][]) {
switch (key) {
switch (tag) {
case "?xml": {
const xmlHeader = element as XmlHeader
assert.strictEqual(xmlHeader["@encoding"], "UTF-8", filePath)
assert.strictEqual(xmlHeader["@version"], "1.0", filePath)
break
}
case "ARTICLE": {
const [article, error] = auditChain(auditLegiArticle, auditRequire)(
strictAudit,
element,
) as [LegiArticle, unknown]
assert.strictEqual(
error,
null,
`Unexpected format for ARTICLE:\n${JSON.stringify(
article,
case "ARTICLE":
if (categoryTag === undefined || categoryTag === tag) {
const [article, error] = auditChain(
auditLegiArticle,
auditRequire,
)(strictAudit, element) as [LegiArticle, unknown]
assert.strictEqual(
error,
null,
2,
)}\nError:\n${JSON.stringify(error, null, 2)}`,
)
await db`
INSERT INTO article (
id,
data
) VALUES (
${article.META.META_COMMUN.ID},
${db.json(article as unknown as JSONValue)}
`Unexpected format for ARTICLE:\n${JSON.stringify(
article,
null,
2,
)}\nError:\n${JSON.stringify(error, null, 2)}`,
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(article as unknown as JSONValue)}
`
articleRemainingIds.delete(article.META.META_COMMUN.ID)
await db`
INSERT INTO article (
id,
data
) VALUES (
${article.META.META_COMMUN.ID},
${db.json(article as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(article as unknown as JSONValue)}
`
articleRemainingIds.delete(article.META.META_COMMUN.ID)
}
break
}
case "ID": {
assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/")
const [id, idError] = auditChain(auditId, auditRequire)(
strictAudit,
element,
)
assert.strictEqual(
idError,
null,
`Unexpected format for ID:\n${JSON.stringify(
id,
case "ID":
if (categoryTag === undefined || categoryTag === tag) {
assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/")
const [id, idError] = auditChain(auditId, auditRequire)(
strictAudit,
element,
)
assert.strictEqual(
idError,
null,
2,
)}\nError:\n${JSON.stringify(idError, null, 2)}`,
)
assert
await db`
INSERT INTO id (
eli,
id
) VALUES (
${eli},
${id}
`Unexpected format for ID:\n${JSON.stringify(
id,
null,
2,
)}\nError:\n${JSON.stringify(idError, null, 2)}`,
)
ON CONFLICT (eli)
DO UPDATE SET
id = ${id}
`
idRemainingElis.delete(eli)
assert
await db`
INSERT INTO id (
eli,
id
) VALUES (
${eli},
${id}
)
ON CONFLICT (eli)
DO UPDATE SET
id = ${id}
`
idRemainingElis.delete(eli)
}
break
}
case "SECTION_TA": {
const section = element as SectionTa
await db`
INSERT INTO section_ta (
id,
data
) VALUES (
${section.ID},
${db.json(section as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(section as unknown as JSONValue)}
`
sectionTaRemainingIds.delete(section.ID)
case "SECTION_TA":
if (categoryTag === undefined || categoryTag === tag) {
const section = element as SectionTa
await db`
INSERT INTO section_ta (
id,
data
) VALUES (
${section.ID},
${db.json(section as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(section as unknown as JSONValue)}
`
sectionTaRemainingIds.delete(section.ID)
}
break
}
case "TEXTE_VERSION": {
const texteVersion = element as TexteVersion
const textAFragments = [
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE,
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITREFULL,
]
await db`
INSERT INTO texte_version (
id,
data,
nature,
text_search
) VALUES (
${texteVersion.META.META_COMMUN.ID},
${db.json(texteVersion as unknown as JSONValue)},
${texteVersion.META.META_COMMUN.NATURE},
setweight(to_tsvector('french', ${textAFragments.join(
" ",
)}), 'A')
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(texteVersion as unknown as JSONValue)},
nature = ${texteVersion.META.META_COMMUN.NATURE},
text_search = setweight(to_tsvector('french', ${textAFragments.join(
" ",
)}), 'A')
`
texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID)
case "TEXTE_VERSION":
if (categoryTag === undefined || categoryTag === tag) {
const texteVersion = element as TexteVersion
const textAFragments = [
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE,
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITREFULL,
]
await db`
INSERT INTO texte_version (
id,
data,
nature,
text_search
) VALUES (
${texteVersion.META.META_COMMUN.ID},
${db.json(texteVersion as unknown as JSONValue)},
${texteVersion.META.META_COMMUN.NATURE},
setweight(to_tsvector('french', ${textAFragments.join(
" ",
)}), 'A')
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(texteVersion as unknown as JSONValue)},
nature = ${texteVersion.META.META_COMMUN.NATURE},
text_search = setweight(to_tsvector('french', ${textAFragments.join(
" ",
)}), 'A')
`
texteVersionRemainingIds.delete(texteVersion.META.META_COMMUN.ID)
}
break
}
case "TEXTELR": {
const textelr = element as Textelr
await db`
INSERT INTO textelr (
id,
data
) VALUES (
${textelr.META.META_COMMUN.ID},
${db.json(textelr as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(textelr as unknown as JSONValue)}
`
textelrRemainingIds.delete(textelr.META.META_COMMUN.ID)
case "TEXTELR":
if (categoryTag === undefined || categoryTag === tag) {
const textelr = element as Textelr
await db`
INSERT INTO textelr (
id,
data
) VALUES (
${textelr.META.META_COMMUN.ID},
${db.json(textelr as unknown as JSONValue)}
)
ON CONFLICT (id)
DO UPDATE SET
data = ${db.json(textelr as unknown as JSONValue)}
`
textelrRemainingIds.delete(textelr.META.META_COMMUN.ID)
}
break
}
case "VERSIONS": {
assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/")
const [versions, versionsError] = auditChain(
auditVersions,
auditRequire,
)(strictAudit, element)
assert.strictEqual(
versionsError,
null,
`Unexpected format for VERSIONS:\n${JSON.stringify(
versions,
case "VERSIONS":
if (categoryTag === undefined || categoryTag === tag) {
assert.strictEqual(relativeSplitPath[0], "global")
assert.strictEqual(relativeSplitPath[1], "eli")
const eli = relativeSplitPath.slice(2, -1).join("/")
const [versions, versionsError] = auditChain(
auditVersions,
auditRequire,
)(strictAudit, element)
assert.strictEqual(
versionsError,
null,
2,
)}\nError:\n${JSON.stringify(versionsError, null, 2)}`,
)
const id = versions.VERSION["@id"]
await db`
INSERT INTO versions (
eli,
id,
data
) VALUES (
${eli},
${id},
${db.json(versions as unknown as JSONValue)}
`Unexpected format for VERSIONS:\n${JSON.stringify(
versions,
null,
2,
)}\nError:\n${JSON.stringify(versionsError, null, 2)}`,
)
ON CONFLICT (eli)
DO UPDATE SET
id = ${id},
data = ${db.json(versions as unknown as JSONValue)}
`
versionsRemainingElis.delete(id)
const id = versions.VERSION["@id"]
await db`
INSERT INTO versions (
eli,
id,
data
) VALUES (
${eli},
${id},
${db.json(versions as unknown as JSONValue)}
)
ON CONFLICT (eli)
DO UPDATE SET
id = ${id},
data = ${db.json(versions as unknown as JSONValue)}
`
versionsRemainingElis.delete(id)
}
break
}
default: {
console.warn(
`Unexpected root element "${key}" in XML file: ${filePath}`,
`Unexpected root element "${tag}" in XML file: ${filePath}`,
)
break iterXmlFiles
}