Validate & clean schedule of legislative files. Add export of schedules

This commit is contained in:
Emmanuel 2023-02-03 09:40:18 +01:00
parent 4948510195
commit 2ad909e36e
8 changed files with 415 additions and 14 deletions

32
package-lock.json generated
View file

@ -20,6 +20,7 @@
"@types/fs-extra": "^11.0.1",
"@types/he": "^1.1.2",
"@types/morgan": "^1.9.3",
"@types/papaparse": "^5.3.7",
"@typescript-eslint/eslint-plugin": "^5.27.0",
"@typescript-eslint/parser": "^5.27.0",
"augmented-data-viewer": "^0.2.0",
@ -34,6 +35,7 @@
"he": "^1.2.0",
"iconify-icon": "^1.0.3",
"morgan": "^1.10.0",
"papaparse": "^5.3.2",
"postcss": "^8.4.14",
"postcss-load-config": "^4.0.1",
"postgres": "^3.2.4",
@ -867,6 +869,15 @@
"integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==",
"dev": true
},
"node_modules/@types/papaparse": {
"version": "5.3.7",
"resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.7.tgz",
"integrity": "sha512-f2HKmlnPdCvS0WI33WtCs5GD7X1cxzzS/aduaxSu3I7TbhWlENjSPs6z5TaB9K0J+BH1jbmqTaM+ja5puis4wg==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/ps-tree": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@types/ps-tree/-/ps-tree-1.1.2.tgz",
@ -3380,6 +3391,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/papaparse": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.2.tgz",
"integrity": "sha512-6dNZu0Ki+gyV0eBsFKJhYr+MdQYAzFUGlBMNj3GNrmHxmz1lfRa24CjFObPXtjcetlOv5Ad299MhIK0znp3afw==",
"dev": true
},
"node_modules/parent-module": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
@ -5409,6 +5426,15 @@
"integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==",
"dev": true
},
"@types/papaparse": {
"version": "5.3.7",
"resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.7.tgz",
"integrity": "sha512-f2HKmlnPdCvS0WI33WtCs5GD7X1cxzzS/aduaxSu3I7TbhWlENjSPs6z5TaB9K0J+BH1jbmqTaM+ja5puis4wg==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/ps-tree": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@types/ps-tree/-/ps-tree-1.1.2.tgz",
@ -7142,6 +7168,12 @@
"p-limit": "^3.0.2"
}
},
"papaparse": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.2.tgz",
"integrity": "sha512-6dNZu0Ki+gyV0eBsFKJhYr+MdQYAzFUGlBMNj3GNrmHxmz1lfRa24CjFObPXtjcetlOv5Ad299MhIK0znp3afw==",
"dev": true
},
"parent-module": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",

View file

@ -33,6 +33,7 @@
"@types/fs-extra": "^11.0.1",
"@types/he": "^1.1.2",
"@types/morgan": "^1.9.3",
"@types/papaparse": "^5.3.7",
"@typescript-eslint/eslint-plugin": "^5.27.0",
"@typescript-eslint/parser": "^5.27.0",
"augmented-data-viewer": "^0.2.0",
@ -47,6 +48,7 @@
"he": "^1.2.0",
"iconify-icon": "^1.0.3",
"morgan": "^1.10.0",
"papaparse": "^5.3.2",
"postcss": "^8.4.14",
"postcss-load-config": "^4.0.1",
"postgres": "^3.2.4",

View file

@ -0,0 +1,258 @@
import {
type Audit,
auditRequire,
auditDateIso8601String,
auditArray,
auditTrimString,
auditInteger,
auditFunction,
auditStringToNumber,
auditEmptyToNull,
auditNullish,
auditSwitch,
auditString,
auditTest,
} from "@auditors/core"
function auditContenu(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"ECHEANCIER",
true,
errors,
remainingKeys,
auditSwitch(
[auditTrimString, auditEmptyToNull, auditNullish],
auditEcheancier,
),
)
// TODO
// TODO
// return audit.reduceRemaining(data, errors, remainingKeys)
return audit.reduceErrors(data, errors)
}
export function auditDossierLegislatif(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"META",
true,
errors,
remainingKeys,
auditMeta,
auditRequire,
)
audit.attribute(
data,
"CONTENU",
true,
errors,
remainingKeys,
auditContenu,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditEcheancier(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"@derniere_maj",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditDateIso8601String,
)
audit.attribute(
data,
"LIGNE",
true,
errors,
remainingKeys,
auditFunction((ligne) => (Array.isArray(ligne) ? ligne : [ligne])),
auditArray(auditLigne, auditRequire),
auditRequire,
)
// TODO
// TODO
// return audit.reduceRemaining(data, errors, remainingKeys)
return audit.reduceErrors(data, errors)
}
function auditLienArticle(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"@id",
true,
errors,
remainingKeys,
auditString,
auditEmptyToNull,
auditTest((id) => /^JORFARTI\d{12}$/.test(id), "Invalid ID"),
auditRequire,
)
audit.attribute(
data,
"#text",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditLigne(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
for (const key of ["ARTICLE", "BASE_LEGALE", "DECRET", "OBJET"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
)
}
audit.attribute(
data,
"CID_LOI_CIBLE",
true,
errors,
remainingKeys,
auditString,
auditEmptyToNull,
auditTest((id) => /^JORFTEXT\d{12}$/.test(id), "Invalid ID"),
)
audit.attribute(
data,
"DATE_PREVUE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditDateIso8601String,
)
audit.attribute(
data,
"LIEN_ARTICLE",
true,
errors,
remainingKeys,
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
auditArray(auditLienArticle, auditRequire),
)
audit.attribute(
data,
"NUMERO_ORDRE",
true,
errors,
remainingKeys,
auditSwitch(
[auditInteger, auditFunction((numero) => [numero])],
[
auditString,
auditFunction((numeros) => numeros.split(",")),
auditArray(auditStringToNumber, auditInteger),
],
),
auditArray(auditRequire),
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMeta(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
// TODO
// TODO
// return audit.reduceRemaining(data, errors, remainingKeys)
return audit.reduceErrors(data, errors)
}

View file

@ -7,6 +7,7 @@ import {
auditNullish,
auditOptions,
auditSetNullish,
auditSingleton,
auditStringToNumber,
auditSwitch,
auditTest,
@ -219,15 +220,3 @@ export function auditSearchQueryContent(
auditOffsetQueryParameter(audit, data, errors, remainingKeys)
auditQQueryParameter(audit, data, errors, remainingKeys)
}
export function auditSingleton(...auditors: Auditor[]): Auditor {
return auditChain(
auditArray(),
auditTest(
(values) => values.length <= 1,
"Parameter must be present only once in query",
),
auditFunction((value) => value[0]),
...auditors,
)
}

View file

@ -55,6 +55,12 @@ export interface DossierLegislatif {
TITRE: string
}
}
CONTENU: {
ECHEANCIER?: {
"@derniere_maj"?: string
LIGNE: Ligne[]
}
}
}
export type Etat =
@ -200,6 +206,20 @@ export interface LienSectionTa {
"#text": string
}
export interface Ligne {
ARTICLE?: string
BASE_LEGALE?: string
CID_LOI_CIBLE?: string // Always a JORFTEXT ID
DATE_PREVUE?: string
DECRET?: string
LIEN_ARTICLE?: Array<{
"@id": string // Always a JORFARTI ID
"#text": string
}>
NUMERO_ORDRE: number[]
OBJET?: string
}
export interface MetaCommun {
ID: string
URL: string

View file

@ -1,6 +1,7 @@
import {
type Audit,
auditSetNullish,
auditSingleton,
auditTrimString,
cleanAudit,
} from "@auditors/core"
@ -13,7 +14,6 @@ import {
auditLimitQueryParameter,
auditOffsetQueryParameter,
auditQQueryParameter,
auditSingleton,
} from "$lib/auditors/queries"
import type { TexteVersion } from "$lib/legal"
import { Aggregator } from "$lib/server/aggregates"

View file

@ -0,0 +1,86 @@
import fs from "fs-extra"
import Papa from "papaparse"
import sade from "sade"
import type { DossierLegislatif } from "$lib/legal"
import { db } from "$lib/server/database"
async function exportEcheanciers(csvFilePath: string): Promise<void> {
const data: Array<{
article?: string
base_legale?: string
cid_loi_cible?: string
date_prevue?: string
decret?: string
derniere_maj?: string
id: string
lien_article_id?: string
lien_article_texte?: string
numero_ordre?: string
objet?: string
titre: string
}> = []
for (const { data: dossierLegislatif } of await db<
{ data: DossierLegislatif }[]
>`
SELECT data FROM dossier_legislatif
`) {
const echeancier = dossierLegislatif.CONTENU.ECHEANCIER
if (echeancier === undefined) {
data.push({
id: dossierLegislatif.META.META_COMMUN.ID,
titre: dossierLegislatif.META.META_DOSSIER_LEGISLATIF.TITRE,
})
continue
}
for (const ligne of echeancier.LIGNE) {
for (const lienArticle of ligne.LIEN_ARTICLE ?? [
{ "@id": undefined, "#text": undefined },
]) {
data.push({
article: ligne.ARTICLE,
base_legale: ligne.BASE_LEGALE,
cid_loi_cible: ligne.CID_LOI_CIBLE,
date_prevue: ligne.DATE_PREVUE,
decret: ligne.DECRET,
lien_article_id: lienArticle["@id"],
lien_article_texte: lienArticle["#text"],
derniere_maj: echeancier["@derniere_maj"],
id: dossierLegislatif.META.META_COMMUN.ID,
numero_ordre: ligne.NUMERO_ORDRE.map((numero) =>
numero.toString(),
).join(", "),
objet: ligne.OBJET,
titre: dossierLegislatif.META.META_DOSSIER_LEGISLATIF.TITRE,
})
}
}
}
const csvFile = await fs.writeFile(
csvFilePath,
Papa.unparse(data, {
columns: [
"id",
"titre",
"derniere_maj",
"article",
"lien_article_id",
"lien_article_texte",
"base_legale",
"objet",
"date_prevue",
"decret",
"cid_loi_cible",
"numero_ordre",
],
}),
)
}
sade("export_dole_echeancier_to_csv <csvFilePath>", true)
.describe("Export schedules of legislative files from Dila's DOLE database")
.action(async (csvFilePath) => {
await exportEcheanciers(csvFilePath)
process.exit(0)
})
.parse(process.argv)

View file

@ -1,3 +1,4 @@
import { auditChain, auditRequire, strictAudit } from "@auditors/core"
import assert from "assert"
import { XMLParser } from "fast-xml-parser"
import fs from "fs-extra"
@ -6,6 +7,7 @@ import path from "path"
import type { JSONValue } from "postgres"
import sade from "sade"
import { auditDossierLegislatif } from "$lib/auditors/dossiers_legislatifs"
import type { DossierLegislatif, XmlHeader } from "$lib/legal"
import { db } from "$lib/server/database"
import { walkDir } from "$lib/server/file_systems"
@ -73,7 +75,19 @@ async function importDole(
break
}
case "DOSSIER_LEGISLATIF": {
const dossierLegislatif = element as DossierLegislatif
const [dossierLegislatif, error] = auditChain(
auditDossierLegislatif,
auditRequire,
)(strictAudit, element) as [DossierLegislatif, unknown]
assert.strictEqual(
error,
null,
`Unexpected format for DOSSIER_LEGISLATIF:\n${JSON.stringify(
dossierLegislatif,
null,
2,
)}\nError:\n${JSON.stringify(error, null, 2)}`,
)
await db`
INSERT INTO dossier_legislatif (
id,