Fully validate & clean DOLE content

This commit is contained in:
Emmanuel 2023-02-07 17:47:38 +01:00
parent 23edd79999
commit 5730714865
3 changed files with 474 additions and 28 deletions

View file

@ -2,19 +2,178 @@ import {
type Audit,
auditRequire,
auditDateIso8601String,
auditArray,
auditTrimString,
auditInteger,
auditFunction,
auditStringToNumber,
auditEmptyToNull,
auditNullish,
auditSwitch,
auditString,
auditTest,
auditNumber,
auditCleanArray,
auditHttpUrl,
auditOptions,
} from "@auditors/core"
function auditArborescence(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"LIEN",
true,
errors,
remainingKeys,
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
auditCleanArray(auditArborescenceLien, auditRequire),
)
audit.attribute(
data,
"NIVEAU",
true,
errors,
remainingKeys,
auditFunction((niveau) => (Array.isArray(niveau) ? niveau : [niveau])),
auditCleanArray(auditArborescenceNiveau, auditRequire),
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditArborescenceLien(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"#text",
true,
errors,
remainingKeys,
auditSwitch(
[
auditNumber,
auditInteger,
auditFunction((article) => article.toString()),
],
auditTrimString,
),
auditEmptyToNull,
)
audit.attribute(
data,
"@id",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
audit.attribute(
data,
"@libelle",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
)
audit.attribute(
data,
"@lien",
true,
errors,
remainingKeys,
// auditFunction((url) => url.replace(//g, "")),
// auditTrimString,
// auditFunction((url) =>
// url
// .replace(/^http:\/\/webdim\//, "http://www.assemblee-nationale.fr/")
// .replace(/^https?:\/\/.*(https?):\/\//, "$1://"),
// ),
// auditEmptyToNull,
// auditHttpUrl,
auditTrimString,
auditEmptyToNull,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditArborescenceNiveau(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
for (const key of ["@id", "@libelle"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
}
audit.attribute(
data,
"LIEN",
true,
errors,
remainingKeys,
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
auditCleanArray(auditArborescenceLien, auditRequire),
)
audit.attribute(
data,
"NIVEAU",
true,
errors,
remainingKeys,
auditFunction((niveau) => (Array.isArray(niveau) ? niveau : [niveau])),
auditCleanArray(auditArborescenceNiveau, auditRequire),
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditContenu(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
@ -27,6 +186,38 @@ function auditContenu(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"ARBORESCENCE",
true,
errors,
remainingKeys,
auditArborescence,
auditRequire,
)
for (const key of [
"CONTENU_DOSSIER_1",
"CONTENU_DOSSIER_2",
"CONTENU_DOSSIER_3",
"CONTENU_DOSSIER_4",
"CONTENU_DOSSIER_5",
"EXPOSE_MOTIF",
"LIBELLE_TEXTE_1",
"LIBELLE_TEXTE_2",
"LIBELLE_TEXTE_3",
"LIBELLE_TEXTE_4",
"LIBELLE_TEXTE_5",
]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
)
}
audit.attribute(
data,
"ECHEANCIER",
@ -39,11 +230,7 @@ function auditContenu(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
),
)
// TODO
// TODO
// return audit.reduceRemaining(data, errors, remainingKeys)
return audit.reduceErrors(data, errors)
return audit.reduceRemaining(data, errors, remainingKeys)
}
export function auditDossierLegislatif(
@ -115,15 +302,60 @@ function auditEcheancier(
errors,
remainingKeys,
auditFunction((ligne) => (Array.isArray(ligne) ? ligne : [ligne])),
auditArray(auditLigne, auditRequire),
auditCleanArray(auditLigne, auditRequire),
auditRequire,
)
// TODO
return audit.reduceRemaining(data, errors, remainingKeys)
}
// TODO
// return audit.reduceRemaining(data, errors, remainingKeys)
return audit.reduceErrors(data, errors)
function auditLegislature(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
for (const key of ["DATE_DEBUT", "DATE_FIN"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditDateIso8601String,
auditRequire,
)
}
audit.attribute(
data,
"LIBELLE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
audit.attribute(
data,
"NUMERO",
true,
errors,
remainingKeys,
auditInteger,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditLienArticle(
@ -141,6 +373,16 @@ function auditLienArticle(
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"#text",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
audit.attribute(
data,
"@id",
@ -152,16 +394,6 @@ function auditLienArticle(
auditTest((id) => /^JORFARTI\d{12}$/.test(id), "Invalid ID"),
auditRequire,
)
audit.attribute(
data,
"#text",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
@ -232,7 +464,7 @@ function auditLigne(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
errors,
remainingKeys,
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
auditArray(auditLienArticle, auditRequire),
auditCleanArray(auditLienArticle, auditRequire),
)
audit.attribute(
data,
@ -262,9 +494,166 @@ function auditMeta(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
// TODO
audit.attribute(
data,
"META_COMMUN",
true,
errors,
remainingKeys,
auditMetaCommun,
auditRequire,
)
audit.attribute(
data,
"META_DOSSIER_LEGISLATIF",
true,
errors,
remainingKeys,
auditMetaDossierLegislatif,
auditRequire,
)
// TODO
// return audit.reduceRemaining(data, errors, remainingKeys)
return audit.reduceErrors(data, errors)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMetaCommun(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"ANCIEN_ID",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
)
for (const key of ["ID", "URL"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
}
audit.attribute(
data,
"NATURE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditNullish,
)
audit.attribute(
data,
"ORIGINE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditOptions(["JORF"]),
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMetaDossierLegislatif(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
for (const key of ["DATE_CREATION", "DATE_DERNIERE_MODIFICATION"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditDateIso8601String,
auditRequire,
)
}
for (const key of [
"ID_TEXTE_1",
"ID_TEXTE_2",
"ID_TEXTE_3",
"ID_TEXTE_4",
"ID_TEXTE_5",
]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
)
}
audit.attribute(
data,
"LEGISLATURE",
true,
errors,
remainingKeys,
auditLegislature,
auditRequire,
)
audit.attribute(
data,
"TITRE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
audit.attribute(
data,
"TYPE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditOptions([
"LOI_PUBLIEE",
"PROJET_LOI",
"PROPOSITION_LOI",
"ORDONNANCE_PUBLIEE",
]),
)
return audit.reduceRemaining(data, errors, remainingKeys)
}

View file

@ -1,5 +1,19 @@
import type { MenuItem } from "@tricoteuses/explorer-tools"
export interface ArborescenceLien {
"#text"?: string
"@id": string
"@libelle"?: string
"@lien"?: string
}
export interface ArborescenceNiveau {
"@id": string
"@libelle": string
LIEN?: ArborescenceLien[]
NIVEAU?: ArborescenceNiveau
}
export interface Article {
META: {
META_COMMUN: MetaCommun
@ -50,16 +64,56 @@ export interface Contexte {
export interface DossierLegislatif {
META: {
META_COMMUN: MetaCommun
META_COMMUN: {
// MetaCommun
ANCIEN_ID?: string
ID: string
URL: string
// NATURE?: Nature // Never used
ORIGINE: "JORF" // Origine
}
META_DOSSIER_LEGISLATIF: {
DATE_CREATION: string
DATE_DERNIERE_MODIFICATION: string
ID_TEXTE_1?: string
ID_TEXTE_2?: string
ID_TEXTE_3?: string
ID_TEXTE_4?: string
ID_TEXTE_5?: string
LEGISLATURE: {
DATE_DEBUT: string
DATE_FIN: string
LIBELLE: string
NUMERO: number
}
TITRE: string
TYPE?:
| "LOI_PUBLIEE"
| "PROJET_LOI"
| "PROPOSITION_LOI"
| "ORDONNANCE_PUBLIEE"
}
}
CONTENU: {
ARBORESCENCE: {
LIEN?: ArborescenceLien[]
NIVEAU?: ArborescenceNiveau
}
CONTENU_DOSSIER_1?: string
CONTENU_DOSSIER_2?: string
CONTENU_DOSSIER_3?: string
CONTENU_DOSSIER_4?: string
CONTENU_DOSSIER_5?: string
ECHEANCIER?: {
"@derniere_maj"?: string
LIGNE: Ligne[]
}
EXPOSE_MOTIF?: string
LIBELLE_TEXTE_1?: string
LIBELLE_TEXTE_2?: string
LIBELLE_TEXTE_3?: string
LIBELLE_TEXTE_4?: string
LIBELLE_TEXTE_5?: string
}
}

View file

@ -18,6 +18,9 @@ const xmlParser = new XMLParser({
stopNodes: [
"DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_1",
"DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_2",
"DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_3",
"DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_4",
"DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_5",
"DOSSIER_LEGISLATIF.CONTENU.EXPOSE_MOTIF",
],
tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue),