Fully validate & clean DOLE content

This commit is contained in:
Emmanuel 2023-02-07 17:47:38 +01:00
parent 23edd79999
commit 5730714865
3 changed files with 474 additions and 28 deletions

View file

@ -2,19 +2,178 @@ import {
type Audit, type Audit,
auditRequire, auditRequire,
auditDateIso8601String, auditDateIso8601String,
auditArray,
auditTrimString, auditTrimString,
auditInteger, auditInteger,
auditFunction, auditFunction,
auditStringToNumber,
auditEmptyToNull, auditEmptyToNull,
auditNullish, auditNullish,
auditSwitch, auditSwitch,
auditString, auditString,
auditTest, auditTest,
auditNumber, auditNumber,
auditCleanArray,
auditHttpUrl,
auditOptions,
} from "@auditors/core" } from "@auditors/core"
function auditArborescence(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"LIEN",
true,
errors,
remainingKeys,
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
auditCleanArray(auditArborescenceLien, auditRequire),
)
audit.attribute(
data,
"NIVEAU",
true,
errors,
remainingKeys,
auditFunction((niveau) => (Array.isArray(niveau) ? niveau : [niveau])),
auditCleanArray(auditArborescenceNiveau, auditRequire),
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditArborescenceLien(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"#text",
true,
errors,
remainingKeys,
auditSwitch(
[
auditNumber,
auditInteger,
auditFunction((article) => article.toString()),
],
auditTrimString,
),
auditEmptyToNull,
)
audit.attribute(
data,
"@id",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
audit.attribute(
data,
"@libelle",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
)
audit.attribute(
data,
"@lien",
true,
errors,
remainingKeys,
// auditFunction((url) => url.replace(//g, "")),
// auditTrimString,
// auditFunction((url) =>
// url
// .replace(/^http:\/\/webdim\//, "http://www.assemblee-nationale.fr/")
// .replace(/^https?:\/\/.*(https?):\/\//, "$1://"),
// ),
// auditEmptyToNull,
// auditHttpUrl,
auditTrimString,
auditEmptyToNull,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditArborescenceNiveau(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
for (const key of ["@id", "@libelle"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
}
audit.attribute(
data,
"LIEN",
true,
errors,
remainingKeys,
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
auditCleanArray(auditArborescenceLien, auditRequire),
)
audit.attribute(
data,
"NIVEAU",
true,
errors,
remainingKeys,
auditFunction((niveau) => (Array.isArray(niveau) ? niveau : [niveau])),
auditCleanArray(auditArborescenceNiveau, auditRequire),
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditContenu(audit: Audit, dataUnknown: unknown): [unknown, unknown] { function auditContenu(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
if (dataUnknown == null) { if (dataUnknown == null) {
return [dataUnknown, null] return [dataUnknown, null]
@ -27,6 +186,38 @@ function auditContenu(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
const errors: { [key: string]: unknown } = {} const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data)) const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"ARBORESCENCE",
true,
errors,
remainingKeys,
auditArborescence,
auditRequire,
)
for (const key of [
"CONTENU_DOSSIER_1",
"CONTENU_DOSSIER_2",
"CONTENU_DOSSIER_3",
"CONTENU_DOSSIER_4",
"CONTENU_DOSSIER_5",
"EXPOSE_MOTIF",
"LIBELLE_TEXTE_1",
"LIBELLE_TEXTE_2",
"LIBELLE_TEXTE_3",
"LIBELLE_TEXTE_4",
"LIBELLE_TEXTE_5",
]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
)
}
audit.attribute( audit.attribute(
data, data,
"ECHEANCIER", "ECHEANCIER",
@ -39,11 +230,7 @@ function auditContenu(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
), ),
) )
// TODO return audit.reduceRemaining(data, errors, remainingKeys)
// TODO
// return audit.reduceRemaining(data, errors, remainingKeys)
return audit.reduceErrors(data, errors)
} }
export function auditDossierLegislatif( export function auditDossierLegislatif(
@ -115,15 +302,60 @@ function auditEcheancier(
errors, errors,
remainingKeys, remainingKeys,
auditFunction((ligne) => (Array.isArray(ligne) ? ligne : [ligne])), auditFunction((ligne) => (Array.isArray(ligne) ? ligne : [ligne])),
auditArray(auditLigne, auditRequire), auditCleanArray(auditLigne, auditRequire),
auditRequire, auditRequire,
) )
// TODO return audit.reduceRemaining(data, errors, remainingKeys)
}
// TODO function auditLegislature(
// return audit.reduceRemaining(data, errors, remainingKeys) audit: Audit,
return audit.reduceErrors(data, errors) dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
for (const key of ["DATE_DEBUT", "DATE_FIN"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditDateIso8601String,
auditRequire,
)
}
audit.attribute(
data,
"LIBELLE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
audit.attribute(
data,
"NUMERO",
true,
errors,
remainingKeys,
auditInteger,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
} }
function auditLienArticle( function auditLienArticle(
@ -141,6 +373,16 @@ function auditLienArticle(
const errors: { [key: string]: unknown } = {} const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data)) const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"#text",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
audit.attribute( audit.attribute(
data, data,
"@id", "@id",
@ -152,16 +394,6 @@ function auditLienArticle(
auditTest((id) => /^JORFARTI\d{12}$/.test(id), "Invalid ID"), auditTest((id) => /^JORFARTI\d{12}$/.test(id), "Invalid ID"),
auditRequire, auditRequire,
) )
audit.attribute(
data,
"#text",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys) return audit.reduceRemaining(data, errors, remainingKeys)
} }
@ -232,7 +464,7 @@ function auditLigne(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
errors, errors,
remainingKeys, remainingKeys,
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])), auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
auditArray(auditLienArticle, auditRequire), auditCleanArray(auditLienArticle, auditRequire),
) )
audit.attribute( audit.attribute(
data, data,
@ -262,9 +494,166 @@ function auditMeta(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
const errors: { [key: string]: unknown } = {} const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data)) const remainingKeys = new Set(Object.keys(data))
// TODO audit.attribute(
data,
"META_COMMUN",
true,
errors,
remainingKeys,
auditMetaCommun,
auditRequire,
)
audit.attribute(
data,
"META_DOSSIER_LEGISLATIF",
true,
errors,
remainingKeys,
auditMetaDossierLegislatif,
auditRequire,
)
// TODO return audit.reduceRemaining(data, errors, remainingKeys)
// return audit.reduceRemaining(data, errors, remainingKeys) }
return audit.reduceErrors(data, errors)
function auditMetaCommun(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"ANCIEN_ID",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
)
for (const key of ["ID", "URL"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
}
audit.attribute(
data,
"NATURE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditNullish,
)
audit.attribute(
data,
"ORIGINE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditOptions(["JORF"]),
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
function auditMetaDossierLegislatif(
audit: Audit,
dataUnknown: unknown,
): [unknown, unknown] {
if (dataUnknown == null) {
return [dataUnknown, null]
}
if (typeof dataUnknown !== "object") {
return audit.unexpectedType(dataUnknown, "object")
}
const data = { ...dataUnknown }
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
for (const key of ["DATE_CREATION", "DATE_DERNIERE_MODIFICATION"]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditDateIso8601String,
auditRequire,
)
}
for (const key of [
"ID_TEXTE_1",
"ID_TEXTE_2",
"ID_TEXTE_3",
"ID_TEXTE_4",
"ID_TEXTE_5",
]) {
audit.attribute(
data,
key,
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
)
}
audit.attribute(
data,
"LEGISLATURE",
true,
errors,
remainingKeys,
auditLegislature,
auditRequire,
)
audit.attribute(
data,
"TITRE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditRequire,
)
audit.attribute(
data,
"TYPE",
true,
errors,
remainingKeys,
auditTrimString,
auditEmptyToNull,
auditOptions([
"LOI_PUBLIEE",
"PROJET_LOI",
"PROPOSITION_LOI",
"ORDONNANCE_PUBLIEE",
]),
)
return audit.reduceRemaining(data, errors, remainingKeys)
} }

View file

@ -1,5 +1,19 @@
import type { MenuItem } from "@tricoteuses/explorer-tools" import type { MenuItem } from "@tricoteuses/explorer-tools"
export interface ArborescenceLien {
"#text"?: string
"@id": string
"@libelle"?: string
"@lien"?: string
}
export interface ArborescenceNiveau {
"@id": string
"@libelle": string
LIEN?: ArborescenceLien[]
NIVEAU?: ArborescenceNiveau
}
export interface Article { export interface Article {
META: { META: {
META_COMMUN: MetaCommun META_COMMUN: MetaCommun
@ -50,16 +64,56 @@ export interface Contexte {
export interface DossierLegislatif { export interface DossierLegislatif {
META: { META: {
META_COMMUN: MetaCommun META_COMMUN: {
// MetaCommun
ANCIEN_ID?: string
ID: string
URL: string
// NATURE?: Nature // Never used
ORIGINE: "JORF" // Origine
}
META_DOSSIER_LEGISLATIF: { META_DOSSIER_LEGISLATIF: {
DATE_CREATION: string
DATE_DERNIERE_MODIFICATION: string
ID_TEXTE_1?: string
ID_TEXTE_2?: string
ID_TEXTE_3?: string
ID_TEXTE_4?: string
ID_TEXTE_5?: string
LEGISLATURE: {
DATE_DEBUT: string
DATE_FIN: string
LIBELLE: string
NUMERO: number
}
TITRE: string TITRE: string
TYPE?:
| "LOI_PUBLIEE"
| "PROJET_LOI"
| "PROPOSITION_LOI"
| "ORDONNANCE_PUBLIEE"
} }
} }
CONTENU: { CONTENU: {
ARBORESCENCE: {
LIEN?: ArborescenceLien[]
NIVEAU?: ArborescenceNiveau
}
CONTENU_DOSSIER_1?: string
CONTENU_DOSSIER_2?: string
CONTENU_DOSSIER_3?: string
CONTENU_DOSSIER_4?: string
CONTENU_DOSSIER_5?: string
ECHEANCIER?: { ECHEANCIER?: {
"@derniere_maj"?: string "@derniere_maj"?: string
LIGNE: Ligne[] LIGNE: Ligne[]
} }
EXPOSE_MOTIF?: string
LIBELLE_TEXTE_1?: string
LIBELLE_TEXTE_2?: string
LIBELLE_TEXTE_3?: string
LIBELLE_TEXTE_4?: string
LIBELLE_TEXTE_5?: string
} }
} }

View file

@ -18,6 +18,9 @@ const xmlParser = new XMLParser({
stopNodes: [ stopNodes: [
"DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_1", "DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_1",
"DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_2", "DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_2",
"DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_3",
"DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_4",
"DOSSIER_LEGISLATIF.CONTENU.CONTENU_DOSSIER_5",
"DOSSIER_LEGISLATIF.CONTENU.EXPOSE_MOTIF", "DOSSIER_LEGISLATIF.CONTENU.EXPOSE_MOTIF",
], ],
tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue), tagValueProcessor: (_tagName, tagValue) => he.decode(tagValue),