Validate & clean schedule of legislative files. Add export of schedules
This commit is contained in:
parent
4948510195
commit
2ad909e36e
8 changed files with 415 additions and 14 deletions
32
package-lock.json
generated
32
package-lock.json
generated
|
@ -20,6 +20,7 @@
|
|||
"@types/fs-extra": "^11.0.1",
|
||||
"@types/he": "^1.1.2",
|
||||
"@types/morgan": "^1.9.3",
|
||||
"@types/papaparse": "^5.3.7",
|
||||
"@typescript-eslint/eslint-plugin": "^5.27.0",
|
||||
"@typescript-eslint/parser": "^5.27.0",
|
||||
"augmented-data-viewer": "^0.2.0",
|
||||
|
@ -34,6 +35,7 @@
|
|||
"he": "^1.2.0",
|
||||
"iconify-icon": "^1.0.3",
|
||||
"morgan": "^1.10.0",
|
||||
"papaparse": "^5.3.2",
|
||||
"postcss": "^8.4.14",
|
||||
"postcss-load-config": "^4.0.1",
|
||||
"postgres": "^3.2.4",
|
||||
|
@ -867,6 +869,15 @@
|
|||
"integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/papaparse": {
|
||||
"version": "5.3.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.7.tgz",
|
||||
"integrity": "sha512-f2HKmlnPdCvS0WI33WtCs5GD7X1cxzzS/aduaxSu3I7TbhWlENjSPs6z5TaB9K0J+BH1jbmqTaM+ja5puis4wg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/ps-tree": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/ps-tree/-/ps-tree-1.1.2.tgz",
|
||||
|
@ -3380,6 +3391,12 @@
|
|||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/papaparse": {
|
||||
"version": "5.3.2",
|
||||
"resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.2.tgz",
|
||||
"integrity": "sha512-6dNZu0Ki+gyV0eBsFKJhYr+MdQYAzFUGlBMNj3GNrmHxmz1lfRa24CjFObPXtjcetlOv5Ad299MhIK0znp3afw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/parent-module": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
|
||||
|
@ -5409,6 +5426,15 @@
|
|||
"integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/papaparse": {
|
||||
"version": "5.3.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.7.tgz",
|
||||
"integrity": "sha512-f2HKmlnPdCvS0WI33WtCs5GD7X1cxzzS/aduaxSu3I7TbhWlENjSPs6z5TaB9K0J+BH1jbmqTaM+ja5puis4wg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/ps-tree": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/ps-tree/-/ps-tree-1.1.2.tgz",
|
||||
|
@ -7142,6 +7168,12 @@
|
|||
"p-limit": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"papaparse": {
|
||||
"version": "5.3.2",
|
||||
"resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.2.tgz",
|
||||
"integrity": "sha512-6dNZu0Ki+gyV0eBsFKJhYr+MdQYAzFUGlBMNj3GNrmHxmz1lfRa24CjFObPXtjcetlOv5Ad299MhIK0znp3afw==",
|
||||
"dev": true
|
||||
},
|
||||
"parent-module": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
|
||||
|
|
|
@ -33,6 +33,7 @@
|
|||
"@types/fs-extra": "^11.0.1",
|
||||
"@types/he": "^1.1.2",
|
||||
"@types/morgan": "^1.9.3",
|
||||
"@types/papaparse": "^5.3.7",
|
||||
"@typescript-eslint/eslint-plugin": "^5.27.0",
|
||||
"@typescript-eslint/parser": "^5.27.0",
|
||||
"augmented-data-viewer": "^0.2.0",
|
||||
|
@ -47,6 +48,7 @@
|
|||
"he": "^1.2.0",
|
||||
"iconify-icon": "^1.0.3",
|
||||
"morgan": "^1.10.0",
|
||||
"papaparse": "^5.3.2",
|
||||
"postcss": "^8.4.14",
|
||||
"postcss-load-config": "^4.0.1",
|
||||
"postgres": "^3.2.4",
|
||||
|
|
258
src/lib/auditors/dossiers_legislatifs.ts
Normal file
258
src/lib/auditors/dossiers_legislatifs.ts
Normal file
|
@ -0,0 +1,258 @@
|
|||
import {
|
||||
type Audit,
|
||||
auditRequire,
|
||||
auditDateIso8601String,
|
||||
auditArray,
|
||||
auditTrimString,
|
||||
auditInteger,
|
||||
auditFunction,
|
||||
auditStringToNumber,
|
||||
auditEmptyToNull,
|
||||
auditNullish,
|
||||
auditSwitch,
|
||||
auditString,
|
||||
auditTest,
|
||||
} from "@auditors/core"
|
||||
|
||||
function auditContenu(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
|
||||
if (dataUnknown == null) {
|
||||
return [dataUnknown, null]
|
||||
}
|
||||
if (typeof dataUnknown !== "object") {
|
||||
return audit.unexpectedType(dataUnknown, "object")
|
||||
}
|
||||
|
||||
const data = { ...dataUnknown }
|
||||
const errors: { [key: string]: unknown } = {}
|
||||
const remainingKeys = new Set(Object.keys(data))
|
||||
|
||||
audit.attribute(
|
||||
data,
|
||||
"ECHEANCIER",
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditSwitch(
|
||||
[auditTrimString, auditEmptyToNull, auditNullish],
|
||||
auditEcheancier,
|
||||
),
|
||||
)
|
||||
|
||||
// TODO
|
||||
|
||||
// TODO
|
||||
// return audit.reduceRemaining(data, errors, remainingKeys)
|
||||
return audit.reduceErrors(data, errors)
|
||||
}
|
||||
|
||||
export function auditDossierLegislatif(
|
||||
audit: Audit,
|
||||
dataUnknown: unknown,
|
||||
): [unknown, unknown] {
|
||||
if (dataUnknown == null) {
|
||||
return [dataUnknown, null]
|
||||
}
|
||||
if (typeof dataUnknown !== "object") {
|
||||
return audit.unexpectedType(dataUnknown, "object")
|
||||
}
|
||||
|
||||
const data = { ...dataUnknown }
|
||||
const errors: { [key: string]: unknown } = {}
|
||||
const remainingKeys = new Set(Object.keys(data))
|
||||
|
||||
audit.attribute(
|
||||
data,
|
||||
"META",
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditMeta,
|
||||
auditRequire,
|
||||
)
|
||||
audit.attribute(
|
||||
data,
|
||||
"CONTENU",
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditContenu,
|
||||
auditRequire,
|
||||
)
|
||||
|
||||
return audit.reduceRemaining(data, errors, remainingKeys)
|
||||
}
|
||||
|
||||
function auditEcheancier(
|
||||
audit: Audit,
|
||||
dataUnknown: unknown,
|
||||
): [unknown, unknown] {
|
||||
if (dataUnknown == null) {
|
||||
return [dataUnknown, null]
|
||||
}
|
||||
if (typeof dataUnknown !== "object") {
|
||||
return audit.unexpectedType(dataUnknown, "object")
|
||||
}
|
||||
|
||||
const data = { ...dataUnknown }
|
||||
const errors: { [key: string]: unknown } = {}
|
||||
const remainingKeys = new Set(Object.keys(data))
|
||||
|
||||
audit.attribute(
|
||||
data,
|
||||
"@derniere_maj",
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditTrimString,
|
||||
auditEmptyToNull,
|
||||
auditDateIso8601String,
|
||||
)
|
||||
audit.attribute(
|
||||
data,
|
||||
"LIGNE",
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditFunction((ligne) => (Array.isArray(ligne) ? ligne : [ligne])),
|
||||
auditArray(auditLigne, auditRequire),
|
||||
auditRequire,
|
||||
)
|
||||
|
||||
// TODO
|
||||
|
||||
// TODO
|
||||
// return audit.reduceRemaining(data, errors, remainingKeys)
|
||||
return audit.reduceErrors(data, errors)
|
||||
}
|
||||
|
||||
function auditLienArticle(
|
||||
audit: Audit,
|
||||
dataUnknown: unknown,
|
||||
): [unknown, unknown] {
|
||||
if (dataUnknown == null) {
|
||||
return [dataUnknown, null]
|
||||
}
|
||||
if (typeof dataUnknown !== "object") {
|
||||
return audit.unexpectedType(dataUnknown, "object")
|
||||
}
|
||||
|
||||
const data = { ...dataUnknown }
|
||||
const errors: { [key: string]: unknown } = {}
|
||||
const remainingKeys = new Set(Object.keys(data))
|
||||
|
||||
audit.attribute(
|
||||
data,
|
||||
"@id",
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditString,
|
||||
auditEmptyToNull,
|
||||
auditTest((id) => /^JORFARTI\d{12}$/.test(id), "Invalid ID"),
|
||||
auditRequire,
|
||||
)
|
||||
audit.attribute(
|
||||
data,
|
||||
"#text",
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditTrimString,
|
||||
auditEmptyToNull,
|
||||
auditRequire,
|
||||
)
|
||||
|
||||
return audit.reduceRemaining(data, errors, remainingKeys)
|
||||
}
|
||||
|
||||
function auditLigne(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
|
||||
if (dataUnknown == null) {
|
||||
return [dataUnknown, null]
|
||||
}
|
||||
if (typeof dataUnknown !== "object") {
|
||||
return audit.unexpectedType(dataUnknown, "object")
|
||||
}
|
||||
|
||||
const data = { ...dataUnknown }
|
||||
const errors: { [key: string]: unknown } = {}
|
||||
const remainingKeys = new Set(Object.keys(data))
|
||||
|
||||
for (const key of ["ARTICLE", "BASE_LEGALE", "DECRET", "OBJET"]) {
|
||||
audit.attribute(
|
||||
data,
|
||||
key,
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditTrimString,
|
||||
auditEmptyToNull,
|
||||
)
|
||||
}
|
||||
audit.attribute(
|
||||
data,
|
||||
"CID_LOI_CIBLE",
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditString,
|
||||
auditEmptyToNull,
|
||||
auditTest((id) => /^JORFTEXT\d{12}$/.test(id), "Invalid ID"),
|
||||
)
|
||||
audit.attribute(
|
||||
data,
|
||||
"DATE_PREVUE",
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditTrimString,
|
||||
auditEmptyToNull,
|
||||
auditDateIso8601String,
|
||||
)
|
||||
audit.attribute(
|
||||
data,
|
||||
"LIEN_ARTICLE",
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditFunction((lien) => (Array.isArray(lien) ? lien : [lien])),
|
||||
auditArray(auditLienArticle, auditRequire),
|
||||
)
|
||||
audit.attribute(
|
||||
data,
|
||||
"NUMERO_ORDRE",
|
||||
true,
|
||||
errors,
|
||||
remainingKeys,
|
||||
auditSwitch(
|
||||
[auditInteger, auditFunction((numero) => [numero])],
|
||||
[
|
||||
auditString,
|
||||
auditFunction((numeros) => numeros.split(",")),
|
||||
auditArray(auditStringToNumber, auditInteger),
|
||||
],
|
||||
),
|
||||
auditArray(auditRequire),
|
||||
auditRequire,
|
||||
)
|
||||
|
||||
return audit.reduceRemaining(data, errors, remainingKeys)
|
||||
}
|
||||
|
||||
function auditMeta(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
|
||||
if (dataUnknown == null) {
|
||||
return [dataUnknown, null]
|
||||
}
|
||||
if (typeof dataUnknown !== "object") {
|
||||
return audit.unexpectedType(dataUnknown, "object")
|
||||
}
|
||||
|
||||
const data = { ...dataUnknown }
|
||||
const errors: { [key: string]: unknown } = {}
|
||||
const remainingKeys = new Set(Object.keys(data))
|
||||
|
||||
// TODO
|
||||
|
||||
// TODO
|
||||
// return audit.reduceRemaining(data, errors, remainingKeys)
|
||||
return audit.reduceErrors(data, errors)
|
||||
}
|
|
@ -7,6 +7,7 @@ import {
|
|||
auditNullish,
|
||||
auditOptions,
|
||||
auditSetNullish,
|
||||
auditSingleton,
|
||||
auditStringToNumber,
|
||||
auditSwitch,
|
||||
auditTest,
|
||||
|
@ -219,15 +220,3 @@ export function auditSearchQueryContent(
|
|||
auditOffsetQueryParameter(audit, data, errors, remainingKeys)
|
||||
auditQQueryParameter(audit, data, errors, remainingKeys)
|
||||
}
|
||||
|
||||
export function auditSingleton(...auditors: Auditor[]): Auditor {
|
||||
return auditChain(
|
||||
auditArray(),
|
||||
auditTest(
|
||||
(values) => values.length <= 1,
|
||||
"Parameter must be present only once in query",
|
||||
),
|
||||
auditFunction((value) => value[0]),
|
||||
...auditors,
|
||||
)
|
||||
}
|
||||
|
|
|
@ -55,6 +55,12 @@ export interface DossierLegislatif {
|
|||
TITRE: string
|
||||
}
|
||||
}
|
||||
CONTENU: {
|
||||
ECHEANCIER?: {
|
||||
"@derniere_maj"?: string
|
||||
LIGNE: Ligne[]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export type Etat =
|
||||
|
@ -200,6 +206,20 @@ export interface LienSectionTa {
|
|||
"#text": string
|
||||
}
|
||||
|
||||
export interface Ligne {
|
||||
ARTICLE?: string
|
||||
BASE_LEGALE?: string
|
||||
CID_LOI_CIBLE?: string // Always a JORFTEXT ID
|
||||
DATE_PREVUE?: string
|
||||
DECRET?: string
|
||||
LIEN_ARTICLE?: Array<{
|
||||
"@id": string // Always a JORFARTI ID
|
||||
"#text": string
|
||||
}>
|
||||
NUMERO_ORDRE: number[]
|
||||
OBJET?: string
|
||||
}
|
||||
|
||||
export interface MetaCommun {
|
||||
ID: string
|
||||
URL: string
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
import {
|
||||
type Audit,
|
||||
auditSetNullish,
|
||||
auditSingleton,
|
||||
auditTrimString,
|
||||
cleanAudit,
|
||||
} from "@auditors/core"
|
||||
|
@ -13,7 +14,6 @@ import {
|
|||
auditLimitQueryParameter,
|
||||
auditOffsetQueryParameter,
|
||||
auditQQueryParameter,
|
||||
auditSingleton,
|
||||
} from "$lib/auditors/queries"
|
||||
import type { TexteVersion } from "$lib/legal"
|
||||
import { Aggregator } from "$lib/server/aggregates"
|
||||
|
|
86
src/scripts/export_dole_echeancier_to_csv.ts
Normal file
86
src/scripts/export_dole_echeancier_to_csv.ts
Normal file
|
@ -0,0 +1,86 @@
|
|||
import fs from "fs-extra"
|
||||
import Papa from "papaparse"
|
||||
import sade from "sade"
|
||||
|
||||
import type { DossierLegislatif } from "$lib/legal"
|
||||
import { db } from "$lib/server/database"
|
||||
|
||||
async function exportEcheanciers(csvFilePath: string): Promise<void> {
|
||||
const data: Array<{
|
||||
article?: string
|
||||
base_legale?: string
|
||||
cid_loi_cible?: string
|
||||
date_prevue?: string
|
||||
decret?: string
|
||||
derniere_maj?: string
|
||||
id: string
|
||||
lien_article_id?: string
|
||||
lien_article_texte?: string
|
||||
numero_ordre?: string
|
||||
objet?: string
|
||||
titre: string
|
||||
}> = []
|
||||
for (const { data: dossierLegislatif } of await db<
|
||||
{ data: DossierLegislatif }[]
|
||||
>`
|
||||
SELECT data FROM dossier_legislatif
|
||||
`) {
|
||||
const echeancier = dossierLegislatif.CONTENU.ECHEANCIER
|
||||
if (echeancier === undefined) {
|
||||
data.push({
|
||||
id: dossierLegislatif.META.META_COMMUN.ID,
|
||||
titre: dossierLegislatif.META.META_DOSSIER_LEGISLATIF.TITRE,
|
||||
})
|
||||
continue
|
||||
}
|
||||
for (const ligne of echeancier.LIGNE) {
|
||||
for (const lienArticle of ligne.LIEN_ARTICLE ?? [
|
||||
{ "@id": undefined, "#text": undefined },
|
||||
]) {
|
||||
data.push({
|
||||
article: ligne.ARTICLE,
|
||||
base_legale: ligne.BASE_LEGALE,
|
||||
cid_loi_cible: ligne.CID_LOI_CIBLE,
|
||||
date_prevue: ligne.DATE_PREVUE,
|
||||
decret: ligne.DECRET,
|
||||
lien_article_id: lienArticle["@id"],
|
||||
lien_article_texte: lienArticle["#text"],
|
||||
derniere_maj: echeancier["@derniere_maj"],
|
||||
id: dossierLegislatif.META.META_COMMUN.ID,
|
||||
numero_ordre: ligne.NUMERO_ORDRE.map((numero) =>
|
||||
numero.toString(),
|
||||
).join(", "),
|
||||
objet: ligne.OBJET,
|
||||
titre: dossierLegislatif.META.META_DOSSIER_LEGISLATIF.TITRE,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
const csvFile = await fs.writeFile(
|
||||
csvFilePath,
|
||||
Papa.unparse(data, {
|
||||
columns: [
|
||||
"id",
|
||||
"titre",
|
||||
"derniere_maj",
|
||||
"article",
|
||||
"lien_article_id",
|
||||
"lien_article_texte",
|
||||
"base_legale",
|
||||
"objet",
|
||||
"date_prevue",
|
||||
"decret",
|
||||
"cid_loi_cible",
|
||||
"numero_ordre",
|
||||
],
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
sade("export_dole_echeancier_to_csv <csvFilePath>", true)
|
||||
.describe("Export schedules of legislative files from Dila's DOLE database")
|
||||
.action(async (csvFilePath) => {
|
||||
await exportEcheanciers(csvFilePath)
|
||||
process.exit(0)
|
||||
})
|
||||
.parse(process.argv)
|
|
@ -1,3 +1,4 @@
|
|||
import { auditChain, auditRequire, strictAudit } from "@auditors/core"
|
||||
import assert from "assert"
|
||||
import { XMLParser } from "fast-xml-parser"
|
||||
import fs from "fs-extra"
|
||||
|
@ -6,6 +7,7 @@ import path from "path"
|
|||
import type { JSONValue } from "postgres"
|
||||
import sade from "sade"
|
||||
|
||||
import { auditDossierLegislatif } from "$lib/auditors/dossiers_legislatifs"
|
||||
import type { DossierLegislatif, XmlHeader } from "$lib/legal"
|
||||
import { db } from "$lib/server/database"
|
||||
import { walkDir } from "$lib/server/file_systems"
|
||||
|
@ -73,7 +75,19 @@ async function importDole(
|
|||
break
|
||||
}
|
||||
case "DOSSIER_LEGISLATIF": {
|
||||
const dossierLegislatif = element as DossierLegislatif
|
||||
const [dossierLegislatif, error] = auditChain(
|
||||
auditDossierLegislatif,
|
||||
auditRequire,
|
||||
)(strictAudit, element) as [DossierLegislatif, unknown]
|
||||
assert.strictEqual(
|
||||
error,
|
||||
null,
|
||||
`Unexpected format for DOSSIER_LEGISLATIF:\n${JSON.stringify(
|
||||
dossierLegislatif,
|
||||
null,
|
||||
2,
|
||||
)}\nError:\n${JSON.stringify(error, null, 2)}`,
|
||||
)
|
||||
await db`
|
||||
INSERT INTO dossier_legislatif (
|
||||
id,
|
||||
|
|
Loading…
Add table
Reference in a new issue