Add associations between Légifrance & Assemblée dossiers législatifs

This commit is contained in:
Emmanuel 2023-03-05 19:32:37 +01:00
parent 5c3f3ceb20
commit 1ceff78750
44 changed files with 1829 additions and 45 deletions

View file

@ -54,25 +54,21 @@ mkdir -p ../dila-data/dole
cd ../dila-data/dole
git init
cd -
npx tsx src/scripts/download_dila_dataset.ts dole
mkdir -p ../dila-data/jorf
cd ../dila-data/jorf
git init
cd -
npx tsx src/scripts/download_dila_dataset.ts jorf
mkdir -p ../dila-data/kali
cd ../dila-data/kali
git init
cd -
npx tsx src/scripts/download_dila_dataset.ts kali
mkdir -p ../dila-data/legi
cd ../dila-data/legi
git init
cd -
npx tsx src/scripts/download_dila_dataset.ts legi
```
## Datasets Update
@ -91,6 +87,9 @@ npx tsx src/scripts/import_dole.ts ../dila-data/
npx tsx src/scripts/import_jorf.ts ../dila-data/
npx tsx src/scripts/import_kali.ts ../dila-data/
npx tsx src/scripts/import_legi.ts ../dila-data/
# Note: Assemblee database must be updated before launching this script:
npx tsx src/scripts/associate_dossiers_legislatifs_with_assemblee.ts
```
## Server Launch

View file

@ -1,3 +1,11 @@
# PostgreSQL database configuration
ASSEMBLEE_DB_NAME="assemblee"
ASSEMBLEE_DB_HOST="localhost"
ASSEMBLEE_DB_PORT=5432
ASSEMBLEE_DB_USER="assemblee"
# Change value of DB_PASSWORD!
ASSEMBLEE_DB_PASSWORD="assemblee"
# PostgreSQL database configuration
DB_NAME="legi"
DB_HOST="localhost"

1559
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -33,6 +33,7 @@
"@sveltejs/kit": "^1.10.0",
"@sveltejs/package": "^2.0.2",
"@tailwindcss/typography": "^0.5.3",
"@tricoteuses/assemblee": "^1.1.8",
"@types/fs-extra": "^11.0.1",
"@types/he": "^1.1.2",
"@types/morgan": "^1.9.3",

View file

@ -46,11 +46,11 @@ export interface DossierLegislatif {
META_DOSSIER_LEGISLATIF: {
DATE_CREATION: string
DATE_DERNIERE_MODIFICATION: string
ID_TEXTE_1?: string
ID_TEXTE_2?: string
ID_TEXTE_3?: string
ID_TEXTE_4?: string
ID_TEXTE_5?: string
ID_TEXTE_1?: string // Always starts with JORFTEXT when present
ID_TEXTE_2?: string // Always starts with JORFTEXT when present
ID_TEXTE_3?: string // Always starts with JORFTEXT when present
ID_TEXTE_4?: string // Always starts with JORFTEXT when present
ID_TEXTE_5?: string // Always starts with JORFTEXT when present
LEGISLATURE: {
DATE_DEBUT: string
DATE_FIN: string

View file

@ -10,7 +10,7 @@ import {
type Textelr,
type TexteVersion,
} from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
export class Aggregator {
article: { [id: string]: Article } = {}

View file

@ -23,6 +23,15 @@ export function auditConfig(
const errors: { [key: string]: unknown } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"assembleeDb",
true,
errors,
remainingKeys,
auditDb,
auditRequire,
)
audit.attribute(
data,
"db",

View file

@ -3,6 +3,13 @@ import "dotenv/config"
import { validateConfig } from "$lib/server/auditors/config"
export interface Config {
assembleeDb: {
host: string
port: number
database: string
user: string
password: string
}
db: {
host: string
port: number
@ -14,6 +21,13 @@ export interface Config {
}
const [config, error] = validateConfig({
assembleeDb: {
host: process.env.ASSEMBLEE_DB_HOST,
port: process.env.ASSEMBLEE_DB_PORT,
database: process.env.ASSEMBLEE_DB_NAME,
user: process.env.ASSEMBLEE_DB_USER,
password: process.env.ASSEMBLEE_DB_PASSWORD,
},
db: {
host: process.env.DB_HOST,
port: process.env.DB_PORT,

View file

@ -1,7 +1,7 @@
import assert from "assert"
import type { TexteVersion } from "$lib/legal"
import { db, type Version, versionNumber } from "$lib/server/database"
import { db, type Version, versionNumber } from "$lib/server/databases"
export async function configureDatabase() {
// Table: version
@ -91,6 +91,14 @@ export async function configureDatabase() {
)
`
// Table: dossier_legislatif_assemblee_association
await db`
CREATE TABLE IF NOT EXISTS dossier_legislatif_assemblee_associations (
id char(20) PRIMARY KEY REFERENCES dossier_legislatif(id),
assemblee_uid char(13) NOT NULL
)
`
// Table: id
await db`
CREATE TABLE IF NOT EXISTS id (

View file

@ -7,6 +7,15 @@ export interface Version {
number: number
}
export const assembleeDb = postgres({
database: config.assembleeDb.database,
host: config.assembleeDb.host,
password: config.assembleeDb.password,
port: config.assembleeDb.port,
user: config.assembleeDb.user,
})
export const assembleeVersionNumber = 7
export const db = postgres({
database: config.db.database,
host: config.db.host,
@ -14,7 +23,34 @@ export const db = postgres({
port: config.db.port,
user: config.db.user,
})
export const versionNumber = 4
export const versionNumber = 5
/// Check that assemblee database exists and is up to date.
export async function checkAssembleeDb(): Promise<void> {
assert(
(
await assembleeDb`SELECT EXISTS (
SELECT * FROM information_schema.tables WHERE table_name='version'
)`
)[0]?.exists,
"Assemblee database is not initialized.",
)
const version = (await assembleeDb<Version[]>`SELECT * FROM version`)[0]
assert.notStrictEqual(
version,
undefined,
"Assemblee database has no version number.",
)
assert(
version.number <= assembleeVersionNumber,
"Assemblee database format is too recent.",
)
assert.strictEqual(
version.number,
assembleeVersionNumber,
"Assemblee database must be upgraded.",
)
}
/// Check that database exists and is up to date.
export async function checkDb(): Promise<void> {

View file

@ -1,6 +1,6 @@
import type { PendingQuery, Row } from "postgres"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
export function joinSqlClauses(
joiner: PendingQuery<Row[]>,

View file

@ -6,7 +6,7 @@ import { auditFollowQuery } from "$lib/auditors/queries"
import type { Follow } from "$lib/aggregates"
import type { Article } from "$lib/legal"
import { Aggregator } from "$lib/server/aggregates"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { RequestHandler } from "./$types"

View file

@ -5,7 +5,7 @@ import type { Follow } from "$lib/aggregates"
import { auditFollowQuery, auditQQueryParameter } from "$lib/auditors/queries"
import type { Article } from "$lib/legal"
import { Aggregator } from "$lib/server/aggregates"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { RequestHandler } from "./$types"

View file

@ -17,7 +17,7 @@ import {
} from "$lib/auditors/queries"
import type { TexteVersion } from "$lib/legal"
import { Aggregator } from "$lib/server/aggregates"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import { joinSqlClauses } from "$lib/server/sql"
import type { RequestHandler } from "./$types"

View file

@ -6,7 +6,7 @@ import { auditFollowQuery } from "$lib/auditors/queries"
import type { Follow } from "$lib/aggregates"
import type { TexteVersion } from "$lib/legal"
import { Aggregator } from "$lib/server/aggregates"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { RequestHandler } from "./$types"

View file

@ -1,4 +1,4 @@
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { RequestHandler } from "./$types"

View file

@ -3,7 +3,7 @@ import { error } from "@sveltejs/kit"
import { auditSearchQueryContent } from "$lib/auditors/queries"
import type { Article } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -1,7 +1,7 @@
import { error } from "@sveltejs/kit"
import type { Article } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -3,7 +3,7 @@ import { error } from "@sveltejs/kit"
import { auditSearchQueryContent } from "$lib/auditors/queries"
import type { DossierLegislatif } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -1,7 +1,7 @@
import { error } from "@sveltejs/kit"
import type { DossierLegislatif } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -3,7 +3,7 @@ import { error } from "@sveltejs/kit"
import { auditSearchQueryContent } from "$lib/auditors/queries"
import type { IdWrapper } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -1,7 +1,7 @@
import { error } from "@sveltejs/kit"
import type { IdWrapper } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -3,7 +3,7 @@ import { error } from "@sveltejs/kit"
import { auditSearchQueryContent } from "$lib/auditors/queries"
import type { Idcc } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -1,7 +1,7 @@
import { error } from "@sveltejs/kit"
import type { Idcc } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -3,7 +3,7 @@ import { error } from "@sveltejs/kit"
import { auditSearchQueryContent } from "$lib/auditors/queries"
import type { Jo } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -1,7 +1,7 @@
import { error } from "@sveltejs/kit"
import type { Jo } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -3,7 +3,7 @@ import { error } from "@sveltejs/kit"
import { auditSearchQueryContent } from "$lib/auditors/queries"
import type { SectionTa } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -1,7 +1,7 @@
import { error } from "@sveltejs/kit"
import type { SectionTa } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -3,7 +3,7 @@ import { error } from "@sveltejs/kit"
import { auditSearchQueryContent } from "$lib/auditors/queries"
import type { TexteVersion } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -1,7 +1,7 @@
import { error } from "@sveltejs/kit"
import type { TexteVersion } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -3,7 +3,7 @@ import { error } from "@sveltejs/kit"
import { auditSearchQueryContent } from "$lib/auditors/queries"
import type { Textekali } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -1,7 +1,7 @@
import { error } from "@sveltejs/kit"
import type { Textekali } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -3,7 +3,7 @@ import { error } from "@sveltejs/kit"
import { auditSearchQueryContent } from "$lib/auditors/queries"
import type { Textelr } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -1,7 +1,7 @@
import { error } from "@sveltejs/kit"
import type { Textelr } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -3,7 +3,7 @@ import { error } from "@sveltejs/kit"
import { auditSearchQueryContent } from "$lib/auditors/queries"
import type { VersionsWrapper } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -1,7 +1,7 @@
import { error } from "@sveltejs/kit"
import type { VersionsWrapper } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import type { PageServerLoad } from "./$types"

View file

@ -2,7 +2,7 @@ import metslesliens from "metslesliens"
import sade from "sade"
import type { JorfTexteVersion } from "$lib"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
async function associateDecretsToLois(): Promise<void> {
for (const texteVersion of (

View file

@ -0,0 +1,150 @@
import { CodeActe, type DossierParlementaire } from "@tricoteuses/assemblee"
import assert from "assert"
import sade from "sade"
import type { DossierLegislatif, JorfTextelr } from "$lib/legal"
import { assembleeDb, db } from "$lib/server/databases"
async function associateDossiersLegislatifsWithAssemblee({
verbose,
}: { verbose?: boolean } = {}): Promise<void> {
const dossierAssociationRemainingIds = new Set(
(
await db<{ id: string }[]>`
SELECT id
FROM dossier_legislatif_assemblee_associations
`
).map(({ id }) => id),
)
const dossierIdByLawId: { [nor: string]: string } = {}
const lawIdByNor: { [nor: string]: string } = {}
for (const dossier of (
await db<{ data: DossierLegislatif }[]>`
SELECT data
FROM dossier_legislatif
WHERE
data -> 'META' -> 'META_DOSSIER_LEGISLATIF' ->> 'TYPE' IN ('LOI_PUBLIEE', 'ORDONNANCE_PUBLIEE')
`
).map(({ data }) => data)) {
const lawsId = new Set<string>()
const metaDossierLegislatif = dossier.META.META_DOSSIER_LEGISLATIF
for (const idTexteName of [
"ID_TEXTE_1",
"ID_TEXTE_2",
"ID_TEXTE_3",
"ID_TEXTE_4",
"ID_TEXTE_5",
] as Array<keyof DossierLegislatif["META"]["META_DOSSIER_LEGISLATIF"]>) {
const idTexte = metaDossierLegislatif[idTexteName] as string | undefined
if (idTexte === undefined) {
continue
}
assert(idTexte.startsWith("JORFTEXT"))
const textelr = (
await db<{ data: JorfTextelr }[]>`
SELECT data
FROM textelr
WHERE id = ${idTexte}
`
).map(({ data }) => data)[0]
if (textelr === undefined) {
console.warn(
`In dossier législatif ${dossier.META.META_COMMUN.ID}, field META.META_DOSSIER_LEGISLATIF.${idTexteName} has value ${idTexte} that is not a valid JORF reference`,
)
continue
}
if (
["LOI", "LOI_CONSTIT", "LOI_ORGANIQUE", "ORDONNANCE"].includes(
textelr.META.META_COMMUN.NATURE as string,
)
) {
lawsId.add(idTexte)
const nor = textelr.META.META_SPEC.META_TEXTE_CHRONICLE.NOR
assert.notStrictEqual(nor, undefined)
lawIdByNor[nor as string] = idTexte
dossierIdByLawId[idTexte] = dossier.META.META_COMMUN.ID
}
}
if (lawsId.size === 0) {
console.warn(
`In dossier législatif ${dossier.META.META_COMMUN.ID}, fields META.META_DOSSIER_LEGISLATIF.ID_TEXTE_n don't reference any valid law`,
)
continue
}
}
const assembleeDossiersPromulgues = (
await assembleeDb<{ data: DossierParlementaire }[]>`
SELECT data
FROM dossiers
WHERE data @? '$.actesLegislatifs[*].codeActe ? (@ == "PROM")'
`
).map(({ data }) => data)
for (const assembleeDossier of assembleeDossiersPromulgues) {
const promActe = assembleeDossier.actesLegislatifs?.find(
(acte) => (acte.codeActe as unknown as string) === CodeActe.Prom,
)
assert.notStrictEqual(promActe, undefined)
const promPubActe = promActe?.actesLegislatifs?.find(
(acte) => (acte.codeActe as unknown as string) === CodeActe.PromPub,
)
assert.notStrictEqual(promPubActe, undefined)
const nor = promPubActe?.infoJo?.referenceNor ?? promPubActe?.referenceNor
if (nor === undefined) {
console.log(
`The Assemblée "dossier législatif" ${assembleeDossier.uid} doesn't contain a NOR yet`,
)
continue
}
const lawId = lawIdByNor[nor as string]
if (lawId === undefined) {
// Law has no Légifrance "dossier législatif".
// This is normal for example for "lois de ratification d'accords entre pays".
if (verbose) {
console.log(
`The Assemblée "dossier législatif" ${assembleeDossier.uid} has a NOR ${nor} that is not found in Légifrance "dossiers législatifs"`,
)
}
continue
}
assert.notStrictEqual(lawId, undefined)
const dossierId = dossierIdByLawId[lawId]
assert.notStrictEqual(dossierId, undefined)
await db`
INSERT INTO dossier_legislatif_assemblee_associations (
id,
assemblee_uid
) VALUES (
${dossierId},
${assembleeDossier.uid}
)
ON CONFLICT (id)
DO UPDATE SET
assemblee_uid = ${assembleeDossier.uid}
`
dossierAssociationRemainingIds.delete(dossierId)
}
for (const id of dossierAssociationRemainingIds) {
console.log(
`Deleting association ${id} with "dossier législatif" of Assemblée…`,
)
await db`
DELETE FROM dossier_legislatif_assemblee_associations
WHERE id = ${id}
`
}
}
sade("associate_dossiers_legislatifs_with_assemblee", true)
.describe(
'Associate Légifrance "dossiers législatifs" with those of Assemblée nationale',
)
.option("--verbose", "Log all messages")
.action(async (options) => {
await associateDossiersLegislatifsWithAssemblee(options)
process.exit(0)
})
.parse(process.argv)

View file

@ -1,4 +1,4 @@
import { configureDatabase } from "$lib/server/database/configuration"
import { configureDatabase } from "$lib/server/databases/configuration"
export async function configure(): Promise<void> {
await configureDatabase()

View file

@ -3,7 +3,7 @@ import Papa from "papaparse"
import sade from "sade"
import type { DossierLegislatif } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
async function exportEcheanciers(csvFilePath: string): Promise<void> {
const data: Array<{

View file

@ -9,7 +9,7 @@ import sade from "sade"
import { auditDossierLegislatif } from "$lib/auditors/dole"
import type { DossierLegislatif, XmlHeader } from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import { walkDir } from "$lib/server/file_systems"
const xmlParser = new XMLParser({

View file

@ -34,7 +34,7 @@ import type {
Versions,
XmlHeader,
} from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import { walkDir } from "$lib/server/file_systems"
type CategoryTag = (typeof allCategoriesCode)[number]

View file

@ -15,7 +15,7 @@ import type {
TexteVersion,
XmlHeader,
} from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import { walkDir } from "$lib/server/file_systems"
const xmlParser = new XMLParser({

View file

@ -31,7 +31,7 @@ import type {
Versions,
XmlHeader,
} from "$lib/legal"
import { db } from "$lib/server/database"
import { db } from "$lib/server/databases"
import { walkDir } from "$lib/server/file_systems"
type CategoryTag = (typeof allCategoriesCode)[number]