Use jsdom instead of fast-xml-parser to parse opendata HTML page

This commit is contained in:
Emmanuel 2023-09-27 12:33:03 +02:00
parent 3bbf078134
commit f93db9d697
3 changed files with 43 additions and 78 deletions

87
package-lock.json generated
View file

@ -12,7 +12,8 @@
"@auditors/core": "^0.7.0",
"@tricoteuses/explorer-tools": "^0.4.3",
"fast-xml-parser": "^4.2.2",
"he": "^1.2.0"
"he": "^1.2.0",
"jsdom": "^22.1.0"
},
"devDependencies": {
"@iconify-icons/codicon": "^1.2.15",
@ -23,6 +24,7 @@
"@tricoteuses/assemblee": "^1.1.8",
"@types/fs-extra": "^11.0.1",
"@types/he": "^1.1.2",
"@types/jsdom": "^21.1.3",
"@types/morgan": "^1.9.4",
"@types/papaparse": "^5.3.7",
"@typescript-eslint/eslint-plugin": "^6.1.0",
@ -942,7 +944,6 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
"integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==",
"dev": true,
"engines": {
"node": ">= 10"
}
@ -1059,6 +1060,17 @@
"integrity": "sha512-CdNmJMcSqX1BiP3iSsWt+VgixndRIDGzWyaGpBnW3i5heATSk5bJu2j3buutsoBQNjyryqxaNpr8M7fRsGL15w==",
"dev": true
},
"node_modules/@types/jsdom": {
"version": "21.1.3",
"resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.3.tgz",
"integrity": "sha512-1zzqSP+iHJYV4lB3lZhNBa012pubABkj9yG/GuXuf6LZH1cSPIJBqFDrm5JX65HHt6VOnNYdTui/0ySerRbMgA==",
"dev": true,
"dependencies": {
"@types/node": "*",
"@types/tough-cookie": "*",
"parse5": "^7.0.0"
}
},
"node_modules/@types/json-schema": {
"version": "7.0.13",
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.13.tgz",
@ -1128,6 +1140,12 @@
"integrity": "sha512-OxepLK9EuNEIPxWNME+C6WwbRAOOI2o2BaQEGzz5Lu2e4Z5eDnEo+/aVEDMIXywoJitJ7xWd641wrGLZdtwRyw==",
"dev": true
},
"node_modules/@types/tough-cookie": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.3.tgz",
"integrity": "sha512-THo502dA5PzG/sfQH+42Lw3fvmYkceefOspdCwpHRul8ik2Jv1K8I5OZz1AT3/rs46kwgMCe9bSBmDLYkkOMGg==",
"dev": true
},
"node_modules/@types/which": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@types/which/-/which-3.0.0.tgz",
@ -1326,8 +1344,7 @@
"node_modules/abab": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
"integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==",
"dev": true
"integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA=="
},
"node_modules/acorn": {
"version": "8.10.0",
@ -1353,7 +1370,6 @@
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"dev": true,
"dependencies": {
"debug": "4"
},
@ -1475,8 +1491,7 @@
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
"dev": true
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
},
"node_modules/augmented-data-viewer": {
"version": "0.3.1",
@ -1819,7 +1834,6 @@
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"dev": true,
"dependencies": {
"delayed-stream": "~1.0.0"
},
@ -1964,7 +1978,6 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-3.0.0.tgz",
"integrity": "sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==",
"dev": true,
"dependencies": {
"rrweb-cssom": "^0.6.0"
},
@ -2005,7 +2018,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz",
"integrity": "sha512-/mMTei/JXPqvFqQtfyTowxmJVwr2PVAeCcDxyFf6LhoOu/09TX2OX3kb2wzi4DMXcfj4OItwDOnhl5oziPnT6g==",
"dev": true,
"dependencies": {
"abab": "^2.0.6",
"whatwg-mimetype": "^3.0.0",
@ -2050,8 +2062,7 @@
"node_modules/decimal.js": {
"version": "10.4.3",
"resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz",
"integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==",
"dev": true
"integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA=="
},
"node_modules/dedent-js": {
"version": "1.0.1",
@ -2143,7 +2154,6 @@
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
"dev": true,
"engines": {
"node": ">=0.4.0"
}
@ -2247,7 +2257,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz",
"integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==",
"dev": true,
"dependencies": {
"webidl-conversions": "^7.0.0"
},
@ -2330,7 +2339,6 @@
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
"dev": true,
"engines": {
"node": ">=0.12"
},
@ -2914,7 +2922,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
"integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
"dev": true,
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
@ -3300,7 +3307,6 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",
"integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==",
"dev": true,
"dependencies": {
"whatwg-encoding": "^2.0.0"
},
@ -3312,7 +3318,6 @@
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
"integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",
"dev": true,
"dependencies": {
"@tootallnate/once": "2",
"agent-base": "6",
@ -3326,7 +3331,6 @@
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
"integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
"dev": true,
"dependencies": {
"agent-base": "6",
"debug": "4"
@ -3350,7 +3354,6 @@
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
"dev": true,
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
},
@ -3627,8 +3630,7 @@
"node_modules/is-potential-custom-element-name": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
"integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==",
"dev": true
"integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ=="
},
"node_modules/is-reference": {
"version": "1.2.1",
@ -3798,7 +3800,6 @@
"version": "22.1.0",
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-22.1.0.tgz",
"integrity": "sha512-/9AVW7xNbsBv6GfWho4TTNjEo9fe6Zhf9O7s0Fhhr3u+awPwAJMKwAMXnkk5vBxflqLW9hTHX/0cs+P3gW+cQw==",
"dev": true,
"dependencies": {
"abab": "^2.0.6",
"cssstyle": "^3.0.0",
@ -4057,7 +4058,6 @@
"version": "1.52.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"dev": true,
"engines": {
"node": ">= 0.6"
}
@ -4066,7 +4066,6 @@
"version": "2.1.35",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"dev": true,
"dependencies": {
"mime-db": "1.52.0"
},
@ -4321,8 +4320,7 @@
"node_modules/nwsapi": {
"version": "2.2.7",
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.7.tgz",
"integrity": "sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==",
"dev": true
"integrity": "sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ=="
},
"node_modules/object-assign": {
"version": "4.1.1",
@ -4493,7 +4491,6 @@
"version": "7.1.2",
"resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz",
"integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==",
"dev": true,
"dependencies": {
"entities": "^4.4.0"
},
@ -5020,14 +5017,12 @@
"node_modules/psl": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
"integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==",
"dev": true
"integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag=="
},
"node_modules/punycode": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
"integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
"dev": true,
"engines": {
"node": ">=6"
}
@ -5035,8 +5030,7 @@
"node_modules/querystringify": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
"integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
"dev": true
"integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
},
"node_modules/queue-microtask": {
"version": "1.2.3",
@ -5105,8 +5099,7 @@
"node_modules/requires-port": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
"integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==",
"dev": true
"integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
},
"node_modules/resolve": {
"version": "1.22.6",
@ -5206,8 +5199,7 @@
"node_modules/rrweb-cssom": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz",
"integrity": "sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==",
"dev": true
"integrity": "sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw=="
},
"node_modules/run-parallel": {
"version": "1.2.0",
@ -5252,8 +5244,7 @@
"node_modules/safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
"dev": true
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"node_modules/sander": {
"version": "0.5.1",
@ -5309,7 +5300,6 @@
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz",
"integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==",
"dev": true,
"dependencies": {
"xmlchars": "^2.2.0"
},
@ -5891,8 +5881,7 @@
"node_modules/symbol-tree": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
"integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==",
"dev": true
"integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="
},
"node_modules/tailwindcss": {
"version": "3.3.3",
@ -6024,7 +6013,6 @@
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
"integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
"dev": true,
"dependencies": {
"psl": "^1.1.33",
"punycode": "^2.1.1",
@ -6039,7 +6027,6 @@
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
"integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==",
"dev": true,
"engines": {
"node": ">= 4.0.0"
}
@ -6048,7 +6035,6 @@
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz",
"integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==",
"dev": true,
"dependencies": {
"punycode": "^2.3.0"
},
@ -6207,7 +6193,6 @@
"version": "1.5.10",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
"integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
"dev": true,
"dependencies": {
"querystringify": "^2.1.1",
"requires-port": "^1.0.0"
@ -6291,7 +6276,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz",
"integrity": "sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==",
"dev": true,
"dependencies": {
"xml-name-validator": "^4.0.0"
},
@ -6312,7 +6296,6 @@
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
"integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
"dev": true,
"engines": {
"node": ">=12"
}
@ -6330,7 +6313,6 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz",
"integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==",
"dev": true,
"dependencies": {
"iconv-lite": "0.6.3"
},
@ -6342,7 +6324,6 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz",
"integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==",
"dev": true,
"engines": {
"node": ">=12"
}
@ -6351,7 +6332,6 @@
"version": "12.0.1",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-12.0.1.tgz",
"integrity": "sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==",
"dev": true,
"dependencies": {
"tr46": "^4.1.1",
"webidl-conversions": "^7.0.0"
@ -6529,7 +6509,6 @@
"version": "8.14.2",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
"integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
"dev": true,
"engines": {
"node": ">=10.0.0"
},
@ -6550,7 +6529,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz",
"integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==",
"dev": true,
"engines": {
"node": ">=12"
}
@ -6580,8 +6558,7 @@
"node_modules/xmlchars": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
"integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==",
"dev": true
"integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="
},
"node_modules/yallist": {
"version": "4.0.0",

View file

@ -45,7 +45,8 @@
"@auditors/core": "^0.7.0",
"@tricoteuses/explorer-tools": "^0.4.3",
"fast-xml-parser": "^4.2.2",
"he": "^1.2.0"
"he": "^1.2.0",
"jsdom": "^22.1.0"
},
"devDependencies": {
"@iconify-icons/codicon": "^1.2.15",
@ -56,6 +57,7 @@
"@tricoteuses/assemblee": "^1.1.8",
"@types/fs-extra": "^11.0.1",
"@types/he": "^1.1.2",
"@types/jsdom": "^21.1.3",
"@types/morgan": "^1.9.4",
"@types/papaparse": "^5.3.7",
"@typescript-eslint/eslint-plugin": "^6.1.0",

View file

@ -1,6 +1,6 @@
import assert from "assert"
import { XMLParser } from "fast-xml-parser"
import fs from "fs-extra"
import { JSDOM } from "jsdom"
import path from "path"
import sade from "sade"
import { $, cd } from "zx"
@ -21,32 +21,18 @@ async function downloadDataset(
const response = await fetch(archivesUrl)
assert(response.ok)
const html = await response.text()
const parsingOptions = {
attributeNamePrefix: "@",
htmlEntities: true,
ignoreAttributes: false,
// preserveOrder: true,
processEntities: true,
// In Dila server <pre> encapsulates the files list.
// => It must not be a stop node.
// stopNodes: ["*.pre", "*.script"],
stopNodes: ["*.script"],
// In Dila server <img> is not closed.
// unpairedTags: ["br", "hr", "link", "meta"],
unpairedTags: ["br", "hr", "img", "link", "meta"],
}
const parser = new XMLParser(parsingOptions)
const dom = parser.parse(html)
const { document } = new JSDOM(html).window
const archivesA = document.querySelectorAll("body pre a, body table tr td a")
assert.notStrictEqual(archivesA.length, 0)
let archivesA = dom.html.body.pre.a
if (!Array.isArray(archivesA)) {
archivesA = [archivesA]
}
const archiveNameByDate: { [date: string]: string } = {}
let newDatasetVersionsCount = 0
let latestFullArchiveDate: string | undefined = undefined
for (const { "@href": filename } of archivesA) {
for (const a of archivesA) {
const filename = a.getAttribute("href")
if (
filename === null ||
filename.startsWith("?") ||
filename.endsWith(".pdf") ||
filename === "/OPENDATA/"