Add ArticleCache because prettier is the slowest part of git generation

This commit is contained in:
Emmanuel 2024-12-21 23:29:10 +01:00
parent e68b29aa1a
commit d84a130bab

View file

@ -36,8 +36,15 @@ import { walkDir } from "$lib/server/file_systems"
type Action = "CREATE" | "DELETE" type Action = "CREATE" | "DELETE"
interface ArticleCache {
id: string
markdown: string
repositoryRelativeFilePath: string
}
interface Context { interface Context {
articleById: Record<string, JorfArticle | LegiArticle> articleById: Record<string, JorfArticle | LegiArticle>
articleCacheByNumber: Record<string, ArticleCache>
consolidatedIdsByActionByModifyingTextIdByDate: Record< consolidatedIdsByActionByModifyingTextIdByDate: Record<
string, string,
Record<string, Partial<Record<Action, Set<string>>>> Record<string, Partial<Record<Action, Set<string>>>>
@ -395,6 +402,7 @@ async function exportConsolidatedTextToGit(
): Promise<void> { ): Promise<void> {
const context: Context = { const context: Context = {
articleById: {}, articleById: {},
articleCacheByNumber: {},
consolidatedIdsByActionByModifyingTextIdByDate: {}, consolidatedIdsByActionByModifyingTextIdByDate: {},
consolidatedTextCid: consolidatedTextId, // Temporary value, overrided below consolidatedTextCid: consolidatedTextId, // Temporary value, overrided below
consolidatedTextInternalIds: new Set([consolidatedTextId]), consolidatedTextInternalIds: new Set([consolidatedTextId]),
@ -868,7 +876,7 @@ async function exportConsolidatedTextToGit(
} }
const t2 = performance.now() const t2 = performance.now()
const changedFilesCount = await generateTextGitDirectory( const changedFilesCount = await generateTextGit(
context, context,
1, 1,
tree, tree,
@ -989,40 +997,27 @@ async function exportConsolidatedTextToGit(
} }
} }
async function generateSectionTaGitDirectory( async function generateLiensArticlesGit(
context: Context, context: Context,
depth: number, liensArticles: (JorfSectionTaLienArt | LegiSectionTaLienArt)[] | undefined,
sectionTaNode: SectionTaNode, repositoryRelativeDir: string,
sectionTa: LegiSectionTa,
parentRepositoryRelativeDir: string,
modifyingTextId: string,
obsoleteRepositoryRelativeFilesPaths: Set<string>, obsoleteRepositoryRelativeFilesPaths: Set<string>,
): Promise<number> { ): Promise<{
changedFilesCount: number
readmeLinks: Array<{ href: string; title: string }>
}> {
let changedFilesCount = 0 let changedFilesCount = 0
const sectionTaTitle = sectionTa.TITRE_TA ?? sectionTa.ID
let sectionTaSlug = slugify(sectionTaTitle.split(":")[0].trim(), "_")
if (sectionTaSlug.length > 255) {
sectionTaSlug = sectionTaSlug.slice(0, 254)
if (sectionTaSlug.at(-1) !== "_") {
sectionTaSlug += "_"
}
}
const sectionTaDirName = sectionTaSlug
const repositoryRelativeDir = path.join(
parentRepositoryRelativeDir,
sectionTaDirName,
)
await fs.ensureDir(path.join(context.targetDir, repositoryRelativeDir))
const readmeLinks: Array<{ href: string; title: string }> = [] const readmeLinks: Array<{ href: string; title: string }> = []
if (sectionTaNode.liensArticles !== undefined) { if (liensArticles !== undefined) {
for (const lienArticle of sectionTaNode.liensArticles) { for (const lienArticle of liensArticles) {
const articleId = lienArticle["@id"] const articleId = lienArticle["@id"]
const article = (await getOrLoadArticle( const article = (await getOrLoadArticle(
context, context,
articleId, articleId,
)) as LegiArticle )) as LegiArticle
const articleTitle = `Article ${article.META.META_SPEC.META_ARTICLE.NUM ?? articleId}` const articleNumber = article.META.META_SPEC.META_ARTICLE.NUM
const articleTitle = `Article ${articleNumber ?? articleId}`
let articleSlug = slugify(articleTitle, "_") let articleSlug = slugify(articleTitle, "_")
if (articleSlug.length > 252) { if (articleSlug.length > 252) {
articleSlug = articleSlug.slice(0, 251) articleSlug = articleSlug.slice(0, 251)
@ -1035,9 +1030,17 @@ async function generateSectionTaGitDirectory(
repositoryRelativeDir, repositoryRelativeDir,
articleFilename, articleFilename,
) )
const fileChanged = await writeTextFileIfChanged( const articleCache =
path.join(context.targetDir, articleRepositoryRelativeFilePath), articleNumber === undefined
dedent` ? undefined
: context.articleCacheByNumber[articleNumber]
if (
articleCache === undefined ||
articleId !== articleCache.id ||
articleRepositoryRelativeFilePath !==
articleCache.repositoryRelativeFilePath
) {
const articleMarkdown = dedent`
--- ---
${[ ${[
["État", article.META.META_SPEC.META_ARTICLE.ETAT], ["État", article.META.META_SPEC.META_ARTICLE.ETAT],
@ -1057,7 +1060,17 @@ async function generateSectionTaGitDirectory(
###### ${articleTitle} ###### ${articleTitle}
${await cleanHtmlFragment(article.BLOC_TEXTUEL?.CONTENU)} ${await cleanHtmlFragment(article.BLOC_TEXTUEL?.CONTENU)}
`, `
if (articleNumber !== undefined) {
context.articleCacheByNumber[articleNumber] = {
id: articleId,
markdown: articleMarkdown,
repositoryRelativeFilePath: articleRepositoryRelativeFilePath,
}
}
const fileChanged = await writeTextFileIfChanged(
path.join(context.targetDir, articleRepositoryRelativeFilePath),
articleMarkdown,
) )
if (fileChanged) { if (fileChanged) {
await git.add({ await git.add({
@ -1067,12 +1080,51 @@ async function generateSectionTaGitDirectory(
}) })
changedFilesCount++ changedFilesCount++
} }
}
obsoleteRepositoryRelativeFilesPaths.delete( obsoleteRepositoryRelativeFilesPaths.delete(
articleRepositoryRelativeFilePath, articleRepositoryRelativeFilePath,
) )
readmeLinks.push({ href: articleFilename, title: articleTitle }) readmeLinks.push({ href: articleFilename, title: articleTitle })
} }
} }
return {
changedFilesCount,
readmeLinks,
}
}
async function generateSectionTaGit(
context: Context,
depth: number,
sectionTaNode: SectionTaNode,
sectionTa: LegiSectionTa,
parentRepositoryRelativeDir: string,
modifyingTextId: string,
obsoleteRepositoryRelativeFilesPaths: Set<string>,
): Promise<number> {
const sectionTaTitle = sectionTa.TITRE_TA ?? sectionTa.ID
let sectionTaSlug = slugify(sectionTaTitle.split(":")[0].trim(), "_")
if (sectionTaSlug.length > 255) {
sectionTaSlug = sectionTaSlug.slice(0, 254)
if (sectionTaSlug.at(-1) !== "_") {
sectionTaSlug += "_"
}
}
const sectionTaDirName = sectionTaSlug
const repositoryRelativeDir = path.join(
parentRepositoryRelativeDir,
sectionTaDirName,
)
await fs.ensureDir(path.join(context.targetDir, repositoryRelativeDir))
const liensArticlesGitGeneration = await generateLiensArticlesGit(
context,
sectionTaNode.liensArticles,
repositoryRelativeDir,
obsoleteRepositoryRelativeFilesPaths,
)
let { changedFilesCount } = liensArticlesGitGeneration
const { readmeLinks } = liensArticlesGitGeneration
if (sectionTaNode.children !== undefined) { if (sectionTaNode.children !== undefined) {
for (const child of sectionTaNode.children) { for (const child of sectionTaNode.children) {
@ -1092,7 +1144,7 @@ async function generateSectionTaGitDirectory(
const sectionTaDirName = sectionTaSlug const sectionTaDirName = sectionTaSlug
readmeLinks.push({ href: sectionTaDirName, title: sectionTaTitle }) readmeLinks.push({ href: sectionTaDirName, title: sectionTaTitle })
changedFilesCount += await generateSectionTaGitDirectory( changedFilesCount += await generateSectionTaGit(
context, context,
depth + 1, depth + 1,
child, child,
@ -1144,14 +1196,13 @@ async function generateSectionTaGitDirectory(
return changedFilesCount return changedFilesCount
} }
async function generateTextGitDirectory( async function generateTextGit(
context: Context, context: Context,
depth: number, depth: number,
tree: TextelrNode, tree: TextelrNode,
texteVersion: LegiTexteVersion, texteVersion: LegiTexteVersion,
modifyingTextId: string, modifyingTextId: string,
): Promise<number> { ): Promise<number> {
let changedFilesCount = 0
const texteTitle = ( const texteTitle = (
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITREFULL ?? texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITREFULL ??
texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE ?? texteVersion.META.META_SPEC.META_TEXTE_VERSION.TITRE ??
@ -1168,62 +1219,14 @@ async function generateTextGitDirectory(
) )
obsoleteRepositoryRelativeFilesPaths.delete("LICENCE.md") obsoleteRepositoryRelativeFilesPaths.delete("LICENCE.md")
const readmeLinks: Array<{ href: string; title: string }> = [] const liensArticlesGitGeneration = await generateLiensArticlesGit(
if (tree.liensArticles !== undefined) {
for (const lienArticle of tree.liensArticles) {
const articleId = lienArticle["@id"]
const article = (await getOrLoadArticle(
context, context,
articleId, tree.liensArticles,
)) as LegiArticle "",
const articleTitle = `Article ${article.META.META_SPEC.META_ARTICLE.NUM ?? articleId}` obsoleteRepositoryRelativeFilesPaths,
let articleSlug = slugify(articleTitle, "_")
if (articleSlug.length > 252) {
articleSlug = articleSlug.slice(0, 251)
if (articleSlug.at(-1) !== "_") {
articleSlug += "_"
}
}
const articleFilename = `${articleSlug}.md`
const articleRepositoryRelativeFilePath = articleFilename
const fileChanged = await writeTextFileIfChanged(
path.join(repositoryDir, articleRepositoryRelativeFilePath),
dedent`
---
${[
["État", article.META.META_SPEC.META_ARTICLE.ETAT],
["Type", article.META.META_SPEC.META_ARTICLE.TYPE],
["Date de début", article.META.META_SPEC.META_ARTICLE.DATE_DEBUT],
["Date de fin", article.META.META_SPEC.META_ARTICLE.DATE_FIN],
["Identifiant", articleId],
["Ancien identifiant", article.META.META_COMMUN.ANCIEN_ID],
// TODO: Mettre l'URL dans le Git Tricoteuses
["URL", article.META.META_COMMUN.URL],
]
.filter(([, value]) => value !== undefined)
.map(([key, value]) => `${key}: ${value}`)
.join("\n")}
---
###### ${articleTitle}
${await cleanHtmlFragment(article.BLOC_TEXTUEL?.CONTENU)}
`,
) )
if (fileChanged) { let { changedFilesCount } = liensArticlesGitGeneration
await git.add({ const { readmeLinks } = liensArticlesGitGeneration
dir: repositoryDir,
filepath: articleRepositoryRelativeFilePath,
fs,
})
changedFilesCount++
}
obsoleteRepositoryRelativeFilesPaths.delete(
articleRepositoryRelativeFilePath,
)
readmeLinks.push({ href: articleFilename, title: articleTitle })
}
}
if (tree.children !== undefined) { if (tree.children !== undefined) {
for (const child of tree.children) { for (const child of tree.children) {
@ -1243,7 +1246,7 @@ async function generateTextGitDirectory(
const sectionTaDirName = sectionTaSlug const sectionTaDirName = sectionTaSlug
readmeLinks.push({ href: sectionTaDirName, title: sectionTaTitle }) readmeLinks.push({ href: sectionTaDirName, title: sectionTaTitle })
changedFilesCount += await generateSectionTaGitDirectory( changedFilesCount += await generateSectionTaGit(
context, context,
depth + 1, depth + 1,
child, child,