From 79d1c9f5c7e216bea386dbe16a68b9bba9299a51 Mon Sep 17 00:00:00 2001
From: thiloho <123883702+thiloho@users.noreply.github.com>
Date: Fri, 13 Sep 2024 19:30:56 +0200
Subject: [PATCH] Use a more robust slugify function

---
 web-app/package-lock.json                     |  7 ---
 web-app/package.json                          |  1 -
 .../src/lib/templates/blog/BlogIndex.svelte   |  5 +-
 web-app/src/lib/templates/common/Nav.svelte   |  5 +-
 web-app/src/lib/utils.ts                      | 54 +++++++------------
 .../[websiteId]/publish/+page.server.ts       |  6 +--
 6 files changed, 25 insertions(+), 53 deletions(-)

diff --git a/web-app/package-lock.json b/web-app/package-lock.json
index 810a9fe..97b6ede 100644
--- a/web-app/package-lock.json
+++ b/web-app/package-lock.json
@@ -9,7 +9,6 @@
       "version": "0.0.1",
       "dependencies": {
         "fast-diff": "1.3.0",
-        "github-slugger": "2.0.0",
         "highlight.js": "11.10.0",
         "isomorphic-dompurify": "2.14.0",
         "marked": "14.0.0",
@@ -2734,12 +2733,6 @@
         "node": "6.* || 8.* || >= 10.*"
       }
     },
-    "node_modules/github-slugger": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-2.0.0.tgz",
-      "integrity": "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==",
-      "license": "ISC"
-    },
     "node_modules/glob": {
       "version": "10.4.5",
       "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz",
diff --git a/web-app/package.json b/web-app/package.json
index 2a62160..44c0c95 100644
--- a/web-app/package.json
+++ b/web-app/package.json
@@ -39,7 +39,6 @@
   "type": "module",
   "dependencies": {
     "fast-diff": "1.3.0",
-    "github-slugger": "2.0.0",
     "highlight.js": "11.10.0",
     "isomorphic-dompurify": "2.14.0",
     "marked": "14.0.0",
diff --git a/web-app/src/lib/templates/blog/BlogIndex.svelte b/web-app/src/lib/templates/blog/BlogIndex.svelte
index a0033b2..604cb8e 100644
--- a/web-app/src/lib/templates/blog/BlogIndex.svelte
+++ b/web-app/src/lib/templates/blog/BlogIndex.svelte
@@ -2,7 +2,7 @@
   import Head from "../common/Head.svelte";
   import Nav from "../common/Nav.svelte";
   import Footer from "../common/Footer.svelte";
-  import { md, type WebsiteOverview } from "../../utils";
+  import { md, slugify, type WebsiteOverview } from "$lib/utils";
 
   const {
     websiteOverview,
@@ -42,14 +42,13 @@
 
         <ul class="unpadded">
           {#each websiteOverview.article as article}
-            {@const articleFileName = article.title.toLowerCase().split(" ").join("-")}
             <li>
               {#if article.publication_date}
                 <p>{article.publication_date}</p>
               {/if}
               <p>
                 <strong>
-                  <a href="./articles/{articleFileName}">{article.title}</a>
+                  <a href="./articles/{slugify(article.title)}">{article.title}</a>
                 </strong>
               </p>
               {#if article.meta_description}
diff --git a/web-app/src/lib/templates/common/Nav.svelte b/web-app/src/lib/templates/common/Nav.svelte
index 8c5c2e1..5d66a8e 100644
--- a/web-app/src/lib/templates/common/Nav.svelte
+++ b/web-app/src/lib/templates/common/Nav.svelte
@@ -1,5 +1,5 @@
 <script lang="ts">
-  import type { WebsiteOverview } from "../../utils";
+  import { type WebsiteOverview, slugify } from "../../utils";
   import type { Article } from "../../db-schema";
 
   const {
@@ -60,9 +60,8 @@
               <strong>{key}</strong>
               <ul>
                 {#each categorizedArticles[key] as { title }}
-                  {@const articleFileName = title.toLowerCase().split(" ").join("-")}
                   <li>
-                    <a href="{isIndexPage ? './articles' : '.'}/{articleFileName}">{title}</a>
+                    <a href="{isIndexPage ? './articles' : '.'}/{slugify(title)}">{title}</a>
                   </li>
                 {/each}
               </ul>
diff --git a/web-app/src/lib/utils.ts b/web-app/src/lib/utils.ts
index 68b9314..45dc7fa 100644
--- a/web-app/src/lib/utils.ts
+++ b/web-app/src/lib/utils.ts
@@ -2,7 +2,6 @@ import { Marked } from "marked";
 import type { Renderer, Token } from "marked";
 import { markedHighlight } from "marked-highlight";
 import hljs from "highlight.js";
-import GithubSlugger from "github-slugger";
 import DOMPurify from "isomorphic-dompurify";
 import { applyAction, deserialize } from "$app/forms";
 import type {
@@ -18,6 +17,19 @@ import type {
 
 export const ALLOWED_MIME_TYPES = ["image/jpeg", "image/png", "image/svg+xml", "image/webp"];
 
+export const slugify = (string: string) => {
+  return string
+    .toString()
+    .normalize("NFKD") // Normalize Unicode characters
+    .toLowerCase() // Convert to lowercase
+    .trim() // Trim leading and trailing whitespace
+    .replace(/\s+/g, "-") // Replace spaces with hyphens
+    .replace(/[^\w\-]+/g, "") // Remove non-word characters (except hyphens)
+    .replace(/\-\-+/g, "-") // Replace multiple hyphens with single hyphen
+    .replace(/^-+/, "") // Remove leading hyphens
+    .replace(/-+$/, ""); // Remove trailing hyphens
+};
+
 const createMarkdownParser = (showToc = true) => {
   const marked = new Marked();
 
@@ -38,36 +50,17 @@ const createMarkdownParser = (showToc = true) => {
     })
   );
 
-  const unescapeTest = /&(#(?:\d+)|(?:#x[0-9A-Fa-f]+)|(?:\w+));?/gi;
-
-  const unescape = (html: string) => {
-    return html.replace(unescapeTest, (_, n) => {
-      n = n.toLowerCase();
-      if (n === "colon") return ":";
-      if (n.charAt(0) === "#") {
-        return n.charAt(1) === "x"
-          ? String.fromCharCode(parseInt(n.substring(2), 16))
-          : String.fromCharCode(+n.substring(1));
-      }
-      return "";
-    });
-  };
-
-  let slugger = new GithubSlugger();
-  let headings: { text: string; raw: string; level: number; id: string }[] = [];
-  let sectionStack: { level: number; id: string }[] = [];
-
   const gfmHeadingId = ({ prefix = "", showToc = true } = {}) => {
+    let headings: { text: string; level: number; id: string }[] = [];
+    let sectionStack: { level: number; id: string }[] = [];
+
     return {
       renderer: {
         heading(this: Renderer, { tokens, depth }: { tokens: Token[]; depth: number }) {
           const text = this.parser.parseInline(tokens);
-          const raw = unescape(this.parser.parseInline(tokens, this.parser.textRenderer))
-            .trim()
-            .replace(/<[!a-z].*?>/gi, "");
           const level = depth;
-          const id = `${prefix}${slugger.slug(raw.toLowerCase())}`;
-          const heading = { level, text, id, raw };
+          const id = `${prefix}${slugify(text)}`;
+          const heading = { level, text, id };
           headings.push(heading);
 
           let closingSections = "";
@@ -89,16 +82,7 @@ const createMarkdownParser = (showToc = true) => {
         }
       },
       hooks: {
-        preprocess(src: string) {
-          headings = [];
-          sectionStack = [];
-          slugger = new GithubSlugger();
-
-          return src;
-        },
         postprocess(html: string) {
-          const closingRemainingSection = "</section>".repeat(sectionStack.length);
-
           let tableOfContents = "";
           if (showToc && headings.length > 0) {
             const tocItems = [];
@@ -140,7 +124,7 @@ const createMarkdownParser = (showToc = true) => {
           return `
             ${tableOfContents}
             ${html}
-            ${closingRemainingSection}
+            ${"</section>".repeat(sectionStack.length)}
           `;
         }
       }
diff --git a/web-app/src/routes/(authenticated)/website/[websiteId]/publish/+page.server.ts b/web-app/src/routes/(authenticated)/website/[websiteId]/publish/+page.server.ts
index 998d4cf..674e6e7 100644
--- a/web-app/src/routes/(authenticated)/website/[websiteId]/publish/+page.server.ts
+++ b/web-app/src/routes/(authenticated)/website/[websiteId]/publish/+page.server.ts
@@ -1,6 +1,6 @@
 import { readFile, mkdir, writeFile } from "node:fs/promises";
 import { join } from "node:path";
-import { type WebsiteOverview } from "$lib/utils";
+import { type WebsiteOverview, slugify } from "$lib/utils";
 import type { Actions, PageServerLoad } from "./$types";
 import { API_BASE_PREFIX } from "$lib/server/utils";
 import { render } from "svelte/server";
@@ -122,8 +122,6 @@ const generateStaticFiles = async (websiteData: WebsiteOverview, isPreview: bool
   });
 
   for (const article of websiteData.article ?? []) {
-    const articleFileName = article.title.toLowerCase().split(" ").join("-");
-
     const { head, body } = render(websiteData.content_type === "Blog" ? BlogArticle : DocsArticle, {
       props: {
         websiteOverview: websiteData,
@@ -133,7 +131,7 @@ const generateStaticFiles = async (websiteData: WebsiteOverview, isPreview: bool
     });
 
     await writeFile(
-      join(uploadDir, "articles", `${articleFileName}.html`),
+      join(uploadDir, "articles", `${slugify(article.title)}.html`),
       fileContents(head, body)
     );
   }