Use marked for markdown parsing

2025-11-22 10:51:36 +01:00 · 2024-08-23 18:43:52 +02:00
parent a833d0307c
commit 5e4ee45004
14 changed files with 94 additions and 214 deletions
--- a/rest-api/db/migrations/20240810115846_image_upload_function.sql.sql
+++ b/rest-api/db/migrations/20240810115846_image_upload_function.sql.sql
--- a/web-app/package-lock.json
+++ b/web-app/package-lock.json
@@ -9,14 +9,14 @@
      "version": "0.0.1",
      "dependencies": {
        "highlight.js": "11.10.0",
-        "markdown-it": "14.1.0"
+        "marked": "14.0.0",
+        "marked-highlight": "2.1.4"
      },
      "devDependencies": {
        "@sveltejs/adapter-auto": "3.2.4",
        "@sveltejs/adapter-node": "5.2.2",
        "@sveltejs/kit": "2.5.22",
        "@sveltejs/vite-plugin-svelte": "3.1.1",
-        "@types/markdown-it": "14.1.2",
        "@types/node": "22.2.0",
        "prettier": "3.3.3",
        "prettier-plugin-svelte": "3.2.6",
@@ -970,31 +970,6 @@
      "dev": true,
      "license": "MIT"
    },
-    "node_modules/@types/linkify-it": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-5.0.0.tgz",
-      "integrity": "sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==",
-      "dev": true,
-      "license": "MIT"
-    },
-    "node_modules/@types/markdown-it": {
-      "version": "14.1.2",
-      "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-14.1.2.tgz",
-      "integrity": "sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@types/linkify-it": "^5",
-        "@types/mdurl": "^2"
-      }
-    },
-    "node_modules/@types/mdurl": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-2.0.0.tgz",
-      "integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==",
-      "dev": true,
-      "license": "MIT"
-    },
    "node_modules/@types/node": {
      "version": "22.2.0",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.2.0.tgz",
@@ -1082,12 +1057,6 @@
        "node": ">= 8"
      }
    },
-    "node_modules/argparse": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
-      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
-      "license": "Python-2.0"
-    },
    "node_modules/aria-query": {
      "version": "5.3.0",
      "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz",
@@ -1327,18 +1296,6 @@
      "dev": true,
      "license": "MIT"
    },
-    "node_modules/entities": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
-      "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
-      "license": "BSD-2-Clause",
-      "engines": {
-        "node": ">=0.12"
-      },
-      "funding": {
-        "url": "https://github.com/fb55/entities?sponsor=1"
-      }
-    },
    "node_modules/es6-promise": {
      "version": "3.3.1",
      "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.3.1.tgz",
@@ -1717,15 +1674,6 @@
        "node": ">=6"
      }
    },
-    "node_modules/linkify-it": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz",
-      "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==",
-      "license": "MIT",
-      "dependencies": {
-        "uc.micro": "^2.0.0"
-      }
-    },
    "node_modules/locate-character": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz",
@@ -1750,28 +1698,26 @@
        "@jridgewell/sourcemap-codec": "^1.5.0"
      }
    },
-    "node_modules/markdown-it": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz",
-      "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==",
+    "node_modules/marked": {
+      "version": "14.0.0",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz",
+      "integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==",
      "license": "MIT",
-      "dependencies": {
-        "argparse": "^2.0.1",
-        "entities": "^4.4.0",
-        "linkify-it": "^5.0.0",
-        "mdurl": "^2.0.0",
-        "punycode.js": "^2.3.1",
-        "uc.micro": "^2.1.0"
-      },
      "bin": {
-        "markdown-it": "bin/markdown-it.mjs"
+        "marked": "bin/marked.js"
+      },
+      "engines": {
+        "node": ">= 18"
      }
    },
-    "node_modules/mdurl": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz",
-      "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==",
-      "license": "MIT"
+    "node_modules/marked-highlight": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/marked-highlight/-/marked-highlight-2.1.4.tgz",
+      "integrity": "sha512-D1GOkcdzP+1dzjoColL7umojefFrASDuLeyaHS0Zr/Uo9jkr1V6vpLRCzfi1djmEaWyK0SYMFtHnpkZ+cwFT1w==",
+      "license": "MIT",
+      "peerDependencies": {
+        "marked": ">=4 <15"
+      }
    },
    "node_modules/min-indent": {
      "version": "1.0.1",
@@ -2025,15 +1971,6 @@
        "svelte": "^3.2.0 || ^4.0.0-next.0 || ^5.0.0-next.0"
      }
    },
-    "node_modules/punycode.js": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz",
-      "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=6"
-      }
-    },
    "node_modules/readdirp": {
      "version": "3.6.0",
      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
@@ -2568,12 +2505,6 @@
        "node": ">=14.17"
      }
    },
-    "node_modules/uc.micro": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz",
-      "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
-      "license": "MIT"
-    },
    "node_modules/undici-types": {
      "version": "6.13.0",
      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.13.0.tgz",
--- a/web-app/package.json
+++ b/web-app/package.json
@@ -16,7 +16,6 @@
    "@sveltejs/adapter-node": "5.2.2",
    "@sveltejs/kit": "2.5.22",
    "@sveltejs/vite-plugin-svelte": "3.1.1",
-    "@types/markdown-it": "14.1.2",
    "@types/node": "22.2.0",
    "prettier": "3.3.3",
    "prettier-plugin-svelte": "3.2.6",
@@ -28,6 +27,7 @@
  "type": "module",
  "dependencies": {
    "highlight.js": "11.10.0",
-    "markdown-it": "14.1.0"
+    "marked": "14.0.0",
+    "marked-highlight": "2.1.4"
  }
 }
--- a/web-app/src/app.css
+++ b/web-app/src/app.css
@@ -1,7 +1,3 @@
-section + section {
-  margin-block-start: var(--space-l);
-}
-
 button,
 label,
 select,
@@ -72,11 +68,6 @@ summary {
  outline-offset: 0.25rem;
 }

-ul,
-ol {
-  list-style: none;
-}
-
 form {
  display: flex;
  flex-direction: column;
--- a/web-app/src/lib/components/WebsiteEditor.svelte
+++ b/web-app/src/lib/components/WebsiteEditor.svelte
@@ -33,7 +33,7 @@
  <h1>{title}</h1>

  <nav class="operations__nav">
-    <ul>
+    <ul class="unpadded">
      <li>
        <a href="/website/{id}">Settings</a>
      </li>
@@ -56,7 +56,11 @@
  {#if fullPreview}
    <iframe src={previewContent} title="Preview"></iframe>
  {:else}
-    {@html md.render(previewContent)}
+    {#await md(previewContent)}
+      <p>Loading preview...</p>
+    {:then content}
+      {@html content}
+    {/await}
  {/if}
 </div>

--- a/web-app/src/lib/templates/blog/BlogIndex.svelte
+++ b/web-app/src/lib/templates/blog/BlogIndex.svelte
@@ -37,7 +37,7 @@
      <section class="articles">
        <h2>Articles</h2>

-        <ul>
+        <ul class="unpadded">
          {#each articles as article}
            {@const articleFileName = article.title.toLowerCase().split(" ").join("-")}
            <li>
--- a/web-app/src/lib/utils.ts
+++ b/web-app/src/lib/utils.ts
@@ -1,6 +1,6 @@
-import markdownit from "markdown-it";
+import { Marked } from "marked";
 import hljs from "highlight.js";
-import type { StateCore } from "markdown-it/index.js";
+import { markedHighlight } from "marked-highlight";

 export const sortOptions = [
  { value: "creation-time", text: "Creation time" },
@@ -11,106 +11,33 @@ export const sortOptions = [

 export const ALLOWED_MIME_TYPES = ["image/jpeg", "image/png", "image/svg+xml", "image/webp"];

-export const md = markdownit({
-  linkify: true,
-  typographer: true,
-  highlight: (str: string, lang: string) => {
-    if (lang && hljs.getLanguage(lang)) {
-      try {
-        return hljs.highlight(str, { language: lang }).value;
-      } catch (_) {}
-    }
-    return "";
-  }
-}).use((md) => {
-  const addSections = (state: StateCore) => {
-    const tokens = [];
-    const Token = state.Token;
-    const sections: { header: number; nesting: number }[] = [];
-    let nestedLevel = 0;
-
-    const slugify = (text: string) => {
-      return text
-        .toLowerCase()
-        .replace(/\s+/g, "-")
-        .replace(/[^\w-]+/g, "")
-        .replace(/--+/g, "-")
-        .replace(/^-+/, "")
-        .replace(/-+$/, "");
-    };
-
-    const openSection = (attrs: [string, string][] | null, headingText: string) => {
-      const t = new Token("section_open", "section", 1);
-      t.block = true;
-      t.attrs = attrs ? attrs.map((attr) => [attr[0], attr[1]]) : [];
-      t.attrs.push(["id", slugify(headingText)]);
-      return t;
-    };
-
-    const closeSection = () => {
-      const t = new Token("section_close", "section", -1);
-      t.block = true;
-      return t;
-    };
-
-    const closeSections = (section: { header: number; nesting: number }) => {
-      while (sections.length && section.header <= sections[sections.length - 1].header) {
-        sections.pop();
-        tokens.push(closeSection());
+const createMarkdownParser = () => {
+  const marked = new Marked(
+    markedHighlight({
+      langPrefix: "hljs language-",
+      highlight(code, lang) {
+        const language = hljs.getLanguage(lang) ? lang : "plaintext";
+        return hljs.highlight(code, { language }).value;
      }
-    };
+    })
+  );

-    const closeSectionsToCurrentNesting = (nesting: number) => {
-      while (sections.length && nesting < sections[sections.length - 1].nesting) {
-        sections.pop();
-        tokens.push(closeSection());
-      }
-    };
+  marked.use({
+    async: true,
+    pedantic: false,
+    gfm: true
+  });

-    const closeAllSections = () => {
-      while (sections.pop()) {
-        tokens.push(closeSection());
-      }
-    };
+  return marked;
+};

-    for (let i = 0; i < state.tokens.length; i++) {
-      const token = state.tokens[i];
-      if (token.type.search("heading") !== 0) {
-        nestedLevel += token.nesting;
-      }
-      if (sections.length && nestedLevel < sections[sections.length - 1].nesting) {
-        closeSectionsToCurrentNesting(nestedLevel);
-      }
+const marked = createMarkdownParser();

-      if (token.type === "heading_open") {
-        const section: { header: number; nesting: number } = {
-          header: parseInt(token.tag.charAt(1)),
-          nesting: nestedLevel
-        };
-        if (sections.length && section.header <= sections[sections.length - 1].header) {
-          closeSections(section);
-        }
+export const md = async (markdownContent: string) => {
+  const html = await marked.parse(markdownContent);

-        const headingTextToken = state.tokens[i + 1];
-        const headingText = headingTextToken.content;
-
-        tokens.push(openSection(token.attrs, headingText));
-        const idIndex = token.attrIndex("id");
-        if (idIndex !== -1) {
-          token.attrs?.splice(idIndex, 1);
-        }
-        sections.push(section);
-      }
-
-      tokens.push(token);
-    }
-    closeAllSections();
-
-    state.tokens = tokens;
-  };
-
-  md.core.ruler.push("header_sections", addSections);
-});
+  return html;
+};

 export const handleImagePaste = async (event: ClipboardEvent, API_BASE_PREFIX: string) => {
  const clipboardItems = Array.from(event.clipboardData?.items || []);
--- a/web-app/src/routes/(authenticated)/+page.svelte
+++ b/web-app/src/routes/(authenticated)/+page.svelte
@@ -82,7 +82,7 @@
      </form>
    </details>

-    <ul class="website-grid">
+    <ul class="website-grid unpadded">
      {#each data.websites as { id, content_type, title, created_at } (id)}
        <li class="website-card">
          <p>
--- a/web-app/src/routes/(authenticated)/website/[websiteId]/articles/+page.svelte
+++ b/web-app/src/routes/(authenticated)/website/[websiteId]/articles/+page.svelte
@@ -80,7 +80,7 @@
        </form>
      </details>

-      <ul>
+      <ul class="unpadded">
        {#each data.articles as { id, title } (id)}
          <li class="article-card">
            <p>
--- a/web-app/src/routes/(authenticated)/website/[websiteId]/collaborators/+page.svelte
+++ b/web-app/src/routes/(authenticated)/website/[websiteId]/collaborators/+page.svelte
@@ -54,7 +54,7 @@
    <section>
      <h2>All collaborators</h2>

-      <ul>
+      <ul class="unpadded">
        {#each data.collaborators as { website_id, user_id, permission_level, user: { username } } (`${website_id}-${user_id}`)}
          <li class="collaborator-card">
            <p>
--- a/web-app/src/routes/(authenticated)/website/[websiteId]/publish/+page.server.ts
+++ b/web-app/src/routes/(authenticated)/website/[websiteId]/publish/+page.server.ts
@@ -60,7 +60,7 @@ const generateStaticFiles = async (websiteData: any, isPreview: boolean = true)
            title: websiteData.title,
            logoType: websiteData.logo_type,
            logo: websiteData.logo_text,
-            mainContent: md.render(websiteData.main_content ?? ""),
+            mainContent: await md(websiteData.main_content ?? ""),
            articles: websiteData.articles ?? [],
            footerAdditionalText: websiteData.additional_text ?? ""
          }
@@ -74,7 +74,7 @@ const generateStaticFiles = async (websiteData: any, isPreview: boolean = true)
            title: websiteData.title,
            logoType: websiteData.logo_type,
            logo: websiteData.logo_text,
-            mainContent: md.render(websiteData.main_content ?? ""),
+            mainContent: await md(websiteData.main_content ?? ""),
            articles: websiteData.articles ?? [],
            footerAdditionalText: websiteData.additional_text ?? ""
          }
@@ -117,7 +117,7 @@ const generateStaticFiles = async (websiteData: any, isPreview: boolean = true)
                ? `${API_BASE_PREFIX}/rpc/retrieve_file?id=${article.cover_image}`
                : "",
              publicationDate: article.publication_date,
-              mainContent: md.render(article.main_content ?? ""),
+              mainContent: await md(article.main_content ?? ""),
              footerAdditionalText: websiteData.additional_text ?? ""
            }
          }));
@@ -134,7 +134,7 @@ const generateStaticFiles = async (websiteData: any, isPreview: boolean = true)
                ? `${API_BASE_PREFIX}/rpc/retrieve_file?id=${article.cover_image}`
                : "",
              publicationDate: article.publication_date,
-              mainContent: md.render(article.main_content ?? ""),
+              mainContent: await md(article.main_content ?? ""),
              footerAdditionalText: websiteData.additional_text ?? ""
            }
          }));
--- a/web-app/src/routes/+layout.svelte
+++ b/web-app/src/routes/+layout.svelte
@@ -21,7 +21,7 @@

 <nav>
  <img src="/favicon.svg" width="24" height="24" alt="" />
-  <ul class="link-wrapper">
+  <ul class="link-wrapper unpadded">
    {#if data.user}
      <li>
        <a href="/">Dashboard</a>
--- a/web-app/template-styles/blog-styles.css
+++ b/web-app/template-styles/blog-styles.css
@@ -1,8 +1,3 @@
-ul,
-ol {
-  list-style: inside;
-}
-
 .container {
  margin-inline: auto;
  inline-size: min(100% - var(--space-m), 75ch);
@@ -21,10 +16,6 @@ footer {
  padding-block: var(--space-s);
 }

-section:has(> h2) + section:has(> h2) {
-  margin-block-start: var(--space-l);
-}
-
 .articles ul {
  display: grid;
  list-style: none;
--- a/web-app/template-styles/common-styles.css
+++ b/web-app/template-styles/common-styles.css
@@ -108,6 +108,10 @@ section {
  gap: var(--space-s);
 }

+section:has(> h2) + section:has(> h2) {
+  margin-block-start: var(--space-l);
+}
+
 a {
  color: var(--color-accent);
 }
@@ -173,3 +177,35 @@ code {
  border: var(--border-primary);
  padding-inline: var(--space-3xs);
 }
+
+:is(ul, ol):not(.unpadded) {
+  padding-inline-start: var(--space-s);
+}
+
+.unpadded {
+  list-style: none;
+}
+
+hr {
+  block-size: 0.125rem;
+  background-color: var(--bg-tertiary);
+  border: none;
+}
+
+.scroll-container {
+  overflow-x: auto;
+}
+
+table {
+  border-collapse: collapse;
+  inline-size: 100%;
+  border: var(--border-primary);
+}
+
+th,
+td {
+  text-align: start;
+  padding-inline: var(--space-2xs);
+  padding-block: var(--space-3xs);
+  border: var(--border-primary);
+}