render readme html scrapped via the server (#781)

svelte/src/components/markdown/markdown.svelte

@@ -6,13 +6,12 @@
   import { onMount } from "svelte";
   import { tokenizeMarkdown } from "./md";
   import Preloader from "$components/preloader/preloader.svelte";
+  import { shellOpenExternal } from "@native";
 
-  export let source: { data: string; type: "md" | "rst" };
-
+  export let source: { data: string; type: "md" | "rst" | "html" };
   let markDownRoot: HTMLElement;
 
   export let hook = (node: HTMLElement): { destroy: () => void } => {
-    console.log("hook", node);
     return {
       destroy() {
         console.log("destroy");
@@ -24,7 +23,15 @@
     link: Link
   };
 
-  $: html = source.type === "rst" ? rst2html(source.data) : "";
+  $: html = source.type === "rst" ? rst2html(source.data) : source.data;
+
+  // TODO: support gitlab too
+  const changeSrcIfNeeded = (element: HTMLElement | Element) => {
+    const src = element.getAttribute("src");
+    if (src?.startsWith("/")) {
+      element.setAttribute("src", `https://raw.githubusercontent.com${src}`);
+    }
+  };
 
   onMount(() => {
     // Need to override the height/width STYLE with the old-school height/width ATTRIBUTE to make it work with the markdown
@@ -40,6 +47,24 @@
         if (width) {
           element.style.width = width;
         }
+        changeSrcIfNeeded(element);
+      });
+    }
+
+    if (source.type === "html" && html) {
+      // TODO: fix this hack, seems to be redundant from the hooks above
+      document.querySelectorAll(".html-content a").forEach((element: Element) => {
+        const href = element.getAttribute("href");
+        if (!href?.startsWith("#") && href) {
+          element.addEventListener("click", (e) => {
+            e.preventDefault();
+            shellOpenExternal(href!);
+          });
+        }
+      });
+
+      document.querySelectorAll(".html-content img").forEach((element: Element) => {
+        changeSrcIfNeeded(element);
       });
     }
   });
@@ -50,8 +75,10 @@
     <div class="w-full" bind:this={markDownRoot}>
       <SvelteMarkdown source={tokenizeMarkdown(source.data)} {renderers} />
     </div>
-  {:else if source.type === "rst" && html}
-    {@html html}
+  {:else if ["html", "rst"].includes(source.type) && html}
+    <div class="html-content w-full">
+      {@html html}
+    </div>
   {:else}
     <Preloader />
   {/if}

svelte/src/components/package-metas/package-metas.svelte

@@ -4,6 +4,7 @@
   import dayjs from "dayjs";
   import relativeTime from "dayjs/plugin/relativeTime";
   import { shellOpenExternal } from "@native";
+  import { getRepoLabel } from "$libs/repo";
   dayjs.extend(relativeTime);
 
   export let pkg: Package;
@@ -71,11 +72,11 @@
     </ul>
   {/if}
   {#if pkg.github_url}
-    <h1 class="text-primary">{$t("common.github-repository").toLowerCase()}</h1>
+    <h1 class="text-primary">{$t("common.repository").toLowerCase()}</h1>
     <ul class="mb-10 flex flex-col gap-2">
       <!-- svelte-ignore a11y-click-events-have-key-events -->
-      <li on:click={() => shellOpenExternal(`https://github.com/${pkg.github_url}`)}>
-        <span class="hover:text-primary cursor-pointer">{pkg.github_url}</span>
+      <li on:click={() => shellOpenExternal(pkg.github_url)}>
+        <span class="hover:text-primary cursor-pointer">{getRepoLabel(pkg.github_url)}</span>
       </li>
     </ul>
   {/if}

svelte/src/components/packages/package.svelte

@@ -6,7 +6,7 @@
   import PackageCard from "$components/package-card/package-card.svelte";
   import LocalPackageCard from "$components/package-card/local-package-card.svelte";
 
-  export let tab = "all";
+  export let tab = "discover";
   export let pkg: GUIPackage;
   export let layout: "bottom" | "left" | "right" = "bottom";
 

svelte/src/libs/repo.ts

@@ -0,0 +1,133 @@
+import axios from "axios";
+import type { Contributor, GUIPackage, Package } from "$libs/types";
+import yaml from "js-yaml";
+import { isDev } from "@native";
+import log from "./logger";
+export async function getPackageYaml(pkgYamlUrl: string) {
+  const url = pkgYamlUrl.replace("/github.com", "/raw.githubusercontent.com").replace("/blob", "");
+
+  const { data: rawYaml } = await axios.get(url);
+
+  const data = await yaml.load(rawYaml);
+
+  return data;
+}
+
+export async function getReadme(
+  pkg: Partial<GUIPackage>
+): Promise<{ data: string; type: "md" | "rst" | "html" }> {
+  let type: "md" | "rst" | "html" = "md";
+  let data = "";
+  try {
+    const stage = (await isDev()) ? "dev" : "prod";
+    const reqHTML = await axios.get(`https://gui.tea.xyz/${stage}/${pkg.full_name!}/readme.html`);
+    if (reqHTML.status === 200 && reqHTML.data) {
+      return {
+        data: reqHTML.data,
+        type: "html"
+      };
+    }
+
+    const repo = repoCloudProvider(pkg.github_url!);
+    const { namespace, projectName } = getNamespaceProject(pkg.github_url!); // TODO: replace with repo_url
+
+    if (repo.isGithub) {
+      const reqGithub = await axios.get(
+        `https://github.com/repos/${namespace}/${projectName}/readme`
+      );
+      type = reqGithub.data.name.endsWith(".rst") ? "rst" : "md";
+      const reqDl = await axios.get(reqGithub.data.download_url);
+      data = reqDl.data;
+    } else if (repo.isGitlab) {
+      const gitlabApiUrl = `https://gitlab.com/api/v4/projects/${encodeURIComponent(
+        namespace + "/" + projectName
+      )}`;
+      const reqForId = await axios.get(gitlabApiUrl);
+      const projectId = reqForId.data.id;
+      const reqGitlab = await axios.get(
+        `https://gitlab.com/api/v4/projects/${projectId}/repository/files/README.md/raw`
+      );
+      data = reqGitlab.data;
+    }
+  } catch (error) {
+    log.error(error);
+  }
+  return { data, type };
+}
+
+export async function getContributors(pkg: Pick<GUIPackage, "github_url">): Promise<Contributor[]> {
+  // TODO: replace with repo_url
+  // TODO: work with gitlab api GET /projects/:id/repository/contributors or index this in the db too
+  const { namespace, projectName } = getNamespaceProject(pkg.github_url!);
+  // maintainer/repo
+  let contributors: Contributor[] = [];
+  try {
+    const repo = repoCloudProvider(pkg.github_url!);
+    if (repo.isGithub) {
+      const req = await axios.get(
+        `https://github.com/repos/${namespace}/${projectName}/contributors`
+      );
+      if (req.data) {
+        contributors = req.data.map((c: Contributor & { id: number }) => ({
+          login: c.login,
+          avatar_url: c.avatar_url,
+          name: c.name || "",
+          github_id: c.id,
+          contributions: c.contributions
+        }));
+      }
+    }
+  } catch (error) {
+    log.error(error);
+  }
+  return contributors;
+}
+
+export async function getRepoAsPackage(
+  pkg: Pick<GUIPackage, "github_url">
+): Promise<Partial<Package>> {
+  const { namespace, projectName } = getNamespaceProject(pkg.github_url!);
+
+  const newPkg: Partial<Package> = {};
+  try {
+    const repo = repoCloudProvider(pkg.github_url!);
+    if (repo.isGithub) {
+      const req = await axios.get(`https://github.com/repos/${namespace}/${projectName}`);
+      if (req.data) {
+        newPkg.license = req.data?.license?.name || "";
+      }
+    }
+  } catch (error) {
+    log.error(error);
+  }
+  return newPkg;
+}
+
+export const trimGithubSlug = (slug: string): string => {
+  const gh = slug.replace("https://github.com/", "");
+  const [owner, repo] = gh.split("/");
+  return [owner, repo].join("/");
+};
+
+export const getNamespaceProject = (repoURL: string) => {
+  const parsedUrl = new URL(repoURL);
+  const pathParts = parsedUrl.pathname.split("/").filter((part) => part);
+  const namespace = pathParts[0];
+  const projectName = pathParts[1];
+  return {
+    namespace,
+    projectName
+  };
+};
+
+export const getRepoLabel = (repoURL: string) => {
+  const { namespace, projectName } = getNamespaceProject(repoURL);
+  return `${namespace}/${projectName}`;
+};
+
+export const repoCloudProvider = (repoURL: string) => {
+  return {
+    isGithub: repoURL.includes("github.com"),
+    isGitlab: repoURL.includes("gitlab")
+  };
+};

svelte/src/libs/stores/pkgs.ts

@@ -21,7 +21,7 @@ import {
   getPantryDetails
 } from "@native";
 
-import { getReadme, getContributors, getRepoAsPackage } from "$libs/github";
+import { getReadme, getContributors, getRepoAsPackage, trimGithubSlug } from "$libs/repo";
 import { trackInstall, trackInstallFailed } from "$libs/analytics";
 import {
   addInstalledVersion,
@@ -30,7 +30,6 @@ import {
   packageWasUpdated
 } from "$libs/packages/pkg-utils";
 import withDebounce from "$libs/utils/debounce";
-import { trimGithubSlug } from "$libs/github";
 import { notificationStore } from "$libs/stores";
 import withRetry from "$libs/utils/retry";
 
@@ -126,7 +125,9 @@ const syncPackageData = async (guiPkg: Partial<GUIPackage> | undefined) => {
 
   const pkg = await getPackage(guiPkg.full_name!); // ATM: pkg only bottles and github:string
   const readmeMd = `# ${guiPkg.full_name} #
-To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
+To read more about this package go to [${guiPkg.homepage || guiPkg.github_url}](${
+    guiPkg.homepage || guiPkg.github_url
+  }).
 		`;
 
   const updatedPackage: Partial<GUIPackage> = {
@@ -135,19 +136,14 @@ To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
       data: readmeMd,
       type: "md"
     },
-    synced: true,
-    github_url: pkg.github_url
-      ? trimGithubSlug(pkg.github_url)
-      : pkg.full_name?.includes("github.com")
-      ? trimGithubSlug(pkg.full_name.split("github.com/")[1])
-      : ""
+    synced: true
   };
-  if (updatedPackage.github_url) {
-    const [owner, repo] = updatedPackage.github_url.split("/");
+  // TODO: fix this with repo_url
+  if (pkg.github_url) {
     const [readme, contributors, repoData] = await Promise.all([
-      getReadme(owner, repo),
-      getContributors(owner, repo),
-      getRepoAsPackage(owner, repo)
+      getReadme(pkg),
+      getContributors(pkg),
+      getRepoAsPackage(pkg)
     ]);
     if (readme) {
       updatedPackage.readme = readme;

svelte/src/libs/translations/languages/de.json

@@ -51,6 +51,7 @@
       "homepage": "Startseite",
       "documentation": "Dokumentation",
       "github-repository": "Github Repository",
+      "repository": "Repository",
       "contributors": "Mitwirkende",
       "view-on-github": "AUF GITHUB ANSEHEN"
     },

svelte/src/libs/translations/languages/en.json

@@ -51,6 +51,7 @@
       "homepage": "Homepage",
       "documentation": "Documentation",
       "github-repository": "Github Repository",
+      "repository": "Repository",
       "contributors": "Contributors",
       "view-on-github": "VIEW ON GITHUB"
     },

svelte/src/libs/translations/languages/ptbr.json

@@ -51,6 +51,7 @@
       "homepage": "Página inicial",
       "documentation": "Documentação",
       "github-repository": "Repositório Github",
+      "repository": "Repositório",
       "contributors": "Contribuidores",
       "view-on-github": "VER NO GITHUB"
     },

svelte/src/libs/translations/languages/ru.json

@@ -51,6 +51,7 @@
       "homepage": "Домашняя страница",
       "documentation": "Документация",
       "github-repository": "Репозиторий на Github",
+      "repository": "Репозиторий",
       "contributors": "Участники",
       "view-on-github": "ПОСМОТРЕТЬ НА GITHUB"
     },

svelte/src/libs/translations/languages/uk.json

@@ -51,6 +51,7 @@
       "homepage": "Головна",
       "documentation": "Документація",
       "github-repository": "Github Репозитарій",
+      "repository": "Репозитарій",
       "contributors": "Учасники",
       "view-on-github": "ДИВИТИСЯ НА GITHUB"
     },

svelte/src/libs/translations/languages/zh.json

@@ -51,6 +51,7 @@
       "homepage": "主页",
       "documentation": "文档",
       "github-repository": "Github 仓库",
+      "repository": "仓库",
       "contributors": "贡献者",
       "view-on-github": "在 Github 上查看"
     },