refactor(site): Remove untar dep and support nested folders on template version page (#6244)

BrunoQuaresma · web-flow · commit 2c309194e979 · 2023-02-16T18:38:14.000Z
diff --git a/site/js-untar.d.ts b/site/js-untar.d.ts
diff --git a/site/package.json b/site/package.json
@@ -59,7 +59,6 @@
     "history": "5.3.0",
     "i18next": "21.9.1",
     "jest-location-mock": "1.0.9",
-    "js-untar": "2.0.0",
     "just-debounce-it": "3.1.1",
     "lodash": "4.17.21",
     "playwright": "^1.29.2",
@@ -103,7 +102,6 @@
     "@types/react-helmet": "6.1.5",
     "@types/react-syntax-highlighter": "15.5.5",
     "@types/semver": "7.3.12",
-    "@types/tar-js": "^0.3.2",
     "@types/ua-parser-js": "0.7.36",
     "@types/uuid": "8.3.4",
     "@typescript-eslint/eslint-plugin": "5.50.0",
diff --git a/site/site.go b/site/site.go
@@ -326,8 +326,6 @@ func cspHeaders(next http.Handler) http.Handler {
 			// Report all violations back to the server to log
 			CSPDirectiveReportURI: {"/api/v2/csp/reports"},
 			CSPFrameAncestors:     {"'none'"},
-			// worker for loading the .tar files on FE using js-untar
-			CSPDirectiveWorkerSrc: {"'self' blob:"},
 
 			// Only scripts can manipulate the dom. This prevents someone from
 			// naming themselves something like '<svg onload="alert(/cross-site-scripting/)" />'.
diff --git a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx
@@ -27,7 +27,7 @@ export const TemplateVersionEditorPage: FC = () => {
     },
     {
       onSuccess(data) {
-        sendEvent({ type: "INITIALIZE", untarFiles: data.untarFiles })
+        sendEvent({ type: "INITIALIZE", tarReader: data.tarReader })
       },
     },
   )
diff --git a/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts b/site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts
@@ -1,7 +1,7 @@
 import { useQuery, UseQueryOptions } from "@tanstack/react-query"
 import { getFile, getTemplateByName, getTemplateVersionByName } from "api/api"
+import { TarReader } from "util/tar"
 import { createTemplateVersionFileTree } from "util/templateVersion"
-import untar, { File as UntarFile } from "js-untar"
 
 const getTemplateVersionData = async (
   orgId: string,
@@ -13,17 +13,15 @@ const getTemplateVersionData = async (
     getTemplateVersionByName(orgId, templateName, versionName),
   ])
   const tarFile = await getFile(version.job.file_id)
-  let untarFiles: UntarFile[] = []
-  await untar(tarFile).then((files) => {
-    untarFiles = files
-  })
-  const fileTree = await createTemplateVersionFileTree(untarFiles)
+  const tarReader = new TarReader()
+  await tarReader.readFile(tarFile)
+  const fileTree = await createTemplateVersionFileTree(tarReader)
 
   return {
     template,
     version,
     fileTree,
-    untarFiles,
+    tarReader,
   }
 }
 
diff --git a/site/src/util/tar.test.ts b/site/src/util/tar.test.ts
@@ -9,7 +9,12 @@ test("tar", async () => {
   writer.addFile("b.txt", new Blob(["world"]), { mtime })
   writer.addFile("c.txt", "", { mtime })
   writer.addFolder("etc", { mtime })
-  writer.addFile("etc/d.txt", "", { mtime })
+  writer.addFile("etc/d.txt", "Some text content", {
+    mtime,
+    user: "coder",
+    group: "codergroup",
+    mode: parseInt("777", 8),
+  })
   const blob = await writer.write()
 
   // Read
@@ -32,8 +37,11 @@ test("tar", async () => {
   })
   verifyFile(fileInfos[4], reader.getTextFile(fileInfos[4].name) as string, {
     name: "etc/d.txt",
-    content: "",
+    content: "Some text content",
   })
+  expect(fileInfos[4].group).toEqual("codergroup")
+  expect(fileInfos[4].user).toEqual("coder")
+  expect(fileInfos[4].mode).toEqual(parseInt("777", 8))
 })
 
 function verifyFile(
diff --git a/site/src/util/tar.ts b/site/src/util/tar.ts
@@ -13,6 +13,10 @@ export interface ITarFileInfo {
   name: string
   type: TarFileType
   size: number
+  mode: number
+  mtime: number
+  user: string
+  group: string
   headerOffset: number
 }
 
@@ -34,7 +38,7 @@ export interface ITarWriteOptions {
 }
 
 export class TarReader {
-  private fileInfo: ITarFileInfo[] = []
+  public fileInfo: ITarFileInfo[] = []
   private _buffer: ArrayBuffer | null = null
 
   constructor() {
@@ -64,22 +68,28 @@ export class TarReader {
   private readFileInfo() {
     this.fileInfo = []
     let offset = 0
-    let fileSize = 0
-    let fileName = ""
-    let fileType: TarFileType
+
     while (offset < this.buffer.byteLength - 512) {
-      fileName = this.readFileName(offset)
+      const fileName = this.readFileName(offset)
       if (!fileName) {
         break
       }
-      fileType = this.readFileType(offset)
-      fileSize = this.readFileSize(offset)
+      const fileType = this.readFileType(offset)
+      const fileSize = this.readFileSize(offset)
+      const fileMode = this.readFileMode(offset)
+      const fileMtime = this.readFileMtime(offset)
+      const fileUser = this.readFileUser(offset)
+      const fileGroup = this.readFileGroup(offset)
 
       this.fileInfo.push({
         name: fileName,
         type: fileType,
         size: fileSize,
         headerOffset: offset,
+        mode: fileMode,
+        mtime: fileMtime,
+        user: fileUser,
+        group: fileGroup,
       })
 
       offset += 512 + 512 * Math.floor((fileSize + 511) / 512)
@@ -100,6 +110,24 @@ export class TarReader {
     return this.readString(offset, 100)
   }
 
+  private readFileMode(offset: number) {
+    const mode = this.readString(offset + 100, 8)
+    return parseInt(mode, 8)
+  }
+
+  private readFileMtime(offset: number) {
+    const mtime = this.readString(offset + 136, 12)
+    return parseInt(mtime, 8)
+  }
+
+  private readFileUser(offset: number) {
+    return this.readString(offset + 265, 32)
+  }
+
+  private readFileGroup(offset: number) {
+    return this.readString(offset + 297, 32)
+  }
+
   private readFileType(offset: number) {
     const typeView = new Uint8Array(this.buffer, offset + 156, 1)
     const typeStr = String.fromCharCode(typeView[0])
diff --git a/site/src/util/templateVersion.ts b/site/src/util/templateVersion.ts
@@ -1,7 +1,7 @@
 import * as API from "api/api"
 import { TemplateVersion } from "api/typesGenerated"
-import untar, { File as UntarFile } from "js-untar"
 import { FileTree, setFile } from "./filetree"
+import { TarReader } from "./tar"
 
 /**
  * Content by filename
@@ -10,32 +10,16 @@ export type TemplateVersionFiles = Record<string, string>
 
 export const getTemplateVersionFiles = async (
   version: TemplateVersion,
-  allowedExtensions: string[],
-  allowedFiles: string[],
 ): Promise<TemplateVersionFiles> => {
   const files: TemplateVersionFiles = {}
   const tarFile = await API.getFile(version.job.file_id)
-  const blobs: Record<string, Blob> = {}
-
-  await untar(tarFile).then(undefined, undefined, async (file) => {
-    const paths = file.name.split("/")
-    const filename = paths[paths.length - 1]
-    const [_, extension] = filename.split(".")
-
-    if (
-      allowedExtensions.includes(extension) ||
-      allowedFiles.includes(filename)
-    ) {
-      blobs[filename] = file.blob
+  const tarReader = new TarReader()
+  await tarReader.readFile(tarFile)
+  for (const file of tarReader.fileInfo) {
+    if (isAllowedFile(file.name)) {
+      files[file.name] = tarReader.getTextFile(file.name) as string
     }
-  })
-
-  await Promise.all(
-    Object.entries(blobs).map(async ([filename, blob]) => {
-      files[filename] = await blob.text()
-    }),
-  )
-
+  }
   return files
 }
 
@@ -46,23 +30,17 @@ export const isAllowedFile = (name: string) => {
 }
 
 export const createTemplateVersionFileTree = async (
-  untarFiles: UntarFile[],
+  tarReader: TarReader,
 ): Promise<FileTree> => {
   let fileTree: FileTree = {}
-  const blobs: Record<string, Blob> = {}
-
-  for (const untarFile of untarFiles) {
-    if (isAllowedFile(untarFile.name)) {
-      blobs[untarFile.name] = untarFile.blob
+  for (const file of tarReader.fileInfo) {
+    if (isAllowedFile(file.name)) {
+      fileTree = setFile(
+        file.name,
+        tarReader.getTextFile(file.name) as string,
+        fileTree,
+      )
     }
   }
-
-  await Promise.all(
-    Object.entries(blobs).map(async ([fullPath, blob]) => {
-      const content = await blob.text()
-      fileTree = setFile(fullPath, content, fileTree)
-    }),
-  )
-
   return fileTree
 }
diff --git a/site/src/xServices/templateVersion/templateVersionXService.ts b/site/src/xServices/templateVersion/templateVersionXService.ts
@@ -162,23 +162,9 @@ export const templateVersionMachine = createMachine(
         }
         const loadFilesPromises: ReturnType<typeof getTemplateVersionFiles>[] =
           []
-        const allowedExtensions = ["tf", "md"]
-        const allowedFiles = ["Dockerfile"]
-        loadFilesPromises.push(
-          getTemplateVersionFiles(
-            currentVersion,
-            allowedExtensions,
-            allowedFiles,
-          ),
-        )
+        loadFilesPromises.push(getTemplateVersionFiles(currentVersion))
         if (previousVersion) {
-          loadFilesPromises.push(
-            getTemplateVersionFiles(
-              previousVersion,
-              allowedExtensions,
-              allowedFiles,
-            ),
-          )
+          loadFilesPromises.push(getTemplateVersionFiles(previousVersion))
         }
         const [currentFiles, previousFiles] = await Promise.all(
           loadFilesPromises,
diff --git a/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts b/site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts
diff --git a/site/yarn.lock b/site/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ export const TemplateVersionEditorPage: FC = () => {`
`27`	`27`	`},`
`28`	`28`	`{`
`29`	`29`	`onSuccess(data) {`
`30`		`- sendEvent({ type: "INITIALIZE", untarFiles: data.untarFiles })`
	`30`	`+ sendEvent({ type: "INITIALIZE", tarReader: data.tarReader })`
`31`	`31`	`},`
`32`	`32`	`},`
`33`	`33`	`)`