Skip to content

Commit df34858

Browse files
johnstcnjohnstcn
authored
chore(coderd): extract fileszip to package archive for reuse (#15229)
Related to #15087 As part of sniffing the workspace tags from an uploaded file, we need to be able to handle both zip and tar files. Extracting the functions to a separate `archive` package will be helpful here.
1 parent 5ad4747 commit df34858

File tree

8 files changed

+156
-126
lines changed

8 files changed

+156
-126
lines changed

‎coderd/fileszip.go renamed to ‎archive/archive.go

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package coderd
1+
package archive
22

33
import (
44
"archive/tar"
@@ -10,29 +10,30 @@ import (
1010
"strings"
1111
)
1212

13-
func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) {
13+
// CreateTarFromZip converts the given zipReader to a tar archive.
14+
func CreateTarFromZip(zipReader *zip.Reader, maxSize int64) ([]byte, error) {
1415
var tarBuffer bytes.Buffer
15-
err := writeTarArchive(&tarBuffer, zipReader)
16+
err := writeTarArchive(&tarBuffer, zipReader, maxSize)
1617
if err != nil {
1718
return nil, err
1819
}
1920
return tarBuffer.Bytes(), nil
2021
}
2122

22-
func writeTarArchive(w io.Writer, zipReader *zip.Reader) error {
23+
func writeTarArchive(w io.Writer, zipReader *zip.Reader, maxSize int64) error {
2324
tarWriter := tar.NewWriter(w)
2425
defer tarWriter.Close()
2526

2627
for _, file := range zipReader.File {
27-
err := processFileInZipArchive(file, tarWriter)
28+
err := processFileInZipArchive(file, tarWriter, maxSize)
2829
if err != nil {
2930
return err
3031
}
3132
}
3233
return nil
3334
}
3435

35-
func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
36+
func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer, maxSize int64) error {
3637
fileReader, err := file.Open()
3738
if err != nil {
3839
return err
@@ -52,24 +53,26 @@ func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
5253
return err
5354
}
5455

55-
n, err := io.CopyN(tarWriter, fileReader, httpFileMaxBytes)
56+
n, err := io.CopyN(tarWriter, fileReader, maxSize)
5657
log.Println(file.Name, n, err)
5758
if errors.Is(err, io.EOF) {
5859
err = nil
5960
}
6061
return err
6162
}
6263

63-
func CreateZipFromTar(tarReader *tar.Reader) ([]byte, error) {
64+
// CreateZipFromTar converts the given tarReader to a zip archive.
65+
func CreateZipFromTar(tarReader *tar.Reader, maxSize int64) ([]byte, error) {
6466
var zipBuffer bytes.Buffer
65-
err := WriteZipArchive(&zipBuffer, tarReader)
67+
err := WriteZip(&zipBuffer, tarReader, maxSize)
6668
if err != nil {
6769
return nil, err
6870
}
6971
return zipBuffer.Bytes(), nil
7072
}
7173

72-
func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
74+
// WriteZip writes the given tarReader to w.
75+
func WriteZip(w io.Writer, tarReader *tar.Reader, maxSize int64) error {
7376
zipWriter := zip.NewWriter(w)
7477
defer zipWriter.Close()
7578

@@ -100,7 +103,7 @@ func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
100103
return err
101104
}
102105

103-
_, err = io.CopyN(zipEntry, tarReader, httpFileMaxBytes)
106+
_, err = io.CopyN(zipEntry, tarReader, maxSize)
104107
if errors.Is(err, io.EOF) {
105108
err = nil
106109
}

‎coderd/fileszip_test.go renamed to ‎archive/archive_test.go

Lines changed: 12 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,22 @@
1-
package coderd_test
1+
package archive_test
22

33
import (
44
"archive/tar"
55
"archive/zip"
66
"bytes"
7-
"io"
87
"io/fs"
98
"os"
109
"os/exec"
1110
"path/filepath"
1211
"runtime"
1312
"strings"
1413
"testing"
15-
"time"
1614

1715
"github.com/stretchr/testify/assert"
1816
"github.com/stretchr/testify/require"
19-
"golang.org/x/xerrors"
2017

21-
"github.com/coder/coder/v2/coderd"
18+
"github.com/coder/coder/v2/archive"
19+
"github.com/coder/coder/v2/archive/archivetest"
2220
"github.com/coder/coder/v2/testutil"
2321
)
2422

@@ -30,18 +28,17 @@ func TestCreateTarFromZip(t *testing.T) {
3028

3129
// Read a zip file we prepared earlier
3230
ctx := testutil.Context(t, testutil.WaitShort)
33-
zipBytes, err := os.ReadFile(filepath.Join("testdata", "test.zip"))
34-
require.NoError(t, err, "failed to read sample zip file")
31+
zipBytes := archivetest.TestZipFileBytes()
3532
// Assert invariant
36-
assertSampleZipFile(t, zipBytes)
33+
archivetest.AssertSampleZipFile(t, zipBytes)
3734

3835
zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
3936
require.NoError(t, err, "failed to parse sample zip file")
4037

41-
tarBytes, err := coderd.CreateTarFromZip(zr)
38+
tarBytes, err := archive.CreateTarFromZip(zr, int64(len(zipBytes)))
4239
require.NoError(t, err, "failed to convert zip to tar")
4340

44-
assertSampleTarFile(t, tarBytes)
41+
archivetest.AssertSampleTarFile(t, tarBytes)
4542

4643
tempDir := t.TempDir()
4744
tempFilePath := filepath.Join(tempDir, "test.tar")
@@ -60,14 +57,13 @@ func TestCreateZipFromTar(t *testing.T) {
6057
}
6158
t.Run("OK", func(t *testing.T) {
6259
t.Parallel()
63-
tarBytes, err := os.ReadFile(filepath.Join(".", "testdata", "test.tar"))
64-
require.NoError(t, err, "failed to read sample tar file")
60+
tarBytes := archivetest.TestTarFileBytes()
6561

6662
tr := tar.NewReader(bytes.NewReader(tarBytes))
67-
zipBytes, err := coderd.CreateZipFromTar(tr)
63+
zipBytes, err := archive.CreateZipFromTar(tr, int64(len(tarBytes)))
6864
require.NoError(t, err)
6965

70-
assertSampleZipFile(t, zipBytes)
66+
archivetest.AssertSampleZipFile(t, zipBytes)
7167

7268
tempDir := t.TempDir()
7369
tempFilePath := filepath.Join(tempDir, "test.zip")
@@ -99,7 +95,7 @@ func TestCreateZipFromTar(t *testing.T) {
9995

10096
// When: we convert this to a zip
10197
tr := tar.NewReader(&tarBytes)
102-
zipBytes, err := coderd.CreateZipFromTar(tr)
98+
zipBytes, err := archive.CreateZipFromTar(tr, int64(tarBytes.Len()))
10399
require.NoError(t, err)
104100

105101
// Then: the resulting zip should contain a corresponding directory
@@ -133,7 +129,7 @@ func assertExtractedFiles(t *testing.T, dir string, checkModePerm bool) {
133129
if checkModePerm {
134130
assert.Equal(t, fs.ModePerm&0o755, stat.Mode().Perm(), "expected mode 0755 on directory")
135131
}
136-
assert.Equal(t, archiveRefTime(t).UTC(), stat.ModTime().UTC(), "unexpected modtime of %q", path)
132+
assert.Equal(t, archivetest.ArchiveRefTime(t).UTC(), stat.ModTime().UTC(), "unexpected modtime of %q", path)
137133
case "/test/hello.txt":
138134
stat, err := os.Stat(path)
139135
assert.NoError(t, err, "failed to stat path %q", path)
@@ -168,84 +164,3 @@ func assertExtractedFiles(t *testing.T, dir string, checkModePerm bool) {
168164
return nil
169165
})
170166
}
171-
172-
func assertSampleTarFile(t *testing.T, tarBytes []byte) {
173-
t.Helper()
174-
175-
tr := tar.NewReader(bytes.NewReader(tarBytes))
176-
for {
177-
hdr, err := tr.Next()
178-
if err != nil {
179-
if err == io.EOF {
180-
return
181-
}
182-
require.NoError(t, err)
183-
}
184-
185-
// Note: ignoring timezones here.
186-
require.Equal(t, archiveRefTime(t).UTC(), hdr.ModTime.UTC())
187-
188-
switch hdr.Name {
189-
case "test/":
190-
require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
191-
case "test/hello.txt":
192-
require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
193-
bs, err := io.ReadAll(tr)
194-
if err != nil && !xerrors.Is(err, io.EOF) {
195-
require.NoError(t, err)
196-
}
197-
require.Equal(t, "hello", string(bs))
198-
case "test/dir/":
199-
require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
200-
case "test/dir/world.txt":
201-
require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
202-
bs, err := io.ReadAll(tr)
203-
if err != nil && !xerrors.Is(err, io.EOF) {
204-
require.NoError(t, err)
205-
}
206-
require.Equal(t, "world", string(bs))
207-
default:
208-
require.Failf(t, "unexpected file in tar", hdr.Name)
209-
}
210-
}
211-
}
212-
213-
func assertSampleZipFile(t *testing.T, zipBytes []byte) {
214-
t.Helper()
215-
216-
zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
217-
require.NoError(t, err)
218-
219-
for _, f := range zr.File {
220-
// Note: ignoring timezones here.
221-
require.Equal(t, archiveRefTime(t).UTC(), f.Modified.UTC())
222-
switch f.Name {
223-
case "test/", "test/dir/":
224-
// directory
225-
case "test/hello.txt":
226-
rc, err := f.Open()
227-
require.NoError(t, err)
228-
bs, err := io.ReadAll(rc)
229-
_ = rc.Close()
230-
require.NoError(t, err)
231-
require.Equal(t, "hello", string(bs))
232-
case "test/dir/world.txt":
233-
rc, err := f.Open()
234-
require.NoError(t, err)
235-
bs, err := io.ReadAll(rc)
236-
_ = rc.Close()
237-
require.NoError(t, err)
238-
require.Equal(t, "world", string(bs))
239-
default:
240-
require.Failf(t, "unexpected file in zip", f.Name)
241-
}
242-
}
243-
}
244-
245-
// archiveRefTime is the Go reference time. The contents of the sample tar and zip files
246-
// in testdata/ all have their modtimes set to the below in some timezone.
247-
func archiveRefTime(t *testing.T) time.Time {
248-
locMST, err := time.LoadLocation("MST")
249-
require.NoError(t, err, "failed to load MST timezone")
250-
return time.Date(2006, 1, 2, 3, 4, 5, 0, locMST)
251-
}

‎archive/archivetest/archivetest.go

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package archivetest
2+
3+
import (
4+
"archive/tar"
5+
"archive/zip"
6+
"bytes"
7+
_ "embed"
8+
"io"
9+
"testing"
10+
"time"
11+
12+
"github.com/stretchr/testify/require"
13+
"golang.org/x/xerrors"
14+
)
15+
16+
//go:embed testdata/test.tar
17+
var testTarFileBytes []byte
18+
19+
//go:embed testdata/test.zip
20+
var testZipFileBytes []byte
21+
22+
// TestTarFileBytes returns the content of testdata/test.tar
23+
func TestTarFileBytes() []byte {
24+
return append([]byte{}, testTarFileBytes...)
25+
}
26+
27+
// TestZipFileBytes returns the content of testdata/test.zip
28+
func TestZipFileBytes() []byte {
29+
return append([]byte{}, testZipFileBytes...)
30+
}
31+
32+
// AssertSampleTarfile compares the content of tarBytes against testdata/test.tar.
33+
func AssertSampleTarFile(t *testing.T, tarBytes []byte) {
34+
t.Helper()
35+
36+
tr := tar.NewReader(bytes.NewReader(tarBytes))
37+
for {
38+
hdr, err := tr.Next()
39+
if err != nil {
40+
if err == io.EOF {
41+
return
42+
}
43+
require.NoError(t, err)
44+
}
45+
46+
// Note: ignoring timezones here.
47+
require.Equal(t, ArchiveRefTime(t).UTC(), hdr.ModTime.UTC())
48+
49+
switch hdr.Name {
50+
case "test/":
51+
require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
52+
case "test/hello.txt":
53+
require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
54+
bs, err := io.ReadAll(tr)
55+
if err != nil && !xerrors.Is(err, io.EOF) {
56+
require.NoError(t, err)
57+
}
58+
require.Equal(t, "hello", string(bs))
59+
case "test/dir/":
60+
require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
61+
case "test/dir/world.txt":
62+
require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
63+
bs, err := io.ReadAll(tr)
64+
if err != nil && !xerrors.Is(err, io.EOF) {
65+
require.NoError(t, err)
66+
}
67+
require.Equal(t, "world", string(bs))
68+
default:
69+
require.Failf(t, "unexpected file in tar", hdr.Name)
70+
}
71+
}
72+
}
73+
74+
// AssertSampleZipFile compares the content of zipBytes against testdata/test.zip.
75+
func AssertSampleZipFile(t *testing.T, zipBytes []byte) {
76+
t.Helper()
77+
78+
zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
79+
require.NoError(t, err)
80+
81+
for _, f := range zr.File {
82+
// Note: ignoring timezones here.
83+
require.Equal(t, ArchiveRefTime(t).UTC(), f.Modified.UTC())
84+
switch f.Name {
85+
case "test/", "test/dir/":
86+
// directory
87+
case "test/hello.txt":
88+
rc, err := f.Open()
89+
require.NoError(t, err)
90+
bs, err := io.ReadAll(rc)
91+
_ = rc.Close()
92+
require.NoError(t, err)
93+
require.Equal(t, "hello", string(bs))
94+
case "test/dir/world.txt":
95+
rc, err := f.Open()
96+
require.NoError(t, err)
97+
bs, err := io.ReadAll(rc)
98+
_ = rc.Close()
99+
require.NoError(t, err)
100+
require.Equal(t, "world", string(bs))
101+
default:
102+
require.Failf(t, "unexpected file in zip", f.Name)
103+
}
104+
}
105+
}
106+
107+
// archiveRefTime is the Go reference time. The contents of the sample tar and zip files
108+
// in testdata/ all have their modtimes set to the below in some timezone.
109+
func ArchiveRefTime(t *testing.T) time.Time {
110+
locMST, err := time.LoadLocation("MST")
111+
require.NoError(t, err, "failed to load MST timezone")
112+
return time.Date(2006, 1, 2, 3, 4, 5, 0, locMST)
113+
}

‎coderd/testdata/test.tar renamed to ‎archive/archivetest/testdata/test.tar

File renamed without changes.

‎coderd/testdata/test.zip renamed to ‎archive/archivetest/testdata/test.zip

File renamed without changes.

‎cli/templatepull_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/google/uuid"
1414
"github.com/stretchr/testify/require"
1515

16+
"github.com/coder/coder/v2/archive"
1617
"github.com/coder/coder/v2/cli/clitest"
1718
"github.com/coder/coder/v2/coderd"
1819
"github.com/coder/coder/v2/coderd/coderdtest"
@@ -95,7 +96,7 @@ func TestTemplatePull_Stdout(t *testing.T) {
9596

9697
// Verify .zip format
9798
tarReader := tar.NewReader(bytes.NewReader(expected))
98-
expectedZip, err := coderd.CreateZipFromTar(tarReader)
99+
expectedZip, err := archive.CreateZipFromTar(tarReader, coderd.HTTPFileMaxBytes)
99100
require.NoError(t, err)
100101

101102
inv, root = clitest.New(t, "templates", "pull", "--zip", template.Name)

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy