Skip to content

Commit 7364933

Browse files
kylecarbskylecarbs
authored
refactor: Allow provisioner jobs to be disconnected from projects (#194)
* Nest jobs under an organization * Rename project parameter to parameter schema * Update references when computing project parameters * Add files endpoint * Allow one-off project import jobs * Allow variables to be injected that are not defined by the schema * Update API to use jobs first * Fix CLI tests * Fix linting * Fix hex length for files table * Reduce memory allocation for windows
1 parent 4c5e443 commit 7364933

37 files changed

+1373
-988
lines changed

‎coderd/coderd.go

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,14 +102,27 @@ func New(options *Options) http.Handler {
102102
})
103103
})
104104

105+
r.Route("/files", func(r chi.Router) {
106+
r.Use(httpmw.ExtractAPIKey(options.Database, nil))
107+
r.Post("/", api.postFiles)
108+
})
109+
105110
r.Route("/provisioners", func(r chi.Router) {
106111
r.Route("/daemons", func(r chi.Router) {
107112
r.Get("/", api.provisionerDaemons)
108113
r.Get("/serve", api.provisionerDaemonsServe)
109114
})
110-
r.Route("/jobs/{provisionerjob}", func(r chi.Router) {
111-
r.Use(httpmw.ExtractProvisionerJobParam(options.Database))
112-
r.Get("/logs", api.provisionerJobLogsByID)
115+
r.Route("/jobs/{organization}", func(r chi.Router) {
116+
r.Use(
117+
httpmw.ExtractAPIKey(options.Database, nil),
118+
httpmw.ExtractOrganizationParam(options.Database),
119+
)
120+
r.Post("/import", api.postProvisionerImportJobByOrganization)
121+
r.Route("/{provisionerjob}", func(r chi.Router) {
122+
r.Use(httpmw.ExtractProvisionerJobParam(options.Database))
123+
r.Get("/", api.provisionerJobByOrganization)
124+
r.Get("/logs", api.provisionerJobLogsByID)
125+
})
113126
})
114127
})
115128
})

‎coderd/coderdtest/coderdtest.go

Lines changed: 26 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -122,40 +122,44 @@ func CreateInitialUser(t *testing.T, client *codersdk.Client) coderd.CreateIniti
122122
return req
123123
}
124124

125-
// CreateProject creates a project with the "echo" provisioner for
126-
// compatibility with testing. The name assigned is randomly generated.
127-
func CreateProject(t *testing.T, client *codersdk.Client, organization string) coderd.Project {
128-
project, err := client.CreateProject(context.Background(), organization, coderd.CreateProjectRequest{
129-
Name: randomUsername(),
130-
Provisioner: database.ProvisionerTypeEcho,
125+
// CreateProjectImportProvisionerJob creates a project import provisioner job
126+
// with the responses provided. It uses the "echo" provisioner for compatibility
127+
// with testing.
128+
func CreateProjectImportProvisionerJob(t *testing.T, client *codersdk.Client, organization string, res *echo.Responses) coderd.ProvisionerJob {
129+
data, err := echo.Tar(res)
130+
require.NoError(t, err)
131+
file, err := client.UploadFile(context.Background(), codersdk.ContentTypeTar, data)
132+
require.NoError(t, err)
133+
job, err := client.CreateProjectVersionImportProvisionerJob(context.Background(), organization, coderd.CreateProjectImportJobRequest{
134+
StorageSource: file.Hash,
135+
StorageMethod: database.ProvisionerStorageMethodFile,
136+
Provisioner: database.ProvisionerTypeEcho,
131137
})
132138
require.NoError(t, err)
133-
return project
139+
return job
134140
}
135141

136-
// CreateProjectVersion creates a project version for the "echo" provisioner
137-
// for compatibility with testing.
138-
func CreateProjectVersion(t *testing.T, client *codersdk.Client, organization, project string, responses *echo.Responses) coderd.ProjectVersion {
139-
data, err := echo.Tar(responses)
140-
require.NoError(t, err)
141-
version, err := client.CreateProjectVersion(context.Background(), organization, project, coderd.CreateProjectVersionRequest{
142-
StorageMethod: database.ProjectStorageMethodInlineArchive,
143-
StorageSource: data,
142+
// CreateProject creates a project with the "echo" provisioner for
143+
// compatibility with testing. The name assigned is randomly generated.
144+
func CreateProject(t *testing.T, client *codersdk.Client, organization string, job uuid.UUID) coderd.Project {
145+
project, err := client.CreateProject(context.Background(), organization, coderd.CreateProjectRequest{
146+
Name: randomUsername(),
147+
VersionImportJobID: job,
144148
})
145149
require.NoError(t, err)
146-
return version
150+
return project
147151
}
148152

149-
// AwaitProjectVersionImported awaits for the project import job to reach completed status.
150-
func AwaitProjectVersionImported(t *testing.T, client *codersdk.Client, organization, project, version string) coderd.ProjectVersion {
151-
var projectVersion coderd.ProjectVersion
153+
// AwaitProvisionerJob awaits for a job to reach completed status.
154+
func AwaitProvisionerJob(t *testing.T, client *codersdk.Client, organization string, job uuid.UUID) coderd.ProvisionerJob {
155+
var provisionerJob coderd.ProvisionerJob
152156
require.Eventually(t, func() bool {
153157
var err error
154-
projectVersion, err = client.ProjectVersion(context.Background(), organization, project, version)
158+
provisionerJob, err = client.ProvisionerJob(context.Background(), organization, job)
155159
require.NoError(t, err)
156-
return projectVersion.Import.Status.Completed()
160+
return provisionerJob.Status.Completed()
157161
}, 3*time.Second, 25*time.Millisecond)
158-
return projectVersion
162+
return provisionerJob
159163
}
160164

161165
// CreateWorkspace creates a workspace for the user and project provided.
@@ -169,18 +173,6 @@ func CreateWorkspace(t *testing.T, client *codersdk.Client, user string, project
169173
return workspace
170174
}
171175

172-
// AwaitWorkspaceHistoryProvisioned awaits for the workspace provision job to reach completed status.
173-
func AwaitWorkspaceHistoryProvisioned(t *testing.T, client *codersdk.Client, user, workspace, history string) coderd.WorkspaceHistory {
174-
var workspaceHistory coderd.WorkspaceHistory
175-
require.Eventually(t, func() bool {
176-
var err error
177-
workspaceHistory, err = client.WorkspaceHistory(context.Background(), user, workspace, history)
178-
require.NoError(t, err)
179-
return workspaceHistory.Provision.Status.Completed()
180-
}, 3*time.Second, 25*time.Millisecond)
181-
return workspaceHistory
182-
}
183-
184176
func randomUsername() string {
185177
return strings.ReplaceAll(namesgenerator.GetRandomName(0), "_", "-")
186178
}

‎coderd/coderdtest/coderdtest_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,15 @@ func TestNew(t *testing.T) {
2222
client := coderdtest.New(t)
2323
user := coderdtest.CreateInitialUser(t, client)
2424
closer := coderdtest.NewProvisionerDaemon(t, client)
25-
project := coderdtest.CreateProject(t, client, user.Organization)
26-
version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
27-
coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
25+
job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
26+
coderdtest.AwaitProvisionerJob(t, client, user.Organization, job.ID)
27+
project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
2828
workspace := coderdtest.CreateWorkspace(t, client, "me", project.ID)
2929
history, err := client.CreateWorkspaceHistory(context.Background(), "me", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
30-
ProjectVersionID: version.ID,
30+
ProjectVersionID: project.ActiveVersionID,
3131
Transition: database.WorkspaceTransitionStart,
3232
})
3333
require.NoError(t, err)
34-
coderdtest.AwaitWorkspaceHistoryProvisioned(t, client, "me", workspace.Name, history.Name)
34+
coderdtest.AwaitProvisionerJob(t, client, user.Organization, history.ProvisionJobID)
3535
closer.Close()
3636
}

‎coderd/files.go

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package coderd
2+
3+
import (
4+
"crypto/sha256"
5+
"encoding/hex"
6+
"fmt"
7+
"io"
8+
"net/http"
9+
10+
"github.com/go-chi/render"
11+
12+
"github.com/coder/coder/database"
13+
"github.com/coder/coder/httpapi"
14+
"github.com/coder/coder/httpmw"
15+
)
16+
17+
type UploadFileResponse struct {
18+
Hash string `json:"hash"`
19+
}
20+
21+
func (api *api) postFiles(rw http.ResponseWriter, r *http.Request) {
22+
apiKey := httpmw.APIKey(r)
23+
contentType := r.Header.Get("Content-Type")
24+
25+
switch contentType {
26+
case "application/x-tar":
27+
default:
28+
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
29+
Message: fmt.Sprintf("unsupported content type: %s", contentType),
30+
})
31+
return
32+
}
33+
34+
r.Body = http.MaxBytesReader(rw, r.Body, 10*(10<<20))
35+
data, err := io.ReadAll(r.Body)
36+
if err != nil {
37+
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
38+
Message: fmt.Sprintf("read file: %s", err),
39+
})
40+
return
41+
}
42+
hashBytes := sha256.Sum256(data)
43+
file, err := api.Database.InsertFile(r.Context(), database.InsertFileParams{
44+
Hash: hex.EncodeToString(hashBytes[:]),
45+
CreatedBy: apiKey.UserID,
46+
CreatedAt: database.Now(),
47+
Mimetype: contentType,
48+
Data: data,
49+
})
50+
if err != nil {
51+
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
52+
Message: fmt.Sprintf("insert file: %s", err),
53+
})
54+
return
55+
}
56+
render.Status(r, http.StatusCreated)
57+
render.JSON(rw, r, UploadFileResponse{
58+
Hash: file.Hash,
59+
})
60+
}

‎coderd/files_test.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package coderd_test
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
"github.com/stretchr/testify/require"
8+
9+
"github.com/coder/coder/coderd/coderdtest"
10+
"github.com/coder/coder/codersdk"
11+
)
12+
13+
func TestPostFiles(t *testing.T) {
14+
t.Parallel()
15+
t.Run("BadContentType", func(t *testing.T) {
16+
t.Parallel()
17+
client := coderdtest.New(t)
18+
_ = coderdtest.CreateInitialUser(t, client)
19+
_, err := client.UploadFile(context.Background(), "bad", []byte{'a'})
20+
require.Error(t, err)
21+
})
22+
23+
t.Run("Insert", func(t *testing.T) {
24+
t.Parallel()
25+
client := coderdtest.New(t)
26+
_ = coderdtest.CreateInitialUser(t, client)
27+
_, err := client.UploadFile(context.Background(), codersdk.ContentTypeTar, make([]byte, 1024))
28+
require.NoError(t, err)
29+
})
30+
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy