From 944eb3a52649ca807cf1d200fed36f2ad94e8439 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 18 Nov 2024 13:34:02 +0000 Subject: [PATCH 01/18] feat: extract provisioner tags from coder_workspace_tags data source when creating a template version --- cli/templatepush_test.go | 67 +++++++++++++++++++++++++++++++++ coderd/apidoc/docs.go | 6 ++- coderd/apidoc/swagger.json | 7 +++- coderd/templateversions.go | 69 +++++++++++++++++++++++++++++++++- codersdk/templateversions.go | 3 +- docs/reference/api/schemas.md | 7 ++-- site/src/api/typesGenerated.ts | 4 +- 7 files changed, 151 insertions(+), 12 deletions(-) diff --git a/cli/templatepush_test.go b/cli/templatepush_test.go index 4e9c8613961e5..2ddae2ed90662 100644 --- a/cli/templatepush_test.go +++ b/cli/templatepush_test.go @@ -8,6 +8,7 @@ import ( "runtime" "strings" "testing" + "time" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -16,9 +17,12 @@ import ( "github.com/coder/coder/v2/cli/clitest" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/rbac" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/provisioner/echo" + "github.com/coder/coder/v2/provisioner/terraform/tfparse" + "github.com/coder/coder/v2/provisionersdk" "github.com/coder/coder/v2/provisionersdk/proto" "github.com/coder/coder/v2/pty/ptytest" "github.com/coder/coder/v2/testutil" @@ -406,6 +410,69 @@ func TestTemplatePush(t *testing.T) { t.Run("ProvisionerTags", func(t *testing.T) { t.Parallel() + t.Run("WorkspaceTagsTerraform", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitShort) + + // Start an instance **without** a built-in provisioner. + // We're not actually testing that the Terraform applies. + // What we test is that a provisioner job is created with the expected + // tags based on the __content__ of the Terraform. + store, ps := dbtestutil.NewDB(t) + client := coderdtest.New(t, &coderdtest.Options{ + Database: store, + Pubsub: ps, + }) + + owner := coderdtest.CreateFirstUser(t, client) + templateAdmin, _ := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID, rbac.RoleTemplateAdmin()) + + // Create a tar file with some pre-defined content + tarFile := testutil.CreateTar(t, map[string]string{ + "main.tf": `data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar" + } + }`, + }) + + // Write the tar file to disk. + tempDir := t.TempDir() + err := tfparse.WriteArchive(tarFile, "application/x-tar", tempDir) + require.NoError(t, err) + + // Run `coder templates push` + // TODO: assert warning about no available provisioners in output. + // This is returned from the API. + templateName := strings.ReplaceAll(testutil.GetRandomName(t), "_", "-") + inv, root := clitest.New(t, "templates", "push", templateName, "-d", tempDir, "--yes") + clitest.SetupConfig(t, templateAdmin, root) + + // Don't forget to clean up! + cancelCtx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) + done := make(chan error) + go func() { + done <- inv.WithContext(cancelCtx).Run() + }() + + // Assert that a provisioner job was created with the desired tags. + wantTags := database.StringMap(provisionersdk.MutateTags(uuid.Nil, map[string]string{"foo": "bar"})) + require.Eventually(t, func() bool { + jobs, err := store.GetProvisionerJobsCreatedAfter(ctx, time.Time{}) + if !assert.NoError(t, err) { + return false + } + if len(jobs) == 0 { + return false + } + return assert.EqualValues(t, wantTags, jobs[0].Tags) + }, testutil.WaitShort, testutil.IntervalSlow) + + cancel() + <-done + }) + t.Run("ChangeTags", func(t *testing.T) { t.Parallel() diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 2c046dc8d997c..bd7b0ef9e5805 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -13725,10 +13725,12 @@ const docTemplate = `{ "codersdk.TemplateVersionWarning": { "type": "string", "enum": [ - "UNSUPPORTED_WORKSPACES" + "UNSUPPORTED_WORKSPACES", + "NO_MATCHING_PROVISIONERS" ], "x-enum-varnames": [ - "TemplateVersionWarningUnsupportedWorkspaces" + "TemplateVersionWarningUnsupportedWorkspaces", + "TemplateVersionWarningNoMatchingProvisioners" ] }, "codersdk.TimingStage": { diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 4baae0c3568c3..47121ca222419 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -12455,8 +12455,11 @@ }, "codersdk.TemplateVersionWarning": { "type": "string", - "enum": ["UNSUPPORTED_WORKSPACES"], - "x-enum-varnames": ["TemplateVersionWarningUnsupportedWorkspaces"] + "enum": ["UNSUPPORTED_WORKSPACES", "NO_MATCHING_PROVISIONERS"], + "x-enum-varnames": [ + "TemplateVersionWarningUnsupportedWorkspaces", + "TemplateVersionWarningNoMatchingProvisioners" + ] }, "codersdk.TimingStage": { "type": "string", diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 85e60a1dfff07..12a4c86757943 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "net/http" + "os" "github.com/go-chi/chi/v5" "github.com/google/uuid" @@ -32,6 +33,7 @@ import ( "github.com/coder/coder/v2/coderd/tracing" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/examples" + "github.com/coder/coder/v2/provisioner/terraform/tfparse" "github.com/coder/coder/v2/provisionersdk" sdkproto "github.com/coder/coder/v2/provisionersdk/proto" ) @@ -1342,7 +1344,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht } // Ensures the "owner" is properly applied. - tags := provisionersdk.MutateTags(apiKey.UserID, req.ProvisionerTags) + // tags := provisionersdk.MutateTags(apiKey.UserID, req.ProvisionerTags) if req.ExampleID != "" && req.FileID != uuid.Nil { httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ @@ -1437,8 +1439,59 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht } } + // Try to sniff template tags from the given file. + tempDir, err := os.MkdirTemp("", "tfparse-*") + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error checking workspace tags", + Detail: "create tempdir: " + err.Error(), + }) + return + } + defer func() { + if err := os.RemoveAll(tempDir); err != nil { + api.Logger.Error(ctx, "failed to remove temporary tfparse dir", slog.Error(err)) + } + }() + + if err := tfparse.WriteArchive(file.Data, file.Mimetype, tempDir); err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error checking workspace tags", + Detail: "extract archive to tempdir: " + err.Error(), + }) + return + } + + parser, diags := tfparse.New(tempDir, tfparse.WithLogger(api.Logger.Named("tfparse"))) + if diags.HasErrors() { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error checking workspace tags", + Detail: "parse module: " + diags.Error(), + }) + return + } + + sniffedTags, err := parser.WorkspaceTagDefaults(ctx) + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error checking workspace tags", + Detail: "evaluate default values of workspace tags: " + err.Error(), + }) + return + } + + // Tag order precedence: + // 1) User-specified tags in the request + // 2) Tags sniffed automatically from template file + // OLD + // tags := provisionersdk.MutateTags(apiKey.UserID, req.ProvisionerTags) + // NEW + tags := provisionersdk.MutateTags(apiKey.UserID, req.ProvisionerTags, sniffedTags) + var templateVersion database.TemplateVersion var provisionerJob database.ProvisionerJob + var eligibleProvisioners []database.ProvisionerDaemon + var warnings []codersdk.TemplateVersionWarning err = api.Database.InTx(func(tx database.Store) error { jobID := uuid.New() @@ -1463,6 +1516,18 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht return err } + if eligibleProvisioners, err = tx.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{ + OrganizationID: organization.ID, + WantTags: tags, + }); err != nil { + api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err)) + } + + // If there are no eligible provisioners at the time of insertion, add a warning. + if len(eligibleProvisioners) == 0 { + warnings = append(warnings, codersdk.TemplateVersionWarningNoMatchingProvisioners) + } + provisionerJob, err = tx.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{ ID: jobID, CreatedAt: dbtime.Now(), @@ -1555,7 +1620,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht httpapi.Write(ctx, rw, http.StatusCreated, convertTemplateVersion(templateVersion, convertProvisionerJob(database.GetProvisionerJobsByIDsWithQueuePositionRow{ ProvisionerJob: provisionerJob, QueuePosition: 0, - }), nil)) + }), warnings)) } // templateVersionResources returns the workspace agent resources associated diff --git a/codersdk/templateversions.go b/codersdk/templateversions.go index a6f9bbe1a2a49..a43a8f79d1245 100644 --- a/codersdk/templateversions.go +++ b/codersdk/templateversions.go @@ -14,7 +14,8 @@ import ( type TemplateVersionWarning string const ( - TemplateVersionWarningUnsupportedWorkspaces TemplateVersionWarning = "UNSUPPORTED_WORKSPACES" + TemplateVersionWarningUnsupportedWorkspaces TemplateVersionWarning = "UNSUPPORTED_WORKSPACES" + TemplateVersionWarningNoMatchingProvisioners TemplateVersionWarning = "NO_MATCHING_PROVISIONERS" ) // TemplateVersion represents a single version of a template. diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index fe8db822aafb5..eab9fa8079c33 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -5761,9 +5761,10 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o #### Enumerated Values -| Value | -| ------------------------ | -| `UNSUPPORTED_WORKSPACES` | +| Value | +| -------------------------- | +| `UNSUPPORTED_WORKSPACES` | +| `NO_MATCHING_PROVISIONERS` | ## codersdk.TimingStage diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index bd00dbda353c3..6e7934ee7f09e 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -2253,8 +2253,8 @@ export type TemplateRole = "" | "admin" | "use" export const TemplateRoles: TemplateRole[] = ["", "admin", "use"] // From codersdk/templateversions.go -export type TemplateVersionWarning = "UNSUPPORTED_WORKSPACES" -export const TemplateVersionWarnings: TemplateVersionWarning[] = ["UNSUPPORTED_WORKSPACES"] +export type TemplateVersionWarning = "NO_MATCHING_PROVISIONERS" | "UNSUPPORTED_WORKSPACES" +export const TemplateVersionWarnings: TemplateVersionWarning[] = ["NO_MATCHING_PROVISIONERS", "UNSUPPORTED_WORKSPACES"] // From codersdk/workspacebuilds.go export type TimingStage = "apply" | "connect" | "cron" | "graph" | "init" | "plan" | "start" | "stop" From 4653638d865364bf6d008df3bbe8d29fdf2b6414 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 20 Nov 2024 14:25:50 +0000 Subject: [PATCH 02/18] add initial framework for API tests --- coderd/templateversions_test.go | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/coderd/templateversions_test.go b/coderd/templateversions_test.go index a03a1c619871e..57d6ba8ee6d13 100644 --- a/coderd/templateversions_test.go +++ b/coderd/templateversions_test.go @@ -16,6 +16,7 @@ import ( "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/externalauth" "github.com/coder/coder/v2/coderd/rbac" "github.com/coder/coder/v2/coderd/rbac/policy" @@ -221,6 +222,57 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { }) require.NoError(t, err) }) + + t.Run("WorkspaceTags", func(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + store, ps := dbtestutil.NewDB(t) + client := coderdtest.New(t, &coderdtest.Options{ + Database: store, + Pubsub: ps, + }) + owner := coderdtest.CreateFirstUser(t, client) + templateAdmin, _ := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID, rbac.RoleTemplateAdmin()) + + for _, tt := range []struct { + name string + files map[string]string + wantTags map[string]string + }{ + { + name: "empty", + wantTags: map[string]string{"owner": "", "scope": "organization"}, + }, + // TODO(cian): add more test cases. + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // Create an archive from the files provided in the test case. + tarFile := testutil.CreateTar(t, tt.files) + + // Post the archive file + fi, err := templateAdmin.Upload(ctx, "application/x-tar", bytes.NewReader(tarFile)) + require.NoError(t, err) + + // Create a template version from the archive + tvName := strings.ReplaceAll(testutil.GetRandomName(t), "_", "-") + tv, err := templateAdmin.CreateTemplateVersion(ctx, owner.OrganizationID, codersdk.CreateTemplateVersionRequest{ + Name: tvName, + StorageMethod: codersdk.ProvisionerStorageMethodFile, + Provisioner: codersdk.ProvisionerTypeTerraform, + FileID: fi.ID, + }) + require.NoError(t, err) + + // Assert the expected provisioner job is created from the template version import + pj, err := store.GetProvisionerJobByID(ctx, tv.Job.ID) + require.NoError(t, err) + require.EqualValues(t, tt.wantTags, pj.Tags) + }) + } + }) } func TestPatchCancelTemplateVersion(t *testing.T) { From 85c2c8fde1f78be2e4722ecbdfa34e42fc1213e2 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 20 Nov 2024 16:29:08 +0000 Subject: [PATCH 03/18] update tests --- coderd/templateversions.go | 3 - coderd/templateversions_test.go | 121 ++++++++++++++++++++++++++++---- 2 files changed, 107 insertions(+), 17 deletions(-) diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 12a4c86757943..6498b317ce151 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -1483,9 +1483,6 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht // Tag order precedence: // 1) User-specified tags in the request // 2) Tags sniffed automatically from template file - // OLD - // tags := provisionersdk.MutateTags(apiKey.UserID, req.ProvisionerTags) - // NEW tags := provisionersdk.MutateTags(apiKey.UserID, req.ProvisionerTags, sniffedTags) var templateVersion database.TemplateVersion diff --git a/coderd/templateversions_test.go b/coderd/templateversions_test.go index 57d6ba8ee6d13..293de6ed521c2 100644 --- a/coderd/templateversions_test.go +++ b/coderd/templateversions_test.go @@ -233,18 +233,105 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { Pubsub: ps, }) owner := coderdtest.CreateFirstUser(t, client) - templateAdmin, _ := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID, rbac.RoleTemplateAdmin()) + templateAdmin, templateAdminUser := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID, rbac.RoleTemplateAdmin()) for _, tt := range []struct { - name string - files map[string]string - wantTags map[string]string + name string + files map[string]string + reqTags map[string]string + wantTags map[string]string + expectError string }{ { name: "empty", wantTags: map[string]string{"owner": "", "scope": "organization"}, }, - // TODO(cian): add more test cases. + { + name: "main.tf with no tags", + files: map[string]string{ + `main.tf`: `resource "null_resource" "test" {}`, + }, + wantTags: map[string]string{"owner": "", "scope": "organization"}, + }, + { + name: "main.tf with empty workspace tags", + files: map[string]string{ + `main.tf`: `resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = {} + }`, + }, + wantTags: map[string]string{"owner": "", "scope": "organization"}, + }, + { + name: "main.tf with workspace tags", + files: map[string]string{ + `main.tf`: `resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + } + }`, + }, + wantTags: map[string]string{"owner": "", "scope": "organization", "foo": "bar"}, + }, + { + name: "main.tf with workspace tags and request tags", + files: map[string]string{ + `main.tf`: `resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + } + }`, + }, + reqTags: map[string]string{"baz": "zap", "foo": "noclobber"}, + wantTags: map[string]string{"owner": "", "scope": "organization", "foo": "bar", "baz": "zap"}, + }, + { + name: "main.tf with disallowed workspace tag value", + files: map[string]string{ + `main.tf`: `resource "null_resource" "test" { + name = "foo" + } + data "coder_workspace_tags" "tags" { + tags = { + "foo": null_resource.test.name, + } + }`, + }, + expectError: ` Unknown variable; There is no variable named "null_resource".`, + }, + // We will allow coder_workspace_tags to set the scope on a template version import job + // BUT the user ID will be ultimately determined by the API key in the scope. + // TODO(Cian): Is this what we want? Or should we just ignore these provisioner + // tags entirely? + { + name: "main.tf with workspace tags that attempts to set user scope", + files: map[string]string{ + `main.tf`: `resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "scope": "user", + "owner": "12345678-1234-1234-1234-1234567890ab", + } + }`, + }, + wantTags: map[string]string{"owner": templateAdminUser.ID.String(), "scope": "user"}, + }, + { + name: "main.tf with workspace tags that attempt to clobber org ID", + files: map[string]string{ + `main.tf`: `resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "scope": "organization", + "owner": "12345678-1234-1234-1234-1234567890ab", + } + }`, + }, + wantTags: map[string]string{"owner": "", "scope": "organization"}, + }, } { tt := tt t.Run(tt.name, func(t *testing.T) { @@ -259,17 +346,23 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { // Create a template version from the archive tvName := strings.ReplaceAll(testutil.GetRandomName(t), "_", "-") tv, err := templateAdmin.CreateTemplateVersion(ctx, owner.OrganizationID, codersdk.CreateTemplateVersionRequest{ - Name: tvName, - StorageMethod: codersdk.ProvisionerStorageMethodFile, - Provisioner: codersdk.ProvisionerTypeTerraform, - FileID: fi.ID, + Name: tvName, + StorageMethod: codersdk.ProvisionerStorageMethodFile, + Provisioner: codersdk.ProvisionerTypeTerraform, + FileID: fi.ID, + ProvisionerTags: tt.reqTags, }) - require.NoError(t, err) - // Assert the expected provisioner job is created from the template version import - pj, err := store.GetProvisionerJobByID(ctx, tv.Job.ID) - require.NoError(t, err) - require.EqualValues(t, tt.wantTags, pj.Tags) + if tt.expectError == "" { + require.NoError(t, err) + + // Assert the expected provisioner job is created from the template version import + pj, err := store.GetProvisionerJobByID(ctx, tv.Job.ID) + require.NoError(t, err) + require.EqualValues(t, tt.wantTags, pj.Tags) + } else { + require.ErrorContains(t, err, tt.expectError) + } }) } }) From 2e9526b3b5948c9558d9fa1427e5e361beff4a2b Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 20 Nov 2024 17:39:00 +0000 Subject: [PATCH 04/18] drop warn log --- coderd/templateversions.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 6498b317ce151..a023ade42c7ba 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -1522,6 +1522,12 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht // If there are no eligible provisioners at the time of insertion, add a warning. if len(eligibleProvisioners) == 0 { + api.Logger.Warn(ctx, "no matching provisioners found for job", + slog.F("user_id", apiKey.UserID), + slog.F("job_id", jobID), + slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport), + slog.F("tags", tags), + ) warnings = append(warnings, codersdk.TemplateVersionWarningNoMatchingProvisioners) } From 01c8e9971e83a41217c58faad27260941b1b1aac Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 20 Nov 2024 17:40:31 +0000 Subject: [PATCH 05/18] var naming --- coderd/templateversions.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/coderd/templateversions.go b/coderd/templateversions.go index a023ade42c7ba..67270d2c77106 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -1439,7 +1439,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht } } - // Try to sniff template tags from the given file. + // Try to parse template tags from the given file. tempDir, err := os.MkdirTemp("", "tfparse-*") if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ @@ -1471,7 +1471,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht return } - sniffedTags, err := parser.WorkspaceTagDefaults(ctx) + parsedTags, err := parser.WorkspaceTagDefaults(ctx) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error checking workspace tags", @@ -1482,8 +1482,9 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht // Tag order precedence: // 1) User-specified tags in the request - // 2) Tags sniffed automatically from template file - tags := provisionersdk.MutateTags(apiKey.UserID, req.ProvisionerTags, sniffedTags) + // 2) Tags parsed from coder_workspace_tags data source in template file + // 2 may clobber 1. + tags := provisionersdk.MutateTags(apiKey.UserID, req.ProvisionerTags, parsedTags) var templateVersion database.TemplateVersion var provisionerJob database.ProvisionerJob From 5a53d86fe48953281c9b4df497b4cdd3cd96e9b4 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 20 Nov 2024 18:06:12 +0000 Subject: [PATCH 06/18] adjust control flow in error case --- coderd/templateversions.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 67270d2c77106..1b179116944a1 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -1519,10 +1519,9 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht WantTags: tags, }); err != nil { api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err)) - } - - // If there are no eligible provisioners at the time of insertion, add a warning. - if len(eligibleProvisioners) == 0 { + } else if len(eligibleProvisioners) == 0 { + // If there are no eligible provisioners at the time of insertion, add a warning. + // TODO(Cian): check provisioner last_seen? api.Logger.Warn(ctx, "no matching provisioners found for job", slog.F("user_id", apiKey.UserID), slog.F("job_id", jobID), From c1e321d434bffbe031122bdc3621914a2150d7bd Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 20 Nov 2024 18:14:57 +0000 Subject: [PATCH 07/18] show warning in templates push --- cli/templatepush.go | 13 +++++++++++++ cli/templatepush_test.go | 7 +++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cli/templatepush.go b/cli/templatepush.go index 22a77791c5f77..3e1988543a923 100644 --- a/cli/templatepush.go +++ b/cli/templatepush.go @@ -2,6 +2,7 @@ package cli import ( "bufio" + "encoding/json" "errors" "fmt" "io" @@ -16,6 +17,7 @@ import ( "golang.org/x/xerrors" "github.com/coder/coder/v2/cli/cliui" + "github.com/coder/coder/v2/coderd/util/slice" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/provisionersdk" "github.com/coder/pretty" @@ -416,6 +418,17 @@ func createValidTemplateVersion(inv *serpent.Invocation, args createValidTemplat return nil, err } + if slice.Contains(version.Warnings, codersdk.TemplateVersionWarningNoMatchingProvisioners) { + var tagsJSON strings.Builder + _ = json.NewEncoder(&tagsJSON).Encode(version.Job.Tags) + cliui.Warnf(inv.Stderr, `No provisioners are available to handle the job! +Please contact your deployment administrator for assistance. +Details: + Provisioner Job ID : %s + Requested tags : %s +`, version.Job.ID, tagsJSON.String()) + } + err = cliui.ProvisionerJob(inv.Context(), inv.Stdout, cliui.ProvisionerJobOptions{ Fetch: func() (codersdk.ProvisionerJob, error) { version, err := client.TemplateVersion(inv.Context(), version.ID) diff --git a/cli/templatepush_test.go b/cli/templatepush_test.go index 2ddae2ed90662..f9d261ce60a88 100644 --- a/cli/templatepush_test.go +++ b/cli/templatepush_test.go @@ -442,10 +442,11 @@ func TestTemplatePush(t *testing.T) { require.NoError(t, err) // Run `coder templates push` - // TODO: assert warning about no available provisioners in output. - // This is returned from the API. templateName := strings.ReplaceAll(testutil.GetRandomName(t), "_", "-") + var stdout, stderr strings.Builder inv, root := clitest.New(t, "templates", "push", templateName, "-d", tempDir, "--yes") + inv.Stdout = &stdout + inv.Stderr = &stderr clitest.SetupConfig(t, templateAdmin, root) // Don't forget to clean up! @@ -471,6 +472,8 @@ func TestTemplatePush(t *testing.T) { cancel() <-done + + require.Contains(t, stderr.String(), "No provisioners are available to handle the job!") }) t.Run("ChangeTags", func(t *testing.T) { From ed534bda6dee5d73d50915e5d3018801c90115b6 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 20 Nov 2024 18:15:19 +0000 Subject: [PATCH 08/18] bump test timeout --- coderd/templateversions_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/templateversions_test.go b/coderd/templateversions_test.go index 293de6ed521c2..165afbfada7b4 100644 --- a/coderd/templateversions_test.go +++ b/coderd/templateversions_test.go @@ -226,7 +226,7 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { t.Run("WorkspaceTags", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitShort) + ctx := testutil.Context(t, testutil.WaitLong) store, ps := dbtestutil.NewDB(t) client := coderdtest.New(t, &coderdtest.Options{ Database: store, From 42f39a761a9c9eb475e573182e7e3d7c34ec7b03 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 21 Nov 2024 11:57:57 +0000 Subject: [PATCH 09/18] improve test coverage --- coderd/templateversions_test.go | 183 ++++++++++++++++++++++++-------- 1 file changed, 141 insertions(+), 42 deletions(-) diff --git a/coderd/templateversions_test.go b/coderd/templateversions_test.go index 165afbfada7b4..fb7579d0ab6d7 100644 --- a/coderd/templateversions_test.go +++ b/coderd/templateversions_test.go @@ -225,6 +225,11 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { t.Run("WorkspaceTags", func(t *testing.T) { t.Parallel() + // This test ensures that when creating a template version from an archive continaining a coder_workspace_tags + // data source, we automatically assign some "reasonable" provisioner tag values to the resulting template + // import job. + // TODO(Cian): I'd also like to assert that the correct raw tag values are stored in the database, + // but in order to do this, we need to actually run the job! This isn't straightforward right now. ctx := testutil.Context(t, testutil.WaitLong) store, ps := dbtestutil.NewDB(t) @@ -249,58 +254,136 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { { name: "main.tf with no tags", files: map[string]string{ - `main.tf`: `resource "null_resource" "test" {}`, + `main.tf`: ` +variable "a" { + type = string + default = "1" +} +data "coder_parameter" "b" { + type = string + default = "2" +} +resource "null_resource" "test" {}`, }, wantTags: map[string]string{"owner": "", "scope": "organization"}, }, { name: "main.tf with empty workspace tags", files: map[string]string{ - `main.tf`: `resource "null_resource" "test" {} - data "coder_workspace_tags" "tags" { - tags = {} - }`, + `main.tf`: ` +variable "a" { + type = string + default = "1" +} +data "coder_parameter" "b" { + type = string + default = "2" +} +resource "null_resource" "test" {} +data "coder_workspace_tags" "tags" { + tags = {} +}`, }, wantTags: map[string]string{"owner": "", "scope": "organization"}, }, { name: "main.tf with workspace tags", files: map[string]string{ - `main.tf`: `resource "null_resource" "test" {} - data "coder_workspace_tags" "tags" { - tags = { - "foo": "bar", - } - }`, + `main.tf`: ` +variable "a" { + type = string + default = "1" +} +data "coder_parameter" "b" { + type = string + default = "2" +} +resource "null_resource" "test" {} +data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + } +}`, }, - wantTags: map[string]string{"owner": "", "scope": "organization", "foo": "bar"}, + wantTags: map[string]string{"owner": "", "scope": "organization", "foo": "bar", "a": "1", "b": "2"}, }, { name: "main.tf with workspace tags and request tags", files: map[string]string{ - `main.tf`: `resource "null_resource" "test" {} - data "coder_workspace_tags" "tags" { - tags = { - "foo": "bar", - } - }`, + `main.tf`: ` +variable "a" { + type = string + default = "1" +} +data "coder_parameter" "b" { + type = string + default = "2" +} +resource "null_resource" "test" {} +data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + } +}`, }, reqTags: map[string]string{"baz": "zap", "foo": "noclobber"}, - wantTags: map[string]string{"owner": "", "scope": "organization", "foo": "bar", "baz": "zap"}, + wantTags: map[string]string{"owner": "", "scope": "organization", "foo": "bar", "baz": "zap", "a": "1", "b": "2"}, }, { name: "main.tf with disallowed workspace tag value", files: map[string]string{ - `main.tf`: `resource "null_resource" "test" { - name = "foo" - } - data "coder_workspace_tags" "tags" { - tags = { - "foo": null_resource.test.name, - } - }`, + `main.tf`: ` +variable "a" { + type = string + default = "1" +} +data "coder_parameter" "b" { + type = string + default = "2" +} +resource "null_resource" "test" { + name = "foo" +} +data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + "test": null_resource.test.name, + } +}`, }, - expectError: ` Unknown variable; There is no variable named "null_resource".`, + expectError: `Unknown variable; There is no variable named "null_resource".`, + }, + { + name: "main.tf with disallowed function in tag value", + files: map[string]string{ + `main.tf`: ` +variable "a" { + type = string + default = "1" +} +data "coder_parameter" "b" { + type = string + default = "2" +} +resource "null_resource" "test" { + name = "foo" +} +data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + "test": try(null_resource.test.name, "whatever"), + } +}`, + }, + expectError: `Function calls not allowed; Functions may not be called here.`, }, // We will allow coder_workspace_tags to set the scope on a template version import job // BUT the user ID will be ultimately determined by the API key in the scope. @@ -309,29 +392,44 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { { name: "main.tf with workspace tags that attempts to set user scope", files: map[string]string{ - `main.tf`: `resource "null_resource" "test" {} - data "coder_workspace_tags" "tags" { - tags = { - "scope": "user", - "owner": "12345678-1234-1234-1234-1234567890ab", - } - }`, + `main.tf`: ` +resource "null_resource" "test" {} +data "coder_workspace_tags" "tags" { + tags = { + "scope": "user", + "owner": "12345678-1234-1234-1234-1234567890ab", + } +}`, }, wantTags: map[string]string{"owner": templateAdminUser.ID.String(), "scope": "user"}, }, { name: "main.tf with workspace tags that attempt to clobber org ID", files: map[string]string{ - `main.tf`: `resource "null_resource" "test" {} - data "coder_workspace_tags" "tags" { - tags = { - "scope": "organization", - "owner": "12345678-1234-1234-1234-1234567890ab", - } - }`, + `main.tf`: ` +resource "null_resource" "test" {} +data "coder_workspace_tags" "tags" { + tags = { + "scope": "organization", + "owner": "12345678-1234-1234-1234-1234567890ab", + } +}`, }, wantTags: map[string]string{"owner": "", "scope": "organization"}, }, + { + name: "main.tf with workspace tags that set scope=user", + files: map[string]string{ + `main.tf`: ` +resource "null_resource" "test" {} +data "coder_workspace_tags" "tags" { + tags = { + "scope": "user", + } +}`, + }, + wantTags: map[string]string{"owner": templateAdminUser.ID.String(), "scope": "user"}, + }, } { tt := tt t.Run(tt.name, func(t *testing.T) { @@ -363,6 +461,7 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { } else { require.ErrorContains(t, err, tt.expectError) } + }) } }) From c6f33f7520cf60a87fd4d66abc64d4856afa7a19 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 21 Nov 2024 12:14:47 +0000 Subject: [PATCH 10/18] fix tests --- coderd/templateversions.go | 4 +--- coderd/templateversions_test.go | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 1b179116944a1..42d11932c2d06 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -1343,9 +1343,6 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht } } - // Ensures the "owner" is properly applied. - // tags := provisionersdk.MutateTags(apiKey.UserID, req.ProvisionerTags) - if req.ExampleID != "" && req.FileID != uuid.Nil { httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ Message: "You cannot specify both an example_id and a file_id.", @@ -1480,6 +1477,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht return } + // Ensure the "owner" tag is properly applied in addition to request tags and coder_workspace_tags. // Tag order precedence: // 1) User-specified tags in the request // 2) Tags parsed from coder_workspace_tags data source in template file diff --git a/coderd/templateversions_test.go b/coderd/templateversions_test.go index fb7579d0ab6d7..2257e7c4e86ea 100644 --- a/coderd/templateversions_test.go +++ b/coderd/templateversions_test.go @@ -231,7 +231,6 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { // TODO(Cian): I'd also like to assert that the correct raw tag values are stored in the database, // but in order to do this, we need to actually run the job! This isn't straightforward right now. - ctx := testutil.Context(t, testutil.WaitLong) store, ps := dbtestutil.NewDB(t) client := coderdtest.New(t, &coderdtest.Options{ Database: store, @@ -434,6 +433,8 @@ data "coder_workspace_tags" "tags" { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() + ctx := testutil.Context(t, testutil.WaitShort) + // Create an archive from the files provided in the test case. tarFile := testutil.CreateTar(t, tt.files) @@ -453,7 +454,6 @@ data "coder_workspace_tags" "tags" { if tt.expectError == "" { require.NoError(t, err) - // Assert the expected provisioner job is created from the template version import pj, err := store.GetProvisionerJobByID(ctx, tv.Job.ID) require.NoError(t, err) @@ -461,7 +461,6 @@ data "coder_workspace_tags" "tags" { } else { require.ErrorContains(t, err, tt.expectError) } - }) } }) From b22e89016f702dc6d5dfe60bc13e67ab9a7903a3 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 21 Nov 2024 17:24:08 +0000 Subject: [PATCH 11/18] move to client-side checking for warnings --- cli/templatepush.go | 47 ++++++++++++++++++++++++++++++++--- cli/templatepush_test.go | 30 ++++++++++++++++------ coderd/templateversions.go | 48 +++++++++++++++++++++++++++++------- codersdk/templateversions.go | 3 +-- 4 files changed, 105 insertions(+), 23 deletions(-) diff --git a/cli/templatepush.go b/cli/templatepush.go index 3e1988543a923..35b5156af476e 100644 --- a/cli/templatepush.go +++ b/cli/templatepush.go @@ -2,6 +2,7 @@ package cli import ( "bufio" + "context" "encoding/json" "errors" "fmt" @@ -17,7 +18,6 @@ import ( "golang.org/x/xerrors" "github.com/coder/coder/v2/cli/cliui" - "github.com/coder/coder/v2/coderd/util/slice" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/provisionersdk" "github.com/coder/pretty" @@ -417,12 +417,28 @@ func createValidTemplateVersion(inv *serpent.Invocation, args createValidTemplat if err != nil { return nil, err } + var tagsJSON strings.Builder + _ = json.NewEncoder(&tagsJSON).Encode(version.Job.Tags) + foundProvisioners, activeProvisioners, err := checkProvisioners(inv.Context(), client, args.Organization.ID, version.Job.Tags) + if err != nil { + var apiErr *codersdk.Error + // Unfortunately this is an enterprise endpoint, so check for that first. + if errors.As(err, &apiErr) && apiErr.StatusCode() != http.StatusNotFound { + cliui.Warnf(inv.Stderr, "Unable to check for available provisioners: %s", err.Error()) + } + } - if slice.Contains(version.Warnings, codersdk.TemplateVersionWarningNoMatchingProvisioners) { - var tagsJSON strings.Builder - _ = json.NewEncoder(&tagsJSON).Encode(version.Job.Tags) + if foundProvisioners == 0 { cliui.Warnf(inv.Stderr, `No provisioners are available to handle the job! Please contact your deployment administrator for assistance. +Details: + Provisioner Job ID : %s + Requested tags : %s +`, version.Job.ID, tagsJSON.String()) + } else if activeProvisioners == 0 { + cliui.Warnf(inv.Stderr, `All available provisioner daemons have been silent for a while. +Your build will proceed once they become available. +If this persists, please contact your deployment administrator for assistance. Details: Provisioner Job ID : %s Requested tags : %s @@ -510,3 +526,26 @@ func prettyDirectoryPath(dir string) string { } return prettyDir } + +func checkProvisioners(ctx context.Context, client *codersdk.Client, orgID uuid.UUID, wantTags map[string]string) (found, active int, err error) { + // Check for eligible provisioners. This allows us to return a warning to the user if they + // submit a job for which no provisioner is available. + provisioners, err := client.OrganizationProvisionerDaemons(ctx, orgID, wantTags) + if err != nil { + return -1, -1, xerrors.Errorf("get organization provisioner daemons: %w", err) + } + + oneMinuteAgo := time.Now().Add(-time.Minute) + for _, provisioner := range provisioners { + if !provisioner.LastSeenAt.Valid { + continue + } + found++ + if provisioner.LastSeenAt.Time.Before(oneMinuteAgo) { + continue + } + active++ + } + + return found, active, nil +} diff --git a/cli/templatepush_test.go b/cli/templatepush_test.go index f9d261ce60a88..d7b055468072c 100644 --- a/cli/templatepush_test.go +++ b/cli/templatepush_test.go @@ -429,11 +429,23 @@ func TestTemplatePush(t *testing.T) { // Create a tar file with some pre-defined content tarFile := testutil.CreateTar(t, map[string]string{ - "main.tf": `data "coder_workspace_tags" "tags" { - tags = { - "foo": "bar" - } - }`, + "main.tf": ` +variable "a" { + type = string + default = "1" +} +data "coder_parameter" "b" { + type = string + default = "2" +} +resource "null_resource" "test" {} +data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + } +}`, }) // Write the tar file to disk. @@ -458,7 +470,11 @@ func TestTemplatePush(t *testing.T) { }() // Assert that a provisioner job was created with the desired tags. - wantTags := database.StringMap(provisionersdk.MutateTags(uuid.Nil, map[string]string{"foo": "bar"})) + wantTags := database.StringMap(provisionersdk.MutateTags(uuid.Nil, map[string]string{ + "foo": "bar", + "a": "1", + "b": "2", + })) require.Eventually(t, func() bool { jobs, err := store.GetProvisionerJobsCreatedAfter(ctx, time.Time{}) if !assert.NoError(t, err) { @@ -472,8 +488,6 @@ func TestTemplatePush(t *testing.T) { cancel() <-done - - require.Contains(t, stderr.String(), "No provisioners are available to handle the job!") }) t.Run("ChangeTags", func(t *testing.T) { diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 42d11932c2d06..64bbfdc84c9e7 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -10,6 +10,7 @@ import ( "fmt" "net/http" "os" + "time" "github.com/go-chi/chi/v5" "github.com/google/uuid" @@ -1486,7 +1487,6 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht var templateVersion database.TemplateVersion var provisionerJob database.ProvisionerJob - var eligibleProvisioners []database.ProvisionerDaemon var warnings []codersdk.TemplateVersionWarning err = api.Database.InTx(func(tx database.Store) error { jobID := uuid.New() @@ -1512,21 +1512,25 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht return err } - if eligibleProvisioners, err = tx.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{ - OrganizationID: organization.ID, - WantTags: tags, - }); err != nil { + // Check for eligible provisioners. This allows us to log a message warning deployment administrators + // of users submitting jobs for which no provisioners are available. + allProvisioners, activeProvisioners, err := checkProvisioners(ctx, tx, organization.ID, tags, api.DeploymentValues.Provisioner.DaemonPollInterval.Value()) + if err != nil { api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err)) - } else if len(eligibleProvisioners) == 0 { - // If there are no eligible provisioners at the time of insertion, add a warning. - // TODO(Cian): check provisioner last_seen? + } else if activeProvisioners == 0 { api.Logger.Warn(ctx, "no matching provisioners found for job", slog.F("user_id", apiKey.UserID), slog.F("job_id", jobID), slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport), slog.F("tags", tags), ) - warnings = append(warnings, codersdk.TemplateVersionWarningNoMatchingProvisioners) + } else if allProvisioners == 0 { + api.Logger.Warn(ctx, "no active provisioners found for job", + slog.F("user_id", apiKey.UserID), + slog.F("job_id", jobID), + slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport), + slog.F("tags", tags), + ) } provisionerJob, err = tx.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{ @@ -1808,3 +1812,29 @@ func (api *API) publishTemplateUpdate(ctx context.Context, templateID uuid.UUID) slog.F("template_id", templateID), slog.Error(err)) } } + +func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string, pollInterval time.Duration) (found, active int, err error) { + // Check for eligible provisioners. This allows us to return a warning to the user if they + // submit a job for which no provisioner is available. + eligibleProvisioners, err := store.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{ + OrganizationID: orgID, + WantTags: wantTags, + }) + if err != nil { + // Log the error but do not return any warnings. This is purely advisory and we should not block. + return -1, -1, xerrors.Errorf("get provisioner daemons by organization: %w", err) + } + + onePollAgo := time.Now().Add(-pollInterval) + for _, provisioner := range eligibleProvisioners { + if !provisioner.LastSeenAt.Valid { + continue + } + found++ + if provisioner.LastSeenAt.Time.Before(onePollAgo) { + continue + } + active++ + } + return found, active, nil +} diff --git a/codersdk/templateversions.go b/codersdk/templateversions.go index a43a8f79d1245..a6f9bbe1a2a49 100644 --- a/codersdk/templateversions.go +++ b/codersdk/templateversions.go @@ -14,8 +14,7 @@ import ( type TemplateVersionWarning string const ( - TemplateVersionWarningUnsupportedWorkspaces TemplateVersionWarning = "UNSUPPORTED_WORKSPACES" - TemplateVersionWarningNoMatchingProvisioners TemplateVersionWarning = "NO_MATCHING_PROVISIONERS" + TemplateVersionWarningUnsupportedWorkspaces TemplateVersionWarning = "UNSUPPORTED_WORKSPACES" ) // TemplateVersion represents a single version of a template. From a8a620ff888037cb135b86c204b454db081c7b76 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 21 Nov 2024 17:26:28 +0000 Subject: [PATCH 12/18] make gen --- coderd/apidoc/docs.go | 6 ++---- coderd/apidoc/swagger.json | 7 ++----- docs/reference/api/schemas.md | 7 +++---- site/src/api/typesGenerated.ts | 4 ++-- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 9f0ea5368e431..7c96af792e13c 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -13721,12 +13721,10 @@ const docTemplate = `{ "codersdk.TemplateVersionWarning": { "type": "string", "enum": [ - "UNSUPPORTED_WORKSPACES", - "NO_MATCHING_PROVISIONERS" + "UNSUPPORTED_WORKSPACES" ], "x-enum-varnames": [ - "TemplateVersionWarningUnsupportedWorkspaces", - "TemplateVersionWarningNoMatchingProvisioners" + "TemplateVersionWarningUnsupportedWorkspaces" ] }, "codersdk.TimingStage": { diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index e9f0e2f23a6d3..56b36a4580a16 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -12451,11 +12451,8 @@ }, "codersdk.TemplateVersionWarning": { "type": "string", - "enum": ["UNSUPPORTED_WORKSPACES", "NO_MATCHING_PROVISIONERS"], - "x-enum-varnames": [ - "TemplateVersionWarningUnsupportedWorkspaces", - "TemplateVersionWarningNoMatchingProvisioners" - ] + "enum": ["UNSUPPORTED_WORKSPACES"], + "x-enum-varnames": ["TemplateVersionWarningUnsupportedWorkspaces"] }, "codersdk.TimingStage": { "type": "string", diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index fd816fed31709..d40fe8e240005 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -5749,10 +5749,9 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o #### Enumerated Values -| Value | -| -------------------------- | -| `UNSUPPORTED_WORKSPACES` | -| `NO_MATCHING_PROVISIONERS` | +| Value | +| ------------------------ | +| `UNSUPPORTED_WORKSPACES` | ## codersdk.TimingStage diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 6e7934ee7f09e..bd00dbda353c3 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -2253,8 +2253,8 @@ export type TemplateRole = "" | "admin" | "use" export const TemplateRoles: TemplateRole[] = ["", "admin", "use"] // From codersdk/templateversions.go -export type TemplateVersionWarning = "NO_MATCHING_PROVISIONERS" | "UNSUPPORTED_WORKSPACES" -export const TemplateVersionWarnings: TemplateVersionWarning[] = ["NO_MATCHING_PROVISIONERS", "UNSUPPORTED_WORKSPACES"] +export type TemplateVersionWarning = "UNSUPPORTED_WORKSPACES" +export const TemplateVersionWarnings: TemplateVersionWarning[] = ["UNSUPPORTED_WORKSPACES"] // From codersdk/workspacebuilds.go export type TimingStage = "apply" | "connect" | "cron" | "graph" | "init" | "plan" | "start" | "stop" From 8ec672c849110e17d87fc326a86c090759e8b1a8 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 14:27:02 +0000 Subject: [PATCH 13/18] address indentation --- coderd/templateversions_test.go | 222 ++++++++++++++++---------------- 1 file changed, 111 insertions(+), 111 deletions(-) diff --git a/coderd/templateversions_test.go b/coderd/templateversions_test.go index 2257e7c4e86ea..2d5fa5052694d 100644 --- a/coderd/templateversions_test.go +++ b/coderd/templateversions_test.go @@ -254,15 +254,15 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { name: "main.tf with no tags", files: map[string]string{ `main.tf`: ` -variable "a" { - type = string - default = "1" -} -data "coder_parameter" "b" { - type = string - default = "2" -} -resource "null_resource" "test" {}`, + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" {}`, }, wantTags: map[string]string{"owner": "", "scope": "organization"}, }, @@ -270,18 +270,18 @@ resource "null_resource" "test" {}`, name: "main.tf with empty workspace tags", files: map[string]string{ `main.tf`: ` -variable "a" { - type = string - default = "1" -} -data "coder_parameter" "b" { - type = string - default = "2" -} -resource "null_resource" "test" {} -data "coder_workspace_tags" "tags" { - tags = {} -}`, + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = {} + }`, }, wantTags: map[string]string{"owner": "", "scope": "organization"}, }, @@ -289,22 +289,22 @@ data "coder_workspace_tags" "tags" { name: "main.tf with workspace tags", files: map[string]string{ `main.tf`: ` -variable "a" { - type = string - default = "1" -} -data "coder_parameter" "b" { - type = string - default = "2" -} -resource "null_resource" "test" {} -data "coder_workspace_tags" "tags" { - tags = { - "foo": "bar", - "a": var.a, - "b": data.coder_parameter.b.value, - } -}`, + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + } + }`, }, wantTags: map[string]string{"owner": "", "scope": "organization", "foo": "bar", "a": "1", "b": "2"}, }, @@ -312,22 +312,22 @@ data "coder_workspace_tags" "tags" { name: "main.tf with workspace tags and request tags", files: map[string]string{ `main.tf`: ` -variable "a" { - type = string - default = "1" -} -data "coder_parameter" "b" { - type = string - default = "2" -} -resource "null_resource" "test" {} -data "coder_workspace_tags" "tags" { - tags = { - "foo": "bar", - "a": var.a, - "b": data.coder_parameter.b.value, - } -}`, + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + } + }`, }, reqTags: map[string]string{"baz": "zap", "foo": "noclobber"}, wantTags: map[string]string{"owner": "", "scope": "organization", "foo": "bar", "baz": "zap", "a": "1", "b": "2"}, @@ -336,25 +336,25 @@ data "coder_workspace_tags" "tags" { name: "main.tf with disallowed workspace tag value", files: map[string]string{ `main.tf`: ` -variable "a" { - type = string - default = "1" -} -data "coder_parameter" "b" { - type = string - default = "2" -} -resource "null_resource" "test" { - name = "foo" -} -data "coder_workspace_tags" "tags" { - tags = { - "foo": "bar", - "a": var.a, - "b": data.coder_parameter.b.value, - "test": null_resource.test.name, - } -}`, + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" { + name = "foo" + } + data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + "test": null_resource.test.name, + } + }`, }, expectError: `Unknown variable; There is no variable named "null_resource".`, }, @@ -362,25 +362,25 @@ data "coder_workspace_tags" "tags" { name: "main.tf with disallowed function in tag value", files: map[string]string{ `main.tf`: ` -variable "a" { - type = string - default = "1" -} -data "coder_parameter" "b" { - type = string - default = "2" -} -resource "null_resource" "test" { - name = "foo" -} -data "coder_workspace_tags" "tags" { - tags = { - "foo": "bar", - "a": var.a, - "b": data.coder_parameter.b.value, - "test": try(null_resource.test.name, "whatever"), - } -}`, + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" { + name = "foo" + } + data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + "test": try(null_resource.test.name, "whatever"), + } + }`, }, expectError: `Function calls not allowed; Functions may not be called here.`, }, @@ -392,13 +392,13 @@ data "coder_workspace_tags" "tags" { name: "main.tf with workspace tags that attempts to set user scope", files: map[string]string{ `main.tf`: ` -resource "null_resource" "test" {} -data "coder_workspace_tags" "tags" { - tags = { - "scope": "user", - "owner": "12345678-1234-1234-1234-1234567890ab", - } -}`, + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "scope": "user", + "owner": "12345678-1234-1234-1234-1234567890ab", + } + }`, }, wantTags: map[string]string{"owner": templateAdminUser.ID.String(), "scope": "user"}, }, @@ -406,13 +406,13 @@ data "coder_workspace_tags" "tags" { name: "main.tf with workspace tags that attempt to clobber org ID", files: map[string]string{ `main.tf`: ` -resource "null_resource" "test" {} -data "coder_workspace_tags" "tags" { - tags = { - "scope": "organization", - "owner": "12345678-1234-1234-1234-1234567890ab", - } -}`, + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "scope": "organization", + "owner": "12345678-1234-1234-1234-1234567890ab", + } + }`, }, wantTags: map[string]string{"owner": "", "scope": "organization"}, }, @@ -420,12 +420,12 @@ data "coder_workspace_tags" "tags" { name: "main.tf with workspace tags that set scope=user", files: map[string]string{ `main.tf`: ` -resource "null_resource" "test" {} -data "coder_workspace_tags" "tags" { - tags = { - "scope": "user", - } -}`, + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "scope": "user", + } + }`, }, wantTags: map[string]string{"owner": templateAdminUser.ID.String(), "scope": "user"}, }, From 7b7aa4036dcd1eaa93ab0c9546db8d58182e4573 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 14:27:11 +0000 Subject: [PATCH 14/18] use CacheDir for tfparse instead --- coderd/templateversions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 64bbfdc84c9e7..7508c354be043 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -1438,7 +1438,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht } // Try to parse template tags from the given file. - tempDir, err := os.MkdirTemp("", "tfparse-*") + tempDir, err := os.MkdirTemp(api.Options.CacheDir, "tfparse-*") if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error checking workspace tags", From 80dc6c211b3929b5222f50ab10342b7601e87642 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 17:10:57 +0000 Subject: [PATCH 15/18] feat: add codersdk.MatchedProvisioners --- coderd/apidoc/docs.go | 21 ++++ coderd/apidoc/swagger.json | 21 ++++ coderd/templateversions.go | 59 ++++++----- coderd/templateversions_test.go | 5 + codersdk/provisionerdaemons.go | 16 +++ codersdk/templateversions.go | 3 +- docs/reference/api/schemas.md | 52 +++++++--- docs/reference/api/templates.md | 171 ++++++++++++++++++++------------ site/src/api/typesGenerated.ts | 8 ++ 9 files changed, 253 insertions(+), 103 deletions(-) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 7c96af792e13c..7152ca12fb270 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -11349,6 +11349,24 @@ const docTemplate = `{ } } }, + "codersdk.MatchedProvisioners": { + "type": "object", + "properties": { + "available": { + "description": "Available is the number of provisioner daemons that are available to\ntake jobs. This may be less than the count if some provisioners are\nbusy or have been stopped.", + "type": "integer" + }, + "count": { + "description": "Count is the number of provisioner daemons that matched the given\ntags. If the count is 0, it means no provisioner daemons matched the\nrequested tags.", + "type": "integer" + }, + "most_recently_seen": { + "description": "MostRecentlySeen is the most recently seen time of the set of matched\nprovisioners. If no provisioners matched, this field will be null.", + "type": "string", + "format": "date-time" + } + } + }, "codersdk.MinimalOrganization": { "type": "object", "required": [ @@ -13542,6 +13560,9 @@ const docTemplate = `{ "job": { "$ref": "#/definitions/codersdk.ProvisionerJob" }, + "matched_provisioners": { + "$ref": "#/definitions/codersdk.MatchedProvisioners" + }, "message": { "type": "string" }, diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 56b36a4580a16..0f1fab2b0ef53 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -10171,6 +10171,24 @@ } } }, + "codersdk.MatchedProvisioners": { + "type": "object", + "properties": { + "available": { + "description": "Available is the number of provisioner daemons that are available to\ntake jobs. This may be less than the count if some provisioners are\nbusy or have been stopped.", + "type": "integer" + }, + "count": { + "description": "Count is the number of provisioner daemons that matched the given\ntags. If the count is 0, it means no provisioner daemons matched the\nrequested tags.", + "type": "integer" + }, + "most_recently_seen": { + "description": "MostRecentlySeen is the most recently seen time of the set of matched\nprovisioners. If no provisioners matched, this field will be null.", + "type": "string", + "format": "date-time" + } + } + }, "codersdk.MinimalOrganization": { "type": "object", "required": ["id"], @@ -12287,6 +12305,9 @@ "job": { "$ref": "#/definitions/codersdk.ProvisionerJob" }, + "matched_provisioners": { + "$ref": "#/definitions/codersdk.MatchedProvisioners" + }, "message": { "type": "string" }, diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 7508c354be043..a0609c42c33f9 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -77,7 +77,7 @@ func (api *API) templateVersion(rw http.ResponseWriter, r *http.Request) { warnings = append(warnings, codersdk.TemplateVersionWarningUnsupportedWorkspaces) } - httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), warnings)) + httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), codersdk.MatchedProvisioners{}, warnings)) } // @Summary Patch template version by ID @@ -173,7 +173,7 @@ func (api *API) patchTemplateVersion(rw http.ResponseWriter, r *http.Request) { return } - httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(updatedTemplateVersion, convertProvisionerJob(jobs[0]), nil)) + httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(updatedTemplateVersion, convertProvisionerJob(jobs[0]), codersdk.MatchedProvisioners{}, nil)) } // @Summary Cancel template version by ID @@ -814,7 +814,7 @@ func (api *API) templateVersionsByTemplate(rw http.ResponseWriter, r *http.Reque return err } - apiVersions = append(apiVersions, convertTemplateVersion(version, convertProvisionerJob(job), nil)) + apiVersions = append(apiVersions, convertTemplateVersion(version, convertProvisionerJob(job), codersdk.MatchedProvisioners{}, nil)) } return nil @@ -869,7 +869,7 @@ func (api *API) templateVersionByName(rw http.ResponseWriter, r *http.Request) { return } - httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), nil)) + httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), codersdk.MatchedProvisioners{}, nil)) } // @Summary Get template version by organization, template, and name @@ -934,7 +934,7 @@ func (api *API) templateVersionByOrganizationTemplateAndName(rw http.ResponseWri return } - httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), nil)) + httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), codersdk.MatchedProvisioners{}, nil)) } // @Summary Get previous template version by organization, template, and name @@ -1020,7 +1020,7 @@ func (api *API) previousTemplateVersionByOrganizationTemplateAndName(rw http.Res return } - httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(previousTemplateVersion, convertProvisionerJob(jobs[0]), nil)) + httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(previousTemplateVersion, convertProvisionerJob(jobs[0]), codersdk.MatchedProvisioners{}, nil)) } // @Summary Archive template unused versions by template id @@ -1488,6 +1488,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht var templateVersion database.TemplateVersion var provisionerJob database.ProvisionerJob var warnings []codersdk.TemplateVersionWarning + var matchedProvisioners codersdk.MatchedProvisioners err = api.Database.InTx(func(tx database.Store) error { jobID := uuid.New() @@ -1514,17 +1515,17 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht // Check for eligible provisioners. This allows us to log a message warning deployment administrators // of users submitting jobs for which no provisioners are available. - allProvisioners, activeProvisioners, err := checkProvisioners(ctx, tx, organization.ID, tags, api.DeploymentValues.Provisioner.DaemonPollInterval.Value()) + matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags, api.DeploymentValues.Provisioner.DaemonPollInterval.Value()) if err != nil { api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err)) - } else if activeProvisioners == 0 { + } else if matchedProvisioners.Count == 0 { api.Logger.Warn(ctx, "no matching provisioners found for job", slog.F("user_id", apiKey.UserID), slog.F("job_id", jobID), slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport), slog.F("tags", tags), ) - } else if allProvisioners == 0 { + } else if matchedProvisioners.Available == 0 { api.Logger.Warn(ctx, "no active provisioners found for job", slog.F("user_id", apiKey.UserID), slog.F("job_id", jobID), @@ -1622,10 +1623,14 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht api.Logger.Error(ctx, "failed to post provisioner job to pubsub", slog.Error(err)) } - httpapi.Write(ctx, rw, http.StatusCreated, convertTemplateVersion(templateVersion, convertProvisionerJob(database.GetProvisionerJobsByIDsWithQueuePositionRow{ - ProvisionerJob: provisionerJob, - QueuePosition: 0, - }), warnings)) + httpapi.Write(ctx, rw, http.StatusCreated, convertTemplateVersion( + templateVersion, + convertProvisionerJob(database.GetProvisionerJobsByIDsWithQueuePositionRow{ + ProvisionerJob: provisionerJob, + QueuePosition: 0, + }), + matchedProvisioners, + warnings)) } // templateVersionResources returns the workspace agent resources associated @@ -1692,7 +1697,7 @@ func (api *API) templateVersionLogs(rw http.ResponseWriter, r *http.Request) { api.provisionerJobLogs(rw, r, job) } -func convertTemplateVersion(version database.TemplateVersion, job codersdk.ProvisionerJob, warnings []codersdk.TemplateVersionWarning) codersdk.TemplateVersion { +func convertTemplateVersion(version database.TemplateVersion, job codersdk.ProvisionerJob, matchedProvisioners codersdk.MatchedProvisioners, warnings []codersdk.TemplateVersionWarning) codersdk.TemplateVersion { return codersdk.TemplateVersion{ ID: version.ID, TemplateID: &version.TemplateID.UUID, @@ -1708,8 +1713,9 @@ func convertTemplateVersion(version database.TemplateVersion, job codersdk.Provi Username: version.CreatedByUsername, AvatarURL: version.CreatedByAvatarURL, }, - Archived: version.Archived, - Warnings: warnings, + Archived: version.Archived, + Warnings: warnings, + MatchedProvisioners: matchedProvisioners, } } @@ -1813,7 +1819,7 @@ func (api *API) publishTemplateUpdate(ctx context.Context, templateID uuid.UUID) } } -func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string, pollInterval time.Duration) (found, active int, err error) { +func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string, pollInterval time.Duration) (codersdk.MatchedProvisioners, error) { // Check for eligible provisioners. This allows us to return a warning to the user if they // submit a job for which no provisioner is available. eligibleProvisioners, err := store.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{ @@ -1822,19 +1828,24 @@ func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUI }) if err != nil { // Log the error but do not return any warnings. This is purely advisory and we should not block. - return -1, -1, xerrors.Errorf("get provisioner daemons by organization: %w", err) + return codersdk.MatchedProvisioners{}, xerrors.Errorf("provisioner daemons by organization: %w", err) } - onePollAgo := time.Now().Add(-pollInterval) + threePollsAgo := time.Now().Add(-3 * pollInterval) + mostRecentlySeen := codersdk.NullTime{} + var matched codersdk.MatchedProvisioners for _, provisioner := range eligibleProvisioners { if !provisioner.LastSeenAt.Valid { continue } - found++ - if provisioner.LastSeenAt.Time.Before(onePollAgo) { - continue + matched.Count++ + if provisioner.LastSeenAt.Time.After(threePollsAgo) { + matched.Available++ + } + if provisioner.LastSeenAt.Time.After(mostRecentlySeen.Time) { + matched.MostRecentlySeen.Valid = true + matched.MostRecentlySeen.Time = provisioner.LastSeenAt.Time } - active++ } - return found, active, nil + return matched, nil } diff --git a/coderd/templateversions_test.go b/coderd/templateversions_test.go index 2d5fa5052694d..5ebbd0f41804f 100644 --- a/coderd/templateversions_test.go +++ b/coderd/templateversions_test.go @@ -461,6 +461,11 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { } else { require.ErrorContains(t, err, tt.expectError) } + + // Also assert that we get the expected information back from the API endpoint + require.Zero(t, tv.MatchedProvisioners.Count) + require.Zero(t, tv.MatchedProvisioners.Available) + require.Zero(t, tv.MatchedProvisioners.MostRecentlySeen.Time) }) } }) diff --git a/codersdk/provisionerdaemons.go b/codersdk/provisionerdaemons.go index 7b14afbbb285a..edb3bef89b0e4 100644 --- a/codersdk/provisionerdaemons.go +++ b/codersdk/provisionerdaemons.go @@ -51,6 +51,22 @@ type ProvisionerDaemon struct { Tags map[string]string `json:"tags"` } +// MatchedProvisioners represents the number of provisioner daemons +// available to take a job at a specific point in time. +type MatchedProvisioners struct { + // Count is the number of provisioner daemons that matched the given + // tags. If the count is 0, it means no provisioner daemons matched the + // requested tags. + Count int `json:"count"` + // Available is the number of provisioner daemons that are available to + // take jobs. This may be less than the count if some provisioners are + // busy or have been stopped. + Available int `json:"available"` + // MostRecentlySeen is the most recently seen time of the set of matched + // provisioners. If no provisioners matched, this field will be null. + MostRecentlySeen NullTime `json:"most_recently_seen,omitempty" format:"date-time"` +} + // ProvisionerJobStatus represents the at-time state of a job. type ProvisionerJobStatus string diff --git a/codersdk/templateversions.go b/codersdk/templateversions.go index a6f9bbe1a2a49..5bda52daf3dfe 100644 --- a/codersdk/templateversions.go +++ b/codersdk/templateversions.go @@ -31,7 +31,8 @@ type TemplateVersion struct { CreatedBy MinimalUser `json:"created_by"` Archived bool `json:"archived"` - Warnings []TemplateVersionWarning `json:"warnings,omitempty" enums:"DEPRECATED_PARAMETERS"` + Warnings []TemplateVersionWarning `json:"warnings,omitempty" enums:"DEPRECATED_PARAMETERS"` + MatchedProvisioners MatchedProvisioners `json:"matched_provisioners,omitempty"` } type TemplateVersionExternalAuth struct { diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index d40fe8e240005..39c4bb61f6b3f 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -3296,6 +3296,24 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o | --------------- | ------ | -------- | ------------ | ----------- | | `session_token` | string | true | | | +## codersdk.MatchedProvisioners + +```json +{ + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" +} +``` + +### Properties + +| Name | Type | Required | Restrictions | Description | +| -------------------- | ------- | -------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `available` | integer | false | | Available is the number of provisioner daemons that are available to take jobs. This may be less than the count if some provisioners are busy or have been stopped. | +| `count` | integer | false | | Count is the number of provisioner daemons that matched the given tags. If the count is 0, it means no provisioner daemons matched the requested tags. | +| `most_recently_seen` | string | false | | Most recently seen is the most recently seen time of the set of matched provisioners. If no provisioners matched, this field will be null. | + ## codersdk.MinimalOrganization ```json @@ -5570,6 +5588,11 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -5582,20 +5605,21 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o ### Properties -| Name | Type | Required | Restrictions | Description | -| ----------------- | --------------------------------------------------------------------------- | -------- | ------------ | ----------- | -| `archived` | boolean | false | | | -| `created_at` | string | false | | | -| `created_by` | [codersdk.MinimalUser](#codersdkminimaluser) | false | | | -| `id` | string | false | | | -| `job` | [codersdk.ProvisionerJob](#codersdkprovisionerjob) | false | | | -| `message` | string | false | | | -| `name` | string | false | | | -| `organization_id` | string | false | | | -| `readme` | string | false | | | -| `template_id` | string | false | | | -| `updated_at` | string | false | | | -| `warnings` | array of [codersdk.TemplateVersionWarning](#codersdktemplateversionwarning) | false | | | +| Name | Type | Required | Restrictions | Description | +| ---------------------- | --------------------------------------------------------------------------- | -------- | ------------ | ----------- | +| `archived` | boolean | false | | | +| `created_at` | string | false | | | +| `created_by` | [codersdk.MinimalUser](#codersdkminimaluser) | false | | | +| `id` | string | false | | | +| `job` | [codersdk.ProvisionerJob](#codersdkprovisionerjob) | false | | | +| `matched_provisioners` | [codersdk.MatchedProvisioners](#codersdkmatchedprovisioners) | false | | | +| `message` | string | false | | | +| `name` | string | false | | | +| `organization_id` | string | false | | | +| `readme` | string | false | | | +| `template_id` | string | false | | | +| `updated_at` | string | false | | | +| `warnings` | array of [codersdk.TemplateVersionWarning](#codersdktemplateversionwarning) | false | | | ## codersdk.TemplateVersionExternalAuth diff --git a/docs/reference/api/templates.md b/docs/reference/api/templates.md index ceda61533ef5b..d7da209e94771 100644 --- a/docs/reference/api/templates.md +++ b/docs/reference/api/templates.md @@ -446,6 +446,11 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -517,6 +522,11 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -612,6 +622,11 @@ curl -X POST http://coder-server:8080/api/v2/organizations/{organization}/templa }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -1121,6 +1136,11 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions \ }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -1142,38 +1162,42 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions \ Status Code **200** -| Name | Type | Required | Restrictions | Description | -| -------------------- | ------------------------------------------------------------------------ | -------- | ------------ | ----------- | -| `[array item]` | array | false | | | -| `» archived` | boolean | false | | | -| `» created_at` | string(date-time) | false | | | -| `» created_by` | [codersdk.MinimalUser](schemas.md#codersdkminimaluser) | false | | | -| `»» avatar_url` | string(uri) | false | | | -| `»» id` | string(uuid) | true | | | -| `»» username` | string | true | | | -| `» id` | string(uuid) | false | | | -| `» job` | [codersdk.ProvisionerJob](schemas.md#codersdkprovisionerjob) | false | | | -| `»» canceled_at` | string(date-time) | false | | | -| `»» completed_at` | string(date-time) | false | | | -| `»» created_at` | string(date-time) | false | | | -| `»» error` | string | false | | | -| `»» error_code` | [codersdk.JobErrorCode](schemas.md#codersdkjoberrorcode) | false | | | -| `»» file_id` | string(uuid) | false | | | -| `»» id` | string(uuid) | false | | | -| `»» queue_position` | integer | false | | | -| `»» queue_size` | integer | false | | | -| `»» started_at` | string(date-time) | false | | | -| `»» status` | [codersdk.ProvisionerJobStatus](schemas.md#codersdkprovisionerjobstatus) | false | | | -| `»» tags` | object | false | | | -| `»»» [any property]` | string | false | | | -| `»» worker_id` | string(uuid) | false | | | -| `» message` | string | false | | | -| `» name` | string | false | | | -| `» organization_id` | string(uuid) | false | | | -| `» readme` | string | false | | | -| `» template_id` | string(uuid) | false | | | -| `» updated_at` | string(date-time) | false | | | -| `» warnings` | array | false | | | +| Name | Type | Required | Restrictions | Description | +| ------------------------ | ------------------------------------------------------------------------ | -------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `[array item]` | array | false | | | +| `» archived` | boolean | false | | | +| `» created_at` | string(date-time) | false | | | +| `» created_by` | [codersdk.MinimalUser](schemas.md#codersdkminimaluser) | false | | | +| `»» avatar_url` | string(uri) | false | | | +| `»» id` | string(uuid) | true | | | +| `»» username` | string | true | | | +| `» id` | string(uuid) | false | | | +| `» job` | [codersdk.ProvisionerJob](schemas.md#codersdkprovisionerjob) | false | | | +| `»» canceled_at` | string(date-time) | false | | | +| `»» completed_at` | string(date-time) | false | | | +| `»» created_at` | string(date-time) | false | | | +| `»» error` | string | false | | | +| `»» error_code` | [codersdk.JobErrorCode](schemas.md#codersdkjoberrorcode) | false | | | +| `»» file_id` | string(uuid) | false | | | +| `»» id` | string(uuid) | false | | | +| `»» queue_position` | integer | false | | | +| `»» queue_size` | integer | false | | | +| `»» started_at` | string(date-time) | false | | | +| `»» status` | [codersdk.ProvisionerJobStatus](schemas.md#codersdkprovisionerjobstatus) | false | | | +| `»» tags` | object | false | | | +| `»»» [any property]` | string | false | | | +| `»» worker_id` | string(uuid) | false | | | +| `» matched_provisioners` | [codersdk.MatchedProvisioners](schemas.md#codersdkmatchedprovisioners) | false | | | +| `»» available` | integer | false | | Available is the number of provisioner daemons that are available to take jobs. This may be less than the count if some provisioners are busy or have been stopped. | +| `»» count` | integer | false | | Count is the number of provisioner daemons that matched the given tags. If the count is 0, it means no provisioner daemons matched the requested tags. | +| `»» most_recently_seen` | string(date-time) | false | | Most recently seen is the most recently seen time of the set of matched provisioners. If no provisioners matched, this field will be null. | +| `» message` | string | false | | | +| `» name` | string | false | | | +| `» organization_id` | string(uuid) | false | | | +| `» readme` | string | false | | | +| `» template_id` | string(uuid) | false | | | +| `» updated_at` | string(date-time) | false | | | +| `» warnings` | array | false | | | #### Enumerated Values @@ -1350,6 +1374,11 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions/{templ }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -1371,38 +1400,42 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions/{templ Status Code **200** -| Name | Type | Required | Restrictions | Description | -| -------------------- | ------------------------------------------------------------------------ | -------- | ------------ | ----------- | -| `[array item]` | array | false | | | -| `» archived` | boolean | false | | | -| `» created_at` | string(date-time) | false | | | -| `» created_by` | [codersdk.MinimalUser](schemas.md#codersdkminimaluser) | false | | | -| `»» avatar_url` | string(uri) | false | | | -| `»» id` | string(uuid) | true | | | -| `»» username` | string | true | | | -| `» id` | string(uuid) | false | | | -| `» job` | [codersdk.ProvisionerJob](schemas.md#codersdkprovisionerjob) | false | | | -| `»» canceled_at` | string(date-time) | false | | | -| `»» completed_at` | string(date-time) | false | | | -| `»» created_at` | string(date-time) | false | | | -| `»» error` | string | false | | | -| `»» error_code` | [codersdk.JobErrorCode](schemas.md#codersdkjoberrorcode) | false | | | -| `»» file_id` | string(uuid) | false | | | -| `»» id` | string(uuid) | false | | | -| `»» queue_position` | integer | false | | | -| `»» queue_size` | integer | false | | | -| `»» started_at` | string(date-time) | false | | | -| `»» status` | [codersdk.ProvisionerJobStatus](schemas.md#codersdkprovisionerjobstatus) | false | | | -| `»» tags` | object | false | | | -| `»»» [any property]` | string | false | | | -| `»» worker_id` | string(uuid) | false | | | -| `» message` | string | false | | | -| `» name` | string | false | | | -| `» organization_id` | string(uuid) | false | | | -| `» readme` | string | false | | | -| `» template_id` | string(uuid) | false | | | -| `» updated_at` | string(date-time) | false | | | -| `» warnings` | array | false | | | +| Name | Type | Required | Restrictions | Description | +| ------------------------ | ------------------------------------------------------------------------ | -------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `[array item]` | array | false | | | +| `» archived` | boolean | false | | | +| `» created_at` | string(date-time) | false | | | +| `» created_by` | [codersdk.MinimalUser](schemas.md#codersdkminimaluser) | false | | | +| `»» avatar_url` | string(uri) | false | | | +| `»» id` | string(uuid) | true | | | +| `»» username` | string | true | | | +| `» id` | string(uuid) | false | | | +| `» job` | [codersdk.ProvisionerJob](schemas.md#codersdkprovisionerjob) | false | | | +| `»» canceled_at` | string(date-time) | false | | | +| `»» completed_at` | string(date-time) | false | | | +| `»» created_at` | string(date-time) | false | | | +| `»» error` | string | false | | | +| `»» error_code` | [codersdk.JobErrorCode](schemas.md#codersdkjoberrorcode) | false | | | +| `»» file_id` | string(uuid) | false | | | +| `»» id` | string(uuid) | false | | | +| `»» queue_position` | integer | false | | | +| `»» queue_size` | integer | false | | | +| `»» started_at` | string(date-time) | false | | | +| `»» status` | [codersdk.ProvisionerJobStatus](schemas.md#codersdkprovisionerjobstatus) | false | | | +| `»» tags` | object | false | | | +| `»»» [any property]` | string | false | | | +| `»» worker_id` | string(uuid) | false | | | +| `» matched_provisioners` | [codersdk.MatchedProvisioners](schemas.md#codersdkmatchedprovisioners) | false | | | +| `»» available` | integer | false | | Available is the number of provisioner daemons that are available to take jobs. This may be less than the count if some provisioners are busy or have been stopped. | +| `»» count` | integer | false | | Count is the number of provisioner daemons that matched the given tags. If the count is 0, it means no provisioner daemons matched the requested tags. | +| `»» most_recently_seen` | string(date-time) | false | | Most recently seen is the most recently seen time of the set of matched provisioners. If no provisioners matched, this field will be null. | +| `» message` | string | false | | | +| `» name` | string | false | | | +| `» organization_id` | string(uuid) | false | | | +| `» readme` | string | false | | | +| `» template_id` | string(uuid) | false | | | +| `» updated_at` | string(date-time) | false | | | +| `» warnings` | array | false | | | #### Enumerated Values @@ -1469,6 +1502,11 @@ curl -X GET http://coder-server:8080/api/v2/templateversions/{templateversion} \ }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -1549,6 +1587,11 @@ curl -X PATCH http://coder-server:8080/api/v2/templateversions/{templateversion} }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index bd00dbda353c3..45186586df98d 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -758,6 +758,13 @@ export interface LoginWithPasswordResponse { readonly session_token: string; } +// From codersdk/provisionerdaemons.go +export interface MatchedProvisioners { + readonly count: number; + readonly available: number; + readonly most_recently_seen?: string; +} + // From codersdk/organizations.go export interface MinimalOrganization { readonly id: string; @@ -1463,6 +1470,7 @@ export interface TemplateVersion { readonly created_by: MinimalUser; readonly archived: boolean; readonly warnings?: Readonly>; + readonly matched_provisioners?: MatchedProvisioners; } // From codersdk/templateversions.go From 56315777e0910fcd1f82bb7f2ba6e1d157c94382 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 17:22:15 +0000 Subject: [PATCH 16/18] update cli template push --- cli/templatepush.go | 40 ++++------------------------------------ cli/templatepush_test.go | 2 ++ 2 files changed, 6 insertions(+), 36 deletions(-) diff --git a/cli/templatepush.go b/cli/templatepush.go index 35b5156af476e..e4698bd0a8c8c 100644 --- a/cli/templatepush.go +++ b/cli/templatepush.go @@ -2,7 +2,6 @@ package cli import ( "bufio" - "context" "encoding/json" "errors" "fmt" @@ -419,30 +418,22 @@ func createValidTemplateVersion(inv *serpent.Invocation, args createValidTemplat } var tagsJSON strings.Builder _ = json.NewEncoder(&tagsJSON).Encode(version.Job.Tags) - foundProvisioners, activeProvisioners, err := checkProvisioners(inv.Context(), client, args.Organization.ID, version.Job.Tags) - if err != nil { - var apiErr *codersdk.Error - // Unfortunately this is an enterprise endpoint, so check for that first. - if errors.As(err, &apiErr) && apiErr.StatusCode() != http.StatusNotFound { - cliui.Warnf(inv.Stderr, "Unable to check for available provisioners: %s", err.Error()) - } - } - - if foundProvisioners == 0 { + if version.MatchedProvisioners.Count == 0 { cliui.Warnf(inv.Stderr, `No provisioners are available to handle the job! Please contact your deployment administrator for assistance. Details: Provisioner Job ID : %s Requested tags : %s `, version.Job.ID, tagsJSON.String()) - } else if activeProvisioners == 0 { + } else if version.MatchedProvisioners.Available == 0 { cliui.Warnf(inv.Stderr, `All available provisioner daemons have been silent for a while. Your build will proceed once they become available. If this persists, please contact your deployment administrator for assistance. Details: Provisioner Job ID : %s Requested tags : %s -`, version.Job.ID, tagsJSON.String()) + Most Recently Seen : %s +`, version.Job.ID, tagsJSON.String(), version.MatchedProvisioners.MostRecentlySeen.Time) } err = cliui.ProvisionerJob(inv.Context(), inv.Stdout, cliui.ProvisionerJobOptions{ @@ -526,26 +517,3 @@ func prettyDirectoryPath(dir string) string { } return prettyDir } - -func checkProvisioners(ctx context.Context, client *codersdk.Client, orgID uuid.UUID, wantTags map[string]string) (found, active int, err error) { - // Check for eligible provisioners. This allows us to return a warning to the user if they - // submit a job for which no provisioner is available. - provisioners, err := client.OrganizationProvisionerDaemons(ctx, orgID, wantTags) - if err != nil { - return -1, -1, xerrors.Errorf("get organization provisioner daemons: %w", err) - } - - oneMinuteAgo := time.Now().Add(-time.Minute) - for _, provisioner := range provisioners { - if !provisioner.LastSeenAt.Valid { - continue - } - found++ - if provisioner.LastSeenAt.Time.Before(oneMinuteAgo) { - continue - } - active++ - } - - return found, active, nil -} diff --git a/cli/templatepush_test.go b/cli/templatepush_test.go index d7b055468072c..a20e3070740a8 100644 --- a/cli/templatepush_test.go +++ b/cli/templatepush_test.go @@ -488,6 +488,8 @@ data "coder_workspace_tags" "tags" { cancel() <-done + + require.Contains(t, stderr.String(), "No provisioners are available to handle the job!") }) t.Run("ChangeTags", func(t *testing.T) { From 58fae9718fecc60005e3d306338db834ee77a83b Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Nov 2024 17:34:22 +0000 Subject: [PATCH 17/18] cleanup and error handling --- cli/templatepush.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cli/templatepush.go b/cli/templatepush.go index e4698bd0a8c8c..6ccce0168e70a 100644 --- a/cli/templatepush.go +++ b/cli/templatepush.go @@ -417,7 +417,11 @@ func createValidTemplateVersion(inv *serpent.Invocation, args createValidTemplat return nil, err } var tagsJSON strings.Builder - _ = json.NewEncoder(&tagsJSON).Encode(version.Job.Tags) + if err := json.NewEncoder(&tagsJSON).Encode(version.Job.Tags); err != nil { + // Fall back to the less-pretty string representation. + tagsJSON.Reset() + _, _ = tagsJSON.WriteString(fmt.Sprintf("%v", version.Job.Tags)) + } if version.MatchedProvisioners.Count == 0 { cliui.Warnf(inv.Stderr, `No provisioners are available to handle the job! Please contact your deployment administrator for assistance. @@ -433,7 +437,7 @@ Details: Provisioner Job ID : %s Requested tags : %s Most Recently Seen : %s -`, version.Job.ID, tagsJSON.String(), version.MatchedProvisioners.MostRecentlySeen.Time) +`, version.Job.ID, strings.TrimSpace(tagsJSON.String()), version.MatchedProvisioners.MostRecentlySeen.Time) } err = cliui.ProvisionerJob(inv.Context(), inv.Stdout, cliui.ProvisionerJobOptions{ From 90991bd2d00d17e39192c34ef6e65883c21f98ff Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 25 Nov 2024 11:08:54 +0000 Subject: [PATCH 18/18] Apply suggestions from code review Co-authored-by: Mathias Fredriksson --- cli/templatepush.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/templatepush.go b/cli/templatepush.go index 6ccce0168e70a..8516d7f9c1310 100644 --- a/cli/templatepush.go +++ b/cli/templatepush.go @@ -426,7 +426,7 @@ func createValidTemplateVersion(inv *serpent.Invocation, args createValidTemplat cliui.Warnf(inv.Stderr, `No provisioners are available to handle the job! Please contact your deployment administrator for assistance. Details: - Provisioner Job ID : %s + Provisioner job ID : %s Requested tags : %s `, version.Job.ID, tagsJSON.String()) } else if version.MatchedProvisioners.Available == 0 { @@ -434,9 +434,9 @@ Details: Your build will proceed once they become available. If this persists, please contact your deployment administrator for assistance. Details: - Provisioner Job ID : %s + Provisioner job ID : %s Requested tags : %s - Most Recently Seen : %s + Most recently seen : %s `, version.Job.ID, strings.TrimSpace(tagsJSON.String()), version.MatchedProvisioners.MostRecentlySeen.Time) } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy