From 2b236e81f419a917ef4c15773f959853c8f682cb Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Wed, 23 Oct 2024 10:45:54 +0000 Subject: [PATCH 01/11] feat: notify users on template deprecation --- coderd/database/dbauthz/dbauthz.go | 10 +++++ coderd/database/dbauthz/dbauthz_test.go | 4 ++ coderd/database/dbmem/dbmem.go | 27 +++++++++++++ coderd/database/dbmetrics/dbmetrics.go | 7 ++++ coderd/database/dbmock/dbmock.go | 15 ++++++++ ...template_deprecation_notification.down.sql | 1 + ...9_template_deprecation_notification.up.sql | 13 +++++++ coderd/database/querier.go | 1 + coderd/database/queries.sql.go | 38 +++++++++++++++++++ coderd/database/queries/templates.sql | 15 ++++++++ coderd/notifications/events.go | 3 +- coderd/templates.go | 34 +++++++++++++++++ 12 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 coderd/database/migrations/000269_template_deprecation_notification.down.sql create mode 100644 coderd/database/migrations/000269_template_deprecation_notification.up.sql diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 052f25450e6a5..99c4abcd0d27d 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2339,6 +2339,16 @@ func (q *querier) GetUsersByIDs(ctx context.Context, ids []uuid.UUID) ([]databas return q.db.GetUsersByIDs(ctx, ids) } +func (q *querier) GetUsersWithAccessToTemplateByID(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) { + // Ensure we have permission to access this template. + _, err := q.GetTemplateByID(ctx, id) + if err != nil { + return nil, err + } + + return q.db.GetUsersWithAccessToTemplateByID(ctx, id) +} + func (q *querier) GetWorkspaceAgentAndLatestBuildByAuthToken(ctx context.Context, authToken uuid.UUID) (database.GetWorkspaceAgentAndLatestBuildByAuthTokenRow, error) { // This is a system function if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 6a34e88104ce1..4032b41526573 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1131,6 +1131,10 @@ func (s *MethodTestSuite) TestUser() { Asserts(a, policy.ActionRead, b, policy.ActionRead). Returns(slice.New(a, b)) })) + s.Run("GetUsersWithAccessToTemplateByID", s.Subtest(func(db database.Store, check *expects) { + a := dbgen.Template(s.T(), db, database.Template{}) + check.Args(a.ID).Asserts(a, policy.ActionRead) + })) s.Run("GetUsers", s.Subtest(func(db database.Store, check *expects) { dbgen.User(s.T(), db, database.User{Username: "GetUsers-a-user"}) dbgen.User(s.T(), db, database.User{Username: "GetUsers-b-user"}) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 24498d88c9dbc..5dabfa3258ebf 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -5669,6 +5669,33 @@ func (q *FakeQuerier) GetUsersByIDs(_ context.Context, ids []uuid.UUID) ([]datab return users, nil } +func (q *FakeQuerier) GetUsersWithAccessToTemplateByID(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + groups := make(map[string]bool, 0) + for _, template := range q.templates { + if template.ID != id { + continue + } + + for group := range template.GroupACL { + groups[group] = true + } + } + + users := make([]uuid.UUID, 0) + for _, member := range q.organizationMembers { + if _, ok := groups[member.OrganizationID.String()]; !ok { + continue + } + + users = append(users, member.UserID) + } + + return users, nil +} + func (q *FakeQuerier) GetWorkspaceAgentAndLatestBuildByAuthToken(_ context.Context, authToken uuid.UUID) (database.GetWorkspaceAgentAndLatestBuildByAuthTokenRow, error) { q.mutex.RLock() defer q.mutex.RUnlock() diff --git a/coderd/database/dbmetrics/dbmetrics.go b/coderd/database/dbmetrics/dbmetrics.go index c3e9de22fb0d8..79f5dbb7d7f30 100644 --- a/coderd/database/dbmetrics/dbmetrics.go +++ b/coderd/database/dbmetrics/dbmetrics.go @@ -1348,6 +1348,13 @@ func (m metricsStore) GetUsersByIDs(ctx context.Context, ids []uuid.UUID) ([]dat return users, err } +func (m metricsStore) GetUsersWithAccessToTemplateByID(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) { + start := time.Now() + r0, r1 := m.s.GetUsersWithAccessToTemplateByID(ctx, id) + m.queryLatencies.WithLabelValues("GetUsersWithAccessToTemplateByID").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m metricsStore) GetWorkspaceAgentAndLatestBuildByAuthToken(ctx context.Context, authToken uuid.UUID) (database.GetWorkspaceAgentAndLatestBuildByAuthTokenRow, error) { start := time.Now() r0, r1 := m.s.GetWorkspaceAgentAndLatestBuildByAuthToken(ctx, authToken) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index b3c7b9e7615d3..d2a4b5b2df8fe 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2813,6 +2813,21 @@ func (mr *MockStoreMockRecorder) GetUsersByIDs(arg0, arg1 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUsersByIDs", reflect.TypeOf((*MockStore)(nil).GetUsersByIDs), arg0, arg1) } +// GetUsersWithAccessToTemplateByID mocks base method. +func (m *MockStore) GetUsersWithAccessToTemplateByID(arg0 context.Context, arg1 uuid.UUID) ([]uuid.UUID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUsersWithAccessToTemplateByID", arg0, arg1) + ret0, _ := ret[0].([]uuid.UUID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetUsersWithAccessToTemplateByID indicates an expected call of GetUsersWithAccessToTemplateByID. +func (mr *MockStoreMockRecorder) GetUsersWithAccessToTemplateByID(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUsersWithAccessToTemplateByID", reflect.TypeOf((*MockStore)(nil).GetUsersWithAccessToTemplateByID), arg0, arg1) +} + // GetWorkspaceAgentAndLatestBuildByAuthToken mocks base method. func (m *MockStore) GetWorkspaceAgentAndLatestBuildByAuthToken(arg0 context.Context, arg1 uuid.UUID) (database.GetWorkspaceAgentAndLatestBuildByAuthTokenRow, error) { m.ctrl.T.Helper() diff --git a/coderd/database/migrations/000269_template_deprecation_notification.down.sql b/coderd/database/migrations/000269_template_deprecation_notification.down.sql new file mode 100644 index 0000000000000..b3f9abc0133bd --- /dev/null +++ b/coderd/database/migrations/000269_template_deprecation_notification.down.sql @@ -0,0 +1 @@ +DELETE FROM notification_templates WHERE id = 'f40fae84-55a2-42cd-99fa-b41c1ca64894'; diff --git a/coderd/database/migrations/000269_template_deprecation_notification.up.sql b/coderd/database/migrations/000269_template_deprecation_notification.up.sql new file mode 100644 index 0000000000000..cb8dcf4272571 --- /dev/null +++ b/coderd/database/migrations/000269_template_deprecation_notification.up.sql @@ -0,0 +1,13 @@ +INSERT INTO notification_templates + (id, name, title_template, body_template, "group", actions) +VALUES ( + 'f40fae84-55a2-42cd-99fa-b41c1ca64894', + 'Template Deprecated', + E'Template **{{.Labels.template}}** has been deprecated', + E'Hello {{.UserName}},\n\n'|| + E'The template **{{.Labels.template}}** has been deprecated with the following message:\n\n' || + E'**{{.Labels.message}}**\n\n' || + E'New workspaces may not be created from this template. Existing workspaces will continue to function normally.', + 'Template Events', + '[]'::jsonb +); diff --git a/coderd/database/querier.go b/coderd/database/querier.go index fcb58a7d6e305..1e8f6909b1d86 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -290,6 +290,7 @@ type sqlcQuerier interface { // to look up references to actions. eg. a user could build a workspace // for another user, then be deleted... we still want them to appear! GetUsersByIDs(ctx context.Context, ids []uuid.UUID) ([]User, error) + GetUsersWithAccessToTemplateByID(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) GetWorkspaceAgentAndLatestBuildByAuthToken(ctx context.Context, authToken uuid.UUID) (GetWorkspaceAgentAndLatestBuildByAuthTokenRow, error) GetWorkspaceAgentByID(ctx context.Context, id uuid.UUID) (WorkspaceAgent, error) GetWorkspaceAgentByInstanceID(ctx context.Context, authInstanceID string) (WorkspaceAgent, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 45cbef3f5e1d8..f0a290d2a2f8b 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8557,6 +8557,44 @@ func (q *sqlQuerier) GetTemplatesWithFilter(ctx context.Context, arg GetTemplate return items, nil } +const getUsersWithAccessToTemplateByID = `-- name: GetUsersWithAccessToTemplateByID :many +SELECT + user_id +FROM + organization_members +WHERE + organization_members.organization_id::text IN ( + SELECT + jsonb_object_keys(group_acl) + FROM + templates + WHERE templates.id = $1 + ) +` + +func (q *sqlQuerier) GetUsersWithAccessToTemplateByID(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) { + rows, err := q.db.QueryContext(ctx, getUsersWithAccessToTemplateByID, id) + if err != nil { + return nil, err + } + defer rows.Close() + var items []uuid.UUID + for rows.Next() { + var user_id uuid.UUID + if err := rows.Scan(&user_id); err != nil { + return nil, err + } + items = append(items, user_id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertTemplate = `-- name: InsertTemplate :exec INSERT INTO templates ( diff --git a/coderd/database/queries/templates.sql b/coderd/database/queries/templates.sql index 84df9633a1a53..eb270d4d491a6 100644 --- a/coderd/database/queries/templates.sql +++ b/coderd/database/queries/templates.sql @@ -199,3 +199,18 @@ SET WHERE id = $1 ; + +-- name: GetUsersWithAccessToTemplateByID :many +SELECT + user_id +FROM + organization_members +WHERE + organization_members.organization_id::text IN ( + SELECT + jsonb_object_keys(group_acl) + FROM + templates + WHERE templates.id = $1 + ) +; diff --git a/coderd/notifications/events.go b/coderd/notifications/events.go index c2e0f442e0623..e33a85b523db2 100644 --- a/coderd/notifications/events.go +++ b/coderd/notifications/events.go @@ -30,7 +30,8 @@ var ( // Template-related events. var ( - TemplateTemplateDeleted = uuid.MustParse("29a09665-2a4c-403f-9648-54301670e7be") + TemplateTemplateDeleted = uuid.MustParse("29a09665-2a4c-403f-9648-54301670e7be") + TemplateTemplateDeprecated = uuid.MustParse("f40fae84-55a2-42cd-99fa-b41c1ca64894") TemplateWorkspaceBuildsFailedReport = uuid.MustParse("34a20db2-e9cc-4a93-b0e4-8569699d7a00") ) diff --git a/coderd/templates.go b/coderd/templates.go index 907a4d1265836..4ea0196d4fced 100644 --- a/coderd/templates.go +++ b/coderd/templates.go @@ -845,6 +845,12 @@ func (api *API) patchTemplateMeta(rw http.ResponseWriter, r *http.Request) { return } + if template.Deprecated != updated.Deprecated && updated.Deprecated != "" { + if err := api.notifyUsersOfTemplateDeprecation(ctx, updated); err != nil { + api.Logger.Error(ctx, "failed to notify users of template deprecation", slog.Error(err)) + } + } + if updated.UpdatedAt.IsZero() { aReq.New = template rw.WriteHeader(http.StatusNotModified) @@ -855,6 +861,34 @@ func (api *API) patchTemplateMeta(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusOK, api.convertTemplate(updated)) } +func (api *API) notifyUsersOfTemplateDeprecation(ctx context.Context, template database.Template) error { + users, err := api.Database.GetUsersWithAccessToTemplateByID(ctx, template.ID) + if err != nil { + return xerrors.Errorf("get users with access to template by id: %w", err) + } + + errs := []error{} + + for _, userID := range users { + _, err = api.NotificationsEnqueuer.Enqueue( + //nolint:gocritic // We need the system auth context to be able to send the deprecation notification. + dbauthz.AsSystemRestricted(ctx), + userID, + notifications.TemplateTemplateDeprecated, + map[string]string{ + "template": template.Name, + "message": template.Deprecated, + }, + "notify-users-of-template-deprecation", + ) + if err != nil { + errs = append(errs, xerrors.Errorf("enqueue notification: %w", err)) + } + } + + return errors.Join(errs...) +} + // @Summary Get template DAUs by ID // @ID get-template-daus-by-id // @Security CoderSessionToken From 640138acc07fdaf124ad6fce0cb3b30952d53cb4 Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Wed, 23 Oct 2024 12:30:49 +0000 Subject: [PATCH 02/11] test: test behaviour --- coderd/database/dbmem/dbmem.go | 4 ++-- enterprise/coderd/templates_test.go | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 5dabfa3258ebf..17115af3214c9 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -5673,14 +5673,14 @@ func (q *FakeQuerier) GetUsersWithAccessToTemplateByID(ctx context.Context, id u q.mutex.RLock() defer q.mutex.RUnlock() - groups := make(map[string]bool, 0) + groups := make(map[string]struct{}, 0) for _, template := range q.templates { if template.ID != id { continue } for group := range template.GroupACL { - groups[group] = true + groups[group] = struct{}{} } } diff --git a/enterprise/coderd/templates_test.go b/enterprise/coderd/templates_test.go index 5d9cb8ee9fa35..290fd4454b941 100644 --- a/enterprise/coderd/templates_test.go +++ b/enterprise/coderd/templates_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "net/http" + "slices" "testing" "time" @@ -38,9 +39,11 @@ func TestTemplates(t *testing.T) { t.Run("Deprecated", func(t *testing.T) { t.Parallel() + notifyEnq := &testutil.FakeNotificationsEnqueuer{} owner, user := coderdenttest.New(t, &coderdenttest.Options{ Options: &coderdtest.Options{ IncludeProvisionerDaemon: true, + NotificationsEnqueuer: notifyEnq, }, LicenseOptions: &coderdenttest.LicenseOptions{ Features: license.Features{ @@ -48,7 +51,7 @@ func TestTemplates(t *testing.T) { }, }, }) - client, _ := coderdtest.CreateAnotherUser(t, owner, user.OrganizationID, rbac.RoleTemplateAdmin()) + client, anotherUser := coderdtest.CreateAnotherUser(t, owner, user.OrganizationID, rbac.RoleTemplateAdmin()) version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil) template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) @@ -65,6 +68,25 @@ func TestTemplates(t *testing.T) { assert.True(t, updated.Deprecated) assert.NotEmpty(t, updated.DeprecationMessage) + notifs := []*testutil.Notification{} + for _, notif := range notifyEnq.Sent { + if notif.TemplateID == notifications.TemplateTemplateDeprecated { + notifs = append(notifs, notif) + } + } + require.Equal(t, 2, len(notifs)) + + expectedSentTo := []string{user.UserID.String(), anotherUser.ID.String()} + slices.Sort(expectedSentTo) + + sentTo := []string{} + for _, notif := range notifs { + sentTo = append(sentTo, notif.UserID.String()) + } + slices.Sort(sentTo) + + assert.Equal(t, expectedSentTo, sentTo) + _, err = client.CreateWorkspace(ctx, user.OrganizationID, codersdk.Me, codersdk.CreateWorkspaceRequest{ TemplateID: template.ID, Name: "foobar", From 22ec59dd1a07d03da5481cd040e78a569b1a0530 Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Wed, 23 Oct 2024 13:05:20 +0000 Subject: [PATCH 03/11] fix: ci --- coderd/database/dbmem/dbmem.go | 2 +- ...wn.sql => 000270_template_deprecation_notification.down.sql} | 0 ...n.up.sql => 000270_template_deprecation_notification.up.sql} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename coderd/database/migrations/{000269_template_deprecation_notification.down.sql => 000270_template_deprecation_notification.down.sql} (100%) rename coderd/database/migrations/{000269_template_deprecation_notification.up.sql => 000270_template_deprecation_notification.up.sql} (100%) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 17115af3214c9..1ac9074006949 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -5669,7 +5669,7 @@ func (q *FakeQuerier) GetUsersByIDs(_ context.Context, ids []uuid.UUID) ([]datab return users, nil } -func (q *FakeQuerier) GetUsersWithAccessToTemplateByID(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) { +func (q *FakeQuerier) GetUsersWithAccessToTemplateByID(_ context.Context, id uuid.UUID) ([]uuid.UUID, error) { q.mutex.RLock() defer q.mutex.RUnlock() diff --git a/coderd/database/migrations/000269_template_deprecation_notification.down.sql b/coderd/database/migrations/000270_template_deprecation_notification.down.sql similarity index 100% rename from coderd/database/migrations/000269_template_deprecation_notification.down.sql rename to coderd/database/migrations/000270_template_deprecation_notification.down.sql diff --git a/coderd/database/migrations/000269_template_deprecation_notification.up.sql b/coderd/database/migrations/000270_template_deprecation_notification.up.sql similarity index 100% rename from coderd/database/migrations/000269_template_deprecation_notification.up.sql rename to coderd/database/migrations/000270_template_deprecation_notification.up.sql From cd97285cdd0e9eea6f515ba8141ec24058a0b2bc Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Wed, 23 Oct 2024 14:41:18 +0000 Subject: [PATCH 04/11] feat: add CTAs --- .../000270_template_deprecation_notification.up.sql | 13 +++++++++++-- coderd/templates.go | 5 +++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/coderd/database/migrations/000270_template_deprecation_notification.up.sql b/coderd/database/migrations/000270_template_deprecation_notification.up.sql index cb8dcf4272571..1a289f448f5d0 100644 --- a/coderd/database/migrations/000270_template_deprecation_notification.up.sql +++ b/coderd/database/migrations/000270_template_deprecation_notification.up.sql @@ -3,11 +3,20 @@ INSERT INTO notification_templates VALUES ( 'f40fae84-55a2-42cd-99fa-b41c1ca64894', 'Template Deprecated', - E'Template **{{.Labels.template}}** has been deprecated', + E'Template ''{{.Labels.template}}'' has been deprecated', E'Hello {{.UserName}},\n\n'|| E'The template **{{.Labels.template}}** has been deprecated with the following message:\n\n' || E'**{{.Labels.message}}**\n\n' || E'New workspaces may not be created from this template. Existing workspaces will continue to function normally.', 'Template Events', - '[]'::jsonb + '[ + { + "label": "See workspaces", + "url": "{{base_url}}/workspaces?filter=owner%3Ame+template%3A{{.Labels.template}}" + }, + { + "label": "View template", + "url": "{{base_url}}/templates/{{.Labels.organization}}/{{.Labels.template}}" + } + ]'::jsonb ); diff --git a/coderd/templates.go b/coderd/templates.go index 4ea0196d4fced..14dced44a68ae 100644 --- a/coderd/templates.go +++ b/coderd/templates.go @@ -876,8 +876,9 @@ func (api *API) notifyUsersOfTemplateDeprecation(ctx context.Context, template d userID, notifications.TemplateTemplateDeprecated, map[string]string{ - "template": template.Name, - "message": template.Deprecated, + "template": template.Name, + "message": template.Deprecated, + "organization": template.OrganizationName, }, "notify-users-of-template-deprecation", ) From 5732b2656ae5ba4b6402596b399e6cddb8aa74be Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Wed, 23 Oct 2024 15:53:00 +0000 Subject: [PATCH 05/11] test: add rendered-template test --- coderd/notifications/notifications_test.go | 14 +++ .../TemplateTemplateDeprecated.html.golden | 98 +++++++++++++++++++ .../TemplateTemplateDeprecated.json.golden | 33 +++++++ 3 files changed, 145 insertions(+) create mode 100644 coderd/notifications/testdata/rendered-templates/smtp/TemplateTemplateDeprecated.html.golden create mode 100644 coderd/notifications/testdata/rendered-templates/webhook/TemplateTemplateDeprecated.json.golden diff --git a/coderd/notifications/notifications_test.go b/coderd/notifications/notifications_test.go index 4a6978b5024fe..86ed14fe90957 100644 --- a/coderd/notifications/notifications_test.go +++ b/coderd/notifications/notifications_test.go @@ -1021,6 +1021,20 @@ func TestNotificationTemplates_Golden(t *testing.T) { appName: "Custom Application Name", logoURL: "https://custom.application/logo.png", }, + { + name: "TemplateTemplateDeprecated", + id: notifications.TemplateTemplateDeprecated, + payload: types.MessagePayload{ + UserName: "Bobby", + UserEmail: "bobby@coder.com", + UserUsername: "bobby", + Labels: map[string]string{ + "template": "alpha", + "message": "This template has been replaced by beta", + "organization": "coder", + }, + }, + }, } // We must have a test case for every notification_template. This is enforced below: diff --git a/coderd/notifications/testdata/rendered-templates/smtp/TemplateTemplateDeprecated.html.golden b/coderd/notifications/testdata/rendered-templates/smtp/TemplateTemplateDeprecated.html.golden new file mode 100644 index 0000000000000..b627c6f8aafcf --- /dev/null +++ b/coderd/notifications/testdata/rendered-templates/smtp/TemplateTemplateDeprecated.html.golden @@ -0,0 +1,98 @@ +From: system@coder.com +To: bobby@coder.com +Subject: Template 'alpha' has been deprecated +Message-Id: 02ee4935-73be-4fa1-a290-ff9999026b13@blush-whale-48 +Date: Fri, 11 Oct 2024 09:03:06 +0000 +Content-Type: multipart/alternative; boundary=bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4 +MIME-Version: 1.0 + +--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4 +Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset=UTF-8 + +Hello Bobby, + +The template alpha has been deprecated with the following message: + +This template has been replaced by beta + +New workspaces may not be created from this template. Existing workspaces w= +ill continue to function normally. + + +See workspaces: http://test.com/workspaces?filter=3Downer%3Ame+template%3Aa= +lpha + +View template: http://test.com/templates/coder/alpha + +--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4 +Content-Transfer-Encoding: quoted-printable +Content-Type: text/html; charset=UTF-8 + + + + + + + Template 'alpha' has been deprecated + + +
+
+ 3D"Cod= +
+

+ Template 'alpha' has been deprecated +

+
+

Hello Bobby,

+ +

The template alpha has been deprecated with the followi= +ng message:

+ +

This template has been replaced by beta

+ +

New workspaces may not be created from this template. Existing workspace= +s will continue to function normally.

+
+ + +
+ + + +--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4-- diff --git a/coderd/notifications/testdata/rendered-templates/webhook/TemplateTemplateDeprecated.json.golden b/coderd/notifications/testdata/rendered-templates/webhook/TemplateTemplateDeprecated.json.golden new file mode 100644 index 0000000000000..206cbba5e1abb --- /dev/null +++ b/coderd/notifications/testdata/rendered-templates/webhook/TemplateTemplateDeprecated.json.golden @@ -0,0 +1,33 @@ +{ + "_version": "1.1", + "msg_id": "00000000-0000-0000-0000-000000000000", + "payload": { + "_version": "1.1", + "notification_name": "Template Deprecated", + "notification_template_id": "00000000-0000-0000-0000-000000000000", + "user_id": "00000000-0000-0000-0000-000000000000", + "user_email": "bobby@coder.com", + "user_name": "Bobby", + "user_username": "bobby", + "actions": [ + { + "label": "See workspaces", + "url": "http://test.com/workspaces?filter=owner%3Ame+template%3Aalpha" + }, + { + "label": "View template", + "url": "http://test.com/templates/coder/alpha" + } + ], + "labels": { + "message": "This template has been replaced by beta", + "organization": "coder", + "template": "alpha" + }, + "data": null + }, + "title": "Template 'alpha' has been deprecated", + "title_markdown": "Template 'alpha' has been deprecated", + "body": "Hello Bobby,\n\nThe template alpha has been deprecated with the following message:\n\nThis template has been replaced by beta\n\nNew workspaces may not be created from this template. Existing workspaces will continue to function normally.", + "body_markdown": "Hello Bobby,\n\nThe template **alpha** has been deprecated with the following message:\n\n**This template has been replaced by beta**\n\nNew workspaces may not be created from this template. Existing workspaces will continue to function normally." +} \ No newline at end of file From a9717121fa56d441fc6f9498d07feacd0644562f Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Thu, 24 Oct 2024 09:31:48 +0000 Subject: [PATCH 06/11] test: ensure notification goes to correct users --- enterprise/coderd/templates_test.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/enterprise/coderd/templates_test.go b/enterprise/coderd/templates_test.go index 290fd4454b941..1d0cb13306fe8 100644 --- a/enterprise/coderd/templates_test.go +++ b/enterprise/coderd/templates_test.go @@ -47,7 +47,8 @@ func TestTemplates(t *testing.T) { }, LicenseOptions: &coderdenttest.LicenseOptions{ Features: license.Features{ - codersdk.FeatureAccessControl: 1, + codersdk.FeatureAccessControl: 1, + codersdk.FeatureMultipleOrganizations: 1, }, }, }) @@ -56,6 +57,9 @@ func TestTemplates(t *testing.T) { template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + org := coderdenttest.CreateOrganization(t, owner, coderdenttest.CreateOrganizationOptions{}) + _, thirdUser := coderdtest.CreateAnotherUser(t, owner, org.ID, rbac.RoleTemplateAdmin()) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() @@ -85,8 +89,15 @@ func TestTemplates(t *testing.T) { } slices.Sort(sentTo) + // Require the notification to have only been sent to the expected users assert.Equal(t, expectedSentTo, sentTo) + // The previous check should verify this but we're double checking that + // the notification wasn't sent to a user in another org. + for _, notif := range notifs { + assert.NotEqual(t, thirdUser.ID, notif.UserID) + } + _, err = client.CreateWorkspace(ctx, user.OrganizationID, codersdk.Me, codersdk.CreateWorkspaceRequest{ TemplateID: template.ID, Name: "foobar", From 800adadb1f959ab7dcb57104bc5104381f5b00d4 Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Thu, 24 Oct 2024 10:34:59 +0000 Subject: [PATCH 07/11] refactor: only notify users of template --- coderd/database/dbauthz/dbauthz.go | 10 ------- coderd/database/dbmem/dbmem.go | 27 ------------------ coderd/database/dbmetrics/dbmetrics.go | 7 ----- coderd/database/dbmock/dbmock.go | 15 ---------- coderd/database/querier.go | 1 - coderd/database/queries.sql.go | 38 -------------------------- coderd/database/queries/templates.sql | 15 ---------- coderd/templates.go | 13 +++++++-- enterprise/coderd/templates_test.go | 14 +++++----- 9 files changed, 17 insertions(+), 123 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 99c4abcd0d27d..052f25450e6a5 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2339,16 +2339,6 @@ func (q *querier) GetUsersByIDs(ctx context.Context, ids []uuid.UUID) ([]databas return q.db.GetUsersByIDs(ctx, ids) } -func (q *querier) GetUsersWithAccessToTemplateByID(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) { - // Ensure we have permission to access this template. - _, err := q.GetTemplateByID(ctx, id) - if err != nil { - return nil, err - } - - return q.db.GetUsersWithAccessToTemplateByID(ctx, id) -} - func (q *querier) GetWorkspaceAgentAndLatestBuildByAuthToken(ctx context.Context, authToken uuid.UUID) (database.GetWorkspaceAgentAndLatestBuildByAuthTokenRow, error) { // This is a system function if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 1ac9074006949..24498d88c9dbc 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -5669,33 +5669,6 @@ func (q *FakeQuerier) GetUsersByIDs(_ context.Context, ids []uuid.UUID) ([]datab return users, nil } -func (q *FakeQuerier) GetUsersWithAccessToTemplateByID(_ context.Context, id uuid.UUID) ([]uuid.UUID, error) { - q.mutex.RLock() - defer q.mutex.RUnlock() - - groups := make(map[string]struct{}, 0) - for _, template := range q.templates { - if template.ID != id { - continue - } - - for group := range template.GroupACL { - groups[group] = struct{}{} - } - } - - users := make([]uuid.UUID, 0) - for _, member := range q.organizationMembers { - if _, ok := groups[member.OrganizationID.String()]; !ok { - continue - } - - users = append(users, member.UserID) - } - - return users, nil -} - func (q *FakeQuerier) GetWorkspaceAgentAndLatestBuildByAuthToken(_ context.Context, authToken uuid.UUID) (database.GetWorkspaceAgentAndLatestBuildByAuthTokenRow, error) { q.mutex.RLock() defer q.mutex.RUnlock() diff --git a/coderd/database/dbmetrics/dbmetrics.go b/coderd/database/dbmetrics/dbmetrics.go index 79f5dbb7d7f30..c3e9de22fb0d8 100644 --- a/coderd/database/dbmetrics/dbmetrics.go +++ b/coderd/database/dbmetrics/dbmetrics.go @@ -1348,13 +1348,6 @@ func (m metricsStore) GetUsersByIDs(ctx context.Context, ids []uuid.UUID) ([]dat return users, err } -func (m metricsStore) GetUsersWithAccessToTemplateByID(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) { - start := time.Now() - r0, r1 := m.s.GetUsersWithAccessToTemplateByID(ctx, id) - m.queryLatencies.WithLabelValues("GetUsersWithAccessToTemplateByID").Observe(time.Since(start).Seconds()) - return r0, r1 -} - func (m metricsStore) GetWorkspaceAgentAndLatestBuildByAuthToken(ctx context.Context, authToken uuid.UUID) (database.GetWorkspaceAgentAndLatestBuildByAuthTokenRow, error) { start := time.Now() r0, r1 := m.s.GetWorkspaceAgentAndLatestBuildByAuthToken(ctx, authToken) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index d2a4b5b2df8fe..b3c7b9e7615d3 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2813,21 +2813,6 @@ func (mr *MockStoreMockRecorder) GetUsersByIDs(arg0, arg1 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUsersByIDs", reflect.TypeOf((*MockStore)(nil).GetUsersByIDs), arg0, arg1) } -// GetUsersWithAccessToTemplateByID mocks base method. -func (m *MockStore) GetUsersWithAccessToTemplateByID(arg0 context.Context, arg1 uuid.UUID) ([]uuid.UUID, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetUsersWithAccessToTemplateByID", arg0, arg1) - ret0, _ := ret[0].([]uuid.UUID) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetUsersWithAccessToTemplateByID indicates an expected call of GetUsersWithAccessToTemplateByID. -func (mr *MockStoreMockRecorder) GetUsersWithAccessToTemplateByID(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUsersWithAccessToTemplateByID", reflect.TypeOf((*MockStore)(nil).GetUsersWithAccessToTemplateByID), arg0, arg1) -} - // GetWorkspaceAgentAndLatestBuildByAuthToken mocks base method. func (m *MockStore) GetWorkspaceAgentAndLatestBuildByAuthToken(arg0 context.Context, arg1 uuid.UUID) (database.GetWorkspaceAgentAndLatestBuildByAuthTokenRow, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 1e8f6909b1d86..fcb58a7d6e305 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -290,7 +290,6 @@ type sqlcQuerier interface { // to look up references to actions. eg. a user could build a workspace // for another user, then be deleted... we still want them to appear! GetUsersByIDs(ctx context.Context, ids []uuid.UUID) ([]User, error) - GetUsersWithAccessToTemplateByID(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) GetWorkspaceAgentAndLatestBuildByAuthToken(ctx context.Context, authToken uuid.UUID) (GetWorkspaceAgentAndLatestBuildByAuthTokenRow, error) GetWorkspaceAgentByID(ctx context.Context, id uuid.UUID) (WorkspaceAgent, error) GetWorkspaceAgentByInstanceID(ctx context.Context, authInstanceID string) (WorkspaceAgent, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index f0a290d2a2f8b..45cbef3f5e1d8 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8557,44 +8557,6 @@ func (q *sqlQuerier) GetTemplatesWithFilter(ctx context.Context, arg GetTemplate return items, nil } -const getUsersWithAccessToTemplateByID = `-- name: GetUsersWithAccessToTemplateByID :many -SELECT - user_id -FROM - organization_members -WHERE - organization_members.organization_id::text IN ( - SELECT - jsonb_object_keys(group_acl) - FROM - templates - WHERE templates.id = $1 - ) -` - -func (q *sqlQuerier) GetUsersWithAccessToTemplateByID(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) { - rows, err := q.db.QueryContext(ctx, getUsersWithAccessToTemplateByID, id) - if err != nil { - return nil, err - } - defer rows.Close() - var items []uuid.UUID - for rows.Next() { - var user_id uuid.UUID - if err := rows.Scan(&user_id); err != nil { - return nil, err - } - items = append(items, user_id) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const insertTemplate = `-- name: InsertTemplate :exec INSERT INTO templates ( diff --git a/coderd/database/queries/templates.sql b/coderd/database/queries/templates.sql index eb270d4d491a6..84df9633a1a53 100644 --- a/coderd/database/queries/templates.sql +++ b/coderd/database/queries/templates.sql @@ -199,18 +199,3 @@ SET WHERE id = $1 ; - --- name: GetUsersWithAccessToTemplateByID :many -SELECT - user_id -FROM - organization_members -WHERE - organization_members.organization_id::text IN ( - SELECT - jsonb_object_keys(group_acl) - FROM - templates - WHERE templates.id = $1 - ) -; diff --git a/coderd/templates.go b/coderd/templates.go index 14dced44a68ae..cbc6eb784d2e4 100644 --- a/coderd/templates.go +++ b/coderd/templates.go @@ -862,14 +862,21 @@ func (api *API) patchTemplateMeta(rw http.ResponseWriter, r *http.Request) { } func (api *API) notifyUsersOfTemplateDeprecation(ctx context.Context, template database.Template) error { - users, err := api.Database.GetUsersWithAccessToTemplateByID(ctx, template.ID) + workspaces, err := api.Database.GetWorkspaces(ctx, database.GetWorkspacesParams{ + TemplateIDs: []uuid.UUID{template.ID}, + }) if err != nil { - return xerrors.Errorf("get users with access to template by id: %w", err) + return xerrors.Errorf("get workspaces by template id: %w", err) + } + + users := make(map[uuid.UUID]struct{}) + for _, workspace := range workspaces { + users[workspace.OwnerID] = struct{}{} } errs := []error{} - for _, userID := range users { + for userID := range users { _, err = api.NotificationsEnqueuer.Enqueue( //nolint:gocritic // We need the system auth context to be able to send the deprecation notification. dbauthz.AsSystemRestricted(ctx), diff --git a/enterprise/coderd/templates_test.go b/enterprise/coderd/templates_test.go index 1d0cb13306fe8..5f981b83bbdb5 100644 --- a/enterprise/coderd/templates_test.go +++ b/enterprise/coderd/templates_test.go @@ -47,18 +47,18 @@ func TestTemplates(t *testing.T) { }, LicenseOptions: &coderdenttest.LicenseOptions{ Features: license.Features{ - codersdk.FeatureAccessControl: 1, - codersdk.FeatureMultipleOrganizations: 1, + codersdk.FeatureAccessControl: 1, }, }, }) - client, anotherUser := coderdtest.CreateAnotherUser(t, owner, user.OrganizationID, rbac.RoleTemplateAdmin()) + client, secondUser := coderdtest.CreateAnotherUser(t, owner, user.OrganizationID, rbac.RoleTemplateAdmin()) + _, thirdUser := coderdtest.CreateAnotherUser(t, owner, user.OrganizationID, rbac.RoleTemplateAdmin()) version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil) template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) - org := coderdenttest.CreateOrganization(t, owner, coderdenttest.CreateOrganizationOptions{}) - _, thirdUser := coderdtest.CreateAnotherUser(t, owner, org.ID, rbac.RoleTemplateAdmin()) + _ = coderdtest.CreateWorkspace(t, owner, template.ID) + _ = coderdtest.CreateWorkspace(t, client, template.ID) ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() @@ -80,7 +80,7 @@ func TestTemplates(t *testing.T) { } require.Equal(t, 2, len(notifs)) - expectedSentTo := []string{user.UserID.String(), anotherUser.ID.String()} + expectedSentTo := []string{user.UserID.String(), secondUser.ID.String()} slices.Sort(expectedSentTo) sentTo := []string{} @@ -93,7 +93,7 @@ func TestTemplates(t *testing.T) { assert.Equal(t, expectedSentTo, sentTo) // The previous check should verify this but we're double checking that - // the notification wasn't sent to a user in another org. + // the notification wasn't sent to users not using the template. for _, notif := range notifs { assert.NotEqual(t, thirdUser.ID, notif.UserID) } From c63dbff5b760f29d96b087388cbb0a98112c7b99 Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Thu, 24 Oct 2024 10:39:30 +0000 Subject: [PATCH 08/11] test: create another template+workspace --- enterprise/coderd/templates_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/enterprise/coderd/templates_test.go b/enterprise/coderd/templates_test.go index 5f981b83bbdb5..cde01553e349c 100644 --- a/enterprise/coderd/templates_test.go +++ b/enterprise/coderd/templates_test.go @@ -52,7 +52,8 @@ func TestTemplates(t *testing.T) { }, }) client, secondUser := coderdtest.CreateAnotherUser(t, owner, user.OrganizationID, rbac.RoleTemplateAdmin()) - _, thirdUser := coderdtest.CreateAnotherUser(t, owner, user.OrganizationID, rbac.RoleTemplateAdmin()) + otherClient, otherUser := coderdtest.CreateAnotherUser(t, owner, user.OrganizationID, rbac.RoleTemplateAdmin()) + version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil) template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) @@ -60,6 +61,14 @@ func TestTemplates(t *testing.T) { _ = coderdtest.CreateWorkspace(t, owner, template.ID) _ = coderdtest.CreateWorkspace(t, client, template.ID) + // Create another template for testing that users of another template do not + // get a notification. + secondVersion := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil) + secondTemplate := coderdtest.CreateTemplate(t, client, user.OrganizationID, secondVersion.ID) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, secondVersion.ID) + + _ = coderdtest.CreateWorkspace(t, otherClient, secondTemplate.ID) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() @@ -95,7 +104,7 @@ func TestTemplates(t *testing.T) { // The previous check should verify this but we're double checking that // the notification wasn't sent to users not using the template. for _, notif := range notifs { - assert.NotEqual(t, thirdUser.ID, notif.UserID) + assert.NotEqual(t, otherUser.ID, notif.UserID) } _, err = client.CreateWorkspace(ctx, user.OrganizationID, codersdk.Me, codersdk.CreateWorkspaceRequest{ From 468ba87954fe00140ecb1f8bc4748234e98b9059 Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Thu, 24 Oct 2024 10:47:05 +0000 Subject: [PATCH 09/11] fix: remove unused test --- coderd/database/dbauthz/dbauthz_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 4032b41526573..6a34e88104ce1 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1131,10 +1131,6 @@ func (s *MethodTestSuite) TestUser() { Asserts(a, policy.ActionRead, b, policy.ActionRead). Returns(slice.New(a, b)) })) - s.Run("GetUsersWithAccessToTemplateByID", s.Subtest(func(db database.Store, check *expects) { - a := dbgen.Template(s.T(), db, database.Template{}) - check.Args(a.ID).Asserts(a, policy.ActionRead) - })) s.Run("GetUsers", s.Subtest(func(db database.Store, check *expects) { dbgen.User(s.T(), db, database.User{Username: "GetUsers-a-user"}) dbgen.User(s.T(), db, database.User{Username: "GetUsers-b-user"}) From c1e0fb24482bffc555f571c0f33a32ad4ec6d792 Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Thu, 24 Oct 2024 11:25:20 +0000 Subject: [PATCH 10/11] chore: rename 'See workspaces' to 'See affected workspaces' --- .../migrations/000270_template_deprecation_notification.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/database/migrations/000270_template_deprecation_notification.up.sql b/coderd/database/migrations/000270_template_deprecation_notification.up.sql index 1a289f448f5d0..e98f852c8b4e1 100644 --- a/coderd/database/migrations/000270_template_deprecation_notification.up.sql +++ b/coderd/database/migrations/000270_template_deprecation_notification.up.sql @@ -11,7 +11,7 @@ VALUES ( 'Template Events', '[ { - "label": "See workspaces", + "label": "See affected workspaces", "url": "{{base_url}}/workspaces?filter=owner%3Ame+template%3A{{.Labels.template}}" }, { From b210680ef49ef7ea84bd49453d80a297666a01ba Mon Sep 17 00:00:00 2001 From: Danielle Maywood Date: Thu, 24 Oct 2024 11:33:44 +0000 Subject: [PATCH 11/11] fix: update golden files --- .../smtp/TemplateTemplateDeprecated.html.golden | 6 +++--- .../webhook/TemplateTemplateDeprecated.json.golden | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/coderd/notifications/testdata/rendered-templates/smtp/TemplateTemplateDeprecated.html.golden b/coderd/notifications/testdata/rendered-templates/smtp/TemplateTemplateDeprecated.html.golden index b627c6f8aafcf..1393acc4bc60a 100644 --- a/coderd/notifications/testdata/rendered-templates/smtp/TemplateTemplateDeprecated.html.golden +++ b/coderd/notifications/testdata/rendered-templates/smtp/TemplateTemplateDeprecated.html.golden @@ -20,8 +20,8 @@ New workspaces may not be created from this template. Existing workspaces w= ill continue to function normally. -See workspaces: http://test.com/workspaces?filter=3Downer%3Ame+template%3Aa= -lpha +See affected workspaces: http://test.com/workspaces?filter=3Downer%3Ame+tem= +plate%3Aalpha View template: http://test.com/templates/coder/alpha @@ -69,7 +69,7 @@ s will continue to function normally.

3Aalpha" style=3D"display: inline-block; padding: 13px 24px; background-col= or: #020617; color: #f8fafc; text-decoration: none; border-radius: 8px; mar= gin: 0 4px;"> - See workspaces + See affected workspaces =20 pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy