Skip to content

Commit 97ce44a

Browse files
hugodutkahugodutka
authored
chore: track terraform module source type in telemetry (#15590)
1 parent fbe2fa6 commit 97ce44a

File tree

2 files changed

+170
-2
lines changed

2 files changed

+170
-2
lines changed

‎coderd/telemetry/telemetry.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"net/http"
1212
"net/url"
1313
"os"
14+
"regexp"
1415
"runtime"
1516
"slices"
1617
"strings"
@@ -680,9 +681,95 @@ func shouldSendRawModuleSource(source string) bool {
680681
return strings.Contains(source, "registry.coder.com")
681682
}
682683

684+
// ModuleSourceType is the type of source for a module.
685+
// For reference, see https://developer.hashicorp.com/terraform/language/modules/sources
686+
type ModuleSourceType string
687+
688+
const (
689+
ModuleSourceTypeLocal ModuleSourceType = "local"
690+
ModuleSourceTypeLocalAbs ModuleSourceType = "local_absolute"
691+
ModuleSourceTypePublicRegistry ModuleSourceType = "public_registry"
692+
ModuleSourceTypePrivateRegistry ModuleSourceType = "private_registry"
693+
ModuleSourceTypeCoderRegistry ModuleSourceType = "coder_registry"
694+
ModuleSourceTypeGitHub ModuleSourceType = "github"
695+
ModuleSourceTypeBitbucket ModuleSourceType = "bitbucket"
696+
ModuleSourceTypeGit ModuleSourceType = "git"
697+
ModuleSourceTypeMercurial ModuleSourceType = "mercurial"
698+
ModuleSourceTypeHTTP ModuleSourceType = "http"
699+
ModuleSourceTypeS3 ModuleSourceType = "s3"
700+
ModuleSourceTypeGCS ModuleSourceType = "gcs"
701+
ModuleSourceTypeUnknown ModuleSourceType = "unknown"
702+
)
703+
704+
// Terraform supports a variety of module source types, like:
705+
// - local paths (./ or ../)
706+
// - absolute local paths (/)
707+
// - git URLs (git:: or git@)
708+
// - http URLs
709+
// - s3 URLs
710+
//
711+
// and more!
712+
//
713+
// See https://developer.hashicorp.com/terraform/language/modules/sources for an overview.
714+
//
715+
// This function attempts to classify the source type of a module. It's imperfect,
716+
// as checks that terraform actually does are pretty complicated.
717+
// See e.g. https://github.com/hashicorp/go-getter/blob/842d6c379e5e70d23905b8f6b5a25a80290acb66/detect.go#L47
718+
// if you're interested in the complexity.
719+
func GetModuleSourceType(source string) ModuleSourceType {
720+
source = strings.TrimSpace(source)
721+
source = strings.ToLower(source)
722+
if strings.HasPrefix(source, "./") || strings.HasPrefix(source, "../") {
723+
return ModuleSourceTypeLocal
724+
}
725+
if strings.HasPrefix(source, "/") {
726+
return ModuleSourceTypeLocalAbs
727+
}
728+
// Match public registry modules in the format <NAMESPACE>/<NAME>/<PROVIDER>
729+
// Sources can have a `//...` suffix, which signifies a subdirectory.
730+
// The allowed characters are based on
731+
// https://developer.hashicorp.com/terraform/cloud-docs/api-docs/private-registry/modules#request-body-1
732+
// because Hashicorp's documentation about module sources doesn't mention it.
733+
if matched, _ := regexp.MatchString(`^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+(//.*)?$`, source); matched {
734+
return ModuleSourceTypePublicRegistry
735+
}
736+
if strings.Contains(source, "github.com") {
737+
return ModuleSourceTypeGitHub
738+
}
739+
if strings.Contains(source, "bitbucket.org") {
740+
return ModuleSourceTypeBitbucket
741+
}
742+
if strings.HasPrefix(source, "git::") || strings.HasPrefix(source, "git@") {
743+
return ModuleSourceTypeGit
744+
}
745+
if strings.HasPrefix(source, "hg::") {
746+
return ModuleSourceTypeMercurial
747+
}
748+
if strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://") {
749+
return ModuleSourceTypeHTTP
750+
}
751+
if strings.HasPrefix(source, "s3::") {
752+
return ModuleSourceTypeS3
753+
}
754+
if strings.HasPrefix(source, "gcs::") {
755+
return ModuleSourceTypeGCS
756+
}
757+
if strings.Contains(source, "registry.terraform.io") {
758+
return ModuleSourceTypePublicRegistry
759+
}
760+
if strings.Contains(source, "app.terraform.io") || strings.Contains(source, "localterraform.com") {
761+
return ModuleSourceTypePrivateRegistry
762+
}
763+
if strings.Contains(source, "registry.coder.com") {
764+
return ModuleSourceTypeCoderRegistry
765+
}
766+
return ModuleSourceTypeUnknown
767+
}
768+
683769
func ConvertWorkspaceModule(module database.WorkspaceModule) WorkspaceModule {
684770
source := module.Source
685771
version := module.Version
772+
sourceType := GetModuleSourceType(source)
686773
if !shouldSendRawModuleSource(source) {
687774
source = fmt.Sprintf("%x", sha256.Sum256([]byte(source)))
688775
version = fmt.Sprintf("%x", sha256.Sum256([]byte(version)))
@@ -694,6 +781,7 @@ func ConvertWorkspaceModule(module database.WorkspaceModule) WorkspaceModule {
694781
Transition: module.Transition,
695782
Source: source,
696783
Version: version,
784+
SourceType: sourceType,
697785
Key: module.Key,
698786
CreatedAt: module.CreatedAt,
699787
}
@@ -938,6 +1026,7 @@ type WorkspaceModule struct {
9381026
Key string `json:"key"`
9391027
Version string `json:"version"`
9401028
Source string `json:"source"`
1029+
SourceType ModuleSourceType `json:"source_type"`
9411030
}
9421031

9431032
type WorkspaceAgent struct {

‎coderd/telemetry/telemetry_test.go

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ func TestTelemetry(t *testing.T) {
133133
})
134134
_ = dbgen.WorkspaceModule(t, db, database.WorkspaceModule{
135135
JobID: pj.ID,
136-
Source: "internal-url.com/some-module",
136+
Source: "https://internal-url.com/some-module",
137137
Version: "1.0.0",
138138
})
139139
_, snapshot := collectSnapshot(t, db, nil)
@@ -142,10 +142,89 @@ func TestTelemetry(t *testing.T) {
142142
sort.Slice(modules, func(i, j int) bool {
143143
return modules[i].Source < modules[j].Source
144144
})
145-
require.Equal(t, modules[0].Source, "921c61d6f3eef5118f3cae658d1518b378c5b02a4955a766c791440894d989c5")
145+
require.Equal(t, modules[0].Source, "ed662ec0396db67e77119f14afcb9253574cc925b04a51d4374bcb1eae299f5d")
146146
require.Equal(t, modules[0].Version, "92521fc3cbd964bdc9f584a991b89fddaa5754ed1cc96d6d42445338669c1305")
147+
require.Equal(t, modules[0].SourceType, telemetry.ModuleSourceTypeHTTP)
147148
require.Equal(t, modules[1].Source, "registry.coder.com/terraform/aws")
148149
require.Equal(t, modules[1].Version, "1.0.0")
150+
require.Equal(t, modules[1].SourceType, telemetry.ModuleSourceTypeCoderRegistry)
151+
})
152+
t.Run("ModuleSourceType", func(t *testing.T) {
153+
t.Parallel()
154+
cases := []struct {
155+
source string
156+
want telemetry.ModuleSourceType
157+
}{
158+
// Local relative paths
159+
{source: "./modules/terraform-aws-vpc", want: telemetry.ModuleSourceTypeLocal},
160+
{source: "../shared/modules/vpc", want: telemetry.ModuleSourceTypeLocal},
161+
{source: " ./my-module ", want: telemetry.ModuleSourceTypeLocal}, // with whitespace
162+
163+
// Local absolute paths
164+
{source: "/opt/terraform/modules/vpc", want: telemetry.ModuleSourceTypeLocalAbs},
165+
{source: "/Users/dev/modules/app", want: telemetry.ModuleSourceTypeLocalAbs},
166+
{source: "/etc/terraform/modules/network", want: telemetry.ModuleSourceTypeLocalAbs},
167+
168+
// Public registry
169+
{source: "hashicorp/consul/aws", want: telemetry.ModuleSourceTypePublicRegistry},
170+
{source: "registry.terraform.io/hashicorp/aws", want: telemetry.ModuleSourceTypePublicRegistry},
171+
{source: "terraform-aws-modules/vpc/aws", want: telemetry.ModuleSourceTypePublicRegistry},
172+
{source: "hashicorp/consul/aws//modules/consul-cluster", want: telemetry.ModuleSourceTypePublicRegistry},
173+
{source: "hashicorp/co-nsul/aw_s//modules/consul-cluster", want: telemetry.ModuleSourceTypePublicRegistry},
174+
175+
// Private registry
176+
{source: "app.terraform.io/company/vpc/aws", want: telemetry.ModuleSourceTypePrivateRegistry},
177+
{source: "localterraform.com/org/module", want: telemetry.ModuleSourceTypePrivateRegistry},
178+
{source: "APP.TERRAFORM.IO/test/module", want: telemetry.ModuleSourceTypePrivateRegistry}, // case insensitive
179+
180+
// Coder registry
181+
{source: "registry.coder.com/terraform/aws", want: telemetry.ModuleSourceTypeCoderRegistry},
182+
{source: "registry.coder.com/modules/base", want: telemetry.ModuleSourceTypeCoderRegistry},
183+
{source: "REGISTRY.CODER.COM/test/module", want: telemetry.ModuleSourceTypeCoderRegistry}, // case insensitive
184+
185+
// GitHub
186+
{source: "github.com/hashicorp/terraform-aws-vpc", want: telemetry.ModuleSourceTypeGitHub},
187+
{source: "git::https://github.com/org/repo.git", want: telemetry.ModuleSourceTypeGitHub},
188+
{source: "git::https://github.com/org/repo//modules/vpc", want: telemetry.ModuleSourceTypeGitHub},
189+
190+
// Bitbucket
191+
{source: "bitbucket.org/hashicorp/terraform-aws-vpc", want: telemetry.ModuleSourceTypeBitbucket},
192+
{source: "git::https://bitbucket.org/org/repo.git", want: telemetry.ModuleSourceTypeBitbucket},
193+
{source: "https://bitbucket.org/org/repo//modules/vpc", want: telemetry.ModuleSourceTypeBitbucket},
194+
195+
// Generic Git
196+
{source: "git::ssh://git.internal.com/repo.git", want: telemetry.ModuleSourceTypeGit},
197+
{source: "git@gitlab.com:org/repo.git", want: telemetry.ModuleSourceTypeGit},
198+
{source: "git::https://git.internal.com/repo.git?ref=v1.0.0", want: telemetry.ModuleSourceTypeGit},
199+
200+
// Mercurial
201+
{source: "hg::https://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
202+
{source: "hg::http://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
203+
{source: "hg::ssh://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
204+
205+
// HTTP
206+
{source: "https://example.com/vpc-module.zip", want: telemetry.ModuleSourceTypeHTTP},
207+
{source: "http://example.com/modules/vpc", want: telemetry.ModuleSourceTypeHTTP},
208+
{source: "https://internal.network/terraform/modules", want: telemetry.ModuleSourceTypeHTTP},
209+
210+
// S3
211+
{source: "s3::https://s3-eu-west-1.amazonaws.com/bucket/vpc", want: telemetry.ModuleSourceTypeS3},
212+
{source: "s3::https://bucket.s3.amazonaws.com/vpc", want: telemetry.ModuleSourceTypeS3},
213+
{source: "s3::http://bucket.s3.amazonaws.com/vpc?version=1", want: telemetry.ModuleSourceTypeS3},
214+
215+
// GCS
216+
{source: "gcs::https://www.googleapis.com/storage/v1/bucket/vpc", want: telemetry.ModuleSourceTypeGCS},
217+
{source: "gcs::https://storage.googleapis.com/bucket/vpc", want: telemetry.ModuleSourceTypeGCS},
218+
{source: "gcs::https://bucket.storage.googleapis.com/vpc", want: telemetry.ModuleSourceTypeGCS},
219+
220+
// Unknown
221+
{source: "custom://example.com/vpc", want: telemetry.ModuleSourceTypeUnknown},
222+
{source: "something-random", want: telemetry.ModuleSourceTypeUnknown},
223+
{source: "", want: telemetry.ModuleSourceTypeUnknown},
224+
}
225+
for _, c := range cases {
226+
require.Equal(t, c.want, telemetry.GetModuleSourceType(c.source))
227+
}
149228
})
150229
}
151230

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy