Skip to content

Commit 30204ec

Browse files
apparentlymartapparentlymart
committed
command/cliconfig: Allow development overrides for providers
For normal provider installation we want to associate each provider with a selected version number and find a suitable package for that version that conforms to the official hashes for that release. Those requirements are very onerous for a provider developer currently testing a not-yet-released build, though. To allow for that case this new CLI configuration feature allows overriding specific providers to refer to give local filesystem directories. Any provider overridden in this way is not subject to the usual restrictions about selected versions or checksum conformance, and activating an override won't cause any changes to the selections recorded in the lock file because it's intended to be a temporary setting for one developer only. This is, in a sense, a spiritual successor of an old capability we had to override specific plugins in the CLI configuration file. There were some vestiges of that left in the main package and CLI config package but nothing has actually been honoring them for several versions now and so this commit removes them to avoid confusion with the new mechanism.
1 parent 98dc399 commit 30204ec

17 files changed

Lines changed: 389 additions & 28 deletions

‎command/apply.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,11 @@ func (c *ApplyCommand) Run(args []string) int {
131131
return 1
132132
}
133133

134+
// Applying changes with dev overrides in effect could make it impossible
135+
// to switch back to a release version if the schema isn't compatible,
136+
// so we'll warn about it.
137+
diags = diags.Append(c.providerDevOverrideWarnings())
138+
134139
// Before we delegate to the backend, we'll print any warning diagnostics
135140
// we've accumulated here, since the backend will start fresh with its own
136141
// diagnostics.

‎command/cliconfig/provider_installation.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,36 @@ package cliconfig
22

33
import (
44
"fmt"
5+
"path/filepath"
56

67
"github.com/hashicorp/hcl"
78
hclast "github.com/hashicorp/hcl/hcl/ast"
9+
"github.com/hashicorp/terraform/addrs"
10+
"github.com/hashicorp/terraform/internal/getproviders"
811
"github.com/hashicorp/terraform/tfdiags"
912
)
1013

1114
// ProviderInstallation is the structure of the "provider_installation"
1215
// nested block within the CLI configuration.
1316
type ProviderInstallation struct {
1417
Methods []*ProviderInstallationMethod
18+
19+
// DevOverrides allows overriding the normal selection process for
20+
// a particular subset of providers to force using a particular
21+
// local directory and disregard version numbering altogether.
22+
// This is here to allow provider developers to conveniently test
23+
// local builds of their plugins in a development environment, without
24+
// having to fuss with version constraints, dependency lock files, and
25+
// so forth.
26+
//
27+
// This is _not_ intended for "production" use because it bypasses the
28+
// usual version selection and checksum verification mechanisms for
29+
// the providers in question. To make that intent/effect clearer, some
30+
// Terraform commands emit warnings when overrides are present. Local
31+
// mirror directories are a better way to distribute "released"
32+
// providers, because they are still subject to version constraints and
33+
// checksum verification.
34+
DevOverrides map[addrs.Provider]getproviders.PackageLocalDir
1535
}
1636

1737
// decodeProviderInstallationFromConfig uses the HCL AST API directly to
@@ -65,6 +85,7 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst
6585
}
6686

6787
pi := &ProviderInstallation{}
88+
devOverrides := make(map[addrs.Provider]getproviders.PackageLocalDir)
6889

6990
body, ok := block.Val.(*hclast.ObjectType)
7091
if !ok {
@@ -188,6 +209,53 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst
188209
location = ProviderInstallationNetworkMirror(bodyContent.URL)
189210
include = bodyContent.Include
190211
exclude = bodyContent.Exclude
212+
case "dev_overrides":
213+
if len(pi.Methods) > 0 {
214+
// We require dev_overrides to appear first if it's present,
215+
// because dev_overrides effectively bypass the normal
216+
// selection process for a particular provider altogether,
217+
// and so they don't participate in the usual
218+
// include/exclude arguments and priority ordering.
219+
diags = diags.Append(tfdiags.Sourceless(
220+
tfdiags.Error,
221+
"Invalid provider_installation method block",
222+
fmt.Sprintf("The dev_overrides block at at %s must appear before all other installation methods, because development overrides always have the highest priority.", methodBlock.Pos()),
223+
))
224+
continue
225+
}
226+
227+
// The content of a dev_overrides block is a mapping from
228+
// provider source addresses to local filesystem paths. To get
229+
// our decoding started, we'll use the normal HCL decoder to
230+
// populate a map of strings and then decode further from
231+
// that.
232+
var rawItems map[string]string
233+
err := hcl.DecodeObject(&rawItems, methodBody)
234+
if err != nil {
235+
diags = diags.Append(tfdiags.Sourceless(
236+
tfdiags.Error,
237+
"Invalid provider_installation method block",
238+
fmt.Sprintf("Invalid %s block at %s: %s.", methodTypeStr, block.Pos(), err),
239+
))
240+
continue
241+
}
242+
243+
for rawAddr, rawPath := range rawItems {
244+
addr, moreDiags := addrs.ParseProviderSourceString(rawAddr)
245+
if moreDiags.HasErrors() {
246+
diags = diags.Append(tfdiags.Sourceless(
247+
tfdiags.Error,
248+
"Invalid provider installation dev overrides",
249+
fmt.Sprintf("The entry %q in %s is not a valid provider source string.", rawAddr, block.Pos()),
250+
))
251+
continue
252+
}
253+
dirPath := filepath.Clean(rawPath)
254+
devOverrides[addr] = getproviders.PackageLocalDir(dirPath)
255+
}
256+
257+
continue // We won't add anything to pi.Methods for this one
258+
191259
default:
192260
diags = diags.Append(tfdiags.Sourceless(
193261
tfdiags.Error,
@@ -204,6 +272,10 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst
204272
})
205273
}
206274

275+
if len(devOverrides) > 0 {
276+
pi.DevOverrides = devOverrides
277+
}
278+
207279
ret = append(ret, pi)
208280
}
209281

‎command/cliconfig/provider_installation_test.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"testing"
66

77
"github.com/google/go-cmp/cmp"
8+
"github.com/hashicorp/terraform/addrs"
9+
"github.com/hashicorp/terraform/internal/getproviders"
810
)
911

1012
func TestLoadConfig_providerInstallation(t *testing.T) {
@@ -36,6 +38,11 @@ func TestLoadConfig_providerInstallation(t *testing.T) {
3638
Exclude: []string{"example.com/*/*"},
3739
},
3840
},
41+
42+
DevOverrides: map[addrs.Provider]getproviders.PackageLocalDir{
43+
addrs.MustParseProviderSourceString("hashicorp/boop"): getproviders.PackageLocalDir(filepath.FromSlash("/tmp/boop")),
44+
addrs.MustParseProviderSourceString("hashicorp/blorp"): getproviders.PackageLocalDir(filepath.FromSlash("/tmp/blorp")),
45+
},
3946
},
4047
},
4148
}

‎command/cliconfig/testdata/provider-installation‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
provider_installation {
2+
dev_overrides {
3+
"hashicorp/boop" = "/tmp/bloop/../boop"
4+
"hashicorp/blorp" = "/tmp/blorp"
5+
}
26
filesystem_mirror {
37
path = "/tmp/example1"
48
include = ["example.com/*/*"]

‎command/cliconfig/testdata/provider-installation.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
{
22
"provider_installation": {
3+
"dev_overrides": {
4+
"hashicorp/boop": "/tmp/bloop/../boop",
5+
"hashicorp/blorp": "/tmp/blorp"
6+
},
37
"filesystem_mirror": [{
48
"path": "/tmp/example1",
59
"include": ["example.com/*/*"]

‎command/e2etest/provider_dev_test.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package e2etest
2+
3+
import (
4+
"fmt"
5+
"io/ioutil"
6+
"os"
7+
"path/filepath"
8+
"strings"
9+
"testing"
10+
11+
"github.com/hashicorp/terraform/e2e"
12+
)
13+
14+
// TestProviderDevOverrides is a test for the special dev_overrides setting
15+
// in the provider_installation section of the CLI configuration file, which
16+
// is our current answer to smoothing provider development by allowing
17+
// developers to opt out of the version number and checksum verification
18+
// we normally do, so they can just overwrite the same local executable
19+
// in-place to iterate faster.
20+
func TestProviderDevOverrides(t *testing.T) {
21+
t.Parallel()
22+
23+
tf := e2e.NewBinary(terraformBin, "testdata/provider-dev-override")
24+
defer tf.Close()
25+
26+
// In order to do a decent end-to-end test for this case we will need a
27+
// real enough provider plugin to try to run and make sure we are able
28+
// to actually run it. For now we'll use the "test" provider for that,
29+
// because it happens to be in this repository and therefore allows
30+
// us to avoid drawing in anything external, but we might revisit this
31+
// strategy in future if other needs cause us to evolve the test
32+
// provider in a way that makes it less suitable for this particular test,
33+
// such as if it stops being buildable into an independent executable.
34+
providerExeDir := filepath.Join(tf.WorkDir(), "pkgdir")
35+
providerExePrefix := filepath.Join(providerExeDir, "terraform-provider-test_")
36+
providerExe := e2e.GoBuild("github.com/hashicorp/terraform/builtin/bins/provider-test", providerExePrefix)
37+
t.Logf("temporary provider executable is %s", providerExe)
38+
39+
err := ioutil.WriteFile(filepath.Join(tf.WorkDir(), "dev.tfrc"), []byte(fmt.Sprintf(`
40+
provider_installation {
41+
dev_overrides {
42+
"example.com/test/test" = %q
43+
}
44+
}
45+
`, providerExeDir)), os.ModePerm)
46+
if err != nil {
47+
t.Fatal(err)
48+
}
49+
50+
tf.AddEnv("TF_CLI_CONFIG_FILE=dev.tfrc")
51+
52+
stdout, stderr, err := tf.Run("providers")
53+
if err != nil {
54+
t.Fatalf("unexpected error: %s\n%s", err, stderr)
55+
}
56+
if got, want := stdout, `provider[example.com/test/test]`; !strings.Contains(got, want) {
57+
t.Errorf("configuration should depend on %s, but doesn't\n%s", want, got)
58+
}
59+
60+
// NOTE: We're intentionally not running "terraform init" here, because
61+
// dev overrides are always ready to use and don't need any special action
62+
// to "install" them. This test is mimicking the a happy path of going
63+
// directly from "go build" to validate/plan/apply without interacting
64+
// with any registries, mirrors, lock files, etc.
65+
stdout, stderr, err = tf.Run("validate")
66+
if err != nil {
67+
t.Fatalf("unexpected error: %s\n%s", err, stderr)
68+
}
69+
70+
if got, want := stdout, `The configuration is valid, but`; !strings.Contains(got, want) {
71+
t.Errorf("stdout doesn't include the success message\nwant: %s\n%s", want, got)
72+
}
73+
if got, want := stdout, `Provider development overrides are in effect`; !strings.Contains(got, want) {
74+
t.Errorf("stdout doesn't include the warning about development overrides\nwant: %s\n%s", want, got)
75+
}
76+
}

‎command/e2etest/testdata/provider-dev-override/pkgdir/.exists‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
This is where the test will place the temporary build of the test provider.

‎command/e2etest/testdata/provider-dev-override/provider-dev-override.tf‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
terraform {
2+
required_providers {
3+
test = {
4+
source = "example.com/test/test"
5+
version = "2.0.0"
6+
}
7+
}
8+
}
9+
10+
provider "test" {
11+
}
12+
13+
data "test_data_source" "test" {
14+
}

‎command/init.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -730,6 +730,12 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State,
730730
},
731731
}
732732

733+
// Dev overrides cause the result of "terraform init" to be irrelevant for
734+
// any overridden providers, so we'll warn about it to avoid later
735+
// confusion when Terraform ends up using a different provider than the
736+
// lock file called for.
737+
diags = diags.Append(c.providerDevOverrideWarnings())
738+
733739
mode := providercache.InstallNewProvidersOnly
734740
if upgrade {
735741
mode = providercache.InstallUpgrades

‎command/meta.go‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,9 @@ type Meta struct {
5050
// for some reason.
5151
OriginalWorkingDir string
5252

53-
Color bool // True if output should be colored
54-
GlobalPluginDirs []string // Additional paths to search for plugins
55-
PluginOverrides *PluginOverrides // legacy overrides from .terraformrc file
56-
Ui cli.Ui // Ui for output
53+
Color bool // True if output should be colored
54+
GlobalPluginDirs []string // Additional paths to search for plugins
55+
Ui cli.Ui // Ui for output
5756

5857
// ExtraHooks are extra hooks to add to the context.
5958
ExtraHooks []terraform.Hook
@@ -104,7 +103,21 @@ type Meta struct {
104103
// When this channel is closed, the command will be cancelled.
105104
ShutdownCh <-chan struct{}
106105

107-
// UnmanagedProviders are a set of providers that exist as processes predating Terraform, which Terraform should use but not worry about the lifecycle of.
106+
// ProviderDevOverrides are providers where we ignore the lock file, the
107+
// configured version constraints, and the local cache directory and just
108+
// always use exactly the path specified. This is intended to allow
109+
// provider developers to easily test local builds without worrying about
110+
// what version number they might eventually be released as, or what
111+
// checksums they have.
112+
ProviderDevOverrides map[addrs.Provider]getproviders.PackageLocalDir
113+
114+
// UnmanagedProviders are a set of providers that exist as processes
115+
// predating Terraform, which Terraform should use but not worry about the
116+
// lifecycle of.
117+
//
118+
// This is essentially a more extreme version of ProviderDevOverrides where
119+
// Terraform doesn't even worry about how the provider server gets launched,
120+
// just trusting that someone else did it before running Terraform.
108121
UnmanagedProviders map[addrs.Provider]*plugin.ReattachConfig
109122

110123
//----------------------------------------------------------
@@ -194,11 +207,6 @@ type Meta struct {
194207
allowMissingConfig bool
195208
}
196209

197-
type PluginOverrides struct {
198-
Providers map[string]string
199-
Provisioners map[string]string
200-
}
201-
202210
type testingOverrides struct {
203211
Providers map[addrs.Provider]providers.Factory
204212
Provisioners map[string]provisioners.Factory

0 commit comments

Comments
 (0)