Skip to content

Commit 52c4bfb

Browse files
committed
core: fix deadlock when dependable node replaced with non-dependable one
In hashicorp#2884, Terraform would hang on graphs with an orphaned resource depended on an orphaned module. This is because orphan module nodes (which are dependable) were getting expanded (replaced) with GraphNodeBasicSubgraph nodes (which are _not_ dependable). The old `graph.Replace()` code resulted in GraphNodeBasicSubgraph being entered into the lookaside table, even though it is not dependable. This resulted in an untraversable edge in the graph, so the graph would hang and wait forever. Now, we remove entries from the lookaside table when a dependable node is being replaced with a non-dependable node. This means we lose an edge, but we can move forward. It's ~probably~ never correct to be replacing depenable nodes with non-dependable ones, but this tweak seemed preferable to tossing a panic in there.
1 parent 7a464b1 commit 52c4bfb

6 files changed

Lines changed: 157 additions & 4 deletions

File tree

‎terraform/context_refresh_test.go‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"reflect"
55
"sort"
66
"strings"
7+
"sync"
78
"testing"
89
)
910

@@ -685,6 +686,100 @@ func TestContext2Refresh_vars(t *testing.T) {
685686
}
686687
}
687688

689+
func TestContext2Refresh_orphanModule(t *testing.T) {
690+
p := testProvider("aws")
691+
m := testModule(t, "refresh-module-orphan")
692+
693+
// Create a custom refresh function to track the order they were visited
694+
var order []string
695+
var orderLock sync.Mutex
696+
p.RefreshFn = func(
697+
info *InstanceInfo,
698+
is *InstanceState) (*InstanceState, error) {
699+
orderLock.Lock()
700+
defer orderLock.Unlock()
701+
702+
order = append(order, is.ID)
703+
return is, nil
704+
}
705+
706+
state := &State{
707+
Modules: []*ModuleState{
708+
&ModuleState{
709+
Path: rootModulePath,
710+
Resources: map[string]*ResourceState{
711+
"aws_instance.foo": &ResourceState{
712+
Primary: &InstanceState{
713+
ID: "i-abc123",
714+
Attributes: map[string]string{
715+
"childid": "i-bcd234",
716+
"grandchildid": "i-cde345",
717+
},
718+
},
719+
Dependencies: []string{
720+
"module.child",
721+
"module.child",
722+
},
723+
},
724+
},
725+
},
726+
&ModuleState{
727+
Path: append(rootModulePath, "child"),
728+
Resources: map[string]*ResourceState{
729+
"aws_instance.bar": &ResourceState{
730+
Primary: &InstanceState{
731+
ID: "i-bcd234",
732+
Attributes: map[string]string{
733+
"grandchildid": "i-cde345",
734+
},
735+
},
736+
Dependencies: []string{
737+
"module.grandchild",
738+
},
739+
},
740+
},
741+
Outputs: map[string]string{
742+
"id": "i-bcd234",
743+
"grandchild_id": "i-cde345",
744+
},
745+
},
746+
&ModuleState{
747+
Path: append(rootModulePath, "child", "grandchild"),
748+
Resources: map[string]*ResourceState{
749+
"aws_instance.baz": &ResourceState{
750+
Primary: &InstanceState{
751+
ID: "i-cde345",
752+
},
753+
},
754+
},
755+
Outputs: map[string]string{
756+
"id": "i-cde345",
757+
},
758+
},
759+
},
760+
}
761+
ctx := testContext2(t, &ContextOpts{
762+
Module: m,
763+
Providers: map[string]ResourceProviderFactory{
764+
"aws": testProviderFuncFixed(p),
765+
},
766+
State: state,
767+
})
768+
769+
testCheckDeadlock(t, func() {
770+
_, err := ctx.Refresh()
771+
if err != nil {
772+
t.Fatalf("err: %s", err)
773+
}
774+
775+
// TODO: handle order properly for orphaned modules / resources
776+
// expected := []string{"i-abc123", "i-bcd234", "i-cde345"}
777+
// if !reflect.DeepEqual(order, expected) {
778+
// t.Fatalf("expected: %#v, got: %#v", expected, order)
779+
// }
780+
})
781+
}
782+
688783
func TestContext2Validate(t *testing.T) {
689784
p := testProvider("aws")
690785
m := testModule(t, "validate-good")

‎terraform/graph.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,11 @@ func (g *Graph) Replace(o, n dag.Vertex) bool {
7474
// Go through and update our lookaside to point to the new vertex
7575
for k, v := range g.dependableMap {
7676
if v == o {
77-
g.dependableMap[k] = n
77+
if _, ok := n.(GraphNodeDependable); ok {
78+
g.dependableMap[k] = n
79+
} else {
80+
delete(g.dependableMap, k)
81+
}
7882
}
7983
}
8084

‎terraform/graph_test.go‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package terraform
22

33
import (
4+
"reflect"
45
"strings"
56
"testing"
67
)
@@ -20,7 +21,7 @@ func TestGraphAdd(t *testing.T) {
2021

2122
func TestGraphConnectDependent(t *testing.T) {
2223
var g Graph
23-
g.Add(&testGraphDependable{VertexName: "a", Mock: []string{"a"}})
24+
g.Add(&testGraphDependable{VertexName: "a"})
2425
b := g.Add(&testGraphDependable{
2526
VertexName: "b",
2627
DependentOnMock: []string{"a"},
@@ -37,18 +38,48 @@ func TestGraphConnectDependent(t *testing.T) {
3738
}
3839
}
3940

41+
func TestGraphReplace_DependableWithNonDependable(t *testing.T) {
42+
var g Graph
43+
a := g.Add(&testGraphDependable{VertexName: "a"})
44+
b := g.Add(&testGraphDependable{
45+
VertexName: "b",
46+
DependentOnMock: []string{"a"},
47+
})
48+
newA := "non-dependable-a"
49+
50+
if missing := g.ConnectDependent(b); len(missing) > 0 {
51+
t.Fatalf("bad: %#v", missing)
52+
}
53+
54+
if !g.Replace(a, newA) {
55+
t.Fatalf("failed to replace")
56+
}
57+
58+
c := g.Add(&testGraphDependable{
59+
VertexName: "c",
60+
DependentOnMock: []string{"a"},
61+
})
62+
63+
// This should fail by reporting missing, since a node with dependable
64+
// name "a" is no longer in the graph.
65+
missing := g.ConnectDependent(c)
66+
expected := []string{"a"}
67+
if !reflect.DeepEqual(expected, missing) {
68+
t.Fatalf("expected: %#v, got: %#v", expected, missing)
69+
}
70+
}
71+
4072
type testGraphDependable struct {
4173
VertexName string
4274
DependentOnMock []string
43-
Mock []string
4475
}
4576

4677
func (v *testGraphDependable) Name() string {
4778
return v.VertexName
4879
}
4980

5081
func (v *testGraphDependable) DependableName() []string {
51-
return v.Mock
82+
return []string{v.VertexName}
5283
}
5384

5485
func (v *testGraphDependable) DependentOn() []string {
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
resource "aws_instance" "baz" {}
2+
3+
output "id" { value = "${aws_instance.baz.id}" }
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
module "grandchild" {
2+
source = "./grandchild"
3+
}
4+
5+
resource "aws_instance" "bar" {
6+
grandchildid = "${module.grandchild.id}"
7+
}
8+
9+
output "id" { value = "${aws_instance.bar.id}" }
10+
output "grandchild_id" { value = "${module.grandchild.id}" }
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
/*
2+
module "child" {
3+
source = "./child"
4+
}
5+
6+
resource "aws_instance" "bar" {
7+
childid = "${module.child.id}"
8+
grandchildid = "${module.child.grandchild_id}"
9+
}
10+
*/

0 commit comments

Comments
 (0)