diff --git a/go.mod b/go.mod index 400435df1..4378b8cb1 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20170113022742-e6dbea820a9f github.com/antchfx/htmlquery v1.0.0 // indirect github.com/antchfx/xmlquery v1.0.0 // indirect - github.com/antchfx/xpath v0.0.0-20170728053731-b5c552e1acbd // indirect github.com/antchfx/xquery v0.0.0-20170730121040-eb8c3c172607 // indirect github.com/approvals/go-approval-tests v0.0.0-20160714161514-ad96e53bea43 @@ -153,7 +152,7 @@ require ( github.com/xanzy/go-cloudstack v0.0.0-20190526095453-42f262b63ed0 github.com/yandex-cloud/go-genproto v0.0.0-20190916101622-7617782d381e github.com/yandex-cloud/go-sdk v0.0.0-20190916101744-c781afa45829 - github.com/zclconf/go-cty v1.2.1 + github.com/zclconf/go-cty v1.3.1 github.com/zclconf/go-cty-yaml v1.0.1 go.opencensus.io v0.22.2 // indirect golang.org/x/crypto v0.0.0-20200117160349-530e935923ad @@ -180,6 +179,4 @@ replace git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110 replace github.com/gofrs/flock => github.com/azr/flock v0.0.0-20190823144736-958d66434653 -replace github.com/zclconf/go-cty => github.com/azr/go-cty v1.1.1-0.20200203143058-28fcda2fe0cc - go 1.13 diff --git a/go.sum b/go.sum index 36a39c483..85d9d39b8 100644 --- a/go.sum +++ b/go.sum @@ -488,6 +488,11 @@ github.com/yandex-cloud/go-genproto v0.0.0-20190916101622-7617782d381e h1:hzwq5G github.com/yandex-cloud/go-genproto v0.0.0-20190916101622-7617782d381e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= github.com/yandex-cloud/go-sdk v0.0.0-20190916101744-c781afa45829 h1:2FGwbx03GpP1Ulzg/L46tSoKh9t4yg8BhMKQl/Ff1x8= github.com/yandex-cloud/go-sdk v0.0.0-20190916101744-c781afa45829/go.mod h1:Eml0jFLU4VVHgIN8zPHMuNwZXVzUMILyO6lQZSfz854= +github.com/zclconf/go-cty v1.0.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= +github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= +github.com/zclconf/go-cty v1.2.1/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= +github.com/zclconf/go-cty v1.3.1 h1:QIOZl+CKKdkv4l2w3lG23nNzXgLoxsWLSEdg1MlX4p0= +github.com/zclconf/go-cty v1.3.1/go.mod h1:YO23e2L18AG+ZYQfSobnY4G65nvwvprPCxBHkufUH1k= github.com/zclconf/go-cty-yaml v1.0.1 h1:up11wlgAaDvlAGENcFDnZgkn0qUJurso7k6EpURKNF8= github.com/zclconf/go-cty-yaml v1.0.1/go.mod h1:IP3Ylp0wQpYm50IHK8OZWKMu6sPJIUgKa8XhiVHura0= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= @@ -541,6 +546,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zH golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index afa7687e6..b0dfedb82 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/packer/builder/null" . "github.com/hashicorp/packer/hcl2template/internal" "github.com/hashicorp/packer/helper/config" "github.com/hashicorp/packer/packer" @@ -21,6 +22,7 @@ func getBasicParser() *Parser { BuilderSchemas: packer.MapOfBuilder{ "amazon-ebs": func() (packer.Builder, error) { return &MockBuilder{}, nil }, "virtualbox-iso": func() (packer.Builder, error) { return &MockBuilder{}, nil }, + "null": func() (packer.Builder, error) { return &null.Builder{}, nil }, }, ProvisionersSchemas: packer.MapOfProvisioner{ "shell": func() (packer.Provisioner, error) { return &MockProvisioner{}, nil }, @@ -58,7 +60,7 @@ func testParse(t *testing.T, tests []parseTest) { t.Run(tt.name, func(t *testing.T) { gotCfg, gotDiags := tt.parser.parse(tt.args.filename, tt.args.vars) if tt.parseWantDiags == (gotDiags == nil) { - t.Fatalf("Parser.parse() unexpected diagnostics. %s", gotDiags) + t.Fatalf("Parser.parse() unexpected %q diagnostics.", gotDiags) } if tt.parseWantDiagHasErrors != gotDiags.HasErrors() { t.Fatalf("Parser.parse() unexpected diagnostics HasErrors. %s", gotDiags) @@ -120,6 +122,7 @@ func testParse(t *testing.T, tests []parseTest) { packer.CoreBuild{}, packer.CoreBuildProvisioner{}, packer.CoreBuildPostProcessor{}, + null.Builder{}, ), ); diff != "" { t.Fatalf("Parser.getBuilds() wrong packer builds. %s", diff) diff --git a/hcl2template/internal/mock.go b/hcl2template/internal/mock.go index 1c33d8f99..3c36528f3 100644 --- a/hcl2template/internal/mock.go +++ b/hcl2template/internal/mock.go @@ -9,6 +9,8 @@ import ( "github.com/hashicorp/hcl/v2/hcldec" "github.com/hashicorp/packer/helper/config" "github.com/hashicorp/packer/packer" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/json" ) type NestedMockConfig struct { @@ -38,6 +40,27 @@ type MockConfig struct { NestedSlice []NestedMockConfig `mapstructure:"nested_slice"` } +func (b *MockConfig) Prepare(raws ...interface{}) error { + for i, raw := range raws { + cval, ok := raw.(cty.Value) + if !ok { + continue + } + b, err := json.Marshal(cval, cty.DynamicPseudoType) + if err != nil { + return err + } + ccval, err := json.Unmarshal(b, cty.DynamicPseudoType) + if err != nil { + return err + } + raws[i] = ccval + } + return config.Decode(b, &config.DecodeOpts{ + Interpolate: true, + }, raws...) +} + ////// // MockBuilder ////// @@ -51,9 +74,7 @@ var _ packer.Builder = new(MockBuilder) func (b *MockBuilder) ConfigSpec() hcldec.ObjectSpec { return b.Config.FlatMapstructure().HCL2Spec() } func (b *MockBuilder) Prepare(raws ...interface{}) ([]string, []string, error) { - return nil, nil, config.Decode(&b.Config, &config.DecodeOpts{ - Interpolate: true, - }, raws...) + return nil, nil, b.Config.Prepare(raws...) } func (b *MockBuilder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) { @@ -75,9 +96,7 @@ func (b *MockProvisioner) ConfigSpec() hcldec.ObjectSpec { } func (b *MockProvisioner) Prepare(raws ...interface{}) error { - return config.Decode(&b.Config, &config.DecodeOpts{ - Interpolate: true, - }, raws...) + return b.Config.Prepare(raws...) } func (b *MockProvisioner) Provision(ctx context.Context, ui packer.Ui, comm packer.Communicator, _ map[string]interface{}) error { @@ -99,9 +118,7 @@ func (b *MockPostProcessor) ConfigSpec() hcldec.ObjectSpec { } func (b *MockPostProcessor) Configure(raws ...interface{}) error { - return config.Decode(&b.Config, &config.DecodeOpts{ - Interpolate: true, - }, raws...) + return b.Config.Prepare(raws...) } func (b *MockPostProcessor) PostProcess(ctx context.Context, ui packer.Ui, a packer.Artifact) (packer.Artifact, bool, bool, error) { @@ -124,9 +141,7 @@ func (b *MockCommunicator) ConfigSpec() hcldec.ObjectSpec { } func (b *MockCommunicator) Configure(raws ...interface{}) ([]string, error) { - return nil, config.Decode(&b.Config, &config.DecodeOpts{ - Interpolate: true, - }, raws...) + return nil, b.Config.Prepare(raws...) } ////// diff --git a/hcl2template/parser.go b/hcl2template/parser.go index 9ecbde9a2..3bf48294e 100644 --- a/hcl2template/parser.go +++ b/hcl2template/parser.go @@ -123,9 +123,14 @@ func (p *Parser) parse(filename string, vars map[string]string) (*PackerConfig, varFiles = append(varFiles, f) } - diags = append(diags, cfg.InputVariables.collectVariableValues(os.Environ(), varFiles, vars)...) + diags = append(diags, cfg.collectInputVariableValues(os.Environ(), varFiles, vars)...) } + _, moreDiags := cfg.InputVariables.Values() + diags = append(diags, moreDiags...) + _, moreDiags = cfg.LocalVariables.Values() + diags = append(diags, moreDiags...) + // decode the actual content for _, file := range files { diags = append(diags, p.decodeConfig(file, cfg)...) diff --git a/hcl2template/testdata/variables/basic.pkr.hcl b/hcl2template/testdata/variables/basic.pkr.hcl index c495ed741..8fb119b0c 100644 --- a/hcl2template/testdata/variables/basic.pkr.hcl +++ b/hcl2template/testdata/variables/basic.pkr.hcl @@ -29,6 +29,7 @@ variable "super_secret_password" { description = < 0 && (mapCt+dynamicCt) == len(types): + return unifyMapTypes(types, unsafe, dynamicCt > 0) case objectCt > 0 && (objectCt+dynamicCt) == len(types): return unifyObjectTypes(types, unsafe, dynamicCt > 0) case tupleCt > 0 && (tupleCt+dynamicCt) == len(types): @@ -95,6 +100,44 @@ Preferences: return cty.NilType, nil } +func unifyMapTypes(types []cty.Type, unsafe bool, hasDynamic bool) (cty.Type, []Conversion) { + // If we had any dynamic types in the input here then we can't predict + // what path we'll take through here once these become known types, so + // we'll conservatively produce DynamicVal for these. + if hasDynamic { + return unifyAllAsDynamic(types) + } + + elemTypes := make([]cty.Type, 0, len(types)) + for _, ty := range types { + elemTypes = append(elemTypes, ty.ElementType()) + } + retElemType, _ := unify(elemTypes, unsafe) + if retElemType == cty.NilType { + return cty.NilType, nil + } + + retTy := cty.Map(retElemType) + + conversions := make([]Conversion, len(types)) + for i, ty := range types { + if ty.Equals(retTy) { + continue + } + if unsafe { + conversions[i] = GetConversionUnsafe(ty, retTy) + } else { + conversions[i] = GetConversion(ty, retTy) + } + if conversions[i] == nil { + // Shouldn't be reachable, since we were able to unify + return cty.NilType, nil + } + } + + return retTy, conversions +} + func unifyObjectTypes(types []cty.Type, unsafe bool, hasDynamic bool) (cty.Type, []Conversion) { // If we had any dynamic types in the input here then we can't predict // what path we'll take through here once these become known types, so diff --git a/vendor/github.com/zclconf/go-cty/cty/function/stdlib/collection.go b/vendor/github.com/zclconf/go-cty/cty/function/stdlib/collection.go index 4b96f9dd3..b2ce062a6 100644 --- a/vendor/github.com/zclconf/go-cty/cty/function/stdlib/collection.go +++ b/vendor/github.com/zclconf/go-cty/cty/function/stdlib/collection.go @@ -299,7 +299,7 @@ var ContainsFunc = function.New(&function.Spec{ }, }, Type: function.StaticReturnType(cty.Bool), - Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { arg := args[0] ty := arg.Type() @@ -307,12 +307,39 @@ var ContainsFunc = function.New(&function.Spec{ return cty.NilVal, errors.New("argument must be list, tuple, or set") } - _, err = Index(cty.TupleVal(arg.AsValueSlice()), args[1]) - if err != nil { + if args[0].IsNull() { + return cty.NilVal, errors.New("cannot search a nil list or set") + } + + if args[0].LengthInt() == 0 { return cty.False, nil } - return cty.True, nil + if !args[0].IsKnown() || !args[1].IsKnown() { + return cty.UnknownVal(cty.Bool), nil + } + + containsUnknown := false + for it := args[0].ElementIterator(); it.Next(); { + _, v := it.Element() + eq := args[1].Equals(v) + if !eq.IsKnown() { + // We may have an unknown value which could match later, but we + // first need to continue checking all values for an exact + // match. + containsUnknown = true + continue + } + if eq.True() { + return cty.True, nil + } + } + + if containsUnknown { + return cty.UnknownVal(cty.Bool), nil + } + + return cty.False, nil }, }) @@ -566,19 +593,12 @@ var LookupFunc = function.New(&function.Spec{ Name: "key", Type: cty.String, }, - }, - VarParam: &function.Parameter{ - Name: "default", - Type: cty.DynamicPseudoType, - AllowUnknown: true, - AllowDynamicType: true, - AllowNull: true, + { + Name: "default", + Type: cty.DynamicPseudoType, + }, }, Type: func(args []cty.Value) (ret cty.Type, err error) { - if len(args) < 1 || len(args) > 3 { - return cty.NilType, fmt.Errorf("lookup() takes two or three arguments, got %d", len(args)) - } - ty := args[0].Type() switch { @@ -609,13 +629,7 @@ var LookupFunc = function.New(&function.Spec{ } }, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { - var defaultVal cty.Value - defaultValueSet := false - - if len(args) == 3 { - defaultVal = args[2] - defaultValueSet = true - } + defaultVal := args[2] mapVar := args[0] lookupKey := args[1].AsString() @@ -632,48 +646,128 @@ var LookupFunc = function.New(&function.Spec{ return mapVar.Index(cty.StringVal(lookupKey)), nil } - if defaultValueSet { - defaultVal, err = convert.Convert(defaultVal, retType) - if err != nil { - return cty.NilVal, err - } - return defaultVal, nil + defaultVal, err = convert.Convert(defaultVal, retType) + if err != nil { + return cty.NilVal, err } - - return cty.UnknownVal(cty.DynamicPseudoType), fmt.Errorf( - "lookup failed to find '%s'", lookupKey) + return defaultVal, nil }, }) -// MergeFunc is a function that takes an arbitrary number of maps and -// returns a single map that contains a merged set of elements from all of the maps. +// MergeFunc constructs a function that takes an arbitrary number of maps or +// objects, and returns a single value that contains a merged set of keys and +// values from all of the inputs. // -// If more than one given map defines the same key then the one that is later in -// the argument sequence takes precedence. +// If more than one given map or object defines the same key then the one that +// is later in the argument sequence takes precedence. var MergeFunc = function.New(&function.Spec{ Params: []function.Parameter{}, VarParam: &function.Parameter{ Name: "maps", Type: cty.DynamicPseudoType, AllowDynamicType: true, + AllowNull: true, + }, + Type: func(args []cty.Value) (cty.Type, error) { + // empty args is accepted, so assume an empty object since we have no + // key-value types. + if len(args) == 0 { + return cty.EmptyObject, nil + } + + // collect the possible object attrs + attrs := map[string]cty.Type{} + + first := cty.NilType + matching := true + attrsKnown := true + for i, arg := range args { + ty := arg.Type() + // any dynamic args mean we can't compute a type + if ty.Equals(cty.DynamicPseudoType) { + return cty.DynamicPseudoType, nil + } + + // check for invalid arguments + if !ty.IsMapType() && !ty.IsObjectType() { + return cty.NilType, fmt.Errorf("arguments must be maps or objects, got %#v", ty.FriendlyName()) + } + + switch { + case ty.IsObjectType() && !arg.IsNull(): + for attr, aty := range ty.AttributeTypes() { + attrs[attr] = aty + } + case ty.IsMapType(): + switch { + case arg.IsNull(): + // pass, nothing to add + case arg.IsKnown(): + ety := arg.Type().ElementType() + for it := arg.ElementIterator(); it.Next(); { + attr, _ := it.Element() + attrs[attr.AsString()] = ety + } + default: + // any unknown maps means we don't know all possible attrs + // for the return type + attrsKnown = false + } + } + + // record the first argument type for comparison + if i == 0 { + first = arg.Type() + continue + } + + if !ty.Equals(first) && matching { + matching = false + } + } + + // the types all match, so use the first argument type + if matching { + return first, nil + } + + // We had a mix of unknown maps and objects, so we can't predict the + // attributes + if !attrsKnown { + return cty.DynamicPseudoType, nil + } + + return cty.Object(attrs), nil }, - Type: function.StaticReturnType(cty.DynamicPseudoType), Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { outputMap := make(map[string]cty.Value) + // if all inputs are null, return a null value rather than an object + // with null attributes + allNull := true for _, arg := range args { - if !arg.IsWhollyKnown() { - return cty.UnknownVal(retType), nil - } - if !arg.Type().IsObjectType() && !arg.Type().IsMapType() { - return cty.NilVal, fmt.Errorf("arguments must be maps or objects, got %#v", arg.Type().FriendlyName()) + if arg.IsNull() { + continue + } else { + allNull = false } + for it := arg.ElementIterator(); it.Next(); { k, v := it.Element() outputMap[k.AsString()] = v } } - return cty.ObjectVal(outputMap), nil + + switch { + case allNull: + return cty.NullVal(retType), nil + case retType.IsMapType(): + return cty.MapVal(outputMap), nil + case retType.IsObjectType(), retType.Equals(cty.DynamicPseudoType): + return cty.ObjectVal(outputMap), nil + default: + panic(fmt.Sprintf("unexpected return type: %#v", retType)) + } }, }) @@ -1184,8 +1278,8 @@ func Keys(inputMap cty.Value) (cty.Value, error) { // Lookup performs a dynamic lookup into a map. // There are two required arguments, map and key, plus an optional default, // which is a value to return if no key is found in map. -func Lookup(args ...cty.Value) (cty.Value, error) { - return LookupFunc.Call(args) +func Lookup(inputMap, key, defaultValue cty.Value) (cty.Value, error) { + return LookupFunc.Call([]cty.Value{inputMap, key, defaultValue}) } // Merge takes an arbitrary number of maps and returns a single map that contains diff --git a/vendor/github.com/zclconf/go-cty/cty/function/stdlib/datetime.go b/vendor/github.com/zclconf/go-cty/cty/function/stdlib/datetime.go index a1aa5f4b5..3ce41ba9d 100644 --- a/vendor/github.com/zclconf/go-cty/cty/function/stdlib/datetime.go +++ b/vendor/github.com/zclconf/go-cty/cty/function/stdlib/datetime.go @@ -217,7 +217,7 @@ var TimeAddFunc = function.New(&function.Spec{ }, Type: function.StaticReturnType(cty.String), Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { - ts, err := time.Parse(time.RFC3339, args[0].AsString()) + ts, err := parseTimestamp(args[0].AsString()) if err != nil { return cty.UnknownVal(cty.String), err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 65491cc03..12fae6eca 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -638,7 +638,7 @@ github.com/yandex-cloud/go-sdk/pkg/retry github.com/yandex-cloud/go-sdk/pkg/sdkerrors github.com/yandex-cloud/go-sdk/pkg/singleflight github.com/yandex-cloud/go-sdk/sdkresolvers -# github.com/zclconf/go-cty v1.2.1 => github.com/azr/go-cty v1.1.1-0.20200203143058-28fcda2fe0cc +# github.com/zclconf/go-cty v1.3.1 github.com/zclconf/go-cty/cty github.com/zclconf/go-cty/cty/convert github.com/zclconf/go-cty/cty/function diff --git a/website/source/docs/configuration/from-1.5/variables.html.md b/website/source/docs/configuration/from-1.5/variables.html.md index dda0c11d2..809383226 100644 --- a/website/source/docs/configuration/from-1.5/variables.html.md +++ b/website/source/docs/configuration/from-1.5/variables.html.md @@ -278,3 +278,48 @@ precedence over earlier ones: ~> **Important:** Variables with map and object values behave the same way as other variables: the last value found overrides the previous values. + +### A variable value must be known : + +Take the following variable for example: + +``` hcl +variable "foo" { + type = string +``` + +Here `foo` must have a known value but you can default it to `null` to make +this behavior optional : + +| | no default | `default = null` | `default = "xy"` | +|:---------------------------:|:----------------------------:|:----------------:|:----------------:| +| foo unused | error, "foo needs to be set" | - | - | +| var.foo | error, "foo needs to be set" | null¹ | xy | +| `PKR_VAR_foo=yz`
var.foo | yz | yz | yz | +| `-var foo=yz`
var.foo | yz | yz | yz | + +1: Null is a valid value. Packer will only error when the receiving field needs +a value, example: + +``` hcl +variable "example" { + type = string + default = null +} + +source "example" "foo" { + arg = var.example +} +``` + +In the above case, as long as "arg" is optional for an "example" source, there is no error and arg won’t be set. + + +### Setting an unknown variable will not always fail : + +| Usage | packer validate | any other packer command | +|:------------------------------:|:-----------------------:|:-------------------------:| +| `bar=yz` in .pkrvars.hcl file. | error, "bar undeclared" | warning, "bar undeclared" | +| `var.bar` in .pkr.hcl file | error, "bar undeclared" | error, "bar undeclared" | +| `-var bar=yz` argument | error, "bar undeclared" | error, "bar undeclared" | +| `export PKR_VAR_bar=yz` | - | - |