Some new features and bugfixes

- Fixed parse bug which would result in an 'unrecognised }' when a comment appeared before a }
- Added support for ${} variables interpolation in strings
- Added support for $() for sub-expression interoplation in strings
- Fixed bug which was preventing dot dereferencing in array and hash literals
- Defined error type for when the result is not convertable to go

ucl/ast.go

@@ -14,10 +14,12 @@ type astStringStringSpan struct {
 }
 
 type astDoubleStringSpan struct {
-	Pos      lexer.Position
+	Pos          lexer.Position
-	Chars    *string `parser:"@Char"`
+	Chars        *string      `parser:"@Char"`
-	Escaped  *string `parser:"| @Escaped"`
+	Escaped      *string      `parser:"| @Escaped"`
-	IdentRef *string `parser:"| @IdentRef"`
+	IdentRef     *string      `parser:"| @IdentRef"`
+	LongIdentRef *string      `parser:"| @LongIdentRef"`
+	SubExpr      *astPipeline `parser:"| StartSubExpr @@ RP"`
 }
 
 type astDoubleString struct {
@@ -50,8 +52,8 @@ func (ai *astIdentNames) String() string {
 }
 
 type astElementPair struct {
-	Left  astCmdArg  `parser:"@@"`
+	Left  astDot  `parser:"@@"`
-	Right *astCmdArg `parser:"( COLON @@ )? NL?"`
+	Right *astDot `parser:"( COLON @@ )? NL?"`
 }
 
 type astListOrHash struct {
@@ -111,15 +113,15 @@ type astScript struct {
 var scanner = lexer.MustStateful(lexer.Rules{
 	"Root": {
 		{"Whitespace", `[ \t]+`, nil},
-		{"Comment", `[#].*`, nil},
+		{"Comment", `[#].*\s*`, nil},
 		{"StringStart", `"`, lexer.Push("String")},
 		{"SingleStringStart", `'`, lexer.Push("SingleString")},
 		{"Int", `[-]?[0-9][0-9]*`, nil},
 		{"DOLLAR", `\$`, nil},
 		{"COLON", `\:`, nil},
 		{"DOT", `[.]`, nil},
-		{"LP", `\(`, nil},
+		{"LP", `\(`, lexer.Push("Root")},
-		{"RP", `\)`, nil},
+		{"RP", `\)`, lexer.Pop()},
 		{"LS", `\[`, nil},
 		{"RS", `\]`, nil},
 		{"LC", `\{`, nil},
@@ -132,6 +134,8 @@ var scanner = lexer.MustStateful(lexer.Rules{
 		{"Escaped", `\\.`, nil},
 		{"StringEnd", `"`, lexer.Pop()},
 		{"IdentRef", `\$[-]*[a-zA-Z_][\w-]*`, nil},
+		{"LongIdentRef", `\$[{][^}]*[}]`, nil},
+		{"StartSubExpr", `\$[(]`, lexer.Push("Root")},
 		{"Char", `[^$"\\]+`, nil},
 	},
 	"SingleString": {

ucl/errors.go

@@ -1,10 +1,15 @@
 package ucl
 
 import (
+	"errors"
 	"fmt"
 	"github.com/alecthomas/participle/v2/lexer"
 )
 
+var (
+	ErrNotConvertable = errors.New("result not convertable to go")
+)
+
 var (
 	tooManyFinallyBlocksError = newBadUsage("try needs at most 1 finally")
 )

ucl/eval.go

@@ -214,12 +214,12 @@ func (e evaluator) evalListOrHash(ctx context.Context, ec *evalCtx, loh *astList
 				return nil, errors.New("miss-match of lists and hash")
 			}
 
-			n, err := e.evalArg(ctx, ec, el.Left)
+			n, err := e.evalDot(ctx, ec, el.Left)
 			if err != nil {
 				return nil, err
 			}
 
-			v, err := e.evalArg(ctx, ec, *el.Right)
+			v, err := e.evalDot(ctx, ec, *el.Right)
 			if err != nil {
 				return nil, err
 			}
@@ -234,7 +234,7 @@ func (e evaluator) evalListOrHash(ctx context.Context, ec *evalCtx, loh *astList
 		if el.Right != nil {
 			return nil, errors.New("miss-match of lists and hash")
 		}
-		v, err := e.evalArg(ctx, ec, el.Left)
+		v, err := e.evalDot(ctx, ec, el.Left)
 		if err != nil {
 			return nil, err
 		}
@@ -298,6 +298,19 @@ func (e evaluator) interpolateDoubleQuotedString(ctx context.Context, ec *evalCt
 			if v, ok := ec.getVar(identVal); ok && v != nil {
 				sb.WriteString(v.String())
 			}
+		case n.LongIdentRef != nil:
+			identVal := (*n.LongIdentRef)[2 : len(*n.LongIdentRef)-1]
+			if v, ok := ec.getVar(identVal); ok && v != nil {
+				sb.WriteString(v.String())
+			}
+		case n.SubExpr != nil:
+			res, err := e.evalPipeline(ctx, ec, n.SubExpr)
+			if err != nil {
+				return nil, err
+			}
+			if res != nil {
+				sb.WriteString(res.String())
+			}
 		}
 	}
 	return StringObject(sb.String()), nil

ucl/inst.go

@@ -137,7 +137,7 @@ func (inst *Inst) Eval(ctx context.Context, expr string) (any, error) {
 
 	goRes, ok := toGoValue(res)
 	if !ok {
-		return nil, errors.New("result not convertable to go")
+		return nil, ErrNotConvertable
 	}
 
 	return goRes, nil

ucl/inst_test.go

@@ -11,15 +11,26 @@ import (
 
 func TestInst_Eval(t *testing.T) {
 	tests := []struct {
-		desc string
+		desc    string
-		expr string
+		expr    string
-		want any
+		want    any
+		wantErr error
 	}{
 		{desc: "simple string", expr: `firstarg "hello"`, want: "hello"},
 		{desc: "simple int 1", expr: `firstarg 123`, want: 123},
 		{desc: "simple int 2", expr: `firstarg -234`, want: -234},
 		{desc: "simple ident", expr: `firstarg a-test`, want: "a-test"},
 
+		// String interpolation
+		{desc: "interpolate string 1", expr: `set what "world" ; firstarg "hello $what"`, want: "hello world"},
+		{desc: "interpolate string 2", expr: `set what "world" ; set when "now" ; firstarg "$when, hello $what"`, want: "now, hello world"},
+		{desc: "interpolate string 3", expr: `set what "world" ; set when "now" ; firstarg "${when}, hello ${what}"`, want: "now, hello world"},
+		{desc: "interpolate string 4", expr: `set "crazy var" "unknown" ; firstarg "hello ${crazy var}"`, want: "hello unknown"},
+		{desc: "interpolate string 5", expr: `set what "world" ; firstarg "hello $($what)"`, want: "hello world"},
+		{desc: "interpolate string 6", expr: `firstarg "hello $([1 2 3] | len)"`, want: "hello 3"},
+		{desc: "interpolate string 7", expr: `firstarg "hello $(add (add 1 2) 3)"`, want: "hello 6"},
+		{desc: "interpolate string 8", expr: `firstarg ("$(add 2 (add 1 1)) + $([1 2 3].(1) | cat ("$("")")) = $(("$(add 2 (4))"))")`, want: "4 + 2 = 6"},
+
 		// Sub-expressions
 		{desc: "sub expression 1", expr: `firstarg (sjoin "hello")`, want: "hello"},
 		{desc: "sub expression 2", expr: `firstarg (sjoin "hello " "world")`, want: "hello world"},
@@ -48,6 +59,7 @@ func TestInst_Eval(t *testing.T) {
 		{desc: "list 1", expr: `firstarg ["1" "2" "3"]`, want: []any{"1", "2", "3"}},
 		{desc: "list 2", expr: `set one "one" ; firstarg [$one (list "two" | map { |x| toUpper $x } | head) "three"]`, want: []any{"one", "TWO", "three"}},
 		{desc: "list 3", expr: `firstarg []`, want: []any{}},
+		{desc: "list 4", expr: `set x ["a" "b" "c"] ; firstarg [$x.(2) $x.(1) $x.(0)]`, want: []any{"c", "b", "a"}},
 
 		// Maps
 		{desc: "map 1", expr: `firstarg [one:"1" two:"2" three:"3"]`, want: map[string]any{"one": "1", "two": "2", "three": "3"}},
@@ -60,6 +72,8 @@ func TestInst_Eval(t *testing.T) {
 				three:"3"
 			]`, want: map[string]any{"one": "1", "TWO": "2", "three": "3"}},
 		{desc: "map 4", expr: `firstarg [:]`, want: map[string]any{}},
+		{desc: "map 5", expr: `set x ["a" "b" "c"] ; firstarg ["one":$x.(2) "two":$x.(1) "three":$x.(0)]`, want: map[string]any{"one": "c", "two": "b", "three": "a"}},
+		{desc: "map 6", expr: `set x [a:"A" b:"B" c:"C"] ; firstarg ["one":$x.c "two":$x.b "three":$x.a]`, want: map[string]any{"one": "C", "two": "B", "three": "A"}},
 
 		// Dots
 		{desc: "dot 1", expr: `set x [1 2 3] ; $x.(0)`, want: 1},
@@ -76,6 +90,11 @@ func TestInst_Eval(t *testing.T) {
 		{desc: "dot 12", expr: `set x [MORE:"stuff"] ; $x.("more" | toUpper)`, want: "stuff"},
 		{desc: "dot 13", expr: `set x [MORE:"stuff"] ; $x.(toUpper ("more"))`, want: "stuff"},
 		{desc: "dot 14", expr: `set x [MORE:"stuff"] ; x.y`, want: nil},
+
+		{desc: "parse comments 1", expr: parseComments1, wantErr: ucl.ErrNotConvertable},
+		{desc: "parse comments 2", expr: parseComments2, wantErr: ucl.ErrNotConvertable},
+		{desc: "parse comments 3", expr: parseComments3, wantErr: ucl.ErrNotConvertable},
+		{desc: "parse comments 4", expr: parseComments4, wantErr: ucl.ErrNotConvertable},
 	}
 
 	for _, tt := range tests {
@@ -86,8 +105,51 @@ func TestInst_Eval(t *testing.T) {
 			inst := ucl.New(ucl.WithOut(outW), ucl.WithTestBuiltin())
 			res, err := inst.Eval(ctx, tt.expr)
 
-			assert.NoError(t, err)
+			if tt.wantErr != nil {
-			assert.Equal(t, tt.want, res)
+				assert.ErrorIs(t, err, tt.wantErr)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.want, res)
+			}
 		})
 	}
 }
+
+var parseComments1 = `
+	proc lookup { |file|
+		foreach { |toks|
+		}
+		# this use to fail
+	}
+`
+
+var parseComments2 = `
+	proc lookup { |file|
+		foreach { |toks|
+		}
+
+		# this use to fail
+		#
+		# And so did this
+
+	}
+`
+
+var parseComments3 = `
+	proc lookup { |file|
+		foreach { |toks|
+		}
+
+		# this use to fail
+		#
+		# And so did this
+    
+	}
+`
+
+var parseComments4 = `
+	proc lookup { |file|
+		foreach { |toks|
+		}	
+	}
+# this use to fail`

ucl/testbuiltins_test.go

@@ -785,6 +785,41 @@ func TestBuiltins_Return(t *testing.T) {
 				}
 			}
 
+			set hello "xx"
+			foreach (test-thing) { |y| call $y ; echo $hello }
+		`, want: "1\nxx\n2\nxx\n3\nxx\n(nil)\n"},
+		{desc: "check closure 7", expr: `
+			proc do-thing { |p|
+				call $p
+			}
+
+			proc test-thing {
+				set f 0
+				[1 2 3] | map { |x|
+					set myProc (proc { echo $f })
+					set f (add $f 1)
+					proc { do-thing $myProc }	
+				}
+			}
+
+			set hello "xx"
+			foreach (test-thing) { |y| call $y ; echo $hello }
+		`, want: "3\nxx\n3\nxx\n3\nxx\n(nil)\n"},
+		{desc: "check closure 7", expr: `
+			proc do-thing { |p|
+				call $p
+			}
+
+			proc test-thing {
+				set f 1
+				[1 2 3] | map { |x|
+					set g $f
+					set myProc (proc { echo $g })	
+					set f (add $f 1)
+					proc { do-thing $myProc }	
+				}
+			}
+
 			set hello "xx"
 			foreach (test-thing) { |y| call $y ; echo $hello }
 		`, want: "1\nxx\n2\nxx\n3\nxx\n(nil)\n"},
@@ -1504,6 +1539,7 @@ func TestBuiltins_Cat(t *testing.T) {
 		{desc: "cat 6", expr: `cat "array = " [1 3 2 4]`, want: "array = [1 3 2 4]"},
 		{desc: "cat 7", expr: `cat 1 $true 3 [4]`, want: "1true3[4]"},
 		{desc: "cat 8", expr: `cat`, want: ""},
+		{desc: "cat 9", expr: `set x ["a" "b" "c"] ; cat "array = " [1 $x.(0) $x.(2) $x.(1)]`, want: "array = [1 a c b]"},
 	}
 
 	for _, tt := range tests {