Added iterators

Iterators are an unbounded sequence of elements that can only be consumed one-by-one.

_docs/mod/core.md

@@ -74,14 +74,17 @@ Returns the length of COL. If COL is a list or hash, COL will be the number of
 elements. If COL is a string, COL will be the string's length. All other values will
 return a length of 0.
 
+If COL is an iterator, `len` will consume the values of the iterator and return the number of items consumed. 
+
 ### map
 
 ```
 map COL BLOCK
 ```
 
-Returns a new list of elements mapped from COL according to the result of BLOCK. COL can be any listable data
+Returns a new list of elements mapped from COL according to the result of BLOCK. COL can be any list or hash
-structure, however the result will always be a concrete list.
+with the result being a concrete list. COL can be an iterator, in which case the result will be an iterator
+which will call BLOCK for every consumed value.
 
 ```
 map [1 2 3] { |x| str $x | len }
@@ -107,7 +110,7 @@ reduce COL [INIT] BLOCK
 Returns the result of reducing the elements of COL with the passed in block.
 
 BLOCK will receive at least two argument, with the current value of the accumulator always being the last argument.
-If COL is a list, the arguments will be _|element accumulator|_, and if COL is a hash, the arguments will be
+If COL is a list or iterator, the arguments will be _|element accumulator|_, and if COL is a hash, the arguments will be
 _|key value accumulator|_. 
 
 The block result will be set as the value of the accumulator for the next iteration. Once all elements are process

_docs/mod/index.md

@@ -5,4 +5,5 @@ Modules of the standard library:
 - [core](/mod/core): Core builtins
 - [csv](/mod/csv): Functions for operating over CSV data.
 - [fs](/mod/fs): File system functions
+- [itrs](/mod/itrs): Iterator utilities
 - [os](/mod/os): Operating system functions 

_docs/mod/itrs.md

@@ -0,0 +1,20 @@
+---
+---
+
+# Iterator Builtins
+
+### from
+
+```
+itrs:from LIST
+```
+
+Returns an iterator which will step through the elements of LIST.
+
+### to-list
+
+```
+lists:to-list ITR
+```
+
+Consume the elements of the iterator ITR and return the elements as a list.

cmd/cmsh/main.go

@@ -21,6 +21,7 @@ func main() {
 		ucl.WithModule(builtins.CSV(nil)),
 		ucl.WithModule(builtins.FS(nil)),
 		ucl.WithModule(builtins.Log(nil)),
+		ucl.WithModule(builtins.Itrs()),
 		ucl.WithModule(builtins.OS()),
 		ucl.WithModule(builtins.Strs()),
 		ucl.WithModule(builtins.Time()),

cmd/playwasm/jsiter.go

@@ -26,6 +26,8 @@ func initJS(ctx context.Context) {
 		ucl.WithModule(builtins.Log(nil)),
 		ucl.WithModule(builtins.Strs()),
 		ucl.WithModule(builtins.Time()),
+		ucl.WithModule(builtins.Itrs()),
+		ucl.WithModule(builtins.Lists()),
 		ucl.WithOut(ucl.LineHandler(func(line string) {
 			invokeUCLCallback("onOutLine", line)
 		})),

ucl/builtins.go

@@ -273,7 +273,7 @@ func andBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 
 	for _, a := range args.args {
 		if a == nil || !a.Truthy() {
-			return boolObject(false), nil
+			return a, nil
 		}
 	}
 	return args.args[len(args.args)-1], nil
@@ -284,12 +284,12 @@ func orBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 		return nil, err
 	}
 
-	for _, a := range args.args {
+	for _, a := range args.args[:len(args.args)-1] {
 		if a != nil && a.Truthy() {
 			return a, nil
 		}
 	}
-	return boolObject(false), nil
+	return args.args[len(args.args)-1], nil
 }
 
 func notBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
@@ -452,6 +452,16 @@ func lenBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 		return IntObject(v.Len()), nil
 	case hashable:
 		return IntObject(v.Len()), nil
+	case Iterable:
+		cnt := 0
+		for v.HasNext() {
+			_, err := v.Next(ctx)
+			if err != nil {
+				return nil, err
+			}
+			cnt++
+		}
+		return IntObject(cnt), nil
 	}
 
 	return IntObject(0), nil
@@ -503,19 +513,45 @@ func keysBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 	val := args.args[0]
 	switch v := val.(type) {
 	case hashable:
-		keys := make(listObject, 0, v.Len())
+		keys := make(ListObject, 0, v.Len())
 		if err := v.Each(func(k string, _ Object) error {
 			keys = append(keys, StringObject(k))
 			return nil
 		}); err != nil {
 			return nil, err
 		}
-		return keys, nil
+		return &keys, nil
 	}
 
 	return nil, nil
 }
 
+type mappedIter struct {
+	src  Iterable
+	inv  invokable
+	args invocationArgs
+}
+
+func (mi mappedIter) String() string {
+	return "mappedIter{}"
+}
+
+func (mi mappedIter) Truthy() bool {
+	return mi.src.HasNext()
+}
+
+func (mi mappedIter) HasNext() bool {
+	return mi.src.HasNext()
+}
+
+func (mi mappedIter) Next(ctx context.Context) (Object, error) {
+	v, err := mi.src.Next(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return mi.inv.invoke(ctx, mi.args.fork([]Object{v}))
+}
+
 func mapBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 	if err := args.expectArgn(2); err != nil {
 		return nil, err
@@ -529,7 +565,7 @@ func mapBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 	switch t := args.args[0].(type) {
 	case Listable:
 		l := t.Len()
-		newList := listObject{}
+		newList := ListObject{}
 		for i := 0; i < l; i++ {
 			v := t.Index(i)
 			m, err := inv.invoke(ctx, args.fork([]Object{v}))
@@ -538,12 +574,186 @@ func mapBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 			}
 			newList = append(newList, m)
 		}
-		return newList, nil
+		return &newList, nil
+	case Iterable:
+		return mappedIter{src: t, inv: inv, args: args}, nil
 	}
 	return nil, errors.New("expected listable")
 }
 
-func firstBuiltin(ctx context.Context, args invocationArgs) (object, error) {
+type filterIter struct {
+	src  Iterable
+	inv  invokable
+	args invocationArgs
+
+	hasNext bool
+	next    Object
+	err     error
+}
+
+func (mi *filterIter) prime(ctx context.Context) {
+	for mi.src.HasNext() {
+		v, err := mi.src.Next(ctx)
+		if err != nil {
+			mi.err = err
+			mi.hasNext = false
+			return
+		}
+
+		fv, err := mi.inv.invoke(ctx, mi.args.fork([]Object{v}))
+		if err != nil {
+			mi.err = err
+			mi.hasNext = false
+			return
+		} else if isTruthy(fv) {
+			mi.next = v
+			mi.hasNext = true
+			return
+		}
+	}
+	mi.hasNext = false
+	mi.err = nil
+}
+
+func (mi *filterIter) String() string {
+	return "filterIter{}"
+}
+
+func (mi *filterIter) Truthy() bool {
+	return mi.HasNext()
+}
+
+func (mi *filterIter) HasNext() bool {
+	return mi.hasNext
+}
+
+func (mi *filterIter) Next(ctx context.Context) (Object, error) {
+	next, err := mi.next, mi.err
+	mi.prime(ctx)
+	return next, err
+}
+
+func filterBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
+	if err := args.expectArgn(2); err != nil {
+		return nil, err
+	}
+
+	inv, err := args.invokableArg(1)
+	if err != nil {
+		return nil, err
+	}
+
+	switch t := args.args[0].(type) {
+	case Listable:
+		l := t.Len()
+		newList := ListObject{}
+		for i := 0; i < l; i++ {
+			v := t.Index(i)
+			m, err := inv.invoke(ctx, args.fork([]Object{v}))
+			if err != nil {
+				return nil, err
+			} else if m != nil && m.Truthy() {
+				newList = append(newList, v)
+			}
+		}
+		return &newList, nil
+	case hashable:
+		newHash := hashObject{}
+		if err := t.Each(func(k string, v Object) error {
+			if m, err := inv.invoke(ctx, args.fork([]Object{StringObject(k), v})); err != nil {
+				return err
+			} else if m != nil && m.Truthy() {
+				newHash[k] = v
+			}
+			return nil
+		}); err != nil {
+			return nil, err
+		}
+		return newHash, nil
+	case Iterable:
+		fi := &filterIter{src: t, inv: inv, args: args}
+		fi.prime(ctx)
+		return fi, nil
+	}
+	return nil, errors.New("expected listable")
+}
+
+func reduceBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
+	var err error
+	if err = args.expectArgn(2); err != nil {
+		return nil, err
+	}
+
+	var (
+		accum    Object
+		setFirst bool
+		block    invokable
+	)
+	if len(args.args) == 3 {
+		accum = args.args[1]
+		block, err = args.invokableArg(2)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		setFirst = true
+		block, err = args.invokableArg(1)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	switch t := args.args[0].(type) {
+	case Listable:
+		l := t.Len()
+		for i := 0; i < l; i++ {
+			v := t.Index(i)
+			if setFirst {
+				accum = v
+				setFirst = false
+				continue
+			}
+
+			newAccum, err := block.invoke(ctx, args.fork([]Object{v, accum}))
+			if err != nil {
+				return nil, err
+			}
+
+			accum = newAccum
+		}
+		return accum, nil
+	case hashable:
+		// TODO: should raise error?
+		if err := t.Each(func(k string, v Object) error {
+			newAccum, err := block.invoke(ctx, args.fork([]Object{StringObject(k), v, accum}))
+			if err != nil {
+				return err
+			}
+			accum = newAccum
+			return nil
+		}); err != nil {
+			return nil, err
+		}
+		return accum, nil
+	case Iterable:
+		for t.HasNext() {
+			v, err := t.Next(ctx)
+			if err != nil {
+				return nil, err
+			}
+
+			newAccum, err := block.invoke(ctx, args.fork([]Object{v, accum}))
+			if err != nil {
+				return nil, err
+			}
+			accum = newAccum
+		}
+		return accum, nil
+	}
+	return nil, errors.New("expected listable")
+}
+
+func firstBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 	if err := args.expectArgn(1); err != nil {
 		return nil, err
 	}
@@ -554,6 +764,11 @@ func firstBuiltin(ctx context.Context, args invocationArgs) (object, error) {
 			return nil, nil
 		}
 		return t.Index(0), nil
+	case Iterable:
+		if t.HasNext() {
+			return t.Next(ctx)
+		}
+		return nil, nil
 	}
 	return nil, errors.New("expected listable")
 }
@@ -720,6 +935,25 @@ func foreachBuiltin(ctx context.Context, args macroArgs) (object, error) {
 		} else {
 			return nil, err
 		}
+	case Iterable:
+		for t.HasNext() {
+			v, err := t.Next(ctx)
+			if err != nil {
+				return nil, err
+			}
+
+			last, err = args.evalBlock(ctx, blockIdx, []Object{v}, true) // TO INCLUDE: the index
+			if err != nil {
+				if errors.As(err, &breakErr) {
+					if !breakErr.isCont {
+						return breakErr.ret, nil
+					}
+				} else {
+					return nil, err
+				}
+			}
+		}
+		return nil, nil
 	}
 
 	return last, nil

ucl/builtins/itrs.go

@@ -0,0 +1,68 @@
+package builtins
+
+import (
+	"context"
+	"ucl.lmika.dev/ucl"
+)
+
+func Itrs() ucl.Module {
+	return ucl.Module{
+		Name: "itrs",
+		Builtins: map[string]ucl.BuiltinHandler{
+			"from":    iterFrom,
+			"to-list": iterToList,
+		},
+	}
+}
+
+func iterFrom(ctx context.Context, args ucl.CallArgs) (any, error) {
+	var listable ucl.Listable
+
+	if err := args.Bind(&listable); err != nil {
+		return nil, err
+	}
+
+	return &fromIterator{listable: listable}, nil
+}
+
+type fromIterator struct {
+	idx      int
+	listable ucl.Listable
+}
+
+func (f *fromIterator) String() string {
+	return "fromIterator{}"
+}
+
+func (f *fromIterator) HasNext() bool {
+	return f.idx < f.listable.Len()
+}
+
+func (f *fromIterator) Next(ctx context.Context) (ucl.Object, error) {
+	if f.idx >= f.listable.Len() {
+		return nil, nil
+	}
+	v := f.listable.Index(f.idx)
+	f.idx++
+	return v, nil
+}
+
+func iterToList(ctx context.Context, args ucl.CallArgs) (any, error) {
+	var itr ucl.Iterable
+	if err := args.Bind(&itr); err != nil {
+		return nil, err
+	}
+
+	target := ucl.NewListObject()
+	for itr.HasNext() {
+		v, err := itr.Next(ctx)
+		if err != nil {
+			return nil, err
+		}
+		if err := target.Insert(-1, v); err != nil {
+			return nil, err
+		}
+	}
+
+	return target, nil
+}

ucl/builtins/itrs_test.go

@@ -0,0 +1,38 @@
+package builtins_test
+
+import (
+	"context"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"ucl.lmika.dev/ucl"
+	"ucl.lmika.dev/ucl/builtins"
+)
+
+func TestItrs_ToList(t *testing.T) {
+	tests := []struct {
+		desc    string
+		eval    string
+		want    any
+		wantErr bool
+	}{
+		{desc: "to-list 1", eval: `itrs:from (seq 5) | itrs:to-list`, want: []any{0, 1, 2, 3, 4}},
+		{desc: "to-list 2", eval: `itrs:from (seq 10) | filter { |x| eq (mod $x 2) 0 } | itrs:to-list`, want: []any{0, 2, 4, 6, 8}},
+		{desc: "to-list 3", eval: `itrs:from (seq 10) | filter { |x| eq (mod $x 2) 0 } | map { |x| (add $x 2) } | itrs:to-list`, want: []any{2, 4, 6, 8, 10}},
+		{desc: "to-list 4", eval: `itrs:from (seq 10) | filter { |x| () } | map { |x| (add $x 2) } | itrs:to-list`, want: []any{}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			inst := ucl.New(
+				ucl.WithModule(builtins.Itrs()),
+			)
+			res, err := inst.Eval(context.Background(), tt.eval)
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.want, res)
+			}
+		})
+	}
+}

ucl/objs.go

@@ -22,32 +22,74 @@ type Listable interface {
 	Index(i int) Object
 }
 
+type Iterable interface {
+	HasNext() bool
+
+	// Next returns the next object from the iterable if one exists, otherwise
+	// returns nil, false.
+	Next(ctx context.Context) (Object, error)
+}
+
+type ModListable interface {
+	Listable
+
+	// Insert adds a new item to the list. idx can be a positive
+	// number from 0 to len(), in which case the object will be inserted
+	// at that position.  If idx is negative, then the item will be inserted
+	// at that position from the right.
+	Insert(idx int, obj Object) error
+}
+
 type hashable interface {
 	Len() int
 	Value(k string) Object
 	Each(func(k string, v Object) error) error
 }
 
-type listObject []Object
+type ListObject []Object
 
-func (lo *listObject) Append(o Object) {
+func NewListObject() *ListObject {
+	return &ListObject{}
+}
+
+func (lo *ListObject) Append(o Object) {
 	*lo = append(*lo, o)
 }
 
-func (s listObject) String() string {
+func (lo *ListObject) Insert(idx int, obj Object) error {
-	return fmt.Sprintf("%v", []Object(s))
+	if idx != -1 {
+		return errors.New("not supported")
+	}
+	*lo = append(*lo, obj)
+	return nil
 }
 
-func (s listObject) Truthy() bool {
+func (s *ListObject) String() string {
-	return len(s) > 0
+	return fmt.Sprintf("%v", []Object(*s))
 }
 
-func (s listObject) Len() int {
+func (s *ListObject) Truthy() bool {
-	return len(s)
+	return len(*s) > 0
 }
 
-func (s listObject) Index(i int) Object {
+func (s *ListObject) Len() int {
-	return s[i]
+	return len(*s)
+}
+
+func (s *ListObject) Index(i int) Object {
+	return (*s)[i]
+}
+
+type iteratorObject struct {
+	Iterable
+}
+
+func (i iteratorObject) String() string {
+	return "iterator{}"
+}
+
+func (i iteratorObject) Truthy() bool {
+	return i.Iterable.HasNext()
 }
 
 type hashObject map[string]Object
@@ -147,9 +189,9 @@ func toGoValue(obj Object) (interface{}, bool) {
 		return bool(v), true
 	case timeObject:
 		return time.Time(v), true
-	case listObject:
+	case *ListObject:
-		xs := make([]interface{}, 0, len(v))
+		xs := make([]interface{}, 0, len(*v))
-		for _, va := range v {
+		for _, va := range *v {
 			x, ok := toGoValue(va)
 			if !ok {
 				continue
@@ -167,6 +209,10 @@ func toGoValue(obj Object) (interface{}, bool) {
 			xs[k] = x
 		}
 		return xs, true
+	case iteratorObject:
+		return v.Iterable, true
+	case Iterable:
+		return v, true
 	case proxyObject:
 		return v.p, true
 	case listableProxyObject:
@@ -182,6 +228,8 @@ func fromGoValue(v any) (Object, error) {
 	switch t := v.(type) {
 	case Object:
 		return t, nil
+	case Iterable:
+		return iteratorObject{t}, nil
 	case nil:
 		return nil, nil
 	case string:
@@ -331,7 +379,7 @@ type invocationArgs struct {
 	inst   *Inst
 	ec     *evalCtx
 	args   []Object
-	kwargs map[string]*listObject
+	kwargs map[string]*ListObject
 }
 
 func (ia invocationArgs) expectArgn(x int) error {
@@ -389,7 +437,7 @@ func (ia invocationArgs) fork(args []Object) invocationArgs {
 		inst:   ia.inst,
 		ec:     ia.ec,
 		args:   args,
-		kwargs: make(map[string]*listObject),
+		kwargs: make(map[string]*ListObject),
 	}
 }
 

ucl/testbuiltins_test.go

@@ -9,20 +9,35 @@ import (
 	"testing"
 )
 
+type testIterator struct {
+	cnt int
+	max int
+	err error
+}
+
+func (ti *testIterator) HasNext() bool {
+	return ti.cnt < ti.max
+}
+
+func (ti *testIterator) Next(ctx context.Context) (Object, error) {
+	ti.cnt++
+	return IntObject(ti.cnt), nil
+}
+
 // Builtins used for test
 func WithTestBuiltin() InstOption {
 	return func(i *Inst) {
-		i.rootEC.addCmd("firstarg", invokableFunc(func(ctx context.Context, args invocationArgs) (object, error) {
+		i.rootEC.addCmd("firstarg", invokableFunc(func(ctx context.Context, args invocationArgs) (Object, error) {
 			return args.args[0], nil
 		}))
 
-		i.rootEC.addCmd("toUpper", invokableFunc(func(ctx context.Context, args invocationArgs) (object, error) {
+		i.rootEC.addCmd("toUpper", invokableFunc(func(ctx context.Context, args invocationArgs) (Object, error) {
-			return strObject(strings.ToUpper(args.args[0].String())), nil
+			return StringObject(strings.ToUpper(args.args[0].String())), nil
 		}))
 
-		i.rootEC.addCmd("sjoin", invokableFunc(func(ctx context.Context, args invocationArgs) (object, error) {
+		i.rootEC.addCmd("sjoin", invokableFunc(func(ctx context.Context, args invocationArgs) (Object, error) {
 			if len(args.args) == 0 {
-				return strObject(""), nil
+				return StringObject(""), nil
 			}
 
 			var line strings.Builder
@@ -32,19 +47,28 @@ func WithTestBuiltin() InstOption {
 				}
 			}
 
-			return strObject(line.String()), nil
+			return StringObject(line.String()), nil
 		}))
 
-		i.rootEC.addCmd("list", invokableFunc(func(ctx context.Context, args invocationArgs) (object, error) {
+		i.rootEC.addCmd("list", invokableFunc(func(ctx context.Context, args invocationArgs) (Object, error) {
-			return listObject(args.args), nil
+			var a ListObject = make([]Object, len(args.args))
+			copy(a, args.args)
+			return &a, nil
 		}))
 
-		i.rootEC.addCmd("joinpipe", invokableFunc(func(ctx context.Context, args invocationArgs) (object, error) {
+		i.rootEC.addCmd("error", invokableFunc(func(ctx context.Context, args invocationArgs) (Object, error) {
+			if len(args.args) == 0 {
+				return nil, errors.New("an error occurred")
+			}
+			return nil, errors.New(args.args[0].String())
+		}))
+
+		i.rootEC.addCmd("joinpipe", invokableFunc(func(ctx context.Context, args invocationArgs) (Object, error) {
 			sb := strings.Builder{}
 
-			lst, ok := args.args[0].(listable)
+			lst, ok := args.args[0].(Listable)
 			if !ok {
-				return strObject(""), nil
+				return StringObject(""), nil
 			}
 
 			l := lst.Len()
@@ -54,11 +78,15 @@ func WithTestBuiltin() InstOption {
 				}
 				sb.WriteString(lst.Index(x).String())
 			}
-			return strObject(sb.String()), nil
+			return StringObject(sb.String()), nil
 		}))
 
-		i.rootEC.setOrDefineVar("a", strObject("alpha"))
+		i.rootEC.addCmd("itr", invokableFunc(func(ctx context.Context, args invocationArgs) (Object, error) {
-		i.rootEC.setOrDefineVar("bee", strObject("buzz"))
+			return iteratorObject{Iterable: &testIterator{max: 3}}, nil
+		}))
+
+		i.rootEC.setOrDefineVar("a", StringObject("alpha"))
+		i.rootEC.setOrDefineVar("bee", StringObject("buzz"))
 	}
 }
 
@@ -187,6 +215,12 @@ func TestBuiltins_If(t *testing.T) {
 		{desc: "compressed then", expr: `set x "Hello" ; if $x { echo "true" }`, want: "true\n(nil)\n"},
 		{desc: "compressed else", expr: `if $x { echo "true" } else { echo "false" }`, want: "false\n(nil)\n"},
 		{desc: "compressed if", expr: `if $x { echo "x" } elif $y { echo "y" } else { echo "false" }`, want: "false\n(nil)\n"},
+		{desc: "if of itr 1", expr: `set i (itr) ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"},
+		{desc: "if of itr 2", expr: `set i (itr) ; foreach (seq 1) { head $i } ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"},
+		{desc: "if of itr 3", expr: `set i (itr) ; foreach (seq 3) { head $i } ; if $i { echo "more" } else { echo "none" }`, want: "none\n(nil)\n"},
+		{desc: "if of itr 4", expr: `set i (itr | map { |x| add 2 $x }) ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"},
+		{desc: "if of itr 5", expr: `set i (itr | filter { |x| () }) ; if $i { echo "more" } else { echo "none" }`, want: "none\n(nil)\n"},
+		{desc: "if of itr 6", expr: `set i (itr | filter { |x| 1 }) ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"},
 	}
 
 	for _, tt := range tests {
@@ -195,7 +229,7 @@ func TestBuiltins_If(t *testing.T) {
 			outW := bytes.NewBuffer(nil)
 
 			inst := New(WithOut(outW), WithTestBuiltin())
-			err := EvalAndDisplay(ctx, inst, tt.expr)
+			err := evalAndDisplay(ctx, inst, tt.expr)
 
 			assert.NoError(t, err)
 			assert.Equal(t, tt.want, outW.String())
@@ -203,6 +237,7 @@ func TestBuiltins_If(t *testing.T) {
 	}
 }
 
+
 func TestBuiltins_ForEach(t *testing.T) {
 	tests := []struct {
 		desc string
@@ -222,6 +257,7 @@ func TestBuiltins_ForEach(t *testing.T) {
 		{desc: "iterate over map 2", expr: `
 			foreach [a:"1"] echo`, want: "a1\n(nil)\n"},
 		{desc: "iterate via pipe", expr: `["2" "4" "6"] | foreach { |x| echo $x }`, want: "2\n4\n6\n(nil)\n"},
+		{desc: "iterate from iterator 1", expr: `itr | foreach { |x| echo $x }`, want: "1\n2\n3\n(nil)\n"},
 	}
 
 	for _, tt := range tests {
@@ -230,7 +266,7 @@ func TestBuiltins_ForEach(t *testing.T) {
 			outW := bytes.NewBuffer(nil)
 
 			inst := New(WithOut(outW), WithTestBuiltin())
-			err := EvalAndDisplay(ctx, inst, tt.expr)
+			err := evalAndDisplay(ctx, inst, tt.expr)
 
 			assert.NoError(t, err)
 			assert.Equal(t, tt.want, outW.String())
@@ -261,11 +297,16 @@ func TestBuiltins_Break(t *testing.T) {
 					if (eq $v "2") { break }
 				}
 			}`, want: "a1\na2\nb1\nb2\n(nil)\n"},
-		{desc: "break returning value", expr: `
+		{desc: "break returning value 1", expr: `
 			echo (foreach ["1" "2" "3"] { |v|
 				echo $v
 				if (eq $v "2") { break "hello" }
 			})`, want: "1\n2\nhello\n(nil)\n"},
+		{desc: "break returning value 2", expr: `
+			echo (foreach (itr) { |v|
+				echo $v
+				if (eq $v 2) { break "hello" }
+			})`, want: "1\n2\nhello\n(nil)\n"},
 	}
 
 	for _, tt := range tests {
@@ -274,7 +315,7 @@ func TestBuiltins_Break(t *testing.T) {
 			outW := bytes.NewBuffer(nil)
 
 			inst := New(WithOut(outW), WithTestBuiltin())
-			err := EvalAndDisplay(ctx, inst, tt.expr)
+			err := evalAndDisplay(ctx, inst, tt.expr)
 
 			assert.NoError(t, err)
 			assert.Equal(t, tt.want, outW.String())
@@ -315,7 +356,7 @@ func TestBuiltins_Continue(t *testing.T) {
 			outW := bytes.NewBuffer(nil)
 
 			inst := New(WithOut(outW), WithTestBuiltin())
-			err := EvalAndDisplay(ctx, inst, tt.expr)
+			err := evalAndDisplay(ctx, inst, tt.expr)
 
 			assert.NoError(t, err)
 			assert.Equal(t, tt.want, outW.String())
@@ -395,7 +436,7 @@ func TestBuiltins_Procs(t *testing.T) {
 			outW := bytes.NewBuffer(nil)
 
 			inst := New(WithOut(outW), WithTestBuiltin())
-			err := EvalAndDisplay(ctx, inst, tt.expr)
+			err := evalAndDisplay(ctx, inst, tt.expr)
 
 			assert.NoError(t, err)
 			assert.Equal(t, tt.want, outW.String())
@@ -627,7 +668,7 @@ func TestBuiltins_Return(t *testing.T) {
 			outW := bytes.NewBuffer(nil)
 
 			inst := New(WithOut(outW), WithTestBuiltin())
-			err := EvalAndDisplay(ctx, inst, tt.expr)
+			err := evalAndDisplay(ctx, inst, tt.expr)
 
 			assert.NoError(t, err)
 			assert.Equal(t, tt.want, outW.String())
@@ -689,7 +730,7 @@ func TestBuiltins_Seq(t *testing.T) {
 			outW := bytes.NewBuffer(nil)
 
 			inst := New(WithOut(outW), WithTestBuiltin())
-			err := EvalAndDisplay(ctx, inst, tt.expr)
+			err := evalAndDisplay(ctx, inst, tt.expr)
 
 			assert.NoError(t, err)
 			assert.Equal(t, tt.want, outW.String())
@@ -728,6 +769,12 @@ func TestBuiltins_Map(t *testing.T) {
 			set l (["a" "b" "c"] | map $makeUpper)
 			echo $l
 			`, want: "[A B C]\n(nil)\n"},
+		{desc: "map itr stream", expr: `
+			set add2 (proc { |x| add $x 2 })
+		
+			set l (itr | map $add2)
+			foreach $l { |x| echo $x }
+			`, want: "3\n4\n5\n(nil)\n"},
 	}
 
 	for _, tt := range tests {
@@ -736,7 +783,7 @@ func TestBuiltins_Map(t *testing.T) {
 			outW := bytes.NewBuffer(nil)
 
 			inst := New(WithOut(outW), WithTestBuiltin())
-			err := EvalAndDisplay(ctx, inst, tt.expr)
+			err := evalAndDisplay(ctx, inst, tt.expr)
 
 			assert.NoError(t, err)
 			assert.Equal(t, tt.want, outW.String())
@@ -765,11 +812,18 @@ func TestBuiltins_Index(t *testing.T) {
 		{desc: "list of hash 1", expr: `index [["id":"abc"] ["id":"123"]] 0 id`, want: "abc\n"},
 		{desc: "list of hash 2", expr: `index [["id":"abc"] ["id":"123"]] 1 id`, want: "123\n"},
 
-		{desc: "go list 1", expr: `goInt | index 1`, want: "5\n"},
+		{desc: "go int 1", expr: `goInt | index 1`, want: "5\n"},
-		{desc: "go list 2", expr: `goInt | index 2`, want: "4\n"},
+		{desc: "go int 2", expr: `goInt | index 2`, want: "4\n"},
-		{desc: "go list 3", expr: `goInt | index 555`, want: "(nil)\n"},
+		{desc: "go int 3", expr: `goInt | index 555`, want: "(nil)\n"},
-		{desc: "go list 4", expr: `goInt | index -12`, want: "(nil)\n"},
+		{desc: "go int 4", expr: `goInt | index -12`, want: "(nil)\n"},
-		{desc: "go list 5", expr: `goInt | index NotAnIndex`, want: "(nil)\n"},
+		{desc: "go int 5", expr: `goInt | index NotAnIndex`, want: "(nil)\n"},
+
+		{desc: "go list 1", expr: `goList | index 0 This`, want: "thing 1\n"},
+		{desc: "go list 2", expr: `goList | index 1 This`, want: "thing 2\n"},
+		{desc: "go list 3", expr: `goList | index 2`, want: "(nil)\n"},
+		{desc: "go list 4", expr: `goList | index 2 This`, want: "(nil)\n"},
+		{desc: "go list 5", expr: `goList | index 30`, want: "(nil)\n"},
+
 		{desc: "go struct 1", expr: `goStruct | index Alpha`, want: "foo\n"},
 		{desc: "go struct 2", expr: `goStruct | index Beta`, want: "bar\n"},
 		{desc: "go struct 3", expr: `goStruct | index Gamma 1`, want: "33\n"},
@@ -829,7 +883,7 @@ func TestBuiltins_Index(t *testing.T) {
 					},
 				}, nil
 			})
-			err := EvalAndDisplay(ctx, inst, tt.expr)
+			err := evalAndDisplay(ctx, inst, tt.expr)
 
 			assert.NoError(t, err)
 			assert.Equal(t, tt.want, outW.String())
@@ -859,6 +913,8 @@ func TestBuiltins_Len(t *testing.T) {
 		{desc: "len of int", expr: `len 1232`, want: "0\n"},
 		{desc: "len of nil", expr: `len ()`, want: "0\n"},
 
+		{desc: "len of itr 1", expr: `len (itr)`, want: "3\n"},
+
 		{desc: "go list 1", expr: `goInt | len`, want: "3\n"},
 		{desc: "go struct 1", expr: `goStruct | len`, want: "3\n"},
 		{desc: "go struct 2", expr: `index (goStruct) Gamma | len`, want: "2\n"},
@@ -888,7 +944,7 @@ func TestBuiltins_Len(t *testing.T) {
 					missing: "missing",
 				}, nil
 			})
-			err := EvalAndDisplay(ctx, inst, tt.expr)
+			err := evalAndDisplay(ctx, inst, tt.expr)
 
 			assert.NoError(t, err)
 			assert.Equal(t, tt.want, outW.String())
@@ -959,6 +1015,8 @@ func TestBuiltins_Filter(t *testing.T) {
 		{desc: "filter list 1", expr: `filter [1 2 3] { |x| eq $x 2 }`, want: []any{2}},
 		{desc: "filter list 2", expr: `filter ["flim" "flam" "fla"] { |x| eq $x "flam" }`, want: []any{"flam"}},
 		{desc: "filter list 3", expr: `filter ["flim" "flam" "fla"] { |x| eq $x "bogie" }`, want: []any{}},
+		{desc: "filter list 4", expr: `filter [() () ()] { |x| $x }`, want: []any{}},
+		{desc: "filter list 5", expr: `filter [] { |x| $x }`, want: []any{}},
 
 		{desc: "filter map 1", expr: `filter [alpha:"hello" bravo:"world"] { |k v| eq $k "alpha" }`, want: map[string]any{
 			"alpha": "hello",
@@ -967,6 +1025,8 @@ func TestBuiltins_Filter(t *testing.T) {
 			"bravo": "world",
 		}},
 		{desc: "filter map 3", expr: `filter [alpha:"hello" bravo:"world"] { |k v| eq $v "alpha" }`, want: map[string]any{}},
+
+		{desc: "filter itr 1", expr: `set s "" ; itr | filter { |x| ne $x 2 } | foreach { |x| set s "$s $x" }; $s`, want: " 1 3"},
 	}
 
 	for _, tt := range tests {
@@ -991,6 +1051,37 @@ func TestBuiltins_Reduce(t *testing.T) {
 	}{
 		{desc: "reduce list 1", expr: `reduce [1 1 1] { |x a| add $x $a }`, want: 3},
 		{desc: "reduce list 2", expr: `reduce [1 1 1] 20 { |x a| add $x $a }`, want: 23},
+
+		{desc: "reduce itr 1", expr: `reduce (itr) 1 { |x a| add $x $a }`, want: 7},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			ctx := context.Background()
+			outW := bytes.NewBuffer(nil)
+
+			inst := New(WithOut(outW), WithTestBuiltin())
+
+			res, err := inst.Eval(ctx, tt.expr)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.want, res)
+		})
+	}
+}
+
+func TestBuiltins_Head(t *testing.T) {
+	tests := []struct {
+		desc string
+		expr string
+		want any
+	}{
+		{desc: "head list 1", expr: `head [1 2 3]`, want: 1},
+
+		{desc: "head itr 1", expr: `head (itr)`, want: 1},
+		{desc: "head itr 2", expr: `set h (itr) ; head $h`, want: 1},
+		{desc: "head itr 3", expr: `set h (itr) ; head $h ; head $h`, want: 2},
+		{desc: "head itr 4", expr: `set h (itr) ; head $h ; head $h ; head $h`, want: 3},
+		{desc: "head itr 5", expr: `set h (itr) ; head $h ; head $h ; head $h ; head $h`, want: nil},
 	}
 
 	for _, tt := range tests {
@@ -1282,9 +1373,11 @@ func TestBuiltins_AndOrNot(t *testing.T) {
 		{desc: "not 3", expr: `not $false $true`, want: true},
 
 		{desc: "short circuit and 1", expr: `and "hello" "world"`, want: "world"},
-		{desc: "short circuit and 2", expr: `and () "world"`, want: false},
+		{desc: "short circuit and 2", expr: `and () "world"`, want: nil},
+		{desc: "short circuit and 3", expr: `and [] "world"`, want: []any{}},
 		{desc: "short circuit or 1", expr: `or "hello" "world"`, want: "hello"},
 		{desc: "short circuit or 2", expr: `or () "world"`, want: "world"},
+		{desc: "short circuit or 3", expr: `or () []`, want: []any{}},
 
 		{desc: "bad and 1", expr: `and "one"`, wantErr: true},
 		{desc: "bad and 2", expr: `and`, wantErr: true},
@@ -1345,3 +1438,32 @@ func TestBuiltins_Cat(t *testing.T) {
 		})
 	}
 }
+
+func evalAndDisplay(ctx context.Context, inst *Inst, expr string) error {
+	res, err := inst.eval(ctx, expr)
+	if err != nil {
+		return err
+	}
+
+	return displayResult(ctx, inst, res)
+}
+
+func displayResult(ctx context.Context, inst *Inst, res Object) (err error) {
+	switch v := res.(type) {
+	case nil:
+		if _, err = fmt.Fprintln(inst.out, "(nil)"); err != nil {
+			return err
+		}
+	case Listable:
+		for i := 0; i < v.Len(); i++ {
+			if err = displayResult(ctx, inst, v.Index(i)); err != nil {
+				return err
+			}
+		}
+	default:
+		if _, err = fmt.Fprintln(inst.out, v.String()); err != nil {
+			return err
+		}
+	}
+	return nil
+}

ucl/userbuiltin.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"reflect"
+
+	"github.com/lmika/gopkgs/fp/slices"
 )
 
 type BuiltinHandler func(ctx context.Context, args CallArgs) (any, error)
@@ -72,7 +74,7 @@ func (ca CallArgs) BindSwitch(name string, val interface{}) error {
 		return nil
 	}
 
-	return bindArg(val, (*vars)[0])
+	return ca.bindArg(val, (*vars)[0])
 }
 
 func (inst *Inst) SetBuiltin(name string, fn BuiltinHandler) {
@@ -83,7 +85,7 @@ type userBuiltin struct {
 	fn func(ctx context.Context, args CallArgs) (any, error)
 }
 
-func (u userBuiltin) invoke(ctx context.Context, args invocationArgs) (object, error) {
+func (u userBuiltin) invoke(ctx context.Context, args invocationArgs) (Object, error) {
 	v, err := u.fn(ctx, CallArgs{args: args})
 	if err != nil {
 		return nil, err
@@ -92,13 +94,14 @@ func (u userBuiltin) invoke(ctx context.Context, args invocationArgs) (object, e
 	return fromGoValue(v)
 }
 
-func bindArg(v interface{}, arg object) error {
+func (ca CallArgs) bindArg(v interface{}, arg Object) error {
 	switch t := v.(type) {
 	case *Object:
 		*t = arg
 		return nil
 	case *interface{}:
 		*t, _ = toGoValue(arg)
+		return nil
 	case *Invokable:
 		i, ok := arg.(invokable)
 		if !ok {
@@ -118,6 +121,18 @@ func bindArg(v interface{}, arg object) error {
 		}
 		*t = i
 		return nil
+	case *ModListable:
+		if i, ok := arg.(ModListable); ok {
+			*t = i
+			return nil
+		}
+		return errors.New("exepected listable")
+	case *Iterable:
+		if i, ok := arg.(Iterable); ok {
+			*t = i
+			return nil
+		}
+		return errors.New("exepected iterable")
 	case *string:
 		if arg != nil {
 			*t = arg.String()
@@ -146,7 +161,7 @@ func bindArg(v interface{}, arg object) error {
 	return nil
 }
 
-func canBindArg(v interface{}, arg object) bool {
+func canBindArg(v interface{}, arg Object) bool {
 	switch v.(type) {
 	case *string:
 		return true
@@ -214,7 +229,7 @@ type missingHandlerInvokable struct {
 	handler MissingBuiltinHandler
 }
 
-func (m missingHandlerInvokable) invoke(ctx context.Context, args invocationArgs) (object, error) {
+func (m missingHandlerInvokable) invoke(ctx context.Context, args invocationArgs) (Object, error) {
 	v, err := m.handler(ctx, m.name, CallArgs{args: args})
 	if err != nil {
 		return nil, err