Added iterators

Iterators are an unbounded sequence of elements that can only be consumed one-by-one.

_docs/mod/core.md

@@ -74,14 +74,17 @@ Returns the length of COL. If COL is a list or hash, COL will be the number of
 elements. If COL is a string, COL will be the string's length. All other values will
 return a length of 0.
 
+If COL is an iterator, `len` will consume the values of the iterator and return the number of items consumed. 
+
 ### map
 
 ```
 map COL BLOCK
 ```
 
-Returns a new list of elements mapped from COL according to the result of BLOCK. COL can be any listable data
-structure, however the result will always be a concrete list.
+Returns a new list of elements mapped from COL according to the result of BLOCK. COL can be any list or hash
+with the result being a concrete list. COL can be an iterator, in which case the result will be an iterator
+which will call BLOCK for every consumed value.
 
 ```
 map [1 2 3] { |x| str $x | len }
@@ -107,7 +110,7 @@ reduce COL [INIT] BLOCK
 Returns the result of reducing the elements of COL with the passed in block.
 
 BLOCK will receive at least two argument, with the current value of the accumulator always being the last argument.
-If COL is a list, the arguments will be _|element accumulator|_, and if COL is a hash, the arguments will be
+If COL is a list or iterator, the arguments will be _|element accumulator|_, and if COL is a hash, the arguments will be
 _|key value accumulator|_. 
 
 The block result will be set as the value of the accumulator for the next iteration. Once all elements are process

_docs/mod/index.md

@@ -5,5 +5,6 @@ Modules of the standard library:
 - [core](/mod/core): Core builtins
 - [csv](/mod/csv): Functions for operating over CSV data.
 - [fs](/mod/fs): File system functions
+- [itrs](/mod/itrs): Iterator utilities
 - [lists](/mod/lists): List utilities
 - [os](/mod/os): Operating system functions 

_docs/mod/itrs.md

@@ -0,0 +1,20 @@
+---
+---
+
+# Iterator Builtins
+
+### from
+
+```
+itrs:from LIST
+```
+
+Returns an iterator which will step through the elements of LIST.
+
+### to-list
+
+```
+lists:to-list ITR
+```
+
+Consume the elements of the iterator ITR and return the elements as a list.

cmd/cmsh/main.go

@@ -21,6 +21,7 @@ func main() {
 		ucl.WithModule(builtins.CSV(nil)),
 		ucl.WithModule(builtins.FS(nil)),
 		ucl.WithModule(builtins.Log(nil)),
+		ucl.WithModule(builtins.Itrs()),
 		ucl.WithModule(builtins.Lists()),
 		ucl.WithModule(builtins.OS()),
 		ucl.WithModule(builtins.Strs()),

cmd/playwasm/jsiter.go

@@ -26,6 +26,8 @@ func initJS(ctx context.Context) {
 		ucl.WithModule(builtins.Log(nil)),
 		ucl.WithModule(builtins.Strs()),
 		ucl.WithModule(builtins.Time()),
+		ucl.WithModule(builtins.Itrs()),
+		ucl.WithModule(builtins.Lists()),
 		ucl.WithOut(ucl.LineHandler(func(line string) {
 			invokeUCLCallback("onOutLine", line)
 		})),

ucl/builtins.go

@@ -459,6 +459,16 @@ func lenBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 		return IntObject(v.Len()), nil
 	case hashable:
 		return IntObject(v.Len()), nil
+	case Iterable:
+		cnt := 0
+		for v.HasNext() {
+			_, err := v.Next(ctx)
+			if err != nil {
+				return nil, err
+			}
+			cnt++
+		}
+		return IntObject(cnt), nil
 	}
 
 	return IntObject(0), nil
@@ -523,6 +533,32 @@ func keysBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 	return nil, nil
 }
 
+type mappedIter struct {
+	src  Iterable
+	inv  invokable
+	args invocationArgs
+}
+
+func (mi mappedIter) String() string {
+	return "mappedIter{}"
+}
+
+func (mi mappedIter) Truthy() bool {
+	return mi.src.HasNext()
+}
+
+func (mi mappedIter) HasNext() bool {
+	return mi.src.HasNext()
+}
+
+func (mi mappedIter) Next(ctx context.Context) (Object, error) {
+	v, err := mi.src.Next(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return mi.inv.invoke(ctx, mi.args.fork([]Object{v}))
+}
+
 func mapBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 	if err := args.expectArgn(2); err != nil {
 		return nil, err
@@ -546,10 +582,64 @@ func mapBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 			newList = append(newList, m)
 		}
 		return &newList, nil
+	case Iterable:
+		return mappedIter{src: t, inv: inv, args: args}, nil
 	}
 	return nil, errors.New("expected listable")
 }
 
+type filterIter struct {
+	src  Iterable
+	inv  invokable
+	args invocationArgs
+
+	hasNext bool
+	next    Object
+	err     error
+}
+
+func (mi *filterIter) prime(ctx context.Context) {
+	for mi.src.HasNext() {
+		v, err := mi.src.Next(ctx)
+		if err != nil {
+			mi.err = err
+			mi.hasNext = false
+			return
+		}
+
+		fv, err := mi.inv.invoke(ctx, mi.args.fork([]Object{v}))
+		if err != nil {
+			mi.err = err
+			mi.hasNext = false
+			return
+		} else if isTruthy(fv) {
+			mi.next = v
+			mi.hasNext = true
+			return
+		}
+	}
+	mi.hasNext = false
+	mi.err = nil
+}
+
+func (mi *filterIter) String() string {
+	return "filterIter{}"
+}
+
+func (mi *filterIter) Truthy() bool {
+	return mi.HasNext()
+}
+
+func (mi *filterIter) HasNext() bool {
+	return mi.hasNext
+}
+
+func (mi *filterIter) Next(ctx context.Context) (Object, error) {
+	next, err := mi.next, mi.err
+	mi.prime(ctx)
+	return next, err
+}
+
 func filterBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 	if err := args.expectArgn(2); err != nil {
 		return nil, err
@@ -587,6 +677,10 @@ func filterBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 			return nil, err
 		}
 		return newHash, nil
+	case Iterable:
+		fi := &filterIter{src: t, inv: inv, args: args}
+		fi.prime(ctx)
+		return fi, nil
 	}
 	return nil, errors.New("expected listable")
 }
@@ -648,6 +742,20 @@ func reduceBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 			return nil, err
 		}
 		return accum, nil
+	case Iterable:
+		for t.HasNext() {
+			v, err := t.Next(ctx)
+			if err != nil {
+				return nil, err
+			}
+
+			newAccum, err := block.invoke(ctx, args.fork([]Object{v, accum}))
+			if err != nil {
+				return nil, err
+			}
+			accum = newAccum
+		}
+		return accum, nil
 	}
 	return nil, errors.New("expected listable")
 }
@@ -663,6 +771,11 @@ func firstBuiltin(ctx context.Context, args invocationArgs) (Object, error) {
 			return nil, nil
 		}
 		return t.Index(0), nil
+	case Iterable:
+		if t.HasNext() {
+			return t.Next(ctx)
+		}
+		return nil, nil
 	}
 	return nil, errors.New("expected listable")
 }
@@ -884,6 +997,25 @@ func foreachBuiltin(ctx context.Context, args macroArgs) (Object, error) {
 		} else {
 			return nil, err
 		}
+	case Iterable:
+		for t.HasNext() {
+			v, err := t.Next(ctx)
+			if err != nil {
+				return nil, err
+			}
+
+			last, err = args.evalBlock(ctx, blockIdx, []Object{v}, true) // TO INCLUDE: the index
+			if err != nil {
+				if errors.As(err, &breakErr) {
+					if !breakErr.isCont {
+						return breakErr.ret, nil
+					}
+				} else {
+					return nil, err
+				}
+			}
+		}
+		return nil, nil
 	}
 
 	return last, nil

ucl/builtins/itrs.go

@@ -0,0 +1,68 @@
+package builtins
+
+import (
+	"context"
+	"ucl.lmika.dev/ucl"
+)
+
+func Itrs() ucl.Module {
+	return ucl.Module{
+		Name: "itrs",
+		Builtins: map[string]ucl.BuiltinHandler{
+			"from":    iterFrom,
+			"to-list": iterToList,
+		},
+	}
+}
+
+func iterFrom(ctx context.Context, args ucl.CallArgs) (any, error) {
+	var listable ucl.Listable
+
+	if err := args.Bind(&listable); err != nil {
+		return nil, err
+	}
+
+	return &fromIterator{listable: listable}, nil
+}
+
+type fromIterator struct {
+	idx      int
+	listable ucl.Listable
+}
+
+func (f *fromIterator) String() string {
+	return "fromIterator{}"
+}
+
+func (f *fromIterator) HasNext() bool {
+	return f.idx < f.listable.Len()
+}
+
+func (f *fromIterator) Next(ctx context.Context) (ucl.Object, error) {
+	if f.idx >= f.listable.Len() {
+		return nil, nil
+	}
+	v := f.listable.Index(f.idx)
+	f.idx++
+	return v, nil
+}
+
+func iterToList(ctx context.Context, args ucl.CallArgs) (any, error) {
+	var itr ucl.Iterable
+	if err := args.Bind(&itr); err != nil {
+		return nil, err
+	}
+
+	target := ucl.NewListObject()
+	for itr.HasNext() {
+		v, err := itr.Next(ctx)
+		if err != nil {
+			return nil, err
+		}
+		if err := target.Insert(-1, v); err != nil {
+			return nil, err
+		}
+	}
+
+	return target, nil
+}

ucl/builtins/itrs_test.go

@@ -0,0 +1,38 @@
+package builtins_test
+
+import (
+	"context"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"ucl.lmika.dev/ucl"
+	"ucl.lmika.dev/ucl/builtins"
+)
+
+func TestItrs_ToList(t *testing.T) {
+	tests := []struct {
+		desc    string
+		eval    string
+		want    any
+		wantErr bool
+	}{
+		{desc: "to-list 1", eval: `itrs:from (seq 5) | itrs:to-list`, want: []any{0, 1, 2, 3, 4}},
+		{desc: "to-list 2", eval: `itrs:from (seq 10) | filter { |x| eq (mod $x 2) 0 } | itrs:to-list`, want: []any{0, 2, 4, 6, 8}},
+		{desc: "to-list 3", eval: `itrs:from (seq 10) | filter { |x| eq (mod $x 2) 0 } | map { |x| (add $x 2) } | itrs:to-list`, want: []any{2, 4, 6, 8, 10}},
+		{desc: "to-list 4", eval: `itrs:from (seq 10) | filter { |x| () } | map { |x| (add $x 2) } | itrs:to-list`, want: []any{}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			inst := ucl.New(
+				ucl.WithModule(builtins.Itrs()),
+			)
+			res, err := inst.Eval(context.Background(), tt.eval)
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.want, res)
+			}
+		})
+	}
+}

ucl/objs.go

@@ -22,6 +22,14 @@ type Listable interface {
 	Index(i int) Object
 }
 
+type Iterable interface {
+	HasNext() bool
+
+	// Next returns the next object from the iterable if one exists, otherwise
+	// returns nil, false.
+	Next(ctx context.Context) (Object, error)
+}
+
 type ModListable interface {
 	Listable
 
@@ -72,6 +80,18 @@ func (s *ListObject) Index(i int) Object {
 	return (*s)[i]
 }
 
+type iteratorObject struct {
+	Iterable
+}
+
+func (i iteratorObject) String() string {
+	return "iterator{}"
+}
+
+func (i iteratorObject) Truthy() bool {
+	return i.Iterable.HasNext()
+}
+
 type hashObject map[string]Object
 
 func (s hashObject) String() string {
@@ -191,6 +211,10 @@ func toGoValue(obj Object) (interface{}, bool) {
 			xs[k] = x
 		}
 		return xs, true
+	case iteratorObject:
+		return v.Iterable, true
+	case Iterable:
+		return v, true
 	case proxyObject:
 		return v.p, true
 	case listableProxyObject:
@@ -208,6 +232,8 @@ func fromGoValue(v any) (Object, error) {
 		return t, nil
 	case OpaqueObject:
 		return t, nil
+	case Iterable:
+		return iteratorObject{t}, nil
 	case nil:
 		return nil, nil
 	case string:

ucl/testbuiltins_test.go

@@ -11,6 +11,21 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+type testIterator struct {
+	cnt int
+	max int
+	err error
+}
+
+func (ti *testIterator) HasNext() bool {
+	return ti.cnt < ti.max
+}
+
+func (ti *testIterator) Next(ctx context.Context) (Object, error) {
+	ti.cnt++
+	return IntObject(ti.cnt), nil
+}
+
 // Builtins used for test
 func WithTestBuiltin() InstOption {
 	return func(i *Inst) {
@@ -68,6 +83,10 @@ func WithTestBuiltin() InstOption {
 			return StringObject(sb.String()), nil
 		}))
 
+		i.rootEC.addCmd("itr", invokableFunc(func(ctx context.Context, args invocationArgs) (Object, error) {
+			return iteratorObject{Iterable: &testIterator{max: 3}}, nil
+		}))
+
 		i.rootEC.setOrDefineVar("a", StringObject("alpha"))
 		i.rootEC.setOrDefineVar("bee", StringObject("buzz"))
 	}
@@ -198,6 +217,12 @@ func TestBuiltins_If(t *testing.T) {
 		{desc: "compressed then", expr: `set x "Hello" ; if $x { echo "true" }`, want: "true\n(nil)\n"},
 		{desc: "compressed else", expr: `if $x { echo "true" } else { echo "false" }`, want: "false\n(nil)\n"},
 		{desc: "compressed if", expr: `if $x { echo "x" } elif $y { echo "y" } else { echo "false" }`, want: "false\n(nil)\n"},
+		{desc: "if of itr 1", expr: `set i (itr) ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"},
+		{desc: "if of itr 2", expr: `set i (itr) ; foreach (seq 1) { head $i } ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"},
+		{desc: "if of itr 3", expr: `set i (itr) ; foreach (seq 3) { head $i } ; if $i { echo "more" } else { echo "none" }`, want: "none\n(nil)\n"},
+		{desc: "if of itr 4", expr: `set i (itr | map { |x| add 2 $x }) ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"},
+		{desc: "if of itr 5", expr: `set i (itr | filter { |x| () }) ; if $i { echo "more" } else { echo "none" }`, want: "none\n(nil)\n"},
+		{desc: "if of itr 6", expr: `set i (itr | filter { |x| 1 }) ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"},
 	}
 
 	for _, tt := range tests {
@@ -428,6 +453,7 @@ func TestBuiltins_ForEach(t *testing.T) {
 		{desc: "iterate over map 2", expr: `
 			foreach [a:"1"] echo`, want: "a1\n(nil)\n"},
 		{desc: "iterate via pipe", expr: `["2" "4" "6"] | foreach { |x| echo $x }`, want: "2\n4\n6\n(nil)\n"},
+		{desc: "iterate from iterator 1", expr: `itr | foreach { |x| echo $x }`, want: "1\n2\n3\n(nil)\n"},
 	}
 
 	for _, tt := range tests {
@@ -467,11 +493,16 @@ func TestBuiltins_Break(t *testing.T) {
 					if (eq $v "2") { break }
 				}
 			}`, want: "a1\na2\nb1\nb2\n(nil)\n"},
-		{desc: "break returning value", expr: `
+		{desc: "break returning value 1", expr: `
 			echo (foreach ["1" "2" "3"] { |v|
 				echo $v
 				if (eq $v "2") { break "hello" }
 			})`, want: "1\n2\nhello\n(nil)\n"},
+		{desc: "break returning value 2", expr: `
+			echo (foreach (itr) { |v|
+				echo $v
+				if (eq $v 2) { break "hello" }
+			})`, want: "1\n2\nhello\n(nil)\n"},
 	}
 
 	for _, tt := range tests {
@@ -934,6 +965,12 @@ func TestBuiltins_Map(t *testing.T) {
 			set l (["a" "b" "c"] | map $makeUpper)
 			echo $l
 			`, want: "[A B C]\n(nil)\n"},
+		{desc: "map itr stream", expr: `
+			set add2 (proc { |x| add $x 2 })
+		
+			set l (itr | map $add2)
+			foreach $l { |x| echo $x }
+			`, want: "3\n4\n5\n(nil)\n"},
 	}
 
 	for _, tt := range tests {
@@ -1072,6 +1109,8 @@ func TestBuiltins_Len(t *testing.T) {
 		{desc: "len of int", expr: `len 1232`, want: "0\n"},
 		{desc: "len of nil", expr: `len ()`, want: "0\n"},
 
+		{desc: "len of itr 1", expr: `len (itr)`, want: "3\n"},
+
 		{desc: "go list 1", expr: `goInt | len`, want: "3\n"},
 		{desc: "go struct 1", expr: `goStruct | len`, want: "3\n"},
 		{desc: "go struct 2", expr: `index (goStruct) Gamma | len`, want: "2\n"},
@@ -1182,6 +1221,8 @@ func TestBuiltins_Filter(t *testing.T) {
 			"bravo": "world",
 		}},
 		{desc: "filter map 3", expr: `filter [alpha:"hello" bravo:"world"] { |k v| eq $v "alpha" }`, want: map[string]any{}},
+
+		{desc: "filter itr 1", expr: `set s "" ; itr | filter { |x| ne $x 2 } | foreach { |x| set s "$s $x" }; $s`, want: " 1 3"},
 	}
 
 	for _, tt := range tests {
@@ -1206,6 +1247,37 @@ func TestBuiltins_Reduce(t *testing.T) {
 	}{
 		{desc: "reduce list 1", expr: `reduce [1 1 1] { |x a| add $x $a }`, want: 3},
 		{desc: "reduce list 2", expr: `reduce [1 1 1] 20 { |x a| add $x $a }`, want: 23},
+
+		{desc: "reduce itr 1", expr: `reduce (itr) 1 { |x a| add $x $a }`, want: 7},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			ctx := context.Background()
+			outW := bytes.NewBuffer(nil)
+
+			inst := New(WithOut(outW), WithTestBuiltin())
+
+			res, err := inst.Eval(ctx, tt.expr)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.want, res)
+		})
+	}
+}
+
+func TestBuiltins_Head(t *testing.T) {
+	tests := []struct {
+		desc string
+		expr string
+		want any
+	}{
+		{desc: "head list 1", expr: `head [1 2 3]`, want: 1},
+
+		{desc: "head itr 1", expr: `head (itr)`, want: 1},
+		{desc: "head itr 2", expr: `set h (itr) ; head $h`, want: 1},
+		{desc: "head itr 3", expr: `set h (itr) ; head $h ; head $h`, want: 2},
+		{desc: "head itr 4", expr: `set h (itr) ; head $h ; head $h ; head $h`, want: 3},
+		{desc: "head itr 5", expr: `set h (itr) ; head $h ; head $h ; head $h ; head $h`, want: nil},
 	}
 
 	for _, tt := range tests {

ucl/userbuiltin.go

@@ -127,6 +127,12 @@ func (ca CallArgs) bindArg(v interface{}, arg Object) error {
 			return nil
 		}
 		return errors.New("exepected listable")
+	case *Iterable:
+		if i, ok := arg.(Iterable); ok {
+			*t = i
+			return nil
+		}
+		return errors.New("exepected iterable")
 	case *string:
 		if arg != nil {
 			*t = arg.String()