From 2fcfe9d5400e0eacc61c6e539e2395823409f016 Mon Sep 17 00:00:00 2001 From: Leon Mika Date: Thu, 30 Jan 2025 22:15:38 +1100 Subject: [PATCH] Added iterators Iterators are an unbounded sequence of elements that can only be consumed one-by-one. --- _docs/mod/core.md | 9 ++- _docs/mod/index.md | 1 + _docs/mod/itrs.md | 20 +++++ _docs/mod/{list.md => lists.md} | 0 cmd/cmsh/main.go | 1 + cmd/playwasm/jsiter.go | 2 + ucl/builtins.go | 132 ++++++++++++++++++++++++++++++++ ucl/builtins/itrs.go | 68 ++++++++++++++++ ucl/builtins/itrs_test.go | 38 +++++++++ ucl/objs.go | 26 +++++++ ucl/testbuiltins_test.go | 74 +++++++++++++++++- ucl/userbuiltin.go | 6 ++ 12 files changed, 373 insertions(+), 4 deletions(-) create mode 100644 _docs/mod/itrs.md rename _docs/mod/{list.md => lists.md} (100%) create mode 100644 ucl/builtins/itrs.go create mode 100644 ucl/builtins/itrs_test.go diff --git a/_docs/mod/core.md b/_docs/mod/core.md index 711ccd7..c705504 100644 --- a/_docs/mod/core.md +++ b/_docs/mod/core.md @@ -74,14 +74,17 @@ Returns the length of COL. If COL is a list or hash, COL will be the number of elements. If COL is a string, COL will be the string's length. All other values will return a length of 0. +If COL is an iterator, `len` will consume the values of the iterator and return the number of items consumed. + ### map ``` map COL BLOCK ``` -Returns a new list of elements mapped from COL according to the result of BLOCK. COL can be any listable data -structure, however the result will always be a concrete list. +Returns a new list of elements mapped from COL according to the result of BLOCK. COL can be any list or hash +with the result being a concrete list. COL can be an iterator, in which case the result will be an iterator +which will call BLOCK for every consumed value. ``` map [1 2 3] { |x| str $x | len } @@ -107,7 +110,7 @@ reduce COL [INIT] BLOCK Returns the result of reducing the elements of COL with the passed in block. BLOCK will receive at least two argument, with the current value of the accumulator always being the last argument. -If COL is a list, the arguments will be _|element accumulator|_, and if COL is a hash, the arguments will be +If COL is a list or iterator, the arguments will be _|element accumulator|_, and if COL is a hash, the arguments will be _|key value accumulator|_. The block result will be set as the value of the accumulator for the next iteration. Once all elements are process diff --git a/_docs/mod/index.md b/_docs/mod/index.md index 9c01f8a..c7aac79 100644 --- a/_docs/mod/index.md +++ b/_docs/mod/index.md @@ -5,5 +5,6 @@ Modules of the standard library: - [core](/mod/core): Core builtins - [csv](/mod/csv): Functions for operating over CSV data. - [fs](/mod/fs): File system functions +- [itrs](/mod/itrs): Iterator utilities - [lists](/mod/lists): List utilities - [os](/mod/os): Operating system functions \ No newline at end of file diff --git a/_docs/mod/itrs.md b/_docs/mod/itrs.md new file mode 100644 index 0000000..df4fc9b --- /dev/null +++ b/_docs/mod/itrs.md @@ -0,0 +1,20 @@ +--- +--- + +# Iterator Builtins + +### from + +``` +itrs:from LIST +``` + +Returns an iterator which will step through the elements of LIST. + +### to-list + +``` +lists:to-list ITR +``` + +Consume the elements of the iterator ITR and return the elements as a list. \ No newline at end of file diff --git a/_docs/mod/list.md b/_docs/mod/lists.md similarity index 100% rename from _docs/mod/list.md rename to _docs/mod/lists.md diff --git a/cmd/cmsh/main.go b/cmd/cmsh/main.go index f2fd688..a1c1664 100644 --- a/cmd/cmsh/main.go +++ b/cmd/cmsh/main.go @@ -21,6 +21,7 @@ func main() { ucl.WithModule(builtins.CSV(nil)), ucl.WithModule(builtins.FS(nil)), ucl.WithModule(builtins.Log(nil)), + ucl.WithModule(builtins.Itrs()), ucl.WithModule(builtins.Lists()), ucl.WithModule(builtins.OS()), ucl.WithModule(builtins.Strs()), diff --git a/cmd/playwasm/jsiter.go b/cmd/playwasm/jsiter.go index f2dd726..72723cd 100644 --- a/cmd/playwasm/jsiter.go +++ b/cmd/playwasm/jsiter.go @@ -26,6 +26,8 @@ func initJS(ctx context.Context) { ucl.WithModule(builtins.Log(nil)), ucl.WithModule(builtins.Strs()), ucl.WithModule(builtins.Time()), + ucl.WithModule(builtins.Itrs()), + ucl.WithModule(builtins.Lists()), ucl.WithOut(ucl.LineHandler(func(line string) { invokeUCLCallback("onOutLine", line) })), diff --git a/ucl/builtins.go b/ucl/builtins.go index 539d119..4e3b9ad 100644 --- a/ucl/builtins.go +++ b/ucl/builtins.go @@ -459,6 +459,16 @@ func lenBuiltin(ctx context.Context, args invocationArgs) (Object, error) { return IntObject(v.Len()), nil case hashable: return IntObject(v.Len()), nil + case Iterable: + cnt := 0 + for v.HasNext() { + _, err := v.Next(ctx) + if err != nil { + return nil, err + } + cnt++ + } + return IntObject(cnt), nil } return IntObject(0), nil @@ -523,6 +533,32 @@ func keysBuiltin(ctx context.Context, args invocationArgs) (Object, error) { return nil, nil } +type mappedIter struct { + src Iterable + inv invokable + args invocationArgs +} + +func (mi mappedIter) String() string { + return "mappedIter{}" +} + +func (mi mappedIter) Truthy() bool { + return mi.src.HasNext() +} + +func (mi mappedIter) HasNext() bool { + return mi.src.HasNext() +} + +func (mi mappedIter) Next(ctx context.Context) (Object, error) { + v, err := mi.src.Next(ctx) + if err != nil { + return nil, err + } + return mi.inv.invoke(ctx, mi.args.fork([]Object{v})) +} + func mapBuiltin(ctx context.Context, args invocationArgs) (Object, error) { if err := args.expectArgn(2); err != nil { return nil, err @@ -546,10 +582,64 @@ func mapBuiltin(ctx context.Context, args invocationArgs) (Object, error) { newList = append(newList, m) } return &newList, nil + case Iterable: + return mappedIter{src: t, inv: inv, args: args}, nil } return nil, errors.New("expected listable") } +type filterIter struct { + src Iterable + inv invokable + args invocationArgs + + hasNext bool + next Object + err error +} + +func (mi *filterIter) prime(ctx context.Context) { + for mi.src.HasNext() { + v, err := mi.src.Next(ctx) + if err != nil { + mi.err = err + mi.hasNext = false + return + } + + fv, err := mi.inv.invoke(ctx, mi.args.fork([]Object{v})) + if err != nil { + mi.err = err + mi.hasNext = false + return + } else if isTruthy(fv) { + mi.next = v + mi.hasNext = true + return + } + } + mi.hasNext = false + mi.err = nil +} + +func (mi *filterIter) String() string { + return "filterIter{}" +} + +func (mi *filterIter) Truthy() bool { + return mi.HasNext() +} + +func (mi *filterIter) HasNext() bool { + return mi.hasNext +} + +func (mi *filterIter) Next(ctx context.Context) (Object, error) { + next, err := mi.next, mi.err + mi.prime(ctx) + return next, err +} + func filterBuiltin(ctx context.Context, args invocationArgs) (Object, error) { if err := args.expectArgn(2); err != nil { return nil, err @@ -587,6 +677,10 @@ func filterBuiltin(ctx context.Context, args invocationArgs) (Object, error) { return nil, err } return newHash, nil + case Iterable: + fi := &filterIter{src: t, inv: inv, args: args} + fi.prime(ctx) + return fi, nil } return nil, errors.New("expected listable") } @@ -648,6 +742,20 @@ func reduceBuiltin(ctx context.Context, args invocationArgs) (Object, error) { return nil, err } return accum, nil + case Iterable: + for t.HasNext() { + v, err := t.Next(ctx) + if err != nil { + return nil, err + } + + newAccum, err := block.invoke(ctx, args.fork([]Object{v, accum})) + if err != nil { + return nil, err + } + accum = newAccum + } + return accum, nil } return nil, errors.New("expected listable") } @@ -663,6 +771,11 @@ func firstBuiltin(ctx context.Context, args invocationArgs) (Object, error) { return nil, nil } return t.Index(0), nil + case Iterable: + if t.HasNext() { + return t.Next(ctx) + } + return nil, nil } return nil, errors.New("expected listable") } @@ -884,6 +997,25 @@ func foreachBuiltin(ctx context.Context, args macroArgs) (Object, error) { } else { return nil, err } + case Iterable: + for t.HasNext() { + v, err := t.Next(ctx) + if err != nil { + return nil, err + } + + last, err = args.evalBlock(ctx, blockIdx, []Object{v}, true) // TO INCLUDE: the index + if err != nil { + if errors.As(err, &breakErr) { + if !breakErr.isCont { + return breakErr.ret, nil + } + } else { + return nil, err + } + } + } + return nil, nil } return last, nil diff --git a/ucl/builtins/itrs.go b/ucl/builtins/itrs.go new file mode 100644 index 0000000..a58bb50 --- /dev/null +++ b/ucl/builtins/itrs.go @@ -0,0 +1,68 @@ +package builtins + +import ( + "context" + "ucl.lmika.dev/ucl" +) + +func Itrs() ucl.Module { + return ucl.Module{ + Name: "itrs", + Builtins: map[string]ucl.BuiltinHandler{ + "from": iterFrom, + "to-list": iterToList, + }, + } +} + +func iterFrom(ctx context.Context, args ucl.CallArgs) (any, error) { + var listable ucl.Listable + + if err := args.Bind(&listable); err != nil { + return nil, err + } + + return &fromIterator{listable: listable}, nil +} + +type fromIterator struct { + idx int + listable ucl.Listable +} + +func (f *fromIterator) String() string { + return "fromIterator{}" +} + +func (f *fromIterator) HasNext() bool { + return f.idx < f.listable.Len() +} + +func (f *fromIterator) Next(ctx context.Context) (ucl.Object, error) { + if f.idx >= f.listable.Len() { + return nil, nil + } + v := f.listable.Index(f.idx) + f.idx++ + return v, nil +} + +func iterToList(ctx context.Context, args ucl.CallArgs) (any, error) { + var itr ucl.Iterable + if err := args.Bind(&itr); err != nil { + return nil, err + } + + target := ucl.NewListObject() + for itr.HasNext() { + v, err := itr.Next(ctx) + if err != nil { + return nil, err + } + if err := target.Insert(-1, v); err != nil { + return nil, err + } + } + + return target, nil +} diff --git a/ucl/builtins/itrs_test.go b/ucl/builtins/itrs_test.go new file mode 100644 index 0000000..e852209 --- /dev/null +++ b/ucl/builtins/itrs_test.go @@ -0,0 +1,38 @@ +package builtins_test + +import ( + "context" + "github.com/stretchr/testify/assert" + "testing" + "ucl.lmika.dev/ucl" + "ucl.lmika.dev/ucl/builtins" +) + +func TestItrs_ToList(t *testing.T) { + tests := []struct { + desc string + eval string + want any + wantErr bool + }{ + {desc: "to-list 1", eval: `itrs:from (seq 5) | itrs:to-list`, want: []any{0, 1, 2, 3, 4}}, + {desc: "to-list 2", eval: `itrs:from (seq 10) | filter { |x| eq (mod $x 2) 0 } | itrs:to-list`, want: []any{0, 2, 4, 6, 8}}, + {desc: "to-list 3", eval: `itrs:from (seq 10) | filter { |x| eq (mod $x 2) 0 } | map { |x| (add $x 2) } | itrs:to-list`, want: []any{2, 4, 6, 8, 10}}, + {desc: "to-list 4", eval: `itrs:from (seq 10) | filter { |x| () } | map { |x| (add $x 2) } | itrs:to-list`, want: []any{}}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + inst := ucl.New( + ucl.WithModule(builtins.Itrs()), + ) + res, err := inst.Eval(context.Background(), tt.eval) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, res) + } + }) + } +} diff --git a/ucl/objs.go b/ucl/objs.go index 8cdd011..e6c0fc0 100644 --- a/ucl/objs.go +++ b/ucl/objs.go @@ -22,6 +22,14 @@ type Listable interface { Index(i int) Object } +type Iterable interface { + HasNext() bool + + // Next returns the next object from the iterable if one exists, otherwise + // returns nil, false. + Next(ctx context.Context) (Object, error) +} + type ModListable interface { Listable @@ -72,6 +80,18 @@ func (s *ListObject) Index(i int) Object { return (*s)[i] } +type iteratorObject struct { + Iterable +} + +func (i iteratorObject) String() string { + return "iterator{}" +} + +func (i iteratorObject) Truthy() bool { + return i.Iterable.HasNext() +} + type hashObject map[string]Object func (s hashObject) String() string { @@ -191,6 +211,10 @@ func toGoValue(obj Object) (interface{}, bool) { xs[k] = x } return xs, true + case iteratorObject: + return v.Iterable, true + case Iterable: + return v, true case proxyObject: return v.p, true case listableProxyObject: @@ -208,6 +232,8 @@ func fromGoValue(v any) (Object, error) { return t, nil case OpaqueObject: return t, nil + case Iterable: + return iteratorObject{t}, nil case nil: return nil, nil case string: diff --git a/ucl/testbuiltins_test.go b/ucl/testbuiltins_test.go index 455680f..31eb775 100644 --- a/ucl/testbuiltins_test.go +++ b/ucl/testbuiltins_test.go @@ -11,6 +11,21 @@ import ( "github.com/stretchr/testify/assert" ) +type testIterator struct { + cnt int + max int + err error +} + +func (ti *testIterator) HasNext() bool { + return ti.cnt < ti.max +} + +func (ti *testIterator) Next(ctx context.Context) (Object, error) { + ti.cnt++ + return IntObject(ti.cnt), nil +} + // Builtins used for test func WithTestBuiltin() InstOption { return func(i *Inst) { @@ -68,6 +83,10 @@ func WithTestBuiltin() InstOption { return StringObject(sb.String()), nil })) + i.rootEC.addCmd("itr", invokableFunc(func(ctx context.Context, args invocationArgs) (Object, error) { + return iteratorObject{Iterable: &testIterator{max: 3}}, nil + })) + i.rootEC.setOrDefineVar("a", StringObject("alpha")) i.rootEC.setOrDefineVar("bee", StringObject("buzz")) } @@ -198,6 +217,12 @@ func TestBuiltins_If(t *testing.T) { {desc: "compressed then", expr: `set x "Hello" ; if $x { echo "true" }`, want: "true\n(nil)\n"}, {desc: "compressed else", expr: `if $x { echo "true" } else { echo "false" }`, want: "false\n(nil)\n"}, {desc: "compressed if", expr: `if $x { echo "x" } elif $y { echo "y" } else { echo "false" }`, want: "false\n(nil)\n"}, + {desc: "if of itr 1", expr: `set i (itr) ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"}, + {desc: "if of itr 2", expr: `set i (itr) ; foreach (seq 1) { head $i } ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"}, + {desc: "if of itr 3", expr: `set i (itr) ; foreach (seq 3) { head $i } ; if $i { echo "more" } else { echo "none" }`, want: "none\n(nil)\n"}, + {desc: "if of itr 4", expr: `set i (itr | map { |x| add 2 $x }) ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"}, + {desc: "if of itr 5", expr: `set i (itr | filter { |x| () }) ; if $i { echo "more" } else { echo "none" }`, want: "none\n(nil)\n"}, + {desc: "if of itr 6", expr: `set i (itr | filter { |x| 1 }) ; if $i { echo "more" } else { echo "none" }`, want: "more\n(nil)\n"}, } for _, tt := range tests { @@ -428,6 +453,7 @@ func TestBuiltins_ForEach(t *testing.T) { {desc: "iterate over map 2", expr: ` foreach [a:"1"] echo`, want: "a1\n(nil)\n"}, {desc: "iterate via pipe", expr: `["2" "4" "6"] | foreach { |x| echo $x }`, want: "2\n4\n6\n(nil)\n"}, + {desc: "iterate from iterator 1", expr: `itr | foreach { |x| echo $x }`, want: "1\n2\n3\n(nil)\n"}, } for _, tt := range tests { @@ -467,11 +493,16 @@ func TestBuiltins_Break(t *testing.T) { if (eq $v "2") { break } } }`, want: "a1\na2\nb1\nb2\n(nil)\n"}, - {desc: "break returning value", expr: ` + {desc: "break returning value 1", expr: ` echo (foreach ["1" "2" "3"] { |v| echo $v if (eq $v "2") { break "hello" } })`, want: "1\n2\nhello\n(nil)\n"}, + {desc: "break returning value 2", expr: ` + echo (foreach (itr) { |v| + echo $v + if (eq $v 2) { break "hello" } + })`, want: "1\n2\nhello\n(nil)\n"}, } for _, tt := range tests { @@ -934,6 +965,12 @@ func TestBuiltins_Map(t *testing.T) { set l (["a" "b" "c"] | map $makeUpper) echo $l `, want: "[A B C]\n(nil)\n"}, + {desc: "map itr stream", expr: ` + set add2 (proc { |x| add $x 2 }) + + set l (itr | map $add2) + foreach $l { |x| echo $x } + `, want: "3\n4\n5\n(nil)\n"}, } for _, tt := range tests { @@ -1072,6 +1109,8 @@ func TestBuiltins_Len(t *testing.T) { {desc: "len of int", expr: `len 1232`, want: "0\n"}, {desc: "len of nil", expr: `len ()`, want: "0\n"}, + {desc: "len of itr 1", expr: `len (itr)`, want: "3\n"}, + {desc: "go list 1", expr: `goInt | len`, want: "3\n"}, {desc: "go struct 1", expr: `goStruct | len`, want: "3\n"}, {desc: "go struct 2", expr: `index (goStruct) Gamma | len`, want: "2\n"}, @@ -1182,6 +1221,8 @@ func TestBuiltins_Filter(t *testing.T) { "bravo": "world", }}, {desc: "filter map 3", expr: `filter [alpha:"hello" bravo:"world"] { |k v| eq $v "alpha" }`, want: map[string]any{}}, + + {desc: "filter itr 1", expr: `set s "" ; itr | filter { |x| ne $x 2 } | foreach { |x| set s "$s $x" }; $s`, want: " 1 3"}, } for _, tt := range tests { @@ -1206,6 +1247,37 @@ func TestBuiltins_Reduce(t *testing.T) { }{ {desc: "reduce list 1", expr: `reduce [1 1 1] { |x a| add $x $a }`, want: 3}, {desc: "reduce list 2", expr: `reduce [1 1 1] 20 { |x a| add $x $a }`, want: 23}, + + {desc: "reduce itr 1", expr: `reduce (itr) 1 { |x a| add $x $a }`, want: 7}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + ctx := context.Background() + outW := bytes.NewBuffer(nil) + + inst := New(WithOut(outW), WithTestBuiltin()) + + res, err := inst.Eval(ctx, tt.expr) + assert.NoError(t, err) + assert.Equal(t, tt.want, res) + }) + } +} + +func TestBuiltins_Head(t *testing.T) { + tests := []struct { + desc string + expr string + want any + }{ + {desc: "head list 1", expr: `head [1 2 3]`, want: 1}, + + {desc: "head itr 1", expr: `head (itr)`, want: 1}, + {desc: "head itr 2", expr: `set h (itr) ; head $h`, want: 1}, + {desc: "head itr 3", expr: `set h (itr) ; head $h ; head $h`, want: 2}, + {desc: "head itr 4", expr: `set h (itr) ; head $h ; head $h ; head $h`, want: 3}, + {desc: "head itr 5", expr: `set h (itr) ; head $h ; head $h ; head $h ; head $h`, want: nil}, } for _, tt := range tests { diff --git a/ucl/userbuiltin.go b/ucl/userbuiltin.go index ce26e40..a576b45 100644 --- a/ucl/userbuiltin.go +++ b/ucl/userbuiltin.go @@ -127,6 +127,12 @@ func (ca CallArgs) bindArg(v interface{}, arg Object) error { return nil } return errors.New("exepected listable") + case *Iterable: + if i, ok := arg.(Iterable); ok { + *t = i + return nil + } + return errors.New("exepected iterable") case *string: if arg != nil { *t = arg.String()