From 922acd37510b50247b0e7a5d3c816a8818bda61e Mon Sep 17 00:00:00 2001 From: Leon Mika Date: Thu, 11 Apr 2024 22:05:05 +1000 Subject: [PATCH] Started writing some tests --- cmdlang/ast.go | 9 +++- cmdlang/builtins.go | 17 +++++--- cmdlang/eval.go | 40 ++++++++++++++---- cmdlang/inst.go | 55 ++++++++++++++++++++---- cmdlang/inst_test.go | 81 ++++++++++++++++++++++++++++++++++++ cmdlang/objs.go | 13 ++++++ cmdlang/streams.go | 45 ++++++++++++++++++-- cmdlang/testbuiltins_test.go | 38 +++++++++++++++++ go.mod | 4 ++ go.sum | 9 ++++ 10 files changed, 284 insertions(+), 27 deletions(-) create mode 100644 cmdlang/inst_test.go create mode 100644 cmdlang/testbuiltins_test.go diff --git a/cmdlang/ast.go b/cmdlang/ast.go index a305692..4adbc2c 100644 --- a/cmdlang/ast.go +++ b/cmdlang/ast.go @@ -26,8 +26,13 @@ type astPipeline struct { Rest []*astCmd `parser:"( '|' @@ )*"` } -var parser = participle.MustBuild[astPipeline]() +type astStatements struct { + First *astPipeline `parser:"@@"` + Rest []*astPipeline `parser:"( ';' @@ )*"` // TODO: also add support for newlines +} -func parse(r io.Reader) (*astPipeline, error) { +var parser = participle.MustBuild[astStatements]() + +func parse(r io.Reader) (*astStatements, error) { return parser.Parse("test", r) } diff --git a/cmdlang/builtins.go b/cmdlang/builtins.go index 404622a..6499942 100644 --- a/cmdlang/builtins.go +++ b/cmdlang/builtins.go @@ -3,7 +3,6 @@ package cmdlang import ( "bufio" "context" - "errors" "fmt" "io" "os" @@ -12,7 +11,7 @@ import ( func echoBuiltin(ctx context.Context, args invocationArgs) (object, error) { if len(args.args) == 0 { - return asStream(""), nil + return asStream(strObject("")), nil } var line strings.Builder @@ -22,7 +21,7 @@ func echoBuiltin(ctx context.Context, args invocationArgs) (object, error) { } } - return asStream(line.String()), nil + return asStream(strObject(line.String())), nil } func setBuiltin(ctx context.Context, args invocationArgs) (object, error) { @@ -47,11 +46,11 @@ func toUpperBuiltin(ctx context.Context, inStream stream, args invocationArgs) ( return mapFilterStream{ in: inStream, mapFn: func(x object) (object, bool) { - s, ok := x.(string) + s, ok := x.(strObject) if !ok { return nil, false } - return strings.ToUpper(s), true + return strObject(strings.ToUpper(string(s))), true }, }, nil } @@ -75,6 +74,10 @@ type fileLinesStream struct { scnr *bufio.Scanner } +func (f *fileLinesStream) String() string { + return fmt.Sprintf("fileLinesStream{file: %v}", f.filename) +} + func (f *fileLinesStream) next() (object, error) { var err error @@ -88,7 +91,7 @@ func (f *fileLinesStream) next() (object, error) { } if f.scnr.Scan() { - return f.scnr.Text(), nil + return strObject(f.scnr.Text()), nil } if f.scnr.Err() == nil { return nil, io.EOF @@ -103,6 +106,7 @@ func (f *fileLinesStream) close() error { return nil } +/* func errorTestBuiltin(ctx context.Context, inStream stream, args invocationArgs) (object, error) { return &timeBombStream{inStream, 2}, nil } @@ -123,3 +127,4 @@ func (ms *timeBombStream) next() (object, error) { func (ms *timeBombStream) close() error { return ms.in.close() } +*/ diff --git a/cmdlang/eval.go b/cmdlang/eval.go index 9658508..62514fc 100644 --- a/cmdlang/eval.go +++ b/cmdlang/eval.go @@ -12,7 +12,33 @@ import ( type evaluator struct { } -func (e evaluator) evaluate(ctx context.Context, ec *evalCtx, n *astPipeline) (object, error) { +func (e evaluator) evalStatement(ctx context.Context, ec *evalCtx, n *astStatements) (object, error) { + res, err := e.evalPipeline(ctx, ec, n.First) + if err != nil { + return nil, err + } + if len(n.Rest) == 0 { + return res, nil + } + + for _, rest := range n.Rest { + // Discard and close unused streams + if s, isStream := res.(stream); isStream { + if err := s.close(); err != nil { + return nil, err + } + } + + out, err := e.evalPipeline(ctx, ec, rest) + if err != nil { + return nil, err + } + res = out + } + return res, nil +} + +func (e evaluator) evalPipeline(ctx context.Context, ec *evalCtx, n *astPipeline) (object, error) { res, err := e.evalCmd(ctx, ec, n.First) if err != nil { return nil, err @@ -81,31 +107,31 @@ func (e evaluator) evalLiteral(ctx context.Context, ec *evalCtx, n *astLiteral) case n.Str != nil: uq, err := strconv.Unquote(*n.Str) if err != nil { - return "", err + return nil, err } return strObject(uq), nil case n.Ident != nil: return strObject(*n.Ident), nil } - return "", errors.New("unhandled literal type") + return nil, errors.New("unhandled literal type") } func (e evaluator) evalSub(ctx context.Context, ec *evalCtx, n *astPipeline) (object, error) { - pipelineRes, err := e.evaluate(ctx, ec, n) + pipelineRes, err := e.evalPipeline(ctx, ec, n) if err != nil { - return "", err + return nil, err } switch v := pipelineRes.(type) { case stream: // TODO: use proper lists here, not a string join sb := strings.Builder{} - if err := forEach(v, func(o object) error { + if err := forEach(v, func(o object, _ int) error { // TODO: use o.String() sb.WriteString(fmt.Sprint(o)) return nil }); err != nil { - return "", err + return nil, err } return strObject(sb.String()), nil diff --git a/cmdlang/inst.go b/cmdlang/inst.go index 3d15844..05ea127 100644 --- a/cmdlang/inst.go +++ b/cmdlang/inst.go @@ -2,43 +2,78 @@ package cmdlang import ( "context" + "errors" "fmt" + "io" + "os" "strings" ) type Inst struct { + out io.Writer + rootEC *evalCtx } -func New() *Inst { +type InstOption func(*Inst) + +func WithOut(out io.Writer) InstOption { + return func(i *Inst) { + i.out = out + } +} + +func New(opts ...InstOption) *Inst { rootEC := evalCtx{} + rootEC.addCmd("echo", invokableFunc(echoBuiltin)) rootEC.addCmd("set", invokableFunc(setBuiltin)) rootEC.addCmd("toUpper", invokableStreamFunc(toUpperBuiltin)) rootEC.addCmd("cat", invokableFunc(catBuiltin)) - rootEC.addCmd("testTimebomb", invokableStreamFunc(errorTestBuiltin)) + //rootEC.addCmd("testTimebomb", invokableStreamFunc(errorTestBuiltin)) rootEC.setVar("hello", strObject("world")) - return &Inst{ + inst := &Inst{ + out: os.Stdout, rootEC: &rootEC, } + + for _, opt := range opts { + opt(inst) + } + + return inst } -// TODO: return value? func (inst *Inst) Eval(ctx context.Context, expr string) (any, error) { + res, err := inst.eval(ctx, expr) + if err != nil { + return nil, err + } + + goRes, ok := toGoValue(res) + if !ok { + return nil, errors.New("result not convertable to go") + } + + return goRes, nil +} + +func (inst *Inst) eval(ctx context.Context, expr string) (object, error) { ast, err := parse(strings.NewReader(expr)) if err != nil { return nil, err } eval := evaluator{} - return eval.evaluate(ctx, inst.rootEC, ast) + + return eval.evalStatement(ctx, inst.rootEC, ast) } func (inst *Inst) EvalAndDisplay(ctx context.Context, expr string) error { - res, err := inst.Eval(ctx, expr) + res, err := inst.eval(ctx, expr) if err != nil { return err } @@ -49,9 +84,11 @@ func (inst *Inst) EvalAndDisplay(ctx context.Context, expr string) error { func (inst *Inst) display(ctx context.Context, res object) (err error) { switch v := res.(type) { case stream: - return forEach(v, func(o object) error { return inst.display(ctx, o) }) - case string: - fmt.Println(v) + return forEach(v, func(o object, _ int) error { return inst.display(ctx, o) }) + default: + if _, err = fmt.Fprintln(inst.out, v.String()); err != nil { + return err + } } return nil } diff --git a/cmdlang/inst_test.go b/cmdlang/inst_test.go new file mode 100644 index 0000000..51db215 --- /dev/null +++ b/cmdlang/inst_test.go @@ -0,0 +1,81 @@ +package cmdlang_test + +import ( + "bytes" + "context" + "github.com/lmika/cmdlang-proto/cmdlang" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestInst_Eval(t *testing.T) { + tests := []struct { + desc string + expr string + want string + }{ + {desc: "simple string", expr: `firstarg "hello"`, want: "hello"}, + + // Sub-expressions + {desc: "sub expression 1", expr: `firstarg (echo "hello")`, want: "hello"}, + {desc: "sub expression 2", expr: `firstarg (echo "hello " "world")`, want: "hello world"}, + {desc: "sub expression 3", expr: `firstarg (echo "hello" (echo " ") (echo "world"))`, want: "hello world"}, + + // Variables + {desc: "var 1", expr: `firstarg $a`, want: "alpha"}, + {desc: "var 2", expr: `firstarg $bee`, want: "buzz"}, + {desc: "var 3", expr: `firstarg (echo $bee " " $bee " " $bee)`, want: "buzz buzz buzz"}, + + // Pipeline + {desc: "pipe 1", expr: `pipe "aye" "bee" "see" | joinpipe`, want: "aye,bee,see"}, + {desc: "pipe 2", expr: `pipe "aye" "bee" "see" | toUpper | joinpipe`, want: "AYE,BEE,SEE"}, + + {desc: "ignored pipe", expr: `pipe "aye" "bee" "see" | firstarg "ignore me"`, want: "ignore me"}, // TODO: check for leaks + + // Multi-statements + {desc: "multi 1", expr: `firstarg "hello" ; firstarg "world"`, want: "world"}, + {desc: "multi 2", expr: `pipe "hello" | toUpper ; firstarg "world"`, want: "world"}, // TODO: assert for leaks + {desc: "multi 3", expr: `set new "this is new" ; firstarg $new`, want: "this is new"}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + ctx := context.Background() + outW := bytes.NewBuffer(nil) + + inst := cmdlang.New(cmdlang.WithOut(outW), cmdlang.WithTestBuiltin()) + res, err := inst.Eval(ctx, tt.expr) + + assert.NoError(t, err) + assert.Equal(t, tt.want, res) + }) + } +} + +func TestInst_Builtins(t *testing.T) { + t.Run("echo", func(t *testing.T) { + tests := []struct { + desc string + expr string + want string + }{ + {desc: "no args", expr: `echo`, want: "\n"}, + {desc: "single arg", expr: `echo "hello"`, want: "hello\n"}, + {desc: "dual args", expr: `echo "hello " "world"`, want: "hello world\n"}, + {desc: "args to singleton stream", expr: `echo "aye" "bee" "see" | toUpper`, want: "AYEBEESEE\n"}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + ctx := context.Background() + outW := bytes.NewBuffer(nil) + + inst := cmdlang.New(cmdlang.WithOut(outW), cmdlang.WithTestBuiltin()) + err := inst.EvalAndDisplay(ctx, tt.expr) + + assert.NoError(t, err) + assert.Equal(t, tt.want, outW.String()) + }) + } + }) +} diff --git a/cmdlang/objs.go b/cmdlang/objs.go index 1faa5a1..a8696d0 100644 --- a/cmdlang/objs.go +++ b/cmdlang/objs.go @@ -8,6 +8,7 @@ import ( ) type object interface { + String() string } type strObject string @@ -16,7 +17,19 @@ func (s strObject) String() string { return string(s) } +func toGoValue(obj object) (interface{}, bool) { + switch v := obj.(type) { + case nil: + return nil, true + case strObject: + return string(v), true + } + + return nil, false +} + type invocationArgs struct { + inst *Inst ec *evalCtx args []object } diff --git a/cmdlang/streams.go b/cmdlang/streams.go index b7f4e5c..5fd71f9 100644 --- a/cmdlang/streams.go +++ b/cmdlang/streams.go @@ -2,6 +2,7 @@ package cmdlang import ( "errors" + "fmt" "io" ) @@ -14,6 +15,8 @@ import ( // It is the job of the final iterator to call close. Any steam that consumes from another stream must // implement this, and call close on the parent stream. type stream interface { + object + // next pulls the next object from the stream. If an object is available, the result is the // object and a nil error. If no more objects are available, error returns io.EOF. // Otherwise, an error is returned. @@ -24,14 +27,16 @@ type stream interface { // forEach will iterate over all the items of a stream. The iterating function can return an error, which will // be returned as is. A stream that has consumed every item will return nil. The stream will automatically be closed. -func forEach(s stream, f func(object) error) (err error) { +func forEach(s stream, f func(object, int) error) (err error) { defer s.close() var sv object + i := 0 for sv, err = s.next(); err == nil; sv, err = s.next() { - if err := f(sv); err != nil { + if err := f(sv, i); err != nil { return err } + i += 1 } if !errors.Is(err, io.EOF) { return err @@ -50,6 +55,10 @@ func asStream(v object) stream { type emptyStream struct{} +func (s *emptyStream) String() string { + return "(nil)" +} + func (s emptyStream) next() (object, error) { return nil, io.EOF } @@ -57,10 +66,14 @@ func (s emptyStream) next() (object, error) { func (s emptyStream) close() error { return nil } type singletonStream struct { - t any + t object consumed bool } +func (s *singletonStream) String() string { + return s.t.String() +} + func (s *singletonStream) next() (object, error) { if s.consumed { return nil, io.EOF @@ -71,11 +84,37 @@ func (s *singletonStream) next() (object, error) { func (s *singletonStream) close() error { return nil } +type listIterStream struct { + list []object + cusr int +} + +func (s *listIterStream) String() string { + return fmt.Sprintf("listIterStream{list: %v}", s.list) +} + +func (s *listIterStream) next() (o object, err error) { + if s.cusr >= len(s.list) { + return nil, io.EOF + } + + o = s.list[s.cusr] + s.cusr += 1 + + return o, nil +} + +func (s *listIterStream) close() error { return nil } + type mapFilterStream struct { in stream mapFn func(x object) (object, bool) } +func (ms mapFilterStream) String() string { + return fmt.Sprintf("mapFilterStream{in: %v}", ms.in) +} + func (ms mapFilterStream) next() (object, error) { for { u, err := ms.in.next() diff --git a/cmdlang/testbuiltins_test.go b/cmdlang/testbuiltins_test.go new file mode 100644 index 0000000..e879a1e --- /dev/null +++ b/cmdlang/testbuiltins_test.go @@ -0,0 +1,38 @@ +package cmdlang + +import ( + "context" + "strings" +) + +// Builtins used for test +func WithTestBuiltin() InstOption { + return func(i *Inst) { + i.rootEC.addCmd("firstarg", invokableFunc(func(ctx context.Context, args invocationArgs) (object, error) { + return args.args[0], nil + })) + + i.rootEC.addCmd("pipe", invokableFunc(func(ctx context.Context, args invocationArgs) (object, error) { + return &listIterStream{ + list: args.args, + }, nil + })) + + i.rootEC.addCmd("joinpipe", invokableStreamFunc(func(ctx context.Context, inStream stream, args invocationArgs) (object, error) { + sb := strings.Builder{} + if err := forEach(inStream, func(o object, i int) error { + if i > 0 { + sb.WriteString(",") + } + sb.WriteString(o.String()) + return nil + }); err != nil { + return nil, err + } + return strObject(sb.String()), nil + })) + + i.rootEC.setVar("a", strObject("alpha")) + i.rootEC.setVar("bee", strObject("buzz")) + } +} diff --git a/go.mod b/go.mod index 70190e5..be04ff5 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,10 @@ go 1.21.1 require ( github.com/alecthomas/participle/v2 v2.1.1 // indirect github.com/chzyer/readline v1.5.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/lmika/gopkgs v0.0.0-20240408110817-a02f6fc67d1f // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f982ff2..21d3bb6 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,16 @@ github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwys github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/lmika/gopkgs v0.0.0-20240408110817-a02f6fc67d1f h1:tz68Lhc1oR15HVz69IGbtdukdH0x70kBDEvvj5pTXyE= github.com/lmika/gopkgs v0.0.0-20240408110817-a02f6fc67d1f/go.mod h1:zHQvhjGXRro/Xp2C9dbC+ZUpE0gL4GYW75x1lk7hwzI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=