lmika/ucl
1
0
Fork 0
Code Issues 3 Pull requests Actions Packages Projects Releases Wiki Activity

dramatic simplification of command evaluation

Browse source 
- Removed the notion of streams
- Modified pipes such that any object can be passed through pipe. Now, the first argument will be used if the command is invoked via a pipe
This commit is contained in:
Leon Mika 2024-04-23 22:02:06 +10:00
parent 4c532e5005
commit 63762e633c
7 changed files with 165 additions and 332 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
cmdlang/builtins.go
View file

@ -1,12 +1,9 @@
package cmdlang
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"os"
"strings"
)
@ -48,18 +45,15 @@ func setBuiltin(ctx context.Context, args invocationArgs) (object, error) {
return newVal, nil
}
func toUpperBuiltin(ctx context.Context, inStream stream, args invocationArgs) (object, error) {
// Handle args
return mapFilterStream{
in: inStream,
mapFn: func(x object) (object, bool, error) {
s, ok := x.(strObject)
if !ok {
return nil, false, nil
}
return strObject(strings.ToUpper(string(s))), true, nil
},
}, nil
func toUpperBuiltin(ctx context.Context, args invocationArgs) (object, error) {
if err := args.expectArgn(1); err != nil {
return nil, err
}
sarg, err := args.stringArg(0)
if err != nil {
return nil, err
}
return strObject(strings.ToUpper(sarg)), nil
}
func eqBuiltin(ctx context.Context, args invocationArgs) (object, error) {
@ -92,18 +86,19 @@ func concatBuiltin(ctx context.Context, args invocationArgs) (object, error) {
return strObject(sb.String()), nil
}
func catBuiltin(ctx context.Context, args invocationArgs) (object, error) {
if err := args.expectArgn(1); err != nil {
return nil, err
}
filename, err := args.stringArg(0)
if err != nil {
return nil, err
}
return &fileLinesStream{filename: filename}, nil
}
//
//func catBuiltin(ctx context.Context, args invocationArgs) (object, error) {
// if err := args.expectArgn(1); err != nil {
// return nil, err
// }
//
// filename, err := args.stringArg(0)
// if err != nil {
// return nil, err
// }
//
// return &fileLinesStream{filename: filename}, nil
//}
func callBuiltin(ctx context.Context, args invocationArgs) (object, error) {
if err := args.expectArgn(1); err != nil {
@ -118,47 +113,49 @@ func callBuiltin(ctx context.Context, args invocationArgs) (object, error) {
return inv.invoke(ctx, args.shift(1))
}
func mapBuiltin(ctx context.Context, inStream stream, args invocationArgs) (object, error) {
args, strm, err := args.streamableSource(inStream)
func mapBuiltin(ctx context.Context, args invocationArgs) (object, error) {
if err := args.expectArgn(2); err != nil {
return nil, err
}
inv, err := args.invokableArg(1)
if err != nil {
return nil, err
}
switch t := args.args[0].(type) {
case listable:
l := t.Len()
newList := listObject{}
for i := 0; i < l; i++ {
v := t.Index(i)
m, err := inv.invoke(ctx, args.fork([]object{v}))
if err != nil {
return nil, err
}
newList = append(newList, m)
}
return newList, nil
}
return nil, errors.New("expected listable")
}
func firstBuiltin(ctx context.Context, args invocationArgs) (object, error) {
if err := args.expectArgn(1); err != nil {
return nil, err
}
inv, ok := args.args[0].(invokable)
if !ok {
return nil, errors.New("expected invokable")
switch t := args.args[0].(type) {
case listable:
if t.Len() == 0 {
return nil, nil
}
return t.Index(0), nil
}
return mapFilterStream{
in: strm,
mapFn: func(x object) (object, bool, error) {
y, err := inv.invoke(ctx, args.fork(nil, []object{x}))
return y, true, err
},
}, nil
}
func firstBuiltin(ctx context.Context, inStream stream, args invocationArgs) (object, error) {
args, strm, err := args.streamableSource(inStream)
if err != nil {
return nil, err
}
defer strm.close()
x, err := strm.next()
if errors.Is(err, io.EOF) {
return nil, nil
} else if err != nil {
return x, nil
}
return x, nil
return nil, errors.New("expected listable")
}
/*
type fileLinesStream struct {
filename string
f *os.File
@ -200,6 +197,7 @@ func (f *fileLinesStream) close() error {
}
return nil
}
*/
func ifBuiltin(ctx context.Context, args macroArgs) (object, error) {
if args.nargs() < 2 {
@ -268,7 +266,6 @@ func foreachBuiltin(ctx context.Context, args macroArgs) (object, error) {
return nil, err
}
}
// TODO: streams
}
return last, nil

58
cmdlang/eval.go
View file

@ -36,13 +36,6 @@ func (e evaluator) evalStatement(ctx context.Context, ec *evalCtx, n *astStateme
}
for _, rest := range n.Rest {
// Discard and close unused streams
if s, isStream := res.(stream); isStream {
if err := s.close(); err != nil {
return nil, err
}
}
out, err := e.evalPipeline(ctx, ec, rest)
if err != nil {
return nil, err
@ -63,7 +56,7 @@ func (e evaluator) evalPipeline(ctx context.Context, ec *evalCtx, n *astPipeline
// Command is a pipeline, so build it out
for _, rest := range n.Rest {
out, err := e.evalCmd(ctx, ec, asStream(res), rest)
out, err := e.evalCmd(ctx, ec, res, rest)
if err != nil {
return nil, err
}
@ -72,16 +65,16 @@ func (e evaluator) evalPipeline(ctx context.Context, ec *evalCtx, n *astPipeline
return res, nil
}
func (e evaluator) evalCmd(ctx context.Context, ec *evalCtx, currentStream stream, ast *astCmd) (object, error) {
func (e evaluator) evalCmd(ctx context.Context, ec *evalCtx, currentPipe object, ast *astCmd) (object, error) {
switch {
case ast.Name.Ident != nil:
name := *ast.Name.Ident
// Regular command
if cmd := ec.lookupInvokable(name); cmd != nil {
return e.evalInvokable(ctx, ec, currentStream, ast, cmd)
return e.evalInvokable(ctx, ec, currentPipe, ast, cmd)
} else if macro := ec.lookupMacro(name); macro != nil {
return e.evalMacro(ctx, ec, currentStream, ast, macro)
return e.evalMacro(ctx, ec, currentPipe, ast, macro)
} else {
return nil, errors.New("unknown command: " + name)
}
@ -96,7 +89,7 @@ func (e evaluator) evalCmd(ctx context.Context, ec *evalCtx, currentStream strea
return nil, errors.New("command is not invokable")
}
return e.evalInvokable(ctx, ec, currentStream, ast, inv)
return e.evalInvokable(ctx, ec, currentPipe, ast, inv)
}
nameElem, err := e.evalArg(ctx, ec, ast.Name)
@ -106,7 +99,7 @@ func (e evaluator) evalCmd(ctx context.Context, ec *evalCtx, currentStream strea
return nameElem, nil
}
func (e evaluator) evalInvokable(ctx context.Context, ec *evalCtx, currentStream stream, ast *astCmd, cmd invokable) (object, error) {
func (e evaluator) evalInvokable(ctx context.Context, ec *evalCtx, currentPipe object, ast *astCmd, cmd invokable) (object, error) {
var (
pargs listObject
kwargs map[string]*listObject
@ -114,6 +107,9 @@ func (e evaluator) evalInvokable(ctx context.Context, ec *evalCtx, currentStream
)
argsPtr = &pargs
if currentPipe != nil {
argsPtr.Append(currentPipe)
}
for _, arg := range ast.Args {
if ident := arg.Ident; ident != nil && (*ident)[0] == '-' {
// Arg switch
@ -132,27 +128,16 @@ func (e evaluator) evalInvokable(ctx context.Context, ec *evalCtx, currentStream
}
}
invArgs := invocationArgs{ec: ec, inst: e.inst, args: pargs, kwargs: kwargs, currentStream: currentStream}
if currentStream != nil {
if si, ok := cmd.(streamInvokable); ok {
return si.invokeWithStream(ctx, currentStream, invArgs)
} else {
if err := currentStream.close(); err != nil {
return nil, err
}
}
}
invArgs := invocationArgs{eval: e, ec: ec, inst: e.inst, args: pargs, kwargs: kwargs}
return cmd.invoke(ctx, invArgs)
}
func (e evaluator) evalMacro(ctx context.Context, ec *evalCtx, currentStream stream, ast *astCmd, cmd macroable) (object, error) {
func (e evaluator) evalMacro(ctx context.Context, ec *evalCtx, pipeArg object, ast *astCmd, cmd macroable) (object, error) {
return cmd.invokeMacro(ctx, macroArgs{
eval: e,
ec: ec,
currentStream: currentStream,
ast: ast,
eval: e,
ec: ec,
pipeArg: pipeArg,
ast: ast,
})
}
@ -241,18 +226,5 @@ func (e evaluator) evalSub(ctx context.Context, ec *evalCtx, n *astPipeline) (ob
if err != nil {
return nil, err
}
switch v := pipelineRes.(type) {
case stream:
list := listObject{}
if err := forEach(v, func(o object, _ int) error {
list = append(list, o)
return nil
}); err != nil {
return nil, err
}
return list, nil
}
return pipelineRes, nil
}

8
cmdlang/inst.go
View file

@ -29,12 +29,12 @@ func New(opts ...InstOption) *Inst {
rootEC.addCmd("echo", invokableFunc(echoBuiltin))
rootEC.addCmd("set", invokableFunc(setBuiltin))
rootEC.addCmd("toUpper", invokableStreamFunc(toUpperBuiltin))
rootEC.addCmd("toUpper", invokableFunc(toUpperBuiltin))
//rootEC.addCmd("cat", invokableFunc(catBuiltin))
rootEC.addCmd("call", invokableFunc(callBuiltin))
rootEC.addCmd("map", invokableStreamFunc(mapBuiltin))
rootEC.addCmd("head", invokableStreamFunc(firstBuiltin))
rootEC.addCmd("map", invokableFunc(mapBuiltin))
rootEC.addCmd("head", invokableFunc(firstBuiltin))
rootEC.addCmd("eq", invokableFunc(eqBuiltin))
rootEC.addCmd("cat", invokableFunc(concatBuiltin))
@ -107,8 +107,6 @@ func (inst *Inst) display(ctx context.Context, res object) (err error) {
if _, err = fmt.Fprintln(inst.out, "(nil)"); err != nil {
return err
}
case stream:
return forEach(v, func(o object, _ int) error { return inst.display(ctx, o) })
default:
if _, err = fmt.Fprintln(inst.out, v.String()); err != nil {
return err

16
cmdlang/inst_test.go
View file

@ -29,20 +29,22 @@ func TestInst_Eval(t *testing.T) {
{desc: "var 3", expr: `firstarg (sjoin $bee " " $bee " " $bee)`, want: "buzz buzz buzz"},
// Pipeline
{desc: "pipe 1", expr: `pipe "aye" "bee" "see" | joinpipe`, want: "aye,bee,see"},
{desc: "pipe 2", expr: `pipe "aye" "bee" "see" | toUpper | joinpipe`, want: "AYE,BEE,SEE"},
{desc: "pipe 3", expr: `firstarg "normal" | toUpper | joinpipe`, want: "NORMAL"},
{desc: "pipe 1", expr: `list "aye" "bee" "see" | joinpipe`, want: "aye,bee,see"},
{desc: "pipe 2", expr: `list "aye" "bee" "see" | map { |x| toUpper $x } | joinpipe`, want: "AYE,BEE,SEE"},
{desc: "pipe 3", expr: `firstarg ["normal"] | map { |x| toUpper $x } | joinpipe`, want: "NORMAL"},
{desc: "pipe literal 1", expr: `"hello" | firstarg`, want: "hello"},
{desc: "pipe literal 2", expr: `["hello" "world"] | joinpipe`, want: "hello,world"},
{desc: "ignored pipe", expr: `pipe "aye" "bee" "see" | firstarg "ignore me"`, want: "ignore me"}, // TODO: check for leaks
{desc: "ignored pipe", expr: `(list "aye" | firstarg "ignore me") | joinpipe`, want: "aye"},
// Multi-statements
{desc: "multi 1", expr: `firstarg "hello" ; firstarg "world"`, want: "world"},
{desc: "multi 2", expr: `pipe "hello" | toUpper ; firstarg "world"`, want: "world"}, // TODO: assert for leaks
{desc: "multi 2", expr: `list "hello" | toUpper ; firstarg "world"`, want: "world"},
{desc: "multi 3", expr: `set new "this is new" ; firstarg $new`, want: "this is new"},
// Lists
{desc: "list 1", expr: `firstarg ["1" "2" "3"]`, want: []any{"1", "2", "3"}},
{desc: "list 2", expr: `set one "one" ; firstarg [$one (pipe "two" | toUpper | head) "three"]`, want: []any{"one", "TWO", "three"}},
{desc: "list 2", expr: `set one "one" ; firstarg [$one (list "two" | map { |x| toUpper $x } | head) "three"]`, want: []any{"one", "TWO", "three"}},
{desc: "list 3", expr: `firstarg []`, want: []any{}},
// Maps
@ -52,7 +54,7 @@ func TestInst_Eval(t *testing.T) {
set one "one" ; set n1 "1"
firstarg [
$one:$n1
(firstarg "two" | toUpper | head):(firstarg "2" | toUpper | head)
(list "two" | map { |x| toUpper $x } | head):(list "2" | map { |x| toUpper $x } | head)
three:"3"
]`, want: map[string]any{"one": "1", "TWO": "2", "three": "3"}},
{desc: "map 4", expr: `firstarg [:]`, want: map[string]any{}},

115
cmdlang/objs.go
View file

@ -18,6 +18,11 @@ type listable interface {
Index(i int) object
}
type hashable interface {
Len() int
Each(func(k string, v object) error) error
}
type listObject []object
func (lo *listObject) Append(o object) {
@ -50,6 +55,19 @@ func (s hashObject) Truthy() bool {
return len(s) > 0
}
func (s hashObject) Len() int {
return len(s)
}
func (s hashObject) Each(fn func(k string, v object) error) error {
for k, v := range s {
if err := fn(k, v); err != nil {
return err
}
}
return nil
}
type strObject string
func (s strObject) String() string {
@ -125,11 +143,11 @@ func fromGoValue(v any) (object, error) {
}
type macroArgs struct {
eval evaluator
ec *evalCtx
currentStream stream
ast *astCmd
argShift int
eval evaluator
ec *evalCtx
pipeArg object
ast *astCmd
argShift int
}
func (ma macroArgs) nargs() int {
@ -199,30 +217,11 @@ func (ma macroArgs) evalBlock(ctx context.Context, n int, args []object, pushSco
}
type invocationArgs struct {
inst *Inst
ec *evalCtx
currentStream stream
args []object
kwargs map[string]*listObject
}
// streamableSource takes a stream. If the stream is set, the inStream and invocation arguments are consumed as is.
// If not, then the first argument is consumed and returned as a stream.
func (ia invocationArgs) streamableSource(inStream stream) (invocationArgs, stream, error) {
if inStream != nil {
return ia, inStream, nil
}
if len(ia.args) < 1 {
return ia, nil, errors.New("expected at least 1 argument")
}
switch v := ia.args[0].(type) {
case listObject:
return ia.shift(1), &listIterStream{list: v}, nil
}
return ia, nil, errors.New("expected arg 0 to be streamable")
eval evaluator
inst *Inst
ec *evalCtx
args []object
kwargs map[string]*listObject
}
func (ia invocationArgs) expectArgn(x int) error {
@ -243,23 +242,35 @@ func (ia invocationArgs) stringArg(i int) (string, error) {
return s.String(), nil
}
func (ia invocationArgs) fork(currentStr stream, args []object) invocationArgs {
func (ia invocationArgs) invokableArg(i int) (invokable, error) {
if len(ia.args) < i {
return nil, errors.New("expected at least " + strconv.Itoa(i) + " args")
}
switch v := ia.args[i].(type) {
case invokable:
return v, nil
}
return nil, errors.New("expected an invokable arg")
}
func (ia invocationArgs) fork(args []object) invocationArgs {
return invocationArgs{
inst: ia.inst,
ec: ia.ec,
currentStream: currentStr,
args: args,
kwargs: make(map[string]*listObject),
eval: ia.eval,
inst: ia.inst,
ec: ia.ec,
args: args,
kwargs: make(map[string]*listObject),
}
}
func (ia invocationArgs) shift(i int) invocationArgs {
return invocationArgs{
inst: ia.inst,
ec: ia.ec,
currentStream: ia.currentStream,
args: ia.args[i:],
kwargs: ia.kwargs,
eval: ia.eval,
inst: ia.inst,
ec: ia.ec,
args: ia.args[i:],
kwargs: ia.kwargs,
}
}
@ -272,9 +283,8 @@ type macroable interface {
invokeMacro(ctx context.Context, args macroArgs) (object, error)
}
type streamInvokable interface {
type pipeInvokable interface {
invokable
invokeWithStream(context.Context, stream, invocationArgs) (object, error)
}
type invokableFunc func(ctx context.Context, args invocationArgs) (object, error)
@ -283,16 +293,6 @@ func (i invokableFunc) invoke(ctx context.Context, args invocationArgs) (object,
return i(ctx, args)
}
type invokableStreamFunc func(ctx context.Context, inStream stream, args invocationArgs) (object, error)
func (i invokableStreamFunc) invoke(ctx context.Context, args invocationArgs) (object, error) {
return i(ctx, nil, args)
}
func (i invokableStreamFunc) invokeWithStream(ctx context.Context, inStream stream, args invocationArgs) (object, error) {
return i(ctx, inStream, args)
}
type blockObject struct {
block *astBlock
}
@ -305,6 +305,17 @@ func (bo blockObject) Truthy() bool {
return len(bo.block.Statements) > 0
}
func (bo blockObject) invoke(ctx context.Context, args invocationArgs) (object, error) {
ec := args.ec.fork()
for i, n := range bo.block.Names {
if i < len(args.args) {
ec.setVar(n, args.args[i])
}
}
return args.eval.evalBlock(ctx, ec, bo.block)
}
type macroFunc func(ctx context.Context, args macroArgs) (object, error)
func (i macroFunc) invokeMacro(ctx context.Context, args macroArgs) (object, error) {

152
cmdlang/streams.go
View file

@ -1,152 +0,0 @@
package cmdlang
import (
"errors"
"fmt"
"io"
)
// stream is an object which returns a collection of objects from a source.
// These are used to create pipelines
//
// The stream implementation can expect close to be called if at least one next() call is made. Otherwise
// closableStream cannot assume that close will be called (the pipe may be left unconsumed, for example).
//
// It is the job of the final iterator to call close. Any steam that consumes from another stream must
// implement this, and call close on the parent stream.
type stream interface {
object
// next pulls the next object from the stream. If an object is available, the result is the
// object and a nil error. If no more objects are available, error returns io.EOF.
// Otherwise, an error is returned.
next() (object, error)
close() error
}
// forEach will iterate over all the items of a stream. The iterating function can return an error, which will
// be returned as is. A stream that has consumed every item will return nil. The stream will automatically be closed.
func forEach(s stream, f func(object, int) error) (err error) {
defer s.close()
var sv object
i := 0
for sv, err = s.next(); err == nil; sv, err = s.next() {
if err := f(sv, i); err != nil {
return err
}
i += 1
}
if !errors.Is(err, io.EOF) {
return err
}
return nil
}
// asStream converts an object to a stream. If t is already a stream, it's returned as is.
// Otherwise, a singleton stream is returned.
func asStream(v object) stream {
switch s := v.(type) {
case stream:
return s
case listObject:
return &listIterStream{list: s}
}
return &singletonStream{t: v}
}
type emptyStream struct{}
func (s *emptyStream) String() string {
return "(nil)"
}
func (s emptyStream) next() (object, error) {
return nil, io.EOF
}
func (s emptyStream) close() error { return nil }
type singletonStream struct {
t object
consumed bool
}
func (s *singletonStream) String() string {
return s.t.String()
}
func (s *singletonStream) Truthy() bool {
return !s.consumed
}
func (s *singletonStream) next() (object, error) {
if s.consumed {
return nil, io.EOF
}
s.consumed = true
return s.t, nil
}
func (s *singletonStream) close() error { return nil }
type listIterStream struct {
list []object
cusr int
}
func (s *listIterStream) String() string {
return fmt.Sprintf("listIterStream{list: %v}", s.list)
}
func (s *listIterStream) Truthy() bool {
return len(s.list) > s.cusr
}
func (s *listIterStream) next() (o object, err error) {
if s.cusr >= len(s.list) {
return nil, io.EOF
}
o = s.list[s.cusr]
s.cusr += 1
return o, nil
}
func (s *listIterStream) close() error { return nil }
type mapFilterStream struct {
in stream
mapFn func(x object) (object, bool, error)
}
func (ms mapFilterStream) String() string {
return fmt.Sprintf("mapFilterStream{in: %v}", ms.in)
}
func (ms mapFilterStream) Truthy() bool {
return true // ???
}
func (ms mapFilterStream) next() (object, error) {
for {
u, err := ms.in.next()
if err != nil {
return nil, err
}
t, ok, err := ms.mapFn(u)
if err != nil {
return nil, err
} else if ok {
return t, nil
}
}
}
func (ms mapFilterStream) close() error {
return ms.in.close()
}

35
cmdlang/testbuiltins_test.go
View file

@ -31,22 +31,24 @@ func WithTestBuiltin() InstOption {
return strObject(line.String()), nil
}))
i.rootEC.addCmd("pipe", invokableFunc(func(ctx context.Context, args invocationArgs) (object, error) {
return &listIterStream{
list: args.args,
}, nil
i.rootEC.addCmd("list", invokableFunc(func(ctx context.Context, args invocationArgs) (object, error) {
return listObject(args.args), nil
}))
i.rootEC.addCmd("joinpipe", invokableStreamFunc(func(ctx context.Context, inStream stream, args invocationArgs) (object, error) {
i.rootEC.addCmd("joinpipe", invokableFunc(func(ctx context.Context, args invocationArgs) (object, error) {
sb := strings.Builder{}
if err := forEach(inStream, func(o object, i int) error {
if i > 0 {
lst, ok := args.args[0].(listable)
if !ok {
return strObject(""), nil
}
l := lst.Len()
for x := 0; x < l; x++ {
if x > 0 {
sb.WriteString(",")
}
sb.WriteString(o.String())
return nil
}); err != nil {
return nil, err
sb.WriteString(lst.Index(x).String())
}
return strObject(sb.String()), nil
}))
@ -292,17 +294,20 @@ func TestBuiltins_Map(t *testing.T) {
proc makeUpper { |x| $x | toUpper }
map ["a" "b" "c"] (proc { |x| makeUpper $x })
`, want: "A\nB\nC\n"},
`, want: "[A B C]\n"},
{desc: "map list 2", expr: `
set makeUpper (proc { |x| $x | toUpper })
map ["a" "b" "c"] $makeUpper
`, want: "A\nB\nC\n"},
{desc: "map list with stream", expr: `
`, want: "[A B C]\n"},
{desc: "map list with pipe", expr: `
set makeUpper (proc { |x| $x | toUpper })
["a" "b" "c"] | map $makeUpper
`, want: "A\nB\nC\n"},
`, want: "[A B C]\n"},
{desc: "map list with block", expr: `
map ["a" "b" "c"] { |x| toUpper $x }
`, want: "[A B C]\n"},
//{desc: "map list with stream", expr: `
// set makeUpper (proc { |x| $x | toUpper })
//