From 98f5f773a744df485ec989b48580f662286f76ec Mon Sep 17 00:00:00 2001 From: Leon Mika Date: Sat, 18 Jan 2025 10:30:20 +1100 Subject: [PATCH] Added a CSV module --- cmd/cmsh/main.go | 4 +- cmd/playwasm/jsiter.go | 4 +- ucl/builtins.go | 62 +++++++++++------------ ucl/builtins/csv.go | 104 +++++++++++++++++++++++++++++++++++++++ ucl/builtins/csv_test.go | 53 ++++++++++++++++++++ ucl/eval.go | 2 +- ucl/objs.go | 12 ++--- ucl/userbuiltin.go | 4 +- 8 files changed, 203 insertions(+), 42 deletions(-) create mode 100644 ucl/builtins/csv.go create mode 100644 ucl/builtins/csv_test.go diff --git a/cmd/cmsh/main.go b/cmd/cmsh/main.go index 8296224..baaf1aa 100644 --- a/cmd/cmsh/main.go +++ b/cmd/cmsh/main.go @@ -18,10 +18,12 @@ func main() { defer rl.Close() instRepl := repl.New( - ucl.WithModule(builtins.OS()), + ucl.WithModule(builtins.CSV(nil)), ucl.WithModule(builtins.FS(nil)), ucl.WithModule(builtins.Log(nil)), + ucl.WithModule(builtins.OS()), ucl.WithModule(builtins.Strs()), + ucl.WithModule(builtins.Time()), ) ctx := context.Background() diff --git a/cmd/playwasm/jsiter.go b/cmd/playwasm/jsiter.go index c7fcead..f2dd726 100644 --- a/cmd/playwasm/jsiter.go +++ b/cmd/playwasm/jsiter.go @@ -25,9 +25,11 @@ func initJS(ctx context.Context) { replInst := repl.New( ucl.WithModule(builtins.Log(nil)), ucl.WithModule(builtins.Strs()), + ucl.WithModule(builtins.Time()), ucl.WithOut(ucl.LineHandler(func(line string) { invokeUCLCallback("onOutLine", line) - }))) + })), + ) uclObj["eval"] = js.FuncOf(func(this js.Value, args []js.Value) any { if len(args) != 2 { diff --git a/ucl/builtins.go b/ucl/builtins.go index 381abac..3b3fbd0 100644 --- a/ucl/builtins.go +++ b/ucl/builtins.go @@ -44,13 +44,13 @@ func echoBuiltin(ctx context.Context, args invocationArgs) (Object, error) { func addBuiltin(ctx context.Context, args invocationArgs) (Object, error) { if len(args.args) == 0 { - return intObject(0), nil + return IntObject(0), nil } n := 0 for i, a := range args.args { switch t := a.(type) { - case intObject: + case IntObject: n += int(t) case StringObject: v, err := strconv.Atoi(string(t)) @@ -63,19 +63,19 @@ func addBuiltin(ctx context.Context, args invocationArgs) (Object, error) { } } - return intObject(n), nil + return IntObject(n), nil } func subBuiltin(ctx context.Context, args invocationArgs) (Object, error) { if len(args.args) == 0 { - return intObject(0), nil + return IntObject(0), nil } n := 0 for i, a := range args.args { var p int switch t := a.(type) { - case intObject: + case IntObject: p = int(t) case StringObject: v, err := strconv.Atoi(string(t)) @@ -93,18 +93,18 @@ func subBuiltin(ctx context.Context, args invocationArgs) (Object, error) { } } - return intObject(n), nil + return IntObject(n), nil } func mupBuiltin(ctx context.Context, args invocationArgs) (Object, error) { if len(args.args) == 0 { - return intObject(1), nil + return IntObject(1), nil } n := 1 for i, a := range args.args { switch t := a.(type) { - case intObject: + case IntObject: n *= int(t) case StringObject: v, err := strconv.Atoi(string(t)) @@ -117,19 +117,19 @@ func mupBuiltin(ctx context.Context, args invocationArgs) (Object, error) { } } - return intObject(n), nil + return IntObject(n), nil } func divBuiltin(ctx context.Context, args invocationArgs) (Object, error) { if len(args.args) == 0 { - return intObject(1), nil + return IntObject(1), nil } n := 1 for i, a := range args.args { var p int switch t := a.(type) { - case intObject: + case IntObject: p = int(t) case StringObject: v, err := strconv.Atoi(string(t)) @@ -147,19 +147,19 @@ func divBuiltin(ctx context.Context, args invocationArgs) (Object, error) { } } - return intObject(n), nil + return IntObject(n), nil } func modBuiltin(ctx context.Context, args invocationArgs) (Object, error) { if len(args.args) == 0 { - return intObject(0), nil + return IntObject(0), nil } n := 0 for i, a := range args.args { var p int switch t := a.(type) { - case intObject: + case IntObject: p = int(t) case StringObject: v, err := strconv.Atoi(string(t)) @@ -177,7 +177,7 @@ func modBuiltin(ctx context.Context, args invocationArgs) (Object, error) { } } - return intObject(n), nil + return IntObject(n), nil } func setBuiltin(ctx context.Context, args invocationArgs) (Object, error) { @@ -323,8 +323,8 @@ func objectsEqual(l, r Object) bool { if rv, ok := r.(StringObject); ok { return lv == rv } - case intObject: - if rv, ok := r.(intObject); ok { + case IntObject: + if rv, ok := r.(IntObject); ok { return lv == rv } case boolObject: @@ -384,8 +384,8 @@ func objectsLessThan(l, r Object) (bool, error) { if rv, ok := r.(StringObject); ok { return lv < rv, nil } - case intObject: - if rv, ok := r.(intObject); ok { + case IntObject: + if rv, ok := r.(IntObject); ok { return lv < rv, nil } } @@ -410,23 +410,23 @@ func intBuiltin(ctx context.Context, args invocationArgs) (Object, error) { } if args.args[0] == nil { - return intObject(0), nil + return IntObject(0), nil } switch v := args.args[0].(type) { - case intObject: + case IntObject: return v, nil case StringObject: i, err := strconv.Atoi(string(v)) if err != nil { return nil, errors.New("cannot convert to int") } - return intObject(i), nil + return IntObject(i), nil case boolObject: if v { - return intObject(1), nil + return IntObject(1), nil } - return intObject(0), nil + return IntObject(0), nil } return nil, errors.New("cannot convert to int") @@ -465,20 +465,20 @@ func lenBuiltin(ctx context.Context, args invocationArgs) (Object, error) { switch v := args.args[0].(type) { case StringObject: - return intObject(len(string(v))), nil + return IntObject(len(string(v))), nil case Listable: - return intObject(v.Len()), nil + return IntObject(v.Len()), nil case hashable: - return intObject(v.Len()), nil + return IntObject(v.Len()), nil } - return intObject(0), nil + return IntObject(0), nil } func indexLookup(ctx context.Context, obj, elem Object) (Object, error) { switch v := obj.(type) { case Listable: - intIdx, ok := elem.(intObject) + intIdx, ok := elem.(IntObject) if !ok { return nil, nil } @@ -711,9 +711,9 @@ func (s seqObject) Index(i int) Object { return nil } if s.from > s.to { - return intObject(s.from - i) + return IntObject(s.from - i) } - return intObject(s.from + i) + return IntObject(s.from + i) } func seqBuiltin(ctx context.Context, args invocationArgs) (Object, error) { diff --git a/ucl/builtins/csv.go b/ucl/builtins/csv.go new file mode 100644 index 0000000..3b00cf4 --- /dev/null +++ b/ucl/builtins/csv.go @@ -0,0 +1,104 @@ +package builtins + +import ( + "context" + "encoding/csv" + "errors" + "io" + "io/fs" + "strings" + "ucl.lmika.dev/ucl" +) + +type csvHandlers struct { + fs fs.FS +} + +func CSV(fs fs.FS) ucl.Module { + fsh := csvHandlers{fs: fs} + + return ucl.Module{ + Name: "csv", + Builtins: map[string]ucl.BuiltinHandler{ + "each-record": fsh.eachRecord, + }, + } +} + +func (h csvHandlers) eachRecord(ctx context.Context, args ucl.CallArgs) (any, error) { + var ( + filename string + closure ucl.Invokable + ) + + if err := args.Bind(&filename, &closure); err != nil { + return nil, err + } + + f, err := h.fs.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + cr := csv.NewReader(f) + + header, err := cr.Read() + if err != nil { + return nil, err + } + hio := make(headerIndexObject) + for i, h := range header { + hio[h] = i + } + + for { + record, err := cr.Read() + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + + if _, err := closure.Invoke(ctx, stringSlice(record), hio); err != nil { + return nil, err + } + } + + return nil, nil +} + +type headerIndexObject map[string]int + +func (hio headerIndexObject) String() string { + strs := make([]string, len(hio)) + for h, i := range hio { + strs[i] = h + } + return strings.Join(strs, ",") +} + +func (hio headerIndexObject) Truthy() bool { + return len(hio) > 0 +} + +func (hio headerIndexObject) Len() int { + return len(hio) +} + +func (hio headerIndexObject) Value(k string) ucl.Object { + v, ok := hio[k] + if !ok { + return nil + } + return ucl.IntObject(v) +} + +func (hio headerIndexObject) Each(fn func(k string, v ucl.Object) error) error { + for k, v := range hio { + if err := fn(k, ucl.IntObject(v)); err != nil { + return err + } + } + return nil +} diff --git a/ucl/builtins/csv_test.go b/ucl/builtins/csv_test.go new file mode 100644 index 0000000..5b60834 --- /dev/null +++ b/ucl/builtins/csv_test.go @@ -0,0 +1,53 @@ +package builtins_test + +import ( + "bytes" + "context" + "github.com/stretchr/testify/assert" + "strings" + "testing" + "testing/fstest" + "ucl.lmika.dev/ucl" + "ucl.lmika.dev/ucl/builtins" +) + +var testCsvFS = fstest.MapFS{ + "test.csv": &fstest.MapFile{ + Data: []byte(strings.Join([]string{ + "wind,dir,bearing", + "north,N,0", + "south,S,180", + "east,E,90", + "west,W,270", + }, "\n")), + }, +} + +func TestCSV_ReadRecord(t *testing.T) { + tests := []struct { + descr string + eval string + wantOut string + }{ + {descr: "read csv 1", eval: `csv:each-record "test.csv" { |r h| echo $r.(0) }`, wantOut: "north\nsouth\neast\nwest\n"}, + {descr: "read csv 2", eval: `csv:each-record "test.csv" { |r h| echo $r.($h.dir) }`, wantOut: "N\nS\nE\nW\n"}, + {descr: "read csv 3", eval: `csv:each-record "test.csv" { |r h| echo $r.($h.bearing) "-" $r.($h.dir) }`, wantOut: "0-N\n180-S\n90-E\n270-W\n"}, + {descr: "read csv 4", eval: `csv:each-record "test.csv" { |r h| echo $h.bearing }`, wantOut: "2\n2\n2\n2\n"}, + {descr: "read csv 5", eval: `csv:each-record "test.csv" {}`, wantOut: ""}, + } + + for _, tt := range tests { + t.Run(tt.descr, func(t *testing.T) { + var bfr bytes.Buffer + + inst := ucl.New( + ucl.WithModule(builtins.CSV(testCsvFS)), + ucl.WithOut(&bfr), + ) + + _, err := inst.Eval(context.Background(), tt.eval) + assert.NoError(t, err) + assert.Equal(t, tt.wantOut, bfr.String()) + }) + } +} diff --git a/ucl/eval.go b/ucl/eval.go index ad33e63..4f96ce3 100644 --- a/ucl/eval.go +++ b/ucl/eval.go @@ -259,7 +259,7 @@ func (e evaluator) evalLiteral(ctx context.Context, ec *evalCtx, n *astLiteral) } return sval, nil case n.Int != nil: - return intObject(*n.Int), nil + return IntObject(*n.Int), nil } return nil, errors.New("unhandled literal type") } diff --git a/ucl/objs.go b/ucl/objs.go index a533d99..59ff80b 100644 --- a/ucl/objs.go +++ b/ucl/objs.go @@ -102,13 +102,13 @@ func (s StringObject) Truthy() bool { return string(s) != "" } -type intObject int +type IntObject int -func (i intObject) String() string { +func (i IntObject) String() string { return strconv.Itoa(int(i)) } -func (i intObject) Truthy() bool { +func (i IntObject) Truthy() bool { return i != 0 } @@ -143,7 +143,7 @@ func toGoValue(obj Object) (interface{}, bool) { return nil, true case StringObject: return string(v), true - case intObject: + case IntObject: return int(v), true case boolObject: return bool(v), true @@ -191,7 +191,7 @@ func fromGoValue(v any) (Object, error) { case string: return StringObject(t), nil case int: - return intObject(t), nil + return IntObject(t), nil case bool: return boolObject(t), nil case time.Time: @@ -362,7 +362,7 @@ func (ia invocationArgs) intArg(i int) (int, error) { } switch v := ia.args[i].(type) { - case intObject: + case IntObject: return int(v), nil default: return 0, errors.New("expected an int arg") diff --git a/ucl/userbuiltin.go b/ucl/userbuiltin.go index 78f3d47..1294e47 100644 --- a/ucl/userbuiltin.go +++ b/ucl/userbuiltin.go @@ -129,7 +129,7 @@ func (ca CallArgs) bindArg(v interface{}, arg Object) error { } return nil case *int: - if iArg, ok := arg.(intObject); ok { + if iArg, ok := arg.(IntObject); ok { *t = int(iArg) } else { return errors.New("invalid arg") @@ -171,7 +171,7 @@ func canBindArg(v interface{}, arg Object) bool { case *string: return true case *int: - _, ok := arg.(intObject) + _, ok := arg.(IntObject) return ok }