From 143ca8d5551141255f8047a1eb737f2637dc6622 Mon Sep 17 00:00:00 2001 From: Leon Mika Date: Tue, 8 Sep 2026 19:50:08 +1000 Subject: [PATCH] Added bytes builtin --- .beads/issues.jsonl | 0 ucl/builtins.go | 4 + ucl/builtins/bytes.go | 98 ++++++++++++++++++++ ucl/builtins/bytes_test.go | 85 ++++++++++++++++++ ucl/builtins/os_test.go | 1 + ucl/builtins/urls_test.go | 30 ++++++- ucl/objs.go | 22 +++++ ucl/objs_test.go | 177 +++++++++++++++++++++++++++++++++++++ 8 files changed, 415 insertions(+), 2 deletions(-) delete mode 100644 .beads/issues.jsonl create mode 100644 ucl/builtins/bytes.go create mode 100644 ucl/builtins/bytes_test.go create mode 100644 ucl/objs_test.go diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl deleted file mode 100644 index e69de29..0000000 diff --git a/ucl/builtins.go b/ucl/builtins.go index c984394..4c012bc 100644 --- a/ucl/builtins.go +++ b/ucl/builtins.go @@ -369,6 +369,10 @@ func strBuiltin(ctx context.Context, args invocationArgs) (Object, error) { return StringObject(""), nil } + if bs, isBS := args.args[0].(BytesObject); isBS { + return StringObject(bs), nil + } + return StringObject(args.args[0].String()), nil } diff --git a/ucl/builtins/bytes.go b/ucl/builtins/bytes.go new file mode 100644 index 0000000..98d1a3c --- /dev/null +++ b/ucl/builtins/bytes.go @@ -0,0 +1,98 @@ +package builtins + +import ( + "context" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "errors" + + "ucl.lmika.dev/ucl" +) + +type bytesHandlers struct{} + +/// :module bytes +// +// Perform operations over byte slices. + +func Bytes() ucl.Module { + return ucl.Module{ + Name: "bytes", + Builtins: map[string]ucl.BuiltinHandler{ + "hash": bytesHandlers{}.hash, + "base64": bytesHandlers{}.base64, + }, + } +} + +// :fn hash +// +// :syntax INPUT ALGORITHM +// +// Returns a byte slice containing the result of hashing the input with the given algorithm. +// +// The input can either be a byte slice or a string. The supported algorithms are as follows: +// +// - md5 +// - sha1 +// - sha256 +func (b bytesHandlers) hash(ctx context.Context, args ucl.CallArgs) (any, error) { + inputBts, args, err := consumeStringOrBytes(args) + if err != nil { + return nil, err + } + + var algor string + if err := args.Bind(&algor); err != nil { + return nil, err + } + + switch algor { + case "md5": + res := md5.Sum(inputBts) + return ucl.BytesObject(res[:]), nil + case "sha1": + res := sha1.Sum(inputBts) + return ucl.BytesObject(res[:]), nil + case "sha256": + res := sha256.Sum256(inputBts) + return ucl.BytesObject(res[:]), nil + } + + return nil, errors.New("unsupported algorithm") +} + +// :fn base64 +// +// :syntax INPUT +// +// Returns a string containing the base64 encoding of the input. +// +// The input can either be a byte slice or a string. The result will be a base64 encoded string +// using standard encoding. +func (c bytesHandlers) base64(ctx context.Context, args ucl.CallArgs) (any, error) { + inputBts, args, err := consumeStringOrBytes(args) + if err != nil { + return nil, err + } + + res := base64.StdEncoding.EncodeToString(inputBts) + return res, nil +} + +func consumeStringOrBytes(args ucl.CallArgs) (bs ucl.BytesObject, _ ucl.CallArgs, _ error) { + if objs := args.RestAsObjects(); len(objs) > 0 { + if b, ok := objs[0].(ucl.BytesObject); ok { + args.Shift(1) + return b, args, nil + } + } + + var s string + if err := args.Bind(&s); err != nil { + return nil, ucl.CallArgs{}, err + } + return ucl.BytesObject(s), args, nil +} diff --git a/ucl/builtins/bytes_test.go b/ucl/builtins/bytes_test.go new file mode 100644 index 0000000..1c5554e --- /dev/null +++ b/ucl/builtins/bytes_test.go @@ -0,0 +1,85 @@ +package builtins_test + +import ( + "context" + "encoding/hex" + "testing" + + "github.com/stretchr/testify/assert" + "ucl.lmika.dev/ucl" + "ucl.lmika.dev/ucl/builtins" +) + +func TestBytes_Hash(t *testing.T) { + tests := []struct { + desc string + eval string + wantHex string + wantErr bool + }{ + {desc: "md5 of string", eval: `bytes:hash "hello" md5`, wantHex: "5d41402abc4b2a76b9719d911017c592"}, + {desc: "sha1 of string", eval: `bytes:hash "hello" sha1`, wantHex: "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d"}, + {desc: "sha256 of string", eval: `bytes:hash "hello" sha256`, wantHex: "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"}, + + {desc: "md5 of empty string", eval: `bytes:hash "" md5`, wantHex: "d41d8cd98f00b204e9800998ecf8427e"}, + {desc: "sha256 of empty string", eval: `bytes:hash "" sha256`, wantHex: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + + {desc: "md5 of bytes input", eval: `bytes:hash (bytes:hash "hello" md5) md5`, wantHex: "62109206880d38a4010a98e11243924a"}, + {desc: "sha1 of bytes input", eval: `bytes:hash (bytes:hash "hello" md5) sha1`, wantHex: "7eb14e07d62722cb2e338faa6060b4f780e80b2a"}, + + {desc: "err unsupported algorithm", eval: `bytes:hash "hello" sha512`, wantErr: true}, + {desc: "err missing algorithm", eval: `bytes:hash "hello"`, wantErr: true}, + {desc: "err missing args", eval: `bytes:hash`, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + inst := ucl.New( + ucl.WithModule(builtins.Bytes()), + ) + res, err := inst.EvalString(context.Background(), tt.eval) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + + resBts, ok := res.([]byte) + if assert.True(t, ok, "expected []byte result, got %T", res) { + assert.Equal(t, tt.wantHex, hex.EncodeToString(resBts)) + } + } + }) + } +} + +func TestBytes_Base64(t *testing.T) { + tests := []struct { + desc string + eval string + want string + wantErr bool + }{ + {desc: "encode string", eval: `bytes:base64 "hello, world"`, want: "aGVsbG8sIHdvcmxk"}, + {desc: "encode string with padding", eval: `bytes:base64 "hello"`, want: "aGVsbG8="}, + {desc: "encode empty string", eval: `bytes:base64 ""`, want: ""}, + + {desc: "encode bytes input", eval: `bytes:base64 (bytes:hash "hello" md5)`, want: "XUFAKrxLKna5cZ2REBfFkg=="}, + + {desc: "err missing args", eval: `bytes:base64`, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + inst := ucl.New( + ucl.WithModule(builtins.Bytes()), + ) + res, err := inst.EvalString(context.Background(), tt.eval) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, res) + } + }) + } +} diff --git a/ucl/builtins/os_test.go b/ucl/builtins/os_test.go index b54b59f..78cee7f 100644 --- a/ucl/builtins/os_test.go +++ b/ucl/builtins/os_test.go @@ -46,6 +46,7 @@ func TestOS_Exec(t *testing.T) { {descr: "run command 1", eval: `os:exec "echo" "hello, world"`, want: "hello, world\n"}, {descr: "run command 2", eval: `os:exec "date" "+%Y%m%d"`, want: time.Now().Format("20060102") + "\n"}, {descr: "run command 3", eval: `os:exec "tr" "[a-z]" "[A-Z]" -in "hello"`, want: "HELLO"}, + {descr: "run command 4", eval: `os:exec "tr" "-d" "e" -in "hello"`, want: "hllo"}, } for _, tt := range tests { diff --git a/ucl/builtins/urls_test.go b/ucl/builtins/urls_test.go index 47d46a9..c6d9078 100644 --- a/ucl/builtins/urls_test.go +++ b/ucl/builtins/urls_test.go @@ -2,6 +2,9 @@ package builtins_test import ( "context" + "fmt" + "net/http" + "net/http/httptest" "testing" "github.com/stretchr/testify/assert" @@ -10,19 +13,42 @@ import ( ) func TestURLs_Fetch_http(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/text", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprint(w, "Hello, world") + }) + mux.HandleFunc("/html", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + fmt.Fprint(w, "

Example Domain

") + }) + mux.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"title":"Example Domain"}`) + }) + mux.HandleFunc("/missing", func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "not found", http.StatusNotFound) + }) + + srv := httptest.NewServer(mux) + defer srv.Close() + tests := []struct { desc string eval string want any wantErr bool }{ - {desc: "fetch 1", eval: `in (urls:fetch "https://www.example.com") "Example Domain"`, want: true}, + {desc: "fetch text", eval: fmt.Sprintf(`urls:fetch "%v/text"`, srv.URL), want: "Hello, world"}, + {desc: "fetch html", eval: fmt.Sprintf(`in (urls:fetch "%v/html") "Example Domain"`, srv.URL), want: true}, + {desc: "non-200 status code", eval: fmt.Sprintf(`urls:fetch "%v/missing"`, srv.URL), wantErr: true}, + {desc: "unsupported content type", eval: fmt.Sprintf(`urls:fetch "%v/json"`, srv.URL), wantErr: true}, + {desc: "unsupported scheme", eval: `urls:fetch "ftp://example.com/file.txt"`, wantErr: true}, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { inst := ucl.New( - ucl.WithModule(builtins.Strs()), ucl.WithModule(builtins.URLs()), ) res, err := inst.EvalString(context.Background(), tt.eval) diff --git a/ucl/objs.go b/ucl/objs.go index d77f13f..df0872d 100644 --- a/ucl/objs.go +++ b/ucl/objs.go @@ -118,6 +118,24 @@ func (ss StringListObject) Index(i int) Object { return StringObject(ss[i]) } +type BytesObject []byte + +func (bs BytesObject) String() string { + return fmt.Sprintf("[%v]bytes", len(bs)) +} + +func (bs BytesObject) Truthy() bool { + return len(bs) > 0 +} + +func (bs BytesObject) Len() int { + return len(bs) +} + +func (bs BytesObject) Index(i int) Object { + return IntObject(bs[i]) +} + type iteratorObject struct { Iterable } @@ -242,6 +260,8 @@ func toGoValue(obj Object) (interface{}, bool) { return bool(v), true case TimeObject: return time.Time(v), true + case BytesObject: + return []byte(v), true case *ListObject: xs := make([]interface{}, 0, len(*v)) for _, va := range *v { @@ -293,6 +313,8 @@ func fromGoValue(v any) (Object, error) { return IntObject(t), nil case bool: return BoolObject(t), nil + case []byte: + return BytesObject(t), nil case time.Time: return TimeObject(t), nil } diff --git a/ucl/objs_test.go b/ucl/objs_test.go new file mode 100644 index 0000000..e84d772 --- /dev/null +++ b/ucl/objs_test.go @@ -0,0 +1,177 @@ +package ucl + +import ( + "bytes" + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBytesObject_String(t *testing.T) { + tests := []struct { + desc string + obj BytesObject + want string + }{ + {desc: "empty", obj: BytesObject{}, want: "[0]bytes"}, + {desc: "single byte", obj: BytesObject("a"), want: "[1]bytes"}, + {desc: "multiple bytes", obj: BytesObject("hello"), want: "[5]bytes"}, + {desc: "binary bytes", obj: BytesObject{0x00, 0xff, 0x10}, want: "[3]bytes"}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + assert.Equal(t, tt.want, tt.obj.String()) + }) + } +} + +func TestBytesObject_Truthy(t *testing.T) { + tests := []struct { + desc string + obj BytesObject + want bool + }{ + {desc: "empty", obj: BytesObject{}, want: false}, + {desc: "nil", obj: nil, want: false}, + {desc: "single byte", obj: BytesObject("a"), want: true}, + {desc: "multiple bytes", obj: BytesObject("hello"), want: true}, + {desc: "single zero byte", obj: BytesObject{0x00}, want: true}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + assert.Equal(t, tt.want, tt.obj.Truthy()) + }) + } +} + +func TestBytesObject_Len(t *testing.T) { + tests := []struct { + desc string + obj BytesObject + want int + }{ + {desc: "empty", obj: BytesObject{}, want: 0}, + {desc: "single byte", obj: BytesObject("a"), want: 1}, + {desc: "multiple bytes", obj: BytesObject("hello"), want: 5}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + assert.Equal(t, tt.want, tt.obj.Len()) + }) + } +} + +func TestBytesObject_Index(t *testing.T) { + tests := []struct { + desc string + obj BytesObject + idx int + want Object + }{ + {desc: "first byte", obj: BytesObject("hello"), idx: 0, want: IntObject(104)}, + {desc: "last byte", obj: BytesObject("hello"), idx: 4, want: IntObject(111)}, + {desc: "binary byte", obj: BytesObject{0x00, 0xff}, idx: 1, want: IntObject(255)}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + assert.Equal(t, tt.want, tt.obj.Index(tt.idx)) + }) + } +} + +func TestBytesObject_GoConversion(t *testing.T) { + t.Run("from go value", func(t *testing.T) { + obj, err := fromGoValue([]byte("hello")) + assert.NoError(t, err) + assert.Equal(t, BytesObject("hello"), obj) + }) + + t.Run("to go value", func(t *testing.T) { + v, ok := toGoValue(BytesObject("hello")) + assert.True(t, ok) + assert.Equal(t, []byte("hello"), v) + }) + + t.Run("round trip through script", func(t *testing.T) { + inst := New(WithTestBuiltin()) + inst.SetVar("b", []byte("hello")) + + res, err := inst.EvalString(context.Background(), `$b`) + assert.NoError(t, err) + assert.Equal(t, []byte("hello"), res) + }) +} + +func TestBytesObject_Script(t *testing.T) { + tests := []struct { + desc string + expr string + want any + }{ + {desc: "str of bytes", expr: `str $b`, want: "hello"}, + {desc: "str of empty bytes", expr: `str $e`, want: ""}, + {desc: "str of binary bytes", expr: `str $bin`, want: "\x00\xff"}, + + {desc: "len of bytes", expr: `len $b`, want: 5}, + {desc: "len of empty bytes", expr: `len $e`, want: 0}, + + {desc: "index first byte", expr: `index $b 0`, want: 104}, + {desc: "index last byte", expr: `index $b 4`, want: 111}, + {desc: "index out of range", expr: `index $b 555`, want: nil}, + {desc: "index via pipe", expr: `$b | index 1`, want: 101}, + + {desc: "truthy non-empty", expr: `if $b { "yes" } else { "no" }`, want: "yes"}, + {desc: "truthy empty", expr: `if $e { "yes" } else { "no" }`, want: "no"}, + + {desc: "iterate bytes yields ints", expr: `s = "" ; for $b { |x| s = cat $s $x "," } ; $s`, want: "104,101,108,108,111,"}, + {desc: "iterate empty bytes", expr: `s = "" ; for $e { |x| s = cat $s $x "," } ; $s`, want: ""}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + ctx := context.Background() + + inst := New(WithTestBuiltin()) + inst.SetVar("b", []byte("hello")) + inst.SetVar("e", []byte{}) + inst.SetVar("bin", []byte{0x00, 0xff}) + + res, err := inst.EvalString(ctx, tt.expr) + assert.NoError(t, err) + assert.Equal(t, tt.want, res) + }) + } +} + +func TestBytesObject_Display(t *testing.T) { + tests := []struct { + desc string + expr string + want string + }{ + {desc: "echo bytes", expr: `echo $b`, want: "[5]bytes\n(nil)\n"}, + {desc: "echo empty bytes", expr: `echo $e`, want: "[0]bytes\n(nil)\n"}, + {desc: "display bytes iterates as ints", expr: `$b`, want: "104\n101\n108\n108\n111\n"}, + {desc: "display empty bytes", expr: `$e`, want: ""}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + ctx := context.Background() + outW := bytes.NewBuffer(nil) + + inst := New(WithOut(outW), WithTestBuiltin()) + inst.SetVar("b", []byte("hello")) + inst.SetVar("e", []byte{}) + + err := evalAndDisplay(ctx, inst, tt.expr) + assert.NoError(t, err) + assert.Equal(t, tt.want, outW.String()) + }) + } +}