lmika/dynamo-browse
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Issue 32: Fixed some TODOs in query expressions

Browse source 
- Fixed the gaps in conjunctions, disjunctions, and equality operator for expression value evaluation.
- Fixed the issue in which '^=' was treated as two separate tokens, it's now a single token.
This commit is contained in:
Leon Mika 2022-10-11 22:16:20 +11:00 committed by GitHub
parent 79692302af
commit b51c13dfb1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 240 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -3,7 +3,7 @@ module github.com/lmika/audax
go 1.18
require (
github.com/alecthomas/participle/v2 v2.0.0-alpha7
github.com/alecthomas/participle/v2 v2.0.0-beta.5
github.com/asdine/storm v2.1.2+incompatible
github.com/aws/aws-sdk-go-v2 v1.16.5
github.com/aws/aws-sdk-go-v2/config v1.13.1

4
go.sum
View file

@ -3,10 +3,14 @@ github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/Sereal/Sereal v0.0.0-20220220040404-e0d1e550e879 h1:M5ptEKnqKqpFTKbe+p5zEf3ro1deJ6opUz5j3g3/ErQ=
github.com/Sereal/Sereal v0.0.0-20220220040404-e0d1e550e879/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM=
github.com/alecthomas/participle v0.7.1 h1:2bN7reTw//5f0cugJcTOnY/NYZcWQOaajW+BwZB5xWs=
github.com/alecthomas/participle/v2 v2.0.0-alpha7 h1:cK4vjj0VSgb3lN1nuKA5F7dw+1s1pWBe5bx7nNCnN+c=
github.com/alecthomas/participle/v2 v2.0.0-alpha7/go.mod h1:NumScqsC42o9x+dGj8/YqsIfhrIQjFEOFovxotbBirA=
github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo=
github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
github.com/asdine/storm v2.1.2+incompatible h1:dczuIkyqwY2LrtXPz8ixMrU/OFgZp71kbKTHGrXYt/Q=
github.com/asdine/storm v2.1.2+incompatible/go.mod h1:RarYDc9hq1UPLImuiXK3BIWPJLdIygvV3PsInK0FbVQ=
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=

4
internal/dynamo-browse/controllers/export.go
View file

@ -4,7 +4,7 @@ import (
"encoding/csv"
tea "github.com/charmbracelet/bubbletea"
"github.com/lmika/audax/internal/common/ui/events"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/lmika/audax/internal/dynamo-browse/models/attrutils"
"github.com/pkg/errors"
"os"
)
@ -46,7 +46,7 @@ func (c *ExportController) ExportCSV(filename string) tea.Msg {
row := make([]string, len(columns))
for _, item := range resultSet.Items() {
for i, col := range columns {
row[i], _ = models.AttributeToString(col.Evaluator.EvaluateForItem(item))
row[i], _ = attrutils.AttributeToString(col.Evaluator.EvaluateForItem(item))
}
if err := cw.Write(row); err != nil {
return events.Error(errors.Wrapf(err, "cannot export to '%v'", filename))

4
internal/dynamo-browse/models/attrutils.go → internal/dynamo-browse/models/attrutils/attrutils.go
View file

@ -1,4 +1,4 @@
package models
package attrutils
import (
"math/big"
@ -6,7 +6,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
)
func compareScalarAttributes(x, y types.AttributeValue) (int, bool) {
func CompareScalarAttributes(x, y types.AttributeValue) (int, bool) {
switch xVal := x.(type) {
case *types.AttributeValueMemberS:
if yVal, ok := y.(*types.AttributeValueMemberS); ok {

3
internal/dynamo-browse/models/items.go
View file

@ -2,6 +2,7 @@ package models
import (
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models/attrutils"
)
type ItemIndex struct {
@ -33,5 +34,5 @@ func (i Item) KeyValue(info *TableInfo) map[string]types.AttributeValue {
}
func (i Item) AttributeValueAsString(key string) (string, bool) {
return AttributeToString(i[key])
return attrutils.AttributeToString(i[key])
}

9
internal/dynamo-browse/models/modexpr/ast.go
View file

@ -22,14 +22,13 @@ type astLiteralValue struct {
String string `parser:"@String"`
}
var parser = participle.MustBuild(&astExpr{})
var parser = participle.MustBuild[astExpr]()
func Parse(expr string) (*ModExpr, error) {
var ast astExpr
if err := parser.ParseString("expr", expr, &ast); err != nil {
ast, err := parser.ParseString("expr", expr)
if err != nil {
return nil, errors.Wrapf(err, "cannot parse expression: '%v'", expr)
}
return &ModExpr{ast: &ast}, nil
return &ModExpr{ast: ast}, nil
}

30
internal/dynamo-browse/models/queryexpr/ast.go
View file

@ -2,6 +2,7 @@ package queryexpr
import (
"github.com/alecthomas/participle/v2"
"github.com/alecthomas/participle/v2/lexer"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/pkg/errors"
@ -27,13 +28,12 @@ type astDisjunction struct {
}
type astConjunction struct {
Operands []*astBinOp `parser:"@@ ('and' @@)*"`
Operands []*astEqualityOp `parser:"@@ ('and' @@)*"`
}
// TODO: do this properly
type astBinOp struct {
type astEqualityOp struct {
Ref *astDot `parser:"@@"`
Op string `parser:"( @('^' '=' | '=')"`
Op string `parser:"( @('^=' | '=')"`
Value *astLiteralValue `parser:"@@ )?"`
}
@ -43,17 +43,27 @@ type astDot struct {
}
type astLiteralValue struct {
StringVal string `parser:"@String"`
StringVal *string `parser:"@String"`
IntVal *int64 `parser:"| @Int"`
}
var parser = participle.MustBuild(&astExpr{})
var scanner = lexer.MustSimple([]lexer.SimpleRule{
{Name: "Eq", Pattern: `=|[\\^]=`},
{Name: "String", Pattern: `"(\\"|[^"])*"`},
{Name: "Int", Pattern: `[-+]?(\d*\.)?\d+`},
{Name: "Number", Pattern: `[-+]?(\d*\.)?\d+`},
{Name: "Ident", Pattern: `[a-zA-Z_][a-zA-Z0-9_-]*`},
{Name: "Punct", Pattern: `[-[!@#$%^&*()+_={}\|:;"'<,>.?/]|][=]?`},
{Name: "EOL", Pattern: `[\n\r]+`},
{Name: "whitespace", Pattern: `[ \t]+`},
})
var parser = participle.MustBuild[astExpr](participle.Lexer(scanner))
func Parse(expr string) (*QueryExpr, error) {
var ast astExpr
if err := parser.ParseString("expr", expr, &ast); err != nil {
ast, err := parser.ParseString("expr", expr)
if err != nil {
return nil, errors.Wrapf(err, "cannot parse expression: '%v'", expr)
}
return &QueryExpr{ast: &ast}, nil
return &QueryExpr{ast: ast}, nil
}

40
internal/dynamo-browse/models/queryexpr/binops.go
View file

@ -4,10 +4,12 @@ import (
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/lmika/audax/internal/dynamo-browse/models/attrutils"
"github.com/pkg/errors"
"strings"
)
func (a *astBinOp) evalToIR(info *models.TableInfo) (irAtom, error) {
func (a *astEqualityOp) evalToIR(info *models.TableInfo) (irAtom, error) {
v, err := a.Value.goValue()
if err != nil {
return nil, err
@ -32,7 +34,7 @@ func (a *astBinOp) evalToIR(info *models.TableInfo) (irAtom, error) {
return nil, errors.Errorf("unrecognised operator: %v", a.Op)
}
func (a *astBinOp) evalItem(item models.Item) (types.AttributeValue, error) {
func (a *astEqualityOp) evalItem(item models.Item) (types.AttributeValue, error) {
left, err := a.Ref.evalItem(item)
if err != nil {
return nil, err
@ -42,10 +44,40 @@ func (a *astBinOp) evalItem(item models.Item) (types.AttributeValue, error) {
return left, nil
}
return nil, errors.New("TODO")
right, err := a.Value.dynamoValue()
if err != nil {
return nil, err
}
func (a *astBinOp) String() string {
switch a.Op {
case "=":
cmp, isComparable := attrutils.CompareScalarAttributes(left, right)
if !isComparable {
return nil, ValuesNotComparable{Left: left, Right: right}
}
return &types.AttributeValueMemberBOOL{Value: cmp == 0}, nil
case "^=":
rightVal, err := a.Value.goValue()
if err != nil {
return nil, err
}
strValue, isStrValue := rightVal.(string)
if !isStrValue {
return nil, errors.New("operand '^=' must be string")
}
leftAsStr, canBeString := attrutils.AttributeToString(left)
if !canBeString {
return nil, ValueNotConvertableToString{Val: left}
}
return &types.AttributeValueMemberBOOL{Value: strings.HasPrefix(leftAsStr, strValue)}, nil
}
return nil, errors.Errorf("unrecognised operator: %v", a.Op)
}
func (a *astEqualityOp) String() string {
return a.Ref.String() + a.Op + a.Value.String()
}

33
internal/dynamo-browse/models/queryexpr/conj.go
View file

@ -22,11 +22,26 @@ func (a *astConjunction) evalToIR(tableInfo *models.TableInfo) (*irConjunction,
}
func (a *astConjunction) evalItem(item models.Item) (types.AttributeValue, error) {
val, err := a.Operands[0].evalItem(item)
if err != nil {
return nil, err
}
if len(a.Operands) == 1 {
return a.Operands[0].evalItem(item)
return val, nil
}
return nil, errors.New("TODO")
for _, opr := range a.Operands[1:] {
if !isAttributeTrue(val) {
return &types.AttributeValueMemberBOOL{Value: false}, nil
}
val, err = opr.evalItem(item)
if err != nil {
return nil, err
}
}
return &types.AttributeValueMemberBOOL{Value: isAttributeTrue(val)}, nil
}
func (d *astConjunction) String() string {
@ -96,3 +111,17 @@ func (d *irConjunction) calcQueryForScan(info *models.TableInfo) (expression.Con
conjExpr := expression.And(conds[0], conds[1], conds[2:]...)
return conjExpr, nil
}
func isAttributeTrue(attr types.AttributeValue) bool {
switch val := attr.(type) {
case *types.AttributeValueMemberS:
return val.Value != ""
case *types.AttributeValueMemberN:
return val.Value != "0"
case *types.AttributeValueMemberBOOL:
return val.Value
case *types.AttributeValueMemberNULL:
return false
}
return true
}

19
internal/dynamo-browse/models/queryexpr/disj.go
View file

@ -22,11 +22,26 @@ func (a *astDisjunction) evalToIR(tableInfo *models.TableInfo) (*irDisjunction,
}
func (a *astDisjunction) evalItem(item models.Item) (types.AttributeValue, error) {
val, err := a.Operands[0].evalItem(item)
if err != nil {
return nil, err
}
if len(a.Operands) == 1 {
return a.Operands[0].evalItem(item)
return val, nil
}
return nil, errors.New("TODO")
for _, opr := range a.Operands[1:] {
if isAttributeTrue(val) {
return &types.AttributeValueMemberBOOL{Value: true}, nil
}
val, err = opr.evalItem(item)
if err != nil {
return nil, err
}
}
return &types.AttributeValueMemberBOOL{Value: isAttributeTrue(val)}, nil
}
func (d *astDisjunction) String() string {

24
internal/dynamo-browse/models/queryexpr/errors.go
View file

@ -2,6 +2,9 @@ package queryexpr
import (
"fmt"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models/attrutils"
"github.com/lmika/audax/internal/dynamo-browse/models/itemrender"
"strings"
)
@ -18,3 +21,24 @@ type ValueNotAMapError []string
func (n ValueNotAMapError) Error() string {
return fmt.Sprintf("%v: name is not a map", strings.Join(n, "."))
}
// ValuesNotComparable indicates that two values are not comparable
type ValuesNotComparable struct {
Left, Right types.AttributeValue
}
func (n ValuesNotComparable) Error() string {
leftStr, _ := attrutils.AttributeToString(n.Left)
rightStr, _ := attrutils.AttributeToString(n.Right)
return fmt.Sprintf("values '%v' and '%v' are not comparable", leftStr, rightStr)
}
// ValueNotConvertableToString indicates that a value is not convertable to a string
type ValueNotConvertableToString struct {
Val types.AttributeValue
}
func (n ValueNotConvertableToString) Error() string {
render := itemrender.ToRenderer(n.Val)
return fmt.Sprintf("values '%v', type %v, is not convertable to string", render.StringValue(), render.TypeName())
}

63
internal/dynamo-browse/models/queryexpr/expr_test.go
View file

@ -77,7 +77,7 @@ func TestModExpr_Query(t *testing.T) {
t.Run("as scans", func(t *testing.T) {
t.Run("when request pk prefix", func(t *testing.T) {
modExpr, err := queryexpr.Parse(`pk^="prefix"`) // TODO: fix this so that '^ =' is invalid
modExpr, err := queryexpr.Parse(`pk^="prefix"`)
assert.NoError(t, err)
plan, err := modExpr.Plan(tableInfo)
@ -117,6 +117,23 @@ func TestModExpr_Query(t *testing.T) {
assert.Equal(t, "another", plan.Expression.Values()[":1"].(*types.AttributeValueMemberS).Value)
})
t.Run("with disjunctions with numbers", func(t *testing.T) {
modExpr, err := queryexpr.Parse(`pk="prefix" or num=123 and negnum=-131`)
assert.NoError(t, err)
plan, err := modExpr.Plan(tableInfo)
assert.NoError(t, err)
assert.False(t, plan.CanQuery)
assert.Equal(t, "(#0 = :0) OR ((#1 = :1) AND (#2 = :2))", aws.ToString(plan.Expression.Filter()))
assert.Equal(t, "pk", plan.Expression.Names()["#0"])
assert.Equal(t, "num", plan.Expression.Names()["#1"])
assert.Equal(t, "negnum", plan.Expression.Names()["#2"])
assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value)
assert.Equal(t, "123", plan.Expression.Values()[":1"].(*types.AttributeValueMemberN).Value)
assert.Equal(t, "-131", plan.Expression.Values()[":2"].(*types.AttributeValueMemberN).Value)
})
t.Run("with disjunctions if pk is present twice in expression", func(t *testing.T) {
modExpr, err := queryexpr.Parse(`pk="prefix" and pk="another"`)
assert.NoError(t, err)
@ -157,9 +174,37 @@ func TestQueryExpr_EvalItem(t *testing.T) {
{expr: `bravo`, expected: &types.AttributeValueMemberN{Value: "123"}},
{expr: `charlie`, expected: item["charlie"]},
// Equality with literal
{expr: `alpha="alpha"`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `bravo=123`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `charlie.tree="green"`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `alpha^="al"`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `alpha="foobar"`, expected: &types.AttributeValueMemberBOOL{Value: false}},
{expr: `alpha^="need-something"`, expected: &types.AttributeValueMemberBOOL{Value: false}},
// Dot values
{expr: `charlie.door`, expected: &types.AttributeValueMemberS{Value: "red"}},
{expr: `charlie.tree`, expected: &types.AttributeValueMemberS{Value: "green"}},
// Conjunction
{expr: `alpha="alpha" and bravo=123`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `alpha="alpha" and bravo=321`, expected: &types.AttributeValueMemberBOOL{Value: false}},
{expr: `alpha="bravo" and bravo=123`, expected: &types.AttributeValueMemberBOOL{Value: false}},
{expr: `alpha="bravo" and bravo=321`, expected: &types.AttributeValueMemberBOOL{Value: false}},
{expr: `alpha="alpha" and bravo=123 and charlie.door="red"`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `alpha="alpha" and bravo=123 and charlie.door^="green"`, expected: &types.AttributeValueMemberBOOL{Value: false}},
// Disjunction
{expr: `alpha="alpha" or bravo=123`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `alpha="alpha" or bravo=321`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `alpha="bravo" or bravo=123`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `alpha="bravo" or bravo=321`, expected: &types.AttributeValueMemberBOOL{Value: false}},
{expr: `alpha="alpha" or bravo=123 or charlie.tree="green"`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `alpha="bravo" or bravo=321 or charlie.tree^="red"`, expected: &types.AttributeValueMemberBOOL{Value: false}},
// Order of operation
{expr: `alpha="alpha" and bravo=123 or charlie.door="green"`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `alpha="bravo" or bravo=321 and charlie.door="green"`, expected: &types.AttributeValueMemberBOOL{Value: false}},
}
for _, scenario := range scenarios {
t.Run(scenario.expr, func(t *testing.T) {
@ -174,6 +219,22 @@ func TestQueryExpr_EvalItem(t *testing.T) {
}
})
t.Run("unparsed expression", func(t *testing.T) {
scenarios := []struct {
expr string
expectedError error
}{
{expr: `bla ^ = "something"`},
}
for _, scenario := range scenarios {
t.Run(scenario.expr, func(t *testing.T) {
_, err := queryexpr.Parse(scenario.expr)
assert.Error(t, err)
})
}
})
t.Run("expression errors", func(t *testing.T) {
scenarios := []struct {
expr string

31
internal/dynamo-browse/models/queryexpr/values.go
View file

@ -12,11 +12,19 @@ func (a *astLiteralValue) dynamoValue() (types.AttributeValue, error) {
return nil, nil
}
s, err := strconv.Unquote(a.StringVal)
goValue, err := a.goValue()
if err != nil {
return nil, errors.Wrap(err, "cannot unquote string")
return nil, err
}
return &types.AttributeValueMemberS{Value: s}, nil
switch v := goValue.(type) {
case string:
return &types.AttributeValueMemberS{Value: v}, nil
case int64:
return &types.AttributeValueMemberN{Value: strconv.FormatInt(v, 10)}, nil
}
return nil, errors.New("unrecognised type")
}
func (a *astLiteralValue) goValue() (any, error) {
@ -24,16 +32,29 @@ func (a *astLiteralValue) goValue() (any, error) {
return nil, nil
}
s, err := strconv.Unquote(a.StringVal)
switch {
case a.StringVal != nil:
s, err := strconv.Unquote(*a.StringVal)
if err != nil {
return nil, errors.Wrap(err, "cannot unquote string")
}
return s, nil
case a.IntVal != nil:
return *a.IntVal, nil
}
return nil, errors.New("unrecognised type")
}
func (a *astLiteralValue) String() string {
if a == nil {
return ""
}
return a.StringVal
switch {
case a.StringVal != nil:
return *a.StringVal
case a.IntVal != nil:
return strconv.FormatInt(*a.IntVal, 10)
}
return ""
}

9
internal/dynamo-browse/models/sorted.go
View file

@ -1,6 +1,9 @@
package models
import "sort"
import (
"github.com/lmika/audax/internal/dynamo-browse/models/attrutils"
"sort"
)
// sortedItems is a collection of items that is sorted.
// Items are sorted based on the PK, and SK in ascending order
@ -22,7 +25,7 @@ func (si *sortedItems) Len() int {
func (si *sortedItems) Less(i, j int) bool {
// Compare primary keys
pv1, pv2 := si.items[i][si.tableInfo.Keys.PartitionKey], si.items[j][si.tableInfo.Keys.PartitionKey]
pc, ok := compareScalarAttributes(pv1, pv2)
pc, ok := attrutils.CompareScalarAttributes(pv1, pv2)
if !ok {
return i < j
}
@ -36,7 +39,7 @@ func (si *sortedItems) Less(i, j int) bool {
// Partition keys are equal, compare sort key
if sortKey := si.tableInfo.Keys.SortKey; sortKey != "" {
sv1, sv2 := si.items[i][sortKey], si.items[j][sortKey]
sc, ok := compareScalarAttributes(sv1, sv2)
sc, ok := attrutils.CompareScalarAttributes(sv1, sv2)
if !ok {
return i < j
}