From 917663fac0d0c7b489657845977981e7335b0417 Mon Sep 17 00:00:00 2001 From: Leon Mika Date: Fri, 18 Nov 2022 07:31:15 +1100 Subject: [PATCH] Issue 33: Finished most aspects of the expression language (#38) - Most aspects of scans and queries can now be represented using the expression language - All constructs of the expression language can be used to evaluate items --- internal/common/sliceutils/map.go | 13 + .../dynamo-browse/controllers/tableread.go | 4 + .../dynamo-browse/models/queryexpr/ast.go | 114 ++++- .../dynamo-browse/models/queryexpr/atom.go | 63 +++ .../dynamo-browse/models/queryexpr/binops.go | 132 ----- .../dynamo-browse/models/queryexpr/boolnot.go | 56 +++ .../models/queryexpr/builtins.go | 39 ++ .../models/queryexpr/calcquery.go | 47 -- .../dynamo-browse/models/queryexpr/comp.go | 177 +++++++ .../dynamo-browse/models/queryexpr/conj.go | 82 +++- .../dynamo-browse/models/queryexpr/disj.go | 27 +- .../dynamo-browse/models/queryexpr/dot.go | 37 +- .../models/queryexpr/equality.go | 202 ++++++++ .../dynamo-browse/models/queryexpr/errors.go | 66 +++ .../dynamo-browse/models/queryexpr/expr.go | 20 +- .../models/queryexpr/expr_test.go | 462 +++++++++++++----- .../dynamo-browse/models/queryexpr/fncall.go | 125 +++++ internal/dynamo-browse/models/queryexpr/in.go | 269 ++++++++++ internal/dynamo-browse/models/queryexpr/ir.go | 44 +- internal/dynamo-browse/models/queryexpr/is.go | 172 +++++++ .../dynamo-browse/models/queryexpr/types.go | 1 + .../dynamo-browse/models/queryexpr/values.go | 26 + .../dynamo-browse/services/tables/service.go | 5 +- .../ui/teamodels/colselector/tblmodel.go | 2 +- 24 files changed, 1815 insertions(+), 370 deletions(-) create mode 100644 internal/dynamo-browse/models/queryexpr/atom.go delete mode 100644 internal/dynamo-browse/models/queryexpr/binops.go create mode 100644 internal/dynamo-browse/models/queryexpr/boolnot.go create mode 100644 internal/dynamo-browse/models/queryexpr/builtins.go delete mode 100644 internal/dynamo-browse/models/queryexpr/calcquery.go create mode 100644 internal/dynamo-browse/models/queryexpr/comp.go create mode 100644 internal/dynamo-browse/models/queryexpr/equality.go create mode 100644 internal/dynamo-browse/models/queryexpr/fncall.go create mode 100644 internal/dynamo-browse/models/queryexpr/in.go create mode 100644 internal/dynamo-browse/models/queryexpr/is.go create mode 100644 internal/dynamo-browse/models/queryexpr/types.go diff --git a/internal/common/sliceutils/map.go b/internal/common/sliceutils/map.go index 8d34dd8..0d233b3 100644 --- a/internal/common/sliceutils/map.go +++ b/internal/common/sliceutils/map.go @@ -8,6 +8,19 @@ func Map[T, U any](ts []T, fn func(t T) U) []U { return us } +func MapWithError[T, U any](ts []T, fn func(t T) (U, error)) ([]U, error) { + us := make([]U, len(ts)) + + for i, t := range ts { + var err error + us[i], err = fn(t) + if err != nil { + return nil, err + } + } + return us, nil +} + func Filter[T any](ts []T, fn func(t T) bool) []T { us := make([]T, 0) for _, t := range ts { diff --git a/internal/dynamo-browse/controllers/tableread.go b/internal/dynamo-browse/controllers/tableread.go index e73d6cf..48c99ac 100644 --- a/internal/dynamo-browse/controllers/tableread.go +++ b/internal/dynamo-browse/controllers/tableread.go @@ -291,6 +291,10 @@ func (c *TableReadController) handleResultSetFromJobResult(filter string, pushba return events.StatusMsg("Operation cancelled") }) } + + if newResultSet != nil { + return c.setResultSetAndFilter(newResultSet, filter, pushbackStack, op) + } return events.Error(err) } } diff --git a/internal/dynamo-browse/models/queryexpr/ast.go b/internal/dynamo-browse/models/queryexpr/ast.go index 12d3fd2..7355fb4 100644 --- a/internal/dynamo-browse/models/queryexpr/ast.go +++ b/internal/dynamo-browse/models/queryexpr/ast.go @@ -3,6 +3,7 @@ package queryexpr import ( "github.com/alecthomas/participle/v2" "github.com/alecthomas/participle/v2/lexer" + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/lmika/audax/internal/dynamo-browse/models" "github.com/pkg/errors" @@ -15,26 +16,54 @@ type astExpr struct { Root *astDisjunction `parser:"@@"` } -func (a *astExpr) evalToIR(tableInfo *models.TableInfo) (*irDisjunction, error) { - return a.Root.evalToIR(tableInfo) -} - -func (a *astExpr) evalItem(item models.Item) (types.AttributeValue, error) { - return a.Root.evalItem(item) -} - type astDisjunction struct { Operands []*astConjunction `parser:"@@ ('or' @@)*"` } type astConjunction struct { - Operands []*astEqualityOp `parser:"@@ ('and' @@)*"` + Operands []*astBooleanNot `parser:"@@ ('and' @@)*"` +} + +type astBooleanNot struct { + HasNot bool `parser:"@'not'? "` + Operand *astIn `parser:"@@"` +} + +type astIn struct { + Ref *astComparisonOp `parser:"@@ ("` + HasNot bool `parser:"@'not'? 'in' "` + Operand []*astExpr `parser:"('(' @@ (',' @@ )* ')' |"` + SingleOperand *astComparisonOp `parser:"@@ ))?"` +} + +type astComparisonOp struct { + Ref *astEqualityOp `parser:"@@"` + Op string `parser:"( @('<' | '<=' | '>' | '>=')"` + Value *astEqualityOp `parser:"@@ )?"` } type astEqualityOp struct { - Ref *astDot `parser:"@@"` - Op string `parser:"( @('^=' | '=')"` - Value *astLiteralValue `parser:"@@ )?"` + Ref *astIsOp `parser:"@@"` + Op string `parser:"( @('^=' | '=' | '!=')"` + Value *astIsOp `parser:"@@ )?"` +} + +type astIsOp struct { + Ref *astFunctionCall `parser:"@@ ( 'is' "` + HasNot bool `parser:"@'not'?"` + Value *astFunctionCall `parser:"@@ )?"` +} + +type astFunctionCall struct { + Caller *astAtom `parser:"@@"` + IsCall bool `parser:"( @'(' "` + Args []*astExpr `parser:"( @@ (',' @@ )*)? ')' )?"` +} + +type astAtom struct { + Ref *astDot `parser:"@@ | "` + Literal *astLiteralValue `parser:"@@ | "` + Paren *astExpr `parser:"'(' @@ ')'"` } type astDot struct { @@ -48,7 +77,8 @@ type astLiteralValue struct { } var scanner = lexer.MustSimple([]lexer.SimpleRule{ - {Name: "Eq", Pattern: `=|[\\^]=`}, + {Name: "Eq", Pattern: `=|[\\^]=|[!]=`}, + {Name: "Cmp", Pattern: `<[=]?|>[=]?`}, {Name: "String", Pattern: `"(\\"|[^"])*"`}, {Name: "Int", Pattern: `[-+]?(\d*\.)?\d+`}, {Name: "Number", Pattern: `[-+]?(\d*\.)?\d+`}, @@ -57,7 +87,9 @@ var scanner = lexer.MustSimple([]lexer.SimpleRule{ {Name: "EOL", Pattern: `[\n\r]+`}, {Name: "whitespace", Pattern: `[ \t]+`}, }) -var parser = participle.MustBuild[astExpr](participle.Lexer(scanner)) +var parser = participle.MustBuild[astExpr]( + participle.Lexer(scanner), +) func Parse(expr string) (*QueryExpr, error) { ast, err := parser.ParseString("expr", expr) @@ -67,3 +99,57 @@ func Parse(expr string) (*QueryExpr, error) { return &QueryExpr{ast: ast}, nil } + +func (a *astExpr) calcQuery(info *models.TableInfo) (*models.QueryExecutionPlan, error) { + ir, err := a.evalToIR(info) + if err != nil { + return nil, err + } + + var qci queryCalcInfo + if canExecuteAsQuery(ir, info, &qci) { + ke, err := ir.(queryableIRAtom).calcQueryForQuery(info) + if err != nil { + return nil, err + } + + builder := expression.NewBuilder() + builder = builder.WithKeyCondition(ke) + + expr, err := builder.Build() + if err != nil { + return nil, err + } + + return &models.QueryExecutionPlan{ + CanQuery: true, + Expression: expr, + }, nil + } + + cb, err := ir.calcQueryForScan(info) + if err != nil { + return nil, err + } + + builder := expression.NewBuilder() + builder = builder.WithFilter(cb) + + expr, err := builder.Build() + if err != nil { + return nil, err + } + + return &models.QueryExecutionPlan{ + CanQuery: false, + Expression: expr, + }, nil +} + +func (a *astExpr) evalToIR(tableInfo *models.TableInfo) (irAtom, error) { + return a.Root.evalToIR(tableInfo) +} + +func (a *astExpr) evalItem(item models.Item) (types.AttributeValue, error) { + return a.Root.evalItem(item) +} diff --git a/internal/dynamo-browse/models/queryexpr/atom.go b/internal/dynamo-browse/models/queryexpr/atom.go new file mode 100644 index 0000000..8b26858 --- /dev/null +++ b/internal/dynamo-browse/models/queryexpr/atom.go @@ -0,0 +1,63 @@ +package queryexpr + +import ( + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/lmika/audax/internal/dynamo-browse/models" + "github.com/pkg/errors" +) + +func (a *astAtom) evalToIR(info *models.TableInfo) (irAtom, error) { + switch { + case a.Ref != nil: + return a.Ref.evalToIR(info) + case a.Literal != nil: + return a.Literal.evalToIR(info) + case a.Paren != nil: + return a.Paren.evalToIR(info) + } + + return nil, errors.New("unhandled atom case") +} + +func (a *astAtom) rightOperandDynamoValue() (types.AttributeValue, error) { + switch { + case a.Literal != nil: + return a.Literal.dynamoValue() + } + + return nil, errors.New("unhandled atom case") +} + +func (a *astAtom) unqualifiedName() (string, bool) { + switch { + case a.Ref != nil: + return a.Ref.unqualifiedName() + } + + return "", false +} + +func (a *astAtom) evalItem(item models.Item) (types.AttributeValue, error) { + switch { + case a.Ref != nil: + return a.Ref.evalItem(item) + case a.Literal != nil: + return a.Literal.dynamoValue() + case a.Paren != nil: + return a.Paren.evalItem(item) + } + + return nil, errors.New("unhandled atom case") +} + +func (a *astAtom) String() string { + switch { + case a.Ref != nil: + return a.Ref.String() + case a.Literal != nil: + return a.Literal.String() + case a.Paren != nil: + return "(" + a.Paren.String() + ")" + } + return "" +} diff --git a/internal/dynamo-browse/models/queryexpr/binops.go b/internal/dynamo-browse/models/queryexpr/binops.go deleted file mode 100644 index 078f705..0000000 --- a/internal/dynamo-browse/models/queryexpr/binops.go +++ /dev/null @@ -1,132 +0,0 @@ -package queryexpr - -import ( - "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" - "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/lmika/audax/internal/dynamo-browse/models" - "github.com/lmika/audax/internal/dynamo-browse/models/attrutils" - "github.com/pkg/errors" - "strings" -) - -func (a *astEqualityOp) evalToIR(info *models.TableInfo) (irAtom, error) { - v, err := a.Value.goValue() - if err != nil { - return nil, err - } - - singleName, isSingleName := a.Ref.unqualifiedName() - if !isSingleName { - return nil, errors.Errorf("%v: cannot use dereferences", singleName) - } - - switch a.Op { - case "=": - return irFieldEq{name: singleName, value: v}, nil - case "^=": - strValue, isStrValue := v.(string) - if !isStrValue { - return nil, errors.New("operand '^=' must be string") - } - return irFieldBeginsWith{name: singleName, prefix: strValue}, nil - } - - return nil, errors.Errorf("unrecognised operator: %v", a.Op) -} - -func (a *astEqualityOp) evalItem(item models.Item) (types.AttributeValue, error) { - left, err := a.Ref.evalItem(item) - if err != nil { - return nil, err - } - - if a.Op == "" { - return left, nil - } - - right, err := a.Value.dynamoValue() - if err != nil { - return nil, err - } - - switch a.Op { - case "=": - cmp, isComparable := attrutils.CompareScalarAttributes(left, right) - if !isComparable { - return nil, ValuesNotComparable{Left: left, Right: right} - } - return &types.AttributeValueMemberBOOL{Value: cmp == 0}, nil - case "^=": - rightVal, err := a.Value.goValue() - if err != nil { - return nil, err - } - - strValue, isStrValue := rightVal.(string) - if !isStrValue { - return nil, errors.New("operand '^=' must be string") - } - - leftAsStr, canBeString := attrutils.AttributeToString(left) - if !canBeString { - return nil, ValueNotConvertableToString{Val: left} - } - return &types.AttributeValueMemberBOOL{Value: strings.HasPrefix(leftAsStr, strValue)}, nil - } - - return nil, errors.Errorf("unrecognised operator: %v", a.Op) -} - -func (a *astEqualityOp) String() string { - return a.Ref.String() + a.Op + a.Value.String() -} - -type irFieldEq struct { - name string - value any -} - -func (a irFieldEq) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool { - if a.name == info.Keys.PartitionKey || a.name == info.Keys.SortKey { - return qci.addKey(info, a.name) - } - - return false -} - -func (a irFieldEq) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { - return expression.Name(a.name).Equal(expression.Value(a.value)), nil -} - -func (a irFieldEq) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) { - return expression.Key(a.name).Equal(expression.Value(a.value)), nil -} - -func (a irFieldEq) operandFieldName() string { - return a.name -} - -type irFieldBeginsWith struct { - name string - prefix string -} - -func (a irFieldBeginsWith) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool { - if a.name == info.Keys.SortKey { - return qci.addKey(info, a.name) - } - - return false -} - -func (a irFieldBeginsWith) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { - return expression.Name(a.name).BeginsWith(a.prefix), nil -} - -func (a irFieldBeginsWith) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) { - return expression.Key(a.name).BeginsWith(a.prefix), nil -} - -func (a irFieldBeginsWith) operandFieldName() string { - return a.name -} diff --git a/internal/dynamo-browse/models/queryexpr/boolnot.go b/internal/dynamo-browse/models/queryexpr/boolnot.go new file mode 100644 index 0000000..32eee69 --- /dev/null +++ b/internal/dynamo-browse/models/queryexpr/boolnot.go @@ -0,0 +1,56 @@ +package queryexpr + +import ( + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/lmika/audax/internal/dynamo-browse/models" + "strings" +) + +func (a *astBooleanNot) evalToIR(tableInfo *models.TableInfo) (irAtom, error) { + irNode, err := a.Operand.evalToIR(tableInfo) + if err != nil { + return nil, err + } + + if !a.HasNot { + return irNode, nil + } + + return &irBoolNot{atom: irNode}, nil +} + +func (a *astBooleanNot) evalItem(item models.Item) (types.AttributeValue, error) { + val, err := a.Operand.evalItem(item) + if err != nil { + return nil, err + } + + if !a.HasNot { + return val, nil + } + + return &types.AttributeValueMemberBOOL{Value: !isAttributeTrue(val)}, nil +} + +func (d *astBooleanNot) String() string { + sb := new(strings.Builder) + if d.HasNot { + sb.WriteString(" not ") + } + sb.WriteString(d.Operand.String()) + return sb.String() +} + +type irBoolNot struct { + atom irAtom +} + +func (d *irBoolNot) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + cb, err := d.atom.calcQueryForScan(info) + if err != nil { + return expression.ConditionBuilder{}, err + } + + return expression.Not(cb), nil +} diff --git a/internal/dynamo-browse/models/queryexpr/builtins.go b/internal/dynamo-browse/models/queryexpr/builtins.go new file mode 100644 index 0000000..a42f0cc --- /dev/null +++ b/internal/dynamo-browse/models/queryexpr/builtins.go @@ -0,0 +1,39 @@ +package queryexpr + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/pkg/errors" + "strconv" +) + +type nativeFunc func(ctx context.Context, args []types.AttributeValue) (types.AttributeValue, error) + +var nativeFuncs = map[string]nativeFunc{ + "size": func(ctx context.Context, args []types.AttributeValue) (types.AttributeValue, error) { + if len(args) != 1 { + return nil, InvalidArgumentNumberError{Name: "size", Expected: 1, Actual: len(args)} + } + + var l int + switch t := args[0].(type) { + case *types.AttributeValueMemberB: + l = len(t.Value) + case *types.AttributeValueMemberS: + l = len(t.Value) + case *types.AttributeValueMemberL: + l = len(t.Value) + case *types.AttributeValueMemberM: + l = len(t.Value) + case *types.AttributeValueMemberSS: + l = len(t.Value) + case *types.AttributeValueMemberNS: + l = len(t.Value) + case *types.AttributeValueMemberBS: + l = len(t.Value) + default: + return nil, errors.New("cannot take size of arg") + } + return &types.AttributeValueMemberN{Value: strconv.Itoa(l)}, nil + }, +} diff --git a/internal/dynamo-browse/models/queryexpr/calcquery.go b/internal/dynamo-browse/models/queryexpr/calcquery.go deleted file mode 100644 index 45e3986..0000000 --- a/internal/dynamo-browse/models/queryexpr/calcquery.go +++ /dev/null @@ -1,47 +0,0 @@ -package queryexpr - -import ( - "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" - "github.com/lmika/audax/internal/dynamo-browse/models" -) - -func (a *irDisjunction) calcQuery(info *models.TableInfo) (*models.QueryExecutionPlan, error) { - var qci queryCalcInfo - if a.canBeExecutedAsQuery(info, &qci) { - ke, err := a.calcQueryForQuery(info) - if err != nil { - return nil, err - } - - builder := expression.NewBuilder() - builder = builder.WithKeyCondition(ke) - - expr, err := builder.Build() - if err != nil { - return nil, err - } - - return &models.QueryExecutionPlan{ - CanQuery: true, - Expression: expr, - }, nil - } - - cb, err := a.calcQueryForScan(info) - if err != nil { - return nil, err - } - - builder := expression.NewBuilder() - builder = builder.WithFilter(cb) - - expr, err := builder.Build() - if err != nil { - return nil, err - } - - return &models.QueryExecutionPlan{ - CanQuery: false, - Expression: expr, - }, nil -} diff --git a/internal/dynamo-browse/models/queryexpr/comp.go b/internal/dynamo-browse/models/queryexpr/comp.go new file mode 100644 index 0000000..a90b858 --- /dev/null +++ b/internal/dynamo-browse/models/queryexpr/comp.go @@ -0,0 +1,177 @@ +package queryexpr + +import ( + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/lmika/audax/internal/dynamo-browse/models" + "github.com/lmika/audax/internal/dynamo-browse/models/attrutils" + "github.com/pkg/errors" +) + +func (a *astComparisonOp) evalToIR(info *models.TableInfo) (irAtom, error) { + leftIR, err := a.Ref.evalToIR(info) + if err != nil { + return nil, err + } + + if a.Op == "" { + return leftIR, nil + } + + cmpType, hasCmpType := opToCmdType[a.Op] + if !hasCmpType { + return nil, errors.Errorf("unrecognised operator: %v", a.Op) + } + + leftOpr, isLeftOpr := leftIR.(oprIRAtom) + if !isLeftOpr { + return nil, OperandNotAnOperandError{} + } + + rightIR, err := a.Value.evalToIR(info) + if err != nil { + return nil, err + } + + rightOpr, isRightIR := rightIR.(oprIRAtom) + if !isRightIR { + return nil, OperandNotAnOperandError{} + } + + nameIR, isNameIR := leftIR.(nameIRAtom) + valueIR, isValueIR := rightIR.(valueIRAtom) + if isNameIR && isValueIR { + return irKeyFieldCmp{nameIR, valueIR, cmpType}, nil + } + + return irGenericCmp{leftOpr, rightOpr, cmpType}, nil +} + +func (a *astComparisonOp) evalItem(item models.Item) (types.AttributeValue, error) { + left, err := a.Ref.evalItem(item) + if err != nil { + return nil, err + } + if a.Op == "" { + return left, nil + } + + right, err := a.Value.evalItem(item) + if err != nil { + return nil, err + } + + cmp, isComparable := attrutils.CompareScalarAttributes(left, right) + if !isComparable { + return nil, ValuesNotComparable{Left: left, Right: right} + } + + switch opToCmdType[a.Op] { + case cmpTypeLt: + return &types.AttributeValueMemberBOOL{Value: cmp < 0}, nil + case cmpTypeLe: + return &types.AttributeValueMemberBOOL{Value: cmp <= 0}, nil + case cmpTypeGt: + return &types.AttributeValueMemberBOOL{Value: cmp > 0}, nil + case cmpTypeGe: + return &types.AttributeValueMemberBOOL{Value: cmp >= 0}, nil + } + return nil, errors.Errorf("unrecognised operator: %v", a.Op) +} + +func (a *astComparisonOp) String() string { + if a.Op == "" { + return a.Ref.String() + } + return a.Ref.String() + a.Op + a.Value.String() +} + +const ( + cmpTypeLt int = 0 + cmpTypeLe int = 1 + cmpTypeGt int = 2 + cmpTypeGe int = 3 +) + +var opToCmdType = map[string]int{ + "<": cmpTypeLt, + "<=": cmpTypeLe, + ">": cmpTypeGt, + ">=": cmpTypeGe, +} + +type irKeyFieldCmp struct { + name nameIRAtom + value valueIRAtom + cmpType int +} + +func (a irKeyFieldCmp) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool { + keyName := a.name.keyName() + if keyName == "" { + return false + } + + if keyName == info.Keys.SortKey { + return qci.addKey(info, keyName) + } + + return false +} + +func (a irKeyFieldCmp) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + nb := a.name.calcName(info) + vb := a.value.goValue() + + switch a.cmpType { + case cmpTypeLt: + return nb.LessThan(expression.Value(vb)), nil + case cmpTypeLe: + return nb.LessThanEqual(expression.Value(vb)), nil + case cmpTypeGt: + return nb.GreaterThan(expression.Value(vb)), nil + case cmpTypeGe: + return nb.GreaterThanEqual(expression.Value(vb)), nil + } + return expression.ConditionBuilder{}, errors.New("unsupported cmp type") +} + +func (a irKeyFieldCmp) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) { + keyName := a.name.keyName() + vb := a.value.goValue() + + switch a.cmpType { + case cmpTypeLt: + return expression.Key(keyName).LessThan(expression.Value(vb)), nil + case cmpTypeLe: + return expression.Key(keyName).LessThanEqual(expression.Value(vb)), nil + case cmpTypeGt: + return expression.Key(keyName).GreaterThan(expression.Value(vb)), nil + case cmpTypeGe: + return expression.Key(keyName).GreaterThanEqual(expression.Value(vb)), nil + } + return expression.KeyConditionBuilder{}, errors.New("unsupported cmp type") +} + +type irGenericCmp struct { + left oprIRAtom + right oprIRAtom + cmpType int +} + +func (a irGenericCmp) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + nb := a.left.calcOperand(info) + vb := a.right.calcOperand(info) + + switch a.cmpType { + case cmpTypeLt: + return expression.LessThan(nb, vb), nil + case cmpTypeLe: + return expression.LessThanEqual(nb, vb), nil + case cmpTypeGt: + return expression.GreaterThan(nb, vb), nil + case cmpTypeGe: + return expression.GreaterThanEqual(nb, vb), nil + } + return expression.ConditionBuilder{}, errors.New("unsupported cmp type") +} diff --git a/internal/dynamo-browse/models/queryexpr/conj.go b/internal/dynamo-browse/models/queryexpr/conj.go index 1de0e77..614bf0f 100644 --- a/internal/dynamo-browse/models/queryexpr/conj.go +++ b/internal/dynamo-browse/models/queryexpr/conj.go @@ -4,11 +4,26 @@ import ( "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/lmika/audax/internal/dynamo-browse/models" - "github.com/pkg/errors" "strings" ) -func (a *astConjunction) evalToIR(tableInfo *models.TableInfo) (*irConjunction, error) { +func (a *astConjunction) evalToIR(tableInfo *models.TableInfo) (irAtom, error) { + if len(a.Operands) == 1 { + return a.Operands[0].evalToIR(tableInfo) + } else if len(a.Operands) == 2 { + left, err := a.Operands[0].evalToIR(tableInfo) + if err != nil { + return nil, err + } + + right, err := a.Operands[1].evalToIR(tableInfo) + if err != nil { + return nil, err + } + + return &irDualConjunction{left: left, right: right}, nil + } + atoms := make([]irAtom, len(a.Operands)) for i, op := range a.Operands { var err error @@ -18,7 +33,7 @@ func (a *astConjunction) evalToIR(tableInfo *models.TableInfo) (*irConjunction, } } - return &irConjunction{atoms: atoms}, nil + return &irMultiConjunction{atoms: atoms}, nil } func (a *astConjunction) evalItem(item models.Item) (types.AttributeValue, error) { @@ -55,49 +70,66 @@ func (d *astConjunction) String() string { return sb.String() } -type irConjunction struct { - atoms []irAtom +type irDualConjunction struct { + left irAtom + right irAtom + leftIsPK bool } -func (d *irConjunction) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool { - switch len(d.atoms) { - case 1: - return d.atoms[0].operandFieldName() == info.Keys.PartitionKey && d.atoms[0].canBeExecutedAsQuery(info, qci) - case 2: - return d.atoms[0].canBeExecutedAsQuery(info, qci) && d.atoms[1].canBeExecutedAsQuery(info, qci) +func (i *irDualConjunction) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool { + qciCopy := qci.clone() + + leftCanExecuteAsQuery := canExecuteAsQuery(i.left, info, qci) + if leftCanExecuteAsQuery { + i.leftIsPK = qci.hasSeenPrimaryKey(info) + return canExecuteAsQuery(i.right, info, qci) } + + // Might be that the right is the partition key, so test again with them swapped + rightCanExecuteAsQuery := canExecuteAsQuery(i.right, info, qciCopy) + if rightCanExecuteAsQuery { + return canExecuteAsQuery(i.left, info, qciCopy) + } + return false } -func (d *irConjunction) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) { - if len(d.atoms) == 1 { - return d.atoms[0].calcQueryForQuery(info) - } else if len(d.atoms) != 2 { - return expression.KeyConditionBuilder{}, errors.Errorf("internal error: expected len to be either 1 or 2, but was %v", len(d.atoms)) - } - - left, err := d.atoms[0].calcQueryForQuery(info) +func (i *irDualConjunction) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) { + left, err := i.left.(queryableIRAtom).calcQueryForQuery(info) if err != nil { return expression.KeyConditionBuilder{}, err } - right, err := d.atoms[1].calcQueryForQuery(info) + right, err := i.right.(queryableIRAtom).calcQueryForQuery(info) if err != nil { return expression.KeyConditionBuilder{}, err } - if d.atoms[0].operandFieldName() == info.Keys.PartitionKey { + if i.leftIsPK { return expression.KeyAnd(left, right), nil } return expression.KeyAnd(right, left), nil } -func (d *irConjunction) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { - if len(d.atoms) == 1 { - return d.atoms[0].calcQueryForScan(info) +func (i *irDualConjunction) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + left, err := i.left.calcQueryForScan(info) + if err != nil { + return expression.ConditionBuilder{}, err } - // TODO: check if can be query + right, err := i.right.calcQueryForScan(info) + if err != nil { + return expression.ConditionBuilder{}, err + } + + return expression.And(left, right), nil +} + +type irMultiConjunction struct { + atoms []irAtom +} + +func (d *irMultiConjunction) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { conds := make([]expression.ConditionBuilder, len(d.atoms)) for i, operand := range d.atoms { cond, err := operand.calcQueryForScan(info) diff --git a/internal/dynamo-browse/models/queryexpr/disj.go b/internal/dynamo-browse/models/queryexpr/disj.go index a5d1f6c..30b5a6f 100644 --- a/internal/dynamo-browse/models/queryexpr/disj.go +++ b/internal/dynamo-browse/models/queryexpr/disj.go @@ -4,12 +4,15 @@ import ( "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/lmika/audax/internal/dynamo-browse/models" - "github.com/pkg/errors" "strings" ) -func (a *astDisjunction) evalToIR(tableInfo *models.TableInfo) (*irDisjunction, error) { - conj := make([]*irConjunction, len(a.Operands)) +func (a *astDisjunction) evalToIR(tableInfo *models.TableInfo) (irAtom, error) { + if len(a.Operands) == 1 { + return a.Operands[0].evalToIR(tableInfo) + } + + conj := make([]irAtom, len(a.Operands)) for i, op := range a.Operands { var err error conj[i], err = op.evalToIR(tableInfo) @@ -56,23 +59,7 @@ func (d *astDisjunction) String() string { } type irDisjunction struct { - conj []*irConjunction -} - -func (d *irDisjunction) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool { - // TODO: not entire accurate, as filter expressions are also possible - if len(d.conj) == 1 { - return d.conj[0].canBeExecutedAsQuery(info, qci) - } - return false -} - -func (d *irDisjunction) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) { - if len(d.conj) == 1 { - return d.conj[0].calcQueryForQuery(info) - } - - return expression.KeyConditionBuilder{}, errors.New("expected exactly 1 operand for query") + conj []irAtom } func (d *irDisjunction) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { diff --git a/internal/dynamo-browse/models/queryexpr/dot.go b/internal/dynamo-browse/models/queryexpr/dot.go index 001227a..f6906ba 100644 --- a/internal/dynamo-browse/models/queryexpr/dot.go +++ b/internal/dynamo-browse/models/queryexpr/dot.go @@ -1,11 +1,16 @@ package queryexpr import ( + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/lmika/audax/internal/dynamo-browse/models" "strings" ) +func (dt *astDot) evalToIR(info *models.TableInfo) (irAtom, error) { + return irNamePath{dt.Name, dt.Quals}, nil +} + func (dt *astDot) unqualifiedName() (string, bool) { if len(dt.Quals) == 0 { return dt.Name, true @@ -16,7 +21,7 @@ func (dt *astDot) unqualifiedName() (string, bool) { func (dt *astDot) evalItem(item models.Item) (types.AttributeValue, error) { res, hasV := item[dt.Name] if !hasV { - return nil, NameNotFoundError(dt.String()) + return nil, nil } for i, qualName := range dt.Quals { @@ -27,7 +32,7 @@ func (dt *astDot) evalItem(item models.Item) (types.AttributeValue, error) { res, hasV = mapRes.Value[qualName] if !hasV { - return nil, NameNotFoundError(dt.String()) + return nil, nil } } @@ -45,3 +50,31 @@ func (a *astDot) String() string { return sb.String() } + +type irNamePath struct { + name string + quals []string +} + +func (i irNamePath) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + return expression.ConditionBuilder{}, NodeCannotBeConvertedToQueryError{} +} + +func (i irNamePath) calcOperand(info *models.TableInfo) expression.OperandBuilder { + return i.calcName(info) +} + +func (i irNamePath) keyName() string { + if len(i.quals) > 0 { + return "" + } + return i.name +} + +func (i irNamePath) calcName(info *models.TableInfo) expression.NameBuilder { + nb := expression.Name(i.name) + for _, qual := range i.quals { + nb = nb.AppendName(expression.Name(qual)) + } + return nb +} diff --git a/internal/dynamo-browse/models/queryexpr/equality.go b/internal/dynamo-browse/models/queryexpr/equality.go new file mode 100644 index 0000000..284e9f5 --- /dev/null +++ b/internal/dynamo-browse/models/queryexpr/equality.go @@ -0,0 +1,202 @@ +package queryexpr + +import ( + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/lmika/audax/internal/dynamo-browse/models" + "github.com/lmika/audax/internal/dynamo-browse/models/attrutils" + "github.com/pkg/errors" + "strings" +) + +func (a *astEqualityOp) evalToIR(info *models.TableInfo) (irAtom, error) { + leftIR, err := a.Ref.evalToIR(info) + if err != nil { + return nil, err + } + + if a.Op == "" { + return leftIR, nil + } + + leftOpr, isLeftOpr := leftIR.(oprIRAtom) + if !isLeftOpr { + return nil, OperandNotAnOperandError{} + } + + rightIR, err := a.Value.evalToIR(info) + if err != nil { + return nil, err + } + + rightOpr, isRightIR := rightIR.(oprIRAtom) + if !isRightIR { + return nil, OperandNotAnOperandError{} + } + + switch a.Op { + case "=": + nameIR, isNameIR := leftIR.(nameIRAtom) + valueIR, isValueIR := rightIR.(valueIRAtom) + if isNameIR && isValueIR { + return irKeyFieldEq{name: nameIR, value: valueIR}, nil + } + return irGenericEq{name: leftOpr, value: rightOpr}, nil + case "!=": + return irFieldNe{name: leftOpr, value: rightOpr}, nil + case "^=": + nameIR, isNameIR := leftIR.(nameIRAtom) + if !isNameIR { + return nil, OperandNotANameError(a.Ref.String()) + } + realValueIR, isRealValueIR := rightIR.(irValue) + if !isRealValueIR { + return nil, ValueMustBeLiteralError{} + } + return irFieldBeginsWith{name: nameIR, value: realValueIR}, nil + } + + return nil, errors.Errorf("unrecognised operator: %v", a.Op) +} + +func (a *astEqualityOp) evalItem(item models.Item) (types.AttributeValue, error) { + left, err := a.Ref.evalItem(item) + if err != nil { + return nil, err + } + + if a.Op == "" { + return left, nil + } + + right, err := a.Value.evalItem(item) + if err != nil { + return nil, err + } + + switch a.Op { + case "=": + cmp, isComparable := attrutils.CompareScalarAttributes(left, right) + if !isComparable { + return nil, ValuesNotComparable{Left: left, Right: right} + } + return &types.AttributeValueMemberBOOL{Value: cmp == 0}, nil + case "!=": + cmp, isComparable := attrutils.CompareScalarAttributes(left, right) + if !isComparable { + return nil, ValuesNotComparable{Left: left, Right: right} + } + return &types.AttributeValueMemberBOOL{Value: cmp != 0}, nil + case "^=": + strValue, isStrValue := right.(*types.AttributeValueMemberS) + if !isStrValue { + return nil, errors.New("operand '^=' must be string") + } + + leftAsStr, canBeString := attrutils.AttributeToString(left) + if !canBeString { + return nil, ValueNotConvertableToString{Val: left} + } + return &types.AttributeValueMemberBOOL{Value: strings.HasPrefix(leftAsStr, strValue.Value)}, nil + } + + return nil, errors.Errorf("unrecognised operator: %v", a.Op) +} + +func (a *astEqualityOp) String() string { + if a.Op == "" { + return a.Ref.String() + } + return a.Ref.String() + a.Op + a.Value.String() +} + +type irKeyFieldEq struct { + name nameIRAtom + value valueIRAtom +} + +func (a irKeyFieldEq) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool { + keyName := a.name.keyName() + if keyName == "" { + return false + } + + if keyName == info.Keys.PartitionKey || + (keyName == info.Keys.SortKey && qci.hasSeenPrimaryKey(info)) { + return qci.addKey(info, keyName) + } + + return false +} + +func (a irKeyFieldEq) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + nb := a.name.calcName(info) + vb := a.value.calcOperand(info) + return nb.Equal(vb), nil +} + +func (a irKeyFieldEq) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) { + vb := a.value.goValue() + return expression.Key(a.name.keyName()).Equal(expression.Value(vb)), nil +} + +type irGenericEq struct { + name oprIRAtom + value oprIRAtom +} + +func (a irGenericEq) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + nb := a.name.calcOperand(info) + vb := a.value.calcOperand(info) + return expression.Equal(nb, vb), nil +} + +type irFieldNe struct { + name oprIRAtom + value oprIRAtom +} + +func (a irFieldNe) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + nb := a.name.calcOperand(info) + vb := a.value.calcOperand(info) + return expression.NotEqual(nb, vb), nil +} + +type irFieldBeginsWith struct { + name nameIRAtom + value irValue +} + +func (a irFieldBeginsWith) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool { + keyName := a.name.keyName() + if keyName == "" { + return false + } + + if keyName == info.Keys.SortKey { + return qci.addKey(info, a.name.keyName()) + } + + return false +} + +func (a irFieldBeginsWith) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + nb := a.name.calcName(info) + vb := a.value.goValue() + strValue, isStrValue := vb.(string) + if !isStrValue { + return expression.ConditionBuilder{}, errors.New("operand '^=' must be string") + } + + return nb.BeginsWith(strValue), nil +} + +func (a irFieldBeginsWith) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) { + vb := a.value.goValue() + strValue, isStrValue := vb.(string) + if !isStrValue { + return expression.KeyConditionBuilder{}, errors.New("operand '^=' must be string") + } + + return expression.Key(a.name.keyName()).BeginsWith(strValue), nil +} diff --git a/internal/dynamo-browse/models/queryexpr/errors.go b/internal/dynamo-browse/models/queryexpr/errors.go index 97e3241..9df428f 100644 --- a/internal/dynamo-browse/models/queryexpr/errors.go +++ b/internal/dynamo-browse/models/queryexpr/errors.go @@ -15,6 +15,18 @@ func (n NameNotFoundError) Error() string { return fmt.Sprintf("%v: name not found", string(n)) } +type OperandNotANameError string + +func (n OperandNotANameError) Error() string { + return fmt.Sprintf("operand '%v' is not a name", string(n)) +} + +type OperandNotAnOperandError struct{} + +func (n OperandNotAnOperandError) Error() string { + return "element must be an operand" +} + // ValueNotAMapError is return if the given name is not a map type ValueNotAMapError []string @@ -33,6 +45,16 @@ func (n ValuesNotComparable) Error() string { return fmt.Sprintf("values '%v' and '%v' are not comparable", leftStr, rightStr) } +// ValuesNotInnable indicates that a values cannot be used on the right side of an in +type ValuesNotInnableError struct { + Val types.AttributeValue +} + +func (n ValuesNotInnableError) Error() string { + leftStr, _ := attrutils.AttributeToString(n.Val) + return fmt.Sprintf("values '%v' cannot be used as the right side of an 'in'", leftStr) +} + // ValueNotConvertableToString indicates that a value is not convertable to a string type ValueNotConvertableToString struct { Val types.AttributeValue @@ -42,3 +64,47 @@ func (n ValueNotConvertableToString) Error() string { render := itemrender.ToRenderer(n.Val) return fmt.Sprintf("values '%v', type %v, is not convertable to string", render.StringValue(), render.TypeName()) } + +type NodeCannotBeConvertedToQueryError struct{} + +func (n NodeCannotBeConvertedToQueryError) Error() string { + return "node cannot be converted to query" +} + +type ValueMustBeLiteralError struct{} + +func (n ValueMustBeLiteralError) Error() string { + return "value must be a literal" +} + +type ValueMustBeStringError struct{} + +func (n ValueMustBeStringError) Error() string { + return "value must be a string" +} + +type InvalidTypeForIsError struct { + TypeName string +} + +func (n InvalidTypeForIsError) Error() string { + return "invalid type for 'is': " + n.TypeName +} + +type InvalidArgumentNumberError struct { + Name string + Expected int + Actual int +} + +func (e InvalidArgumentNumberError) Error() string { + return fmt.Sprintf("function '%v' expected %v args but received %v", e.Name, e.Expected, e.Actual) +} + +type UnrecognisedFunctionError struct { + Name string +} + +func (e UnrecognisedFunctionError) Error() string { + return "unrecognised function '" + e.Name + "'" +} diff --git a/internal/dynamo-browse/models/queryexpr/expr.go b/internal/dynamo-browse/models/queryexpr/expr.go index ebb6bb7..4e3d945 100644 --- a/internal/dynamo-browse/models/queryexpr/expr.go +++ b/internal/dynamo-browse/models/queryexpr/expr.go @@ -10,12 +10,7 @@ type QueryExpr struct { } func (md *QueryExpr) Plan(tableInfo *models.TableInfo) (*models.QueryExecutionPlan, error) { - ir, err := md.ast.evalToIR(tableInfo) - if err != nil { - return nil, err - } - - return ir.calcQuery(tableInfo) + return md.ast.calcQuery(tableInfo) } func (md *QueryExpr) EvalItem(item models.Item) (types.AttributeValue, error) { @@ -34,6 +29,19 @@ type queryCalcInfo struct { seenKeys map[string]struct{} } +func (qc *queryCalcInfo) clone() *queryCalcInfo { + newKeys := make(map[string]struct{}) + for k, v := range qc.seenKeys { + newKeys[k] = v + } + return &queryCalcInfo{seenKeys: newKeys} +} + +func (qc *queryCalcInfo) hasSeenPrimaryKey(tableInfo *models.TableInfo) bool { + _, hasKey := qc.seenKeys[tableInfo.Keys.PartitionKey] + return hasKey +} + func (qc *queryCalcInfo) addKey(tableInfo *models.TableInfo, key string) bool { if tableInfo.Keys.PartitionKey != key && tableInfo.Keys.SortKey != key { return false diff --git a/internal/dynamo-browse/models/queryexpr/expr_test.go b/internal/dynamo-browse/models/queryexpr/expr_test.go index b203851..b559b6c 100644 --- a/internal/dynamo-browse/models/queryexpr/expr_test.go +++ b/internal/dynamo-browse/models/queryexpr/expr_test.go @@ -1,6 +1,7 @@ package queryexpr_test import ( + "fmt" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/lmika/audax/internal/dynamo-browse/models/queryexpr" @@ -20,133 +21,248 @@ func TestModExpr_Query(t *testing.T) { } t.Run("as queries", func(t *testing.T) { - t.Run("perform query when request pk is fixed", func(t *testing.T) { - modExpr, err := queryexpr.Parse(`pk="prefix"`) - assert.NoError(t, err) + scenarios := []scanScenario{ + scanCase("when request pk is fixed", + `pk="prefix"`, + `#0 = :0`, + exprNameIsString(0, 0, "pk", "prefix"), + ), + scanCase("when request pk is fixed in parens #1", + `(pk="prefix")`, + `#0 = :0`, + exprNameIsString(0, 0, "pk", "prefix"), + ), + scanCase("when request pk is fixed in parens #2", + `(pk)="prefix"`, + `#0 = :0`, + exprNameIsString(0, 0, "pk", "prefix"), + ), + scanCase("when request pk is fixed in parens #3", + `pk=("prefix")`, + `#0 = :0`, + exprNameIsString(0, 0, "pk", "prefix"), + ), + scanCase("when request pk is in with a single value", + `pk in ("prefix")`, + `#0 = :0`, + exprNameIsString(0, 0, "pk", "prefix"), + ), + scanCase("when request pk and sk is fixed", + `pk="prefix" and sk="another"`, + `(#0 = :0) AND (#1 = :1)`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsString(1, 1, "sk", "another"), + ), + scanCase("when request pk and sk is fixed (using 'in')", + `pk in ("prefix") and sk in ("another")`, + `(#0 = :0) AND (#1 = :1)`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsString(1, 1, "sk", "another"), + ), + scanCase("when request pk is equals and sk is prefix #1", + `pk="prefix" and sk^="another"`, + `(#0 = :0) AND (begins_with (#1, :1))`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsString(1, 1, "sk", "another"), + ), + scanCase("when request pk is equals and sk is prefix #2", + `sk^="another" and pk="prefix"`, + `(#0 = :0) AND (begins_with (#1, :1))`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsString(1, 1, "sk", "another"), + ), + scanCase("when request pk is equals and sk is less than", + `pk="prefix" and sk < 100`, + `(#0 = :0) AND (#1 < :1)`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsNumber(1, 1, "sk", "100"), + ), + scanCase("when request pk is equals and sk is less or equal to", + `pk="prefix" and sk <= 100`, + `(#0 = :0) AND (#1 <= :1)`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsNumber(1, 1, "sk", "100"), + ), + scanCase("when request pk is equals and sk is greater than", + `pk="prefix" and sk > 100`, + `(#0 = :0) AND (#1 > :1)`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsNumber(1, 1, "sk", "100"), + ), + scanCase("when request pk is equals and sk is greater or equal to", + `pk="prefix" and sk >= 100`, + `(#0 = :0) AND (#1 >= :1)`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsNumber(1, 1, "sk", "100"), + ), + } - plan, err := modExpr.Plan(tableInfo) - assert.NoError(t, err) + for _, scenario := range scenarios { + t.Run(scenario.description, func(t *testing.T) { + modExpr, err := queryexpr.Parse(scenario.expression) + assert.NoError(t, err) - assert.True(t, plan.CanQuery) - assert.Equal(t, "#0 = :0", aws.ToString(plan.Expression.KeyCondition())) - assert.Equal(t, "pk", plan.Expression.Names()["#0"]) - assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value) - }) + plan, err := modExpr.Plan(tableInfo) + assert.NoError(t, err) - t.Run("perform query when request pk and sk is fixed", func(t *testing.T) { - modExpr, err := queryexpr.Parse(`pk="prefix" and sk="another"`) - assert.NoError(t, err) - - plan, err := modExpr.Plan(tableInfo) - assert.NoError(t, err) - - assert.True(t, plan.CanQuery) - assert.Equal(t, "(#0 = :0) AND (#1 = :1)", aws.ToString(plan.Expression.KeyCondition())) - assert.Equal(t, "pk", plan.Expression.Names()["#0"]) - assert.Equal(t, "sk", plan.Expression.Names()["#1"]) - assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value) - assert.Equal(t, "another", plan.Expression.Values()[":1"].(*types.AttributeValueMemberS).Value) - }) - - t.Run("perform query when request pk is equals and sk is prefix", func(t *testing.T) { - scenarios := []struct { - expr string - }{ - {expr: `pk="prefix" and sk^="another"`}, - {expr: `sk^="another" and pk="prefix"`}, - } - - for _, scenario := range scenarios { - t.Run(scenario.expr, func(t *testing.T) { - modExpr, err := queryexpr.Parse(scenario.expr) - assert.NoError(t, err) - - plan, err := modExpr.Plan(tableInfo) - assert.NoError(t, err) - - assert.True(t, plan.CanQuery) - assert.Equal(t, "(#0 = :0) AND (begins_with (#1, :1))", aws.ToString(plan.Expression.KeyCondition())) - assert.Equal(t, "pk", plan.Expression.Names()["#0"]) - assert.Equal(t, "sk", plan.Expression.Names()["#1"]) - assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value) - assert.Equal(t, "another", plan.Expression.Values()[":1"].(*types.AttributeValueMemberS).Value) - }) - } - }) + assert.True(t, plan.CanQuery) + assert.Equal(t, scenario.expectedFilter, aws.ToString(plan.Expression.KeyCondition())) + for k, v := range scenario.expectedNames { + assert.Equal(t, v, plan.Expression.Names()[k]) + } + for k, v := range scenario.expectedValues { + assert.Equal(t, v, plan.Expression.Values()[k]) + } + }) + } }) t.Run("as scans", func(t *testing.T) { - t.Run("when request pk prefix", func(t *testing.T) { - modExpr, err := queryexpr.Parse(`pk^="prefix"`) - assert.NoError(t, err) + scenarios := []scanScenario{ + scanCase("when request pk prefix", `pk^="prefix"`, `begins_with (#0, :0)`, + exprNameIsString(0, 0, "pk", "prefix"), + ), + scanCase("when request sk equals something", `sk="something"`, `#0 = :0`, + exprNameIsString(0, 0, "sk", "something"), + ), + scanCase("with not equal", `sk != "something"`, `#0 <> :0`, + exprNameIsString(0, 0, "sk", "something"), + ), + scanCase("less than value", `num < 100`, `#0 < :0`, + exprNameIsNumber(0, 0, "num", "100"), + ), + scanCase("less or equal to value", `num <= 100`, `#0 <= :0`, + exprNameIsNumber(0, 0, "num", "100"), + ), + scanCase("greater than value", `num > 100`, `#0 > :0`, + exprNameIsNumber(0, 0, "num", "100"), + ), + scanCase("greater or equal to value", `num >= 100`, `#0 >= :0`, + exprNameIsNumber(0, 0, "num", "100"), + ), + scanCase("with disjunctions", + `pk="prefix" or sk="another"`, + `(#0 = :0) OR (#1 = :1)`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsString(1, 1, "sk", "another"), + ), + scanCase("with disjunctions with numbers", + `pk="prefix" or num=123 and negnum=-131`, + `(#0 = :0) OR ((#1 = :1) AND (#2 = :2))`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsNumber(1, 1, "num", "123"), + exprNameIsNumber(2, 2, "negnum", "-131"), + ), + scanCase("with disjunctions with numbers (different priority)", + `(pk="prefix" or num=123) and negnum=-131`, + `((#0 = :0) OR (#1 = :1)) AND (#2 = :2)`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsNumber(1, 1, "num", "123"), + exprNameIsNumber(2, 2, "negnum", "-131"), + ), + scanCase("with disjunctions if pk is present twice in expression", + `pk="prefix" and pk="another"`, + `(#0 = :0) AND (#0 = :1)`, + exprNameIsString(0, 0, "pk", "prefix"), + exprNameIsString(0, 1, "pk", "another"), + ), + scanCase("with not", `not pk="prefix"`, `NOT (#0 = :0)`, + exprNameIsString(0, 0, "pk", "prefix"), + ), - plan, err := modExpr.Plan(tableInfo) - assert.NoError(t, err) + scanCase("with in", `pk in ("alpha", "bravo", "charlie")`, + `#0 IN (:0, :1, :2)`, + exprName(0, "pk"), + exprValueIsString(0, "alpha"), + exprValueIsString(1, "bravo"), + exprValueIsString(2, "charlie"), + ), + scanCase("with not in", `pk not in ("alpha", "bravo", "charlie")`, + `NOT (#0 IN (:0, :1, :2))`, + exprName(0, "pk"), + exprValueIsString(0, "alpha"), + exprValueIsString(1, "bravo"), + exprValueIsString(2, "charlie"), + ), + scanCase("with in with single operand returning a sequence", `pk in range(1, 5)`, + `#0 IN (:0, :1, :2, :3, :4)`, + exprName(0, "pk"), + exprValueIsNumber(0, "1"), + exprValueIsNumber(1, "2"), + exprValueIsNumber(2, "3"), + exprValueIsNumber(3, "4"), + exprValueIsNumber(4, "5"), + ), + scanCase("with in with single operand not returning a literal", `"foobar" in pk`, + `contains (#0, :0)`, + exprNameIsString(0, 0, "pk", "foobar"), + ), + // TODO: in > 100 items ==> items OR items - assert.False(t, plan.CanQuery) - assert.Equal(t, "begins_with (#0, :0)", aws.ToString(plan.Expression.Filter())) - assert.Equal(t, "pk", plan.Expression.Names()["#0"]) - assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value) - }) + scanCase("with is S", `pk is "S"`, + `attribute_type (#0, :0)`, + exprNameIsString(0, 0, "pk", "S"), + ), + scanCase("with is N", `pk is "N"`, + `attribute_type (#0, :0)`, + exprNameIsString(0, 0, "pk", "N"), + ), + scanCase("with is not N", `pk is not "SS"`, + `NOT (attribute_type (#0, :0))`, + exprNameIsString(0, 0, "pk", "SS"), + ), + scanCase("with is any", `pk is "any"`, + `attribute_exists (#0)`, + exprName(0, "pk"), + ), + scanCase("with is not any", `pk is not "any"`, + `attribute_not_exists (#0)`, + exprName(0, "pk"), + ), - t.Run("when request sk equals something", func(t *testing.T) { - modExpr, err := queryexpr.Parse(`sk="something"`) - assert.NoError(t, err) + scanCase("the size function as a right-side operand", `ln=size(pk)`, + `#0 = size (#1)`, + exprName(0, "ln"), + exprName(1, "pk"), + ), + scanCase("the size function as a left-side operand #1", `size(pk) = 123`, + `size (#0) = :0`, + exprNameIsNumber(0, 0, "pk", "123"), + ), + scanCase("the size function as a left-side operand #2", `size(pk) > 123`, + `size (#0) > :0`, + exprNameIsNumber(0, 0, "pk", "123"), + ), + scanCase("the size function on both sizes", + `size(pk) != size(sk) and size(pk) > size(third) and size(pk) = 131`, + `(size (#0) <> size (#1)) AND (size (#0) > size (#2)) AND (size (#0) = :0)`, + exprName(0, "pk"), + exprName(1, "sk"), + exprName(2, "third"), + exprValueIsNumber(0, "131"), + ), - plan, err := modExpr.Plan(tableInfo) - assert.NoError(t, err) + // TODO: the contains function + } - assert.False(t, plan.CanQuery) - assert.Equal(t, "#0 = :0", aws.ToString(plan.Expression.Filter())) - assert.Equal(t, "sk", plan.Expression.Names()["#0"]) - assert.Equal(t, "something", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value) - }) + for _, scenario := range scenarios { + t.Run(scenario.description, func(t *testing.T) { + modExpr, err := queryexpr.Parse(scenario.expression) + assert.NoError(t, err) - t.Run("with disjunctions", func(t *testing.T) { - modExpr, err := queryexpr.Parse(`pk="prefix" or sk="another"`) - assert.NoError(t, err) + plan, err := modExpr.Plan(tableInfo) + assert.NoError(t, err) - plan, err := modExpr.Plan(tableInfo) - assert.NoError(t, err) - - assert.False(t, plan.CanQuery) - assert.Equal(t, "(#0 = :0) OR (#1 = :1)", aws.ToString(plan.Expression.Filter())) - assert.Equal(t, "pk", plan.Expression.Names()["#0"]) - assert.Equal(t, "sk", plan.Expression.Names()["#1"]) - assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value) - assert.Equal(t, "another", plan.Expression.Values()[":1"].(*types.AttributeValueMemberS).Value) - }) - - t.Run("with disjunctions with numbers", func(t *testing.T) { - modExpr, err := queryexpr.Parse(`pk="prefix" or num=123 and negnum=-131`) - assert.NoError(t, err) - - plan, err := modExpr.Plan(tableInfo) - assert.NoError(t, err) - - assert.False(t, plan.CanQuery) - assert.Equal(t, "(#0 = :0) OR ((#1 = :1) AND (#2 = :2))", aws.ToString(plan.Expression.Filter())) - assert.Equal(t, "pk", plan.Expression.Names()["#0"]) - assert.Equal(t, "num", plan.Expression.Names()["#1"]) - assert.Equal(t, "negnum", plan.Expression.Names()["#2"]) - assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value) - assert.Equal(t, "123", plan.Expression.Values()[":1"].(*types.AttributeValueMemberN).Value) - assert.Equal(t, "-131", plan.Expression.Values()[":2"].(*types.AttributeValueMemberN).Value) - }) - - t.Run("with disjunctions if pk is present twice in expression", func(t *testing.T) { - modExpr, err := queryexpr.Parse(`pk="prefix" and pk="another"`) - assert.NoError(t, err) - - plan, err := modExpr.Plan(tableInfo) - assert.NoError(t, err) - - assert.False(t, plan.CanQuery) - assert.Equal(t, "(#0 = :0) AND (#0 = :1)", aws.ToString(plan.Expression.Filter())) - assert.Equal(t, "pk", plan.Expression.Names()["#0"]) - assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value) - assert.Equal(t, "another", plan.Expression.Values()[":1"].(*types.AttributeValueMemberS).Value) - }) + assert.False(t, plan.CanQuery) + assert.Equal(t, scenario.expectedFilter, aws.ToString(plan.Expression.Filter())) + for k, v := range scenario.expectedNames { + assert.Equal(t, v, plan.Expression.Names()[k]) + } + for k, v := range scenario.expectedValues { + assert.Equal(t, v, plan.Expression.Values()[k]) + } + }) + } }) } @@ -161,6 +277,15 @@ func TestQueryExpr_EvalItem(t *testing.T) { "tree": &types.AttributeValueMemberS{Value: "green"}, }, }, + "prime": &types.AttributeValueMemberL{ + Value: []types.AttributeValue{ + &types.AttributeValueMemberN{Value: "2"}, + &types.AttributeValueMemberN{Value: "3"}, + &types.AttributeValueMemberN{Value: "5"}, + &types.AttributeValueMemberN{Value: "7"}, + }, + }, + "three": &types.AttributeValueMemberN{Value: "3"}, } ) @@ -173,15 +298,65 @@ func TestQueryExpr_EvalItem(t *testing.T) { {expr: `alpha`, expected: &types.AttributeValueMemberS{Value: "alpha"}}, {expr: `bravo`, expected: &types.AttributeValueMemberN{Value: "123"}}, {expr: `charlie`, expected: item["charlie"]}, + {expr: `missing`, expected: nil}, // Equality with literal {expr: `alpha="alpha"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `alpha!="not alpha"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, {expr: `bravo=123`, expected: &types.AttributeValueMemberBOOL{Value: true}}, {expr: `charlie.tree="green"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, {expr: `alpha^="al"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, {expr: `alpha="foobar"`, expected: &types.AttributeValueMemberBOOL{Value: false}}, {expr: `alpha^="need-something"`, expected: &types.AttributeValueMemberBOOL{Value: false}}, + // Comparison + {expr: "three > 4", expected: &types.AttributeValueMemberBOOL{Value: false}}, + {expr: "three >= 4", expected: &types.AttributeValueMemberBOOL{Value: false}}, + {expr: "three < 4", expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: "three <= 4", expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: "three > 3", expected: &types.AttributeValueMemberBOOL{Value: false}}, + {expr: "three >= 3", expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: "three < 3", expected: &types.AttributeValueMemberBOOL{Value: false}}, + {expr: "three <= 3", expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: "three > 2", expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: "three >= 2", expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: "three < 2", expected: &types.AttributeValueMemberBOOL{Value: false}}, + {expr: "three <= 2", expected: &types.AttributeValueMemberBOOL{Value: false}}, + + // In + {expr: "three in (2, 3, 4, 5)", expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: "three in (20, 30, 40)", expected: &types.AttributeValueMemberBOOL{Value: false}}, + {expr: `alpha in ("alpha", "beta", "gamma", "delta")`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `alpha in ("ey", "be", "see")`, expected: &types.AttributeValueMemberBOOL{Value: false}}, + {expr: `three in prime`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `1 in prime`, expected: &types.AttributeValueMemberBOOL{Value: false}}, + {expr: `"door" in charlie`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `"sky" in charlie`, expected: &types.AttributeValueMemberBOOL{Value: false}}, + {expr: `"al" in alpha`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `"cent" in "percentage"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + + // Is + {expr: `alpha is "S"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `alpha is not "N"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `three is "N"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `three is not "S"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `(three = 3) is "BOOL"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `prime is "L"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `charlie is "M"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + + {expr: `alpha is "any"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `three is "any"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `(three = 3) is "any"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `charlie is "any"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `prime is "any"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + {expr: `undef is not "any"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + + // Size + {expr: `size(alpha)`, expected: &types.AttributeValueMemberN{Value: "5"}}, + {expr: `size("This is a test")`, expected: &types.AttributeValueMemberN{Value: "14"}}, + {expr: `size(charlie)`, expected: &types.AttributeValueMemberN{Value: "2"}}, + {expr: `size(prime)`, expected: &types.AttributeValueMemberN{Value: "4"}}, + // Dot values {expr: `charlie.door`, expected: &types.AttributeValueMemberS{Value: "red"}}, {expr: `charlie.tree`, expected: &types.AttributeValueMemberS{Value: "green"}}, @@ -202,6 +377,10 @@ func TestQueryExpr_EvalItem(t *testing.T) { {expr: `alpha="alpha" or bravo=123 or charlie.tree="green"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, {expr: `alpha="bravo" or bravo=321 or charlie.tree^="red"`, expected: &types.AttributeValueMemberBOOL{Value: false}}, + // Bool negation + {expr: `not alpha="alpha"`, expected: &types.AttributeValueMemberBOOL{Value: false}}, + {expr: `not alpha!="alpha"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, + // Order of operation {expr: `alpha="alpha" and bravo=123 or charlie.door="green"`, expected: &types.AttributeValueMemberBOOL{Value: true}}, {expr: `alpha="bravo" or bravo=321 and charlie.door="green"`, expected: &types.AttributeValueMemberBOOL{Value: false}}, @@ -240,7 +419,6 @@ func TestQueryExpr_EvalItem(t *testing.T) { expr string expectedError error }{ - {expr: `not_present`, expectedError: queryexpr.NameNotFoundError("not_present")}, {expr: `alpha.bravo`, expectedError: queryexpr.ValueNotAMapError([]string{"alpha", "bravo"})}, {expr: `charlie.tree.bla`, expectedError: queryexpr.ValueNotAMapError([]string{"charlie", "tree", "bla"})}, } @@ -257,3 +435,57 @@ func TestQueryExpr_EvalItem(t *testing.T) { } }) } + +type scanScenario struct { + description string + expression string + expectedFilter string + expectedNames map[string]string + expectedValues map[string]types.AttributeValue +} + +func scanCase(description, expression, expectedFilter string, options ...func(ss *scanScenario)) scanScenario { + ss := scanScenario{ + description: description, + expression: expression, + expectedFilter: expectedFilter, + expectedNames: map[string]string{}, + expectedValues: map[string]types.AttributeValue{}, + } + for _, opt := range options { + opt(&ss) + } + return ss +} + +func exprName(idx int, name string) func(ss *scanScenario) { + return func(ss *scanScenario) { + ss.expectedNames[fmt.Sprintf("#%d", idx)] = name + } +} + +func exprValueIsString(valIdx int, expected string) func(ss *scanScenario) { + return func(ss *scanScenario) { + ss.expectedValues[fmt.Sprintf(":%d", valIdx)] = &types.AttributeValueMemberS{Value: expected} + } +} + +func exprValueIsNumber(valIdx int, expected string) func(ss *scanScenario) { + return func(ss *scanScenario) { + ss.expectedValues[fmt.Sprintf(":%d", valIdx)] = &types.AttributeValueMemberN{Value: expected} + } +} + +func exprNameIsString(idx, valIdx int, name string, expected string) func(ss *scanScenario) { + return func(ss *scanScenario) { + ss.expectedNames[fmt.Sprintf("#%d", idx)] = name + ss.expectedValues[fmt.Sprintf(":%d", valIdx)] = &types.AttributeValueMemberS{Value: expected} + } +} + +func exprNameIsNumber(idx, valIdx int, name string, expected string) func(ss *scanScenario) { + return func(ss *scanScenario) { + ss.expectedNames[fmt.Sprintf("#%d", idx)] = name + ss.expectedValues[fmt.Sprintf(":%d", valIdx)] = &types.AttributeValueMemberN{Value: expected} + } +} diff --git a/internal/dynamo-browse/models/queryexpr/fncall.go b/internal/dynamo-browse/models/queryexpr/fncall.go new file mode 100644 index 0000000..a9034d0 --- /dev/null +++ b/internal/dynamo-browse/models/queryexpr/fncall.go @@ -0,0 +1,125 @@ +package queryexpr + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/lmika/audax/internal/common/sliceutils" + "github.com/lmika/audax/internal/dynamo-browse/models" + "github.com/pkg/errors" + "strings" +) + +func (a *astFunctionCall) evalToIR(info *models.TableInfo) (irAtom, error) { + callerIr, err := a.Caller.evalToIR(info) + if err != nil { + return nil, err + } + if !a.IsCall { + return callerIr, nil + } + + nameIr, isNameIr := callerIr.(nameIRAtom) + if !isNameIr || nameIr.keyName() == "" { + return nil, OperandNotANameError("") + } + + irNodes, err := sliceutils.MapWithError(a.Args, func(x *astExpr) (irAtom, error) { return x.evalToIR(info) }) + if err != nil { + return nil, err + } + + // TODO: do this properly + switch nameIr.keyName() { + case "size": + if len(irNodes) != 1 { + return nil, InvalidArgumentNumberError{Name: "size", Expected: 1, Actual: len(irNodes)} + } + name, isName := irNodes[0].(nameIRAtom) + if !isName { + return nil, OperandNotANameError(a.Args[0].String()) + } + return irSizeFn{name}, nil + case "range": + if len(irNodes) != 2 { + return nil, InvalidArgumentNumberError{Name: "range", Expected: 2, Actual: len(irNodes)} + } + + // TEMP + fromVal := irNodes[0].(valueIRAtom).goValue().(int64) + toVal := irNodes[1].(valueIRAtom).goValue().(int64) + return irRangeFn{fromVal, toVal}, nil + } + return nil, UnrecognisedFunctionError{Name: nameIr.keyName()} +} + +func (a *astFunctionCall) evalItem(item models.Item) (types.AttributeValue, error) { + if !a.IsCall { + return a.Caller.evalItem(item) + } + + name, isName := a.Caller.unqualifiedName() + if !isName { + return nil, OperandNotANameError(a.Args[0].String()) + } + fn, isFn := nativeFuncs[name] + if !isFn { + return nil, UnrecognisedFunctionError{Name: name} + } + + args, err := sliceutils.MapWithError(a.Args, func(a *astExpr) (types.AttributeValue, error) { + return a.evalItem(item) + }) + if err != nil { + return nil, err + } + + return fn(context.Background(), args) +} + +func (a *astFunctionCall) String() string { + var sb strings.Builder + + sb.WriteString(a.Caller.String()) + if a.IsCall { + sb.WriteRune('(') + for i, q := range a.Args { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(q.String()) + } + sb.WriteRune(')') + } + return sb.String() +} + +type irSizeFn struct { + arg nameIRAtom +} + +func (i irSizeFn) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + return expression.ConditionBuilder{}, errors.New("cannot run as scan") +} + +func (i irSizeFn) calcOperand(info *models.TableInfo) expression.OperandBuilder { + name := i.arg.calcName(info) + return name.Size() +} + +type irRangeFn struct { + fromIdx int64 + toIdx int64 +} + +func (i irRangeFn) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + return expression.ConditionBuilder{}, errors.New("cannot run as scan") +} + +func (i irRangeFn) calcGoValues(info *models.TableInfo) ([]any, error) { + xs := make([]any, 0) + for x := i.fromIdx; x <= i.toIdx; x++ { + xs = append(xs, x) + } + return xs, nil +} diff --git a/internal/dynamo-browse/models/queryexpr/in.go b/internal/dynamo-browse/models/queryexpr/in.go new file mode 100644 index 0000000..6cc3f51 --- /dev/null +++ b/internal/dynamo-browse/models/queryexpr/in.go @@ -0,0 +1,269 @@ +package queryexpr + +import ( + "bytes" + "fmt" + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/lmika/audax/internal/common/sliceutils" + "github.com/lmika/audax/internal/dynamo-browse/models" + "github.com/lmika/audax/internal/dynamo-browse/models/attrutils" + "github.com/pkg/errors" + "strings" +) + +func (a *astIn) evalToIR(info *models.TableInfo) (irAtom, error) { + leftIR, err := a.Ref.evalToIR(info) + if err != nil { + return nil, err + } + + if len(a.Operand) == 0 && a.SingleOperand == nil { + return leftIR, nil + } + + var ir irAtom + switch { + case len(a.Operand) > 0: + nameIR, isNameIR := leftIR.(irNamePath) + if !isNameIR { + return nil, OperandNotANameError(a.Ref.String()) + } + + oprValues := make([]oprIRAtom, len(a.Operand)) + for i, o := range a.Operand { + v, err := o.evalToIR(info) + if err != nil { + return nil, err + } + + valueIR, isValueIR := v.(oprIRAtom) + if !isValueIR { + return nil, errors.Wrapf(ValueMustBeLiteralError{}, "'in' operand %v", i) + } + oprValues[i] = valueIR + } + + // If there is a single operand value, and the name is either the partition or sort key, then + // convert this to an equality so that it could be run as a query + if len(oprValues) == 1 && (nameIR.keyName() == info.Keys.PartitionKey || nameIR.keyName() == info.Keys.SortKey) { + if a.HasNot { + return irFieldNe{name: nameIR, value: oprValues[0]}, nil + } + + if valueIR, isValueIR := oprValues[0].(valueIRAtom); isValueIR { + return irKeyFieldEq{name: nameIR, value: valueIR}, nil + } + return irGenericEq{name: nameIR, value: oprValues[0]}, nil + } + + ir = irIn{name: nameIR, values: oprValues} + case a.SingleOperand != nil: + oprs, err := a.SingleOperand.evalToIR(info) + if err != nil { + return nil, err + } + + switch t := oprs.(type) { + case irNamePath: + lit, isLit := leftIR.(valueIRAtom) + if !isLit { + return nil, OperandNotANameError(a.Ref.String()) + } + ir = irContains{needle: lit, haystack: t} + case oprIRAtom: + nameIR, isNameIR := leftIR.(irNamePath) + if !isNameIR { + return nil, OperandNotANameError(a.Ref.String()) + } + + ir = irIn{name: nameIR, values: []oprIRAtom{t}} + case multiValueIRAtom: + nameIR, isNameIR := leftIR.(irNamePath) + if !isNameIR { + return nil, OperandNotANameError(a.Ref.String()) + } + + ir = irLiteralValues{name: nameIR, values: t} + default: + return nil, OperandNotAnOperandError{} + } + } + + if a.HasNot { + return &irBoolNot{atom: ir}, nil + } + return ir, nil +} + +func (a *astIn) evalItem(item models.Item) (types.AttributeValue, error) { + val, err := a.Ref.evalItem(item) + if err != nil { + return nil, err + } + if len(a.Operand) == 0 && a.SingleOperand == nil { + return val, nil + } + + switch { + case len(a.Operand) > 0: + for _, opr := range a.Operand { + evalOp, err := opr.evalItem(item) + if err != nil { + return nil, err + } + cmp, isComparable := attrutils.CompareScalarAttributes(val, evalOp) + if !isComparable { + continue + } else if cmp == 0 { + return &types.AttributeValueMemberBOOL{Value: true}, nil + } + } + return &types.AttributeValueMemberBOOL{Value: false}, nil + case a.SingleOperand != nil: + evalOp, err := a.SingleOperand.evalItem(item) + if err != nil { + return nil, err + } + + switch t := evalOp.(type) { + case *types.AttributeValueMemberS: + str, canToStr := attrutils.AttributeToString(val) + if !canToStr { + return &types.AttributeValueMemberBOOL{Value: false}, nil + } + + return &types.AttributeValueMemberBOOL{Value: strings.Contains(t.Value, str)}, nil + case *types.AttributeValueMemberL: + for _, listItem := range t.Value { + cmp, isComparable := attrutils.CompareScalarAttributes(val, listItem) + if !isComparable { + continue + } else if cmp == 0 { + return &types.AttributeValueMemberBOOL{Value: true}, nil + } + } + return &types.AttributeValueMemberBOOL{Value: false}, nil + case *types.AttributeValueMemberSS: + str, canToStr := attrutils.AttributeToString(val) + if !canToStr { + return &types.AttributeValueMemberBOOL{Value: false}, nil + } + + for _, listItem := range t.Value { + if str != listItem { + return &types.AttributeValueMemberBOOL{Value: false}, nil + } + } + return &types.AttributeValueMemberBOOL{Value: true}, nil + case *types.AttributeValueMemberBS: + b, isB := val.(*types.AttributeValueMemberB) + if !isB { + return &types.AttributeValueMemberBOOL{Value: false}, nil + } + + for _, listItem := range t.Value { + if !bytes.Equal(b.Value, listItem) { + return &types.AttributeValueMemberBOOL{Value: false}, nil + } + } + return &types.AttributeValueMemberBOOL{Value: true}, nil + case *types.AttributeValueMemberNS: + n, isN := val.(*types.AttributeValueMemberN) + if !isN { + return &types.AttributeValueMemberBOOL{Value: false}, nil + } + + for _, listItem := range t.Value { + // TODO: this is not actually right + if n.Value != listItem { + return &types.AttributeValueMemberBOOL{Value: false}, nil + } + } + return &types.AttributeValueMemberBOOL{Value: true}, nil + case *types.AttributeValueMemberM: + str, canToStr := attrutils.AttributeToString(val) + if !canToStr { + return &types.AttributeValueMemberBOOL{Value: false}, nil + } + _, hasItem := t.Value[str] + return &types.AttributeValueMemberBOOL{Value: hasItem}, nil + } + return nil, ValuesNotInnableError{Val: evalOp} + } + return nil, errors.New("internal error: unhandled 'in' case") +} + +func (a *astIn) String() string { + if len(a.Operand) == 0 && a.SingleOperand == nil { + return a.Ref.String() + } + + var sb strings.Builder + + sb.WriteString(a.Ref.String()) + if a.HasNot { + sb.WriteString(" not in ") + } else { + sb.WriteString(" in ") + } + + switch { + case len(a.Operand) > 0: + sb.WriteString("(") + for i, o := range a.Operand { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(o.String()) + } + sb.WriteString(")") + case a.SingleOperand != nil: + sb.WriteString(a.SingleOperand.String()) + } + + return sb.String() +} + +type irIn struct { + name nameIRAtom + values []oprIRAtom +} + +func (i irIn) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + right := i.values[0].calcOperand(info) + others := sliceutils.Map(i.values[1:], func(x oprIRAtom) expression.OperandBuilder { + return x.calcOperand(info) + }) + + return i.name.calcName(info).In(right, others...), nil +} + +type irLiteralValues struct { + name nameIRAtom + values multiValueIRAtom +} + +func (i irLiteralValues) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + vals, err := i.values.calcGoValues(info) + if err != nil { + return expression.ConditionBuilder{}, err + } + + oprValues := sliceutils.Map(vals, func(t any) expression.OperandBuilder { + return expression.Value(t) + }) + return i.name.calcName(info).In(oprValues[0], oprValues[1:]...), nil +} + +type irContains struct { + needle valueIRAtom + haystack nameIRAtom +} + +func (i irContains) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + needle := i.needle.goValue() + haystack := i.haystack.calcName(info) + + return haystack.Contains(fmt.Sprint(needle)), nil +} diff --git a/internal/dynamo-browse/models/queryexpr/ir.go b/internal/dynamo-browse/models/queryexpr/ir.go index b42550f..335314e 100644 --- a/internal/dynamo-browse/models/queryexpr/ir.go +++ b/internal/dynamo-browse/models/queryexpr/ir.go @@ -5,18 +5,48 @@ import ( "github.com/lmika/audax/internal/dynamo-browse/models" ) +// TO DELETE = operandFieldName() string + type irAtom interface { - // operandFieldName returns the field that this atom operates on. For example, - // if this IR node represents 'a = "b"', this should return "a". - // If this does not operate on a definitive field name, this returns null - operandFieldName() string + // calcQueryForScan returns the condition builder for this atom to include in a scan + calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) +} + +type queryableIRAtom interface { + irAtom // canBeExecutedAsQuery returns true if the atom is capable of being executed as a query canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool // calcQueryForQuery returns a key condition builder for this atom to include in a query calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) - - // calcQueryForScan returns the condition builder for this atom to include in a scan - calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) +} + +type oprIRAtom interface { + calcOperand(info *models.TableInfo) expression.OperandBuilder +} + +type nameIRAtom interface { + oprIRAtom + + // keyName returns the name as key if it can be a DB key. Returns "" if this name cannot be a key + keyName() string + calcName(info *models.TableInfo) expression.NameBuilder +} + +type valueIRAtom interface { + oprIRAtom + goValue() any +} + +type multiValueIRAtom interface { + calcGoValues(info *models.TableInfo) ([]any, error) +} + +func canExecuteAsQuery(ir irAtom, info *models.TableInfo, qci *queryCalcInfo) bool { + queryable, isQuearyable := ir.(queryableIRAtom) + if !isQuearyable { + return false + } + return queryable.canBeExecutedAsQuery(info, qci) } diff --git a/internal/dynamo-browse/models/queryexpr/is.go b/internal/dynamo-browse/models/queryexpr/is.go new file mode 100644 index 0000000..b4f0ba5 --- /dev/null +++ b/internal/dynamo-browse/models/queryexpr/is.go @@ -0,0 +1,172 @@ +package queryexpr + +import ( + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/lmika/audax/internal/dynamo-browse/models" + "github.com/lmika/audax/internal/dynamo-browse/models/attrutils" + "reflect" + "strings" +) + +type isTypeInfo struct { + isAny bool + attributeType expression.DynamoDBAttributeType + goType reflect.Type +} + +var validIsTypeNames = map[string]isTypeInfo{ + "ANY": {isAny: true}, + "B": { + attributeType: expression.Binary, + goType: reflect.TypeOf(&types.AttributeValueMemberB{}), + }, + "BOOL": { + attributeType: expression.Boolean, + goType: reflect.TypeOf(&types.AttributeValueMemberBOOL{}), + }, + "S": { + attributeType: expression.String, + goType: reflect.TypeOf(&types.AttributeValueMemberS{}), + }, + "N": { + attributeType: expression.Number, + goType: reflect.TypeOf(&types.AttributeValueMemberN{}), + }, + "NULL": { + attributeType: expression.Null, + goType: reflect.TypeOf(&types.AttributeValueMemberNULL{}), + }, + "L": { + attributeType: expression.List, + goType: reflect.TypeOf(&types.AttributeValueMemberL{}), + }, + "M": { + attributeType: expression.Map, + goType: reflect.TypeOf(&types.AttributeValueMemberM{}), + }, + "BS": { + attributeType: expression.BinarySet, + goType: reflect.TypeOf(&types.AttributeValueMemberBS{}), + }, + "NS": { + attributeType: expression.NumberSet, + goType: reflect.TypeOf(&types.AttributeValueMemberNS{}), + }, + "SS": { + attributeType: expression.StringSet, + goType: reflect.TypeOf(&types.AttributeValueMemberSS{}), + }, +} + +func (a *astIsOp) evalToIR(info *models.TableInfo) (irAtom, error) { + leftIR, err := a.Ref.evalToIR(info) + if err != nil { + return nil, err + } + + if a.Value == nil { + return leftIR, nil + } + + nameIR, isNameIR := leftIR.(irNamePath) + if !isNameIR { + return nil, OperandNotANameError(a.Ref.String()) + } + + rightIR, err := a.Value.evalToIR(info) + if err != nil { + return nil, err + } + + valueIR, isValueIR := rightIR.(irValue) + if !isValueIR { + return nil, ValueMustBeLiteralError{} + } + strValue, isStringValue := valueIR.goValue().(string) + if !isStringValue { + return nil, ValueMustBeStringError{} + } + + typeInfo, isValidType := validIsTypeNames[strings.ToUpper(strValue)] + if !isValidType { + return nil, InvalidTypeForIsError{TypeName: strValue} + } + + var ir = irIs{name: nameIR, typeInfo: typeInfo} + if a.HasNot { + if typeInfo.isAny { + ir.hasNot = true + } else { + return &irBoolNot{atom: ir}, nil + } + } + return ir, nil +} + +func (a *astIsOp) evalItem(item models.Item) (types.AttributeValue, error) { + ref, err := a.Ref.evalItem(item) + if err != nil { + return nil, err + } + + if a.Value == nil { + return ref, nil + } + + expTypeVal, err := a.Value.evalItem(item) + if err != nil { + return nil, err + } + str, canToStr := attrutils.AttributeToString(expTypeVal) + if !canToStr { + return nil, ValueMustBeStringError{} + } + typeInfo, hasTypeInfo := validIsTypeNames[strings.ToUpper(str)] + if !hasTypeInfo { + return nil, InvalidTypeForIsError{TypeName: str} + } + + var resultOfIs bool + if typeInfo.isAny { + resultOfIs = ref != nil + } else { + refType := reflect.TypeOf(ref) + resultOfIs = typeInfo.goType.AssignableTo(refType) + } + if a.HasNot { + resultOfIs = !resultOfIs + } + return &types.AttributeValueMemberBOOL{Value: resultOfIs}, nil +} + +func (a *astIsOp) String() string { + var sb strings.Builder + + sb.WriteString(a.Ref.String()) + if a.Value != nil { + sb.WriteString(" is ") + if a.HasNot { + sb.WriteString("not ") + } + sb.WriteString(a.Value.String()) + } + return sb.String() +} + +type irIs struct { + name nameIRAtom + hasNot bool + typeInfo isTypeInfo +} + +func (i irIs) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + nb := i.name.calcName(info) + if i.typeInfo.isAny { + if i.hasNot { + return expression.AttributeNotExists(nb), nil + } + return expression.AttributeExists(nb), nil + } + return expression.AttributeType(nb, i.typeInfo.attributeType), nil +} diff --git a/internal/dynamo-browse/models/queryexpr/types.go b/internal/dynamo-browse/models/queryexpr/types.go new file mode 100644 index 0000000..7175665 --- /dev/null +++ b/internal/dynamo-browse/models/queryexpr/types.go @@ -0,0 +1 @@ +package queryexpr diff --git a/internal/dynamo-browse/models/queryexpr/values.go b/internal/dynamo-browse/models/queryexpr/values.go index 61cfd36..7ef60da 100644 --- a/internal/dynamo-browse/models/queryexpr/values.go +++ b/internal/dynamo-browse/models/queryexpr/values.go @@ -1,12 +1,22 @@ package queryexpr import ( + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" + "github.com/lmika/audax/internal/dynamo-browse/models" "strconv" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/pkg/errors" ) +func (a *astLiteralValue) evalToIR(info *models.TableInfo) (irAtom, error) { + v, err := a.goValue() + if err != nil { + return nil, err + } + return irValue{value: v}, nil +} + func (a *astLiteralValue) dynamoValue() (types.AttributeValue, error) { if a == nil { return nil, nil @@ -58,3 +68,19 @@ func (a *astLiteralValue) String() string { } return "" } + +type irValue struct { + value any +} + +func (i irValue) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) { + return expression.ConditionBuilder{}, NodeCannotBeConvertedToQueryError{} +} + +func (i irValue) goValue() any { + return i.value +} + +func (a irValue) calcOperand(info *models.TableInfo) expression.OperandBuilder { + return expression.Value(a.goValue()) +} diff --git a/internal/dynamo-browse/services/tables/service.go b/internal/dynamo-browse/services/tables/service.go index a87d660..1c8c090 100644 --- a/internal/dynamo-browse/services/tables/service.go +++ b/internal/dynamo-browse/services/tables/service.go @@ -64,7 +64,10 @@ func (s *Service) doScan(ctx context.Context, tableInfo *models.TableInfo, expr } if err != nil && len(results) == 0 { - return nil, errors.Wrapf(err, "unable to scan table %v", tableInfo.Name) + return &models.ResultSet{ + TableInfo: tableInfo, + Query: expr, + }, errors.Wrapf(err, "unable to scan table %v", tableInfo.Name) } models.Sort(results, tableInfo) diff --git a/internal/dynamo-browse/ui/teamodels/colselector/tblmodel.go b/internal/dynamo-browse/ui/teamodels/colselector/tblmodel.go index 0884428..505d2e7 100644 --- a/internal/dynamo-browse/ui/teamodels/colselector/tblmodel.go +++ b/internal/dynamo-browse/ui/teamodels/colselector/tblmodel.go @@ -24,7 +24,7 @@ func (clr colListRowModel) Render(w io.Writer, model table.Model, index int) { col := clr.m.colController.Columns().Columns[index] if !col.Hidden { - fmt.Fprintln(w, style.Render(fmt.Sprintf(".\t%v", col.Name))) + fmt.Fprintln(w, style.Render(fmt.Sprintf("⋅\t%v", col.Name))) } else { fmt.Fprintln(w, style.Render(fmt.Sprintf("✕\t%v", col.Name))) }