lmika/dynamo-browse
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Added a few changes to query expressions (#51)

Browse source 
- Added the between operator to query expressions.
- Added the using query expression suffix to specify which index to query (or force a scan). This is required if query planning has found multiple indices that can potentially be used.
- Rewrote the types of the query expressions to allow for functions to be defined once, and be useful in queries that result in DynamoDB queries, and evaluation.
- Added some test functions around time and summing numbers.
- Fixed a bug in the del-attr which was not honouring marked rows in a similar way to set-attr: it was only deleting attributes from the first row.
- Added the -to type flag to set-attr which will set the attribute to the value of a query expression.
This commit is contained in:
Leon Mika 2023-04-14 15:35:43 +10:00 committed by GitHub
parent 835ddd5630
commit 4b4d515ade
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 1284 additions and 339 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
internal/common/maputils/map.go
View file

@ -8,6 +8,16 @@ func Values[K comparable, T any](ts map[K]T) []T {
return values
}
func MapValues[K comparable, T, U any](ts map[K]T, fn func(t T) U) map[K]U {
us := make(map[K]U)
for k, t := range ts {
us[k] = fn(t)
}
return us
}
func MapValuesWithError[K comparable, T, U any](ts map[K]T, fn func(t T) (U, error)) (map[K]U, error) {
us := make(map[K]U)

19
internal/common/sliceutils/map.go
View file

@ -39,3 +39,22 @@ func Filter[T any](ts []T, fn func(t T) bool) []T {
}
return us
}
func FindFirst[T any](ts []T, fn func(t T) bool) (returnedT T, found bool) {
for _, t := range ts {
if fn(t) {
return t, true
}
}
return returnedT, false
}
func FindLast[T any](ts []T, fn func(t T) bool) (returnedT T, found bool) {
for i := len(ts) - 1; i >= 0; i-- {
t := ts[i]
if fn(t) {
return t, true
}
}
return returnedT, false
}

3
internal/dynamo-browse/controllers/scripts.go
View file

@ -182,6 +182,9 @@ func (s *sessionImpl) Query(ctx context.Context, query string, opts scriptmanage
if opts.ValuePlaceholders != nil {
expr = expr.WithValueParams(opts.ValuePlaceholders)
}
if opts.IndexName != "" {
expr = expr.WithIndex(opts.IndexName)
}
// Get the table info
var tableInfo *models.TableInfo

46
internal/dynamo-browse/controllers/tablewrite.go
View file

@ -122,6 +122,8 @@ func (twc *TableWriteController) SetAttributeValue(idx int, itemType models.Item
return twc.setBoolValue(idx, path)
case models.NullItemType:
return twc.setNullValue(idx, path)
case models.ExprValueItemType:
return twc.setToExpressionValue(idx, path)
default:
return events.Error(errors.New("unsupported attribute type"))
}
@ -151,6 +153,39 @@ func (twc *TableWriteController) setStringValue(idx int, attr *queryexpr.QueryEx
}
}
func (twc *TableWriteController) setToExpressionValue(idx int, attr *queryexpr.QueryExpr) tea.Msg {
return events.PromptForInputMsg{
Prompt: "expr value: ",
OnDone: func(value string) tea.Msg {
valueExpr, err := queryexpr.Parse(value)
if err != nil {
return events.Error(err)
}
if err := twc.state.withResultSetReturningError(func(set *models.ResultSet) error {
if err := applyToMarkedItems(set, idx, func(idx int, item models.Item) error {
newValue, err := valueExpr.EvalItem(item)
if err != nil {
return err
}
if err := attr.SetEvalItem(item, newValue); err != nil {
return err
}
set.SetDirty(idx, true)
return nil
}); err != nil {
return err
}
set.RefreshColumns()
return nil
}); err != nil {
return events.Error(err)
}
return ResultSetUpdated{}
},
}
}
func (twc *TableWriteController) setNumberValue(idx int, attr *queryexpr.QueryExpr) tea.Msg {
return events.PromptForInputMsg{
Prompt: "number value: ",
@ -239,12 +274,17 @@ func (twc *TableWriteController) DeleteAttribute(idx int, key string) tea.Msg {
}
if err := twc.state.withResultSetReturningError(func(set *models.ResultSet) error {
err := path.DeleteAttribute(set.Items()[idx])
if err != nil {
if err := applyToMarkedItems(set, idx, func(idx int, item models.Item) error {
if err := path.DeleteAttribute(set.Items()[idx]); err != nil {
return err
}
set.SetDirty(idx, true)
return nil
}); err != nil {
return err
}
set.SetDirty(idx, true)
set.RefreshColumns()
return nil
}); err != nil {

2
internal/dynamo-browse/models/itemtype.go
View file

@ -8,4 +8,6 @@ const (
NumberItemType ItemType = "N"
BoolItemType ItemType = "BOOL"
NullItemType ItemType = "NULL"
ExprValueItemType ItemType = "exprvalue"
)

89
internal/dynamo-browse/models/queryexpr/ast.go
View file

@ -4,17 +4,23 @@ import (
"github.com/alecthomas/participle/v2"
"github.com/alecthomas/participle/v2/lexer"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/common/sliceutils"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/pkg/errors"
"strconv"
)
// Modelled on the expression language here
// https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html
type astExpr struct {
Root *astDisjunction `parser:"@@"`
Root *astDisjunction `parser:"@@"`
Options *astOptions `parser:"( 'using' @@ )?"`
}
type astOptions struct {
Scan bool `parser:"@'scan'"`
Index string `parser:" | 'index' '(' @String ')'"`
}
type astDisjunction struct {
@ -38,9 +44,15 @@ type astIn struct {
}
type astComparisonOp struct {
Ref *astEqualityOp `parser:"@@"`
Op string `parser:"( @('<' | '<=' | '>' | '>=')"`
Value *astEqualityOp `parser:"@@ )?"`
Ref *astBetweenOp `parser:"@@"`
Op string `parser:"( @('<' | '<=' | '>' | '>=')"`
Value *astBetweenOp `parser:"@@ )?"`
}
type astBetweenOp struct {
Ref *astEqualityOp `parser:"@@"`
From *astEqualityOp `parser:"( 'between' @@ "`
To *astEqualityOp `parser:" 'and' @@ )?"`
}
type astEqualityOp struct {
@ -58,7 +70,6 @@ type astIsOp struct {
type astSubRef struct {
Ref *astFunctionCall `parser:"@@"`
SubRefs []*astSubRefType `parser:"@@*"`
//Quals []string `parser:"('.' @Ident)*"`
}
type astSubRefType struct {
@ -121,7 +132,58 @@ func Parse(expr string) (*QueryExpr, error) {
return &QueryExpr{ast: ast}, nil
}
func (a *astExpr) calcQuery(ctx *evalContext, info *models.TableInfo) (*models.QueryExecutionPlan, error) {
func (a *astExpr) calcQuery(ctx *evalContext, info *models.TableInfo, preferredIndex string) (*models.QueryExecutionPlan, error) {
plans, err := a.determinePlausibleExecutionPlans(ctx, info)
if err != nil {
return nil, err
}
scanPlan, _ := sliceutils.FindLast(plans, func(p *models.QueryExecutionPlan) bool {
return !p.CanQuery
})
queryPlans := sliceutils.Filter(plans, func(p *models.QueryExecutionPlan) bool {
if !p.CanQuery {
return false
}
return true
})
if len(queryPlans) == 0 || (a.Options != nil && a.Options.Scan) {
if preferredIndex != "" {
return nil, NoPlausiblePlanWithIndexError{
PreferredIndex: preferredIndex,
PossibleIndices: sliceutils.Map(queryPlans, func(p *models.QueryExecutionPlan) string { return p.IndexName }),
}
}
return scanPlan, nil
}
if preferredIndex == "" && (a.Options != nil && a.Options.Index != "") {
preferredIndex, err = strconv.Unquote(a.Options.Index)
if err != nil {
return nil, err
}
}
if preferredIndex != "" {
queryPlans = sliceutils.Filter(queryPlans, func(p *models.QueryExecutionPlan) bool { return p.IndexName == preferredIndex })
}
if len(queryPlans) == 1 {
return queryPlans[0], nil
} else if len(queryPlans) == 0 {
return nil, NoPlausiblePlanWithIndexError{
PreferredIndex: preferredIndex,
}
}
return nil, MultiplePlansWithIndexError{
PossibleIndices: sliceutils.Map(queryPlans, func(p *models.QueryExecutionPlan) string { return p.IndexName }),
}
}
func (a *astExpr) determinePlausibleExecutionPlans(ctx *evalContext, info *models.TableInfo) ([]*models.QueryExecutionPlan, error) {
plans := make([]*models.QueryExecutionPlan, 0)
type queryTestAttempt struct {
index string
keysUnderTest models.KeyAttribute
@ -153,11 +215,11 @@ func (a *astExpr) calcQuery(ctx *evalContext, info *models.TableInfo) (*models.Q
return nil, err
}
return &models.QueryExecutionPlan{
plans = append(plans, &models.QueryExecutionPlan{
CanQuery: true,
IndexName: attempt.index,
Expression: expr,
}, nil
})
}
}
@ -174,21 +236,22 @@ func (a *astExpr) calcQuery(ctx *evalContext, info *models.TableInfo) (*models.Q
return nil, err
}
return &models.QueryExecutionPlan{
plans = append(plans, &models.QueryExecutionPlan{
CanQuery: false,
Expression: expr,
}, nil
})
return plans, nil
}
func (a *astExpr) evalToIR(ctx *evalContext, tableInfo *models.TableInfo) (irAtom, error) {
return a.Root.evalToIR(ctx, tableInfo)
}
func (a *astExpr) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (a *astExpr) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
return a.Root.evalItem(ctx, item)
}
func (a *astExpr) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (a *astExpr) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
return a.Root.setEvalItem(ctx, item, value)
}

16
internal/dynamo-browse/models/queryexpr/atom.go
View file

@ -1,7 +1,6 @@
package queryexpr
import (
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/pkg/errors"
)
@ -21,15 +20,6 @@ func (a *astAtom) evalToIR(ctx *evalContext, info *models.TableInfo) (irAtom, er
return nil, errors.New("unhandled atom case")
}
func (a *astAtom) rightOperandDynamoValue() (types.AttributeValue, error) {
switch {
case a.Literal != nil:
return a.Literal.dynamoValue()
}
return nil, errors.New("unhandled atom case")
}
func (a *astAtom) unqualifiedName() (string, bool) {
switch {
case a.Ref != nil:
@ -39,12 +29,12 @@ func (a *astAtom) unqualifiedName() (string, bool) {
return "", false
}
func (a *astAtom) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (a *astAtom) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
switch {
case a.Ref != nil:
return a.Ref.evalItem(ctx, item)
case a.Literal != nil:
return a.Literal.dynamoValue()
return a.Literal.exprValue()
case a.Placeholder != nil:
return a.Placeholder.evalItem(ctx, item)
case a.Paren != nil:
@ -66,7 +56,7 @@ func (a *astAtom) canModifyItem(ctx *evalContext, item models.Item) bool {
return false
}
func (a *astAtom) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (a *astAtom) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
switch {
case a.Ref != nil:
return a.Ref.setEvalItem(ctx, item, value)

155
internal/dynamo-browse/models/queryexpr/between.go Normal file
View file

@ -0,0 +1,155 @@
package queryexpr
import (
"fmt"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/lmika/audax/internal/dynamo-browse/models"
)
func (a *astBetweenOp) evalToIR(ctx *evalContext, info *models.TableInfo) (irAtom, error) {
leftIR, err := a.Ref.evalToIR(ctx, info)
if err != nil {
return nil, err
}
if a.From == nil {
return leftIR, nil
}
nameIR, isNameIR := leftIR.(nameIRAtom)
if !isNameIR {
return nil, OperandNotANameError(a.Ref.String())
}
fromIR, err := a.From.evalToIR(ctx, info)
if err != nil {
return nil, err
}
toIR, err := a.To.evalToIR(ctx, info)
if err != nil {
return nil, err
}
fromOprIR, isFromOprIR := fromIR.(valueIRAtom)
if !isFromOprIR {
return nil, OperandNotAnOperandError{}
}
toOprIR, isToOprIR := toIR.(valueIRAtom)
if !isToOprIR {
return nil, OperandNotAnOperandError{}
}
return irBetween{name: nameIR, from: fromOprIR, to: toOprIR}, nil
}
func (a *astBetweenOp) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
val, err := a.Ref.evalItem(ctx, item)
if a.From == nil {
return val, err
}
fromIR, err := a.From.evalItem(ctx, item)
if err != nil {
return nil, err
}
toIR, err := a.To.evalItem(ctx, item)
if err != nil {
return nil, err
}
switch v := val.(type) {
case stringableExprValue:
fromNumVal, isFromNumVal := fromIR.(stringableExprValue)
if !isFromNumVal {
return nil, ValuesNotComparable{Left: val.asAttributeValue(), Right: fromIR.asAttributeValue()}
}
toNumVal, isToNumVal := toIR.(stringableExprValue)
if !isToNumVal {
return nil, ValuesNotComparable{Left: val.asAttributeValue(), Right: toNumVal.asAttributeValue()}
}
return boolExprValue(v.asString() >= fromNumVal.asString() && v.asString() <= toNumVal.asString()), nil
case numberableExprValue:
fromNumVal, isFromNumVal := fromIR.(numberableExprValue)
if !isFromNumVal {
return nil, ValuesNotComparable{Left: val.asAttributeValue(), Right: fromIR.asAttributeValue()}
}
toNumVal, isToNumVal := toIR.(numberableExprValue)
if !isToNumVal {
return nil, ValuesNotComparable{Left: val.asAttributeValue(), Right: toNumVal.asAttributeValue()}
}
fromCmp := v.asBigFloat().Cmp(fromNumVal.asBigFloat())
toCmp := v.asBigFloat().Cmp(toNumVal.asBigFloat())
return boolExprValue(fromCmp >= 0 && toCmp <= 0), nil
}
return nil, InvalidTypeForBetweenError{TypeName: val.typeName()}
}
func (a *astBetweenOp) canModifyItem(ctx *evalContext, item models.Item) bool {
if a.From != nil {
return false
}
return a.Ref.canModifyItem(ctx, item)
}
func (a *astBetweenOp) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
if a.From != nil {
return PathNotSettableError{}
}
return a.Ref.setEvalItem(ctx, item, value)
}
func (a *astBetweenOp) deleteAttribute(ctx *evalContext, item models.Item) error {
if a.From != nil {
return PathNotSettableError{}
}
return a.Ref.deleteAttribute(ctx, item)
}
func (a *astBetweenOp) String() string {
name := a.Ref.String()
if a.From != nil {
return fmt.Sprintf("%v between %v and %v", name, a.From.String(), a.To.String())
}
return name
}
type irBetween struct {
name nameIRAtom
from valueIRAtom
to valueIRAtom
}
func (i irBetween) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
nb := i.name.calcName(info)
fb := i.from.calcOperand(info)
tb := i.to.calcOperand(info)
return nb.Between(fb, tb), nil
}
func (i irBetween) canBeExecutedAsQuery(qci *queryCalcInfo) bool {
keyName := i.name.keyName()
if keyName == "" {
return false
}
if keyName == qci.keysUnderTest.SortKey {
return qci.addKey(keyName)
}
return false
}
func (i irBetween) calcQueryForQuery() (expression.KeyConditionBuilder, error) {
nb := i.name.keyName()
fb := i.from.exprValue()
tb := i.to.exprValue()
return expression.Key(nb).Between(buildExpressionFromValue(fb), buildExpressionFromValue(tb)), nil
}

7
internal/dynamo-browse/models/queryexpr/boolnot.go
View file

@ -2,7 +2,6 @@ package queryexpr
import (
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models"
"strings"
)
@ -20,7 +19,7 @@ func (a *astBooleanNot) evalToIR(ctx *evalContext, tableInfo *models.TableInfo)
return &irBoolNot{atom: irNode}, nil
}
func (a *astBooleanNot) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (a *astBooleanNot) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
val, err := a.Operand.evalItem(ctx, item)
if err != nil {
return nil, err
@ -30,7 +29,7 @@ func (a *astBooleanNot) evalItem(ctx *evalContext, item models.Item) (types.Attr
return val, nil
}
return &types.AttributeValueMemberBOOL{Value: !isAttributeTrue(val)}, nil
return boolExprValue(!isAttributeTrue(val)), nil
}
func (a *astBooleanNot) canModifyItem(ctx *evalContext, item models.Item) bool {
@ -40,7 +39,7 @@ func (a *astBooleanNot) canModifyItem(ctx *evalContext, item models.Item) bool {
return a.Operand.canModifyItem(ctx, item)
}
func (a *astBooleanNot) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (a *astBooleanNot) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
if a.HasNot {
return PathNotSettableError{}
}

90
internal/dynamo-browse/models/queryexpr/builtins.go
View file

@ -2,38 +2,90 @@ package queryexpr
import (
"context"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/pkg/errors"
"strconv"
)
type nativeFunc func(ctx context.Context, args []types.AttributeValue) (types.AttributeValue, error)
type nativeFunc func(ctx context.Context, args []exprValue) (exprValue, error)
var nativeFuncs = map[string]nativeFunc{
"size": func(ctx context.Context, args []types.AttributeValue) (types.AttributeValue, error) {
"size": func(ctx context.Context, args []exprValue) (exprValue, error) {
if len(args) != 1 {
return nil, InvalidArgumentNumberError{Name: "size", Expected: 1, Actual: len(args)}
}
var l int
switch t := args[0].(type) {
case *types.AttributeValueMemberB:
l = len(t.Value)
case *types.AttributeValueMemberS:
l = len(t.Value)
case *types.AttributeValueMemberL:
l = len(t.Value)
case *types.AttributeValueMemberM:
l = len(t.Value)
case *types.AttributeValueMemberSS:
l = len(t.Value)
case *types.AttributeValueMemberNS:
l = len(t.Value)
case *types.AttributeValueMemberBS:
l = len(t.Value)
case stringExprValue:
l = len(t)
case mappableExprValue:
l = t.len()
case slicableExprValue:
l = t.len()
default:
return nil, errors.New("cannot take size of arg")
}
return &types.AttributeValueMemberN{Value: strconv.Itoa(l)}, nil
return int64ExprValue(l), nil
},
"range": func(ctx context.Context, args []exprValue) (exprValue, error) {
if len(args) != 2 {
return nil, InvalidArgumentNumberError{Name: "range", Expected: 2, Actual: len(args)}
}
xVal, isXNum := args[0].(numberableExprValue)
if !isXNum {
return nil, InvalidArgumentTypeError{Name: "range", ArgIndex: 0, Expected: "N"}
}
yVal, isYNum := args[1].(numberableExprValue)
if !isYNum {
return nil, InvalidArgumentTypeError{Name: "range", ArgIndex: 1, Expected: "N"}
}
xInt, _ := xVal.asBigFloat().Int64()
yInt, _ := yVal.asBigFloat().Int64()
xs := make([]exprValue, 0)
for x := xInt; x <= yInt; x++ {
xs = append(xs, int64ExprValue(x))
}
return listExprValue(xs), nil
},
"_x_now": func(ctx context.Context, args []exprValue) (exprValue, error) {
now := timeSourceFromContext(ctx).now().Unix()
return int64ExprValue(now), nil
},
"_x_add": func(ctx context.Context, args []exprValue) (exprValue, error) {
if len(args) != 2 {
return nil, InvalidArgumentNumberError{Name: "_x_add", Expected: 2, Actual: len(args)}
}
xVal, isXNum := args[0].(numberableExprValue)
if !isXNum {
return nil, InvalidArgumentTypeError{Name: "_x_add", ArgIndex: 0, Expected: "N"}
}
yVal, isYNum := args[1].(numberableExprValue)
if !isYNum {
return nil, InvalidArgumentTypeError{Name: "_x_add", ArgIndex: 1, Expected: "N"}
}
return bigNumExprValue{num: xVal.asBigFloat().Add(xVal.asBigFloat(), yVal.asBigFloat())}, nil
},
"_x_concat": func(ctx context.Context, args []exprValue) (exprValue, error) {
if len(args) != 2 {
return nil, InvalidArgumentNumberError{Name: "_x_concat", Expected: 2, Actual: len(args)}
}
xVal, isXNum := args[0].(stringableExprValue)
if !isXNum {
return nil, InvalidArgumentTypeError{Name: "_x_concat", ArgIndex: 0, Expected: "S"}
}
yVal, isYNum := args[1].(stringableExprValue)
if !isYNum {
return nil, InvalidArgumentTypeError{Name: "_x_concat", ArgIndex: 1, Expected: "S"}
}
return stringExprValue(xVal.asString() + yVal.asString()), nil
},
}

38
internal/dynamo-browse/models/queryexpr/comp.go
View file

@ -2,7 +2,6 @@ package queryexpr
import (
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/lmika/audax/internal/dynamo-browse/models/attrutils"
"github.com/pkg/errors"
@ -47,7 +46,7 @@ func (a *astComparisonOp) evalToIR(ctx *evalContext, info *models.TableInfo) (ir
return irGenericCmp{leftOpr, rightOpr, cmpType}, nil
}
func (a *astComparisonOp) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (a *astComparisonOp) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
left, err := a.Ref.evalItem(ctx, item)
if err != nil {
return nil, err
@ -61,20 +60,21 @@ func (a *astComparisonOp) evalItem(ctx *evalContext, item models.Item) (types.At
return nil, err
}
cmp, isComparable := attrutils.CompareScalarAttributes(left, right)
// TODO: use expr value here
cmp, isComparable := attrutils.CompareScalarAttributes(left.asAttributeValue(), right.asAttributeValue())
if !isComparable {
return nil, ValuesNotComparable{Left: left, Right: right}
return nil, ValuesNotComparable{Left: left.asAttributeValue(), Right: right.asAttributeValue()}
}
switch opToCmdType[a.Op] {
case cmpTypeLt:
return &types.AttributeValueMemberBOOL{Value: cmp < 0}, nil
return boolExprValue(cmp < 0), nil
case cmpTypeLe:
return &types.AttributeValueMemberBOOL{Value: cmp <= 0}, nil
return boolExprValue(cmp <= 0), nil
case cmpTypeGt:
return &types.AttributeValueMemberBOOL{Value: cmp > 0}, nil
return boolExprValue(cmp > 0), nil
case cmpTypeGe:
return &types.AttributeValueMemberBOOL{Value: cmp >= 0}, nil
return boolExprValue(cmp >= 0), nil
}
return nil, errors.Errorf("unrecognised operator: %v", a.Op)
}
@ -86,7 +86,7 @@ func (a *astComparisonOp) canModifyItem(ctx *evalContext, item models.Item) bool
return a.Ref.canModifyItem(ctx, item)
}
func (a *astComparisonOp) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (a *astComparisonOp) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
if a.Op != "" {
return PathNotSettableError{}
}
@ -143,34 +143,34 @@ func (a irKeyFieldCmp) canBeExecutedAsQuery(qci *queryCalcInfo) bool {
func (a irKeyFieldCmp) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
nb := a.name.calcName(info)
vb := a.value.goValue()
vb := a.value.exprValue()
switch a.cmpType {
case cmpTypeLt:
return nb.LessThan(expression.Value(vb)), nil
return nb.LessThan(buildExpressionFromValue(vb)), nil
case cmpTypeLe:
return nb.LessThanEqual(expression.Value(vb)), nil
return nb.LessThanEqual(buildExpressionFromValue(vb)), nil
case cmpTypeGt:
return nb.GreaterThan(expression.Value(vb)), nil
return nb.GreaterThan(buildExpressionFromValue(vb)), nil
case cmpTypeGe:
return nb.GreaterThanEqual(expression.Value(vb)), nil
return nb.GreaterThanEqual(buildExpressionFromValue(vb)), nil
}
return expression.ConditionBuilder{}, errors.New("unsupported cmp type")
}
func (a irKeyFieldCmp) calcQueryForQuery() (expression.KeyConditionBuilder, error) {
keyName := a.name.keyName()
vb := a.value.goValue()
vb := a.value.exprValue()
switch a.cmpType {
case cmpTypeLt:
return expression.Key(keyName).LessThan(expression.Value(vb)), nil
return expression.Key(keyName).LessThan(buildExpressionFromValue(vb)), nil
case cmpTypeLe:
return expression.Key(keyName).LessThanEqual(expression.Value(vb)), nil
return expression.Key(keyName).LessThanEqual(buildExpressionFromValue(vb)), nil
case cmpTypeGt:
return expression.Key(keyName).GreaterThan(expression.Value(vb)), nil
return expression.Key(keyName).GreaterThan(buildExpressionFromValue(vb)), nil
case cmpTypeGe:
return expression.Key(keyName).GreaterThanEqual(expression.Value(vb)), nil
return expression.Key(keyName).GreaterThanEqual(buildExpressionFromValue(vb)), nil
}
return expression.KeyConditionBuilder{}, errors.New("unsupported cmp type")
}

26
internal/dynamo-browse/models/queryexpr/conj.go
View file

@ -2,8 +2,8 @@ package queryexpr
import (
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models"
"math/big"
"strings"
)
@ -36,7 +36,7 @@ func (a *astConjunction) evalToIR(ctx *evalContext, tableInfo *models.TableInfo)
return &irMultiConjunction{atoms: atoms}, nil
}
func (a *astConjunction) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (a *astConjunction) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
val, err := a.Operands[0].evalItem(ctx, item)
if err != nil {
return nil, err
@ -47,7 +47,7 @@ func (a *astConjunction) evalItem(ctx *evalContext, item models.Item) (types.Att
for _, opr := range a.Operands[1:] {
if !isAttributeTrue(val) {
return &types.AttributeValueMemberBOOL{Value: false}, nil
return boolExprValue(false), nil
}
val, err = opr.evalItem(ctx, item)
@ -56,7 +56,7 @@ func (a *astConjunction) evalItem(ctx *evalContext, item models.Item) (types.Att
}
}
return &types.AttributeValueMemberBOOL{Value: isAttributeTrue(val)}, nil
return boolExprValue(isAttributeTrue(val)), nil
}
func (a *astConjunction) canModifyItem(ctx *evalContext, item models.Item) bool {
@ -67,7 +67,7 @@ func (a *astConjunction) canModifyItem(ctx *evalContext, item models.Item) bool
return false
}
func (a *astConjunction) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (a *astConjunction) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
if len(a.Operands) == 1 {
return a.Operands[0].setEvalItem(ctx, item, value)
}
@ -168,16 +168,16 @@ func (d *irMultiConjunction) calcQueryForScan(info *models.TableInfo) (expressio
return conjExpr, nil
}
func isAttributeTrue(attr types.AttributeValue) bool {
func isAttributeTrue(attr exprValue) bool {
switch val := attr.(type) {
case *types.AttributeValueMemberS:
return val.Value != ""
case *types.AttributeValueMemberN:
return val.Value != "0"
case *types.AttributeValueMemberBOOL:
return val.Value
case *types.AttributeValueMemberNULL:
case nullExprValue:
return false
case boolExprValue:
return bool(val)
case stringableExprValue:
return val.asString() != ""
case numberableExprValue:
return val.asBigFloat().Cmp(&big.Float{}) != 0
}
return true
}

27
internal/dynamo-browse/models/queryexpr/context.go Normal file
View file

@ -0,0 +1,27 @@
package queryexpr
import (
"context"
"time"
)
type timeSource interface {
now() time.Time
}
type defaultTimeSource struct{}
func (tds defaultTimeSource) now() time.Time {
return time.Now()
}
type timeSourceContextKeyType struct{}
var timeSourceContextKey = timeSourceContextKeyType{}
func timeSourceFromContext(ctx context.Context) timeSource {
if tts, ok := ctx.Value(timeSourceContextKey).(timeSource); ok {
return tts
}
return defaultTimeSource{}
}

9
internal/dynamo-browse/models/queryexpr/disj.go
View file

@ -2,7 +2,6 @@ package queryexpr
import (
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models"
"strings"
)
@ -24,7 +23,7 @@ func (a *astDisjunction) evalToIR(ctx *evalContext, tableInfo *models.TableInfo)
return &irDisjunction{conj: conj}, nil
}
func (a *astDisjunction) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (a *astDisjunction) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
val, err := a.Operands[0].evalItem(ctx, item)
if err != nil {
return nil, err
@ -35,7 +34,7 @@ func (a *astDisjunction) evalItem(ctx *evalContext, item models.Item) (types.Att
for _, opr := range a.Operands[1:] {
if isAttributeTrue(val) {
return &types.AttributeValueMemberBOOL{Value: true}, nil
return boolExprValue(true), nil
}
val, err = opr.evalItem(ctx, item)
@ -44,7 +43,7 @@ func (a *astDisjunction) evalItem(ctx *evalContext, item models.Item) (types.Att
}
}
return &types.AttributeValueMemberBOOL{Value: isAttributeTrue(val)}, nil
return boolExprValue(isAttributeTrue(val)), nil
}
func (a *astDisjunction) canModifyItem(ctx *evalContext, item models.Item) bool {
@ -55,7 +54,7 @@ func (a *astDisjunction) canModifyItem(ctx *evalContext, item models.Item) bool
return false
}
func (a *astDisjunction) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (a *astDisjunction) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
if len(a.Operands) == 1 {
return a.Operands[0].setEvalItem(ctx, item, value)
}

11
internal/dynamo-browse/models/queryexpr/dot.go
View file

@ -3,7 +3,6 @@ package queryexpr
import (
"fmt"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models"
"strings"
)
@ -16,21 +15,21 @@ func (dt *astRef) unqualifiedName() (string, bool) {
return dt.Name, true
}
func (dt *astRef) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (dt *astRef) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
res, hasV := item[dt.Name]
if !hasV {
return nil, nil
}
return res, nil
return newExprValueFromAttributeValue(res)
}
func (dt *astRef) canModifyItem(ctx *evalContext, item models.Item) bool {
return true
}
func (dt *astRef) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
item[dt.Name] = value
func (dt *astRef) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
item[dt.Name] = value.asAttributeValue()
return nil
}
@ -71,7 +70,7 @@ func (i irNamePath) calcName(info *models.TableInfo) expression.NameBuilder {
switch v := qual.(type) {
case string:
fullName.WriteString("." + v)
case int:
case int64:
fullName.WriteString(fmt.Sprintf("[%v]", qual))
}
}

43
internal/dynamo-browse/models/queryexpr/equality.go
View file

@ -2,7 +2,6 @@ package queryexpr
import (
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/lmika/audax/internal/dynamo-browse/models/attrutils"
"github.com/pkg/errors"
@ -59,7 +58,7 @@ func (a *astEqualityOp) evalToIR(ctx *evalContext, info *models.TableInfo) (irAt
return nil, errors.Errorf("unrecognised operator: %v", a.Op)
}
func (a *astEqualityOp) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (a *astEqualityOp) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
left, err := a.Ref.evalItem(ctx, item)
if err != nil {
return nil, err
@ -74,30 +73,32 @@ func (a *astEqualityOp) evalItem(ctx *evalContext, item models.Item) (types.Attr
return nil, err
}
// TODO: use expr values here
switch a.Op {
case "=":
cmp, isComparable := attrutils.CompareScalarAttributes(left, right)
cmp, isComparable := attrutils.CompareScalarAttributes(left.asAttributeValue(), right.asAttributeValue())
if !isComparable {
return nil, ValuesNotComparable{Left: left, Right: right}
return nil, ValuesNotComparable{Left: left.asAttributeValue(), Right: right.asAttributeValue()}
}
return &types.AttributeValueMemberBOOL{Value: cmp == 0}, nil
return boolExprValue(cmp == 0), nil
case "!=":
cmp, isComparable := attrutils.CompareScalarAttributes(left, right)
cmp, isComparable := attrutils.CompareScalarAttributes(left.asAttributeValue(), right.asAttributeValue())
if !isComparable {
return nil, ValuesNotComparable{Left: left, Right: right}
return nil, ValuesNotComparable{Left: left.asAttributeValue(), Right: right.asAttributeValue()}
}
return &types.AttributeValueMemberBOOL{Value: cmp != 0}, nil
return boolExprValue(cmp != 0), nil
case "^=":
strValue, isStrValue := right.(*types.AttributeValueMemberS)
strValue, isStrValue := right.(stringableExprValue)
if !isStrValue {
return nil, errors.New("operand '^=' must be string")
}
leftAsStr, canBeString := attrutils.AttributeToString(left)
leftAsStr, canBeString := left.(stringableExprValue)
if !canBeString {
return nil, ValueNotConvertableToString{Val: left}
return nil, ValueNotConvertableToString{Val: leftAsStr.asAttributeValue()}
}
return &types.AttributeValueMemberBOOL{Value: strings.HasPrefix(leftAsStr, strValue.Value)}, nil
return boolExprValue(strings.HasPrefix(leftAsStr.asString(), strValue.asString())), nil
}
return nil, errors.Errorf("unrecognised operator: %v", a.Op)
@ -110,7 +111,7 @@ func (a *astEqualityOp) canModifyItem(ctx *evalContext, item models.Item) bool {
return a.Ref.canModifyItem(ctx, item)
}
func (a *astEqualityOp) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (a *astEqualityOp) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
if a.Op != "" {
return PathNotSettableError{}
}
@ -157,8 +158,8 @@ func (a irKeyFieldEq) calcQueryForScan(info *models.TableInfo) (expression.Condi
}
func (a irKeyFieldEq) calcQueryForQuery() (expression.KeyConditionBuilder, error) {
vb := a.value.goValue()
return expression.Key(a.name.keyName()).Equal(expression.Value(vb)), nil
vb := a.value.exprValue()
return expression.Key(a.name.keyName()).Equal(buildExpressionFromValue(vb)), nil
}
type irGenericEq struct {
@ -203,21 +204,21 @@ func (a irFieldBeginsWith) canBeExecutedAsQuery(qci *queryCalcInfo) bool {
func (a irFieldBeginsWith) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
nb := a.name.calcName(info)
vb := a.value.goValue()
strValue, isStrValue := vb.(string)
vb := a.value.exprValue()
strValue, isStrValue := vb.(stringableExprValue)
if !isStrValue {
return expression.ConditionBuilder{}, errors.New("operand '^=' must be string")
}
return nb.BeginsWith(strValue), nil
return nb.BeginsWith(strValue.asString()), nil
}
func (a irFieldBeginsWith) calcQueryForQuery() (expression.KeyConditionBuilder, error) {
vb := a.value.goValue()
strValue, isStrValue := vb.(string)
vb := a.value.exprValue()
strValue, isStrValue := vb.(stringableExprValue)
if !isStrValue {
return expression.KeyConditionBuilder{}, errors.New("operand '^=' must be string")
}
return expression.Key(a.name.keyName()).BeginsWith(strValue), nil
return expression.Key(a.name.keyName()).BeginsWith(strValue.asString()), nil
}

35
internal/dynamo-browse/models/queryexpr/errors.go
View file

@ -98,6 +98,14 @@ func (n InvalidTypeForIsError) Error() string {
return "invalid type for 'is': " + n.TypeName
}
type InvalidTypeForBetweenError struct {
TypeName string
}
func (n InvalidTypeForBetweenError) Error() string {
return "invalid type for 'between': " + n.TypeName
}
type InvalidArgumentNumberError struct {
Name string
Expected int
@ -108,6 +116,16 @@ func (e InvalidArgumentNumberError) Error() string {
return fmt.Sprintf("function '%v' expected %v args but received %v", e.Name, e.Expected, e.Actual)
}
type InvalidArgumentTypeError struct {
Name string
ArgIndex int
Expected string
}
func (e InvalidArgumentTypeError) Error() string {
return fmt.Sprintf("function '%v' expected arg %v to be of type %v", e.Name, e.ArgIndex, e.Expected)
}
type UnrecognisedFunctionError struct {
Name string
}
@ -137,3 +155,20 @@ type ValueNotUsableAsASubref struct {
func (e ValueNotUsableAsASubref) Error() string {
return "value cannot be used as a subref"
}
type MultiplePlansWithIndexError struct {
PossibleIndices []string
}
func (e MultiplePlansWithIndexError) Error() string {
return fmt.Sprintf("multiple plans with index found. Specify index or scan with 'using' clause: possible indices are %v", e.PossibleIndices)
}
type NoPlausiblePlanWithIndexError struct {
PreferredIndex string
PossibleIndices []string
}
func (e NoPlausiblePlanWithIndexError) Error() string {
return fmt.Sprintf("no plan with index '%v' found: possible indices are %v", e.PreferredIndex, e.PossibleIndices)
}

46
internal/dynamo-browse/models/queryexpr/expr.go
View file

@ -16,12 +16,17 @@ import (
type QueryExpr struct {
ast *astExpr
index string
names map[string]string
values map[string]types.AttributeValue
// tests fields only
timeSource timeSource
}
type serializedExpr struct {
Expr string
Index string
Names map[string]string
Values []byte
}
@ -39,6 +44,7 @@ func DeserializeFrom(r io.Reader) (*QueryExpr, error) {
}
qe.names = se.Names
qe.index = se.Index
if len(se.Values) > 0 {
vals, err := attrcodec.NewDecoder(bytes.NewReader(se.Values)).Decode()
@ -56,7 +62,7 @@ func DeserializeFrom(r io.Reader) (*QueryExpr, error) {
}
func (md *QueryExpr) SerializeTo(w io.Writer) error {
se := serializedExpr{Expr: md.String(), Names: md.names}
se := serializedExpr{Expr: md.String(), Index: md.index, Names: md.names}
if md.values != nil {
var bts bytes.Buffer
if err := attrcodec.NewEncoder(&bts).Encode(&types.AttributeValueMemberM{Value: md.values}); err != nil {
@ -90,6 +96,7 @@ func (md *QueryExpr) Equal(other *QueryExpr) bool {
}
return md.ast.String() == other.ast.String() &&
md.index == other.index &&
maps.Equal(md.names, other.names) &&
maps.EqualFunc(md.values, md.values, attrutils.Equals)
}
@ -104,6 +111,7 @@ func (md *QueryExpr) HashCode() uint64 {
h := fnv.New64a()
h.Write([]byte(md.ast.String()))
h.Write([]byte(md.index))
// the names must be in sorted order to maintain consistant key ordering
if len(md.names) > 0 {
@ -134,6 +142,7 @@ func (md *QueryExpr) HashCode() uint64 {
func (md *QueryExpr) WithNameParams(value map[string]string) *QueryExpr {
return &QueryExpr{
ast: md.ast,
index: md.index,
names: value,
values: md.values,
}
@ -158,17 +167,34 @@ func (md *QueryExpr) ValueParamOrNil(name string) types.AttributeValue {
func (md *QueryExpr) WithValueParams(value map[string]types.AttributeValue) *QueryExpr {
return &QueryExpr{
ast: md.ast,
index: md.index,
names: md.names,
values: value,
}
}
func (md *QueryExpr) WithIndex(index string) *QueryExpr {
return &QueryExpr{
ast: md.ast,
index: index,
names: md.names,
values: md.values,
}
}
func (md *QueryExpr) Plan(tableInfo *models.TableInfo) (*models.QueryExecutionPlan, error) {
return md.ast.calcQuery(md.evalContext(), tableInfo)
return md.ast.calcQuery(md.evalContext(), tableInfo, md.index)
}
func (md *QueryExpr) EvalItem(item models.Item) (types.AttributeValue, error) {
return md.ast.evalItem(md.evalContext(), item)
val, err := md.ast.evalItem(md.evalContext(), item)
if err != nil {
return nil, err
}
if val == nil {
return nil, nil
}
return val.asAttributeValue(), nil
}
func (md *QueryExpr) DeleteAttribute(item models.Item) error {
@ -176,7 +202,11 @@ func (md *QueryExpr) DeleteAttribute(item models.Item) error {
}
func (md *QueryExpr) SetEvalItem(item models.Item, newValue types.AttributeValue) error {
return md.ast.setEvalItem(md.evalContext(), item, newValue)
val, err := newExprValueFromAttributeValue(newValue)
if err != nil {
return err
}
return md.ast.setEvalItem(md.evalContext(), item, val)
}
func (md *QueryExpr) IsModifiablePath(item models.Item) bool {
@ -237,6 +267,7 @@ type evalContext struct {
nameLookup func(string) (string, bool)
valuePlaceholders map[string]types.AttributeValue
valueLookup func(string) (types.AttributeValue, bool)
timeSource timeSource
}
func (ec *evalContext) lookupName(name string) (string, bool) {
@ -264,3 +295,10 @@ func (ec *evalContext) lookupValue(name string) (types.AttributeValue, bool) {
return nil, false
}
func (ec *evalContext) getTimeSource() timeSource {
if ts := ec.timeSource; ts != nil {
return ts
}
return defaultTimeSource{}
}

114
internal/dynamo-browse/models/queryexpr/expr_test.go
View file

@ -7,6 +7,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models/queryexpr"
"testing"
"time"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/stretchr/testify/assert"
@ -34,6 +35,13 @@ func TestModExpr_Query(t *testing.T) {
SortKey: "sk",
},
},
{
Name: "with-apples-and-oranges",
Keys: models.KeyAttribute{
PartitionKey: "apples",
SortKey: "oranges",
},
},
},
}
@ -112,6 +120,13 @@ func TestModExpr_Query(t *testing.T) {
exprNameIsString(0, 0, "pk", "prefix"),
exprNameIsNumber(1, 1, "sk", "100"),
),
scanCase("when request pk is equals and sk is greater or equal to",
`pk="prefix" and sk between 100 and 200`,
`(#0 = :0) AND (#1 BETWEEN :1 AND :2)`,
exprNameIsString(0, 0, "pk", "prefix"),
exprNameIsNumber(1, 1, "sk", "100"),
exprValueIsNumber(2, "200"),
),
scanCase("with placeholders",
`:partition=$valuePrefix and :sort=$valueAnother`,
@ -149,6 +164,13 @@ func TestModExpr_Query(t *testing.T) {
exprNameIsString(0, 0, "color", "yellow"),
exprNameIsString(1, 1, "shade", "dark"),
),
// Function calls
scanCase("use the value of fn call in query",
`pk = _x_concat("Hello ", "world")`,
`#0 = :0`,
exprNameIsString(0, 0, "pk", "Hello world"),
),
}
for _, scenario := range scenarios {
@ -238,6 +260,13 @@ func TestModExpr_Query(t *testing.T) {
exprNameIsString(0, 0, "pk", "prefix"),
),
scanCase("with between", `pk between "a" and "z"`,
`#0 BETWEEN :0 AND :1`,
exprName(0, "pk"),
exprValueIsString(0, "a"),
exprValueIsString(1, "z"),
),
scanCase("with in", `pk in ("alpha", "bravo", "charlie")`,
`#0 IN (:0, :1, :2)`,
exprName(0, "pk"),
@ -374,6 +403,53 @@ func TestModExpr_Query(t *testing.T) {
})
}
})
t.Run("with index clash", func(t *testing.T) {
t.Run("should return error if attempt to run query with two indices that can be chosen", func(t *testing.T) {
modExpr, err := queryexpr.Parse(`apples="this"`)
assert.NoError(t, err)
_, err = modExpr.Plan(tableInfo)
assert.Error(t, err)
})
t.Run("should run as scan if explicitly forced to", func(t *testing.T) {
modExpr, err := queryexpr.Parse(`apples="this" using scan`)
assert.NoError(t, err)
plan, err := modExpr.Plan(tableInfo)
assert.NoError(t, err)
assert.False(t, plan.CanQuery)
})
t.Run("should run as query with the 'with-apples' index", func(t *testing.T) {
modExpr, err := queryexpr.Parse(`apples="this" using index("with-apples")`)
assert.NoError(t, err)
plan, err := modExpr.Plan(tableInfo)
assert.NoError(t, err)
assert.True(t, plan.CanQuery)
assert.Equal(t, "with-apples", plan.IndexName)
})
t.Run("should run as query with the 'with-apples-and-oranges' index", func(t *testing.T) {
modExpr, err := queryexpr.Parse(`apples="this" using index("with-apples-and-oranges")`)
assert.NoError(t, err)
plan, err := modExpr.Plan(tableInfo)
assert.NoError(t, err)
assert.True(t, plan.CanQuery)
assert.Equal(t, "with-apples-and-oranges", plan.IndexName)
})
t.Run("should return error if the chosen index can't be used", func(t *testing.T) {
modExpr, err := queryexpr.Parse(`apples="this" using index("with-missing")`)
assert.NoError(t, err)
_, err = modExpr.Plan(tableInfo)
assert.Error(t, err)
})
})
}
func TestQueryExpr_EvalItem(t *testing.T) {
@ -395,7 +471,9 @@ func TestQueryExpr_EvalItem(t *testing.T) {
&types.AttributeValueMemberN{Value: "7"},
},
},
"one": &types.AttributeValueMemberN{Value: "1"},
"three": &types.AttributeValueMemberN{Value: "3"},
"five": &types.AttributeValueMemberN{Value: "5"},
}
)
@ -433,6 +511,19 @@ func TestQueryExpr_EvalItem(t *testing.T) {
{expr: "three < 2", expected: &types.AttributeValueMemberBOOL{Value: false}},
{expr: "three <= 2", expected: &types.AttributeValueMemberBOOL{Value: false}},
// Between
{expr: "three between 1 and 5", expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: "three between one and five", expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: "three between 10 and 15", expected: &types.AttributeValueMemberBOOL{Value: false}},
{expr: "three between 1 and 2", expected: &types.AttributeValueMemberBOOL{Value: false}},
{expr: "8 between five and 10", expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: "three between 1 and 3", expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: "three between 3 and 5", expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `"e" between "a" and "z"`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `"eee" between "aaa" and "zzz"`, expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: `"e" between "between" and "beyond"`, expected: &types.AttributeValueMemberBOOL{Value: false}},
// In
{expr: "three in (2, 3, 4, 5)", expected: &types.AttributeValueMemberBOOL{Value: true}},
{expr: "three in (20, 30, 40)", expected: &types.AttributeValueMemberBOOL{Value: false}},
@ -509,6 +600,29 @@ func TestQueryExpr_EvalItem(t *testing.T) {
}
})
t.Run("functions", func(t *testing.T) {
timeNow := time.Now()
scenarios := []struct {
expr string
expected types.AttributeValue
}{
// _x_now() -- unreleased version of now
{expr: `_x_now()`, expected: &types.AttributeValueMemberN{Value: fmt.Sprint(timeNow.Unix())}},
}
for _, scenario := range scenarios {
t.Run(scenario.expr, func(t *testing.T) {
modExpr, err := queryexpr.Parse(scenario.expr)
assert.NoError(t, err)
res, err := modExpr.WithTestTimeSource(timeNow).EvalItem(item)
assert.NoError(t, err)
assert.Equal(t, scenario.expected, res)
})
}
})
t.Run("unparsed expression", func(t *testing.T) {
scenarios := []struct {
expr string

58
internal/dynamo-browse/models/queryexpr/fncall.go
View file

@ -3,7 +3,6 @@ package queryexpr
import (
"context"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/common/sliceutils"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/pkg/errors"
@ -29,7 +28,7 @@ func (a *astFunctionCall) evalToIR(ctx *evalContext, info *models.TableInfo) (ir
return nil, err
}
// TODO: do this properly
// Special handling of functions that have IR nodes
switch nameIr.keyName() {
case "size":
if len(irNodes) != 1 {
@ -40,20 +39,34 @@ func (a *astFunctionCall) evalToIR(ctx *evalContext, info *models.TableInfo) (ir
return nil, OperandNotANameError(a.Args[0].String())
}
return irSizeFn{name}, nil
case "range":
if len(irNodes) != 2 {
return nil, InvalidArgumentNumberError{Name: "range", Expected: 2, Actual: len(irNodes)}
}
// TEMP
fromVal := irNodes[0].(valueIRAtom).goValue().(int64)
toVal := irNodes[1].(valueIRAtom).goValue().(int64)
return irRangeFn{fromVal, toVal}, nil
}
return nil, UnrecognisedFunctionError{Name: nameIr.keyName()}
builtinFn, hasBuiltin := nativeFuncs[nameIr.keyName()]
if !hasBuiltin {
return nil, UnrecognisedFunctionError{Name: nameIr.keyName()}
}
// Normal functions which are evaluated to regular values
irValues, err := sliceutils.MapWithError(irNodes, func(a irAtom) (exprValue, error) {
v, isV := a.(valueIRAtom)
if !isV {
return nil, errors.New("cannot use value")
}
return v.exprValue(), nil
})
if err != nil {
return nil, err
}
val, err := builtinFn(context.Background(), irValues)
if err != nil {
return nil, err
}
return irValue{value: val}, nil
}
func (a *astFunctionCall) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (a *astFunctionCall) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
if !a.IsCall {
return a.Caller.evalItem(ctx, item)
}
@ -67,14 +80,15 @@ func (a *astFunctionCall) evalItem(ctx *evalContext, item models.Item) (types.At
return nil, UnrecognisedFunctionError{Name: name}
}
args, err := sliceutils.MapWithError(a.Args, func(a *astExpr) (types.AttributeValue, error) {
args, err := sliceutils.MapWithError(a.Args, func(a *astExpr) (exprValue, error) {
return a.evalItem(ctx, item)
})
if err != nil {
return nil, err
}
return fn(context.Background(), args)
cCtx := context.WithValue(context.Background(), timeSourceContextKey, ctx.timeSource)
return fn(cCtx, args)
}
func (a *astFunctionCall) canModifyItem(ctx *evalContext, item models.Item) bool {
@ -85,7 +99,7 @@ func (a *astFunctionCall) canModifyItem(ctx *evalContext, item models.Item) bool
return a.Caller.canModifyItem(ctx, item)
}
func (a *astFunctionCall) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (a *astFunctionCall) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
// TODO: Should a function vall return an item?
if a.IsCall {
return PathNotSettableError{}
@ -147,3 +161,15 @@ func (i irRangeFn) calcGoValues(info *models.TableInfo) ([]any, error) {
}
return xs, nil
}
type multiValueFnResult struct {
items []any
}
func (i multiValueFnResult) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
return expression.ConditionBuilder{}, errors.New("cannot run as scan")
}
func (i multiValueFnResult) calcGoValues(info *models.TableInfo) ([]any, error) {
return i.items, nil
}

16
internal/dynamo-browse/models/queryexpr/helpers_test.go Normal file
View file

@ -0,0 +1,16 @@
package queryexpr
import (
"time"
)
type testTimeSource time.Time
func (tds testTimeSource) now() time.Time {
return time.Time(tds)
}
func (a *QueryExpr) WithTestTimeSource(timeNow time.Time) *QueryExpr {
a.timeSource = testTimeSource(timeNow)
return a
}

147
internal/dynamo-browse/models/queryexpr/in.go
View file

@ -1,10 +1,7 @@
package queryexpr
import (
"bytes"
"fmt"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/common/sliceutils"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/lmika/audax/internal/dynamo-browse/models/attrutils"
@ -71,6 +68,13 @@ func (a *astIn) evalToIR(ctx *evalContext, info *models.TableInfo) (irAtom, erro
return nil, OperandNotANameError(a.Ref.String())
}
ir = irContains{needle: lit, haystack: t}
case valueIRAtom:
nameIR, isNameIR := leftIR.(irNamePath)
if !isNameIR {
return nil, OperandNotANameError(a.Ref.String())
}
ir = irLiteralValues{name: nameIR, values: t}
case oprIRAtom:
nameIR, isNameIR := leftIR.(irNamePath)
if !isNameIR {
@ -78,13 +82,6 @@ func (a *astIn) evalToIR(ctx *evalContext, info *models.TableInfo) (irAtom, erro
}
ir = irIn{name: nameIR, values: []oprIRAtom{t}}
case multiValueIRAtom:
nameIR, isNameIR := leftIR.(irNamePath)
if !isNameIR {
return nil, OperandNotANameError(a.Ref.String())
}
ir = irLiteralValues{name: nameIR, values: t}
default:
return nil, OperandNotAnOperandError{}
}
@ -96,7 +93,7 @@ func (a *astIn) evalToIR(ctx *evalContext, info *models.TableInfo) (irAtom, erro
return ir, nil
}
func (a *astIn) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (a *astIn) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
val, err := a.Ref.evalItem(ctx, item)
if err != nil {
return nil, err
@ -112,14 +109,15 @@ func (a *astIn) evalItem(ctx *evalContext, item models.Item) (types.AttributeVal
if err != nil {
return nil, err
}
cmp, isComparable := attrutils.CompareScalarAttributes(val, evalOp)
// TODO: use native types here
cmp, isComparable := attrutils.CompareScalarAttributes(val.asAttributeValue(), evalOp.asAttributeValue())
if !isComparable {
continue
} else if cmp == 0 {
return &types.AttributeValueMemberBOOL{Value: true}, nil
return boolExprValue(true), nil
}
}
return &types.AttributeValueMemberBOOL{Value: false}, nil
return boolExprValue(false), nil
case a.SingleOperand != nil:
evalOp, err := a.SingleOperand.evalItem(ctx, item)
if err != nil {
@ -127,69 +125,38 @@ func (a *astIn) evalItem(ctx *evalContext, item models.Item) (types.AttributeVal
}
switch t := evalOp.(type) {
case *types.AttributeValueMemberS:
str, canToStr := attrutils.AttributeToString(val)
case stringableExprValue:
str, canToStr := val.(stringableExprValue)
if !canToStr {
return &types.AttributeValueMemberBOOL{Value: false}, nil
return boolExprValue(false), nil
}
return &types.AttributeValueMemberBOOL{Value: strings.Contains(t.Value, str)}, nil
case *types.AttributeValueMemberL:
for _, listItem := range t.Value {
cmp, isComparable := attrutils.CompareScalarAttributes(val, listItem)
return boolExprValue(strings.Contains(t.asString(), str.asString())), nil
case slicableExprValue:
for i := 0; i < t.len(); i++ {
va, err := t.valueAt(i)
if err != nil {
return nil, err
}
// TODO: use expr value types here
cmp, isComparable := attrutils.CompareScalarAttributes(val.asAttributeValue(), va.asAttributeValue())
if !isComparable {
continue
} else if cmp == 0 {
return &types.AttributeValueMemberBOOL{Value: true}, nil
return boolExprValue(true), nil
}
}
return &types.AttributeValueMemberBOOL{Value: false}, nil
case *types.AttributeValueMemberSS:
str, canToStr := attrutils.AttributeToString(val)
return boolExprValue(false), nil
case mappableExprValue:
str, canToStr := val.(stringableExprValue)
if !canToStr {
return &types.AttributeValueMemberBOOL{Value: false}, nil
return boolExprValue(false), nil
}
for _, listItem := range t.Value {
if str != listItem {
return &types.AttributeValueMemberBOOL{Value: false}, nil
}
}
return &types.AttributeValueMemberBOOL{Value: true}, nil
case *types.AttributeValueMemberBS:
b, isB := val.(*types.AttributeValueMemberB)
if !isB {
return &types.AttributeValueMemberBOOL{Value: false}, nil
}
for _, listItem := range t.Value {
if !bytes.Equal(b.Value, listItem) {
return &types.AttributeValueMemberBOOL{Value: false}, nil
}
}
return &types.AttributeValueMemberBOOL{Value: true}, nil
case *types.AttributeValueMemberNS:
n, isN := val.(*types.AttributeValueMemberN)
if !isN {
return &types.AttributeValueMemberBOOL{Value: false}, nil
}
for _, listItem := range t.Value {
// TODO: this is not actually right
if n.Value != listItem {
return &types.AttributeValueMemberBOOL{Value: false}, nil
}
}
return &types.AttributeValueMemberBOOL{Value: true}, nil
case *types.AttributeValueMemberM:
str, canToStr := attrutils.AttributeToString(val)
if !canToStr {
return &types.AttributeValueMemberBOOL{Value: false}, nil
}
_, hasItem := t.Value[str]
return &types.AttributeValueMemberBOOL{Value: hasItem}, nil
hasKey := t.hasKey(str.asString())
return boolExprValue(hasKey), nil
}
return nil, ValuesNotInnableError{Val: evalOp}
return nil, ValuesNotInnableError{Val: evalOp.asAttributeValue()}
}
return nil, errors.New("internal error: unhandled 'in' case")
}
@ -201,7 +168,7 @@ func (a *astIn) canModifyItem(ctx *evalContext, item models.Item) bool {
return a.Ref.canModifyItem(ctx, item)
}
func (a *astIn) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (a *astIn) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
if len(a.Operand) != 0 || a.SingleOperand != nil {
return PathNotSettableError{}
}
@ -263,19 +230,38 @@ func (i irIn) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuil
type irLiteralValues struct {
name nameIRAtom
values multiValueIRAtom
values valueIRAtom
}
func (i irLiteralValues) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
vals, err := i.values.calcGoValues(info)
if err != nil {
return expression.ConditionBuilder{}, err
func (iv irLiteralValues) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
if sliceable, isSliceable := iv.values.exprValue().(slicableExprValue); isSliceable {
if sliceable.len() == 1 {
va, err := sliceable.valueAt(0)
if err != nil {
return expression.ConditionBuilder{}, err
}
return iv.name.calcName(info).In(buildExpressionFromValue(va)), nil
} else if sliceable.len() == 0 {
// name is not in an empty slice, so this branch always evaluates to false
// TODO: would be better to not even include this branch in some way?
return expression.Equal(expression.Value(false), expression.Value(true)), nil
}
items := make([]expression.OperandBuilder, sliceable.len())
for i := 0; i < sliceable.len(); i++ {
va, err := sliceable.valueAt(i)
if err != nil {
return expression.ConditionBuilder{}, err
}
items[i] = buildExpressionFromValue(va)
}
return iv.name.calcName(info).In(items[0], items[1:]...), nil
}
oprValues := sliceutils.Map(vals, func(t any) expression.OperandBuilder {
return expression.Value(t)
})
return i.name.calcName(info).In(oprValues[0], oprValues[1:]...), nil
return iv.name.calcName(info).In(buildExpressionFromValue(iv.values.exprValue())), nil
}
type irContains struct {
@ -284,8 +270,11 @@ type irContains struct {
}
func (i irContains) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
needle := i.needle.goValue()
haystack := i.haystack.calcName(info)
strNeedle, isString := i.needle.exprValue().(stringableExprValue)
if !isString {
return expression.ConditionBuilder{}, errors.New("value cannot be converted to string")
}
return haystack.Contains(fmt.Sprint(needle)), nil
haystack := i.haystack.calcName(info)
return haystack.Contains(strNeedle.asString()), nil
}

6
internal/dynamo-browse/models/queryexpr/ir.go
View file

@ -36,11 +36,7 @@ type nameIRAtom interface {
type valueIRAtom interface {
oprIRAtom
goValue() any
}
type multiValueIRAtom interface {
calcGoValues(info *models.TableInfo) ([]any, error)
exprValue() exprValue
}
func canExecuteAsQuery(ir irAtom, qci *queryCalcInfo) bool {

50
internal/dynamo-browse/models/queryexpr/is.go
View file

@ -2,9 +2,7 @@ package queryexpr
import (
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/lmika/audax/internal/dynamo-browse/models/attrutils"
"reflect"
"strings"
)
@ -12,50 +10,50 @@ import (
type isTypeInfo struct {
isAny bool
attributeType expression.DynamoDBAttributeType
goType reflect.Type
goTypes []reflect.Type
}
var validIsTypeNames = map[string]isTypeInfo{
"ANY": {isAny: true},
"B": {
attributeType: expression.Binary,
goType: reflect.TypeOf(&types.AttributeValueMemberB{}),
// TODO
},
"BOOL": {
attributeType: expression.Boolean,
goType: reflect.TypeOf(&types.AttributeValueMemberBOOL{}),
goTypes: []reflect.Type{reflect.TypeOf(boolExprValue(false))},
},
"S": {
attributeType: expression.String,
goType: reflect.TypeOf(&types.AttributeValueMemberS{}),
goTypes: []reflect.Type{reflect.TypeOf(stringExprValue(""))},
},
"N": {
attributeType: expression.Number,
goType: reflect.TypeOf(&types.AttributeValueMemberN{}),
goTypes: []reflect.Type{reflect.TypeOf(int64ExprValue(0)), reflect.TypeOf(bigNumExprValue{})},
},
"NULL": {
attributeType: expression.Null,
goType: reflect.TypeOf(&types.AttributeValueMemberNULL{}),
goTypes: []reflect.Type{reflect.TypeOf(nullExprValue{})},
},
"L": {
attributeType: expression.List,
goType: reflect.TypeOf(&types.AttributeValueMemberL{}),
goTypes: []reflect.Type{reflect.TypeOf(listExprValue{}), reflect.TypeOf(listProxyValue{})},
},
"M": {
attributeType: expression.Map,
goType: reflect.TypeOf(&types.AttributeValueMemberM{}),
goTypes: []reflect.Type{reflect.TypeOf(mapExprValue{}), reflect.TypeOf(mapProxyValue{})},
},
"BS": {
attributeType: expression.BinarySet,
goType: reflect.TypeOf(&types.AttributeValueMemberBS{}),
// TODO
},
"NS": {
attributeType: expression.NumberSet,
goType: reflect.TypeOf(&types.AttributeValueMemberNS{}),
// TODO
},
"SS": {
attributeType: expression.StringSet,
goType: reflect.TypeOf(&types.AttributeValueMemberSS{}),
// TODO
},
}
@ -83,14 +81,14 @@ func (a *astIsOp) evalToIR(ctx *evalContext, info *models.TableInfo) (irAtom, er
if !isValueIR {
return nil, ValueMustBeLiteralError{}
}
strValue, isStringValue := valueIR.goValue().(string)
strValue, isStringValue := valueIR.exprValue().(stringableExprValue)
if !isStringValue {
return nil, ValueMustBeStringError{}
}
typeInfo, isValidType := validIsTypeNames[strings.ToUpper(strValue)]
typeInfo, isValidType := validIsTypeNames[strings.ToUpper(strValue.asString())]
if !isValidType {
return nil, InvalidTypeForIsError{TypeName: strValue}
return nil, InvalidTypeForIsError{TypeName: strValue.asString()}
}
var ir = irIs{name: nameIR, typeInfo: typeInfo}
@ -104,7 +102,7 @@ func (a *astIsOp) evalToIR(ctx *evalContext, info *models.TableInfo) (irAtom, er
return ir, nil
}
func (a *astIsOp) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (a *astIsOp) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
ref, err := a.Ref.evalItem(ctx, item)
if err != nil {
return nil, err
@ -118,13 +116,13 @@ func (a *astIsOp) evalItem(ctx *evalContext, item models.Item) (types.AttributeV
if err != nil {
return nil, err
}
str, canToStr := attrutils.AttributeToString(expTypeVal)
str, canToStr := expTypeVal.(stringableExprValue)
if !canToStr {
return nil, ValueMustBeStringError{}
}
typeInfo, hasTypeInfo := validIsTypeNames[strings.ToUpper(str)]
typeInfo, hasTypeInfo := validIsTypeNames[strings.ToUpper(str.asString())]
if !hasTypeInfo {
return nil, InvalidTypeForIsError{TypeName: str}
return nil, InvalidTypeForIsError{TypeName: str.asString()}
}
var resultOfIs bool
@ -132,12 +130,18 @@ func (a *astIsOp) evalItem(ctx *evalContext, item models.Item) (types.AttributeV
resultOfIs = ref != nil
} else {
refType := reflect.TypeOf(ref)
resultOfIs = typeInfo.goType.AssignableTo(refType)
for _, t := range typeInfo.goTypes {
if t.AssignableTo(refType) {
resultOfIs = true
break
}
}
}
if a.HasNot {
resultOfIs = !resultOfIs
}
return &types.AttributeValueMemberBOOL{Value: resultOfIs}, nil
return boolExprValue(resultOfIs), nil
}
func (a *astIsOp) canModifyItem(ctx *evalContext, item models.Item) bool {
@ -147,7 +151,7 @@ func (a *astIsOp) canModifyItem(ctx *evalContext, item models.Item) bool {
return a.Ref.canModifyItem(ctx, item)
}
func (a *astIsOp) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (a *astIsOp) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
if a.Value != nil {
return PathNotSettableError{}
}

18
internal/dynamo-browse/models/queryexpr/placeholder.go
View file

@ -1,7 +1,6 @@
package queryexpr
import (
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/dynamo-browse/models"
"github.com/pkg/errors"
)
@ -21,7 +20,12 @@ func (p *astPlaceholder) evalToIR(ctx *evalContext, info *models.TableInfo) (irA
return nil, MissingPlaceholderError{Placeholder: p.Placeholder}
}
return irValue{value: val}, nil
ev, err := newExprValueFromAttributeValue(val)
if err != nil {
return nil, err
}
return irValue{value: ev}, nil
} else if placeholderType == namePlaceholderPrefix {
name, hasName := ctx.lookupName(placeholder)
if !hasName {
@ -34,7 +38,7 @@ func (p *astPlaceholder) evalToIR(ctx *evalContext, info *models.TableInfo) (irA
return nil, errors.New("unrecognised placeholder")
}
func (p *astPlaceholder) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (p *astPlaceholder) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
placeholderType := p.Placeholder[0]
placeholder := p.Placeholder[1:]
@ -43,7 +47,7 @@ func (p *astPlaceholder) evalItem(ctx *evalContext, item models.Item) (types.Att
if !hasVal {
return nil, MissingPlaceholderError{Placeholder: p.Placeholder}
}
return val, nil
return newExprValueFromAttributeValue(val)
} else if placeholderType == namePlaceholderPrefix {
name, hasName := ctx.lookupName(placeholder)
if !hasName {
@ -55,7 +59,7 @@ func (p *astPlaceholder) evalItem(ctx *evalContext, item models.Item) (types.Att
return nil, nil
}
return res, nil
return newExprValueFromAttributeValue(res)
}
return nil, errors.New("unrecognised placeholder")
@ -66,7 +70,7 @@ func (p *astPlaceholder) canModifyItem(ctx *evalContext, item models.Item) bool
return placeholderType == namePlaceholderPrefix
}
func (p *astPlaceholder) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (p *astPlaceholder) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
placeholderType := p.Placeholder[0]
placeholder := p.Placeholder[1:]
@ -78,7 +82,7 @@ func (p *astPlaceholder) setEvalItem(ctx *evalContext, item models.Item, value t
return MissingPlaceholderError{Placeholder: p.Placeholder}
}
item[name] = value
item[name] = value.asAttributeValue()
return nil
}

86
internal/dynamo-browse/models/queryexpr/subref.go
View file

@ -1,10 +1,8 @@
package queryexpr
import (
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/common/sliceutils"
"github.com/lmika/audax/internal/dynamo-browse/models"
"strconv"
"strings"
)
@ -34,7 +32,7 @@ func (r *astSubRef) evalToIR(ctx *evalContext, info *models.TableInfo) (irAtom,
return irNamePath{name: namePath.name, quals: quals}, nil
}
func (r *astSubRef) evalItem(ctx *evalContext, item models.Item) (types.AttributeValue, error) {
func (r *astSubRef) evalItem(ctx *evalContext, item models.Item) (exprValue, error) {
res, err := r.Ref.evalItem(ctx, item)
if err != nil {
return nil, err
@ -48,7 +46,7 @@ func (r *astSubRef) evalItem(ctx *evalContext, item models.Item) (types.Attribut
return res, nil
}
func (r *astSubRef) evalSubRefs(ctx *evalContext, item models.Item, res types.AttributeValue, subRefs []*astSubRefType) (types.AttributeValue, error) {
func (r *astSubRef) evalSubRefs(ctx *evalContext, item models.Item, res exprValue, subRefs []*astSubRefType) (exprValue, error) {
for i, sr := range subRefs {
sv, err := sr.evalToStrOrInt(ctx, nil)
if err != nil {
@ -57,24 +55,30 @@ func (r *astSubRef) evalSubRefs(ctx *evalContext, item models.Item, res types.At
switch val := sv.(type) {
case string:
var hasV bool
mapRes, isMapRes := res.(*types.AttributeValueMemberM)
mapRes, isMapRes := res.(mappableExprValue)
if !isMapRes {
return nil, newValueNotAMapError(r, subRefs[:i+1])
}
res, hasV = mapRes.Value[val]
if !hasV {
return nil, nil
if mapRes.hasKey(val) {
res, err = mapRes.valueOf(val)
if err != nil {
return nil, err
}
} else {
res = nil
}
case int:
listRes, isMapRes := res.(*types.AttributeValueMemberL)
case int64:
listRes, isMapRes := res.(slicableExprValue)
if !isMapRes {
return nil, newValueNotAListError(r, subRefs[:i+1])
}
// TODO - deal with index properly
res = listRes.Value[val]
// TODO - deal with index properly (i.e. error handling)
res, err = listRes.valueAt(int(val))
if err != nil {
return nil, err
}
}
}
return res, nil
@ -84,7 +88,7 @@ func (r *astSubRef) canModifyItem(ctx *evalContext, item models.Item) bool {
return r.Ref.canModifyItem(ctx, item)
}
func (r *astSubRef) setEvalItem(ctx *evalContext, item models.Item, value types.AttributeValue) error {
func (r *astSubRef) setEvalItem(ctx *evalContext, item models.Item, value exprValue) error {
if len(r.SubRefs) == 0 {
return r.Ref.setEvalItem(ctx, item, value)
}
@ -108,20 +112,19 @@ func (r *astSubRef) setEvalItem(ctx *evalContext, item models.Item, value types.
switch val := sv.(type) {
case string:
mapRes, isMapRes := parentItem.(*types.AttributeValueMemberM)
mapRes, isMapRes := parentItem.(modifiableMapExprValue)
if !isMapRes {
return newValueNotAMapError(r, r.SubRefs)
}
mapRes.Value[val] = value
case int:
listRes, isMapRes := parentItem.(*types.AttributeValueMemberL)
mapRes.setValueOf(val, value)
case int64:
listRes, isMapRes := parentItem.(modifiableSliceExprValue)
if !isMapRes {
return newValueNotAListError(r, r.SubRefs)
}
// TODO: handle indexes
listRes.Value[val] = value
listRes.setValueAt(int(val), value)
}
return nil
}
@ -136,20 +139,6 @@ func (r *astSubRef) deleteAttribute(ctx *evalContext, item models.Item) error {
return err
}
/*
for i, key := range r.Quals {
mapItem, isMapItem := parentItem.(*types.AttributeValueMemberM)
if !isMapItem {
return PathNotSettableError{}
}
if isLast := i == len(r.Quals)-1; isLast {
delete(mapItem.Value, key)
} else {
parentItem = mapItem.Value[key]
}
}
*/
if len(r.SubRefs) > 1 {
parentItem, err = r.evalSubRefs(ctx, item, parentItem, r.SubRefs[0:len(r.SubRefs)-1])
if err != nil {
@ -164,23 +153,20 @@ func (r *astSubRef) deleteAttribute(ctx *evalContext, item models.Item) error {
switch val := sv.(type) {
case string:
mapRes, isMapRes := parentItem.(*types.AttributeValueMemberM)
mapRes, isMapRes := parentItem.(modifiableMapExprValue)
if !isMapRes {
return newValueNotAMapError(r, r.SubRefs)
}
delete(mapRes.Value, val)
case int:
listRes, isMapRes := parentItem.(*types.AttributeValueMemberL)
mapRes.deleteValueOf(val)
case int64:
listRes, isMapRes := parentItem.(modifiableSliceExprValue)
if !isMapRes {
return newValueNotAListError(r, r.SubRefs)
}
// TODO: handle indexes out of bounds
oldList := listRes.Value
newList := append([]types.AttributeValue{}, oldList[:val]...)
newList = append(newList, oldList[val+1:]...)
listRes.Value = newList
listRes.deleteValueAt(int(val))
}
return nil
}
@ -214,18 +200,10 @@ func (sr *astSubRefType) evalToStrOrInt(ctx *evalContext, item models.Item) (any
return nil, err
}
switch v := subEvalItem.(type) {
case *types.AttributeValueMemberS:
return v.Value, nil
case *types.AttributeValueMemberN:
intVal, err := strconv.Atoi(v.Value)
if err == nil {
return intVal, nil
}
flVal, err := strconv.ParseFloat(v.Value, 64)
if err == nil {
return int(flVal), nil
}
return nil, err
case stringableExprValue:
return v.asString(), nil
case numberableExprValue:
return v.asInt(), nil
}
return nil, ValueNotUsableAsASubref{}
}

399
internal/dynamo-browse/models/queryexpr/types.go
View file

@ -1 +1,400 @@
package queryexpr
import (
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/lmika/audax/internal/common/maputils"
"github.com/lmika/audax/internal/common/sliceutils"
"github.com/pkg/errors"
"math/big"
"strconv"
)
type exprValue interface {
typeName() string
asGoValue() any
asAttributeValue() types.AttributeValue
}
type stringableExprValue interface {
exprValue
asString() string
}
type numberableExprValue interface {
exprValue
asBigFloat() *big.Float
asInt() int64
}
type slicableExprValue interface {
exprValue
len() int
valueAt(idx int) (exprValue, error)
}
type modifiableSliceExprValue interface {
setValueAt(idx int, value exprValue)
deleteValueAt(idx int)
}
type mappableExprValue interface {
len() int
hasKey(name string) bool
valueOf(name string) (exprValue, error)
}
type modifiableMapExprValue interface {
setValueOf(name string, value exprValue)
deleteValueOf(name string)
}
func buildExpressionFromValue(ev exprValue) expression.ValueBuilder {
return expression.Value(ev)
}
func newExprValueFromAttributeValue(ev types.AttributeValue) (exprValue, error) {
if ev == nil {
return nil, nil
}
switch xVal := ev.(type) {
case *types.AttributeValueMemberS:
return stringExprValue(xVal.Value), nil
case *types.AttributeValueMemberN:
xNumVal, _, err := big.ParseFloat(xVal.Value, 10, 63, big.ToNearestEven)
if err != nil {
return nil, err
}
return bigNumExprValue{num: xNumVal}, nil
case *types.AttributeValueMemberBOOL:
return boolExprValue(xVal.Value), nil
case *types.AttributeValueMemberNULL:
return nullExprValue{}, nil
case *types.AttributeValueMemberL:
return listProxyValue{list: xVal}, nil
case *types.AttributeValueMemberM:
return mapProxyValue{mapValue: xVal}, nil
case *types.AttributeValueMemberSS:
return stringSetProxyValue{stringSet: xVal}, nil
case *types.AttributeValueMemberNS:
return numberSetProxyValue{numberSet: xVal}, nil
}
return nil, errors.New("cannot convert to expr value")
}
type stringExprValue string
func (s stringExprValue) asGoValue() any {
return string(s)
}
func (s stringExprValue) asAttributeValue() types.AttributeValue {
return &types.AttributeValueMemberS{Value: string(s)}
}
func (s stringExprValue) asString() string {
return string(s)
}
func (s stringExprValue) typeName() string {
return "S"
}
type int64ExprValue int64
func (i int64ExprValue) asGoValue() any {
return int(i)
}
func (i int64ExprValue) asAttributeValue() types.AttributeValue {
return &types.AttributeValueMemberN{Value: strconv.Itoa(int(i))}
}
func (i int64ExprValue) asInt() int64 {
return int64(i)
}
func (i int64ExprValue) asBigFloat() *big.Float {
var f big.Float
f.SetInt64(int64(i))
return &f
}
func (s int64ExprValue) typeName() string {
return "N"
}
type bigNumExprValue struct {
num *big.Float
}
func (i bigNumExprValue) asGoValue() any {
return i.num
}
func (i bigNumExprValue) asAttributeValue() types.AttributeValue {
return &types.AttributeValueMemberN{Value: i.num.String()}
}
func (i bigNumExprValue) asInt() int64 {
x, _ := i.num.Int64()
return x
}
func (i bigNumExprValue) asBigFloat() *big.Float {
return i.num
}
func (s bigNumExprValue) typeName() string {
return "N"
}
type boolExprValue bool
func (b boolExprValue) asGoValue() any {
return bool(b)
}
func (b boolExprValue) asAttributeValue() types.AttributeValue {
return &types.AttributeValueMemberBOOL{Value: bool(b)}
}
func (s boolExprValue) typeName() string {
return "BOOL"
}
type nullExprValue struct{}
func (b nullExprValue) asGoValue() any {
return nil
}
func (b nullExprValue) asAttributeValue() types.AttributeValue {
return &types.AttributeValueMemberNULL{Value: true}
}
func (s nullExprValue) typeName() string {
return "NULL"
}
type listExprValue []exprValue
func (bs listExprValue) asGoValue() any {
return sliceutils.Map(bs, func(t exprValue) any {
return t.asGoValue()
})
}
func (bs listExprValue) asAttributeValue() types.AttributeValue {
return &types.AttributeValueMemberL{Value: sliceutils.Map(bs, func(t exprValue) types.AttributeValue {
return t.asAttributeValue()
})}
}
func (bs listExprValue) len() int {
return len(bs)
}
func (bs listExprValue) valueAt(i int) (exprValue, error) {
return bs[i], nil
}
func (s listExprValue) typeName() string {
return "L"
}
type mapExprValue map[string]exprValue
func (bs mapExprValue) asGoValue() any {
return maputils.MapValues(bs, func(t exprValue) any {
return t.asGoValue()
})
}
func (bs mapExprValue) asAttributeValue() types.AttributeValue {
return &types.AttributeValueMemberM{Value: maputils.MapValues(bs, func(t exprValue) types.AttributeValue {
return t.asAttributeValue()
})}
}
func (bs mapExprValue) len() int {
return len(bs)
}
func (bs mapExprValue) hasKey(name string) bool {
_, ok := bs[name]
return ok
}
func (bs mapExprValue) valueOf(name string) (exprValue, error) {
return bs[name], nil
}
func (s mapExprValue) typeName() string {
return "M"
}
type listProxyValue struct {
list *types.AttributeValueMemberL
}
func (bs listProxyValue) asGoValue() any {
resultingList := make([]any, len(bs.list.Value))
for i, item := range bs.list.Value {
if av, _ := newExprValueFromAttributeValue(item); av != nil {
resultingList[i] = av.asGoValue()
} else {
resultingList[i] = nil
}
}
return resultingList
}
func (bs listProxyValue) asAttributeValue() types.AttributeValue {
return bs.list
}
func (bs listProxyValue) len() int {
return len(bs.list.Value)
}
func (bs listProxyValue) valueAt(i int) (exprValue, error) {
return newExprValueFromAttributeValue(bs.list.Value[i])
}
func (bs listProxyValue) setValueAt(i int, newVal exprValue) {
bs.list.Value[i] = newVal.asAttributeValue()
}
func (bs listProxyValue) deleteValueAt(idx int) {
newList := append([]types.AttributeValue{}, bs.list.Value[:idx]...)
newList = append(newList, bs.list.Value[idx+1:]...)
bs.list = &types.AttributeValueMemberL{Value: newList}
}
func (s listProxyValue) typeName() string {
return "L"
}
type mapProxyValue struct {
mapValue *types.AttributeValueMemberM
}
func (bs mapProxyValue) asGoValue() any {
resultingMap := make(map[string]any)
for k, item := range bs.mapValue.Value {
if av, _ := newExprValueFromAttributeValue(item); av != nil {
resultingMap[k] = av.asGoValue()
} else {
resultingMap[k] = nil
}
}
return resultingMap
}
func (bs mapProxyValue) asAttributeValue() types.AttributeValue {
return bs.mapValue
}
func (bs mapProxyValue) len() int {
return len(bs.mapValue.Value)
}
func (bs mapProxyValue) hasKey(name string) bool {
_, ok := bs.mapValue.Value[name]
return ok
}
func (bs mapProxyValue) valueOf(name string) (exprValue, error) {
return newExprValueFromAttributeValue(bs.mapValue.Value[name])
}
func (bs mapProxyValue) setValueOf(name string, newVal exprValue) {
bs.mapValue.Value[name] = newVal.asAttributeValue()
}
func (bs mapProxyValue) deleteValueOf(name string) {
delete(bs.mapValue.Value, name)
}
func (s mapProxyValue) typeName() string {
return "M"
}
type stringSetProxyValue struct {
stringSet *types.AttributeValueMemberSS
}
func (bs stringSetProxyValue) asGoValue() any {
return bs.stringSet.Value
}
func (bs stringSetProxyValue) asAttributeValue() types.AttributeValue {
return bs.stringSet
}
func (bs stringSetProxyValue) len() int {
return len(bs.stringSet.Value)
}
func (bs stringSetProxyValue) valueAt(i int) (exprValue, error) {
return stringExprValue(bs.stringSet.Value[i]), nil
}
func (bs stringSetProxyValue) setValueAt(i int, newVal exprValue) {
if str, isStr := newVal.(stringableExprValue); isStr {
bs.stringSet.Value[i] = str.asString()
}
}
func (bs stringSetProxyValue) deleteValueAt(idx int) {
newList := append([]string{}, bs.stringSet.Value[:idx]...)
newList = append(newList, bs.stringSet.Value[idx+1:]...)
bs.stringSet = &types.AttributeValueMemberSS{Value: newList}
}
func (s stringSetProxyValue) typeName() string {
return "SS"
}
type numberSetProxyValue struct {
numberSet *types.AttributeValueMemberNS
}
func (bs numberSetProxyValue) asGoValue() any {
return bs.numberSet.Value
}
func (bs numberSetProxyValue) asAttributeValue() types.AttributeValue {
return bs.numberSet
}
func (bs numberSetProxyValue) len() int {
return len(bs.numberSet.Value)
}
func (bs numberSetProxyValue) valueAt(i int) (exprValue, error) {
fs, _, err := big.ParseFloat(bs.numberSet.Value[i], 10, 63, big.ToNearestEven)
if err != nil {
return nil, err
}
return bigNumExprValue{fs}, nil
}
func (bs numberSetProxyValue) setValueAt(i int, newVal exprValue) {
if str, isStr := newVal.(numberableExprValue); isStr {
bs.numberSet.Value[i] = str.asBigFloat().String()
}
}
func (bs numberSetProxyValue) deleteValueAt(idx int) {
newList := append([]string{}, bs.numberSet.Value[:idx]...)
newList = append(newList, bs.numberSet.Value[idx+1:]...)
bs.numberSet = &types.AttributeValueMemberNS{Value: newList}
}
func (s numberSetProxyValue) typeName() string {
return "NS"
}

43
internal/dynamo-browse/models/queryexpr/values.go
View file

@ -5,56 +5,31 @@ import (
"github.com/lmika/audax/internal/dynamo-browse/models"
"strconv"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/pkg/errors"
)
func (a *astLiteralValue) evalToIR(ctx *evalContext, info *models.TableInfo) (irAtom, error) {
v, err := a.goValue()
v, err := a.exprValue()
if err != nil {
return nil, err
}
return irValue{value: v}, nil
}
func (a *astLiteralValue) dynamoValue() (types.AttributeValue, error) {
if a == nil {
return nil, nil
}
goValue, err := a.goValue()
if err != nil {
return nil, err
}
switch v := goValue.(type) {
case string:
return &types.AttributeValueMemberS{Value: v}, nil
case int64:
return &types.AttributeValueMemberN{Value: strconv.FormatInt(v, 10)}, nil
}
return nil, errors.New("unrecognised type")
}
func (a *astLiteralValue) goValue() (any, error) {
if a == nil {
return nil, nil
}
func (a *astLiteralValue) exprValue() (exprValue, error) {
switch {
case a.StringVal != nil:
s, err := strconv.Unquote(*a.StringVal)
if err != nil {
return nil, errors.Wrap(err, "cannot unquote string")
}
return s, nil
return stringExprValue(s), nil
case a.IntVal != nil:
return *a.IntVal, nil
return int64ExprValue(*a.IntVal), nil
case a.TrueBoolValue:
return true, nil
return boolExprValue(true), nil
case a.FalseBoolValue:
return false, nil
return boolExprValue(false), nil
}
return nil, errors.New("unrecognised type")
}
@ -78,17 +53,17 @@ func (a *astLiteralValue) String() string {
}
type irValue struct {
value any
value exprValue
}
func (i irValue) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
return expression.ConditionBuilder{}, NodeCannotBeConvertedToQueryError{}
}
func (i irValue) goValue() any {
func (i irValue) exprValue() exprValue {
return i.value
}
func (a irValue) calcOperand(info *models.TableInfo) expression.OperandBuilder {
return expression.Value(a.goValue())
return expression.Value(a.value.asGoValue())
}

1
internal/dynamo-browse/services/scriptmanager/iface.go
View file

@ -32,6 +32,7 @@ type SessionService interface {
type QueryOptions struct {
TableName string
IndexName string
NamePlaceholders map[string]string
ValuePlaceholders map[string]types.AttributeValue
}

5
internal/dynamo-browse/services/scriptmanager/modsession.go
View file

@ -44,6 +44,11 @@ func (um *sessionModule) query(ctx context.Context, args ...object.Object) objec
}
}
// Index name
if val, isStr := objMap.Get("index").(*object.String); isStr {
options.IndexName = val.Value()
}
// Placeholders
if argsVal, isArgsValMap := objMap.Get("args").(*object.Map); isArgsValMap {
options.NamePlaceholders = make(map[string]string)

2
internal/dynamo-browse/ui/model.go
View file

@ -144,6 +144,8 @@ func NewModel(
itemType = models.BoolItemType
case "-NULL":
itemType = models.NullItemType
case "-TO":
itemType = models.ExprValueItemType
default:
return events.Error(errors.New("unrecognised item type"))
}

6
test/cmd/load-test-table/main.go
View file

@ -91,10 +91,14 @@ func main() {
}
}
key := gofakeit.UUID()
var key = gofakeit.UUID()
for i := 0; i < totalItems; i++ {
if i%50 == 0 {
key = gofakeit.UUID()
}
if err := tableService.Put(ctx, inventoryTableInfo, models.Item{
"pk": &types.AttributeValueMemberS{Value: key},
"sk": &types.AttributeValueMemberN{Value: fmt.Sprint(i % 50)},
"uuid": &types.AttributeValueMemberS{Value: gofakeit.UUID()},
}); err != nil {
log.Fatalln(err)