issue-22: Fixed expressions so that queries will be executed as queries (#25)
Augmented expressions so that queries that can be executed as queries on DynamoDB can be done so. Also added an IR tree which is a simplified representation of the AST and will be used to plan the query.
This commit is contained in:
parent
0063d7c6d5
commit
a1717572c5
|
@ -2,11 +2,27 @@ package queryexpr
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/alecthomas/participle/v2"
|
"github.com/alecthomas/participle/v2"
|
||||||
|
"github.com/lmika/audax/internal/dynamo-browse/models"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Modelled on the expression language here
|
||||||
|
// https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html
|
||||||
|
|
||||||
type astExpr struct {
|
type astExpr struct {
|
||||||
Equality *astBinOp `parser:"@@"`
|
Root *astDisjunction `parser:"@@"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *astExpr) evalToIR(tableInfo *models.TableInfo) (*irDisjunction, error) {
|
||||||
|
return a.Root.evalToIR(tableInfo)
|
||||||
|
}
|
||||||
|
|
||||||
|
type astDisjunction struct {
|
||||||
|
Operands []*astConjunction `parser:"@@ ('or' @@)*"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type astConjunction struct {
|
||||||
|
Operands []*astBinOp `parser:"@@ ('and' @@)*"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type astBinOp struct {
|
type astBinOp struct {
|
||||||
|
|
81
internal/dynamo-browse/models/queryexpr/binops.go
Normal file
81
internal/dynamo-browse/models/queryexpr/binops.go
Normal file
|
@ -0,0 +1,81 @@
|
||||||
|
package queryexpr
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
|
||||||
|
"github.com/lmika/audax/internal/dynamo-browse/models"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (a *astBinOp) evalToIR(info *models.TableInfo) (irAtom, error) {
|
||||||
|
v, err := a.Value.goValue()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
switch a.Op {
|
||||||
|
case "=":
|
||||||
|
return irFieldEq{name: a.Name, value: v}, nil
|
||||||
|
case "^=":
|
||||||
|
strValue, isStrValue := v.(string)
|
||||||
|
if !isStrValue {
|
||||||
|
return nil, errors.New("operand '^=' must be string")
|
||||||
|
}
|
||||||
|
return irFieldBeginsWith{name: a.Name, prefix: strValue}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, errors.Errorf("unrecognised operator: %v", a.Op)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *astBinOp) String() string {
|
||||||
|
return a.Name + a.Op + a.Value.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
type irFieldEq struct {
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a irFieldEq) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool {
|
||||||
|
if a.name == info.Keys.PartitionKey || a.name == info.Keys.SortKey {
|
||||||
|
return qci.addKey(info, a.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a irFieldEq) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
|
||||||
|
return expression.Name(a.name).Equal(expression.Value(a.value)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a irFieldEq) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) {
|
||||||
|
return expression.Key(a.name).Equal(expression.Value(a.value)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a irFieldEq) operandFieldName() string {
|
||||||
|
return a.name
|
||||||
|
}
|
||||||
|
|
||||||
|
type irFieldBeginsWith struct {
|
||||||
|
name string
|
||||||
|
prefix string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a irFieldBeginsWith) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool {
|
||||||
|
if a.name == info.Keys.SortKey {
|
||||||
|
return qci.addKey(info, a.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a irFieldBeginsWith) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
|
||||||
|
return expression.Name(a.name).BeginsWith(a.prefix), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a irFieldBeginsWith) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) {
|
||||||
|
return expression.Key(a.name).BeginsWith(a.prefix), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a irFieldBeginsWith) operandFieldName() string {
|
||||||
|
return a.name
|
||||||
|
}
|
|
@ -3,15 +3,30 @@ package queryexpr
|
||||||
import (
|
import (
|
||||||
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
|
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
|
||||||
"github.com/lmika/audax/internal/dynamo-browse/models"
|
"github.com/lmika/audax/internal/dynamo-browse/models"
|
||||||
"github.com/pkg/errors"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a *astExpr) calcQuery(tableInfo *models.TableInfo) (*models.QueryExecutionPlan, error) {
|
func (a *irDisjunction) calcQuery(info *models.TableInfo) (*models.QueryExecutionPlan, error) {
|
||||||
return a.Equality.calcQuery(tableInfo)
|
var qci queryCalcInfo
|
||||||
}
|
if a.canBeExecutedAsQuery(info, &qci) {
|
||||||
|
ke, err := a.calcQueryForQuery(info)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
builder := expression.NewBuilder()
|
||||||
|
builder = builder.WithKeyCondition(ke)
|
||||||
|
|
||||||
|
expr, err := builder.Build()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &models.QueryExecutionPlan{
|
||||||
|
CanQuery: true,
|
||||||
|
Expression: expr,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (a *astBinOp) calcQuery(info *models.TableInfo) (*models.QueryExecutionPlan, error) {
|
|
||||||
// TODO: check if can be a query
|
|
||||||
cb, err := a.calcQueryForScan(info)
|
cb, err := a.calcQueryForScan(info)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -30,23 +45,3 @@ func (a *astBinOp) calcQuery(info *models.TableInfo) (*models.QueryExecutionPlan
|
||||||
Expression: expr,
|
Expression: expr,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *astBinOp) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
|
|
||||||
v, err := a.Value.goValue()
|
|
||||||
if err != nil {
|
|
||||||
return expression.ConditionBuilder{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
switch a.Op {
|
|
||||||
case "=":
|
|
||||||
return expression.Name(a.Name).Equal(expression.Value(v)), nil
|
|
||||||
case "^=":
|
|
||||||
strValue, isStrValue := v.(string)
|
|
||||||
if !isStrValue {
|
|
||||||
return expression.ConditionBuilder{}, errors.New("operand '^=' must be string")
|
|
||||||
}
|
|
||||||
return expression.Name(a.Name).BeginsWith(strValue), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return expression.ConditionBuilder{}, errors.Errorf("unrecognised operator: %v", a.Op)
|
|
||||||
}
|
|
||||||
|
|
89
internal/dynamo-browse/models/queryexpr/conj.go
Normal file
89
internal/dynamo-browse/models/queryexpr/conj.go
Normal file
|
@ -0,0 +1,89 @@
|
||||||
|
package queryexpr
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
|
||||||
|
"github.com/lmika/audax/internal/dynamo-browse/models"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (a *astConjunction) evalToIR(tableInfo *models.TableInfo) (*irConjunction, error) {
|
||||||
|
atoms := make([]irAtom, len(a.Operands))
|
||||||
|
for i, op := range a.Operands {
|
||||||
|
var err error
|
||||||
|
atoms[i], err = op.evalToIR(tableInfo)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return &irConjunction{atoms: atoms}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *astConjunction) String() string {
|
||||||
|
sb := new(strings.Builder)
|
||||||
|
for i, operand := range d.Operands {
|
||||||
|
if i > 0 {
|
||||||
|
sb.WriteString(" and ")
|
||||||
|
}
|
||||||
|
sb.WriteString(operand.String())
|
||||||
|
}
|
||||||
|
return sb.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
type irConjunction struct {
|
||||||
|
atoms []irAtom
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *irConjunction) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool {
|
||||||
|
switch len(d.atoms) {
|
||||||
|
case 1:
|
||||||
|
return d.atoms[0].operandFieldName() == info.Keys.PartitionKey && d.atoms[0].canBeExecutedAsQuery(info, qci)
|
||||||
|
case 2:
|
||||||
|
return d.atoms[0].canBeExecutedAsQuery(info, qci) && d.atoms[1].canBeExecutedAsQuery(info, qci)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *irConjunction) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) {
|
||||||
|
if len(d.atoms) == 1 {
|
||||||
|
return d.atoms[0].calcQueryForQuery(info)
|
||||||
|
} else if len(d.atoms) != 2 {
|
||||||
|
return expression.KeyConditionBuilder{}, errors.Errorf("internal error: expected len to be either 1 or 2, but was %v", len(d.atoms))
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err := d.atoms[0].calcQueryForQuery(info)
|
||||||
|
if err != nil {
|
||||||
|
return expression.KeyConditionBuilder{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
right, err := d.atoms[1].calcQueryForQuery(info)
|
||||||
|
if err != nil {
|
||||||
|
return expression.KeyConditionBuilder{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if d.atoms[0].operandFieldName() == info.Keys.PartitionKey {
|
||||||
|
return expression.KeyAnd(left, right), nil
|
||||||
|
}
|
||||||
|
return expression.KeyAnd(right, left), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *irConjunction) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
|
||||||
|
if len(d.atoms) == 1 {
|
||||||
|
return d.atoms[0].calcQueryForScan(info)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: check if can be query
|
||||||
|
conds := make([]expression.ConditionBuilder, len(d.atoms))
|
||||||
|
for i, operand := range d.atoms {
|
||||||
|
cond, err := operand.calcQueryForScan(info)
|
||||||
|
if err != nil {
|
||||||
|
return expression.ConditionBuilder{}, err
|
||||||
|
}
|
||||||
|
conds[i] = cond
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build conjunction
|
||||||
|
conjExpr := expression.And(conds[0], conds[1], conds[2:]...)
|
||||||
|
return conjExpr, nil
|
||||||
|
}
|
72
internal/dynamo-browse/models/queryexpr/disj.go
Normal file
72
internal/dynamo-browse/models/queryexpr/disj.go
Normal file
|
@ -0,0 +1,72 @@
|
||||||
|
package queryexpr
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
|
||||||
|
"github.com/lmika/audax/internal/dynamo-browse/models"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (a *astDisjunction) evalToIR(tableInfo *models.TableInfo) (*irDisjunction, error) {
|
||||||
|
conj := make([]*irConjunction, len(a.Operands))
|
||||||
|
for i, op := range a.Operands {
|
||||||
|
var err error
|
||||||
|
conj[i], err = op.evalToIR(tableInfo)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return &irDisjunction{conj: conj}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *astDisjunction) String() string {
|
||||||
|
sb := new(strings.Builder)
|
||||||
|
for i, operand := range d.Operands {
|
||||||
|
if i > 0 {
|
||||||
|
sb.WriteString(" or ")
|
||||||
|
}
|
||||||
|
sb.WriteString(operand.String())
|
||||||
|
}
|
||||||
|
return sb.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
type irDisjunction struct {
|
||||||
|
conj []*irConjunction
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *irDisjunction) canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool {
|
||||||
|
// TODO: not entire accurate, as filter expressions are also possible
|
||||||
|
if len(d.conj) == 1 {
|
||||||
|
return d.conj[0].canBeExecutedAsQuery(info, qci)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *irDisjunction) calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error) {
|
||||||
|
if len(d.conj) == 1 {
|
||||||
|
return d.conj[0].calcQueryForQuery(info)
|
||||||
|
}
|
||||||
|
|
||||||
|
return expression.KeyConditionBuilder{}, errors.New("expected exactly 1 operand for query")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *irDisjunction) calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error) {
|
||||||
|
if len(d.conj) == 1 {
|
||||||
|
return d.conj[0].calcQueryForScan(info)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: check if can be query
|
||||||
|
conds := make([]expression.ConditionBuilder, len(d.conj))
|
||||||
|
for i, operand := range d.conj {
|
||||||
|
cond, err := operand.calcQueryForScan(info)
|
||||||
|
if err != nil {
|
||||||
|
return expression.ConditionBuilder{}, err
|
||||||
|
}
|
||||||
|
conds[i] = cond
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build disjunction
|
||||||
|
disjExpr := expression.Or(conds[0], conds[1], conds[2:]...)
|
||||||
|
return disjExpr, nil
|
||||||
|
}
|
|
@ -7,9 +7,38 @@ type QueryExpr struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (md *QueryExpr) Plan(tableInfo *models.TableInfo) (*models.QueryExecutionPlan, error) {
|
func (md *QueryExpr) Plan(tableInfo *models.TableInfo) (*models.QueryExecutionPlan, error) {
|
||||||
return md.ast.calcQuery(tableInfo)
|
ir, err := md.ast.evalToIR(tableInfo)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ir.calcQuery(tableInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (md *QueryExpr) String() string {
|
func (md *QueryExpr) String() string {
|
||||||
return md.ast.String()
|
return md.ast.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a *astExpr) String() string {
|
||||||
|
return a.Root.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
type queryCalcInfo struct {
|
||||||
|
seenKeys map[string]struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (qc *queryCalcInfo) addKey(tableInfo *models.TableInfo, key string) bool {
|
||||||
|
if tableInfo.Keys.PartitionKey != key && tableInfo.Keys.SortKey != key {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if qc.seenKeys == nil {
|
||||||
|
qc.seenKeys = make(map[string]struct{})
|
||||||
|
}
|
||||||
|
if _, hasSeenKey := qc.seenKeys[key]; hasSeenKey {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
qc.seenKeys[key] = struct{}{}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
|
@ -19,29 +19,116 @@ func TestModExpr_Query(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("perform query when request pk is fixed", func(t *testing.T) {
|
t.Run("as queries", func(t *testing.T) {
|
||||||
modExpr, err := queryexpr.Parse(`pk="prefix"`)
|
t.Run("perform query when request pk is fixed", func(t *testing.T) {
|
||||||
assert.NoError(t, err)
|
modExpr, err := queryexpr.Parse(`pk="prefix"`)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
plan, err := modExpr.Plan(tableInfo)
|
plan, err := modExpr.Plan(tableInfo)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
assert.False(t, plan.CanQuery)
|
assert.True(t, plan.CanQuery)
|
||||||
assert.Equal(t, "#0 = :0", aws.ToString(plan.Expression.Filter()))
|
assert.Equal(t, "#0 = :0", aws.ToString(plan.Expression.KeyCondition()))
|
||||||
assert.Equal(t, "pk", plan.Expression.Names()["#0"])
|
assert.Equal(t, "pk", plan.Expression.Names()["#0"])
|
||||||
assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value)
|
assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("perform query when request pk and sk is fixed", func(t *testing.T) {
|
||||||
|
modExpr, err := queryexpr.Parse(`pk="prefix" and sk="another"`)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
plan, err := modExpr.Plan(tableInfo)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
assert.True(t, plan.CanQuery)
|
||||||
|
assert.Equal(t, "(#0 = :0) AND (#1 = :1)", aws.ToString(plan.Expression.KeyCondition()))
|
||||||
|
assert.Equal(t, "pk", plan.Expression.Names()["#0"])
|
||||||
|
assert.Equal(t, "sk", plan.Expression.Names()["#1"])
|
||||||
|
assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value)
|
||||||
|
assert.Equal(t, "another", plan.Expression.Values()[":1"].(*types.AttributeValueMemberS).Value)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("perform query when request pk is equals and sk is prefix", func(t *testing.T) {
|
||||||
|
scenarios := []struct {
|
||||||
|
expr string
|
||||||
|
}{
|
||||||
|
{expr: `pk="prefix" and sk^="another"`},
|
||||||
|
{expr: `sk^="another" and pk="prefix"`},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, scenario := range scenarios {
|
||||||
|
t.Run(scenario.expr, func(t *testing.T) {
|
||||||
|
modExpr, err := queryexpr.Parse(scenario.expr)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
plan, err := modExpr.Plan(tableInfo)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
assert.True(t, plan.CanQuery)
|
||||||
|
assert.Equal(t, "(#0 = :0) AND (begins_with (#1, :1))", aws.ToString(plan.Expression.KeyCondition()))
|
||||||
|
assert.Equal(t, "pk", plan.Expression.Names()["#0"])
|
||||||
|
assert.Equal(t, "sk", plan.Expression.Names()["#1"])
|
||||||
|
assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value)
|
||||||
|
assert.Equal(t, "another", plan.Expression.Values()[":1"].(*types.AttributeValueMemberS).Value)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("perform scan when request pk prefix", func(t *testing.T) {
|
t.Run("as scans", func(t *testing.T) {
|
||||||
modExpr, err := queryexpr.Parse(`pk^="prefix"`) // TODO: fix this so that '^ =' is invalid
|
t.Run("when request pk prefix", func(t *testing.T) {
|
||||||
assert.NoError(t, err)
|
modExpr, err := queryexpr.Parse(`pk^="prefix"`) // TODO: fix this so that '^ =' is invalid
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
plan, err := modExpr.Plan(tableInfo)
|
plan, err := modExpr.Plan(tableInfo)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
assert.False(t, plan.CanQuery)
|
assert.False(t, plan.CanQuery)
|
||||||
assert.Equal(t, "begins_with (#0, :0)", aws.ToString(plan.Expression.Filter()))
|
assert.Equal(t, "begins_with (#0, :0)", aws.ToString(plan.Expression.Filter()))
|
||||||
assert.Equal(t, "pk", plan.Expression.Names()["#0"])
|
assert.Equal(t, "pk", plan.Expression.Names()["#0"])
|
||||||
assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value)
|
assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("when request sk equals something", func(t *testing.T) {
|
||||||
|
modExpr, err := queryexpr.Parse(`sk="something"`)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
plan, err := modExpr.Plan(tableInfo)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
assert.False(t, plan.CanQuery)
|
||||||
|
assert.Equal(t, "#0 = :0", aws.ToString(plan.Expression.Filter()))
|
||||||
|
assert.Equal(t, "sk", plan.Expression.Names()["#0"])
|
||||||
|
assert.Equal(t, "something", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("with disjunctions", func(t *testing.T) {
|
||||||
|
modExpr, err := queryexpr.Parse(`pk="prefix" or sk="another"`)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
plan, err := modExpr.Plan(tableInfo)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
assert.False(t, plan.CanQuery)
|
||||||
|
assert.Equal(t, "(#0 = :0) OR (#1 = :1)", aws.ToString(plan.Expression.Filter()))
|
||||||
|
assert.Equal(t, "pk", plan.Expression.Names()["#0"])
|
||||||
|
assert.Equal(t, "sk", plan.Expression.Names()["#1"])
|
||||||
|
assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value)
|
||||||
|
assert.Equal(t, "another", plan.Expression.Values()[":1"].(*types.AttributeValueMemberS).Value)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("with disjunctions if pk is present twice in expression", func(t *testing.T) {
|
||||||
|
modExpr, err := queryexpr.Parse(`pk="prefix" and pk="another"`)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
plan, err := modExpr.Plan(tableInfo)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
assert.False(t, plan.CanQuery)
|
||||||
|
assert.Equal(t, "(#0 = :0) AND (#0 = :1)", aws.ToString(plan.Expression.Filter()))
|
||||||
|
assert.Equal(t, "pk", plan.Expression.Names()["#0"])
|
||||||
|
assert.Equal(t, "prefix", plan.Expression.Values()[":0"].(*types.AttributeValueMemberS).Value)
|
||||||
|
assert.Equal(t, "another", plan.Expression.Values()[":1"].(*types.AttributeValueMemberS).Value)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
22
internal/dynamo-browse/models/queryexpr/ir.go
Normal file
22
internal/dynamo-browse/models/queryexpr/ir.go
Normal file
|
@ -0,0 +1,22 @@
|
||||||
|
package queryexpr
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
|
||||||
|
"github.com/lmika/audax/internal/dynamo-browse/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
type irAtom interface {
|
||||||
|
// operandFieldName returns the field that this atom operates on. For example,
|
||||||
|
// if this IR node represents 'a = "b"', this should return "a".
|
||||||
|
// If this does not operate on a definitive field name, this returns null
|
||||||
|
operandFieldName() string
|
||||||
|
|
||||||
|
// canBeExecutedAsQuery returns true if the atom is capable of being executed as a query
|
||||||
|
canBeExecutedAsQuery(info *models.TableInfo, qci *queryCalcInfo) bool
|
||||||
|
|
||||||
|
// calcQueryForQuery returns a key condition builder for this atom to include in a query
|
||||||
|
calcQueryForQuery(info *models.TableInfo) (expression.KeyConditionBuilder, error)
|
||||||
|
|
||||||
|
// calcQueryForScan returns the condition builder for this atom to include in a scan
|
||||||
|
calcQueryForScan(info *models.TableInfo) (expression.ConditionBuilder, error)
|
||||||
|
}
|
|
@ -1,13 +0,0 @@
|
||||||
package queryexpr
|
|
||||||
|
|
||||||
func (a *astExpr) String() string {
|
|
||||||
return a.Equality.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *astBinOp) String() string {
|
|
||||||
return a.Name + a.Op + a.Value.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *astLiteralValue) String() string {
|
|
||||||
return a.StringVal
|
|
||||||
}
|
|
|
@ -22,3 +22,7 @@ func (a *astLiteralValue) goValue() (any, error) {
|
||||||
}
|
}
|
||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a *astLiteralValue) String() string {
|
||||||
|
return a.StringVal
|
||||||
|
}
|
||||||
|
|
|
@ -127,6 +127,40 @@ outer:
|
||||||
return items, nil
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *Provider) QueryItems(ctx context.Context, tableName string, filterExpr *expression.Expression, maxItems int) ([]models.Item, error) {
|
||||||
|
input := &dynamodb.QueryInput{
|
||||||
|
TableName: aws.String(tableName),
|
||||||
|
Limit: aws.Int32(int32(maxItems)),
|
||||||
|
}
|
||||||
|
if filterExpr != nil {
|
||||||
|
input.KeyConditionExpression = filterExpr.KeyCondition()
|
||||||
|
input.FilterExpression = filterExpr.Filter()
|
||||||
|
input.ExpressionAttributeNames = filterExpr.Names()
|
||||||
|
input.ExpressionAttributeValues = filterExpr.Values()
|
||||||
|
}
|
||||||
|
|
||||||
|
paginator := dynamodb.NewQueryPaginator(p.client, input)
|
||||||
|
|
||||||
|
items := make([]models.Item, 0)
|
||||||
|
|
||||||
|
outer:
|
||||||
|
for paginator.HasMorePages() {
|
||||||
|
res, err := paginator.NextPage(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "cannot execute query on table %v", tableName)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, itm := range res.Items {
|
||||||
|
items = append(items, itm)
|
||||||
|
if len(items) >= maxItems {
|
||||||
|
break outer
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (p *Provider) DeleteItem(ctx context.Context, tableName string, key map[string]types.AttributeValue) error {
|
func (p *Provider) DeleteItem(ctx context.Context, tableName string, key map[string]types.AttributeValue) error {
|
||||||
_, err := p.client.DeleteItem(ctx, &dynamodb.DeleteItemInput{
|
_, err := p.client.DeleteItem(ctx, &dynamodb.DeleteItemInput{
|
||||||
TableName: aws.String(tableName),
|
TableName: aws.String(tableName),
|
||||||
|
|
|
@ -11,6 +11,7 @@ import (
|
||||||
type TableProvider interface {
|
type TableProvider interface {
|
||||||
ListTables(ctx context.Context) ([]string, error)
|
ListTables(ctx context.Context) ([]string, error)
|
||||||
DescribeTable(ctx context.Context, tableName string) (*models.TableInfo, error)
|
DescribeTable(ctx context.Context, tableName string) (*models.TableInfo, error)
|
||||||
|
QueryItems(ctx context.Context, tableName string, filterExpr *expression.Expression, maxItems int) ([]models.Item, error)
|
||||||
ScanItems(ctx context.Context, tableName string, filterExpr *expression.Expression, maxItems int) ([]models.Item, error)
|
ScanItems(ctx context.Context, tableName string, filterExpr *expression.Expression, maxItems int) ([]models.Item, error)
|
||||||
DeleteItem(ctx context.Context, tableName string, key map[string]types.AttributeValue) error
|
DeleteItem(ctx context.Context, tableName string, key map[string]types.AttributeValue) error
|
||||||
PutItem(ctx context.Context, name string, item models.Item) error
|
PutItem(ctx context.Context, name string, item models.Item) error
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
|
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
|
||||||
"github.com/lmika/audax/internal/common/sliceutils"
|
"github.com/lmika/audax/internal/common/sliceutils"
|
||||||
|
"log"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/lmika/audax/internal/dynamo-browse/models"
|
"github.com/lmika/audax/internal/dynamo-browse/models"
|
||||||
|
@ -33,66 +34,39 @@ func (s *Service) Scan(ctx context.Context, tableInfo *models.TableInfo) (*model
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) doScan(ctx context.Context, tableInfo *models.TableInfo, expr models.Queryable) (*models.ResultSet, error) {
|
func (s *Service) doScan(ctx context.Context, tableInfo *models.TableInfo, expr models.Queryable) (*models.ResultSet, error) {
|
||||||
var filterExpr *expression.Expression
|
var (
|
||||||
|
filterExpr *expression.Expression
|
||||||
|
runAsQuery bool
|
||||||
|
err error
|
||||||
|
)
|
||||||
if expr != nil {
|
if expr != nil {
|
||||||
plan, err := expr.Plan(tableInfo)
|
plan, err := expr.Plan(tableInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// TEMP
|
runAsQuery = plan.CanQuery
|
||||||
if plan.CanQuery {
|
|
||||||
return nil, errors.Errorf("queries not yet supported")
|
|
||||||
}
|
|
||||||
|
|
||||||
filterExpr = &plan.Expression
|
filterExpr = &plan.Expression
|
||||||
}
|
}
|
||||||
|
|
||||||
results, err := s.provider.ScanItems(ctx, tableInfo.Name, filterExpr, 1000)
|
var results []models.Item
|
||||||
|
if runAsQuery {
|
||||||
|
log.Printf("executing query")
|
||||||
|
results, err = s.provider.QueryItems(ctx, tableInfo.Name, filterExpr, 1000)
|
||||||
|
} else {
|
||||||
|
log.Printf("executing scan")
|
||||||
|
results, err = s.provider.ScanItems(ctx, tableInfo.Name, filterExpr, 1000)
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrapf(err, "unable to scan table %v", tableInfo.Name)
|
return nil, errors.Wrapf(err, "unable to scan table %v", tableInfo.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the columns
|
|
||||||
//seenColumns := make(map[string]int)
|
|
||||||
//seenColumns[tableInfo.Keys.PartitionKey] = 0
|
|
||||||
//if tableInfo.Keys.SortKey != "" {
|
|
||||||
// seenColumns[tableInfo.Keys.SortKey] = 1
|
|
||||||
//}
|
|
||||||
//
|
|
||||||
//for _, definedAttribute := range tableInfo.DefinedAttributes {
|
|
||||||
// if _, seen := seenColumns[definedAttribute]; !seen {
|
|
||||||
// seenColumns[definedAttribute] = len(seenColumns)
|
|
||||||
// }
|
|
||||||
//}
|
|
||||||
//
|
|
||||||
//otherColsRank := len(seenColumns)
|
|
||||||
//for _, result := range results {
|
|
||||||
// for k := range result {
|
|
||||||
// if _, isSeen := seenColumns[k]; !isSeen {
|
|
||||||
// seenColumns[k] = otherColsRank
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
//}
|
|
||||||
//
|
|
||||||
//columns := make([]string, 0, len(seenColumns))
|
|
||||||
//for k := range seenColumns {
|
|
||||||
// columns = append(columns, k)
|
|
||||||
//}
|
|
||||||
//sort.Slice(columns, func(i, j int) bool {
|
|
||||||
// if seenColumns[columns[i]] == seenColumns[columns[j]] {
|
|
||||||
// return columns[i] < columns[j]
|
|
||||||
// }
|
|
||||||
// return seenColumns[columns[i]] < seenColumns[columns[j]]
|
|
||||||
//})
|
|
||||||
|
|
||||||
models.Sort(results, tableInfo)
|
models.Sort(results, tableInfo)
|
||||||
|
|
||||||
resultSet := &models.ResultSet{
|
resultSet := &models.ResultSet{
|
||||||
TableInfo: tableInfo,
|
TableInfo: tableInfo,
|
||||||
Query: expr,
|
Query: expr,
|
||||||
//Columns: columns,
|
|
||||||
}
|
}
|
||||||
resultSet.SetItems(results)
|
resultSet.SetItems(results)
|
||||||
resultSet.RefreshColumns()
|
resultSet.RefreshColumns()
|
||||||
|
|
Loading…
Reference in a new issue