Commit 4339c078 authored by Lucas Fernandes de Oliveira's avatar Lucas Fernandes de Oliveira

Merge branch 'issue/66' into 'develop'

Issue #66: Refactor engine and adapter relationship

See merge request !64
parents ba1a7642 1e2b68b0
Pipeline #15786 failed with stages
in 1 minute and 6 seconds
......@@ -350,7 +350,7 @@ describe("postgres adapter", () => {
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(5);
expect(result).to.have.length(2);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
......@@ -369,7 +369,7 @@ describe("postgres adapter", () => {
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(5);
expect(result).to.have.length(1);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
......
This diff is collapsed.
/*
* Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { View } from "../core/view";
export enum Opcode {
PUSH,
JOIN,
REDUCE
}
export interface Operation {
opcode: Opcode;
values: View[];
}
......@@ -50,10 +50,8 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions");
expect(optimalView).to.have.property("childViews");
expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics).to.have.length(12);
expect(optimalView.dimensions).to.have.length(12);
});
......@@ -92,13 +90,13 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions");
expect(optimalView).to.have.property("childViews");
expect(optimalView).to.have.property("operation");
expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array");
expect(optimalView.operation).to.be.an("object");
expect(optimalView.metrics).to.have.length(4);
expect(optimalView.dimensions).to.have.length(2);
expect(optimalView.childViews).to.have.length(0);
expect(optimalView.operation).to.have.property("opcode");
expect(optimalView.id).to.be.equal(views[0].id);
});
......@@ -120,15 +118,13 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions");
expect(optimalView).to.have.property("childViews");
expect(optimalView).to.have.property("materialized");
expect(optimalView).to.have.property("operation");
expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array");
expect(optimalView.operation).to.be.an("object");
expect(optimalView.metrics).to.have.length(3);
expect(optimalView.dimensions).to.have.length(1);
expect(optimalView.childViews).to.have.length(0);
expect(optimalView.materialized).to.be.true;
expect(optimalView.operation).to.have.property("opcode");
expect(optimalView.id).to.be.equal(views[9].id);
});
......@@ -143,13 +139,10 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions");
expect(optimalView).to.have.property("childViews");
expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics).to.have.length(0);
expect(optimalView.dimensions).to.have.length(2);
expect(optimalView.childViews).to.have.length(3);
expect(optimalView).satisfy((optView: View) => {
return optView.dimensions.some((item) => item.name === subdim[0].name);
......@@ -169,13 +162,10 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions");
expect(optimalView).to.have.property("childViews");
expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics).to.have.length(0);
expect(optimalView.dimensions).to.have.length(2);
expect(optimalView.childViews).to.have.length(3);
expect(optimalView).satisfy((optView: View) => {
return optView.dimensions.some((item) => item.name === subdim[2].name);
......
......@@ -27,6 +27,16 @@ import { Query } from "../common/query";
import { Graph } from "../util/graph";
import { EnumType, EnumTypeOptions} from "./enumType";
import { Source , SourceStrOptions } from "./source";
import { ViewHandler } from "../util/viewHandler";
interface Score {
[key: string]: number;
}
interface ViewsAndClauses {
views: View[];
clauses: Clause[];
}
export class Engine {
private views: View[] = [];
......@@ -197,67 +207,51 @@ export class Engine {
}
private selectOptimalView (q: Query): View {
let optimalViews = this.graph.cover(q);
if (optimalViews.length === 0) {
throw new Error ("Engine views cannot cover the query");
}
let queries: Query[] = [];
if (q.metrics.length > 0) {
for (let i = 0; i < q.metrics.length; ++i) {
queries.push({
metrics: [q.metrics[i]],
dimensions: q.dimensions,
clauses: (q.clauses) ? q.clauses : [],
sort: (q.sort) ? q.sort : []
});
}
const views = queries.map((query) => {
return ViewHandler.growView(query, this.getCover(query));
});
// If all the metrics and dimensions are the same and only exist one child view
// return this single child view
const metrics = q.metrics;
const dimensions = q.dimensions;
const clauses = ((q.clauses) ? q.clauses : []);
const sort = ((q.sort) ? q.sort : []);
if (optimalViews.length === 1 &&
optimalViews[0].metrics.length === metrics.length &&
optimalViews[0].dimensions.length === dimensions.length &&
optimalViews[0].clauses.length === clauses.length &&
optimalViews[0].sort.length === sort.length &&
optimalViews[0].metrics.every((item) => metrics.indexOf(item) !== -1) &&
optimalViews[0].dimensions.every((item) => dimensions.indexOf(item) !== -1) &&
perfectMatch(optimalViews[0].clauses, clauses) &&
perfectOrder(optimalViews[0].sort, sort)) {
return optimalViews[0];
return ViewHandler.queryJoin(q, views);
}
else {
let options = {
metrics: metrics,
dimensions: dimensions,
clauses: clauses,
sort: sort,
materialized: false,
origin: false, // Never a dynamic generated view will be origin
childViews: optimalViews
let query = {
metrics: q.metrics,
dimensions: q.dimensions,
clauses: (q.clauses) ? q.clauses : [],
sort: (q.sort) ? q.sort : []
};
let view = new View(options);
// this.addView(view);
/*
This line has been removed for now because not all views can be
re-used by other views (unmaterializeble views), and so far this
is only detected in the adapter, when this can be detected in
engine, than the queries can be added again to the engine
*/
return view;
return ViewHandler.growView(query, this.getCover(query));
}
}
}
function perfectMatch (array1: Clause[],
array2: Clause[]): boolean {
return array1.every((item: Clause) => {
return array2.some((otherItem: Clause) => item.id === otherItem.id);
});
}
private getCover (q: Query): View[] {
const optimalViews = this.graph.cover(q);
if (optimalViews.length === 0) {
throw new Error ("Engine views cannot cover the query");
}
function perfectOrder (array1: (Metric|Dimension)[],
array2: (Metric|Dimension)[]): boolean {
// Assuming that the arrays have the same length
for (let i = 0; i < array1.length; ++i) {
if (array1[i].name !== array2[i].name) {
return false;
let matViews: View[] = optimalViews.sort((a, b) => (a.id < b.id) ? -1 : 1);
let noRepeat: View[] = [matViews[0]];
for (let i = 1; i < matViews.length; ++i) {
if (matViews[i - 1].id !== matViews[i].id) {
noRepeat.push(matViews[i]);
}
}
}
return true;
return noRepeat;
}
}
......@@ -22,6 +22,7 @@ import { Dimension } from "./dimension";
import { Metric } from "./metric";
import { Hash } from "../util/hash";
import { Clause } from "./clause";
import { Operation, Opcode } from "../common/expression";
export interface LoadView {
view: View;
......@@ -31,38 +32,36 @@ export interface LoadView {
export interface ViewOptions {
metrics: Metric[];
dimensions: Dimension[];
keys?: Dimension[];
origin: boolean;
clauses?: Clause[];
sort?: (Metric|Dimension)[];
materialized?: boolean;
childViews?: View[];
operation?: Operation;
}
export class View {
public readonly id: string;
public readonly metrics: Metric[];
public readonly dimensions: Dimension[];
public readonly keys: Dimension[];
public readonly clauses: Clause[];
public readonly sort: (Metric|Dimension)[];
public readonly materialized: boolean;
public readonly origin: boolean;
public childViews: View[];
public readonly operation: Operation;
constructor (options: ViewOptions) {
this.metrics = options.metrics.sort();
this.dimensions = options.dimensions.sort();
this.clauses = (options.clauses) ? options.clauses.sort() : [];
this.sort = (options.sort) ? options.sort : [];
this.materialized = options.materialized || false;
this.origin = options.origin || false;
this.childViews = (options.childViews) ? options.childViews : [];
if (options.keys && options.keys.length > 0) {
this.keys = options.keys.sort();
if (options.operation) {
this.operation = options.operation;
}
else {
this.keys = this.dimensions;
this.operation = {
opcode: Opcode.PUSH,
values: []
};
}
// calculate the id of the view based on it's metrics and dimensions
......
......@@ -23,6 +23,7 @@ import { Dimension, DimensionOptions, DimensionStrOptions } from "../core/dimens
import { View, ViewOptions, LoadView } from "../core/view";
import { EnumType, EnumTypeOptions } from "../core/enumType";
import { RelationType, DataType } from "../common/types";
import { Opcode } from "../common/expression";
import { Filter } from "../core/filter";
import { Clause } from "../core/clause";
import { Source, SourceOptions, SourceStrOptions} from "../core/source";
......@@ -38,7 +39,6 @@ export interface ViewParsingOptions {
dimensions: string[];
metrics: string[];
clauses?: string[];
keys?: string[];
}
interface ConfigSchema {
......@@ -222,15 +222,14 @@ export class ConfigParser {
let viewOpt: ViewOptions = {
metrics: [],
dimensions: [],
materialized: true,
origin: opts.origin,
childViews: [],
clauses: [],
keys: []
operation: {
opcode: Opcode.PUSH,
values: []
}
};
const keys = (opts.keys) ? opts.keys : [];
for (let i = 0; i < opts.metrics.length; ++i) {
if (metMap[opts.metrics[i]]) {
viewOpt.metrics.push(metMap[opts.metrics[i]]);
......@@ -251,15 +250,6 @@ export class ConfigParser {
throw new Error("[Parsing error] Non exist dimension set to view " + opts.alias);
}
}
for (let i = 0; i < keys.length; ++i) {
if (dimMap[keys[i]]) {
viewOpt.keys.push(dimMap[opts.keys[i]]);
}
else {
throw new Error("[Parsing error] Non exist key set to view " + opts.alias);
}
}
if (opts.clauses) {
for (let i = 0; i < opts.clauses.length; ++i) {
......
......@@ -96,20 +96,17 @@ describe("graph class", () => {
new View({
metrics: [],
dimensions: [dims[0], dims[1]],
origin: true,
materialized: true
origin: true
}),
new View({
metrics: [],
dimensions: [dims[2], dims[3]],
origin: true,
materialized: true
origin: true
}),
new View({
metrics: [],
dimensions: dims,
origin: true,
materialized: true
origin: true
})
];
......@@ -133,8 +130,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: [dims[0], dims[1]],
origin: true,
materialized: true
origin: true
});
expect(g.addView(view)).to.be.true;
......@@ -155,8 +151,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: dims,
origin: true,
materialized: true
origin: true
});
expect(g.addView(view)).to.be.false;
......@@ -177,8 +172,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: [dim],
origin: true,
materialized: true
origin: true
});
expect(g.addView(view)).to.be.true;
......@@ -195,8 +189,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: [dim],
origin: true,
materialized: true
origin: true
});
expect(g.addView(view)).to.be.true;
......@@ -231,38 +224,32 @@ describe("graph class", () => {
new View({
metrics: [mets[0]],
dimensions: [dims[0]],
origin: true,
materialized: true
origin: true
}),
new View({
metrics: [mets[1]],
dimensions: [dims[1]],
origin: true,
materialized: true
origin: true
}),
new View({
metrics: [mets[2]],
dimensions: [dims[2]],
origin: true,
materialized: true
origin: true
}),
new View({
metrics: [],
dimensions: dims,
origin: true,
materialized: true
origin: true
}),
new View({
metrics: mets,
dimensions: dims,
origin: true,
materialized: true
origin: true
}),
new View({
metrics: [mets[0], mets[1]],
dimensions: [dims[0], dims[1]],
origin: true,
materialized: true
origin: true
}),
];
......@@ -304,8 +291,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: [dims[0]],
origin: true,
materialized: true
origin: true
});
expect(g.addView(view)).to.be.true;
......@@ -344,8 +330,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: [dims[0]],
origin: true,
materialized: true
origin: true
});
expect(g.addView(view)).to.be.true;
......@@ -385,23 +370,19 @@ describe("graph class", () => {
let view1 = new View({
metrics: [],
dimensions: [dims[0], dims[1]],
origin: true,
materialized: true
origin: true
});
let view2 = new View({
metrics: [],
dimensions: [dims[1], dims[2]],
origin: true,
materialized: true
origin: true
});
let view3 = new View({
metrics: [],
dimensions: dims,
origin: false,
materialized: false,
childViews: [view1, view2],
clauses: [clause1]
});
......@@ -517,7 +498,6 @@ describe("graph class", () => {
metrics: [],
dimensions: dims,
origin: false,
materialized: true,
clauses: clauses
});
......@@ -630,8 +610,7 @@ describe("graph class", () => {
new View({
metrics: [],
dimensions: dims,
origin: false,
materialized: true
origin: false
})
];
......@@ -640,7 +619,6 @@ describe("graph class", () => {
metrics: [],
dimensions: dims,
origin: false,
materialized: true,
clauses: [clauses[i]]
}));
}
......
This diff is collapsed.
......@@ -26,6 +26,7 @@ import { Filter, FilterOperator } from "../src/core/filter";
import { Clause } from "../src/core/clause";
import { AggregationType, RelationType , DataType} from "../src/common/types";
import { Query} from "../src/common/query";
import { ViewHandler } from "../src/util/viewHandler";
interface EngineScenario {
metrics: Metric[];
......@@ -215,158 +216,135 @@ const dateSubDim = [
})
];
const dateView = new View({
const dateView = ViewHandler.queryReduce({
metrics: [],
dimensions: dateSubDim,
materialized: false,
origin: false,
childViews: [views[0]]
});
}, views[0]);
const aggrView = new View({
const aggrView = ViewHandler.queryJoin({
metrics: [mets[0], mets[1], mets[6], mets[10], mets[11]],
dimensions: [],
materialized: false,
origin: false,
childViews: [views[0], views[2], views[3]]
});
const clauseView = new View({
}, [
ViewHandler.queryReduce({
metrics: [mets[0], mets[1], mets[10]],
dimensions: []
}, views[0]),
ViewHandler.queryReduce({
metrics: [mets[6], mets[11]],
dimensions: []
}, views[2])
]);
const clauseView = ViewHandler.queryReduce({
metrics: [mets[0], mets[1], mets[2]],
dimensions: [dims[0]],
materialized: false,
origin: false,
childViews: [views[0]],
clauses: [clauses.view0dim7]
});
}, views[0]);
const multiFilterView = new View({
const multiFilterView = ViewHandler.queryReduce({
metrics: [mets[0], mets[1]],
dimensions: [dims[0]],
materialized: false,
origin: false,
childViews: [views[0]],
clauses: [clauses.view0dim0]
});
}, views[0]);
const multiClauseView = new View({
const multiClauseView = ViewHandler.queryReduce({
metrics: [mets[0], mets[1]],
dimensions: [dims[0]],
materialized: false,
origin: false,
childViews: [views[0]],
clauses: [clauses.view0dim0, clauses.view0dim7]
});
}, views[0]);
const notEqualView = new View({
const notEqualView = ViewHandler.queryReduce({
metrics: [],
dimensions: [dims[4], dims[5]],
materialized: false,
origin: false,
childViews: [views[7]],
clauses: [clauses.view7dim5]
});
}, views[7]);
const gtltView = new View({
const gtltView = ViewHandler.queryReduce({
metrics: [],
dimensions: [dims[0]],
materialized: false,
origin: false,
childViews: [views[0]],
clauses: [clauses.view0gt, clauses.view0lt]
});
}, views[0]);
const geleView = new View({
const geleView = ViewHandler.queryReduce({
metrics: [],
dimensions: [dims[0]],
materialized: false,
origin: false,
childViews: [views[0]],
clauses: [clauses.view0ge, clauses.view0le]
});
}, views[0]);
const notMatchFilterView = new View({
const notMatchFilterView = ViewHandler.queryReduce({
metrics: [mets[0]],
dimensions: [dims[0]],
materialized: false,
origin: false,
childViews: [views[0]],
clauses: [clauses.view7dim5]
});
}, views[0]);
const subDimView = new View({
const subDimView = ViewHandler.queryJoin({
metrics: [mets[0]],
dimensions: [subdims[0], subdims[1], dims[7], dims[8]],
materialized: false,
origin: false,
childViews: [views[0], views[1], views[4]]
});
const join4View = new View({
dimensions: [subdims[0], subdims[1], dims[7], dims[8]]
}, [
ViewHandler.queryReduce({
metrics: [mets[0]],
dimensions: [subdims[0], dims[7]]
}, views[0]),
ViewHandler.queryJoin({
metrics: [],
dimensions: [subdims[1], dims[7], dims[8]]
}, [views[1], views[4]])
]);
const join4View = ViewHandler.queryJoin({
metrics: [mets[0], mets[1], mets[2], mets[3], mets[4], mets[5]],
dimensions: [dims[2], dims[7], dims[8]],
materialized: false,
origin: false,
childViews: [views[0], views[1], views[2], views[4]]
});
dimensions: [dims[2], dims[7], dims[8]]
}, [views[0], views[1], views[2], views[4]]);
const noSelView = new View({
const noSelView = ViewHandler.queryJoin({
metrics: [mets[0], mets[3]],
dimensions: [],
materialized: false,
origin: false,
childViews: [views[0], views[1]]
});
dimensions: []
}, [views[0], views[1]]);
const withSelView = new View({
const withSelView = ViewHandler.queryJoin({
metrics: [mets[0], mets[1]],
dimensions: [dims[7], dims[8]],
materialized: false,
origin: false,
childViews: [views[0], views[4]]
});
dimensions: [dims[7], dims[8]]
}, [views[0], views[4]]);
const notOriginCount = new View({
metrics: [mets[5], mets[6], mets[7]],
dimensions: [dims[2]],
materialized: true,
origin: false,
clauses: [clauses.view9dim2]