Issue #66: Refactor engine and adapter relationship

Signed-off-by: Lucas Fernandes de Oliveira's avatarLucas Fernandes de Oliveira <lfoliveira@inf.ufpr.br>
parent ba1a7642
Pipeline #15720 passed with stages
in 1 minute and 8 seconds
...@@ -350,7 +350,7 @@ describe("postgres adapter", () => { ...@@ -350,7 +350,7 @@ describe("postgres adapter", () => {
adapter.getDataFromView(view, (err, result) => { adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null"); expect(err).to.be.a("null");
expect(result).to.be.an("array"); expect(result).to.be.an("array");
expect(result).to.have.length(5); expect(result).to.have.length(2);
expect(result[0]).to.be.an("object"); expect(result[0]).to.be.an("object");
let keys: string[] = []; let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name)); keys = keys.concat(view.metrics.map((item) => item.name));
...@@ -369,7 +369,7 @@ describe("postgres adapter", () => { ...@@ -369,7 +369,7 @@ describe("postgres adapter", () => {
adapter.getDataFromView(view, (err, result) => { adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null"); expect(err).to.be.a("null");
expect(result).to.be.an("array"); expect(result).to.be.an("array");
expect(result).to.have.length(5); expect(result).to.have.length(1);
expect(result[0]).to.be.an("object"); expect(result[0]).to.be.an("object");
let keys: string[] = []; let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name)); keys = keys.concat(view.metrics.map((item) => item.name));
......
This diff is collapsed.
/*
* Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { View } from "../core/view";
export enum Opcode {
PUSH,
JOIN,
REDUCE
}
export interface Operation {
opcode: Opcode;
values: View[];
}
...@@ -50,10 +50,8 @@ describe("engine class", () => { ...@@ -50,10 +50,8 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object"); expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics"); expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions"); expect(optimalView).to.have.property("dimensions");
expect(optimalView).to.have.property("childViews");
expect(optimalView.metrics).to.be.an("array"); expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array"); expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics).to.have.length(12); expect(optimalView.metrics).to.have.length(12);
expect(optimalView.dimensions).to.have.length(12); expect(optimalView.dimensions).to.have.length(12);
}); });
...@@ -92,13 +90,13 @@ describe("engine class", () => { ...@@ -92,13 +90,13 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object"); expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics"); expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions"); expect(optimalView).to.have.property("dimensions");
expect(optimalView).to.have.property("childViews"); expect(optimalView).to.have.property("operation");
expect(optimalView.metrics).to.be.an("array"); expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array"); expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array"); expect(optimalView.operation).to.be.an("object");
expect(optimalView.metrics).to.have.length(4); expect(optimalView.metrics).to.have.length(4);
expect(optimalView.dimensions).to.have.length(2); expect(optimalView.dimensions).to.have.length(2);
expect(optimalView.childViews).to.have.length(0); expect(optimalView.operation).to.have.property("opcode");
expect(optimalView.id).to.be.equal(views[0].id); expect(optimalView.id).to.be.equal(views[0].id);
}); });
...@@ -120,15 +118,13 @@ describe("engine class", () => { ...@@ -120,15 +118,13 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object"); expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics"); expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions"); expect(optimalView).to.have.property("dimensions");
expect(optimalView).to.have.property("childViews"); expect(optimalView).to.have.property("operation");
expect(optimalView).to.have.property("materialized");
expect(optimalView.metrics).to.be.an("array"); expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array"); expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array"); expect(optimalView.operation).to.be.an("object");
expect(optimalView.metrics).to.have.length(3); expect(optimalView.metrics).to.have.length(3);
expect(optimalView.dimensions).to.have.length(1); expect(optimalView.dimensions).to.have.length(1);
expect(optimalView.childViews).to.have.length(0); expect(optimalView.operation).to.have.property("opcode");
expect(optimalView.materialized).to.be.true;
expect(optimalView.id).to.be.equal(views[9].id); expect(optimalView.id).to.be.equal(views[9].id);
}); });
...@@ -143,13 +139,10 @@ describe("engine class", () => { ...@@ -143,13 +139,10 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object"); expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics"); expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions"); expect(optimalView).to.have.property("dimensions");
expect(optimalView).to.have.property("childViews");
expect(optimalView.metrics).to.be.an("array"); expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array"); expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics).to.have.length(0); expect(optimalView.metrics).to.have.length(0);
expect(optimalView.dimensions).to.have.length(2); expect(optimalView.dimensions).to.have.length(2);
expect(optimalView.childViews).to.have.length(3);
expect(optimalView).satisfy((optView: View) => { expect(optimalView).satisfy((optView: View) => {
return optView.dimensions.some((item) => item.name === subdim[0].name); return optView.dimensions.some((item) => item.name === subdim[0].name);
...@@ -169,13 +162,10 @@ describe("engine class", () => { ...@@ -169,13 +162,10 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object"); expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics"); expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions"); expect(optimalView).to.have.property("dimensions");
expect(optimalView).to.have.property("childViews");
expect(optimalView.metrics).to.be.an("array"); expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array"); expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics).to.have.length(0); expect(optimalView.metrics).to.have.length(0);
expect(optimalView.dimensions).to.have.length(2); expect(optimalView.dimensions).to.have.length(2);
expect(optimalView.childViews).to.have.length(3);
expect(optimalView).satisfy((optView: View) => { expect(optimalView).satisfy((optView: View) => {
return optView.dimensions.some((item) => item.name === subdim[2].name); return optView.dimensions.some((item) => item.name === subdim[2].name);
......
...@@ -27,6 +27,16 @@ import { Query } from "../common/query"; ...@@ -27,6 +27,16 @@ import { Query } from "../common/query";
import { Graph } from "../util/graph"; import { Graph } from "../util/graph";
import { EnumType, EnumTypeOptions} from "./enumType"; import { EnumType, EnumTypeOptions} from "./enumType";
import { Source , SourceStrOptions } from "./source"; import { Source , SourceStrOptions } from "./source";
import { ViewHandler } from "../util/viewHandler";
interface Score {
[key: string]: number;
}
interface ViewsAndClauses {
views: View[];
clauses: Clause[];
}
export class Engine { export class Engine {
private views: View[] = []; private views: View[] = [];
...@@ -197,67 +207,51 @@ export class Engine { ...@@ -197,67 +207,51 @@ export class Engine {
} }
private selectOptimalView (q: Query): View { private selectOptimalView (q: Query): View {
let optimalViews = this.graph.cover(q); let queries: Query[] = [];
if (optimalViews.length === 0) { if (q.metrics.length > 0) {
throw new Error ("Engine views cannot cover the query"); for (let i = 0; i < q.metrics.length; ++i) {
} queries.push({
metrics: [q.metrics[i]],
dimensions: q.dimensions,
clauses: (q.clauses) ? q.clauses : [],
sort: (q.sort) ? q.sort : []
});
}
const views = queries.map((query) => {
return ViewHandler.growView(query, this.getCover(query));
});
// If all the metrics and dimensions are the same and only exist one child view return ViewHandler.queryJoin(q, views);
// return this single child view
const metrics = q.metrics;
const dimensions = q.dimensions;
const clauses = ((q.clauses) ? q.clauses : []);
const sort = ((q.sort) ? q.sort : []);
if (optimalViews.length === 1 &&
optimalViews[0].metrics.length === metrics.length &&
optimalViews[0].dimensions.length === dimensions.length &&
optimalViews[0].clauses.length === clauses.length &&
optimalViews[0].sort.length === sort.length &&
optimalViews[0].metrics.every((item) => metrics.indexOf(item) !== -1) &&
optimalViews[0].dimensions.every((item) => dimensions.indexOf(item) !== -1) &&
perfectMatch(optimalViews[0].clauses, clauses) &&
perfectOrder(optimalViews[0].sort, sort)) {
return optimalViews[0];
} }
else { else {
let options = { let query = {
metrics: metrics, metrics: q.metrics,
dimensions: dimensions, dimensions: q.dimensions,
clauses: clauses, clauses: (q.clauses) ? q.clauses : [],
sort: sort, sort: (q.sort) ? q.sort : []
materialized: false,
origin: false, // Never a dynamic generated view will be origin
childViews: optimalViews
}; };
return ViewHandler.growView(query, this.getCover(query));
let view = new View(options);
// this.addView(view);
/*
This line has been removed for now because not all views can be
re-used by other views (unmaterializeble views), and so far this
is only detected in the adapter, when this can be detected in
engine, than the queries can be added again to the engine
*/
return view;
} }
} }
}
function perfectMatch (array1: Clause[], private getCover (q: Query): View[] {
array2: Clause[]): boolean { const optimalViews = this.graph.cover(q);
return array1.every((item: Clause) => { if (optimalViews.length === 0) {
return array2.some((otherItem: Clause) => item.id === otherItem.id); throw new Error ("Engine views cannot cover the query");
}); }
}
function perfectOrder (array1: (Metric|Dimension)[], let matViews: View[] = optimalViews.sort((a, b) => (a.id < b.id) ? -1 : 1);
array2: (Metric|Dimension)[]): boolean {
// Assuming that the arrays have the same length let noRepeat: View[] = [matViews[0]];
for (let i = 0; i < array1.length; ++i) { for (let i = 1; i < matViews.length; ++i) {
if (array1[i].name !== array2[i].name) { if (matViews[i - 1].id !== matViews[i].id) {
return false; noRepeat.push(matViews[i]);
}
} }
}
return true; return noRepeat;
}
} }
...@@ -22,6 +22,7 @@ import { Dimension } from "./dimension"; ...@@ -22,6 +22,7 @@ import { Dimension } from "./dimension";
import { Metric } from "./metric"; import { Metric } from "./metric";
import { Hash } from "../util/hash"; import { Hash } from "../util/hash";
import { Clause } from "./clause"; import { Clause } from "./clause";
import { Operation, Opcode } from "../common/expression";
export interface LoadView { export interface LoadView {
view: View; view: View;
...@@ -31,38 +32,36 @@ export interface LoadView { ...@@ -31,38 +32,36 @@ export interface LoadView {
export interface ViewOptions { export interface ViewOptions {
metrics: Metric[]; metrics: Metric[];
dimensions: Dimension[]; dimensions: Dimension[];
keys?: Dimension[];
origin: boolean; origin: boolean;
clauses?: Clause[]; clauses?: Clause[];
sort?: (Metric|Dimension)[]; sort?: (Metric|Dimension)[];
materialized?: boolean; operation?: Operation;
childViews?: View[];
} }
export class View { export class View {
public readonly id: string; public readonly id: string;
public readonly metrics: Metric[]; public readonly metrics: Metric[];
public readonly dimensions: Dimension[]; public readonly dimensions: Dimension[];
public readonly keys: Dimension[];
public readonly clauses: Clause[]; public readonly clauses: Clause[];
public readonly sort: (Metric|Dimension)[]; public readonly sort: (Metric|Dimension)[];
public readonly materialized: boolean;
public readonly origin: boolean; public readonly origin: boolean;
public childViews: View[]; public readonly operation: Operation;
constructor (options: ViewOptions) { constructor (options: ViewOptions) {
this.metrics = options.metrics.sort(); this.metrics = options.metrics.sort();
this.dimensions = options.dimensions.sort(); this.dimensions = options.dimensions.sort();
this.clauses = (options.clauses) ? options.clauses.sort() : []; this.clauses = (options.clauses) ? options.clauses.sort() : [];
this.sort = (options.sort) ? options.sort : []; this.sort = (options.sort) ? options.sort : [];
this.materialized = options.materialized || false;
this.origin = options.origin || false; this.origin = options.origin || false;
this.childViews = (options.childViews) ? options.childViews : []; if (options.operation) {
if (options.keys && options.keys.length > 0) { this.operation = options.operation;
this.keys = options.keys.sort();
} }
else { else {
this.keys = this.dimensions; this.operation = {
opcode: Opcode.PUSH,
values: []
};
} }
// calculate the id of the view based on it's metrics and dimensions // calculate the id of the view based on it's metrics and dimensions
......
...@@ -23,6 +23,7 @@ import { Dimension, DimensionOptions, DimensionStrOptions } from "../core/dimens ...@@ -23,6 +23,7 @@ import { Dimension, DimensionOptions, DimensionStrOptions } from "../core/dimens
import { View, ViewOptions, LoadView } from "../core/view"; import { View, ViewOptions, LoadView } from "../core/view";
import { EnumType, EnumTypeOptions } from "../core/enumType"; import { EnumType, EnumTypeOptions } from "../core/enumType";
import { RelationType, DataType } from "../common/types"; import { RelationType, DataType } from "../common/types";
import { Opcode } from "../common/expression";
import { Filter } from "../core/filter"; import { Filter } from "../core/filter";
import { Clause } from "../core/clause"; import { Clause } from "../core/clause";
import { Source, SourceOptions, SourceStrOptions} from "../core/source"; import { Source, SourceOptions, SourceStrOptions} from "../core/source";
...@@ -38,7 +39,6 @@ export interface ViewParsingOptions { ...@@ -38,7 +39,6 @@ export interface ViewParsingOptions {
dimensions: string[]; dimensions: string[];
metrics: string[]; metrics: string[];
clauses?: string[]; clauses?: string[];
keys?: string[];
} }
interface ConfigSchema { interface ConfigSchema {
...@@ -222,15 +222,14 @@ export class ConfigParser { ...@@ -222,15 +222,14 @@ export class ConfigParser {
let viewOpt: ViewOptions = { let viewOpt: ViewOptions = {
metrics: [], metrics: [],
dimensions: [], dimensions: [],
materialized: true,
origin: opts.origin, origin: opts.origin,
childViews: [],
clauses: [], clauses: [],
keys: [] operation: {
opcode: Opcode.PUSH,
values: []
}
}; };
const keys = (opts.keys) ? opts.keys : [];
for (let i = 0; i < opts.metrics.length; ++i) { for (let i = 0; i < opts.metrics.length; ++i) {
if (metMap[opts.metrics[i]]) { if (metMap[opts.metrics[i]]) {
viewOpt.metrics.push(metMap[opts.metrics[i]]); viewOpt.metrics.push(metMap[opts.metrics[i]]);
...@@ -251,15 +250,6 @@ export class ConfigParser { ...@@ -251,15 +250,6 @@ export class ConfigParser {
throw new Error("[Parsing error] Non exist dimension set to view " + opts.alias); throw new Error("[Parsing error] Non exist dimension set to view " + opts.alias);
} }
} }
for (let i = 0; i < keys.length; ++i) {
if (dimMap[keys[i]]) {
viewOpt.keys.push(dimMap[opts.keys[i]]);
}
else {
throw new Error("[Parsing error] Non exist key set to view " + opts.alias);
}
}
if (opts.clauses) { if (opts.clauses) {
for (let i = 0; i < opts.clauses.length; ++i) { for (let i = 0; i < opts.clauses.length; ++i) {
......
...@@ -96,20 +96,17 @@ describe("graph class", () => { ...@@ -96,20 +96,17 @@ describe("graph class", () => {
new View({ new View({
metrics: [], metrics: [],
dimensions: [dims[0], dims[1]], dimensions: [dims[0], dims[1]],
origin: true, origin: true
materialized: true
}), }),
new View({ new View({
metrics: [], metrics: [],
dimensions: [dims[2], dims[3]], dimensions: [dims[2], dims[3]],
origin: true, origin: true
materialized: true
}), }),
new View({ new View({
metrics: [], metrics: [],
dimensions: dims, dimensions: dims,
origin: true, origin: true
materialized: true
}) })
]; ];
...@@ -133,8 +130,7 @@ describe("graph class", () => { ...@@ -133,8 +130,7 @@ describe("graph class", () => {
let view = new View({ let view = new View({
metrics: [], metrics: [],
dimensions: [dims[0], dims[1]], dimensions: [dims[0], dims[1]],
origin: true, origin: true
materialized: true
}); });
expect(g.addView(view)).to.be.true; expect(g.addView(view)).to.be.true;
...@@ -155,8 +151,7 @@ describe("graph class", () => { ...@@ -155,8 +151,7 @@ describe("graph class", () => {
let view = new View({ let view = new View({
metrics: [], metrics: [],
dimensions: dims, dimensions: dims,
origin: true, origin: true
materialized: true
}); });
expect(g.addView(view)).to.be.false; expect(g.addView(view)).to.be.false;
...@@ -177,8 +172,7 @@ describe("graph class", () => { ...@@ -177,8 +172,7 @@ describe("graph class", () => {
let view = new View({ let view = new View({
metrics: [], metrics: [],
dimensions: [dim], dimensions: [dim],
origin: true, origin: true
materialized: true
}); });
expect(g.addView(view)).to.be.true; expect(g.addView(view)).to.be.true;
...@@ -195,8 +189,7 @@ describe("graph class", () => { ...@@ -195,8 +189,7 @@ describe("graph class", () => {
let view = new View({ let view = new View({
metrics: [], metrics: [],
dimensions: [dim], dimensions: [dim],
origin: true, origin: true
materialized: true
}); });
expect(g.addView(view)).to.be.true; expect(g.addView(view)).to.be.true;
...@@ -231,38 +224,32 @@ describe("graph class", () => { ...@@ -231,38 +224,32 @@ describe("graph class", () => {
new View({ new View({
metrics: [mets[0]], metrics: [mets[0]],
dimensions: [dims[0]], dimensions: [dims[0]],
origin: true, origin: true
materialized: true
}), }),
new View({ new View({
metrics: [mets[1]], metrics: [mets[1]],
dimensions: [dims[1]], dimensions: [dims[1]],
origin: true, origin: true
materialized: true
}), }),
new View({ new View({
metrics: [mets[2]], metrics: [mets[2]],
dimensions: [dims[2]], dimensions: [dims[2]],
origin: true, origin: true
materialized: true
}), }),
new View({ new View({
metrics: [], metrics: [],
dimensions: dims, dimensions: dims,
origin: true, origin: true
materialized: true
}), }),
new View({ new View({
metrics: mets, metrics: mets,
dimensions: dims, dimensions: dims,
origin: true, origin: true
materialized: true
}), }),
new View({ new View({
metrics: [mets[0], mets[1]], metrics: [mets[0], mets[1]],
dimensions: [dims[0], dims[1]], dimensions: [dims[0], dims[1]],
origin: true, origin: true
materialized: true
}), }),
]; ];
...@@ -304,8 +291,7 @@ describe("graph class", () => { ...@@ -304,8 +291,7 @@ describe("graph class", () => {
let view = new View({ let view = new View({
metrics: [], metrics: [],
dimensions: [dims[0]], dimensions: [dims[0]],
origin: true, origin: true
materialized: true
}); });
expect(g.addView(view)).to.be.true; expect(g.addView(view)).to.be.true;
...@@ -344,8 +330,7 @@ describe("graph class", () => { ...@@ -344,8 +330,7 @@ describe("graph class", () => {
let view = new View({ let view = new View({
metrics: [], metrics: [],
dimensions: [dims[0]], dimensions: [dims[0]],
origin: true, origin: true
materialized: true
}); });
expect(g.addView(view)).to.be.true; expect(g.addView(view)).to.be.true;
...@@ -385,23 +370,19 @@ describe("graph class", () => { ...@@ -385,23 +370,19 @@ describe("graph class", () => {
let view1 = new View({ let view1 = new View({
metrics: [], metrics: [],
dimensions: [dims[0], dims[1]], dimensions: [dims[0], dims[1]],
origin: true, origin: true
materialized: true
}); });
let view2 = new View({ let view2 = new View({
metrics: [], metrics: [],
dimensions: [dims[1], dims[2]], dimensions: [dims[1], dims[2]],
origin: true, origin: true
materialized: true
}); });
let view3 = new View({ let view3 = new View({
metrics: [], metrics: [],
dimensions: dims, dimensions: dims,
origin: false, origin: false,
materialized: false,
childViews: [view1, view2],
clauses: [clause1] clauses: [clause1]
}); });
...@@ -517,7 +498,6 @@ describe("graph class", () => { ...@@ -517,7 +498,6 @@ describe("graph class", () => {
metrics: [], metrics: [],
dimensions: dims, dimensions: dims,
origin: false, origin: false,
materialized: true,
clauses: clauses clauses: clauses
}); });
...@@ -630,8 +610,7 @@ describe("graph class", () => { ...@@ -630,8 +610,7 @@ describe("graph class", () => {
new View({ new View({
metrics: [], metrics: [],
dimensions: dims, dimensions: dims,
origin: false, origin: false
materialized: true
})