Commit c4092417 authored by Eduardo L. Buratti's avatar Eduardo L. Buratti

Merge branch 'issue/6' into 'master'

Issue #6: Create selector of optimal View and sql query generator for Postgres adpter

Foram criados os Arquivos para views, queries e adapters e engine

Na definição do adapter, as funções que deveriam ter como retorno promise e Stream estão como string pois esse tipos não foram encontrados

Apenas o arquivo engine ganhou um script de teste.
Apenas a string gerada pelo Adapter foi testada, mas ela foi deixada de fora do teste da engine.
Signed-off-by: Lucas Fernandes de Oliveira's avatarLucas Fernandes de Oliveira <lfo14@inf.ufpr.br>

See merge request !2
parents 6dc7ebfc 0aca2492
Pipeline #7527 failed with stage
in 1 minute and 27 seconds
/*
* Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { View } from "../core/view";
import { Adapter } from "../core/adapter";
import { AggregationType } from "../common/aggregationType";
interface ParsedView {
query: string;
view: View;
};
export class PostgresAdapter extends Adapter{
public getDataFromView(view: View): string {
// buildQueryFromView does not put the final ;, it need to be put apart
return this.buildQueryFromView(view) + ";\n";
}
public materializeView(view: View): string {
return null;
}
private buildQueryFromView (view: View): string {
let sql = "(\n";
let metrics = view.metrics.map((metric: string) => {
let aggrType = view.getAggregationtype(metric);
let func = this.getAggregateFunction(aggrType);
let extMetric = func + "(" + metric + ")";
return extMetric;
});
if (view.materialized) {
sql += "SELECT " + metrics.join(", ") + ", " + dimensions.join(", ") + "\n";
sql += "FROM " + "view_" + view.id + "\n";
sql += "GROUP BY " + view.dimensions.join(", ") + "\n";
sql += ")\n";
return sql;
}
else {
let children: ParsedView[] = view.childViews.map((item: View, idx: number) => {
return {
query: this.queryfyView(item, childAlias, degree + 1),
view: item
};
});
let covered = new Map();
view.dimensions.forEach((item: string) => covered.set(item, ""));
view.metrics.forEach((item: string) => covered.set(item, ""));
let projection = "SELECT ";
let viewsFrom = "FROM";
let selection = "WHERE ";
let grouping = "GROUP BY ";
let elements = [];
let group = [];
children.forEach((child: ParsedView) => {
let selected = [];
child.view.dimensions.forEach((dimension: string) => {
let first = covered.get(dimension);
let extDimension = "view_" + child.view.id + "." + dimension;
if (first === "") {
covered.set(dimension, child.view.id);
elements.push(extDimension);
group.push(extDimension);
}
else {
let extFirst = "view_" + first + "." + dimension;
selected.push(extDimension + " = " + extFirst);
}
});
child.view.metrics.forEach((metric: string) => {
let first = covered.get(metric);
let aggrType = child.view.getAggregateFunction(metric);
let func = this.geAggregateFunction(aggrType);
let extMetric = func + "(view_" + child.view.id + "." + metric + ")";
if (first === "") {
covered.set(metric, child.view.id);
elements.push(extMetric);
}
});
viewsFrom += "\n" + child.query;
if (selected.length > 0) {
selection += selected.join(" AND ");
}
});
projection += elements.join(", ") + "\n";
selection += "\n";
grouping += group.join(", ") + "\n";
sql += projection + viewsFrom + selection + grouping + ")";
return sql;
}
return sql;
}
private getAggregateFunction(aggrType: AggregationType): string {
switch (aggrType) {
case AggregationType.SUM:
return "SUM";
case AggregationType.AVG:
return "AVG";
case AggregationType.COUNT:
return "COUNT";
default:
return "";
}
}
}
/*
* Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
export enum AggregationType {
SUM,
AVG,
COUNT,
NONE
};
/*
* Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
export interface Query {
public metrics: string[];
public dimensions: string[];
}
/*
* Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { View } from "./view";
export abstract class Adapter {
public abstract getDataFromView(view: View): string;
public abstract materializeView(view: View): string;
}
/*
* Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { expect } from "chai";
import { Engine } from "./engine";
import { View } from "./view";
import { ViewOptions } from "./view";
import { Query } from "../common/query";
import { AggregationType } from "../common/aggregationType";
describe("engine class", () => {
let query: Query = {
metrics: [],
dimensions: []
};
let views: View[] = [];
let viewBuilder: Query[] = [
{ metrics: ["met:1", "met:2", "met:3"], dimensions: ["dim:1", "dim:2"]},
{ metrics: ["met:1", "met:3", "met:5"], dimensions: ["dim:1", "dim:2"]},
{ metrics: ["met:3", "met:4", "met:7"], dimensions: ["dim:4", "dim:5"]},
{ metrics: ["met:6", "met:7"], dimensions: ["dim:3", "dim:4", "dim:5", "dim:6"]},
{ metrics: ["met:8", "met:2", "met:3"], dimensions: ["dim:1", "dim:2", "dim:7"]},
{ metrics: ["met:1", "met:2", "met:3"], dimensions: ["dim:1", "dim:2"]},
{ metrics: ["met:2", "met:4"], dimensions: ["dim:1", "dim:2"]},
{ metrics: ["met:8"], dimensions: ["dim:8", "dim:9", "dim:10"]},
{ metrics: ["met:9"], dimensions: ["dim:8", "dim:9", "dim:10"]},
{ metrics: ["met:10"], dimensions: ["dim:8", "dim:9", "dim:10"]}
];
let iterable: [string, AggregationType][] = [
["met:1", AggregationType.SUM],
["met:2", AggregationType.AVG],
["met:3", AggregationType.AVG],
["met:4", AggregationType.SUM],
["met:5", AggregationType.SUM],
["met:6", AggregationType.AVG],
["met:7", AggregationType.COUNT],
["met:8", AggregationType.COUNT],
["met:9", AggregationType.SUM],
["met:10", AggregationType.COUNT],
];
let map: Map<string, AggregationType> = new Map(iterable);
for (let i: number = 0; i < 10; ++ i) {
query.metrics.push("met:" + (i + 1));
query.dimensions.push("dim:" + (i + 1));
let options: ViewOptions = {
metrics: viewBuilder[i].metrics,
dimensions: viewBuilder[i].dimensions,
materialized: true,
aggregationMap: map,
childViews: []
};
views.push(new View (options));
}
views.push(new View({
metrics: ["met:1", "met:2", "met:3", "met:4", "met:5"],
dimensions: ["dim:1", "dim:2"],
materialized: false,
aggregationMap: map,
childViews: [views[0], views[6]]
}));
views.push(new View({
metrics: ["met:8", "met:9", "met:10"],
dimensions: ["dim:8", "dim:9", "dim:10"],
materialized: false,
aggregationMap: map,
childViews: [views[7], views[8], views[9]]
}));
it("should be create a fill that cover the query and has 4 children", () => {
let engine: Engine = new Engine (views);
let optimalView = engine.query(query);
expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions");
expect(optimalView).to.have.property("childViews");
expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics.length === 10);
expect(optimalView.dimensions.length === 10);
expect(optimalView.childViews.length === 4);
let metAux: number[] = optimalView.metrics.sort().map((item: string) => {
return Number(item.split(":")[1]);
});
let dimAux: number[] = optimalView.dimensions.sort().map((item: string) => {
return Number(item.split(":")[1]);
});
for (let i: number = 1; i <= 10; ++i) {
expect(dimAux[i] === i);
expect(metAux[i] === i);
}
});
it("should throw an exception, query with non-existent metric", () => {
let engine: Engine = new Engine (views);
let error: boolean = false;
try {
engine.query({metrics: ["met:11"], dimensions: ["dim:1"]});
}
catch (e){
error = true;
expect(e.message).to.be.equal("Engine views cannot cover the query");
}
expect(error);
});
it("should throw an exception, query with non-existent dimension", () => {
let engine: Engine = new Engine (views);
let error: boolean = false;
try {
engine.query({metrics: ["met:1"], dimensions: ["dim:11"]});
}
catch (e){
error = true;
expect(e.message).to.be.equal("Engine views cannot cover the query");
}
expect(error);
});
});
/*
* Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { Query } from "../common/query";
import { View } from "./view";
import { AggregationType } from "../common/aggregationType";
export class Engine {
private views: View[];
constructor (v: View[]) {
this.views = v;
}
public query (q: Query) { return this.selectOptimalView(q); }
public insert (data: any) { /*to Implement*/ }
private selectOptimalView (q: Query) {
let objective = q.metrics.concat(q.dimensions);
let optimalViews: View[] = [];
let activeViews = this.views;
// Run this block until all metrics and dimmensions are covered
while (objective.length > 0) {
let bestView: View;
let shortestDistance = objective.length + 1;
activeViews = activeViews.filter((view: View) => {
let cover = view.metrics.concat(view.dimensions);
let intersection = cover.filter((item: string) => {
return objective.indexOf(item) !== -1;
});
if (intersection.length > 0) {
let distance = objective.length - intersection.length;
if (distance < shortestDistance) {
bestView = view;
shortestDistance = distance;
}
return true;
}
/*If the intersection is empty,
remove this element from future searches*/
return false;
});
if (shortestDistance === objective.length + 1) {
throw new Error ("Engine views cannot cover the query");
}
optimalViews.push(bestView);
//remove metrics and dimensions corevered by the bestView
objective = objective.filter((item: string) => {
let cover = bestView.dimensions.concat(bestView.metrics);
return cover.indexOf(item) === -1;
});
}
if (optimalViews.length === 1) {
return optimalViews.pop();
}
else {
let map = new Map();
optimalViews.forEach((view: View) => {
view.aggregationMap.forEach((value: AggregationType, key: string) => {
map.set(key, value);
});
});
let options = {
metrics: q.metrics,
dimensions: q.dimensions,
materialized: false,
aggregationMap: map,
childViews: optimalViews
};
let view = new View(options);
this.views.push(view);
return view;
}
}
}
/*
* Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { Hash } from "../util/hash";
import { AggregationType } from "../common/aggregationType";
export interface ViewOptions {
metrics: string[];
dimensions: string[];
materialized: boolean;
aggregationMap: Map <string, AggregationType>;
childViews: View[];
}
export class View {
public readonly id: string;
public readonly metrics: string[];
public readonly dimensions: string[];
public readonly materialized: boolean;
public readonly aggregationMap: Map<string, AggregationType>;
public childViews: View[];
constructor (options: ViewOptions) {
this.metrics = options.metrics;
this.dimensions = options.dimensions;
this.materialized = options.materialized;
this.childViews = options.childViews;
this.aggregationMap = options.aggregationMap;
this.id = Hash.sha1(options.metrics.sort(), options.dimensions.sort());
}
public getAggregatationType(metric: string): AggregationType {
if (this.aggregationMap.has(metric)) {
return AggregationType.NONE;
}
return this.aggregationMap.get(metric);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment