Commit b0e89bb9 authored by Lucas Fernandes de Oliveira's avatar Lucas Fernandes de Oliveira
Browse files

Issue #23: Add filters to API


Signed-off-by: Lucas Fernandes de Oliveira's avatarLucas Fernandes de Oliveira <lfo14@inf.ufpr.br>
parent 42215793
Pipeline #11266 passed with stage
in 43 seconds
......@@ -169,4 +169,76 @@ describe("postgres adapter", () => {
done();
});
});
it("should get data from view when a single clause exists", (done) => {
let view = adapterScenario.clauseView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(1);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
expect(parseInt(result[0]["met:0"], 10)).to.be.equal(1);
expect(parseInt(result[0]["met:1"], 10)).to.be.equal(1);
expect(parseInt(result[0]["met:2"], 10)).to.be.equal(1);
expect(result[0]["dim:0"].getDate()).to.be.equal(1);
done();
});
});
it("should get data from view with single clause, with more than on filter", (done) => {
let view = adapterScenario.multiFilterView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(2);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from view with multiple clauses", (done) => {
let view = adapterScenario.multiClauseView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(0);
done();
});
});
it("should get data from view with a clause with not equal operator", (done) => {
let view = adapterScenario.notEqualView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(4);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
});
......@@ -21,6 +21,8 @@
import { Adapter } from "../core/adapter";
import { Metric } from "../core/metric";
import { Dimension } from "../core/dimension";
import { Clause } from "../core/clause";
import { Filter, FilterOperator } from "../core/filter";
import { AggregationType, RelationType } from "../common/types";
import { View } from "../core/view";
import { Pool, PoolConfig } from "pg";
......@@ -38,9 +40,6 @@ export class PostgresAdapter extends Adapter {
this.pool = new Pool(config);
}
public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void {
// buildQueryFromView does not put the final ;, it need to be put apart
// let query = this.buildQueryFromView(view, view.metrics, view.dimensions) + ";\n";
const materialized = this.searchMaterializedViews(view).sort((a, b) => {
return (a.id < b.id) ? -1 : 1;
});
......@@ -70,54 +69,6 @@ export class PostgresAdapter extends Adapter {
return false;
}
private getAggregateFunction(aggrType: AggregationType, origin: boolean): string {
switch (aggrType) {
case AggregationType.SUM:
return "SUM";
case AggregationType.AVG:
return "AVG";
case AggregationType.COUNT:
return (origin) ? "COUNT" : "SUM";
default:
return "";
}
}
private translateRelation(relation: RelationType, arg: string): string {
switch (relation) {
case RelationType.DAY:
return this.applyRelation("EXTRACT", ["DAY FROM "], [arg]);
case RelationType.MONTH:
return this.applyRelation("EXTRACT", ["MONTH FROM "], [arg]);
case RelationType.YEAR:
return this.applyRelation("EXTRACT", ["YEAR FROM "], [arg]);
case RelationType.DAYOFWEEK:
return this.applyRelation("EXTRACT", ["DOW FROM "], [arg]);
default:
return "";
}
}
private applyRelation(name: string, args: string[], values: string[]): string {
/*
This adapter uses the concept of functions in Postgres to
implement BLENDB sub-dimention relations, this functions
applys the transformation to build the call of a Postgres
funtion. Note that this function can be native from postgres,
like EXTRACT, or even implemented on the database.
This function is short and only used in the translateRelation
method however is a bit complex and is possible to be used
several times, because of that is puted appart to make easyer update
and avoid problems
Example
applyRelation ("EXTRACT", "["DAY FROM"]", ["view_0.date"])
output: EXTRACT(DAY FROM view_0.date)
*/
return name + "(" + args.map((item, idx) => item + values[idx]).join(",") + ")";
}
private searchMaterializedViews(view: View): View[] {
let r: View[] = [];
if (view.materialized) {
......@@ -137,9 +88,11 @@ export class PostgresAdapter extends Adapter {
private buildQuery(target: View, views: View[]) {
const metrics = target.metrics;
const dimensions = target.dimensions;
const clauses = target.clauses;
let dimMap: {[key: string]: DimInfo} = {};
let metMap: {[key: string]: View[]} = {};
let nameMap: {[key: string]: View} = {};
for (let i = 0; i < views.length; ++i) {
const mets = views[i].metrics;
......@@ -147,6 +100,7 @@ export class PostgresAdapter extends Adapter {
for (let j = 0; j < mets.length; ++j) {
if (!metMap[mets[j].name]) {
metMap[mets[j].name] = [views[i]];
nameMap[mets[j].name] = views[i];
}
else {
......@@ -160,6 +114,7 @@ export class PostgresAdapter extends Adapter {
dim: dims[j],
views: [views[i]]
};
nameMap[dims[j].name] = views[i];
}
else {
......@@ -168,24 +123,23 @@ export class PostgresAdapter extends Adapter {
}
}
// Projection
const strMetrics = metrics.map((metric) => {
const view = metMap[metric.name][0];
const view = nameMap[metric.name];
let func = this.getAggregateFunction(metric.aggregation, view.origin);
let quotedName = "\"" + metric.name + "\"";
let extMetric = func + "(view_" + view.id + "." + quotedName + ")";
let extMetric = func + "(" + this.buildColumn(metric, view.id) + ")";
return extMetric + " AS " + quotedName;
});
const parsedDimensions = dimensions.map((dimension) => {
let dim = dimension;
while (!dimMap[dim.name]) {
// Checar exeção
while (!nameMap[dim.name]) {
dim = dim.parent;
}
const view = dimMap[dim.name].views[0];
const quotedDim = "\"" + dim.name + "\"";
const view = nameMap[dim.name];
const quotedName = "\"" + dimension.name + "\"";
let extDimension = "view_" + view.id + "." + quotedDim;
let extDimension = this.buildColumn(dim, view.id);
let aux = dimension;
while (aux.name !== dim.name) {
extDimension = this.translateRelation(aux.relation, extDimension);
......@@ -198,7 +152,8 @@ export class PostgresAdapter extends Adapter {
const grouped = parsedDimensions.map((item) => item.noalias);
const elements = strMetrics.concat(strDimensions);
let joins = [];
// Joins
let conds: string[] = [];
for (let i in dimMap) {
let remainViews = dimMap[i].views.slice();
let dim = dimMap[i].dim;
......@@ -207,15 +162,30 @@ export class PostgresAdapter extends Adapter {
while (remainViews.length > 0) {
const id = remainViews.shift().id;
const rightSide = this.buildColumn(dim, id);
joins.push(leftSide + " = " + rightSide);
conds.push(leftSide + " = " + rightSide);
}
}
}
// Selection
let covered: Clause[] = [];
for (let i = 0; i < views.length; ++i) {
// Get the clauses that children already cover
covered = covered.concat(views[i].clauses);
}
const toCover = clauses.filter((item) => !covered.some ((clause) => {
return clause.id === item.id;
}));
toCover.forEach((item) => conds.push("(" + this.translateClause(item, nameMap) + ")"));
// Assembly
const projection = "SELECT " + elements.join(",");
const source = " FROM " + views.map((view) => "view_" + view.id).join(",");
const selection = (joins.length > 0) ? " WHERE " + joins.join(" AND ") : "";
const selection = (conds.length > 0) ? " WHERE " + conds.join(" AND ") : "";
let grouping = "";
if (grouped.length > 0) {
grouping = " GROUP BY " + grouped.join(",");
......@@ -225,8 +195,93 @@ export class PostgresAdapter extends Adapter {
}
private getAggregateFunction(aggrType: AggregationType, origin: boolean): string {
switch (aggrType) {
case AggregationType.SUM:
return "SUM";
case AggregationType.AVG:
return "AVG";
case AggregationType.COUNT:
return (origin) ? "COUNT" : "SUM";
default:
return "";
}
}
private translateRelation(relation: RelationType, arg: string): string {
switch (relation) {
case RelationType.DAY:
return this.applyRelation("EXTRACT", ["DAY FROM "], [arg]);
case RelationType.MONTH:
return this.applyRelation("EXTRACT", ["MONTH FROM "], [arg]);
case RelationType.YEAR:
return this.applyRelation("EXTRACT", ["YEAR FROM "], [arg]);
case RelationType.DAYOFWEEK:
return this.applyRelation("EXTRACT", ["DOW FROM "], [arg]);
default:
return "";
}
}
private applyRelation(name: string, args: string[], values: string[]): string {
/*
This adapter uses the concept of functions in Postgres to
implement BLENDB sub-dimention relations, this functions
applys the transformation to build the call of a Postgres
funtion. Note that this function can be native from postgres,
like EXTRACT, or even implemented on the database.
This function is short and only used in the translateRelation
method however is a bit complex and is possible to be used
several times, because of that is puted appart to make easyer update
and avoid problems
Example
applyRelation ("EXTRACT", "["DAY FROM"]", ["view_0.date"])
output: EXTRACT(DAY FROM view_0.date)
*/
return name + "(" + args.map((item, idx) => item + values[idx]).join(",") + ")";
}
private buildColumn (item: Metric|Dimension, id: string): string {
const quotedName = "\"" + item.name + "\"";
return "view_" + id + "." + quotedName;
}
private translateClause(clause: Clause, map: {[key: string]: View}): string {
let r = clause.filters.map((item) => this.translateFilter(item, map));
return r.join(" OR ");
}
private translateFilter(filter: Filter, map: {[key: string]: View}): string {
const viewId = map[filter.target.name].id;
const leftSide = this.buildColumn(filter.target, viewId);
const op = this.translateOperator(filter.operator);
const dataType = this.translateDataType(filter.target.dataType);
const quotedValue = "'" + filter.value + "'";
return leftSide + op + quotedValue + dataType;
}
private translateOperator(op: FilterOperator): string {
switch (op) {
case FilterOperator.EQUAL:
return " = ";
case FilterOperator.NOTEQUAL:
return " != ";
default:
return "";
}
}
private translateDataType(dt: string ): string {
switch (dt) {
case "date":
return "::DATE";
case "integer":
return "::INTEGER";
default:
return "";
}
}
}
......@@ -28,6 +28,7 @@ import { Query } from "../../common/query";
interface StrQuery {
metrics: string;
dimensions: string;
filters?: string;
}
function parseQuery(obj: Query): StrQuery {
......@@ -94,4 +95,79 @@ describe("API data controller", () => {
.end(done);
});
it("should respond 200 and get some data, using a single filter", (done) => {
// Clause does not come to scenario besause is a lot of work for
// only a single test
let query = parseQuery(tests.clausal);
query.filters = "dim:7==1";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
expect(result).to.have.length(1);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(tests.clausal.metrics.map((item) => item.name));
keys = keys.concat(tests.clausal.dimensions.map((item) => item.name));
result.forEach((row: any) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
})
.end(done);
});
it("should respond 200 and get some data, using filters with OR", (done) => {
// Clause does not come to scenario besause is a lot of work for
// only a single test
let query = parseQuery(tests.clausal);
query.filters = "dim:7==1,dim:7==2";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
expect(result).to.have.length(2);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(tests.clausal.metrics.map((item) => item.name));
keys = keys.concat(tests.clausal.dimensions.map((item) => item.name));
result.forEach((row: any) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
})
.end(done);
});
it("should respond 200 and get some data, using filters with AND", (done) => {
// Clause does not come to scenario besause is a lot of work for
// only a single test
let query = parseQuery(tests.clausal);
query.filters = "dim:7!=1;dim:0!=2017-01-01";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
expect(result).to.have.length(4);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(tests.clausal.metrics.map((item) => item.name));
keys = keys.concat(tests.clausal.dimensions.map((item) => item.name));
result.forEach((row: any) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
})
.end(done);
});
});
......@@ -26,10 +26,14 @@ export class DataCtrl {
public static read(req: Request, res: express.Response, next: express.NextFunction) {
let metrics = req.query.metrics.split(",").filter((item: string) => item !== "");
let dimensions = req.query.dimensions.split(",").filter((item: string) => item !== "");
let clauses = [];
if (req.query.filters) {
clauses = req.query.filters.split(";").filter((item: string) => item !== "");
}
let view;
try {
let query: Query = { metrics: [], dimensions: [] };
let query: Query = { metrics: [], dimensions: [], clauses: [] };
for (let i = 0; i < metrics.length; ++i) {
query.metrics.push(req.engine.getMetricByName(metrics[i]));
}
......@@ -37,6 +41,10 @@ export class DataCtrl {
for (let i = 0; i < dimensions.length; ++i) {
query.dimensions.push(req.engine.getDimensionByName(dimensions[i]));
}
for (let i = 0; i < clauses.length; ++i) {
query.clauses.push(req.engine.parseClause(clauses[i]));
}
view = req.engine.query(query);
}
catch (e) {
......
......@@ -20,9 +20,10 @@
import { Metric } from "../core/metric";
import { Dimension } from "../core/dimension";
import { Clause } from "../core/clause";
export interface Query {
public metrics: Metric[];
public dimensions: Dimension[];
public clauses?: Clause[];
}
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { Filter } from "./filter";
import { Hash } from "../util/hash";
export interface ClauseOptions {
filters: Filter[];
}
export class Clause {
public readonly id: string;
public readonly filters: Filter[];
constructor (options: ClauseOptions) {
this.filters = options.filters;
const filtersIds = this.filters.map((item) => item.id);
this.id = Hash.sha1(filtersIds.sort());
}
}
......@@ -23,6 +23,7 @@ import { expect } from "chai";
import { Engine } from "./engine";
import { Metric } from "./metric";
import { Dimension } from "./dimension";
import { FilterOperator } from "./filter";
import { View } from "./view";
import { engineScenario } from "../../test/scenario";
......@@ -182,4 +183,91 @@ describe("engine class", () => {
}
expect(error).to.be.true;
});
it("should parse a clause, with target as dimension and operator as ==", () => {
const strFilter = "dim:0==0";
const clause = engine.parseClause(strFilter);
expect(clause).to.be.an("object");
expect(clause).to.have.property("filters");
expect(clause).to.have.property("id");
expect(clause.filters).to.be.an("array");
expect(clause.filters).to.have.length(1);
expect(clause.filters[0]).to.have.property("id");
expect(clause.filters[0]).to.have.property("target");
expect(clause.filters[0]).to.have.property("value");
expect(clause.filters[0]).to.have.property("operator");
expect(clause.filters[0].target).to.be.equal(dim[0]);
expect(clause.filters[0].value).to.be.equal("0");
expect(clause.filters[0].operator).to.be.equal(FilterOperator.EQUAL);
});
it("should parse a clause, with target as metric and operator as !=", () => {
const strFilter = "met:0!=0";
const clause = engine.parseClause(strFilter);
expect(clause).to.be.an("object");
expect(clause).to.have.property("filters");
expect(clause).to.have.property("id");
expect(clause.filters).to.be.an("array");
expect(clause.filters).to.have.length(1);
expect(clause.filters[0]).to.have.property("id");
expect(clause.filters[0]).to.have.property("target");
expect(clause.filters[0]).to.have.property("operator");
expect(clause.filters[0]).to.have.property("value");
expect(clause.filters[0].target).to.be.equal(met[0]);
expect(clause.filters[0].value).to.be.equal("0");
expect(clause.filters[0].operator).to.be.equal(FilterOperator.NOTEQUAL);
});
it("should throw an exception, when a dimension is not found", () => {
let error: boolean = false;
const strFilter = "dim:-1==0";
const exeption = "Filter could not be created: \"dim:-1\" was not found";
try {
engine.parseClause(strFilter);
}
catch (e){
error = true;
expect(e.message).to.be.equal(exeption);
}
expect(error).to.be.true;
});
it("should throw an exception, when a metric is not found", () => {
let error: boolean = false;
const strFilter = "met:-1==0";
const exeption = "Filter could not be created: \"met:-1\" was not found";
try {
engine.parseClause(strFilter);
}
catch (e){
error = true;
expect(e.message).to.be.equal(exeption);
}
expect(error).to.be.true;
});
it("should throw an exception, when a operator is not found", () => {
let error: boolean = false;