Issue #23: Add filters to API

Signed-off-by: Lucas Fernandes de Oliveira's avatarLucas Fernandes de Oliveira <lfo14@inf.ufpr.br>
parent 42215793
Pipeline #11266 passed with stage
in 43 seconds
......@@ -169,4 +169,76 @@ describe("postgres adapter", () => {
done();
});
});
it("should get data from view when a single clause exists", (done) => {
let view = adapterScenario.clauseView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(1);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
expect(parseInt(result[0]["met:0"], 10)).to.be.equal(1);
expect(parseInt(result[0]["met:1"], 10)).to.be.equal(1);
expect(parseInt(result[0]["met:2"], 10)).to.be.equal(1);
expect(result[0]["dim:0"].getDate()).to.be.equal(1);
done();
});
});
it("should get data from view with single clause, with more than on filter", (done) => {
let view = adapterScenario.multiFilterView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(2);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from view with multiple clauses", (done) => {
let view = adapterScenario.multiClauseView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(0);
done();
});
});
it("should get data from view with a clause with not equal operator", (done) => {
let view = adapterScenario.notEqualView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(4);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
});
This diff is collapsed.
......@@ -28,6 +28,7 @@ import { Query } from "../../common/query";
interface StrQuery {
metrics: string;
dimensions: string;
filters?: string;
}
function parseQuery(obj: Query): StrQuery {
......@@ -94,4 +95,79 @@ describe("API data controller", () => {
.end(done);
});
it("should respond 200 and get some data, using a single filter", (done) => {
// Clause does not come to scenario besause is a lot of work for
// only a single test
let query = parseQuery(tests.clausal);
query.filters = "dim:7==1";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
expect(result).to.have.length(1);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(tests.clausal.metrics.map((item) => item.name));
keys = keys.concat(tests.clausal.dimensions.map((item) => item.name));
result.forEach((row: any) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
})
.end(done);
});
it("should respond 200 and get some data, using filters with OR", (done) => {
// Clause does not come to scenario besause is a lot of work for
// only a single test
let query = parseQuery(tests.clausal);
query.filters = "dim:7==1,dim:7==2";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
expect(result).to.have.length(2);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(tests.clausal.metrics.map((item) => item.name));
keys = keys.concat(tests.clausal.dimensions.map((item) => item.name));
result.forEach((row: any) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
})
.end(done);
});
it("should respond 200 and get some data, using filters with AND", (done) => {
// Clause does not come to scenario besause is a lot of work for
// only a single test
let query = parseQuery(tests.clausal);
query.filters = "dim:7!=1;dim:0!=2017-01-01";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
expect(result).to.have.length(4);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(tests.clausal.metrics.map((item) => item.name));
keys = keys.concat(tests.clausal.dimensions.map((item) => item.name));
result.forEach((row: any) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
})
.end(done);
});
});
......@@ -26,10 +26,14 @@ export class DataCtrl {
public static read(req: Request, res: express.Response, next: express.NextFunction) {
let metrics = req.query.metrics.split(",").filter((item: string) => item !== "");
let dimensions = req.query.dimensions.split(",").filter((item: string) => item !== "");
let clauses = [];
if (req.query.filters) {
clauses = req.query.filters.split(";").filter((item: string) => item !== "");
}
let view;
try {
let query: Query = { metrics: [], dimensions: [] };
let query: Query = { metrics: [], dimensions: [], clauses: [] };
for (let i = 0; i < metrics.length; ++i) {
query.metrics.push(req.engine.getMetricByName(metrics[i]));
}
......@@ -37,6 +41,10 @@ export class DataCtrl {
for (let i = 0; i < dimensions.length; ++i) {
query.dimensions.push(req.engine.getDimensionByName(dimensions[i]));
}
for (let i = 0; i < clauses.length; ++i) {
query.clauses.push(req.engine.parseClause(clauses[i]));
}
view = req.engine.query(query);
}
catch (e) {
......
......@@ -20,9 +20,10 @@
import { Metric } from "../core/metric";
import { Dimension } from "../core/dimension";
import { Clause } from "../core/clause";
export interface Query {
public metrics: Metric[];
public dimensions: Dimension[];
public clauses?: Clause[];
}
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { Filter } from "./filter";
import { Hash } from "../util/hash";
export interface ClauseOptions {
filters: Filter[];
}
export class Clause {
public readonly id: string;
public readonly filters: Filter[];
constructor (options: ClauseOptions) {
this.filters = options.filters;
const filtersIds = this.filters.map((item) => item.id);
this.id = Hash.sha1(filtersIds.sort());
}
}
......@@ -23,6 +23,7 @@ import { expect } from "chai";
import { Engine } from "./engine";
import { Metric } from "./metric";
import { Dimension } from "./dimension";
import { FilterOperator } from "./filter";
import { View } from "./view";
import { engineScenario } from "../../test/scenario";
......@@ -182,4 +183,91 @@ describe("engine class", () => {
}
expect(error).to.be.true;
});
it("should parse a clause, with target as dimension and operator as ==", () => {
const strFilter = "dim:0==0";
const clause = engine.parseClause(strFilter);
expect(clause).to.be.an("object");
expect(clause).to.have.property("filters");
expect(clause).to.have.property("id");
expect(clause.filters).to.be.an("array");
expect(clause.filters).to.have.length(1);
expect(clause.filters[0]).to.have.property("id");
expect(clause.filters[0]).to.have.property("target");
expect(clause.filters[0]).to.have.property("value");
expect(clause.filters[0]).to.have.property("operator");
expect(clause.filters[0].target).to.be.equal(dim[0]);
expect(clause.filters[0].value).to.be.equal("0");
expect(clause.filters[0].operator).to.be.equal(FilterOperator.EQUAL);
});
it("should parse a clause, with target as metric and operator as !=", () => {
const strFilter = "met:0!=0";
const clause = engine.parseClause(strFilter);
expect(clause).to.be.an("object");
expect(clause).to.have.property("filters");
expect(clause).to.have.property("id");
expect(clause.filters).to.be.an("array");
expect(clause.filters).to.have.length(1);
expect(clause.filters[0]).to.have.property("id");
expect(clause.filters[0]).to.have.property("target");
expect(clause.filters[0]).to.have.property("operator");
expect(clause.filters[0]).to.have.property("value");
expect(clause.filters[0].target).to.be.equal(met[0]);
expect(clause.filters[0].value).to.be.equal("0");
expect(clause.filters[0].operator).to.be.equal(FilterOperator.NOTEQUAL);
});
it("should throw an exception, when a dimension is not found", () => {
let error: boolean = false;
const strFilter = "dim:-1==0";
const exeption = "Filter could not be created: \"dim:-1\" was not found";
try {
engine.parseClause(strFilter);
}
catch (e){
error = true;
expect(e.message).to.be.equal(exeption);
}
expect(error).to.be.true;
});
it("should throw an exception, when a metric is not found", () => {
let error: boolean = false;
const strFilter = "met:-1==0";
const exeption = "Filter could not be created: \"met:-1\" was not found";
try {
engine.parseClause(strFilter);
}
catch (e){
error = true;
expect(e.message).to.be.equal(exeption);
}
expect(error).to.be.true;
});
it("should throw an exception, when a operator is not found", () => {
let error: boolean = false;
let strFilter = "met:-1=?0";
let exeption = "Filter could not be created: Operator on \"" + strFilter + "\" could not be extracted";
try {
engine.parseClause(strFilter);
}
catch (e){
error = true;
expect(e.message).to.be.equal(exeption);
}
expect(error).to.be.true;
error = false;
strFilter = "met:-1!?0";
exeption = "Filter could not be created: Operator on \"" + strFilter + "\" could not be extracted";
try {
engine.parseClause(strFilter);
}
catch (e){
error = true;
expect(e.message).to.be.equal(exeption);
}
expect(error).to.be.true;
});
});
......@@ -20,6 +20,8 @@
import { Dimension } from "./dimension";
import { Metric } from "./metric";
import { Clause } from "./clause";
import { Filter } from "./filter";
import { View } from "./view";
import { Query } from "../common/query";
import { Graph } from "../util/graph";
......@@ -88,12 +90,56 @@ export class Engine {
return result;
}
public parseClause(strClause: string): Clause {
let strFilters = strClause.split(",").filter((item: string) => item !== "");
let filters: Filter[] = [];
for (let i = 0; i < strFilters.length; ++i) {
filters.push(this.parseFilter(strFilters[i]));
}
return new Clause({filters: filters});
}
public parseFilter(strFilter: string): Filter {
let segment = Filter.segment(strFilter);
if (segment) {
// Segment never returns NONE
let op = Filter.parseOperator(segment.operator);
let target: Metric|Dimension = null;
try {
target = this.getDimensionByName(segment.target);
}
catch (e) {
try {
target = this.getMetricByName(segment.target);
}
catch (e) {
target = null;
}
}
if (!target) {
throw new Error("Filter could not be created: \"" + segment.target + "\" was not found");
}
return new Filter({
target: target,
operator: op,
value: segment.value
});
}
else {
throw new Error("Filter could not be created: Operator on \"" + strFilter + "\" could not be extracted");
}
}
public query (q: Query) {
return this.selectOptimalView(q);
}
private selectOptimalView (q: Query) {
let optimalViews = this.graph.cover(q.metrics, q.dimensions);
let optimalViews = this.graph.cover(q);
if (optimalViews.length === 0) {
throw new Error ("Engine views cannot cover the query");
}
......@@ -111,6 +157,7 @@ export class Engine {
let options = {
metrics: q.metrics,
dimensions: q.dimensions,
clauses: ((q.clauses) ? q.clauses : []),
materialized: false,
origin: false, // Never a dynamic generated view will be origin
childViews: optimalViews
......
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { expect } from "chai";
import { FilterOperator, Filter } from "./filter";
describe("filter class", () => {
it("should correctly parse the operators", () => {
expect(Filter.parseOperator("==")).to.be.equal(FilterOperator.EQUAL);
expect(Filter.parseOperator("!=")).to.be.equal(FilterOperator.NOTEQUAL);
expect(Filter.parseOperator("?=")).to.be.equal(FilterOperator.NONE);
});
});
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { Dimension } from "./dimension";
import { Metric } from "./metric";
import { Hash } from "../util/hash";
export interface FilterOptions {
target: Metric|Dimension;
operator: FilterOperator;
value: string;
}
export interface StrFilterOptions {
target: string;
operator: string;
value: string;
}
export enum FilterOperator {
NONE,
EQUAL,
NOTEQUAL
}
export class Filter {
public readonly id: string;
public readonly target: Metric|Dimension;
public readonly operator: FilterOperator;
public readonly value: string;
constructor (options: FilterOptions) {
this.target = options.target;
this.operator = options.operator;
this.value = options.value;
this.id = Hash.sha1(options.target.name + options.operator + options.value);
}
public static parseOperator(op: string): FilterOperator {
switch (op) {
case "==":
return FilterOperator.EQUAL;
case "!=":
return FilterOperator.NOTEQUAL;
default:
return FilterOperator.NONE;
}
}
public static segment(strFilter: string): StrFilterOptions {
for (let i = 0; i < strFilter.length; ++i) {
if (strFilter[i] === "=") {
if (strFilter[i + 1] === "=") {
return {
target: strFilter.slice(0, i),
operator: "==",
value: strFilter.slice(i + 2)
};
}
}
if (strFilter[i] === "!") {
if (strFilter[i + 1] === "=") {
return {
target: strFilter.slice(0, i),
operator: "!=",
value: strFilter.slice(i + 2)
};
}
}
}
return null;
}
}
......@@ -21,6 +21,7 @@
import { Dimension } from "./dimension";
import { Metric } from "./metric";
import { Hash } from "../util/hash";
import { Clause } from "./clause";
export interface LoadView {
view: View;
......@@ -31,6 +32,7 @@ export interface ViewOptions {
metrics: Metric[];
dimensions: Dimension[];
origin: boolean;
clauses?: Clause[];
materialized?: boolean;
childViews?: View[];
}
......@@ -39,6 +41,7 @@ export class View {
public readonly id: string;
public readonly metrics: Metric[];
public readonly dimensions: Dimension[];
public readonly clauses: Clause[];
public readonly materialized: boolean;
public readonly origin: boolean;
public childViews: View[];
......@@ -46,13 +49,16 @@ export class View {
constructor (options: ViewOptions) {
this.metrics = options.metrics.sort();
this.dimensions = options.dimensions.sort();
this.clauses = (options.clauses) ? options.clauses.sort() : [];
this.materialized = options.materialized || false;
this.origin = options.origin || false;
this.childViews = (options.childViews) ? options.childViews : [];
// calculate the id of the view based on it's metrics and dimensions
let metricsNames = options.metrics.map(metric => metric.name);
let dimensionsNames = options.dimensions.map(dimension => dimension.name);
this.id = Hash.sha1(metricsNames.concat(dimensionsNames).sort());
const metNames = this.metrics.map((metric) => metric.name);
const dimNames = this.dimensions.map((dimension) => dimension.name);
const clausesIds = this.clauses.map((clause) => clause.id);
const join = metNames.concat(dimNames).concat(clausesIds).sort();
this.id = Hash.sha1(join);
}
}
......@@ -23,8 +23,11 @@ import { expect } from "chai";
import { Metric } from "../core/metric";
import { Dimension } from "../core/dimension";
import { View } from "../core/view";
import { Filter, FilterOperator } from "../core/filter";
import { Clause } from "../core/clause";
import { Graph } from "./graph";
import { AggregationType, RelationType } from "../common/types";
import { Query } from "../common/query";
describe("graph class", () => {
......@@ -197,7 +200,8 @@ describe("graph class", () => {
});
expect(g.addView(view)).to.be.true;
let children = g.cover([], [dim]);
const query: Query = { metrics: [], dimensions: [dim] };
let children = g.cover(query);
expect(children).to.be.an("array");
expect(children).to.have.length(1);
expect(children[0].id).to.be.equal(view.id);
......@@ -266,7 +270,8 @@ describe("graph class", () => {
expect(g.addView(views[i])).to.be.true;
}
let children = g.cover([mets[0], mets[1]], [dims[0], dims[1]]);
const query: Query = { metrics: [mets[0], mets[1]], dimensions: [dims[0], dims[1]] };
let children = g.cover(query);
expect(children).to.be.an("array");
expect(children).to.have.length(1);
expect(children[0].id).to.be.equal(views[views.length - 1].id);
......@@ -305,7 +310,8 @@ describe("graph class", () => {
expect(g.addView(view)).to.be.true;
let children = g.cover([], [dims[1], dims[2]]);
const query: Query = { metrics: [], dimensions: [dims[1], dims[2]] };
let children = g.cover(query);
expect(children).to.be.an("array");
expect(children).to.have.length(1);
expect(children[0].id).to.be.equal(view.id);
......@@ -344,8 +350,70 @@ describe("graph class", () => {
expect(g.addView(view)).to.be.true;
let children = g.cover([], []);
const query: Query = { metrics: [], dimensions: [] };
let children = g.cover(query);
expect(children).to.be.an("array");
expect(children).to.be.empty;
});
it("should cover the graph, even when a constraint edge can not be used", () => {
let dims = [
new Dimension({name: "dim:0", dataType: "date"}),
new Dimension({name: "dim:1", dataType: "date"}),
new Dimension({name: "dim:2", dataType: "date"}),
];
let filter1 = new Filter({
target: dims[0],
operator: FilterOperator.EQUAL,
value: "01/01/01"
});
let filter2 = new Filter({
target: dims[0],
operator: FilterOperator.EQUAL,
value: "01/01/02"
});
let clause1 = new Clause({filters: [filter1]});
let clause2 = new Clause({filters: [filter2]});
let g = new Graph();
for (let i = 0; i < 3; ++i) {
expect(g.addDimension(dims[i])).to.be.true;
}
let view1 = new View({
metrics: [],
dimensions: [dims[0], dims[1]],
origin: true,
materialized: true
});
let view2 = new View({
metrics: [],
dimensions: [dims[1], dims[2]],
origin: true,
materialized: true
});
let view3 = new View({
metrics: [],
dimensions: dims,
origin: false,
materialized: false,
childViews: [view1, view2],
clauses: [clause1]
});
expect(g.addView(view1)).to.be.true;
expect(g.addView(view2)).to.be.true;
expect(g.addView(view3)).to.be.true;
const query: Query = { metrics: [], dimensions: dims, clauses: [clause2] };
let children = g.cover(query);