Issue #15: Connect API with Engine

Signed-off-by: Lucas Fernandes de Oliveira's avatarLucas Fernandes de Oliveira <lfo14@inf.ufpr.br>
parent 4ce265d9
Pipeline #9901 passed with stage
in 52 seconds
......@@ -38,10 +38,12 @@
"typescript": "^2.0.3"
},
"devDependencies": {
"@types/supertest": "^2.0.0",
"chai": "^3.4.33",
"istanbul": "1.1.0-alpha.1",
"mocha": "^3.1.0",
"raml2html": "^3.0.1",
"supertest": "^3.0.0",
"tslint": "^3.15.1",
"tslint-stylish": "^2.1.0-beta"
}
......
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import * as request from "supertest";
import { expect } from "chai";
import * as server from "../../main";
describe("API collect controller", () => {
it("should respond 400 when _id field is in request body", (done) => {
request(server)
.post("/v1/collect/class")
.send({"_id": 1})
.expect(400)
.expect((res: any) => {
const message = "Property named \"_id\" is protected.";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
expect(res.body.message).to.be.eql(message);
})
.end(done);
});
it("should respond 500 on error on writing in the database", (done) => {
request(server)
.post("/v1/collect/class")
.send({"field1": 1, "field2": 2})
.expect(500)
.expect((res: any) => {
const message = "Error while writing to the database.";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
expect(res.body.message).to.be.eql(message);
})
.end(done);
});
});
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import * as request from "supertest";
import { expect } from "chai";
import * as server from "../../main";
import { dataCtrlScenario as tests } from "../../../test/scenario";
import { Query } from "../../common/query";
interface StrQuery {
metrics: string;
dimensions: string;
}
function parseQuery(obj: Query): StrQuery {
let r: StrQuery = {
metrics: "",
dimensions: ""
};
r.metrics = obj.metrics.map ((item) => item.name).join(",");
r.dimensions = obj.dimensions.map ((item) => item.name).join(",");
return r;
}
describe("API data controller", () => {
it("should respond 500 when query has inexistent metric", (done) => {
request(server)
.get("/v1/data")
.query(parseQuery(tests.wrongMet))
.expect(500)
.expect((res: any) => {
const message = "Query execution failed: " +
"Could not construct query with the paramters given.";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
expect(res.body.message).to.be.eql(message);
})
.end(done);
});
it("should respond 500 when query has inexistent dimension", (done) => {
request(server)
.get("/v1/data")
.query(parseQuery(tests.wrongDim))
.expect(500)
.expect((res: any) => {
const message = "Query execution failed: " +
"Could not construct query with the paramters given.";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
expect(res.body.message).to.be.eql(message);
})
.end(done);
});
it("should respond 200 and get some data", (done) => {
request(server)
.get("/v1/data")
.query(parseQuery(tests.correct))
.expect(200)
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
expect(result).to.have.length(5);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(tests.correct.metrics.map((item) => item.name));
keys = keys.concat(tests.correct.dimensions.map((item) => item.name));
result.forEach((row: any) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
})
.end(done);
});
});
......@@ -19,13 +19,41 @@
*/
import * as express from "express";
import { Request } from "../types";
import { Query } from "../../common/query";
export class DataCtrl {
public static read(req: express.Request, res: express.Response, next: express.NextFunction) {
public static read(req: Request, res: express.Response, next: express.NextFunction) {
let metrics = req.query.metrics.split(",");
let dimensions = req.query.dimensions.split(",");
let view;
res.status(500).json({ message: "Query execution failed " +
"because of an unknown error." });
try {
let query: Query = { metrics: [], dimensions: [] };
for (let i = 0; i < metrics.length; ++i) {
query.metrics.push(req.engine.getMetricByName(metrics[i]));
}
for (let i = 0; i < metrics.length; ++i) {
query.dimensions.push(req.engine.getDimensionByName(dimensions[i]));
}
view = req.engine.query(query);
}
catch (e) {
res.status(500).json({ message: "Query execution failed: " +
"Could not construct query with the paramters given." });
return;
}
req.adapter.getDataFromView(view, (err: Error, result: any[]) => {
if (err) {
res.status(500).json({ message: "Query execution failed " +
"failed on execute query on database." });
return;
}
res.status(200).json(result);
return;
});
}
}
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { Middleware } from "../types";
import { Adapter } from "../../core/adapter";
import { PostgresAdapter } from "../../adapter/postgres";
import { PoolConfig } from "pg";
export function PostgresMw(config: PoolConfig): Middleware {
let adapter: Adapter = new PostgresAdapter(config);
return function postgresMiddleware(req, res, next) {
req.adapter = adapter;
next();
};
}
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { Engine } from "../../core/engine";
import { ParsedConfig } from "../../util/configParser";
import { Middleware } from "../types";
export function EngineMw (config: ParsedConfig): Middleware {
let engine: Engine = new Engine();
config.metrics.forEach ((met) => engine.addMetric(met));
config.dimensions.forEach ((dim) => engine.addDimension(dim));
config.views.forEach ((view) => engine.addView(view));
return function engineMiddleware(req, res, next) {
req.engine = engine;
next();
};
}
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import * as express from "express";
import { Engine } from "../core/engine";
import { Adapter} from "../core/adpater";
export interface Request extends express.Request {
engine: Engine;
adapter: Adapter;
}
export interface Middleware {
(req: Request, res: express.Response, next: express.NextFunction): void;
}
......@@ -30,6 +30,19 @@ import { router } from "./api/router-v1";
// create a new express app
const app = module.exports = express();
// Load configuration file
import { ConfigParser } from "./util/configParser";
const configPath = (app.get("env") === "test") ? "config/test.yaml" : "config/config.yaml";
const config = ConfigParser.parse(configPath);
// include middlewares
import { EngineMw } from "./api/middlewares/engine";
import { PostgresMw } from "./api/middlewares/adapter";
app.use(EngineMw(config));
app.use(PostgresMw(config.connection));
// parse the RAML spec and load osprey middleware
ramlParser.loadFile("specs/blendb-api-v1.raml")
.then((raml: any) => {
......
......@@ -21,6 +21,7 @@
import { Metric, MetricOptions } from "../core/metric";
import { Dimension, DimensionOptions } from "../core/dimension";
import { View, ViewOptions, LoadView } from "../core/view";
import { AggregationType, RelationType } from "../common/types";
import { PoolConfig } from "pg";
import * as fs from "fs";
import * as yaml from "js-yaml";
......@@ -32,10 +33,23 @@ export interface ViewParsingOptions {
metrics: string[];
}
interface MetricStrOptions {
name: string;
aggregation: string;
dataType: string;
}
interface DimensionStrOptions {
name: string;
dataType: string;
parent?: string;
relation?: string;
}
interface ConfigSchema {
views: ViewParsingOptions[];
metrics: MetricOptions[];
dimensions: DimensionOptions[];
metrics: MetricStrOptions[];
dimensions: DimensionStrOptions[];
}
interface ConfigFile {
......@@ -80,18 +94,21 @@ export class ConfigParser {
let dimMap: Map<string, Dimension> = new Map();
for (let i = 0; i < metricsOpts.length; ++i) {
let met = new Metric(metricsOpts[i]);
let met = new Metric(this.parseMetOpts(metricsOpts[i]));
parsed.metrics.push(met);
metMap.set(met.name, met);
}
for (let i = 0; i < dimensionsOpts.length; ++i) {
let dim = new Dimension(dimensionsOpts[i]);
let dim = new Dimension(this.parseDimOpts(dimensionsOpts[i]));
parsed.dimensions.push(dim);
dimMap.set(dim.name, dim);
}
for (let i = 0; i < viewsOpts.length; ++i) {
if (!viewsOpts[i].metrics) {
viewsOpts[i].metrics = [];
}
let viewOpts = ConfigParser.parseViewOpt(viewsOpts[i], metMap, dimMap);
let view = new View(viewOpts);
parsed.views.push(view);
......@@ -134,4 +151,37 @@ export class ConfigParser {
}
return viewOpt;
}
private static parseDimOpts (opts: DimensionStrOptions): DimensionOptions {
if (opts.parent || opts.relation) {
throw new Error("Sub dimensions not implemented in config parser !!!");
}
return {
name: opts.name,
dataType: opts.dataType,
parent: null,
relation: RelationType.NONE
};
}
private static parseMetOpts (opts: MetricStrOptions): MetricOptions {
return {
name: opts.name,
aggregation: this.strToAggregationType(opts.aggregation),
dataType : opts.dataType
};
}
private static strToAggregationType (str: string): AggregationType {
switch (str) {
case "sum":
return AggregationType.SUM;
case "avg":
return AggregationType.AVG;
case "count":
return AggregationType.COUNT;
default:
return AggregationType.NONE;
}
}
}
......@@ -18,10 +18,12 @@
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { ConfigParser } from "../src/util/configParser";
import { Metric } from "../src/core/metric";
import { Dimension } from "../src/core/dimension";
import { View } from "../src/core/view";
import { AggregationType, RelationType } from "../src/common/types";
import { Query} from "../src/common/query";
interface EngineScenario {
metrics: Metric[];
......@@ -39,31 +41,17 @@ interface AdapterScenario {
dateView: View;
}
const mets = [
new Metric({ name: "met:1", aggregation: AggregationType.SUM, dataType: "integer" }),
new Metric({ name: "met:2", aggregation: AggregationType.AVG, dataType: "integer" }),
new Metric({ name: "met:3", aggregation: AggregationType.AVG, dataType: "integer" }),
new Metric({ name: "met:4", aggregation: AggregationType.SUM, dataType: "integer" }),
new Metric({ name: "met:5", aggregation: AggregationType.SUM, dataType: "integer" }),
new Metric({ name: "met:6", aggregation: AggregationType.AVG, dataType: "integer" }),
new Metric({ name: "met:7", aggregation: AggregationType.COUNT, dataType: "integer" }),
new Metric({ name: "met:8", aggregation: AggregationType.COUNT, dataType: "integer" }),
new Metric({ name: "met:9", aggregation: AggregationType.SUM, dataType: "integer" }),
new Metric({ name: "met:10", aggregation: AggregationType.COUNT, dataType: "integer" }),
];
interface DataCtrlScenario {
wrongMet: Query;
wrongDim: Query;
correct: Query;
}
const dims = [
new Dimension({ name: "dim:1", dataType: "date" }),
new Dimension({ name: "dim:2", dataType: "date" }),
new Dimension({ name: "dim:3", dataType: "integer" }),
new Dimension({ name: "dim:4", dataType: "string" }),
new Dimension({ name: "dim:5", dataType: "string" }),
new Dimension({ name: "dim:6", dataType: "boolean" }),
new Dimension({ name: "dim:7", dataType: "integer" }),
new Dimension({ name: "dim:8", dataType: "integer" }),
new Dimension({ name: "dim:9", dataType: "date" }),
new Dimension({ name: "dim:10", dataType: "string" }),
];
const config = ConfigParser.parse("config/test.yaml");
const mets = config.metrics;
const dims = config.dimensions;
const views = config.views;
const wrongMet = new Metric({
name: "met:11",
......@@ -134,54 +122,6 @@ const dateSubDim = [
}),
];
const views = [
new View({
metrics: [mets[0], mets[1], mets[2]],
dimensions: [dims[0], dims[1]],
materialized: true
}),
new View({
metrics: [mets[0], mets[2], mets[4]],
dimensions: [dims[0], dims[1]],
materialized: true
}),
new View({
metrics: [mets[2], mets[3], mets[6]],
dimensions: [dims[3], dims[4]],
materialized: true
}),
new View({
metrics: [mets[5], mets[6]],
dimensions: [dims[2], dims[3], dims[4], dims[5]],
materialized: true
}),
new View({
metrics: [mets[7], mets[1], mets[2]],
dimensions: [dims[0], dims[1], dims[6]],
materialized: true
}),
new View({
metrics: [mets[0], mets[3]],
dimensions: [dims[0], dims[1]],
materialized: true
}),
new View({
metrics: [mets[7]],
dimensions: [dims[7], dims[8], dims[9]],
materialized: true
}),
new View({
metrics: [mets[8]],
dimensions: [dims[7], dims[8], dims[9]],
materialized: true
}),
new View({
metrics: [mets[9]],
dimensions: [dims[7], dims[8], dims[9]],
materialized: true
})
];
const engAuxView = [
new View({
metrics: [mets[0], mets[1], mets[2], mets[3], mets[4]],
......@@ -356,3 +296,9 @@ export const adapterScenario: AdapterScenario = {
join4View: join4View,
dateView: dateView
};
export const dataCtrlScenario: DataCtrlScenario = {
wrongMet: { metrics: [wrongMet], dimensions: [dims[0]] },
wrongDim: { metrics: [mets[0]], dimensions: [wrongDim] },
correct: { metrics: [mets[0]], dimensions: [dims[0]] }
};
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment