Commit d7beb242 authored by Lucas Fernandes de Oliveira's avatar Lucas Fernandes de Oliveira

Merge branch 'issue/56' into 'master'

Issue #56: Add monet adapter

See merge request !52
parents b3310950 247adaa2
Pipeline #14259 passed with stages
in 1 minute and 4 seconds
......@@ -211,9 +211,3 @@ dimensions:
parent: "dim:0"
relation: "year"
description: "A dimension of Blendb. Has 1 possible value."
-
name: "dim:12"
dataType: "integer"
parent: "dim:0"
relation: "dayofweek"
description: "A dimension of Blendb. Has 7 possible values."
/*
* Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { SQLAdapter } from "./sql";
import { View } from "../core/view";
import { FilterOperator } from "../core/filter";
const MDB = require("monetdb")();
export interface MonetConfig {
host: string;
port: number;
dbname: string;
user: string;
password: string;
}
interface MonetResult {
data: any[];
structure: {type: string, column: string, index: number}[];
}
export class MonetAdapter extends SQLAdapter {
private config: MonetConfig;
constructor (conf: MonetConfig) {
super();
this.config = conf;
}
public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void {
const query = this.getQueryFromView(view);
let pool: any = new MDB(this.config);
pool.connect();
pool.query(query).then((result: MonetResult) => {
if (result) {
let res = result.data.map((item) => {
let obj: any = {};
for (let i = 0; i < result.structure.length; ++i) {
let struct = result.structure[i];
if (struct.type === "timestamp") {
obj[struct.column] = new Date(item[struct.index]);
}
else {
obj[struct.column] = item[struct.index];
}
}
return obj;
});
cb(null, res);
}
else {
cb(null, null);
}
}).fail((err: Error) => {
cb(err, null);
});
pool.close();
}
public materializeView(view: View): boolean {
return false;
}
protected typeCast(quotedValue: string, dt: string): string {
switch (dt) {
case "date":
return "CAST(" + quotedValue + " AS TIMESTAMP)";
case "integer":
return "CAST(" + quotedValue + " AS INTEGER)";
case "boolean":
return "CAST(" + quotedValue + " AS BOOLEAN)";
default:
return quotedValue;
}
}
protected applyOperator(lSide: string, rSide: string, op: FilterOperator): string {
switch (op) {
case FilterOperator.EQUAL:
return lSide + " = " + rSide;
case FilterOperator.NOTEQUAL:
return "NOT(" + lSide + " = " + rSide + ")";
case FilterOperator.GREATER:
return lSide + " > " + rSide;
case FilterOperator.LOWER:
return lSide + " < " + rSide;
case FilterOperator.GREATEREQ:
return lSide + " >= " + rSide;
case FilterOperator.LOWEREQ:
return lSide + " <= " + rSide;
default:
return "";
}
}
}
......@@ -21,8 +21,10 @@
import { expect } from "chai";
import { PostgresAdapter } from "./postgres";
import { MonetAdapter, MonetConfig } from "./monet";
import { Adapter } from "../core/adapter";
import { Fixture } from "../../test/postgres/fixture";
import { Fixture as FixPostgres } from "../../test/postgres/fixture";
import { Fixture as FixMonet } from "../../test/monet/fixture";
import { ConfigParser } from "../util/configParser";
import { adapterScenario } from "../../test/scenario";
......@@ -33,16 +35,39 @@ describe("postgres adapter", () => {
let config: any;
let adapter: Adapter;
let fixture;
before((done) => {
before(function (done) {
// Arrow function not used to get acces to this and skip the test
config = ConfigParser.parse("config/test.yaml");
fixture = new Fixture(config.connection);
fixture.load(config.loadViews, config.struct.create, (err) => {
if (err) {
throw err;
}
adapter = new PostgresAdapter(config.connection);
done();
});
if (config.adapter === "postgres") {
fixture = new FixPostgres(config.connection);
fixture.load(config.loadViews, config.struct.create, (err) => {
if (err) {
throw err;
}
adapter = new PostgresAdapter(config.connection);
done();
});
}
else if (config.adapter === "monet") {
fixture = new FixMonet(config.connection);
fixture.load(config.loadViews, config.struct.create, (err) => {
if (err) {
throw err;
}
let parsedConfig: MonetConfig = {
user: config.connection.user,
dbname: config.connection.database,
password: config.connection.password,
host: config.connection.host,
port: config.connection.port
};
adapter = new MonetAdapter(parsedConfig);
done();
});
}
else {
this.skip();
}
});
// Tests
it("should get data from single materialized view", (done) => {
......
This diff is collapsed.
This diff is collapsed.
......@@ -44,7 +44,7 @@ describe("API engine controller", () => {
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
expect(result).to.have.length(13);
expect(result).to.have.length(12);
})
.end(done);
});
......
......@@ -21,6 +21,7 @@
import { Middleware } from "../types";
import { Adapter } from "../../core/adapter";
import { PostgresAdapter } from "../../adapter/postgres";
import { MonetAdapter, MonetConfig } from "../../adapter/monet";
import { PoolConfig } from "pg";
import { Connection } from "../../util/configParser";
......@@ -41,3 +42,19 @@ export function PostgresMw(config: Connection): Middleware {
};
}
export function MonetMw(config: Connection): Middleware {
let parsedConfig: MonetConfig = {
user: config.user,
dbname: config.database,
password: config.password,
host: config.host,
port: config.port,
};
let adapter: Adapter = new MonetAdapter(parsedConfig);
return function monetMiddleware(req, res, next) {
req.adapter = adapter;
next();
};
}
......@@ -32,5 +32,4 @@ export enum RelationType {
DAY,
MONTH,
YEAR,
DAYOFWEEK
};
......@@ -79,8 +79,6 @@ export class Dimension {
return RelationType.MONTH;
case "year":
return RelationType.YEAR;
case "dayofweek":
return RelationType.DAYOFWEEK;
default:
return RelationType.NONE;
}
......@@ -94,8 +92,6 @@ export class Dimension {
return "month";
case RelationType.YEAR:
return "year";
case RelationType.DAYOFWEEK:
return "dayofweek";
default:
return "";
}
......
......@@ -55,7 +55,7 @@ describe("engine class", () => {
expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics).to.have.length(12);
expect(optimalView.dimensions).to.have.length(13);
expect(optimalView.dimensions).to.have.length(12);
});
it("should throw an exception, query with non-existent metric", () => {
let error: boolean = false;
......
......@@ -38,7 +38,7 @@ const config = ConfigParser.parse(configPath);
// include middlewares
import { EngineMw } from "./api/middlewares/engine";
import { PostgresMw } from "./api/middlewares/adapter";
import { PostgresMw, MonetMw } from "./api/middlewares/adapter";
import { ErrorMw } from "./api/middlewares/error";
app.use(EngineMw(config));
......@@ -46,8 +46,12 @@ if (config.adapter === "postgres") {
app.use(PostgresMw(config.connection));
}
else if (config.adapter === "monet") {
app.use(MonetMw(config.connection));
}
else {
console.error("Invalid adapter. Options available are: postgres");
console.error("Invalid adapter. Options available are: postgres and monet");
process.exit(1);
}
......
......@@ -32,8 +32,6 @@ function strToRelationType (str: string): RelationType {
return RelationType.MONTH;
case "year":
return RelationType.YEAR;
case "dayofweek":
return RelationType.DAYOFWEEK;
default:
return RelationType.NONE;
}
......@@ -181,18 +179,12 @@ describe("configParser utility library", () => {
parent: "dim:0",
relation: "year"
},
{
name: "dim:dayofweek",
dataType: "integer",
parent: "dim:0",
relation: "dayofweek"
},
{
name: "dim:none",
dataType: "integer",
parent: "dim:0",
relation: "none"
},
}
];
let dims: Dimension[] = [
......
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
const MDB = require("monetdb")();
import { View, LoadView } from "../../src/core/view";
import { each, series } from "async";
import { Connection } from "../../src/util/configParser";
import * as fs from "fs";
interface TransSet {
init: string;
data: string[];
}
export interface Schema {
alias?: string;
query?: string;
data?: string;
fields: any[];
}
export class Fixture {
private database: string;
private config: Connection;
constructor(config: Connection) {
this.config = config;
this.database = config.database;
}
public load(schemas: LoadView[], create: boolean, cb: (err: Error) => void) {
let client = new MDB({
user: this.config.user,
dbname: this.config.database,
password: this.config.password,
host: this.config.host,
port: this.config.port,
});
/*
Loading data has 2 steps:
1 - Create a table or truncate a existing one.
Create or truncate dependes from a parameter in the configure
file, this parameter reaches this function as the create
parameter.
2 - Insert data.
*/
let init: string[] = [];
let data: string[] = [];
for (let i = 0; i < schemas.length; ++i) {
let table: TransSet = this.createTransSet(schemas[i].view, schemas[i].data, create);
init.push(table.init);
data = data.concat(table.data);
}
client.connect();
/*
Tables must be creates before data could be inserted, so the
queries that create and insert are splited in 2 arrays.
To garantee that tables exists before start insert data, series
is used. Inside series each query is executed, using each;
*/
series([(callback: (err: Error) => void) => {
each(init, (query, cback) => {
return client.query(query).then((result: any) => cback())
.fail((err: Error) => (cback(err)));
}, (errQuery: Error) => callback(errQuery));
}, (callback: (err: Error) => void) => {
each(data, (query, cback) => {
return client.query(query).then((result: any) => cback())
.fail((err: Error) => (cback(err)));
}, (errQuery: Error) => callback(errQuery));
}], (errQuery: Error) => {
client.close();
cb(errQuery);
});
}
private typeConvertion(t: string) {
switch (t) {
case "integer":
return "INTEGER";
case "date":
return "TIMESTAMP";
case "string":
return "TEXT";
case "boolean":
return "BOOLEAN";
default:
return "";
}
}
private createTransSet(view: View, filePath: string, create: boolean) {
let props = [];
for (let i = 0; i < view.metrics.length; ++i) {
let met = view.metrics[i];
props.push("\"" + met.name + "\" " + this.typeConvertion(met.dataType));
}
for (let i = 0; i < view.dimensions.length; ++i) {
let dim = view.dimensions[i];
props.push("\"" + dim.name + "\" " + this.typeConvertion(dim.dataType));
}
let name = "view_" + view.id;
let transaction: TransSet = {init: "", data: []};
if (create) {
transaction.init = "CREATE TABLE " + name + "(" + props.join(", ") + ")";
}
else {
transaction.init = "DELETE FROM " + name;
}
transaction.data = [];
let rows = JSON.parse(fs.readFileSync(filePath, {encoding : "utf8"}));
for (let i = 0; i < rows.length; ++i) {
let values = [];
let keys = [];
for (let key of Object.keys(rows[i])) {
keys.push("\"" + key + "\"");
values.push("'" + this.boolCast(rows[i][key]) + "'");
}
transaction.data.push("INSERT INTO " + name +
"(" + keys.join(", ") + ") " +
"VALUES (" + values.join(", ") + ")");
}
return transaction;
}
private boolCast(text: string): string {
if (text === "t") {
return "true";
}
else if (text === "f") {
return "false";
}
else {
return text;
}
}
}
../postgres/fixtures/
\ No newline at end of file
......@@ -212,13 +212,7 @@ const dateSubDim = [
dataType: "integer",
parent: dims[0],
relation: RelationType.YEAR
}),
new Dimension ({
name: "dim:0:dow",
dataType: "integer",
parent: dims[0],
relation: RelationType.DAYOFWEEK
}),
})
];
const dateView = new View({
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment