Commit 89bc66a8 authored by Lucas Fernandes de Oliveira's avatar Lucas Fernandes de Oliveira
Browse files

Merge branch 'issue/12' into 'master'

Issue #12: Add new config files, postgres adpater tests and connection

See merge request !9
parents 5a23cb47 e38572b2
Pipeline #9727 passed with stage
in 40 seconds
image: node:6.2
services:
- postgres:latest
variables:
POSTGRES_DB: 'blendb_fixture'
POSTGRES_USER: 'runner'
POSTGRES_PASSWORD: ''
cache:
paths:
- node_modules
......@@ -9,6 +17,8 @@ before_script:
run_tests:
script:
- mv config/ci_test.yaml.example config/test.yaml
- npm test
tags:
- node
- postgres
# gitignore ignores files.yaml in this folder
# however a config file for tests in CI is required
# so this example file in fact is the CI test file
connection:
database: 'blendb_fixture'
user: 'runner'
password: ''
host: 'postgres'
port: 5432
max: 10
idleTimeoutMillis: 30000
struct:
create: true
insert: true
schema:
views:
-
alias: "View 1"
data: "test/postgres/fixtures/view1.json"
dimensions:
- "dim:1"
- "dim:2"
metrics:
- "met:1"
- "met:2"
- "met:3"
-
alias: "View 2"
data: "test/postgres/fixtures/view2.json"
dimensions:
- "dim:1"
- "dim:2"
metrics:
- "met:1"
- "met:3"
- "met:5"
-
alias: "View 3"
data: "test/postgres/fixtures/view3.json"
dimensions:
- "dim:4"
- "dim:5"
metrics:
- "met:3"
- "met:4"
- "met:7"
-
alias: "View 4"
data: "test/postgres/fixtures/view4.json"
dimensions:
- "dim:3"
- "dim:4"
- "dim:5"
- "dim:6"
metrics:
- "met:6"
- "met:7"
-
alias: "View 5"
data: "test/postgres/fixtures/view5.json"
dimensions:
- "dim:1"
- "dim:2"
- "dim:7"
metrics:
- "met:2"
- "met:3"
- "met:8"
-
alias: "View 6"
data: "test/postgres/fixtures/view6.json"
dimensions:
- "dim:1"
- "dim:2"
metrics:
- "met:1"
- "met:4"
-
alias: "View 7"
data: "test/postgres/fixtures/view7.json"
dimensions:
- "dim:8"
- "dim:9"
- "dim:10"
metrics:
- "met:8"
-
alias: "View 8"
data: "test/postgres/fixtures/view8.json"
dimensions:
- "dim:8"
- "dim:9"
- "dim:10"
metrics:
- "met:9"
-
alias: "View 9"
data: "test/postgres/fixtures/view9.json"
dimensions:
- "dim:8"
- "dim:9"
- "dim:10"
metrics:
- "met:10"
metrics:
-
name: "met:1"
dataType: "integer"
aggregation: "sum"
-
name: "met:2"
dataType: "integer"
aggregation: "avg"
-
name: "met:3"
dataType: "integer"
aggregation: "avg"
-
name: "met:4"
dataType: "integer"
aggregation: "sum"
-
name: "met:5"
dataType: "integer"
aggregation: "sum"
-
name: "met:6"
dataType: "integer"
aggregation: "avg"
-
name: "met:7"
dataType: "integer"
aggregation: "count"
-
name: "met:8"
dataType: "integer"
aggregation: "count"
-
name: "met:9"
dataType: "integer"
aggregation: "sum"
-
name: "met:10"
dataType: "integer"
aggregation: "count"
dimensions:
-
name: "dim:1"
dataType: "date"
-
name: "dim:2"
dataType: "date"
-
name: "dim:3"
dataType: "integer"
-
name: "dim:4"
dataType: "string"
-
name: "dim:5"
dataType: "string"
-
name: "dim:6"
dataType: "boolean"
-
name: "dim:7"
dataType: "integer"
-
name: "dim:8"
dataType: "integer"
-
name: "dim:9"
dataType: "date"
-
name: "dim:10"
dataType: "string"
connection:
user: 'blendb'
database: 'blendb-test'
password: 'secret'
host: 'localhost'
port: 5432
max: 10
idleTimeoutMillis: 30000
struct:
create: false
insert: false
schema:
views:
metrics:
dimensions:
......@@ -83,7 +83,7 @@ function createView(view: View): string {
let keys = [];
for (let field in view.fields) {
props.push("\"" + field + "\" " + typeConvertion(view.fields[field]));
keys.push(field);
keys.push(field.name);
}
keys.sort();
let name = "view_" + Hash.sha1(keys);
......
......@@ -18,15 +18,21 @@
"author": "Centro de Computação Científica e Software Livre (C3SL)",
"license": "GPL-3.0",
"dependencies": {
"@types/async": "^2.0.40",
"@types/chai": "^3.4.33",
"@types/d3": "^3.5.36",
"@types/express": "^4.0.33",
"@types/js-yaml": "^3.5.29",
"@types/mocha": "^2.2.32",
"@types/pg": "^6.1.38",
"@types/pug": "^2.0.1",
"async": "^2.3.0",
"express": "^4.0.33",
"js-yaml": "^3.8.2",
"mississippi": "^1.2.0",
"node-uuid": "^1.4.7",
"osprey": "^0.3.2",
"pg": "^6.1.5",
"pug": "^2.0.0-beta6",
"ts-node": "^1.3.0",
"typescript": "^2.0.3"
......
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { expect } from "chai";
import { PostgresAdapter } from "./postgres";
import { Adapter } from "../core/adapter";
import { Fixture } from "../../test/postgres/fixture";
import { ConfigParser } from "../util/configParser";
import { adapterScenario } from "../../test/scenario";
describe("postgres adapter", () => {
// Initializing
let config: any;
let adapter: Adapter;
let fixture;
before((done) => {
config = ConfigParser.parse("config/test.yaml");
fixture = new Fixture(config.connection);
fixture.load(config.loadViews, config.struct.create, (err) => {
if (err) {
throw err;
}
adapter = new PostgresAdapter(config.connection);
done();
});
});
// Tests
it("should get data from single materialized view", (done) => {
let view = adapterScenario.materializedView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(5);
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from join of 2 views (without selection)", (done) => {
let view = adapterScenario.noSelectionView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(1);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from join of 2 views (with selection)", (done) => {
let view = adapterScenario.withSelectionView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(5);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from single view (with sub-dimension)", (done) => {
let view = adapterScenario.subDimensionView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(25);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from join of 4 views (with selection)", (done) => {
let view = adapterScenario.join4View;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(125);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from different sub dimensions with same parent", (done) => {
let view = adapterScenario.dateView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(5);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
expect(row).to.have.property("dim:2:year", 2017);
});
done();
});
});
});
......@@ -19,136 +19,171 @@
*/
import { Adapter } from "../core/adapter";
import { Metric } from "../core/metric";
import { Dimension } from "../core/dimension";
import { AggregationType, RelationType } from "../common/types";
import { View, ChildView } from "../core/view";
import { Pool, PoolConfig } from "pg";
interface ParsedView {
interface ParsedChild {
query: string;
view: View;
dimensions: Dimension[];
metrics: Metric[];
alias: string;
};
export class PostgresAdapter extends Adapter{
public getDataFromView(view: View): string {
export class PostgresAdapter extends Adapter {
private pool: Pool;
constructor (config: PoolConfig) {
super();
this.pool = new Pool(config);
}
public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void {
// buildQueryFromView does not put the final ;, it need to be put apart
return this.buildQueryFromView(view, view.metrics, view.dimensions) + ";\n";
let query = this.buildQueryFromView(view, view.metrics, view.dimensions) + ";\n";
this.pool.connect((err, client, done) => {
if (err) {
cb (err);
}
client.query(query, [], (error, result) => {
// call 'done()' to release client back to pool
done();
cb(error, (result) ? result.rows : null);
});
});
}
public materializeView(view: View): string {
return null;
public materializeView(view: View): boolean {
return false;
}
private buildQueryFromView (view: View, metrics: Metric[], dimensions: Dimession[]): string {
private buildQueryFromView (view: View, metrics: Metric[], dimensions: Dimension[]): string {
/*
Reduce metrics and dimensions array to the intersection with the
view. So is possible only get useful data in the sub-querys.
*/
let strMetrics = metrics.map((metric) => {
let func = this.getAggregateFunction(metric.aggregation);
let extMetric = func + "(" + metric.name + ")";
let quotedName = "\"" + metric.name + "\"";
let extMetric = func + "(" + quotedName + ") AS " + quotedName;
return extMetric;
});
if (view.materialized) {
let strDimensions = dimensions.map((dimmension) => dimension.name );
let sql = "(SELECT " + strMetrics.join(", ") + ", " + strDimensions.join(", ");
let strDimensions = dimensions.map((dimension) => "\"" + dimension.name + "\"");
let sql = "(SELECT " + strMetrics.concat(strDimensions).join(", ");
sql += " FROM " + "view_" + view.id;
sql += " GROUP BY " + strDimensions.join(", ");
sql += ")\n";
if (strDimensions.length > 0 && strMetrics.length > 0) {
sql += " GROUP BY " + strDimensions.join(", ");
}
sql += ")";
return sql;
}
else {
let children: ParsedView[] = view.childViews.map((item: ChildView) => {
return {
query: this.buildQueryFromView(item.view, item.metrics, item.dimensions),
view: item.view
};
});
let covered = new Map();
dimensions.forEach((item) => covered.set(item.name, ""));
let matchable: any[] = [] ;
dimensions.forEach((item) => {
covered.set(item.name, "");
/*
For each dimension that must be covered
create a match in the array.
If a more than onde view match to the same
dimension, cretes a WHERE clause
*/
matchable.push({match: item.name, sub: item});
let dim = item;
/*
Sub dimensions also have parents that can match
with then too.
*/
while (dim.relation !== RelationType.NONE) {
dim = dim.parent;
matchable.push({
match: dim.name,
sub: item,
});
}
});
metrics.forEach((item) => covered.set(item.name, ""));
let projection = "SELECT ";
let viewsFrom = "FROM ";
let selection = "WHERE ";
let grouping = "GROUP BY ";
let elements = [];
let group = [];
let elements: string[] = [];
let group: string[] = [];
let viewsQuery: string[] = [];
let selected: string[] = [];
children.forEach((child: ParsedView) => {
let selected = [];
let matchable = child.dimensions.map((item) => {
return { match: item, sub: item, viewId: child.view.id };
let children: ParsedChild[] = view.childViews.map((item: ChildView) => {
let dims = item.view.dimensions.filter((dim) => {
return matchable.some((match) => match.match === dim.name);
});
// Running over the dimensions that cover some part of the query
child.dimensions.forEach((dimension: Dimension) => {
/*
If a dimension has a parent, the parent must match with
the sub dimension, so must be added in the matchable
array, for the selection (WHERE statement).
*/
let dim = dimension;
while (dim.relation !== RelationType.NONE) {
matchable.push({
match: dim.name,
sub: dimension,
viewId: child.view.id
});
dim = dim.parent;
}
dim = dimension;
let extDimension = "view_" + child.view.id + ".\"" + dim.name + "\"";
// If the view dimension does not match, search for parent
while (!child.dimensions.some((item) => item.name === dim.name)) {
dim = dim.parent;
extDimension = this.translateRelation(dim.relation, extDimension);
}
covered.set(dimension.name, extDimension);
elements.push(extDimension + " AS " + dimension.name);
group.push(dimension.name);
let mets = item.metrics.filter((met) => {
return metrics.some((elem) => elem.name === met.name);
});
let query = "";
if (dims.length !== 0 || mets.length !== 0) {
query = this.buildQueryFromView(item.view, mets, dims);
}
return {
query: query,
view: item.view,
dimensions : dims,
metrics: mets,
alias: "alias_" + item.view.id
};
}).filter ((item) => item.query !== "");
children.forEach((child: ParsedChild) => {
child.view.dimensions.forEach((dimension: Dimension) => {
/*
Make selection. Search for dimensions, that are in
matchable array.
*/
matchable.filter((item) => {
return item.viewId !== child.view.id &&
item.match === dimension.name;
return item.match === dimension.name;
})
.forEach((item) => {