Issue #12: Add new config files, postgres adpater tests and connection

Signed-off-by: Lucas Fernandes de Oliveira's avatarLucas Fernandes de Oliveira <lfo14@inf.ufpr.br>
parent 5a23cb47
Pipeline #9725 passed with stage
in 40 seconds
image: node:6.2
services:
- postgres:latest
variables:
POSTGRES_DB: 'blendb_fixture'
POSTGRES_USER: 'runner'
POSTGRES_PASSWORD: ''
cache:
paths:
- node_modules
......@@ -9,6 +17,8 @@ before_script:
run_tests:
script:
- mv config/ci_test.yaml.example config/test.yaml
- npm test
tags:
- node
- postgres
# gitignore ignores files.yaml in this folder
# however a config file for tests in CI is required
# so this example file in fact is the CI test file
connection:
database: 'blendb_fixture'
user: 'runner'
password: ''
host: 'postgres'
port: 5432
max: 10
idleTimeoutMillis: 30000
struct:
create: true
insert: true
schema:
views:
-
alias: "View 1"
data: "test/postgres/fixtures/view1.json"
dimensions:
- "dim:1"
- "dim:2"
metrics:
- "met:1"
- "met:2"
- "met:3"
-
alias: "View 2"
data: "test/postgres/fixtures/view2.json"
dimensions:
- "dim:1"
- "dim:2"
metrics:
- "met:1"
- "met:3"
- "met:5"
-
alias: "View 3"
data: "test/postgres/fixtures/view3.json"
dimensions:
- "dim:4"
- "dim:5"
metrics:
- "met:3"
- "met:4"
- "met:7"
-
alias: "View 4"
data: "test/postgres/fixtures/view4.json"
dimensions:
- "dim:3"
- "dim:4"
- "dim:5"
- "dim:6"
metrics:
- "met:6"
- "met:7"
-
alias: "View 5"
data: "test/postgres/fixtures/view5.json"
dimensions:
- "dim:1"
- "dim:2"
- "dim:7"
metrics:
- "met:2"
- "met:3"
- "met:8"
-
alias: "View 6"
data: "test/postgres/fixtures/view6.json"
dimensions:
- "dim:1"
- "dim:2"
metrics:
- "met:1"
- "met:4"
-
alias: "View 7"
data: "test/postgres/fixtures/view7.json"
dimensions:
- "dim:8"
- "dim:9"
- "dim:10"
metrics:
- "met:8"
-
alias: "View 8"
data: "test/postgres/fixtures/view8.json"
dimensions:
- "dim:8"
- "dim:9"
- "dim:10"
metrics:
- "met:9"
-
alias: "View 9"
data: "test/postgres/fixtures/view9.json"
dimensions:
- "dim:8"
- "dim:9"
- "dim:10"
metrics:
- "met:10"
metrics:
-
name: "met:1"
dataType: "integer"
aggregation: "sum"
-
name: "met:2"
dataType: "integer"
aggregation: "avg"
-
name: "met:3"
dataType: "integer"
aggregation: "avg"
-
name: "met:4"
dataType: "integer"
aggregation: "sum"
-
name: "met:5"
dataType: "integer"
aggregation: "sum"
-
name: "met:6"
dataType: "integer"
aggregation: "avg"
-
name: "met:7"
dataType: "integer"
aggregation: "count"
-
name: "met:8"
dataType: "integer"
aggregation: "count"
-
name: "met:9"
dataType: "integer"
aggregation: "sum"
-
name: "met:10"
dataType: "integer"
aggregation: "count"
dimensions:
-
name: "dim:1"
dataType: "date"
-
name: "dim:2"
dataType: "date"
-
name: "dim:3"
dataType: "integer"
-
name: "dim:4"
dataType: "string"
-
name: "dim:5"
dataType: "string"
-
name: "dim:6"
dataType: "boolean"
-
name: "dim:7"
dataType: "integer"
-
name: "dim:8"
dataType: "integer"
-
name: "dim:9"
dataType: "date"
-
name: "dim:10"
dataType: "string"
connection:
user: 'blendb'
database: 'blendb-test'
password: 'secret'
host: 'localhost'
port: 5432
max: 10
idleTimeoutMillis: 30000
struct:
create: false
insert: false
schema:
views:
metrics:
dimensions:
......@@ -83,7 +83,7 @@ function createView(view: View): string {
let keys = [];
for (let field in view.fields) {
props.push("\"" + field + "\" " + typeConvertion(view.fields[field]));
keys.push(field);
keys.push(field.name);
}
keys.sort();
let name = "view_" + Hash.sha1(keys);
......
......@@ -18,15 +18,21 @@
"author": "Centro de Computação Científica e Software Livre (C3SL)",
"license": "GPL-3.0",
"dependencies": {
"@types/async": "^2.0.40",
"@types/chai": "^3.4.33",
"@types/d3": "^3.5.36",
"@types/express": "^4.0.33",
"@types/js-yaml": "^3.5.29",
"@types/mocha": "^2.2.32",
"@types/pg": "^6.1.38",
"@types/pug": "^2.0.1",
"async": "^2.3.0",
"express": "^4.0.33",
"js-yaml": "^3.8.2",
"mississippi": "^1.2.0",
"node-uuid": "^1.4.7",
"osprey": "^0.3.2",
"pg": "^6.1.5",
"pug": "^2.0.0-beta6",
"ts-node": "^1.3.0",
"typescript": "^2.0.3"
......
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { expect } from "chai";
import { PostgresAdapter } from "./postgres";
import { Adapter } from "../core/adapter";
import { Fixture } from "../../test/postgres/fixture";
import { ConfigParser } from "../util/configParser";
import { adapterScenario } from "../../test/scenario";
describe("postgres adapter", () => {
// Initializing
let config: any;
let adapter: Adapter;
let fixture;
before((done) => {
config = ConfigParser.parse("config/test.yaml");
fixture = new Fixture(config.connection);
fixture.load(config.loadViews, config.struct.create, (err) => {
if (err) {
throw err;
}
adapter = new PostgresAdapter(config.connection);
done();
});
});
// Tests
it("should get data from single materialized view", (done) => {
let view = adapterScenario.materializedView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(5);
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from join of 2 views (without selection)", (done) => {
let view = adapterScenario.noSelectionView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(1);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from join of 2 views (with selection)", (done) => {
let view = adapterScenario.withSelectionView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(5);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from single view (with sub-dimension)", (done) => {
let view = adapterScenario.subDimensionView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(25);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from join of 4 views (with selection)", (done) => {
let view = adapterScenario.join4View;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(125);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
});
done();
});
});
it("should get data from different sub dimensions with same parent", (done) => {
let view = adapterScenario.dateView;
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
expect(result).to.have.length(5);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
keys = keys.concat(view.dimensions.map((item) => item.name));
result.forEach((row) => {
expect(row).to.be.an("object");
expect(row).to.have.all.keys(keys);
expect(row).to.have.property("dim:2:year", 2017);
});
done();
});
});
});
This diff is collapsed.
......@@ -21,6 +21,6 @@
import { View } from "./view";
export abstract class Adapter {
public abstract getDataFromView(view: View): string;
public abstract materializeView(view: View): string;
public abstract getDataFromView(view: View, cb: (err: Error, result: any[]) => void): void;
public abstract materializeView(view: View): boolean;
}
......@@ -22,17 +22,20 @@ import { RelationType } from "../common/types";
export interface DimensionOptions {
name: string;
dataType: string;
parent?: Dimension;
relation?: RelationType;
}
export class Dimension {
public readonly name: string;
public readonly dataType: string;
public readonly parent: Dimension;
public readonly relation: RelationType;
constructor(options: DimensionOptions) {
this.name = options.name;
this.dataType = options.dataType;
this.relation = (options.relation) ? options.relation : RelationType.NONE;
this.parent = (options.parent) ? options.parent : null;
}
......
This diff is collapsed.
......@@ -23,14 +23,17 @@ import { AggregationType } from "../common/types";
export interface MetricOptions {
name: string;
aggregation: AggregationType;
dataType: string;
}
export class Metric {
public readonly name: string;
public readonly aggregation: AggregationType;
public readonly dataType: string;
constructor(options: MetricOptions) {
this.name = options.name;
this.aggregation = options.aggregation;
this.dataType = options.dataType;
}
}
......@@ -22,6 +22,11 @@ import { Dimension } from "./dimension";
import { Metric } from "./metric";
import { Hash } from "../util/hash";
export interface LoadView {
view: View;
data: string;
}
export interface ChildView {
metrics: Metric[];
dimensions: Dimension[];
......@@ -45,8 +50,8 @@ export class View {
constructor (options: ViewOptions) {
this.metrics = options.metrics.sort();
this.dimensions = options.dimensions.sort();
this.materialized = options.materialized || true;
this.childViews = options.childViews || [];
this.materialized = options.materialized || false;
this.childViews = (options.childViews) ? options.childViews : [];
// calculate the id of the view based on it's metrics and dimensions
let metricsNames = options.metrics.map(metric => metric.name);
......
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { expect } from "chai";
import { ConfigParser, ViewParsingOptions } from "./configParser";
describe("configParser utility library", () => {
let metMap = new Map();
let dimMap = new Map();
it("should throw expection for inexistent metric", () => {
let opts: ViewParsingOptions = {
alias: "Test",
data: "test",
dimensions: [],
metrics: ["met:-1"]
};
let error: boolean = false;
try {
ConfigParser.parseViewOpt(opts, metMap, dimMap);
}
catch (e) {
error = true;
expect(e.message).to.be
.equal("[Parsing error] Non exist metric set to view " + opts.alias);
}
expect(error).to.be.true;
});
it("should throw expection for inexistent metric", () => {
let opts: ViewParsingOptions = {
alias: "Test",
data: "test",
dimensions: ["dim:-1"],
metrics: []
};
let error: boolean = false;
try {
ConfigParser.parseViewOpt(opts, metMap, dimMap);
}
catch (e) {
error = true;
expect(e.message).to.be
.equal("[Parsing error] Non exist dimension set to view " + opts.alias);
}
expect(error).to.be.true;
});
});
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { Metric, MetricOptions } from "../core/metric";
import { Dimension, DimensionOptions } from "../core/dimension";
import { View, ViewOptions, LoadView } from "../core/view";
import { PoolConfig } from "pg";
import * as fs from "fs";
import * as yaml from "js-yaml";
export interface ViewParsingOptions {
alias: string;
data: string;
dimensions: string[];
metrics: string[];
}
interface ConfigSchema {
views: ViewParsingOptions[];
metrics: MetricOptions[];
dimensions: DimensionOptions[];
}
interface ConfigFile {
connection: PoolConfig;
struct: LoadStruct;
schema: ConfigSchema;
}
export interface LoadStruct{
create: boolean;
insert: boolean;
}
export interface ParsedConfig {
connection: PoolConfig;
views: View[];
metrics: Metric[];
dimensions: Dimension[];
struct: LoadStruct;
loadViews: LoadView[];
}
export class ConfigParser {
public static parse(configPath: string): ParsedConfig {
let config: ConfigFile = yaml.safeLoad(fs.readFileSync(configPath, {
encoding: "utf-8"
}));
let metricsOpts = config.schema.metrics;
let viewsOpts = config.schema.views;
let dimensionsOpts = config.schema.dimensions;
let parsed: ParsedConfig = {
connection: config.connection,
views: [],
metrics: [],
dimensions: [],
struct: config.struct,
loadViews: []
};
let metMap: Map<string, Metric> = new Map();
let dimMap: Map<string, Dimension> = new Map();
for (let i = 0; i < metricsOpts.length; ++i) {
let met = new Metric(metricsOpts[i]);
parsed.metrics.push(met);
metMap.set(met.name, met);
}
for (let i = 0; i < dimensionsOpts.length; ++i) {
let dim = new Dimension(dimensionsOpts[i]);
parsed.dimensions.push(dim);
dimMap.set(dim.name, dim);
}
for (let i = 0; i < viewsOpts.length; ++i) {
let viewOpts = ConfigParser.parseViewOpt(viewsOpts[i], metMap, dimMap);
let view = new View(viewOpts);
parsed.views.push(view);
let loadView: LoadView = {view: view, data: viewsOpts[i].data};
parsed.loadViews.push(loadView);
}
return parsed;
}
public static parseViewOpt(opts: ViewParsingOptions,
metMap: Map<string, Metric>,
dimMap: Map<string, Dimension>): ViewOptions {
let viewOpt: ViewOptions = {
metrics: [],
dimensions: [],
materialized: true,
childViews: [],
};
for (let i = 0; i < opts.metrics.length; ++i) {
if (metMap.has(opts.metrics[i])) {
viewOpt.metrics.push(metMap.get(opts.metrics[i]));
}
else {
throw new Error("[Parsing error] Non exist metric set to view " + opts.alias);
}
}
for (let i = 0; i < opts.dimensions.length; ++i) {
if (dimMap.has(opts.dimensions[i])) {
viewOpt.dimensions.push(dimMap.get(opts.dimensions[i]));
}
else {
throw new Error("[Parsing error] Non exist dimension set to view " + opts.alias);
}
}
return viewOpt;
}
}
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { Client, PoolConfig } from "pg";
import { View, LoadView } from "../../src/core/view";
import { each, series } from "async";
import * as fs from "fs";
interface TransSet {
init: string;
data: string[];
}
export interface Schema {
alias?: string;
query?: string;
data?: string;