Commit 69a6dc23 authored by Lucas Fernandes de Oliveira's avatar Lucas Fernandes de Oliveira

Merge branch 'issue/37' into 'master'

Issue #37: Add script to create database views

See merge request !28
parents 781a6f59 7a8e2133
Pipeline #11496 passed with stage
in 44 seconds
#!/usr/bin/env node
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { Hash } from "../src/util/hash";
import * as fs from "fs";
interface View {
alias: string;
query: string;
fields: any;
}
/*interface Table {
create: string;
load: string;
}*/
function typeConvertion(t: string) {
switch (t) {
case "integer":
return "INTEGER";
case "date":
return "DATE";
case "string":
return "TEXT";
case "boolean":
return "BOOLEAN";
default:
return "";
}
}
/*function createTable(view: View) {
let output = "-- View: " + view.alias + "\n";
let props = [];
let keys = [];
for (let field in view.fields) {
props.push("\"" + field + "\" " + typeConvertion(view.fields[field]));
keys.push(field);
}
keys.sort();
let name = "view_" + Hash.sha1(keys);
output += "CREATE TABLE " + name + "(" + props.join(",") + ");\n";
output += "GRANT ALL ON " + name + " TO :DBUSER;\n";
output += "GRANT ALL ON SEQUENCE " + name + "_id_seq TO :DBUSER;\n";
let pathProp = view.alias.toUpperCase() + "_CSV_PATH";
let path = "\\set " + pathProp +
" '\\'' :SCRIPT_PATH/postgres/create/data/" +
view.loadFile + "'\\''";
let nickProp = view.alias.toUpperCase() + "_VIEW";
let wrappedKeys = keys.map((item) => "\"" + item + "\"");
let nick = "\\set " + nickProp + " " + name + "";
let copy = "COPY :" + nickProp + "(" + wrappedKeys.join(",") + ") FROM :" +
pathProp + " DELIMITER ','";
let load = path + "\n" + nick + "\n" + copy + "\n\n";
return { create: output, load: load };
}*/
function createView(view: View): string {
let output = "-- View: " + view.alias + "\n";
let props = [];
let keys = [];
for (let field in view.fields) {
props.push("\"" + field + "\" " + typeConvertion(view.fields[field]));
keys.push(field);
}
keys.sort();
let name = "view_" + Hash.sha1(keys);
output += "DROP VIEW IF EXISTS " + name + " CASCADE;\n";
output += "CREATE OR REPLACE VIEW " + name + " AS\n";
output += fs.readFileSync(filePath + "/views/" + view.query, {encoding: "utf-8" });
return output;
}
// let configFile = process.argv[2];
// let schemaFile = process.argv[3];
// let loadFile = process.argv[4];
let filePath = process.argv[2];
let configFile = filePath + "/schema.json";
let schemaFile = filePath + "/schema.sql";
fs.readFile(configFile, {encoding: "utf-8" }, (err, data) => {
if (err) {
console.log("The config file could not be read. Abort");
process.exitCode = 1;
return;
}
let config: View[] = JSON.parse(data);
let schema = "\\set ON_ERROR_STOP\n\n";
// let load = "\\set ON_ERROR_STOP\n\n";
/*for (let i = 0; i < config.length; ++i) {
let view = config[i];
let table: Table = createTable(view);
schema += table.create;
load += table.load;
}
fs.writeFile(schemaFile, schema, (error) => {
if (error) {
console.log("The schema file could not be written. Abort");
process.exitCode = -1;
return;
}
});
fs.writeFile(loadFile, load, (error) => {
if (error) {
console.log("The load file could not be written. Abort");
process.exitCode = -1;
return;
}
});*/
for (let i = 0; i < config.length; ++i) {
let view = config[i];
let stringfyied: string = createView(view);
schema += stringfyied;
}
fs.writeFile(schemaFile, schema, (error) => {
if (error) {
console.log("The schema file could not be written. Abort");
process.exitCode = -1;
return;
}
});
});
#!/usr/bin/env node
/*
* Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { Engine } from "../src/core/engine";
import { PostgresAdapter } from "../src/adapter/postgres";
import { ConfigParser } from "../src/util/configParser";
import * as path from "path";
import * as fs from "fs";
if (process.argv.length !== 4) {
console.log("Wrong arguments number.");
console.log("Arguments: <configFile> <outputFile>");
process.exitCode = 1;
process.exit();
}
const configFile = process.argv[2];
const schemaFile = process.argv[3];
const config = ConfigParser.parse(configFile);
const referencePath = path.dirname(configFile);
const engine = new Engine();
const adapter = new PostgresAdapter(config.connection);
let schema = "";
config.metrics.forEach ((met) => engine.addMetric(met));
config.dimensions.forEach ((dim) => engine.addDimension(dim));
for (let i = 0; i < config.buildViews.length; ++i) {
const view = config.buildViews[i].view;
const filePath = config.buildViews[i].file;
const alias = config.buildViews[i].alias;
if (view.origin) {
// Cria uma tabela
let output = "-- View: " + alias + "\n";
output += "DROP VIEW IF EXISTS view_" + view.id + " CASCADE;\n";
output += "CREATE OR REPLACE VIEW view_" + view.id + " AS\n";
output += fs.readFileSync(referencePath + "/" + filePath, {encoding: "utf-8" });
schema += output + "\n";
engine.addView(view);
}
}
for (let i = 0; i < config.buildViews.length; ++i) {
const view = config.buildViews[i].view;
const alias = config.buildViews[i].alias;
if (!view.origin) {
const materializedView = engine.query({
metrics: view.metrics,
dimensions: view.dimensions,
clauses: view.clauses
});
const table = adapter.getQueryFromView(materializedView);
const query = "-- View: " + alias + "\n" +
"DROP MATERIALIZED VIEW IF EXISTS view_" + view.id + " CASCADE;\n" +
"CREATE MATERIALIZED VIEW view_" + materializedView.id +
" AS " + table + "\n\n";
schema += query;
engine.addView(materializedView);
}
}
fs.writeFile(schemaFile, schema, (error) => {
if (error) {
console.log("The schema file could not be written. Abort");
process.exitCode = 1;
return;
}
});
[
{
"alias" : "net_usage"
, "query" : "fnu.sql"
, "fields" : {
"dim:contactDate" : "date",
"dim:point" : "integer",
"dim:macAddr" : "string",
"met:avg:download" : "integer",
"met:avg:upload" : "integer",
"met:count:contact" : "integer",
"met:max:contact" : "date",
"met:max:download" : "integer",
"met:max:upload" : "integer",
"met:sum:download" : "integer",
"met:sum:upload" : "integer"
}
}
, {
"alias" : "city"
, "query" : "city.sql"
, "fields" : {
"dim:city" : "integer",
"dim:cityName" : "string",
"dim:region" : "string",
"dim:state" : "string",
"met:count:city" : "integer"
}
}
, {
"alias" : "point"
, "query" : "point.sql"
, "fields" : {
"dim:active" : "boolean",
"dim:city" : "integer",
"dim:digitalCity" : "boolean",
"dim:gesac" : "boolean",
"dim:point" : "integer",
"dim:pointDate" : "date",
"dim:pointName" : "string",
"dim:project" : "string",
"dim:telecenter" : "boolean",
"met:count:point" : "integer",
"met:avg:hiredDownload" : "integer",
"met:avg:hiredUpload" : "integer"
}
}
, {
"alias" : "inventory_changes"
, "query" : "inv_changes.sql"
, "fields" : {
"dim:disk" : "integer",
"dim:inventoryDate" : "date",
"dim:macAddr" : "string",
"dim:memory" : "string",
"dim:oldDisk" : "string",
"dim:oldInventoryDate" : "date",
"dim:oldMemory" : "string",
"dim:oldOs" : "string",
"dim:oldProcessor" : "string",
"dim:os" : "string",
"dim:point" : "integer",
"dim:processor" : "string",
"met:count:machines" : "integer",
"met:count:modified_disk" : "integer",
"met:count:modified_memory" : "integer"
}
}
, {
"alias" : "inventory"
, "query" : "inv.sql"
, "fields" : {
"dim:currentInventory" : "boolean",
"dim:inventoryDate" : "date",
"dim:point" : "integer"
}
}
]
......@@ -40,18 +40,8 @@ export class PostgresAdapter extends Adapter {
this.pool = new Pool(config);
}
public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void {
const materialized = this.searchMaterializedViews(view).sort((a, b) => {
return (a.id < b.id) ? -1 : 1;
});
const unique = [materialized[0]];
for (let i = 1; i < materialized.length; ++i) {
if (materialized[i - 1].id !== materialized[i].id) {
unique.push(materialized[i]);
}
}
const query = this.getQueryFromView(view);
const query = this.buildQuery(view, unique);
this.pool.connect((err, client, done) => {
if (err) {
cb (err);
......@@ -69,6 +59,21 @@ export class PostgresAdapter extends Adapter {
return false;
}
public getQueryFromView(view: View): string {
const materialized = this.searchMaterializedViews(view).sort((a, b) => {
return (a.id < b.id) ? -1 : 1;
});
const unique = [materialized[0]];
for (let i = 1; i < materialized.length; ++i) {
if (materialized[i - 1].id !== materialized[i].id) {
unique.push(materialized[i]);
}
}
return this.buildQuery(view, unique);
}
private searchMaterializedViews(view: View): View[] {
let r: View[] = [];
if (view.materialized) {
......
......@@ -31,6 +31,7 @@ import * as yaml from "js-yaml";
export interface ViewParsingOptions {
alias: string;
data: string;
file?: string;
origin?: boolean;
dimensions: string[];
metrics: string[];
......@@ -62,6 +63,12 @@ interface ConfigFile {
schema: ConfigSchema;
}
interface BuildView {
view: View;
file: string;
alias: string;
}
export interface LoadStruct{
create: boolean;
insert: boolean;
......@@ -74,6 +81,7 @@ export interface ParsedConfig {
dimensions: Dimension[];
struct: LoadStruct;
loadViews: LoadView[];
buildViews: BuildView[];
}
interface DimensionMap {
......@@ -99,7 +107,8 @@ export class ConfigParser {
metrics: [],
dimensions: [],
struct: config.struct,
loadViews: []
loadViews: [],
buildViews: []
};
let metMap: MetricMap = {};
......@@ -126,6 +135,12 @@ export class ConfigParser {
parsed.views.push(view);
let loadView: LoadView = {view: view, data: viewsOpts[i].data};
parsed.loadViews.push(loadView);
const buildView: BuildView = {
view: view,
file: viewsOpts[i].file,
alias: viewsOpts[i].alias
};
parsed.buildViews.push(buildView);
}
return parsed;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment