Issue simmc/SCRUM#22: Cube model for SIMMC

Signed-off-by: Lucas Fernandes de Oliveira <lfo14@inf.ufpr.br>

.gitignore

@@ -2,6 +2,7 @@
 *.out
 *.pid
 *.map
+*.swp
 /.trash
 /pids
 /logs

database/config.ts

+#!/usr/bin/env node
+
+/*
+ * Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Hash } from "../src/util/hash";
+import * as fs from "fs";
+
+interface View {
+    alias: string;
+    query: string;
+    fields: any;
+}
+
+/*interface Table {
+    create: string;
+    load: string;
+}*/
+
+function typeConvertion(t: string) {
+    switch (t) {
+        case "integer":
+            return "INTEGER";
+        case "date":
+            return "DATE";
+        case "string":
+            return "TEXT";
+        case "boolean":
+            return "BOOLEAN";
+        default:
+            return "";
+    }
+}
+
+/*function createTable(view: View) {
+    let output = "-- View: " + view.alias + "\n";
+    let props = [];
+    let keys = [];
+    for (let field in view.fields) {
+        props.push("\"" + field + "\" " + typeConvertion(view.fields[field]));
+        keys.push(field);
+    }
+    keys.sort();
+    let name = "view_" + Hash.sha1(keys);
+    output += "CREATE TABLE " + name + "(" + props.join(",") + ");\n";
+    output += "GRANT ALL ON " + name + " TO :DBUSER;\n";
+    output += "GRANT ALL ON SEQUENCE " + name + "_id_seq TO :DBUSER;\n";
+
+    let pathProp = view.alias.toUpperCase() + "_CSV_PATH";
+    let path = "\\set " + pathProp +
+                 " '\\'' :SCRIPT_PATH/postgres/create/data/" +
+                 view.loadFile + "'\\''";
+    let nickProp = view.alias.toUpperCase() + "_VIEW";
+    let wrappedKeys = keys.map((item) => "\"" + item + "\"");
+    let nick = "\\set " + nickProp + " " + name + "";
+    let copy = "COPY :" + nickProp + "(" + wrappedKeys.join(",") + ") FROM :" +
+                pathProp + " DELIMITER ','";
+    let load = path + "\n" + nick + "\n" + copy + "\n\n";
+
+    return { create: output, load: load };
+}*/
+
+function createView(view: View): string {
+    let output = "-- View: " + view.alias + "\n";
+    let props = [];
+    let keys = [];
+    for (let field in view.fields) {
+        props.push("\"" + field + "\" " + typeConvertion(view.fields[field]));
+        keys.push(field);
+    }
+    keys.sort();
+    let name = "view_" + Hash.sha1(keys);
+    output += "DROP VIEW IF EXISTS " + name + " CASCADE;\n";
+    output += "CREATE OR REPLACE VIEW " +  name + " AS\n";
+    output += fs.readFileSync(filePath + "/views/" + view.query, {encoding: "utf-8" });
+
+    return output;
+}
+
+// let configFile = process.argv[2];
+// let schemaFile = process.argv[3];
+// let loadFile = process.argv[4];
+let filePath = process.argv[2];
+
+let configFile = filePath + "/schema.json";
+let schemaFile = filePath + "/schema.sql";
+
+fs.readFile(configFile, {encoding: "utf-8" },  (err, data) => {
+    if (err) {
+        console.log("The config file could not be read. Abort");
+        process.exitCode = 1;
+        return;
+    }
+
+    let config: View[] = JSON.parse(data);
+    let schema = "\\set ON_ERROR_STOP\n\n";
+    // let load = "\\set ON_ERROR_STOP\n\n";
+
+    /*for (let i = 0; i < config.length; ++i) {
+        let view = config[i];
+        let table: Table = createTable(view);
+        schema += table.create;
+        load += table.load;
+    }
+
+    fs.writeFile(schemaFile, schema, (error) => {
+        if (error) {
+            console.log("The schema file could not be written. Abort");
+            process.exitCode = -1;
+            return;
+        }
+    });
+
+    fs.writeFile(loadFile, load, (error) => {
+        if (error) {
+            console.log("The load file could not be written. Abort");
+            process.exitCode = -1;
+            return;
+        }
+    });*/
+
+    for (let i = 0; i < config.length; ++i) {
+        let view = config[i];
+        let stringfyied: string = createView(view);
+        schema += stringfyied;
+    }
+
+    fs.writeFile(schemaFile, schema, (error) => {
+        if (error) {
+            console.log("The schema file could not be written. Abort");
+            process.exitCode = -1;
+            return;
+        }
+    });
+});

database/schema.json

+[
+    {
+        "alias" : "net_usage"
+        , "query" : "fnu.sql"
+        , "fields" : {
+            "dim:contact_date" : "date",
+            "dim:point" : "integer",
+            "dim:mac_addr" : "string",
+            "met:avg_download" : "integer",
+            "met:avg_upload" : "integer",
+            "met:count_countact" : "integer",
+            "met:last_contact" : "date",
+            "met:max_upload" : "integer",
+            "met:max_download" : "integer",
+            "met:sum_download" : "integer",
+            "met:sum_upload" : "integer"
+        }
+    }
+    , {
+        "alias" : "city"
+        , "query" : "city.sql"
+        , "fields" : {
+             "dim:city" : "integer",
+             "dim:city_name" : "string",
+             "dim:region" : "string",
+             "dim:state" : "string",
+             "met:count_city" : "integer"
+        }
+    }
+    , {
+        "alias" : "point"
+        , "query" : "point.sql"
+        , "fields" : {
+             "dim:active" : "boolean",
+             "dim:city" : "integer",
+             "dim:digital_city" : "boolean",
+             "dim:gesac" : "boolean",
+             "dim:point" : "integer",
+             "dim:point_date" : "date",
+             "dim:point_name" : "string",
+             "dim:project" : "string",
+             "dim:telecenter" : "boolean",
+             "met:count_point" : "integer",
+             "met:hired_download" : "integer",
+             "met:hired_upload" : "integer"
+        }
+    }
+    , {
+        "alias" : "inventory_changes"
+        , "query" : "inv_changes.sql"
+        , "fields" : {
+             "dim:disk" : "integer",
+             "dim:inventory_date" : "date",
+             "dim:mac_addr" : "string",
+             "dim:memory" : "string",
+             "dim:old_disk" : "string",
+             "dim:old_inventory_date" : "date",
+             "dim:old_memory" : "string",
+             "dim:old_os" : "string",
+             "dim:old_processor" : "string",
+             "dim:os" : "string",
+             "dim:point" : "integer",
+             "dim:processor" : "string",
+             "met:machines" : "integer",
+             "met:modified_disk" : "integer",
+             "met:modified_memory" : "integer"
+        }
+    }
+    , {
+        "alias" : "inventory"
+        , "query" : "inv.sql"
+        , "fields" : {
+             "dim:current_inventory" : "boolean",
+             "dim:inventory_date" : "date",
+             "dim:point" : "integer"
+        }
+    }
+]

database/views/city.sql

+SELECT
+    id AS "dim:city"
+    , name AS "dim:city_name"
+    , region AS "dim:region"
+    , state AS "dim:state"
+    , id AS "met:count_city"
+FROM
+    city
+;

database/views/fnu.sql

+SELECT
+    contact_date AS "dim:contact_date"
+    , id_point AS "dim:point"
+    , macaddr AS "dim:mac_addr"
+    , down_bytes AS "met:avg_download"
+    , up_bytes AS "met:avg_upload"
+    , '1' AS "met:count_countact"
+    , contact_date AS "met:last_contact"
+    , down_bytes AS "met:max_upload"
+    , up_bytes AS "met:max_download"
+    , down_bytes AS "met:sum_download"
+    , up_bytes AS "met:sum_upload"
+FROM
+    fact_net_usage
+;

database/views/inv.sql

+SELECT
+     is_current AS "dim:current_inventory"
+     , load_date AS "dim:inventory_date"
+     , id_point AS "dim:point"
+FROM
+    dim_inventory
+WHERE
+    is_current = '1'
+;

database/views/inv_changes.sql

+SELECT
+    d.new_hd_size AS "dim:disk"
+    , d.new_date AS "dim:inventory_date"
+    , d.machine AS "dim:mac_addr"
+    , d.new_memory_size AS "dim:memory"
+    , d.old_hd_size AS "dim:old_disk"
+    , d.old_date AS "dim:old_inventory_date"
+    , d.old_memory_size AS"dim:old_memory"
+    , d.os_distro AS "dim:os"
+    , d.id_point AS "dim:point"
+    , d.processor AS "dim:processor"
+    , '1' AS "met:machines"
+    , d.hd_alert AS "met:modified_disk"
+    , d.memory_alert AS "met:modified_memory"
+FROM
+    (
+        SELECT
+            pt.id AS id_point
+            , c.id AS id_city
+            , c.region AS region
+            , c.state AS state
+            , initcap(c.name) AS city
+            , initcap(a.establishment) AS telecenter
+            , i1.macaddr AS machine
+            , i1.load_date AS old_date
+            , i2.load_date AS new_date
+            , i1.os_distro AS os_distro
+            , i1.processor AS processor
+            , simplify_memory(i1.memory) AS old_memory_size
+            , simplify_memory(i2.memory) AS new_memory_size
+            , simplify_hd(i1.disk1_size) AS old_hd_size
+            , simplify_hd(i2.disk1_size) AS new_hd_size
+            , alert_check_memory(i1.memory, i2.memory) AS memory_alert
+            , alert_check_hd(i1, i2) AS hd_alert
+        FROM
+            dim_inventory i1
+            INNER JOIN dim_inventory i2
+            ON i1.id_point = i2.id_point
+                AND i1.macaddr = i2.macaddr
+                AND i1.is_first = '1'
+                AND i2.is_current = '1'
+            INNER JOIN point pt ON pt.id = i1.id_point
+            INNER JOIN address a ON a.id_point = pt.id
+            INNER JOIN city c ON pt.id_city = c.id
+        WHERE
+            pt.is_telecenter = '1'
+    ) AS d
+WHERE
+    d.memory_alert = 'true'
+    OR d.hd_alert = 'true'
+;

database/views/point.sql

+SELECT
+    p.is_active AS "dim:active"
+    , p.id_city  AS "dim:city"
+    , p.is_digital_city AS "dim:digital_city"
+    , p.is_gesac AS "dim:gesac"
+    , p.id AS"dim:point"
+    , current_date AS "dim:point_date"
+    , a.establishment AS "dim:point_name"
+    , CASE
+        WHEN (p.is_digital_city = TRUE) THEN 'DC'
+        WHEN (p.is_telecenter = TRUE AND p.is_gesac = FALSE) THEN 'TLBR'
+        WHEN (p.is_telecenter = TRUE AND p.is_gesac = TRUE) THEN 'TLBR/GESAC'
+        ELSE 'GESAC'
+    END AS"dim:project"
+    , p.is_telecenter AS "dim:telecenter"
+    , '1' AS "met:count_point"
+    , h.hired_download AS "met:hired_download"
+    , h.hired_upload AS "met:hired_upload"
+FROM
+    point p
+    INNER JOIN address a ON p.id = a.id_point
+    LEFT OUTER JOIN (
+        SELECT
+            c.id_point
+            , ct.download_kbps AS hired_download
+            , ct.upload_kbps as hired_upload
+        FROM
+            convention c
+            INNER JOIN connection_type ct ON ct.id = c.id_connection_type
+    ) h ON p.id = h.id_point
+;

package.json

@@ -8,7 +8,8 @@
     "lint": "tslint -s node_modules/tslint-stylish -t stylish src/**/*.ts test/**/*.ts",
     "test": "ts-node node_modules/istanbul/lib/cli.js cover -x \"**/*.spec.ts\" -e .ts _mocha",
     "show-coverage": "xdg-open coverage/lcov-report/index.html",
-    "doc-api": "raml2html -i specs/blendb-api-v1.raml -o doc/api-v1-reference.html"
+    "doc-api": "raml2html -i specs/blendb-api-v1.raml -o doc/api-v1-reference.html",
+    "database": "ts-node database/config.ts database"
   },
   "repository": {
     "type": "git",