From be4945c8d5fc09db9c3d5a94ff8afdd64ce53471 Mon Sep 17 00:00:00 2001
From: bhm15 <bruno.meyer@ufpr.br>
Date: Thu, 22 Feb 2018 11:51:10 -0300
Subject: [PATCH] ISSUE#9: Data access optimization
---
config.js.example | 4 +
config_test.js.example | 5 ++
config_test_error.js | 4 +
database/create/schema/bd.sql | 24 +++---
src/db/query_builder.js | 7 +-
src/middleware/db.js | 12 +--
src/middleware/populate.js | 34 ++++++---
src/routes/data.js | 20 +++--
src/server.js | 16 ++--
src/static/exemplo.html | 140 ++++++++++++++--------------------
src/static/pinsis.ejs | 18 ++++-
11 files changed, 158 insertions(+), 126 deletions(-)
diff --git a/config.js.example b/config.js.example
index ae636fc..1846bae 100644
--- a/config.js.example
+++ b/config.js.example
@@ -8,4 +8,8 @@ config.db_config = {
port: 5432
};
+config.webservice_config = {
+ ip: "127.0.0.1"
+}
+
config.agentVersion = 0.0;
diff --git a/config_test.js.example b/config_test.js.example
index bc4bb7e..3a80276 100644
--- a/config_test.js.example
+++ b/config_test.js.example
@@ -7,4 +7,9 @@ config.db_config = {
host: 'localhost',
port: 5432
};
+
+config.webservice_config = {
+ ip: "127.0.0.1"
+}
+
config.agentVersion = 0.0;
diff --git a/config_test_error.js b/config_test_error.js
index 5a00d0e..7e9c26e 100644
--- a/config_test_error.js
+++ b/config_test_error.js
@@ -8,5 +8,9 @@ config.db_config = {
port: 5432
};
+config.webservice_config = {
+ ip: "127.0.0.1"
+}
+
config.agentVersion = 0.0;
diff --git a/database/create/schema/bd.sql b/database/create/schema/bd.sql
index 4dcdcd1..f3cfd94 100755
--- a/database/create/schema/bd.sql
+++ b/database/create/schema/bd.sql
@@ -63,30 +63,34 @@ CREATE TABLE Course(
);
CREATE TABLE MachineUse(
- -- ~ Triagem DATE,
- -- ~ Consulta DATE,
- -- ~ Simulador DATE,
- -- ~ Tratamento DATE,
- -- ~ Tri_Cons int, -- DIAS
- -- ~ Cons_Simulador int, -- DIAS
- -- ~ Simulador_Trat int, -- DIAS
- -- ~ CID VARCHAR(4),
Pacient_id VARCHAR(254),
Pacient_name VARCHAR(254),
Energy VARCHAR(254),
Use_date TIMESTAMP,
Field_name VARCHAR(254),
Machine_id VARCHAR(254),
- Fk_Agent int
+ Fk_agent int
);
CREATE TABLE Agent (
Id BIGSERIAL primary key,
- Hospital VARCHAR(16),
+ Hospital_name VARCHAR(16),
Geo_long FLOAT,
Geo_lat FLOAT
);
+
+CREATE MATERIALIZED VIEW ResumeUsesMonth AS
+ SELECT date_trunc('month', use_date) AS date, count(*) as uses, fk_agent
+ FROM machineuse
+ GROUP BY date, fk_agent;
+
+CREATE MATERIALIZED VIEW ResumeUsesDay AS
+ SELECT date_trunc('day', use_date) AS date, count(*) as uses, fk_agent
+ FROM machineuse
+ GROUP BY date, fk_agent;
+
+
ALTER TABLE session ADD CONSTRAINT fk_session_1 FOREIGN KEY(CourseSer) REFERENCES Course(CourseSer);
ALTER TABLE SessionProcedurePart ADD CONSTRAINT fk_SessionProcedurePart_2 FOREIGN KEY(SessionProcedurePartSer) REFERENCES session(sessionSer);
ALTER TABLE SessionProcedurePart ADD CONSTRAINT unique_SessionProcedurePart UNIQUE(SessionProcedurePartSer);
diff --git a/src/db/query_builder.js b/src/db/query_builder.js
index c997202..57a5697 100644
--- a/src/db/query_builder.js
+++ b/src/db/query_builder.js
@@ -16,7 +16,10 @@ module.exports = {
},
select: function(table, conditions){
- var q = squel.select().from(table);
- return q;
+ var q = squel.select().from(table);
+ for(c in conditions){
+ q = q.where(c+" = "+"'"+conditions[c]+"'");
+ }
+ return q;
}
};
diff --git a/src/middleware/db.js b/src/middleware/db.js
index 308c4db..c506f7b 100644
--- a/src/middleware/db.js
+++ b/src/middleware/db.js
@@ -66,16 +66,16 @@ function namedPatch(client) {
exports.pool = function (config) {
return function (req, res, next) {
+ var pool = new pg.Pool(config);
req.db = {
connect: function (callback) {
- pg.on('error', function(err) {
- pg.end();
- throw(err);
+ pool.on('error', function(err) {
+ pool.end();
return;
});
- pg.connect(config, function (err, client, done) {
+ pool.connect(function (err, client, done) {
if (err) {
- setTimeout(req.db.connect, 5000, callback);
+ callback(err, null, done);
return;
}
@@ -83,7 +83,7 @@ exports.pool = function (config) {
callback(null, new Connection(client), done);
});
- pg.end();
+ pool.end();
}
};
diff --git a/src/middleware/populate.js b/src/middleware/populate.js
index e8d7403..44f23f6 100644
--- a/src/middleware/populate.js
+++ b/src/middleware/populate.js
@@ -8,9 +8,7 @@ var async = require('async');
var XLSX = require('xlsx');
-function namedPatch(client) {
-
-
+function namedPatch(client) {
var originalQuery = client.query;
if (originalQuery.patched) return client;
@@ -127,18 +125,17 @@ var sheet2arr = function(sheet){
};
function waterfall_queries(queries, config, cb_w){
+ var pool = new pg.Pool(config);
var temp_db = {
connect: function (callback) {
- pg.on('error', function(err) {
+ pool.on('error', function(err) {
throw(err);
cb_w();
- return;
});
- pg.connect(config, function (err, client, done) {
+ pool.connect(function (err, client, done) {
if (err) {
setTimeout(temp_db.connect, 5000, callback);
- cb_w();
- throw(err);
+ done();
return;
}
@@ -146,15 +143,13 @@ function waterfall_queries(queries, config, cb_w){
callback(null, new Connection(client), done);
});
- pg.end();
+ pool.end();
}
};
temp_db.connect(function(err, conn, done) {
if (err) {
throw(err);
- cb_w();
- return;
}
async.waterfall([
@@ -191,8 +186,9 @@ exports.populateAgent = function(file, config, callback = function(){}){
var queries = [];
+ //TODO: Create a config file that will be used to populate this table
var q = qb.insert("agent",{
- Hospital: "Erasto Gaertner",
+ Hospital_name: "Erasto Gaertner",
Geo_lat: -25.4696528,
Geo_long: -49.261403
});
@@ -217,6 +213,20 @@ exports.populeMachineuse = function (file,config, callback = function(){}){
queries.push(q);
}
+ var refresh_views_query = {
+ toParam: function refresh_views(){
+ return {
+ values: [],
+ text:
+ "REFRESH materialized view ResumeUsesMonth;"+
+ "REFRESH materialized view ResumeUsesDay;"
+ }
+ }
+ }
+
+ queries.push(refresh_views_query);
waterfall_queries(queries, config, callback);
+
+
}
diff --git a/src/routes/data.js b/src/routes/data.js
index 0ced3c0..f405333 100644
--- a/src/routes/data.js
+++ b/src/routes/data.js
@@ -27,22 +27,32 @@ module.exports = function(agentVersion) {
return {
listTable : function (req, res, table) {
-
req.db.connect(function(err, conn, done) {
+
if (err) {
req.log.error(err);
res.status(500).json({error: 'db_connection_failed'});
+ done();
return;
}
+
+ var parameters = {};
+
+ switch(table){
+ case 'ResumeUsesMonth':
+ case 'ResumeUsesDay':
+ if(req.query['hospital_id']) parameters["fk_agent"] = parseInt(req.query['hospital_id']);
+ break;
+ }
- var q = qb.select(table,{});
+ var q = qb.select(table,parameters);
+
conn.query(q.toParam().text,q.toParam().values, function(err,data,fields){
- if (err) throw err;
- res.setHeader('Access-Control-Allow-Origin', 'file:///home/bhm15/c3sl/webservice/src/static/exemplo.html');
+ if (err) req.log.error(err);
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "X-Requested-With");
-
+ done();
return res.status(200).json(data.rows);
});
diff --git a/src/server.js b/src/server.js
index 3664857..c236018 100644
--- a/src/server.js
+++ b/src/server.js
@@ -27,7 +27,6 @@
/*jshint esversion: 6*/
var os = require('os');
-var ip = require("ip");
var fs = require('fs');
var cluster = require('cluster');
var bodyParser = require('body-parser');
@@ -176,16 +175,21 @@ app.get('/api/data/listAgents', function(req, res) {
data.listTable(req,res,"agent");
});
-app.get('/api/data/listMachineUses', function(req, res) {
- data.listTable(req,res,"machineuse");
+app.get('/api/data/ResumeUsesDay', function(req, res) {
+ data.listTable(req,res,"ResumeUsesDay");
});
+app.get('/api/data/ResumeUsesMonth', function(req, res) {
+ data.listTable(req,res,"ResumeUsesMonth");
+});
+
+
app.get('/pinsis.js', function(req, res) {
- res.render(path.join(__dirname + '/static/pinsis.ejs'),{ip:ip.address()});
+ res.render(path.join(__dirname + '/static/pinsis.ejs'),{ip:config.webservice_config.ip});
});
app.get('/exemplo.html', function(req, res) {
- res.render(path.join(__dirname + '/static/exemplo.html'),{ip:ip.address()});
+ res.render(path.join(__dirname + '/static/exemplo.html'),{ip:config.webservice_config.ip});
});
@@ -200,6 +204,8 @@ var exit = function exit() {
// Fork process
//
if (cluster.isMaster) {
+ //Here is where should be populated the tables
+
// daemonize the process (useful for init.d scripts and similar)
if (argv.daemon) {
diff --git a/src/static/exemplo.html b/src/static/exemplo.html
index 279e296..7c78735 100644
--- a/src/static/exemplo.html
+++ b/src/static/exemplo.html
@@ -18,73 +18,57 @@
<script src="http://<%= ip %>:3000/pinsis.js"></script>
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
- <script src="https://cdnjs.cloudflare.com/ajax/libs/underscore.js/1.8.3/underscore.js"></script>
- <script src="https://momentjs.com/downloads/moment.js"></script>
<script>
+
+ function get_attribute_list(object_list, key){
+ return object_list.map(function(x){return x[key];});
+ }
function plot_uses_by(uses,groupby, divname){
- var dates = uses.map(function(x){
- return x["use_date"];
- });
-
- var occurrenceDay = function(occurrence){
- return moment(occurrence).startOf(groupby).format();
- };
-
- var groupToDay = function(group, day){
- return {
- day: day,
- times: group
- }
- };
-
- var result = _.chain(dates)
- .groupBy(occurrenceDay)
- .map(groupToDay)
- .sortBy('day')
- .value();
-
+ var plot1_data_x = get_attribute_list(uses,"date");
+ var plot2_data_x = plot1_data_x;
- var plot1_data_x = [];
- var plot1_data_y = [];
- var plot2_data_x = [];
+ var plot1_data_y = get_attribute_list(uses,"uses");
var plot2_data_y = [];
- for(x in result){
- if(Math.round(Math.random()) == 1){
- plot1_data_x.push(result[x].day);
- plot1_data_y.push(result[x].times.length);
- }
- else{
- plot2_data_x.push(result[x].day);
- plot2_data_y.push(result[x].times.length);
- }
+ for(i in plot1_data_x){
+ plot2_data_y[i] = plot1_data_y[i] - parseInt(plot1_data_y[i]*Math.random());
+ plot1_data_y[i] -= plot2_data_y[i];
}
+
+ var susColor = 'green';
+ var othersColor = 'blue';
+
var plot_data = [
{
x: plot1_data_x,
y: plot1_data_y,
- type: 'scatter',
- name: 'Outros'
+ type: 'bar',
+ name: 'Outros',
+ marker: {color: othersColor},
},
{
x: plot2_data_x,
y: plot2_data_y,
- type: 'scatter',
- name: 'SUS'
+ type: 'bar',
+ name: 'SUS',
+ marker: {color: susColor},
}
];
- Plotly.newPlot(divname, plot_data);
+ Plotly.newPlot(divname, plot_data, {barmode: 'group'});
+ var sum = function(x,y){return x+y;}
+
var data = [{
- values: [plot1_data_x.length,plot2_data_x.length],
- labels: ['Outros', 'SUS'],
- type: 'pie'
+ values: [plot2_data_y.reduce(sum),plot1_data_y.reduce(sum)],
+ labels: ['SUS', 'Outros'],
+ type: 'pie',
+ marker:{colors:[susColor, othersColor]},
}];
var layout = {
@@ -96,43 +80,46 @@
}
- function plot(groupby){
- Pinsis.lastUses(function(err, data){
- console.log(err);
- //~ console.log(data);
- plot_uses_by(data,groupby,"myDiv");
-
- data_m = data.filter(function(x){
- return x["machine_id"].indexOf("CX") !== -1
- });
-
- data_h = data.filter(function(x){
- return x["machine_id"].indexOf("CX") === -1
- });
-
-
- plot_uses_by(data_m, groupby,"myDiv2");
- plot_uses_by(data_h, groupby,"myDiv3");
- });
+ var hospitals;
+
+ function plot(groupby, hospital){
+ switch(groupby){
+ case "days":
+ Pinsis.lastUsesByDay(function(err, data){
+ if(err) console.log(err);
+ plot_uses_by(data,groupby,"myDiv");
+ }, hospital.id);
+ break;
+ case "months":
+ Pinsis.lastUsesByMonth(function(err, data){
+ if(err) console.log(err);
+ plot_uses_by(data,groupby,"myDiv");
+ },hospital.id);
+ break;
+ }
+
+ document.getElementById("hospital_name").innerHTML = hospital.hospital_name;
}
- Pinsis.agents(function(err, data){
- console.log(err);
+
+ Pinsis.hospitals(function(err, data){
+ if(err) console.log(err);
+ hospitals = data;
+ console.log(data);
+ plot("days",hospitals[0]);
});
- plot("day");
+
</script>
</header>
<body>
<center>
- <button onClick="plot('years')" > ANOS </button>
- <button onClick="plot('months')" > MESES </button>
- <button onClick="plot('days')" > DIAS </button>
- <button onClick="plot('hours')" > HORAS </button>
- <button onClick="plot('minutes')" > MINUTOS </button>
+ <button onClick="plot('months', hospitals[0])" > MESES </button>
+ <button onClick="plot('days', hospitals[0])" > DIAS </button>
</center>
- <center><h1>Todas as máquinas</h1>
+ <center><h1>Monitoramento: <span id="hospital_name"></span></h1>
+ <div class="SEPARADOR"></div>
<div id="myDiv"></div>
<div id="myDiv_pizza"></div>
<div class="SEPARADOR"></div>
@@ -140,19 +127,6 @@
<br><br>
- <center><h1>Máquinas do Ministério</h1>
- <div id="myDiv2"></div>
- <div id="myDiv2_pizza"></div>
- <div class="SEPARADOR"></div>
- </center>
-
- <br><br>
-
- <center><h1>Máquinas do Hospital</h1>
- <div id="myDiv3"></div>
- <div id="myDiv3_pizza"></div>
- <div class="SEPARADOR"></div>
- </center>
</body>
diff --git a/src/static/pinsis.ejs b/src/static/pinsis.ejs
index 23974fb..724d23f 100644
--- a/src/static/pinsis.ejs
+++ b/src/static/pinsis.ejs
@@ -17,14 +17,26 @@ var Pinsis = {
xmlhttp.send();
},
- lastUses: function(callback){
+ lastUsesByDay: function(callback, hospital_id){
+ var url = "http://<%= ip %>:3000/api/data/ResumeUsesDay";
+ if(hospital_id) url+="?hospital_id="+hospital_id;
this.getJson(
callback,
- "http://<%= ip %>:3000/api/data/listMachineUses"
+ url
);
},
- agents: function(callback){
+ lastUsesByMonth: function(callback, hospital_id){
+ var url = "http://<%= ip %>:3000/api/data/ResumeUsesMonth";
+ if(hospital_id) url+="?hospital_id="+hospital_id;
+ this.getJson(
+ callback,
+ url
+ );
+
+ },
+
+ hospitals: function(callback){
this.getJson(
callback,
"http://<%= ip %>:3000/api/data/listAgents"
--
GitLab