WIP: Merge branch 'workerpool_feature' into development

.gitignore

+.vscode
 .DS_Store
 .idea
 lib-cov
@@ -20,4 +21,4 @@ build/*
 
 config.json
 docs/
-.vscode/
\ No newline at end of file
+.vscode/

config.json.example

@@ -7,19 +7,32 @@
         "monetdb": {
             "host": "simcaqdb3.c3sl.ufpr.br",
             "port": 50000,
-            "dbname": "simcaq_dev",
+            "dbname": "simcaq_dev3",
             "user": "monetdb",
             "password":"monetdb",
             "nrConnections": "4"
         },
+        "cdn" : {
+            "url": "http://simcaqdb3.c3sl.ufpr.br:3000"
+        },
         "mongodb" : {
-            "uri": "mongodb://localhost/users",
+            "uri": "mongodb://localhost/dev_users",
             "secret": "SimCAQC3SL"
         },
+        "monq": {
+            "uri": "mongodb://localhost/dev_monq"
+        },
         "default": {
             "api": {
                 "version" : "v1"
             }
+        },
+        "email": {
+            "port": 25,
+            "host": "mx.c3sl.ufpr.br",
+            "secure": false,
+            "ignoreTLS": true,
+            "from": "\"Laboratório de Dados Educacionais\" <lde@c3sl.ufpr.br>"
         }
     },
     "test":
@@ -30,20 +43,27 @@
         "monetdb": {
             "host": "simcaqdb3.c3sl.ufpr.br",
             "port": 50000,
-            "dbname": "simcaq_dev",
+            "dbname": "simcaq_dev3",
             "user": "monetdb",
             "password":"monetdb",
             "nrConnections": "4"
         },
+        "cdn" : {
+            "url": "http://simcaqdb3.c3sl.ufpr.br:3000"
+        },
         "mongodb" : {
             "uri": "mongodb://localhost/test_users",
             "secret": "SimCAQC3SL"
         },
+        "monq": {
+            "uri": "mongodb://localhost/test_monq"
+        },
         "default": {
             "api": {
                 "version" : "v1"
             }
-        }
+        },
+        "email": "simcaq@c3sl.ufpr.br"
     },
     "production":
     {
@@ -53,19 +73,26 @@
         "monetdb": {
             "host": "simcaqdb3.c3sl.ufpr.br",
             "port": 50000,
-            "dbname": "simcaq_dev",
+            "dbname": "simcaq_dev3",
             "user": "monetdb",
             "password":"monetdb",
             "nrConnections": "4"
         },
+        "cdn" : {
+            "url": "http://simcaqdb3.c3sl.ufpr.br:7000"
+        },
         "mongodb" : {
             "uri": "mongodb://localhost/users",
             "secret": "SimCAQC3SL"
         },
+        "monq": {
+            "uri": "mongodb://localhost/monq"
+        },
         "default": {
             "api": {
                 "version" : "v1"
             }
-        }
+        },
+        "email": "simcaq@c3sl.ufpr.br"
     }
 }

package.json

@@ -13,6 +13,7 @@
     "test": "cd build && mocha"
   },
   "dependencies": {
+    "agenda": "^0.9.1",
     "apicache": "0.7.0",
     "bcrypt-nodejs": "0.0.3",
     "body-parser": "^1.13.1",
@@ -27,6 +28,7 @@
     "express": "^4.13.0",
     "forever": "^0.15.2",
     "js2xmlparser": "^2.0.2",
+    "jsonexport": "^2.0.9",
     "jwt-simple": "^0.5.0",
     "lodash": "^4.17.2",
     "method-override": "^2.3.3",
@@ -34,7 +36,10 @@
     "monetdb-pool": "0.0.8",
     "mongoose": "^4.6.0",
     "nconf": "^0.8.x",
+    "nodemailer": "^4.0.1",
+    "nodemailer-html-to-text": "^2.1.0",
     "passport": "^0.3.2",
+    "request": "^2.81.0",
     "squel": "^5.4.2",
     "winston": "^2.2.0"
   },

src/libs/agenda.js

+const libs = `${process.cwd()}/libs`;
+const config = require(`${libs}/config`);
+
+const Agenda = require('agenda');
+
+const agenda = new Agenda({
+    db: {
+        address: config.monq.uri,
+        collection: 'agendaJobs'
+    },
+    maxConcurrency: 1,
+    defaultLockLimit: 0,
+    defaultLockLifetime: 86400*1000
+});
+
+require(`${libs}/jobs/download`)(agenda);
+
+agenda.on('ready', function() {
+  agenda.start();
+});
+
+module.exports = agenda;

src/libs/db/mongoose.js

@@ -11,7 +11,7 @@ mongoose.Promise = global.Promise;
 module.exports = () => {
     // Get mongodb URI (ip and port) in config file
     const mongoUri = process.env.MONGO_URI || config.mongodb.uri;
-    log.debug(`Connecting to MongoDB on URI ${mongoUri}`);
+    log.info(`Connecting to MongoDB on URI ${mongoUri}`);
     // Connection singleton
     const db = mongoose.connect(mongoUri);
 

src/libs/jobs/download.js

+const libs = `${process.cwd()}/libs`;
+const log = require(`${libs}/log`)(module);
+const jsonexport = require('jsonexport');
+const request = require('request');
+const execQuery = require(`${libs}/db/query_exec`)
+const fs = require('fs');
+const config = require(`${libs}/config`);
+const mail = require(`${libs}/middlewares/email.js`);
+
+module.exports = function(agenda) {
+    agenda.define('download database', (job, done) => {
+        let sql = job.attrs.data.sql;
+        let table = job.attrs.data.table;
+        // Roda o COPY INTO
+        // Cria o registro no mongo
+        // Manda email notificando o usuário
+        let path = '/tmp/' + table + Date.now() + '.csv';
+        log.debug(sql);
+        let query = `COPY ${sql} INTO '${path}' USING DELIMITERS ';'`;
+
+        execQuery(query, []).then((result) => {
+            // If we're here then everything is ok. Result is empty
+            let file = {
+                name: table,
+                path
+            }
+            request.post(config.cdn.url + '/api/v1/file', {form: file}, (err, response, body) => {
+                if(err) {
+                    log.error(err);
+                    return done();
+                }
+                console.log(body);
+
+                let mailOptions = {
+                    to: 'Teste <vsbc14@inf.ufpr.br>',
+                    subject: 'DOWNLOAD!!!!!',
+                    text: `Oi!\n\n ${body}`
+                };
+                mail(mailOptions, (err, info) => {
+                    if (err) {
+                        log.error(err);
+                        return done();
+                    }
+                    console.log('Message %s sent: %s', info.messageId, info.response);
+                    done();
+                });
+            });
+        }, (err) => {
+            log.error('Error creating dump from DB');
+            log.error(err);
+            return done();
+        });
+    });
+}

src/libs/middlewares/email.js

+const libs = `${process.cwd()}/libs`;
+const log = require(`${libs}/log`)(module);
+const config = require(`${libs}/config`);
+const nodemailer = require('nodemailer');
+const htmlToText = require('nodemailer-html-to-text').htmlToText;
+
+let transporter = nodemailer.createTransport({
+    host: config.email.host,
+    port: config.email.port,
+    secure: config.email.secure,
+    ignoreTLS: config.email.ignoreTLS
+});
+
+transporter.use('compile', htmlToText());
+
+// verify connection configuration
+transporter.verify(function(error, success) {
+   if (error) {
+        log.error(error);
+   } else {
+        log.info('Email server is ready to take our messages');
+   }
+});
+
+let mailOptions = {
+    from: config.email.from
+};
+
+module.exports = function send(options, cb) {
+    Object.assign(options, mailOptions);
+    transporter.sendMail(options, (err, info) => {
+        if(err) {
+            return cb(err);
+        }
+        cb(null, info);
+    });
+};
\ No newline at end of file

src/libs/routes/region.js

@@ -4,6 +4,8 @@ const regionApp = express.Router();
 
 const libs = `${process.cwd()}/libs`;
 
+const log = require(`${libs}/log`)(module);
+
 const squel = require('squel');
 
 const query = require(`${libs}/middlewares/query`);
@@ -52,4 +54,13 @@ regionApp.get('/', rqf.parse(), (req, res, next) => {
     next();
 }, rqf.build(), query, response('region'));
 
+const agenda = require(`${libs}/agenda`);
+
+regionApp.get('/download', (req, res) => {
+    req.sql.from('regiao').field('*');
+    let params = req.sql;
+    agenda.now('download database', {sql: req.sql.toString(), table: req.sql.tableFrom});
+    res.json({msg: 'Wait for it...'});
+});
+
 module.exports = regionApp;