Merge branch 'refactor_enrollments' into 'development'

Refactor enrollments route to include query building

Changes:
- Add error handling in the database query function. When a database error
occurs, it is logged in the server and a plain text response is returned to the
user.
- Implement building of SQL queries inside the API, which removes the
  need to have stored functions/procedures on the database, since the former
approach is not flexible or efficient to maintain/extend.
- Use route chaining to determine which function will build the query
  that needs to be executed. The last function is the one that actually
  sends the query to the database and send the response.

Future Work:
- Instead of returning a plain text response, return a HTTP 501 status,
  which is currently not possible. When one attempts to send such status
  the framework throws an error that the headers were already sent for
  the current request.
- Chaining based on the route might not be the best solution here.

See merge request !7

.babelrc

+{ 'presets': ['es2015'] }

.eslintignore

+[
+  "**/test/*.js"
+]

.eslintrc.json

+{
+    "extends": "airbnb",
+    "root": true,
+    "plugins": [
+        "react",
+        "jsx-a11y",
+        "import"
+    ],
+    "rules": {
+        "indent": [ "error", 4 ],
+        "no-unused-vars": [ "error", { "args": "none" }],
+        "no-param-reassign": [ "off" ]
+    }
+}

.gitignore

@@ -10,8 +10,10 @@ lib-cov
 *.gz
 
 pids
-logs
 results
 
 npm-debug.log
 node_modules/
+build/*
+
+.gulp-cache

.gitlab-ci.yml

@@ -2,11 +2,12 @@ stages:
   - test
 
 before_script:
+  - npm install --global gulp gulp-cli babel babel-cli babel-core babel-register mocha
   - npm install
 
 run_tests:
   stage: test
   script:
-    - npm test
+    - gulp test
   tags:
     - node

README.md

 # SIMCAQ
+
+# Dependencies
+
+Previous versions of Node.js do not support ECMAScript6, it is recommended to use at least version 4.5.0LTS.
+
+1) Install [NVM (Node Version Manager)](https://github.com/creationix/nvm)
+
+2) Install Node.js via NVM
+
+> source ~/.bashrc
+
+> nvm install v4.5.0
+
+3) Enable Node.js
+
+> nvm use v4.5.0
+
+4) Install babel and gulp globally
+
+> npm install -g gulp gulp-cli babel babel-cli babel-core babel-register mocha
+
+5) Install project dependencies
+
+> npm install

config.json

 {
     "port": 3000,
+    "debug" : false,
     "monetdb": {
         "host": "simcaqdb1",
         "port": 50000,
         "dbname": "simcaq_dev",
         "user": "monetdb",
-        "password":"monetdb"
+        "password":"monetdb",
+        "nrConnections": "16"
     },
     "default": {
         "api": {

gulpfile.babel.js

+const gulp = require('gulp');
+
+const babel = require('gulp-babel');
+
+const eslint = require('gulp-eslint');
+
+const mocha = require('gulp-mocha');
+
+const nodemon = require('gulp-nodemon');
+
+const Cache = require('gulp-file-cache');
+
+const cache = new Cache();
+
+/**
+ * Compile source files
+ */
+function compile() {
+    // run ESLint
+    gulp.src('src/**/*.js')
+        .pipe(eslint())
+        .pipe(eslint.format());
+
+    // compile source to ES5
+    gulp.src('src/**/*.js')
+        .pipe(cache.filter())       // cache source files
+        .pipe(babel())      // compile only modified files
+        .pipe(cache.cache())        // cache compiled files
+        .pipe(gulp.dest('build'));  // move compiled files to build directory
+
+    // copy configuration file to build directory
+    gulp.src('config.json')
+        .pipe(gulp.dest('build'));
+
+}
+
+gulp.task('build', compile);
+
+gulp.task('test', () => {
+  gulp.src('test/test.js', {read: false})
+    .pipe(mocha())
+    .once('error', () => {
+      process.exit(1);
+    })
+    .once('end', () => {
+      process.exit();
+    });
+});
+
+gulp.task('watch', [], () => {
+    console.log('Watching source directory for changes');
+    compile();
+    gulp.watch('src/**/*.js').on('change', () => {
+        console.log('Recompiling source');
+        compile();
+        console.log('Source recompilation done')
+    });
+});
+
+gulp.task('run', () => {
+  process.chdir('build');
+  nodemon({
+    script: 'server.js',
+    tasks: ['watch'],
+    ignore: ["test/test.js", "gulpfile.babel.js"],
+    ext: 'js html json',
+    env: { 'NODE_ENV': 'development' }
+  });
+});
+
+gulp.task('default', ['run']);
+

libs/app.js

-var express = require('express')
-var path = require('path')
-var cookieParser = require('cookie-parser')
-var bodyParser = require('body-parser')
-var csv = require('csv-express')
-var xml = require('js2xmlparser')
-var methodOverride = require('method-override')
-var cors = require('cors')
-var apicache = require('apicache').options({ debug: true }).middleware
-
-var libs = process.cwd() + '/libs/'
-
-var config = require('./config')
-var log = require('./log')(module)
-
-var api = require('./routes/api')
-var states = require('./routes/states')
-var regions = require('./routes/regions')
-var cities = require('./routes/cities')
-
-var app = express()
-
-app.use(bodyParser.json())
-app.use(bodyParser.urlencoded({ extended: false }))
-app.use(cookieParser())
-app.use(cors())
-app.use(methodOverride())
-app.use(apicache('1 hour'))
-
-app.use('/api/v1/', api)
-app.use('/api/v1/states', states)
-app.use('/api/v1/regions', regions)
-app.use('/api/v1/cities', cities)
-
-// catch 404 and forward to error handler
-app.use(function(req, res, next){
-    res.status(404)
-    log.debug('%s %d %s', req.method, res.statusCode, req.url)
-    res.json({
-    	error: 'Not found'
-    })
-    return
-})
-
-// error handlers
-app.use(function(err, req, res, next){
-    res.status(err.status || 500)
-    log.error('%s %d %s', req.method, res.statusCode, err.message)
-    res.json({
-    	error: err.message
-    })
-    return
-})
-
-module.exports = app

libs/config.js

-var nconf = require('nconf')
-
-nconf.argv()
-	.env()
-	.file({
-		file: process.cwd() + '/config.json'
-	})
-
-module.exports = nconf

libs/middlewares/query.js

-var libs = process.cwd() + '/libs/'
-var conn = require(libs + 'db/monet')
-var log = require(libs + 'log')(module)
-
-function query(req, res, next) {
-    log.debug(req.query)
-    conn.query(
-        req.query.text, req.query.values, true
-    ).then(function(result) {
-        // log.debug(result)
-        req.result = result
-        next()
-    })
-}
-
-module.exports = query

libs/routes/api.js

-var express = require('express')
-var xml = require('js2xmlparser')
-var router = express.Router()
-
-var libs = process.cwd() + '/libs/'
-
-var log = require(libs + 'log')(module)
-var config = require(libs + 'config')
-
-var conn = require(libs + 'db/monet')
-
-router.get('/', function (req, res) {
-    res.json({
-        msg: 'SimCAQ API is running'
-    })
-})
-
-/**
- * Complete range of the enrollments dataset
- *
- * Returns a tuple of start and ending years of the complete enrollments dataset.
- */
-router.get('/year_range', function(req, res) {
-    var yearSql = "SELECT MIN(t.ano_censo) AS start_year, MAX(t.ano_censo) AS end_year FROM turmas AS t";
-    conn.query(yearSql, true).then(function(result) {
-      if (req.query.format === 'csv') {
-        res.csv(result.data);
-      } else if (req.query.format === 'xml') {
-        res.send(xml("result", JSON.stringify({year_range: result.data})))
-      }
-      else {
-        res.json({
-            result: result.data
-        });
-      }
-    });
-})
-
-router.get('/data', function(req, res) {
-    log.debug(req.query)
-    log.debug(req.query.met)
-    log.debug(req.query.dim)
-    conn.query(
-        'SELECT * FROM turmas'
-    ).then(function(result) {
-      if (req.query.format === 'csv') {
-        res.csv(result.data);
-      } else if (req.query.format === 'xml') {
-        res.send(xml("result", JSON.stringify({data: result.data})))
-      }
-      else {
-        res.json({
-            result: result.data
-        });
-      }
-    })
-})
-
-router.get('/enrollments', function(req, res) {
-    var params = req.query;
-    var id = 0;
-    var location_id = 0;
-    var adm_dependency_id = 0;
-    var census_year = 0;
-    var enrollmentSql = "";
-
-    if (params.id)
-    {
-        id = parseInt(params.id, 10);
-    }
-
-    if (params.location_id)
-    {
-        location_id = parseInt(params.location_id, 10);
-    }
-
-    if (params.adm_dependency_id)
-    {
-        adm_dependency_id = parseInt(params.adm_dependency_id, 10);
-    }
-
-    if (params.census_year)
-    {
-        census_year = parseInt(params.census_year, 10);
-    }
-
-    /**
-     * FIXME: parameter substitution in the queries is not safe (vulnerable to
-     * SQL injection). Substitution from MonetDB module is not working for some
-     * reason.
-     */
-    switch(params.aggregate)
-    {
-    case "city":
-        if (id) {
-            enrollmentSql = "SELECT nome AS name, total FROM mat_municipio(" + id + "," + census_year + "," + adm_dependency_id + "," + location_id + ")";
-        } else {
-            enrollmentSql = "SELECT nome AS name, total FROM mat_municipios(" + census_year + "," + adm_dependency_id + "," + location_id + ")";
-        }
-        break;
-    case "state":
-        if (id) {
-            enrollmentSql = "SELECT nome AS name, total FROM mat_estado(" + id + "," + census_year + "," + adm_dependency_id + "," + location_id + ")";
-        } else {
-            enrollmentSql = "SELECT nome AS name, total FROM mat_estados(" + census_year + "," + adm_dependency_id + "," + location_id + ")";
-        }
-        break;
-    case "region":
-        if (id) {
-            enrollmentSql = "SELECT nome AS name, total FROM mat_regiao(" + id + "," + census_year + "," + adm_dependency_id + "," + location_id + ")";
-        } else {
-            enrollmentSql = "SELECT nome AS name, total FROM mat_regioes(" + census_year + "," + adm_dependency_id + "," + location_id + ")";
-        }
-        break;
-    default:
-        enrollmentSql = "SELECT nome AS name, total FROM mat_brasil(" + census_year + "," + adm_dependency_id + "," + location_id + ")";
-    }
-
-    log.debug(params);
-    log.debug("Executing query: " + enrollmentSql);
-
-    conn.query(enrollmentSql, true).then(function(result) {
-        log.debug(result);
-        if (req.query.format === 'csv') {
-          res.csv(result.data);
-        } else if (req.query.format === 'xml') {
-          res.send(xml("result", JSON.stringify({enrollments: result.data})))
-        }
-        else {
-          res.json({
-              result: result.data
-          });
-        }
-        log.debug("All resources were released");
-    });
-})
-
-module.exports = router

libs/routes/cities.js

-var express = require('express')
-var xml = require('js2xmlparser')
-var router = express.Router()
-var squel = require('squel')
-
-var libs = process.cwd() + '/libs/'
-
-var log = require(libs + 'log')(module)
-var config = require(libs + 'config')
-
-var conn = require(libs + 'db/monet')
-var query = require(libs + 'middlewares/query')
-
-function response(req, res) {
-    if (req.query.format === 'csv') {
-        res.csv(req.result.data)
-    } else if (req.query.format === 'xml') {
-        res.send(xml("result", JSON.stringify({city: req.result.data})))
-    }
-    else {
-        res.json({
-            result: req.result.data
-        })
-    }
-}
-
-router.get('/', function(req, res, next) {
-    req.query = squel.select().from('municipios').toParam()
-    next()
-}, query, response)
-
-router.get('/:id', function(req, res, next) {
-    req.query = squel.select().from('municipios').where('pk_municipio_id=?', parseInt(req.params.id, 10)).toParam()
-    next()
-}, query, response)
-
-router.get('/state/:id', function(req, res, next) {
-    req.query = squel.select().from('municipios').where('fk_estado_id=?', parseInt(req.params.id, 10)).toParam()
-    next()
-}, query, response)
-
-module.exports = router

libs/routes/regions.js

-var express = require('express')
-var xml = require('js2xmlparser')
-var router = express.Router()
-var squel = require('squel')
-
-var libs = process.cwd() + '/libs/'
-
-var log = require(libs + 'log')(module)
-var config = require(libs + 'config')
-
-var conn = require(libs + 'db/monet')
-
-var query = require(libs + 'middlewares/query')
-
-function response(req, res) {
-    if (req.query.format === 'csv') {
-        res.csv(req.result.data)
-    } else if (req.query.format === 'xml') {
-        res.send(xml("result", JSON.stringify({state: req.result.data})))
-    }
-    else {
-        res.json({
-            result: req.result.data
-        })
-    }
-}
-
-router.get('/', function(req, res, next) {
-    req.query = squel.select().from('regioes').toParam()
-    next()
-}, query, response)
-
-router.get('/:id', function(req, res, next) {
-    req.query = squel.select().from('regioes').where('pk_regiao_id=?', parseInt(req.params.id, 10)).toParam()
-    next()
-}, query, response)
-
-module.exports = router

libs/routes/states.js

-var express = require('express')
-var xml = require('js2xmlparser')
-var router = express.Router()
-var squel = require('squel')
-
-var libs = process.cwd() + '/libs/'
-
-var log = require(libs + 'log')(module)
-var config = require(libs + 'config')
-
-var conn = require(libs + 'db/monet')
-
-var query = require(libs + 'middlewares/query')
-
-function response(req, res) {
-    console.log('respostas :)')
-    if (req.query.format === 'csv') {
-        res.csv(req.result.data)
-    } else if (req.query.format === 'xml') {
-        res.send(xml("result", JSON.stringify({state: req.result.data})))
-    }
-    else {
-        res.json({
-            result: req.result.data
-        })
-    }
-}
-
-router.get('/', function(req, res, next) {
-    req.query = squel.select().from('estados').toParam()
-    next()
-}, query, response)
-
-router.get('/:id', function(req, res, next) {
-    req.query = squel.select().from('estados').where('pk_estado_id = ?', parseInt(req.params.id, 10)).toParam()
-    next()
-}, query, response)
-
-router.get('/region/:id', function(req, res, next) {
-    req.query = squel.select().from('estados').where('fk_regiao_id = ?', parseInt(req.params.id, 10)).toParam()
-    next()
-}, query, response)
-
-module.exports = router

package.json

@@ -5,30 +5,52 @@
   "description": "Simulador custo aluno-qualidade",
   "private": true,
   "scripts": {
-    "start": "forever start server.js || node server.js",
-    "test": "mocha"
+    "start": "cd build && forever start server.js || node server.js",
+    "test": "cd build && mocha"
   },
   "dependencies": {
     "apicache": "0.0.14",
     "body-parser": "^1.13.1",
+    "chai": "^3.5.0",
+    "chai-http": "^3.0.0",
     "cookie-parser": "^1.3.5",
     "cors": "^2.7.1",
     "csv-express": "^1.1.0",
     "debug": "~2.0.x",
+    "dirty-chai": "^1.2.2",
     "express": "^4.13.0",
     "faker": "^2.1.5",
     "forever": "^0.15.2",
     "js2xmlparser": "^1.0.0",
     "method-override": "^2.3.3",
-    "monetdb": "^1.1.2",
+    "mocha": "^2.5.3",
+    "monetdb-pool": "0.0.8",
     "nconf": "^0.6.x",
     "squel": "^5.4.2",
     "winston": "^2.2.0"
   },
   "license": "MIT",
   "devDependencies": {
-    "chai": "^3.5.0",
-    "chai-http": "^3.0.0",
-    "mocha": "^2.5.3"
+    "babel-cli": "^6.11.4",
+    "babel-core": "^6.14.0",
+    "babel-preset-es2015": "^6.13.2",
+    "babel-register": "^6.14.0",
+    "babelify": "^7.3.0",
+    "browserify": "^13.1.0",
+    "eslint": "^3.3.1",
+    "eslint-config-airbnb": "^10.0.1",
+    "eslint-plugin-import": "^1.13.0",
+    "eslint-plugin-jsx-a11y": "^2.1.0",
+    "eslint-plugin-react": "^6.1.1",
+    "gulp": "^3.9.1",
+    "gulp-babel": "^6.1.2",
+    "gulp-cli": "^1.2.2",
+    "gulp-eslint": "^3.0.1",
+    "gulp-file-cache": "0.0.1",
+    "gulp-mocha": "^3.0.1",
+    "gulp-nodemon": "^2.1.0",
+    "gulp-plumber": "^1.1.0",
+    "gulp-rename": "^1.2.2",
+    "gulp-uglify": "^2.0.0"
   }
 }

server.js

-var debug = require('debug')('node-express-base')
-
-var libs = process.cwd() + '/libs/'
-var config = require(libs + 'config')
-var log = require(libs + 'log')(module)
-var app = require(libs + 'app')
-
-app.set('port', process.env.PORT || config.get('port') || 3000)
-
-var server = app.listen(app.get('port'), function() {
-  debug('Express server listening on port ' + server.address().port)
-  log.info('Express server listening on port ' + server.address().port)
-})

src/libs/app.js

+const express = require('express');
+const cookieParser = require('cookie-parser');
+const bodyParser = require('body-parser');
+const methodOverride = require('method-override');
+const cors = require('cors');
+
+const log = require('./log')(module);
+
+const app = express();
+
+const api = require('./routes/api');
+
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: false }));
+app.use(cookieParser());
+app.use(cors());
+app.use(methodOverride());
+app.use(api);
+
+// catch 404 and forward to error handler
+app.use((req, res, next) => {
+    res.status(404);
+    log.debug('%s %d %s', req.method, res.statusCode, req.url);
+    res.json({ error: 'Not found' }).end();
+});
+
+// error handlers
+app.use((err, req, res, next) => {
+    res.status(err.status || 500);
+    log.error('%s %d %s', req.method, res.statusCode, err.message);
+    res.json({ error: err.message }).end();
+});
+
+module.exports = app;

src/libs/config.js

+const nconf = require('nconf');
+
+nconf.argv()
+    .env()
+    .file({ file: `${process.cwd()}/config.json` });
+
+module.exports = nconf;

libs/db/monet.js → src/libs/db/monet.js

-var mdb = require('monetdb')()
+const MonetDBPool = require('monetdb-pool');
 
-var libs = process.cwd() + '/libs/'
+const libs = `${process.cwd()}/libs`;
 
-var log = require(libs + 'log')(module)
-var config = require(libs + 'config')
+const config = require(`${libs}/config`);
 
-var options = {
+const poolOptions = {
+    nrConnections: config.get('monetdb:nrConnections'),
+};
+
+const options = {
     host: config.get('monetdb:host'),
     port: config.get('monetdb:port'),
     dbname: config.get('monetdb:dbname'),
     user: config.get('monetdb:user'),
-    password: config.get('monetdb:password')
-}
+    password: config.get('monetdb:password'),
+};
 
-var conn = new mdb(options)
-conn.connect()
+const conn = new MonetDBPool(poolOptions, options);
+conn.connect();
 
-module.exports = conn
+module.exports = conn;