Commit 315402f1 authored by Rafael Dias's avatar Rafael Dias

Issue #108: Add Log

Signed-off-by: Rafael Dias's avatarrafaelcosc <rpd17@inf.ufpr.br>
parent 83bc0295
Pipeline #21472 passed with stages
in 1 minute and 16 seconds
......@@ -21,3 +21,4 @@
/service
schema.sql
/doc/code
.nyc_output/*
BLENDB_SCHEMA_FILE=config/config.yaml.example
PORT=3000
BLENDB_LOG_FILE=/var/log/blendb.log
BLENDB_LOG_LEVEL=debug
BLENDB_N_DB=1
BLENDB_DB0_USER=blendb
BLENDB_DB0_NAME=blendb-test
......
PORT=3000
BLENDB_N_DB=2
BLENDB_LOG_FILE=/var/log/blendb.log
BLENDB_LOG_LEVEL=debug
BLENDB_DB0_USER=runner
BLENDB_DB0_NAME=blendb_fixture
BLENDB_DB0_PASSWORD=
......
......@@ -15,10 +15,20 @@
"doc-code": "typedoc --mode 'file' --module 'commonjs' --target 'ES6' --ignoreCompilerErrors --exclude '**/*.spec.ts' --out 'doc/code' 'src'"
},
"nyc": {
"include": ["src/**/*.ts"],
"extension": [".ts"],
"require": ["ts-node/register"],
"reporter": ["text-summary", "html"],
"include": [
"src/**/*.ts"
],
"extension": [
".ts"
],
"require": [
"ts-node/register"
],
"reporter": [
"text-summary",
"text",
"lcov"
],
"sourceMap": true,
"instrument": true
},
......@@ -38,6 +48,7 @@
"express": "^4.17.1",
"js-yaml": "^3.13.1",
"json-2-csv": "^3.5.6",
"log4js": "^5.1.0",
"monetdb": "^1.1.4",
"osprey": "^0.3.2",
"pg": "^7.12.1",
......@@ -60,4 +71,5 @@
"engines": {
"node": "^10.16.3"
}
}
......@@ -61,6 +61,9 @@ echo "To set different user and port use npm run service -- <port> [<user>]"
echo "Run this commands, as root (or sudo) to finish the process and start blendb"
SYSTEMD_PATH=/etc/systemd/system/blendb.service
mkdir -p /var/log/
touch /var/log/blendb.log
chown root:$REAL_USER /var/log/blendb.log
echo -n "rm -f $SYSTEMD_PATH && "
echo -n "ln -s $WORKSPACE/service/blendb.service $SYSTEMD_PATH && "
echo "systemctl daemon-reload && systemctl restart blendb.service"
......
......@@ -133,9 +133,11 @@ export class CollectCtrl {
// true/false, it must guarantee that it isn't a boolean
// then it'll test if it's empty
if (!(typeof(data[i]) === "boolean") && !data[i]){
throw new Error("[Collect error] '" + fields[i].name +
"' is mandatory, but no data was received. Review the data sent.");
const message = "[Collect error] '" + fields[i].name +
"' is mandatory, but no data was received. Review the data sent."
throw new Error(message);
}
req.log.debug("Sucessfuly accepted the data: " + data[i] + " from source: ",source.name);
}
for (let i = 0; i < fields.length; i++){
......@@ -151,22 +153,27 @@ export class CollectCtrl {
}
}
if (!found) {
throw new Error("[Collect error] EnumType: '" + data[i] + "' from '" + fields[i].name +
"' isn't allowed on " + fields[i].enumType +
". Review configuration files.");
const message = "[Collect error] EnumType: '" + data[i] + "' from '" + fields[i].name +
"' isn't allowed on " + fields[i].enumType +
". Review configuration files."
throw new Error(message);
}
}else if (!validador[EnumHandler.stringfyDataType(fields[i].dataType)](data[i]) === true){
throw new Error("[Collect error] Datatype: '" + data[i] + "' from '" + fields[i].name +
"' could not be converted to type: " + [EnumHandler.stringfyDataType(fields[i].dataType)] +
". Review configuration files.");
const message = "[Collect error] Datatype: '" + data[i] + "' from '" + fields[i].name +
"' could not be converted to type: " + [EnumHandler.stringfyDataType(fields[i].dataType)] +
". Review configuration files."
throw new Error(message);
}
req.log.debug("Sucessfuly accepted the enumType data: " + data[i] + " from source: ",source.name);
}
}
catch (e) {
const message = "Query execution failed: " +
"Could not construct query with the given parameters."
req.log.warn(message);
res.status(500).json({
message: "Query execution failed: " +
"Could not construct query with the given parameters.",
message: message,
error: e.message
});
return;
......@@ -174,14 +181,18 @@ export class CollectCtrl {
req.adapter.insertIntoSource(source, req.body, (err: Error, result: any[]) => {
if (err) {
const message = "Insertion has failed";
req.log.error(message,err);
res.status(500).json({
message: "Insertion has failed",
message: message,
error: err
});
return;
}
else{
res.status(200).json({message: "Data has been successfully received and stored by the server"});
const message = "Data has been successfully received and stored by the server";
req.log.info(message);
res.status(200).json({message: message});
return;
}
......
......@@ -35,6 +35,7 @@ export class DataCtrl {
* by typescript definition of route.
*/
public static read(req: Request, res: express.Response, next: express.NextFunction) {
req.log.info("Query: ",req.query);
let metrics = req.query.metrics.split(",").filter((item: string) => item !== "");
let dimensions = req.query.dimensions.split(",").filter((item: string) => item !== "");
let clauses = [];
......@@ -45,7 +46,6 @@ export class DataCtrl {
if (req.query.sort) {
sort = req.query.sort.split(",").filter((item: string) => item !== "");
}
let format = "json";
if (req.query.format) {
format = req.query.format;
......@@ -90,9 +90,11 @@ export class DataCtrl {
view = req.engine.query(query);
}
catch (e) {
const message = "Query execution failed: " +
"Could not construct query with the given parameters."
req.log.warn(message,e);
res.status(500).json({
message: "Query execution failed: " +
"Could not construct query with the given parameters.",
message: message,
error: e.message
});
return;
......@@ -100,29 +102,35 @@ export class DataCtrl {
req.adapter.getDataFromView(view, (err: Error, result: any[]) => {
if (err) {
const message = "Query execution failed " +
"failed on execute query on database."
req.log.error(message,err);
res.status(500).json({
message: "Query execution failed " +
"failed on execute query on database.",
message: message,
error: err
});
return;
}
if (format === "json") {
req.log.info("Response (json) send with success");
res.status(200).json(result);
}
else {
req.csvParser(result, format, (error: Error, csv: string) => {
if (error) {
const message = "Error generating csv file. " +
"Try json format."
req.log.error(message,error);
res.status(500).json({
message: "Error generating csv file. " +
"Try json format.",
message: message,
error: error
});
return;
}
req.log.info("Response (csv) send with success");
res.setHeader("Content-Type", "text/csv");
res.setHeader("Content-disposition", "attachment;filename=data.csv");
res.status(200).send(csv);
......
......@@ -42,12 +42,14 @@ export class EngineCtrl {
}
if (format === "json") {
req.log.info("Response (json) send with success");
res.status(200).json(list);
}
else {
req.csvParser(list, format, (error: Error, csv: string) => {
if (error) {
req.log.error("Error generating csv file: ",error);
res.status(500).json({
message: "Error generating csv file. " +
"Try json format.",
......@@ -57,6 +59,7 @@ export class EngineCtrl {
}
const disposition = "attachment;filename=" + fileName + ".csv";
req.log.info("Response (csv) send with success");
res.setHeader("Content-Type", "text/csv");
res.setHeader("Content-disposition", disposition);
res.status(200).send(csv);
......
/*
* Copyright (C) 2019 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { Log } from "../../util/log";
import { Middleware } from "../types";
/**
* Creates a log and middleware that
* inserts the log into the request objects.
*/
export function LogMw (): Middleware {
let log: Log = new Log();
return function logMiddleware(req, res, next) {
req.log = log;
next();
};
}
......@@ -21,6 +21,7 @@
import * as express from "express";
import { Engine } from "../core/engine";
import { Adapter} from "../core/adapter";
import { Log } from "../core/log";
/**
* Extension of Express requests that suports the addition
......@@ -37,6 +38,8 @@ export interface Request extends express.Request {
adapter: Adapter;
/** A csvParser function. Used to parse json object into csv file. */
csvParser: (json: any, format: string, cb: (err: Error, csv?: string));
/** A log object. Used store logs into file. */
log: Log;
}
/**
......
......@@ -55,9 +55,11 @@ import { EngineMw } from "./api/middlewares/engine";
import { PostgresMw, MonetMw } from "./api/middlewares/adapter";
import { ErrorMw } from "./api/middlewares/error";
import { CsvMw } from "./api/middlewares/csv";
import { LogMw } from "./api/middlewares/log"
app.use(EngineMw(config));
app.use(CsvMw());
app.use(LogMw());
if (config.adapters[0] === "postgres") {
app.use(PostgresMw(config.connections[0]));
}
......
/*
* Copyright (C) 2019 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { expect } from "chai";
import { Log } from "./log";
describe("log class", () => {
it("should modifiy loglevel to error if wrong level", () => {
process.env.BLENDB_LOG_LEVEL = "test";
const logLevelWrong = new Log();
expect(logLevelWrong.getLogLevel()).to.be.equals("error");
});
it("should modifiy loglevel to error if empty level", () => {
process.env.BLENDB_LOG_LEVEL = "";
const logLevelEmpty = new Log();
expect(logLevelEmpty.getLogLevel()).to.be.equals("error");
});
it("should modifiy logFile to stderr if empty file", () => {
process.env.BLENDB_LOG_FILE = "";
const LogFileEmpty = new Log();
expect(LogFileEmpty.getLogFile()).to.be.equals("stderr");
});
it("should modifiy logFile to stderr if file does not exist", () => {
process.env.BLENDB_LOG_FILE = "norepo/test/shalnotexist";
const LogFileEmpty = new Log();
expect(LogFileEmpty.getLogFile()).to.be.equals("stderr");
});
});
/*
* Copyright (C) 2019 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { configure, getLogger } from "log4js";
import * as fs from "fs";
/**
* Define the levels that BlenDB log must have
*/
interface LogInterface {
debug(msg: string, ...additionalInfo: any[]): void;
info(msg: string, ...additionalInfo: any[]): void;
warn(msg: string, ...additionalInfo: any[]): void;
error(msg: string, ...additionalInfo: any[]): void;
}
/**
* Logging for BlenDB using the framework log4js,
* using debug, info, warn and error as levels.
* Hierarchy of levels for showing messages are:
* debug > info > warn > error
*/
export class Log implements LogInterface {
private levels = ["debug" , "info" , "warn" , "error"];
private file: string;
private level: string;
private logger: any;
/**
* Definitions to use log4js, such as file destination,
* level of log and configure log4js.
*/
constructor(){
this.file = (process.env["BLENDB_LOG_FILE"]) ? process.env["BLENDB_LOG_FILE"] : "stderr";
this.level = (process.env["BLENDB_LOG_LEVEL"]) ? process.env["BLENDB_LOG_LEVEL"] : "error";
let find = this.levels.some( element => {
return element === this.level;
});
if (!find) {
this.level = "error";
}
if (!(this.file === "stderr")) {
try {
fs.appendFileSync(this.file, "");
}
catch (e) {
this.file = "stderr";
}
}
/**
* The reason for the duplication of configure,
* is because the filename flag from appenders it start
* the empty file if doesn't exist, even if it's not
* called from getLogger. Then it won't create an empty
* file called stderr.
*/
if (this.file === "stderr") {
configure({
appenders: { defaultLog: { type : "stderr" }
},
categories: {
default: { appenders: ["defaultLog"], level: this.level },
}
});
this.logger = getLogger("defaultLog");
}else{
configure({
appenders: { blendb: { type: "file", layout: {
type: "pattern",
pattern: "%d [%p] host: [%h] user: [%x{user}] %n %[%m%n%]",
tokens: {
user: function(logEvent) {
return process.env.USER;
}
}
},
filename: this.file }
},
categories: {
default: { appenders: ["blendb"], level: this.level }
}
});
this.logger = getLogger("blendb");
}
this.logger.level = this.level;
}
/**
* Get log level
*/
public getLogLevel(){
return this.level;
}
/**
* Get lof filename
*/
public getLogFile(){
return this.file;
}
/**
* Log level debug
* @param msg description of log
* @param additionalInfo useful additional information
*/
public debug(msg: string, ...additionalInfo: any[]){
this.emtigLogMessage("debug", msg, additionalInfo);
}
/**
* Log level info
* @param msg description of log
* @param additionalInfo useful additional information
*/
public info(msg: string, ...additionalInfo: any[]){
this.emtigLogMessage("info", msg, additionalInfo);
}
/**
* Log level warning
* @param msg description of log
* @param additionalInfo useful additional information
*/
public warn(msg: string, ...additionalInfo: any[]){
this.emtigLogMessage("warn", msg, additionalInfo);
}
/**
* Log level error
* @param msg description of log
* @param additionalInfo useful additional information
*/
public error(msg: string, ...additionalInfo: any[]){
this.emtigLogMessage("error", msg, additionalInfo);
}
/**
* Handle data between log class and log4js framework
* @param msgType level of log being use
* @param msg description of log
* @param additionalInfo useful additional information
*/
private emtigLogMessage(logLevel: "debug" | "info" | "warn" | "error" , msg: string, additionalInfo: any[]){
if (additionalInfo.length){
this.logger[logLevel](msg, additionalInfo);
}else{
this.logger[logLevel](msg);
}
}
}
......@@ -991,6 +991,11 @@ date-and-time@0.7.0:
resolved "https://registry.yarnpkg.com/date-and-time/-/date-and-time-0.7.0.tgz#26273355558877799f9c95888293fccee92fdb94"
integrity sha512-qPHBPG0AQqbjP7wVf7vLv25/0bZRjYPiJiJtE0t6RqTswJR/6ExCXQLDnL5w4986j7i6470TMtalJxC8/UHrww==
date-format@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/date-format/-/date-format-2.1.0.tgz#31d5b5ea211cf5fd764cd38baf9d033df7e125cf"
integrity sha512-bYQuGLeFxhkxNOF3rcMtiZxvCBAquGzZm6oWA1oZ0g2THUzivaRhv8uOhdr19LmoobSOLoIAxeUK2RdbM8IFTA==
debug@2.6.9, debug@2.x.x, debug@^2.2.0, debug@^2.3.3:
version "2.6.9"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
......@@ -1544,6 +1549,11 @@ flat@^4.1.0:
dependencies:
is-buffer "~2.0.3"
flatted@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.1.tgz#69e57caa8f0eacbc281d2e2cb458d46fdb449e08"
integrity sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==
for-in@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
......@@ -2486,6 +2496,17 @@ log-symbols@^1.0.2:
dependencies:
chalk "^1.0.0"
log4js@^5.1.0:
version "5.1.0"
resolved "https://registry.yarnpkg.com/log4js/-/log4js-5.1.0.tgz#3fa5372055a4c2611ab92d80496bffc100841508"
integrity sha512-QtXrBGZiIwfwBrH9zF2uQarvBuJ5+Icqx9fW+nQL4pnmPITJw8n6kh3bck5IkcTDBQatDeKqUMXXX41fp0TIqw==
dependencies:
date-format "^2.1.0"
debug "^4.1.1"
flatted "^2.0.1"
rfdc "^1.1.4"
streamroller "^2.1.0"
loophole@1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/loophole/-/loophole-1.1.0.tgz#37949fea453b6256acc725c320ce0c5a7f70a2bd"
......@@ -4000,6 +4021,11 @@ ret@~0.1.10:
resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc"
integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==
rfdc@^1.1.4:
version "1.1.4"
resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.1.4.tgz#ba72cc1367a0ccd9cf81a870b3b58bd3ad07f8c2"
integrity sha512-5C9HXdzK8EAqN7JDif30jqsBzavB7wLpaubisuQIGHWf2gUXSpzy6ArX/+Da8RjFpagWsCn+pIgxTMAmKw9Zug==
rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3:
version "2.7.1"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec"
......@@ -4007,20 +4033,7 @@ rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3:
dependencies:
glob "^7.1.3"
"router@git+https://github.com/blakeembrey/router#router-engine":
version "1.1.4"
uid "5eb68560e91b302251ff17a70cd1b6af1fc36d30"
resolved "git+https://github.com/blakeembrey/router#5eb68560e91b302251ff17a70cd1b6af1fc36d30"
dependencies:
array-flatten "2.0.0"
debug "^3.1.0"
methods "~1.1.2"
parseurl "~1.3.1"
path-to-regexp "0.1.7"
setprototypeof "1.0.0"
utils-merge "1.0.0"
"router@github:blakeembrey/router#router-engine":
router@blakeembrey/router#router-engine:
version "1.1.4"
resolved "https://codeload.github.com/blakeembrey/router/tar.gz/5eb68560e91b302251ff17a70cd1b6af1fc36d30"
dependencies:
......@@ -4332,6 +4345,15 @@ stream-shift@^1.0.0:
resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.0.tgz#d5c752825e5367e786f78e18e445ea223a155952"
integrity sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=
streamroller@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/streamroller/-/streamroller-2.1.0.tgz#702de4dbba428c82ed3ffc87a75a21a61027e461"
integrity sha512-Ps7CuQL0RRG0YAigxNehrGfHrLu+jKSSnhiZBwF8uWi62WmtHDQV1OG5gVgV5SAzitcz1GrM3QVgnRO0mXV2hg==
dependencies:
date-format "^2.1.0"
debug "^4.1.1"
fs-extra "^8.1.0"
streamsearch@0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment