Commit 2a499400 authored by Lucas Fernandes de Oliveira's avatar Lucas Fernandes de Oliveira

Merge branch 'issue/106' into 'develop'

Issue #106: Add data return as csv in several routes

See merge request !87
parents e60dbecd a5d7a309
Pipeline #21032 passed with stages
in 1 minute and 21 seconds
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
"async": "=2.4.1", "async": "=2.4.1",
"express": "^4.0.33", "express": "^4.0.33",
"js-yaml": "^3.8.2", "js-yaml": "^3.8.2",
"json-2-csv": "^3.5.5",
"monetdb": "^1.1.4", "monetdb": "^1.1.4",
"osprey": "^0.3.2", "osprey": "^0.3.2",
"pg": "^6.1.5", "pg": "^6.1.5",
......
...@@ -271,6 +271,18 @@ traits: ...@@ -271,6 +271,18 @@ traits:
description: | description: |
Fields to be returned. Fields to be returned.
type: string type: string
- formatable:
queryParameters:
format:
description: |
Response format. Defines if the response objects will be a
json or a csv-like file. The default value is json.
The csv-like formats are: csv, ssv and tsv which the
separator is comma, semi-colon and tab respectively.
example: "ssv+"
required: false
pattern: "^json$|^csv$|^ssv$|^tsv$"
type: string
/metrics: /metrics:
description: | description: |
...@@ -279,6 +291,7 @@ traits: ...@@ -279,6 +291,7 @@ traits:
system and their descriptions. system and their descriptions.
securedBy: [ null, oauth_2_0 ] securedBy: [ null, oauth_2_0 ]
get: get:
is: [ formatable ]
/sources: /sources:
description: | description: |
A Source represents a type of object that can be inserted in the database. A Source represents a type of object that can be inserted in the database.
...@@ -286,6 +299,7 @@ traits: ...@@ -286,6 +299,7 @@ traits:
system and their descriptions system and their descriptions
securedBy: [ null, oauth_2_0 ] securedBy: [ null, oauth_2_0 ]
get: get:
is: [ formatable ]
/dimensions: /dimensions:
description: | description: |
...@@ -294,12 +308,14 @@ traits: ...@@ -294,12 +308,14 @@ traits:
the system and their descriptions. the system and their descriptions.
securedBy: [ null, oauth_2_0 ] securedBy: [ null, oauth_2_0 ]
get: get:
is: [ formatable ]
/enumtypes: /enumtypes:
description: | description: |
A EnumType is short for enumerable type. This is a special data type that only accepts a few possible values. This A EnumType is short for enumerable type. This is a special data type that only accepts a few possible values. This
collection allows the user to list all the enumerable types available in the system, their descriptions and possible collection allows the user to list all the enumerable types available in the system, their descriptions and possible
values. values.
get: get:
is: [ formatable ]
securedBy: [ null, oauth_2_0 ] securedBy: [ null, oauth_2_0 ]
/data: /data:
description: | description: |
...@@ -309,7 +325,7 @@ traits: ...@@ -309,7 +325,7 @@ traits:
start/end dates to refine your query. start/end dates to refine your query.
type: base type: base
get: get:
is: [ filtered ] is: [ filtered, formatable ]
queryParameters: queryParameters:
metrics: metrics:
description: | description: |
......
...@@ -23,12 +23,15 @@ import { expect } from "chai"; ...@@ -23,12 +23,15 @@ import { expect } from "chai";
import * as server from "../../main"; import * as server from "../../main";
import { dataCtrlScenario as tests } from "../../../test/scenario"; import { dataCtrlScenario as tests } from "../../../test/scenario";
import { Query } from "../../common/query"; import { Query } from "../../common/query";
import { waterfall } from "async";
import * as fs from "fs";
interface StrQuery { interface StrQuery {
metrics: string; metrics: string;
dimensions: string; dimensions: string;
filters?: string; filters?: string;
sort?: string; sort?: string;
format?: string;
} }
function parseQuery(obj: Query): StrQuery { function parseQuery(obj: Query): StrQuery {
...@@ -307,4 +310,72 @@ describe("API data controller", () => { ...@@ -307,4 +310,72 @@ describe("API data controller", () => {
.end(done); .end(done);
}); });
it("should respond 200 and get some data with format as csv", (done) => {
waterfall([(cb: (err: Error, data: string) => void) => {
fs.readFile("test/files/data.csv", "utf8", (err, data) => {
cb(err, data);
});
}
, (file: string, cb: (err: Error) => void) => {
let query = parseQuery(tests.csv);
query.format = "csv";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
expect(res.text).to.be.eql(file);
})
.end(cb);
}], (err) => {
expect(err).to.be.eql(null);
done();
});
});
it("should respond 200 and get some data with format as ssv", (done) => {
waterfall([(cb: (err: Error, data: string) => void) => {
fs.readFile("test/files/data.ssv", "utf8", (err, data) => {
cb(err, data);
});
}
, (file: string, cb: (err: Error) => void) => {
let query = parseQuery(tests.csv);
query.format = "ssv";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
expect(res.text).to.be.eql(file);
})
.end(cb);
}], (err) => {
expect(err).to.be.eql(null);
done();
});
});
it("should respond 200 and get some data with format as tsv", (done) => {
waterfall([(cb: (err: Error, data: string) => void) => {
fs.readFile("test/files/data.tsv", "utf8", (err, data) => {
cb(err, data);
});
}
, (file: string, cb: (err: Error) => void) => {
let query = parseQuery(tests.csv);
query.format = "tsv";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
expect(res.text).to.be.eql(file);
})
.end(cb);
}], (err) => {
expect(err).to.be.eql(null);
done();
});
});
}); });
...@@ -45,6 +45,12 @@ export class DataCtrl { ...@@ -45,6 +45,12 @@ export class DataCtrl {
if (req.query.sort) { if (req.query.sort) {
sort = req.query.sort.split(",").filter((item: string) => item !== ""); sort = req.query.sort.split(",").filter((item: string) => item !== "");
} }
let format = "json";
if (req.query.format) {
format = req.query.format;
}
let view; let view;
try { try {
...@@ -102,7 +108,26 @@ export class DataCtrl { ...@@ -102,7 +108,26 @@ export class DataCtrl {
return; return;
} }
res.status(200).json(result); if (format === "json") {
res.status(200).json(result);
}
else {
req.csvParser(result, format, (error: Error, csv: string) => {
if (error) {
res.status(500).json({
message: "Error generating csv file. " +
"Try json format.",
error: error
});
return;
}
res.setHeader("Content-Type", "text/csv");
res.setHeader("Content-disposition", "attachment;filename=data.csv");
res.status(200).send(csv);
});
}
return; return;
}); });
} }
......
...@@ -19,8 +19,10 @@ ...@@ -19,8 +19,10 @@
*/ */
import * as request from "supertest"; import * as request from "supertest";
import * as fs from "fs";
import { expect } from "chai"; import { expect } from "chai";
import * as server from "../../main"; import * as server from "../../main";
import { waterfall } from "async";
describe("API engine controller", () => { describe("API engine controller", () => {
...@@ -68,4 +70,25 @@ describe("API engine controller", () => { ...@@ -68,4 +70,25 @@ describe("API engine controller", () => {
.end(done); .end(done);
}); });
it("should respond 200 and the list of metrics (in csv)", (done) => {
waterfall([(cb: (err: Error, data: string) => void) => {
fs.readFile("test/files/metrics.csv", "utf8", (err, data) => {
cb(err, data);
});
}
, (file: string, cb: (err: Error) => void) => {
request(server)
.get("/v1/metrics")
.query({format: "ssv"})
.expect(200)
.expect((res: any) => {
expect(res.text).to.be.eql(file);
})
.end(cb);
}], (err) => {
expect(err).to.be.eql(null);
done();
});
});
}); });
...@@ -27,6 +27,42 @@ import { Request } from "../types"; ...@@ -27,6 +27,42 @@ import { Request } from "../types";
* engine object that API users can use to create queries. * engine object that API users can use to create queries.
*/ */
export class EngineCtrl { export class EngineCtrl {
/**
* Auxiliary function that returns engine information.
* @param list - List of objects to return
* @param req - Object with request information
* @param res - Object used to create and send the response
* @param next - Call next middleware or controller. Not used but required
* by typescript definition of route.
*/
private static respondList(list: any[], fileName: string, req: Request, res: express.Response, next: express.NextFunction) {
let format = "json";
if (req.query.format) {
format = req.query.format;
}
if (format === "json") {
res.status(200).json(list);
}
else {
req.csvParser(list, format, (error: Error, csv: string) => {
if (error) {
res.status(500).json({
message: "Error generating csv file. " +
"Try json format.",
error: error
});
return;
}
const disposition = "attachment;filename=" + fileName + ".csv";
res.setHeader("Content-Type", "text/csv");
res.setHeader("Content-disposition", disposition);
res.status(200).send(csv);
});
}
}
/** /**
* Route that returns the list of available metrics. * Route that returns the list of available metrics.
* @param req - Object with request information * @param req - Object with request information
...@@ -35,7 +71,8 @@ export class EngineCtrl { ...@@ -35,7 +71,8 @@ export class EngineCtrl {
* by typescript definition of route. * by typescript definition of route.
*/ */
public static metrics(req: Request, res: express.Response, next: express.NextFunction) { public static metrics(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getMetricsDescription()); const metrics = req.engine.getMetricsDescription();
EngineCtrl.respondList(metrics, "metrics", req, res, next);
} }
/** /**
...@@ -46,7 +83,8 @@ export class EngineCtrl { ...@@ -46,7 +83,8 @@ export class EngineCtrl {
* by typescript definition of route. * by typescript definition of route.
*/ */
public static dimensions(req: Request, res: express.Response, next: express.NextFunction) { public static dimensions(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getDimensionsDescription()); const dimensions = req.engine.getDimensionsDescription();
EngineCtrl.respondList(dimensions, "dimensions", req, res, next);
} }
/** /**
...@@ -57,7 +95,8 @@ export class EngineCtrl { ...@@ -57,7 +95,8 @@ export class EngineCtrl {
* by typescript definition of route. * by typescript definition of route.
*/ */
public static enumTypes(req: Request, res: express.Response, next: express.NextFunction) { public static enumTypes(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getEnumTypesDescription()); const enumTypes = req.engine.getEnumTypesDescription();
EngineCtrl.respondList(enumTypes, "enums", req, res, next);
} }
/** /**
...@@ -68,6 +107,7 @@ export class EngineCtrl { ...@@ -68,6 +107,7 @@ export class EngineCtrl {
* by typescript definition of route. * by typescript definition of route.
*/ */
public static sources(req: Request, res: express.Response, next: express.NextFunction) { public static sources(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getSourcesDescription()); const sources = req.engine.getSourcesDescription();
EngineCtrl.respondList(sources, "sources", req, res, next);
} }
} }
/*
* Copyright (C) 2015-2019 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import * as json2csv from "json-2-csv";
import { Middleware } from "../types";
/**
* Creates a csv parser and middleaew that
* inserts the parser into the request objects.
*/
export function CsvMw(): Middleware {
return function csvMiddleware(req, res, next) {
req.csvParser = function parseCsv(json: any, format: string, cb) {
const separator = format.substring(0, 1);
let sep = ",";
if (separator === "s") {
sep = ";";
}
else if (separator === "t"){
sep = "\t";
}
json2csv.json2csv(json, cb, {
delimiter: {
field: sep
, wrap: "\""
, eol: "\n"
}
, prependHeader: true
});
};
next();
};
}
...@@ -35,6 +35,8 @@ export interface Request extends express.Request { ...@@ -35,6 +35,8 @@ export interface Request extends express.Request {
engine: Engine; engine: Engine;
/** A adapter object. Used to communicate with the database in use. */ /** A adapter object. Used to communicate with the database in use. */
adapter: Adapter; adapter: Adapter;
/** A csvParser function. Used to parse json object into csv file. */
csvParser: (json: any, format: string, cb: (err: Error, csv?: string));
} }
/** /**
......
...@@ -40,7 +40,7 @@ import { ConfigParser } from "./util/configParser"; ...@@ -40,7 +40,7 @@ import { ConfigParser } from "./util/configParser";
let configPath; let configPath;
/** @hidden */ /** @hidden */
if(process.env.BLENDB_SCHEMA_FILE){ if (process.env.BLENDB_SCHEMA_FILE) {
configPath = process.env.BLENDB_SCHEMA_FILE; configPath = process.env.BLENDB_SCHEMA_FILE;
} }
else{ else{
...@@ -54,8 +54,10 @@ const config = ConfigParser.parse(configPath); ...@@ -54,8 +54,10 @@ const config = ConfigParser.parse(configPath);
import { EngineMw } from "./api/middlewares/engine"; import { EngineMw } from "./api/middlewares/engine";
import { PostgresMw, MonetMw } from "./api/middlewares/adapter"; import { PostgresMw, MonetMw } from "./api/middlewares/adapter";
import { ErrorMw } from "./api/middlewares/error"; import { ErrorMw } from "./api/middlewares/error";
import { CsvMw } from "./api/middlewares/csv";
app.use(EngineMw(config)); app.use(EngineMw(config));
app.use(CsvMw());
if (config.adapters[0] === "postgres") { if (config.adapters[0] === "postgres") {
app.use(PostgresMw(config.connections[0])); app.use(PostgresMw(config.connections[0]));
} }
......
...@@ -173,4 +173,8 @@ export interface DataCtrlScenario { ...@@ -173,4 +173,8 @@ export interface DataCtrlScenario {
* The products that are the most expensive * The products that are the most expensive
*/ */
expensive: Query; expensive: Query;
/**
* The sellers average age by cpf and sex
*/
csv: Query;
} }
dim:seller:sex,dim:seller:cpf,met:seller:avg:age
male,604.424.718-07,27
female,575.657.111-60,26
nonbinary,977.221.375-39,24
undecided,344.805.128-45,23
female,885.517.020-17,25
\ No newline at end of file
dim:seller:sex;dim:seller:cpf;met:seller:avg:age
male;604.424.718-07;27
female;575.657.111-60;26
nonbinary;977.221.375-39;24
undecided;344.805.128-45;23
female;885.517.020-17;25
\ No newline at end of file
dim:seller:sex dim:seller:cpf met:seller:avg:age
male 604.424.718-07 27
female 575.657.111-60 26
nonbinary 977.221.375-39 24
undecided 344.805.128-45 23
female 885.517.020-17 25
\ No newline at end of file
name;aggregation;dataType;description
met:seller:avg:age;avg;float;The seller average age
met:seller:max:age;max;integer;The seller highest age
met:seller:min:age;min;integer;The seller lowest age
met:seller:count:age;count;integer;The number of seller's
met:product:avg:pricein;avg;float;The average product pricein
met:product:max:pricein;max;float;The highest product pricein
met:product:min:pricein;min;float;The lowest product pricein
met:product:avg:priceout;avg;float;The average product priceout
met:product:max:priceout;max;float;The highest product priceout
met:product:min:priceout;min;float;The lowest product priceout
met:sell:sum:quantity;sum;integer;The sum of sales quantity
met:sell:avg:quantity;avg;float;The average of sales quantity
met:sell:count:quantity;count;integer;The total number of sales
met:buyout:avg:quantity;avg;float;The average of quantity bought
met:buyout:max:quantity;max;integer;The highest quantity bought
met:buyout:min:quantity;min;integer;The lowest quantity bought
\ No newline at end of file
...@@ -334,6 +334,10 @@ let qOpts : {[key: string]: QueryOpts} = { ...@@ -334,6 +334,10 @@ let qOpts : {[key: string]: QueryOpts} = {
metrics: [mets["met:seller:count:age"]], metrics: [mets["met:seller:count:age"]],
dimensions: [] dimensions: []
}, },
"csv": {
metrics: [mets["met:seller:avg:age"]],
dimensions: [dims["dim:seller:cpf"], dims["dim:seller:sex"]]
}
} }
const queries : {[key: string]: Query} = {}; const queries : {[key: string]: Query} = {};
...@@ -437,4 +441,5 @@ export const dataCtrlScenario: DataCtrlScenario = { ...@@ -437,4 +441,5 @@ export const dataCtrlScenario: DataCtrlScenario = {
correct: queries["correct"], correct: queries["correct"],
clausal: queries["clausal"], clausal: queries["clausal"],
expensive: queries["expensive"], expensive: queries["expensive"],
csv: queries["csv"],
}; };
...@@ -725,6 +725,13 @@ decamelize@^1.0.0, decamelize@^1.1.1: ...@@ -725,6 +725,13 @@ decamelize@^1.0.0, decamelize@^1.1.1:
version "1.2.0" version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
deeks@2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/deeks/-/deeks-2.2.1.tgz#4de360652998fd0a681153a9ffde5dcc6c983f4a"
integrity sha512-D2Qu3Fv5zBtBzXjXIUgWPRYn30d/IG6SCPOKIz42+LVTwhPeRI5+DAxdzap0vI2zYheiErIpVLwaUQBoA/iENw==
dependencies:
underscore "1.9.1"
deep-eql@^0.1.3: deep-eql@^0.1.3:
version "0.1.3" version "0.1.3"
resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2" resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2"
...@@ -810,6 +817,11 @@ digest-header@^0.0.1: ...@@ -810,6 +817,11 @@ digest-header@^0.0.1:
dependencies: dependencies:
utility "0.1.11" utility "0.1.11"
doc-path@2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/doc-path/-/doc-path-2.0.1.tgz#d2adb8bcd31c895b17b92f61eff39aaec04430d2"
integrity sha512-/CCG157H//3l513omROUzaREChY/OYpxqYXvQcv7gsrwGfjVOh5d/1gJigHJ6iTnO77pA8rMLZ63CgEPEM6+9Q==
dom-serializer@0: dom-serializer@0:
version "0.1.0" version "0.1.0"
resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.0.tgz#073c697546ce0780ce23be4a28e293e40bc30c82" resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.0.tgz#073c697546ce0780ce23be4a28e293e40bc30c82"
...@@ -1593,6 +1605,15 @@ jsesc@^1.3.0: ...@@ -1593,6 +1605,15 @@ jsesc@^1.3.0:
version "1.3.0" version "1.3.0"
resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-1.3.0.tgz#46c3fec8c1892b12b0833db9bc7622176dbab34b" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-1.3.0.tgz#46c3fec8c1892b12b0833db9bc7622176dbab34b"
json-2-csv@^3.5.5:
version "3.5.5"
resolved "https://registry.yarnpkg.com/json-2-csv/-/json-2-csv-3.5.5.tgz#555d493547086a34da6e36672f5ea1849ebb6789"
integrity sha512-7BlKbejl42pmzoamfH+OFsNkVtPtWUuBJCCh/rhjf32fPTol5UwQGp051cf+qFfDHu0iQssBEzwRIDoTcD3MAw==
dependencies:
deeks "2.2.1"
doc-path "2.0.1"
underscore "1.9.1"
json-schema-compatibility@1.1.0, json-schema-compatibility@^1.0.1: json-schema-compatibility@1.1.0, json-schema-compatibility@^1.0.1:
version "1.1.0" version "1.1.0"
resolved "https://registry.yarnpkg.com/json-schema-compatibility/-/json-schema-compatibility-1.1.0.tgz#1a8981778cda0c38187298d999d089e51af282df" resolved "https://registry.yarnpkg.com/json-schema-compatibility/-/json-schema-compatibility-1.1.0.tgz#1a8981778cda0c38187298d999d089e51af282df"
...@@ -3078,6 +3099,11 @@ underscore@1.8.3: ...@@ -3078,6 +3099,11 @@ underscore@1.8.3:
version "1.8.3" version "1.8.3"
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022"
underscore@1.9.1:
version "1.9.1"
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.9.1.tgz#06dce34a0e68a7babc29b365b8e74b8925203961"
integrity sha512-5/4etnCkd9c8gwgowi5/om/mYO5ajCaOgdzj/oW+0eQV9WxKBDZw5+ycmKmeaTXjInS/W0BzpGLo2xR2aBwZdg==
universalify@^0.1.0: universalify@^0.1.0:
version "0.1.1" version "0.1.1"
resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.1.tgz#fa71badd4437af4c148841e3b3b165f9e9e590b7" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.1.tgz#fa71badd4437af4c148841e3b3b165f9e9e590b7"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment