Issue #106: Add data return as csv in several routes

Signed-off-by: Lucas Fernandes de Oliveira's avatarLucas Fernandes de Oliveira <lfoliveira@inf.ufpr.br>
parent e60dbecd
Pipeline #20691 passed with stages
in 56 seconds
......@@ -29,6 +29,7 @@
"async": "=2.4.1",
"express": "^4.0.33",
"js-yaml": "^3.8.2",
"json-2-csv": "^3.5.5",
"monetdb": "^1.1.4",
"osprey": "^0.3.2",
"pg": "^6.1.5",
......
......@@ -271,6 +271,18 @@ traits:
description: |
Fields to be returned.
type: string
- formatable:
queryParameters:
format:
description: |
Response format. Defines if the response objects will be a
json or a csv-like file. The default value is json.
The csv-like formats are: csv, ssv and tsv which the
separator is comma, semi-colon and tab respectively.
example: "ssv+"
required: false
pattern: "^json$|^csv$|^ssv$|^tsv$"
type: string
/metrics:
description: |
......@@ -279,6 +291,7 @@ traits:
system and their descriptions.
securedBy: [ null, oauth_2_0 ]
get:
is: [ formatable ]
/sources:
description: |
A Source represents a type of object that can be inserted in the database.
......@@ -286,6 +299,7 @@ traits:
system and their descriptions
securedBy: [ null, oauth_2_0 ]
get:
is: [ formatable ]
/dimensions:
description: |
......@@ -294,12 +308,14 @@ traits:
the system and their descriptions.
securedBy: [ null, oauth_2_0 ]
get:
is: [ formatable ]
/enumtypes:
description: |
A EnumType is short for enumerable type. This is a special data type that only accepts a few possible values. This
collection allows the user to list all the enumerable types available in the system, their descriptions and possible
values.
get:
is: [ formatable ]
securedBy: [ null, oauth_2_0 ]
/data:
description: |
......@@ -309,7 +325,7 @@ traits:
start/end dates to refine your query.
type: base
get:
is: [ filtered ]
is: [ filtered, formatable ]
queryParameters:
metrics:
description: |
......
......@@ -23,12 +23,15 @@ import { expect } from "chai";
import * as server from "../../main";
import { dataCtrlScenario as tests } from "../../../test/scenario";
import { Query } from "../../common/query";
import { waterfall } from "async";
import * as fs from "fs";
interface StrQuery {
metrics: string;
dimensions: string;
filters?: string;
sort?: string;
format?: string;
}
function parseQuery(obj: Query): StrQuery {
......@@ -307,4 +310,72 @@ describe("API data controller", () => {
.end(done);
});
it("should respond 200 and get some data with format as csv", (done) => {
waterfall([(cb: (err: Error, data: string) => void) => {
fs.readFile("test/files/data.csv", "utf8", (err, data) => {
cb(err, data);
});
}
, (file: string, cb: (err: Error) => void) => {
let query = parseQuery(tests.csv);
query.format = "csv";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
expect(res.text).to.be.eql(file);
})
.end(cb);
}], (err) => {
expect(err).to.be.eql(null);
done();
});
});
it("should respond 200 and get some data with format as ssv", (done) => {
waterfall([(cb: (err: Error, data: string) => void) => {
fs.readFile("test/files/data.ssv", "utf8", (err, data) => {
cb(err, data);
});
}
, (file: string, cb: (err: Error) => void) => {
let query = parseQuery(tests.csv);
query.format = "ssv";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
expect(res.text).to.be.eql(file);
})
.end(cb);
}], (err) => {
expect(err).to.be.eql(null);
done();
});
});
it("should respond 200 and get some data with format as tsv", (done) => {
waterfall([(cb: (err: Error, data: string) => void) => {
fs.readFile("test/files/data.tsv", "utf8", (err, data) => {
cb(err, data);
});
}
, (file: string, cb: (err: Error) => void) => {
let query = parseQuery(tests.csv);
query.format = "tsv";
request(server)
.get("/v1/data")
.query(query)
.expect(200)
.expect((res: any) => {
expect(res.text).to.be.eql(file);
})
.end(cb);
}], (err) => {
expect(err).to.be.eql(null);
done();
});
});
});
......@@ -45,6 +45,12 @@ export class DataCtrl {
if (req.query.sort) {
sort = req.query.sort.split(",").filter((item: string) => item !== "");
}
let format = "json";
if (req.query.format) {
format = req.query.format;
}
let view;
try {
......@@ -102,7 +108,26 @@ export class DataCtrl {
return;
}
res.status(200).json(result);
if (format === "json") {
res.status(200).json(result);
}
else {
req.csvParser(result, format, (error: Error, csv: string) => {
if (error) {
res.status(500).json({
message: "Error generating csv file. " +
"Try json format.",
error: error
});
return;
}
res.setHeader("Content-Type", "text/csv");
res.setHeader("Content-disposition", "attachment;filename=data.csv");
res.status(200).send(csv);
});
}
return;
});
}
......
......@@ -19,8 +19,10 @@
*/
import * as request from "supertest";
import * as fs from "fs";
import { expect } from "chai";
import * as server from "../../main";
import { waterfall } from "async";
describe("API engine controller", () => {
......@@ -68,4 +70,25 @@ describe("API engine controller", () => {
.end(done);
});
it("should respond 200 and the list of metrics (in csv)", (done) => {
waterfall([(cb: (err: Error, data: string) => void) => {
fs.readFile("test/files/metrics.csv", "utf8", (err, data) => {
cb(err, data);
});
}
, (file: string, cb: (err: Error) => void) => {
request(server)
.get("/v1/metrics")
.query({format: "ssv"})
.expect(200)
.expect((res: any) => {
expect(res.text).to.be.eql(file);
})
.end(cb);
}], (err) => {
expect(err).to.be.eql(null);
done();
});
});
});
......@@ -27,6 +27,42 @@ import { Request } from "../types";
* engine object that API users can use to create queries.
*/
export class EngineCtrl {
/**
* Auxiliary function that returns engine information.
* @param list - List of objects to return
* @param req - Object with request information
* @param res - Object used to create and send the response
* @param next - Call next middleware or controller. Not used but required
* by typescript definition of route.
*/
private static respondList(list: any[], fileName: string, req: Request, res: express.Response, next: express.NextFunction) {
let format = "json";
if (req.query.format) {
format = req.query.format;
}
if (format === "json") {
res.status(200).json(list);
}
else {
req.csvParser(list, format, (error: Error, csv: string) => {
if (error) {
res.status(500).json({
message: "Error generating csv file. " +
"Try json format.",
error: error
});
return;
}
const disposition = "attachment;filename=" + fileName + ".csv";
res.setHeader("Content-Type", "text/csv");
res.setHeader("Content-disposition", disposition);
res.status(200).send(csv);
});
}
}
/**
* Route that returns the list of available metrics.
* @param req - Object with request information
......@@ -35,7 +71,8 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static metrics(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getMetricsDescription());
const metrics = req.engine.getMetricsDescription();
EngineCtrl.respondList(metrics, "metrics", req, res, next);
}
/**
......@@ -46,7 +83,8 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static dimensions(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getDimensionsDescription());
const dimensions = req.engine.getDimensionsDescription();
EngineCtrl.respondList(dimensions, "dimensions", req, res, next);
}
/**
......@@ -57,7 +95,8 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static enumTypes(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getEnumTypesDescription());
const enumTypes = req.engine.getEnumTypesDescription();
EngineCtrl.respondList(enumTypes, "enums", req, res, next);
}
/**
......@@ -68,6 +107,7 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static sources(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getSourcesDescription());
const sources = req.engine.getSourcesDescription();
EngineCtrl.respondList(sources, "sources", req, res, next);
}
}
/*
* Copyright (C) 2015-2019 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import * as json2csv from "json-2-csv";
import { Middleware } from "../types";
/**
* Creates a csv parser and middleaew that
* inserts the parser into the request objects.
*/
export function CsvMw(): Middleware {
return function csvMiddleware(req, res, next) {
req.csvParser = function parseCsv(json: any, format: string, cb) {
const separator = format.substring(0, 1);
let sep = ",";
if (separator === "s") {
sep = ";";
}
else if (separator === "t"){
sep = "\t";
}
json2csv.json2csv(json, cb, {
delimiter: {
field: sep
, wrap: "\""
, eol: "\n"
}
, prependHeader: true
});
};
next();
};
}
......@@ -35,6 +35,8 @@ export interface Request extends express.Request {
engine: Engine;
/** A adapter object. Used to communicate with the database in use. */
adapter: Adapter;
/** A csvParser function. Used to parse json object into csv file. */
csvParser: (json: any, format: string, cb: (err: Error, csv?: string));
}
/**
......
......@@ -40,7 +40,7 @@ import { ConfigParser } from "./util/configParser";
let configPath;
/** @hidden */
if(process.env.BLENDB_SCHEMA_FILE){
if (process.env.BLENDB_SCHEMA_FILE) {
configPath = process.env.BLENDB_SCHEMA_FILE;
}
else{
......@@ -54,8 +54,10 @@ const config = ConfigParser.parse(configPath);
import { EngineMw } from "./api/middlewares/engine";
import { PostgresMw, MonetMw } from "./api/middlewares/adapter";
import { ErrorMw } from "./api/middlewares/error";
import { CsvMw } from "./api/middlewares/csv";
app.use(EngineMw(config));
app.use(CsvMw());
if (config.adapters[0] === "postgres") {
app.use(PostgresMw(config.connections[0]));
}
......
......@@ -173,4 +173,8 @@ export interface DataCtrlScenario {
* The products that are the most expensive
*/
expensive: Query;
/**
* The sellers average age by cpf and sex
*/
csv: Query;
}
dim:seller:sex,dim:seller:cpf,met:seller:avg:age
male,604.424.718-07,27
female,575.657.111-60,26
nonbinary,977.221.375-39,24
undecided,344.805.128-45,23
female,885.517.020-17,25
\ No newline at end of file
dim:seller:sex;dim:seller:cpf;met:seller:avg:age
male;604.424.718-07;27
female;575.657.111-60;26
nonbinary;977.221.375-39;24
undecided;344.805.128-45;23
female;885.517.020-17;25
\ No newline at end of file
dim:seller:sex dim:seller:cpf met:seller:avg:age
male 604.424.718-07 27
female 575.657.111-60 26
nonbinary 977.221.375-39 24
undecided 344.805.128-45 23
female 885.517.020-17 25
\ No newline at end of file
name;aggregation;dataType;description
met:seller:avg:age;avg;float;The seller average age
met:seller:max:age;max;integer;The seller highest age
met:seller:min:age;min;integer;The seller lowest age
met:seller:count:age;count;integer;The number of seller's
met:product:avg:pricein;avg;float;The average product pricein
met:product:max:pricein;max;float;The highest product pricein
met:product:min:pricein;min;float;The lowest product pricein
met:product:avg:priceout;avg;float;The average product priceout
met:product:max:priceout;max;float;The highest product priceout
met:product:min:priceout;min;float;The lowest product priceout
met:sell:sum:quantity;sum;integer;The sum of sales quantity
met:sell:avg:quantity;avg;float;The average of sales quantity
met:sell:count:quantity;count;integer;The total number of sales
met:buyout:avg:quantity;avg;float;The average of quantity bought
met:buyout:max:quantity;max;integer;The highest quantity bought
met:buyout:min:quantity;min;integer;The lowest quantity bought
\ No newline at end of file
......@@ -334,6 +334,10 @@ let qOpts : {[key: string]: QueryOpts} = {
metrics: [mets["met:seller:count:age"]],
dimensions: []
},
"csv": {
metrics: [mets["met:seller:avg:age"]],
dimensions: [dims["dim:seller:cpf"], dims["dim:seller:sex"]]
}
}
const queries : {[key: string]: Query} = {};
......@@ -437,4 +441,5 @@ export const dataCtrlScenario: DataCtrlScenario = {
correct: queries["correct"],
clausal: queries["clausal"],
expensive: queries["expensive"],
csv: queries["csv"],
};
......@@ -725,6 +725,13 @@ decamelize@^1.0.0, decamelize@^1.1.1:
version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
deeks@2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/deeks/-/deeks-2.2.1.tgz#4de360652998fd0a681153a9ffde5dcc6c983f4a"
integrity sha512-D2Qu3Fv5zBtBzXjXIUgWPRYn30d/IG6SCPOKIz42+LVTwhPeRI5+DAxdzap0vI2zYheiErIpVLwaUQBoA/iENw==
dependencies:
underscore "1.9.1"
deep-eql@^0.1.3:
version "0.1.3"
resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2"
......@@ -810,6 +817,11 @@ digest-header@^0.0.1:
dependencies:
utility "0.1.11"
doc-path@2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/doc-path/-/doc-path-2.0.1.tgz#d2adb8bcd31c895b17b92f61eff39aaec04430d2"
integrity sha512-/CCG157H//3l513omROUzaREChY/OYpxqYXvQcv7gsrwGfjVOh5d/1gJigHJ6iTnO77pA8rMLZ63CgEPEM6+9Q==
dom-serializer@0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.0.tgz#073c697546ce0780ce23be4a28e293e40bc30c82"
......@@ -1593,6 +1605,15 @@ jsesc@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-1.3.0.tgz#46c3fec8c1892b12b0833db9bc7622176dbab34b"
json-2-csv@^3.5.5:
version "3.5.5"
resolved "https://registry.yarnpkg.com/json-2-csv/-/json-2-csv-3.5.5.tgz#555d493547086a34da6e36672f5ea1849ebb6789"
integrity sha512-7BlKbejl42pmzoamfH+OFsNkVtPtWUuBJCCh/rhjf32fPTol5UwQGp051cf+qFfDHu0iQssBEzwRIDoTcD3MAw==
dependencies:
deeks "2.2.1"
doc-path "2.0.1"
underscore "1.9.1"
json-schema-compatibility@1.1.0, json-schema-compatibility@^1.0.1:
version "1.1.0"
resolved "https://registry.yarnpkg.com/json-schema-compatibility/-/json-schema-compatibility-1.1.0.tgz#1a8981778cda0c38187298d999d089e51af282df"
......@@ -3078,6 +3099,11 @@ underscore@1.8.3:
version "1.8.3"
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022"
underscore@1.9.1:
version "1.9.1"
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.9.1.tgz#06dce34a0e68a7babc29b365b8e74b8925203961"
integrity sha512-5/4etnCkd9c8gwgowi5/om/mYO5ajCaOgdzj/oW+0eQV9WxKBDZw5+ycmKmeaTXjInS/W0BzpGLo2xR2aBwZdg==
universalify@^0.1.0:
version "0.1.1"
resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.1.tgz#fa71badd4437af4c148841e3b3b165f9e9e590b7"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment