diff --git a/package.json b/package.json
index 06d658259063631455bef4d624ded367f1e8acce..f1b829d7a7f021dd7d4fbf76e6c5b7198de84b0f 100644
--- a/package.json
+++ b/package.json
@@ -29,6 +29,7 @@
"async": "=2.4.1",
"express": "^4.0.33",
"js-yaml": "^3.8.2",
+ "json-2-csv": "^3.5.5",
"monetdb": "^1.1.4",
"osprey": "^0.3.2",
"pg": "^6.1.5",
diff --git a/specs/blendb-api-v1.raml b/specs/blendb-api-v1.raml
index e5c69cb53a278be1c4194339e913e13da5e89690..4de37301acda02380488b2d3cbe51e9862e143b1 100644
--- a/specs/blendb-api-v1.raml
+++ b/specs/blendb-api-v1.raml
@@ -271,6 +271,18 @@ traits:
description: |
Fields to be returned.
type: string
+ - formatable:
+ queryParameters:
+ format:
+ description: |
+ Response format. Defines if the response objects will be a
+ json or a csv-like file. The default value is json.
+ The csv-like formats are: csv, ssv and tsv which the
+ separator is comma, semi-colon and tab respectively.
+ example: "ssv+"
+ required: false
+ pattern: "^json$|^csv$|^ssv$|^tsv$"
+ type: string
/metrics:
description: |
@@ -279,6 +291,7 @@ traits:
system and their descriptions.
securedBy: [ null, oauth_2_0 ]
get:
+ is: [ formatable ]
/sources:
description: |
A Source represents a type of object that can be inserted in the database.
@@ -286,6 +299,7 @@ traits:
system and their descriptions
securedBy: [ null, oauth_2_0 ]
get:
+ is: [ formatable ]
/dimensions:
description: |
@@ -294,12 +308,14 @@ traits:
the system and their descriptions.
securedBy: [ null, oauth_2_0 ]
get:
+ is: [ formatable ]
/enumtypes:
description: |
A EnumType is short for enumerable type. This is a special data type that only accepts a few possible values. This
collection allows the user to list all the enumerable types available in the system, their descriptions and possible
values.
get:
+ is: [ formatable ]
securedBy: [ null, oauth_2_0 ]
/data:
description: |
@@ -309,7 +325,7 @@ traits:
start/end dates to refine your query.
type: base
get:
- is: [ filtered ]
+ is: [ filtered, formatable ]
queryParameters:
metrics:
description: |
diff --git a/src/api/controllers/data.spec.ts b/src/api/controllers/data.spec.ts
index 83710e99d13d7bc3041bea4bcf85d19c6b9fea18..2e1f58cb5a770807a8a373bb7414087733b2dd5e 100644
--- a/src/api/controllers/data.spec.ts
+++ b/src/api/controllers/data.spec.ts
@@ -23,12 +23,15 @@ import { expect } from "chai";
import * as server from "../../main";
import { dataCtrlScenario as tests } from "../../../test/scenario";
import { Query } from "../../common/query";
+import { waterfall } from "async";
+import * as fs from "fs";
interface StrQuery {
metrics: string;
dimensions: string;
filters?: string;
sort?: string;
+ format?: string;
}
function parseQuery(obj: Query): StrQuery {
@@ -307,4 +310,72 @@ describe("API data controller", () => {
.end(done);
});
+ it("should respond 200 and get some data with format as csv", (done) => {
+ waterfall([(cb: (err: Error, data: string) => void) => {
+ fs.readFile("test/files/data.csv", "utf8", (err, data) => {
+ cb(err, data);
+ });
+ }
+ , (file: string, cb: (err: Error) => void) => {
+ let query = parseQuery(tests.csv);
+ query.format = "csv";
+ request(server)
+ .get("/v1/data")
+ .query(query)
+ .expect(200)
+ .expect((res: any) => {
+ expect(res.text).to.be.eql(file);
+ })
+ .end(cb);
+ }], (err) => {
+ expect(err).to.be.eql(null);
+ done();
+ });
+ });
+
+ it("should respond 200 and get some data with format as ssv", (done) => {
+ waterfall([(cb: (err: Error, data: string) => void) => {
+ fs.readFile("test/files/data.ssv", "utf8", (err, data) => {
+ cb(err, data);
+ });
+ }
+ , (file: string, cb: (err: Error) => void) => {
+ let query = parseQuery(tests.csv);
+ query.format = "ssv";
+ request(server)
+ .get("/v1/data")
+ .query(query)
+ .expect(200)
+ .expect((res: any) => {
+ expect(res.text).to.be.eql(file);
+ })
+ .end(cb);
+ }], (err) => {
+ expect(err).to.be.eql(null);
+ done();
+ });
+ });
+
+ it("should respond 200 and get some data with format as tsv", (done) => {
+ waterfall([(cb: (err: Error, data: string) => void) => {
+ fs.readFile("test/files/data.tsv", "utf8", (err, data) => {
+ cb(err, data);
+ });
+ }
+ , (file: string, cb: (err: Error) => void) => {
+ let query = parseQuery(tests.csv);
+ query.format = "tsv";
+ request(server)
+ .get("/v1/data")
+ .query(query)
+ .expect(200)
+ .expect((res: any) => {
+ expect(res.text).to.be.eql(file);
+ })
+ .end(cb);
+ }], (err) => {
+ expect(err).to.be.eql(null);
+ done();
+ });
+ });
});
diff --git a/src/api/controllers/data.ts b/src/api/controllers/data.ts
index a6e4b1f343f4baf3252f347cc5c717ccea9391dd..3cd155c3cb14cb044a8f3c43dfffaf9658c7c321 100644
--- a/src/api/controllers/data.ts
+++ b/src/api/controllers/data.ts
@@ -45,6 +45,12 @@ export class DataCtrl {
if (req.query.sort) {
sort = req.query.sort.split(",").filter((item: string) => item !== "");
}
+
+ let format = "json";
+ if (req.query.format) {
+ format = req.query.format;
+ }
+
let view;
try {
@@ -102,7 +108,26 @@ export class DataCtrl {
return;
}
- res.status(200).json(result);
+ if (format === "json") {
+ res.status(200).json(result);
+ }
+
+ else {
+ req.csvParser(result, format, (error: Error, csv: string) => {
+ if (error) {
+ res.status(500).json({
+ message: "Error generating csv file. " +
+ "Try json format.",
+ error: error
+ });
+ return;
+ }
+
+ res.setHeader("Content-Type", "text/csv");
+ res.setHeader("Content-disposition", "attachment;filename=data.csv");
+ res.status(200).send(csv);
+ });
+ }
return;
});
}
diff --git a/src/api/controllers/engine.spec.ts b/src/api/controllers/engine.spec.ts
index 28f71adfc05b58429e9802e8b74a6dc0ab0beb5b..7290c349888a8623892738e7f32748dd73854344 100644
--- a/src/api/controllers/engine.spec.ts
+++ b/src/api/controllers/engine.spec.ts
@@ -19,8 +19,10 @@
*/
import * as request from "supertest";
+import * as fs from "fs";
import { expect } from "chai";
import * as server from "../../main";
+import { waterfall } from "async";
describe("API engine controller", () => {
@@ -68,4 +70,25 @@ describe("API engine controller", () => {
.end(done);
});
+ it("should respond 200 and the list of metrics (in csv)", (done) => {
+ waterfall([(cb: (err: Error, data: string) => void) => {
+ fs.readFile("test/files/metrics.csv", "utf8", (err, data) => {
+ cb(err, data);
+ });
+ }
+ , (file: string, cb: (err: Error) => void) => {
+ request(server)
+ .get("/v1/metrics")
+ .query({format: "ssv"})
+ .expect(200)
+ .expect((res: any) => {
+ expect(res.text).to.be.eql(file);
+ })
+ .end(cb);
+ }], (err) => {
+ expect(err).to.be.eql(null);
+ done();
+ });
+ });
+
});
diff --git a/src/api/controllers/engine.ts b/src/api/controllers/engine.ts
index 4fde6269537dca47013104a91200a53527965e91..802acc1ae49f898cc3fc32568d968ab09ac88003 100644
--- a/src/api/controllers/engine.ts
+++ b/src/api/controllers/engine.ts
@@ -27,6 +27,42 @@ import { Request } from "../types";
* engine object that API users can use to create queries.
*/
export class EngineCtrl {
+ /**
+ * Auxiliary function that returns engine information.
+ * @param list - List of objects to return
+ * @param req - Object with request information
+ * @param res - Object used to create and send the response
+ * @param next - Call next middleware or controller. Not used but required
+ * by typescript definition of route.
+ */
+ private static respondList(list: any[], fileName: string, req: Request, res: express.Response, next: express.NextFunction) {
+ let format = "json";
+ if (req.query.format) {
+ format = req.query.format;
+ }
+
+ if (format === "json") {
+ res.status(200).json(list);
+ }
+
+ else {
+ req.csvParser(list, format, (error: Error, csv: string) => {
+ if (error) {
+ res.status(500).json({
+ message: "Error generating csv file. " +
+ "Try json format.",
+ error: error
+ });
+ return;
+ }
+
+ const disposition = "attachment;filename=" + fileName + ".csv";
+ res.setHeader("Content-Type", "text/csv");
+ res.setHeader("Content-disposition", disposition);
+ res.status(200).send(csv);
+ });
+ }
+ }
/**
* Route that returns the list of available metrics.
* @param req - Object with request information
@@ -35,7 +71,8 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static metrics(req: Request, res: express.Response, next: express.NextFunction) {
- res.status(200).json(req.engine.getMetricsDescription());
+ const metrics = req.engine.getMetricsDescription();
+ EngineCtrl.respondList(metrics, "metrics", req, res, next);
}
/**
@@ -46,7 +83,8 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static dimensions(req: Request, res: express.Response, next: express.NextFunction) {
- res.status(200).json(req.engine.getDimensionsDescription());
+ const dimensions = req.engine.getDimensionsDescription();
+ EngineCtrl.respondList(dimensions, "dimensions", req, res, next);
}
/**
@@ -57,7 +95,8 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static enumTypes(req: Request, res: express.Response, next: express.NextFunction) {
- res.status(200).json(req.engine.getEnumTypesDescription());
+ const enumTypes = req.engine.getEnumTypesDescription();
+ EngineCtrl.respondList(enumTypes, "enums", req, res, next);
}
/**
@@ -68,6 +107,7 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static sources(req: Request, res: express.Response, next: express.NextFunction) {
- res.status(200).json(req.engine.getSourcesDescription());
+ const sources = req.engine.getSourcesDescription();
+ EngineCtrl.respondList(sources, "sources", req, res, next);
}
}
diff --git a/src/api/middlewares/csv.ts b/src/api/middlewares/csv.ts
new file mode 100644
index 0000000000000000000000000000000000000000..42540aae3cfc6d3765a40e0d6e48f1c7f289c343
--- /dev/null
+++ b/src/api/middlewares/csv.ts
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2015-2019 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as json2csv from "json-2-csv";
+import { Middleware } from "../types";
+
+/**
+ * Creates a csv parser and middleaew that
+ * inserts the parser into the request objects.
+ */
+export function CsvMw(): Middleware {
+ return function csvMiddleware(req, res, next) {
+ req.csvParser = function parseCsv(json: any, format: string, cb) {
+ const separator = format.substring(0, 1);
+
+ let sep = ",";
+ if (separator === "s") {
+ sep = ";";
+ }
+
+ else if (separator === "t"){
+ sep = "\t";
+ }
+
+ json2csv.json2csv(json, cb, {
+ delimiter: {
+ field: sep
+ , wrap: "\""
+ , eol: "\n"
+ }
+ , prependHeader: true
+ });
+ };
+ next();
+ };
+
+}
diff --git a/src/api/types.ts b/src/api/types.ts
index 0fe099f9690f47667205d7bb723da6168f9b984b..469b6adc6b48b53ca773fd0c9bf42556e96a99c4 100644
--- a/src/api/types.ts
+++ b/src/api/types.ts
@@ -35,6 +35,8 @@ export interface Request extends express.Request {
engine: Engine;
/** A adapter object. Used to communicate with the database in use. */
adapter: Adapter;
+ /** A csvParser function. Used to parse json object into csv file. */
+ csvParser: (json: any, format: string, cb: (err: Error, csv?: string));
}
/**
diff --git a/src/main.ts b/src/main.ts
index 0710b3cc5db45c3b2d378e021a10c0bc00c798f4..4eb837ddb67c69e9fe7439a721ce670cbd5ddce9 100755
--- a/src/main.ts
+++ b/src/main.ts
@@ -40,7 +40,7 @@ import { ConfigParser } from "./util/configParser";
let configPath;
/** @hidden */
-if(process.env.BLENDB_SCHEMA_FILE){
+if (process.env.BLENDB_SCHEMA_FILE) {
configPath = process.env.BLENDB_SCHEMA_FILE;
}
else{
@@ -54,8 +54,10 @@ const config = ConfigParser.parse(configPath);
import { EngineMw } from "./api/middlewares/engine";
import { PostgresMw, MonetMw } from "./api/middlewares/adapter";
import { ErrorMw } from "./api/middlewares/error";
+import { CsvMw } from "./api/middlewares/csv";
app.use(EngineMw(config));
+app.use(CsvMw());
if (config.adapters[0] === "postgres") {
app.use(PostgresMw(config.connections[0]));
}
diff --git a/src/util/scenarioHandler.ts b/src/util/scenarioHandler.ts
index ae981185caef05afaf3a9488cd0f5dea29b9a7da..bb2de85715ff1e36320035cfd58071ec05f3b9d7 100644
--- a/src/util/scenarioHandler.ts
+++ b/src/util/scenarioHandler.ts
@@ -173,4 +173,8 @@ export interface DataCtrlScenario {
* The products that are the most expensive
*/
expensive: Query;
+ /**
+ * The sellers average age by cpf and sex
+ */
+ csv: Query;
}
diff --git a/test/files/data.csv b/test/files/data.csv
new file mode 100644
index 0000000000000000000000000000000000000000..47783fd0ee7db1c1425741032efe42f9605048e0
--- /dev/null
+++ b/test/files/data.csv
@@ -0,0 +1,6 @@
+dim:seller:sex,dim:seller:cpf,met:seller:avg:age
+male,604.424.718-07,27
+female,575.657.111-60,26
+nonbinary,977.221.375-39,24
+undecided,344.805.128-45,23
+female,885.517.020-17,25
\ No newline at end of file
diff --git a/test/files/data.ssv b/test/files/data.ssv
new file mode 100644
index 0000000000000000000000000000000000000000..fa76b7ef2d198c5031c824bfd2c3d91aad9d68e9
--- /dev/null
+++ b/test/files/data.ssv
@@ -0,0 +1,6 @@
+dim:seller:sex;dim:seller:cpf;met:seller:avg:age
+male;604.424.718-07;27
+female;575.657.111-60;26
+nonbinary;977.221.375-39;24
+undecided;344.805.128-45;23
+female;885.517.020-17;25
\ No newline at end of file
diff --git a/test/files/data.tsv b/test/files/data.tsv
new file mode 100644
index 0000000000000000000000000000000000000000..70d90f86559c0d52b753065211101ec9f914b8f3
--- /dev/null
+++ b/test/files/data.tsv
@@ -0,0 +1,6 @@
+dim:seller:sex dim:seller:cpf met:seller:avg:age
+male 604.424.718-07 27
+female 575.657.111-60 26
+nonbinary 977.221.375-39 24
+undecided 344.805.128-45 23
+female 885.517.020-17 25
\ No newline at end of file
diff --git a/test/files/metrics.csv b/test/files/metrics.csv
new file mode 100644
index 0000000000000000000000000000000000000000..468d068440ee978ed90cc992e803228f9a833497
--- /dev/null
+++ b/test/files/metrics.csv
@@ -0,0 +1,17 @@
+name;aggregation;dataType;description
+met:seller:avg:age;avg;float;The seller average age
+met:seller:max:age;max;integer;The seller highest age
+met:seller:min:age;min;integer;The seller lowest age
+met:seller:count:age;count;integer;The number of seller's
+met:product:avg:pricein;avg;float;The average product pricein
+met:product:max:pricein;max;float;The highest product pricein
+met:product:min:pricein;min;float;The lowest product pricein
+met:product:avg:priceout;avg;float;The average product priceout
+met:product:max:priceout;max;float;The highest product priceout
+met:product:min:priceout;min;float;The lowest product priceout
+met:sell:sum:quantity;sum;integer;The sum of sales quantity
+met:sell:avg:quantity;avg;float;The average of sales quantity
+met:sell:count:quantity;count;integer;The total number of sales
+met:buyout:avg:quantity;avg;float;The average of quantity bought
+met:buyout:max:quantity;max;integer;The highest quantity bought
+met:buyout:min:quantity;min;integer;The lowest quantity bought
\ No newline at end of file
diff --git a/test/scenario.ts b/test/scenario.ts
index ad807b8260b1ebd863689835d6785b63ead2eb92..d598f79ecd548b1ea24b87d8cf42048bede546b8 100644
--- a/test/scenario.ts
+++ b/test/scenario.ts
@@ -334,6 +334,10 @@ let qOpts : {[key: string]: QueryOpts} = {
metrics: [mets["met:seller:count:age"]],
dimensions: []
},
+ "csv": {
+ metrics: [mets["met:seller:avg:age"]],
+ dimensions: [dims["dim:seller:cpf"], dims["dim:seller:sex"]]
+ }
}
const queries : {[key: string]: Query} = {};
@@ -437,4 +441,5 @@ export const dataCtrlScenario: DataCtrlScenario = {
correct: queries["correct"],
clausal: queries["clausal"],
expensive: queries["expensive"],
+ csv: queries["csv"],
};
diff --git a/yarn.lock b/yarn.lock
index 96d8f20df54ea37887898695adfa575f4fa30f9a..5bfc5e3d1b634079e6de4c157136677df15d7d83 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -725,6 +725,13 @@ decamelize@^1.0.0, decamelize@^1.1.1:
version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
+deeks@2.2.1:
+ version "2.2.1"
+ resolved "https://registry.yarnpkg.com/deeks/-/deeks-2.2.1.tgz#4de360652998fd0a681153a9ffde5dcc6c983f4a"
+ integrity sha512-D2Qu3Fv5zBtBzXjXIUgWPRYn30d/IG6SCPOKIz42+LVTwhPeRI5+DAxdzap0vI2zYheiErIpVLwaUQBoA/iENw==
+ dependencies:
+ underscore "1.9.1"
+
deep-eql@^0.1.3:
version "0.1.3"
resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2"
@@ -810,6 +817,11 @@ digest-header@^0.0.1:
dependencies:
utility "0.1.11"
+doc-path@2.0.1:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/doc-path/-/doc-path-2.0.1.tgz#d2adb8bcd31c895b17b92f61eff39aaec04430d2"
+ integrity sha512-/CCG157H//3l513omROUzaREChY/OYpxqYXvQcv7gsrwGfjVOh5d/1gJigHJ6iTnO77pA8rMLZ63CgEPEM6+9Q==
+
dom-serializer@0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.0.tgz#073c697546ce0780ce23be4a28e293e40bc30c82"
@@ -1593,6 +1605,15 @@ jsesc@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-1.3.0.tgz#46c3fec8c1892b12b0833db9bc7622176dbab34b"
+json-2-csv@^3.5.5:
+ version "3.5.5"
+ resolved "https://registry.yarnpkg.com/json-2-csv/-/json-2-csv-3.5.5.tgz#555d493547086a34da6e36672f5ea1849ebb6789"
+ integrity sha512-7BlKbejl42pmzoamfH+OFsNkVtPtWUuBJCCh/rhjf32fPTol5UwQGp051cf+qFfDHu0iQssBEzwRIDoTcD3MAw==
+ dependencies:
+ deeks "2.2.1"
+ doc-path "2.0.1"
+ underscore "1.9.1"
+
json-schema-compatibility@1.1.0, json-schema-compatibility@^1.0.1:
version "1.1.0"
resolved "https://registry.yarnpkg.com/json-schema-compatibility/-/json-schema-compatibility-1.1.0.tgz#1a8981778cda0c38187298d999d089e51af282df"
@@ -3078,6 +3099,11 @@ underscore@1.8.3:
version "1.8.3"
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022"
+underscore@1.9.1:
+ version "1.9.1"
+ resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.9.1.tgz#06dce34a0e68a7babc29b365b8e74b8925203961"
+ integrity sha512-5/4etnCkd9c8gwgowi5/om/mYO5ajCaOgdzj/oW+0eQV9WxKBDZw5+ycmKmeaTXjInS/W0BzpGLo2xR2aBwZdg==
+
universalify@^0.1.0:
version "0.1.1"
resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.1.tgz#fa71badd4437af4c148841e3b3b165f9e9e590b7"