diff --git a/config/ci_test.yaml.example b/config/ci_test.yaml.example index 30192280141dbd5e14709a3b925873543fadd32b..b319223d69cb93affeed18a80eb000e2d8514a05 100644 --- a/config/ci_test.yaml.example +++ b/config/ci_test.yaml.example @@ -213,30 +213,31 @@ dimensions: description: "A dimension of Blendb. Has 1 possible value." enumTypes: - - name: "enum type:0" + name: "enumtype:0" values: - - "test:0" - - "test:1" - - "test:2" - - "test:3" + - "male" + - "female" + - "binary" + - "undecided" - - name: "enum type:1" + name: "enumtype:1" values: - - "test:4" - - "test:5" - - "test:6" + - "test_4" + - "test_5" + - "test_6" + - "string" - - name: "enum type:2" + name: "enumtype:2" values: - - "test:7" - - "test:8" + - "test_7" + - "test_8" - - name: "enum type:3" + name: "enumtype:3" values: - - "test:9" + - "test_9" sources: - - name: "source:0" + name: "source_0" description: "source with 3 entries" fields: - @@ -252,30 +253,55 @@ sources: description: "third entry" dataType: "string" - - name: "source:1" + name: "source_1" description: "source with 2 entries" fields: - name: "fields:0" description: "first entry" - dataType: "string" + dataType: "enumtype:0" - name: "fields:1" description: "second entry" dataType: "string" - - name: "source:2" - description: "source with one entry" + name: "source_2" + description: "source with one entry and undefined dataType" fields: - name: "fields:0" description: "first entry" - dataType: "string" + dataType: "notValid" - - name: "source:3" + name: "source_3" description: "source with one entry and without description" fields: - - name: "fields:3" + name: "fields:0" + dataType: "string" + - + name: "source_4" + description: "source with all core types from blendb" + fields: + - + name: "fields:0" + description: "first entry" + dataType: "integer" + - + name: "fields:1" + description: "second entry" + dataType: "float" + - + name: "fields:2" + description: "third entry" dataType: "string" + - + name: "fields:3" + description: "fourth entry" + dataType: "boolean" + - + name: "fields:4" + description: "fifth entry" + dataType: "date" + \ No newline at end of file diff --git a/src/adapter/monet.ts b/src/adapter/monet.ts index aa01de01d99b7bc5d833340799886ba6e670e14a..321a3872ebd25930f537f062b7a2aee61a5e5b63 100644 --- a/src/adapter/monet.ts +++ b/src/adapter/monet.ts @@ -20,6 +20,7 @@ import { SQLAdapter } from "./sql"; import { View } from "../core/view"; +import { Source } from "../core/source"; import { FilterOperator } from "../core/filter"; const MDB = require("monetdb")(); @@ -45,7 +46,10 @@ export class MonetAdapter extends SQLAdapter { } public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void { const query = this.getQueryFromView(view); + this.executeQuery(query, cb); + } + private executeQuery(query: string, cb: (error: Error, result?: any[]) => void): void { let pool: any = new MDB(this.config); pool.connect(); pool.query(query).then((result: MonetResult) => { @@ -78,6 +82,11 @@ export class MonetAdapter extends SQLAdapter { return false; } + public insertIntoSource(source: Source, data: any[], cb: (err: Error, result: any[]) => void): void { + const query = this.getQueryFromSource(source, data); + this.executeQuery(query, cb); + } + protected typeCast(quotedValue: string, dt: string): string { switch (dt) { case "date": diff --git a/src/adapter/postgres.ts b/src/adapter/postgres.ts index 3566542b2d1111ecaa8b8d625d1fc914925ff94b..e01b2bb32064cd0da0078df23935297e5dbac5ce 100644 --- a/src/adapter/postgres.ts +++ b/src/adapter/postgres.ts @@ -20,6 +20,7 @@ import { SQLAdapter } from "./sql"; import { View } from "../core/view"; +import { Source } from "../core/source"; import { FilterOperator } from "../core/filter"; import { Pool, PoolConfig } from "pg"; @@ -32,7 +33,10 @@ export class PostgresAdapter extends SQLAdapter { } public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void { const query = this.getQueryFromView(view); + this.executeQuery(query, cb); + } + private executeQuery(query: string, cb: (err: Error, result?: any[]) => void): void{ this.pool.connect((err, client, done) => { if (err) { cb (err); @@ -45,6 +49,10 @@ export class PostgresAdapter extends SQLAdapter { }); }); } + public insertIntoSource(source: Source, data: any[], cb: (err: Error, result?: any[]) => void): void { + const query = this.getQueryFromSource(source, data); + this.executeQuery(query, cb); + } public materializeView(view: View): boolean { return false; diff --git a/src/adapter/sql.ts b/src/adapter/sql.ts index 0ef7ca229ae05aafe5bccd7a111d2430ac71ad27..dc6c3cdc02393349ffbcafb1be81143ba1ec7cc9 100644 --- a/src/adapter/sql.ts +++ b/src/adapter/sql.ts @@ -20,6 +20,7 @@ import { Adapter } from "../core/adapter"; import { Metric } from "../core/metric"; +import { Source } from "../core/source"; import { Dimension } from "../core/dimension"; import { Clause } from "../core/clause"; import { Filter, FilterOperator } from "../core/filter"; @@ -844,4 +845,22 @@ export abstract class SQLAdapter extends Adapter { } return dims; } + + public getQueryFromSource(source: Source, data: any[]): string { + let consult: string; + let colums: any[] = []; + let values: string[] = []; + let tam = Object.keys(data).length; + colums = source.fields.map(o => o.name); + for (let i = 0; i < tam; i++){ + values[i] = data[colums[i]]; + } + consult = "INSERT INTO " + source.name + " (\""; + consult = consult.concat(colums.join("\",\"")); + consult = consult.concat("\") VALUES ('"); + consult = consult.concat(values.join("' , '")); + consult = consult.concat("');"); + + return consult; + } } diff --git a/src/api/controllers/collect.spec.ts b/src/api/controllers/collect.spec.ts index 958196db90dbc26854be8d8b37959ca7a0e980d2..d4b57f0efc470560bc5a4f6732d80e85645dfea9 100644 --- a/src/api/controllers/collect.spec.ts +++ b/src/api/controllers/collect.spec.ts @@ -20,37 +20,259 @@ import * as request from "supertest"; import { expect } from "chai"; - import * as server from "../../main"; +import { Adapter } from "../../core/adapter"; +import { ConfigParser } from "../../util/configParser"; +import { Fixture as FixPostgres } from "../../../test/postgres/fixture"; +import { Fixture as FixMonet } from "../../../test/monet/fixture"; +import { MonetAdapter, MonetConfig } from "../../adapter/monet"; +import { PostgresAdapter } from "../../adapter/postgres"; describe("API collect controller", () => { - it("should respond 400 when _id field is in request body", (done) => { + // Initializing + let config: any; + let adapter: Adapter; + let fixture; + before(function (done): void { + // Arrow function not used to get acces to this and skip the test + config = ConfigParser.parse("config/test.yaml"); + if (config.adapter === "postgres") { + fixture = new FixPostgres(config.connection); + fixture.LoadSource(config.sources, config.struct.create, (err) => { + if (err) { + throw err; + } + adapter = new PostgresAdapter(config.connection); + done(); + }); + } + else if (config.adapter === "monet") { + fixture = new FixMonet(config.connection); + fixture.load(config.sources, config.struct.create, (err) => { + if (err) { + throw err; + } + let parsedConfig: MonetConfig = { + user: config.connection.user, + dbname: config.connection.database, + password: config.connection.password, + host: config.connection.host, + port: config.connection.port + }; + adapter = new MonetAdapter(parsedConfig); + done(); + }); + } + else { + this.skip(); + } + }); + it("should respond 500 when req.params.class does not exist on Sources", (done) => { + request(server) + .post("/v1/collect/thisisjustatest") + .send({"fields:1": 1, "fields:2": 2}) + .expect(500) + .expect((res: any) => { + + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The source named 'thisisjustatest' was not found"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 500 when fields:0 does not exist on Source", (done) => { + request(server) + .post("/v1/collect/source_0") + .send({"fields:1": 1, "fields:2": 2}) + .expect(500) + .expect((res: any) => { + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The 'fields:0' wasn't informed on json"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 200 when data has been stored on source_0", (done) => { + request(server) + .post("/v1/collect/source_0") + .send({"fields:0": "teste", "fields:1": "test1", "fields:2": "teste2"}) + .expect(200) + .expect((res: any) => { + const message = "Data has been successfully received and stored by the server"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + + }) + .end(done); + }); + it("should respond 500 when value isn't defined on enumtype:0 ", (done) => { request(server) - .post("/v1/collect/class") - .send({"_id": 1}) - .expect(400) + .post("/v1/collect/source_1") + .send({"fields:0": 1, "fields:1": 2}) + .expect(500) .expect((res: any) => { - const message = "Property named \"_id\" is protected."; + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value '1' from 'fields:0' isn't listed on enumtype:0"; expect(res.body).to.be.an("object"); expect(res.body).to.have.property("message"); expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + }) .end(done); }); + it("should respond 200 when data has been stored on source_1", (done) => { + request(server) + .post("/v1/collect/source_1") + .send({"fields:0": "male", "fields:1": "test1"}) + .expect(200) + .expect((res: any) => { + const message = "Data has been successfully received and stored by the server"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); - it("should respond 500 on error on writing in the database", (done) => { + }) + .end(done); + }); + it("should respond 500 when dataType from fields:0 isn't integer", (done) => { request(server) - .post("/v1/collect/class") - .send({"field1": 1, "field2": 2}) + .post("/v1/collect/source_4") + .send({"fields:0" : "nope", "fields:1" : 95.5 , "fields:2" : "justabacon" + , "fields:3" : "1991-25-03" , "fields:4" : 1}) .expect(500) .expect((res: any) => { - const message = "Error while writing to the database."; + + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value 'nope' from 'fields:0' isn't a type integer"; expect(res.body).to.be.an("object"); expect(res.body).to.have.property("message"); expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + }) .end(done); }); + it("should respond 500 when dataType from fields:1 isn't float", (done) => { + request(server) + .post("/v1/collect/source_4") + .send({"fields:0" : 1 , "fields:1" : "notafloat" , "fields:2" : "justabacon" + , "fields:3" : "1991-25-03" , "fields:4" : 1}) + .expect(500) + .expect((res: any) => { + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value 'notafloat' from 'fields:1' isn't a type float"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 500 when dataType from fields:2 isn't string", (done) => { + request(server) + .post("/v1/collect/source_4") + .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : 1 + , "fields:3" : "1991-25-03" , "fields:4" : 1}) + .expect(500) + .expect((res: any) => { + + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value '1' from 'fields:2' isn't a type string"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 500 when dataType from fields:3 isn't boolean", (done) => { + request(server) + .post("/v1/collect/source_4") + .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : "teste" + , "fields:3" : "notaboolean" , "fields:4" : "1999-10-10"}) + .expect(500) + .expect((res: any) => { + + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value 'notaboolean' from 'fields:3' isn't a type boolean"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 500 when the first dataType from fields:4 isn't date", (done) => { + request(server) + .post("/v1/collect/source_4") + .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : "teste" + , "fields:3" : "true" , "fields:4" : "1999-25-25"}) + .expect(500) + .expect((res: any) => { + + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value '1999-25-25' from 'fields:4' isn't a type date"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 200 when sucessfull insert data on source_4", (done) => { + request(server) + .post("/v1/collect/source_4") + .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : "teste" + , "fields:3" : "true" , "fields:4" : "1999-10-10"}) + .expect(200) + .expect((res: any) => { + const message = "Data has been successfully received and stored by the server"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + + }) + .end(done); + }); + it("should respond 500 when dataType does not exist", (done) => { + request(server) + .post("/v1/collect/source_2") + .send({"fields:0" : 1 }) + .expect(500) + .expect((res: any) => { + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The dataType named 'notValid' was not found"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); }); diff --git a/src/api/controllers/collect.ts b/src/api/controllers/collect.ts index a273be52624a964fb57ccc8dcd158d6f9ff265fc..5fc3207e34207912fe0a55a334920782f88c06ad 100644 --- a/src/api/controllers/collect.ts +++ b/src/api/controllers/collect.ts @@ -19,16 +19,152 @@ */ import * as express from "express"; +import { Request } from "../types"; +import { Source, Field } from "../../core/source"; +import { EnumType } from "../../core/enumType"; + +interface Valid{ + [key: string]: (value: any) => boolean; +} export class CollectCtrl { - public static write(req: express.Request, res: express.Response, next: express.NextFunction) { - if ("_id" in req.body) { - res.status(400) - .json({ message: "Property named \"_id\" is protected." }); + public static write(req: Request, res: express.Response, next: express.NextFunction) { + + const validador: Valid = { + "integer": function(value: any) { + let test = parseInt(value, 10); + if (test === undefined || isNaN(value) ) { + return false; + } + + else { + return true; + } + }, + "float": function(value: any) { + let test = parseFloat(value); + if (test === undefined || isNaN(value) ) { + return false; + } + + else { + return true; + } + }, + "string": function(value: any) { + let test = typeof(value); + if (test === "string") { + return true; + } + + else{ + return false; + } + }, + "date": function(value: any) { + + let test: string[] = []; + let date = new Date(value); + try { + test = date.toISOString().split("T"); + } + catch (e) { + return false; + } + if (test[0] === value){ + return true; + } + else{ + return false; + } + + }, + "boolean": function(value: any) { + let test: string = value; + test = test.toLocaleLowerCase(); + if (test === "true" || test === "false"){ + return true; + } + + else{ + return false; + } + } + }; + + let fields: Field[] = []; + let data: string[] = []; + let types: string[] = []; + let source: Source; + let enumType: EnumType; + + const id: string = req.params.class; + try { + source = req.engine.getSourceByName(id); + // If source does not exist them return error + + fields = source.fields; + + for (let i = 0; i < fields.length; i++){ + data[i] = req.body[fields[i].name]; + if (!data[i]){ + throw new Error( + "The '" + fields[i].name + "' wasn't informed on json"); + } + } + + for (let i = 0; i < fields.length; i++){ + if (validador[fields[i].dataType] !== undefined){ + if (!validador[fields[i].dataType](data[i]) === true){ + throw new Error( + "The value '" + data[i] + "' from '" + fields[i].name + + "' isn't a type " + fields[i].dataType); + } + + } + else { + enumType = req.engine.getEnumTypeByName(fields[i].dataType); + + types = enumType.values; + let found: boolean = false; + for (let j = 0; j < types.length; j++){ + if (data[i] === types[j]){ + found = true; + break; + } + } + if (!found) { + throw new Error( + "The value '" + data[i] + "' from '" + fields[i].name + + "' isn't listed on " + fields[i].dataType); + } + } + } + } + + catch (e) { + res.status(500).json({ + message: "Query execution failed: " + + "Could not construct query with the paramters given.", + error: e.message + }); return; } - res.status(500) - .json({ message: "Error while writing to the database." }); + req.adapter.insertIntoSource(source, req.body, (err: Error, result: any[]) => { + if (err) { + res.status(500).json({ + message: "Insertion has failed", + error: err + }); + return; + } + else{ + res.status(200).json({message: "Data has been successfully received and stored by the server"}); + return; + } + + }); + } } diff --git a/src/api/controllers/engine.spec.ts b/src/api/controllers/engine.spec.ts index 79e8c1800600dbee9254307811867bd0c9022b57..45cb983c89f2a8e09c8e71c0b327ef0a24567f02 100644 --- a/src/api/controllers/engine.spec.ts +++ b/src/api/controllers/engine.spec.ts @@ -43,7 +43,7 @@ describe("API engine controller", () => { .expect((res: any) => { let result = res.body; expect(result).to.be.an("array"); - expect(result).to.have.length(4); + expect(result).to.have.length(5); }) .end(done); }); diff --git a/src/core/adapter.ts b/src/core/adapter.ts index 20cb90cfa5b995959f0aa7fb6b1d9381d97e0cdf..d70f56151307c936d1c183b5855d9c29742252e2 100644 --- a/src/core/adapter.ts +++ b/src/core/adapter.ts @@ -19,8 +19,10 @@ */ import { View } from "./view"; +import { Source } from "./source"; export abstract class Adapter { public abstract getDataFromView(view: View, cb: (err: Error, result: any[]) => void): void; public abstract materializeView(view: View): boolean; + public abstract insertIntoSource(source: Source, data: any[], cb: (err: Error, result: any[]) => void): void; } diff --git a/src/core/engine.ts b/src/core/engine.ts index e2d4ef146e992b6c3c1b26f239579020953d31b2..633319491575118dcaf994af73caff9723cc2e6e 100644 --- a/src/core/engine.ts +++ b/src/core/engine.ts @@ -103,6 +103,26 @@ export class Engine { return result; } + public getEnumTypeByName(name: string): EnumType { + let result = this.enumTypes.find(EnumType => EnumType.name === name); + + if (!result) { + throw new Error("The dataType named '" + name + "' was not found"); + } + + return result; + } + + public getSourceByName(name: string): Source { + let result = this.sources.find(source => source.name === name); + + if (!result) { + throw new Error("The source named '" + name + "' was not found"); + } + + return result; + } + public addDimension(dimension: Dimension): Dimension { if (this.graph.addDimension(dimension)) { this.dimensions.push(dimension); diff --git a/src/util/configParser.ts b/src/util/configParser.ts index 21cad7f9067886795c429a145ddf2ac3762ff736..6e15a86fed21aa5b573c1c3cfc126ca2785b55d0 100644 --- a/src/util/configParser.ts +++ b/src/util/configParser.ts @@ -105,7 +105,7 @@ export class ConfigParser { public static parse(configPath: string): ParsedConfig { let config: ConfigSchema = yaml.safeLoad(fs.readFileSync(configPath, { encoding: "utf-8" - })); + })) as ConfigSchema; let connection: Connection = { user: process.env.BLENDB_DB_USER, @@ -212,6 +212,7 @@ export class ConfigParser { } return parsed; + } public static parseViewOpt(opts: ViewParsingOptions, diff --git a/test/postgres/fixture.ts b/test/postgres/fixture.ts index 4fdc013438221fe3305ec3c929f3aeb3f16a9bca..3233453e6bd5a1233fc827dbf8995d621d0ecb8e 100644 --- a/test/postgres/fixture.ts +++ b/test/postgres/fixture.ts @@ -20,6 +20,7 @@ import { Client, PoolConfig } from "pg"; import { View, LoadView } from "../../src/core/view"; +import { Source } from "../../src/core/source"; import { each, series } from "async"; import * as fs from "fs"; @@ -28,6 +29,12 @@ interface TransSet { data: string[]; } +interface SoucerParse{ + name: string; + type: string[]; + fields: string[]; +} + export interface Schema { alias?: string; query?: string; @@ -107,7 +114,7 @@ export class Fixture { case "boolean": return "BOOLEAN"; default: - return ""; + return "TEXT"; } } @@ -148,4 +155,59 @@ export class Fixture { } return transaction; } + public LoadSource(source: Source[], create: boolean , cb: (err: Error) => void): void { + + let client = new Client(this.config); + + let query: string[] = []; + for (let i = 0; i < source.length; i++ ){ + query[i] = this.ExtractData(source[i], create); + } + + client.connect((error) => { + if (error) { + cb(error); + return; + } + + series([(callback: (err: Error) => void) => { + each(query, (insere, cback) => { + return client.query(insere , [], (err: Error) => cback(err)); + }, (errQuery: Error) => callback(errQuery)); + }], (errQuery: Error) => { + if (errQuery) { + client.end(); + cb(errQuery); + } + client.end((err) => { + cb(err); + }); + }); + }); + + } + private ExtractData(data: Source , create: boolean): string{ + let name: string; + let type: string[]; + let fields: string[]; + let consult: string; + + name = data.name; + type = data.fields.map((item) => item.dataType); + fields = data.fields.map((item) => item.name); + + if (create){ + consult = "CREATE TABLE " + name + " (" + '"'; + for (let i = 0; i < fields.length; i++){ + fields[i] = fields[i].concat('"' + " " + this.typeConvertion(type[i])); + } + consult = consult.concat(fields.join(", " + '"')); + consult = consult.concat(");"); + } + else{ + consult = "TRUNCATE TABLE " + name + ";"; + } + return consult; + + } }