Commit ba1a7642 authored by Lucas Fernandes de Oliveira's avatar Lucas Fernandes de Oliveira

Merge branch 'issue/67' into 'develop'

Issue #67: Add dataType type check

See merge request !63
parents 3da7d2b4 cf4e4912
Pipeline #15693 passed with stages
in 1 minute and 15 seconds
......@@ -271,7 +271,7 @@ sources:
-
name: "fields:0"
description: "first entry"
dataType: "notValid"
dataType: "string"
-
name: "source_3"
description: "source with one entry and without description"
......
......@@ -22,6 +22,7 @@ import { SQLAdapter } from "./sql";
import { View } from "../core/view";
import { Source } from "../core/source";
import { FilterOperator } from "../core/filter";
import { DataType } from "../common/types";
const MDB = require("monetdb")();
export interface MonetConfig {
......@@ -87,13 +88,13 @@ export class MonetAdapter extends SQLAdapter {
this.executeQuery(query, cb);
}
protected typeCast(quotedValue: string, dt: string): string {
protected typeCast(quotedValue: string, dt: DataType): string {
switch (dt) {
case "date":
case DataType.DATE:
return "CAST(" + quotedValue + " AS TIMESTAMP)";
case "integer":
case DataType.INTEGER:
return "CAST(" + quotedValue + " AS INTEGER)";
case "boolean":
case DataType.BOOLEAN:
return "CAST(" + quotedValue + " AS BOOLEAN)";
default:
return quotedValue;
......
......@@ -23,6 +23,7 @@ import { View } from "../core/view";
import { Source } from "../core/source";
import { FilterOperator } from "../core/filter";
import { Pool, PoolConfig } from "pg";
import { DataType } from "../common/types";
export class PostgresAdapter extends SQLAdapter {
private pool: Pool;
......@@ -58,13 +59,13 @@ export class PostgresAdapter extends SQLAdapter {
return false;
}
protected typeCast(quotedValue: string, dt: string): string {
protected typeCast(quotedValue: string, dt: DataType): string {
switch (dt) {
case "date":
case DataType.DATE:
return quotedValue + "::DATE";
case "integer":
case DataType.INTEGER:
return quotedValue + "::INTEGER";
case "boolean":
case DataType.BOOLEAN:
return quotedValue + "::BOOLEAN";
default:
return quotedValue;
......
......@@ -24,7 +24,7 @@ import { Source } from "../core/source";
import { Dimension } from "../core/dimension";
import { Clause } from "../core/clause";
import { Filter, FilterOperator } from "../core/filter";
import { AggregationType, RelationType } from "../common/types";
import { AggregationType, RelationType, DataType } from "../common/types";
import { View } from "../core/view";
interface ExpandedView {
......@@ -793,7 +793,7 @@ export abstract class SQLAdapter extends Adapter {
protected abstract applyOperator(leftSide: string, rightSide: string, op: FilterOperator): string;
protected abstract typeCast(quotedValue: string, dt: string): string;
protected abstract typeCast(quotedValue: string, dt: DataType): string;
private compareKeys(a: Dimension[], b: Dimension[], blackList: string[]): number {
/*
......
......@@ -76,7 +76,7 @@ describe("API collect controller", () => {
.expect((res: any) => {
const message = "Query execution failed: " +
"Could not construct query with the paramters given.";
"Could not construct query with the parameters given.";
const error = "The source named 'thisisjustatest' was not found";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
......@@ -93,7 +93,7 @@ describe("API collect controller", () => {
.expect(500)
.expect((res: any) => {
const message = "Query execution failed: " +
"Could not construct query with the paramters given.";
"Could not construct query with the parameters given.";
const error = "The 'fields:0' wasn't informed on json";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
......@@ -124,7 +124,7 @@ describe("API collect controller", () => {
.expect(500)
.expect((res: any) => {
const message = "Query execution failed: " +
"Could not construct query with the paramters given.";
"Could not construct query with the parameters given.";
const error = "The value '1' from 'fields:0' isn't listed on enumtype:0";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
......@@ -157,7 +157,7 @@ describe("API collect controller", () => {
.expect((res: any) => {
const message = "Query execution failed: " +
"Could not construct query with the paramters given.";
"Could not construct query with the parameters given.";
const error = "The value 'nope' from 'fields:0' isn't a type integer";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
......@@ -176,7 +176,7 @@ describe("API collect controller", () => {
.expect((res: any) => {
const message = "Query execution failed: " +
"Could not construct query with the paramters given.";
"Could not construct query with the parameters given.";
const error = "The value 'notafloat' from 'fields:1' isn't a type float";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
......@@ -195,7 +195,7 @@ describe("API collect controller", () => {
.expect((res: any) => {
const message = "Query execution failed: " +
"Could not construct query with the paramters given.";
"Could not construct query with the parameters given.";
const error = "The value '1' from 'fields:2' isn't a type string";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
......@@ -214,7 +214,7 @@ describe("API collect controller", () => {
.expect((res: any) => {
const message = "Query execution failed: " +
"Could not construct query with the paramters given.";
"Could not construct query with the parameters given.";
const error = "The value 'notaboolean' from 'fields:3' isn't a type boolean";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
......@@ -233,7 +233,7 @@ describe("API collect controller", () => {
.expect((res: any) => {
const message = "Query execution failed: " +
"Could not construct query with the paramters given.";
"Could not construct query with the parameters given.";
const error = "The value '1999-25-25' from 'fields:4' isn't a type date";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
......@@ -258,21 +258,4 @@ describe("API collect controller", () => {
})
.end(done);
});
it("should respond 500 when dataType does not exist", (done) => {
request(server)
.post("/v1/collect/source_2")
.send({"fields:0" : 1 })
.expect(500)
.expect((res: any) => {
const message = "Query execution failed: " +
"Could not construct query with the paramters given.";
const error = "The dataType named 'notValid' was not found";
expect(res.body).to.be.an("object");
expect(res.body).to.have.property("message");
expect(res.body.message).to.be.eql(message);
expect(res.body.error).to.be.eql(error);
})
.end(done);
});
});
......@@ -22,6 +22,7 @@ import * as express from "express";
import { Request } from "../types";
import { Source, Field } from "../../core/source";
import { EnumType } from "../../core/enumType";
import { DataType } from "../../common/types";
interface Valid{
[key: string]: (value: any) => boolean;
......@@ -114,17 +115,16 @@ export class CollectCtrl {
}
for (let i = 0; i < fields.length; i++){
if (validador[fields[i].dataType] !== undefined){
if (!validador[fields[i].dataType](data[i]) === true){
if (fields[i].dataType !== DataType.NONE){
if (!validador[EnumType.stringfyDataType(fields[i].dataType)](data[i]) === true){
throw new Error(
"The value '" + data[i] + "' from '" + fields[i].name +
"' isn't a type " + fields[i].dataType);
"' isn't a type " + [EnumType.stringfyDataType(fields[i].dataType)]);
}
}
else {
enumType = req.engine.getEnumTypeByName(fields[i].dataType);
enumType = req.engine.getEnumTypeByName(fields[i].enumType);
types = enumType.values;
let found: boolean = false;
for (let j = 0; j < types.length; j++){
......@@ -136,7 +136,7 @@ export class CollectCtrl {
if (!found) {
throw new Error(
"The value '" + data[i] + "' from '" + fields[i].name +
"' isn't listed on " + fields[i].dataType);
"' isn't listed on " + fields[i].enumType);
}
}
}
......@@ -145,7 +145,7 @@ export class CollectCtrl {
catch (e) {
res.status(500).json({
message: "Query execution failed: " +
"Could not construct query with the paramters given.",
"Could not construct query with the parameters given.",
error: e.message
});
return;
......
......@@ -33,3 +33,13 @@ export enum RelationType {
MONTH,
YEAR,
};
export enum DataType {
NONE,
INTEGER,
FLOAT,
STRING,
DATE,
BOOLEAN,
ENUMTYPE,
};
......@@ -18,14 +18,16 @@
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { RelationType } from "../common/types";
import { RelationType, DataType } from "../common/types";
import { EnumType } from "./enumType";
export interface DimensionOptions {
name: string;
dataType: string;
dataType: DataType;
parent?: Dimension;
relation?: RelationType;
description?: string;
enumType?: string;
}
export interface DimensionStrOptions {
......@@ -38,10 +40,11 @@ export interface DimensionStrOptions {
export class Dimension {
public readonly name: string;
public readonly dataType: string;
public readonly dataType: DataType;
public readonly parent: Dimension;
public readonly relation: RelationType;
public readonly description: string;
public readonly enumType: string;
constructor(options: DimensionOptions) {
this.name = options.name;
......@@ -49,21 +52,21 @@ export class Dimension {
this.relation = (options.relation) ? options.relation : RelationType.NONE;
this.parent = (options.parent) ? options.parent : null;
this.description = (options.description) ? options.description : "";
this.enumType = (options.enumType) ? (options.enumType) : "";
}
public strOptions(): DimensionStrOptions {
if (this.relation === RelationType.NONE) {
return {
name: this.name,
dataType: this.dataType,
dataType: (this.dataType !== DataType.NONE) ? EnumType.stringfyDataType(this.dataType) : this.enumType ,
description: this.description
};
}
else {
return {
name: this.name,
dataType: this.dataType,
dataType: (this.dataType !== DataType.NONE) ? EnumType.stringfyDataType(this.dataType) : this.enumType ,
parent: this.parent.name,
relation: Dimension.stringifyRelationType(this.relation),
description: this.description
......
......@@ -26,7 +26,7 @@ import { View } from "./view";
import { Query } from "../common/query";
import { Graph } from "../util/graph";
import { EnumType, EnumTypeOptions} from "./enumType";
import { Source , SourceOptions } from "./source";
import { Source , SourceStrOptions } from "./source";
export class Engine {
private views: View[] = [];
......@@ -57,7 +57,7 @@ export class Engine {
return this.enumTypes.map((i) => i.strOptions());
}
public getSourcesDescription(): SourceOptions[] {
public getSourcesDescription(): SourceStrOptions[] {
return this.sources.map((i) => i.strOptions());
}
......
......@@ -18,6 +18,8 @@
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { DataType } from "../common/types";
export interface EnumTypeOptions {
name: string;
values: string[];
......@@ -30,10 +32,45 @@ export class EnumType {
this.name = options.name;
this.values = options.values;
}
public strOptions(): EnumTypeOptions {
return{
name: this.name,
values: this.values
};
}
public static stringfyDataType(a: DataType): string {
switch (a) {
case DataType.INTEGER:
return "integer";
case DataType.FLOAT:
return "float";
case DataType.STRING:
return "string";
case DataType.DATE:
return "date";
case DataType.BOOLEAN:
return "boolean";
default:
return "";
}
}
public static parseDataType (str: string): DataType {
str = str.toLocaleLowerCase();
switch (str) {
case "integer":
return DataType.INTEGER;
case "float":
return DataType.FLOAT;
case "string":
return DataType.STRING;
case "date":
return DataType.DATE;
case "boolean":
return DataType.BOOLEAN;
default:
return DataType.NONE;
}
}
}
......@@ -21,6 +21,7 @@
import { Dimension } from "./dimension";
import { Metric } from "./metric";
import { Hash } from "../util/hash";
import { DataType } from "../common/types";
export interface FilterOptions {
target: Metric|Dimension;
......@@ -146,7 +147,7 @@ export class Filter {
op.operator === FilterOperator.LOWER ||
op.operator === FilterOperator.GREATEREQ ||
op.operator === FilterOperator.LOWEREQ) {
if (op.target.dataType === "date" || op.target.dataType === "integer") {
if (op.target.dataType === DataType.DATE || op.target.dataType === DataType.INTEGER) {
return true;
}
......
......@@ -18,12 +18,13 @@
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
import { AggregationType } from "../common/types";
import { AggregationType, DataType } from "../common/types";
import { EnumType } from "./enumType";
export interface MetricOptions {
name: string;
aggregation: AggregationType;
dataType: string;
dataType: DataType;
description?: string;
}
......@@ -37,7 +38,7 @@ export interface MetricStrOptions {
export class Metric {
public readonly name: string;
public readonly aggregation: AggregationType;
public readonly dataType: string;
public readonly dataType: DataType;
public readonly description: string;
constructor(options: MetricOptions) {
......@@ -51,7 +52,7 @@ export class Metric {
return {
name: this.name,
aggregation: Metric.stringifyAggrType(this.aggregation),
dataType: this.dataType,
dataType: EnumType.stringfyDataType(this.dataType),
description: this.description
};
}
......
......@@ -18,7 +18,17 @@
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
import { EnumType } from "./enumType";
import { DataType } from "../common/types";
export interface Field {
name: string;
description?: string;
dataType: DataType;
enumType?: string;
}
export interface FieldStr {
name: string;
description?: string;
dataType: string;
......@@ -30,6 +40,12 @@ export interface SourceOptions {
fields: Field[];
}
export interface SourceStrOptions {
name: string;
description?: string;
fields: FieldStr[];
}
export class Source {
public readonly name: string;
public readonly description: string;
......@@ -42,16 +58,42 @@ export class Source {
return {
name: item.name,
description: (item.description) ? item.description : "",
dataType: item.dataType
dataType: item.dataType,
enumType: (item.enumType) ? item.enumType : ""
};
});
}
public strOptions(): SourceOptions {
public strOptions(): SourceStrOptions {
return {
name: this.name,
description: this.description,
fields: this.fields
fields: Source.stringfyFieldDataType(this.fields),
};
}
public static stringfyFieldDataType(opts: Field[]): FieldStr[] {
let str: FieldStr[];
str = opts.map((i) => {
return {
name : i.name,
description: i.description,
dataType: (i.dataType !== DataType.NONE) ? EnumType.stringfyDataType(i.dataType) : i.enumType
};
});
return str;
}
public static parseFieldDataType(opts: FieldStr[]): Field[] {
let str: Field[];
str = opts.map((i) => {
return {
name : i.name,
description: i.description,
dataType: EnumType.parseDataType(i.dataType),
enumType: (EnumType.parseDataType(i.dataType) === DataType.NONE) ? i.dataType : ""
};
});
return str;
}
}
......@@ -22,7 +22,10 @@ import { expect } from "chai";
import { ConfigParser, ViewParsingOptions } from "./configParser";
import { Dimension, DimensionStrOptions } from "../core/dimension";
import { RelationType } from "../common/types";
import { RelationType , DataType} from "../common/types";
import { EnumType } from "../core/enumType";
import { MetricStrOptions } from "../core/metric";
import { SourceStrOptions } from "../core/source";
function strToRelationType (str: string): RelationType {
switch (str) {
......@@ -92,7 +95,7 @@ describe("configParser utility library", () => {
let error: boolean = false;
try {
ConfigParser.parseDimOpts(opts, dims);
ConfigParser.parseDimOpts(opts, dims, null);
}
catch (e) {
error = true;
......@@ -114,7 +117,7 @@ describe("configParser utility library", () => {
};
let dimMap: {[key: string]: Dimension} = {
"dim:0" : new Dimension({name: "dim:0", dataType: "integer"})
"dim:0" : new Dimension({name: "dim:0", dataType: DataType.INTEGER})
};
let error: boolean = false;
......@@ -142,7 +145,7 @@ describe("configParser utility library", () => {
};
let dimMap: {[key: string]: Dimension} = {
"dim:0" : new Dimension({name: "dim:0", dataType: "integer"})
"dim:0" : new Dimension({name: "dim:0", dataType: DataType.INTEGER})
};
let error: boolean = false;
......@@ -188,17 +191,113 @@ describe("configParser utility library", () => {
];
let dims: Dimension[] = [
new Dimension({name: "dim:a", dataType: "date"}),
new Dimension({name: "dim:0", dataType: "date"})
new Dimension({name: "dim:a", dataType: DataType.DATE}),
new Dimension({name: "dim:0", dataType: DataType.DATE})
];
for (let i = 0; i < opts.length; ++i) {
let parsed = ConfigParser.parseDimOpts(opts[i], dims);
let parsed = ConfigParser.parseDimOpts(opts[i], dims, null);
expect(parsed.name).to.be.equal(opts[i].name);
expect(parsed.dataType).to.be.equal(opts[i].dataType);
expect(EnumType.stringfyDataType(parsed.dataType)).to.be.equal(opts[i].dataType);
expect(parsed.parent).to.be.equal(dims[1]);
expect(parsed.relation).to.be.equal(strToRelationType(opts[i].relation));
}
});
it("should parse correctly enumType with dimension", () => {
let opts: DimensionStrOptions = {
name: "dim:day",
dataType: "enumtype:5",
parent: "dim:0",
relation: "day"
};
let dims: Dimension[] = [
new Dimension({name: "dim:0", dataType: DataType.DATE})
];
let enumMap: {[key: string]: EnumType} = {
"enumtype:5" : new EnumType({name: "enumtype:5", values: ["nope", "test"]})
};
let parsed = ConfigParser.parseDimOpts(opts, dims, enumMap);
expect(parsed.enumType).to.be.equal(enumMap["enumtype:5"].name);
});
it("should fail to parse enumType with dimension", () => {
let opts: DimensionStrOptions = {
name: "dim:day",
dataType: "enumtype:4",
parent: "dim:0",
relation: "day"
};
let dims: Dimension[] = [
new Dimension({name: "dim:0", dataType: DataType.INTEGER})
];
let enumMap: {[key: string]: EnumType} = {
"enumtype:5" : new EnumType({name: "enumtype:5", values: ["nope", "test"]})
};
let error: boolean = false;
try {
ConfigParser.parseDimOpts(opts, dims, enumMap);
}
catch (e) {
error = true;
expect(e.message).to.be
.equal("[Parsing error] DataType: '" + opts.dataType + "' does not exist on Dimension");
}
expect(error).to.be.true;
});
it("should fail to parse dataType with Metrics", () => {
let met: MetricStrOptions;
met = {
name: "met:0",
aggregation: "sum",
dataType: "string",
description: "just for test"
};
let error: boolean = false;
try {
ConfigParser.parseMetOpts(met);
}
catch (e) {
error = true;
expect(e.message).to.be
.equal("[Parsing error] DataType: '" + met.dataType + "' does not exist on Metric");
}
expect(error).to.be.true;
});
it("should fail to parse enumType with Source", () => {
let sourc: SourceStrOptions;
sourc = {
name: "source_10",
description: "source used for test only",
fields: [
{name: "fields:0",
description: "first entry",
dataType: "string"},
{name: "fields:1",
description: "second entry",
dataType: "enumtype:10"}
]
};
let enumMap: {[key: string]: EnumType} = {
"enumtype:5" : new EnumType({name: "enumtype:5", values: ["nope", "test"]})
};
let error: boolean = false;
try {
ConfigParser.parseSourceOpts(sourc, enumMap);
}
catch (e) {
error = true;
expect(e.message).to.be
.equal("[Parsing error] DataType: '" + sourc.fields[1].dataType + "' does not exist on Source");
}
expect(error).to.be.true;
});
});
......@@ -22,10 +22,10 @@ import { Metric, MetricOptions, MetricStrOptions } from "../core/metric";
import { Dimension, DimensionOptions, DimensionStrOptions } from "../core/dimension";
import { View, ViewOptions, LoadView } from "../core/view";
import { EnumType, EnumTypeOptions } from "../core/enumType";
import { RelationType } from "../common/types";
import { RelationType, DataType } from "../common/types";
import { Filter } from "../core/filter";
import { Clause } from "../core/clause";
import { Source, SourceOptions} from "../core/source";
import { Source, SourceOptions, SourceStrOptions} from "../core/source";
import { Tsort, TsortDep } from "./tsort";
import * as fs from "fs";
import * as yaml from "js-yaml";
......@@ -42,7 +42,7 @@ export interface ViewParsingOptions {
}
interface ConfigSchema {
sources: SourceOptions[];
sources: SourceStrOptions[];
views: ViewParsingOptions[];
metrics: MetricStrOptions[];
dimensions: DimensionStrOptions[];
......@@ -147,16 +147,16 @@ export class ConfigParser {
let sourcMap: SourceMap = {};
let dimOptsMap: DimensionOptsMap = {};
for (let i = 0; i < metricsOpts.length; ++i) {
let met = new Metric(this.parseMetOpts(metricsOpts[i]));
parsed.metrics.push(met);
metMap[met.name] = met;
}
for (let i = 0; i < enumTypesOpts.length; i++) {
let enumT = new EnumType((enumTypesOpts[i]));
parsed.enumTypes.push(enumT);
enumMap[enumT.name] = enumT;
}
for (let i = 0; i < metricsOpts.length; ++i) {
let met = new Metric(this.parseMetOpts(metricsOpts[i]));
parsed.metrics.push(met);
metMap[met.name] = met;
}
let toSort: TsortDep[] = [];
for (let i = 0; i < dimensionsOpts.length; ++i) {
......@@ -183,13 +183,13 @@ export class ConfigParser {
});
for (let i = 0; i < dimensionsOpts.length; ++i) {
let dim = new Dimension(this.parseDimOpts(dimensionsOpts[i], parsed.dimensions));
let dim = new Dimension(this.parseDimOpts(dimensionsOpts[i], parsed.dimensions, enumMap));
parsed.dimensions.push(dim);
dimMap[dim.name] = dim;
}
for (let i = 0; i < sourcesOpts.length; i++) {
let sourc = new Source((sourcesOpts[i]));
let sourc = new Source(this.parseSourceOpts(sourcesOpts[i], enumMap));
parsed.sources.push(sourc);
sourcMap[sourc.name] = sourc;