Issue #107: Make metrics and dimensions routes filterable by tags

Signed-off-by: Lucas Fernandes de Oliveira's avatarLucas Fernandes de Oliveira <lfoliveira@inf.ufpr.br>
parent 2a499400
Pipeline #21137 passed with stages
in 1 minute and 1 second
# gitignore ignores files.yaml in this folder
# however a config file for tests in CI is required
# so this example file in fact is the CI test file
tags:
links:
- config/market_tags.yaml.example
obj:
-
name: "noDescription"
description: "Related with seller"
views:
links:
- config/market_views.yaml.example
......@@ -30,6 +37,9 @@ metrics:
dataType: "float"
aggregation: "avg"
description: "The seller average age"
tags:
- "seller"
- "age"
dimensions:
links:
- config/market_dimensions.yaml.example
......@@ -38,6 +48,8 @@ dimensions:
name: "dim:seller:name"
dataType: "string"
description: "Name of the seller from market"
tags:
- "seller"
enumTypes:
links:
- config/market_enum.yaml.example
......
# gitignore ignores files.yaml in this folder
# however a config file for tests in CI is required
# so this example file in fact is the CI test file
tags:
links: []
obj: []
views:
links:
- config/market_views.yaml.example
......
-
name: "dim:seller:name"
dataType: "string"
description: "The name of the seller from market"
-
name: "dim:seller:sex"
dataType: "enumtype"
......
-
name: "met:seller:avg:age"
dataType: "float"
aggregation: "avg"
description: "The seller average age"
-
name: "met:seller:max:age"
dataType: "integer"
aggregation: "max"
description: "The seller highest age"
tags:
- "seller"
- "age"
- "max"
-
name: "met:seller:min:age"
dataType: "integer"
aggregation: "min"
description: "The seller lowest age"
tags:
- "seller"
- "age"
-
name: "met:seller:count:age"
dataType: "integer"
aggregation: "count"
description: "The number of seller's"
tags:
- "seller"
- "age"
-
name: "met:product:avg:pricein"
dataType: "float"
aggregation: "avg"
description: "The average product pricein"
tags:
- "product"
-
name: "met:product:max:pricein"
dataType: "float"
aggregation: "max"
description: "The highest product pricein"
tags:
- "product"
- "max"
-
name: "met:product:min:pricein"
dataType: "float"
aggregation: "min"
description: "The lowest product pricein"
tags:
- "product"
-
name: "met:product:avg:priceout"
dataType: "float"
aggregation: "avg"
description: "The average product priceout"
tags:
- "product"
-
name: "met:product:max:priceout"
dataType: "float"
aggregation: "max"
description: "The highest product priceout"
tags:
- "product"
- "max"
-
name: "met:product:min:priceout"
dataType: "float"
aggregation: "min"
description: "The lowest product priceout"
tags:
- "product"
-
name: "met:sell:sum:quantity"
dataType: "integer"
......@@ -68,13 +87,20 @@
dataType: "float"
aggregation: "avg"
description: "The average of quantity bought"
tags:
- "buyout"
-
name: "met:buyout:max:quantity"
dataType: "integer"
aggregation: "max"
description: "The highest quantity bought"
tags:
- "buyout"
- "max"
-
name: "met:buyout:min:quantity"
dataType: "integer"
aggregation: "min"
description: "The lowest quantity bought"
tags:
- "buyout"
-
name: "seller"
description: "Related with seller"
-
name: "age"
description: "Related with age"
-
name: "product"
description: "Related with product"
-
name: "client"
description: "Related with client"
-
name: "buyout"
description: "Related with buyout"
-
name: "provider"
description: "Related with provider"
-
name: "max"
description: "Aggregation Max"
......@@ -283,6 +283,15 @@ traits:
required: false
pattern: "^json$|^csv$|^ssv$|^tsv$"
type: string
- taggable:
queryParameters:
tags:
description: |
Tags that restrict the elements returned for your request.
Similar to a filter, but used in Blendb elements, not in
query results.
required: false
type: string
/metrics:
description: |
......@@ -291,13 +300,13 @@ traits:
system and their descriptions.
securedBy: [ null, oauth_2_0 ]
get:
is: [ formatable ]
is: [ formatable, taggable ]
/sources:
description: |
A Source represents a type of object that can be inserted in the database.
This collection allows the user to list all the sources available in the
system and their descriptions
securedBy: [ null, oauth_2_0 ]
securedBy: [ null, oauth_2_0 ]
get:
is: [ formatable ]
......@@ -308,15 +317,23 @@ traits:
the system and their descriptions.
securedBy: [ null, oauth_2_0 ]
get:
is: [ formatable ]
is: [ formatable, taggable ]
/enumtypes:
description: |
A EnumType is short for enumerable type. This is a special data type that only accepts a few possible values. This
collection allows the user to list all the enumerable types available in the system, their descriptions and possible
values.
get:
get:
is: [ formatable ]
securedBy: [ null, oauth_2_0 ]
/tags:
description: |
A Tag can be placed in a metric or dimension to add some extra meaning
to it. Tags can be used to filter the amount of elements returned by a
route. Tags are like filters, but instead of filtering query results,
filter blendb elements.
get:
is: [ formatable ]
securedBy: [ null, oauth_2_0 ]
/data:
description: |
This is the main part of the API. You may query it for report
......
......@@ -70,7 +70,18 @@ describe("API engine controller", () => {
.end(done);
});
it("should respond 200 and the list of metrics (in csv)", (done) => {
it("should respond 200 and the list of tags", (done) => {
request(server)
.get("/v1/tags")
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
expect(result).to.have.length(8);
})
.end(done);
});
it("should respond 200 and the list of metrics (in ssv)", (done) => {
waterfall([(cb: (err: Error, data: string) => void) => {
fs.readFile("test/files/metrics.csv", "utf8", (err, data) => {
cb(err, data);
......@@ -91,4 +102,28 @@ describe("API engine controller", () => {
});
});
it("should respond 200 and the filtered list of metrics", (done) => {
request(server)
.get("/v1/metrics")
.query({tags: "seller,buyout"})
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
expect(result).to.have.length(7);
})
.end(done);
});
it("should respond 200 and the filtered list of dimensions", (done) => {
request(server)
.get("/v1/dimensions")
.query({tags: "seller"})
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
expect(result).to.have.length(1);
})
.end(done);
});
});
......@@ -63,6 +63,7 @@ export class EngineCtrl {
});
}
}
/**
* Route that returns the list of available metrics.
* @param req - Object with request information
......@@ -71,7 +72,7 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static metrics(req: Request, res: express.Response, next: express.NextFunction) {
const metrics = req.engine.getMetricsDescription();
const metrics = req.engine.getMetricsDescription(req.query.tags);
EngineCtrl.respondList(metrics, "metrics", req, res, next);
}
......@@ -83,7 +84,7 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static dimensions(req: Request, res: express.Response, next: express.NextFunction) {
const dimensions = req.engine.getDimensionsDescription();
const dimensions = req.engine.getDimensionsDescription(req.query.tags);
EngineCtrl.respondList(dimensions, "dimensions", req, res, next);
}
......@@ -110,4 +111,16 @@ export class EngineCtrl {
const sources = req.engine.getSourcesDescription();
EngineCtrl.respondList(sources, "sources", req, res, next);
}
/**
* Route that returns the list of available tags.
* @param req - Object with request information
* @param res - Object used to create and send the response
* @param next - Call next middleware or controller. Not used but required
* by typescript definition of route.
*/
public static tags(req: Request, res: express.Response, next: express.NextFunction) {
const tags = req.engine.getTagsDescription();
EngineCtrl.respondList(tags, "tags", req, res, next);
}
}
......@@ -33,5 +33,6 @@ router.get("/metrics", EngineCtrl.metrics);
router.get("/sources", EngineCtrl.sources);
router.get("/dimensions", EngineCtrl.dimensions);
router.get("/enumtypes", EngineCtrl.enumTypes);
router.get("/tags", EngineCtrl.tags);
router.get("/data", DataCtrl.read);
router.post("/collect/{class}", CollectCtrl.write);
/*
* Copyright (C) 2019 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Parameters used to create a Tag object.
* Parameters used to define tag object in the configuration file.
* Also the string description of a tag.
*/
export interface TagOptions {
/** Tag name. */
name: string;
/** Breif description of what this tag represents. */
description?: string;
}
/**
* A Tag can be attached to other elements (such as metrics and dimensions)
* to create groups of similar elements. A tag can also be used to filter
* these elements, search for itens that only contain the given tags.
*/
export class Tag {
/** Tag name. */
public readonly name: string;
/** Breif description of what this tag represents. */
public readonly description: string;
/**
* Creates a tag.
* @param options - Parameters required to create a tag.
*/
constructor(options: TagOptions) {
this.name = options.name;
this.description = (options.description) ? options.description : "";
}
/**
* Creates a object with the same options used to create this
* tag as strings. Used to inform the API users.
*/
public strOptions(): TagOptions {
return {
name: this.name
, description: this.description
};
}
}
......@@ -20,6 +20,7 @@
import { RelationType, DataType } from "../common/types";
import { EnumHandler } from "../util/enumHandler";
import { Tag } from "../common/tag";
/** Parameters used to create a Dimension object. */
export interface DimensionOptions {
......@@ -33,8 +34,10 @@ export interface DimensionOptions {
relation?: RelationType;
/** Breif description of what this dimension represents. */
description?: string;
/* Enumerable type name, used if data type is enumerable type. */
/** Enumerable type name, used if data type is enumerable type. */
enumType?: string;
/** List of tags, a complement to attribute description. */
tags?: Tag[];
}
/**
......@@ -54,6 +57,8 @@ export interface DimensionStrOptions {
description?: string;
/** Dimension enum type */
enumType?: string;
/** List of tag names, a complement to attribute description. */
tags?: string[];
}
/**
......@@ -74,8 +79,10 @@ export class Dimension {
public readonly relation: RelationType;
/** Breif description of what this dimension represents. */
public readonly description: string;
/* Enumerable type name, used if data type is enumerable type. */
/** Enumerable type name, used if data type is enumerable type. */
public readonly enumType: string;
/** List of tags, a complement to attribute description. */
public readonly tags: Tag[];
/**
* Creates a dimension.
......@@ -88,6 +95,7 @@ export class Dimension {
this.parent = (options.parent) ? options.parent : null;
this.description = (options.description) ? options.description : "";
this.enumType = (options.enumType) ? options.enumType : "";
this.tags = (options.tags) ? options.tags.map((i) => i) : [];
}
/**
......@@ -98,16 +106,19 @@ export class Dimension {
let o: DimensionStrOptions = {
name: this.name,
dataType: EnumHandler.stringfyDataType(this.dataType),
description: this.description
description: this.description,
tags: this.tags.map ((i) => i.name)
};
if (this.relation !== RelationType.NONE){
if (this.relation !== RelationType.NONE) {
o.relation = EnumHandler.stringifyRelationType(this.relation);
o.parent = this.parent.name;
}
if (this.dataType === DataType.ENUMTYPE){
if (this.dataType === DataType.ENUMTYPE) {
o.enumType = this.enumType;
}
return o;
}
}
......@@ -20,6 +20,7 @@
import { Dimension, DimensionStrOptions } from "./dimension";
import { Metric, MetricStrOptions } from "./metric";
import { Tag, TagOptions } from "../common/tag";
import { Clause } from "./clause";
import { Filter } from "./filter";
import { View } from "./view";
......@@ -49,6 +50,8 @@ export class Engine {
private dimensions: Dimension[];
/** Set of sources available in the database. */
private sources: Source[];
/** Set of tags available in the database. */
private tags: Tag[];
/** Graph which represents the database schema. */
private graph: Graph;
......@@ -62,12 +65,14 @@ export class Engine {
this.metrics = [];
this.dimensions = [];
this.sources = [];
this.tags = [];
config.metrics.forEach ((met) => this.addMetric(met));
config.dimensions.forEach ((dim) => this.addDimension(dim));
config.views.forEach ((view) => this.addView(view));
config.enumTypes.forEach ((enumt) => this.addEnumType(enumt));
config.sources.forEach ((sourc) => this.addSource(sourc));
config.tags.forEach ((tag) => this.addTag(tag));
}
......@@ -76,9 +81,26 @@ export class Engine {
return this.views;
}
/** Gets a string description for all the available metrics. */
public getMetricsDescription(): MetricStrOptions[] {
return this.metrics.map((i) => i.strOptions());
/**
* Gets a string description for all the available metrics.
* @param expression - Expression to be used in the tags
*/
public getMetricsDescription(expression: string): MetricStrOptions[] {
let list = this.metrics.map((i) => i.strOptions());
if (!expression) {
return list;
}
const clauses = expression.split(";").filter((item: string) => item !== "");
for (let i = 0; i < clauses.length; ++i) {
const tags = clauses[i].split(",").filter((item: string) => item !== "");
list = list.filter((item) => {
return item.tags.some((o) => tags.some((t) => t === o));
});
}
return list;
}
/** Gets a string description for all the available enumerable types. */
......@@ -91,9 +113,31 @@ export class Engine {
return this.sources.map((i) => i.strOptions());
}
/** Gets a string description for all the available dimensions. */
public getDimensionsDescription(): DimensionStrOptions[] {
return this.dimensions.map((i) => i.strOptions());
/**
* Gets a string description for all the available dimensions.
* @param expression - Expression to be used in the tags
*/
public getDimensionsDescription(expression: string): DimensionStrOptions[] {
let list = this.dimensions.map((i) => i.strOptions());
if (!expression) {
return list;
}
const clauses = expression.split(";").filter((item: string) => item !== "");
for (let i = 0; i < clauses.length; ++i) {
const tags = clauses[i].split(",").filter((item: string) => item !== "");
list = list.filter((item) => {
return item.tags.some((o) => tags.some((t) => t === o));
});
}
return list;
}
/** Gets a string description for all the available tags. */
public getTagsDescription(): TagOptions[] {
return this.tags.map((i) => i.strOptions());
}
/**
......@@ -118,6 +162,15 @@ export class Engine {
return enumType;
}
/**
* Adds a new tag to the database schema (engine).
* @param enumType - Enumerable type to be added.
*/
public addTag(tag: Tag): Tag {
this.tags.push(tag);
return tag;
}
/**
* Adds a new source to the database schema (engine).
* @param source - Source to be added.
......@@ -340,4 +393,5 @@ export class Engine {
return noRepeat;
}
}
......@@ -20,6 +20,7 @@
import { AggregationType, DataType } from "../common/types";
import { EnumHandler } from "../util/enumHandler";
import { Tag } from "../common/tag";
/** Parameters used to create a metric object. */
export interface MetricOptions {
......@@ -31,6 +32,8 @@ export interface MetricOptions {
dataType: DataType;
/** Breif description of what this metric represents. */
description?: string;
/** List of tags, a complement to attribute description. */
tags?: Tag[];
}
/**
......@@ -46,6 +49,8 @@ export interface MetricStrOptions {
dataType: string;
/** Breif description of what this metric represents. */
description?: string;
/** List of tag names, a complement to attribute description. */
tags?: string[];
}
/**
......@@ -64,6 +69,8 @@ export class Metric {
public readonly dataType: DataType;
/** Breif description of what this metric represents. */
public readonly description: string;
/** List of tags, a complement to attribute description. */
public readonly tags: Tag[];
/**
* Create a metric.
......@@ -74,6 +81,7 @@ export class Metric {
this.aggregation = options.aggregation;
this.dataType = options.dataType;
this.description = (options.description) ? options.description : "";
this.tags = (options.tags) ? options.tags.map((i) => i) : [];
}
/**
......@@ -85,7 +93,8 @@ export class Metric {
name: this.name,
aggregation: EnumHandler.stringifyAggrType(this.aggregation),
dataType: EnumHandler.stringfyDataType(this.dataType),
description: this.description
description: this.description,
tags: this.tags.map ((i) => i.name)
};
}
}
......@@ -95,7 +95,7 @@ describe("configParser utility library", () => {
let error: boolean = false;
try {
ConfigParser.parseDimOpts(opts, dims, null);
ConfigParser.parseDimOpts(opts, dims, null, null);
}
catch (e) {
error = true;
......@@ -196,7 +196,7 @@ describe("configParser utility library", () => {
];
for (let i = 0; i < opts.length; ++i) {
let parsed = ConfigParser.parseDimOpts(opts[i], dims, null);
let parsed = ConfigParser.parseDimOpts(opts[i], dims, null, null);
expect(parsed.name).to.be.equal(opts[i].name);
expect(EnumHandler.stringfyDataType(parsed.dataType)).to.be.equal(opts[i].dataType);
expect(parsed.parent).to.be.equal(dims[1]);
......@@ -219,7 +219,7 @@ describe("configParser utility library", () => {
let enumMap: {[key: string]: EnumType} = {
"enumtype:5" : new EnumType({name: "enumtype:5", values: ["nope", "test"]})
};
let parsed = ConfigParser.parseDimOpts(opts, dims, enumMap);
let parsed = ConfigParser.parseDimOpts(opts, dims, enumMap, null);
expect(parsed.enumType).to.be.equal(enumMap["enumtype:5"].name);
});
......@@ -240,7 +240,7 @@ describe("configParser utility library", () => {
};
let error: boolean = false;
try {
ConfigParser.parseDimOpts(opts, dims, enumMap);
ConfigParser.parseDimOpts(opts, dims, enumMap, null);
}
catch (e) {
error = true;
......@@ -261,7 +261,7 @@ describe("configParser utility library", () => {
};
let error: boolean = false;
try {
ConfigParser.parseMetOpts(met);
ConfigParser.parseMetOpts(met, null);
}
catch (e) {
error = true;
......@@ -301,4 +301,47 @@ describe("configParser utility library", () => {
expect(error).to.be.true;
});
it("should throw expection for inexistent tag in a Dimension", () => {
let opts: DimensionStrOptions = {
name: "dim:0",
dataType: "integer",
tags: ["none"]
};
let dims: Dimension[] = [];
let error: boolean = false;
try {
ConfigParser.parseDimOpts(opts, dims, null, {});
}
catch (e) {
error = true;
expect(e.message).to.be
.equal("[Parsing error] Tag: 'none' used in dimension: '" + opts.name + "' was not defined. Check tag spelling and configuration files.");
}
expect(error).to.be.true;
});
it("should throw expection for inexistent tag in a Metric", () => {
let opts: MetricStrOptions = {
name: "met:0",
dataType: "integer",
aggregation: "avg",
tags: ["none"]
};
let error: boolean = false;
try {
ConfigParser.parseMetOpts(opts, {});
}
catch (e) {
error = true;
expect(e.message).to.be
.equal("[Parsing error] Tag: 'none' used in metric: '" + opts.name + "' was not defined. Check tag spelling and configuration files.");
}
expect(error).to.be.true;
});
});
......@@ -24,6 +24,7 @@ import { View, ViewOptions, LoadView } from "../core/view";
import { EnumType, EnumTypeOptions } from "../core/enumType";
import { RelationType, DataType } from "../common/types";
import { Opcode } from "../common/expression";
import { Tag, TagOptions } from "../common/tag";
import { Filter } from "../core/filter";
import { Clause } from "../core/clause";
import { Source, SourceOptions, SourceStrOptions} from "../core/source";
......@@ -50,7 +51,7 @@ export interface ViewParsingOptions {
metrics: string[];
/** Set of (stringified) clauses applied to the view. */
clauses?: string[];
/** Inform if the view's name will be it's alias or id */
/** Inform if the view's name will be it's alias or id. */
aliasAsName?: boolean;
}
......@@ -78,6 +79,10 @@ interface ConfigSchema {
enumTypes: { obj: EnumTypeOptions[],
links: string[],
};
/** Options of all tags types available */
tags: { obj: TagOptions[],
links: string[],