Issue #107: Make metrics and dimensions routes filterable by tags

Signed-off-by: Lucas Fernandes de Oliveira <lfoliveira@inf.ufpr.br>

config/ci_test.yaml.example

 # gitignore ignores files.yaml in this folder
 # however a config file for tests in CI is required
 # so this example file in fact is the CI test file
+tags:
+    links:
+        - config/market_tags.yaml.example
+    obj:
+        -
+            name: "noDescription"
+            description: "Related with seller"
 views:
     links:
         - config/market_views.yaml.example
@@ -30,6 +37,9 @@ metrics:
             dataType: "float"
             aggregation: "avg"
             description: "The seller average age"
+            tags:
+                - "seller"
+                - "age"
 dimensions:
     links:
         - config/market_dimensions.yaml.example
@@ -38,6 +48,8 @@ dimensions:
             name: "dim:seller:name"
             dataType: "string"
             description: "Name of the seller from market"
+            tags:
+                - "seller"
 enumTypes:
     links:
         - config/market_enum.yaml.example

config/config.yaml.example

 # gitignore ignores files.yaml in this folder
 # however a config file for tests in CI is required
 # so this example file in fact is the CI test file
+tags:
+    links: []
+    obj: []
 views:
     links:
         - config/market_views.yaml.example

config/market_dimensions.yaml.example

--
-    name: "dim:seller:name"
-    dataType: "string"
-    description: "The name of the seller from market"
 -
     name: "dim:seller:sex"
     dataType: "enumtype"

config/market_metrics.yaml.example

--
-    name: "met:seller:avg:age"
-    dataType: "float"
-    aggregation: "avg"
-    description: "The seller average age"
 -     
     name: "met:seller:max:age"                        
     dataType: "integer"
     aggregation: "max"
     description: "The seller highest age"
+    tags:
+        - "seller"
+        - "age"
+        - "max"
 - 
     name: "met:seller:min:age"
     dataType: "integer"
     aggregation: "min"
     description: "The seller lowest age"
+    tags:
+        - "seller"
+        - "age"
 - 
     name: "met:seller:count:age"
     dataType: "integer"
     aggregation: "count"
     description: "The number of seller's"
+    tags:
+        - "seller"
+        - "age"
 - 
     name: "met:product:avg:pricein"
     dataType: "float"
     aggregation: "avg"
     description: "The average product pricein"
+    tags:
+        - "product"
 - 
     name: "met:product:max:pricein"
     dataType: "float"
     aggregation: "max"
     description: "The highest product pricein"
+    tags:
+        - "product"
+        - "max"
 - 
     name: "met:product:min:pricein"
     dataType: "float"
     aggregation: "min"
     description: "The lowest product pricein"
+    tags:
+        - "product"
 - 
     name: "met:product:avg:priceout"
     dataType: "float"
     aggregation: "avg"
     description: "The average product priceout"
+    tags:
+        - "product"
 - 
     name: "met:product:max:priceout"
     dataType: "float"
     aggregation: "max"
     description: "The highest product priceout"
+    tags:
+        - "product"
+        - "max"
 - 
     name: "met:product:min:priceout"
     dataType: "float"
     aggregation: "min"
     description: "The lowest product priceout"
+    tags:
+        - "product"
 - 
     name: "met:sell:sum:quantity"
     dataType: "integer"
@@ -68,13 +87,20 @@
     dataType: "float"
     aggregation: "avg"
     description: "The average of quantity bought"
+    tags:
+        - "buyout"
 - 
     name: "met:buyout:max:quantity"
     dataType: "integer"
     aggregation: "max"
     description: "The highest quantity bought"
+    tags:
+        - "buyout"
+        - "max"
 - 
     name: "met:buyout:min:quantity"
     dataType: "integer"
     aggregation: "min"
     description: "The lowest quantity bought"
+    tags:
+        - "buyout"

config/market_tags.yaml.example

+-
+    name: "seller"
+    description: "Related with seller"
+-
+    name: "age"
+    description: "Related with age"
+-
+    name: "product"
+    description: "Related with product"
+-
+    name: "client"
+    description: "Related with client"
+-
+    name: "buyout"
+    description: "Related with buyout"
+-
+    name: "provider"
+    description: "Related with provider"
+-
+    name: "max"
+    description: "Aggregation Max"

specs/blendb-api-v1.raml

@@ -283,6 +283,15 @@ traits:
                 required: false
                 pattern: "^json$|^csv$|^ssv$|^tsv$"
                 type: string
+    - taggable:
+        queryParameters:
+            tags:
+                description: |
+                    Tags that restrict the elements returned for your request.
+                    Similar to a filter, but used in Blendb elements, not in
+                    query results.
+                required: false
+                type: string
 
 /metrics:
     description: |
@@ -291,13 +300,13 @@ traits:
         system and their descriptions.
     securedBy: [ null, oauth_2_0 ]
     get:
-        is: [ formatable ]
+        is: [ formatable, taggable ]
 /sources:
     description: |
         A Source represents a type of object that can be inserted in the database.
         This collection allows the user to list all the sources available in the
         system and their descriptions
-    securedBy: [ null, oauth_2_0 ]    
+    securedBy: [ null, oauth_2_0 ]
     get:
         is: [ formatable ]
 
@@ -308,15 +317,23 @@ traits:
         the system and their descriptions.
     securedBy: [ null, oauth_2_0 ]
     get:
-        is: [ formatable ]
+        is: [ formatable, taggable ]
 /enumtypes:
     description: |
         A EnumType is short for enumerable type. This is a special data type that only accepts a few possible values. This
         collection allows the user to list all the enumerable types available in the system, their descriptions and possible
         values.
-    get:    
+    get:
+        is: [ formatable ]
+    securedBy: [ null, oauth_2_0 ]
+/tags:
+    description: |
+        A Tag can be placed in a metric or dimension to add some extra meaning
+        to it. Tags can be used to filter the amount of elements returned by a
+        route. Tags are like filters, but instead of filtering query results,
+        filter blendb elements.
+    get:
         is: [ formatable ]
-    securedBy: [ null, oauth_2_0 ]    
 /data:
     description: |
       This is the main part of the API. You may query it for report

src/api/controllers/engine.spec.ts

@@ -70,7 +70,18 @@ describe("API engine controller", () => {
             .end(done);
     });
 
-    it("should respond 200 and the list of metrics (in csv)", (done) => {
+    it("should respond 200 and the list of tags", (done) => {
+        request(server)
+            .get("/v1/tags")
+            .expect((res: any) => {
+                let result = res.body;
+                expect(result).to.be.an("array");
+                expect(result).to.have.length(8);
+            })
+            .end(done);
+    });
+
+    it("should respond 200 and the list of metrics (in ssv)", (done) => {
         waterfall([(cb: (err: Error, data: string) => void) => {
             fs.readFile("test/files/metrics.csv", "utf8", (err, data) => {
                 cb(err, data);
@@ -91,4 +102,28 @@ describe("API engine controller", () => {
         });
     });
 
+    it("should respond 200 and the filtered list of metrics", (done) => {
+        request(server)
+            .get("/v1/metrics")
+            .query({tags: "seller,buyout"})
+            .expect((res: any) => {
+                let result = res.body;
+                expect(result).to.be.an("array");
+                expect(result).to.have.length(7);
+            })
+            .end(done);
+    });
+
+    it("should respond 200 and the filtered list of dimensions", (done) => {
+        request(server)
+            .get("/v1/dimensions")
+            .query({tags: "seller"})
+            .expect((res: any) => {
+                let result = res.body;
+                expect(result).to.be.an("array");
+                expect(result).to.have.length(1);
+            })
+            .end(done);
+    });
+
 });

src/api/controllers/engine.ts

@@ -63,6 +63,7 @@ export class EngineCtrl {
             });
         }
     }
+
     /**
      * Route that returns the list of available metrics.
      * @param req - Object with request information
@@ -71,7 +72,7 @@ export class EngineCtrl {
      * by typescript definition of route.
      */
     public static metrics(req: Request, res: express.Response, next: express.NextFunction) {
-        const metrics = req.engine.getMetricsDescription();
+        const metrics = req.engine.getMetricsDescription(req.query.tags);
         EngineCtrl.respondList(metrics, "metrics", req, res, next);
     }
 
@@ -83,7 +84,7 @@ export class EngineCtrl {
      * by typescript definition of route.
      */
     public static dimensions(req: Request, res: express.Response, next: express.NextFunction) {
-        const dimensions = req.engine.getDimensionsDescription();
+        const dimensions = req.engine.getDimensionsDescription(req.query.tags);
         EngineCtrl.respondList(dimensions, "dimensions", req, res, next);
     }
 
@@ -110,4 +111,16 @@ export class EngineCtrl {
        const sources = req.engine.getSourcesDescription();
        EngineCtrl.respondList(sources, "sources", req, res, next);
     }
+
+    /**
+     * Route that returns the list of available tags.
+     * @param req - Object with request information
+     * @param res - Object used to create and send the response
+     * @param next - Call next middleware or controller. Not used but required
+     * by typescript definition of route.
+     */
+    public static tags(req: Request, res: express.Response, next: express.NextFunction) {
+       const tags = req.engine.getTagsDescription();
+       EngineCtrl.respondList(tags, "tags", req, res, next);
+    }
 }

src/api/router-v1.ts

@@ -33,5 +33,6 @@ router.get("/metrics", EngineCtrl.metrics);
 router.get("/sources", EngineCtrl.sources);
 router.get("/dimensions", EngineCtrl.dimensions);
 router.get("/enumtypes", EngineCtrl.enumTypes);
+router.get("/tags", EngineCtrl.tags);
 router.get("/data", DataCtrl.read);
 router.post("/collect/{class}", CollectCtrl.write);

src/common/tag.ts

+/*
+ * Copyright (C) 2019 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blend is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Parameters used to create a Tag object.
+ * Parameters used to define tag object in the configuration file.
+ * Also the string description of a tag.
+ */
+export interface TagOptions {
+    /**  Tag name. */
+    name: string;
+    /** Breif description of what this tag represents. */
+    description?: string;
+}
+
+/**
+ * A Tag can be attached to other elements (such as metrics and dimensions)
+ * to create groups of similar elements. A tag can also be used to filter
+ * these elements, search for itens that only contain the given tags.
+ */
+export class Tag {
+    /**  Tag name. */
+    public readonly name: string;
+    /** Breif description of what this tag represents. */
+    public readonly description: string;
+
+    /**
+     * Creates a tag.
+     * @param options - Parameters required to create a tag.
+     */
+    constructor(options: TagOptions) {
+        this.name = options.name;
+        this.description = (options.description) ? options.description : "";
+    }
+
+    /**
+     * Creates a object with the same options used to create this
+     * tag as strings. Used to inform the API users.
+     */
+    public strOptions(): TagOptions {
+        return {
+            name: this.name
+            , description: this.description
+        };
+    }
+}

src/core/dimension.ts

@@ -20,6 +20,7 @@
 
 import { RelationType, DataType } from "../common/types";
 import { EnumHandler } from "../util/enumHandler";
+import { Tag } from "../common/tag";
 
 /** Parameters used to create a Dimension object. */
 export interface DimensionOptions {
@@ -33,8 +34,10 @@ export interface DimensionOptions {
     relation?: RelationType;
     /** Breif description of what this dimension represents. */
     description?: string;
-    /* Enumerable type name, used if data type is enumerable type. */
+    /** Enumerable type name, used if data type is enumerable type. */
     enumType?: string;
+    /** List of tags, a complement to attribute description. */
+    tags?: Tag[];
 }
 
 /**
@@ -54,6 +57,8 @@ export interface DimensionStrOptions {
     description?: string;
     /** Dimension enum type */
     enumType?: string;
+    /** List of tag names, a complement to attribute description. */
+    tags?: string[];
 }
 
 /**
@@ -74,8 +79,10 @@ export class Dimension {
     public readonly relation: RelationType;
     /** Breif description of what this dimension represents. */
     public readonly description: string;
-    /* Enumerable type name, used if data type is enumerable type. */
+    /** Enumerable type name, used if data type is enumerable type. */
     public readonly enumType: string;
+    /** List of tags, a complement to attribute description. */
+    public readonly tags: Tag[];
 
     /**
      * Creates a dimension.
@@ -88,6 +95,7 @@ export class Dimension {
         this.parent = (options.parent) ? options.parent : null;
         this.description = (options.description) ? options.description : "";
         this.enumType = (options.enumType) ? options.enumType : "";
+        this.tags = (options.tags) ? options.tags.map((i) => i) : [];
     }
 
     /**
@@ -98,16 +106,19 @@ export class Dimension {
         let o: DimensionStrOptions = {
             name: this.name,
             dataType: EnumHandler.stringfyDataType(this.dataType),
-            description: this.description
-
+            description: this.description,
+            tags: this.tags.map ((i) => i.name)
         };
-        if (this.relation !== RelationType.NONE){
+
+        if (this.relation !== RelationType.NONE) {
             o.relation =  EnumHandler.stringifyRelationType(this.relation);
             o.parent = this.parent.name;
         }
-        if (this.dataType === DataType.ENUMTYPE){
+
+        if (this.dataType === DataType.ENUMTYPE) {
             o.enumType = this.enumType;
         }
+
         return o;
     }
 }

src/core/engine.ts

@@ -20,6 +20,7 @@
 
 import { Dimension, DimensionStrOptions } from "./dimension";
 import { Metric, MetricStrOptions } from "./metric";
+import { Tag, TagOptions } from "../common/tag";
 import { Clause } from "./clause";
 import { Filter } from "./filter";
 import { View } from "./view";
@@ -49,6 +50,8 @@ export class Engine {
     private dimensions: Dimension[];
     /** Set of sources available in the database. */
     private sources: Source[];
+    /** Set of tags available in the database. */
+    private tags: Tag[];
     /** Graph which represents the database schema. */
     private graph: Graph;
 
@@ -62,12 +65,14 @@ export class Engine {
         this.metrics = [];
         this.dimensions = [];
         this.sources = [];
+        this.tags = [];
 
         config.metrics.forEach ((met) => this.addMetric(met));
         config.dimensions.forEach ((dim) => this.addDimension(dim));
         config.views.forEach ((view) => this.addView(view));
         config.enumTypes.forEach ((enumt) => this.addEnumType(enumt));
         config.sources.forEach ((sourc) => this.addSource(sourc));
+        config.tags.forEach ((tag) => this.addTag(tag));
 
     }
 
@@ -76,9 +81,26 @@ export class Engine {
         return this.views;
     }
 
-    /** Gets a string description for all the available metrics. */
-    public getMetricsDescription(): MetricStrOptions[] {
-       return this.metrics.map((i) => i.strOptions());
+    /**
+     * Gets a string description for all the available metrics.
+     * @param expression - Expression to be used in the tags
+     */
+    public getMetricsDescription(expression: string): MetricStrOptions[] {
+        let list = this.metrics.map((i) => i.strOptions());
+        if (!expression) {
+            return list;
+        }
+
+        const clauses = expression.split(";").filter((item: string) => item !== "");
+
+        for (let i = 0; i < clauses.length; ++i) {
+            const tags = clauses[i].split(",").filter((item: string) => item !== "");
+            list = list.filter((item) => {
+                return item.tags.some((o) => tags.some((t) => t === o));
+            });
+        }
+
+        return list;
     }
 
     /** Gets a string description for all the available enumerable types. */
@@ -91,9 +113,31 @@ export class Engine {
         return this.sources.map((i) => i.strOptions());
     }
 
-    /** Gets a string description for all the available dimensions. */
-    public getDimensionsDescription(): DimensionStrOptions[] {
-       return this.dimensions.map((i) => i.strOptions());
+    /**
+     *  Gets a string description for all the available dimensions.
+     * @param expression - Expression to be used in the tags
+     */
+    public getDimensionsDescription(expression: string): DimensionStrOptions[] {
+        let list =  this.dimensions.map((i) => i.strOptions());
+        if (!expression) {
+            return list;
+        }
+
+        const clauses = expression.split(";").filter((item: string) => item !== "");
+
+        for (let i = 0; i < clauses.length; ++i) {
+            const tags = clauses[i].split(",").filter((item: string) => item !== "");
+            list = list.filter((item) => {
+                return item.tags.some((o) => tags.some((t) => t === o));
+            });
+        }
+
+        return list;
+    }
+
+    /** Gets a string description for all the available tags. */
+    public getTagsDescription(): TagOptions[] {
+       return this.tags.map((i) => i.strOptions());
     }
 
     /**
@@ -118,6 +162,15 @@ export class Engine {
         return enumType;
     }
 
+    /**
+     * Adds a new tag to the database schema (engine).
+     * @param enumType - Enumerable type to be added.
+     */
+    public addTag(tag: Tag): Tag {
+        this.tags.push(tag);
+        return tag;
+    }
+
     /**
      * Adds a new source to the database schema (engine).
      * @param source - Source to be added.
@@ -340,4 +393,5 @@ export class Engine {
         return noRepeat;
 
     }
+
 }

src/core/metric.ts

@@ -20,6 +20,7 @@
 
 import { AggregationType, DataType } from "../common/types";
 import { EnumHandler } from "../util/enumHandler";
+import { Tag } from "../common/tag";
 
 /** Parameters used to create a metric object. */
 export interface MetricOptions {
@@ -31,6 +32,8 @@ export interface MetricOptions {
     dataType: DataType;
     /** Breif description of what this metric represents. */
     description?: string;
+    /** List of tags, a complement to attribute description. */
+    tags?: Tag[];
 }
 
 /**
@@ -46,6 +49,8 @@ export interface MetricStrOptions {
     dataType: string;
     /** Breif description of what this metric represents. */
     description?: string;
+    /** List of tag names, a complement to attribute description. */
+    tags?: string[];
 }
 
 /**
@@ -64,6 +69,8 @@ export class Metric {
     public readonly dataType: DataType;
     /** Breif description of what this metric represents. */
     public readonly description: string;
+    /** List of tags, a complement to attribute description. */
+    public readonly tags: Tag[];
 
     /**
      * Create a metric.
@@ -74,6 +81,7 @@ export class Metric {
         this.aggregation = options.aggregation;
         this.dataType = options.dataType;
         this.description = (options.description) ? options.description : "";
+        this.tags = (options.tags) ? options.tags.map((i) => i) : [];
     }
 
     /**
@@ -85,7 +93,8 @@ export class Metric {
             name: this.name,
             aggregation: EnumHandler.stringifyAggrType(this.aggregation),
             dataType: EnumHandler.stringfyDataType(this.dataType),
-            description: this.description
+            description: this.description,
+            tags: this.tags.map ((i) => i.name)
         };
     }
 }

src/util/configParser.spec.ts

@@ -95,7 +95,7 @@ describe("configParser utility library", () => {
 
         let error: boolean = false;
         try {
-            ConfigParser.parseDimOpts(opts, dims, null);
+            ConfigParser.parseDimOpts(opts, dims, null, null);
         }
         catch (e) {
             error = true;
@@ -196,7 +196,7 @@ describe("configParser utility library", () => {
         ];
 
         for (let i = 0; i < opts.length; ++i) {
-            let parsed = ConfigParser.parseDimOpts(opts[i], dims, null);
+            let parsed = ConfigParser.parseDimOpts(opts[i], dims, null, null);
             expect(parsed.name).to.be.equal(opts[i].name);
             expect(EnumHandler.stringfyDataType(parsed.dataType)).to.be.equal(opts[i].dataType);
             expect(parsed.parent).to.be.equal(dims[1]);
@@ -219,7 +219,7 @@ describe("configParser utility library", () => {
         let enumMap: {[key: string]: EnumType} = {
             "enumtype:5" : new EnumType({name: "enumtype:5", values: ["nope", "test"]})
         };
-        let parsed = ConfigParser.parseDimOpts(opts, dims, enumMap);
+        let parsed = ConfigParser.parseDimOpts(opts, dims, enumMap, null);