Issue #41: Fix join in adapter

Signed-off-by: Lucas Fernandes de Oliveira <lfo14@inf.ufpr.br>

config/ci_test.yaml.example

@@ -25,6 +25,7 @@ schema:
                 - "met:0"
                 - "met:1"
                 - "met:2"
+                - "met:10"
         -
             alias: "View 1"
             data: "test/postgres/fixtures/view1.json"
@@ -44,6 +45,7 @@ schema:
             metrics:
                 - "met:5"
                 - "met:6"
+                - "met:11"
         -
             alias: "View 3"
             data: "test/postgres/fixtures/view3.json"
@@ -93,8 +95,6 @@ schema:
                 - "dim:6"
             metrics:
                 - "met:9"
-                - "met:10"
-                - "met:11"
         -
             alias: "view 9"
             data: "test/postgres/fixtures/view9.json"

src/adapter/postgres.spec.ts

@@ -319,4 +319,42 @@ describe("postgres adapter", () => {
             done();
         });
     });
+
+    it("should get data from a unmaterializeble view", (done) => {
+        let view = adapterScenario.unMaterializebleView;
+        adapter.getDataFromView(view, (err, result) => {
+            expect(err).to.be.a("null");
+            expect(result).to.be.an("array");
+            expect(result).to.have.length(5);
+            expect(result[0]).to.be.an("object");
+            let keys: string[] = [];
+            keys = keys.concat(view.metrics.map((item) => item.name));
+            keys = keys.concat(view.dimensions.map((item) => item.name));
+            result.forEach((row) => {
+                expect(row).to.be.an("object");
+                expect(row).to.have.all.keys(keys);
+            });
+
+            done();
+        });
+    });
+
+    it("should get data from partial joins", (done) => {
+        let view = adapterScenario.partialJoinView;
+        adapter.getDataFromView(view, (err, result) => {
+            expect(err).to.be.a("null");
+            expect(result).to.be.an("array");
+            expect(result).to.have.length(5);
+            expect(result[0]).to.be.an("object");
+            let keys: string[] = [];
+            keys = keys.concat(view.metrics.map((item) => item.name));
+            keys = keys.concat(view.dimensions.map((item) => item.name));
+            result.forEach((row) => {
+                expect(row).to.be.an("object");
+                expect(row).to.have.all.keys(keys);
+            });
+
+            done();
+        });
+    });
 });

src/core/engine.spec.ts

@@ -85,7 +85,7 @@ describe("engine class", () => {
 
     it("should be create a fill that cover the query, that match perfectly with a existent view", () => {
         let query = {
-            metrics : [met[0], met[1], met[2]]
+            metrics : [met[0], met[1], met[2], met[10]]
             , dimensions : [dim[0], dim[7]]
         };
         let optimalView = engine.query(query);
@@ -96,7 +96,7 @@ describe("engine class", () => {
         expect(optimalView.metrics).to.be.an("array");
         expect(optimalView.dimensions).to.be.an("array");
         expect(optimalView.childViews).to.be.an("array");
-        expect(optimalView.metrics).to.have.length(3);
+        expect(optimalView.metrics).to.have.length(4);
         expect(optimalView.dimensions).to.have.length(2);
         expect(optimalView.childViews).to.have.length(0);
 
@@ -149,7 +149,7 @@ describe("engine class", () => {
         expect(optimalView.childViews).to.be.an("array");
         expect(optimalView.metrics).to.have.length(0);
         expect(optimalView.dimensions).to.have.length(2);
-        expect(optimalView.childViews).to.have.length(1);
+        expect(optimalView.childViews).to.have.length(3);
 
         expect(optimalView).satisfy((optView: View) => {
             return optView.dimensions.some((item) => item.name === subdim[0].name);
@@ -175,7 +175,7 @@ describe("engine class", () => {
         expect(optimalView.childViews).to.be.an("array");
         expect(optimalView.metrics).to.have.length(0);
         expect(optimalView.dimensions).to.have.length(2);
-        expect(optimalView.childViews).to.have.length(1);
+        expect(optimalView.childViews).to.have.length(3);
 
         expect(optimalView).satisfy((optView: View) => {
             return optView.dimensions.some((item) => item.name === subdim[2].name);

src/core/engine.ts

@@ -175,7 +175,13 @@ export class Engine {
             };
 
             let view = new View(options);
-            this.addView(view);
+            // this.addView(view);
+            /*
+                This line has been removed for now because not all views can be
+                re-used by other views (unmaterializeble views), and so far this
+                is only detected in the adapter, when this can be detected in
+                engine, than the queries can be added again to the engine
+            */
             return view;
         }
     }

src/core/view.ts

@@ -31,6 +31,7 @@ export interface LoadView {
 export interface ViewOptions {
     metrics: Metric[];
     dimensions: Dimension[];
+    keys?: Dimension[];
     origin: boolean;
     clauses?: Clause[];
     materialized?: boolean;
@@ -41,6 +42,7 @@ export class View {
     public readonly id: string;
     public readonly metrics: Metric[];
     public readonly dimensions: Dimension[];
+    public readonly keys: Dimension[];
     public readonly clauses: Clause[];
     public readonly materialized: boolean;
     public readonly origin: boolean;
@@ -53,6 +55,12 @@ export class View {
         this.materialized = options.materialized || false;
         this.origin = options.origin || false;
         this.childViews = (options.childViews) ? options.childViews : [];
+        if (options.keys && options.keys.length > 0) {
+            this.keys = options.keys.sort();
+        }
+        else {
+            this.keys = this.dimensions;
+        }
 
         // calculate the id of the view based on it's metrics and dimensions
         const metNames = this.metrics.map((metric) => metric.name);

src/util/configParser.ts

@@ -36,6 +36,7 @@ export interface ViewParsingOptions {
     dimensions: string[];
     metrics: string[];
     clauses?: string[];
+    keys?: string[];
 }
 
 interface MetricStrOptions {
@@ -156,9 +157,12 @@ export class ConfigParser {
             materialized: true,
             origin: opts.origin,
             childViews: [],
-            clauses: []
+            clauses: [],
+            keys: []
         };
 
+        const keys = (opts.keys) ? opts.keys : [];
+
         for (let i = 0; i < opts.metrics.length; ++i) {
             if (metMap[opts.metrics[i]]) {
                 viewOpt.metrics.push(metMap[opts.metrics[i]]);
@@ -179,6 +183,16 @@ export class ConfigParser {
             }
         }
 
+        for (let i = 0; i < keys.length; ++i) {
+            if (dimMap[keys[i]]) {
+                viewOpt.keys.push(dimMap[opts.dimensions[i]]);
+            }
+
+            else {
+                throw new Error("[Parsing error] Non exist key set to view " + opts.alias);
+            }
+        }
+
         if (opts.clauses) {
             for (let i = 0; i < opts.clauses.length; ++i) {
                 const clause = this.parseClause(opts.clauses[i], metMap, dimMap);

test/postgres/fixtures/view0.json

 [
-{"dim:0":"2017-01-01","dim:7":"1","met:0":"1","met:1":"1","met:2":"1"},
-{"dim:0":"2017-01-02","dim:7":"2","met:0":"2","met:1":"2","met:2":"2"},
-{"dim:0":"2017-01-03","dim:7":"3","met:0":"3","met:1":"3","met:2":"3"},
-{"dim:0":"2017-01-04","dim:7":"4","met:0":"4","met:1":"4","met:2":"4"},
-{"dim:0":"2017-01-05","dim:7":"5","met:0":"5","met:1":"5","met:2":"5"}
+{"dim:0":"2017-01-01","dim:7":"1","met:0":"1","met:1":"1","met:2":"1","met:10":"1"},
+{"dim:0":"2017-01-02","dim:7":"2","met:0":"2","met:1":"2","met:2":"2","met:10":"2"},
+{"dim:0":"2017-01-03","dim:7":"3","met:0":"3","met:1":"3","met:2":"3","met:10":"3"},
+{"dim:0":"2017-01-04","dim:7":"4","met:0":"4","met:1":"4","met:2":"4","met:10":"4"},
+{"dim:0":"2017-01-05","dim:7":"5","met:0":"5","met:1":"5","met:2":"5","met:10":"5"}
 ]

test/postgres/fixtures/view2.json

 [
-{"dim:2":"1","met:5":"1","met:6":"1"},
-{"dim:2":"2","met:5":"2","met:6":"2"},
-{"dim:2":"3","met:5":"3","met:6":"3"},
-{"dim:2":"4","met:5":"4","met:6":"4"},
-{"dim:2":"5","met:5":"5","met:6":"5"}
+{"dim:2":"1","met:5":"1","met:6":"1","met:11":"1"},
+{"dim:2":"2","met:5":"2","met:6":"2","met:11":"2"},
+{"dim:2":"3","met:5":"3","met:6":"3","met:11":"3"},
+{"dim:2":"4","met:5":"4","met:6":"4","met:11":"4"},
+{"dim:2":"5","met:5":"5","met:6":"5","met:11":"5"}
 ]

test/postgres/fixtures/view8.json

 [
-{"dim:5":"t","dim:6":"1","met:9":"1","met:10":"1","met:11":"1"},
-{"dim:5":"t","dim:6":"2","met:9":"2","met:10":"2","met:11":"2"},
-{"dim:5":"t","dim:6":"3","met:9":"3","met:10":"3","met:11":"3"},
-{"dim:5":"f","dim:6":"4","met:9":"4","met:10":"4","met:11":"4"},
-{"dim:5":"f","dim:6":"5","met:9":"5","met:10":"5","met:11":"5"}
+{"dim:5":"t","dim:6":"1","met:9":"1"},
+{"dim:5":"t","dim:6":"2","met:9":"2"},
+{"dim:5":"t","dim:6":"3","met:9":"3"},
+{"dim:5":"f","dim:6":"4","met:9":"4"},
+{"dim:5":"f","dim:6":"5","met:9":"5"}
 ]

test/scenario.ts

@@ -52,6 +52,8 @@ interface AdapterScenario {
     geleView: View;
     notMatchFilterView: View;
     notOriginCount: View;
+    unMaterializebleView: View;
+    partialJoinView: View;
 }
 
 interface DataCtrlScenario {
@@ -224,7 +226,7 @@ const aggrView = new View({
     dimensions: [],
     materialized: false,
     origin: false,
-    childViews: [views[0], views[2], views[3], views[4], views[7], views[8]]
+    childViews: [views[0], views[2], views[3]]
 });
 
 const clauseView = new View({
@@ -330,6 +332,22 @@ const notOriginCount = new View({
     clauses: [clauses.view9dim2]
 });
 
+const unMaterializebleView = new View({
+    metrics: [mets[7], mets[9]],
+    dimensions: [dims[5]],
+    materialized: false,
+    origin: false,
+    childViews: [views[3], views[5], views[7], views[8]]
+});
+
+const partialJoinView = new View({
+    metrics: [mets[7], mets[8]],
+    dimensions: [],
+    materialized: false,
+    origin: false,
+    childViews: [views[3], views[5], views[6]]
+});
+
 export const engineScenario: EngineScenario = {
     metrics: mets,
     dimensions: dims,
@@ -354,7 +372,9 @@ export const adapterScenario: AdapterScenario = {
     gtltView: gtltView,
     geleView: geleView,
     notMatchFilterView: notMatchFilterView,
-    notOriginCount: notOriginCount
+    notOriginCount: notOriginCount,
+    unMaterializebleView: unMaterializebleView,
+    partialJoinView: partialJoinView
 };
 
 export const dataCtrlScenario: DataCtrlScenario = {