Commit 2e0797f1 authored by Lucas Fernandes de Oliveira's avatar Lucas Fernandes de Oliveira
Browse files

Merge branch 'issue/9' into 'master'

Issue #9: Implement date parent relations in postgres adapter

See merge request !7
parents 1b7527f7 1f5c6c53
Pipeline #9002 passed with stage
in 1 minute and 55 seconds
......@@ -19,8 +19,8 @@
*/
import { Adapter } from "../core/adapter";
import { AggregationType } from "../common/types";
import { View } from "../core/view";
import { AggregationType, RelationType } from "../common/types";
import { View, ChildView } from "../core/view";
interface ParsedView {
query: string;
......@@ -30,44 +30,47 @@ interface ParsedView {
export class PostgresAdapter extends Adapter{
public getDataFromView(view: View): string {
// buildQueryFromView does not put the final ;, it need to be put apart
return this.buildQueryFromView(view) + ";\n";
return this.buildQueryFromView(view, view.metrics, view.dimensions) + ";\n";
}
public materializeView(view: View): string {
return null;
}
private buildQueryFromView (view: View): string {
let sql = "(\n";
let metrics = view.metrics.map((metric: string) => {
let aggrType = view.getAggregationtype(metric);
let func = this.getAggregateFunction(aggrType);
let extMetric = func + "(" + metric + ")";
private buildQueryFromView (view: View, metrics: Metric[], dimensions: Dimession[]): string {
/*
Reduce metrics and dimensions array to the intersection with the
view. So is possible only get useful data in the sub-querys.
*/
let strMetrics = metrics.map((metric) => {
let func = this.getAggregateFunction(metric.aggregation);
let extMetric = func + "(" + metric.name + ")";
return extMetric;
});
if (view.materialized) {
sql += "SELECT " + metrics.join(", ") + ", " + dimensions.join(", ") + "\n";
sql += "FROM " + "view_" + view.id + "\n";
sql += "GROUP BY " + view.dimensions.join(", ") + "\n";
let strDimensions = dimensions.map((dimmension) => dimension.name );
let sql = "(SELECT " + strMetrics.join(", ") + ", " + strDimensions.join(", ");
sql += " FROM " + "view_" + view.id;
sql += " GROUP BY " + strDimensions.join(", ");
sql += ")\n";
return sql;
}
else {
let children: ParsedView[] = view.childViews.map((item: View, idx: number) => {
let children: ParsedView[] = view.childViews.map((item: ChildView) => {
return {
query: this.queryfyView(item, childAlias, degree + 1),
view: item
query: this.buildQueryFromView(item.view, item.metrics, item.dimensions),
view: item.view
};
});
let covered = new Map();
view.dimensions.forEach((item: string) => covered.set(item, ""));
view.metrics.forEach((item: string) => covered.set(item, ""));
dimensions.forEach((item) => covered.set(item.name, ""));
metrics.forEach((item) => covered.set(item.name, ""));
let projection = "SELECT ";
let viewsFrom = "FROM";
let viewsFrom = "FROM ";
let selection = "WHERE ";
let grouping = "GROUP BY ";
......@@ -76,31 +79,64 @@ export class PostgresAdapter extends Adapter{
children.forEach((child: ParsedView) => {
let selected = [];
child.view.dimensions.forEach((dimension: string) => {
let first = covered.get(dimension);
let extDimension = "view_" + child.view.id + "." + dimension;
if (first === "") {
covered.set(dimension, child.view.id);
elements.push(extDimension);
group.push(extDimension);
}
let matchable = child.dimensions.map((item) => {
return { match: item, sub: item, viewId: child.view.id };
});
else {
let extFirst = "view_" + first + "." + dimension;
selected.push(extDimension + " = " + extFirst);
// Running over the dimensions that cover some part of the query
child.dimensions.forEach((dimension: Dimension) => {
/*
If a dimension has a parent, the parent must match with
the sub dimension, so must be added in the matchable
array, for the selection (WHERE statement).
*/
let dim = dimension;
while (dim.relation !== RelationType.NONE) {
matchable.push({
match: dim.name,
sub: dimension,
viewId: child.view.id
});
dim = dim.parent;
}
});
child.view.metrics.forEach((metric: string) => {
let first = covered.get(metric);
let aggrType = child.view.getAggregateFunction(metric);
let func = this.geAggregateFunction(aggrType);
let extMetric = func + "(view_" + child.view.id + "." + metric + ")";
if (first === "") {
covered.set(metric, child.view.id);
elements.push(extMetric);
dim = dimension;
let extDimension = "view_" + child.view.id + ".\"" + dim.name + "\"";
// If the view dimension does not match, search for parent
while (!child.dimensions.some((item) => item.name === dim.name)) {
dim = dim.parent;
extDimension = this.translateRelation(dim.relation, extDimension);
}
covered.set(dimension.name, extDimension);
elements.push(extDimension + " AS " + dimension.name);
group.push(dimension.name);
});
child.view.dimensions.forEach((dimension: Dimension) => {
/*
Make selection. Search for dimensions, that are in
matchable array.
*/
matchable.filter((item) => {
return item.viewId !== child.view.id &&
item.match === dimension.name;
})
.forEach((item) => {
// Expand the sub-dimension until match with a parent
let dim = item.sub;
let extDimension = "view_" + child.view.id + ".\"" + dimension.name + "\"";
while (dim.name !== item.match) {
dim = dim.parent;
extDimension = this.translateRelation(dim, extDimension);
}
selected.push(extDimension + " = " + covered.get(item.sub.name));
});
});
child.metrics.forEach((metric: Metric) => {
let func = this.geAggregateFunction(metric.aggregation);
let extMetric = func + "(view_" + child.view.id + "." + metric.name + ")";
elements.push(extMetric);
});
viewsFrom += "\n" + child.query;
......@@ -114,11 +150,8 @@ export class PostgresAdapter extends Adapter{
selection += "\n";
grouping += group.join(", ") + "\n";
sql += projection + viewsFrom + selection + grouping + ")";
return sql;
return "(" + projection + viewsFrom + selection + grouping + ")";
}
return sql;
}
private getAggregateFunction(aggrType: AggregationType): string {
......@@ -134,4 +167,38 @@ export class PostgresAdapter extends Adapter{
}
}
private translateRelation(relation: RelationType, arg: string): string {
switch (relation) {
case RelationType.DAY:
return this.applyRelation("EXTRACT", ["DAY FROM "], [arg]);
case RelationType.MONTH:
return this.applyRelation("EXTRACT", ["MONTH FROM "], [arg]);
case RelationType.YEAR:
return this.applyRelation("EXTRACT", ["YEAR FROM "], [arg]);
case RelationType.DAYOFWEEK:
return this.applyRelation("EXTRACT", ["DOW FROM "], [arg]);
default:
return "";
}
}
private applyRelation(name, args, values): string {
/*
This adapter uses the concept of functions in Postgres to
implement BLENDB sub-dimention relations, this functions
applys the transformation to build the call of a Postgres
funtion. Note that this function can be native from postgres,
like EXTRACT, or even implemented on the database.
This function is short and only used in the translateRelation
method however is a bit complex and is possible to be used
several times, because of that is puted appart to make easyer update
and avoid problems
Example
applyRelation ("EXTRACT", "["DAY FROM"]", ["view_0.date"])
output: EXTRACT(DAY FROM view_0.date)
*/
return name + "(" + args.map((item, idx) => item + values[idx]).join(",") + ")";
}
}
......@@ -27,5 +27,8 @@ export enum AggregationType {
export enum RelationType {
NONE,
DAY
DAY,
MONTH,
YEAR,
DAYOFWEEK
};
......@@ -22,18 +22,18 @@ import { RelationType } from "../common/types";
export interface DimensionOptions {
name: string;
parent?: string;
parent?: Dimension;
relation?: RelationType;
}
export class Dimension {
public readonly name: string;
public readonly parent: string;
public readonly parent: Dimension;
public readonly relation: RelationType;
constructor(options: DimensionOptions) {
this.name = options.name;
this.relation = (options.relation) ? options.relation : RelationType.NONE;
this.parent = (options.parent) ? options.parent : "";
this.parent = (options.parent) ? options.parent : null;
}
}
......@@ -54,11 +54,11 @@ describe("engine class", () => {
const dim10 = new Dimension({ name: "dim:10" });
const dim11 = new Dimension({ name: "dim:11" });
const subdim1 = new Dimension({ name: "sub:1", parent: "dim:1", relation: RelationType.DAY });
const subdim2 = new Dimension({ name: "sub:2", parent: "dim:9", relation: RelationType.DAY });
const subdim3 = new Dimension({ name: "sub:3", parent: "sub:1", relation: RelationType.DAY });
const subdim4 = new Dimension({ name: "sub:4", parent: "sub:0", relation: RelationType.DAY });
const subdim5 = new Dimension({ name: "sub:5", parent: "dim:2", relation: RelationType.DAY });
const subdim1 = new Dimension({ name: "sub:1", parent: dim1, relation: RelationType.DAY });
const subdim2 = new Dimension({ name: "sub:2", parent: dim9, relation: RelationType.DAY });
const subdim3 = new Dimension({ name: "sub:3", parent: subdim1, relation: RelationType.DAY });
const subdim4 = new Dimension({ name: "sub:4", parent: null, relation: RelationType.DAY });
const subdim5 = new Dimension({ name: "sub:5", parent: dim2, relation: RelationType.DAY });
engine.addMetric(met1);
engine.addMetric(met2);
......@@ -105,21 +105,32 @@ describe("engine class", () => {
metrics: [met1, met2, met3, met4, met5],
dimensions: [dim1, dim2],
materialized: false,
childViews: [views[0], views[6]]
childViews: [
{ view: views[0], metrics: [met1, met2, met3], dimensions: [dim1, dim2]},
{ view: views[6], metrics: [met4], dimensions: []},
{ view: views[1], metrics: [met5], dimensions: []}
]
}));
views.push(new View({
metrics: [met8, met9, met10],
dimensions: [dim8, dim9, dim10],
materialized: false,
childViews: [views[7], views[8], views[9]]
childViews: [
{ view: views[7], metrics: [met8], dimensions: [dim8, dim9, dim10]},
{ view: views[8], metrics: [met9], dimensions: []},
{ view: views[9], metrics: [met10], dimensions: []}
]
}));
views.push(new View({
metrics: [met1],
dimensions: [subdim1, subdim2],
materialized: false,
childViews: [views[0], views[9]]
childViews: [
{ view: views[0], metrics: [met1], dimensions: [dim1]},
{ view: views[9], metrics: [], dimensions: [dim9]}
]
}));
views.forEach((view) => engine.addView(view));
......@@ -247,7 +258,7 @@ describe("engine class", () => {
}
catch (e){
error = true;
expect(e.message).to.be.equal("The dimension named " + subdim4.parent + " was not found");
expect(e.message).to.be.equal("Engine sub-dimention " + subdim4.name + " with no parent");
}
expect(error);
......
......@@ -20,7 +20,7 @@
import { Dimension } from "./dimension";
import { Metric } from "./metric";
import { View } from "./view";
import { View, ChildView } from "./view";
import { Query } from "../common/query";
import { RelationType } from "../common/types";
......@@ -77,12 +77,12 @@ export class Engine {
private selectOptimalView (q: Query) {
let metUncovered = q.metrics.map((met) => met);
let dimUncovered = q.dimensions.map((dim) => dim);
let optimalViews: View[] = [];
let optimalViews: ChildView[] = [];
let activeViews = this.getViews();
// run this block until all metrics and dimmensions are covered
while (metUncovered.length > 0 || dimUncovered.length > 0) {
let bestView: View;
let bestView: ChildView;
let coverLength = metUncovered.length + dimUncovered.length;
let shortestDistance = coverLength + 1;
......@@ -96,8 +96,11 @@ export class Engine {
let dimIntersection = dimUncovered.filter((dim: Dimension) => {
let r: boolean = view.dimensions.some((item) => item.name === dim.name);
while (!r && dim.relation !== RelationType.NONE) {
r = view.dimensions.some((item) => item.name === dim.parent);
dim = this.getDimensionByName(dim.parent);
if (dim.parent === null) {
throw new Error("Engine sub-dimention " + dim.name + " with no parent");
}
r = view.dimensions.some((item) => item.name === dim.parent.name);
dim = dim.parent;
}
return r;
});
......@@ -108,14 +111,22 @@ export class Engine {
let distance = coverLength - intersection;
if (distance < shortestDistance) {
bestView = view;
bestView = {
view: view,
metrics: metIntersection,
dimensions: dimIntersection
};
shortestDistance = distance;
}
else if (distance === shortestDistance &&
view.dimensions.length < bestView.dimensions.length) {
// priorizes views with less dimensions
bestView = view;
bestView = {
view: view,
metrics: metIntersection,
dimensions: dimIntersection
};
}
return true;
......@@ -142,8 +153,11 @@ export class Engine {
dimUncovered = dimUncovered.filter((dim: Dimension) => {
let r: boolean = bestView.dimensions.some((item) => item.name === dim.name);
while (!r && dim.relation !== RelationType.NONE) {
r = bestView.dimensions.some((item) => item.name === dim.parent);
dim = this.getDimensionByName(dim.parent);
if (dim.parent === null) {
throw new Error("Engine sub-dimention " + dim.name + " with no parent");
}
r = bestView.dimensions.some((item) => item.name === dim.parent.name);
dim = dim.parent;
}
return !r;
});
......@@ -152,9 +166,9 @@ export class Engine {
// If all the metrics and dimensions are the same and only exist one child view
// return this single child view
if (optimalViews.length === 1 &&
q.metrics.every((item) => optimalViews[0].metrics.indexOf(item) !== -1) &&
q.dimensions.every((item) => optimalViews[0].dimensions.indexOf(item) !== -1)) {
return optimalViews[0];
q.metrics.every((item) => optimalViews[0].view.metrics.indexOf(item) !== -1) &&
q.dimensions.every((item) => optimalViews[0].view.dimensions.indexOf(item) !== -1)) {
return optimalViews[0].view;
}
else {
let options = {
......
......@@ -22,11 +22,17 @@ import { Dimension } from "./dimension";
import { Metric } from "./metric";
import { Hash } from "../util/hash";
export interface ChildView {
metrics: Metric[];
dimensions: Dimension[];
view: View;
}
export interface ViewOptions {
metrics: Metric[];
dimensions: Dimension[];
materialized?: boolean;
childViews?: View[];
childViews?: ChildView[];
}
export class View {
......@@ -34,7 +40,7 @@ export class View {
public readonly metrics: Metric[];
public readonly dimensions: Dimension[];
public readonly materialized: boolean;
public childViews: View[];
public childViews: ChildView[];
constructor (options: ViewOptions) {
this.metrics = options.metrics.sort();
......@@ -43,8 +49,8 @@ export class View {
this.childViews = options.childViews || [];
// calculate the id of the view based on it's metrics and dimensions
let metricsNames = this.metrics.map(metric => metric.name);
let dimensionsNames = this.dimensions.map(dimension => dimension.name);
let metricsNames = options.metrics.map(metric => metric.name);
let dimensionsNames = options.dimensions.map(dimension => dimension.name);
this.id = Hash.sha1(metricsNames.concat(dimensionsNames).sort());
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment