Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
blendb
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
23
Issues
23
List
Boards
Labels
Service Desk
Milestones
Merge Requests
1
Merge Requests
1
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
C3SL
blendb
Commits
1c178bda
Commit
1c178bda
authored
Sep 12, 2017
by
Lucas Fernandes de Oliveira
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'issue/41' into 'master'
Issue
#41
: Fix join in adapter See merge request
!34
parents
f8295d6d
81658d34
Pipeline
#12076
passed with stage
in 42 seconds
Changes
11
Pipelines
1
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
634 additions
and
56 deletions
+634
-56
config/ci_test.yaml.example
config/ci_test.yaml.example
+2
-2
src/adapter/postgres.spec.ts
src/adapter/postgres.spec.ts
+38
-0
src/adapter/postgres.ts
src/adapter/postgres.ts
+523
-31
src/core/engine.spec.ts
src/core/engine.spec.ts
+4
-4
src/core/engine.ts
src/core/engine.ts
+7
-1
src/core/view.ts
src/core/view.ts
+8
-0
src/util/configParser.ts
src/util/configParser.ts
+15
-1
test/postgres/fixtures/view0.json
test/postgres/fixtures/view0.json
+5
-5
test/postgres/fixtures/view2.json
test/postgres/fixtures/view2.json
+5
-5
test/postgres/fixtures/view8.json
test/postgres/fixtures/view8.json
+5
-5
test/scenario.ts
test/scenario.ts
+22
-2
No files found.
config/ci_test.yaml.example
View file @
1c178bda
...
...
@@ -25,6 +25,7 @@ schema:
- "met:0"
- "met:1"
- "met:2"
- "met:10"
-
alias: "View 1"
data: "test/postgres/fixtures/view1.json"
...
...
@@ -44,6 +45,7 @@ schema:
metrics:
- "met:5"
- "met:6"
- "met:11"
-
alias: "View 3"
data: "test/postgres/fixtures/view3.json"
...
...
@@ -93,8 +95,6 @@ schema:
- "dim:6"
metrics:
- "met:9"
- "met:10"
- "met:11"
-
alias: "view 9"
data: "test/postgres/fixtures/view9.json"
...
...
src/adapter/postgres.spec.ts
View file @
1c178bda
...
...
@@ -319,4 +319,42 @@ describe("postgres adapter", () => {
done
();
});
});
it
(
"
should get data from a unmaterializeble view
"
,
(
done
)
=>
{
let
view
=
adapterScenario
.
unMaterializebleView
;
adapter
.
getDataFromView
(
view
,
(
err
,
result
)
=>
{
expect
(
err
).
to
.
be
.
a
(
"
null
"
);
expect
(
result
).
to
.
be
.
an
(
"
array
"
);
expect
(
result
).
to
.
have
.
length
(
5
);
expect
(
result
[
0
]).
to
.
be
.
an
(
"
object
"
);
let
keys
:
string
[]
=
[];
keys
=
keys
.
concat
(
view
.
metrics
.
map
((
item
)
=>
item
.
name
));
keys
=
keys
.
concat
(
view
.
dimensions
.
map
((
item
)
=>
item
.
name
));
result
.
forEach
((
row
)
=>
{
expect
(
row
).
to
.
be
.
an
(
"
object
"
);
expect
(
row
).
to
.
have
.
all
.
keys
(
keys
);
});
done
();
});
});
it
(
"
should get data from partial joins
"
,
(
done
)
=>
{
let
view
=
adapterScenario
.
partialJoinView
;
adapter
.
getDataFromView
(
view
,
(
err
,
result
)
=>
{
expect
(
err
).
to
.
be
.
a
(
"
null
"
);
expect
(
result
).
to
.
be
.
an
(
"
array
"
);
expect
(
result
).
to
.
have
.
length
(
5
);
expect
(
result
[
0
]).
to
.
be
.
an
(
"
object
"
);
let
keys
:
string
[]
=
[];
keys
=
keys
.
concat
(
view
.
metrics
.
map
((
item
)
=>
item
.
name
));
keys
=
keys
.
concat
(
view
.
dimensions
.
map
((
item
)
=>
item
.
name
));
result
.
forEach
((
row
)
=>
{
expect
(
row
).
to
.
be
.
an
(
"
object
"
);
expect
(
row
).
to
.
have
.
all
.
keys
(
keys
);
});
done
();
});
});
});
src/adapter/postgres.ts
View file @
1c178bda
This diff is collapsed.
Click to expand it.
src/core/engine.spec.ts
View file @
1c178bda
...
...
@@ -85,7 +85,7 @@ describe("engine class", () => {
it
(
"
should be create a fill that cover the query, that match perfectly with a existent view
"
,
()
=>
{
let
query
=
{
metrics
:
[
met
[
0
],
met
[
1
],
met
[
2
]]
metrics
:
[
met
[
0
],
met
[
1
],
met
[
2
]
,
met
[
10
]
]
,
dimensions
:
[
dim
[
0
],
dim
[
7
]]
};
let
optimalView
=
engine
.
query
(
query
);
...
...
@@ -96,7 +96,7 @@ describe("engine class", () => {
expect
(
optimalView
.
metrics
).
to
.
be
.
an
(
"
array
"
);
expect
(
optimalView
.
dimensions
).
to
.
be
.
an
(
"
array
"
);
expect
(
optimalView
.
childViews
).
to
.
be
.
an
(
"
array
"
);
expect
(
optimalView
.
metrics
).
to
.
have
.
length
(
3
);
expect
(
optimalView
.
metrics
).
to
.
have
.
length
(
4
);
expect
(
optimalView
.
dimensions
).
to
.
have
.
length
(
2
);
expect
(
optimalView
.
childViews
).
to
.
have
.
length
(
0
);
...
...
@@ -149,7 +149,7 @@ describe("engine class", () => {
expect
(
optimalView
.
childViews
).
to
.
be
.
an
(
"
array
"
);
expect
(
optimalView
.
metrics
).
to
.
have
.
length
(
0
);
expect
(
optimalView
.
dimensions
).
to
.
have
.
length
(
2
);
expect
(
optimalView
.
childViews
).
to
.
have
.
length
(
1
);
expect
(
optimalView
.
childViews
).
to
.
have
.
length
(
3
);
expect
(
optimalView
).
satisfy
((
optView
:
View
)
=>
{
return
optView
.
dimensions
.
some
((
item
)
=>
item
.
name
===
subdim
[
0
].
name
);
...
...
@@ -175,7 +175,7 @@ describe("engine class", () => {
expect
(
optimalView
.
childViews
).
to
.
be
.
an
(
"
array
"
);
expect
(
optimalView
.
metrics
).
to
.
have
.
length
(
0
);
expect
(
optimalView
.
dimensions
).
to
.
have
.
length
(
2
);
expect
(
optimalView
.
childViews
).
to
.
have
.
length
(
1
);
expect
(
optimalView
.
childViews
).
to
.
have
.
length
(
3
);
expect
(
optimalView
).
satisfy
((
optView
:
View
)
=>
{
return
optView
.
dimensions
.
some
((
item
)
=>
item
.
name
===
subdim
[
2
].
name
);
...
...
src/core/engine.ts
View file @
1c178bda
...
...
@@ -175,7 +175,13 @@ export class Engine {
};
let
view
=
new
View
(
options
);
this
.
addView
(
view
);
// this.addView(view);
/*
This line has been removed for now because not all views can be
re-used by other views (unmaterializeble views), and so far this
is only detected in the adapter, when this can be detected in
engine, than the queries can be added again to the engine
*/
return
view
;
}
}
...
...
src/core/view.ts
View file @
1c178bda
...
...
@@ -31,6 +31,7 @@ export interface LoadView {
export
interface
ViewOptions
{
metrics
:
Metric
[];
dimensions
:
Dimension
[];
keys
?:
Dimension
[];
origin
:
boolean
;
clauses
?:
Clause
[];
materialized
?:
boolean
;
...
...
@@ -41,6 +42,7 @@ export class View {
public
readonly
id
:
string
;
public
readonly
metrics
:
Metric
[];
public
readonly
dimensions
:
Dimension
[];
public
readonly
keys
:
Dimension
[];
public
readonly
clauses
:
Clause
[];
public
readonly
materialized
:
boolean
;
public
readonly
origin
:
boolean
;
...
...
@@ -53,6 +55,12 @@ export class View {
this
.
materialized
=
options
.
materialized
||
false
;
this
.
origin
=
options
.
origin
||
false
;
this
.
childViews
=
(
options
.
childViews
)
?
options
.
childViews
:
[];
if
(
options
.
keys
&&
options
.
keys
.
length
>
0
)
{
this
.
keys
=
options
.
keys
.
sort
();
}
else
{
this
.
keys
=
this
.
dimensions
;
}
// calculate the id of the view based on it's metrics and dimensions
const
metNames
=
this
.
metrics
.
map
((
metric
)
=>
metric
.
name
);
...
...
src/util/configParser.ts
View file @
1c178bda
...
...
@@ -36,6 +36,7 @@ export interface ViewParsingOptions {
dimensions
:
string
[];
metrics
:
string
[];
clauses
?:
string
[];
keys
?:
string
[];
}
interface
MetricStrOptions
{
...
...
@@ -156,9 +157,12 @@ export class ConfigParser {
materialized
:
true
,
origin
:
opts
.
origin
,
childViews
:
[],
clauses
:
[]
clauses
:
[],
keys
:
[]
};
const
keys
=
(
opts
.
keys
)
?
opts
.
keys
:
[];
for
(
let
i
=
0
;
i
<
opts
.
metrics
.
length
;
++
i
)
{
if
(
metMap
[
opts
.
metrics
[
i
]])
{
viewOpt
.
metrics
.
push
(
metMap
[
opts
.
metrics
[
i
]]);
...
...
@@ -179,6 +183,16 @@ export class ConfigParser {
}
}
for
(
let
i
=
0
;
i
<
keys
.
length
;
++
i
)
{
if
(
dimMap
[
keys
[
i
]])
{
viewOpt
.
keys
.
push
(
dimMap
[
opts
.
dimensions
[
i
]]);
}
else
{
throw
new
Error
(
"
[Parsing error] Non exist key set to view
"
+
opts
.
alias
);
}
}
if
(
opts
.
clauses
)
{
for
(
let
i
=
0
;
i
<
opts
.
clauses
.
length
;
++
i
)
{
const
clause
=
this
.
parseClause
(
opts
.
clauses
[
i
],
metMap
,
dimMap
);
...
...
test/postgres/fixtures/view0.json
View file @
1c178bda
[
{
"dim:0"
:
"2017-01-01"
,
"dim:7"
:
"1"
,
"met:0"
:
"1"
,
"met:1"
:
"1"
,
"met:2"
:
"1"
},
{
"dim:0"
:
"2017-01-02"
,
"dim:7"
:
"2"
,
"met:0"
:
"2"
,
"met:1"
:
"2"
,
"met:2"
:
"2"
},
{
"dim:0"
:
"2017-01-03"
,
"dim:7"
:
"3"
,
"met:0"
:
"3"
,
"met:1"
:
"3"
,
"met:2"
:
"3"
},
{
"dim:0"
:
"2017-01-04"
,
"dim:7"
:
"4"
,
"met:0"
:
"4"
,
"met:1"
:
"4"
,
"met:2"
:
"4"
},
{
"dim:0"
:
"2017-01-05"
,
"dim:7"
:
"5"
,
"met:0"
:
"5"
,
"met:1"
:
"5"
,
"met:2"
:
"5"
}
{
"dim:0"
:
"2017-01-01"
,
"dim:7"
:
"1"
,
"met:0"
:
"1"
,
"met:1"
:
"1"
,
"met:2"
:
"1"
,
"met:10"
:
"1"
},
{
"dim:0"
:
"2017-01-02"
,
"dim:7"
:
"2"
,
"met:0"
:
"2"
,
"met:1"
:
"2"
,
"met:2"
:
"2"
,
"met:10"
:
"2"
},
{
"dim:0"
:
"2017-01-03"
,
"dim:7"
:
"3"
,
"met:0"
:
"3"
,
"met:1"
:
"3"
,
"met:2"
:
"3"
,
"met:10"
:
"3"
},
{
"dim:0"
:
"2017-01-04"
,
"dim:7"
:
"4"
,
"met:0"
:
"4"
,
"met:1"
:
"4"
,
"met:2"
:
"4"
,
"met:10"
:
"4"
},
{
"dim:0"
:
"2017-01-05"
,
"dim:7"
:
"5"
,
"met:0"
:
"5"
,
"met:1"
:
"5"
,
"met:2"
:
"5"
,
"met:10"
:
"5"
}
]
test/postgres/fixtures/view2.json
View file @
1c178bda
[
{
"dim:2"
:
"1"
,
"met:5"
:
"1"
,
"met:6"
:
"1"
},
{
"dim:2"
:
"2"
,
"met:5"
:
"2"
,
"met:6"
:
"2"
},
{
"dim:2"
:
"3"
,
"met:5"
:
"3"
,
"met:6"
:
"3"
},
{
"dim:2"
:
"4"
,
"met:5"
:
"4"
,
"met:6"
:
"4"
},
{
"dim:2"
:
"5"
,
"met:5"
:
"5"
,
"met:6"
:
"5"
}
{
"dim:2"
:
"1"
,
"met:5"
:
"1"
,
"met:6"
:
"1"
,
"met:11"
:
"1"
},
{
"dim:2"
:
"2"
,
"met:5"
:
"2"
,
"met:6"
:
"2"
,
"met:11"
:
"2"
},
{
"dim:2"
:
"3"
,
"met:5"
:
"3"
,
"met:6"
:
"3"
,
"met:11"
:
"3"
},
{
"dim:2"
:
"4"
,
"met:5"
:
"4"
,
"met:6"
:
"4"
,
"met:11"
:
"4"
},
{
"dim:2"
:
"5"
,
"met:5"
:
"5"
,
"met:6"
:
"5"
,
"met:11"
:
"5"
}
]
test/postgres/fixtures/view8.json
View file @
1c178bda
[
{
"dim:5"
:
"t"
,
"dim:6"
:
"1"
,
"met:9"
:
"1"
,
"met:10"
:
"1"
,
"met:11"
:
"1"
},
{
"dim:5"
:
"t"
,
"dim:6"
:
"2"
,
"met:9"
:
"2"
,
"met:10"
:
"2"
,
"met:11"
:
"2"
},
{
"dim:5"
:
"t"
,
"dim:6"
:
"3"
,
"met:9"
:
"3"
,
"met:10"
:
"3"
,
"met:11"
:
"3"
},
{
"dim:5"
:
"f"
,
"dim:6"
:
"4"
,
"met:9"
:
"4"
,
"met:10"
:
"4"
,
"met:11"
:
"4"
},
{
"dim:5"
:
"f"
,
"dim:6"
:
"5"
,
"met:9"
:
"5"
,
"met:10"
:
"5"
,
"met:11"
:
"5"
}
{
"dim:5"
:
"t"
,
"dim:6"
:
"1"
,
"met:9"
:
"1"
},
{
"dim:5"
:
"t"
,
"dim:6"
:
"2"
,
"met:9"
:
"2"
},
{
"dim:5"
:
"t"
,
"dim:6"
:
"3"
,
"met:9"
:
"3"
},
{
"dim:5"
:
"f"
,
"dim:6"
:
"4"
,
"met:9"
:
"4"
},
{
"dim:5"
:
"f"
,
"dim:6"
:
"5"
,
"met:9"
:
"5"
}
]
test/scenario.ts
View file @
1c178bda
...
...
@@ -52,6 +52,8 @@ interface AdapterScenario {
geleView
:
View
;
notMatchFilterView
:
View
;
notOriginCount
:
View
;
unMaterializebleView
:
View
;
partialJoinView
:
View
;
}
interface
DataCtrlScenario
{
...
...
@@ -224,7 +226,7 @@ const aggrView = new View({
dimensions
:
[],
materialized
:
false
,
origin
:
false
,
childViews
:
[
views
[
0
],
views
[
2
],
views
[
3
]
,
views
[
4
],
views
[
7
],
views
[
8
]
]
childViews
:
[
views
[
0
],
views
[
2
],
views
[
3
]]
});
const
clauseView
=
new
View
({
...
...
@@ -330,6 +332,22 @@ const notOriginCount = new View({
clauses
:
[
clauses
.
view9dim2
]
});
const
unMaterializebleView
=
new
View
({
metrics
:
[
mets
[
7
],
mets
[
9
]],
dimensions
:
[
dims
[
5
]],
materialized
:
false
,
origin
:
false
,
childViews
:
[
views
[
3
],
views
[
5
],
views
[
7
],
views
[
8
]]
});
const
partialJoinView
=
new
View
({
metrics
:
[
mets
[
7
],
mets
[
8
]],
dimensions
:
[],
materialized
:
false
,
origin
:
false
,
childViews
:
[
views
[
3
],
views
[
5
],
views
[
6
]]
});
export
const
engineScenario
:
EngineScenario
=
{
metrics
:
mets
,
dimensions
:
dims
,
...
...
@@ -354,7 +372,9 @@ export const adapterScenario: AdapterScenario = {
gtltView
:
gtltView
,
geleView
:
geleView
,
notMatchFilterView
:
notMatchFilterView
,
notOriginCount
:
notOriginCount
notOriginCount
:
notOriginCount
,
unMaterializebleView
:
unMaterializebleView
,
partialJoinView
:
partialJoinView
};
export
const
dataCtrlScenario
:
DataCtrlScenario
=
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment