Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
blendb
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
22
Issues
22
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
C3SL
blendb
Commits
2ffe7d55
Commit
2ffe7d55
authored
Apr 09, 2018
by
Rafael Dias
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Issue
#62
: Add insert
Signed-off-by:
Rafael
<
rpd17@inf.ufpr.br
>
parent
30658b05
Pipeline
#15420
passed with stages
in 1 minute and 3 seconds
Changes
11
Pipelines
1
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
546 additions
and
41 deletions
+546
-41
config/ci_test.yaml.example
config/ci_test.yaml.example
+48
-22
src/adapter/monet.ts
src/adapter/monet.ts
+9
-0
src/adapter/postgres.ts
src/adapter/postgres.ts
+8
-0
src/adapter/sql.ts
src/adapter/sql.ts
+19
-0
src/api/controllers/collect.spec.ts
src/api/controllers/collect.spec.ts
+232
-10
src/api/controllers/collect.ts
src/api/controllers/collect.ts
+142
-6
src/api/controllers/engine.spec.ts
src/api/controllers/engine.spec.ts
+1
-1
src/core/adapter.ts
src/core/adapter.ts
+2
-0
src/core/engine.ts
src/core/engine.ts
+20
-0
src/util/configParser.ts
src/util/configParser.ts
+2
-1
test/postgres/fixture.ts
test/postgres/fixture.ts
+63
-1
No files found.
config/ci_test.yaml.example
View file @
2ffe7d55
...
...
@@ -213,30 +213,31 @@ dimensions:
description: "A dimension of Blendb. Has 1 possible value."
enumTypes:
-
name: "enum
type:0"
name: "enumtype:0"
values:
- "
test:0
"
- "
test:1
"
- "
test:2
"
- "
test:3
"
- "
male
"
- "
female
"
- "
binary
"
- "
undecided
"
-
name: "enum
type:1"
name: "enumtype:1"
values:
- "test:4"
- "test:5"
- "test:6"
- "test_4"
- "test_5"
- "test_6"
- "string"
-
name: "enum
type:2"
name: "enumtype:2"
values:
- "test
:
7"
- "test
:
8"
- "test
_
7"
- "test
_
8"
-
name: "enum
type:3"
name: "enumtype:3"
values:
- "test
:9"
- "test
_9"
sources:
-
name: "source
:
0"
name: "source
_
0"
description: "source with 3 entries"
fields:
-
...
...
@@ -252,30 +253,55 @@ sources:
description: "third entry"
dataType: "string"
-
name: "source
:
1"
name: "source
_
1"
description: "source with 2 entries"
fields:
-
name: "fields:0"
description: "first entry"
dataType: "
string
"
dataType: "
enumtype:0
"
-
name: "fields:1"
description: "second entry"
dataType: "string"
-
name: "source
:
2"
description: "source with one entry"
name: "source
_
2"
description: "source with one entry
and undefined dataType
"
fields:
-
name: "fields:0"
description: "first entry"
dataType: "
string
"
dataType: "
notValid
"
-
name: "source
:
3"
name: "source
_
3"
description: "source with one entry and without description"
fields:
-
name: "fields:3"
name: "fields:0"
dataType: "string"
-
name: "source_4"
description: "source with all core types from blendb"
fields:
-
name: "fields:0"
description: "first entry"
dataType: "integer"
-
name: "fields:1"
description: "second entry"
dataType: "float"
-
name: "fields:2"
description: "third entry"
dataType: "string"
-
name: "fields:3"
description: "fourth entry"
dataType: "boolean"
-
name: "fields:4"
description: "fifth entry"
dataType: "date"
\ No newline at end of file
src/adapter/monet.ts
View file @
2ffe7d55
...
...
@@ -20,6 +20,7 @@
import
{
SQLAdapter
}
from
"
./sql
"
;
import
{
View
}
from
"
../core/view
"
;
import
{
Source
}
from
"
../core/source
"
;
import
{
FilterOperator
}
from
"
../core/filter
"
;
const
MDB
=
require
(
"
monetdb
"
)();
...
...
@@ -45,7 +46,10 @@ export class MonetAdapter extends SQLAdapter {
}
public
getDataFromView
(
view
:
View
,
cb
:
(
error
:
Error
,
result
?:
any
[])
=>
void
):
void
{
const
query
=
this
.
getQueryFromView
(
view
);
this
.
executeQuery
(
query
,
cb
);
}
private
executeQuery
(
query
:
string
,
cb
:
(
error
:
Error
,
result
?:
any
[])
=>
void
):
void
{
let
pool
:
any
=
new
MDB
(
this
.
config
);
pool
.
connect
();
pool
.
query
(
query
).
then
((
result
:
MonetResult
)
=>
{
...
...
@@ -78,6 +82,11 @@ export class MonetAdapter extends SQLAdapter {
return
false
;
}
public
insertIntoSource
(
source
:
Source
,
data
:
any
[],
cb
:
(
err
:
Error
,
result
:
any
[])
=>
void
):
void
{
const
query
=
this
.
getQueryFromSource
(
source
,
data
);
this
.
executeQuery
(
query
,
cb
);
}
protected
typeCast
(
quotedValue
:
string
,
dt
:
string
):
string
{
switch
(
dt
)
{
case
"
date
"
:
...
...
src/adapter/postgres.ts
View file @
2ffe7d55
...
...
@@ -20,6 +20,7 @@
import
{
SQLAdapter
}
from
"
./sql
"
;
import
{
View
}
from
"
../core/view
"
;
import
{
Source
}
from
"
../core/source
"
;
import
{
FilterOperator
}
from
"
../core/filter
"
;
import
{
Pool
,
PoolConfig
}
from
"
pg
"
;
...
...
@@ -32,7 +33,10 @@ export class PostgresAdapter extends SQLAdapter {
}
public
getDataFromView
(
view
:
View
,
cb
:
(
error
:
Error
,
result
?:
any
[])
=>
void
):
void
{
const
query
=
this
.
getQueryFromView
(
view
);
this
.
executeQuery
(
query
,
cb
);
}
private
executeQuery
(
query
:
string
,
cb
:
(
err
:
Error
,
result
?:
any
[])
=>
void
):
void
{
this
.
pool
.
connect
((
err
,
client
,
done
)
=>
{
if
(
err
)
{
cb
(
err
);
...
...
@@ -45,6 +49,10 @@ export class PostgresAdapter extends SQLAdapter {
});
});
}
public
insertIntoSource
(
source
:
Source
,
data
:
any
[],
cb
:
(
err
:
Error
,
result
?:
any
[])
=>
void
):
void
{
const
query
=
this
.
getQueryFromSource
(
source
,
data
);
this
.
executeQuery
(
query
,
cb
);
}
public
materializeView
(
view
:
View
):
boolean
{
return
false
;
...
...
src/adapter/sql.ts
View file @
2ffe7d55
...
...
@@ -20,6 +20,7 @@
import
{
Adapter
}
from
"
../core/adapter
"
;
import
{
Metric
}
from
"
../core/metric
"
;
import
{
Source
}
from
"
../core/source
"
;
import
{
Dimension
}
from
"
../core/dimension
"
;
import
{
Clause
}
from
"
../core/clause
"
;
import
{
Filter
,
FilterOperator
}
from
"
../core/filter
"
;
...
...
@@ -844,4 +845,22 @@ export abstract class SQLAdapter extends Adapter {
}
return
dims
;
}
public
getQueryFromSource
(
source
:
Source
,
data
:
any
[]):
string
{
let
consult
:
string
;
let
colums
:
any
[]
=
[];
let
values
:
string
[]
=
[];
let
tam
=
Object
.
keys
(
data
).
length
;
colums
=
source
.
fields
.
map
(
o
=>
o
.
name
);
for
(
let
i
=
0
;
i
<
tam
;
i
++
){
values
[
i
]
=
data
[
colums
[
i
]];
}
consult
=
"
INSERT INTO
"
+
source
.
name
+
"
(
\"
"
;
consult
=
consult
.
concat
(
colums
.
join
(
"
\"
,
\"
"
));
consult
=
consult
.
concat
(
"
\"
) VALUES ('
"
);
consult
=
consult
.
concat
(
values
.
join
(
"
' , '
"
));
consult
=
consult
.
concat
(
"
');
"
);
return
consult
;
}
}
src/api/controllers/collect.spec.ts
View file @
2ffe7d55
This diff is collapsed.
Click to expand it.
src/api/controllers/collect.ts
View file @
2ffe7d55
...
...
@@ -19,16 +19,152 @@
*/
import
*
as
express
from
"
express
"
;
import
{
Request
}
from
"
../types
"
;
import
{
Source
,
Field
}
from
"
../../core/source
"
;
import
{
EnumType
}
from
"
../../core/enumType
"
;
interface
Valid
{
[
key
:
string
]:
(
value
:
any
)
=>
boolean
;
}
export
class
CollectCtrl
{
public
static
write
(
req
:
express
.
Request
,
res
:
express
.
Response
,
next
:
express
.
NextFunction
)
{
if
(
"
_id
"
in
req
.
body
)
{
res
.
status
(
400
)
.
json
({
message
:
"
Property named
\"
_id
\"
is protected.
"
});
public
static
write
(
req
:
Request
,
res
:
express
.
Response
,
next
:
express
.
NextFunction
)
{
const
validador
:
Valid
=
{
"
integer
"
:
function
(
value
:
any
)
{
let
test
=
parseInt
(
value
,
10
);
if
(
test
===
undefined
||
isNaN
(
value
)
)
{
return
false
;
}
else
{
return
true
;
}
},
"
float
"
:
function
(
value
:
any
)
{
let
test
=
parseFloat
(
value
);
if
(
test
===
undefined
||
isNaN
(
value
)
)
{
return
false
;
}
else
{
return
true
;
}
},
"
string
"
:
function
(
value
:
any
)
{
let
test
=
typeof
(
value
);
if
(
test
===
"
string
"
)
{
return
true
;
}
else
{
return
false
;
}
},
"
date
"
:
function
(
value
:
any
)
{
let
test
:
string
[]
=
[];
let
date
=
new
Date
(
value
);
try
{
test
=
date
.
toISOString
().
split
(
"
T
"
);
}
catch
(
e
)
{
return
false
;
}
if
(
test
[
0
]
===
value
){
return
true
;
}
else
{
return
false
;
}
},
"
boolean
"
:
function
(
value
:
any
)
{
let
test
:
string
=
value
;
test
=
test
.
toLocaleLowerCase
();
if
(
test
===
"
true
"
||
test
===
"
false
"
){
return
true
;
}
else
{
return
false
;
}
}
};
let
fields
:
Field
[]
=
[];
let
data
:
string
[]
=
[];
let
types
:
string
[]
=
[];
let
source
:
Source
;
let
enumType
:
EnumType
;
const
id
:
string
=
req
.
params
.
class
;
try
{
source
=
req
.
engine
.
getSourceByName
(
id
);
// If source does not exist them return error
fields
=
source
.
fields
;
for
(
let
i
=
0
;
i
<
fields
.
length
;
i
++
){
data
[
i
]
=
req
.
body
[
fields
[
i
].
name
];
if
(
!
data
[
i
]){
throw
new
Error
(
"
The '
"
+
fields
[
i
].
name
+
"
' wasn't informed on json
"
);
}
}
for
(
let
i
=
0
;
i
<
fields
.
length
;
i
++
){
if
(
validador
[
fields
[
i
].
dataType
]
!==
undefined
){
if
(
!
validador
[
fields
[
i
].
dataType
](
data
[
i
])
===
true
){
throw
new
Error
(
"
The value '
"
+
data
[
i
]
+
"
' from '
"
+
fields
[
i
].
name
+
"
' isn't a type
"
+
fields
[
i
].
dataType
);
}
}
else
{
enumType
=
req
.
engine
.
getEnumTypeByName
(
fields
[
i
].
dataType
);
types
=
enumType
.
values
;
let
found
:
boolean
=
false
;
for
(
let
j
=
0
;
j
<
types
.
length
;
j
++
){
if
(
data
[
i
]
===
types
[
j
]){
found
=
true
;
break
;
}
}
if
(
!
found
)
{
throw
new
Error
(
"
The value '
"
+
data
[
i
]
+
"
' from '
"
+
fields
[
i
].
name
+
"
' isn't listed on
"
+
fields
[
i
].
dataType
);
}
}
}
}
catch
(
e
)
{
res
.
status
(
500
).
json
({
message
:
"
Query execution failed:
"
+
"
Could not construct query with the paramters given.
"
,
error
:
e
.
message
});
return
;
}
res
.
status
(
500
)
.
json
({
message
:
"
Error while writing to the database.
"
});
req
.
adapter
.
insertIntoSource
(
source
,
req
.
body
,
(
err
:
Error
,
result
:
any
[])
=>
{
if
(
err
)
{
res
.
status
(
500
).
json
({
message
:
"
Insertion has failed
"
,
error
:
err
});
return
;
}
else
{
res
.
status
(
200
).
json
({
message
:
"
Data has been successfully received and stored by the server
"
});
return
;
}
});
}
}
src/api/controllers/engine.spec.ts
View file @
2ffe7d55
...
...
@@ -43,7 +43,7 @@ describe("API engine controller", () => {
.
expect
((
res
:
any
)
=>
{
let
result
=
res
.
body
;
expect
(
result
).
to
.
be
.
an
(
"
array
"
);
expect
(
result
).
to
.
have
.
length
(
4
);
expect
(
result
).
to
.
have
.
length
(
5
);
})
.
end
(
done
);
});
...
...
src/core/adapter.ts
View file @
2ffe7d55
...
...
@@ -19,8 +19,10 @@
*/
import
{
View
}
from
"
./view
"
;
import
{
Source
}
from
"
./source
"
;
export
abstract
class
Adapter
{
public
abstract
getDataFromView
(
view
:
View
,
cb
:
(
err
:
Error
,
result
:
any
[])
=>
void
):
void
;
public
abstract
materializeView
(
view
:
View
):
boolean
;
public
abstract
insertIntoSource
(
source
:
Source
,
data
:
any
[],
cb
:
(
err
:
Error
,
result
:
any
[])
=>
void
):
void
;
}
src/core/engine.ts
View file @
2ffe7d55
...
...
@@ -103,6 +103,26 @@ export class Engine {
return
result
;
}
public
getEnumTypeByName
(
name
:
string
):
EnumType
{
let
result
=
this
.
enumTypes
.
find
(
EnumType
=>
EnumType
.
name
===
name
);
if
(
!
result
)
{
throw
new
Error
(
"
The dataType named '
"
+
name
+
"
' was not found
"
);
}
return
result
;
}
public
getSourceByName
(
name
:
string
):
Source
{
let
result
=
this
.
sources
.
find
(
source
=>
source
.
name
===
name
);
if
(
!
result
)
{
throw
new
Error
(
"
The source named '
"
+
name
+
"
' was not found
"
);
}
return
result
;
}
public
addDimension
(
dimension
:
Dimension
):
Dimension
{
if
(
this
.
graph
.
addDimension
(
dimension
))
{
this
.
dimensions
.
push
(
dimension
);
...
...
src/util/configParser.ts
View file @
2ffe7d55
...
...
@@ -105,7 +105,7 @@ export class ConfigParser {
public
static
parse
(
configPath
:
string
):
ParsedConfig
{
let
config
:
ConfigSchema
=
yaml
.
safeLoad
(
fs
.
readFileSync
(
configPath
,
{
encoding
:
"
utf-8
"
}));
}))
as
ConfigSchema
;
let
connection
:
Connection
=
{
user
:
process
.
env
.
BLENDB_DB_USER
,
...
...
@@ -212,6 +212,7 @@ export class ConfigParser {
}
return
parsed
;
}
public
static
parseViewOpt
(
opts
:
ViewParsingOptions
,
...
...
test/postgres/fixture.ts
View file @
2ffe7d55
...
...
@@ -20,6 +20,7 @@
import
{
Client
,
PoolConfig
}
from
"
pg
"
;
import
{
View
,
LoadView
}
from
"
../../src/core/view
"
;
import
{
Source
}
from
"
../../src/core/source
"
;
import
{
each
,
series
}
from
"
async
"
;
import
*
as
fs
from
"
fs
"
;
...
...
@@ -28,6 +29,12 @@ interface TransSet {
data
:
string
[];
}
interface
SoucerParse
{
name
:
string
;
type
:
string
[];
fields
:
string
[];
}
export
interface
Schema
{
alias
?:
string
;
query
?:
string
;
...
...
@@ -107,7 +114,7 @@ export class Fixture {
case
"
boolean
"
:
return
"
BOOLEAN
"
;
default
:
return
""
;
return
"
TEXT
"
;
}
}
...
...
@@ -148,4 +155,59 @@ export class Fixture {
}
return
transaction
;
}
public
LoadSource
(
source
:
Source
[],
create
:
boolean
,
cb
:
(
err
:
Error
)
=>
void
):
void
{
let
client
=
new
Client
(
this
.
config
);
let
query
:
string
[]
=
[];
for
(
let
i
=
0
;
i
<
source
.
length
;
i
++
){
query
[
i
]
=
this
.
ExtractData
(
source
[
i
],
create
);
}
client
.
connect
((
error
)
=>
{
if
(
error
)
{
cb
(
error
);
return
;
}
series
([(
callback
:
(
err
:
Error
)
=>
void
)
=>
{
each
(
query
,
(
insere
,
cback
)
=>
{
return
client
.
query
(
insere
,
[],
(
err
:
Error
)
=>
cback
(
err
));
},
(
errQuery
:
Error
)
=>
callback
(
errQuery
));
}],
(
errQuery
:
Error
)
=>
{
if
(
errQuery
)
{
client
.
end
();
cb
(
errQuery
);
}
client
.
end
((
err
)
=>
{
cb
(
err
);
});
});
});
}
private
ExtractData
(
data
:
Source
,
create
:
boolean
):
string
{
let
name
:
string
;
let
type
:
string
[];
let
fields
:
string
[];
let
consult
:
string
;
name
=
data
.
name
;
type
=
data
.
fields
.
map
((
item
)
=>
item
.
dataType
);
fields
=
data
.
fields
.
map
((
item
)
=>
item
.
name
);
if
(
create
){
consult
=
"
CREATE TABLE
"
+
name
+
"
(
"
+
'
"
'
;
for
(
let
i
=
0
;
i
<
fields
.
length
;
i
++
){
fields
[
i
]
=
fields
[
i
].
concat
(
'
"
'
+
"
"
+
this
.
typeConvertion
(
type
[
i
]));
}
consult
=
consult
.
concat
(
fields
.
join
(
"
,
"
+
'
"
'
));
consult
=
consult
.
concat
(
"
);
"
);
}
else
{
consult
=
"
TRUNCATE TABLE
"
+
name
+
"
;
"
;
}
return
consult
;
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment