Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
blendb
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
22
Issues
22
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
C3SL
blendb
Commits
383e79db
Commit
383e79db
authored
Aug 17, 2016
by
Eduardo L. Buratti
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Refactor core module
parent
23c8b800
Changes
7
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
304 additions
and
190 deletions
+304
-190
src/blendb.js
src/blendb.js
+97
-0
src/core/aggregate.js
src/core/aggregate.js
+40
-0
src/core/aggregator.js
src/core/aggregator.js
+0
-188
src/core/source.js
src/core/source.js
+39
-0
src/core/transformer.js
src/core/transformer.js
+43
-0
src/util/hash.js
src/util/hash.js
+19
-2
test.js
test.js
+66
-0
No files found.
src/blendb.js
0 → 100644
View file @
383e79db
/*
* Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
*
* blend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
'
use strict
'
;
const
hash
=
require
(
'
util/hash
'
);
const
Source
=
require
(
'
core/source
'
);
const
Transformer
=
require
(
'
core/transformer
'
);
const
Aggregate
=
require
(
'
core/aggregate
'
);
class
BlenDB
{
constructor
()
{
this
.
sources
=
new
Map
();
this
.
transformers
=
new
Map
();
this
.
aggregates
=
new
Map
();
}
source
(
name
,
options
)
{
if
(
this
.
sources
.
has
(
name
))
{
return
this
.
sources
.
get
(
name
);
}
else
{
const
source
=
new
Source
(
name
,
options
);
this
.
sources
.
set
(
name
,
source
);
return
source
;
}
}
transformer
(
name
,
options
)
{
if
(
this
.
transformers
.
has
(
name
))
{
return
this
.
transformers
.
get
(
name
);
}
else
{
const
transformer
=
new
Transformer
(
name
,
options
);
this
.
transformers
.
set
(
name
,
transformer
);
return
transformer
;
}
}
aggregate
(
metrics
,
dimensions
,
options
)
{
metrics
=
Array
.
from
(
metrics
);
dimensions
=
Array
.
from
(
dimensions
);
const
id
=
hash
.
sha1
(
metrics
.
sort
()
+
dimensions
.
sort
());
if
(
this
.
aggregates
.
has
(
id
))
{
return
this
.
aggregates
.
get
(
id
);
}
else
{
const
aggregate
=
new
Aggregate
(
metrics
,
dimensions
,
options
);
this
.
aggregates
.
set
(
id
,
aggregate
);
return
aggregate
;
}
}
process
()
{
this
.
transformers
.
forEach
((
transformer
)
=>
{
const
source
=
this
.
source
(
transformer
.
source
);
const
aggr
=
this
.
aggregate
(
transformer
.
metrics
,
transformer
.
dimensions
);
source
.
forEach
((
doc
)
=>
{
aggr
.
push
({
metrics
:
transformer
.
extractMetrics
(
doc
),
dimensions
:
transformer
.
extractDimensions
(
doc
)
});
});
// TODO: stream support
// source.stream()
// .pipe(transformer.stream());
// .pipe(aggregate.stream());
});
console
.
log
(
this
.
aggregates
);
}
}
module
.
exports
=
{
BlenDB
,
Source
,
Transformer
};
src/
util/serializer
.js
→
src/
core/aggregate
.js
View file @
383e79db
...
...
@@ -20,39 +20,21 @@
'
use strict
'
;
class
Serializer
{
dump
(
obj
)
{
return
JSON
.
stringify
(
obj
,
(
key
,
value
)
=>
{
if
(
typeof
value
===
'
function
'
)
{
return
value
.
toString
()
.
replace
(
/
[\n\r\t]
/g
,
''
)
.
replace
(
/ +/g
,
'
'
);
}
class
Aggregate
{
constructor
(
metrics
,
dimensions
,
options
)
{
this
.
metrics
=
metrics
;
this
.
dimensions
=
dimensions
;
return
value
;
});
this
.
data
=
[];
}
load
(
str
)
{
return
JSON
.
parse
(
str
,
(
key
,
value
)
=>
{
if
(
key
===
''
)
{
return
value
;
}
if
(
typeof
value
===
'
string
'
)
{
let
rfunc
=
/function
[^\(]
*
\(([^\)]
*
)\)[^\{]
*
\{(
.*
)\}[^\}]
*$/
;
let
match
=
value
.
replace
(
/
\n
/g
,
''
).
match
(
rfunc
);
if
(
match
)
{
let
args
=
match
[
1
].
split
(
'
,
'
)
.
map
((
arg
)
=>
arg
.
replace
(
/
\s
+/
,
''
));
return
new
Function
(
args
,
match
[
2
]);
}
}
push
(
data
)
{
this
.
data
.
push
(
data
);
}
return
value
;
})
;
truncate
()
{
this
.
data
=
[]
;
}
}
module
.
exports
=
new
Serializer
()
;
module
.
exports
=
Aggregate
;
src/core/aggregator.js
deleted
100644 → 0
View file @
23c8b800
/*
* Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
'
use strict
'
;
const
async
=
require
(
'
async
'
);
const
mongo
=
require
(
'
core/mongo
'
);
const
serializer
=
require
(
'
util/serializer
'
);
const
hash
=
require
(
'
util/hash
'
);
class
Aggregator
{
removeAggregate
(
id
,
callback
)
{
const
aggregates
=
mongo
.
db
.
collection
(
'
meta.aggregates
'
);
aggregates
.
findOneAndDelete
({
_id
:
id
},
(
err
)
=>
{
if
(
err
)
{
callback
(
err
);
return
;
}
let
aggr
=
mongo
.
db
.
collection
(
'
aggr.
'
+
id
);
aggr
.
remove
({},
callback
);
});
}
createAggregate
(
dimensions
,
metrics
,
callback
)
{
const
aggregates
=
mongo
.
db
.
collection
(
'
meta.aggregates
'
);
let
doc
=
{
_id
:
hash
.
sha1
(
serializer
.
dump
(
dimensions
)
+
serializer
.
dump
(
metrics
)),
dimensions
:
dimensions
,
metrics
:
metrics
};
aggregates
.
insert
(
doc
,
(
err
)
=>
{
if
(
err
)
{
callback
(
err
);
return
;
}
callback
(
null
,
doc
);
});
}
rebuildBaseAggregates
(
callback
)
{
this
.
cleanAggregates
((
err
)
=>
{
if
(
err
)
{
callback
(
err
);
return
;
}
this
.
buildBaseAggregates
(
callback
);
});
}
buildBaseAggregates
(
callback
)
{
let
classes
=
mongo
.
db
.
collection
(
'
meta.classes
'
);
classes
.
find
({}).
toArray
((
err
,
result
)
=>
{
if
(
err
)
{
callback
(
err
);
return
;
}
async
.
map
(
result
,
(
cls
,
cb
)
=>
{
this
.
buildBaseAggregateFromClass
(
cls
,
cb
);
},
(
err
)
=>
{
if
(
err
)
{
callback
(
err
);
return
;
}
return
callback
(
null
);
});
});
}
buildBaseAggregateFromClass
(
cls
,
callback
)
{
this
.
createAggregate
(
cls
.
dimensions
,
cls
.
metrics
,
(
err
,
aggr
)
=>
{
const
raw
=
mongo
.
db
.
collection
(
'
raw.
'
+
cls
.
name
);
const
aggrData
=
mongo
.
db
.
collection
(
'
aggr.
'
+
aggr
.
_id
);
const
functions
=
serializer
.
load
(
cls
.
functions
);
raw
.
find
({}).
forEach
((
doc
)
=>
{
let
data
=
{
dimensions
:
functions
.
extractDimensions
.
apply
(
doc
),
metrics
:
functions
.
extractMetrics
.
apply
(
doc
)
};
// TODO: aggrData.insert(data);
},
callback
);
});
}
cleanAggregates
(
callback
)
{
let
aggregates
=
mongo
.
db
.
collection
(
'
meta.aggregates
'
);
aggregates
.
find
({}).
toArray
((
err
,
result
)
=>
{
if
(
err
)
{
callback
(
err
);
return
;
}
async
.
map
(
result
,
(
aggr
,
callback
)
=>
{
let
aggrCol
=
mongo
.
db
.
collection
(
'
aggr.
'
+
aggr
.
name
);
aggrCol
.
remove
({},
(
err
)
=>
{
if
(
err
)
{
callback
(
err
);
return
;
}
aggregates
.
remove
({
_id
:
aggr
.
_id
},
callback
);
});
},
(
err
)
=>
{
if
(
err
)
{
callback
(
err
);
return
;
}
callback
(
null
);
});
});
}
query
(
metrics
,
dimensions
,
callback
)
{
this
.
findClosestAggregate
(
metrics
,
dimensions
,
(
err
,
aggr
)
=>
{
if
(
err
)
{
callback
(
err
);
return
;
}
callback
(
null
,
null
);
});
}
findClosestAggregate
(
metrics
,
dimensions
,
callback
)
{
let
aggregates
=
mongo
.
db
.
collection
(
'
meta.aggregates
'
);
aggregates
.
find
({
metrics
:
{
$all
:
metrics
},
dimensions
:
{
$all
:
dimensions
}
}).
toArray
((
err
,
result
)
=>
{
if
(
err
)
{
callback
(
err
);
return
;
}
if
((
!
result
)
||
(
result
.
length
<=
0
))
{
callback
(
'
Query could not be aswered, no aggregate available.
'
);
return
;
}
// fetch the closest aggregation available
let
closestAggr
;
for
(
const
aggr
of
result
)
{
if
((
!
closestAggr
)
||
(
aggr
.
dimensions
.
length
<
closestAggr
.
dimensions
.
length
))
{
closestAggr
=
aggr
;
}
}
callback
(
null
,
closestAggr
);
});
}
}
module
.
exports
=
new
Aggregator
();
src/core/source.js
0 → 100644
View file @
383e79db
/*
* Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
'
use strict
'
;
class
Source
{
constructor
(
name
,
options
)
{
this
.
name
=
name
;
this
.
data
=
[];
}
push
(
doc
)
{
this
.
data
.
push
(
doc
);
}
forEach
(
callback
)
{
this
.
data
.
forEach
(
callback
);
}
}
module
.
exports
=
Source
;
src/core/transformer.js
0 → 100644
View file @
383e79db
/*
* Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
'
use strict
'
;
class
Transformer
{
constructor
(
name
,
options
)
{
this
.
source
=
options
.
source
||
null
;
this
.
metrics
=
options
.
metrics
||
[];
this
.
dimensions
=
options
.
dimensions
||
[];
this
.
extractors
=
{
metrics
:
options
.
extractors
.
metrics
||
((
doc
)
=>
null
),
dimensions
:
options
.
extractors
.
dimensions
||
((
doc
)
=>
null
)
};
}
extractMetrics
(
doc
)
{
return
this
.
extractors
.
metrics
(
doc
);
}
extractDimensions
(
doc
)
{
return
this
.
extractors
.
dimensions
(
doc
);
}
}
module
.
exports
=
Transformer
;
src/util/hash.js
View file @
383e79db
...
...
@@ -23,9 +23,26 @@
const
crypto
=
require
(
'
crypto
'
);
class
Hash
{
sha1
(
content
)
{
sha1
(
...
objects
)
{
let
hash
=
crypto
.
createHash
(
'
sha1
'
);
hash
.
update
(
content
);
objects
.
map
((
obj
)
=>
{
switch
(
typeof
obj
)
{
case
'
string
'
:
return
obj
;
case
'
object
'
:
return
JSON
.
stringify
(
obj
);
default
:
throw
new
TypeError
(
typeof
obj
+
'
cannot be hashed
'
);
}
})
.
sort
()
.
map
((
objStr
)
=>
{
hash
.
update
(
objStr
);
});
return
hash
.
digest
(
'
hex
'
);
}
}
...
...
test.js
0 → 100644
View file @
383e79db
#!/usr/bin/env node
/*
* Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blendb.
*
* blendb is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* blendb is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
'
use strict
'
;
// Add the ./src directory to require's search path to facilitate import
// modules later on (avoiding the require('../../../../module') problem).
require
(
'
app-module-path
'
).
addPath
(
__dirname
+
'
/src
'
);
// connect to mongodb
// const mongo = require('core/mongo');
// mongo.connect('mongodb://pyke/blend');
const
blendb
=
require
(
'
blendb
'
);
const
db
=
new
blendb
.
BlenDB
();
const
netSource
=
db
.
source
(
'
networkData
'
);
for
(
let
i
=
0
;
i
<
100
;
i
++
)
{
netSource
.
push
({
a
:
i
});
}
db
.
transformer
(
'
networkTraffic
'
,
{
source
:
'
networkData
'
,
metrics
:
[
'
met:downBytes
'
,
'
met:upBytes
'
],
dimensions
:
[
'
dim:date
'
,
'
dim:city
'
,
'
dim:state
'
,
'
dim:point
'
],
extractors
:
{
metrics
:
function
extractMetrics
(
doc
)
{
return
{
'
met:downBytes
'
:
5464
,
'
met:upBytes
'
:
342
};
},
dimensions
:
function
extractDimensions
(
doc
)
{
return
{
'
dim:date
'
:
'
2016/06/12
'
,
'
dim:city
'
:
41442
,
'
dim:state
'
:
41
,
'
dim:point
'
:
5344
};
}
}
});
db
.
process
();
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment