Commit fa0142b6 authored by Lucas Fernandes de Oliveira's avatar Lucas Fernandes de Oliveira

Merge branch 'issue/84' into 'develop'

Issue #84: Add code docs in Typedoc format

See merge request !66
parents 4339c078 433595a1
Pipeline #15848 passed with stages
in 52 seconds
......@@ -20,3 +20,4 @@
/database/views
/service
schema.sql
/doc/code
......@@ -10,7 +10,8 @@
"show-coverage": "xdg-open coverage/lcov-report/index.html",
"doc-api": "raml2html -i specs/blendb-api-v1.raml -o doc/api-v1-reference.html",
"schema": "env $(cat config/config.env ) ts-node scripts/schema.ts config/config.yaml schema.sql",
"service": "./scripts/service.sh"
"service": "./scripts/service.sh",
"doc-code": "typedoc --mode 'file' --module 'commonjs' --target 'ES6' --ignoreCompilerErrors --exclude '**/*.spec.ts' --out 'doc/code' 'src'"
},
"repository": {
"type": "git",
......@@ -42,6 +43,7 @@
"raml2html": "^3.0.1",
"supertest": "^3.0.0",
"tslint": "^3.15.1",
"tslint-stylish": "^2.1.0-beta"
"tslint-stylish": "^2.1.0-beta",
"typedoc": "^0.11.1"
}
}
......@@ -23,33 +23,73 @@ import { View } from "../core/view";
import { Source } from "../core/source";
import { FilterOperator } from "../core/filter";
import { DataType } from "../common/types";
/** @hidden */
const MDB = require("monetdb")();
/**
* Params required to connect with a MonetDB database and
* to create a MonetAdapter object.
*/
export interface MonetConfig {
/** Database hostname */
host: string;
/** Database port */
port: number;
/** Database name */
dbname: string;
/** Database user */
user: string;
/** Datavase password */
password: string;
}
/**
* Represent the data format returned by MonetDB.
* This interface is used to parse this format to the BlenDB Standart.
*/
interface MonetResult {
/** Query result as a list of values */
data: any[];
/** Meta data of each attribute required. */
structure: {type: string, column: string, index: number}[];
}
/**
* Adapter which connects with a MonetDB database.
*/
export class MonetAdapter extends SQLAdapter {
/** Information used to connect with a MonetDB database. */
private config: MonetConfig;
/**
* Creates a new adapter with the database connection configuration.
* @param conf - The information required to create a connection with
* the database.
*/
constructor (conf: MonetConfig) {
super();
this.config = conf;
}
/**
* Asynchronously reads all data from given view.
* In other words perform a SELECT query.
* @param view - "Location" from all data should be read.
* @param cb - Callback function which contains the data read.
* @param cb.error - Error information when the method fails.
* @param cb.result - Data got from view.
*/
public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void {
const query = this.getQueryFromView(view);
this.executeQuery(query, cb);
}
/**
* Asynchronously executes a query and get its result.
* @param query - Query (SQL format) to be executed.
* @param cb - Callback function which contains the data read.
* @param cb.error - Error information when the method fails.
* @param cb.result - Query result.
*/
private executeQuery(query: string, cb: (error: Error, result?: any[]) => void): void {
let pool: any = new MDB(this.config);
pool.connect();
......@@ -79,15 +119,31 @@ export class MonetAdapter extends SQLAdapter {
pool.close();
}
/**
* Materialize a given view.
* @param view - View to be materialized.
*/
public materializeView(view: View): boolean {
return false;
}
/**
* Asynchronously insert one register into a given Source.
* @param source - Insertion "location".
* @param cb - Callback function which contains the query result.
* @param cb.error - Error information when the method fails.
* @param cb.result - Query result.
*/
public insertIntoSource(source: Source, data: any[], cb: (err: Error, result: any[]) => void): void {
const query = this.getQueryFromSource(source, data);
this.executeQuery(query, cb);
}
/**
* Cast BlenDB data types to be used in MonetDB queries.
* @param quotedValue - SQL query attribute wrapped by quotes.
* @param dt - Attribute data type.
*/
protected typeCast(quotedValue: string, dt: DataType): string {
switch (dt) {
case DataType.DATE:
......@@ -101,6 +157,12 @@ export class MonetAdapter extends SQLAdapter {
}
}
/**
* Translate filter operator to be used in MonetDB queries.
* @param lSide - Operation left side operator.
* @param rSide - Operation right side operator.
* @param op - Operation to be performed.
*/
protected applyOperator(lSide: string, rSide: string, op: FilterOperator): string {
switch (op) {
case FilterOperator.EQUAL:
......
......@@ -25,18 +25,41 @@ import { FilterOperator } from "../core/filter";
import { Pool, PoolConfig } from "pg";
import { DataType } from "../common/types";
/** Adapter which connects with a PostgreSQL database. */
export class PostgresAdapter extends SQLAdapter {
/** Information used to connect with a PostgreSQL database. */
private pool: Pool;
/**
* Creates a new adapter with the database connection configuration.
* @param config - The information required to create a connection with
* the database.
*/
constructor (config: PoolConfig) {
super();
this.pool = new Pool(config);
}
/**
* Asynchronously reads all data from given view.
* In other words perform a SELECT query.
* @param view - "Location" from all data should be read.
* @param cb - Callback function which contains the data read.
* @param cb.error - Error information when the method fails.
* @param cb.result - Data got from view.
*/
public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void {
const query = this.getQueryFromView(view);
this.executeQuery(query, cb);
}
/**
* Asynchronously executes a query and get its result.
* @param query - Query (SQL format) to be executed.
* @param cb - Callback function which contains the data read.
* @param cb.error - Error information when the method fails.
* @param cb.result - Query result.
*/
private executeQuery(query: string, cb: (err: Error, result?: any[]) => void): void{
this.pool.connect((err, client, done) => {
if (err) {
......@@ -50,15 +73,32 @@ export class PostgresAdapter extends SQLAdapter {
});
});
}
/**
* Asynchronously insert one register into a given Source.
* @param source - Insertion "location".
* @param cb - Callback function which contains the query result.
* @param cb.error - Error information when the method fails.
* @param cb.result - Query result.
*/
public insertIntoSource(source: Source, data: any[], cb: (err: Error, result?: any[]) => void): void {
const query = this.getQueryFromSource(source, data);
this.executeQuery(query, cb);
}
/**
* Materialize a given view.
* @param view - View to be materialized.
*/
public materializeView(view: View): boolean {
return false;
}
/**
* Cast BlenDB data types to be used in PostgreSQL queries.
* @param quotedValue - SQL query attribute wrapped by quotes.
* @param dt - Attribute data type.
*/
protected typeCast(quotedValue: string, dt: DataType): string {
switch (dt) {
case DataType.DATE:
......@@ -72,6 +112,12 @@ export class PostgresAdapter extends SQLAdapter {
}
}
/**
* Translate filter operator to be used in PostgreSQL queries.
* @param lSide - Operation left side operator.
* @param rSide - Operation right side operator.
* @param op - Operation to be performed.
*/
protected applyOperator(lSide: string, rSide: string, op: FilterOperator): string {
switch (op) {
case FilterOperator.EQUAL:
......
......@@ -28,29 +28,67 @@ import { AggregationType, RelationType, DataType } from "../common/types";
import { Operation, Opcode } from "../common/expression";
import { View } from "../core/view";
/**
* Information required to make a join clause.
* The dimension and views that contain this dimension.
*/
interface DimInfo {
/** Dimension object. */
dim: Dimension;
/** Set of views that contain the dimension. */
views: View[];
}
/**
* Several translations of a dimension that will be used in different
* parts of a query
*/
interface DimTranslation {
/** Translation with the AS clause. Used on SELECT clause. */
aliased: string;
/** Translation without the AS clause. USED on GROUP BY clause. */
noalias: string;
/** The alias of the dimension. The quoted name. */
alias: string;
/** Expansion of a sub dimension. */
expanded: boolean;
}
/** Translated view, the query which gets all data and its name */
interface QueryAndName {
/** Query that returns all the data of a view. */
query: string;
/** The view name. */
name: string;
}
/**
* Two Dictionaries, both indexed with a dimension name.
* Used to get the views where a dimension is.
* The dimensions also have the Dimension object inside.
*/
interface DimAndNameMap {
/** Dictionary indexed by dimension name that returns DimInfo */
dimensions: {[key: string]: DimInfo};
/**
* Dictionary indexed by dimension name that returns
* a view where this dimension is. Used as right side
* of a join clause.
*/
views: {[key: string]: View};
}
/**
* Generic SQL adapter.
* Contains most implementation to perform a SQL query.
* However, which database has its peculiarities, so this
* adapter must be extended to each SGBD.
*/
export abstract class SQLAdapter extends Adapter {
/**
* Translate a view to a SQL query.
* @param view - View to be translated.
*/
public getQueryFromView(view: View): string {
const partials = this.buildPartials(view).filter((i) => {
return i.query !== "";
......@@ -76,6 +114,13 @@ export abstract class SQLAdapter extends Adapter {
return withClause + "SELECT * FROM " + this.viewName(view) + sort + ";";
}
/**
* Translates a view to a SQL sub-query. The main diffence between
* this method and getQueryFromView is that a sub-query cannot have
* sorting properties. Also a sub-query is called a partial and
* partials can only have one metric.
* @param view - View which the partial will be built.
*/
private buildPartials(view: View): QueryAndName[] {
let op = view.operation;
let queue: View[] = op.values.map((i) => i);
......@@ -104,10 +149,19 @@ export abstract class SQLAdapter extends Adapter {
return output;
}
/**
* The translated name of a view into SQL databases.
* @param view - View which the name will be built.
*/
protected viewName(view: View): string {
return "view_" + view.id;
}
/**
* Constructs a query from a view based in a given operation
* @param op - Operation used to construct the view.
* @param view - View to be querified.
*/
private operation(op: Operation, view: View): string {
switch (op.opcode) {
case Opcode.REDUCE:
......@@ -121,6 +175,12 @@ export abstract class SQLAdapter extends Adapter {
}
}
/**
* Constructs a query to a expecific view, from a set of given partials.
* @param view - View to be built.
* @param partials - Partials (other views) required to built the view.
* @param isJoin - Used to check if join clauses must be added.
*/
private buildOperation(view: View, partials: View[], isJoin: boolean): string {
// Mapping, which views the metrics and dimensions are
const mapping = this.buildMaps(partials);
......@@ -183,6 +243,11 @@ export abstract class SQLAdapter extends Adapter {
return projection + sources + selection + grouping;
}
/**
* Constructs and returns the dictionaries which inform
* in which views a given dimension is.
* @param views - Set of views to be scored.
*/
private buildMaps(views: View[]): DimAndNameMap {
let dimMap: {[key: string]: DimInfo} = {};
let nameMap: {[key: string]: View} = {};
......@@ -218,6 +283,13 @@ export abstract class SQLAdapter extends Adapter {
};
}
/**
* Returns a set o clauses of the given view that is not fulfilled
* by any of the partials. In other words, clauses that has not been
* added to the query yet.
* @param view - View with all clauses.
* @param partials - Views with some of the clauses.
*/
private orphanClauses(view: View, partials: View[]): Clause[] {
let parentClauses: Clause[] = [];
for (let i = 0; i < partials.length; ++i) {
......@@ -228,6 +300,12 @@ export abstract class SQLAdapter extends Adapter {
return view.clauses.filter((i) => !parentClauses.some((j) => j.id === i.id));
}
/**
* Parse a BlenDB enumeration of agregation functions to a proper string
* to be used in the SQL query.
* @param aggrType - Aggregation function.
* @param origin - Will be applied in a view with origin flag ?
*/
private getAggregateFunction(aggrType: AggregationType, origin: boolean): string {
switch (aggrType) {
case AggregationType.SUM:
......@@ -246,6 +324,12 @@ export abstract class SQLAdapter extends Adapter {
}
/**
* Parse a BlenDB enumeration of relationships to a proper string
* to be used in the SQL query.
* @param relation - The realtion function that will be used.
* @param arg - The attribute that is the function argument.
*/
private translateRelation(relation: RelationType, arg: string): string {
switch (relation) {
case RelationType.DAY:
......@@ -260,6 +344,12 @@ export abstract class SQLAdapter extends Adapter {
}
/**
* Generic translate a SQL function to a query
* @param name - Function name.
* @param args - Function argument names.
* @param values - Function argument values.
*/
private applyRelation(name: string, args: string[], values: string[]): string {
/*
This adapter uses the concept of functions in Postgres to
......@@ -278,11 +368,22 @@ export abstract class SQLAdapter extends Adapter {
return name + "(" + args.map((item, idx) => item + values[idx]).join(",") + ")";
}
/**
* Add quotes and the proper view name to a attribute in a SQL query.
* @param item - Attribute to get the name quoted.
* @param id - View id used to build the view name.
*/
private buildColumn (item: Metric|Dimension, id: string): string {
const quotedName = "\"" + item.name + "\"";
return "view_" + id + "." + quotedName;
}
/**
* Translate Clause object into a SQL query condition.
* @param clause - Clause to be translated.
* @param map - Dictionary indexed bu attributes name
* containing its locations.
*/
private translateClause(clause: Clause, map: {[key: string]: View}): string {
const r = clause.filters.map((item) => {
return this.translateFilter(item, map);
......@@ -292,6 +393,12 @@ export abstract class SQLAdapter extends Adapter {
return r.join(" OR ");
}
/**
* Translate Filter object into a SQL query condition.
* @param filter - Filter to be translated.
* @param map - Dictionary indexed bu attributes name
* containing its locations.
*/
private translateFilter(filter: Filter, map: {[key: string]: View}): string {
if (!map[filter.target.name]) {
return "";
......@@ -304,6 +411,11 @@ export abstract class SQLAdapter extends Adapter {
return this.applyOperator(leftSide, castedValue, filter.operator);
}
/**
* Translate Metric object into a SQL query attribute.
* @param metric - Metric to be translated.
* @param view - View which contains the metric. Used to create the name
*/
private translateMetric(metric: Metric, view: View): string {
const func = this.getAggregateFunction(metric.aggregation, view.origin);
const quotedName = "\"" + metric.name + "\"";
......@@ -311,6 +423,13 @@ export abstract class SQLAdapter extends Adapter {
return extMetric + " AS " + quotedName;
}
/**
* Translate Dimension object into a SQL query attribute.
* @param dimension - Dimension to be translated.
* @param ancestor - Dimension ancestor found in the view
* @param view - View which contains the dimenion/ancestor.
* Used to create the name.
*/
private translateDimension(dimension: Dimension,
ancestor: Dimension,
view: View): DimTranslation {
......@@ -331,6 +450,11 @@ export abstract class SQLAdapter extends Adapter {
};
}
/**
* Generates a generic insertion query for one register.
* @param source - Data insertion "location".
* @param data - Data to be inserted.
*/
public getQueryFromSource(source: Source, data: any[]): string {
let consult: string;
let colums: any[] = [];
......@@ -349,7 +473,17 @@ export abstract class SQLAdapter extends Adapter {
return consult;
}
/**
* Parse a filter operation. Varies in each database.
* @param lSide - Operation left side operator.
* @param rSide - Operation right side operator.
* @param op - Operation to be performed.
*/
protected abstract applyOperator(leftSide: string, rightSide: string, op: FilterOperator): string;
/**
* Cast BlenDB types to proper database types.
* @param quotedValue - SQL query attribute wrapped by quotes.
* @param dt - Attribute data type.
*/
protected abstract typeCast(quotedValue: string, dt: DataType): string;
}
......@@ -24,11 +24,28 @@ import { Source, Field } from "../../core/source";
import { EnumType } from "../../core/enumType";
import { DataType } from "../../common/types";
/**
* Dictionary indexed by a type name which return a
* validation function that returns true if the
* objetct is a valid object of that type
* or false otherwise.
*/
interface Valid{
[key: string]: (value: any) => boolean;
}
/**
* Constroller responsable for collect part from the API. In other
* words, controller responsable for inserting data in BlenDB.
*/
export class CollectCtrl {
/**
* Route that validates and insert data.
* @param req - Object with request information
* @param res - Object used to create and send the response
* @param next - Call next middleware or controller. Not used but required
* by typescript definition of route.
*/
public static write(req: Request, res: express.Response, next: express.NextFunction) {
const validador: Valid = {
......
......@@ -22,7 +22,18 @@ import * as express from "express";
import { Request } from "../types";
import { Query } from "../../common/query";
/**
* Constroller responsable for data part from the API. In other
* words, controller responsable for reading data in BlenDB.
*/
export class DataCtrl {
/**
* Route that validates a query and returns the query data.
* @param req - Object with request information
* @param res - Object used to create and send the response
* @param next - Call next middleware or controller. Not used but required
* by typescript definition of route.
*/
public static read(req: Request, res: express.Response, next: express.NextFunction) {
let metrics = req.query.metrics.split(",").filter((item: string) => item !== "");
let dimensions = req.query.dimensions.split(",").filter((item: string) => item !== "");
......
......@@ -21,19 +21,52 @@
import * as express from "express";
import { Request } from "../types";
/**
* Constroller responsable for the static part from the API. In other
* words, controller responsable for return the meta data stored in the
* engine object that API users can use to create queries.
*/
export class EngineCtrl {
/**
* Route that returns the list of available metrics.
* @param req - Object with request information
* @param res - Object used to create and send the response
* @param next - Call next middleware or controller. Not used but required
* by typescript definition of route.
*/
public static metrics(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getMetricsDescription());
}
/**
* Route that returns the list of available dimensions.
* @param req - Object with request information
* @param res - Object used to create and send the response
* @param next - Call next middleware or controller. Not used but required
* by typescript definition of route.
*/
public static dimensions(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getDimensionsDescription());
}
/**
* Route that returns the list of available enumerable types.
* @param req - Object with request information
* @param res - Object used to create and send the response
* @param next - Call next middleware or controller. Not used but required
* by typescript definition of route.
*/
public static enumTypes(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getEnumTypesDescription());
}
/**
* Route that returns the list of available data sources for insertion.
* @param req - Object with request information
* @param res - Object used to create and send the response
* @param next - Call next middleware or controller. Not used but required
* by typescript definition of route.
*/
public static sources(req: Request, res: express.Response, next: express.NextFunction) {
res.status(200).json(req.engine.getSourcesDescription());
}
......
......@@ -25,6 +25,11 @@ import { MonetAdapter, MonetConfig } from "../../adapter/monet";
import { PoolConfig } from "pg";
import { Connection } from "../../util/configParser";
/**
* Creates a PostgreSQL adapter and middleware that
* inserts the adapter into the request objects.
* @param config - Parameters required to connect in database.
*/
export function PostgresMw(config: Connection): Middleware {
let parsedConfig: PoolConfig = {
user: config.user,
......@@ -43,6 +48,11 @@ export function PostgresMw(config: Connection): Middleware {
}
/**
* Creates a MonetDB adapter and middleware that
* inserts the adapter into the request objects.
* @param config - Parameters required to connect in database.
*/
export function MonetMw(config: Connection): Middleware {
let parsedConfig: MonetConfig = {
user: config.user,
......
......@@ -22,6 +22,11 @@ import { Engine } from "../../core/engine";
import { ParsedConfig } from "../../util/configParser";
import { Middleware } from "../types";
/**
* Creates a engine and middleware that
* inserts the engine into the request objects.
* @param config - Parsed database schema.
*/
export function EngineMw (config: ParsedConfig): Middleware {
let engine: Engine = new Engine();
......
......@@ -20,6 +20,10 @@
import * as express from "express";
/**
* Creates a middleware to handle errors proper to each API version.
* @param config - API version.
*/
export function ErrorMw(config: string): express.ErrorRequestHandler {
const handlers: { [key: string]: express.ErrorRequestHandler } = {
"v1": function(err, req, res, next) {
......
......@@ -18,6 +18,7 @@
* along with blendb. If not, see <http://www.gnu.org/licenses/>.
*/
/** @hidden */
const osprey = require("osprey");
// import controllers
......@@ -25,6 +26,7 @@ import { DataCtrl } from "./controllers/data";
import { CollectCtrl } from "./controllers/collect";
import { EngineCtrl } from "./controllers/engine";
/** @hidden */
export const router = osprey.Router();
router.get("/metrics", EngineCtrl.metrics);
......
......@@ -22,11 +22,26 @@ import * as express from "express";
import { Engine } from "../core/engine";
import { Adapter} from "../core/adapter";
/**
* Extension of Express requests that suports the addition
* of an engine and an adapter.
* This extension is required because some Middlewares
* add some objetcs, in this case metrics and dimensions,
* to the Request object. To typescript compiler do not
* return a error the extension must be made.
*/
export interface Request extends express.Request {
/** A engine object. Represents the database in BlenDB perspective. */
engine: Engine;
/** A adapter object. Used to communicate with the database in use. */
adapter: Adapter;
}
/**
* Extension Middleware function of ExpressJS Module.
* Uses the custom Request object and is used to define
* the middlewares of BlenDB API.
*/
export interface Middleware {
(req: Request, res: express.Response, next: express.NextFunction): void;
}
......@@ -20,13 +20,41 @@
import { View } from "../core/view";
/**
* Operation codes to each allowed operation over a view.
*/
export enum Opcode {
/**
* Push operation.
* This operation means that the view is materialized.
* This also means that there are no need to create it
* using other views.
* And also means that this view is a valid data location.
*/
PUSH,
/**
* Join operation.
* The view will be created using a set of others view.
* This operation is equivalent a INNER JOIN of SQL
* or as close as possible of JOIN.
*/
JOIN,
/**
* Reduce operation.
* The reduce operation removes irrelevant metrics and
* dimensions and apply clauses. Represents the Projection
* and Selecion operations of Relational Algebra.
*/
REDUCE
}
/**
* Defines how to construct a View with a operation
* and a set of chidren Views.
*/
export interface Operation {
/** Operation code used. */
opcode: Opcode;
/** Set of views required to perform the operation. */
values: View[];
}
......@@ -22,9 +22,16 @@ import { Metric } from "../core/metric";
import { Dimension } from "../core/dimension";
import {