...
 
Commits (92)
Steps of Creating the Database
==============================
1) Type
./create/create_bd.sh
Steps of Loading
1) Type
./load/c_load.sh
2) Type
./cron.sh
#######################################################################
# SEED2 DATABASE
#######################################################################
For create the database read "CREATE".
For load the database read "LOADING".
For new features and other news read "NEWS".
For credits read "AUTHORS".
For copyright information read "COPYING".
PROINFODATA Database
For credits, read "AUTHORS".
For copyright information, read "COPYING".
To create the database, execute:
./create/create_database.sh <database> <user> <password>
To grant permissions to a specific user to read most tables and insert data in
the staging area tables, execute:
./grant_permissions.sh <database> <user>
To load the database, execute:
./load/load.sh
To update the crontab to automatically run the load, execute:
./config_crontab.sh
To migrate data from the old database, execute:
./migrate/migrate_data.sh <old_database> <new_database>
1 - Acrescentar novos dispositivos no banco (uca, projetor, etc);
2 - Otimizar as funções de carregamento.
#!/bin/bash
# Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
# Copyright (C) 2009-2012 Centro de Computacao Cientifica e Software Livre
# Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
#
# This file is part of database
......@@ -19,7 +19,7 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
LOADFILE='/var/lib/postgresql/database/load/load.sh'
LOGFILE="/var/lib/postgresql/database/log/$(date +%d_%m_%y).log"
LOADFILE='/home/postgres/load/load.sh'
LOGFILE="/home/postgres/log/load_\$(date +\%F).log"
echo "59 23 * * * $LOADFILE > $LOGFILE" | crontab -
echo "@daily ${LOADFILE} > ${LOGFILE}" | crontab -
/* Copyright (C) 2009-2012 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
/* -------------------------------------------------------------------------- */
/* Enumerations */
comment on type category_enum is 'Enum type for the components category';
comment on type project_enum is 'Enum type for the MEC project which the '
'machines belong to';
comment on type class_enum is 'Enum type for the classification of the '
'machines according to their availability : green if communication occured '
'at most 10 days ago, yellow if it ocurred between 10 and 30, and red if '
'it has not occured for more than 30 days)';
/* -------------------------------------------------------------------------- */
/* Staging Areas */
comment on table sa_inventory is 'Transition table with temporary machines '
'inventory data';
comment on column sa_inventory.inep is 'Longest INEP has 10 characters';
comment on column sa_inventory.hd_model is 'Longest HD model has 47 characters';
comment on column sa_inventory.processor is 'Longest processor model has 46 '
'characters';
comment on column sa_inventory.os_distro is 'Longest OS distro has 35 '
'characters';
comment on column sa_inventory.kernel is 'Longest kernel version has 24'
'characters';
comment on table sa_net_usage is 'Transition table with temporary network '
'usage data per machine';
comment on column sa_net_usage.insertion_date is 'To avoid using lock during '
'the load';
/* -------------------------------------------------------------------------- */
/* Data Warehouse */
comment on table dim_date is 'Dimension table with dates representing each '
'load done';
comment on table dim_school is 'Dimension table with data about the schools. '
'Since this table is loaded apart from the rest of the tables, this is the '
'only one who would not need the load_date field';
comment on column dim_school.city is 'Longest city name has 32 characters';
comment on column dim_school.school is 'MEC database limit for school names is '
'100 characters';
comment on column dim_school.address is 'Longest address has 70 characters';
comment on table dim_component is 'Dimension table of software and hardware '
'components';
comment on column dim_component.description is 'HD model, OS type, processor '
'model and memory size';
comment on column dim_component.detail is 'HD size and OS distro';
comment on table dim_machine is 'Dimension table with machines inventory';
comment on table fact_inventory is 'Fact table with the history of machines '
'connections and their inventory';
comment on table fact_alert is 'Fact table with the history of machines '
'modifications';
comment on column fact_alert.hd_alert is 'First HD changes';
comment on table fact_net_usage is 'Fact table with the history of network '
'usage per machine and 5 minutes sample';
comment on table control is 'Control table with the starting and ending times '
'of each load process';
/* -------------------------------------------------------------------------- */
/* Data Marts */
comment on table dm_school is 'Data mart table with data about the schools '
'used by fast search';
comment on column dm_school.fts_school is 'Full text search field for schools';
comment on column dm_school.nud_avail is 'It indicates if the school has '
'already sent some network usage data';
comment on table dm_avail_state is 'Data mart with availability per state';
comment on table dm_avail_city is 'Data mart with availability per city';
comment on table dm_avail_school is 'Data mart with availability per school';
comment on table dm_avail_machine is 'Data mart with availability per machine';
comment on column dm_avail_machine.total_contacts is '89 years seems enough';
comment on column dm_avail_machine.days_last_contact is '89 years seems enough';
comment on table dm_avail_state_history is 'Data mart with the history '
'of availability per state for the last 6 months';
comment on table dm_avail_city_history is 'Data mart with the history of '
'availability per city for the last 6 months';
comment on table dm_invent_machine is 'Data mart with the machines inventory';
comment on table dm_audit_zm_school is 'Data mart with schools without any '
'machine communicating';
comment on table dm_alert_state is 'Data mart with alerts per state';
comment on table dm_alert_city is 'Data mart with alerts per city';
comment on table dm_alert_school is 'Data mart with alerts per school';
comment on table dm_alert_machine is 'Data mart with machines alerts';
comment on table dm_alert_state_history is 'Data mart with the history of '
'alerts per state for the last 6 months';
comment on table dm_alert_city_history is 'Data mart with the history of '
'alerts per city for the last 6 months';
comment on table dm_net_usage_monthly is 'Data mart with network usage '
'per school with 4 hours sample';
comment on table dm_net_usage_weekly is 'Data mart with network usage '
'per school with 1 hour sample';
comment on table dm_net_usage_daily is 'Data mart with network usage per '
'school with 5 minutes sample';
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
COMMENT ON TABLE mectb00_staging_area IS 'Transition table with temporary
data';
COMMENT ON TABLE mectb01_escola_dim IS 'Table with data from schools.';
COMMENT ON TABLE mectb02_catalogo_dim IS 'Catalog table of hardware
components.';
COMMENT ON TABLE mectb03_data_dim IS 'Table of dates.';
COMMENT ON TABLE mectb04_maquina_dim IS 'Table with data from machines.';
COMMENT ON TABLE mectb05_disponibilidade_fact IS 'Table with history of
machines connections.';
COMMENT ON TABLE mectb06_uso_fact IS 'Table using the network of machines.';
COMMENT ON TABLE mectb07_inventario_fact IS 'Inventory table of the
machines.';
COMMENT ON TABLE mectb08_dm_disp IS 'Classification table of the
availability of machines.';
COMMENT ON TABLE mectb09_dm_disp_ag IS 'Aggregation by city table
mectb08_dm_disp.';
COMMENT ON TABLE mectb10_dm_inventario IS 'Join table of inventory from
machines.';
COMMENT ON TABLE mectb11_dm_auditoria IS 'Table grouping by the School of
machines installed and expected.';
COMMENT ON TABLE mectb12_dm_instaladas_ag IS 'Table grouping by month and
state of the machines installed, provided, by changing the hard disk and
memory.';
COMMENT ON TABLE mectb13_dm_rede IS 'Table that stores per-school network
usage information.';
COMMENT ON TABLE mectb99_control IS 'Control table of consolidated
database.';
This diff is collapsed.
/* Copyright (C) 2009-2012 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
/* -------------------------------------------------------------------------- */
/* Dimension tables */
/* Dimension table with dates representing each load done */
create table dim_date (
id serial primary key,
load_date date unique not null
);
/* Dimension table with data about the schools. Since this table is loaded apart
* and "manually" from the rest of the tables, this is the only one who would not
* need the load_date field */
create table dim_school (
id serial primary key,
load_date integer references dim_date not null,
--last_date integer references dim_date not null,
inep text unique not null,
region text not null,
state text not null,
city text not null, /* Longest city name has 32 characters */
school text not null, /* MEC database limit for schools is 100 */
code text,
address text /* Longest address has 70 characters */
);
/* Dimension table of software and hardware components */
create table dim_component (
id serial primary key,
load_date integer references dim_date not null,
category category_enum not null,
description text not null, /* HD model, OS type, kernel and
* processor model */
detail text, /* HD size and OS distro */
unique(category, description, detail)
);
/* This index is necessary to treat null values as equals, thus not accepting
* repeated processor models and kernel data, since they have no detail, neither
* accepting HDs without size */
create unique index dim_c_constraint on dim_component (category, description)
where detail is null;
/* Dimension table with machines inventory */
create table dim_machine (
id serial primary key,
load_date integer references dim_date not null,
school_id integer references dim_school not null,
machine macaddr not null,
project project_enum default 'proinfo' not null,
unique(school_id, machine)
);
/* -------------------------------------------------------------------------- */
/* Fact tables */
/* Fact table with the history of machines connections and their inventory */
create table fact_inventory (
id bigserial primary key,
load_date integer references dim_date not null,
contact_date date not null,
machine_id integer references dim_machine not null,
os_id integer references dim_component not null,
kernel_id integer references dim_component not null,
processor_id integer references dim_component not null,
memory_size integer not null,
hd_id integer references dim_component not null,
hd_used integer not null,
hd2_id integer references dim_component,
hd2_used integer,
extra_hds smallint default 0 not null,
hash text not null,
unique(contact_date, machine_id, hd_used, hd2_used, hash)
);
/* Fact table with the history of machines modifications */
create table fact_alert (
id bigserial primary key,
load_date integer references dim_date not null,
old_inventory integer references fact_inventory not null,
inventory integer references fact_inventory not null,
memory_alert boolean default false not null,
hd_alert boolean default false not null, /* First HD changes */
unique(old_inventory, inventory)
);
/* Fact table with the history of network usage per machine and 5 minutes
* sample */
create table fact_net_usage (
id bigserial primary key,
load_date integer references dim_date not null,
machine_id integer references dim_machine not null,
collect_time timestamp not null,
down_kbits bigint not null,
down_packages integer not null,
up_kbits bigint not null,
up_packages integer not null,
unique(machine_id, collect_time)
);
/* -------------------------------------------------------------------------- */
/* Other tables */
/* Control table with the starting and ending times of each load process */
create table control (
id serial primary key,
name text not null,
start_time timestamp not null,
end_time timestamp not null,
total integer not null
);
#!/bin/bash
# Copyright (C) 2009-2012 Centro de Computacao Cientifica e Software Livre
# Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
#
# This file is part of database
#
# database is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
if test $# -ne 3; then
printf "Usage: ./create_database.sh <database> <user> <password>\n"
exit 1
fi
PREFIX="$(dirname $(readlink -f $0))"
database=$1
user=$2
password=$3
createdb ${database}
psql -d postgres -c "create role ${user} login;
alter role ${user} password '${password}';"
psql -d ${database} -c "create language plpgsql;"
psql -d ${database} -f "${PREFIX}/create_types.sql"
psql -d ${database} -f "${PREFIX}/create_staging_areas.sql"
psql -d ${database} -f "${PREFIX}/create_data_warehouse.sql"
psql -d ${database} -f "${PREFIX}/create_data_marts.sql"
psql -d ${database} -f "${PREFIX}/create_indexes.sql"
psql -d ${database} -f "${PREFIX}/add_comments.sql"
bash ${PREFIX}/grant_permissions.sh ${database} ${user}
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
CREATE TABLE mectb08_dm_disp (
dis_inep VARCHAR(18),
dis_maquina VARCHAR(18),
dis_regiao VARCHAR(12),
dis_estado VARCHAR(100),
dis_cidade VARCHAR(100),
dis_escola VARCHAR(100),
dis_data DATE,
dis_data_ult_contato DATE,
dis_qtd_dias_ult_contato INTEGER,
dis_qtd_contato_mes INTEGER,
dis_classificacao VARCHAR(9),
dis_projeto INTEGER
);
CREATE TABLE mectb09_dm_disp_ag (
dis_data DATE,
dis_ano INTEGER,
dis_mes INTEGER,
dis_mes_nm VARCHAR(9),
dis_dia INTEGER,
dis_verde INTEGER,
dis_amarelo INTEGER,
dis_vermelho INTEGER,
dis_regiao VARCHAR(12),
dis_estado VARCHAR(100),
dis_cidade VARCHAR(100),
dis_projeto INTEGER
);
CREATE TABLE mectb10_dm_inventario (
inv_data DATE,
inv_ano INTEGER,
inv_mes INTEGER,
inv_mes_nm VARCHAR(9),
inv_dia INTEGER,
inv_inep VARCHAR(18),
inv_maquina VARCHAR(18),
inv_regiao VARCHAR(12),
inv_estado VARCHAR(100),
inv_cidade VARCHAR(100),
inv_escola VARCHAR(100),
inv_so VARCHAR,
inv_modelo_proc VARCHAR,
inv_qtde_mem INTEGER,
inv_tam_disco INTEGER,
inv_disco_ocupado INTEGER,
inv_alerta_hd BIT,
inv_alerta_mem BIT,
inv_projeto INTEGER
);
CREATE TABLE mectb11_dm_auditoria (
aud_data DATE,
aud_ano INTEGER,
aud_mes INTEGER,
aud_mes_nm VARCHAR(9),
aud_dia INTEGER,
aud_inep VARCHAR(18),
aud_regiao VARCHAR(12),
aud_estado VARCHAR(100),
aud_cidade VARCHAR(100),
aud_escola VARCHAR(100),
aud_instaladas INTEGER,
aud_previstas INTEGER,
aud_projeto INTEGER
);
CREATE TABLE mectb12_dm_instaladas_ag (
inst_data DATE,
inst_ano INTEGER,
inst_mes INTEGER,
inst_mes_nm VARCHAR(9),
inst_regiao VARCHAR(12),
inst_estado VARCHAR(100),
inst_instaladas INTEGER,
inst_previstas INTEGER,
inst_hd_modificado INTEGER,
inst_mem_modificada INTEGER,
inst_projeto INTEGER
);
CREATE TABLE mectb13_dm_rede (
rede_inep VARCHAR(18),
rede_regiao VARCHAR(12),
rede_estado VARCHAR(100),
rede_cidade VARCHAR(100),
rede_escola VARCHAR(100),
rede_data DATE,
rede_time TIMESTAMP WITHOUT TIME ZONE,
rede_pacotes_in INTEGER,
rede_bytes_in REAL,
rede_pacotes_out INTEGER,
rede_bytes_out REAL
);
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
CREATE TABLE mectb01_escola_dim (
esc_id SERIAL PRIMARY KEY,
esc_inep VARCHAR(12) UNIQUE,
esc_regiao VARCHAR(12),
esc_uf VARCHAR(2),
esc_municipio VARCHAR(100),
esc_nome VARCHAR(150),
esc_endereco VARCHAR(100),
esc_cep VARCHAR(10),
esc_maq_prev INTEGER
);
CREATE TABLE mectb02_catalogo_dim (
cat_id serial PRIMARY KEY,
cat_componente VARCHAR(20),
cat_modelo VARCHAR(50),
cat_descricao VARCHAR,
cat_descricao2 VARCHAR UNIQUE
);
CREATE TABLE mectb03_data_dim (
dat_id serial PRIMARY KEY,
dat_data DATE,
dat_ano INTEGER,
dat_mes INTEGER,
dat_dia INTEGER
);
CREATE TABLE mectb04_maquina_dim (
maq_id serial primary key,
maq_identificador VARCHAR UNIQUE,
maq_dat_id INTEGER references mectb03_data_dim,
maq_esc_id INTEGER references mectb01_escola_dim,
maq_mac VARCHAR(18),
maq_inventario VARCHAR,
maq_data_remocao DATE,
maq_origem BIT,
maq_conexao BIT,
maq_projeto INTEGER
);
CREATE TABLE mectb05_disponibilidade_fact (
dis_dat_id INTEGER references mectb03_data_dim,
dis_maq_id INTEGER references mectb04_maquina_dim
);
CREATE TABLE mectb06_uso_fact (
uso_dat_id INTEGER references mectb03_data_dim,
uso_maq_id INTEGER references mectb04_maquina_dim,
uso_carga_rede INTEGER
);
CREATE TABLE mectb07_inventario_fact (
inv_dat_id INTEGER references mectb03_data_dim,
inv_maq_id INTEGER references mectb04_maquina_dim,
inv_cat_id INTEGER references mectb02_catalogo_dim,
inv_id_status CHAR,
inv_alarme BIT,
inv_disco_usado INTEGER
);
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
/* Copyright (C) 2009-2012 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
......@@ -19,10 +19,19 @@
* USA.
*/
CREATE TABLE mectb99_control (
ctr_id SERIAL PRIMARY KEY,
ctr_bd VARCHAR(2),
ctr_inicio VARCHAR,
ctr_fim VARCHAR,
ctr_total INTEGER
);
create index dm_s_index on dm_school using gin (fts_school);
/*create index dim_s_index on dim_school (region, state, city);
create index fact_nu_index on fact_net_usage (school_id, date_id);
create index dm_ash_index on dm_avail_state_history (contact_date desc);
create index dm_ach_index on dm_avail_city_history (contact_date desc);
create index dm_im_index on dm_invent_machine (state, city);
create index dm_azms_index on dm_audit_zm_school (state, city);
create index dm_alm_index on dm_alert_machine (state, city);
create index dm_alsh_index on dm_alert_state_history (contact_date desc);
create index dm_alch_index on dm_alert_city_history (contact_date desc);
create index dm_num_index on dm_net_usage_monthly (inep, collect_date);
create index dm_nuw_index on dm_net_usage_weekly (inep, date_trunc('month',
collect_date));
create index dm_nud_index on dm_net_usage_daily (inep, date_trunc('month',
collect_date));*/
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
/* Copyright (C) 2009-2012 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
......@@ -19,19 +19,37 @@
* USA.
*/
GRANT INSERT ON mectb00_staging_area TO my_user;
/* -------------------------------------------------------------------------- */
/* Staging area tables */
GRANT SELECT ON mectb01_escola_dim TO my_user;
GRANT SELECT ON mectb02_catalogo_dim TO my_user;
GRANT SELECT ON mectb03_data_dim TO my_user;
GRANT SELECT ON mectb04_maquina_dim TO my_user;
GRANT SELECT ON mectb05_disponibilidade_fact TO my_user;
GRANT SELECT ON mectb06_uso_fact TO my_user;
GRANT SELECT ON mectb07_inventario_fact TO my_user;
/* Transition table with temporary machines inventory data */
create table sa_inventory (
contact_date date not null,
project project_enum default 'proinfo' not null,
inep text not null, /* Longest INEP has 10 characters */
machine macaddr not null,
os_type text not null, /* "Linux" or "Windows" */
os_distro text not null, /* Longest OS distro has 35 characters */
kernel text not null, /* Longest kernel version has 24 chars */
processor text not null, /* Longest proc model has 46 chars */
memory_size integer not null,
hd_model text not null, /* Longest HD model has 47 characters */
hd_size integer not null,
hd_used integer not null,
hd2_model text,
hd2_size integer,
hd2_used integer,
extra_hds integer default 0 not null
);
GRANT SELECT ON mectb08_dm_disp TO my_user;
GRANT SELECT ON mectb09_dm_disp_ag TO my_user;
GRANT SELECT ON mectb10_dm_inventario TO my_user;
GRANT SELECT ON mectb11_dm_auditoria TO my_user;
GRANT SELECT ON mectb12_dm_instaladas_ag TO my_user;
GRANT SELECT ON mectb13_dm_rede TO my_user;
/* Transition table with temporary network usage data per machine */
create table sa_net_usage (
insertion_date date not null, /* To avoid using lock during the load */
inep text not null,
machine macaddr not null,
collect_time timestamp not null,
down_kbits bigint not null,
down_packages integer not null,
up_kbits bigint not null,
up_packages integer not null
);
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
/* Copyright (C) 2009-2012 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
......@@ -19,29 +19,29 @@
* USA.
*/
DROP DATABASE mec_seed;
CREATE DATABASE mec_seed;
/* -------------------------------------------------------------------------- */
/* Enumerations */
\c mec_seed;
/* Enum type for the MEC project which the machines belong to */
create type project_enum as enum (
'proinfo',
'uca_classmate',
'uca_server',
'projector'
);
/* Enum type for the components category */
create type category_enum as enum (
'os',
'kernel',
'processor',
'hd'
);
CREATE TABLE mectb00_staging_area (
sa_inep VARCHAR(18),
sa_mac VARCHAR(18),
sa_disco1_modelo TEXT,
sa_disco1_capacidade INTEGER,
sa_disco1_usado INTEGER,
sa_disco2_modelo TEXT,
sa_disco2_capacidade INTEGER,
sa_disco2_usado INTEGER,
sa_discos_ad INTEGER,
sa_memoria INTEGER,
sa_processador TEXT,
sa_so_nome TEXT,
sa_so_distribuicao TEXT,
sa_so_kernel TEXT,
sa_uso_up INTEGER,
sa_uso_down INTEGER,
sa_data TIMESTAMP,
sa_conexao BIT,
sa_projeto INTEGER
/* Enum type for the class_enum of the machines according to their
* availability */
create type class_enum as enum (
'green', /* Communication ocurred at most 10 days ago */
'yellow', /* Communication ocurred between 10 and 30 days */
'red' /* Communication hasn't occured for more than 30 days */
);
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
CREATE USER my_user;
ALTER USER my_user PASSWORD 'password';
#!/bin/bash
# Copyright (C) 2009-2012 Centro de Computacao Cientifica e Software Livre
# Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
#
# This file is part of database
#
# database is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
if test $# -ne 2; then
printf "Usage: ./grant_permissions.sh <database> <user>\n"
exit 1
fi
database=$1
user=$2
psql -d ${database} -c "
grant insert on sa_inventory to ${user};
grant insert on sa_net_usage to ${user};
grant select on dim_date to ${user};
grant select on dim_school to ${user};
grant select on dim_component to ${user};
grant select on dim_machine to ${user};
grant select on fact_inventory to ${user};
grant select on fact_alert to ${user};
grant select on fact_net_usage to ${user};
grant select on control to ${user};
grant select on dm_school to ${user};
grant select on dm_avail_state to ${user};
grant select on dm_avail_city to ${user};
grant select on dm_avail_school to ${user};
grant select on dm_avail_machine to ${user};
grant select on dm_avail_state_history to ${user};
grant select on dm_avail_city_history to ${user};
grant select on dm_invent_machine to ${user};
grant select on dm_alert_state to ${user};
grant select on dm_alert_city to ${user};
grant select on dm_alert_school to ${user};
grant select on dm_alert_machine to ${user};
grant select on dm_alert_state_history to ${user};
grant select on dm_alert_city_history to ${user};
grant select on dm_net_usage_monthly to ${user};
grant select on dm_net_usage_weekly to ${user};
grant select on dm_net_usage_daily to ${user};"
This diff is collapsed.
This diff is collapsed.
/* Copyright (C) 2009-2012 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPosE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
/* This script must be used only when there is new data about the schools or
* machines, or a change in the graphs or reports. This is a recall and an
* indicatation to the changes that must be done. Don't forget to update the
* load scripts and to really load the tables with the new data */
/* -------------------------------------------------------------------------- */
/* Information about the origin of the machine data: if it was from the Internet
* (automatically sent by the agent), or by (e-)mail/in person */
alter table sa_inventory add column internet_con boolean default true not null;
alter table dim_machine add column internet_con boolean default true not null;
/* -------------------------------------------------------------------------- */
/* Information that would come from MEC database, if they had it, regarding the
* expected number of machines */
alter table dim_school add column expected smallint default 0 not null;
alter table dm_avail_state add column expected smallint default 0 not null;
alter table dm_avail_city add column expected smallint default 0 not null;
alter table dm_avail_school add column expected smallint default 0 not null;
alter table dm_avail_state_history add column expected smallint default 0 not
null;
alter table dm_avail_city_history add column expected smallint default 0 not
null;
/* -------------------------------------------------------------------------- */
/* Information to the inventory report about the amount used of the first HD */
alter table dm_invent_machine add column hd_used integer default 0 not null;
/* -------------------------------------------------------------------------- */
/* Information to the inventory report about the other HDs */
alter table dm_invent_machine add column hd2_size integer default 0 not null;
alter table dm_invent_machine add column hd2_used integer default 0 not null;
alter table dm_invent_machine add column extra_hds integer default 0 not null;
/* -------------------------------------------------------------------------- */
/* Information to the inventory report about the HD models */
alter table dm_invent_machine add column hd_model text default 0 not
null;
alter table dm_invent_machine add column hd2_model text default 0 not
null;
/* -------------------------------------------------------------------------- */
/* Information to the alert report about the total of alerts */
alter table dm_alert_state add column total integer default 0 not null;
alter table dm_alert_city add column total integer default 0 not null;
alter table dm_alert_school add column total integer default 0 not null;
alter table dm_alert_state_history add column total integer default 0 not null;
alter table dm_alert_city_history add column total integer default 0 not null;
/* -------------------------------------------------------------------------- */
/* Information to the alert report about the machines that have both alerts at
* the same time (if the others are exclusive) */
alter table dm_alert_state add column both_amount integer default 0 not null;
alter table dm_alert_city add column both_amount integer default 0 not null;
alter table dm_alert_school add column both_amount integer default 0 not null;
alter table dm_alert_state_history add column both_amount integer default 0 not
null;
alter table dm_alert_city_history add column both_amount integer default 0 not
null;
This diff is collapsed.
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
/* Function to get the name of the month
*
* Receives as input the number of month.
*
* Returns the name of the month.
*/
CREATE OR REPLACE FUNCTION get_month_name(integer) RETURNS varchar(9) as $$
DECLARE
array_months VARCHAR[] := ARRAY['Janeiro', 'Feveiro', 'Março','Abril',
'Maio', 'Junho', 'Julho', 'Agosto', 'Setembro', 'Outubro', 'Novembro',
'Dezembro'];
BEGIN
RETURN array_months[$1];
END;
$$ LANGUAGE 'plpgsql';
# Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
#!/bin/bash
# Copyright (C) 2009-2012 Centro de Computacao Cientifica e Software Livre
# Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
#
# This file is part of database
......@@ -18,21 +19,45 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
#!/bin/bash
DUMPDIR='/var/lib/postgresql/database/dump'
LOADDIR='/var/lib/postgresql/database'
DUMPFILE="dump_$(date +%d_%m_%y).sql"
DUMP_PARAM="--format custom --blobs"
cd $DUMPDIR
pg_dump mec_seed $DUMP_PARAM > "$DUMPFILE"
cp $DUMPFILE dump.sql
date > dumpDate
cd $LOADDIR
psql -d mec_seed -f load.sql
# This script executes all the processes needed to load the new data into the
# database in a parallel way whenever possible
if test $# -ne 3; then
printf "Usage: ./load.sh <database> <sa_dump_file> <dump_file>\n"
exit 1
fi
PREFIX="$(dirname $(readlink -f $0))"
DUMPPARAM="-Fc -b"
database=$1
sa_dump_file=$2
dump_file=$3
date +"Backup started at %F %T"
pg_dump ${DUMPPARAM} -t sa_* -f "${sa_dump_file}" ${database}
pg_dump ${DUMPPARAM} -T dm_* -f "${dump_file}" ${database}
date +"Backup completed at %F %T"
date +"Data warehouse load started at %F %T"
psql -d ${database} -f "${PREFIX}/load_dim_dw.sql"
psql -d ${database} -f "${PREFIX}/load_invent_dw.sql"
psql -d ${database} -f "${PREFIX}/load_nu_dw.sql"
psql -d ${database} -c "select load_dim_dw();"
psql -d ${database} -c "select load_invent_dw();" &
psql -d ${database} -c "select load_nu_dw();" &
wait
date +"Data warehouse load finished at %F %T"
date +"Data marts load started at %F %T"
psql -d ${database} -f "${PREFIX}/load_avail_dm.sql"
psql -d ${database} -f "${PREFIX}/load_invent_dm.sql"
psql -d ${database} -f "${PREFIX}/load_audit_dm.sql"
psql -d ${database} -f "${PREFIX}/load_alert_dm.sql"
psql -d ${database} -f "${PREFIX}/load_nu_dm.sql"
psql -d ${database} -c "select load_avail_dm();" &
psql -d ${database} -c "select load_invent_dm();" &
psql -d ${database} -c "select load_audit_dm();" &
psql -d ${database} -c "select load_alert_dm();" &
psql -d ${database} -c "select load_nu_dm();" &
wait
date +"Data marts load finished at %F %T"
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
SELECT f_load_dw();
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
/* Copyright (C) 2009-2012 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
/* Create function that updates the data warehouse dimension tables using the
* new data in the staging area, removing trailing white spaces from both
* extremes of the text types */
create or replace function load_dim_dw() returns integer as $$
declare
count_total integer := 0;
tmp integer;
load_date_id integer;
min_date date;
/* -------------------------------------------------------------------------- */
begin
raise log 'Data warehouse dimension load start point: %', current_timestamp;
/* Create and populate staging area temporary table to be used by the
* loading script as the source */
create temporary table tmp_inventory as
select distinct * from sa_inventory where contact_date < current_date;
/* ---------------------------------------------------------------------- */
/* Update dimension tables */
raise log 'Inserting "current_date" into "dim_date"...';
select id into load_date_id from dim_date where load_date = current_date;
if load_date_id is null then
insert into dim_dat