Commit 6c4a27f5 authored by Erik Alexandre Pucci's avatar Erik Alexandre Pucci

database: Revision

Signed-off-by: default avatarErik Alexandre Pucci <eap08@c3sl.ufpr.br>
parent a8d966af
......@@ -61,6 +61,8 @@ comment on column dim_school.school is 'MEC database limit for school names is '
'100 characters';
comment on column dim_school.address is 'Longest address has 70 characters';
comment on column dim_school.fts_school is 'Full text search field for schools';
comment on column dim_school.nud_avail is 'It indicates if the school has '
'already sent some network usage data';
comment on table dim_component is 'Dimension table of software and hardware '
'components';
comment on column dim_component.description is 'HD model, OS type, processor '
......@@ -75,9 +77,6 @@ comment on column fact_alert.hd_alert is 'Alert representing changes in the '
' first HD';
comment on table fact_net_usage is 'Fact table with the history of network '
'usage per school and 5 minutes sample';
comment on table fact_net_usage_avail is 'Fact table of the list of schools '
'with network usage data collected (i.e. at least one machine sent the net '
'usage data)';
comment on table control is 'Control table with the starting and ending times '
'of each load process';
......
......@@ -40,7 +40,9 @@ create table dim_school (
school text not null, /* MEC database limit for schools is 100 */
code text,
address text, /* Longest address has 70 characters */
fts_school tsvector not null /* Full text search field for schools */
fts_school tsvector not null, /* Full text search field for schools */
nud_avail boolean default false not null
/* It indicates if the school has already sent some network usage data */
);
/* Dimension table of software and hardware components */
......@@ -112,13 +114,6 @@ create table fact_net_usage (
up_packages integer not null
);
/* Fact table of the list of schools with network usage data collected (i.e. at
* least one machine sent the net usage data) */
create table fact_net_usage_avail (
load_date integer references dim_date not null,
school_id integer references dim_school unique not null
);
/* -------------------------------------------------------------------------- */
/* Other tables */
......
......@@ -38,7 +38,6 @@ grant select on dim_machine to ${user};
grant select on fact_inventory to ${user};
grant select on fact_alert to ${user};
grant select on fact_net_usage_avail to ${user};
grant select on fact_net_usage to ${user};
grant select on control to ${user};
......
#!/bin/bash
# Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
# Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
#
# This file is part of database
#
# database is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
cd /home/postgres/load
createlang plpgsql -d proinfodata
psql -d proinfodata -f f_load_dw.sql
psql -d proinfodata -f f_load_dm.sql
psql -d proinfodata -f get_month.sql
psql -d proinfodata -f load_mectb08.sql
psql -d proinfodata -f load_mectb09.sql
psql -d proinfodata -f load_mectb10.sql
psql -d proinfodata -f load_mectb11.sql
psql -d proinfodata -f load_mectb12.sql
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
/* Function to load data marts, related to the data warehouse
*
* It connects to the database and creates the function to get the name of
* the month. Its main action is to create and run the loading functions
* of tables 8 to 11.
*
* Receives as input the loading date.
*/
CREATE FUNCTION f_load_dm(date date) RETURNS void
LANGUAGE plpgsql
AS $$
DECLARE
-- Vector containing the id of schools of each new machine
array_id_esc INTEGER[];
id_date INTEGER;
total INTEGER;
control_id INTEGER;
BEGIN
raise notice 'Comecou F_LOAD_DM()';
INSERT INTO mectb99_control(ctr_bd, ctr_inicio) VALUES ('dm',
timeofday());
SELECT dat_id INTO id_date FROM mectb03_data_dim WHERE
dat_data = date;
SELECT count(*) INTO total FROM mectb05_disponibilidade_fact WHERE
dis_dat_id=id_date;
PERFORM load_mectb08(date);
PERFORM load_mectb09(date);
SELECT load_mectb10(date) INTO array_id_esc;
PERFORM load_mectb11(date, array_id_esc);
PERFORM load_mectb12(date);
SELECT last_value INTO control_id FROM mectb99_control_ctr_id_seq;
UPDATE mectb99_control SET ctr_fim = timeofday(), ctr_total = total
WHERE ctr_id = control_id;
TRUNCATE mectb00_staging_area;
raise notice 'Terminou F_LOAD_DM()';
RETURN;
END;
$$;
\ No newline at end of file
This diff is collapsed.
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
/* Function to get the name of the month
*
* Receives as input the number of month. ############## DATE !!!!!!!!!!!!!!!!!
*
* Returns the name of the month.
*/
CREATE FUNCTION get_month_name(integer) RETURNS character varying
LANGUAGE plpgsql
AS $_$
DECLARE
array_months VARCHAR[] := ARRAY['Janeiro', 'Fevereiro', 'Março', 'Abril',
'Maio', 'Junho', 'Julho', 'Agosto', 'Setembro', 'Outubro', 'Novembro',
'Dezembro'];
BEGIN
RETURN array_months[$1];
END;
$_$;
......@@ -46,6 +46,13 @@ begin
raise log 'Done - row already inserted, using it';
end if;
raise log 'Updating nud_avail field inside "dim_school"...';
update dim_school s set nud_avail = true
from sa_net_usage n where n.inep = s.inep;
get diagnostics tmp = ROW_COUNT;
count_total := count_total + tmp;
raise log 'Done - % rows updated', tmp;
raise log 'Inserting new component data into "dim_component"...';
insert into dim_component (load_date, category, description, detail)
select distinct load_date_id, 'os'::category_enum, trim(both from
......@@ -200,16 +207,6 @@ begin
count_total := count_total + tmp;
raise log 'Done - % rows inserted', tmp;
raise log 'Inserting new school network usage data availability into '
'"fact_net_usage_avail"...';
insert into fact_net_usage_avail (load_date, school_id)
select load_date_id, id from sa_net_usage n, dim_school s where
n.inep = s.inep
except select load_date_id, school_id from fact_net_usage_avail;
get diagnostics tmp = ROW_COUNT;
count_total := count_total + tmp;
raise log 'Done - % rows inserted', tmp;
/* ---------------------------------------------------------------------- */
/* Update control table */
......
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
/* Function that loads table mectb08_dm_disp
*
* Receives as input the loading date.
*/
CREATE FUNCTION load_mectb08(date date) RETURNS void
LANGUAGE plpgsql
AS $$
BEGIN
INSERT INTO
mectb08_dm_disp
(
dis_inep,
dis_maquina,
dis_regiao,
dis_estado,
dis_cidade,
dis_escola,
dis_data,
dis_data_ult_contato,
dis_qtd_dias_ult_contato,
dis_qtd_contato_mes,
dis_classificacao,
dis_projeto
)
SELECT
c.esc_inep as dis_inep,
b.maq_mac as dis_maquina,
c.esc_regiao as dis_regiao,
c.esc_uf as dis_estado,
c.esc_municipio as dis_cidade,
c.esc_nome as dis_escola,
date as dis_data,
a.data_ult_contato as dis_data_ult_contato,
date - a.data_ult_contato as dis_qtd_dias_ult_contato,
a.qtd_contato_mes as dis_qtd_contato_mes,
CASE
WHEN date - a.data_ult_contato > 30 THEN '3_RED'
WHEN date - a.data_ult_contato > 10 THEN '2_YELLOW'
ELSE '1_GREEN'
END as dis_classificacao,
b.maq_projeto as dis_projeto
FROM
(SELECT
a.dis_maq_id,
MAX(b.dat_data) as data_ult_contato,
SUM(
CASE WHEN to_char(date, 'MM-YYYY') = to_char(b.dat_data, 'MM-YYYY')
THEN 1
ELSE 0
END
) as qtd_contato_mes
FROM
mectb05_disponibilidade_fact a
INNER JOIN
mectb03_data_dim b
ON
a.dis_dat_id=b.dat_id
GROUP BY
a.dis_maq_id
) a
INNER JOIN
mectb04_maquina_dim b
ON
a.dis_maq_id=b.maq_id
INNER JOIN
mectb01_escola_dim c
ON
b.maq_esc_id=c.esc_id;
RETURN;
END;
$$;
\ No newline at end of file
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
/* Function that loads table mectb09_dm_disp_ag
*
* Receives as input the loading date.
*/
CREATE FUNCTION load_mectb09(date date) RETURNS void
LANGUAGE plpgsql
AS $$
DECLARE
month_name VARCHAR(9);
year INTEGER;
month INTEGER;
day INTEGER;
BEGIN
year := date_part('year', date);
month := date_part('month', date);
day := date_part('day', date);
-- Function to get the month name
SELECT get_month_name(month) INTO month_name;
INSERT INTO
mectb09_dm_disp_ag
(
dis_data,
dis_ano,
dis_mes,
dis_mes_nm,
dis_dia,
dis_verde,
dis_amarelo,
dis_vermelho,
dis_regiao,
dis_estado,
dis_cidade,
dis_projeto
)
SELECT
date as dis_data,
year as dis_ano,
month as dis_mes,
month_name as dis_mes_nm,
day as dis_mes,
sum(CASE WHEN dis_classificacao='1_GREEN' THEN 1 ELSE 0 END) as dis_verde,
sum(CASE WHEN dis_classificacao='2_YELLOW' THEN 1 ELSE 0 END) as dis_amarelo,
sum(CASE WHEN dis_classificacao='3_RED' THEN 1 ELSE 0 END) as dis_vermelho,
dis_regiao,
dis_estado,
dis_cidade,
dis_projeto
FROM
mectb08_dm_disp
WHERE
dis_data=date
GROUP BY
dis_regiao,dis_estado,dis_cidade,dis_projeto;
raise notice 'Terminou LOAD_MECTB09';
RETURN;
END;
$$;
\ No newline at end of file
This diff is collapsed.
/* Copyright (C) 2004-2010 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of database
*
* database is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA.
*/
/* Function to load the table mectb11_dm_auditoria
*
* Receives as input the loading date and a vector with the schools id for
* each new machine.
*
* Receives an array with the schools id of each new machine.
*/
CREATE FUNCTION load_mectb11(date date, array_id_esc integer[]) RETURNS void
LANGUAGE plpgsql
AS $$
DECLARE
city VARCHAR(100);
state VARCHAR(100);
region VARCHAR(12);
name_school VARCHAR(100);
inep VARCHAR(18);
month_name VARCHAR(9);
year INTEGER;
month INTEGER;
day INTEGER;
num_maq_installed_0 INTEGER;
num_maq_provided_0 INTEGER;
last_num_installed_0 INTEGER;
last_num_provided_0 INTEGER;
num_maq_installed_1 INTEGER;
num_maq_provided_1 INTEGER;
last_num_installed_1 INTEGER;
last_num_provided_1 INTEGER;
num_maq_installed_2 INTEGER;
num_maq_provided_2 INTEGER;
last_num_installed_2 INTEGER;
last_num_provided_2 INTEGER;
num_maq_installed_3 INTEGER;
num_maq_provided_3 INTEGER;
last_num_installed_3 INTEGER;
last_num_provided_3 INTEGER;
controller INTEGER;
-- last_date DATE;
-- Variable indexing of the array_id_esc
i INTEGER;
BEGIN
raise notice 'Iniciou LOAD_MECTB11';
raise notice 'Data = %', date;
-- raise notice 'Entrou no LOAD 11';
year := date_part('year',date);
month := date_part('month', date);
day := date_part('day', date);
-- Function to get month name
SELECT get_month_name(month) INTO month_name;
-- Arrow index of array_id_esc
i := 1;
-- While the school id is not null
WHILE array_id_esc[i] IS NOT NULL
LOOP
controller:=0;
-- Selects information from school
SELECT esc_inep, esc_regiao, esc_uf, esc_municipio, esc_nome INTO
inep, region, state, city, name_school FROM "mectb01_escola_dim"
WHERE esc_id = array_id_esc[i];
-- If the information that school were not updated on the current
-- date, insert them, otherwise do nothing
IF (SELECT aud_inep FROM mectb11_dm_auditoria WHERE aud_inep=inep
AND aud_data=date LIMIT 1) IS NULL THEN
-- Machines ProInfoData
-- Select number of machines provided
SELECT esc_maq_prev INTO num_maq_provided_0 FROM
"mectb01_escola_dim" WHERE esc_id=array_id_esc[i];
IF num_maq_provided_0 IS NULL THEN
num_maq_provided_0 := 0;
END IF;
-- Account number of machines installed
SELECT DISTINCT COUNT(*) INTO num_maq_installed_0 FROM
"mectb04_maquina_dim" WHERE maq_esc_id=array_id_esc[i] AND
maq_projeto=0;
SELECT aud_instaladas, aud_previstas INTO last_num_installed_0,
last_num_provided_0 FROM mectb11_dm_auditoria WHERE aud_inep=inep
AND aud_projeto=0 ORDER BY aud_data DESC LIMIT 1;
IF ((num_maq_installed_0 > last_num_installed_0) OR
(num_maq_provided_0 > last_num_provided_0) OR
(last_num_installed_0 IS NULL)) THEN
controller:=1;
END IF;
-- UCA Clasmates
-- Select number of machines provided
SELECT esc_maq_prev INTO num_maq_provided_1 FROM
"mectb01_escola_dim" WHERE esc_id=array_id_esc[i];
IF num_maq_provided_1 IS NULL THEN
num_maq_provided_1 := 0;
END IF;
-- Account number of machines installed
SELECT DISTINCT COUNT(*) INTO num_maq_installed_1 FROM
"mectb04_maquina_dim" WHERE maq_esc_id=array_id_esc[i] AND
maq_projeto=1;
SELECT aud_instaladas, aud_previstas INTO last_num_installed_1,
last_num_provided_1 FROM mectb11_dm_auditoria WHERE aud_inep=inep
AND aud_projeto=1 ORDER BY aud_data DESC LIMIT 1;
IF ((num_maq_installed_1 > last_num_installed_1) OR
(num_maq_provided_1 > last_num_provided_1) OR
(last_num_installed_1 IS NULL)) THEN
controller:=1;
END IF;
-- UCA Server
-- Select number of machines provided
SELECT esc_maq_prev INTO num_maq_provided_2 FROM
"mectb01_escola_dim" WHERE esc_id=array_id_esc[i];
IF num_maq_provided_2 IS NULL THEN
num_maq_provided_2 := 0;
END IF;
-- Account number of machines installed
SELECT DISTINCT COUNT(*) INTO num_maq_installed_2 FROM
"mectb04_maquina_dim" WHERE maq_esc_id=array_id_esc[i] AND
maq_projeto=2;
SELECT aud_instaladas, aud_previstas INTO last_num_installed_2,
last_num_provided_2 FROM mectb11_dm_auditoria WHERE aud_inep=inep
AND aud_projeto=2 ORDER BY aud_data DESC LIMIT 1;
IF ((num_maq_installed_2 > last_num_installed_2) OR
(num_maq_provided_2 > last_num_provided_2) OR
(last_num_installed_2 IS NULL)) THEN
controller:=1;
END IF;
-- Projector
-- Select number of machines provided
SELECT esc_maq_prev INTO num_maq_provided_3 FROM
"mectb01_escola_dim" WHERE esc_id=array_id_esc[i];
IF num_maq_provided_3 IS NULL THEN
num_maq_provided_3 := 0;
END IF;
-- Account number of machines installed
SELECT DISTINCT COUNT(*) INTO num_maq_installed_3 FROM
"mectb04_maquina_dim" WHERE maq_esc_id=array_id_esc[i] AND
maq_projeto=3;
SELECT aud_instaladas, aud_previstas INTO last_num_installed_3,
last_num_provided_3 FROM mectb11_dm_auditoria WHERE aud_inep=inep
AND aud_projeto=3 ORDER BY aud_data DESC LIMIT 1;
IF ((num_maq_installed_3 > last_num_installed_3) OR
(num_maq_provided_3 > last_num_provided_3) OR
(last_num_installed_3 IS NULL)) THEN
controller:=1;
END IF;
END IF;
IF (controller != 0) THEN
INSERT INTO mectb11_dm_auditoria VALUES(date, year, month,
month_name, day, inep, region, state, city, name_school,
num_maq_installed_0, num_maq_provided_0, 0);
INSERT INTO mectb11_dm_auditoria VALUES(date, year, month,
month_name, day, inep, region, state, city, name_school,
num_maq_installed_1, num_maq_provided_1, 1);
INSERT INTO mectb11_dm_auditoria VALUES(date, year, month,
month_name, day, inep, region, state, city, name_school,
num_maq_installed_2, num_maq_provided_2, 2);
INSERT INTO mectb11_dm_auditoria VALUES(date, year, month,
month_name, day, inep, region, state, city, name_school,
num_maq_installed_3, num_maq_provided_3, 3);
END IF;
-- Increases the array_id_esc index
i := i+1;
END LOOP;
-- raise notice 'Entrou no LOAD 11';
raise notice 'Terminou LOAD_MECTB11';
RETURN;
END;
$$;
\ No newline at end of file
This diff is collapsed.
/*
* Function to load the table mectb13_dm_rede
*/
CREATE FUNCTION load_mectb13() RETURNS void
LANGUAGE plpgsql
AS $$
DECLARE
BEGIN
INSERT INTO
mectb13_dm_rede
(
rede_inep,
rede_regiao,
rede_estado,
rede_cidade,
rede_escola,
rede_data,
rede_time,
rede_pacotes_in,
rede_bytes_in,
rede_pacotes_out,
rede_bytes_out
)
SELECT
a.inep,
b.esc_regiao,
b.esc_uf,
b.esc_municipio,
b.esc_nome,
a.data,
a.hora,
a.package_in,
a.bytes_in,
a.package_out,
a.bytes_out
FROM
(SELECT
net_inep AS inep,
net_hora AS hora,
net_data AS data,
sum(net_pacotes_in) AS package_in,
sum(net_bytes_in) AS bytes_in,
sum(net_pacotes_out) AS package_out,
sum (net_bytes_out) AS bytes_out
FROM
mectb00_net_staging_area
GROUP BY
net_inep,net_hora,net_data
) a
INNER JOIN
mectb01_escola_dim b
ON
a.inep=b.esc_inep;
END;
$$;
\ No newline at end of file
......@@ -71,10 +71,10 @@ begin
raise log 'Migrating "mectb01_escola_dim" data to "dim_school"...';
insert into dim_school (inep, region, state, city, school, code,
address, fts_school)
address, fts_school, nud_avail)
select esc_inep, esc_regiao, esc_uf, trim(both from esc_municipio),
trim(both from esc_nome), esc_cep, trim(both from esc_endereco),
to_tsvector(esc_nome) from mectb01_escola_dim;
to_tsvector(esc_nome), false from mectb01_escola_dim;
get diagnostics tmp = ROW_COUNT;
count_total := count_total + tmp;
raise log 'Done - % rows inserted', tmp;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment