From 3abfa1b80cff5086e65cc490d8b1b70937a51d2e Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Mon, 17 Feb 2025 11:19:28 -0300 Subject: [PATCH 1/5] feat: add student revenue API endpoint --- src/libs/routes_v1/api.js | 3 + src/libs/routes_v1/studentRevenue.js | 132 +++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 src/libs/routes_v1/studentRevenue.js diff --git a/src/libs/routes_v1/api.js b/src/libs/routes_v1/api.js index 9bbd5e93..b9af6769 100644 --- a/src/libs/routes_v1/api.js +++ b/src/libs/routes_v1/api.js @@ -173,6 +173,8 @@ const adjustedLiquidFrequency = require(`${libs}/routes_v1/adjustedLiquidFrequen const iliteracyRate = require(`${libs}/routes_v1/iliteracyRate`); +const studentRevenue = require(`${libs}/routes_v1/studentRevenue`); + api.get('/', (req, res) => { res.json({ msg: 'SimCAQ API v1 is running' }); }); @@ -247,6 +249,7 @@ api.use('/superior_education_conclusion_tax', superiorEducationConclusionTax) api.use('/basic_education_conclusion', basicEducationConclusion); api.use('/adjusted_liquid_frequency', adjustedLiquidFrequency); api.use('/iliteracy_rate', iliteracyRate); +api.use('/student_revenue', studentRevenue); //Publication api.use('/publication', publication); diff --git a/src/libs/routes_v1/studentRevenue.js b/src/libs/routes_v1/studentRevenue.js new file mode 100644 index 00000000..ecc79844 --- /dev/null +++ b/src/libs/routes_v1/studentRevenue.js @@ -0,0 +1,132 @@ +/* +Copyright (C) 2024 Centro de Computacao Cientifica e Software Livre +Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR + +This file is part of simcaq-node. + +simcaq-node is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +simcaq-node is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with simcaq-node. If not, see <https://www.gnu.org/licenses/>. +*/ + +const express = require('express'); + +const studentRevenueApp = express.Router(); + +const libs = `${process.cwd()}/libs`; + +const squel = require('squel'); + +const query = require(`${libs}/middlewares/query`).query; + +const response = require(`${libs}/middlewares/response`); + +const ReqQueryFields = require(`${libs}/middlewares/reqQueryFields`); + +const id2str = require(`${libs}/middlewares/id2str`); + +const config = require(`${libs}/config`); + +const cache = require('apicache').options({ debug: config.debug, statusCodes: {include: [200]} }).middleware; + +let rqf = new ReqQueryFields(); + +studentRevenueApp.use(cache('15 day')); + +studentRevenueApp.get('/years', (req, res, next) => { + req.sql.from('pnad_novo') + .field('DISTINCT receitas.ano', 'year') + next(); +}, query, response('years')); + +rqf.addField({ + name: 'filter', + field: false, + where: true +}).addField({ + name: 'dims', + field: true, + where: false +}).addValue({ + name: 'min_year', + table: 'receitas', + tableField: 'ano', + resultField: 'year', + where: { + relation: '>=', + type: 'integer', + field: 'ano' + } +}).addValue({ + name: 'max_year', + table: 'receitas', + tableField: 'ano', + resultField: 'year', + where: { + relation: '<=', + type: 'integer', + field: 'ano' + } +}).addValue({ + name: 'city', + table: 'receitas', + tableField: 'cod_ibge', + resultField: 'city', + where: { + relation: '=', + type: 'integer', + field: 'cod_ibge' + } +}) + +studentRevenueApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { + + if (req.query.dims && req.query.dims.includes('receitas')) { + req.sql.from('receitas') + .field('receitas.ano', 'year') + .field('receitas.receitas', 'total_receitas') + } + if (req.query.dims && req.query.dims.includes('matriculas_publicas')) { + req.sql.from('receitas') + .field('receitas.ano', 'year') + .field('matriculas_publica', 'matriculas_publica_total') + } + if (req.query.dims && req.query.dims.includes('matriculas_publicas_mais_conveniada')) { + req.sql.from('receitas') + .field('receitas.ano', 'year') + .field('matriculas_publicas_mais_conveniada', 'matriculas_publicas_mais_conveniada_total') + } + if (req.query.dims && req.query.dims.includes('receita_aluno_ano_publica')) { + req.sql.from('receitas') + .field('receitas.ano', 'year') + .field('receuta_aluno_ano_publica', 'receita_aluno_ano_publica_total') + } + if (req.query.dims && req.query.dims.includes('receita_aluno_mes_publica')) { + req.sql.from('receitas') + .field('receitas.ano', 'year') + .field('receita_aluno_mes_publica', 'receita_aluno_mes_publica_total') + } + if (req.query.dims && req.query.dims.includes('receita_aluno_ano_publica_mais_conveniada')) { + req.sql.from('receitas') + .field('receitas.ano', 'year') + .field('receita_aluno_ano_publica_mais_conveniada', 'receita_aluno_ano_publica_mais_conveniada_total') + } + if (req.query.dims && req.query.dims.includes('receita_aluno_mes_publica_mais_conveniada')) { + req.sql.from('receitas') + .field('receitas.ano', 'year') + .field('receita_aluno_mes_publica_mais_conveniada', 'receita_aluno_mes_publica_mais_conveniada_total') + } + + next(); +}, query, id2str.transform(false), response('student_revenue')); + +module.exports = studentRevenueApp; -- GitLab From 28d0694690a5ff8c396cb9c8006ee42b8fe91bb5 Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Wed, 19 Feb 2025 11:52:00 -0300 Subject: [PATCH 2/5] feat: implement revenue conversion and update student revenue route --- src/libs/convert/revenue.js | 38 +++++++++++++++++++++++++ src/libs/middlewares/aggregateData.js | 7 +++-- src/libs/middlewares/id2str.js | 7 +++-- src/libs/routes_v1/studentRevenue.js | 40 ++++++--------------------- src/libs/routes_v1/yearsOfStudy.js | 2 +- 5 files changed, 58 insertions(+), 36 deletions(-) create mode 100644 src/libs/convert/revenue.js diff --git a/src/libs/convert/revenue.js b/src/libs/convert/revenue.js new file mode 100644 index 00000000..c4f5865a --- /dev/null +++ b/src/libs/convert/revenue.js @@ -0,0 +1,38 @@ +/* +Copyright (C) 2024 Centro de Computacao Cientifica e Software Livre +Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR + +This file is part of simcaq-node. + +simcaq-node is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +simcaq-node is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with simcaq-node. If not, see <https://www.gnu.org/licenses/>. +*/ + +module.exports = function regionCode(id) { + switch (id) { + case 1: + return 'Receitas'; + case 2: + return 'MatrÃculas Públicas'; + case 3: + return 'MatrÃculas Públicas Mais Conveniada'; + case 4: + return 'Receita Aluno Ano Pública' + case 5: + return 'Receita Aluno Mês Pública'; + case 6: + return 'Receita Aluno Ano Pública Mais Conveniada'; + case 7: + return 'Receita Aluno Mês Pública Mais Conveniada'; + } +}; diff --git a/src/libs/middlewares/aggregateData.js b/src/libs/middlewares/aggregateData.js index eed373dd..adf21fba 100644 --- a/src/libs/middlewares/aggregateData.js +++ b/src/libs/middlewares/aggregateData.js @@ -35,7 +35,8 @@ const convert = { contract_type_entity: 'contractType', total_doc: "totalDoc", education_degree_entity: "educationDegreeEntity", - government_agreement: "governmentAgreement" + government_agreement: "governmentAgreement", + receitas: "revenue" } function aggregateData(req, res, next) { @@ -62,7 +63,8 @@ function aggregateData(req, res, next) { 'age_range_entity', 'post_graduation_entity', 'contract_type_entity', - 'education_degree_entity' + 'education_degree_entity', + 'receitas' ] let id; const fields = req.query.dims.split(','); @@ -78,6 +80,7 @@ function aggregateData(req, res, next) { for (const property in r) { // Dados agregados são identificados com a substring 'total_' em sua chave if (property.includes('total_')) { + console.log(property); let data = { total: r[property], year: r.year, diff --git a/src/libs/middlewares/id2str.js b/src/libs/middlewares/id2str.js index d6c763c9..60ff424f 100644 --- a/src/libs/middlewares/id2str.js +++ b/src/libs/middlewares/id2str.js @@ -126,6 +126,7 @@ const ageRangeEntity = require(`${libs}/convert/ageRangeEntity`); const postGraduationEntity = require(`${libs}/convert/postGraduationEntity`); const totalDoc = require(`${libs}/convert/totalDoc`); const educationDegreeEntity = require(`${libs}/convert/educationDegreeEntity`); +const revenue = require(`${libs}/convert/revenue`); const ids = { gender_id: gender, @@ -244,7 +245,8 @@ const ids = { post_graduation_entity: postGraduationEntity, special_education_entity: specialEducationEntity, total_doc: totalDoc, - education_degree_entity: educationDegreeEntity + education_degree_entity: educationDegreeEntity, + receitas_id: revenue }; function transform(removeId=false) { @@ -386,5 +388,6 @@ module.exports = { postGraduationEntity, specialEducationEntity, totalDoc, - educationDegreeEntity + educationDegreeEntity, + revenue }; diff --git a/src/libs/routes_v1/studentRevenue.js b/src/libs/routes_v1/studentRevenue.js index ecc79844..93ae6997 100644 --- a/src/libs/routes_v1/studentRevenue.js +++ b/src/libs/routes_v1/studentRevenue.js @@ -32,6 +32,8 @@ const response = require(`${libs}/middlewares/response`); const ReqQueryFields = require(`${libs}/middlewares/reqQueryFields`); +const aggregateData = require(`${libs}/middlewares/aggregateData`); + const id2str = require(`${libs}/middlewares/id2str`); const config = require(`${libs}/config`); @@ -94,39 +96,15 @@ studentRevenueApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { req.sql.from('receitas') .field('receitas.ano', 'year') .field('receitas.receitas', 'total_receitas') - } - if (req.query.dims && req.query.dims.includes('matriculas_publicas')) { - req.sql.from('receitas') - .field('receitas.ano', 'year') - .field('matriculas_publica', 'matriculas_publica_total') - } - if (req.query.dims && req.query.dims.includes('matriculas_publicas_mais_conveniada')) { - req.sql.from('receitas') - .field('receitas.ano', 'year') - .field('matriculas_publicas_mais_conveniada', 'matriculas_publicas_mais_conveniada_total') - } - if (req.query.dims && req.query.dims.includes('receita_aluno_ano_publica')) { - req.sql.from('receitas') - .field('receitas.ano', 'year') - .field('receuta_aluno_ano_publica', 'receita_aluno_ano_publica_total') - } - if (req.query.dims && req.query.dims.includes('receita_aluno_mes_publica')) { - req.sql.from('receitas') - .field('receitas.ano', 'year') - .field('receita_aluno_mes_publica', 'receita_aluno_mes_publica_total') - } - if (req.query.dims && req.query.dims.includes('receita_aluno_ano_publica_mais_conveniada')) { - req.sql.from('receitas') - .field('receitas.ano', 'year') - .field('receita_aluno_ano_publica_mais_conveniada', 'receita_aluno_ano_publica_mais_conveniada_total') - } - if (req.query.dims && req.query.dims.includes('receita_aluno_mes_publica_mais_conveniada')) { - req.sql.from('receitas') - .field('receitas.ano', 'year') - .field('receita_aluno_mes_publica_mais_conveniada', 'receita_aluno_mes_publica_mais_conveniada_total') + .field('matriculas_publica', 'total_matriculas_publica') + .field('matriculas_publicas_mais_conveniada', 'total_matriculas_publicas_mais_conveniada') + .field('receuta_aluno_ano_publica', 'total_receita_aluno_ano_publica') + .field('receita_aluno_mes_publica', 'total_receita_aluno_mes_publica') + .field('receita_aluno_ano_publica_mais_conveniada', 'total_receita_aluno_ano_publica_mais_conveniada') + .field('receita_aluno_mes_publica_mais_conveniada', 'total_receita_aluno_mes_publica_mais_conveniada') } next(); -}, query, id2str.transform(false), response('student_revenue')); +}, query, aggregateData, id2str.transform(false), response('student_revenue')); module.exports = studentRevenueApp; diff --git a/src/libs/routes_v1/yearsOfStudy.js b/src/libs/routes_v1/yearsOfStudy.js index 0a84f206..a87d833a 100644 --- a/src/libs/routes_v1/yearsOfStudy.js +++ b/src/libs/routes_v1/yearsOfStudy.js @@ -538,7 +538,7 @@ PnadNovoApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { req.sql.from('pnad_novo') .field('pnad_novo.ano_ref', 'year') - .field('media.media_anos_estudo', 'mean') + .field('media.media_anos_estudo', 'average') .field('SQRT(SUM(POWER(pnad_novo.anos_de_estudo - media.media_anos_estudo, 2) * pnad_novo.peso_domicilio_pessoas_com_cal) / SUM(pnad_novo.peso_domicilio_pessoas_com_cal))', 'std_dev') .join(mean, 'media', joinQuery) .where('pnad_novo.ano_ref >= 2019 AND pnad_novo.faixa_etaria >= 6 AND pnad_novo.anos_de_estudo <> 99') -- GitLab From ee0fd06ac54ca24c712cedc5ce2109601ac9296f Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Mon, 24 Feb 2025 11:32:22 -0300 Subject: [PATCH 3/5] feat: update revenue conversion logic and enhance student revenue API with new filters --- src/libs/convert/revenue.js | 14 ++++++++------ src/libs/routes_v1/studentRevenue.js | 26 +++++++++++++++----------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/libs/convert/revenue.js b/src/libs/convert/revenue.js index c4f5865a..9278e280 100644 --- a/src/libs/convert/revenue.js +++ b/src/libs/convert/revenue.js @@ -21,18 +21,20 @@ along with simcaq-node. If not, see <https://www.gnu.org/licenses/>. module.exports = function regionCode(id) { switch (id) { case 1: - return 'Receitas'; + return "Nome Ente" case 2: - return 'MatrÃculas Públicas'; + return 'Receitas'; case 3: - return 'MatrÃculas Públicas Mais Conveniada'; + return 'MatrÃculas Públicas'; case 4: - return 'Receita Aluno Ano Pública' + return 'MatrÃculas Públicas Mais Conveniada'; case 5: - return 'Receita Aluno Mês Pública'; + return 'Receita Aluno Ano Pública' case 6: - return 'Receita Aluno Ano Pública Mais Conveniada'; + return 'Receita Aluno Mês Pública'; case 7: + return 'Receita Aluno Ano Pública Mais Conveniada'; + case 8: return 'Receita Aluno Mês Pública Mais Conveniada'; } }; diff --git a/src/libs/routes_v1/studentRevenue.js b/src/libs/routes_v1/studentRevenue.js index 93ae6997..efce47c5 100644 --- a/src/libs/routes_v1/studentRevenue.js +++ b/src/libs/routes_v1/studentRevenue.js @@ -45,8 +45,9 @@ let rqf = new ReqQueryFields(); studentRevenueApp.use(cache('15 day')); studentRevenueApp.get('/years', (req, res, next) => { - req.sql.from('pnad_novo') + req.sql.from('receitas') .field('DISTINCT receitas.ano', 'year') + .where('receitas.ano is not null') next(); }, query, response('years')); @@ -78,23 +79,24 @@ rqf.addField({ type: 'integer', field: 'ano' } -}).addValue({ - name: 'city', - table: 'receitas', - tableField: 'cod_ibge', - resultField: 'city', - where: { - relation: '=', - type: 'integer', - field: 'cod_ibge' - } }) studentRevenueApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { if (req.query.dims && req.query.dims.includes('receitas')) { + let whereCondition = ""; + let filterId; + const filters = req.query.filter.split(","); + filters.forEach((filter) => { + if (filter.includes("state") || filter.includes("city")) { + filterId = Number(filter.split(":")[1].replace(/"/g, "")); + whereCondition = `receitas.cod_ibge = ${filterId}` + } + }) + req.sql.from('receitas') .field('receitas.ano', 'year') + .field('receitas.nome_ente', 'total_nome_ente') .field('receitas.receitas', 'total_receitas') .field('matriculas_publica', 'total_matriculas_publica') .field('matriculas_publicas_mais_conveniada', 'total_matriculas_publicas_mais_conveniada') @@ -102,6 +104,8 @@ studentRevenueApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { .field('receita_aluno_mes_publica', 'total_receita_aluno_mes_publica') .field('receita_aluno_ano_publica_mais_conveniada', 'total_receita_aluno_ano_publica_mais_conveniada') .field('receita_aluno_mes_publica_mais_conveniada', 'total_receita_aluno_mes_publica_mais_conveniada') + .where(`${whereCondition}`) + .order('receitas.cod_ibge') } next(); -- GitLab From fae5308ff17244d26035581305dc1a341a2db5d0 Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Mon, 24 Feb 2025 11:48:50 -0300 Subject: [PATCH 4/5] refactor: remove unnecessary console log --- src/libs/middlewares/aggregateData.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libs/middlewares/aggregateData.js b/src/libs/middlewares/aggregateData.js index adf21fba..cbe55d8c 100644 --- a/src/libs/middlewares/aggregateData.js +++ b/src/libs/middlewares/aggregateData.js @@ -80,7 +80,6 @@ function aggregateData(req, res, next) { for (const property in r) { // Dados agregados são identificados com a substring 'total_' em sua chave if (property.includes('total_')) { - console.log(property); let data = { total: r[property], year: r.year, -- GitLab From 1998c40928f1aa3ce1486bfc43b4887f7ec2bf50 Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Thu, 27 Feb 2025 10:12:16 -0300 Subject: [PATCH 5/5] feat: add studentRevenueParser middleware to handle city and region filter logic --- src/libs/middlewares/studentRevenueParser.js | 23 ++++++++++++++++++++ src/libs/routes_v1/api.js | 3 ++- src/libs/routes_v1/studentRevenue.js | 18 +++++++++++++-- 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 src/libs/middlewares/studentRevenueParser.js diff --git a/src/libs/middlewares/studentRevenueParser.js b/src/libs/middlewares/studentRevenueParser.js new file mode 100644 index 00000000..954c9c82 --- /dev/null +++ b/src/libs/middlewares/studentRevenueParser.js @@ -0,0 +1,23 @@ +/* +Middleware que remove o campo "region" quando o campo "city" está ativo nos filtros. +Devido à limitações na tabela de dados, quando ambos campos estão ativos o filtro por +"região" funciona incorretamente. +*/ +module.exports = function studentRevenueParser(req, res, next) { + console.log(req.query); + + if (req.query && req.query.filter) { + const jsonString = `{${req.query.filter.replace(/(\w+):/g, '"$1":')}}`; + const obj = JSON.parse(jsonString); + + if ("city" in obj && "region" in obj) { + delete obj.region; + const customString = Object.entries(obj) + .map(([key, value]) => `${key}:"${value}"`) + .join(','); + req.query.filter = customString; + } + } + + next(); +}; diff --git a/src/libs/routes_v1/api.js b/src/libs/routes_v1/api.js index b9af6769..84306018 100644 --- a/src/libs/routes_v1/api.js +++ b/src/libs/routes_v1/api.js @@ -174,6 +174,7 @@ const adjustedLiquidFrequency = require(`${libs}/routes_v1/adjustedLiquidFrequen const iliteracyRate = require(`${libs}/routes_v1/iliteracyRate`); const studentRevenue = require(`${libs}/routes_v1/studentRevenue`); +const studentRevenueParser = require(`${libs}/middlewares/studentRevenueParser`); api.get('/', (req, res) => { res.json({ msg: 'SimCAQ API v1 is running' }); @@ -249,7 +250,7 @@ api.use('/superior_education_conclusion_tax', superiorEducationConclusionTax) api.use('/basic_education_conclusion', basicEducationConclusion); api.use('/adjusted_liquid_frequency', adjustedLiquidFrequency); api.use('/iliteracy_rate', iliteracyRate); -api.use('/student_revenue', studentRevenue); +api.use('/student_revenue', studentRevenueParser, studentRevenue); //Publication api.use('/publication', publication); diff --git a/src/libs/routes_v1/studentRevenue.js b/src/libs/routes_v1/studentRevenue.js index efce47c5..4b8c334e 100644 --- a/src/libs/routes_v1/studentRevenue.js +++ b/src/libs/routes_v1/studentRevenue.js @@ -79,12 +79,26 @@ rqf.addField({ type: 'integer', field: 'ano' } +}).addValue({ + name: 'region', + table: 'estado', + tableField: ['nome_ente', 'cod_ibge', 'id', 'regiao_id'], + resultField: ['state_name', 'state_cod_ibge', 'state_id', 'region_id'], + where: { + relation: '=', + type: 'integer', + field: 'regiao_id', + }, + join: { + primary: 'id', + foreign: 'cod_ibge', + foreignTable: 'receitas' + } }) studentRevenueApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { - if (req.query.dims && req.query.dims.includes('receitas')) { - let whereCondition = ""; + let whereCondition = req.query.filter.includes("region") ? "" : "receitas.cod_ibge = 0"; let filterId; const filters = req.query.filter.split(","); filters.forEach((filter) => { -- GitLab