diff --git a/add_registers.sh b/add_registers.sh index 780b46d4e42ed8357e4d1bb91d30afe31ffcb161..938a243e11346e34eaaf3e0e84307bb3025cd8d6 100755 --- a/add_registers.sh +++ b/add_registers.sh @@ -1,52 +1,54 @@ #!/bin/bash -./insert_resgiter_payment.sh 2016 11 30 -./insert_resgiter_payment.sh 2016 10 31 -./insert_resgiter_payment.sh 2016 09 30 -./insert_resgiter_payment.sh 2016 08 31 -./insert_resgiter_payment.sh 2016 07 31 -./insert_resgiter_payment.sh 2016 06 30 -./insert_resgiter_payment.sh 2016 05 31 -./insert_resgiter_payment.sh 2016 04 30 -./insert_resgiter_payment.sh 2016 03 31 -./insert_resgiter_payment.sh 2016 02 29 -./insert_resgiter_payment.sh 2016 01 31 +echo "Usage: $0 <user> <password>" -./insert_resgiter_payment.sh 2015 12 31 -./insert_resgiter_payment.sh 2015 11 30 -./insert_resgiter_payment.sh 2015 10 31 -./insert_resgiter_payment.sh 2015 09 30 -./insert_resgiter_payment.sh 2015 08 31 -./insert_resgiter_payment.sh 2015 07 31 -./insert_resgiter_payment.sh 2015 06 30 -./insert_resgiter_payment.sh 2015 05 31 -./insert_resgiter_payment.sh 2015 04 30 -./insert_resgiter_payment.sh 2015 03 31 -./insert_resgiter_payment.sh 2015 02 28 -./insert_resgiter_payment.sh 2015 01 31 +./insert_register_payment.sh 2016 11 30 $1 $2 +./insert_register_payment.sh 2016 10 31 $1 $2 +./insert_register_payment.sh 2016 09 30 $1 $2 +./insert_register_payment.sh 2016 08 31 $1 $2 +./insert_register_payment.sh 2016 07 31 $1 $2 +./insert_register_payment.sh 2016 06 30 $1 $2 +./insert_register_payment.sh 2016 05 31 $1 $2 +./insert_register_payment.sh 2016 04 30 $1 $2 +./insert_register_payment.sh 2016 03 31 $1 $2 +./insert_register_payment.sh 2016 02 29 $1 $2 +./insert_register_payment.sh 2016 01 31 $1 $2 -./insert_resgiter_payment.sh 2014 12 31 -./insert_resgiter_payment.sh 2014 11 30 -./insert_resgiter_payment.sh 2014 10 31 -./insert_resgiter_payment.sh 2014 09 30 -./insert_resgiter_payment.sh 2014 08 31 -./insert_resgiter_payment.sh 2014 07 31 -./insert_resgiter_payment.sh 2014 06 30 -./insert_resgiter_payment.sh 2014 05 31 -./insert_resgiter_payment.sh 2014 04 30 -./insert_resgiter_payment.sh 2014 03 31 -./insert_resgiter_payment.sh 2014 02 28 -./insert_resgiter_payment.sh 2014 01 31 +./insert_register_payment.sh 2015 12 31 $1 $2 +./insert_register_payment.sh 2015 11 30 $1 $2 +./insert_register_payment.sh 2015 10 31 $1 $2 +./insert_register_payment.sh 2015 09 30 $1 $2 +./insert_register_payment.sh 2015 08 31 $1 $2 +./insert_register_payment.sh 2015 07 31 $1 $2 +./insert_register_payment.sh 2015 06 30 $1 $2 +./insert_register_payment.sh 2015 05 31 $1 $2 +./insert_register_payment.sh 2015 04 30 $1 $2 +./insert_register_payment.sh 2015 03 31 $1 $2 +./insert_register_payment.sh 2015 02 28 $1 $2 +./insert_register_payment.sh 2015 01 31 $1 $2 -./insert_resgiter_payment.sh 2013 12 31 -./insert_resgiter_payment.sh 2013 11 30 -./insert_resgiter_payment.sh 2013 10 31 -./insert_resgiter_payment.sh 2013 09 30 -./insert_resgiter_payment.sh 2013 08 31 -./insert_resgiter_payment.sh 2013 07 31 -./insert_resgiter_payment.sh 2013 06 30 -./insert_resgiter_payment.sh 2013 05 31 -./insert_resgiter_payment.sh 2013 04 30 -./insert_resgiter_payment.sh 2013 03 31 -./insert_resgiter_payment.sh 2013 02 28 -./insert_resgiter_payment.sh 2013 01 31 +./insert_register_payment.sh 2014 12 31 $1 $2 +./insert_register_payment.sh 2014 11 30 $1 $2 +./insert_register_payment.sh 2014 10 31 $1 $2 +./insert_register_payment.sh 2014 09 30 $1 $2 +./insert_register_payment.sh 2014 08 31 $1 $2 +./insert_register_payment.sh 2014 07 31 $1 $2 +./insert_register_payment.sh 2014 06 30 $1 $2 +./insert_register_payment.sh 2014 05 31 $1 $2 +./insert_register_payment.sh 2014 04 30 $1 $2 +./insert_register_payment.sh 2014 03 31 $1 $2 +./insert_register_payment.sh 2014 02 28 $1 $2 +./insert_register_payment.sh 2014 01 31 $1 $2 + +./insert_register_payment.sh 2013 12 31 $1 $2 +./insert_register_payment.sh 2013 11 30 $1 $2 +./insert_register_payment.sh 2013 10 31 $1 $2 +./insert_register_payment.sh 2013 09 30 $1 $2 +./insert_register_payment.sh 2013 08 31 $1 $2 +./insert_register_payment.sh 2013 07 31 $1 $2 +./insert_register_payment.sh 2013 06 30 $1 $2 +./insert_register_payment.sh 2013 05 31 $1 $2 +./insert_register_payment.sh 2013 04 30 $1 $2 +./insert_register_payment.sh 2013 03 31 $1 $2 +./insert_register_payment.sh 2013 02 28 $1 $2 +./insert_register_payment.sh 2013 01 31 $1 $2 diff --git a/create_config.py b/create_config.py index 24424f07bffabeb1483ee322342984b437117402..473bb861e45179f51023c2900d40cbc01fa3885a 100755 --- a/create_config.py +++ b/create_config.py @@ -4,8 +4,8 @@ import sys, csv, json, math, subprocess from pathlib import Path from subprocess import call -if len(sys.argv) != 4: - print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)>") +if len(sys.argv) != 6: + print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)> <username> <password>") sys.exit() data = { @@ -28,7 +28,9 @@ with open('logstash_config.example') as infile: example = infile.read() output = example % { "timestamp": sys.argv[3] + '/' + sys.argv[2] + '/' + sys.argv[1] + ' 00:00:00' - , "date": sys.argv[1] + '-' + sys.argv[2] } + , "date": sys.argv[1] + '-' + sys.argv[2] + , "user": sys.argv[4] + , "password": sys.argv[5] } with open('logstash_configs/config-' + sys.argv[1] + '-' + sys.argv[2], 'w') as outfile: outfile.write(output) diff --git a/insert_resgiter_payment.sh b/insert_register_payment.sh similarity index 64% rename from insert_resgiter_payment.sh rename to insert_register_payment.sh index d1dc65ffdd22c3286f626ec590a91aa3cd50134b..342fe79a16c0d81d3a6b411d3f812c8afd064544 100755 --- a/insert_resgiter_payment.sh +++ b/insert_register_payment.sh @@ -1,11 +1,11 @@ #!/bin/bash -if [ "$#" -ne 3 ]; then - echo "Usage: $0 <year> <month> <day>" +if [ "$#" -ne 5 ]; then + echo "Usage: $0 <year> <month> <day> <user> <password>" echo "Example: $0 2016 12 01" exit fi -./create_config.py $1 $2 $3 +./create_config.py $1 $2 $3 $4 $5 ./merge_files_es.py configs/config-${1}-${2}.json logstash -f logstash_configs/config-${1}-${2} < ~/transparencia/Dados_Servidores/Processados/${1}${2}.csv diff --git a/logstash_config.example b/logstash_config.example index c2d4084be6955d841b43322bf9d809ac2b073101..905ef0d75f0929e8b4451999709f77428d01bcec 100644 --- a/logstash_config.example +++ b/logstash_config.example @@ -56,8 +56,8 @@ filter { output { elasticsearch { action => "index" - user => "cw14" - password => "123mudar" + user => "%(user)s" + password => "%(password)s" hosts => "http://node1.c3sl.ufpr.br:9200" index => "ufpr-csv-%(date)s" workers => 1 diff --git a/scripts/expenses/add_expenses.sh b/scripts/expenses/add_expenses.sh new file mode 100755 index 0000000000000000000000000000000000000000..2435a53a2e36900a00aabd1ceb9dc5d27421e829 --- /dev/null +++ b/scripts/expenses/add_expenses.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 <user> <password>" + echo "Example: $0 myuser mypass" + exit +fi + +./insert_expenses.sh 2016 11 30 $1 $2 +./insert_expenses.sh 2016 10 31 $1 $2 +./insert_expenses.sh 2016 09 30 $1 $2 +./insert_expenses.sh 2016 08 31 $1 $2 +./insert_expenses.sh 2016 07 31 $1 $2 +./insert_expenses.sh 2016 06 30 $1 $2 +./insert_expenses.sh 2016 05 31 $1 $2 +./insert_expenses.sh 2016 04 30 $1 $2 +./insert_expenses.sh 2016 03 31 $1 $2 +./insert_expenses.sh 2016 02 29 $1 $2 +./insert_expenses.sh 2016 01 31 $1 $2 + +./insert_expenses.sh 2015 12 31 $1 $2 +./insert_expenses.sh 2015 11 30 $1 $2 +./insert_expenses.sh 2015 10 31 $1 $2 +./insert_expenses.sh 2015 09 30 $1 $2 +./insert_expenses.sh 2015 08 31 $1 $2 +./insert_expenses.sh 2015 07 31 $1 $2 +./insert_expenses.sh 2015 06 30 $1 $2 +./insert_expenses.sh 2015 05 31 $1 $2 +./insert_expenses.sh 2015 04 30 $1 $2 +./insert_expenses.sh 2015 03 31 $1 $2 +./insert_expenses.sh 2015 02 28 $1 $2 +./insert_expenses.sh 2015 01 31 $1 $2 + +./insert_expenses.sh 2014 12 31 $1 $2 +./insert_expenses.sh 2014 11 30 $1 $2 +./insert_expenses.sh 2014 10 31 $1 $2 +./insert_expenses.sh 2014 09 30 $1 $2 +./insert_expenses.sh 2014 08 31 $1 $2 +./insert_expenses.sh 2014 07 31 $1 $2 +./insert_expenses.sh 2014 06 30 $1 $2 +./insert_expenses.sh 2014 05 31 $1 $2 +./insert_expenses.sh 2014 04 30 $1 $2 +./insert_expenses.sh 2014 03 31 $1 $2 +./insert_expenses.sh 2014 02 28 $1 $2 +./insert_expenses.sh 2014 01 31 $1 $2 + +./insert_expenses.sh 2013 12 31 $1 $2 +./insert_expenses.sh 2013 11 30 $1 $2 +./insert_expenses.sh 2013 10 31 $1 $2 +./insert_expenses.sh 2013 09 30 $1 $2 +./insert_expenses.sh 2013 08 31 $1 $2 +./insert_expenses.sh 2013 07 31 $1 $2 +./insert_expenses.sh 2013 06 30 $1 $2 +./insert_expenses.sh 2013 05 31 $1 $2 +./insert_expenses.sh 2013 04 30 $1 $2 +./insert_expenses.sh 2013 03 31 $1 $2 +./insert_expenses.sh 2013 02 28 $1 $2 +./insert_expenses.sh 2013 01 31 $1 $2 diff --git a/scripts/expenses/create_expenses_config.py b/scripts/expenses/create_expenses_config.py new file mode 100755 index 0000000000000000000000000000000000000000..3f6ac7fdbca7bbb402b179403e8bd4e68694090c --- /dev/null +++ b/scripts/expenses/create_expenses_config.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import sys, csv, json, math, subprocess +from pathlib import Path +from subprocess import call + +if len(sys.argv) != 6: + print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)> <username> <password>") + sys.exit() + +with open('logstash_config.example') as infile: + example = infile.read() + +output = example % { "timestamp": sys.argv[3] + '/' + sys.argv[2] + '/' + sys.argv[1] + ' 00:00:00' + , "date": sys.argv[1] + '-' + sys.argv[2] + , "user": sys.argv[4] + , "password": sys.argv[5] } + +with open('logstash_configs/config-' + sys.argv[1] + '-' + sys.argv[2], 'w') as outfile: + outfile.write(output) diff --git a/scripts/expenses/insert_expenses.sh b/scripts/expenses/insert_expenses.sh new file mode 100755 index 0000000000000000000000000000000000000000..900c96779d0b883948ffa5d09de917e156f7071a --- /dev/null +++ b/scripts/expenses/insert_expenses.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ "$#" -ne 5 ]; then + echo "Usage: $0 <year> <month> <day> <user> <password>" + echo "Example: $0 2016 12 31 myuser mypass" + exit +fi + +./create_expenses_config.py $1 $2 $3 $4 $5 +logstash -f logstash_configs/config-${1}-${2} < ~/transparencia/Favorecidos/Processados/${1}${2}.csv diff --git a/scripts/expenses/logstash_config.example b/scripts/expenses/logstash_config.example new file mode 100644 index 0000000000000000000000000000000000000000..52e0c58be35e4e438422ceb2d846066eb908867c --- /dev/null +++ b/scripts/expenses/logstash_config.example @@ -0,0 +1,45 @@ +input { + stdin { + codec => plain { + charset => "Windows-1252" + } + } +} + +filter { + csv { + columns => ["Código Órgão Superior","Nome Órgão Superior","Código Órgão","Nome Órgao","Código Unidade Gestora","Nome Unidade Gestora","Código Grupo Despesa","Nome Grupo Despesa","Código Elemento Despesa","Nome Elemento Despesa","Código Função","Nome Função","Código Subfunção","Nome Subfunção","Código Programa","Nome Programa","Código Ação","Nome Ação","Linguagem Cidadã","Código Favorecido","Nome Favorecido","Número Documento","Gestão Pagamento","Data Pagamento","Valor"] + separator => " " + add_field => { "timestamp" => "%(timestamp)s" } + } + mutate { + convert => { "Código Órgão Superior" => "integer" } + convert => { "Código Órgão" => "integer" } + convert => { "Código Unidade Gestora" => "integer" } + convert => { "Código Grupo Despesa" => "integer" } + convert => { "Código Elemento Despesa" => "integer" } + convert => { "Código Função" => "integer" } + convert => { "Código Subfunção" => "integer" } + convert => { "Código Programa" => "integer" } + convert => { "Código Ação" => "integer" } + convert => { "Código Favorecido" => "integer" } + convert => { "Gestão Pagamento" => "integer" } + convert => { "Valor" => "float" } + } + date { + match => [ "timestamp", "dd/MM/YYYY HH:mm:ss", "ISO8601" ] + target => [ "@timestamp" ] + } +} + +output { + elasticsearch { + action => "index" + user => "%(user)s" + password => "%(password)s" + hosts => "http://node1.c3sl.ufpr.br:9200" + index => "ufpr-gastos-%(date)s" + workers => 1 + } + stdout {} +}