From f8e05a8108ad0b0303c0319d21ef3b65450c618a Mon Sep 17 00:00:00 2001
From: Cristian Weiland <cw14@inf.ufpr.br>
Date: Tue, 7 Feb 2017 11:24:11 -0200
Subject: [PATCH] Add script to insert expenses

Signed-off-by: Cristian Weiland <cw14@inf.ufpr.br>
---
 add_registers.sh                              | 96 ++++++++++---------
 create_config.py                              |  8 +-
 ...r_payment.sh => insert_register_payment.sh |  6 +-
 logstash_config.example                       |  4 +-
 scripts/expenses/add_expenses.sh              | 58 +++++++++++
 scripts/expenses/create_expenses_config.py    | 20 ++++
 scripts/expenses/insert_expenses.sh           | 10 ++
 scripts/expenses/logstash_config.example      | 45 +++++++++
 8 files changed, 192 insertions(+), 55 deletions(-)
 rename insert_resgiter_payment.sh => insert_register_payment.sh (64%)
 create mode 100755 scripts/expenses/add_expenses.sh
 create mode 100755 scripts/expenses/create_expenses_config.py
 create mode 100755 scripts/expenses/insert_expenses.sh
 create mode 100644 scripts/expenses/logstash_config.example

diff --git a/add_registers.sh b/add_registers.sh
index 780b46d..938a243 100755
--- a/add_registers.sh
+++ b/add_registers.sh
@@ -1,52 +1,54 @@
 #!/bin/bash
 
-./insert_resgiter_payment.sh 2016 11 30
-./insert_resgiter_payment.sh 2016 10 31
-./insert_resgiter_payment.sh 2016 09 30
-./insert_resgiter_payment.sh 2016 08 31
-./insert_resgiter_payment.sh 2016 07 31
-./insert_resgiter_payment.sh 2016 06 30
-./insert_resgiter_payment.sh 2016 05 31
-./insert_resgiter_payment.sh 2016 04 30
-./insert_resgiter_payment.sh 2016 03 31
-./insert_resgiter_payment.sh 2016 02 29
-./insert_resgiter_payment.sh 2016 01 31
+echo "Usage: $0 <user> <password>"
 
-./insert_resgiter_payment.sh 2015 12 31
-./insert_resgiter_payment.sh 2015 11 30
-./insert_resgiter_payment.sh 2015 10 31
-./insert_resgiter_payment.sh 2015 09 30
-./insert_resgiter_payment.sh 2015 08 31
-./insert_resgiter_payment.sh 2015 07 31
-./insert_resgiter_payment.sh 2015 06 30
-./insert_resgiter_payment.sh 2015 05 31
-./insert_resgiter_payment.sh 2015 04 30
-./insert_resgiter_payment.sh 2015 03 31
-./insert_resgiter_payment.sh 2015 02 28
-./insert_resgiter_payment.sh 2015 01 31
+./insert_register_payment.sh 2016 11 30 $1 $2
+./insert_register_payment.sh 2016 10 31 $1 $2
+./insert_register_payment.sh 2016 09 30 $1 $2
+./insert_register_payment.sh 2016 08 31 $1 $2
+./insert_register_payment.sh 2016 07 31 $1 $2
+./insert_register_payment.sh 2016 06 30 $1 $2
+./insert_register_payment.sh 2016 05 31 $1 $2
+./insert_register_payment.sh 2016 04 30 $1 $2
+./insert_register_payment.sh 2016 03 31 $1 $2
+./insert_register_payment.sh 2016 02 29 $1 $2
+./insert_register_payment.sh 2016 01 31 $1 $2
 
-./insert_resgiter_payment.sh 2014 12 31
-./insert_resgiter_payment.sh 2014 11 30
-./insert_resgiter_payment.sh 2014 10 31
-./insert_resgiter_payment.sh 2014 09 30
-./insert_resgiter_payment.sh 2014 08 31
-./insert_resgiter_payment.sh 2014 07 31
-./insert_resgiter_payment.sh 2014 06 30
-./insert_resgiter_payment.sh 2014 05 31
-./insert_resgiter_payment.sh 2014 04 30
-./insert_resgiter_payment.sh 2014 03 31
-./insert_resgiter_payment.sh 2014 02 28
-./insert_resgiter_payment.sh 2014 01 31
+./insert_register_payment.sh 2015 12 31 $1 $2
+./insert_register_payment.sh 2015 11 30 $1 $2
+./insert_register_payment.sh 2015 10 31 $1 $2
+./insert_register_payment.sh 2015 09 30 $1 $2
+./insert_register_payment.sh 2015 08 31 $1 $2
+./insert_register_payment.sh 2015 07 31 $1 $2
+./insert_register_payment.sh 2015 06 30 $1 $2
+./insert_register_payment.sh 2015 05 31 $1 $2
+./insert_register_payment.sh 2015 04 30 $1 $2
+./insert_register_payment.sh 2015 03 31 $1 $2
+./insert_register_payment.sh 2015 02 28 $1 $2
+./insert_register_payment.sh 2015 01 31 $1 $2
 
-./insert_resgiter_payment.sh 2013 12 31
-./insert_resgiter_payment.sh 2013 11 30
-./insert_resgiter_payment.sh 2013 10 31
-./insert_resgiter_payment.sh 2013 09 30
-./insert_resgiter_payment.sh 2013 08 31
-./insert_resgiter_payment.sh 2013 07 31
-./insert_resgiter_payment.sh 2013 06 30
-./insert_resgiter_payment.sh 2013 05 31
-./insert_resgiter_payment.sh 2013 04 30
-./insert_resgiter_payment.sh 2013 03 31
-./insert_resgiter_payment.sh 2013 02 28
-./insert_resgiter_payment.sh 2013 01 31
+./insert_register_payment.sh 2014 12 31 $1 $2
+./insert_register_payment.sh 2014 11 30 $1 $2
+./insert_register_payment.sh 2014 10 31 $1 $2
+./insert_register_payment.sh 2014 09 30 $1 $2
+./insert_register_payment.sh 2014 08 31 $1 $2
+./insert_register_payment.sh 2014 07 31 $1 $2
+./insert_register_payment.sh 2014 06 30 $1 $2
+./insert_register_payment.sh 2014 05 31 $1 $2
+./insert_register_payment.sh 2014 04 30 $1 $2
+./insert_register_payment.sh 2014 03 31 $1 $2
+./insert_register_payment.sh 2014 02 28 $1 $2
+./insert_register_payment.sh 2014 01 31 $1 $2
+
+./insert_register_payment.sh 2013 12 31 $1 $2
+./insert_register_payment.sh 2013 11 30 $1 $2
+./insert_register_payment.sh 2013 10 31 $1 $2
+./insert_register_payment.sh 2013 09 30 $1 $2
+./insert_register_payment.sh 2013 08 31 $1 $2
+./insert_register_payment.sh 2013 07 31 $1 $2
+./insert_register_payment.sh 2013 06 30 $1 $2
+./insert_register_payment.sh 2013 05 31 $1 $2
+./insert_register_payment.sh 2013 04 30 $1 $2
+./insert_register_payment.sh 2013 03 31 $1 $2
+./insert_register_payment.sh 2013 02 28 $1 $2
+./insert_register_payment.sh 2013 01 31 $1 $2
diff --git a/create_config.py b/create_config.py
index 24424f0..473bb86 100755
--- a/create_config.py
+++ b/create_config.py
@@ -4,8 +4,8 @@ import sys, csv, json, math, subprocess
 from pathlib import Path
 from subprocess import call
 
-if len(sys.argv) != 4:
-    print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)>")
+if len(sys.argv) != 6:
+    print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)> <username> <password>")
     sys.exit()
 
 data = {
@@ -28,7 +28,9 @@ with open('logstash_config.example') as infile:
 	example = infile.read()
 
 output = example % { "timestamp": sys.argv[3] + '/' + sys.argv[2] + '/' + sys.argv[1] + ' 00:00:00'
-					 , "date": sys.argv[1] + '-' + sys.argv[2] }
+					 , "date": sys.argv[1] + '-' + sys.argv[2]
+					 , "user": sys.argv[4]
+					 , "password": sys.argv[5] }
 
 with open('logstash_configs/config-' + sys.argv[1] + '-' + sys.argv[2], 'w') as outfile:
 	outfile.write(output)
diff --git a/insert_resgiter_payment.sh b/insert_register_payment.sh
similarity index 64%
rename from insert_resgiter_payment.sh
rename to insert_register_payment.sh
index d1dc65f..342fe79 100755
--- a/insert_resgiter_payment.sh
+++ b/insert_register_payment.sh
@@ -1,11 +1,11 @@
 #!/bin/bash
 
-if [ "$#" -ne 3 ]; then
-	echo "Usage: $0 <year> <month> <day>"
+if [ "$#" -ne 5 ]; then
+	echo "Usage: $0 <year> <month> <day> <user> <password>"
 	echo "Example: $0 2016 12 01"
 	exit
 fi
 
-./create_config.py $1 $2 $3
+./create_config.py $1 $2 $3 $4 $5
 ./merge_files_es.py configs/config-${1}-${2}.json
 logstash -f logstash_configs/config-${1}-${2} < ~/transparencia/Dados_Servidores/Processados/${1}${2}.csv
diff --git a/logstash_config.example b/logstash_config.example
index c2d4084..905ef0d 100644
--- a/logstash_config.example
+++ b/logstash_config.example
@@ -56,8 +56,8 @@ filter {
 output {
 	elasticsearch {
 		action => "index"
-		user => "cw14"
-		password => "123mudar"
+		user => "%(user)s"
+		password => "%(password)s"
 		hosts => "http://node1.c3sl.ufpr.br:9200"
 		index => "ufpr-csv-%(date)s"
 		workers => 1
diff --git a/scripts/expenses/add_expenses.sh b/scripts/expenses/add_expenses.sh
new file mode 100755
index 0000000..2435a53
--- /dev/null
+++ b/scripts/expenses/add_expenses.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+if [ "$#" -ne 2 ]; then
+	echo "Usage: $0 <user> <password>"
+	echo "Example: $0 myuser mypass"
+	exit
+fi
+
+./insert_expenses.sh 2016 11 30 $1 $2
+./insert_expenses.sh 2016 10 31 $1 $2
+./insert_expenses.sh 2016 09 30 $1 $2
+./insert_expenses.sh 2016 08 31 $1 $2
+./insert_expenses.sh 2016 07 31 $1 $2
+./insert_expenses.sh 2016 06 30 $1 $2
+./insert_expenses.sh 2016 05 31 $1 $2
+./insert_expenses.sh 2016 04 30 $1 $2
+./insert_expenses.sh 2016 03 31 $1 $2
+./insert_expenses.sh 2016 02 29 $1 $2
+./insert_expenses.sh 2016 01 31 $1 $2
+
+./insert_expenses.sh 2015 12 31 $1 $2
+./insert_expenses.sh 2015 11 30 $1 $2
+./insert_expenses.sh 2015 10 31 $1 $2
+./insert_expenses.sh 2015 09 30 $1 $2
+./insert_expenses.sh 2015 08 31 $1 $2
+./insert_expenses.sh 2015 07 31 $1 $2
+./insert_expenses.sh 2015 06 30 $1 $2
+./insert_expenses.sh 2015 05 31 $1 $2
+./insert_expenses.sh 2015 04 30 $1 $2
+./insert_expenses.sh 2015 03 31 $1 $2
+./insert_expenses.sh 2015 02 28 $1 $2
+./insert_expenses.sh 2015 01 31 $1 $2
+
+./insert_expenses.sh 2014 12 31 $1 $2
+./insert_expenses.sh 2014 11 30 $1 $2
+./insert_expenses.sh 2014 10 31 $1 $2
+./insert_expenses.sh 2014 09 30 $1 $2
+./insert_expenses.sh 2014 08 31 $1 $2
+./insert_expenses.sh 2014 07 31 $1 $2
+./insert_expenses.sh 2014 06 30 $1 $2
+./insert_expenses.sh 2014 05 31 $1 $2
+./insert_expenses.sh 2014 04 30 $1 $2
+./insert_expenses.sh 2014 03 31 $1 $2
+./insert_expenses.sh 2014 02 28 $1 $2
+./insert_expenses.sh 2014 01 31 $1 $2
+
+./insert_expenses.sh 2013 12 31 $1 $2
+./insert_expenses.sh 2013 11 30 $1 $2
+./insert_expenses.sh 2013 10 31 $1 $2
+./insert_expenses.sh 2013 09 30 $1 $2
+./insert_expenses.sh 2013 08 31 $1 $2
+./insert_expenses.sh 2013 07 31 $1 $2
+./insert_expenses.sh 2013 06 30 $1 $2
+./insert_expenses.sh 2013 05 31 $1 $2
+./insert_expenses.sh 2013 04 30 $1 $2
+./insert_expenses.sh 2013 03 31 $1 $2
+./insert_expenses.sh 2013 02 28 $1 $2
+./insert_expenses.sh 2013 01 31 $1 $2
diff --git a/scripts/expenses/create_expenses_config.py b/scripts/expenses/create_expenses_config.py
new file mode 100755
index 0000000..3f6ac7f
--- /dev/null
+++ b/scripts/expenses/create_expenses_config.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+import sys, csv, json, math, subprocess
+from pathlib import Path
+from subprocess import call
+
+if len(sys.argv) != 6:
+    print("Usage: " + sys.argv[0] + " <year (2016)> <month (01)> <day (31)> <username> <password>")
+    sys.exit()
+
+with open('logstash_config.example') as infile:
+	example = infile.read()
+
+output = example % { "timestamp": sys.argv[3] + '/' + sys.argv[2] + '/' + sys.argv[1] + ' 00:00:00'
+					 , "date": sys.argv[1] + '-' + sys.argv[2]
+					 , "user": sys.argv[4]
+					 , "password": sys.argv[5] }
+
+with open('logstash_configs/config-' + sys.argv[1] + '-' + sys.argv[2], 'w') as outfile:
+	outfile.write(output)
diff --git a/scripts/expenses/insert_expenses.sh b/scripts/expenses/insert_expenses.sh
new file mode 100755
index 0000000..900c967
--- /dev/null
+++ b/scripts/expenses/insert_expenses.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+if [ "$#" -ne 5 ]; then
+	echo "Usage: $0 <year> <month> <day> <user> <password>"
+	echo "Example: $0 2016 12 31 myuser mypass"
+	exit
+fi
+
+./create_expenses_config.py $1 $2 $3 $4 $5
+logstash -f logstash_configs/config-${1}-${2} < ~/transparencia/Favorecidos/Processados/${1}${2}.csv
diff --git a/scripts/expenses/logstash_config.example b/scripts/expenses/logstash_config.example
new file mode 100644
index 0000000..52e0c58
--- /dev/null
+++ b/scripts/expenses/logstash_config.example
@@ -0,0 +1,45 @@
+input {
+	stdin {
+		codec => plain {
+			charset => "Windows-1252"
+		}
+	}
+}
+
+filter {
+	csv {
+		columns => ["Código Órgão Superior","Nome Órgão Superior","Código Órgão","Nome Órgao","Código Unidade Gestora","Nome Unidade Gestora","Código Grupo Despesa","Nome Grupo Despesa","Código Elemento Despesa","Nome Elemento Despesa","Código Função","Nome Função","Código Subfunção","Nome Subfunção","Código Programa","Nome Programa","Código Ação","Nome Ação","Linguagem Cidadã","Código Favorecido","Nome Favorecido","Número Documento","Gestão Pagamento","Data Pagamento","Valor"]
+		separator => "	"
+		add_field => { "timestamp" => "%(timestamp)s" }
+	}
+	mutate {
+		convert => { "Código Órgão Superior" => "integer" }
+		convert => { "Código Órgão" => "integer" }
+		convert => { "Código Unidade Gestora" => "integer" }
+		convert => { "Código Grupo Despesa" => "integer" }
+		convert => { "Código Elemento Despesa" => "integer" }
+		convert => { "Código Função" => "integer" }
+		convert => { "Código Subfunção" => "integer" }
+		convert => { "Código Programa" => "integer" }
+		convert => { "Código Ação" => "integer" }
+		convert => { "Código Favorecido" => "integer" }
+		convert => { "Gestão Pagamento" => "integer" }
+		convert => { "Valor" => "float" }
+	}
+	date {
+		match => [ "timestamp", "dd/MM/YYYY HH:mm:ss", "ISO8601" ]
+		target => [ "@timestamp" ]
+	}
+}
+
+output {
+	elasticsearch {
+		action => "index"
+		user => "%(user)s"
+		password => "%(password)s"
+		hosts => "http://node1.c3sl.ufpr.br:9200"
+		index => "ufpr-gastos-%(date)s"
+		workers => 1
+	}
+	stdout {}
+}
-- 
GitLab