From 1d347d73bba1c8b10ce5d8b94505943fb0d884b7 Mon Sep 17 00:00:00 2001
From: Cristian Weiland <cw14@inf.ufpr.br>
Date: Tue, 14 Mar 2017 11:26:26 -0300
Subject: [PATCH] Issue #20: Create master script to insert data and remove day
parameter from insert_expenses.sh
Signed-off-by: Cristian Weiland <cw14@inf.ufpr.br>
---
scripts/curl/delete_index.sh | 6 ++
scripts/expenses/insert_expenses.sh | 17 +++--
scripts/expenses/logstash_config.example | 8 +-
scripts/expenses/process_expenses.sh | 94 ++++++++++++------------
scripts/insert_data.sh | 16 ++++
5 files changed, 87 insertions(+), 54 deletions(-)
create mode 100755 scripts/curl/delete_index.sh
create mode 100755 scripts/insert_data.sh
diff --git a/scripts/curl/delete_index.sh b/scripts/curl/delete_index.sh
new file mode 100755
index 0000000..ca35320
--- /dev/null
+++ b/scripts/curl/delete_index.sh
@@ -0,0 +1,6 @@
+if [ "$#" -ne 1 ]; then
+ echo "Usage: $0 <index-name>"
+ exit
+fi
+
+curl -XDELETE "localhost:9200/$1?pretty"
diff --git a/scripts/expenses/insert_expenses.sh b/scripts/expenses/insert_expenses.sh
index a826346..a05e7a7 100755
--- a/scripts/expenses/insert_expenses.sh
+++ b/scripts/expenses/insert_expenses.sh
@@ -3,7 +3,7 @@
# This script is the one that should be called to insert data from one month.
# Input: Year, month and day from the data to be inserted, ElasticSearch's user and password. The day should be the last day of the month.
-# Example: ./insert_expenses.sh 2016 10 31 myuser mypass
+# Example: ./insert_expenses.sh 2016 10 myuser mypass
# It has 4 steps:
# 1- Download files and put them in the right location.
# 2- Generate logstash config file via create_expenses_config.py.
@@ -11,12 +11,19 @@
# 4- Insert data in ElasticSearch via logstash, using the config file created and the CSV created by resume_expenses.sh.
# Output: The commands/scripts outputs.
-if [ "$#" -ne 5 ]; then
- echo "Usage: $0 <year> <month> <day> <user> <password>"
- echo "Example: $0 2016 12 31 myuser mypass"
+if [ "$#" -ne 4 ]; then
+ echo "Usage: $0 <year> <month> <user> <password>"
+ echo "Example: $0 2016 12 myuser mypass"
exit
fi
+# Getting the Last day of this month (Using date 2016-05-15 as example):
+# First, get next month (201606).
+aux=$(date +%Y%m -d "$(date +%Y%m15) next month")
+# Append day 01 (20160601).
+temp=$(date -d "${aux}01")
+# Remove 1 day: 20160531, get only day: 31.
+day=$(date -d "$temp - 1 day" "+%d")
ym=$1-$2
dataPath="../../data/"
@@ -44,7 +51,7 @@ unzip $path$ym/${1}${2}_GastosDiretos.zip -d $path$ym/
rm $path$ym/${1}${2}_GastosDiretos.zip
# Step 2:
-./create_expenses_config.py $1 $2 $3 $4 $5
+./create_expenses_config.py $1 $2 $day $3 $4
# Step 3:
./resume_expenses.sh ../../data/expenses/ ${1}-${2}
# Step 4:
diff --git a/scripts/expenses/logstash_config.example b/scripts/expenses/logstash_config.example
index 52e0c58..c5b0ee6 100644
--- a/scripts/expenses/logstash_config.example
+++ b/scripts/expenses/logstash_config.example
@@ -30,6 +30,10 @@ filter {
match => [ "timestamp", "dd/MM/YYYY HH:mm:ss", "ISO8601" ]
target => [ "@timestamp" ]
}
+ date {
+ match => [ "Data Pagamento", "dd/MM/YYYY" ]
+ target => [ "Data Pagamento Timestamp" ]
+ }
}
output {
@@ -37,8 +41,8 @@ output {
action => "index"
user => "%(user)s"
password => "%(password)s"
- hosts => "http://node1.c3sl.ufpr.br:9200"
- index => "ufpr-gastos-%(date)s"
+ hosts => "localhost:9200"
+ index => "ufpr-gastos-pagamentos-%(date)s"
workers => 1
}
stdout {}
diff --git a/scripts/expenses/process_expenses.sh b/scripts/expenses/process_expenses.sh
index b158da8..4acae23 100755
--- a/scripts/expenses/process_expenses.sh
+++ b/scripts/expenses/process_expenses.sh
@@ -12,53 +12,53 @@ if [ "$#" -ne 2 ]; then
exit
fi
-./insert_expenses.sh 2016 11 30 $1 $2
-./insert_expenses.sh 2016 10 31 $1 $2
-./insert_expenses.sh 2016 09 30 $1 $2
-./insert_expenses.sh 2016 08 31 $1 $2
-./insert_expenses.sh 2016 07 31 $1 $2
-./insert_expenses.sh 2016 06 30 $1 $2
-./insert_expenses.sh 2016 05 31 $1 $2
-./insert_expenses.sh 2016 04 30 $1 $2
-./insert_expenses.sh 2016 03 31 $1 $2
-./insert_expenses.sh 2016 02 29 $1 $2
-./insert_expenses.sh 2016 01 31 $1 $2
+./insert_expenses.sh 2016 11 $1 $2
+./insert_expenses.sh 2016 10 $1 $2
+./insert_expenses.sh 2016 09 $1 $2
+./insert_expenses.sh 2016 08 $1 $2
+./insert_expenses.sh 2016 07 $1 $2
+./insert_expenses.sh 2016 06 $1 $2
+./insert_expenses.sh 2016 05 $1 $2
+./insert_expenses.sh 2016 04 $1 $2
+./insert_expenses.sh 2016 03 $1 $2
+./insert_expenses.sh 2016 02 $1 $2
+./insert_expenses.sh 2016 01 $1 $2
-./insert_expenses.sh 2015 12 31 $1 $2
-./insert_expenses.sh 2015 11 30 $1 $2
-./insert_expenses.sh 2015 10 31 $1 $2
-./insert_expenses.sh 2015 09 30 $1 $2
-./insert_expenses.sh 2015 08 31 $1 $2
-./insert_expenses.sh 2015 07 31 $1 $2
-./insert_expenses.sh 2015 06 30 $1 $2
-./insert_expenses.sh 2015 05 31 $1 $2
-./insert_expenses.sh 2015 04 30 $1 $2
-./insert_expenses.sh 2015 03 31 $1 $2
-./insert_expenses.sh 2015 02 28 $1 $2
-./insert_expenses.sh 2015 01 31 $1 $2
+./insert_expenses.sh 2015 12 $1 $2
+./insert_expenses.sh 2015 11 $1 $2
+./insert_expenses.sh 2015 10 $1 $2
+./insert_expenses.sh 2015 09 $1 $2
+./insert_expenses.sh 2015 08 $1 $2
+./insert_expenses.sh 2015 07 $1 $2
+./insert_expenses.sh 2015 06 $1 $2
+./insert_expenses.sh 2015 05 $1 $2
+./insert_expenses.sh 2015 04 $1 $2
+./insert_expenses.sh 2015 03 $1 $2
+./insert_expenses.sh 2015 02 $1 $2
+./insert_expenses.sh 2015 01 $1 $2
-./insert_expenses.sh 2014 12 31 $1 $2
-./insert_expenses.sh 2014 11 30 $1 $2
-./insert_expenses.sh 2014 10 31 $1 $2
-./insert_expenses.sh 2014 09 30 $1 $2
-./insert_expenses.sh 2014 08 31 $1 $2
-./insert_expenses.sh 2014 07 31 $1 $2
-./insert_expenses.sh 2014 06 30 $1 $2
-./insert_expenses.sh 2014 05 31 $1 $2
-./insert_expenses.sh 2014 04 30 $1 $2
-./insert_expenses.sh 2014 03 31 $1 $2
-./insert_expenses.sh 2014 02 28 $1 $2
-./insert_expenses.sh 2014 01 31 $1 $2
+./insert_expenses.sh 2014 12 $1 $2
+./insert_expenses.sh 2014 11 $1 $2
+./insert_expenses.sh 2014 10 $1 $2
+./insert_expenses.sh 2014 09 $1 $2
+./insert_expenses.sh 2014 08 $1 $2
+./insert_expenses.sh 2014 07 $1 $2
+./insert_expenses.sh 2014 06 $1 $2
+./insert_expenses.sh 2014 05 $1 $2
+./insert_expenses.sh 2014 04 $1 $2
+./insert_expenses.sh 2014 03 $1 $2
+./insert_expenses.sh 2014 02 $1 $2
+./insert_expenses.sh 2014 01 $1 $2
-./insert_expenses.sh 2013 12 31 $1 $2
-./insert_expenses.sh 2013 11 30 $1 $2
-./insert_expenses.sh 2013 10 31 $1 $2
-./insert_expenses.sh 2013 09 30 $1 $2
-./insert_expenses.sh 2013 08 31 $1 $2
-./insert_expenses.sh 2013 07 31 $1 $2
-./insert_expenses.sh 2013 06 30 $1 $2
-./insert_expenses.sh 2013 05 31 $1 $2
-./insert_expenses.sh 2013 04 30 $1 $2
-./insert_expenses.sh 2013 03 31 $1 $2
-./insert_expenses.sh 2013 02 28 $1 $2
-./insert_expenses.sh 2013 01 31 $1 $2
+./insert_expenses.sh 2013 12 $1 $2
+./insert_expenses.sh 2013 11 $1 $2
+./insert_expenses.sh 2013 10 $1 $2
+./insert_expenses.sh 2013 09 $1 $2
+./insert_expenses.sh 2013 08 $1 $2
+./insert_expenses.sh 2013 07 $1 $2
+./insert_expenses.sh 2013 06 $1 $2
+./insert_expenses.sh 2013 05 $1 $2
+./insert_expenses.sh 2013 04 $1 $2
+./insert_expenses.sh 2013 03 $1 $2
+./insert_expenses.sh 2013 02 $1 $2
+./insert_expenses.sh 2013 01 $1 $2
diff --git a/scripts/insert_data.sh b/scripts/insert_data.sh
new file mode 100755
index 0000000..55dd4a9
--- /dev/null
+++ b/scripts/insert_data.sh
@@ -0,0 +1,16 @@
+# This script will call every script needed to insert data.
+
+if [ "$#" -ne 4 ]; then
+ echo "Usage: $0 <year> <month> <user> <password>"
+ echo "Example: $0 2016 12 myuser mypass"
+ exit
+fi
+
+# First, insert Expenses data.
+(cd expenses && ./insert_expenses.sh $1 $2 $3 $4)
+
+# Now, insert Workers data.
+(cd workers && ./insert_register_payment.sh $1 $2 $3 $4)
+
+# We should now insert Travel allowance data.
+(cd travel_allowances && ./insert_travel_allowances.sh $1 $2 $3 $4)
--
GitLab