Commit a1047663 authored by Eduardo L. Buratti's avatar Eduardo L. Buratti

Merge remote-tracking branch 'origin/fix-net-collect'

parents d22527d2 83e54b28
......@@ -5,12 +5,10 @@ SRCDIR = $(TOPDIR)/src
LIBDIR = $(TOPDIR)/lib
BINDIR = bin
LIBXMLDIR = lib
NETDIR = $(TOPDIR)/net
all:
cd $(LIBDIR) && make all
cd $(SRCDIR) && make all
cd $(NETDIR) && make all
collect-parser:
cd $(SRCDIR) && make all
......@@ -18,7 +16,6 @@ collect-parser:
install: all
@cd $(LIBDIR) && make install
@cd $(SRCDIR) && make install
@cd $(NETDIR) && make install
@echo "Setting variables in collect.conf..."
@grep -v "XMLPARSER=" collect.conf > temp.conf\
&& cat temp.conf > collect.conf && rm -f temp.conf
......@@ -27,9 +24,7 @@ install: all
clean:
cd $(LIBDIR) && make clean
cd $(SRCDIR) && make clean
cd $(NETDIR) && make clean
distclean:
cd $(LIBDIR) && make distclean
cd $(SRCDIR) && make distclean
cd $(NETDIR) && make distclean
CC = gcc
CCFLAGS = -m32 -static -g -Wall
BIN = discover-network
all:
$(CC) $(CCFLAGS) -o $(BIN) discover-network.c
install: all
clean:
@find $(PWD) -type f \( -name '*~' -o -name '*.bak' -o -name 'core*' \) \
-exec rm -f {} \;
distclean: clean
@rm -f $(PWD)/$(BIN)
......@@ -20,41 +20,56 @@
# USA.
#------------------------------------------------------------------------------
# Function: date_comparator
# This function compares the dates of the collected data files.
function date_comparator()
# Function: check_traffic_date
# Checks if the date written in TRAFFIC file is today or creates a
# new TRAFFIC if its not.
function check_traffic_date()
{
# If traffic.txt file doesn't exist, this may the first time the script is
# running. Thus, traffic.txt and previous-traffic.txt files are created.
# The first line of traffic.txt contains the date of the first execution of
# script
# If there is no TRAFFIC (probably its the first run), then just create a
# new one.
if ! test -f "$TRAFFIC"; then
echo $(date +%F) > $TRAFFIC
touch $PREVIOUS_TRAFFIC
else
# If previous-traffic.txt was deleted, copy the contents of traffic.txt to
# previous-traffic.txt. This is necessary to avoid FILE_DATE returning a
# failure
if ! test -f "$PREVIOUS_TRAFFIC"; then
cp $TRAFFIC $PREVIOUS_TRAFFIC
echo $(date +%F) > $TRAFFIC
# If previous-traffic.txt exists, check if the last modification date is the
# current date or not. If it's the second case, it means it's a new day and
# traffic.txt will be copied to previous-traffic.txt. traffic.txt will
# contain the new collected data
else
FILE_DATE=$(date -d"$(ls -l --time-style="+%F" \
$PREVIOUS_TRAFFIC | awk '{ print $6 }')" +%s)
CURRENT_DATE=$(date -d"$(date +%F)" +%s)
if test $FILE_DATE -ne $CURRENT_DATE; then
cp $TRAFFIC $PREVIOUS_TRAFFIC
echo $(date +%F) > $TRAFFIC
fi
fi
return
fi
local CURRENT_DATE=$(date -d"$(date +%F)" +%s)
local TRAFFIC_DATE=$(date -d"$(head -n1 $TRAFFIC)" +%s)
# If it midnight (INTERVALID = 287), then consider current date as
# yesterday (subtract one day from current date). Thats because the
# data collect belongs to an interval beginning yesterday 23:55 and
# only ending today, at midnight.
if test $INTERVALID -eq 287; then
CURRENT_DATE=$(($CURRENT_DATE - 86400))
fi
# Finally, create a new TRAFFIC (keeping the previous one as
# PREVIOUS_TRAFFIC) if the dates doesnt match.
if test $CURRENT_DATE -ne $TRAFFIC_DATE; then
cp $TRAFFIC $PREVIOUS_TRAFFIC
echo $(date +%F) > $TRAFFIC
fi
}
#------------------------------------------------------------------------------
# Function: get_interval_id
# Returns a number identifying the current five minutes interval
function get_interval_id()
{
INTERVALID=$(date +%H:%M | awk -F':' '{print $1 * 12 + \
int($2 / 5 + 0.5) - 1}')
# INTERVALID equal to -1 means midnight 00:00, but midnight needs
# to be the last id possible, so force it to 287
if test $INTERVALID -eq -1; then
INTERVALID=287
fi
}
#------------------------------------------------------------------------------
# Initialization
# Get directory prefix and path to source.int script
PREFIX="$(dirname $(readlink -f $0))"
......@@ -67,7 +82,6 @@ if test "$(bash $PROJECT)" != "0" || ! sudo -v &> /dev/null; then
exit 0
fi
# Change to the script directory
cd $PREFIX
# Collected data from the current day
......@@ -76,106 +90,72 @@ TRAFFIC="./traffic.txt"
# Collected data on the last day before the machine is disconnected
PREVIOUS_TRAFFIC="./previous-traffic.txt"
# Interfaces that have "Destination" the network 0.0.0.0
INTERFACES="./interfaces"
# Stored interfaces from the last run
INTERFACES="./interfaces.txt"
# Receive the address(es) and mask(s) filtered from "interfaces"
ADDR_MASK="./addr-mask"
# Find binaries
IP=$(which ip)
test $? -ne 0 && IP="/sbin/ip"
# Used in the control of iptables rules. To verify if interfaces have not been
# added or removed
PREVIOUS_NETWORKS="./previous-networks"
NETWORKS="./networks"
IPTABLES=$(which iptables)
test $? -ne 0 && IPTABLES="/sbin/iptables"
# Temporary
TMP1="./tmp1"
TMP2="./tmp2"
TMP3="./tmp3"
test -x $IP || exit 1
test -x $IPTABLES || exit 1
# Comments of iptables rules
INPUT="INPUT_COUNTS"
OUTPUT="OUTPUT_COUNTS"
# Call the function date_comparator to check if traffic.txt already been
# copied to previous-traffic.txt in case of a new day has started
date_comparator
#------------------------------------------------------------------------------
# Calculate bytes and packets counts and print to TRAFFIC
# Capture interfaces that have "Destination " the network 0.0.0.0
/sbin/route -n | awk '/^0.0.0.0/ {print $8}' > $INTERFACES
get_interval_id
if test $? -ne 0; then exit 2; fi
rm -rf $ADDR_MASK $NETWORKS
# Check if TRAFFIC is fresh
check_traffic_date
if test $? -ne 0; then exit 3; fi
# Save on "ADDR_MASK" the address(es) and mask(s) filtered from "interfaces"
while read i; do
/sbin/ifconfig $i | grep 'inet ' > $TMP1
cat $TMP1 | cut -d ":" -f2 > $TMP2
sed s/' '// < $TMP2 > $TMP3
cat $TMP3 | cut -d " " -f1 >> $ADDR_MASK
cat $TMP1 | cut -d ":" -f4 >> $ADDR_MASK
done < $INTERFACES
INPUT_COUNT=$(sudo $IPTABLES -nvxL INPUT | grep INPUT_COUNTS | \
awk 'BEGIN{pkts=0; bytes=0} \
{pkts+=$1; bytes+=$2} \
END{print pkts,bytes}')
rm -rf $TMP1 $TMP2 $TMP3
OUTPUT_COUNT=$(sudo $IPTABLES -nvxL OUTPUT | grep OUTPUT_COUNTS | \
awk 'BEGIN{pkts=0; bytes=0} \
{pkts+=$1; bytes+=$2} \
END{print pkts,bytes}')
# Replace the "." by " " to facilitate on time to read the input
sed -i 's/\./ /g' $ADDR_MASK
echo "$INTERVALID $INPUT_COUNT $OUTPUT_COUNT" >> $TRAFFIC
# Run the discover-network program to discover the local networks
./discover-network
# Check if files are present for verification of local networks
if ! test -f "$PREVIOUS_NETWORKS"; then
cp $NETWORKS $PREVIOUS_NETWORKS
fi
rm -rf $ADDR_MASK $INTERFACES
# Variable to check if rules are included
TEST=$(sudo /sbin/iptables -L INPUT -n -v -x | grep $INPUT | head -n 1)
if test -z "$TEST"; then
# If $TEST is empty, then the rule(s) does not exist. Create rules for each
# internal network
while read net && read int; do
sudo /sbin/iptables -I INPUT -i $int ! -s $net -m comment \
--comment $INPUT
sudo /sbin/iptables -I OUTPUT -o $int ! -d $net -m comment \
--comment $OUTPUT
done < $NETWORKS
else
# If new networks were added, first removes your old networks and then
# added the new ones
TEST_DIFF=$(diff $NETWORKS $PREVIOUS_NETWORKS)
if test -n "$TEST_DIFF"; then
while read net && read int; do
sudo /sbin/iptables -D INPUT -i $int ! -s $net -m comment \
--comment $INPUT
sudo /sbin/iptables -D OUTPUT -o $int ! -d $net -m comment \
--comment $OUTPUT
done < $PREVIOUS_NETWORKS
while read net && read int; do
sudo /sbin/iptables -I INPUT -i $int ! -s $net -m comment \
--comment $INPUT
sudo /sbin/iptables -I OUTPUT -o $int ! -d $net -m comment \
--comment $OUTPUT
done < $NETWORKS
cp $NETWORKS $PREVIOUS_NETWORKS
fi
#------------------------------------------------------------------------------
# Create or update iptables rules (used for counting) based on the interfaces
# Remove all current rules
if test -f "$INTERFACES"; then
while IFS=$' ' read -a net; do
sudo $IPTABLES -D INPUT -i ${net[0]} ! -s ${net[1]} -m comment \
--comment INPUT_COUNTS 2>/dev/null
sudo $IPTABLES -D OUTPUT -o ${net[0]} ! -d ${net[1]} -m comment \
--comment OUTPUT_COUNTS 2>/dev/null
done < $INTERFACES
fi
# Variable that stores the current time
TIMESTAMP=$(date +%H:%M)
# Truncate interfaces file
echo -n > $INTERFACES
# Filter the result of "iptables -L INPUT/OUTPUT -n -v -x"
TRAFFICIN=$(sudo /sbin/iptables -L INPUT -n -v -x | grep $INPUT | head -n 1)
TRAFFICOUT=$(sudo /sbin/iptables -L OUTPUT -n -v -x| grep $OUTPUT | head -n 1)
# Grab all the interfaces in wich DEFAULT traffic gets routed
TMP=$($IP route list | sed -n 's/^default.*dev \([a-zA-Z0-9]*\).*$/\1/p')
# Filter "TRAFFICIN/TRAFFICOUT" preparing to save in the file
IN=$(echo "$TRAFFICIN" | awk '{print $1,$2}')
OUT=$(echo "$TRAFFICOUT" | awk '{print $1,$2}')
# Find IP address and subnet mask for each interface found
for IF in $TMP; do
addr_mask=$($IP addr show $IF | sed -n 's/^.*inet \([0-9.]*\/[0-9]*\).*$/\1/p')
echo "$IF $addr_mask" >> $INTERFACES
done
# Save in the "traffic.txt" the collected data in a period of 5 minutes
echo "$TIMESTAMP $IN $OUT" >> $TRAFFIC
# Reset the counters of iptables rules to the next collection
sudo /sbin/iptables -Z
# Insert new rules
while IFS=$' ' read -a net; do
sudo $IPTABLES -I INPUT -i ${net[0]} ! -s ${net[1]} -m comment \
--comment INPUT_COUNTS
sudo $IPTABLES -I OUTPUT -o ${net[0]} ! -d ${net[1]} -m comment \
--comment OUTPUT_COUNTS
done < $INTERFACES
/* Copyright (C) 2004-2011 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
*
* This file is part of collect-agent
*
* collect-agent is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
* USA. */
/* -------------------------------------------------------------------------- */
#include <stdio.h>
#include <stdlib.h>
/* Function: discover_network_and_save_to_file
* Function that finds and saves the local networks in the "networks".
* Parameters:
* decimal_array: array that contains the addresses and masks of local networks
* array_size: size of the array that contains the addresses and masks of local
* networks
* networkFile: file where the addresses, masks and interfaces of the local
* networks will be saved
* interfaceFile: file that contains the interfaces of local networks */
void discover_network_and_save_to_file(int *decimal_array, int array_size,
FILE *networkFile, FILE *interfaceFile)
{
int i, j, k, count, decimal_addr[4], decimal_mask[4];
char interface[10];
count = 0;
while (count < array_size)
{
/* Fill the array decimal_addr with content present in decimal_array */
for (i = count, j = 0; i < (count + 4); i++, j++)
{
decimal_addr[j] = decimal_array[i];
}
/* Fill the array decimal_mask with content present in decimal_array */
for (j = i, k = 0; j < (i + 4); j++, k++)
{
decimal_mask[k] = decimal_array[j];
}
/* Indicate the next position of the array that contains an address or
* end of the array */
count += 8;
/* Print the network in the networkFile.
* Do the "&"("and" bit by bit) between the address and mask to
* discover the network address */
for (i = 0; i < 3; i++)
{
fprintf(networkFile, "%d.", (decimal_addr[i] & decimal_mask[i]));
}
fprintf(networkFile, "%d/", (decimal_addr[3] & decimal_mask[3]));
/* Print the mask in the networkFile */
for (i = 0; i < 3; i++)
{
fprintf(networkFile, "%d.", decimal_mask[i]);
}
fprintf(networkFile, "%d\n", decimal_mask[3]);
/* Print the network interface discovered on file networkFile */
fgets(interface , 10, interfaceFile);
fprintf(networkFile, "%s", interface);
}
}
int main(int argc, char **argv)
{
int i, array_size;
int *decimal_array = (int *) malloc(1 * sizeof(int));
/* File that contains the addresses and masks of local networks */
FILE *addrMask;
addrMask = fopen("addr-mask", "r");
if (addrMask == NULL)
{
fputs("File error 1\n\n", stderr);
exit(1);
}
/* File to save the address of the discovered networks */
FILE *networkFile;
networkFile = fopen("networks", "w");
if (networkFile == NULL)
{
fputs("File error 2\n\n", stderr);
exit(2);
}
/* File that contains the interfaces of local networks */
FILE *interfaceFile;
interfaceFile = fopen("interfaces", "r");
if (interfaceFile == NULL)
{
fputs("File error 3\n\n", stderr);
exit(3);
}
/* Read the file with the addresses and masks and stores in the array of
* decimals */
i = 1;
while ((fscanf(addrMask, "%d", &decimal_array[i - 1])) != EOF)
{
i++;
decimal_array = realloc(decimal_array, i * sizeof(int));
}
array_size = i - 1;
discover_network_and_save_to_file(decimal_array, array_size, networkFile,
interfaceFile);
fclose(addrMask);
fclose(networkFile);
fclose(interfaceFile);
return 0;
}
......@@ -20,26 +20,28 @@
# USA.
#------------------------------------------------------------------------------
# Function: increase
# Function used to increment the counter used in xml_data_generator.
function increase()
# Function: increment_time
# Increment the time counter by 5 minutes
function increment_time()
{
if test $MINUTE -eq 55; then
MINUTE=0
((HOUR=${HOUR}+1))
else
((MINUTE=${MINUTE}+5))
TRAFFIC_MINUTE=$((TRAFFIC_MINUTE + 5))
if test $TRAFFIC_MINUTE -ge 60; then
TRAFFIC_HOUR=$((TRAFFIC_HOUR + 1))
TRAFFIC_MINUTE=2
fi
}
# Function: print
# Function: xml_print
# Function that prints the contents of XML when there are collected
# data. Print the statistics according to the collection.
function print()
function xml_print()
{
printf "<netuse id=\"$COUNT\">"
printf "<date value=\"$FILE_DATE\" type=\"string\"/>"
printf "<time value=\"$HOUR_COLLECT:$MINUTE_COLLECT\" type=\"string\"/>"
local TRAFFIC_TIME=$(printf "%02d:%02d:30" $TRAFFIC_HOUR $TRAFFIC_MINUTE)
printf "<netuse id=\"$((COUNT+1))\">"
printf "<date value=\"$TRAFFIC_DATE\" type=\"string\"/>"
printf "<time value=\"$TRAFFIC_TIME\" type=\"string\"/>"
printf "<rx>"
printf "<packets value=\"$(awk '{ print $2 }' <<< $line)\" type=\"int\"/>"
printf "<bytes value=\"$(awk '{ print $3 }' <<< $line)\" type=\"int\"/>"
......@@ -49,17 +51,18 @@ function print()
printf "<bytes value=\"$(awk '{ print $5 }' <<< $line)\" type=\"int\"/>"
printf "</tx>"
printf "</netuse>"
((COUNT = ${COUNT} + 1))
}
# Function: print_null
# Function: xml_print_zero
# Function that prints the contents of XML when there was no collected
# data. Print the statistics with zeroed content.
function print_null()
function xml_print_zero()
{
printf "<netuse id=\"$COUNT\">"
printf "<date value=\"$FILE_DATE\" type=\"string\"/>"
printf "<time value=\"$HOUR:$MINUTE\" type=\"string\"/>"
local TRAFFIC_TIME=$(printf "%02d:%02d:30" $TRAFFIC_HOUR $TRAFFIC_MINUTE)
printf "<netuse id=\"$((COUNT+1))\">"
printf "<date value=\"$TRAFFIC_DATE\" type=\"string\"/>"
printf "<time value=\"$TRAFFIC_TIME\" type=\"string\"/>"
printf "<rx>"
printf "<packets value=\"0\" type=\"int\"/>"
printf "<bytes value=\"0\" type=\"int\"/>"
......@@ -69,64 +72,6 @@ function print_null()
printf "<bytes value=\"0\" type=\"int\"/>"
printf "</tx>"
printf "</netuse>"
((COUNT=${COUNT}+1))
}
# Function: generator
# Function responsible for generating an XML containing the collected
# data. Assign zero to the time tags which there wasn't collection.
function generator()
{
TMP="$PREFIX/../../../tmp/usage-tmp"
COUNT=1
HOUR=0
MINUTE=5
END_HOUR=24
cat $PREVIOUS_TRAFFIC | tail -n +2 > $TMP
FILE_DATE=$(head -1 $PREVIOUS_TRAFFIC)
while read line; do
TIMESTAMP=$(cut -d" " -f1 <<< $line)
HOUR_COLLECT=$(cut -d":" -f1 <<< "$TIMESTAMP")
MINUTE_COLLECT=$(cut -d":" -f2 <<< "$TIMESTAMP")
# Exchange the "0" of midnight by "24"
if test $HOUR_COLLECT -eq 0 -a $MINUTE_COLLECT -eq 0; then
HOUR_COLLECT=24
fi
# If the time of the collected data is the same counter, just prints
# and increments the counter
if test $HOUR_COLLECT -eq $HOUR -a $MINUTE_COLLECT -eq $MINUTE; then
print
increase
# else print null until the counter reaches the value of collected data
else
until test $HOUR_COLLECT -eq $HOUR \
-a $MINUTE_COLLECT -eq $MINUTE -o $HOUR -ge 24 ; do
print_null
increase
done
if test $HOUR_COLLECT -eq $HOUR -a $MINUTE_COLLECT -eq $MINUTE; then
print
fi
increase
fi
done < $TMP
rm -rf $TMP
# If the last line of the file isn't 00:00 (24:00), then zero out until
# 24:00
if test $HOUR -lt 24; then
until test $HOUR -eq 24 -a $MINUTE -eq 5; do
print_null
increase
done
fi
}
# Function: xml_data_generator
......@@ -135,15 +80,52 @@ function generator()
# data to the XML file.
function xml_data_generator()
{
# If previous-traffic.txt exists, check if the date of last modification
# is the current date or not. If equals, means the XML data can be
# generated
FILE_DATE=$(date -d"$(ls -l --time-style="+%F" $PREVIOUS_TRAFFIC | \
awk '{ print $6 }')" +%s)
CURRENT_DATE=$(date -d"$(date +%F)" +%s)
if test $FILE_DATE -eq $CURRENT_DATE; then
generator
local CURRENT_TIMESTAMP=$(date -d"$(date +%F)" +%s)
local TRAFFIC_TIMESTAMP=$(date -d"$(head -n1 $TRAFFIC)" +%s)
# Check if date in TRAFFIC file aint older than 15 days
if test $TRAFFIC_TIMESTAMP -ge $(($CURRENT_TIMESTAMP - 1296000)); then
# Since we want the mean time between the five minutes intervals,
# the minute counter starts at 2 and gets 5 minutes increments.
# Also the "seconds counter" is fixed at 30. So we always get
# something like HH:2:30 or HH:7:30 (mean times).
#
# Example: if interval = 10:15~10:20 => mean = 10:17:30
TRAFFIC_HOUR=0
TRAFFIC_MINUTE=2
COUNT=-1
while read line; do
# Check if its the first line (containing the date)
if test $COUNT -eq -1; then
TRAFFIC_DATE=$(head -1 $TRAFFIC)
COUNT=$((${COUNT} + 1))
else
INTERVALID=$(cut -d" " -f1 <<< $line)
while test $COUNT -lt $INTERVALID -a $COUNT -lt 288; do
xml_print_zero
COUNT=$((${COUNT} + 1))
increment_time
done
if test $COUNT -eq $INTERVALID -a $COUNT -lt 288; then
xml_print
COUNT=$((${COUNT} + 1))
increment_time
fi
fi
done < $TRAFFIC
while test $COUNT -lt 288; do
xml_print_zero
COUNT=$((${COUNT} + 1))
increment_time
done
fi
printf "\n"
}
......@@ -160,10 +142,10 @@ if test "$(bash $PROJECT)" != "0" -o "${FIRST_EXECUTION}"; then
exit 0
fi
PREVIOUS_TRAFFIC="${PREFIX}/../../../net/previous-traffic.txt"
TRAFFIC="${PREFIX}/../../../net/previous-traffic.txt"
test -f "${PREVIOUS_TRAFFIC}" || exit 0
test -s $PREVIOUS_TRAFFIC || exit 0
test -f "${TRAFFIC}" || exit 0
test -s $TRAFFIC || exit 0
# Call the function xml_data_generator to generate the data to XML
xml_data_generator
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment