#!/opt/nimsoft/bin/perl
#################################################################
# CodeWizard:  perl
# This code was generated with the NimBUS CodeWizard version 1.42
# Date: Wednesday, July 25, 2007

use strict;
use Getopt::Std;
use Nimbus::API;
use Nimbus::Session;
use Nimbus::CFG;

my $prgname = "mysql";
my $sess;
my $version = "1.9";
my $config;
my $int_option = 0;
my $str_option = "";
my ( $samples, $thread_samples );
my %options;
my $loglevel = 0;
my $logfile  = "$prgname.log";
my $next_run;
my $interval            = 300;
my $suppression_enabled = 0;
my $suppression_hour;
my $suppression_duration;
my $suppression_lastday = 0;
my $qosDefinition            = 0;
my $SLAVE_IO_RUNNING_STATUS  = 0;       # Status tracking variable
my $SLAVE_SQL_RUNNING_STATUS = 0;       # Status tracking variable
my $SEC_BEHIND_STATUS        = 0;
my $SEC_BEHIND               = undef;
my @MysqlClient;
my @SlaveIORunning;
my @SlaveSQLRunning;
my @SecondBehindWarn;
my @SecondBehindCrit;
my @DatabaseConnection;
my @MysqlCreds;
my $MysqlUsername = undef;
my $MysqlPassword = undef;
my $MysqlIP       = undef;
my $MysqlPort     = undef;
my ( $CurrentSampleCount, $CurrentIOSampleCount, $CurrentSQLSampleCount ) =
  ( 0, 0, 0 );
my $first_run             = 1;
my $database_connectivity = 1;
my $mysql_defaults;

###########################################################
# DoWork - function called by dispatcher on timeout
#
sub doWork {
    my $now = time();
    return if ( $now < $next_run );
    $next_run = $now + $interval;
    nimLog( 1, "(doWork) - interval has passed" );

    ###############################
    # Enter your code here
    ###############################

    if ($suppression_enabled) {
        my ( $sec, $min, $hour, $day ) = localtime $now;
        if ( $hour == $suppression_hour and $suppression_lastday != $day) {
            nimLog( 0,
"suppressing monitoring of mysql replication during the specified suppression period"
            );
            $next_run = $now + $suppression_duration;
            $suppression_lastday = $day;
            ( $sec, $min, $hour ) = localtime $next_run;
            nimLog( 0, "monitoring will resume at $hour:$min" );
            return;
        }
    }

    # Get variables out of the config file on every run.
    my $CRITICAL_TIME = $config->{setup}->{CRITICAL} || 30;
    my $WARN_TIME     = $config->{setup}->{WARN}     || 20;
    $samples        = $config->{setup}->{samples}        || 3;
    $thread_samples = $config->{setup}->{thread_samples} || 2;

   #Verify that the mysql client is installed and send apporpriate error if not.
    my $MYSQL_EXEC = undef;
    chomp( $MYSQL_EXEC = `which mysql 2>/dev/null ` );
    if ( length($MYSQL_EXEC) == 0 ) {
        nimLog( 1, "Cannot find the executable mysql" );
        nimAlarm( $MysqlClient[1], $MysqlClient[0], "1.1.1", $MysqlClient[2] );
    }
    else {

        #Building Command to run based on config file info
        #
        #$MysqlCreds[0] = Username
        #$MysqlCreds[1] = Password
        if ( $MysqlCreds[0] ne "none" ) {
            $MysqlUsername = "-u $MysqlCreds[0]";
            if ( $MysqlCreds[1] ne "none" ) {
                $MysqlPassword = "-p$MysqlCreds[1]";
            }
        }

        #This value could potentially be a config file setting.
        if ( -e "/root/.my.cnf" ) {
            nimLog( 2, "Using mysql defaults found in /root/.my.cnf" );
            $mysql_defaults = "--defaults-file=/root/.my.cnf";
        }

        nimLog( 2,
"executing: $MYSQL_EXEC $mysql_defaults -h $MysqlIP -P $MysqlPort $MysqlUsername $MysqlPassword -E -e \"SHOW SLAVE STATUS\""
        );

        #Need to seperate STDERR and log it.
        my @data =
`$MYSQL_EXEC $mysql_defaults -h $MysqlIP -P $MysqlPort $MysqlUsername $MysqlPassword -E -e "SHOW SLAVE STATUS"`;
        chomp(@data);

        my @lines;
        for ( my $x = 0 ; $x < @data ; $x++ ) {
            $lines[$x] = $data[$x];
            $lines[$x] =~ s/^\s+//;
            $lines[$x] =~ s/\s+$//;
        }
        my $lines = join( "\r\n", @lines );

        my $size;
        $size = @data;
        my $line      = undef;
        my @tableLine = undef;

        nimLog( 1, "Size of returned data is $size" );

        #Verify we got something back, if not alert on it
        if ( $size <= 0 ) {
            nimAlarm(
                $DatabaseConnection[1], $DatabaseConnection[0],
                "1.1.1",                $DatabaseConnection[2]
            );
            $database_connectivity = 0;
        }
        else {

            # Clear any existing alarm for database connectivity
            if ($first_run) {
                nimLog( 2, "First run Database Connectivity alert clear" );
                nimAlarm( NIML_CLEAR, "Database connecivity is set to YES",
                    "1.1.1", $DatabaseConnection[2] );
            }

        # next let us see if we had database connectivity issues before this run
            if ( !$database_connectivity ) {
                nimLog( 2, "Clearing database connectivity alarm" );
                nimAlarm( NIML_CLEAR, "Database connectivity restored",
                    "1.1.1", $DatabaseConnection[2]
                );
                $database_connectivity = 1;
            }

   #Iterating through output in case the position of the entries were to change.
            for $line (@data) {
                @tableLine = split /:/, $line;
                if ( $tableLine[0] =~ "Slave_IO_Running" ) {

               #write function to alert when Slave_IO_Running is not yes and set
               #a flag so a clear event can be sent if the status changes
                    if ( $tableLine[1] =~ /no/i ) {

                    # if we make it here slave_io_running should not be running,
                    # need to send an alert and set the flag to true

                        $CurrentIOSampleCount++;
                        nimLog( 2,
"IO Slave not running - count ${CurrentIOSampleCount} of ${thread_samples}"
                        );

                        if ( $CurrentIOSampleCount >= $thread_samples ) {
                            nimLog( 2, "Pushing IO Slave alarm to nimbus" );
                            $SLAVE_IO_RUNNING_STATUS = 1;
                            nimAlarm(
                                $SlaveIORunning[1],
                                $SlaveIORunning[0]
                                  . " (${CurrentIOSampleCount} samples) ",
                                "1.1.1",
                                $SlaveIORunning[2]
                            );
                        }
                    }
                    elsif ( $SLAVE_IO_RUNNING_STATUS == 1 ) {
                        nimLog( 2, "Clearing nimbus IO Slave alert" );

        # if we make it here slave_io_running is yes but the flag is set to true
        # need to send clear event
                        nimAlarm( NIML_CLEAR, "Slave_IO_Running is set to YES",
                            "1.1.1", $SlaveIORunning[2] );
                        $SLAVE_IO_RUNNING_STATUS = 0;
                        $CurrentIOSampleCount    = 0;
                    }
                    elsif ($first_run) {

                        # clear any existing alarm
                        nimLog( 2, "First run IO Slave alert clear" );
                        nimAlarm( NIML_CLEAR, "Slave_IO_Running is set to YES",
                            "1.1.1", $SlaveIORunning[2] );
                    }
                    nimLog( 1, "SLAVE IO RUNNING = $tableLine[1]" );
                }
                elsif ( $tableLine[0] =~ "Slave_SQL_Running" ) {
                    if ( $tableLine[1] =~ /no/i ) {
                        $CurrentSQLSampleCount++;
                        nimLog( 2,
"SQL Slave not running - count ${CurrentSQLSampleCount} of ${thread_samples}"
                        );

                        if ( $CurrentSQLSampleCount >= $thread_samples ) {
                            nimLog( 2, "Pushing SQL Slave alarm to nimbus" );
                            nimAlarm(
                                $SlaveSQLRunning[1],
                                $SlaveSQLRunning[0]
                                  . " (${CurrentSQLSampleCount} samples)",
                                "1.1.1",
                                $SlaveSQLRunning[2]
                            );
                            $SLAVE_SQL_RUNNING_STATUS = 1;
                        }
                    }
                    elsif ( $SLAVE_SQL_RUNNING_STATUS == 1 ) {
                        nimLog( 2, "Clearing nimbus SQL slave alert" );
                        nimAlarm( NIML_CLEAR, "Slave_SQL_Running is set to yes",
                            "1.1.1", $SlaveSQLRunning[2] );
                        $SLAVE_SQL_RUNNING_STATUS = 0;
                        $CurrentSQLSampleCount    = 0;
                    }
                    elsif ($first_run) {

                        # clear any existing alarm
                        nimLog( 2, "First run IO Slave alert clear" );
                        nimAlarm( NIML_CLEAR, "Slave_SQL_Running is set to YES",
                            "1.1.1", $SlaveSQLRunning[2] );
                    }
                    nimLog( 1, "SLAVE SQL RUNNING = $tableLine[1]" );
                }
                elsif ( $tableLine[0] =~ "Seconds_Behind_Master" ) {
                    nimLog( 1, "Seconds behind = $tableLine[1]" );
                    $SEC_BEHIND = $tableLine[1];
                    $SEC_BEHIND =~ s/ //g;

                    if (   $tableLine[1] >= $WARN_TIME
                        && $tableLine[1] < $CRITICAL_TIME )
                    {
                        $CurrentSampleCount++;
                        nimLog( 2,
"Seconds behind (${SEC_BEHIND}) over warn threshold ${CurrentSampleCount} of ${samples}"
                        );
                        if ( $CurrentSampleCount >= $samples ) {
                            nimLog( 2, "Pushing sec_behind warning to nimbus" );
                            my $alert_string = $SecondBehindWarn[0];
                            $alert_string =~ s/\$SEC_BEHIND/$SEC_BEHIND/e;
                            nimAlarm(
                                $SecondBehindWarn[1],
                                $alert_string
                                  . " ($CurrentSampleCount samples)",
                                "1.1.1",
                                $SecondBehindWarn[2]
                            );
                            $SEC_BEHIND_STATUS = 1;
                        }
                    }
                    elsif ( $tableLine[1] >= $CRITICAL_TIME ) {
                        $CurrentSampleCount++;
                        nimLog( 2,
"Seconds behind over crit threshold ${CurrentSampleCount} of ${samples}"
                        );
                        if ( $CurrentSampleCount >= $samples ) {
                            nimLog( 2, "Pushing sec_behind crit to nimbus" );
                            my $alert_string = $SecondBehindCrit[0];
                            $alert_string =~ s/\$SEC_BEHIND/$SEC_BEHIND/e;
                            nimAlarm(
                                $SecondBehindCrit[1],
                                $alert_string
                                  . " ($CurrentSampleCount samples)",
                                "1.1.1",
                                $SecondBehindCrit[2]
                            );

                            $SEC_BEHIND_STATUS = 1;
                        }
                    }
                    elsif ($tableLine[1] < $WARN_TIME
                        && $SEC_BEHIND_STATUS == 1 )
                    {
                        $CurrentSampleCount = 0;
                        nimLog( 2, "Clearning sec_behind alert" );
                        nimAlarm( NIML_CLEAR,
"Server is now  $tableLine[1] seconds behind master",
                            "1.1.1",
                            $SecondBehindCrit[2]
                        );
                        $SEC_BEHIND_STATUS = 0;
                    }
                    elsif ($first_run) {

                        # clear any existing alarm
                        nimLog( 2, "First run sec_behind alert clear" );
                        nimAlarm( NIML_CLEAR,
"Server is now  $tableLine[1] seconds behind master",
                            "1.1.1",
                            $SecondBehindCrit[2]
                        );
                    }
                }
            }
        }

        #Publish the qos to the server
        publishQoS();
    }
    $first_run = 0;
}

###########################################################
# defineQoS - send the QoS definitions
#
sub defineQoS {

    if ( !$qosDefinition ) {

        #We only want to send the definition ONCE!
        nimQoSDefinition(
            "MYSQL_REPL_SECONDS_BEHIND",                # QOS Name
            "QOS_MYSQL",                                # QOS Group
            "Seconds Slave server is behind Master",    # QOS Description
            "Seconds", "s", 0, 0
        );                                              # Unit info and flags
        $qosDefinition = 1;
    }

}

###########################################################
# publishQoS (target, value) - Create and publish QoS message
#
sub publishQoS {

    my $now         = shift || time();
    my $samplevalue = shift;
    my $source      = nimGetVarStr(NIMV_ROBOTNAME);

    #my $source = undef;
    #chomp ($source = `/bin/hostname`);
    my $ival   = $interval;    # Seconds
    my $target = $MysqlIP;     # mysql replication host (default 127.0.0.1)

    if ( !$qosDefinition ) {
        nimLog( 0, "Sending QOSDefinintion" );

        #We only want to send the definition ONCE!
        defineQoS();
        $qosDefinition = 1;
    }
    nimLog( 0, "Sending QoS for Mysql" );

    #Mysql Seconds behind Master
    nimQoSMessage( "MYSQL_REPL_SECONDS_BEHIND", $source, $target, $now,
        $SEC_BEHIND, 0, $ival, -1 );

}

#######################################################################
# Service functions
#
sub restart {
    nimLog( 1, "(restart) - got restarted" );
    $config     = Nimbus::CFG->new("$prgname.cfg");
    $int_option = $config->{setup}->{int_option} || 0;
    $str_option = $config->{setup}->{str_option} || "default";
    $loglevel   = $options{d} || $config->{setup}->{loglevel} || 0;
    $logfile    = $options{l} || $config->{setup}->{logfile} || "$prgname.log";
    $interval   = $options{i} || $config->{setup}->{interval} || 300;
    nimLogSet( $logfile, $prgname, $loglevel, 0 );
}

sub timeout {
    nimLog( 2, "(timeout) - got kicked " );
    doWork();
}

###########################################################
# Signal handler - Ctrl-Break
#
sub ctrlc {
    exit;
}

###########################################################
# MAIN ENTRY
#
getopts( "d:l:i:", \%options );
$SIG{INT} = \&ctrlc;
$config = Nimbus::CFG->new("$prgname.cfg");
$int_option = $config->{setup}->{int_option} || 0;
$str_option = $config->{setup}->{str_option} || "default";
$loglevel = $options{d} || $config->{setup}->{loglevel} || 0;
$logfile  = $options{l} || $config->{setup}->{logfile}  || "$prgname.log";
$interval = $options{i} || $config->{setup}->{interval} || 300;

###########################################################
#Putting alerts into a array so they are easier to use later
@MysqlClient = (
    $config->{messages}->{MysqlClient}->{text},
    $config->{messages}->{MysqlClient}->{level},
    nimSuppToStr( 0, 0, 0, "$prgname/noclient" )
);
@SlaveIORunning = (
    $config->{messages}->{SlaveIORunning}->{text},
    $config->{messages}->{SlaveIORunning}->{level},
    nimSuppToStr( 0, 0, 0, "$prgname/SLAVE_IO_RUNNING" )
);
@SlaveSQLRunning = (
    $config->{messages}->{SlaveSQLRunning}->{text},
    $config->{messages}->{SlaveSQLRunning}->{level},
    nimSuppToStr( 0, 0, 0, "$prgname/SLAVE_SQL_RUNNING" )
);
@SecondBehindCrit = (
    $config->{messages}->{SecondBehindCrit}->{text},
    $config->{messages}->{SecondBehindCrit}->{level},
    nimSuppToStr( 0, 0, 0, "$prgname/sec_behind" )
);
@SecondBehindWarn = (
    $config->{messages}->{SecondBehindWarn}->{text},
    $config->{messages}->{SecondBehindWarn}->{level},
    nimSuppToStr( 0, 0, 0, "$prgname/sec_behind" )
);
@DatabaseConnection = (
    $config->{messages}->{DatabaseConnection}->{text},
    $config->{messages}->{DatabaseConnection}->{level},
    nimSuppToStr( 0, 0, 0, "$prgname/DatabaseConnection" )
);

###########################################################
#Retrieving mysql access info
@MysqlCreds = ( $config->{setup}->{username}, $config->{setup}->{password} );
$MysqlIP = $config->{setup}->{IPaddress};
$MysqlPort = $config->{setup}->{port};

nimLogSet( $logfile, $prgname, $loglevel, 0 );
nimLog( 0, "----------------- Starting  (pid: $$) ------------------" );

nimLog( 0, "Checking for suppression period" );
###########################################################
#alarm suppression to be used during know backup time
if ( $config->{suppression}->{active} =~ /yes/i ) {
    $suppression_enabled  = 1;
    $suppression_hour     = $config->{suppression}->{hour};
    $suppression_duration = $config->{suppression}->{duration};
    if (    $suppression_duration < 30 
        or $suppression_duration > 420 )
    {
        nimLog( 0,
"suppression disabled due to unsupported duration: $suppression_duration"
        );
        nimAlarm(
            NIML_CRITICAL,
"Suppression duration is invalid. A duration between 30 and 420 minutes should be specified. Assign a private ticket to the Lead Engineer for resolution. Suppression disabled.",
            "1.1.1",
            nimSuppToStr( 0, 0, 0, "$prgname/suppressionduration" )
        );
        $suppression_enabled = 0;
    }
    else {
        nimLog( 0,
"suppression enabled starting at $suppression_hour:00 for $suppression_duration minutes"
        );
        $suppression_duration = ( $suppression_duration + 1 ) * 60;
    }
}

$sess = Nimbus::Session->new("$prgname");
$sess->setInfo( $version, "rackspace IT Hosting" );

if ( $sess->server( NIMPORT_ANY, \&timeout, \&restart ) == 0 ) {
    nimLog( 1, "server session is created" );
}
else {
    nimLog( 0, "unable to create server session" );
    exit(1);
}

$sess->dispatch();
nimLog( 0, "Received STOP, terminating program" );
nimLog( 0, "Exiting program" );
exit;
