System/ Process Monitoring in Linux using MONIT

Install and Configure Monit

sudo apt-get install monit
usudo chmod 700 /etc/monit/monitrc
telnet localhost 2812
sudo ufw allow 2812

Sample MONIT Configuration

username@ServerName:/etc/monit$ sudo cat monitrc

set daemon 60
set logfile syslog facility log_daemon
set mailserver localhost
set mail-format { from: monit@ServerName.localnet }
set alert username.shah@cw.com
set httpd port 2812 and
#     SSL ENABLE
#     PEMFILE  /etc/monit/monit.pem
    allow admin:edgeadmin

check system edge.localnet
    if loadavg (1min) > 4 then alert
    if loadavg (5min) > 2 then alert
    if memory usage > 70% then alert
    if swap usage > 20% then alert
    if cpu usage (user) > 70% then alert
    if cpu usage (system) > 30% then alert
    if cpu usage (wait) > 20% then alert

# SENDMAIL
check process sendmail with pidfile /var/run/sendmail/mta/sendmail.pid
  group services
  start program "/etc/init.d/sendmail start"
  stop program "/etc/init.d/sendmail stop"
  if failed port 25 protocol smtp then restart
  if 5 restarts within 5 cycles then timeout

# SSHD
check process sshd with pidfile /var/run/sshd.pid
  group system
  start program  "/sbin/start ssh"
  stop program  "/sbin/stop ssh"
  if failed port 22 protocol ssh then restart
  if 5 restarts within 5 cycles then timeout

# ORACLE XE
check process oracle with pidfile /var/run/oracle-xe.pid
  group services
  start program "/etc/init.d/oracle-xe start"
  stop program "/etc/init.d/oracle-xe stop"
  if cpu usage > 23% for 5 cycles then restart
  if failed host ServerName.localnet port 1521 timeout 60 seconds then restart
  if failed host ServerName.localnet port 8080 timeout 120 seconds then restart
  if 5 restarts within 5 cycles then timeout

# VNC
check process Xvnc4 with pidfile "/home/username/.vnc/ServerName.localnet:1.pid"

  start program "/sbin/initctl start vncserve"
  stop program "/sbin/initctl stop vncserve"
  if failed port 5901 then restart
  if 5 restarts within 5 cycles then timeout

# VNC
#check process Xvnc4 with pidfile "/home/username/.vnc/ServerName.localnet:1.pid"
#   start program "/home/username/vnc.sh"
#   stop program "/home/username/vnc-kill.sh"
#   if failed port 5901 then restart
#   if 5 restarts within 5 cycles then timeout

# SMBD
#check process smbd with pidfile /var/run/samba/smbd.pid
#   group system
#   start program  "/sbin/start smbd"
#   stop program  "/sbin/stop smbd"
#   if failed port 139 then start
#   if 5 restarts within 5 cycles then timeout

# SKYPEVNC
#check process Xvnc4 with pidfile "/home/username/.vnc/servername\:2.pid"
#   start program "/etc/skype-start.sh"
#   stop program "/etc/skype-stop.sh"
#   if failed port 5902 then start
#   if 5 restarts within 5 cycles then timeout

# DNS SERVER
#check process named with pidfile /var/run/bind/run/named.pid
#   group services
#   start program  "/etc/init.d/bind9 start"
#   stop program  "/etc/init.d/bind9 stop"
#   if failed host 127.0.0.1 port 53 type tcp protocol dns then alert
#   if failed host 127.0.0.1 port 53 type udp protocol dns then alert
#   if 5 restarts within 5 cycles then timeout

# MYSQL
#check process mysql with pidfile /var/run/mysqld/mysqld.pid
#   group services
#   start program = "/etc/init.d/mysql start"
#   stop program = "/etc/init.d/mysql stop"
#   if failed host 127.0.0.1 port 3306 then restart
#   if 5 restarts within 5 cycles then timeout

# APACHE
#check process apache with pidfile /var/run/apache2.pid
#   group services
#   start program = "/etc/init.d/apache2 start"
#   stop program  = "/etc/init.d/apache2 stop"
#   if failed host admin.hosting.example.com port 80 protocol http
#      and request "/tools/monin_test_file" then restart
#   if cpu is greater than 60% for 2 cycles then alert
#   if cpu > 90% for 5 cycles then restart
#   if totalmem > 500 MB for 5 cycles then restart
#   if children > 250 then restart
#   if loadavg(5min) greater than 10 for 8 cycles then stop
#   if 3 restarts within 5 cycles then timeout

# POSTFIX
#check process postfix with pidfile /var/spool/postfix/pid/master.pid
#   group services
#   start program = "/etc/init.d/postfix start"
#   stop  program = "/etc/init.d/postfix stop"
#   if failed port 25 protocol smtp then restart
#   if 5 restarts within 5 cycles then timeout

# ISPCP_DAEMON
#check process ispcp_daemon with pidfile /var/run/ispcp-daemon.pid
#   group services
#   start program = "/etc/init.d/ispcp_daemon start"
#   stop  program = "/etc/init.d/ispcp_daemon stop"
## If you active this, your /var/log/daemon will grow every minute,
## so it's not recomended unless you don't want to trust only in .pid
##   if failed port 9876 then restart
#   if 5 restarts within 5 cycles then timeout

Sample VNC Server Startup Script

username@ServerName:~$ cat /home/username/vnc.sh
#!/bin/bash
su -l username -c “/usr/bin/vnc4server :1 -name "my Server Name" -geometry 1280x730”

Sample VNC Server Stop Script

username@ServerName:~$ cat /home/username/vnc-kill.sh
#!/bin/bash
/usr/bin/vncserver -kill :1

Sample Script to create a PID for a non-pid type process

username@ServerName:~$ sudo cat /etc/monit/oracle-xe-pid.sh
#!/bin/bash

pmon=`ps -ef | egrep pmon_XE'\>' | grep -v grep| awk '{ print $2 }'`

if [ "$pmon" = "" ];
then
   rm /var/run/oracle-xe.pid
else
       echo $pmon > /var/run/oracle-xe.pid
fi

Sample Cron to check, create and update PID every minute

username@ServerName:~$ sudo crontab -l
* * * * * /etc/monit/oracle-xe-pid.sh