nhc.conf

# NHC Configuration File (sample)
#
# Lines are in the form "<hostmask>||<check>"
# Hostmask is a glob, /regexp/, or {noderange}
# Comments begin with '#'
#

#######################################################################
###
### NHC Configuration Variables
###
# Explicitly instruct NHC to assume PBS (TORQUE, PBSPro) is the Resource Manager
* || export NHC_RM=slurm

# Do not mark nodes offline
#   * || export MARK_OFFLINE=0

# Activate debugging mode
#   * || export DEBUG=1

# Set watchdog timer to 15 seconds
#   * || export TIMEOUT=15

# In out-of-band contexts, enable all checks
#   * || export NHC_CHECK_ALL=1

# Make sure $PATH contains important directories for diagnostic commands
#   * || export MOABHOMEDIR="/opt/moab"
#   * || export PATH="$MOABHOMEDIR/bin:$PATH"


#######################################################################
###
### Hardware checks
###
# Set these to your correct socket, core, and thread counts.
* || check_hw_cpuinfo $NHC_SOCKETS $NHC_CORES $NHC_THREADS

# Set these to the amount of physical RAM you have (leave the fudge factor).
* || check_hw_physmem $NHC_PHYS_MEM $NHC_PHYS_MEM $NHC_PHYS_MEM_FUDGE

# Set these to the amount of swap you have (leave the fudge factor).
* || check_hw_swap $NHC_SWAP_MEM $NHC_SWAP_MEM $NHC_SWAP_MEM_FUDGE

# If you prefer to use this instead of the previous two, you can.
#   * || check_hw_mem 40g 40g 5%

# Check specifically for free physical memory.
#   * || check_hw_physmem_free 1MB

# Same, but for swap space.
#   * || check_hw_swap_free 1MB

# Check for some sort of free memory of either type.
   * || check_hw_mem_free 1mb

# Checks that there's a QDR IB interface that's ACTIVE and shows LinkUp.
#   * || check_hw_ib 40

# Checks for an active Myrinet interface named "myri0."
#   * || check_hw_gm myri0

# Checks for an active ethernet interface named "eth1."
* || check_hw_eth $NHC_ETH0
* || check_hw_eth $NHC_ETH1

# Make sure we're running the correct BIOS version on all nodes.
#   * || check_dmi_data_match "BIOS Information: Version: 2.0.1"

# Make sure our RAM is running at the correct bus rate.
#   * || check_dmi_data_match -t "Memory Device" "*Speed: 1866 MHz"

# Check the mcelog daemon for any pending errors.
#   * || check_hw_mcelog

# Check that there's an ib interface with the correvt speed
* || if [ ! -z ${NHC_IB_DEV+x} ]; then check_hw_eth $NHC_IB_DEV; fi


#######################################################################
###
### Filesystem checks
###
# All nodes should have their root filesystem mounted read/write.
* || check_fs_mount_rw -f /

# Assert that /tmp is a mounted filesystem of type "tmpfs."
* || check_fs_mount_rw -t $NHC_TMP_FS -f /tmp
* || check_fs_mount_rw -t $NHC_VAR_FS -f /var
* || check_fs_mount_rw -t $NHC_VAR_SPOOL_FS -f /var/spool
* || check_fs_mount_rw -t tmpfs -f /dev/shm

# Controlling TTYs are a good thing!
#   * || check_fs_mount_rw -t devpts -s '/(none|devpts)/' -f /dev/pts

# Make sure the root filesystem doesn't get too full.
* || check_fs_free / 10%

# Free inodes are also important.
* || check_fs_ifree / 10k

# The following illustrates how to assert an NFSv3 mount (or any other specific mount option).
#   * || check_fs_mount -s bluearc0:/home -t nfs -o '/(^|,)vers=3(,|$)/' -f /home


#######################################################################
###
### File/metadata checks
###
# These should always be directories and always be read/write/execute and sticky.
* || check_file_test -r -w -x -d -k /tmp /var/tmp

# These should always be readable and should never be empty.
* || check_file_test -r -s /etc/passwd /etc/group

# Assert common properties for /dev/null (which occasionally gets clobbered).
* || check_file_test -c -r -w /dev/null /dev/zero
#   * || check_file_stat -m 0666 -u 0 -g 0 -t 1 -T 3 /dev/null

# Make sure there's relatively recent activity from the syslog.
* || check_file_stat -n 7200 /var/log/messages

# Validate a couple important accounts in the passwd file.
* || check_file_contents /etc/passwd "/^root:x:0:0:/" "sshd:*"


#######################################################################
###
### Process checks
###
# Everybody needs sshd running, right?  But don't use -r (restart)!
* || check_ps_service -u root -S sshd

# The cron daemon is another useful critter...
* || check_ps_service -r crond

# Check for wulfd but don't manage it.
#   * || check_ps_daemon wulfd root

# Make sure no users are SSH'd in, but don't kill them.
#   * || check_ps_blacklist sshd '!root'

# Flag and kill any processes which are owned by unauthorized users.
#   * || check_ps_unauth_users log syslog kill

# Flag any user processes not properly parented.
#   * || check_ps_userproc_lineage log syslog

# Most systems also need NFS locking services.
#   * || check_ps_service -d rpc.statd -r nfslock

# The audit daemon can sometimes disappear if things get hairy.
#   * || check_ps_service -r auditd

# This is only valid for RHEL6 and similar/newer systems.
* || check_ps_service -d rsyslogd -r rsyslog

# In the case of MySQL, it's typically better to cycle.
#   * || check_ps_service -c mysqld

# Double your core count is a good rule of thumb for load average max.
#   * || check_ps_loadavg 24
# This should work if you place it after one of the check_hw_*() checks.
* || check_ps_loadavg $((${NHC_LOADAVG_MULTIPLIER:-2}*$NHC_CORES))


#######################################################################
###
### TORQUE/Moab checks
###
# Monitor trqauthd to make sure it's always running.
#   * || check_ps_service -u root -r trqauthd

# Same for pbs_mom...just make sure NHC runs out-of-band in some way!
#   * || check_ps_service -u root -r pbs_mom

# On the master node, pbs_server gets monitored too.
#   * || check_ps_service -u root -r pbs_server

# Verify Moab status and version.
#   * || check_moab_sched -t 10 -v 7.2.3 -m '!/PAUSED/'

# RM engine sanity checks.
#   * || check_moab_rm -t 10

# TORQUE configuration sanity checks.
#   * || check_moab_torque -t 10

# Assert specific TORQUE settings that are critical to operation.
#   * || check_file_contents $PBS_SERVER_HOME/mom_priv/config '/^\$pbsserver master$/' '/^\$spool_as_final_name true$/' '!/localhost/'

#######################################################################
###
### Check for groups and users
###
# Check that a group exists and has usabel users
* || check_group_member -t 10 -g $NHC_GROUP_USABLE_USER


#######################################################################
###
### Check for all local filesystems of the -C type and excluding those of
### the -X types that there is at least 10% free space
###
* || check_all_local_fs_free -t 10% -X tmpfs -X devtmpfs -C ext4 -C ext2

## Check that on all filesystems a file can be touched
* || check_all_local_fs_mount_check_found_action -C ext4 -C ext2

#######################################################################
###
### Check that the time has not drifted too much.
### Only actually checks if the env var is defined (in /etc/sysconfig/nhc)
* || if [ ! -z "${NHC_PTPD_STATUS+x}" ]; then check_ptpd_drift; fi
* || if [ ! -z "${NHC_CHRONYC_MAX_DRIFT+x}" ]; then check_chrony_drift; fi

#######################################################################
###
### Other checks
###
# Check to verify that SELinux is disabled.  (Remove the "-r 1" to verify it's enabled.)
#   * || check_cmd_status -t 1 -r 1 selinuxenabled

# Verify settings for an Ethernet interface.
#   * || check_cmd_output -m '/addr:10\.0\.0\.1/' -m '/Bcast:10\.0\.0\.255/' -m '/Mask:255\.255\.255\.0/' -m '/^[[:space:]]*UP /' /sbin/ifconfig eth3

# nVidia HealthMon GPU health checks (requires Tesla Development Kit)
#   * || check_nv_healthmon