#!/usr/bin/sh
###############################################################################
####### NightOwl.sh
###############################################################################
#######	This script has been known as "NightOwl" and "Daily", with many more
#######	incarnations.  It's original conception was intended to help stablilize
#######	systems and make a centralized system administration team aware of
####### issues on the UNIX systems in a proactive manner.
#######
####### The original version was developed for AT&T RD System 5 machines.  The
####### second version was for HP-UX 8.X (yes, you read that correctly), and
####### then on to HP-UX 9.X.  The version you have before you has been tested
####### and used on HP-UX 10.20 systems.  It has been tested on an HP-UX 11.00
####### system, but never went full production.
#######
#######	This script is a reporting tool ONLY -- it does not modify the system.
###############################################################################
####### This script represents the suggestions of Glenn Martin Stafford, to
####### whom I am eternally grateful.  It also represents a compendium of my
####### System Administratin experiences.  However, it only covers the tasks I
#######	feel are generically useful (it avoids topics that could be considered
#######	"site specific" -- e.g. backup monitoring, database monitoring, etc).
#######	I also hope that you find the script useful for learning some handy
#######	shell scripting techinques.
#######							David Lee Totsch
###############################################################################
####### Revision History
####### 09/14/1990	Original Release
#######	    *** Interim releases deleted to protect the guilty and innocent **
#######	02/19/2000	Updated due to demand from an article appearing in
#######			'Enterprise Solutions' (a publication of Interex) and
#######			a subsequent thread on the HPUXADMIN list server.
###############################################################################
####### DECLARATIONS:
ADMINS=""			# a comma-separated list of users to receive
				# e-mail from this script.  SUGGESTION: create
				# a e-mail alias.
HOST=$(uname -n)
PROG=${0##*/}
TMPDIR=/var/tmp/${PROG}_${$}
CHKSUM="1826172462 24840"	# known checksum for SUID/SGID files.
FSCAPTHRESH=80			# Percentage full to begin reporting fs full.
INODETHRESH=80			# Percentage used to begin reporting inodes.
EBA=3				# Excessive Badlogin Attempts
#######	End of DECLARATIONS. ##################################################
#######	FUNCTIONS:
SEPARATOR()
{
print -n "****************************************"
print "***************************************"
}

CHECK_WRITABLE()
{
PERMS=$(ls -ld ${1} | cut -c2-10)
GROUP_PERMS=$(print ${PERMS} | cut -c5)
OTHER_PERMS=$(print ${PERMS} | cut -c8)
if [[ ${GROUP_PERMS} = "w" || ${OTHER_PERMS} = "w" ]]
then
	return 0
else
	return 1
fi
}

#######	End of Functions. #####################################################
###############################################################################
#######	MAIN

####### This script needs root priviledge.  Sigh.
case $(whoami) in
"root")	
	;;
*)	print "${PROG}: Terminated.  Root Priviledge required."
	exit 2
	;;
esac

if mkdir ${TMPDIR}
then
	:
else
	print "${PROG}: EXITING. Creation of ${TMPDIR} failed."
	exit 2
fi

SEPARATOR
print "		Daily Report for\n"
banner ${HOST}
SEPARATOR
print -n "DAILY BEGINS: ";date
SEPARATOR

#######------------------------------------------------------------------------
#######	Last boot time.
print "SYSTEM LAST BOOTED:"
who -rb
SEPARATOR

#######------------------------------------------------------------------------
#######	Check to see if /etc/motd is more than 7 days old.
if [[ -f /etc/motd ]]
then
	MOTD=$(find /etc/motd -mtime +7)
	if [[ -n ${MOTD} ]]
	then
		print "/etc/motd HAS EXPIRED!"
		cat /etc/motd
		SEPARATOR
	fi
	unset MOTD
else
	print "===> WARNING: /etc/motd does NOT exist! <==="
	SEPARATOR
fi

#######------------------------------------------------------------------------
#######	Check Software Distributor fileset states.
UNCONFIGURED=$(swlist -l fileset -a state | 
	awk '$1 != "#" && $2 != "configured" {print $1}' )
if [[ -n ${UNCONFIGURED} ]]
then
	print "THE FOLLOWING FILESETS ARE NOT IN THE STATE \"configured\":"
	print "${UNCONFIGURED}" | pr -t -o8
	SEPARATOR
fi
unset UNCONFIGURED

#######------------------------------------------------------------------------
#######	Check HOME directories

####### NOTE:  This checking does NOT account for NIS.

#######	You might wish to be more restrictive about creating HOMELIST, or
####### desire to create it a different way.  You may also not care that some
####### of these files are writable at the group level.
HOMELIST=$(cut -d: -f1,6 /etc/passwd)
for LINE in ${HOMELIST}
do
	USER=${LINE%%:*}
	HOMEDIR=${LINE##*:}
	
	# check permissions of HOME
	# is it writable by group or other?
	if CHECK_WRITABLE ${HOMEDIR}
	then
		print "${USER} home ${HOMEDIR} is writable by group or other."
	fi

	####### check some well known files in HOME directories
	####### these files are expected and missing ones are reported.
	for FILE in .profile .kshrc
	do
		if [[ -f ${HOMEDIR}/${FILE} ]]
		then
			if CHECK_WRITABLE ${HOMEDIR}/${FILE}
			then
				print -n "${USER} file ${HOMEDIR}/${FILE} is "
				print "writable by group or other."
			fi
		else
			print "${USER} ${HOMEDIR}/${FILE} MISSING!"
		fi
	done

	#######	if this user has a .rhosts, is it writable and check contents.
	if [[ -f ${HOMEDIR}/.rhosts ]]
	then
		if CHECK_WRITABLE ${HOMEDIR}/.rhosts
		then
			print -n "${USER} file ${HOMEDIR}/.rhosts is "
			print "writable by group or other."
		fi

		#######	Check for authorization of user other than self.
		if awk 'NF > 1 {print $2}' ${HOMEDIR}/.rhosts |
			grep -v ${USER} > /dev/null 2>&1
		then
			# we found an entry that enables another user
			print "${USER} has a .rhost that allows other users."
		fi

		#######	Check for a liberal host definition.
		if awk '{print $1}' ${HOMEDIR}/.rhosts| grep "+" >/dev/null 2>&1
		then
			print "${USER} has a .rhosts that uses \"+\" as host."
		fi
	fi

	#######	Does this user have a .netrc?
	if [[ -f ${HOMEDIR}/.netrc ]]
	then
		print "${USER} has a ${HOMEDIR}/.netrc!"
	fi
done > ${TMPDIR}/home_check.out

if [[ -s ${TMPDIR}/home_check.out ]]
then
	print "THE FOLLOWING HOME DIRECTORY PROBLEMS DETECTED:"
	pr -t -o8 ${TMPDIR}/home_check.out
	SEPARATOR
fi

unset HOMELIST
unset USER
unset HOMEDIR

#######------------------------------------------------------------------------
#######	Check File Systems for fullness and inode usage.

####### Get list of MOUNT POINTS
#######		We have to do things this way because bdf can report more
#######		than one line of output when the File System name is longish.

#######	Get list of local file systems, regardless of type...
LOCALLIST=$(bdf -l 2>/dev/null | awk '$1 !~ /Filesystem/ {print $NF}')

#######	Get list of cdfs (CDROM file sytems) so we can ignore them...
CDFSLIST=$(bdf -t cdfs 2>/dev/null | awk '$1 !~ /Filesystem/ {print $NF}')

for MOUNTPOINT in $(print "${LOCALLIST}\n${CDFSLIST}" | sort | uniq -u)
do
	bdf -i ${MOUNTPOINT} | grep -v "^Filesystem" | paste - - | \
		read FS V W X CAP Y Z INOD MP
	#######	we will ignore V W X Y Z...

	####### Check the capacity (file sytem full)
	if [[ ${CAP%\%*} -gt FSCAPTHRESH ]]
	then
		print "${MP} is ${CAP} full."
	fi

	####### Check the inode usage
	#######	NOTE: not much use on a VxFS file system.
	if [[ ${INOD%\%*} -gt INODETHRESH ]]
	then
		print "${MP} inodes used ${INOD}"
	fi
done >> ${TMPDIR}/file_system_full

if [[ -s ${TMPDIR}/file_system_full ]]
then
	print -n  "FILE SYSTEMS GREATER THAN ${FSCAPTHRESH}% FULL OR "
	print "${INODETHRESH}% INODES USED:"
	pr -t -o8 ${TMPDIR}/file_system_full
	SEPARATOR
fi

unset CDFSLIST
unset LOCALLIST

#######------------------------------------------------------------------------
#######	Detect Unmounted file systems.
####### WARNING:	this code does not check by mount points!
#######			If /etc/fstab has /dev/vg00/lvol9 mounting to /data,
#######			but /dev/vg00/lvol9 is currently mounted to /tmpmnt,
#######			this code says it is OK.  If you decide to modify this
#######			behavior, note that the grep statements match to the
#######			beginning of the line (hence, the FS detection and not
#######			mount point).
####### WARNING 2:	If you use the "noauto" option in /etc/fstab
#######			(an undocumented feature allowing you to put a live
#######			entry in /etc/fstab that does not automatically mount
#######			at boot time), you will need to cull them from fstab
#######			when generating the list of file systems that are
#######			supposed to be mounted (TABLIST).


TABLIST=$(grep -v "^#" /etc/fstab | awk '{print $1}')

####### Mount may not agree with /etc/fstab where the root FS is concerned.
if mount -p | grep "^/dev/root" > /dev/null 2>&1
then
	#######	We need to "fix" it.
	####### Get the "correct" FS name:
	ROOT=$(grep -v "^#" /etc/fstab |awk '$2 == "/" {print $1}')
	#######	${ROOT} always contains 1 entry?
	MNTLIST=$(mount -p | sed "s#/dev/root#${ROOT}#" | awk '{print $1}')
	unset ROOT
else
	MNTLIST=$(mount -p | awk '{print $1}')
fi

for FS in $(print "${TABLIST}\n${MNTLIST}" | sort | uniq -u)
do
	if grep "^${FS} " /etc/fstab > /dev/null 2>&1
	then
		MISSING="${MISSING} ${FS}"
	else
		EXTRA="${EXTRA} ${FS}"
	fi
done

if [[ -n ${MISSING} ]]
then
	print "UNMOUNTED FILE SYSTEMS:"
	for FS in ${MISSING}
	do
		grep "^${FS} " /etc/fstab
	done | pr -t -o8
	SEPARATOR
fi

if [[ -n ${EXTRA} ]]
then
	print "MOUNTED FILE SYSTEMS NOT APPEARING IN /etc/fstab:"
	for FS in ${EXTRA}
	do
		mount -p | grep "^${FS} "
	done | pr -t -o8
	SEPARATOR
fi

unset MISSING
unset EXTRA
unset TABLIST
unset MNTLIST

#######------------------------------------------------------------------------
#######	Report "old" users (folks who do not log out).
AGERS=$(who -u | awk '$6 == "old" {print $0}')
if [[ -n ${AGERS} ]]
then
	print "USERS LOGGED IN MORE THAN 24 HOURS:"
	print ${AGERS} | pr -t -o8
	SEPARATOR
fi

#######------------------------------------------------------------------------
#######	Detect shared memory segments with missing creator/last read PID.
SHMSEGS=$(ipcs -mp | awk '$1 == "m" {print $2,$7,$8}' | while read MID CPID LPID
	do
		if ps -fp${LPID} > /dev/null 2>&1
		then
			# the process exists
			:
		else
			# the process is not in the process table!
			if ps -fp${CPID} > /dev/null 2>&1
			then
				# the process exists
				:
			else
				ipcs -mp | grep "^m *${MID} "
			fi
		fi
	done)

if [[ -n ${SHMSEGS} ]]
then
	print "SHARED MEMORY SEGMENTS WITH MISSING CPID and LPID:"
	print "${SHMSEGS}" | pr -t -o8
	SEPARATOR
fi

unset SHMSEGS

#######------------------------------------------------------------------------
#######	Detect message queues with missing last read/last send PID.
MSGQS=$(ipcs -qp | awk '$1 == "m" {print $2,$7,$8}' | while read QID LSPID LRPID
	do
		if ps -fp${LSPID} > /dev/null 2>&1
		then
			# the process exists
			:
		else
			# the process is not in the process table!
			if ps -fp${LRPID} > /dev/null 2>&1
			then
				# the process exists
				:
			else
				ipcs -qp | grep "^q *${QID} "
			fi
		fi
	done)

if [[ -n ${MSGQS} ]]
then
	print "MESSAGE QUEUES MISSING LSPID and LRPID:"
	print "${MSGQS}" | pr -t -o8
	SEPARATOR
fi

unset MSGQS

#######------------------------------------------------------------------------
####### Detect Semaphores without owners that are not running processes.
WHOLIST=$(ps -ef | awk '{print $1}')
WHOLIST=$(print ${WHOLIST} | sed "s/ /|/g")	# now suitable for egrep select
SEMKEYS=$(ipcs -sp | grep "^s " | grep -Ev "${WHOLIST}" | awk '{print $3}')
if [[ -n ${SEMKEYS} ]]
then
	print "POTENTIALLY ORPHANED SEMAPHORES:"
	SEMKEYS=$(print ${SEMKEYS} | sed "s/ /|/g")
	ipcs -sp | grep "^s " | grep -E "${SEMKEYS}" | pr -t -o8
	SEPARATOR
fi

unset WHOLIST
unset SEMKEYS

#######------------------------------------------------------------------------
#######	Looing for orphaned processes (inherited by init PID=1).
ORPHANS=$(ps -ef | grep -Ev "root|lp|daemon" | awk '$3 == 1 {print $0}')
if [[ -n ${ORPHANS} ]]
then
	print "ORPHANED PROCESSES:"
	print "${ORPHANS}"
	SEPARATOR
fi

unset ORPHANS

#######------------------------------------------------------------------------
#######	Looking for runaway processes.
####### NOTE:	If you have a database on this system, more than 1 minute of
#######		accumulated time may be too small.
RUNAWAYS=$(ps -e | awk '	{
				split($3,time,":")
				if ( time[1] > 1 ) print $0
				}')
if [[ -n ${RUNAWAYS} ]]
then
	print "RUNAWAY PROCESSES:"
	print "${RUNAWAYS}"
	SEPARATOR
fi

unset RUNAWAYS

#######------------------------------------------------------------------------
#######	FILES IN THE WRONG PLACE: 
CHARINDEV=$(find /dev -type f)
if [[ -n ${CHARINDEV} ]]
then
	print "CHARACTER FILES IN /dev:"
	print "${CHARINDEV}" | pr -t -o8
	SEPARATOR
fi
unset CHARINDEV

#######------------------------------------------------------------------------
####### DEVICE FILES OUTSIDE OF /dev:
DEVOUTSIDEDEV=$(find / -path /dev -prune -o \( -type b -o -type c \) -print)
if [[ -n ${DEVOUTSIDEDEV} ]]
then
	print "DEVICE FILES FOUND OUTSIDE OF /dev:"
	print "${DEVOUTSIDEDEV}" | pr -t -o8
	SEPARATOR
fi
unset DEVOUTSIDEDEV

#######------------------------------------------------------------------------
#######	Find core files.
CORES=$(find / -name core -print)
if [[ -n ${CORES} ]]
then
	print "CORE FILES:"
	for FILE in ${CORES}
	do
		file ${FILE}
	done
	SEPARATOR
fi
unset CORES

#######------------------------------------------------------------------------
#######	Track SUID & SGID files.
####### 	This is not the best way to do this.  You should really put
#######		a list of these files in a secure place (like a tape) for
#######		comparison.  Otherwise, anyone with SuperUser access can change
#######		the data you are comparing with.
CURSUM=$(find / \( -perm -4000 -o -perm 2000 \) -exec ls -ld {} \; | cksum)
if [[ "${CURSUM}" != ${CHKSUM} ]]
then
	print "CHECKSUM OF SUID/SGID FILES  *F*A*I*L*E*D*!"
	print "\tCURRENT CHECKSUM IS\t${CURSUM}"
	print "\tSAVED CHECKSUM IS\t${CHKSUM}"
	SEPARATOR
fi
unset CURSUM

#######------------------------------------------------------------------------
#######	Track SUID & SGID files.
DISCONNECTED=$(find / \( -nouser -o -nogroup \) -exec ls -ld {} \;)
if [[ -n ${DISCONNECTED} ]]
then
	print "FILES WITHOUT A VALID USER OR GROUP:"
	print "\n${DISCONNECTED}"
	SEPARATOR
fi
unset DISCONNECTED

#######------------------------------------------------------------------------
#######	Look for LOST+FOUND FILES:
MOUNTS=$(bdf -l | awk '$1 != "Filesystem" {print $NF}')
for MP in ${MOUNTS}
do
	if ls ${MP}/lost+found > /dev/null 2>&1
	then
		if [[ $(ls -1 ${MP}/lost+found | wc -l) > 1 ]]
		then
			CONTENTSLFDIRS="${CONTENTSLFDIRS} ${MP}"
		fi
	else
		MISSINGLFDIRS="${MISSINGLFDIRS} ${MP}"
	fi
done

if [[ -n ${MISSINGLFDIRS} ]]
then
	print "FILE SYSTEMS WITHOUT A lost+found DIRECTORY:"
	for I in ${MISSINGLFDIRS}
	do
		print ${I}
	done | pr -t -o8

	SEPARATOR
fi

if [[ -n ${CONTENTSLFDIRS} ]]
then
	print "CONTENTS OF lost+found DIRECTORIES:"
	for I in ${CONTENTSLFDIRS}
	do
		print "${I}/lost+found:"
		ls -l ${I}/lost+found
	done | pr -t -o8
	SEPARATOR
fi

unset MISSINGLFDIRS
unset CONTENTSLFDIRS

#######------------------------------------------------------------------------
#######	Report stuff in syslog.
SYSLOGMSGS=$(grep -Ei "err|crit|warn|panic" /var/adm/syslog/syslog.log)
if [[ -n ${SYSLOGMSGS} ]]
then
	print "THE FOLLOWING MESSAGES FOUND IN syslog:"
	echo "${SYSLOGMSGS}" | pr -t -o8
	SEPARATOR
fi
unset SYSLOGMSGS

#######------------------------------------------------------------------------
#######	Scan for disabled printers.
DISABLED=$(lpstat -p | fgrep disabled)
if [[ -n ${DISABLED} ]]
then
	print "DISABLED PRINTERS:"
	print "${DISABLED}" | pr -t -o8
	SEPARATOR
fi
unset DISABLED

#######------------------------------------------------------------------------
#######	Report Bad Login Attempts

####### NOTE: When you schedule this thing, do it _before_ midnight, or pass
####### yesterday's day to the grep below (which is a bit messy).
cat /var/adm/btmp |
	 /usr/sbin/acct/fwtmp |
		grep "$(date "+%b %d")" > ${TMPDIR}/btmp.fmtd
GOODLOGINS=$(	for I in $(awk '{print $1}' ${TMPDIR}/btmp.fmtd | sort -u)
		do
			if id ${I} > /dev/null 2>&1
			then
				print ${I}
			fi
		done)
BADLOGINS=$(	for I in ${GOODLOGINS}
		do
			ATTEMPTS=$(grep -c "^${I}" ${TMPDIR}/btmp.fmtd)
			if [[ ${ATTEMPTS} > ${EBA} ]]
			then
				print "${ATTEMPTS}\t${I}"
			fi
		done| sort -n)
if [[ -n ${BADLOGINS} ]]
then
	print "EXCESSIVE BAD LOGIN ATTEMPTS FOR THE FOLLOWING USERS TODAY:"
	print "${BADLOGINS}" | pr -t -o8
	SEPARATOR
fi
unset BADLOGINS
unset GOODLOGINS


#######------------------------------------------------------------------------
#######	Clean up after ourselves.
rm -rf ${TMPDIR}

#######------------------------------------------------------------------------
#######	Note finish time.
print -n "DAILY FINISHED: ";date
SEPARATOR
#######	End of NightOwl #######################################################