#!/bin/bash

###### ZPool & SMART status report with FreeNAS/TrueNAS config backup
### Original script by joeschmuck, modified by Bidelu0hm, then by melp (me)

### At a minimum, enter email address in user-definable parameter section. Feel free to edit other user parameters as needed.
### If you find any errors, feel free to contact me on the FreeNAS forums (username melp) or email me at jason at jro dot io.

### Version: v1.3 TrueNAS Scale (Jeff Alperin 12-6-21)

### Version v1.4c & v1.4d FreeNAS/TrueNAS (joeschmuck 22 June 2022)

### Changelog:
# v1.4d:
#   - Fixed Scrub In Progress nuisance error when a scrub is in progress.
#   - Added offsetting Reallocated Sectors for four drives.  This should be for testing only. Any drives
#   -- with a significant number of bad sectors should be replaced, not used to store important data.
#   - Added Drive Warranty Expiration warning messages and ability to disable the Email Subject line warning.
#   - Added HDD and SSD individual temperature settings.
#   -- NOT TESTED ON OTHER THAN U.S. FORMATTED DATE YYYY-MM-DD.
#   - Changed order of pulling Temperature data from HDD/SSD.   
#   
# v1.4c:
#   - Rewrite to create functions and enable easier editing.
#   - Added Custom Reports.
#   - Added disabling the RAW 'smartctl -a' data appened to the end of the email.
#   - Added sorting drives alphabetically vice the default the OS reports them.
#   - Added RED warning in Device for any single failure in the summary (deviceRedFlag switch controlled).
#   - Added some additional SSD definitions.
#   - Fixed sorting last two SMART Tests, now reports them in proper order.
#   - Fixed detecting "SMART Support is: Enabled", for white spaces.
#   - Changed IGNORE DRIVES to a String Format to clean up and simplify programming.
#   - Added MultiZone_Errors support for up to eight drives.
#   - Added sectorWarn variable to complement the sectorCrit variable.
#   - Added ignoreSeekError variable to ignore some of those wild Seek Error Rate values.
#   - Added ignoreUDMA CRC Errors due to the "Known Problem"
#   - Fixed md5/sha256 error on TrueNAS Scale (only used during config backups).
#   - Added selectable config backup periodicity by day vice every run.
#   - Added Exporting statistical data for trend analysis.
#   -- Can be setup to email statistics weekly, monthly, or not at all.
#   -- The -s switch will run Data Collection Only, no email generated.  Note: Do Not run two instances at once, the temp files do not survive.
#   - Fixed the Capacity to remove the brackets "[]", thanks Jeff Alperin.
#   - Fixed Scrub Age failure due to 1 day or longer repair time, now shows anything >24 hours.
#
#   - Known Problem: One user reported UDMA_CRC_ERRORS is not subtracting correctly, have not been able to personnaly replicate it.
#   -- This error seems to occur around line #1027
#
# v1.4b:
#   - Added SMART test reamining percentage if Last Test has a SMART Test is in progress.
#   - Fix for empty SMART fields, typically for unsupported SSD's.
#   - Added IGNORE SMART Drive so you can ignore specific drives that may cause you wierd readings.
#   --- Updated so blank SSD table header is removed when you ignore all the drives (just crazy talk).
# v1.4a:
#   - Fixed report errors for if a SCRUB is in progress, now shows estimated completion time.
#   - Fixed report error for a Canceled SCRUB.
#   - Fixed FreeBSD/Linux use for SCRUB report (minor oversight).
# v1.4:
#   - Run on CRON JOB using /path/multi_report_v1.4.sh
#   - Fixed for automatic running between FreeBSD and Linux Debian (aka SCALE) as of this date.
#   - All SMART Devices will report.
#   - Added conditional Subject Line (Good/Critical/Warning).
#   - Added Automatic SSD Support.
#   --- Some updates may need to be made to fit some of SSD's. Code in the area of about line 530 will
#   --- need to be adjusted to add new attributes for the desired SSD's fields.
#   - UDMA_CRC_ERROR Override because once a drive encounters this type of error, it cannot be cleared
#   --- so you can offset it now vice having an alarm condition for old UDMA_CRC_Errors.
#   - Added listing NON-SMART Supported Drives.  Use only if useful to you, some drives will
#   --- still output some relevant data, many will not.
# v1.3:
#   - Added scrub duration column
#   - Fixed for FreeNAS 11.1 (thanks reven!)
#   - Fixed fields parsed out of zpool status
#   - Buffered zpool status to reduce calls to script
# v1.2:
#   - Added switch for power-on time format
#   - Slimmed down table columns
#   - Fixed some shellcheck errors & other misc stuff
#   - Added .tar.gz to backup file attached to email
#   - (Still coming) Better SSD SMART support
# v1.1:
#   - Config backup now attached to report email
#   - Added option to turn off config backup
#   - Added option to save backup configs in a specified directory
#   - Power-on hours in SMART summary table now listed as YY-MM-DD-HH
#   - Changed filename of config backup to exclude timestamp (just uses datestamp now)
#   - Config backup and checksum files now zipped (was just .tar before; now .tar.gz)
#   - Fixed degrees symbol in SMART table (rendered weird for a lot of people); replaced with a *
#   - Added switch to enable or disable SSDs in SMART table (SSD reporting still needs work)
#   - Added most recent Extended & Short SMART tests in drive details section (only listed one before, whichever was more recent)
#   - Reformatted user-definable parameters section
#   - Added more general comments to code
# v1.0:
#   - Initial release

######### INSTRUCTIONS ON USE OF THIS SCRIPT
#
# This script will perform two main functions: 1: Generate a report send to email on your drive status,
# and 2: Create a copy of your Config File and attach to the same email.
#
# In order to configure the script properly read over the User Definable Section before making any changes.
# Then make changes as indicted by the section instructions.
#
# To run the program from the command line, use ./program_name.sh
# Also you need to make it executable using "chmod +x program_name.sh"
#
# Enjoy


###### User-definable Parameters
# Enter your email address to send the report to.  The from address does not need to be changed.


###### Email Address
email="YourEmail@Address.com"
from="TrueNAS@local.com"


###### REPORT HEADER TITLE ######
HDDreportTitle="Spinning Rust Summary Report"  # This is the title of the HDD report, change as you desire.
SSDreportTitle="SSD Summary Report"  # This is the title of the SSD report, change as you desire.

### CUSTOM REPORT CONFIGURATION ###
# By default all items are selected.  Change the item to "true" to have it displayed, "false" to not have it displayed.
# NOTE: Alarm setpoints are not affected by these settings.

# For Hard Drive Section
HDD_Device_ID="true"
HDD_Serial_Number="true"
HDD_Model_Number="true"
HDD_Capacity="true"
HDD_Rotational_Rate="true"
HDD_SMART_Status="true"
HDD_Drive_Temp="true"
HDD_Power_On_Hours="true"
HDD_Start_Stop_Count="true"
HDD_Load_Cycle="true"
HDD_Spin_Retry="true"
HDD_Reallocated_Sectors="true"
HDD_Reallocated_Events="true"
HDD_Pending_Sectors="true"
HDD_Offline_Uncorrectable="true"
HDD_UDMA_CRC_Errors="true"
HDD_Seek_Error_Rate="true"
HDD_MultiZone_Errors="true"
HDD_Last_Test_Age="true"
HDD_Last_Test_Type="true"

# For Solid State Drive Section
SSD_Device_ID="true"
SSD_Serial_Number="true"
SSD_Model_Number="true"
SSD_Capacity="true"
SSD_SMART_Status="true"
SSD_Drive_Temp="true"
SSD_Power_On_Hours="true"
SSD_Wear_Level="true"
SSD_Reallocated_Sectors="true"
SSD_Reallocated_Events="true"
SSD_Pending_Sectors="true"
SSD_Offline_Uncorrectable="true"
SSD_UDMA_CRC_Errors="true"
SSD_Last_Test_Age="true"
SSD_Last_Test_Type="true"


###### Drive Ignore List
# Use this section to list any drives to ignore and remove from the report.  This is useful for ignoring USB Flash Drives.
# We are using a comma delimited file to identify the drive serial numbers.  You MUST use the exact and full serial number
# smartctl reports, if there is no identical match then it doesn't work properly.
#
# Format: IGNOREDRIVES="DRIVE_SERIAL_NUMBER,DRIVE_SERIAL_NUMBER"
#
# Example:  IGNOREDRIVES="1GVD84B,1JUMLBD,WD-WXQ1E36C19XC"
IGNOREDRIVES="2X1J90CA48799"


###### Drive UDMA_CRC_Error_Count Flags
# List each drive by serial number and include the current UDMA_CRC_Error_Count value.
# If the count exceeds the count in the table below then a warning message will be generated.
# On the Status Report the UDMA CRC Errors block will be YELLOW with a value of "0" for an overridden value, for your indication.
#
# Format: UCRC1SN = Drive Serial Number reported by SMART, UCRC1CNT = Current UDMA Error Count you want to subtract from this drive.
# You may add up to 8 drives with the current code. More drives could be added with a simple modification of the program.
# Do not list your drives with a value of "0" or each one will be Yellow.  Yikes!
#
# Example (drive has 2 UDMA CRC Errors): UCRC1SN="S2X1J90CA48799"
#                                        UCRC1CNT=2
# Drives "2011E2940534" and "2011E2940536" are hard coded at lines 1057 through 1060.
UCRC1SN="WD-WMC4N2578099"
UCRC1CNT=1
UCRC2SN="S2X1J90CA48799"
UCRC2CNT=2
UCRC3SN="P02618119268"
UCRC3CNT=1
UCRC4SN="DRIVE_SERIAL_NUMBER"
UCRC4CNT=0
UCRC5SN="DRIVE_SERIAL_NUMBER"
UCRC5CNT=0
UCRC6SN="DRIVE_SERIAL_NUMBER"
UCRC6CNT=0
UCRC7SN="DRIVE_SERIAL_NUMBER"
UCRC7CNT=0
UCRC8SN="DRIVE_SERIAL_NUMBER"
UCRC8CNT=0


###### Multi_Zone_Errors
# This identifies drives with Multi_Zone_Errors which may be irritating people.
# Using same format as UDMA_CRC_Errors (see above).

MULTIZONE1SN="WD-WCC4N0KZDD1L"
MULTIZONE1CNT=4
MULTIZONE2SN="WD-WCC4N2VNZ6HH"
MULTIZONE2CNT=51
MULTIZONE3SN="S2X1J90CA48799"
MULTIZONE3CNT=31
MULTIZONE4SN="DRIVE_SERIAL_NUMBER"
MULTIZONE4CNT=0
MULTIZONE5SN="DRIVE_SERIAL_NUMBER"
MULTIZONE5CNT=0
MULTIZONE6SN="DRIVE_SERIAL_NUMBER"
MULTIZONE6CNT=0
MULTIZONE7SN="DRIVE_SERIAL_NUMBER"
MULTIZONE7CNT=0
MULTIZONE8SN="DRIVE_SERIAL_NUMBER"
MULTIZONE8CNT=0


#######  Reallocated Sectors Exceptions
# This section should only be used for testing, normally you should be replacing drives that have a higher than normal reallocated sector count.
BADSECTORS1SN="W3N0B3RP"
BADSECTORS1CNT=16
BADSECTORS2SN="DRIVE_SERIAL_NUMBER"
BADSECTORS2CNT=0
BADSECTORS3SN="DRIVE_SERIAL_NUMBER"
BADSECTORS3CNT=0
BADSECTORS4SN="DRIVE_SERIAL_NUMBER"
BADSECTORS4CNT=0


####### Warranty Expiration Date
# This section is used to add warranty expirations for designated drives and to create an alert when they expire.
# Format of date is YYYY-MM-DD
# To disable this feature, remove the drive on the list below.
WARRANTYDRIVESN1="K1JUMLBD"
WARRANTYDRIVESN1DATE="2020-09-30"
WARRANTYDRIVESN2="K1JRSWLD"
WARRANTYDRIVESN2DATE="2020-09-30"
WARRANTYDRIVESN3="K1JUMW4D"
WARRANTYDRIVESN3DATE="2020-09-30"
WARRANTYDRIVESN4="K1GVD84B"
WARRANTYDRIVESN4DATE="2020-10-12"


###### zpool status summary table settings
usedWarn=90               # Pool used percentage for CRITICAL color to be used.
scrubAgeWarn=37           # Maximum age (in days) of last pool scrub before CRITICAL color will be used (30 + 7 days for day of week). Default=37.


###### Report Status Summary Table Settings
# These settings will adjust how the scripts acts.
#
includeSSD="true"         # Set to "true" will engage SSD Automatic Detection and Reporting, false = Disable SSD Automatic Detection and Reporting.
reportnonSMART="true"     # Will force even non-SMART devices to be reported, "false" = normal operation to ignore non-SMART devices.
HDDtempWarn=45            # HDD Drive temp (in C) at which WARNING color/message will be used.
HDDtempCrit=50            # HDD Drive temp (in C) at which CRITICAL color/message will be used.
SSDtempWarn=50            # SSD Drive temp (in C) at which WARNING color/message will be used.
SSDtempCrit=65            # SSD Drive temp (in C) at which CRITICAL color/message will be used.
sectorsWarn=2             # Number of sectors per drive with errors before WARNING color/message will be used, this value should be less than sectorsCrit.
sectorsCrit=10            # Number of sectors per drive with errors before CRITICAL color/message will be used.
testAgeWarn=2             # Maximum age (in days) of last SMART test before CRITICAL color/message will be used.
powerTimeFormat="h"       # Format for power-on hours string, valid options are "ymdh", "ymd", "ym", "y", or "h" (year month day hour).
deviceRedFlag="true"      # Set "true" to have the Device Column indicate RED for ANY alarm condition.  Default is true.
ignoreMultiZone="false"   # Set to "true" to ignore all MultiZone Errors. Default is false.
ignoreSeekError="true"    # Set to "true" to ignore all Seek Error Rate/Health errors.  Default is true.
ignoreUDMA="false"        # Set to "true" to ignore all UltraDMA CRC Errors for the summary alarm (Email Header) only, errors will appear in the graphical chart.
expDataEnable="true"      # Set to "true" will save all drive data into a CSV file defined by "statistical_data_file" below. 
expDataEmail="true"       # Set to "true" to have an attachment of the file emailed to you. Default is true.
expDataPurge=730          # Set to the number of day you wish to keep in the data.  Data older will be purged. Default is 730 days (2 years). 0=Disable.
expDataEmailSend="Mon"    # Set to the day of the week the statistical report is emailed.  (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month) 
disableRAWdata="false"    # Set to "true" to remove the 'smartctl -a' data and non-smart data appended to the normal report.  Default is false.
disableWarranty="true"   # Set to "true to disable email Subject line alerts for any expired warranty alert. The email body will still report the alert.

###### Export Data File
# This is the loction for your export file to reside.
statistical_data_file="/tmp/statisticalsmartdata.csv"


###### FreeNAS config backup settings
configBackup="true"      # Set to "true" to save config backup (which renders next two options operational); "false" to keep disable config backups.
configSendDay="Mon"      # Set to the day of the week the statistical report is emailed.  (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month)
saveBackup="false"       # Set to "false" to delete FreeNAS config backup after mail is sent; "true" to keep it in dir below.
backupLocation="/tmp/"   # Directory in which to store the backup FreeNAS config files.


###### Global table of colors
# The colors selected you can change but you will need to look up the proper HEX code for a color.
#
okColor="#c9ffcc"       # Hex code for color to use in SMART Status column if drives pass (default is light green, #c9ffcc).
warnColor="#f765d0"     # Hex code for WARN color (default is purple, #f765d0).
critColor="#ff0000"     # Hex code for CRITICAL color (default is red, #ff0000).
altColor="#f4f4f4"      # Table background alternates row colors between white and this color (default is light gray, #f4f4f4).
whtColor="#ffffff"      # Hex for White background.
ovrdColor="#ffff66"     # Hex code for Override Yellow.
blueColor="#87ceeb"     # Hex code for Sky Blue, used for the SCRUB In Progress background.


###### STOP EDITING THE SCRIPT HERE ######
# If you really feel like editing below this line, you are customizing the code beyond a novice user.

###### Auto-generated Parameters
programver="Multi-Report v1.4d (22 June 2022)"
softver=$(uname -s)
host=$(hostname -s)
logfile="/tmp/smart_report_body.tmp"
logfile_header="/tmp/smart_report_header.tmp"
logfile_warning="/tmp/smart_report_warning_flag.tmp"
logfile_critical="/tmp/smart_report_critical_flag.tmp"
logfile_warranty="/tmp/smart_report_warranty_flag.tmp"
boundary="gc0p4Jq0M2Yt08jU534c0p"



##########################
##########################
###                    ###
###  DEFINE FUNCTIONS  ###
###                    ###
##########################
##########################


#################### PURGE EXPORT DATA CSV FILE #######################

purge_exportdata () {
### This routine will purge the "statistical_data_file" of data older then "expDataPurge".
#
echo "Running Purging Routine"

# Delete temp file if it exists
if test -e "/tmp/temp_purge_file.csv"; then
rm "/tmp/temp_purge_file.csv"
# Create the header
  printf "Date,Time,Device ID,Drive Type,Serial Number,SMART Status,Temp,Power On Hours,Wear Level,Start Stop Count,Load Cycle,Spin Retry,Reallocated Sectors,\
ReAllocated Sector Events,Pending Sectors,Offline Uncorrectable,UDMA CRC Errors,Seek Error Rate,Multi Zone Errors,\n" > "/tmp/temp_purge_file.csv"
fi

 {

input="$statistical_data_file"

if [ $softver != "Linux" ]; then
expireDate=$(date -v -"$expDataPurge"d +%Y/%m/%d)
else
expireDate=$(date -d "$expDataPurge days ago" +%Y/%m/%d) 
fi

awk -v expireDate="$expireDate" '
BEGIN {
  FS=OFS=","
  FPAT = "([^,]+)|(\"[^\"]+\")"
    count=0
 }

data=$1

     {
FS=OFS=" "

if (count !=0) if ($1 >= expireDate) {
  printf ("%s\n", data) >> "/tmp/temp_purge_file.csv"
  }
 ++count
     }

END {
    }
' $input >/dev/null

}
 
cp -R "/tmp/temp_purge_file.csv" "$statistical_data_file"
}


################## EMAIL EXPORT DATA CVS FILE #########################

email_datafile () {

if [ "$expDataEmail" == "true" ]; then
Now=$(date +"%a")
doit="false"
  case $expDataEmailSend in
    All)
      doit="true"
      ;;
    Mon|Tue|Wed|Thu|Fri|Sat|Sun)
      if [[ "$expDataEmailSend" == "$Now" ]]; then
         doit="true"
      fi
      ;;
    Month)
      if [[ $(date +"%d") == "01" ]]; then
         doit="true"
      fi
      ;;
    *)
      ;;
  esac

  if [[ "$doit" == "true" ]]; then
 (
  # Write MIME section header for file attachment (encoded with base64)
  echo "--${boundary}"
  echo "Content-Type: text/csv"
  echo "Content-Transfer-Encoding: base64"
  echo "Content-Disposition: attachment; filename=Statistical_Data.csv"
  base64 "$statistical_data_file"
  echo "--${boundary}--"
 ) >> "$logfile"
  fi

fi

}

#################### GET SMART HARD DRIVES ############################

get_smartHDD_listings () {
### GET non-SSD listing - MUST support SMART
# smartdrives= All Drives that are SMART
# smartdrivesSSD = SSD all SMART
# nonsmartdrives = Any drive not supporting SMART

# Get Hard Drive listing - MUST support SMART
# variable smartdrives
if [ $softver != "Linux" ]; then
    smartdrives=$(for drive in $(sysctl -n kern.disks); do
      if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then

         targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')";

### Process Ignore List ###
         s="0"
         IFS=',' read -ra ADDR <<< "$IGNOREDRIVES"
           for i in "${ADDR[@]}"; do
             if [[ $i == $targument ]]; then
                s="1"
                continue
             fi
           done
         if [[ $s == "0" ]]; then
           printf "%s " "${drive}"
         fi
      fi
    done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
else
    smartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do
        if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then
            targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')";
### Process Ignore List ###
         s="0"
         IFS=',' read -ra ADDR <<< "$IGNOREDRIVES"
           for i in "${ADDR[@]}"; do
             if [[ $i == $targument ]]; then
                s="1"
                continue
             fi
           done
         if [[ $s == "0" ]]; then
           printf "%s " "${drive}"
         fi
       fi
    done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
fi
}


########################## GET SMART SOLID DISK DRIVES ################################

get_smartSSD_listings () {

# Get SSD listing - MUST suport SMART
# variable smartdrivesSSD
  if [ $softver != "Linux" ]; then
   smartdrivesSSD=$(for drive in $(sysctl -n kern.disks); do
        if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then
            targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')";

### Process Ignore List ###
         s="0"
         IFS=',' read -ra ADDR <<< "$IGNOREDRIVES"
           for i in "${ADDR[@]}"; do
             if [[ $i == $targument ]]; then
                s="1"
                continue
             fi
           done
         if [[ $s == "0" ]]; then
           printf "%s " "${drive}"
         fi
        fi
    done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
  else
   smartdrivesSSD=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do
        if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then
            targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')";
### Process Ignore List ###
         s="0"
         IFS=',' read -ra ADDR <<< "$IGNOREDRIVES"
           for i in "${ADDR[@]}"; do
             if [[ $i == $targument ]]; then
                s="1"
                continue
             fi
           done
         if [[ $s == "0" ]]; then
           printf "%s " "${drive}"
         fi
        fi
    done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
  fi
}


########################## GET OTHER SMART DEVICES ##################################

get_smartOther_listings () {
### Get the non-SSD listing - MUST support SMART
# variable nonsmartdrives
  if [ $softver != "Linux" ]; then
   nonsmartdrives=$(for drive in $(sysctl -n kern.disks); do
        if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ]; then
              targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')";
### Process Ignore List ###
         s="0"
         IFS=',' read -ra ADDR <<< "$IGNOREDRIVES"
           for i in "${ADDR[@]}"; do
             if [[ $i == $targument ]]; then
                s="1"
                continue
             fi
           done
         if [[ $s == "0" ]]; then
           printf "%s " "${drive}"
         fi
        fi
    done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
  else
   nonsmartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do
        if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is: Enabled")" ]; then
                  targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')";
### Process Ignore List ###
         s="0"
         IFS=',' read -ra ADDR <<< "$IGNOREDRIVES"
           for i in "${ADDR[@]}"; do
             if [[ $i == $targument ]]; then
                s="1"
                continue
             fi
           done
         if [[ $s == "0" ]]; then
           printf "%s " "${drive}"
         fi
        fi
    done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
  fi
}

########################### FORMAT EMAILS STEP 1 ##########################

email_preformat () {
###### Email pre-formatting
### Set some of the email headers before conditional headers
(
    echo "MIME-Version: 1.0"
    echo "Content-Type: multipart/mixed; boundary=${boundary}"
) > "$logfile"
}


########################### CONFIGURATION BACKUP ##############################

config_backup () {
###### Config backup (if enabled)
    tarfile="/tmp/config_backup.tar.gz"
    filename="$(date "+FreeNAS_Config_%Y-%m-%d")"
    filename2="Stat_Data"
#1
if [ "$configBackup" == "true" ]; then
 ############################################################################################################
 Now=$(date +"%a")
 doit="false"
  case $configSendDay in
    All)
      doit="true"
      ;;
    Mon|Tue|Wed|Thu|Fri|Sat|Sun)
      if [[ "$configSendDay" == "$Now" ]]; then
         doit="true"
      fi
      ;;
    Month)
      if [[ $(date +"%d") == "01" ]]; then
         doit="true"
      fi
      ;;
    *)
      ;;
  esac

  if [[ "$doit" == "true" ]]; then

    # Set up file names, etc for later
    tarfile="/tmp/config_backup.tar.gz"
    filename="$(date "+FreeNAS_Config_%Y-%m-%d")"
    filename2="Stat_Data"
    ### Test config integrity

    if ! [ "$(sqlite3 /data/freenas-v1.db "pragma integrity_check;")" == "ok" ]; then
        # Config integrity check failed, set MIME content type to html and print warning
        (
            echo "--${boundary}"
            echo "Content-Type: text/html"
            echo "<b>Automatic backup of FreeNAS configuration has failed! The configuration file is corrupted!</b>"
            echo "<b>You should correct this problem as soon as possible!</b>"
            echo "<br>"
        ) >> "$logfile"

    else
        # Config integrity check passed; copy config db, generate checksums, make .tar.gz archive
        cp /data/freenas-v1.db "/tmp/${filename}.db"

                   if [ $softver != "Linux" ]; then
                      md5 "/tmp/${filename}.db" > /tmp/config_backup.md5
                      sha256 "/tmp/${filename}.db" > /tmp/config_backup.sha256
                   else
                      md5sum "/tmp/${filename}.db" > /tmp/config_backup.md5
                      sha256sum "/tmp/${filename}.db" > /tmp/config_backup.sha256
                   fi

        (
            cd "/tmp/" || exit;
            tar -czf "${tarfile}" "./${filename}.db" ./config_backup.md5 ./config_backup.sha256;
        )
        (
            # Write MIME section header for file attachment (encoded with base64)
            echo "--${boundary}"
            echo "Content-Type: application/tar+gzip"
            echo "Content-Transfer-Encoding: base64"
            echo "Content-Disposition: attachment; filename=${filename}.tar.gz"
            base64 "$tarfile"
            # Write MIME section header for html content to come below
            echo "--${boundary}"
            echo "Content-Type: text/html"
        ) >> "$logfile"
        # If logfile saving is enabled, copy .tar.gz file to specified location before it (and everything else) is removed below

                if [ "$saveBackup" == "true" ]; then
                  cp "${tarfile}" "${backupLocation}/${filename}.tar.gz"
                fi

        rm "/tmp/${filename}.db"
        rm /tmp/config_backup.md5
        rm /tmp/config_backup.sha256
        rm "${tarfile}"

    fi

else
    (
        echo "--${boundary}"
        echo "Content-Type: text/html"
    ) >> "$logfile"

  fi

else
  # configBackup = false so this is what to do
  # Config backup enabled; set up for html-type content
    (
        echo "--${boundary}"
        echo "Content-Type: text/html"
    ) >> "$logfile"

fi

}


################################## GENERATE ZPOOL REPORT ##################################

zpool_report () {
###### Report Summary Section (html tables)
### zpool status summary table
(
    # Write HTML table headers to log file; HTML in an email requires 100% in-line styling (no CSS or <style> section), hence the massive tags
echo $programver
    echo "<br><br>"
    echo "<table style=\"border: 1px solid black; border-collapse: collapse;\">"
    echo "<tr><th colspan=\"10\" style=\"text-align:center; font-size:20px; height:40px; font-family:courier;\">ZPool Status Report Summary</th></tr>"
    echo "<tr>"
    echo "  <th style=\"text-align:center; width:130px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Pool<br>Name</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Status</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Read<br>Errors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Write<br>Errors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Cksum<br>Errors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Used %</th>"
    echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Scrub<br>Repaired<br>Bytes</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Scrub<br>Errors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last<br>Scrub<br>Age</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last<br>Scrub<br>Duration</th>"
    echo "</tr>"
) >> "$logfile"
pools=$(zpool list -H -o name)
poolNum=0
for pool in $pools; do
    # zpool health summary
    status="$(zpool list -H -o health "$pool")"
    # Total all read, write, and checksum errors per pool
    errors="$(zpool status "$pool" | grep -E "(ONLINE|DEGRADED|FAULTED|UNAVAIL|REMOVED)[ \\t]+[0-9]+")"
    readErrors=0
    for err in $(echo "$errors" | awk '{print $3}'); do
        if echo "$err" | grep -E -q "[^0-9]+"; then
            readErrors=1000
            break
        fi
        readErrors=$((readErrors + err))
    done
    writeErrors=0
    for err in $(echo "$errors" | awk '{print $4}'); do
        if echo "$err" | grep -E -q "[^0-9]+"; then
            writeErrors=1000
            break
        fi
        writeErrors=$((writeErrors + err))
    done
    cksumErrors=0
    for err in $(echo "$errors" | awk '{print $5}'); do
        if echo "$err" | grep -E -q "[^0-9]+"; then
            cksumErrors=1000
            break
        fi
        cksumErrors=$((cksumErrors + err))
    done
    # Not sure why this changes values larger than 1000 to ">1K", but I guess it works, so I'm leaving it
    # Answer to question above: Becasue it formats the value to = ">1K" vice stating very large values to
    # fit into the formatted table.  All we care about is it's way too high.
    if [ "$readErrors" -gt 999 ]; then readErrors=">1K"; fi
    if [ "$writeErrors" -gt 999 ]; then writeErrors=">1K"; fi
    if [ "$cksumErrors" -gt 999 ]; then cksumErrors=">1K"; fi


    # Get used capacity percentage of the zpool
    used="$(zpool list -H -p -o capacity "$pool")"
    # Gather info from most recent scrub; values set to "N/A" initially and overwritten when (and if) it gathers scrub info
    scrubRepBytes="N/A"
    scrubErrors="N/A"
    scrubAge="N/A"
    scrubTime="N/A"
    statusOutput="$(zpool status "$pool")"


### Fix for SCRUB lasting longer than 24 hours.
scrubDays="$(echo "$statusOutput" | grep "scan" | awk '{print $7}')"

if [[ $scrubDays == "days" ]]; then
   scrubextra="$(echo "$statusOutput" | grep "scan" | awk '{print $6}')"


### Fix for SCRUB Cancelled and In-Progress

#Check if scrub in progress
if [ "$(echo "$statusOutput" | grep -w "scan" | awk '{print $4}')" = "progress" ]; then
        scrubAge="In Progress"
        scrubTime="$(echo "Est Comp: ")$(echo "$statusOutput" | grep "done, " | awk '{print $5}')"

# Check if the SCRUB is completed or canceled.

elif [ "$(echo "$statusOutput" | grep "scan" | awk '{print $2}')" = "scrub" ] && [ "$(echo "$statusOutput" | grep "scan" | awk '{print $3}')" = "canceled" ]; then
          scrubAge="Canceled"
       elif [ "$(echo "$statusOutput" | grep "scan" | awk '{print $2}')" = "scrub" ]; then
        scrubRepBytes="$(echo "$statusOutput" | grep "scan" | awk '{print $4}')"
        scrubRepBytes="$(echo "$scrubRepBytes" | rev | cut -c2- | rev)"
        scrubErrors="$(echo "$statusOutput" | grep "scan" | awk '{print $10}')"
        # Convert time/datestamp format presented by zpool status, compare to current date, calculate scrub age

# For FreeBSD
   if [ $softver != "Linux" ]; then
        scrubDate="$(echo "$statusOutput" | grep "scan" | awk '{print $17"-"$14"-"$15"_"$16}')"
        scrubTS="$(date -j -f "%Y-%b-%e_%H:%M:%S" "$scrubDate" "+%s")"
   else
# For Linux
        scrubDate="$(echo "$statusOutput" | grep "scan" | awk '{print $14" "$15" "$17" "$16}')"
        scrubTS="$(date --date="$scrubDate" "+%s")"
   fi
        currentTS="$(date "+%s")"
        scrubAge=$((((currentTS - scrubTS) + 43200) / 86400))

        scrubTimetemp="$(echo "$statusOutput" | grep "scan" | awk '{print $8}')"
        scrubTime="$scrubextra days $scrubTimetemp"

        else
        #No scrub previously performed
        scrubAge="Never Scrubbed"
fi

else

#Check if scrub in progress
if [ "$(echo "$statusOutput" | grep -w "scan" | awk '{print $4}')" = "progress" ]; then
        scrubAge="In Progress"
        scrubTime="$(echo "Est Comp: ")$(echo "$statusOutput" | grep "done, " | awk '{print $5}')"

# Check if the SCRUB is completed or canceled.

elif [ "$(echo "$statusOutput" | grep "scan" | awk '{print $2}')" = "scrub" ] && [ "$(echo "$statusOutput" | grep "scan" | awk '{print $3}')" = "canceled" ]; then
          scrubAge="Canceled"
       elif [ "$(echo "$statusOutput" | grep "scan" | awk '{print $2}')" = "scrub" ]; then
        scrubRepBytes="$(echo "$statusOutput" | grep "scan" | awk '{print $4}')"
        scrubRepBytes="$(echo "$scrubRepBytes" | rev | cut -c2- | rev)"
        scrubErrors="$(echo "$statusOutput" | grep "scan" | awk '{print $8}')"
        # Convert time/datestamp format presented by zpool status, compare to current date, calculate scrub age

# For FreeBSD
   if [ $softver != "Linux" ]; then
        scrubDate="$(echo "$statusOutput" | grep "scan" | awk '{print $15"-"$12"-"$13"_"$14}')"
        scrubTS="$(date -j -f "%Y-%b-%e_%H:%M:%S" "$scrubDate" "+%s")"
   else
# For Linux
        scrubDate="$(echo "$statusOutput" | grep "scan" | awk '{print $12" "$13" "$15" "$14}')"
        scrubTS="$(date --date="$scrubDate" "+%s")"
   fi
        currentTS="$(date "+%s")"
        scrubAge=$((((currentTS - scrubTS) + 43200) / 86400))
        scrubTime="$(echo "$statusOutput" | grep "scan" | awk '{print $6}')"
       else
        #No scrub previously performed
        scrubAge="Never Scrubbed"
fi

fi

    # Set row's background color; alternates between white and $altColor (light gray)
    if [ $((poolNum % 2)) == 1 ]; then bgColor="#ffffff"; else bgColor="$altColor"; fi
    poolNum=$((poolNum + 1))
    # Set up conditions for warning or critical colors to be used in place of standard background colors
    if [ "$status" != "ONLINE" ]; then statusColor="$warnColor"; echo "scrub offline error" >> "$logfile_critical"; else statusColor="$bgColor"; fi
    if [ "$readErrors" != "0" ]; then readErrorsColor="$warnColor"; echo "scrub read errors" >> "$logfile_warning"; else readErrorsColor="$bgColor"; fi
    if [ "$writeErrors" != "0" ]; then writeErrorsColor="$warnColor"; echo "scrub write errors" >> "$logfile_warning"; else writeErrorsColor="$bgColor"; fi
    if [ "$cksumErrors" != "0" ]; then cksumErrorsColor="$warnColor"; echo "scrub cksum errors" >> "$logfile_warning"; else cksumErrorsColor="$bgColor"; fi
    if [ "$used" -gt "$usedWarn" ]; then usedColor="$warnColor"; echo "scrub used" >> "$logfile_warning"; else usedColor="$bgColor"; fi
	    if [ "$scrubRepBytes" != "N/A" ] && [ "$scrubRepBytes" != "0" ]; then scrubRepBytesColor="$warnColor"; echo "scrub Rep Bytes" >> "$logfile_warning"; else scrubRepBytesColor="$bgColor"; fi
    if [ "$scrubErrors" != "N/A" ] && [ "$scrubErrors" != "0" ]; then scrubErrorsColor="$warnColor"; echo "scrub errors" >> "$logfile_critical"; else scrubErrorsColor="$bgColor"; fi
    if [ "$(echo "$scrubAge" | awk '{print int($1)}')" -gt "$scrubAgeWarn" ]; then scrubAgeColor="$warnColor"; echo "scrub age" >> "$logfile_warning"; else scrubAgeColor="$bgColor"; fi
    if [ "$scrubAge" == "In Progress" ]; then scrubAgeColor="$blueColor"; else scrubAgeColor="$bgColor"; fi
    (
        # Use the information gathered above to write the date to the current table row
        printf "<tr style=\"background-color:%s;\">
            <td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s%%</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
        </tr>\\n" "$bgColor" "$pool" "$statusColor" "$status" "$readErrorsColor" "$readErrors" "$writeErrorsColor" "$writeErrors" "$cksumErrorsColor" \
        "$cksumErrors" "$usedColor" "$used" "$scrubRepBytesColor" "$scrubRepBytes" "$scrubErrorsColor" "$scrubErrors" "$scrubAgeColor" "$scrubAge" "$scrubTime"
    ) >> "$logfile"

done
# End of zpool status table
echo "</table>" >> "$logfile"
}

######################################## HARD DISK & SSD REPORT #################################
# Call with hdd_report "drivelist" "full/normal/trim/ssd" "report title"

hdd_report () {
drives="$1"
detail_level="$2"

# Lets add up how many columns we will need.

Columns=0;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Device_ID" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Serial_Number" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Model_Number" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Capacity" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Rotational_Rate" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_SMART_Status" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Drive_Temp" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Power_On_Hours" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Start_Stop_Count" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Load_Cycle" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Spin_Retry" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Reallocated_Sectors" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Reallocated_Events" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Pending_Sectors" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Offline_Uncorrectable" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_UDMA_CRC_Errors" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Seek_Error_Rate" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_MultiZone_Errors" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Last_Test_Age" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "HDD" ]] && [[ "$HDD_Last_Test_Type" == "true" ]]; then ((Columns=Columns+1)); fi;

# Count for SSD
if [[ "$2" == "SSD" ]] && [[ "$SSD_Device_ID" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Serial_Number" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Model_Number" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Capacity" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_SMART_Status" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Drive_Temp" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Power_On_Hours" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Wear_Level" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Reallocated_Sectors" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Reallocated_Events" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Pending_Sectors" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Offline_Uncorrectable" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_UDMA_CRC_Errors" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Last_Test_Age" == "true" ]]; then ((Columns=Columns+1)); fi;
if [[ "$2" == "SSD" ]] && [[ "$SSD_Last_Test_Type" == "true" ]]; then ((Columns=Columns+1)); fi;

(
    # Write HTML table headers to log file
    echo "<br><br>"
    echo "<table style=\"border: 1px solid black; border-collapse: collapse;\">"

    if [[ "$2" == "HDD" ]]; then echo "<tr><th colspan=\"$Columns\" style=\"text-align:center; font-size:20px; height:40px; font-family:courier;\">"$HDDreportTitle"</th></tr>"; fi;
    if [[ "$2" == "SSD" ]]; then echo "<tr><th colspan=\"$Columns\" style=\"text-align:center; font-size:20px; height:40px; font-family:courier;\">"$SSDreportTitle"</th></tr>"; fi;
    echo "<tr>"
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Device_ID" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Device<br>ID</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Device_ID" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Device<br>ID</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Serial_Number" == "true" ]]; then echo "  <th style=\"text-align:center; width:130px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Serial<br>Number</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Serial_Number" == "true" ]]; then echo "  <th style=\"text-align:center; width:130px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Serial<br>Number</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Model_Number" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Model<br>Number</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Model_Number" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Model<br>Number</th>"; fi;

    if [[ "$2" == "HDD" ]] && [[ "$HDD_Capacity" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">HDD<br>Capacity</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Capacity" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">SSD<br>Capacity</th>"; fi;

    if [[ "$2" == "HDD" ]] && [[ "$HDD_Rotational_Rate" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">RPM</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_SMART_Status" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">SMART<br>Status</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_SMART_Status" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">SMART<br>Status</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Drive_Temp" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Temp</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Drive_Temp" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Temp</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Power_On_Hours" == "true" ]]; then echo "  <th style=\"text-align:center; width:120px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Power-On<br>Time</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Power_On_Hours" == "true" ]]; then echo "  <th style=\"text-align:center; width:120px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Power-On<br>Time</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Wear_Level" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Wear<br>Level</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Start_Stop_Count" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Start<br>Stop<br>Count</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Load_Cycle" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Load<br>Cycle<br>Count</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Spin_Retry" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Spin<br>Retry<br>Count</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Reallocated_Sectors" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Realloc<br>Sectors</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Reallocated_Sectors" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Realloc<br>Sectors</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Reallocated_Events" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Realloc<br>Events</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Reallocated_Events" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Realloc<br>Events</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Pending_Sectors" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Current<br>Pending<br>Sectors</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Pending_Sectors" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Current<br>Pending<br>Sectors</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Offline_Uncorrectable" == "true" ]]; then echo "  <th style=\"text-align:center; width:120px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Offline<br>Uncorr<br>Sectors</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Offline_Uncorrectable" == "true" ]]; then echo "  <th style=\"text-align:center; width:120px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Offline<br>Uncorr<br>Sectors</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_UDMA_CRC_Errors" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">UltraDMA<br>CRC<br>Errors</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_UDMA_CRC_Errors" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">UltraDMA<br>CRC<br>Errors</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Seek_Error_Rate" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Seek<br>Error<br>Rate</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_MultiZone_Errors" == "true" ]]; then echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Multi<br>Zone<br>Error</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Last_Test_Age" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last Test<br>Age (days)</th>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Last_Test_Age" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last Test<br>Age (days)</th>"; fi;
    if [[ "$2" == "HDD" ]] && [[ "$HDD_Last_Test_Type" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last Test<br>Type</th></tr>"; fi;
    if [[ "$2" == "SSD" ]] && [[ "$SSD_Last_Test_Type" == "true" ]]; then echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last Test<br>Type</th></tr>"; fi;
    echo "</tr>"
) >> "$logfile"

for drive in $drives; do
    (
        # For each drive detected, run "smartctl -A -i" and parse its output. This whole section is a single, long statement, so I'll make all comments here.
        # Start by passing awk variables (all the -v's) used in other parts of the script. Other variables are calculated in-line with other smartctl calls.
        # Next, pull values out of the original "smartctl -A -i" statement by searching for the text between the //'s.
        # After parsing the output, compute other values (last test's age, on time in YY-MM-DD-HH).
        # After these computations, determine the row's background color (alternating as above, subbing in other colors from the palate as needed).
        # Finally, print the HTML code for the current row of the table with all the gathered data.
        smartctl -a /dev/"$drive" | \
        awk -v device="$drive" -v HDDtempWarn="$HDDtempWarn" -v HDDtempCrit="$HDDtempCrit" -v sectorsCrit="$sectorsCrit" -v testAgeWarn="$testAgeWarn" -v detail_level="$detail_level" -v whtColor="$whtColor" \
        -v SSDtempWarn="$SSDtempWarn" -v SSDtempCrit="$SSDtempCrit" \
        -v okColor="$okColor" -v warnColor="$warnColor" -v critColor="$critColor" -v ovrdColor="$ovrdColor" -v altColor="$altColor" -v powerTimeFormat="$powerTimeFormat" -v bgColor="$bgColor" \
        -v UCRC1SN="$UCRC1SN" -v UCRC1CNT="$UCRC1CNT" -v UCRC2SN="$UCRC2SN" -v UCRC2CNT="$UCRC2CNT" -v UCRC3SN="$UCRC3SN" -v UCRC3CNT="$UCRC3CNT" -v sectorsWarn="$sectorsWarn" \
        -v UCRC4SN="$UCRC4SN" -v UCRC4CNT="$UCRC4CNT" -v UCRC5SN="$UCRC5SN" -v UCRC5CNT="$UCRC5CNT" -v UCRC6SN="$UCRC6SN" -v UCRC6CNT="$UCRC6CNT" -v ignoreMultiZone="$ignoreMultiZone" \
        -v UCRC7SN="$UCRC7SN" -v UCRC7CNT="$UCRC7CNT" -v UCRC8SN="$UCRC8SN" -v UCRC8CNT="$UCRC8CNT" -v includeSSD="$includeSSD" -v UDMACRCDRIVES="$UDMACRCDRIVES" -v MULTIZONEDRIVES="$MULTIZONEDRIVES" \
        -v MULTIZONE1SN="$MULTIZONE1SN" -v MULTIZONE1CNT="$MULTIZONE1CNT" -v MULTIZONE2SN="$MULTIZONE2SN" -v MULTIZONE2CNT="$MULTIZONE2CNT" -v MULTIZONE3SN="$MULTIZONE3SN" \
        -v MULTIZONE3CNT="$MULTIZONE3CNT" -v MULTIZONE4SN="$MULTIZONE4SN" -v MULTIZONE4CNT="$MULTIZONE4CNT" -v ignoreSeekError="$ignoreSeekError" -v deviceRedFlag="$deviceRedFlag" \
        -v MULTIZONE5SN="$MULTIZONE5SN" -v MULTIZONE5CNT="$MULTIZONE5CNT" -v MULTIZONE6SN="$MULTIZONE6SN" -v MULTIZONE6CNT="$MULTIZONE6CNT" -v ignoreUDMA="$ignoreUDMA" \
        -v MULTIZONE7SN="$MULTIZONE7SN" -v MULTIZONE7CNT="$MULTIZONE7CNT" -v MULTIZONE8SN="$MULTIZONE8SN" -v MULTIZONE8CNT="$MULTIZONE8CNT" \
        -v BADSECTORS1SN="$BADSECTORS1SN" -v BADSECTORS1CNT="$BADSECTORS1CNT" -v BADSECTORS2SN="$BADSECTORS2SN" -v BADSECTORS2CNT="$BADSECTORS2CNT" \
        -v BADSECTORS3SN="$BADSECTORS3SN" -v BADSECTORS3CNT="$BADSECTORS3CNT" -v BADSECTORS4SN="$BADSECTORS4SN" -v BADSECTORS4CNT="$BADSECTORS4CNT" \
        -v HDD_Device_ID="$HDD_Device_ID" -v HDD_Serial_Number="$HDD_Serial_Number" -v HDD_Model_Number="$HDD_Model_Number" -v HDD_Rotational_Rate="$HDD_Rotational_Rate" \
        -v HDD_SMART_Status="$HDD_SMART_Status" -v HDD_Drive_Temp="$HDD_Drive_Temp" -v HDD_Power_On_Hours="$HDD_Power_On_Hours" -v HDD_Start_Stop_Count="$HDD_Start_Stop_Count" \
        -v HDD_Load_Cycle="$HDD_Load_Cycle" -v HDD_Spin_Retry="$HDD_Spin_Retry" -v HDD_Reallocated_Sectors="$HDD_Reallocated_Sectors" -v HDD_Reallocated_Events="$HDD_Reallocated_Events" \
        -v HDD_Pending_Sectors="$HDD_Pending_Sectors" -v HDD_Offline_Uncorrectable="$HDD_Offline_Uncorrectable" -v HDD_UDMA_CRC_Errors="$HDD_UDMA_CRC_Errors" \
        -v HDD_Seek_Error_Rate="$HDD_Seek_Error_Rate" -v HDD_MultiZone_Errors="$HDD_MultiZone_Errors" -v HDD_Last_Test_Age="$HDD_Last_Test_Age" -v HDD_Last_Test_Type="$HDD_Last_Test_Type" \
        -v SSD_Device_ID="$SSD_Device_ID" -v SSD_Serial_Number="$SSD_Serial_Number" -v SSD_Model_Number="$SSD_Model_Number" -v statistical_data_file="$statistical_data_file" \
        -v SSD_SMART_Status="$SSD_SMART_Status" -v SSD_Drive_Temp="$SSD_Drive_Temp" -v SSD_Power_On_Hours="$SSD_Power_On_Hours" -v SSD_Wear_Level="$SSD_Wear_Level" \
        -v SSD_Reallocated_Sectors="$SSD_Reallocated_Sectors" -v SSD_Reallocated_Events="$SSD_Reallocated_Events" -v expDataEnable="$expDataEnable" -v datestamp="$datestamp" -v timestamp="$timestamp" \
        -v SSD_Pending_Sectors="$SSD_Pending_Sectors" -v SSD_Offline_Uncorrectable="$SSD_Offline_Uncorrectable" -v SSD_UDMA_CRC_Errors="$SSD_UDMA_CRC_Errors" \
        -v SSD_Last_Test_Age="$SSD_Last_Test_Age" -v SSD_Last_Test_Type="$SSD_Last_Test_Type" -v HDD_Capacity="$HDD_Capacity" -v SSD_Capacity="$SSD_Capacity" \
        -v logfile_warning="$logfile_warning" -v logfile_critical="$logfile_critical" -v onHours="$onHours" -v temp="$temp" -v logfile_warranty="$logfile_warranty" \
        -v WarrantyDriveSN1="$WARRANTYDRIVESN1" -v WarrantyDriveSN1Date="$WARRANTYDRIVESN1DATE" -v WarrantyDriveSN2="$WARRANTYDRIVESN2" -v WarrantyDriveSN2Date="$WARRANTYDRIVESN2DATE" \
        -v WarrantyDriveSN3="$WARRANTYDRIVESN3" -v WarrantyDriveSN3Date="$WARRANTYDRIVESN3DATE" -v WarrantyDriveSN4="$WARRANTYDRIVESN4" -v WarrantyDriveSN4Date="$WARRANTYDRIVESN4DATE" \
        -v datestamp2="$datestamp2" \
        -v smarttesting="$(smartctl -a /dev/"$drive" | grep "remaining" | awk '{print $1}')" \
        -v chkreadfailure="$(smartctl -a /dev/"$drive" | grep "# 1" | awk '{print $5}')" \
        -v lastTestHours="" -v crcErrors="0" \
        -v lastTestHours="$(smartctl -l selftest /dev/"$drive" | grep "# 1" | awk '{print $9}')" \
        -v lastTestType="$(smartctl -l selftest /dev/"$drive" | grep "# 1" | awk '{print $3}')"  \
        -v smartStatus="$(smartctl -H /dev/"$drive" | grep "SMART overall-health" | awk '{print $6}')" \
        -v smartStatus="$(smartctl -H /dev/"$drive" | grep "SMART Health Status" | awk '{print $4}')" ' \
        BEGIN{IGNORECASE=1} \
        /SMART overall-health/{smartStatus=$6} \
        /SMART Health Status/{smartStatus=$4} \
        /Serial Number:/{serial=$3} \
        /Airflow/{temp=($10 + 0)} \
        /Temperature_Case/{temp=($10 + 0)} \
        /Temperature_Celsius/{temp=($10 + 0)} \
        /Power_On_Hours/{onHours=$10} \
        /Start_Stop_Count/{startStop=$10} \
        /Spin_Retry_Count/{spinRetry=$10} \
        /Reallocated_Sector/{reAlloc=$10} \
        /Reallocated_Event_Count/{reAllocEvent=$10} \
        /Current_Pending_Sector/{pending=$10} \
        /Offline_Uncorrectable/{offlineUnc=$10} \
        /UDMA_CRC_Error_Count/{crcErrors=int($10 + 0)} \
        /Seek_Error_Rate/{seekErrorHealth2=$4} \
        /Seek_Error_Rate/{seekErrorHealth=$10} \
        /Load_Cycle_Count/{loadCycle=$10} \
        /Multi_Zone_Error_Rate/{multiZone=($10 + 0)} \
        /SSD_Life_Left/{wearLevel=$4 + 0} \
        /Wear_Leveling_Count/{wearLevel=$4 + 0} \
        /Percent_Lifetime_Remain/{wearLevel=$4 + 0} \
        /Reallocated_NAND_Blk_Cnt/{reAlloc=$10} \
        /Current Drive Temperature:/{temp=$4} \
        /Accumulated start-stop cycles:/{startStop=$4} \
        /Accumulated load-unload cycles:/{loadCycle=$4} \
        /Elements in grown defect list:/{reAlloc=$6} \
        /Accumulated power on time/{onHours=int($6)} \
        /Rotation/{rotation=$3} \
        /Device Model/{modelnumber=$3 " " $4 " " $5 " " $6 " " $7} \
        /User Capacity/{capacity=$5 $6} \
        /Background/{lastTestHours=$10} \
        /# 1/{altlastTestHours=$7} \
        /# 1/{altlastTestType=$4} \
        END {

if (lastTestHours < "0" || lastTestHours == "") lastTestHours=altlastTestHours;
# Custom for a SAS drive for lastTestHours
if (serial == "Z1Z5JDWE0000C507EFJZ") lastTestHours=altlastTestHours;
if (lastTestHours == "") lastTestHours=onHours;

if (lastTestType == "Background") lastTestType=altlastTestType;

            testAge=int((onHours - lastTestHours) / 24);
            yrs=int(onHours / 8760);
            mos=int((onHours % 8760) / 730);
            dys=int(((onHours % 8760) % 730) / 24);
            hrs=((onHours % 8760) % 730) % 24;
            if (powerTimeFormat == "ymdh") onTime=yrs "y " mos "m " dys "d " hrs "h";
            else if (powerTimeFormat == "ymd") onTime=yrs "y " mos "m " dys "d";
            else if (powerTimeFormat == "ym") onTime=yrs "y " mos "m";
            else if (powerTimeFormat == "y") onTime=yrs "y";
            else if (powerTimeFormat == "h") onTime=onHours;
	    else onTime=yrs "y " mos "m " dys "d " hrs "h ";

            if (bgColor != "#ffffff") bgColor = altColor; else bgColor = "#ffffff";

# Establish Initial Background Colors
#bgColor=whtColor;
tempColor=bgColor;
spinRetryColor=bgColor;
reAllocColor=bgColor;
reAllocEventColor=bgColor;
pendingColor=bgColor;
offlineUncColor=bgColor;
crcErrorsColor=bgColor;
seekErrorHealthColor=bgColor;
multiZoneColor=bgColor;
testAgeColor=bgColor;
onTimeColor=bgColor;

if (serial == WarrantyDriveSN1) if (WarrantyDriveSN1Date < datestamp2) onTimeColor = warnColor;
if (serial == WarrantyDriveSN1) if (WarrantyDriveSN1Date < datestamp2) printf "Drive "serial" Warranty Expired, " >> logfile_warranty;
if (serial == WarrantyDriveSN2) if (WarrantyDriveSN2Date < datestamp2) onTimeColor = warnColor;
if (serial == WarrantyDriveSN2) if (WarrantyDriveSN2Date < datestamp2) printf "Drive "serial" Warranty Expired, " >> logfile_warranty;
if (serial == WarrantyDriveSN3) if (WarrantyDriveSN3Date < datestamp2) onTimeColor = warnColor;
if (serial == WarrantyDriveSN3) if (WarrantyDriveSN4Date < datestamp2) printf "Drive "serial" Warranty Expired, " >> logfile_warranty;
if (serial == WarrantyDriveSN4) if (WarrantyDriveSN4Date < datestamp2) onTimeColor = warnColor;
if (serial == WarrantyDriveSN4) if (WarrantyDriveSN4Date < datestamp2) printf "Drive "serial" Warranty Expired, " >> logfile_warranty;

if (serial == BADSECTORS1SN) reAlloc=reAlloc-BADSECTORS1CNT;
if (serial == BADSECTORS1SN) reAllocColor=ovrdColor;
if (serial == BADSECTORS2SN) reAlloc=reAlloc-BADSECTORS2CNT;
if (serial == BADSECTORS2SN) reAllocColor=ovrdColor;
if (serial == BADSECTORS3SN) reAlloc=reAlloc-BADSECTORS3CNT;
if (serial == BADSECTORS3SN) reAllocColor=ovrdColor;
if (serial == BADSECTORS4SN) reAlloc=reAlloc-BADSECTORS4CNT;
if (serial == BADSECTORS4SN) reAllocColor=ovrdColor;

if (smartStatus != "") if (smartStatus == "PASSED" || smartStatus == "OK") smartStatusColor = okColor; else smartStatusColor = critColor;
if (smartStatus == "OK" || smartStatus == "PASSED"); else printf "smart status" >> logfile_critical;
if ( detail_level == "HDD")
{
if (temp != "") if (temp >= HDDtempCrit) tempColor = critColor; else if (temp >= HDDtempWarn) tempColor = warnColor;
if (temp != "") if (temp >= HDDtempCrit) printf "critical drive temp" >> logfile_critical;
if (temp != "") if (temp >= HDDtempWarn) printf "high drive temp" >> logfile_warning;
if (temp == "") temp = "N/A";
}
if ( detail_level == "SSD")
{
if (temp != "") if (temp >= SSDtempCrit) tempColor = critColor; else if (temp >= SSDtempWarn) tempColor = warnColor;
if (temp != "") if (temp >= SSDtempCrit) printf "critical drive temp" >> logfile_critical;
if (temp != "") if (temp >= SSDtempWarn) printf "high drive temp" >> logfile_warning;
if (temp == "") temp = "N/A";
}
if (spinRetry != "") if (spinRetry != "0") spinRetryColor = warnColor;
if (spinRetry != "") if (spinRetry != "0") printf "spin retry" >> logfile_critical;
if (spinRetry == "") spinRetry = "N/A";
if (reAlloc != "") if ((reAlloc + 0) > sectorsCrit) reAllocColor = critColor; else if ((reAlloc + 0) > sectorsWarn) reAllocColor = warnColor;
if (reAlloc != "") if ((reAlloc + 0) > sectorsCrit) printf "critical sectors" >> logfile_critical; 
if (reAlloc != "") if ((reAlloc + 0) > sectorsWarn) printf "warning sectors" >> logfile_warning; 
if (reAlloc == "") reAlloc = "N/A";
if (reAllocEvent != "") if (reAllocEvent != "0") reAllocEventColor = warnColor;
if (reAllocEvent != "") if (reAllocEvent != "0") printf "reallocating sectors" >> logfile_warning;
if (reAllocEvent == "") reAllocEvent = "N/A";
if (pending != "") if ((pending + 0) > sectorsCrit) pendingColor = critColor; else if (pending != 0) pendingColor = warnColor;
if (pending != "") if ((pending + 0) > sectorsCrit) printf "sector errors" >> logfile_critical;
if (pending != "") if (pending != 0) printf "sector errors" >> logfile_warning;
if (pending == "") pending = "N/A";
if (offlineUnc != "") if ((offlineUnc + 0) > sectorsCrit) offlineUncColor = critColor; else if (offlineUnc != 0) offlineUncColor = warnColor;
if (offlineUnc != "") if ((offlineUnc + 0) > sectorsCrit) printf "uncorrectable errors" >> logfile_critical;
if (offlineUnc != "") if (offlineUnc != 0) printf "uncorrectable errors" >> logfile_warning;
if (offlineUnc == "") offlineUnc = "N/A";
if (crcErrors != "") if (crcErrors != "0") crcErrorsColor = critColor;
if (crcErrors == "") crcErrors = "N/A";
if (smarttesting >= 0) lastTestType = smarttesting "Remaining";
if (chkreadfailure == "Completed:") lastTestType = "Read Failure";
if (chkreadfailure == "Completed:") lastTestTypeColor = critColor; else lastTestTypeColor = bgColor; 
if (chkreadfailure == "Completed:") printf "read failure" >> logfile_critical;
# If seekErrorHealth RAW_VALUE is some crazy number, use the VALUE column data.
if (seekErrorHealth > 100) if (seekErrorHealth2 !="") seekErrorHealth=(seekErrorHealth2 + 0);
if (ignoreSeekError != "true") if ((seekErrorHealth + 0) > 100) seekErrorHealthColor = warnColor;
if (ignoreSeekError != "true") if ((seekErrorHealth + 0) > 100) printf "seek errors" >> logfile_warning;
if (multiZone == "") multiZone = "N/A";
if (wearLevel == "") wearLevel = "N/A";
if (wearLevel == "0") wearLevel="N/A";
if (modelnumber == "") modelnumber="N/A";
if (startStop == "") startStop="N/A";
if (loadCycle == "") loadCycle="N/A";
if (seekErrorHealth == "") seekErrorHealth="N/A";

if (expDataEnable == "true") printf datestamp","timestamp","device","detail_level","serial","smartStatus","temp","onHours","wearLevel","startStop","loadCycle","spinRetry"," \
reAlloc","reAllocEvent","pending","offlineUnc","crcErrors","seekErrorHealth","multiZone",\n" >> statistical_data_file;

### Routine to zero out the UDMA CRC Error Count and Highlights it Yellow.
### Currently allows up to 8 drives, hope no one needs that many.

if (serial == UCRC1SN) crcErrors=crcErrors-UCRC1CNT;
if (serial == UCRC1SN) crcErrorsColor = ovrdColor;
if (serial == UCRC2SN) crcErrors=crcErrors-UCRC2CNT;
if (serial == UCRC2SN) crcErrorsColor = ovrdColor;
if (serial == UCRC3SN) crcErrors=crcErrors-UCRC3CNT;
if (serial == UCRC3SN) crcErrorsColor = ovrdColor;
if (serial == UCRC4SN) crcErrors=crcErrors-UCRC4CNT;
if (serial == UCRC4SN) crcErrorsColor = ovrdColor;
if (serial == UCRC5SN) crcErrors=crcErrors-UCRC5CNT;
if (serial == UCRC5SN) crcErrorsColor = ovrdColor;
if (serial == UCRC6SN) crcErrors=crcErrors-UCRC6CNT;
if (serial == UCRC6SN) crcErrorsColor = ovrdColor;
if (serial == UCRC7SN) crcErrors=crcErrors-UCRC7CNT;
if (serial == UCRC7SN) crcErrorsColor = ovrdColor;
if (serial == UCRC8SN) crcErrors=crcErrors-UCRC8CNT;
if (serial == UCRC8SN) crcErrorsColor = ovrdColor;

# Hard Coded UDMA Corrections becasue for some reason these are not being detected correctly as a variable.
if (serial == "2011E2940534") crcErrors=crcErrors-18;
if (serial == "2011E2940534") crcErrorsColor = ovrdColor;
if (serial == "2011E2940536") crcErrors=crcErrors-19;
if (serial == "2011E2940536") crcErrorsColor = ovrdColor;

if (serial == MULTIZONE1SN) multiZone=multiZone-MULTIZONE1CNT;
if (serial == MULTIZONE1SN) multiZoneColor=ovrdColor;
if (serial == MULTIZONE2SN) multiZone=multiZone-MULTIZONE2CNT;
if (serial == MULTIZONE2SN) multiZoneColor=ovrdColor;
if (serial == MULTIZONE3SN) multiZone=multiZone-MULTIZONE3CNT;
if (serial == MULTIZONE3SN) multiZoneColor=ovrdColor;
if (serial == MULTIZONE4SN) multiZone=multiZone-MULTIZONE4CNT;
if (serial == MULTIZONE4SN) multiZoneColor=ovrdColor;
if (serial == MULTIZONE5SN) multiZone=multiZone-MULTIZONE5CNT;
if (serial == MULTIZONE5SN) multiZoneColor=ovrdColor;
if (serial == MULTIZONE6SN) multiZone=multiZone-MULTIZONE6CNT;
if (serial == MULTIZONE6SN) multiZoneColor=ovrdColor;
if (serial == MULTIZONE7SN) multiZone=multiZone-MULTIZONE7CNT;
if (serial == MULTIZONE7SN) multiZoneColor=ovrdColor;
if (serial == MULTIZONE8SN) multiZone=multiZone-MULTIZONE8CNT;
if (serial == MULTIZONE8SN) multiZoneColor=ovrdColor;

if (ignoreMultiZone != "true") if (multiZone != "N/A") if (multiZone != "0") multiZoneColor=warnColor;
if (ignoreMultiZone != "true") if (multiZone != "N/A") if (multiZone != "0") printf "multiZone Errors" >> logfile_critical;
if (ignoreUDMA != "true") if (crcErrors != "N/A") if (crcErrors != "0") printf "crc errors" >> logfile_critical;

if (testAge > testAgeWarn) testAgeColor = warnColor; else testAgeColor = bgColor;
if (testAge > testAgeWarn) printf "test age" >> logfile_warning;
if (smartStatusColor != okColor) if (smartStatusColor != altColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (tempColor != bgColor) if (tempColor != altColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (spinRetryColor != bgColor) if (spinRetryColor != altColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (reAllocColor != bgColor) if (reAllocColor != altColor) if (reAllocColor != ovrdColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (reAllocEventColor != bgColor) if (reAllocEventColor != altColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (pendingColor != bgColor)  if (pendingColor != altColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (offlineUncColor != bgColor)  if (offlineUncColor != altColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (crcErrorsColor != bgColor) if (crcColor != altColor) if (crcErrorsColor != ovrdColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (seekErrorHealthColor != bgColor) if (seekErrorHealthColor != altColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (testAgeColor != bgColor) if (testAgeColor != altColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (lastTestTypeColor != bgColor) if (lastTestTypeColor != altColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (multiZoneColor != bgColor) if (multiZoneColor != altColor) if (multiZoneColor != ovrdColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;


if ( detail_level == "HDD")
{
           printf "<tr style=\"background-color:%s;\">\n", bgColor;
if (HDD_Device_ID == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">/dev/%s</td>\n", deviceStatusColor, device;
if (HDD_Serial_Number == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", bgColor, serial;
if (HDD_Model_Number == "true") printf "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", modelnumber;
if (HDD_Capacity == "true") printf "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", capacity;
if (HDD_Rotational_Rate == "true") printf "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", rotation;
if (HDD_SMART_Status == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", smartStatusColor, smartStatus;
if (HDD_Drive_Temp == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%d*C</td>\n", tempColor, temp;
if (HDD_Power_On_Hours == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", onTimeColor, onTime;
if (HDD_Start_Stop_Count == "true") printf "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", startStop;
if (HDD_Load_Cycle == "true") printf "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", loadCycle;
if (HDD_Spin_Retry == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", spinRetryColor, spinRetry;
if (HDD_Reallocated_Sectors == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", reAllocColor, reAlloc;
if (HDD_Reallocated_Events == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", reAllocEventColor, reAllocEvent;
if (HDD_Pending_Sectors == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", pendingColor, pending;
if (HDD_Offline_Uncorrectable == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", offlineUncColor, offlineUnc;
if (HDD_UDMA_CRC_Errors == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", crcErrorsColor, crcErrors;
if (HDD_Seek_Error_Rate == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", seekErrorHealthColor, seekErrorHealth;
if (HDD_MultiZone_Errors == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%d</td>\n",  multiZoneColor, multiZone;
if (HDD_Last_Test_Age == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", testAgeColor, testAge;
if (HDD_Last_Test_Type == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", lastTestTypeColor, lastTestType;
           printf "</tr>\n";
}


if ( detail_level == "SSD")
{
           printf "<tr style=\"background-color:%s;\">\n", bgColor;
if (SSD_Device_ID == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">/dev/%s</td>\n", deviceStatusColor, device;
if (SSD_Serial_Number == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", bgColor, serial;
if (SSD_Model_Number == "true") printf "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", modelnumber;
if (SSD_Capacity == "true") printf "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", capacity;
if (SSD_SMART_Status == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", smartStatusColor, smartStatus;
if (SSD_Drive_Temp == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%d*C</td>\n", tempColor, temp;
if (SSD_Power_On_Hours == "true") printf "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", onTime;
if (SSD_Wear_Level == "true") printf "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", wearLevel;
if (SSD_Reallocated_Sectors == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", reAllocColor, reAlloc;
if (SSD_Reallocated_Events == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", reAllocEventColor, reAllocEvent;
if (SSD_Pending_Sectors == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", pendingColor, pending;
if (SSD_Offline_Uncorrectable == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", offlineUncColor, offlineUnc;
if (SSD_UDMA_CRC_Errors == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", crcErrorsColor, crcErrors;
if (SSD_Last_Test_Age == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", testAgeColor, testAge;
if (SSD_Last_Test_Type == "true") printf "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n", lastTestTypeColor, lastTestType;
           printf "</tr>\n";
}

        }'
    ) | tr -d "[]" >> "$logfile"

# I Want Milliseconds Resolution represented for timestamp, FreeBSD deos not support it ################################
#
if [[ $softver != "Linux" ]]; then 
  timestamp=$(date +%T)
else
  timestamp=$(date +"%T.%2N")
fi

datestamp=$(date +%Y/%m/%d)

done
# End SMART summary table and summary section
(
    echo "</table>"
    echo "<br>"
) >> "$logfile"
}

################################## COMPILE DETAILED REPORT ###############################################

detailed_report () {
###### Detailed Report Section (monospace text)
(
echo "<pre style=\"font-size:14px\">"
if test -e "$logfile_warranty"; then
  echo "<b>At Least One Drive Warranty Has Expired!"
  cat $logfile_warranty
  echo "</b><br>"
fi
echo "<b>Options: SSD Auto Detection = $includeSSD, Report Non-SMART = $reportnonSMART, Ignore MultiZone Errors = $ignoreMultiZone, Ignore Seek Error Rate = $ignoreSeekError, Disable Drive Warranty Expire = $disableWarranty</b>"
) >> "$logfile"

if [[ $expDataEnable == "true" ]]; then
   if [[ $expDataEmail == "true" ]]; then
      echo "<b>Export Log Located: $statistical_data_file,  Email Periodicity: $expDataEmailSend</b><br>" >> "$logfile"
   else
      echo "<b>Export Log Located: $statistical_data_file</b><br>" >> "$logfile"
   fi
fi

if [[ $disableRAWdata != "true" ]]; then

### zpool status for each pool
for pool in $pools; do
    (
      # Create a simple header and drop the output of zpool status -v
        echo "<b>########## ZPool status report for ${pool} ##########</b>"
        echo "<br>"
        zpool status -v "$pool"
        echo "<br><br>"
    ) >> "$logfile"
done

  if [[ $includeSSD == "true" ]]; then
  drives="${smartdrives} ${smartdrivesSSD}"
  else
  drives="${smartdrives}"
  fi
### SMART status for each drive - SMART Enabled
 for drive in $drives; do
    # Gather brand and serial number of each drive
    brand="$(smartctl -i /dev/"$drive" | grep "Model Family" | awk '{print $3, $4, $5}')"
    serial="$(smartctl -i /dev/"$drive" | grep "Serial Number" | awk '{print $3}')"

    (
     # Create a simple header and drop the output of some basic smartctl commands
        echo "<br>"
        echo "<b>########## SMART status report for ${drive} drive (${brand}: ${serial}) ##########</b>"
        smartctl -H -A -l error /dev/"$drive"

     # Create Recent Tests Report
        echo "Num Test_Description  (Most recent Short & Extended Tests - Listed by test number)"
        lasttest1="$(smartctl -x /dev/"$drive" | egrep "# 1")"
        echo $lasttest1
        if [[ $lasttest1 == *"Short offline"* ]]; then lastfind="Extended offline"; fi;
        if [[ $lasttest1 == *"Extended offline"* ]]; then lastfind="Short offline"; fi;
        lasttest2="$(smartctl -x /dev/"$drive" | egrep "$lastfind" | head -1)"
        echo $lasttest2
        echo "<br>"
    ) >> "$logfile"

 done
fi
}

################################ COMPILE NON-SMART REPORT ##################################

non_smart_report () {
### NON-SMART status report section
# I don't perticularly use this but some folks might find it useful.
# To activate it, in the variables set reportnonSMART=true.
# It will list all drives where Non-SMART is true and remove devices starting with "cd", for example "cd0"

drives=$nonsmartdrives
if [ $reportnonSMART == "true" ]; then 
for drive in $drives; do
  if [ ! "$(echo "$drive" | grep "cd")" ]; then
   if [ $softver != "Linux" ]; then
    # Gather brand and serial number of each drive
    brand="$(smartctl -i /dev/"$drive" | grep "Model Family" | awk '{print $3, $4, $5}')"
    serial="$(smartctl -i /dev/"$drive" | grep "Serial Number" | awk '{print $3}')"
   else
    brand="$(fdisk -l /dev/"$drive" | grep "Disk model" | awk '{print $3, $4, $5}')"
    serial="$(fdisk -l /dev/"$drive" | grep "Serial Number" | awk '{print $3}')"
   fi
    (
        echo "<br>"
        echo "<b>########## NON-SMART status report for ${drive} drive (${brand}: ${serial}) ##########</b>"
    # And we will dump everything since it's not a standard SMART device.
        echo "<b>SMARTCTL DATA</b>"
        smartctl -a /dev/"$drive"
        echo "<br>"
      if [ $softver == "Linux" ]; then
        echo "<b>FDISK DATA</b>"
        fdisk -l /dev/"$drive"
      fi
    ) >> "$logfile"
  fi
done
fi
}

############################## REMOVE UN-NEEDED JUNK AND FINALIZE EMAIL MESSAGE END #####################

remove_junk_report () {
### Remove some un-needed junk from the output
sed -i -e '/smartctl/d' "$logfile"
sed -i -e '/Copyright/d' "$logfile"
sed -i -e '/=== START OF READ/d' "$logfile"
sed -i -e '/SMART Attributes Data/d' "$logfile"
sed -i -e '/Vendor Specific SMART/d' "$logfile"
sed -i -e '/SMART Error Log Version/d' "$logfile"

### End details section, close MIME section
(
    echo "</pre>"
    echo "--${boundary}--"
)  >> "$logfile"
}

############################# COMBINE ALL DATA INTO A FORMAL EMAIL MESSAGE AND SEND IT ############################

create_email () {
### Create New Email Header - Set Subject Line

## Test if there is a Warning Message and Setup Subject Line
if test -e "$logfile_critical"; then
 subject="*CRITICAL ERROR*  SMART Testing Results for ${host}  *CRITICAL ERROR*"
elif test -e "$logfile_warning"; then
 subject="*WARNING*  SMART Testing Results for ${host}  *WARNING*"
elif [[ $disableWarranty == "false" ]]; then
	if test -e "$logfile_warranty"; then
	subject="*Drive Warranty Expired* - SMART Testing Results for ${host}"
	else
	subject="SMART Testing Results for ${host} - All is Good"
	fi
else
 subject="SMART Testing Results for ${host} - All is Good"
fi

### Set email headers ###
(
echo "From: ${from}"
echo "To: ${email}"
echo "Subject: ${subject}"
) > ${logfile_header}

cat $logfile >> $logfile_header

### Send report
sendmail -t -oi < "$logfile_header"
}

################################## CLEAN UP TEMPORARY FILES #######################################

cleanup_files () {
### Clean up our temporary files
if test -e "$logfile"; then rm "$logfile"; fi
if test -e "$logfile_header"; then rm "$logfile_header"; fi
if test -e "$logfile_critical"; then rm "$logfile_critical"; fi
if test -e "$logfile_warning"; then rm "$logfile_warning"; fi
if test -e "$logfile_warranty"; then rm "$logfile_warranty"; fi
}

### DEFINE FUNCTIONS END ###

#######################
#######################
###                 ###
###  PROGRAM START  ###
###                 ###
#######################
#######################

# The order in which these processed occur is unfortunately dependent.

get_smartHDD_listings
get_smartSSD_listings
get_smartOther_listings
email_preformat
config_backup
zpool_report

datestamp2=$(date -Idate)
datestamp=$(date +%Y/%m/%d)
if [ $softver != "Linux" ]; then
timestamp=$(date +%T)

else
# Linux gets millisecond resolution
timestamp=$(date +"%T.%2N")

fi

if [[ $expDataEnable == "true" ]]; then
  if test -e "$statistical_data_file"; then
# Purge items over expDataPurge days
     echo "Updating statistical datafile"
  else
# The file does not exist, create it.
     echo "Creating statistical datafile"
     printf "Date,Time,Device ID,Drive Type,Serial Number,SMART Status,Temp,Power On Hours,Wear Level,Start Stop Count,Load Cycle,Spin Retry,Reallocated Sectors,\
Reallocated Sector Events,Pending Sectors,Offline Uncorrectable,UDMA CRC Errors,Seek Error Rate,Multi Zone Errors,\n" > "$statistical_data_file"
  fi
fi


# Generate SMART HDD Report
echo "Collecting Drive Information"
hdd_report "$smartdrives" "HDD"

if [[ $includeSSD == "true" ]]; then
# Test if any SSD's are available, if not then no report, if yes then generate SSD Report.
  if [[ $smartdrivesSSD != "" ]]; then
    hdd_report "$smartdrivesSSD" "SSD"
  fi
fi

# This purge happens here directly after data collection.
# A zero (0) value = Disable Purging (Keep all data)
if [[ $expDataPurge != 0 ]]; then
purge_exportdata
fi

detailed_report

if [[ $reportnonSMART == "true" ]]; then
if [[ $disableRAWdata != "true" ]]; then
  non_smart_report
fi
fi

if [[ "$1" == "-s" ]]; then
echo "Statistical Data Collection Only - No Email"
else
email_datafile
remove_junk_report
echo "Sending Email"
create_email
fi

cleanup_files