#!/bin/bash

###### ZPool & SMART status report with FreeNAS config backup
### Original script by joeschmuck, modified by Bidelu0hm, then by melp (me)

### At a minimum, enter email address in user-definable parameter section. Feel free to edit other user parameters as needed.
### If you find any errors, feel free to contact me on the FreeNAS forums (username melp) or email me at jason at jro dot io.

### Version: v1.3 TrueNAS Scale (Jeff Alperin 12-6-21)

### Version v1.4b TrueNAS/FreeNAS (joeschmuck 01/07/2022), v1.4c-beta2 (joeschmuck 02/09/2022)

### Version: v1.4c
### Changelog:
# v1.4c:
#   - Added sorting drives alphabetically vice the default the OS reports them.
#   - Added RED warning in Device for any single failure in the summary (deviceRedFlag switch controlled).
#   - Added some additional SSD definitions.
#
# v1.4b:
#   - Added SMART test reamining percentage if Last Test has a SMART Test is in progress.
#   - Fix for empty SMART fields, typically for unsupported SSD's.
#   - Added IGNORE SMART Drive so you can ignore specific drives that may cause you wierd readings.
#   --- Updated so blank SSD table header is removed when you ignore all the drives (just crazy talk).
# v1.4a:
#   - Fixed report errors for if a SCRUB is in progress, now shows estimated completion time.
#   - Fixed report error for a Canceled SCRUB.
#   - Fixed FreeBSD/Linux use for SCRUB report (minor oversight).
# v1.4:
#   - Run on CRON JOB using /path/multi_report_v1.4.sh
#   - Fixed for automatic running between FreeBSD and Linux Debian (aka SCALE) as of this date.
#   - All SMART Devices will report.
#   - Added conditional Subject Line (Good/Critical/Warning).
#   - Added Automatic SSD Support.
#   --- Some updates may need to be made to fit some of SSD's. Code in the area of about line 530 will
#   --- need to be adjusted to add new attributes for the desired SSD's fields.
#   - UDMA_CRC_ERROR Override because once a drive encounters this type of error, it cannot be cleared
#   --- so you can offset it now vice having an alarm condition for old UDMA_CRC_Errors.
#   - Added listing NON-SMART Supported Drives.  Use only if useful to you, some drives will
#   --- still output some relevant data, many will not.
# v1.3:
#   - Added scrub duration column
#   - Fixed for FreeNAS 11.1 (thanks reven!)
#   - Fixed fields parsed out of zpool status
#   - Buffered zpool status to reduce calls to script
# v1.2:
#   - Added switch for power-on time format
#   - Slimmed down table columns
#   - Fixed some shellcheck errors & other misc stuff
#   - Added .tar.gz to backup file attached to email
#   - (Still coming) Better SSD SMART support
# v1.1:
#   - Config backup now attached to report email
#   - Added option to turn off config backup
#   - Added option to save backup configs in a specified directory
#   - Power-on hours in SMART summary table now listed as YY-MM-DD-HH
#   - Changed filename of config backup to exclude timestamp (just uses datestamp now)
#   - Config backup and checksum files now zipped (was just .tar before; now .tar.gz)
#   - Fixed degrees symbol in SMART table (rendered weird for a lot of people); replaced with a *
#   - Added switch to enable or disable SSDs in SMART table (SSD reporting still needs work)
#   - Added most recent Extended & Short SMART tests in drive details section (only listed one before, whichever was more recent)
#   - Reformatted user-definable parameters section
#   - Added more general comments to code
# v1.0:
#   - Initial release

###### User-definable Parameters

### Email Address
email="youremailaddress.com"
from="TrueNAS@address.com"

### Drive Ignore List
# Use this section to list any drives to ignore checking.  This is useful for ignoring USB Flash Drives.
# I configured this for up to 8 drives, if someone needs more, it's an easy minor code change.
#
# Format: IGNORESN1="DRIVE_SERIAL_NUMBER"
IGNORESN1="DRIVE_SERIAL_NUMBER"
IGNORESN2="DRIVE_SERIAL_NUMBER"
IGNORESN3="DRIVE_SERIAL_NUMBER"
IGNORESN4="DRIVE_SERIAL_NUMBER"
IGNORESN5="DRIVE_SERIAL_NUMBER"
IGNORESN6="DRIVE_SERIAL_NUMBER"
IGNORESN7="DRIVE_SERIAL_NUMBER"
IGNORESN8="DRIVE_SERIAL_NUMBER"

### Drive UDMA_CRC_Error_Count Flags
# List each drive by serial number and include the current UDMA_CRC_Error_Count value.
# If the count exceeds the count in the table below then a warning message will be generated.
# On the Status Report the UDMA CRC Errors block will be YELLOW with a value of "0" for an overridden value, for your indication.
# 
# Format: UCRC1SN = Drive Serial Number reported by SMART, UCRC1CNT = Current UDMA Error Count you want to subtract from this drive.
# You may add up to 8 drives with the current code. More drives could be added with a simple modification of the program.
# Do not list your drives with a value of "0" or each one will be Yellow.  Yikes!
UCRC1SN="WD-WMC4N2578099"
UCRC1CNT=1
UCRC2SN="2011E2940536"
UCRC2CNT=19
UCRC3SN="2011E2940534"
UCRC3CNT=18
UCRC4SN="YOUR_SERIAL_NUMBER_HERE"
UCRC4CNT=1
UCRC5SN="YOUR_SERIAL_NUMBER_HERE"
UCRC5CNT=1
UCRC6SN="YOUR_SERIAL_NUMBER_HERE"
UCRC6CNT=1
UCRC7SN="YOUR_SERIAL_NUMBER_HERE"
UCRC7CNT=1
UCRC8SN="YOUR_SERIAL_NUMBER_HERE"
UCRC8CNT=1

### zpool status summary table settings
usedWarn=90             # Pool used percentage for CRITICAL color to be used
scrubAgeWarn=37         # Maximum age (in days) of last pool scrub before CRITICAL color will be used (30 + 7 days for day of week)

### SMART status summary table settings
includeSSD="true"       # true = SSD Automatic Detection and Reporting, false = Disable SSD Automatic Detection and Reporting.
reportnonSMART="true"   # Will force even non-SMART devices to be reported, "false" = normal operation to ignore non-SMART devices.
tempWarn=42             # Drive temp (in C) at which WARNING color will be used
tempCrit=47             # Drive temp (in C) at which CRITICAL color will be used
sectorsCrit=1           # Number of sectors per drive with errors before CRITICAL color will be used
testAgeWarn=2           # Maximum age (in days) of last SMART test before CRITICAL color will be used
powerTimeFormat="ymdh"  # Format for power-on hours string, valid options are "ymdh", "ymd", "ym", or "y" (year month day hour)
deviceRedFlag="true"    # Set "true" to have the Device Column indicate RED for ANY issue.  Default is disabled.

### FreeNAS config backup settings
configBackup="false"     # Change to "true" to save config backup (which renders next two options operational); "false" to keep disable config backups
saveBackup="false"       # Change to "false" to delete FreeNAS config backup after mail is sent; "true" to keep it in dir below
backupLocation="/tmp/"   # Directory in which to save FreeNAS config backups

###### Auto-generated Parameters
programver="Multi-Report v1.4c-beta2 (9 Feb 2022)"
softver=$(uname -s)
host=$(hostname -s)
logfile="/tmp/smart_report_body.tmp"
logfile_header="/tmp/smart_report_header.tmp"
logfile_warning="/tmp/smart_report_warning_flag.tmp"
logfile_critical="/tmp/smart_report_critical_flag.tmp"
boundary="gc0p4Jq0M2Yt08jU534c0p"

### Global table colors
okColor="#c9ffcc"       # Hex code for color to use in SMART Status column if drives pass (default is light green, #c9ffcc)
warnColor="#ffd6d6"     # Hex code for WARN color (default is light red, #ffd6d6)
critColor="#ff0000"     # Hex code for CRITICAL color (default is bright red, #ff0000)
altColor="#f4f4f4"      # Table background alternates row colors between white and this color (default is light gray, #f4f4f4)
ovrdColor="#ffff66"     # Hex code for Override Yellow
blueColor="#87ceeb"     # Hex code for Sky Blue, used for the SCRUB In Progress background

### GET non-SSD listing - MUST support SMART
# smartdrives= All Drives that are SMART
# smartdrivesSSD = SSD all SMART
# nonsmartdrives = Any drive not supporting SMART

# Get Hard Drive listing - MUST support SMART
# variable smartdrives
if [ $softver != "Linux" ]; then
 smartdrives=$(for drive in $(sysctl -n kern.disks); do
        if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is: Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then
            printf "%s " "${drive}"
        fi
    done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
   else
 smartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do
         if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is: Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then
             printf "%s " "${drive}"
         fi
     done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
fi

# Get SSD listing - MUST suport SMART
# variable smartdrivesSSD
  if [ $softver != "Linux" ]; then
   smartdrivesSSD=$(for drive in $(sysctl -n kern.disks); do
        if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is: Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then
            targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')";
             case $targument in
                  ${IGNORESN1}|${IGNORESN2}|${IGNORESN3}|${IGNORESN4}|${IGNORESN5}|${IGNORESN6}|${IGNORESN7}|${IGNORESN8});;
                  *)
                  printf "%s " "${drive}";;
             esac
        fi
    done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
  else
   smartdrivesSSD=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do
        if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is: Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then
            targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')";
            case $targument in
                 ${IGNORESN1}|${IGNORESN2}|${IGNORESN3}|${IGNORESN4}|${IGNORESN5}|${IGNORESN6}|${IGNORESN7}|${IGNORESN8});;
                 *)
                 printf "%s " "${drive}";;
            esac
        fi
    done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
  fi

### Get the non-SSD listing - MUST support SMART
# variable nonsmartdrives
  if [ $softver != "Linux" ]; then
   nonsmartdrives=$(for drive in $(sysctl -n kern.disks); do
        if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is: Enabled")" ]; then
              targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')";
             case $targument in
                  ${IGNORESN1}|${IGNORESN2}|${IGNORESN3}|${IGNORESN4}|${IGNORESN5}|${IGNORESN6}|${IGNORESN7}|${IGNORESN8});;
                  *)
                  printf "%s " "${drive}";;
             esac
        fi
    done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
  else
   nonsmartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do
        if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is: Enabled")" ]; then
                  targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')";
             case $targument in
                  ${IGNORESN1}|${IGNORESN2}|${IGNORESN3}|${IGNORESN4}|${IGNORESN5}|${IGNORESN6}|${IGNORESN7}|${IGNORESN8});;
                  *)
                  printf "%s " "${drive}";;
             esac
        fi
    done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
  fi

### DONE Getting List of Drives

pools=$(zpool list -H -o name)

###### Email pre-formatting
### Set some of the email headers before conditional headers
(
    echo "MIME-Version: 1.0"
    echo "Content-Type: multipart/mixed; boundary=${boundary}"
) > "$logfile"

###### Config backup (if enabled)
if [ "$configBackup" == "true" ]; then
    # Set up file names, etc for later
    tarfile="/tmp/config_backup.tar.gz"
    filename="$(date "+FreeNAS_Config_%Y-%m-%d")"
    ### Test config integrity
    if ! [ "$(sqlite3 /data/freenas-v1.db "pragma integrity_check;")" == "ok" ]; then
        # Config integrity check failed, set MIME content type to html and print warning
        (
            echo "--${boundary}"
            echo "Content-Type: text/html"
            echo "<b>Automatic backup of FreeNAS configuration has failed! The configuration file is corrupted!</b>"
            echo "<b>You should correct this problem as soon as possible!</b>"
            echo "<br>"
        ) >> "$logfile"
    else
        # Config integrity check passed; copy config db, generate checksums, make .tar.gz archive
        cp /data/freenas-v1.db "/tmp/${filename}.db"
        md5 "/tmp/${filename}.db" > /tmp/config_backup.md5
        sha256 "/tmp/${filename}.db" > /tmp/config_backup.sha256
        (
            cd "/tmp/" || exit;
            tar -czf "${tarfile}" "./${filename}.db" ./config_backup.md5 ./config_backup.sha256;
        )
        (
            # Write MIME section header for file attachment (encoded with base64)
            echo "--${boundary}"
            echo "Content-Type: application/tar+gzip"
            echo "Content-Transfer-Encoding: base64"
            echo "Content-Disposition: attachment; filename=${filename}.tar.gz"
            base64 "$tarfile"
            # Write MIME section header for html content to come below
            echo "--${boundary}"
            echo "Content-Type: text/html"
        ) >> "$logfile"
        # If logfile saving is enabled, copy .tar.gz file to specified location before it (and everything else) is removed below
        if [ "$saveBackup" == "true" ]; then
            cp "${tarfile}" "${backupLocation}/${filename}.tar.gz"
        fi
        rm "/tmp/${filename}.db"
        rm /tmp/config_backup.md5
        rm /tmp/config_backup.sha256
        rm "${tarfile}"
    fi
else
    # Config backup enabled; set up for html-type content
    (
        echo "--${boundary}"
        echo "Content-Type: text/html"
    ) >> "$logfile"
fi

###### Report Summary Section (html tables)
### zpool status summary table
(
    # Write HTML table headers to log file; HTML in an email requires 100% in-line styling (no CSS or <style> section), hence the massive tags
    echo $programver
    echo "<br><br>"
    echo "<table style=\"border: 1px solid black; border-collapse: collapse;\">"
    echo "<tr><th colspan=\"10\" style=\"text-align:center; font-size:20px; height:40px; font-family:courier;\">ZPool Status Report Summary</th></tr>"
    echo "<tr>"
    echo "  <th style=\"text-align:center; width:130px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Pool<br>Name</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Status</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Read<br>Errors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Write<br>Errors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Cksum<br>Errors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Used %</th>"
    echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Scrub<br>Repaired<br>Bytes</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Scrub<br>Errors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last<br>Scrub<br>Age</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last<br>Scrub<br>Duration</th>"
    echo "</tr>"
) >> "$logfile"
poolNum=0
for pool in $pools; do
    # zpool health summary
    status="$(zpool list -H -o health "$pool")"
    # Total all read, write, and checksum errors per pool
    errors="$(zpool status "$pool" | grep -E "(ONLINE|DEGRADED|FAULTED|UNAVAIL|REMOVED)[ \\t]+[0-9]+")"
    readErrors=0
    for err in $(echo "$errors" | awk '{print $3}'); do
        if echo "$err" | grep -E -q "[^0-9]+"; then
            readErrors=1000
            break
        fi
        readErrors=$((readErrors + err))
    done
    writeErrors=0
    for err in $(echo "$errors" | awk '{print $4}'); do
        if echo "$err" | grep -E -q "[^0-9]+"; then
            writeErrors=1000
            break
        fi
        writeErrors=$((writeErrors + err))
    done
    cksumErrors=0
    for err in $(echo "$errors" | awk '{print $5}'); do
        if echo "$err" | grep -E -q "[^0-9]+"; then
            cksumErrors=1000
            break
        fi
        cksumErrors=$((cksumErrors + err))
    done
    # Not sure why this changes values larger than 1000 to ">1K", but I guess it works, so I'm leaving it
    # Answer to question above: Becasue it formats the value to = ">1K" vice stating very large values to
    # fit into the formatted table.  All we care about is it's way too high.
    if [ "$readErrors" -gt 999 ]; then readErrors=">1K"; fi
    if [ "$writeErrors" -gt 999 ]; then writeErrors=">1K"; fi
    if [ "$cksumErrors" -gt 999 ]; then cksumErrors=">1K"; fi


    # Get used capacity percentage of the zpool
    used="$(zpool list -H -p -o capacity "$pool")"
    # Gather info from most recent scrub; values set to "N/A" initially and overwritten when (and if) it gathers scrub info
    scrubRepBytes="N/A"
    scrubErrors="N/A"
    scrubAge="N/A"
    scrubTime="N/A"
    statusOutput="$(zpool status "$pool")"

### Fix for SCRUB Cancelled and In-Progress

#Check if scrub in progress
    if [ "$(echo "$statusOutput" | grep -w "scan" | awk '{print $4}')" = "progress" ]; then
        scrubAge="In Progress"
        scrubTime="$(echo "Est Comp: ")$(echo "$statusOutput" | grep "done, " | awk '{print $5}')"

# Check if the SCRUB is completed or canceled.

   elif [ "$(echo "$statusOutput" | grep "scan" | awk '{print $2}')" = "scrub" ] && [ "$(echo "$statusOutput" | grep "scan" | awk '{print $3}')" = "canceled" ]; then
          scrubAge="Canceled"
       elif [ "$(echo "$statusOutput" | grep "scan" | awk '{print $2}')" = "scrub" ]; then
        scrubRepBytes="$(echo "$statusOutput" | grep "scan" | awk '{print $4}')"
        scrubRepBytes="$(echo "$scrubRepBytes" | rev | cut -c2- | rev)"
        scrubErrors="$(echo "$statusOutput" | grep "scan" | awk '{print $8}')"
        # Convert time/datestamp format presented by zpool status, compare to current date, calculate scrub age
# For FreeBSD
if [ $softver != "Linux" ]; then
        scrubDate="$(echo "$statusOutput" | grep "scan" | awk '{print $15"-"$12"-"$13"_"$14}')"
        scrubTS="$(date -j -f "%Y-%b-%e_%H:%M:%S" "$scrubDate" "+%s")"
else
# For Linux
        scrubDate="$(echo "$statusOutput" | grep "scan" | awk '{print $12" "$13" "$15" "$14}')"
        scrubTS="$(date --date="$scrubDate" "+%s")"
fi
        currentTS="$(date "+%s")"
        scrubAge=$((((currentTS - scrubTS) + 43200) / 86400))
        scrubTime="$(echo "$statusOutput" | grep "scan" | awk '{print $6}')"
       else
        #No scrub previously performed
        scrubAge="Never Scrubbed"
   fi


    # Set row's background color; alternates between white and $altColor (light gray)
    if [ $((poolNum % 2)) == 1 ]; then bgColor="#ffffff"; else bgColor="$altColor"; fi
    poolNum=$((poolNum + 1))
    # Set up conditions for warning or critical colors to be used in place of standard background colors
    if [ "$status" != "ONLINE" ]; then statusColor="$warnColor"; echo "scrub offline error" >> "$logfile_critical"; else statusColor="$bgColor"; fi
    if [ "$readErrors" != "0" ]; then readErrorsColor="$warnColor"; echo "scrub read errors" >> "$logfile_warning"; else readErrorsColor="$bgColor"; fi
    if [ "$writeErrors" != "0" ]; then writeErrorsColor="$warnColor"; echo "scrub write errors" >> "$logfile_warning"; else writeErrorsColor="$bgColor"; fi
    if [ "$cksumErrors" != "0" ]; then cksumErrorsColor="$warnColor"; echo "scrub cksum errors" >> "$logfile_warning"; else cksumErrorsColor="$bgColor"; fi
    if [ "$used" -gt "$usedWarn" ]; then usedColor="$warnColor"; echo "scrub used" >> "$logfile_warning"; else usedColor="$bgColor"; fi
    if [ "$scrubRepBytes" != "N/A" ] && [ "$scrubRepBytes" != "0" ]; then scrubRepBytesColor="$warnColor"; echo "scrub Rep Bytes" >> "$logfile_warning"; else scrubRepBytesColor="$bgColor"; fi
    if [ "$scrubErrors" != "N/A" ] && [ "$scrubErrors" != "0" ]; then scrubErrorsColor="$warnColor"; echo "scrub errors" >> "$logfile_critical"; else scrubErrorsColor="$bgColor"; fi
    if [ "$(echo "$scrubAge" | awk '{print int($1)}')" -gt "$scrubAgeWarn" ]; then scrubAgeColor="$warnColor"; echo "scrub age" >> "$logfile_warning"; else scrubAgeColor="$bgColor"; fi
    if [ "$scrubAge" == "In Progress" ]; then scrubAgeColor="$blueColor"; else scrubAgeColor="$bgColor"; fi
    (
        # Use the information gathered above to write the date to the current table row
        printf "<tr style=\"background-color:%s;\">
            <td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s%%</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
            <td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>
        </tr>\\n" "$bgColor" "$pool" "$statusColor" "$status" "$readErrorsColor" "$readErrors" "$writeErrorsColor" "$writeErrors" "$cksumErrorsColor" \
        "$cksumErrors" "$usedColor" "$used" "$scrubRepBytesColor" "$scrubRepBytes" "$scrubErrorsColor" "$scrubErrors" "$scrubAgeColor" "$scrubAge" "$scrubTime"
    ) >> "$logfile"

done
# End of zpool status table
echo "</table>" >> "$logfile"

### SMART status summary table
# Assign which drives we are looking at
drives=$smartdrives
(
    # Write HTML table headers to log file
    echo "<br><br>"
    echo "<table style=\"border: 1px solid black; border-collapse: collapse;\">"
    echo "<tr><th colspan=\"15\" style=\"text-align:center; font-size:20px; height:40px; font-family:courier;\">Hard Drive - SMART Status Report Summary</th></tr>"
    echo "<tr>"
    echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Device</th>"
    echo "  <th style=\"text-align:center; width:130px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Serial<br>Number</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">SMART<br>Status</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Temp</th>"
    echo "  <th style=\"text-align:center; width:120px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Power-On<br>Time</th>"
    echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Start/Stop<br>Count</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Spin<br>Retry<br>Count</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Realloc'd<br>Sectors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Realloc<br>Events</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Current<br>Pending<br>Sectors</th>"
    echo "  <th style=\"text-align:center; width:120px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Offline<br>Uncorrectable<br>Sectors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">UltraDMA<br>CRC<br>Errors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Seek<br>Error<br>Health</th>"
    echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last Test<br>Age (days)</th>"
    echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last Test<br>Type</th></tr>"
    echo "</tr>"
) >> "$logfile"
for drive in $drives; do
    (
        # For each drive detected, run "smartctl -A -i" and parse its output. This whole section is a single, long statement, so I'll make all comments here.
        # Start by passing awk variables (all the -v's) used in other parts of the script. Other variables are calculated in-line with other smartctl calls.
        # Next, pull values out of the original "smartctl -A -i" statement by searching for the text between the //'s.
        # After parsing the output, compute other values (last test's age, on time in YY-MM-DD-HH).
        # After these computations, determine the row's background color (alternating as above, subbing in other colors from the palate as needed).
        # Finally, print the HTML code for the current row of the table with all the gathered data.
        smartctl -A -i /dev/"$drive" | \
        awk -v device="$drive" -v tempWarn="$tempWarn" -v tempCrit="$tempCrit" -v sectorsCrit="$sectorsCrit" -v testAgeWarn="$testAgeWarn" \
        -v okColor="$okColor" -v warnColor="$warnColor" -v critColor="$critColor" -v ovrdColor="$ovrdColor" -v altColor="$altColor" -v powerTimeFormat="$powerTimeFormat" \
        -v UCRC1SN="$UCRC1SN" -v UCRC1CNT="$UCRC1CNT" -v UCRC2SN="$UCRC2SN" -v UCRC2CNT="$UCRC2CNT" -v UCRC3SN="$UCRC3SN" -v UCRC3CNT="$UCRC3CNT" \
        -v UCRC4SN="$UCRC4SN" -v UCRC4CNT="$UCRC4CNT" -v UCRC5SN="$UCRC5SN" -v UCRC5CNT="$UCRC5CNT" -v UCRC6SN="$UCRC6SN" -v UCRC6CNT="$UCRC6CNT" \
        -v UCRC7SN="$UCRC7SN" -v UCRC8CNT="$UCRC8CNT" -v IGNORESN1="$IGNORESN1" -v IGNORESN2="$IGNORESN2" -v IGNORESN3="$IGNORESN3" -v IGNORESN4="$IGNORESN4" \
        -v IGNORESN4="$IGNORESN4" -v IGNORESN5="$IGNORESN5" -v IGNORESN6="$IGNORESN6" -v IGNORESN7="$IGNORESN7" -v IGNORESN8="$IGNORESN8" -v includeSSD="$includeSSD" \
        -v smarttesting="$(smartctl -a /dev/"$drive" | grep "remaining" | awk '{print $1}')" \
        -v chkreadfailure="$(smartctl -a /dev/"$drive" | grep "# 1" | awk '{print $5}')" \
        -v lastTestHours="$(smartctl -l selftest /dev/"$drive" | grep "# 1" | awk '{print $9}')" \
        -v lastTestType="$(smartctl -l selftest /dev/"$drive" | grep "# 1" | awk '{print $3}')" \
        -v smartStatus="$(smartctl -H /dev/"$drive" | grep "SMART overall-health" | awk '{print $6}')" ' \
        /Serial Number:/{serial=$3} \
        /Temperature_Celsius/{temp=($10 + 0)} \
        /Power_On_Hours/{onHours=$10} \
        /Start_Stop_Count/{startStop=$10} \
        /Spin_Retry_Count/{spinRetry=$10} \
        /Reallocated_Sector/{reAlloc=$10} \
        /Reallocated_Event_Count/{reAllocEvent=$10} \
        /Current_Pending_Sector/{pending=$10} \
        /Offline_Uncorrectable/{offlineUnc=$10} \
        /UDMA_CRC_Error_Count/{crcErrors=$10} \
        /Seek_Error_Rate/{seekErrorHealth=$4} \
        END {

if (serial == IGNORESN1) exit done;
if (serial == IGNORESN2) exit done;
if (serial == IGNORESN3) exit done;
if (serial == IGNORESN4) exit done;
if (serial == IGNORESN5) exit done;
if (serial == IGNORESN6) exit done;
if (serial == IGNORESN7) exit done;
if (serial == IGNORESN8) exit done;

            testAge=int((onHours - lastTestHours) / 24);
            yrs=int(onHours / 8760);
            mos=int((onHours % 8760) / 730);
            dys=int(((onHours % 8760) % 730) / 24);
            hrs=((onHours % 8760) % 730) % 24;
            if (powerTimeFormat == "ymdh") onTime=yrs "y " mos "m " dys "d " hrs "h";
            else if (powerTimeFormat == "ymd") onTime=yrs "y " mos "m " dys "d";
            else if (powerTimeFormat == "ym") onTime=yrs "y " mos "m";
            else if (powerTimeFormat == "y") onTime=yrs "y";
            else onTime=yrs "y " mos "m " dys "d " hrs "h ";
            if ((substr(device,3) + 0) % 2 == 1) bgColor = "#ffffff"; else bgColor = altColor;
if (smartStatus != "") if (smartStatus != "PASSED") smartStatusColor = critColor; else smartStatusColor = okColor;
if (smartStatus != "") if (smartStatus != "PASSED") printf "smart status" >> "/tmp/smart_report_critical_flag.tmp";
if (temp != "") if (temp >= tempCrit) tempColor = critColor; else if (temp >= tempWarn) tempColor = warnColor; else tempColor = bgColor;
if (temp != "") if (temp >= tempCrit) printf "critical drive temp" >> "/tmp/smart_report_critical_flag.tmp";
if (temp != "") if (temp >= tempWarn) printf "high drive temp" >> "/tmp/smart_report_warning_flag.tmp";
if (spinRetry != "") if (spinRetry != "0") spinRetryColor = warnColor; else spinRetryColor = bgColor;
if (spinRetry != "") if (spinRetry != "0") printf "spin retry" >> "/tmp/smart_report_critical_flag.tmp";
if (reAlloc != "") if ((reAlloc + 0) > sectorsCrit) reAllocColor = critColor; else if (reAlloc != 0) reAllocColor = warnColor; else reAllocColor = bgColor;
if (reAlloc != "") if ((reAlloc + 0) > sectorsCrit) printf "critical sectors" >> "/tmp/smart_report_critical_flag.tmp"; 
if (reAlloc != "") if (reAlloc != 0) printf "warning sectors" >> "/tmp/smart_report_warning_flag.tmp"; 
if (reAllocEvent != "") if (reAllocEvent != "0") reAllocEventColor = warnColor; else reAllocEventColor = bgColor;
if (reAllocEvent != "") if (reAllocEvent != "0") printf "reallocating sectors" >> "/tmp/smart_report_warning_flag.tmp";
if (pending != "") if ((pending + 0) > sectorsCrit) pendingColor = critColor; else if (pending != 0) pendingColor = warnColor; else pendingColor = bgColor;
if (pending != "") if ((pending + 0) > sectorsCrit) printf "sector errors" >> "/tmp/smart_report_critical_flag.tmp";
if (pending != "") if (pending != 0) printf "sector errors" >> "/tmp/smart_report_warning_flag.tmp";
if (offlineUnc != "") if ((offlineUnc + 0) > sectorsCrit) offlineUncColor = critColor; else if (offlineUnc != 0) offlineUncColor = warnColor; else offlineUncColor = bgColor;
if (offlineUnc != "") if ((offlineUnc + 0) > sectorsCrit) printf "uncorrectable errors" >> "/tmp/smart_report_critical_flag.tmp";
if (offlineUnc != "") if (offlineUnc != 0) printf "uncorrectable errors" >> "/tmp/smart_report_warning_flag.tmp";
if (crcErrors != "") if (crcErrors != "0") crcErrorsColor = warnColor; else crcErrorsColor = bgColor;

if (smarttesting >= 0) lastTestType = smarttesting "Remaining";
if (chkreadfailure == "Completed:") lastTestType = "Read Failure";
if (chkreadfailure == "Completed:") lastTestTypeColor = critColor; else lastTestTypeColor = bgColor; 
if (chkreadfailure == "Completed:") printf "read failure" >> "/tmp/smart_report_critical_flag.tmp";

### Routine to zero out the UDMA CRC Error Count and Highlights it Yellow.
### Currenlty allows up to 8 drives, hope no one needs that many.

if (serial == UCRC1SN) crcErrors=crcErrors-UCRC1CNT
if (serial == UCRC1SN) crcErrorsColor = ovrdColor;
if (serial == UCRC2SN) crcErrors=crcErrors-UCRC2CNT;
if (serial == UCRC2SN) crcErrorsColor = ovrdColor;
if (serial == UCRC3SN) crcErrors=crcErrors-UCRC3CNT;
if (serial == UCRC2SN) crcErrorsColor = ovrdColor;
if (serial == UCRC4SN) crcErrors=crcErrors-UCRC4CNT;
if (serial == UCRC4SN) crcErrorsColor = ovrdColor;
if (serial == UCRC5SN) crcErrors=crcErrors-UCRC5CNT;
if (serial == UCRC5SN) crcErrorsColor = ovrdColor;
if (serial == UCRC6SN) crcErrors=crcErrors-UCRC6CNT;
if (serial == UCRC6SN) crcErrorsColor = ovrdColor;
if (serial == UCRC7SN) crcErrors=crcErrors-UCRC7CNT;
if (serial == UCRC7SN) crcErrorsColor = ovrdColor;
if (serial == UCRC8SN) crcErrors=crcErrors-UCRC8CNT;
if (serial == UCRC8SN) crcErrorsColor = ovrdColor;

 if (crcErrors != "0") printf "crc errors" >> "/tmp/smart_report_critical_flag.tmp";

           if ((seekErrorHealth + 0) < 100) seekErrorHealthColor = warnColor; else seekErrorHealthColor = bgColor;
 if ((seekErrorHealth + 0) < 100) printf "seek errors" >> "/tmp/smart_report_warning_flag.tmp";
            if (testAge > testAgeWarn) testAgeColor = warnColor; else testAgeColor = bgColor;
 if (testAge > testAgeWarn) printf "test age" >> "/tmp/smart_report_warning_flag.tmp";

  if (smartStatusColor != okColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
  if (tempColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
  if (spinRetryColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
  if (reAllocColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
  if (reAllocEventColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
  if (pendingColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
  if (offlineUncColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
  if (crcErrorsColor != bgColor) if (crcErrorsColor != ovrdColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
  if (seekErrorHealthColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
  if (testAgeColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
  if (lastTestTypeColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;


            printf "<tr style=\"background-color:%s;\">\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">/dev/%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%d*C</td>\n" \
                "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s%%</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%d</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
            "</tr>\n", bgColor, deviceStatusColor, device, bgColor, serial, smartStatusColor, smartStatus, tempColor, temp, onTime, startStop, spinRetryColor, spinRetry, reAllocColor, reAlloc, \
            reAllocEventColor, reAllocEvent, pendingColor, pending, offlineUncColor, offlineUnc, crcErrorsColor, crcErrors, seekErrorHealthColor, seekErrorHealth, \
            testAgeColor, testAge, lastTestTypeColor, lastTestType;
        }'
    ) >> "$logfile"
done
# End SMART summary table and summary section
(
    echo "</table>"
    echo "<br><br>"
) >> "$logfile"


###### SOLID STATE DRIVE SMART SECTION HERE
drives=$smartdrivesSSD
if [ "$includeSSD" == "true" ]; then
echo "SSD Auto Detection Enabled" >> "$logfile"

if [ "$drives" != "" ]; then

### SMART status summary table
(
    # Write HTML table headers to log file
    echo "<table style=\"border: 1px solid black; border-collapse: collapse;\">"
    echo "<tr><th colspan=\"13\" style=\"text-align:center; font-size:20px; height:40px; font-family:courier;\">SSD - SMART Status Report Summary</th></tr>"
    echo "<tr>"
    echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Device</th>"
    echo "  <th style=\"text-align:center; width:130px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Serial<br>Number</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">SMART<br>Status</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Temp</th>"
    echo "  <th style=\"text-align:center; width:120px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Power-On<br>Time</th>"
    echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Wear Level</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Realloc'd<br>Sectors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Realloc<br>Events</th>"
    echo "  <th style=\"text-align:center; width:120px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Offline<br>Uncorrectable<br>Sectors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">UltraDMA<br>CRC<br>Errors</th>"
    echo "  <th style=\"text-align:center; width:80px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Read<br>Error<br>Rate</th>"
    echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last Test<br>Age (days)</th>"
    echo "  <th style=\"text-align:center; width:100px; height:60px; border:1px solid black; border-collapse:collapse; font-family:courier;\">Last Test<br>Type</th></tr>"
    echo "</tr>"
) >> "$logfile"

for drive in $drives; do
    (
        # For each drive detected, run "smartctl -A -i" and parse its output. This whole section is a single, long statement, so I'll make all comments here.
        # Start by passing awk variables (all the -v's) used in other parts of the script. Other variables are calculated in-line with other smartctl calls.
        # Next, pull values out of the original "smartctl -A -i" statement by searching for the text between the //'s.
        # After parsing the output, compute other values (last test's age, on time in YY-MM-DD-HH).
        # After these computations, determine the row's background color (alternating as above, subbing in other colors from the palate as needed).
        # Finally, print the HTML code for the current row of the table with all the gathered data.
        smartctl -A -i /dev/"$drive" | \
        awk -v device="$drive" -v tempWarn="$tempWarn" -v tempCrit="$tempCrit" -v sectorsCrit="$sectorsCrit" -v testAgeWarn="$testAgeWarn" \
        -v okColor="$okColor" -v warnColor="$warnColor" -v critColor="$critColor" -v ovrdColor="$ovrdColor" -v altColor="$altColor" -v powerTimeFormat="$powerTimeFormat" \
        -v UCRC1SN="$UCRC1SN" -v UCRC1CNT="$UCRC1CNT" -v UCRC2SN="$UCRC2SN" -v UCRC2CNT="$UCRC2CNT" -v UCRC3SN="$UCRC3SN" -v UCRC3CNT="$UCRC3CNT" \
        -v UCRC4SN="$UCRC4SN" -v UCRC4CNT="$UCRC4CNT" -v UCRC5SN="$UCRC5SN" -v UCRC5CNT="$UCRC5CNT" -v UCRC6SN="$UCRC6SN" -v UCRC6CNT="$UCRC6CNT" \
        -v UCRC7SN="$UCRC7SN" -v UCRC8CNT="$UCRC8CNT" -v IGNORESN1="$IGNORESN1" -v IGNORESN2="$IGNORESN2" -v IGNORESN3="$IGNORESN3" -v IGNORESN4="$IGNORESN4" \
        -v IGNORESN4="$IGNORESN4" -v IGNORESN5="$IGNORESN5" -v IGNORESN6="$IGNORESN6" -v IGNORESN7="$IGNORESN7" -v IGNORESN8="$IGNORESN8" -v includeSSD="$includeSSD" \
        -v logfile_critical="$logfile_critical" -v logfile_warning="$logfile_warning" \
        -v chkreadfailure="$(smartctl -a /dev/"$drive" | grep "# 1" | awk '{print $5}')" \
        -v smarttesting="$(smartctl -a /dev/"$drive" | grep "test remaining" | awk '{print $1}')" \
        -v lastTestHours="$(smartctl -l selftest /dev/"$drive" | grep "# 1" | awk '{print $9}')" \
        -v lastTestType="$(smartctl -l selftest /dev/"$drive" | grep "# 1" | awk '{print $3}')" \
        -v smartStatus="$(smartctl -H /dev/"$drive" | grep "SMART overall-health" | awk '{print $6}')" ' \
        /Serial Number:/{serial=$3} \
        /Temperature/{temp=($10 + 0)} \
        /Power_On_Hours/{onHours=$10} \
        /Start_Stop_Count/{startStop=$10} \
        /Spin_Retry_Count/{spinRetry=$10} \
        /Reallocated_Sector/{reAlloc=$10} \
        /Reallocated_NAND_Blk_Cnt/{reAlloc=$10} \
        /Reallocated_Event_Count/{reAllocEvent=$10} \
        /Current_Pending_Sector/{pending=$10} \
        /Offline_Uncorrectable/{offlineUnc=$10} \
        /CRC_Error_Count/{crcErrors=$10} \
        /Read_Error_Rate/{readErrorrate=$10} \
        /SSD_Life_Left/{wearLevel=$4 + 0} \
        /Wear_Leveling_Count/{wearLevel=$4 + 0} \
        /Percent_Lifetime_Remaining/{wearLevel=$4 + 0} \

        END {
if (serial == IGNORESN1) exit done;
if (serial == IGNORESN2) exit done;
if (serial == IGNORESN3) exit done;
if (serial == IGNORESN4) exit done;
if (serial == IGNORESN5) exit done;
if (serial == IGNORESN6) exit done;
if (serial == IGNORESN7) exit done;
if (serial == IGNORESN8) exit done;

            testAge=int((onHours - lastTestHours) / 24);
            yrs=int(onHours / 8760);
            mos=int((onHours % 8760) / 730);
            dys=int(((onHours % 8760) % 730) / 24);
            hrs=((onHours % 8760) % 730) % 24;
            if (powerTimeFormat == "ymdh") onTime=yrs "y " mos "m " dys "d " hrs "h";
            else if (powerTimeFormat == "ymd") onTime=yrs "y " mos "m " dys "d";
            else if (powerTimeFormat == "ym") onTime=yrs "y " mos "m";
            else if (powerTimeFormat == "y") onTime=yrs "y";
            else onTime=yrs "y " mos "m " dys "d " hrs "h ";
            if ((substr(device,3) + 0) % 2 == 1) bgColor = "#ffffff"; else bgColor = altColor;

if (smartStatus != "") if (smartStatus != "PASSED") smartStatusColor = critColor; else smartStatusColor = okColor;
if (smartStatus != "") if (smartStatus != "PASSED") printf "smart status" >> "/tmp/smart_report_critical_flag.tmp";
if (temp != "") if (temp >= tempCrit) tempColor = critColor; else if (temp >= tempWarn) tempColor = warnColor; else tempColor = bgColor;
if (temp != "") if (temp >= tempCrit) printf "critical drive temp" >> "/tmp/smart_report_critical_flag.tmp";
if (temp != "") if (temp >= tempWarn) printf "high drive temp" >> "/tmp/smart_report_warning_flag.tmp";
if (spinRetry != "") if (spinRetry != "0") spinRetryColor = warnColor; else spinRetryColor = bgColor;
if (spinRetry != "") if (spinRetry != "0") printf "spin retry" >> "/tmp/smart_report_critical_flag.tmp";
if (reAlloc != "") if ((reAlloc + 0) > sectorsCrit) reAllocColor = critColor; else if (reAlloc != 0) reAllocColor = warnColor; else reAllocColor = bgColor;
if (reAlloc != "") if ((reAlloc + 0) > sectorsCrit) printf "critical sectors" >> "/tmp/smart_report_critical_flag.tmp";
if (reAlloc != "") if (reAlloc != 0) printf "warning sectors" >> "/tmp/smart_report_warning_flag.tmp";
if (reAllocEvent != "") if (reAllocEvent != "0") reAllocEventColor = warnColor; else reAllocEventColor = bgColor;
if (reAllocEvent != "") if (reAllocEvent != "0") printf "reallocating sectors" >> "/tmp/smart_report_warning_flag.tmp";
if (pending != "") if ((pending + 0) > sectorsCrit) pendingColor = critColor; else if (pending != 0) pendingColor = warnColor; else pendingColor = bgColor;
if (pending != "") if ((pending + 0) > sectorsCrit) printf "sector errors" >> "/tmp/smart_report_critical_flag.tmp";
if (pending != "") if (pending != 0) printf "sector errors" >> "/tmp/smart_report_warning_flag.tmp";
if (offlineUnc != "") if ((offlineUnc + 0) > sectorsCrit) offlineUncColor = critColor; else if (offlineUnc != 0) offlineUncColor = warnColor; else offlineUncColor = bgColor;
if (offlineUnc != "") if ((offlineUnc + 0) > sectorsCrit) printf "uncorrectable errors" >> "/tmp/smart_report_critical_flag.tmp";
if (offlineUnc != "") if (offlineUnc != 0) printf "uncorrectable errors" >> "/tmp/smart_report_warning_flag.tmp";
if (crcErrors != "") if (crcErrors != "0") crcErrorsColor = warnColor; else crcErrorsColor = bgColor;

if (smarttesting >= 0) lastTestType = smarttesting "Remaining";
if (chkreadfailure == "Completed:") lastTestType = "Read Failure";
if (chkreadfailure == "Completed:") lastTestTypeColor = critColor; else lastTestTypeColor = bgColor

### Routine to zero out the UDMA CRC Error Count and Highlights it Yellow.
### Currenlty allows up to 8 drives, hope no one needs that many.

if (serial == UCRC1SN) crcErrors=crcErrors-UCRC1CNT
if (serial == UCRC1SN) crcErrorsColor = ovrdColor;
if (serial == UCRC2SN) crcErrors=crcErrors-UCRC2CNT;
if (serial == UCRC2SN) crcErrorsColor = ovrdColor;
if (serial == UCRC3SN) crcErrors=crcErrors-UCRC3CNT;
if (serial == UCRC2SN) crcErrorsColor = ovrdColor;
if (serial == UCRC4SN) crcErrors=crcErrors-UCRC4CNT;
if (serial == UCRC4SN) crcErrorsColor = ovrdColor;
if (serial == UCRC5SN) crcErrors=crcErrors-UCRC5CNT;
if (serial == UCRC5SN) crcErrorsColor = ovrdColor;
if (serial == UCRC6SN) crcErrors=crcErrors-UCRC6CNT;
if (serial == UCRC6SN) crcErrorsColor = ovrdColor;
if (serial == UCRC7SN) crcErrors=crcErrors-UCRC7CNT;
if (serial == UCRC7SN) crcErrorsColor = ovrdColor;
if (serial == UCRC8SN) crcErrors=crcErrors-UCRC8CNT;
if (serial == UCRC8SN) crcErrorsColor = ovrdColor;

if (crcErrors != "") if (crcErrors != "0") printf "crc errors" >> "/tmp/smart_report_critical_flag.tmp";
if (readErrorrate != "") if (readErrorrate != 0) readErrorrateColor = warnColor; else readErrorrateColor = bgColor;
if (readErrorrate != "") if (readErrorrate != 0) printf "rear errors" >> "/tmp/smart_report_warning_flag.tmp";
if (testAge > testAgeWarn) testAgeColor = warnColor; else testAgeColor = bgColor;
if (testAge > testAgeWarn) printf "test age" >> "/tmp/smart_report_warning_flag.tmp";

if (smartStatusColor != okColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
# The Temp is causing an error whit a reading of 0 degrees. Only seems to be affected under Linux right now so this is not the code section in trouble.
if (tempColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (reAllocColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (reAllocEventColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (offlineUncColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (crcErrorsColor != bgColor) if (crcErrorsColor != ovrdColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (readErrorrateColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (testAgeColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;
if (lastTestTypeColor != bgColor) if (deviceRedFlag == "true") deviceStatusColor = critColor;

            printf "<tr style=\"background-color:%s;\">\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">/dev/%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%d*C</td>\n" \
                "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%d</td>\n" \
                "<td style=\"text-align:center; background-color:%s; height:25px; border:1px solid black; border-collapse:collapse; font-family:courier;\">%s</td>\n" \
            "</tr>\n", bgColor, deviceStatusColor, device, bgColor, serial, smartStatusColor, smartStatus, tempColor, temp, onTime, wearLevel, reAllocColor, reAlloc, \
            reAllocEventColor, reAllocEvent, offlineUncColor, offlineUnc, crcErrorsColor, crcErrors, readErrorrateColor, readErrorrate, \
            testAgeColor, testAge, lastTestTypeColor, lastTestType;
        }'
    ) >> "$logfile"
done
# End SMART summary table and summary section
(
    echo "</table>"
    echo "<br><br>"
) >> "$logfile"
else
echo "<br>No SMART Supported SSD's Found" >> "$logfile"
fi

else
echo "SSD Auto Detection Disabled" >> "$logfile"
fi
##### END SSD SECTION #####

###### Detailed Report Section (monospace text)
echo "<pre style=\"font-size:14px\">" >> "$logfile"

### zpool status for each pool
for pool in $pools; do
    (
      # Create a simple header and drop the output of zpool status -v
        echo "<b>########## ZPool status report for ${pool} ##########</b>"
        echo "<br>"
        zpool status -v "$pool"
        echo "<br><br>"
    ) >> "$logfile"
done

drives="${smartdrives} ${smartdrivesSSD}"
### SMART status for each drive - SMART Enabled
for drive in $drives; do
    # Gather brand and serial number of each drive
    brand="$(smartctl -i /dev/"$drive" | grep "Model Family" | awk '{print $3, $4, $5}')"
    serial="$(smartctl -i /dev/"$drive" | grep "Serial Number" | awk '{print $3}')"

    if [ $serial == "$IGNORESN1" ]; then continue; fi
    if [ $serial == "$IGNORESN2" ]; then continue; fi
    if [ $serial == "$IGNORESN3" ]; then continue; fi
    if [ $serial == "$IGNORESN4" ]; then continue; fi
    if [ $serial == "$IGNORESN5" ]; then continue; fi
    if [ $serial == "$IGNORESN6" ]; then continue; fi
    if [ $serial == "$IGNORESN7" ]; then continue; fi
    if [ $serial == "$IGNORESN8" ]; then continue; fi
    (
     # Create a simple header and drop the output of some basic smartctl commands
        echo "<br>"
        echo "<b>########## SMART status report for ${drive} drive (${brand}: ${serial}) ##########</b>"
        smartctl -H -A -l error /dev/"$drive"
        smartctl -l selftest /dev/"$drive" | grep "Extended \\|Num" | cut -c6- | head -2
        smartctl -l selftest /dev/"$drive" | grep "Short \\|Num" | cut -c6- | head -2 | tail -n -1
        echo "<br>"
    ) >> "$logfile"
    
done

### NON-SMART status report section
# I don't perticularly use this but some folks might find it useful.
# To activate it, in the variables set reportnonSMART=true.
# It will list all drives where Non-SMART is true and remove devices starting with "cd", for example "cd0"

drives=$nonsmartdrives
if [ $reportnonSMART == "true" ]; then 
for drive in $drives; do
  if [ ! "$(echo "$drive" | grep "cd")" ]; then
   if [ $softver != "Linux" ]; then
    # Gather brand and serial number of each drive
    brand="$(smartctl -i /dev/"$drive" | grep "Model Family" | awk '{print $3, $4, $5}')"
    serial="$(smartctl -i /dev/"$drive" | grep "Serial Number" | awk '{print $3}')"
   else
    brand="$(fdisk -l /dev/"$drive" | grep "Disk model" | awk '{print $3, $4, $5}')"
    serial="$(fdisk -l /dev/"$drive" | grep "Serial Number" | awk '{print $3}')"
   fi
    (
        echo "<br>"
        echo "<b>########## NON-SMART status report for ${drive} drive (${brand}: ${serial}) ##########</b>"
    # And we will dump everything since it's not a standard SMART device.
        echo "<b>SMARTCTL DATA</b>"
        smartctl -a /dev/"$drive"
        echo "<br>"
      if [ $softver == "Linux" ]; then
        echo "<b>FDISK DATA</b>"
        fdisk -l /dev/"$drive"
      fi
    ) >> "$logfile"
  fi
done
fi

### Remove some un-needed junk from the output
sed -i -e '/smartctl/d' "$logfile"
sed -i -e '/Copyright/d' "$logfile"
sed -i -e '/=== START OF READ/d' "$logfile"
sed -i -e '/SMART Attributes Data/d' "$logfile"
sed -i -e '/Vendor Specific SMART/d' "$logfile"
sed -i -e '/SMART Error Log Version/d' "$logfile"

### End details section, close MIME section
(
    echo "</pre>"
    echo "--${boundary}--"
)  >> "$logfile"

### Create New Email Header - Set Subject Line

## Test if there is a Warning Message and Setup Subject Line
if test -e "$logfile_critical"; then
 subject="Subject: *CRITICAL ERROR*  SMART Testing Results for ${host}  *CRITICAL ERROR*"
elif test -e "$logfile_warning"; then
 subject="Subject: *WARNING*  SMART Testing Results for ${host}  *WARNING*"
else
 subject="Subject: SMART Testing Results for ${host} - All is Good"
fi

### Set email headers ###
(
echo "From: ${from}"
echo "To: ${email}"
echo "Subject: ${subject}"
) > ${logfile_header}

cat $logfile >> $logfile_header

### Send report
sendmail -t -oi < "$logfile_header"

### Clean up our temporary files
if test -e "$logfile"; then rm "$logfile"; fi
if test -e "$logfile_header"; then rm "$logfile_header"; fi
if test -e "$logfile_critical"; then rm "$logfile_critical"; fi
if test -e "$logfile_warning"; then rm "$logfile_warning"; fi