#!/bin/bash
# Set the language to English so the date calculations work correctly with zpool status results.
LANG="en_US.UTF-8"
# With version 1.6c and above you may use an external configuration file.
# Use [-help] to read the Help Section. For a short list of commands use [-h].
# Use [-config] to create a configuration file in the directory this script is run from.
###### ZPool & SMART status report with FreeNAS/TrueNAS config backup
### Original script by joeschmuck, modified by Bidule0hm, then by melp (me)
### Version: v1.3 TrueNAS Scale (Jeff Alperin 12-6-21)
### Version v1.4, v1.5, v1.6, v2.0 FreeNAS/TrueNAS (Core & Scale) (joeschmuck)
### Changelog:
# V2.0.8 (20 February 2023)
# - Added Email/Text Monitoring Feature for NugentS to try out.
# - Edit multi_report_config.txt for additional email address.
# - Use '-m' switch to invoke. Statistical data is recorded each time the script is run.
#
#
# V2.0.7 (7 February 2023)
# - Bug Fix: Nuisance error message, did not impact operation but it doesn't look good.
# -- "Cannot open input file /tmp/zpoollist.txt" and "/tmp/zpoolstatus.txt". Only occurs in Core
# -- during TrueNAS configuration file attachment operation (default is Monday).
#
# V2.0.6 (1 February 2023)
# - Reduced drive data collection. Added 'zpool' data collection.
#
# V2.0.5 (27 January 2023)
# - Adjusted Zpool Status to allow 'resilvering' status message. (Line 1340)
# - Updated '-dump email' command to allow user to enter comments to the author.
#
# V2.0.4 (26 January 2023)
# - Fixed if Zpool does not provide a number for fragmentation, will now display non_exist_value string.
#
# V2.0.3 (24 January 2023)
# - Hacked HDD SMART Testing in progress with "NOW" in the '# 1' line. Will fix better later.
#
# V2.0.2 (24 January 2023)
# - Fix Wear Level that may fail on some drives.
#
# v2.0.1 (21 January 2023)
# - Fixed Zpool Fragmentation Warning for 9% and greater (Hex Math issue again).
#
# v2.0 (21 January 2023)
# - Formatted all -config screens to fit into 80 column x 24 lines.
# - Removed custom builds
# - Fixed Custom Configuration Delete Function.
# - Fixed Zpool Scrub Bytes for FreeNAS 11.x
# - Fixed SMART Test to allow for 'Offline' value.
# - Modified Wear Level script to account for 'Reverse' Adjustment.
# - Added Wear Level Adjustment Reversing to the Custom Drive configuration.
# - Added Output.html to -dump command.
# - Added Mouseover and Alternate '()' to Mouseover for normalized values (Reallocated Sectors, Reallocated Sector Events, UDMA CRC, MultiZone).
# - Updated Testing Code to accept both drive_a and drive_x files.
# - Added Zpool Fragmentation value by request.
# - Added '-dump email' parameter to send joeschmuck2023@hotmail.com an email with the drive data and the multi_report_config.txt file ONLY.
# - Added Drive dump data in JSON format. It looks like a better way to parse the drive data. Still retaining the older file format for now.
#
# The multi_report_config file will automatically update previous versions to add new features.
#
# v1.6f (27 December 2022)
# - Added recognition for WDC SSD "230 Media_Wearout_Indicator".
# - Adjusted the language to English for the Date calculations.
# - Updated datestamp2 "date" command to be compatible with FreeBSD 11 and earlier.
# - Updated Zpool Status Report to display Used Space Capacity when using the "zpool" reporting configuration.
# - Added customizable alarm setpoints for up to 24 drives.
# -- This feature allows for customizing drives which do not fit into the default parameters.
# -- It is not intended to individualize each drive, but you could if you wanted.
# -- This allows the option for removal of the three custom code options.
# - The configuration file in the email will now change to FreeNAS or TrueNAS based
# -- on the software running vice FreeNAS hard-coded.
# - Corrected several spelling errors throughout script.
#
# The multi_report_config file will automatically update previous versions to add new features.
#
# - Future Plan: Remove custom code for snowlucas2022, diedrichg, and mistermanko in version 2.0.
#
# v1.6e (11 November 2022)
# - Fixed gptid not showing in the text section for the cache drive (Scale only affected).
# - Fixed Zpool "Pool Size" - Wasn't calculating correctly under certain circumstances.
# - Added Toshiba MG07+ drive Helium value support.
# - Added Alphabetizing Zpool Names and Device ID's.
# - Added No HDD Chart Generation if no HDD's are identified (nice for SSD/NVMe Only Systems).
# - Added Warranty Column to chart (by request and must have a value in the Drive_Warranty variable).
# - Removed Update option in -config since the script will automatically update now.
# - Updated instructions for multiple email addresses.
# - Updated instructions for "from:" address, some email servers will not accept the default
# -- value and must be changed to the email address of the account sending the email.
# - Added the No Text Section Option (enable_text) to remove the Text Section from the email output
# -- and display the chart only, if the value is not "true".
# - Added Phison Driven SSD attribute for correct Wear Level value.
#
# NOTES: If there is an error such as the host aborts a drive test and an error occurs, the script may
# report a script failure. I do not desire to account for every possible drive error message.
# If you take a look at your drive data, you may notice a problem. Fix the problem and the
# script should work normally. If it still does not, then reach out for assistance.
#
# The multi_report_config file will automatically update previous versions to add new features.
#
# v1.6d-2 (09 October 2022)
# - Bug fix for NVMe power on hours.
# --- Unfortunately as the script gets more complex it's very easy to induce a problem. And since I do not have
# --- a lot of different hardware, I need the users to contact me and tell me there is an issue so I can fix it.
# --- It's unfortunate that I've have two bug fixes already but them's the breaks.
# - Updated to support more drives Min/Max temps and display the non-existent value if nothing is obtained vice "0".
#
# The multi_report_config file is compatible with version back to v1.6d.
#
# v1.6d-1 (08 October 2022)
# - Bug Fix for converting multiple numbers from Octal to Decimal. The previous process worked "most" of the time
# -- but we always aim for 100% working.
#
# The multi_report_config file is compatible with version back to v1.6d.
#
# v1.6d (05 October 2022)
# - Thanks goes out to ChrisRJ for offering some great suggestions to enhance and optimize the script.
# - Updated gptid text and help text areas (clarifying information)
# - Updated the -dump parameter to -dump [all] and included non-SMART attachments.
# - Added Automatic UDMA_CRC, MultiZone, and Reallocated Sector Compensation to -config advanced option K.
# - Fixed Warranty Date always showing as expired.
# - Added Helium and Raw Read Error Rates to statistical data file.
# - Added Raw Read Error Rates chart column.
# - Added compensation for Seagate Seek Error Rates and Raw Read Error Rates.
# - Added Automatic Configuration File Update feature.
# - Added selection between ZFS Pool Size or Zpool Pool Size. ZFS is representative of the actual storage capacity
# -- and updated the Pool Status Report Summary chart.
# - Added ATA Error Log Silencing (by special request).
# - Added 0.1 second delay after writing "$logfile" to eliminate intermittent file creation errors.
# - Fixed Text Report -> Drive Model Number not showing up for some drives.
# - Added option to email copy of multi_report_config.txt upon any automatic script modification and/or by day.
# v1.6c (28 August 2022)
# - Supports external configuration file (but not required).
# - Completely Configurable by running the script -config parameter (this took a lot of work).
# - Added HDD/SSDmaxtempovrd variables to combat some bogus SSD values.
# - Added TLER (SCT) support.
# - Added support for drives which do not support recording over 65536 hours for SMART Tests and rolls over to start at zero again.
# - Added -dump parameter to create and email all of the drives smartctl outputs as text file email attachments.
# - Added support for Helium drives.
# v1.6: (05 August 2022)
# Thanks to Jeff, Simon, and Sean for providing me more test data than I could shake a stick at and friendly constructive opinions/advice.
# - Complete rewrite of the script. More organized and easier for future updates.
# - Almost completely got rid of using AWK, earlier versions had way too much programming within the AWK structure.
# - Reads the drives much less often (3 times each I believe).
# - Added test input file to parse txt files of smartctl -a output. This will allow for a single drive entry and ability
# -- for myself or any script writer to identify additional parameters for unrecognized drives.
# -- Usage: program_name.sh [HDD|SSD|NVM] [inputfile_a.txt] [inputfile_b.txt]
# - Added better support for SAS drives.
# - Fixed NVMe and SAS Power On Hours for statistical data recording, and other things.
# - Added Critical and Warning Logs to email output with better descriptive data.
# - Logs (stored in /tmp/) no longer deleted after execution to aid in troubleshooting, but deleted at the start of the script.
# - Added HELP file, use program_name.sh [-h] [-help]
# - Added SCT Error Recovery to the Text Report section.
# - Added Zpool Size, Free Space, and Temp Min/Max.
# - Added customizable temperature values and customizable Non-Value fields (use to be coded to "N/A").
# - Added support for SandForce SSD.
# v1.5:
# - Added NVMe support
# - Added clearer error reporting for Warning and Critical errors.
# - Known Problems: The NVMe Power On Time has a comma that I can't get rid of, yet. I want to remove the comma when the data is retrieved.
# -- NVMe's are not all standardized so I expect tweaks as different drive data arrives.
# -- onHours that includes a comma will not record correctly in the statistical data file. This is related to the NVMe problem above.
# -- Zpool Summary does not indicate Scrub Age warning, likely the entire summary has issues.
# v1.4d:
# - Fixed Scrub In Progress nuisance error when a scrub is in progress.
# - Added offsetting Reallocated Sectors for four drives. This should be for testing only. Any drives
# -- with a significant number of bad sectors should be replaced, just my opinion.
# - Added Drive Warranty Expiration warning messages and ability to disable the Email Subject line warning.
# -- NOT TESTED ON OTHER THAN U.S. FORMATTED DATE YYYY-MM-DD.
# - Added HDD and SSD individual temperature settings.
# - Changed order of polling Temperature data from HDD/SSD.
# v1.4c:
# - Rewrite to create functions and enable easier editing.
# - Added Custom Reports.
# - Added disabling the RAW 'smartctl -a' data appended to the end of the email.
# - Added sorting drives alphabetically vice the default the OS reports them.
# - Added RED warning in Device for any single failure in the summary (deviceRedFlag switch controlled).
# - Added some additional SSD definitions.
# - Fixed sorting last two SMART Tests, now reports them in proper order.
# - Fixed detecting "SMART Support is: Enabled", for white spaces.
# - Changed IGNORE DRIVES to a String Format to clean up and simplify programming.
# - Added MultiZone_Errors support for up to eight drives.
# - Added sectorWarn variable to complement the sectorCrit variable.
# - Added ignoreSeekError variable to ignore some of those wild Seek Error Rate values.
# - Added ignoreUDMA CRC Errors due to the "Known Problem"
# - Fixed md5/sha256 error on TrueNAS Scale (only used during config backups).
# - Added selectable config backup periodicity by day vice every run.
# - Added Exporting statistical data for trend analysis.
# -- Can be setup to email statistics weekly, monthly, or not at all.
# -- The -s switch will run Data Collection Only, no email generated. Note: Do Not run two instances at once, the temp files do not survive.
# - Fixed the Capacity to remove the brackets "[]", thanks Jeff Alperin.
# - Fixed Scrub Age failure due to 1 day or longer repair time, now shows anything >24 hours.
#
# - Known Problem: One user reported UDMA_CRC_Errors is not subtracting correctly, have not been able to personally replicate it.
# -- This error seems to occur around line #1027
# v1.4b:
# - Added SMART test remaining percentage if Last Test has a SMART Test is in progress.
# - Fix for empty SMART fields, typically for unsupported SSD's.
# - Added IGNORE SMART Drive so you can ignore specific drives that may cause you weird readings.
# --- Updated so blank SSD table header is removed when you ignore all the drives (just crazy talk).
# v1.4a:
# - Fixed report errors for if a SCRUB is in progress, now shows estimated completion time.
# - Fixed report error for a Canceled SCRUB.
# - Fixed FreeBSD/Linux use for SCRUB report (minor oversight).
# v1.4:
# - Run on CRON JOB using /path/multi_report_v1.4.sh
# - Fixed for automatic running between FreeBSD and Linux Debian (aka SCALE) as of this date.
# - All SMART Devices will report.
# - Added conditional Subject Line (Good/Critical/Warning).
# - Added Automatic SSD Support.
# --- Some updates may need to be made to fit some of SSD's. Code in the area of about line 530 will
# --- need to be adjusted to add new attributes for the desired SSD's fields.
# - UDMA_CRC_ERROR Override because once a drive encounters this type of error, it cannot be cleared
# --- so you can offset it now vice having an alarm condition for old UDMA_CRC_Errors.
# - Added listing NON-SMART Supported Drives. Use only if useful to you, some drives will
# --- still output some relevant data, many will not.
# v1.3:
# - Added scrub duration column
# - Fixed for FreeNAS 11.1 (thanks reven!)
# - Fixed fields parsed out of zpool status
# - Buffered zpool status to reduce calls to script
# v1.2:
# - Added switch for power-on time format
# - Slimmed down table columns
# - Fixed some shellcheck errors & other misc stuff
# - Added .tar.gz to backup file attached to email
# - (Still coming) Better SSD SMART support
# v1.1:
# - Config backup now attached to report email
# - Added option to turn off config backup
# - Added option to save backup configs in a specified directory
# - Power-on hours in SMART summary table now listed as YY-MM-DD-HH
# - Changed filename of config backup to exclude timestamp (just uses datestamp now)
# - Config backup and checksum files now zipped (was just .tar before; now .tar.gz)
# - Fixed degrees symbol in SMART table (rendered weird for a lot of people); replaced with a *
# - Added switch to enable or disable SSDs in SMART table (SSD reporting still needs work)
# - Added most recent Extended & Short SMART tests in drive details section (only listed one before, whichever was more recent)
# - Reformatted user-definable parameters section
# - Added more general comments to code
# v1.0:
# - Initial release
######### INSTRUCTIONS ON USE OF THIS SCRIPT
#
# This script will perform three main functions:
# 1: Generate a Pool/Drive report and send an email.
# 2: Create a copy of your FreeNAS/TrueNAS Config File and attach to the email.
# 3: Create a statistical database and optionally attach to the email.
#
# This script has been designed to primarily run using an external reusable configuration file which is
# configured by running the script with the -config parameter.
#
# You may opt to not using the extternal configuration file and manually change the User-definable
# Parameters within this script.
#
# In order to configure the script properly read over the User-definable Parameters before making any changes.
# Make changes as indicated by the section instructions.
#
# To run the program from the command line, use ./program_name.sh [-h] for additional help instructions,
# and [-config] to run the configuration routine (highly recommended).
#
# If you create an external configuration file, you never have to edit the script,
# so how many times do I need to say it is highly recommended? And I may force the
# change to require the external configuration file eventually to make the script a little smaller.
#
# You may need to make the script executable using "chmod +x program_name.sh"
#
###### User-definable Parameters (IF YOU DO NOT WANT TO USE THE EXTERNAL CONFIGURATION FILE) #######
#
# Modifying the script below should ONLY be done if you are not using an External Configuration File.
# The External Configuration File allows the end user to not be required to update the script every time a
# new change comes out. It is highly recommended to use the external configuration file, eventually
# I plan to make the external configuration file a requirment.
#
# The sections below configures the script to your needs. Please follow the instructions as it will matter, you cannot
# just "wing it". Configurations are exact. We use generally three different formats, Variables = true/false,
# Variables = NUMBER, and Variables = Comma Separated Variable (CSV) Strings. Each variable will have a description
# associated with it, read it carefully.
#
# The default configuration will work right out of the box however one item must be changed, your email address.
# I highly recommend you try out the default setup first and then make changes as desired. The only two changes
# I recommend is of course your email address, the second is the location of the 'statistical_data_file.cvs'.
#
# Pay attention to any changes you make, accidentally deleting a quote will cause the entire script to fail.
# Do not continue editing the script after the User Definable Section unless you know what you are doing.
#
# This script will not harm your drives. We are mostly only collecting drive status data. All file writes are
# to /tmp space. One exception: statistical_data_file.cvs is stored in /tmp by default however if you desire
# to maintain this data it must be stored in a dataset (user selected).
###### Email Address ######
# Enter your email address to send the report to. The from address does not need to be changed unless you experience
# an error sending the email. Some email servers only use the email address associated with the email server.
email="YourEmail@Address.com"
urgentemail="YourUrgentEmail@Address.com"
from="TrueNAS@local.com"
### Config File Name and Location ###
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
Config_File_Name="$SCRIPT_DIR/multi_report_config.txt"
###### Zpool Status Summary Table Settings
usedWarn=80 # Pool used percentage for CRITICAL color to be used.
scrubAgeWarn=37 # Maximum age (in days) of last pool scrub before CRITICAL color will be used (30 + 7 days for day of week). Default=37.
###### Temperature Settings
HDDtempWarn=45 # HDD Drive temp (in C) upper OK limit before a WARNING color/message will be used.
HDDtempCrit=50 # HDD Drive temp (in C) upper OK limit before a CRITICAL color/message will be used.
HDDmaxovrd="true" # HDD Max Drive Temp Override. This value when "true" will not alarm on any Current Power Cycle Max Temperature Limit.
SSDtempWarn=45 # SSD Drive temp (in C) upper OK limit before a WARNING color/message will be used.
SSDtempCrit=50 # SSD Drive temp (in C) upper OK limit before a CRITICAL color/message will be used.
SSDmaxovrd="true" # SSD Max Drive Temp Override. This value when "true" will not alarm on any Current Power Cycle Max Temperature Limit.
NVMtempWarn=50 # NVM Drive temp (in C) upper OK limit before a WARNING color/message will be used.
NVMtempCrit=60 # NVM Drive temp (in C) upper OK limit before a CRITICAL color/message will be used.
NVMmaxovrd="true" # NVM Max Drive Temp Override. This value when "true" will not alarm on any Current Power Cycle Max Temperature Limit.
# --- NOTE: NVMe drives currently do not report Min/Max temperatures so this is a future feature.
###### SSD/NVMe Specific Settings
wearLevelCrit=9 # Wear Level Alarm Setpoint lower OK limit before a WARNING color/message, 9% is the default.
###### General Settings
# Output Formats
powerTimeFormat="h" # Format for power-on hours string, valid options are "ymdh", "ymd", "ym", "y", or "h" (year month day hour).
tempdisplay="*C" # The format you desire the temperature to be displayed in. Common formats are: "*C", "^C", or "^c". Choose your own.
non_exist_value="---" # How do you desire non-existent data to be displayed. The Default is "---", popular options are "N/A" or " ".
pool_capacity="zfs" # Select "zfs" or "zpool" for Zpool Status Report - Pool Size and Free Space capacities. zfs is default.
# Ignore or Activate Alarms
ignoreUDMA="false" # Set to "true" to ignore all UltraDMA CRC Errors for the summary alarm (Email Header) only, errors will appear in the graphical chart.
ignoreSeekError="true" # Set to "true" to ignore all Seek Error Rate/Health errors. Default is true.
ignoreReadError="true" # Set to "true" to ignore all Seek Error Rate/Health errors. Default is true.
ignoreMultiZone="false" # Set to "true" to ignore all MultiZone Errors. Default is false.
disableWarranty="true" # Set to "true to disable email Subject line alerts for any expired warranty alert. The email body will still report the alert.
# Disable or Activate Input/Output File Settings
includeSSD="true" # Set to "true" will engage SSD Automatic Detection and Reporting, false = Disable SSD Automatic Detection and Reporting.
includeNVM="true" # Set to "true" will engage NVM Automatic Detection and Reporting, false = Disable NVM Automatic Detection and Reporting.
reportnonSMART="true" # Will force even non-SMART devices to be reported, "true" = normal operation to report non-SMART devices.
disableRAWdata="false" # Set to "true" to remove the smartctl -a data and non-smart data appended to the normal report. Default is false.
ata_auto_enable="false" # Set to "true" to automatically update Log Error count to only display a log error when a new one occurs.
# Media Alarms
sectorsWarn=1 # Number of sectors per drive to allow with errors before WARNING color/message will be used, this value should be less than sectorsCrit.
sectorsCrit=9 # Number of sectors per drive with errors before CRITICAL color/message will be used.
reAllocWarn=0 # Number of Reallocated sector events allowed. Over this amount is an alarm condition.
multiZoneWarn=0 # Number of MultiZone Errors to allow before a Warning color/message will be used. Default is 0.
multiZoneCrit=5 # Number of MultiZone Errors to allow before a Warning color/message will be used. Default is 5.
deviceRedFlag="true" # Set to "true" to have the Device Column indicate RED for ANY alarm condition. Default is true.
heliumAlarm="true" # Set to "true" to set for a critical alarm any He value below "heliumMin" value. Default is true.
heliumMin=100 # Set to 100 for a zero leak helium result. An alert will occur below this value.
rawReadWarn=5 # Number of read errors to allow before WARNING color/message will be used, this value should be less than rawReadCrit.
rawReadCrit=100 # Number of read errors to allow before CRITICAL color/message will be used.
seekErrorsWarn=5 # Number of seek errors to allow before WARNING color/message will be used, this value should be less than seekErrorsCrit.
seekErrorsCrit=100 # Number of seek errors to allow before CRITICAL color/message will be used.
# Time-Limited Error Recovery (TLER)
SCT_Drive_Enable="false" # Set to "true" to send a command to enable SCT on your drives for user defined timeout.
SCT_Warning="TLER_No_Msg" # Set to "all" will generate a Warning Message for all devices not reporting SCT enabled. "TLER" reports only drive which support TLER.
# "TLER_No_Msg" will only report for TLER drives and not report a Warning Message if the drive can set TLER on.
SCT_Read_Timeout=70 # Set to the read threshold. Default = 70 = 7.0 seconds.
SCT_Write_Timeout=70 # Set to the write threshold. Default = 70 = 7.0 seconds.
# SMART Testing Alarm
testAgeWarn=2 # Maximum age (in days) of last SMART test before CRITICAL color/message will be used.
# Zpool Fragmentation Alarm
zpoolFragWarn=80 # Percent of fragmentation before a Warning message occurs.
###### Statistical Data File
statistical_data_file="$SCRIPT_DIR/statisticalsmartdata.csv" # Default location is where the script is located.
expDataEnable="true" # Set to "true" will save all drive data into a CSV file defined by "statistical_data_file" below.
expDataEmail="true" # Set to "true" to have an attachment of the file emailed to you. Default is true.
expDataPurge=730 # Set to the number of day you wish to keep in the data. Older data will be purged. Default is 730 days (2 years). 0=Disable.
expDataEmailSend="Mon" # Set to the day of the week the statistical report is emailed. (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month)
###### FreeNAS config backup settings
configBackup="true" # Set to "true" to save config backup (which renders next two options operational); "false" to keep disable config backups.
configSendDay="Mon" # Set to the day of the week the config is emailed. (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month)
saveBackup="false" # Set to "false" to delete FreeNAS config backup after mail is sent; "true" to keep it in dir below.
backupLocation="/tmp/" # Directory in which to store the backup FreeNAS config files.
###### Attach multi_report_config.txt to Email ######
Config_Email_Enable="true" # Set to "true" to enable periodic email (which renders next two options operational).
Config_Changed_Email="true" # If "true" will attach the updated/changed file to the email.
Config_Backup_Day="Mon" # Set to the day of the week the multi_report_config.txt is emailed. (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month, Never)
########## REPORT CHART CONFIGURATION ##############
###### REPORT HEADER TITLE ######
HDDreportTitle="Spinning Rust Summary Report" # This is the title of the HDD report, change as you desire.
SSDreportTitle="SSD Summary Report" # This is the title of the SSD report, change as you desire.
NVMreportTitle="NVMe Summary Report" # This is the title of the NVMe report, change as you desire.
### CUSTOM REPORT CONFIGURATION ###
# By default most items are selected. Change the item to false to have it not displayed in the graph, true to have it displayed.
# NOTE: Alarm setpoints are not affected by these settings, this is only what columns of data are to be displayed on the graph.
# I would recommend that you remove columns of data that you don't really care about to make the graph less busy.
# For Zpool Status Summary
Zpool_Pool_Name_Title="Pool Name"
Zpool_Status_Title="Status"
Zpool_Pool_Size_Title="Pool Size"
Zpool_Free_Space_Title="Free Space"
Zpool_Used_Space_Title="Used Space"
Zfs_Pool_Size_Title="^Pool Size"
Zfs_Free_Space_Title="^Free Space"
Zfs_Used_Space_Title="^Used Space"
Zpool_Frag_Title="Frag"
Zpool_Read_Errors_Title="Read Errors"
Zpool_Write_Errors_Title="Write Errors"
Zpool_Checksum_Errors_Title="Cksum Errors"
Zpool_Scrub_Repaired_Title="Scrub Repaired Bytes"
Zpool_Scrub_Errors_Title="Scrub Errors"
Zpool_Scrub_Age_Title="Last Scrub Age"
Zpool_Scrub_Duration_Title="Last Scrub Duration"
# For Hard Drive Section
HDD_Device_ID="true"
HDD_Device_ID_Title="Device ID"
HDD_Serial_Number="true"
HDD_Serial_Number_Title="Serial Number"
HDD_Model_Number="true"
HDD_Model_Number_Title="Model Number"
HDD_Capacity="true"
HDD_Capacity_Title="HDD Capacity"
HDD_Rotational_Rate="true"
HDD_Rotational_Rate_Title="RPM"
HDD_SMART_Status="true"
HDD_SMART_Status_Title="SMART Status"
HDD_Warranty_Title="Warr- anty"
HDD_Warranty="true"
HDD_Raw_Read_Error_Rate="true"
HDD_Raw_Read_Error_Rate_Title="Raw Error Rate"
HDD_Drive_Temp="true"
HDD_Drive_Temp_Title="Curr Temp"
HDD_Drive_Temp_Min="true"
HDD_Drive_Temp_Min_Title="Temp Min"
HDD_Drive_Temp_Max="true"
HDD_Drive_Temp_Max_Title="Temp Max"
HDD_Power_On_Hours="true"
HDD_Power_On_Hours_Title="Power On Time"
HDD_Start_Stop_Count="true"
HDD_Start_Stop_Count_Title="Start Stop Count"
HDD_Load_Cycle="true"
HDD_Load_Cycle_Title="Load Cycle Count"
HDD_Spin_Retry="true"
HDD_Spin_Retry_Title="Spin Retry Count"
HDD_Reallocated_Sectors="true"
HDD_Reallocated_Sectors_Title="Re-alloc Sects"
HDD_Reallocated_Events="true"
HDD_Reallocated_Events_Title="Re-alloc Evnt"
HDD_Pending_Sectors="true"
HDD_Pending_Sectors_Title="Curr Pend Sects"
HDD_Offline_Uncorrectable="true"
HDD_Offline_Uncorrectable_Title="Offl Unc Sects"
HDD_UDMA_CRC_Errors="true"
HDD_UDMA_CRC_Errors_Title="UDMA CRC Error"
HDD_Seek_Error_Rate="true"
HDD_Seek_Error_Rate_Title="Seek Error Rate"
HDD_MultiZone_Errors="true"
HDD_MultiZone_Errors_Title="Multi Zone Error"
HDD_Helium_Level="true"
HDD_Helium_Level_Title="He Level"
HDD_Last_Test_Age="true"
HDD_Last_Test_Age_Title="Last Test Age"
HDD_Last_Test_Type="true"
HDD_Last_Test_Type_Title="Last Test Type"
# For Solid State Drive Section
SSD_Device_ID="true"
SSD_Device_ID_Title="Device ID"
SSD_Serial_Number="true"
SSD_Serial_Number_Title="Serial Number"
SSD_Model_Number="true"
SSD_Model_Number_Title="Model Number"
SSD_Capacity="true"
SSD_Capacity_Title="HDD Capacity"
SSD_SMART_Status="true"
SSD_SMART_Status_Title="SMART Status"
SSD_Warranty_Title="Warr- anty"
SSD_Warranty="true"
SSD_Drive_Temp="true"
SSD_Drive_Temp_Title="Curr Temp"
SSD_Drive_Temp_Min="true"
SSD_Drive_Temp_Min_Title="Temp Min"
SSD_Drive_Temp_Max="true"
SSD_Drive_Temp_Max_Title="Temp Max"
SSD_Power_On_Hours="true"
SSD_Power_On_Hours_Title="Power On Time"
SSD_Wear_Level="true"
SSD_Wear_Level_Title="Wear Level"
SSD_Reallocated_Sectors="true"
SSD_Reallocated_Sectors_Title="Re-alloc Sects"
SSD_Reallocated_Events="true"
SSD_Reallocated_Events_Title="Re-alloc Evnt"
SSD_Pending_Sectors="true"
SSD_Pending_Sectors_Title="Curr Pend Sects"
SSD_Offline_Uncorrectable="true"
SSD_Offline_Uncorrectable_Title="Offl Unc Sects"
SSD_UDMA_CRC_Errors="true"
SSD_UDMA_CRC_Errors_Title="UDMA CRC Error"
SSD_Last_Test_Age="true"
SSD_Last_Test_Age_Title="Last Test Age"
SSD_Last_Test_Type="true"
SSD_Last_Test_Type_Title="Last Test Type"
# For NVMe Drive Section
NVM_Device_ID="true"
NVM_Device_ID_Title="Device ID"
NVM_Serial_Number="true"
NVM_Serial_Number_Title="Serial Number"
NVM_Model_Number="true"
NVM_Model_Number_Title="Model Number"
NVM_Capacity="true"
NVM_Capacity_Title="HDD Capacity"
NVM_SMART_Status="true"
NVM_SMART_Status_Title="SMART Status"
NVM_Warranty_Title="Warr- anty"
NVM_Warranty="true"
NVM_Critical_Warning="true"
NVM_Critical_Warning_Title="Critical Warning"
NVM_Drive_Temp="true"
NVM_Drive_Temp_Title="Curr Temp"
NVM_Drive_Temp_Min="false" # I have not found this on an NVMe drive yet, so set to false
NVM_Drive_Temp_Min_Title="Temp Min"
NVM_Drive_Temp_Max="false" # I have not found this on an NVMe drive yet, so set to false
NVM_Drive_Temp_Max_Title="Temp Max"
NVM_Power_On_Hours="true"
NVM_Power_On_Hours_Title="Power On Time"
NVM_Wear_Level="true"
NVM_Wear_Level_Title="Wear Level"
###### Mouseover
# This will display the original value of an overriden value (one in yellow)
# This is a tri-state value as explained below.
# "true" will displaying the actual value via a mouseover html link. Still working to make this a true Mouseover.
# "false" will not generate any special chart and will run as previous versions.
# "alt" will place the actual value within parentheses. An option if your email cliet can't display mouseover.
Mouseover="alt"
###### Drive Ignore List
# What does it do:
# Use this to list any drives to ignore and remove from the report. This is very useful for ignoring USB Flash Drives
# or other drives for which good data is not able to be collected (non-standard).
#
# How to use it:
# We are using a comma delimited file to identify the drive serial numbers. You MUST use the exact and full serial
# number smartctl reports, if there is no identical match then it will not match. Additionally you may list drives
# from other systems and they will not have any effect on a system where the drive does not exist. This is great
# to have one configuration file that can be used on several systems.
#
# Live Example: Ignore_Drives="VMWare,1JUMLBD,21HNSAFC21410E"
Ignore_Drives="none"
###### Drive UDMA_CRC_Error_Count List
# What does it do:
# If you have a drive which has an UDMA count other than 0 (zero), this setting will offset the
# value back to zero for the concerns of monitoring future increases of this specific error. Any match will
# subtract the given value to report a 0 (zero) value and highlight it in yellow to denote it was overridden.
# The Warning Title will not be flagged if this is zero'd out in this manner.
# NOTE: UDMA_CRC_Errors are typically permanently stored in the drive and cannot be reset to zero even though
# they are frequently caused by a data cable communications error.
#
# How to use it:
# List each drive by serial number and include the current UDMA_CRC_Error_Count value.
# The format is very specific and will not work if you wing it, use the Live Example.
#
# Set the FLAG in the FLAGS Section ignoreUDMA to false (the default setting).
#
# If the error count exceeds the limit minus the offset then a warning message will be generated.
# On the Status Report the UDMA CRC Errors block will be YELLOW with a value of 0 for an overridden value.
# -- NOTE: We are using the colon : as the separator between the drive serial number and the value to change.
#
# Format: variable=Drive_Serial_Number:Current_UDMA_Error_Count and add a comma if you have more than one drive.
#
# The below example shows drive WD-WMC4N2578099 has 1 UDMA_CRC_Error, drive S2X1J90CA48799 has 2 errors.
#
# Live Example: "WD-WMC4N2578099:1,S2X1J90CA48799:2,P02618119268:1"
CRC_Errors="none"
###### Multi_Zone_Errors List
# What does it do:
# This identifies drives with Multi_Zone_Errors which may be irritating people.
# Multi_Zone_Errors for some drives, not all drives are pretty much meaningless.
#
# How to use it:
# Use same format as CRC_Errors (see above).
Multi_Zone="none"
####### Reallocated Sectors Exceptions
# What does it do:
# This will offset any Reallocated Sectors count by the value provided.
#
# I do not recommend using this feature as I'm a believer in if you have over 5 bad sectors, odds are the drive will get worse.
# I'd recommend replacing the drive before complete failure. But that is your decision.
#
# Why is it even an option?
# I use it for testing purposes only but you may want to use it.
#
# How to use it:
# Use same format as CRC_Errors (see above).
Bad_Sectors="none"
Bad_Sectors2="none"
######## ATA Error Log Silencing ##################
# What does it do:
# This will ignore error log messages equal to or less than the threshold.
# How to use:
# Same as the CRC_Errors, [drive serial number:error count]
ata_errors="none"
####### Custom Drive Configuration (Experimental)
# Used to define specific alarm values for specific drives by serial number.
# This should only be used for drives where the default alarm settings
# are not proper. Up to 24 unique drive values may be stored.
#
# Use -config to set these values.
Custom_Drives=""
####### Warranty Expiration Date
# What does it do:
# This section is used to add warranty expirations for designated drives and to create an alert when they expire.
# The date format is YYYY-MM-DD.
#
# Below is an example for the format using my own drives, which yes, are expired.
# As previously stated above, drive serial numbers must be an exact match to what smartctl reports to function.
#
# If the drive does not exist, for example my drives are not on your system, then nothing will happen.
#
# How to use it:
# Use the format ="Drive_Serial_Number:YYYY-MM-DD" and add a comma if you have more than one drive.
Drive_Warranty="none"
######## Expired Drive Warranty Setup
expiredWarrantyBoxColor="#000000" # "#000000" = normal box perimeter color.
WarrantyBoxPixels="1" # Box line thickness. 1 = normal, 2 = thick, 3 = Very Thick, used for expired drives only.
WarrantyBackgndColor="#f1ffad" # Hex code or "none" = normal background, Only for expired drives.
######## Enable-Disable Text Portion ########
enable_text="true" # This will display the Text Section when = "true" or remove it when not "true". Default="true"
###### Global table of colors
# The colors selected you can change but you will need to look up the proper HEX code for a color.
okColor="#b5fcb9" # Hex code for color to use in SMART Status column if drives pass (default is darker light green, #b5fcb9).
warnColor="#F38B16" # Hex code for WARN color (default is orange, #F38B16).
critColor="#f44336" # Hex code for CRITICAL color (default is red, #f44336).
altColor="#f4f4f4" # Table background alternates row colors between white and this color (default is light gray, #f4f4f4).
whtColor="#ffffff" # Hex for White background.
ovrdColor="#ffffe4" # Hex code for Override Yellow.
blueColor="#87ceeb" # Hex code for Sky Blue, used for the SCRUB In Progress background.
yellowColor="#f1ffad" # Hex code for pale yellow.
##########################
##########################
### ###
### STOP EDITING THE ###
### SCRIPT HERE ###
### ###
##########################
##########################
###### Auto-generated Parameters
softver=$(uname -s)
host=$(hostname -s)
truenas_ver=$(cat /etc/version)
testdata_path="data"
### temp files have been converted to variable stored, not stored in /tmp/ as a file. ###
logfile="/tmp/smart_report_body.tmp"
logfile_header="/tmp/smart_report_header.tmp"
logfile_warning="/tmp/smart_report_warning_flag.tmp"
logfile_critical="/tmp/smart_report_critical_flag.tmp"
logfile_warranty_temp="/tmp/smart_report_warranty_flag.tmp"
logfile_messages_temp="/tmp/smart_report_messages.tmp"
boundary="gc0p4Jq0M2Yt08jU534c0p"
progname="Multi-Report v2.0.8 dtd:"
progverdate="2023-02-20"
if [[ $softver != "Linux" ]]; then
if [[ "$(cat /etc/version | grep "FreeNAS")" ]]; then
programver=progname$progverdate" (FreeNAS "$(cat /etc/version | cut -d " " -f1 | sed 's/FreeNAS-//')")"
programver2="$(cat /etc/version | cut -d"-" -f1)"
else
programver=$progname$progverdate" (TrueNAS Core "$(cat /etc/version | cut -d " " -f1 | sed 's/TrueNAS-//')")"
programver2="$(cat /etc/version | cut -d"-" -f1)"
fi
else
programver=$progname$progverdate" (TrueNAS Scale "$(cat /etc/version)")"
programver2="$(cat /etc/version | cut -d" " -f1)"
fi
# If the config file format changes, this is the latest working date, anything older must be updated.
# Format must be "yyyy-mm-dd"
valid_config_version_date="2023-02-02"
##########################
##########################
### ###
### PROGRAMING / ###
### TROUBLESHOOTING ###
### HACKS ###
### ###
##########################
##########################
# Unique programming hacks to properly emulate other hardware that is not actually on the system.
VMWareNVME="on" # Set to "off" normally, "on" to assist in incorrect VMWare reporting.
Joes_System="false" # Custom settings for my system and to remove these from your system.
Sample_Test="false" # Setup static test values for testing.
extra_data="false" # Collect extra 'json' data. This is a temporary thing.
##########################
##########################
### ###
### DEFINE FUNCTIONS ###
### ###
##########################
##########################
############## CREATE TESTDATA TEXT FILE ################
create_testdata_text_file () {
echo "Creating testdata variable"
testdata_a="$(ls $testdata_path/*.a)"
testdata_x="$(ls $testdata_path/*.x)"
echo "testdata_a="$testdata_a
echo " "
echo "testdata_x="$testdata_x
}
############## LOAD EXTERNAL CONFIGURATION FILE #############
load_config () {
if test -e "$Config_File_Name"; then
. "$Config_File_Name"
# Lets test if the config file needs to be updated first.
config_version_date="$(cat "$Config_File_Name" | grep "dtd" | cut -d ':' -f 2 | cut -d ' ' -f 1 )"
echo "Configuration File Version Date: "$config_version_date
if [[ $config_version_date < $valid_config_version_date ]]; then echo "Found Old Configuration File"; echo "Automatically updating configuration file..."; update_config_file; echo "Running normal script"; fi
. "$Config_File_Name"
else
echo " "
echo " No Config File Exists --- Checking for a valid email within the script..."
if [[ $email == "YourEmail@Address.com" ]]; then
echo " "
echo " No Valid Email Address..."
echo " Recommend running script with the '-config' switch and selecting"
echo " the N)ew Configuration option."
echo " "
echo "... Exiting"
echo " "
exit 1
else
echo "Valid email within the script = "$email", using script parameters..."
echo " "
External_Config="no"
return
fi
fi
}
#################### Convert to Decimal ##################
convert_to_decimal () {
if [[ "$1" == "" ]]; then return; fi
Converting_Value=${1#0}
#echo "Step 1 -> Converting "$1" to BASE 10 = "$Converting_Value
Converting_Value="${Converting_Value//,}"
#echo "Step 2 -> Removing commas from "$1" to "$Converting_Value
Return_Value=$Converting_Value
if [[ $1 == "0" ]]; then Return_Value=0; fi
#echo "Return_Value="$Return_Value
}
#################### Force Slight Delay ####################
# I think there is a race condition when writing to $logfile, trying to slow this down.
force_delay () {
sleep .1
}
#################### PURGE EXPORT DATA CSV FILE #######################
purge_exportdata () {
### This routine will purge the "statistical_data_file" of data older then "expDataPurge".
# Delete temp file if it exists
if test -e "/tmp/temp_purge_file.csv"; then
rm "/tmp/temp_purge_file.csv"
# Create the header
# printf "Date,Time,Device ID,Drive Type,Serial Number,SMART Status,Temp,Power On Hours,Wear Level,Start Stop Count,Load Cycle,Spin Retry,Reallocated Sectors,\
#ReAllocated Sector Events,Pending Sectors,Offline Uncorrectable,UDMA CRC Errors,Seek Error Rate,Multi Zone Errors,Read Error Rate,Helium Level\n" > "/tmp/temp_purge_file.csv"
fi
if [ $softver != "Linux" ]; then
expireDate=$(date -v -"$expDataPurge"d +%Y/%m/%d)
else
expireDate=$(date -d "$expDataPurge days ago" +%Y/%m/%d)
fi
awk -v expireDate="$expireDate" -F, '{ if($1 >= expireDate) print $0;}' "$statistical_data_file" > "/tmp/temp_purge_file.csv"
cp -R "/tmp/temp_purge_file.csv" "$statistical_data_file"
}
###### Purge ada50 and nvme50 from the logs #######
purge_testdata () {
### This routine will purge the "statistical_data_file" of test data matching "ada50" or "nvme50".
# Delete temp file if it exists
if test -e "/tmp/temp_purge_file.csv"; then
rm "/tmp/temp_purge_file.csv"
fi
awk -F, '{ if($3 != "nvme50") print $0;}' "$statistical_data_file" > "/tmp/temp_purge_file1.csv"
awk -F, '{ if($3 != "ada50") print $0;}' "/tmp/temp_purge_file1.csv" > "/tmp/temp_purge_file.csv"
cp -R "/tmp/temp_purge_file.csv" "$statistical_data_file"
}
################## EMAIL EXPORT DATA CVS FILE #########################
email_datafile () {
if [ "$expDataEmail" == "true" ]; then
Now=$(date +"%a")
doit="false"
case $expDataEmailSend in
All)
doit="true"
;;
Mon|Tue|Wed|Thu|Fri|Sat|Sun)
if [[ "$expDataEmailSend" == "$Now" ]]; then doit="true"; fi
;;
Month)
if [[ $(date +"%d") == "01" ]]; then doit="true"; fi
;;
*)
;;
esac
if [[ "$doit" == "true" ]]; then
(
# Write MIME section header for file attachment (encoded with base64)
echo "--${boundary}"
echo "Content-Type: text/csv"
echo "Content-Transfer-Encoding: base64"
echo "Content-Disposition: attachment; filename=Statistical_Data.csv"
base64 "$statistical_data_file"
) >> "$logfile"
force_delay
doit=""
fi
fi
# This file does not get attached to a -dump file. Joe does not need nor want this file.
if [[ "$Config_Email_Enable" == "true" ]]; then
if [[ "$Attach_Config" == "1" ]]; then doit="true"; fi
Now=$(date +"%a")
case $Config_Backup_Day in
All)
doit="true"
;;
Mon|Tue|Wed|Thu|Fri|Sat|Sun)
if [[ "$Config_Backup_Day" == "$Now" ]]; then doit="true"; fi
;;
Month)
if [[ $(date +"%d") == "01" ]]; then doit="true"; fi
;;
Never)
;;
*)
;;
esac
if [[ "$doit" == "true" ]] || [[ "$dump_all" != "0" ]]; then
#echo "Attching Multi_Report Config"
(
# Write MIME section header for file attachment (encoded with base64)
echo "--${boundary}"
echo "Content-Type: text/html"
echo "Content-Transfer-Encoding: base64"
echo "Content-Disposition: attachment; filename=multi_report_config.txt"
base64 $Config_File_Name
# if [[ "$dump_all" != "0" ]]; then echo "--${boundary}"; fi
# If we are not dumping data, end the boundry to attach files.
if [[ "$dump_all" != "0" ]]; then echo "--${boundary}"; else echo "--${boundary}--"; fi
) >> "$logfile"
force_delay
fi
fi
}
##################### IGNORE DRIVES ROUTINE #################
process_ignore_drives () {
targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')";
### Process Ignore List ###
s="0"
IFS=',' read -ra ADDR <<< "$Ignore_Drives"
for i in "${ADDR[@]}"; do
if [[ $i == $targument ]]; then s="1"; continue; fi
done
if [[ $s == "0" ]]; then printf "%s " "${drive}"; fi
}
################## SORT DRIVES ROUTINE ########################
sort_drives () {
#echo "Incoming Drive List="$sort_list
sort_list=$(for i in `echo $sort_list`; do
echo "$i"
done | sort -V)
#echo "Outgoing Drive List="$sort_list
}
#################### GET SMART HARD DRIVES ############################
get_smartHDD_listings () {
# variable smartdrives
if [[ "$testfile" != "" ]]; then
echo "HDD TEST FILE ROUTINE"
smartdrives="ada50"
return
fi
if [ $softver != "Linux" ]; then
smartdrives=$(for drive in $(sysctl -n kern.disks); do
if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi
done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
else
smartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do
if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi
done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
fi
# Call Sort Routine with the drive string.
if [[ "$smartdrives" != "" ]]; then
sort_list=$smartdrives
sort_drives
smartdrives=$sort_list
fi
}
########################## GET SMART SOLID DISK DRIVES ################################
get_smartSSD_listings () {
# variable smartdrivesSSD
if [[ "$testfile" != "" ]]; then
echo "SSD TEST FILE ROUTINE"
smartdrivesSSD="ada50"
return
fi
if [ $softver != "Linux" ]; then
smartdrivesSSD=$(for drive in $(sysctl -n kern.disks); do
if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi
done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
else
smartdrivesSSD=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do
if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi
done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
fi
# Call Sort Routine with the drive string.
if [[ "$smartdrivesSSD" != "" ]]; then
sort_list=$smartdrivesSSD
sort_drives
smartdrivesSSD=$sort_list
fi
}
########################## GET NVMe DRIVES ################################
get_smartNVM_listings () {
# variable smartdrivesNVM
if [[ "$testfile" != "" ]]; then
echo "NVM TEST FILE ROUTINE"
smartdrivesNVM="nvme50"
return
fi
if [ $softver != "Linux" ]; then
smartdrivesNVM=$(for drive in $(sysctl -n kern.disks); do
if [ "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi
done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
else
smartdrivesNVM=$(for drive in $(fdisk -l | grep "Disk /dev/nvm" | cut -d ':' -f 1 | cut -d '/' -f 3 | tr '\n' ' '); do
if [ "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi
done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
fi
### Convert nvdx to nvmexx in smartdrivesNVM ###
smartdrivesNVM=$( echo "$smartdrivesNVM" | sed 's/nvd/nvme/g' )
# Call Sort Routine with the drive string.
if [[ "$smartdrivesNVM" != "" ]]; then
sort_list=$smartdrivesNVM
sort_drives
smartdrivesNVM=$sort_list
fi
}
########################## GET OTHER SMART DEVICES ##################################
get_smartOther_listings () {
### Get the non-SSD listing - MUST support SMART
# variable nonsmartdrives
if [[ "$testfile" != "" ]]; then
echo "TEST FILE ROUTINE"
nonsmartdrives="ada50"
return
fi
if [ $softver != "Linux" ]; then
nonsmartdrives=$(for drive in $(sysctl -n kern.disks); do
if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ ! "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi
done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
else
nonsmartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do
if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is: Enabled")" ]; then process_ignore_drives; fi
done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ')
fi
# Call Sort Routine with the drive string.
if [[ "$nonsmartdrives" != "" ]]; then
sort_list=$nonsmartdrives
sort_drives
nonsmartdrives=$sort_list
fi
}
########################### FORMAT EMAILS STEP 1 ##########################
email_preformat () {
###### Email pre-formatting
### Set some of the email headers before conditional headers
(
echo "MIME-Version: 1.0"
echo "Content-Type: multipart/mixed; boundary=${boundary}"
) > "$logfile"
force_delay
}
########################### CONFIGURATION BACKUP ##############################
config_backup () {
###### Config backup (if enabled)
tarfile="/tmp/config_backup.tar.gz"
filename="$(date "+"$programver2"_Config_%Y-%m-%d")"
filename2="Stat_Data"
if [ "$configBackup" == "true" ]; then
Now=$(date +"%a")
doit="false"
case $configSendDay in
All)
doit="true"
;;
Mon|Tue|Wed|Thu|Fri|Sat|Sun)
if [[ "$configSendDay" == "$Now" ]]; then
doit="true"
fi
;;
Month)
if [[ $(date +"%d") == "01" ]]; then
doit="true"
fi
;;
*)
;;
esac
if [[ "$doit" == "true" ]]; then
# Set up file names, etc for later
tarfile="/tmp/config_backup.tar.gz"
filename="$(date "+"$programver2"_Config_%Y-%m-%d")"
filename2="Stat_Data"
### Test config integrity
if ! [ "$(sqlite3 /data/freenas-v1.db "pragma integrity_check;")" == "ok" ]; then
# Config integrity check failed, set MIME content type to html and print warning
(
echo "--${boundary}"
echo "Content-Type: text/html"
echo "Automatic backup of FreeNAS configuration has failed! The configuration file is corrupted!"
echo "You should correct this problem as soon as possible!"
echo "
"
) >> "$logfile"
force_delay
else
# Config integrity check passed; copy config db, generate checksums, make .tar.gz archive
cp /data/freenas-v1.db "/tmp/${filename}.db"
if [ $softver != "Linux" ]; then
md5 "/tmp/${filename}.db" > /tmp/config_backup.md5
sha256 "/tmp/${filename}.db" > /tmp/config_backup.sha256
else
md5sum "/tmp/${filename}.db" > /tmp/config_backup.md5
sha256sum "/tmp/${filename}.db" > /tmp/config_backup.sha256
fi
(
cd "/tmp/" || exit;
tar -czf "${tarfile}" "./${filename}.db" ./config_backup.md5 ./config_backup.sha256;
)
(
# Write MIME section header for file attachment (encoded with base64)
echo "--${boundary}"
echo "Content-Type: application/tar+gzip"
echo "Content-Transfer-Encoding: base64"
echo "Content-Disposition: attachment; filename=${filename}.tar.gz"
base64 "$tarfile"
# Write MIME section header for html content to come below
echo "--${boundary}"
echo "Content-Type: text/html"
) >> "$logfile"
force_delay
# If logfile saving is enabled, copy .tar.gz file to specified location before it (and everything else) is removed below
if [ "$saveBackup" == "true" ]; then
cp "${tarfile}" "${backupLocation}/${filename}.tar.gz"
fi
rm "/tmp/${filename}.db"
rm /tmp/config_backup.md5
rm /tmp/config_backup.sha256
rm "${tarfile}"
fi
else
(
echo "--${boundary}"
echo "Content-Type: text/html"
) >> "$logfile"
force_delay
fi
else
# configBackup = false so this is what to do
# Config backup enabled; set up for html-type content
(
echo "--${boundary}"
echo "Content-Type: text/html"
) >> "$logfile"
force_delay
fi
}
################################## GENERATE ZPOOL REPORT ##################################
zpool_report () {
###### Report Summary Section (html tables)
if [[ "$Urgent" == "false" ]]; then # For Urgent
(
# Write HTML table headers to log file; HTML in an email requires 100% in-line styling (no CSS or