#!/bin/bash # With version 1.6c and above you may use an external configuration file. # Use [-h] to read the Help Section. # Use [-config] to create a configuration file in the directory this script is run from. ###### ZPool & SMART status report with FreeNAS/TrueNAS config backup ### Original script by joeschmuck, modified by Bidelu0hm, then by melp (me) ### Version: v1.3 TrueNAS Scale (Jeff Alperin 12-6-21) ### Version v1.4, v1.5, v1.6 FreeNAS/TrueNAS (joeschmuck) ### Changelog: # v1.6d (01 October 2022) # - Thanks goes out to ChrisRJ for offering some great suggestions to enhance and optimize the script. # - Updated gptid text and help text areas (clarifying information) # - Updated the -dump parameter to -dump [all] and included non-SMART attachments. # - Added Automatic UDMA_CRC, MultiZone, and Reallocated Sector Compensation to -config advanced option K. # - Fixed Warranty Date always showing as expired. # - Added Helium and Raw Read Error Rates to statistical data file. # - Added Raw Read Error Rates chart column. # - Added compensation for Seagate Seek Error Rates and Raw Read Error Rates. # - Added Automatic Configuration File Update feature. # - Added selection between ZFS Pool Size or Zpool Pool Size. ZFS is representative of the actual storage capacity # -- and updated the Pool Status Report Summary chart. # - Added ATA Error Log Silencing (by special request). # - Added 0.1 second delay after writing "$logfile" to eliminate intermittent file creation errors. # - Fixed Text Report -> Drive Model Number not showing up for some drives. # -- Future Work # ---- Change all the -config dialog to be consistent. # ---- Optimizing Code # # # v1.6c (28 August 2022) # - Supports external configuration file (but not required). # - Completely Configurable by running the script -config parameter (this took a lot of work). # - Added HDD/SSDmaxtempovrd variables to combat some bogus SSD values. # - Added TLER (SCT) support. # - Added support for drives which do not support recording over 65536 hours for SMART Tests and rolls over to start at zero again. # - Added -dump parameter to create and email all of the drives smartctl outputs as text file email attachments. # - Added support for Helium drives. # # # v1.6: (05 August 2022) # Thanks to Jeff, Simon, and Sean for providing me more test data than I could shake a stick at and friendly constructive opinions/advice. # - Complete rewrite of the script. More organized and easier for future updates. # - Almost completely got rid of using AWK, earlier versions had way too much programming within the AWK structure. # - Reads the drives much less often (3 times each I believe). # - Added test input file to parse txt files of smartctl -a output. This will allow for a single drive entry and ability # -- for myself or any script writer to identify additional parameters for unrecognized drives. # -- Usage: program_name.sh [HDD|SSD|NVM] [inputfile.txt] # - Added better support for SAS drives. # - Fixed NVMe and SAS Power On Hours for statistical data recording, and other things. # - Added Critical and Warning Logs to email output with better descriptive data. # - Logs (stored in /tmp/) no longer deleted after execution to aid in troubleshooting, but deleted at the start of the script. # - Added HELP file, use program_name.sh [-h] [-help] # - Added SCT Error Recovery to the Text Report section. # - Added Zpool Size, Free Space, and Temp Min/Max. # - Added customizable temperature values and customizable Non-Value fields (use to be coded to "N/A"). # - Added support for SandForce SSD. # # v1.5: # - Added NVMe support # - Added clearer error reporting for Warning and Critical errors. # - Known Problems: The NVMe Power On Time has a comma that I can't get rid of, yet. I want to remove the comma when the data is retrieved. # -- NVMe's are not all standardized so I expect tweaks as different drive data arrives. # -- onHours that includes a comma will not record correctly in the statistical data file. This is related to the NVMe problem above. # -- Zpool Summary does not indicate Scrub Age warning, likely the entire summary has issues. # # # v1.4d: # - Fixed Scrub In Progress nuisance error when a scrub is in progress. # - Added offsetting Reallocated Sectors for four drives. This should be for testing only. Any drives # -- with a significant number of bad sectors should be replaced, just my opinion. # - Added Drive Warranty Expiration warning messages and ability to disable the Email Subject line warning. # -- NOT TESTED ON OTHER THAN U.S. FORMATTED DATE YYYY-MM-DD. # - Added HDD and SSD individual temperature settings. # - Changed order of polling Temperature data from HDD/SSD. # # v1.4c: # - Rewrite to create functions and enable easier editing. # - Added Custom Reports. # - Added disabling the RAW 'smartctl -a' data appended to the end of the email. # - Added sorting drives alphabetically vice the default the OS reports them. # - Added RED warning in Device for any single failure in the summary (deviceRedFlag switch controlled). # - Added some additional SSD definitions. # - Fixed sorting last two SMART Tests, now reports them in proper order. # - Fixed detecting "SMART Support is: Enabled", for white spaces. # - Changed IGNORE DRIVES to a String Format to clean up and simplify programming. # - Added MultiZone_Errors support for up to eight drives. # - Added sectorWarn variable to complement the sectorCrit variable. # - Added ignoreSeekError variable to ignore some of those wild Seek Error Rate values. # - Added ignoreUDMA CRC Errors due to the "Known Problem" # - Fixed md5/sha256 error on TrueNAS Scale (only used during config backups). # - Added selectable config backup periodicity by day vice every run. # - Added Exporting statistical data for trend analysis. # -- Can be setup to email statistics weekly, monthly, or not at all. # -- The -s switch will run Data Collection Only, no email generated. Note: Do Not run two instances at once, the temp files do not survive. # - Fixed the Capacity to remove the brackets "[]", thanks Jeff Alperin. # - Fixed Scrub Age failure due to 1 day or longer repair time, now shows anything >24 hours. # # - Known Problem: One user reported UDMA_CRC_Errors is not subtracting correctly, have not been able to personally replicate it. # -- This error seems to occur around line #1027 # # v1.4b: # - Added SMART test remaining percentage if Last Test has a SMART Test is in progress. # - Fix for empty SMART fields, typically for unsupported SSD's. # - Added IGNORE SMART Drive so you can ignore specific drives that may cause you weird readings. # --- Updated so blank SSD table header is removed when you ignore all the drives (just crazy talk). # v1.4a: # - Fixed report errors for if a SCRUB is in progress, now shows estimated completion time. # - Fixed report error for a Canceled SCRUB. # - Fixed FreeBSD/Linux use for SCRUB report (minor oversight). # v1.4: # - Run on CRON JOB using /path/multi_report_v1.4.sh # - Fixed for automatic running between FreeBSD and Linux Debian (aka SCALE) as of this date. # - All SMART Devices will report. # - Added conditional Subject Line (Good/Critical/Warning). # - Added Automatic SSD Support. # --- Some updates may need to be made to fit some of SSD's. Code in the area of about line 530 will # --- need to be adjusted to add new attributes for the desired SSD's fields. # - UDMA_CRC_ERROR Override because once a drive encounters this type of error, it cannot be cleared # --- so you can offset it now vice having an alarm condition for old UDMA_CRC_Errors. # - Added listing NON-SMART Supported Drives. Use only if useful to you, some drives will # --- still output some relevant data, many will not. # v1.3: # - Added scrub duration column # - Fixed for FreeNAS 11.1 (thanks reven!) # - Fixed fields parsed out of zpool status # - Buffered zpool status to reduce calls to script # v1.2: # - Added switch for power-on time format # - Slimmed down table columns # - Fixed some shellcheck errors & other misc stuff # - Added .tar.gz to backup file attached to email # - (Still coming) Better SSD SMART support # v1.1: # - Config backup now attached to report email # - Added option to turn off config backup # - Added option to save backup configs in a specified directory # - Power-on hours in SMART summary table now listed as YY-MM-DD-HH # - Changed filename of config backup to exclude timestamp (just uses datestamp now) # - Config backup and checksum files now zipped (was just .tar before; now .tar.gz) # - Fixed degrees symbol in SMART table (rendered weird for a lot of people); replaced with a * # - Added switch to enable or disable SSDs in SMART table (SSD reporting still needs work) # - Added most recent Extended & Short SMART tests in drive details section (only listed one before, whichever was more recent) # - Reformatted user-definable parameters section # - Added more general comments to code # v1.0: # - Initial release ######### INSTRUCTIONS ON USE OF THIS SCRIPT # # This script will perform three main functions: # 1: Generate a report and send an email on your drive(s) status. # 2: Create a copy of your Config File and attach to the same email. # 3: Create a statistical database and attach to the same email. # # In order to configure the script properly read over the User-definable Parameters before making any changes. # Make changes as indicated by the section instructions. # # To run the program from the command line, use ./program_name.sh [-h] for additional help instructions, # and [-config] to run the configuration routine (highly recommended). # # If you create an external configuration file, you never have to edit the script, # so how many times do I need to say it is highly recommended? And I may force the # change to require the external configuration file. # # You may need to make the script executable using "chmod +x program_name.sh" # ###### User-definable Parameters ####### # The sections below configure the script to your needs. Please follow the instructions as it will matter, you cannot # just "wing it". Configurations are exact. We use basically three different formats, Variables = true/false, # Variables = NUMBER, and Variables = Comma Separated Variable (CSV) Strings. Each variable will have a description # associated with it, read it carefully. # # The default configuration will work right out of the box however one item must be changed, your email address. # I highly recommend to try out the default setup first and then make changes as desired. The only two changes # I recommend is of course your email address, the second is the location of the statistical_data_file.cvs. # # Pay attention to any changes you make, accidentally deleting a quote will cause the entire script to fail. # Do not continue editing the script after the User Definable Section unless you know what you are doing. # # This script will not harm your drives. We are mostly only collecting drive data. All file writes are # to /tmp space. One exception: statistical_data_file.cvs is stored in /tmp by default however if you desire # to maintain this data it must be stored in a dataset (user selected). ###### Email Address ###### # Enter your email address to send the report to. The from address does not need to be changed. email="YourEmail@Address.com" from="TrueNAS@local.com" ###### Custom Hack ###### # Custom Hacks are for users with generally very unsupported drives and the data must be manually manipulated. # The goal is to not have any script customized so I will look for fixes where I can. # # Allowable custom hacks are: mistermanko custom_hack="none" ### Config File Name and Location ### SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) Config_File_Name="$SCRIPT_DIR/multi_report_config.txt" ###### Zpool Status Summary Table Settings usedWarn=80 # Pool used percentage for CRITICAL color to be used. scrubAgeWarn=37 # Maximum age (in days) of last pool scrub before CRITICAL color will be used (30 + 7 days for day of week). Default=37. ###### Temperature Settings HDDtempWarn=45 # HDD Drive temp (in C) upper OK limit before a WARNING color/message will be used. HDDtempCrit=50 # HDD Drive temp (in C) upper OK limit before a CRITICAL color/message will be used. HDDmaxovrd="true" # HDD Max Drive Temp Override. This value when "true" will not alarm on any Current Power Cycle Max Temperature Limit. SSDtempWarn=45 # SSD Drive temp (in C) upper OK limit before a WARNING color/message will be used. SSDtempCrit=50 # SSD Drive temp (in C) upper OK limit before a CRITICAL color/message will be used. SSDmaxovrd="true" # SSD Max Drive Temp Override. This value when "true" will not alarm on any Current Power Cycle Max Temperature Limit. NVMtempWarn=50 # NVM Drive temp (in C) upper OK limit before a WARNING color/message will be used. NVMtempCrit=60 # NVM Drive temp (in C) upper OK limit before a CRITICAL color/message will be used. ###### SSD/NVMe Specific Settings wearLevelCrit=9 # Wear Level Alarm Setpoint lower OK limit before a WARNING color/message, 9% is the default. ###### General Settings # Output Formats powerTimeFormat="h" # Format for power-on hours string, valid options are "ymdh", "ymd", "ym", "y", or "h" (year month day hour). tempdisplay="*C" # The format you desire the temperature to be displayed in. Common formats are: "*C", "^C", or "^c". Choose your own. non_exist_value="---" # How do you desire non-existent data to be displayed. The Default is "---", popular options are "N/A" or " ". pool_capacity="zfs" # Select "zfs" or "zpool" for Zpool Status Report - Pool Size and Free Space capacities. zfs is default. # Ignore or Activate Alarms ignoreUDMA="false" # Set to "true" to ignore all UltraDMA CRC Errors for the summary alarm (Email Header) only, errors will appear in the graphical chart. ignoreSeekError="true" # Set to "true" to ignore all Seek Error Rate/Health errors. Default is true. ignoreReadError="true" # Set to "true" to ignore all Seek Error Rate/Health errors. Default is true. ignoreMultiZone="false" # Set to "true" to ignore all MultiZone Errors. Default is false. disableWarranty="true" # Set to "true to disable email Subject line alerts for any expired warranty alert. The email body will still report the alert. # Disable or Activate Input/Output File Settings includeSSD="true" # Set to "true" will engage SSD Automatic Detection and Reporting, false = Disable SSD Automatic Detection and Reporting. includeNVM="true" # Set to "true" will engage NVM Automatic Detection and Reporting, false = Disable NVM Automatic Detection and Reporting. reportnonSMART="true" # Will force even non-SMART devices to be reported, "true" = normal operation to report non-SMART devices. disableRAWdata="false" # Set to "true" to remove the smartctl -a data and non-smart data appended to the normal report. Default is false. ata_auto_enable="false" # Set to "true" to automatically update Log Error count to only display a log error when a new one occurs. # Media Alarms sectorsWarn=1 # Number of sectors per drive to allow with errors before WARNING color/message will be used, this value should be less than sectorsCrit. sectorsCrit=9 # Number of sectors per drive with errors before CRITICAL color/message will be used. reAllocWarn=0 # Number of Reallocated sector events allowed. Over this amount is an alarm condition. multiZoneWarn=0 # Number of MultiZone Errors to allow before a Warning color/message will be used. Default is 0. multiZoneCrit=5 # Number of MultiZone Errors to allow before a Warning color/message will be used. Default is 5. deviceRedFlag="true" # Set to "true" to have the Device Column indicate RED for ANY alarm condition. Default is true. heliumAlarm="true" # Set to "true" to set for a critical alarm any He value below 100. Default is true. heliumMin=100 # Set to 100 for a zero leak helium result. An alert will occur below this value. rawReadWarn=5 # Number of read errors to allow before WARNING color/message will be used, this value should be less than rawReadCrit. rawReadCrit=100 # Number of read errors to allow before CRITICAL color/message will be used. seekErrorsWarn=5 # Number of seek errors to allow before WARNING color/message will be used, this value should be less than seekErrorsCrit. seekErrorsCrit=100 # Number of seek errors to allow before CRITICAL color/message will be used. # Time-Limited Error Recovery (TLER) SCT_Drive_Enable="false" # Set to "true" to send a command to enable SCT on your drives for user defined timeout. SCT_Warning="TLER_No_Msg" # Set to "all" will generate a Warning Message for all devices not reporting SCT enabled. "TLER" reports only drive which support TLER. # "TLER_No_Msg" will only report for TLER drives and not report a Warning Message if the drive can set TLER on. SCT_Read_Timeout=70 # Set to the read threshold. Default = 70 = 7.0 seconds. SCT_Write_Timeout=70 # Set to the write threshold. Default = 70 = 7.0 seconds. # SMART Testing Alarm testAgeWarn=2 # Maximum age (in days) of last SMART test before CRITICAL color/message will be used. ###### Statistical Data File statistical_data_file="$SCRIPT_DIR/statisticalsmartdata.csv" # Default location is where the script is located. expDataEnable="true" # Set to "true" will save all drive data into a CSV file defined by "statistical_data_file" below. expDataEmail="true" # Set to "true" to have an attachment of the file emailed to you. Default is true. expDataPurge=730 # Set to the number of day you wish to keep in the data. Older data will be purged. Default is 730 days (2 years). 0=Disable. expDataEmailSend="Mon" # Set to the day of the week the statistical report is emailed. (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month) ###### FreeNAS config backup settings configBackup="true" # Set to "true" to save config backup (which renders next two options operational); "false" to keep disable config backups. configSendDay="Mon" # Set to the day of the week the config is emailed. (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month) saveBackup="false" # Set to "false" to delete FreeNAS config backup after mail is sent; "true" to keep it in dir below. backupLocation="/tmp/" # Directory in which to store the backup FreeNAS config files. ########## REPORT CHART CONFIGURATION ############## ###### REPORT HEADER TITLE ###### HDDreportTitle="Spinning Rust Summary Report" # This is the title of the HDD report, change as you desire. SSDreportTitle="SSD Summary Report" # This is the title of the SSD report, change as you desire. NVMreportTitle="NVMe Summary Report" # This is the title of the NVMe report, change as you desire. ### CUSTOM REPORT CONFIGURATION ### # By default most items are selected. Change the item to false to have it not displayed in the graph, true to have it displayed. # NOTE: Alarm setpoints are not affected by these settings, this is only what columns of data are to be displayed on the graph. # I would recommend that you remove columns of data that you don't really care about to make the graph less busy. # For Zpool Status Summary Zpool_Pool_Name_Title="Pool Name" Zpool_Status_Title="Status" Zpool_Pool_Size_Title="Pool Size" Zpool_Free_Space_Title="Free Space" Zpool_Used_Space_Title="Used Space" Zfs_Pool_Size_Title="^Pool Size" Zfs_Free_Space_Title="^Free Space" Zfs_Used_Space_Title="^Used Space" Zpool_Read_Errors_Title="Read Errors" Zpool_Write_Errors_Title="Write Errors" Zpool_Checksum_Errors_Title="Cksum Errors" Zpool_Scrub_Repaired_Title="Scrub Repaired Bytes" Zpool_Scrub_Errors_Title="Scrub Errors" Zpool_Scrub_Age_Title="Last Scrub Age" Zpool_Scrub_Duration_Title="Last Scrub Duration" # For Hard Drive Section HDD_Device_ID="true" HDD_Device_ID_Title="Device ID" HDD_Serial_Number="true" HDD_Serial_Number_Title="Serial Number" HDD_Model_Number="true" HDD_Model_Number_Title="Model Number" HDD_Capacity="true" HDD_Capacity_Title="HDD Capacity" HDD_Rotational_Rate="true" HDD_Rotational_Rate_Title="RPM" HDD_SMART_Status="true" HDD_SMART_Status_Title="SMART Status" HDD_Raw_Read_Error_Rate="true" HDD_Raw_Read_Error_Rate_Title="Raw Error Rate" HDD_Drive_Temp="true" HDD_Drive_Temp_Title="Curr Temp" HDD_Drive_Temp_Min="true" HDD_Drive_Temp_Min_Title="Temp Min" HDD_Drive_Temp_Max="true" HDD_Drive_Temp_Max_Title="Temp Max" HDD_Power_On_Hours="true" HDD_Power_On_Hours_Title="Power On Time" HDD_Start_Stop_Count="true" HDD_Start_Stop_Count_Title="Start Stop Count" HDD_Load_Cycle="true" HDD_Load_Cycle_Title="Load Cycle Count" HDD_Spin_Retry="true" HDD_Spin_Retry_Title="Spin Retry Count" HDD_Reallocated_Sectors="true" HDD_Reallocated_Sectors_Title="Re-alloc Sects" HDD_Reallocated_Events="true" HDD_Reallocated_Events_Title="Re-alloc Evnt" HDD_Pending_Sectors="true" HDD_Pending_Sectors_Title="Curr Pend Sects" HDD_Offline_Uncorrectable="true" HDD_Offline_Uncorrectable_Title="Offl Unc Sects" HDD_UDMA_CRC_Errors="true" HDD_UDMA_CRC_Errors_Title="UDMA CRC Error" HDD_Seek_Error_Rate="true" HDD_Seek_Error_Rate_Title="Seek Error Rate" HDD_MultiZone_Errors="true" HDD_MultiZone_Errors_Title="Multi Zone Error" HDD_Helium_Level="true" HDD_Helium_Level_Title="He Level" HDD_Last_Test_Age="true" HDD_Last_Test_Age_Title="Last Test Age" HDD_Last_Test_Type="true" HDD_Last_Test_Type_Title="Last Test Type" # For Solid State Drive Section SSD_Device_ID="true" SSD_Device_ID_Title="Device ID" SSD_Serial_Number="true" SSD_Serial_Number_Title="Serial Number" SSD_Model_Number="true" SSD_Model_Number_Title="Model Number" SSD_Capacity="true" SSD_Capacity_Title="HDD Capacity" SSD_SMART_Status="true" SSD_SMART_Status_Title="SMART Status" SSD_Drive_Temp="true" SSD_Drive_Temp_Title="Curr Temp" SSD_Drive_Temp_Min="true" SSD_Drive_Temp_Min_Title="Temp Min" SSD_Drive_Temp_Max="true" SSD_Drive_Temp_Max_Title="Temp Max" SSD_Power_On_Hours="true" SSD_Power_On_Hours_Title="Power On Time" SSD_Wear_Level="true" SSD_Wear_Level_Title="Wear Level" SSD_Reallocated_Sectors="true" SSD_Reallocated_Sectors_Title="Re-alloc Sects" SSD_Reallocated_Events="true" SSD_Reallocated_Events_Title="Re-alloc Evnt" SSD_Pending_Sectors="true" SSD_Pending_Sectors_Title="Curr Pend Sects" SSD_Offline_Uncorrectable="true" SSD_Offline_Uncorrectable_Title="Offl Unc Sects" SSD_UDMA_CRC_Errors="true" SSD_UDMA_CRC_Errors_Title="UDMA CRC Error" SSD_Last_Test_Age="true" SSD_Last_Test_Age_Title="Last Test Age" SSD_Last_Test_Type="true" SSD_Last_Test_Type_Title="Last Test Type" # For NVMe Drive Section NVM_Device_ID="true" NVM_Device_ID_Title="Device ID" NVM_Serial_Number="true" NVM_Serial_Number_Title="Serial Number" NVM_Model_Number="true" NVM_Model_Number_Title="Model Number" NVM_Capacity="true" NVM_Capacity_Title="HDD Capacity" NVM_SMART_Status="true" NVM_SMART_Status_Title="SMART Status" NVM_Critical_Warning="true" NVM_Critical_Warning_Title="Critical Warning" NVM_Drive_Temp="true" NVM_Drive_Temp_Title="Curr Temp" NVM_Drive_Temp_Min="false" # I have not found this on an NVMe drive yet, so set to false NVM_Drive_Temp_Min_Title="Temp Min" NVM_Drive_Temp_Max="false" # I have not found this on an NVMe drive yet, so set to false NVM_Drive_Temp_Max_Title="Temp Max" NVM_Power_On_Hours="true" NVM_Power_On_Hours_Title="Power On Time" NVM_Wear_Level="true" NVM_Wear_Level_Title="Wear Level" ###### Drive Ignore List # What does it do: # Use this to list any drives to ignore and remove from the report. This is very useful for ignoring USB Flash Drives # or other drives for which good data is not able to be collected (non-standard). # # How to use it: # We are using a comma delimited file to identify the drive serial numbers. You MUST use the exact and full serial # number smartctl reports, if there is no identical match then it will not match. Additionally you may list drives # from other systems and they will not have any effect on a system where the drive does not exist. This is great # to have one configuration file that can be used on several systems. # # Live Example: Ignore_Drives="VMWare,1JUMLBD,21HNSAFC21410E" Ignore_Drives="none" ###### Drive UDMA_CRC_Error_Count List # What does it do: # If you have a drive which has an UDMA count other than 0 (zero), this setting will offset the # value back to zero for the concerns of monitoring future increases of this specific error. Any match will # subtract the given value to report a 0 (zero) value and highlight it in yellow to denote it was overridden. # The Warning Title will not be flagged if this is zero'd out in this manner. # NOTE: UDMA_CRC_Errors are typically permanently stored in the drive and cannot be reset to zero even though # they are frequently caused by a data cable communications error. # # How to use it: # List each drive by serial number and include the current UDMA_CRC_Error_Count value. # The format is very specific and will not work if you wing it, use the Live EXAMPLE. # # Set the FLAG in the FLAGS Section ignoreUDMA to false (the default setting). # # If the error count exceeds the limit minus the offset then a warning message will be generated. # On the Status Report the UDMA CRC Errors block will be YELLOW with a value of 0 for an overridden value. # -- NOTE: We are using the colon : as the separator between the drive serial number and the value to change. # # Format: variable=Drive_Serial_Number:Current_UDMA_Error_Count and add a comma if you have more than one drive. # # The below example shows drive WD-WMC4N2578099 has 1 UDMA_CRC_Error, drive S2X1J90CA48799 has 2 errors. # # Live Example: "WD-WMC4N2578099:1,S2X1J90CA48799:2,P02618119268:1" CRC_Errors="none" ###### Multi_Zone_Errors List # What does it do: # This identifies drives with Multi_Zone_Errors which may be irritating people. # Multi_Zone_Errors for some drives, not all drives are pretty much meaningless. # # How to use it: # Use same format as CRC_Errors (see above). Multi_Zone="none" ####### Reallocated Sectors Exceptions # What does it do: # This will offset any Reallocated Sectors count by the value provided. # # I do not recommend using this feature as I'm a believer in if you have over 5 bad sectors, odds are the drive will get worse. # I'd recommend replacing the drive before complete failure. But that is your decision. # # Why is it even an option? # I use it for testing purposes only but you may want to use it. # # How to use it: # Use same format as CRC_Errors (see above). Bad_Sectors="none" ######## ATA Error Log Silencing ################## # What does it do: # This will ignore error log messages equal to or less than the threshold. # How to use: # Same as the CRC_Errors, [drive serial number:error count] ata_errors="none" ####### Warranty Expiration Date # What does it do: # This section is used to add warranty expirations for designated drives and to create an alert when they expire. # The date format is YYYY-MM-DD. # # Below is an example for the format using my own drives, which yes, are expired. # As previously stated above, drive serial numbers must be an exact match to what smartctl reports to function. # # If the drive does not exist, for example my drives are not on your system, then nothing will happen. # # How to use it: # Use the format ="Drive_Serial_Number:YYYY-MM-DD" and add a comma if you have more than one drive. Drive_Warranty="none" ###### Global table of colors # The colors selected you can change but you will need to look up the proper HEX code for a color. okColor="#c9ffcc" # Hex code for color to use in SMART Status column if drives pass (default is light green, #c9ffcc). warnColor="#f765d0" # Hex code for WARN color (default is purple, #f765d0). critColor="#ff0000" # Hex code for CRITICAL color (default is red, #ff0000). altColor="#f4f4f4" # Table background alternates row colors between white and this color (default is light gray, #f4f4f4). whtColor="#ffffff" # Hex for White background. ovrdColor="#ffff66" # Hex code for Override Yellow. blueColor="#87ceeb" # Hex code for Sky Blue, used for the SCRUB In Progress background. yellowColor="#f1ffad" # Hex code for pale yellow. ########################## ########################## ### ### ### STOP EDITING THE ### ### SCRIPT HERE ### ### ### ########################## ########################## ###### Auto-generated Parameters softver=$(uname -s) host=$(hostname -s) truenas_ver=$(cat /etc/version) ### temp files have been converted to variable stored, not stored in /tmp/ as a file. ### logfile="/tmp/smart_report_body.tmp" logfile_header="/tmp/smart_report_header.tmp" logfile_warning="/tmp/smart_report_warning_flag.tmp" logfile_critical="/tmp/smart_report_critical_flag.tmp" logfile_warranty_temp="/tmp/smart_report_warranty_flag.tmp" logfile_messages_temp="/tmp/smart_report_messages.tmp" boundary="gc0p4Jq0M2Yt08jU534c0p" if [[ $softver != "Linux" ]]; then programver="Multi-Report v1.6d dtd:2022-10-01 (TrueNAS Core "$(cat /etc/version | cut -d " " -f1 | sed 's/TrueNAS-//')")" else programver="Multi-Report v1.6d dtd:2022-10-01 (TrueNAS Scale "$(cat /etc/version)")" fi #If the config file format changes, this is the latest working date, anything older must be updated. valid_config_version_date="2022-09-28" ########################## ########################## ### ### ### PROGRAMING HACKS ### ### ### ########################## ########################## #Unique progeramming hacks to properly emulate other hardware that is not actually on the system. VMWareNVME="on" # Set to "off" normally, "on" to assist in incorrect VMWare reporting. Silence="on" # Set to "on" normally, "off" to provide Joe Schmuck troubleshooting feedback while running the script. ########################## ########################## ### ### ### DEFINE FUNCTIONS ### ### ### ########################## ########################## load_config () { if test -e "$Config_File_Name"; then . "$Config_File_Name" # Lets test if the config file needs to be updated first. config_version_date="$(cat "$Config_File_Name" | grep "dtd" | cut -d ':' -f 2 | cut -d ' ' -f 1 )" echo "Configuration File Version Date: "$config_version_date if [[ $config_version_date < $valid_config_version_date ]]; then echo "Found Old Configuration File"; echo "Automatically updating configuration file..."; update_config_file; echo "Running normal script"; fi . "$Config_File_Name" else echo "No Config File Exists" echo "Checking for a valid email within the script..." if [[ $email == "YourEmail@Address.com" ]]; then echo "No Valid Email Address..." echo "Recommend running script with the '-config' switch and selecting the N)ew Configuration option." echo "... Aborting" echo " " exit 1 else echo "Valid email within the script = "$email", using script parameters..." echo " " External_Config="no" return fi fi } #################### CHECK OPEN FILE ##################### # Checks if trhe file is open before continuing. # Passes $1=filename # Loop for up to 60 seconds waiting for the file to close. check_open_file () { for (( y=1; y<=60; y++ )) do check_file=$1 result=`fuser -f $check_file 2>&1` pid=`echo $result | cut -d ':' -f 2` if [ -z "$pid" ]; then return; fi echo "File $1 Open - Delayed" sleep .5 done } #################### Force Slight Delay #################### # I think there is a race condition when writing to $logfile, trying to slow this down. force_delay () { sleep .1 } #################### PURGE EXPORT DATA CSV FILE ####################### purge_exportdata () { ### This routine will purge the "statistical_data_file" of data older then "expDataPurge". # Delete temp file if it exists if test -e "/tmp/temp_purge_file.csv"; then rm "/tmp/temp_purge_file.csv" # Create the header printf "Date,Time,Device ID,Drive Type,Serial Number,SMART Status,Temp,Power On Hours,Wear Level,Start Stop Count,Load Cycle,Spin Retry,Reallocated Sectors,\ ReAllocated Sector Events,Pending Sectors,Offline Uncorrectable,UDMA CRC Errors,Seek Error Rate,Multi Zone Errors,Read Error Rate,Helium Level\n" > "/tmp/temp_purge_file.csv" fi { input="$statistical_data_file" if [ $softver != "Linux" ]; then expireDate=$(date -v -"$expDataPurge"d +%Y/%m/%d) else expireDate=$(date -d "$expDataPurge days ago" +%Y/%m/%d) fi awk -v expireDate="$expireDate" ' BEGIN { FS=OFS="," FPAT = "([^,]+)|(\"[^\"]+\")" count=0 } data=$1 { FS=OFS=" " if (count !=0) if ($1 >= expireDate) { printf ("%s\n", data) >> "/tmp/temp_purge_file.csv" } ++count } END { } ' $input >/dev/null } cp -R "/tmp/temp_purge_file.csv" "$statistical_data_file" } ################## EMAIL EXPORT DATA CVS FILE ######################### email_datafile () { if [ "$expDataEmail" == "true" ]; then Now=$(date +"%a") doit="false" case $expDataEmailSend in All) doit="true" ;; Mon|Tue|Wed|Thu|Fri|Sat|Sun) if [[ "$expDataEmailSend" == "$Now" ]]; then doit="true"; fi ;; Month) if [[ $(date +"%d") == "01" ]]; then doit="true"; fi ;; *) ;; esac if [[ "$doit" == "true" ]]; then ( # Write MIME section header for file attachment (encoded with base64) echo "--${boundary}" echo "Content-Type: text/csv" echo "Content-Transfer-Encoding: base64" echo "Content-Disposition: attachment; filename=Statistical_Data.csv" base64 "$statistical_data_file" if [[ "$dump_all" == "1" || "$dump_all" == "2" ]]; then echo "--${boundary}"; else echo "--${boundary}--"; fi ) >> "$logfile" force_delay fi fi } ##################### IGNORE DRIVES ROUTINE ################# process_ignore_drives () { targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')"; ### Process Ignore List ### s="0" IFS=',' read -ra ADDR <<< "$Ignore_Drives" for i in "${ADDR[@]}"; do if [[ $i == $targument ]]; then s="1"; continue; fi done if [[ $s == "0" ]]; then printf "%s " "${drive}"; fi } #################### GET SMART HARD DRIVES ############################ get_smartHDD_listings () { # variable smartdrives if [[ "$testfile" != "" ]]; then echo "HDD TEST FILE ROUTINE" smartdrives="ada50" return fi if [ $softver != "Linux" ]; then smartdrives=$(for drive in $(sysctl -n kern.disks); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') else smartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') fi } ########################## GET SMART SOLID DISK DRIVES ################################ get_smartSSD_listings () { # variable smartdrivesSSD if [[ "$testfile" != "" ]]; then echo "SSD TEST FILE ROUTINE" smartdrivesSSD="ada50" return fi if [ $softver != "Linux" ]; then smartdrivesSSD=$(for drive in $(sysctl -n kern.disks); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') else smartdrivesSSD=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') fi } ########################## GET NVMe DRIVES ################################ get_smartNVM_listings () { # variable smartdrivesNVM if [[ "$testfile" != "" ]]; then echo "NVM TEST FILE ROUTINE" smartdrivesNVM="nvme50" return fi if [ $softver != "Linux" ]; then smartdrivesNVM=$(for drive in $(sysctl -n kern.disks); do if [ "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') else smartdrivesNVM=$(for drive in $(fdisk -l | grep "Disk /dev/nvm" | cut -d ':' -f 1 | cut -d '/' -f 3 | tr '\n' ' '); do if [ "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') fi ### Convert nvdx to nvmexx in smartdrivesNVM ### smartdrivesNVM=$( echo "$smartdrivesNVM" | sed 's/nvd/nvme/g' ) } ########################## GET OTHER SMART DEVICES ################################## get_smartOther_listings () { ### Get the non-SSD listing - MUST support SMART # variable nonsmartdrives if [[ "$testfile" != "" ]]; then echo "TEST FILE ROUTINE" nonsmartdrives="ada50" return fi if [ $softver != "Linux" ]; then nonsmartdrives=$(for drive in $(sysctl -n kern.disks); do if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ ! "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') else nonsmartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is: Enabled")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') fi } ########################### FORMAT EMAILS STEP 1 ########################## email_preformat () { ###### Email pre-formatting ### Set some of the email headers before conditional headers ( echo "MIME-Version: 1.0" echo "Content-Type: multipart/mixed; boundary=${boundary}" ) > "$logfile" force_delay } ########################### CONFIGURATION BACKUP ############################## config_backup () { ###### Config backup (if enabled) tarfile="/tmp/config_backup.tar.gz" filename="$(date "+FreeNAS_Config_%Y-%m-%d")" filename2="Stat_Data" if [ "$configBackup" == "true" ]; then Now=$(date +"%a") doit="false" case $configSendDay in All) doit="true" ;; Mon|Tue|Wed|Thu|Fri|Sat|Sun) if [[ "$configSendDay" == "$Now" ]]; then doit="true" fi ;; Month) if [[ $(date +"%d") == "01" ]]; then doit="true" fi ;; *) ;; esac if [[ "$doit" == "true" ]]; then # Set up file names, etc for later tarfile="/tmp/config_backup.tar.gz" filename="$(date "+FreeNAS_Config_%Y-%m-%d")" filename2="Stat_Data" ### Test config integrity if ! [ "$(sqlite3 /data/freenas-v1.db "pragma integrity_check;")" == "ok" ]; then # Config integrity check failed, set MIME content type to html and print warning ( echo "--${boundary}" echo "Content-Type: text/html" echo "Automatic backup of FreeNAS configuration has failed! The configuration file is corrupted!" echo "You should correct this problem as soon as possible!" echo "
" ) >> "$logfile" force_delay else # Config integrity check passed; copy config db, generate checksums, make .tar.gz archive cp /data/freenas-v1.db "/tmp/${filename}.db" if [ $softver != "Linux" ]; then md5 "/tmp/${filename}.db" > /tmp/config_backup.md5 sha256 "/tmp/${filename}.db" > /tmp/config_backup.sha256 else md5sum "/tmp/${filename}.db" > /tmp/config_backup.md5 sha256sum "/tmp/${filename}.db" > /tmp/config_backup.sha256 fi ( cd "/tmp/" || exit; tar -czf "${tarfile}" "./${filename}.db" ./config_backup.md5 ./config_backup.sha256; ) ( # Write MIME section header for file attachment (encoded with base64) echo "--${boundary}" echo "Content-Type: application/tar+gzip" echo "Content-Transfer-Encoding: base64" echo "Content-Disposition: attachment; filename=${filename}.tar.gz" base64 "$tarfile" # Write MIME section header for html content to come below echo "--${boundary}" echo "Content-Type: text/html" ) >> "$logfile" force_delay # If logfile saving is enabled, copy .tar.gz file to specified location before it (and everything else) is removed below if [ "$saveBackup" == "true" ]; then cp "${tarfile}" "${backupLocation}/${filename}.tar.gz" fi rm "/tmp/${filename}.db" rm /tmp/config_backup.md5 rm /tmp/config_backup.sha256 rm "${tarfile}" fi else ( echo "--${boundary}" echo "Content-Type: text/html" ) >> "$logfile" force_delay fi else # configBackup = false so this is what to do # Config backup enabled; set up for html-type content ( echo "--${boundary}" echo "Content-Type: text/html" ) >> "$logfile" force_delay fi } ################################## GENERATE ZPOOL REPORT ################################## zpool_report () { ###### Report Summary Section (html tables) ### zpool status summary table ( # Write HTML table headers to log file; HTML in an email requires 100% in-line styling (no CSS or