#!/bin/bash # Set the language to English so the date calculations work correctly with zpool status results. LANG="en_US.UTF-8" # With version 1.6c and above you may use an external configuration file. # Use [-help] to read the Help Section. For a short list of commands use [-h]. # Use [-config] to create a configuration file in the directory this script is run from. ###### ZPool & SMART status report with FreeNAS/TrueNAS config backup ### Original script by joeschmuck, modified by Bidule0hm, then by melp (me) ### Version: v1.3 TrueNAS Scale (Jeff Alperin 12-6-21) ### Version v1.4, v1.5, v1.6, v2.0 FreeNAS/TrueNAS (Core & Scale) (joeschmuck) ### Changelog: # V2.0.8 (20 February 2023) # - Added Email/Text Monitoring Feature for NugentS to try out. # - Edit multi_report_config.txt for additional email address. # - Use '-m' switch to invoke. Statistical data is recorded each time the script is run. # # # V2.0.7 (7 February 2023) # - Bug Fix: Nuisance error message, did not impact operation but it doesn't look good. # -- "Cannot open input file /tmp/zpoollist.txt" and "/tmp/zpoolstatus.txt". Only occurs in Core # -- during TrueNAS configuration file attachment operation (default is Monday). # # V2.0.6 (1 February 2023) # - Reduced drive data collection. Added 'zpool' data collection. # # V2.0.5 (27 January 2023) # - Adjusted Zpool Status to allow 'resilvering' status message. (Line 1340) # - Updated '-dump email' command to allow user to enter comments to the author. # # V2.0.4 (26 January 2023) # - Fixed if Zpool does not provide a number for fragmentation, will now display non_exist_value string. # # V2.0.3 (24 January 2023) # - Hacked HDD SMART Testing in progress with "NOW" in the '# 1' line. Will fix better later. # # V2.0.2 (24 January 2023) # - Fix Wear Level that may fail on some drives. # # v2.0.1 (21 January 2023) # - Fixed Zpool Fragmentation Warning for 9% and greater (Hex Math issue again). # # v2.0 (21 January 2023) # - Formatted all -config screens to fit into 80 column x 24 lines. # - Removed custom builds # - Fixed Custom Configuration Delete Function. # - Fixed Zpool Scrub Bytes for FreeNAS 11.x # - Fixed SMART Test to allow for 'Offline' value. # - Modified Wear Level script to account for 'Reverse' Adjustment. # - Added Wear Level Adjustment Reversing to the Custom Drive configuration. # - Added Output.html to -dump command. # - Added Mouseover and Alternate '()' to Mouseover for normalized values (Reallocated Sectors, Reallocated Sector Events, UDMA CRC, MultiZone). # - Updated Testing Code to accept both drive_a and drive_x files. # - Added Zpool Fragmentation value by request. # - Added '-dump email' parameter to send joeschmuck2023@hotmail.com an email with the drive data and the multi_report_config.txt file ONLY. # - Added Drive dump data in JSON format. It looks like a better way to parse the drive data. Still retaining the older file format for now. # # The multi_report_config file will automatically update previous versions to add new features. # # v1.6f (27 December 2022) # - Added recognition for WDC SSD "230 Media_Wearout_Indicator". # - Adjusted the language to English for the Date calculations. # - Updated datestamp2 "date" command to be compatible with FreeBSD 11 and earlier. # - Updated Zpool Status Report to display Used Space Capacity when using the "zpool" reporting configuration. # - Added customizable alarm setpoints for up to 24 drives. # -- This feature allows for customizing drives which do not fit into the default parameters. # -- It is not intended to individualize each drive, but you could if you wanted. # -- This allows the option for removal of the three custom code options. # - The configuration file in the email will now change to FreeNAS or TrueNAS based # -- on the software running vice FreeNAS hard-coded. # - Corrected several spelling errors throughout script. # # The multi_report_config file will automatically update previous versions to add new features. # # - Future Plan: Remove custom code for snowlucas2022, diedrichg, and mistermanko in version 2.0. # # v1.6e (11 November 2022) # - Fixed gptid not showing in the text section for the cache drive (Scale only affected). # - Fixed Zpool "Pool Size" - Wasn't calculating correctly under certain circumstances. # - Added Toshiba MG07+ drive Helium value support. # - Added Alphabetizing Zpool Names and Device ID's. # - Added No HDD Chart Generation if no HDD's are identified (nice for SSD/NVMe Only Systems). # - Added Warranty Column to chart (by request and must have a value in the Drive_Warranty variable). # - Removed Update option in -config since the script will automatically update now. # - Updated instructions for multiple email addresses. # - Updated instructions for "from:" address, some email servers will not accept the default # -- value and must be changed to the email address of the account sending the email. # - Added the No Text Section Option (enable_text) to remove the Text Section from the email output # -- and display the chart only, if the value is not "true". # - Added Phison Driven SSD attribute for correct Wear Level value. # # NOTES: If there is an error such as the host aborts a drive test and an error occurs, the script may # report a script failure. I do not desire to account for every possible drive error message. # If you take a look at your drive data, you may notice a problem. Fix the problem and the # script should work normally. If it still does not, then reach out for assistance. # # The multi_report_config file will automatically update previous versions to add new features. # # v1.6d-2 (09 October 2022) # - Bug fix for NVMe power on hours. # --- Unfortunately as the script gets more complex it's very easy to induce a problem. And since I do not have # --- a lot of different hardware, I need the users to contact me and tell me there is an issue so I can fix it. # --- It's unfortunate that I've have two bug fixes already but them's the breaks. # - Updated to support more drives Min/Max temps and display the non-existent value if nothing is obtained vice "0". # # The multi_report_config file is compatible with version back to v1.6d. # # v1.6d-1 (08 October 2022) # - Bug Fix for converting multiple numbers from Octal to Decimal. The previous process worked "most" of the time # -- but we always aim for 100% working. # # The multi_report_config file is compatible with version back to v1.6d. # # v1.6d (05 October 2022) # - Thanks goes out to ChrisRJ for offering some great suggestions to enhance and optimize the script. # - Updated gptid text and help text areas (clarifying information) # - Updated the -dump parameter to -dump [all] and included non-SMART attachments. # - Added Automatic UDMA_CRC, MultiZone, and Reallocated Sector Compensation to -config advanced option K. # - Fixed Warranty Date always showing as expired. # - Added Helium and Raw Read Error Rates to statistical data file. # - Added Raw Read Error Rates chart column. # - Added compensation for Seagate Seek Error Rates and Raw Read Error Rates. # - Added Automatic Configuration File Update feature. # - Added selection between ZFS Pool Size or Zpool Pool Size. ZFS is representative of the actual storage capacity # -- and updated the Pool Status Report Summary chart. # - Added ATA Error Log Silencing (by special request). # - Added 0.1 second delay after writing "$logfile" to eliminate intermittent file creation errors. # - Fixed Text Report -> Drive Model Number not showing up for some drives. # - Added option to email copy of multi_report_config.txt upon any automatic script modification and/or by day. # v1.6c (28 August 2022) # - Supports external configuration file (but not required). # - Completely Configurable by running the script -config parameter (this took a lot of work). # - Added HDD/SSDmaxtempovrd variables to combat some bogus SSD values. # - Added TLER (SCT) support. # - Added support for drives which do not support recording over 65536 hours for SMART Tests and rolls over to start at zero again. # - Added -dump parameter to create and email all of the drives smartctl outputs as text file email attachments. # - Added support for Helium drives. # v1.6: (05 August 2022) # Thanks to Jeff, Simon, and Sean for providing me more test data than I could shake a stick at and friendly constructive opinions/advice. # - Complete rewrite of the script. More organized and easier for future updates. # - Almost completely got rid of using AWK, earlier versions had way too much programming within the AWK structure. # - Reads the drives much less often (3 times each I believe). # - Added test input file to parse txt files of smartctl -a output. This will allow for a single drive entry and ability # -- for myself or any script writer to identify additional parameters for unrecognized drives. # -- Usage: program_name.sh [HDD|SSD|NVM] [inputfile_a.txt] [inputfile_b.txt] # - Added better support for SAS drives. # - Fixed NVMe and SAS Power On Hours for statistical data recording, and other things. # - Added Critical and Warning Logs to email output with better descriptive data. # - Logs (stored in /tmp/) no longer deleted after execution to aid in troubleshooting, but deleted at the start of the script. # - Added HELP file, use program_name.sh [-h] [-help] # - Added SCT Error Recovery to the Text Report section. # - Added Zpool Size, Free Space, and Temp Min/Max. # - Added customizable temperature values and customizable Non-Value fields (use to be coded to "N/A"). # - Added support for SandForce SSD. # v1.5: # - Added NVMe support # - Added clearer error reporting for Warning and Critical errors. # - Known Problems: The NVMe Power On Time has a comma that I can't get rid of, yet. I want to remove the comma when the data is retrieved. # -- NVMe's are not all standardized so I expect tweaks as different drive data arrives. # -- onHours that includes a comma will not record correctly in the statistical data file. This is related to the NVMe problem above. # -- Zpool Summary does not indicate Scrub Age warning, likely the entire summary has issues. # v1.4d: # - Fixed Scrub In Progress nuisance error when a scrub is in progress. # - Added offsetting Reallocated Sectors for four drives. This should be for testing only. Any drives # -- with a significant number of bad sectors should be replaced, just my opinion. # - Added Drive Warranty Expiration warning messages and ability to disable the Email Subject line warning. # -- NOT TESTED ON OTHER THAN U.S. FORMATTED DATE YYYY-MM-DD. # - Added HDD and SSD individual temperature settings. # - Changed order of polling Temperature data from HDD/SSD. # v1.4c: # - Rewrite to create functions and enable easier editing. # - Added Custom Reports. # - Added disabling the RAW 'smartctl -a' data appended to the end of the email. # - Added sorting drives alphabetically vice the default the OS reports them. # - Added RED warning in Device for any single failure in the summary (deviceRedFlag switch controlled). # - Added some additional SSD definitions. # - Fixed sorting last two SMART Tests, now reports them in proper order. # - Fixed detecting "SMART Support is: Enabled", for white spaces. # - Changed IGNORE DRIVES to a String Format to clean up and simplify programming. # - Added MultiZone_Errors support for up to eight drives. # - Added sectorWarn variable to complement the sectorCrit variable. # - Added ignoreSeekError variable to ignore some of those wild Seek Error Rate values. # - Added ignoreUDMA CRC Errors due to the "Known Problem" # - Fixed md5/sha256 error on TrueNAS Scale (only used during config backups). # - Added selectable config backup periodicity by day vice every run. # - Added Exporting statistical data for trend analysis. # -- Can be setup to email statistics weekly, monthly, or not at all. # -- The -s switch will run Data Collection Only, no email generated. Note: Do Not run two instances at once, the temp files do not survive. # - Fixed the Capacity to remove the brackets "[]", thanks Jeff Alperin. # - Fixed Scrub Age failure due to 1 day or longer repair time, now shows anything >24 hours. # # - Known Problem: One user reported UDMA_CRC_Errors is not subtracting correctly, have not been able to personally replicate it. # -- This error seems to occur around line #1027 # v1.4b: # - Added SMART test remaining percentage if Last Test has a SMART Test is in progress. # - Fix for empty SMART fields, typically for unsupported SSD's. # - Added IGNORE SMART Drive so you can ignore specific drives that may cause you weird readings. # --- Updated so blank SSD table header is removed when you ignore all the drives (just crazy talk). # v1.4a: # - Fixed report errors for if a SCRUB is in progress, now shows estimated completion time. # - Fixed report error for a Canceled SCRUB. # - Fixed FreeBSD/Linux use for SCRUB report (minor oversight). # v1.4: # - Run on CRON JOB using /path/multi_report_v1.4.sh # - Fixed for automatic running between FreeBSD and Linux Debian (aka SCALE) as of this date. # - All SMART Devices will report. # - Added conditional Subject Line (Good/Critical/Warning). # - Added Automatic SSD Support. # --- Some updates may need to be made to fit some of SSD's. Code in the area of about line 530 will # --- need to be adjusted to add new attributes for the desired SSD's fields. # - UDMA_CRC_ERROR Override because once a drive encounters this type of error, it cannot be cleared # --- so you can offset it now vice having an alarm condition for old UDMA_CRC_Errors. # - Added listing NON-SMART Supported Drives. Use only if useful to you, some drives will # --- still output some relevant data, many will not. # v1.3: # - Added scrub duration column # - Fixed for FreeNAS 11.1 (thanks reven!) # - Fixed fields parsed out of zpool status # - Buffered zpool status to reduce calls to script # v1.2: # - Added switch for power-on time format # - Slimmed down table columns # - Fixed some shellcheck errors & other misc stuff # - Added .tar.gz to backup file attached to email # - (Still coming) Better SSD SMART support # v1.1: # - Config backup now attached to report email # - Added option to turn off config backup # - Added option to save backup configs in a specified directory # - Power-on hours in SMART summary table now listed as YY-MM-DD-HH # - Changed filename of config backup to exclude timestamp (just uses datestamp now) # - Config backup and checksum files now zipped (was just .tar before; now .tar.gz) # - Fixed degrees symbol in SMART table (rendered weird for a lot of people); replaced with a * # - Added switch to enable or disable SSDs in SMART table (SSD reporting still needs work) # - Added most recent Extended & Short SMART tests in drive details section (only listed one before, whichever was more recent) # - Reformatted user-definable parameters section # - Added more general comments to code # v1.0: # - Initial release ######### INSTRUCTIONS ON USE OF THIS SCRIPT # # This script will perform three main functions: # 1: Generate a Pool/Drive report and send an email. # 2: Create a copy of your FreeNAS/TrueNAS Config File and attach to the email. # 3: Create a statistical database and optionally attach to the email. # # This script has been designed to primarily run using an external reusable configuration file which is # configured by running the script with the -config parameter. # # You may opt to not using the extternal configuration file and manually change the User-definable # Parameters within this script. # # In order to configure the script properly read over the User-definable Parameters before making any changes. # Make changes as indicated by the section instructions. # # To run the program from the command line, use ./program_name.sh [-h] for additional help instructions, # and [-config] to run the configuration routine (highly recommended). # # If you create an external configuration file, you never have to edit the script, # so how many times do I need to say it is highly recommended? And I may force the # change to require the external configuration file eventually to make the script a little smaller. # # You may need to make the script executable using "chmod +x program_name.sh" # ###### User-definable Parameters (IF YOU DO NOT WANT TO USE THE EXTERNAL CONFIGURATION FILE) ####### # # Modifying the script below should ONLY be done if you are not using an External Configuration File. # The External Configuration File allows the end user to not be required to update the script every time a # new change comes out. It is highly recommended to use the external configuration file, eventually # I plan to make the external configuration file a requirment. # # The sections below configures the script to your needs. Please follow the instructions as it will matter, you cannot # just "wing it". Configurations are exact. We use generally three different formats, Variables = true/false, # Variables = NUMBER, and Variables = Comma Separated Variable (CSV) Strings. Each variable will have a description # associated with it, read it carefully. # # The default configuration will work right out of the box however one item must be changed, your email address. # I highly recommend you try out the default setup first and then make changes as desired. The only two changes # I recommend is of course your email address, the second is the location of the 'statistical_data_file.cvs'. # # Pay attention to any changes you make, accidentally deleting a quote will cause the entire script to fail. # Do not continue editing the script after the User Definable Section unless you know what you are doing. # # This script will not harm your drives. We are mostly only collecting drive status data. All file writes are # to /tmp space. One exception: statistical_data_file.cvs is stored in /tmp by default however if you desire # to maintain this data it must be stored in a dataset (user selected). ###### Email Address ###### # Enter your email address to send the report to. The from address does not need to be changed unless you experience # an error sending the email. Some email servers only use the email address associated with the email server. email="YourEmail@Address.com" urgentemail="YourUrgentEmail@Address.com" from="TrueNAS@local.com" ### Config File Name and Location ### SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) Config_File_Name="$SCRIPT_DIR/multi_report_config.txt" ###### Zpool Status Summary Table Settings usedWarn=80 # Pool used percentage for CRITICAL color to be used. scrubAgeWarn=37 # Maximum age (in days) of last pool scrub before CRITICAL color will be used (30 + 7 days for day of week). Default=37. ###### Temperature Settings HDDtempWarn=45 # HDD Drive temp (in C) upper OK limit before a WARNING color/message will be used. HDDtempCrit=50 # HDD Drive temp (in C) upper OK limit before a CRITICAL color/message will be used. HDDmaxovrd="true" # HDD Max Drive Temp Override. This value when "true" will not alarm on any Current Power Cycle Max Temperature Limit. SSDtempWarn=45 # SSD Drive temp (in C) upper OK limit before a WARNING color/message will be used. SSDtempCrit=50 # SSD Drive temp (in C) upper OK limit before a CRITICAL color/message will be used. SSDmaxovrd="true" # SSD Max Drive Temp Override. This value when "true" will not alarm on any Current Power Cycle Max Temperature Limit. NVMtempWarn=50 # NVM Drive temp (in C) upper OK limit before a WARNING color/message will be used. NVMtempCrit=60 # NVM Drive temp (in C) upper OK limit before a CRITICAL color/message will be used. NVMmaxovrd="true" # NVM Max Drive Temp Override. This value when "true" will not alarm on any Current Power Cycle Max Temperature Limit. # --- NOTE: NVMe drives currently do not report Min/Max temperatures so this is a future feature. ###### SSD/NVMe Specific Settings wearLevelCrit=9 # Wear Level Alarm Setpoint lower OK limit before a WARNING color/message, 9% is the default. ###### General Settings # Output Formats powerTimeFormat="h" # Format for power-on hours string, valid options are "ymdh", "ymd", "ym", "y", or "h" (year month day hour). tempdisplay="*C" # The format you desire the temperature to be displayed in. Common formats are: "*C", "^C", or "^c". Choose your own. non_exist_value="---" # How do you desire non-existent data to be displayed. The Default is "---", popular options are "N/A" or " ". pool_capacity="zfs" # Select "zfs" or "zpool" for Zpool Status Report - Pool Size and Free Space capacities. zfs is default. # Ignore or Activate Alarms ignoreUDMA="false" # Set to "true" to ignore all UltraDMA CRC Errors for the summary alarm (Email Header) only, errors will appear in the graphical chart. ignoreSeekError="true" # Set to "true" to ignore all Seek Error Rate/Health errors. Default is true. ignoreReadError="true" # Set to "true" to ignore all Seek Error Rate/Health errors. Default is true. ignoreMultiZone="false" # Set to "true" to ignore all MultiZone Errors. Default is false. disableWarranty="true" # Set to "true to disable email Subject line alerts for any expired warranty alert. The email body will still report the alert. # Disable or Activate Input/Output File Settings includeSSD="true" # Set to "true" will engage SSD Automatic Detection and Reporting, false = Disable SSD Automatic Detection and Reporting. includeNVM="true" # Set to "true" will engage NVM Automatic Detection and Reporting, false = Disable NVM Automatic Detection and Reporting. reportnonSMART="true" # Will force even non-SMART devices to be reported, "true" = normal operation to report non-SMART devices. disableRAWdata="false" # Set to "true" to remove the smartctl -a data and non-smart data appended to the normal report. Default is false. ata_auto_enable="false" # Set to "true" to automatically update Log Error count to only display a log error when a new one occurs. # Media Alarms sectorsWarn=1 # Number of sectors per drive to allow with errors before WARNING color/message will be used, this value should be less than sectorsCrit. sectorsCrit=9 # Number of sectors per drive with errors before CRITICAL color/message will be used. reAllocWarn=0 # Number of Reallocated sector events allowed. Over this amount is an alarm condition. multiZoneWarn=0 # Number of MultiZone Errors to allow before a Warning color/message will be used. Default is 0. multiZoneCrit=5 # Number of MultiZone Errors to allow before a Warning color/message will be used. Default is 5. deviceRedFlag="true" # Set to "true" to have the Device Column indicate RED for ANY alarm condition. Default is true. heliumAlarm="true" # Set to "true" to set for a critical alarm any He value below "heliumMin" value. Default is true. heliumMin=100 # Set to 100 for a zero leak helium result. An alert will occur below this value. rawReadWarn=5 # Number of read errors to allow before WARNING color/message will be used, this value should be less than rawReadCrit. rawReadCrit=100 # Number of read errors to allow before CRITICAL color/message will be used. seekErrorsWarn=5 # Number of seek errors to allow before WARNING color/message will be used, this value should be less than seekErrorsCrit. seekErrorsCrit=100 # Number of seek errors to allow before CRITICAL color/message will be used. # Time-Limited Error Recovery (TLER) SCT_Drive_Enable="false" # Set to "true" to send a command to enable SCT on your drives for user defined timeout. SCT_Warning="TLER_No_Msg" # Set to "all" will generate a Warning Message for all devices not reporting SCT enabled. "TLER" reports only drive which support TLER. # "TLER_No_Msg" will only report for TLER drives and not report a Warning Message if the drive can set TLER on. SCT_Read_Timeout=70 # Set to the read threshold. Default = 70 = 7.0 seconds. SCT_Write_Timeout=70 # Set to the write threshold. Default = 70 = 7.0 seconds. # SMART Testing Alarm testAgeWarn=2 # Maximum age (in days) of last SMART test before CRITICAL color/message will be used. # Zpool Fragmentation Alarm zpoolFragWarn=80 # Percent of fragmentation before a Warning message occurs. ###### Statistical Data File statistical_data_file="$SCRIPT_DIR/statisticalsmartdata.csv" # Default location is where the script is located. expDataEnable="true" # Set to "true" will save all drive data into a CSV file defined by "statistical_data_file" below. expDataEmail="true" # Set to "true" to have an attachment of the file emailed to you. Default is true. expDataPurge=730 # Set to the number of day you wish to keep in the data. Older data will be purged. Default is 730 days (2 years). 0=Disable. expDataEmailSend="Mon" # Set to the day of the week the statistical report is emailed. (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month) ###### FreeNAS config backup settings configBackup="true" # Set to "true" to save config backup (which renders next two options operational); "false" to keep disable config backups. configSendDay="Mon" # Set to the day of the week the config is emailed. (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month) saveBackup="false" # Set to "false" to delete FreeNAS config backup after mail is sent; "true" to keep it in dir below. backupLocation="/tmp/" # Directory in which to store the backup FreeNAS config files. ###### Attach multi_report_config.txt to Email ###### Config_Email_Enable="true" # Set to "true" to enable periodic email (which renders next two options operational). Config_Changed_Email="true" # If "true" will attach the updated/changed file to the email. Config_Backup_Day="Mon" # Set to the day of the week the multi_report_config.txt is emailed. (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month, Never) ########## REPORT CHART CONFIGURATION ############## ###### REPORT HEADER TITLE ###### HDDreportTitle="Spinning Rust Summary Report" # This is the title of the HDD report, change as you desire. SSDreportTitle="SSD Summary Report" # This is the title of the SSD report, change as you desire. NVMreportTitle="NVMe Summary Report" # This is the title of the NVMe report, change as you desire. ### CUSTOM REPORT CONFIGURATION ### # By default most items are selected. Change the item to false to have it not displayed in the graph, true to have it displayed. # NOTE: Alarm setpoints are not affected by these settings, this is only what columns of data are to be displayed on the graph. # I would recommend that you remove columns of data that you don't really care about to make the graph less busy. # For Zpool Status Summary Zpool_Pool_Name_Title="Pool Name" Zpool_Status_Title="Status" Zpool_Pool_Size_Title="Pool Size" Zpool_Free_Space_Title="Free Space" Zpool_Used_Space_Title="Used Space" Zfs_Pool_Size_Title="^Pool Size" Zfs_Free_Space_Title="^Free Space" Zfs_Used_Space_Title="^Used Space" Zpool_Frag_Title="Frag" Zpool_Read_Errors_Title="Read Errors" Zpool_Write_Errors_Title="Write Errors" Zpool_Checksum_Errors_Title="Cksum Errors" Zpool_Scrub_Repaired_Title="Scrub Repaired Bytes" Zpool_Scrub_Errors_Title="Scrub Errors" Zpool_Scrub_Age_Title="Last Scrub Age" Zpool_Scrub_Duration_Title="Last Scrub Duration" # For Hard Drive Section HDD_Device_ID="true" HDD_Device_ID_Title="Device ID" HDD_Serial_Number="true" HDD_Serial_Number_Title="Serial Number" HDD_Model_Number="true" HDD_Model_Number_Title="Model Number" HDD_Capacity="true" HDD_Capacity_Title="HDD Capacity" HDD_Rotational_Rate="true" HDD_Rotational_Rate_Title="RPM" HDD_SMART_Status="true" HDD_SMART_Status_Title="SMART Status" HDD_Warranty_Title="Warr- anty" HDD_Warranty="true" HDD_Raw_Read_Error_Rate="true" HDD_Raw_Read_Error_Rate_Title="Raw Error Rate" HDD_Drive_Temp="true" HDD_Drive_Temp_Title="Curr Temp" HDD_Drive_Temp_Min="true" HDD_Drive_Temp_Min_Title="Temp Min" HDD_Drive_Temp_Max="true" HDD_Drive_Temp_Max_Title="Temp Max" HDD_Power_On_Hours="true" HDD_Power_On_Hours_Title="Power On Time" HDD_Start_Stop_Count="true" HDD_Start_Stop_Count_Title="Start Stop Count" HDD_Load_Cycle="true" HDD_Load_Cycle_Title="Load Cycle Count" HDD_Spin_Retry="true" HDD_Spin_Retry_Title="Spin Retry Count" HDD_Reallocated_Sectors="true" HDD_Reallocated_Sectors_Title="Re-alloc Sects" HDD_Reallocated_Events="true" HDD_Reallocated_Events_Title="Re-alloc Evnt" HDD_Pending_Sectors="true" HDD_Pending_Sectors_Title="Curr Pend Sects" HDD_Offline_Uncorrectable="true" HDD_Offline_Uncorrectable_Title="Offl Unc Sects" HDD_UDMA_CRC_Errors="true" HDD_UDMA_CRC_Errors_Title="UDMA CRC Error" HDD_Seek_Error_Rate="true" HDD_Seek_Error_Rate_Title="Seek Error Rate" HDD_MultiZone_Errors="true" HDD_MultiZone_Errors_Title="Multi Zone Error" HDD_Helium_Level="true" HDD_Helium_Level_Title="He Level" HDD_Last_Test_Age="true" HDD_Last_Test_Age_Title="Last Test Age" HDD_Last_Test_Type="true" HDD_Last_Test_Type_Title="Last Test Type" # For Solid State Drive Section SSD_Device_ID="true" SSD_Device_ID_Title="Device ID" SSD_Serial_Number="true" SSD_Serial_Number_Title="Serial Number" SSD_Model_Number="true" SSD_Model_Number_Title="Model Number" SSD_Capacity="true" SSD_Capacity_Title="HDD Capacity" SSD_SMART_Status="true" SSD_SMART_Status_Title="SMART Status" SSD_Warranty_Title="Warr- anty" SSD_Warranty="true" SSD_Drive_Temp="true" SSD_Drive_Temp_Title="Curr Temp" SSD_Drive_Temp_Min="true" SSD_Drive_Temp_Min_Title="Temp Min" SSD_Drive_Temp_Max="true" SSD_Drive_Temp_Max_Title="Temp Max" SSD_Power_On_Hours="true" SSD_Power_On_Hours_Title="Power On Time" SSD_Wear_Level="true" SSD_Wear_Level_Title="Wear Level" SSD_Reallocated_Sectors="true" SSD_Reallocated_Sectors_Title="Re-alloc Sects" SSD_Reallocated_Events="true" SSD_Reallocated_Events_Title="Re-alloc Evnt" SSD_Pending_Sectors="true" SSD_Pending_Sectors_Title="Curr Pend Sects" SSD_Offline_Uncorrectable="true" SSD_Offline_Uncorrectable_Title="Offl Unc Sects" SSD_UDMA_CRC_Errors="true" SSD_UDMA_CRC_Errors_Title="UDMA CRC Error" SSD_Last_Test_Age="true" SSD_Last_Test_Age_Title="Last Test Age" SSD_Last_Test_Type="true" SSD_Last_Test_Type_Title="Last Test Type" # For NVMe Drive Section NVM_Device_ID="true" NVM_Device_ID_Title="Device ID" NVM_Serial_Number="true" NVM_Serial_Number_Title="Serial Number" NVM_Model_Number="true" NVM_Model_Number_Title="Model Number" NVM_Capacity="true" NVM_Capacity_Title="HDD Capacity" NVM_SMART_Status="true" NVM_SMART_Status_Title="SMART Status" NVM_Warranty_Title="Warr- anty" NVM_Warranty="true" NVM_Critical_Warning="true" NVM_Critical_Warning_Title="Critical Warning" NVM_Drive_Temp="true" NVM_Drive_Temp_Title="Curr Temp" NVM_Drive_Temp_Min="false" # I have not found this on an NVMe drive yet, so set to false NVM_Drive_Temp_Min_Title="Temp Min" NVM_Drive_Temp_Max="false" # I have not found this on an NVMe drive yet, so set to false NVM_Drive_Temp_Max_Title="Temp Max" NVM_Power_On_Hours="true" NVM_Power_On_Hours_Title="Power On Time" NVM_Wear_Level="true" NVM_Wear_Level_Title="Wear Level" ###### Mouseover # This will display the original value of an overriden value (one in yellow) # This is a tri-state value as explained below. # "true" will displaying the actual value via a mouseover html link. Still working to make this a true Mouseover. # "false" will not generate any special chart and will run as previous versions. # "alt" will place the actual value within parentheses. An option if your email cliet can't display mouseover. Mouseover="alt" ###### Drive Ignore List # What does it do: # Use this to list any drives to ignore and remove from the report. This is very useful for ignoring USB Flash Drives # or other drives for which good data is not able to be collected (non-standard). # # How to use it: # We are using a comma delimited file to identify the drive serial numbers. You MUST use the exact and full serial # number smartctl reports, if there is no identical match then it will not match. Additionally you may list drives # from other systems and they will not have any effect on a system where the drive does not exist. This is great # to have one configuration file that can be used on several systems. # # Live Example: Ignore_Drives="VMWare,1JUMLBD,21HNSAFC21410E" Ignore_Drives="none" ###### Drive UDMA_CRC_Error_Count List # What does it do: # If you have a drive which has an UDMA count other than 0 (zero), this setting will offset the # value back to zero for the concerns of monitoring future increases of this specific error. Any match will # subtract the given value to report a 0 (zero) value and highlight it in yellow to denote it was overridden. # The Warning Title will not be flagged if this is zero'd out in this manner. # NOTE: UDMA_CRC_Errors are typically permanently stored in the drive and cannot be reset to zero even though # they are frequently caused by a data cable communications error. # # How to use it: # List each drive by serial number and include the current UDMA_CRC_Error_Count value. # The format is very specific and will not work if you wing it, use the Live Example. # # Set the FLAG in the FLAGS Section ignoreUDMA to false (the default setting). # # If the error count exceeds the limit minus the offset then a warning message will be generated. # On the Status Report the UDMA CRC Errors block will be YELLOW with a value of 0 for an overridden value. # -- NOTE: We are using the colon : as the separator between the drive serial number and the value to change. # # Format: variable=Drive_Serial_Number:Current_UDMA_Error_Count and add a comma if you have more than one drive. # # The below example shows drive WD-WMC4N2578099 has 1 UDMA_CRC_Error, drive S2X1J90CA48799 has 2 errors. # # Live Example: "WD-WMC4N2578099:1,S2X1J90CA48799:2,P02618119268:1" CRC_Errors="none" ###### Multi_Zone_Errors List # What does it do: # This identifies drives with Multi_Zone_Errors which may be irritating people. # Multi_Zone_Errors for some drives, not all drives are pretty much meaningless. # # How to use it: # Use same format as CRC_Errors (see above). Multi_Zone="none" ####### Reallocated Sectors Exceptions # What does it do: # This will offset any Reallocated Sectors count by the value provided. # # I do not recommend using this feature as I'm a believer in if you have over 5 bad sectors, odds are the drive will get worse. # I'd recommend replacing the drive before complete failure. But that is your decision. # # Why is it even an option? # I use it for testing purposes only but you may want to use it. # # How to use it: # Use same format as CRC_Errors (see above). Bad_Sectors="none" Bad_Sectors2="none" ######## ATA Error Log Silencing ################## # What does it do: # This will ignore error log messages equal to or less than the threshold. # How to use: # Same as the CRC_Errors, [drive serial number:error count] ata_errors="none" ####### Custom Drive Configuration (Experimental) # Used to define specific alarm values for specific drives by serial number. # This should only be used for drives where the default alarm settings # are not proper. Up to 24 unique drive values may be stored. # # Use -config to set these values. Custom_Drives="" ####### Warranty Expiration Date # What does it do: # This section is used to add warranty expirations for designated drives and to create an alert when they expire. # The date format is YYYY-MM-DD. # # Below is an example for the format using my own drives, which yes, are expired. # As previously stated above, drive serial numbers must be an exact match to what smartctl reports to function. # # If the drive does not exist, for example my drives are not on your system, then nothing will happen. # # How to use it: # Use the format ="Drive_Serial_Number:YYYY-MM-DD" and add a comma if you have more than one drive. Drive_Warranty="none" ######## Expired Drive Warranty Setup expiredWarrantyBoxColor="#000000" # "#000000" = normal box perimeter color. WarrantyBoxPixels="1" # Box line thickness. 1 = normal, 2 = thick, 3 = Very Thick, used for expired drives only. WarrantyBackgndColor="#f1ffad" # Hex code or "none" = normal background, Only for expired drives. ######## Enable-Disable Text Portion ######## enable_text="true" # This will display the Text Section when = "true" or remove it when not "true". Default="true" ###### Global table of colors # The colors selected you can change but you will need to look up the proper HEX code for a color. okColor="#b5fcb9" # Hex code for color to use in SMART Status column if drives pass (default is darker light green, #b5fcb9). warnColor="#F38B16" # Hex code for WARN color (default is orange, #F38B16). critColor="#f44336" # Hex code for CRITICAL color (default is red, #f44336). altColor="#f4f4f4" # Table background alternates row colors between white and this color (default is light gray, #f4f4f4). whtColor="#ffffff" # Hex for White background. ovrdColor="#ffffe4" # Hex code for Override Yellow. blueColor="#87ceeb" # Hex code for Sky Blue, used for the SCRUB In Progress background. yellowColor="#f1ffad" # Hex code for pale yellow. ########################## ########################## ### ### ### STOP EDITING THE ### ### SCRIPT HERE ### ### ### ########################## ########################## ###### Auto-generated Parameters softver=$(uname -s) host=$(hostname -s) truenas_ver=$(cat /etc/version) testdata_path="data" ### temp files have been converted to variable stored, not stored in /tmp/ as a file. ### logfile="/tmp/smart_report_body.tmp" logfile_header="/tmp/smart_report_header.tmp" logfile_warning="/tmp/smart_report_warning_flag.tmp" logfile_critical="/tmp/smart_report_critical_flag.tmp" logfile_warranty_temp="/tmp/smart_report_warranty_flag.tmp" logfile_messages_temp="/tmp/smart_report_messages.tmp" boundary="gc0p4Jq0M2Yt08jU534c0p" progname="Multi-Report v2.0.8 dtd:" progverdate="2023-02-20" if [[ $softver != "Linux" ]]; then if [[ "$(cat /etc/version | grep "FreeNAS")" ]]; then programver=progname$progverdate" (FreeNAS "$(cat /etc/version | cut -d " " -f1 | sed 's/FreeNAS-//')")" programver2="$(cat /etc/version | cut -d"-" -f1)" else programver=$progname$progverdate" (TrueNAS Core "$(cat /etc/version | cut -d " " -f1 | sed 's/TrueNAS-//')")" programver2="$(cat /etc/version | cut -d"-" -f1)" fi else programver=$progname$progverdate" (TrueNAS Scale "$(cat /etc/version)")" programver2="$(cat /etc/version | cut -d" " -f1)" fi # If the config file format changes, this is the latest working date, anything older must be updated. # Format must be "yyyy-mm-dd" valid_config_version_date="2023-02-02" ########################## ########################## ### ### ### PROGRAMING / ### ### TROUBLESHOOTING ### ### HACKS ### ### ### ########################## ########################## # Unique programming hacks to properly emulate other hardware that is not actually on the system. VMWareNVME="on" # Set to "off" normally, "on" to assist in incorrect VMWare reporting. Joes_System="false" # Custom settings for my system and to remove these from your system. Sample_Test="false" # Setup static test values for testing. extra_data="false" # Collect extra 'json' data. This is a temporary thing. ########################## ########################## ### ### ### DEFINE FUNCTIONS ### ### ### ########################## ########################## ############## CREATE TESTDATA TEXT FILE ################ create_testdata_text_file () { echo "Creating testdata variable" testdata_a="$(ls $testdata_path/*.a)" testdata_x="$(ls $testdata_path/*.x)" echo "testdata_a="$testdata_a echo " " echo "testdata_x="$testdata_x } ############## LOAD EXTERNAL CONFIGURATION FILE ############# load_config () { if test -e "$Config_File_Name"; then . "$Config_File_Name" # Lets test if the config file needs to be updated first. config_version_date="$(cat "$Config_File_Name" | grep "dtd" | cut -d ':' -f 2 | cut -d ' ' -f 1 )" echo "Configuration File Version Date: "$config_version_date if [[ $config_version_date < $valid_config_version_date ]]; then echo "Found Old Configuration File"; echo "Automatically updating configuration file..."; update_config_file; echo "Running normal script"; fi . "$Config_File_Name" else echo " " echo " No Config File Exists --- Checking for a valid email within the script..." if [[ $email == "YourEmail@Address.com" ]]; then echo " " echo " No Valid Email Address..." echo " Recommend running script with the '-config' switch and selecting" echo " the N)ew Configuration option." echo " " echo "... Exiting" echo " " exit 1 else echo "Valid email within the script = "$email", using script parameters..." echo " " External_Config="no" return fi fi } #################### Convert to Decimal ################## convert_to_decimal () { if [[ "$1" == "" ]]; then return; fi Converting_Value=${1#0} #echo "Step 1 -> Converting "$1" to BASE 10 = "$Converting_Value Converting_Value="${Converting_Value//,}" #echo "Step 2 -> Removing commas from "$1" to "$Converting_Value Return_Value=$Converting_Value if [[ $1 == "0" ]]; then Return_Value=0; fi #echo "Return_Value="$Return_Value } #################### Force Slight Delay #################### # I think there is a race condition when writing to $logfile, trying to slow this down. force_delay () { sleep .1 } #################### PURGE EXPORT DATA CSV FILE ####################### purge_exportdata () { ### This routine will purge the "statistical_data_file" of data older then "expDataPurge". # Delete temp file if it exists if test -e "/tmp/temp_purge_file.csv"; then rm "/tmp/temp_purge_file.csv" # Create the header # printf "Date,Time,Device ID,Drive Type,Serial Number,SMART Status,Temp,Power On Hours,Wear Level,Start Stop Count,Load Cycle,Spin Retry,Reallocated Sectors,\ #ReAllocated Sector Events,Pending Sectors,Offline Uncorrectable,UDMA CRC Errors,Seek Error Rate,Multi Zone Errors,Read Error Rate,Helium Level\n" > "/tmp/temp_purge_file.csv" fi if [ $softver != "Linux" ]; then expireDate=$(date -v -"$expDataPurge"d +%Y/%m/%d) else expireDate=$(date -d "$expDataPurge days ago" +%Y/%m/%d) fi awk -v expireDate="$expireDate" -F, '{ if($1 >= expireDate) print $0;}' "$statistical_data_file" > "/tmp/temp_purge_file.csv" cp -R "/tmp/temp_purge_file.csv" "$statistical_data_file" } ###### Purge ada50 and nvme50 from the logs ####### purge_testdata () { ### This routine will purge the "statistical_data_file" of test data matching "ada50" or "nvme50". # Delete temp file if it exists if test -e "/tmp/temp_purge_file.csv"; then rm "/tmp/temp_purge_file.csv" fi awk -F, '{ if($3 != "nvme50") print $0;}' "$statistical_data_file" > "/tmp/temp_purge_file1.csv" awk -F, '{ if($3 != "ada50") print $0;}' "/tmp/temp_purge_file1.csv" > "/tmp/temp_purge_file.csv" cp -R "/tmp/temp_purge_file.csv" "$statistical_data_file" } ################## EMAIL EXPORT DATA CVS FILE ######################### email_datafile () { if [ "$expDataEmail" == "true" ]; then Now=$(date +"%a") doit="false" case $expDataEmailSend in All) doit="true" ;; Mon|Tue|Wed|Thu|Fri|Sat|Sun) if [[ "$expDataEmailSend" == "$Now" ]]; then doit="true"; fi ;; Month) if [[ $(date +"%d") == "01" ]]; then doit="true"; fi ;; *) ;; esac if [[ "$doit" == "true" ]]; then ( # Write MIME section header for file attachment (encoded with base64) echo "--${boundary}" echo "Content-Type: text/csv" echo "Content-Transfer-Encoding: base64" echo "Content-Disposition: attachment; filename=Statistical_Data.csv" base64 "$statistical_data_file" ) >> "$logfile" force_delay doit="" fi fi # This file does not get attached to a -dump file. Joe does not need nor want this file. if [[ "$Config_Email_Enable" == "true" ]]; then if [[ "$Attach_Config" == "1" ]]; then doit="true"; fi Now=$(date +"%a") case $Config_Backup_Day in All) doit="true" ;; Mon|Tue|Wed|Thu|Fri|Sat|Sun) if [[ "$Config_Backup_Day" == "$Now" ]]; then doit="true"; fi ;; Month) if [[ $(date +"%d") == "01" ]]; then doit="true"; fi ;; Never) ;; *) ;; esac if [[ "$doit" == "true" ]] || [[ "$dump_all" != "0" ]]; then #echo "Attching Multi_Report Config" ( # Write MIME section header for file attachment (encoded with base64) echo "--${boundary}" echo "Content-Type: text/html" echo "Content-Transfer-Encoding: base64" echo "Content-Disposition: attachment; filename=multi_report_config.txt" base64 $Config_File_Name # if [[ "$dump_all" != "0" ]]; then echo "--${boundary}"; fi # If we are not dumping data, end the boundry to attach files. if [[ "$dump_all" != "0" ]]; then echo "--${boundary}"; else echo "--${boundary}--"; fi ) >> "$logfile" force_delay fi fi } ##################### IGNORE DRIVES ROUTINE ################# process_ignore_drives () { targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')"; ### Process Ignore List ### s="0" IFS=',' read -ra ADDR <<< "$Ignore_Drives" for i in "${ADDR[@]}"; do if [[ $i == $targument ]]; then s="1"; continue; fi done if [[ $s == "0" ]]; then printf "%s " "${drive}"; fi } ################## SORT DRIVES ROUTINE ######################## sort_drives () { #echo "Incoming Drive List="$sort_list sort_list=$(for i in `echo $sort_list`; do echo "$i" done | sort -V) #echo "Outgoing Drive List="$sort_list } #################### GET SMART HARD DRIVES ############################ get_smartHDD_listings () { # variable smartdrives if [[ "$testfile" != "" ]]; then echo "HDD TEST FILE ROUTINE" smartdrives="ada50" return fi if [ $softver != "Linux" ]; then smartdrives=$(for drive in $(sysctl -n kern.disks); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') else smartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') fi # Call Sort Routine with the drive string. if [[ "$smartdrives" != "" ]]; then sort_list=$smartdrives sort_drives smartdrives=$sort_list fi } ########################## GET SMART SOLID DISK DRIVES ################################ get_smartSSD_listings () { # variable smartdrivesSSD if [[ "$testfile" != "" ]]; then echo "SSD TEST FILE ROUTINE" smartdrivesSSD="ada50" return fi if [ $softver != "Linux" ]; then smartdrivesSSD=$(for drive in $(sysctl -n kern.disks); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') else smartdrivesSSD=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') fi # Call Sort Routine with the drive string. if [[ "$smartdrivesSSD" != "" ]]; then sort_list=$smartdrivesSSD sort_drives smartdrivesSSD=$sort_list fi } ########################## GET NVMe DRIVES ################################ get_smartNVM_listings () { # variable smartdrivesNVM if [[ "$testfile" != "" ]]; then echo "NVM TEST FILE ROUTINE" smartdrivesNVM="nvme50" return fi if [ $softver != "Linux" ]; then smartdrivesNVM=$(for drive in $(sysctl -n kern.disks); do if [ "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') else smartdrivesNVM=$(for drive in $(fdisk -l | grep "Disk /dev/nvm" | cut -d ':' -f 1 | cut -d '/' -f 3 | tr '\n' ' '); do if [ "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') fi ### Convert nvdx to nvmexx in smartdrivesNVM ### smartdrivesNVM=$( echo "$smartdrivesNVM" | sed 's/nvd/nvme/g' ) # Call Sort Routine with the drive string. if [[ "$smartdrivesNVM" != "" ]]; then sort_list=$smartdrivesNVM sort_drives smartdrivesNVM=$sort_list fi } ########################## GET OTHER SMART DEVICES ################################## get_smartOther_listings () { ### Get the non-SSD listing - MUST support SMART # variable nonsmartdrives if [[ "$testfile" != "" ]]; then echo "TEST FILE ROUTINE" nonsmartdrives="ada50" return fi if [ $softver != "Linux" ]; then nonsmartdrives=$(for drive in $(sysctl -n kern.disks); do if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ ! "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') else nonsmartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is: Enabled")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') fi # Call Sort Routine with the drive string. if [[ "$nonsmartdrives" != "" ]]; then sort_list=$nonsmartdrives sort_drives nonsmartdrives=$sort_list fi } ########################### FORMAT EMAILS STEP 1 ########################## email_preformat () { ###### Email pre-formatting ### Set some of the email headers before conditional headers ( echo "MIME-Version: 1.0" echo "Content-Type: multipart/mixed; boundary=${boundary}" ) > "$logfile" force_delay } ########################### CONFIGURATION BACKUP ############################## config_backup () { ###### Config backup (if enabled) tarfile="/tmp/config_backup.tar.gz" filename="$(date "+"$programver2"_Config_%Y-%m-%d")" filename2="Stat_Data" if [ "$configBackup" == "true" ]; then Now=$(date +"%a") doit="false" case $configSendDay in All) doit="true" ;; Mon|Tue|Wed|Thu|Fri|Sat|Sun) if [[ "$configSendDay" == "$Now" ]]; then doit="true" fi ;; Month) if [[ $(date +"%d") == "01" ]]; then doit="true" fi ;; *) ;; esac if [[ "$doit" == "true" ]]; then # Set up file names, etc for later tarfile="/tmp/config_backup.tar.gz" filename="$(date "+"$programver2"_Config_%Y-%m-%d")" filename2="Stat_Data" ### Test config integrity if ! [ "$(sqlite3 /data/freenas-v1.db "pragma integrity_check;")" == "ok" ]; then # Config integrity check failed, set MIME content type to html and print warning ( echo "--${boundary}" echo "Content-Type: text/html" echo "Automatic backup of FreeNAS configuration has failed! The configuration file is corrupted!" echo "You should correct this problem as soon as possible!" echo "
" ) >> "$logfile" force_delay else # Config integrity check passed; copy config db, generate checksums, make .tar.gz archive cp /data/freenas-v1.db "/tmp/${filename}.db" if [ $softver != "Linux" ]; then md5 "/tmp/${filename}.db" > /tmp/config_backup.md5 sha256 "/tmp/${filename}.db" > /tmp/config_backup.sha256 else md5sum "/tmp/${filename}.db" > /tmp/config_backup.md5 sha256sum "/tmp/${filename}.db" > /tmp/config_backup.sha256 fi ( cd "/tmp/" || exit; tar -czf "${tarfile}" "./${filename}.db" ./config_backup.md5 ./config_backup.sha256; ) ( # Write MIME section header for file attachment (encoded with base64) echo "--${boundary}" echo "Content-Type: application/tar+gzip" echo "Content-Transfer-Encoding: base64" echo "Content-Disposition: attachment; filename=${filename}.tar.gz" base64 "$tarfile" # Write MIME section header for html content to come below echo "--${boundary}" echo "Content-Type: text/html" ) >> "$logfile" force_delay # If logfile saving is enabled, copy .tar.gz file to specified location before it (and everything else) is removed below if [ "$saveBackup" == "true" ]; then cp "${tarfile}" "${backupLocation}/${filename}.tar.gz" fi rm "/tmp/${filename}.db" rm /tmp/config_backup.md5 rm /tmp/config_backup.sha256 rm "${tarfile}" fi else ( echo "--${boundary}" echo "Content-Type: text/html" ) >> "$logfile" force_delay fi else # configBackup = false so this is what to do # Config backup enabled; set up for html-type content ( echo "--${boundary}" echo "Content-Type: text/html" ) >> "$logfile" force_delay fi } ################################## GENERATE ZPOOL REPORT ################################## zpool_report () { ###### Report Summary Section (html tables) if [[ "$Urgent" == "false" ]]; then # For Urgent ( # Write HTML table headers to log file; HTML in an email requires 100% in-line styling (no CSS or