#!/bin/bash # LANG="en_US.UTF-8" # TrueNASConfigEmailEncryption="" # Set this to "" for no encryption or enter some text as your passphrase. # NOTE: Some email providers will not send some encrypted file types, such as GMAIL, but .zip files are okay. # NOTE: 7zip is used for the compression/encryption and will not be installed unless encryption is enabled. ###### Get Config File Name and Location SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) Config_File_Name="$SCRIPT_DIR/multi_report_config.txt" # Use [-help] to read the Help Section. For a short list of commands use [-h]. # Use [-config] to create a configuration file in the directory this script is run from. ###### ZPool & SMART status report with FreeNAS/TrueNAS config backup ### Original script by joeschmuck ### Modified by Bidule0hm, melp, toomuchdata ### Currently maintained by joeschmuck (joeschmuck2023@hotmail.com) ### Changelog: # V3.0 (30 March 2024) # Notable issues: # NVMe - Last SMART Short and Long tests not displaying in text area. It will be solved when Smartmontools 7.4 is installed. # -- This is not an issue in SCALE 23.10.2 as it has Smartmontools 7.4. # # - Fixed checking NVMe drives for if they support Self-tests. # - Added NVME Short and Long Self-test for smartctl 7.3 and below. Monday through Saturday a Short Test, Sunday a Long Test. # --- a Long Test, you may disable either or both options. Once TrueNAS can run NVMe SMART Tests expect this option to go away. # - Updated to list Drive Idents for NVMe in the Text section. # - Added NVME Ignore "Invalid Field in Command", disabled by default. # - Added Wait for SMART Short/Long Self-test to complete before sending the report. # - Added SMART Self-test Failure Recognition for NVMe. # - Updated CORE ability to capture NVMe Last Test Age. # - Updated NVMe routines to ignore real data gathering while in test mode. # - Enhanced SCSI/SAS drive recognition and Power_On_Hours collection. # - Fixed Zpool Reporting of 'Resilvering xx days' incorrectly reporting in SCALE. # - Updated 7zip to only being installed if email is encrypted (See line 5 of this script). # - Updated script for SCALE Dragonfish for installing 7zip if required. # - Updated Configuration Questions to make configuration a little easier. # - Removed 'Mouseover' option and hardcoded it. # - Corrected 'Pool_Capacity_Type' variable missing in config file. # - Added checking for all software commands to respond (thanks dak180 for the idea). # - Added custom wear level alarm value 'i' to the group 'n' 'r' 'd'. 'i' = Ignore. This makes wearLevel="", non-exist. # - Added Email Report ONLY on Alert (any Error Message). # - Updated to send attachments when Email_On_Alarm_Only="true" and Email_On_Alarm_Only_And_Attachments="true". # - Changed Non-Recognized drive power_on_hours from Warning to Caution. # - Adjusted script for multiple LBA reporting on Yucun SSDs. # - Updated script to work in a directory with a 'space character' in the path. # - Removed variables (IncluedSSD and IncludeNVM). # # V2.5.1 (3 December 2023) # - Changed exported configuration file to use .tar/.zip appropriately. # - Moved NVMe power state setting to end of script. # ###### ESTABLISH INITIAL DEFAULT VALUES ###### ### THESE ARE OVERRIDDEN BY THE EXTERNAL CONFIGURATION FILE ###### Email Address(s) Email="YourEmail@Address.com" # Normal email address to send report. From="TrueNAS@local.com" # From address (default works for many) ###### Alert Email Configuration - For Temperature and Critical Error monitoring when you suspect a problem. ### You must use the '-m' switch AlertEmail="YourAlertEmail@Address.com" # Alert email address used with the '-m' switch. AlertOnWarningTemp="true" # Send alert on Warning Temp. Default="true" AlertOnCriticalError="true" # Send alert on Critical Error. Default="true" ###### EMAIL ON ALARM ONLY Email_On_Alarm_Only="false" # When true, an email will only be sent if an alarm condition exists. Default = 'false' Email_On_Alarm_Only_And_Attachments="true" # When true, email attachments will be sent even when no alarm condition exists. Default = 'true' ###### Script Update ### Ensure you understand these options. Defaults check only, will not automatically update. CheckForUpdates="true" # Will check GitHub for updates and include message in next email. AutomaticUpdate="false" # WARNING !!! This option will automatically update the script if a newer version exists on GitHub with no user interaction. ###### Spencer Integration ### Warning Levels are: None, Warning, Critical -- This only affects the Email Subject Line, if any errors are present, an attachment will occur. spencer_new_warning_level="Warning" # What to do if a "new" error occurs. spencer_existing_warning_level="None" # What to do for an existing error. spencer_enable="true" # To call the Spencer.py script if "true" or "false" to not run the Spencer.py script. spencer_script_name="$SCRIPT_DIR/spencer.py" # The default is "spencer.py" located in the default script directory. ##### Multipath Enable/Disable ### When active this will (read below options): Multipath="off" # 'off' = No processing of serial numbers and is the default value. # 'normal' = Automatically remove duplicate serial numbers. # 'Exos2x' = Remove duplicate serial numbers ONLY IF the gptid matches. # 'serial' = Sort by serial numbers. ###### Zpool Status Summary Table Settings PoolUsedWarn=80 # Pool used percentage for CRITICAL color to be used. ScrubAgeWarn=37 # Maximum age (in days) of last pool scrub before CRITICAL color will be used. Default=37. ###### Temperature Settings HDDtempWarn=45 # HDD Drive Warning Temp (in C) when a WARNING message will be used. HDDtempCrit=50 # HDD Drive Critical Temp (in C) when a CRITICAL message will be used. SSDtempWarn=50 # SSD Drive Warning Temp (in C) when a WARNING message will be used. SSDtempCrit=60 # SSD Drive Critical Temp (in C) when a CRITICAL message will be used. NVMtempWarn=55 # NVM Drive Warning Temp (in C) when a WARNING message will be used. NVMtempCrit=65 # NVM Drive Critical Temp (in C) when a CRITICAL message will be used. ##### Current Power Cycle Maximum Temperature Override HDD_Cur_Pwr_Max_Temp_Ovrd="true" # HDD Max Drive Temp Override. This value when "true" will NOT alarm on any Current Power Cycle Max Temperature Limit. SSD_Cur_Pwr_Max_Temp_Ovrd="true" # SSD Max Drive Temp Override. This value when "true" will NOT alarm on any Current Power Cycle Max Temperature Limit. NVM_Cur_Pwr_Max_Temp_Ovrd="true" # NVM Max Drive Temp Override. This value when "true" will NOT alarm on any Current Power Cycle Max Temperature Limit. ###### SSD/NVMe Specific Settings WearLevelCrit=9 # Wear Level Alarm Setpoint when a WARNING message. 9% is the default. NVM_Low_Power="false" # Set the NVMe power level to the minimum setting. This does not mean the NVMe will remain at this power level. NVM_Daily_Short_Selftest="false" # This will run a short test on all NVMe drives everyday of the week, IF smartmontools is not 7.4 or greater. NVM_Weekly_Long_Selftest="false" # This will run a long test on all NVMe drives on Sunday and ignore the short test run if set, IF smartmontools is not 7.4 or greater. NVM_Weekly_Long_Selftest_Day="Sun" # Value must be "Sun, Mon, Tue, Wed, Thu, Fri, Sat". Default is Sun. NVM_Smartmontools_74_Override="disable" # Set to 'enable' to override the check for smartmontools 7.4 and force the script to run the SMART tests. Default is 'disable'. Wait_For_SMART="false" # Set to 'true' to wait for test to complete before continuing the script. Wait_For_SMART_Short_Duration=130 # Set how long to wait for an NVMe SMART Short Test to pause the script, waiting on the test to complete. Default = 2 minutes 10 seconds (130 seconds) Wait_For_SMART_Long_Duration=1200 # Set how long to wait for an NVMe SMART Long Test to pause the script, waiting on the test to complete. Default = 20 minutes (1200 seconds) NVMe_Ignore_Invalid_Errors="disable" # Set to 'enable' to ignore 'Invalid Field in Command' messages. Google this message to see if you are comfortable ignoring it. ###### General Settings ### Output Formats PowerTimeFormat="h" # Format for power-on hours string, valid options are "ymdh", "ymd", "ym", "y", or "h" (year month day hour). TempDisplay="*C" # The format you desire the temperature to be displayed. Common formats are: "*C", "^C", or "^c". Choose your own. Non_Exist_Value="---" # How do you desire non-existent data to be displayed. The Default is "---", popular options are "N/A" or " ". Pool_Capacity_Type="zfs" # Select "zfs" or "zpool" for Zpool Status Report - Pool Size and Free Space capacities. "zfs" is default. ###### Ignore or Activate Alarms IgnoreUDMA="false" # Set to "true" to ignore all UltraDMA CRC Errors for the summary alarm (Email Header) only, errors will appear in the graphical chart. Default is "false". IgnoreSeekError="true" # Set to "true" to ignore all Seek Error Rate/Health errors. Default is true. IgnoreReadError="true" # Set to "true" to ignore all Seek Error Rate/Health errors. Default is true. IgnoreMultiZone="false" # Set to "true" to ignore all MultiZone Errors. Default is false. DisableWarranty="true" # Set to "true to disable Email Subject line alerts for any expired warranty alert. The Email body will still report the alert. Default is "true". ###### Disable or Activate Input/Output File Settings ReportNonSMART="true" # Will force even non-SMART devices to be reported, "true" = normal operation to report non-SMART devices. DisableRAWdata="false" # Set to "true" to remove the smartctl -a data and non-smart data appended to the normal report. Default is false. ATA_Auto_Enable="false" # Set to "true" to automatically update Log Error count to only display a log error when a new one occurs. ###### Media Alarms SectorsWarn=0 # Number of sectors per drive when a WARNING message will be used, this value should be less than SectorsCrit. SectorsCrit=9 # Number of sectors per drive when a CRITICAL message will be used. ReAllocWarn=0 # Number of Reallocated sector events allowed. MultiZoneWarn=0 # Number of MultiZone Errors to allow when a Warning message will be used. Default is 0. MultiZoneCrit=5 # Number of MultiZone Errors to allow when a Warning message will be used. Default is 5. DeviceRedFlag="true" # Set to "true" to have the Device Column indicate RED for ANY alarm condition. Default is true. HeliumAlarm="true" # Set to "true" to set for a critical alarm any He value below "HeliumMin" value. Default is true. HeliumMin=100 # Set to 100 for a zero leak helium result. An alert will occur below this value. RawReadWarn=5 # Number of read errors to allow when a WARNING message will be used, this value should be less than RawReadCrit. RawReadCrit=100 # Number of read errors to allow when a CRITICAL message will be used. SeekErrorsWarn=5 # Number of seek errors to allow when a WARNING message will be used, this value should be less than SeekErrorsCrit. SeekErrorsCrit=100 # Number of seek errors to allow when a CRITICAL message will be used. NVM_Media_Errors=1 # Number of Media Errors to alarm with a CRITICAL message. ###### Time-Limited Error Recovery (TLER) SCT_Enable="false" # Set to "true" to send a command to enable SCT on your drives for user defined timeout. SCT_Warning_Level="TLER_No_Msg" # Set to "all" will generate a Warning Message for all devices not reporting SCT enabled. "TLER" reports only drive which support TLER. # "TLER_No_Msg" will only report for TLER drives and not report a Warning Message if the drive can set TLER on. ###### Read/Write Timeout SCT_Read_Timeout=70 # Set to the read threshold. Default = 70 = 7.0 seconds. SCT_Write_Timeout=70 # Set to the write threshold. Default = 70 = 7.0 seconds. ###### SMART Testing Alarm TestWarnAge=2 # Maximum age (in days) of last SMART test before CRITICAL color/message will be used. ###### Zpool Fragmentation Alarm ZpoolFragWarn=80 # Percent of fragmentation before a Warning message occurs. ###### Enable-Disable Text Portion Enable_Text_Section="true" # This will display the Text Section below the CHART when "true". Default="true" ###### Statistical Data File statistical_data_file="$SCRIPT_DIR/statisticalsmartdata.csv" # Default location is where the script is located. SDF_DataRecordEnable="true" # Set to "true" will save all drive data into a CSV file defined by "statistical_data_file" below. SDF_DataEmail="true" # Set to "true" to have an attachment of the file emailed to you. Default is true. SDF_DataPurgeDays=730 # Set to the number of day you wish to keep in the data. Older data will be purged. Default is 730 days (2 years). 0=Disable. SDF_DataEmailDay="Mon" # Set to the day of the week the statistical report is emailed. (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month) ###### TrueNAS config backup settings TrueNASConfigEmailEnable="true" # Set to "true" to save config backup (which renders next two options operational); "false" to keep disable config backups. TrueNASConfigEmailDay="Mon" # Set to the day of the week the config is emailed. (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month) TrueNASConfigBackupSave="false" # Set to "false" to delete TrueNAS config backup after mail is sent; "true" to keep it in dir below. TrueNASConfigBackupLocation="/tmp/" # Directory in which to store the backup FreeNAS config files. ###### Attach multi_report_config.txt to email ###### MRConfigEmailEnable="true" # Set to "true" to enable periodic email (which renders next two options operational). MRChangedEmailSend="true" # If "true" will attach the updated/changed file to the email. MRConfigEmailDay="Mon" # Set to the day of the week the multi_report_config.txt is emailed. (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month, Never) ###### REPORT CHART CONFIGURATION ###### CUSTOM SUBJECT LINE host=$(hostname -s) # The host name will precede the subject line message. Subject_Line_Critical="*CRITICAL ERROR* SMART Testing Results for ${host} *CRITICAL ERROR*" Subject_Line_Warning="*WARNING* SMART Testing Results for ${host} *WARNING*" Subject_Line_Normal="SMART Testing Results for ${host} - All is Good" ###### REPORT HEADER TITLE HDDreportTitle="Spinning Rust Summary Report" # This is the title of the HDD report, change as you desire. SSDreportTitle="SSD Summary Report" # This is the title of the SSD report, change as you desire. NVMreportTitle="NVMe Summary Report" # This is the title of the NVMe report, change as you desire. ###### CUSTOM REPORT CONFIGURATION ### By default most items are selected. Change the item to false to have it not displayed in the graph, true to have it displayed. ### NOTE: Alarm setpoints are not affected by these settings, this is only what columns of data are to be displayed on the graph. ### I would recommend that you remove columns of data that you don't really care about to make the graph less busy. ###### For Zpool Status Summary Zpool_Pool_Name_Title="Pool Name" Zpool_Status_Title="Status" Zpool_Pool_Size_Title="Pool Size" Zpool_Free_Space_Title="Free Space" Zpool_Used_Space_Title="Used Space" Zfs_Pool_Size_Title="^Pool Size" Zfs_Free_Space_Title="^Free Space" Zfs_Used_Space_Title="^Used Space" Zpool_Frag_Title="Frag" Zpool_Read_Errors_Title="Read Errors" Zpool_Write_Errors_Title="Write Errors" Zpool_Checksum_Errors_Title="Cksum Errors" Zpool_Scrub_Repaired_Title="Scrub Repaired Bytes" Zpool_Scrub_Errors_Title="Scrub Errors" Zpool_Scrub_Age_Title="Last Scrub Age" Zpool_Scrub_Duration_Title="Last Scrub Duration" ###### For Hard Drive Section HDD_Device_ID="true" HDD_Device_ID_Title="Device ID" HDD_Serial_Number="true" HDD_Serial_Number_Title="Serial Number" HDD_Model_Number="true" HDD_Model_Number_Title="Model Number" HDD_Capacity="true" HDD_Capacity_Title="HDD Capacity" HDD_Rotational_Rate="true" HDD_Rotational_Rate_Title="RPM" HDD_SMART_Status="true" HDD_SMART_Status_Title="SMART Status" HDD_Warranty_Title="Warr- anty" HDD_Warranty="true" HDD_Raw_Read_Error_Rate="true" HDD_Raw_Read_Error_Rate_Title="Raw Error Rate" HDD_Drive_Temp="true" HDD_Drive_Temp_Title="Curr Temp" HDD_Drive_Temp_Min="true" HDD_Drive_Temp_Min_Title="Temp Min" HDD_Drive_Temp_Max="true" HDD_Drive_Temp_Max_Title="Temp Max" HDD_Power_On_Hours="true" HDD_Power_On_Hours_Title="Power On Time" HDD_Start_Stop_Count="true" HDD_Start_Stop_Count_Title="Start Stop Count" HDD_Load_Cycle="true" HDD_Load_Cycle_Title="Load Cycle Count" HDD_Spin_Retry="true" HDD_Spin_Retry_Title="Spin Retry Count" HDD_Reallocated_Sectors="true" HDD_Reallocated_Sectors_Title="Re-alloc Sects" HDD_Reallocated_Events="true" HDD_Reallocated_Events_Title="Re-alloc Evnt" HDD_Pending_Sectors="true" HDD_Pending_Sectors_Title="Curr Pend Sects" HDD_Offline_Uncorrectable="true" HDD_Offline_Uncorrectable_Title="Offl Unc Sects" HDD_UDMA_CRC_Errors_List="true" HDD_UDMA_CRC_Errors_List_Title="UDMA CRC Error" HDD_Seek_Error_Rate="true" HDD_Seek_Error_Rate_Title="Seek Error Rate" HDD_MultiZone_Errors="true" HDD_MultiZone_Errors_Title="Multi Zone Error" HDD_Helium_Level="true" HDD_Helium_Level_Title="He Level" HDD_Last_Test_Age="true" HDD_Last_Test_Age_Title="Last Test Age" HDD_Last_Test_Type="true" HDD_Last_Test_Type_Title="Last Test Type" HDD_Total_Data_Written="true" HDD_Total_Data_Written_Title="Total Data Written" ###### For Solid State Drive Section SSD_Device_ID="true" SSD_Device_ID_Title="Device ID" SSD_Serial_Number="true" SSD_Serial_Number_Title="Serial Number" SSD_Model_Number="true" SSD_Model_Number_Title="Model Number" SSD_Capacity="true" SSD_Capacity_Title="SSD Capacity" SSD_SMART_Status="true" SSD_SMART_Status_Title="SMART Status" SSD_Warranty_Title="Warr- anty" SSD_Warranty="true" SSD_Drive_Temp="true" SSD_Drive_Temp_Title="Curr Temp" SSD_Drive_Temp_Min="true" SSD_Drive_Temp_Min_Title="Temp Min" SSD_Drive_Temp_Max="true" SSD_Drive_Temp_Max_Title="Temp Max" SSD_Power_On_Hours="true" SSD_Power_On_Hours_Title="Power On Time" SSD_Wear_Level="true" SSD_Wear_Level_Title="Wear Level" SSD_Reallocated_Sectors="true" SSD_Reallocated_Sectors_Title="Re-alloc Sects" SSD_Reallocated_Events="true" SSD_Reallocated_Events_Title="Re-alloc Evnt" SSD_Pending_Sectors="true" SSD_Pending_Sectors_Title="Curr Pend Sects" SSD_Offline_Uncorrectable="true" SSD_Offline_Uncorrectable_Title="Offl Unc Sects" SSD_UDMA_CRC_Errors_List="true" SSD_UDMA_CRC_Errors_List_Title="UDMA CRC Error" SSD_Last_Test_Age="true" SSD_Last_Test_Age_Title="Last Test Age" SSD_Last_Test_Type="true" SSD_Last_Test_Type_Title="Last Test Type" SSD_Total_Data_Written="true" SSD_Total_Data_Written_Title="Total Data Written" ###### For NVMe Drive Section NVM_Device_ID="true" NVM_Device_ID_Title="Device ID" NVM_Serial_Number="true" NVM_Serial_Number_Title="Serial Number" NVM_Model_Number="true" NVM_Model_Number_Title="Model Number" NVM_Capacity="true" NVM_Capacity_Title="NVMe Capacity" NVM_SMART_Status="true" NVM_SMART_Status_Title="SMART Status" NVM_Warranty_Title="Warr- anty" NVM_Warranty="true" NVM_Critical_Warning="true" NVM_Critical_Warning_Title="Critical Warning" NVM_Drive_Temp="true" NVM_Drive_Temp_Title="Curr Temp" NVM_Drive_Temp_Min="false" # Not NVMe drive yet NVM_Drive_Temp_Min_Title="Temp Min" NVM_Drive_Temp_Max="false" # Not NVMe drive yet NVM_Drive_Temp_Max_Title="Temp Max" NVM_Power_Level="true" NVM_Power_Level_Title="Power State" NVM_Power_On_Hours="true" NVM_Power_On_Hours_Title="Power On Time" NVM_Wear_Level="true" NVM_Wear_Level_Title="Wear Level" NVM_Media_Error="true" NVM_Media_Error_Title="Media Errors" NVM_Last_Test_Age="true" NVM_Last_Test_Age_Title="Last Test Age" NVM_Last_Test_Type="true" NVM_Last_Test_Type_Title="Last Test Type" NVM_Total_Data_Written="true" NVM_Total_Data_Written_Title="Total Data Written" ###### Drive Ignore List ### What does it do: ### Use this to list any drives to ignore and remove from the report. This is ### very useful for ignoring USB Flash Drives or other drives for which good ### data is not able to be collected (non-standard). ### ### How to use it: ### We are using a comma delimited file to identify the drive serial numbers. ### You MUST use the exact and full serial number smartctl reports, if there ### is no identical match then it will not ignore the drive. ### ### Format: Ignore_Drives_List="serial_number,serial_number,serial_number" ### Example: Ignore_Drives_List="VMWare,1JUMLBD,21HNSAFC21410E" Ignore_Drives_List="" ###### Drive UDMA CRC Error Count List, MultiZone List Errors List, ###### Reallocated Sectors Exceptions, Reallocated Sectors Events Exceptions ### ### What does it do: ### If you have a drive which has one of the above errors not a 0 (zero) value, ### this setting will offset the value back to zero for the considerations of ### monitoring future increases of this specific error. This will subtract ### the value by a correction value in order to 0 (zero) the value and ### highlight it in yellow to denote it was overridden. The Warning Title ### will not be flagged if this is zero'd out in this manner. ### ### How to use it: ### List each drive by serial number and include the current error count value. ### The format is very specific and will not work if you wing it, use the Example. ### ### Format: CRC_Errors_List="serial_number:current_udma_error_count,serial_number:current_udma_error_count" ### ### Example: CRC_Errors_List="WD-WMC4N2578099:1,S2X1J90CA48799:2,P02618119268:1" CRC_Errors_List="" MultiZone_List="" ReAllocated_Sector_List="" ReAllocated_Sector_Events_List="" ###### Custom Drive Configuration ### Used to define specific alarm values for specific drives by serial number. ### This should only be used for drives where the default alarm settings ### are not proper or you need to reverse some values where they may be listed ### opposite, for example WearLevel may be listed as 0% vice 100%. ### Up to 24 unique drive values may be stored (tested). ### ### Use -config to set these values. ### ### THE BREAKDOWN OF THIS LINE FORMAT (entire list broken to two lines for viewing, separated using a comma) ### serial:tempwarn:tempcrit:sectorswarn:sectorscrit:reallocwarn:multizonewarn:multizonecrit:rawreadwarn: ### rawreadcrit:seekerrorswarn:seekerrorscrit:testage:testAgeOvrd:heliummin:wearleveladj ### (testAgeOvrd '0'=Default, '1'=Ignore, wearleveladj 'd'=Default 'r'=Reverse value 'n'=Normalized 'i'=Ignore) Custom_Drives_List="" ###### Warranty Expiration Date ### What does it do: ### This section is used to add warranty expiration dates for designated drives ### and to create an alert when they expire. This is good to give you a ### heads-up on when you might need to start looking for a replacement drive. ### ### How to use it: ### Format: Drive_Warranty_List="serial_number:YYYY-MM-DD,serial_number:YYYY-MM-DD" ### Example: Drive_Warranty_List="K1JUMLBD:2020-09-30,K1JRSWLD:2020-09-30,K1JUMW4D:2020-09-30,K1GVD84B:2020-10-12" Drive_Warranty_List="" ###### Expired Drive Warranty Colors expiredWarrantyBoxColor="#000000" # "#000000" = normal box perimeter color. WarrantyBackgndColor="#f1ffad" # Hex code or "none" = normal background. ###### Global table of colors ### The colors selected you can change but you will need to look up the proper ### HEX code for a color. okColor="#b5fcb9" # Hex code for color to use in SMART Status column if drives pass (default is darker light green, #b5fcb9). warnColor="#F38B16" # Hex code for WARN color (default is orange, #F38B16). critColor="#f44336" # Hex code for CRITICAL color (default is red, #f44336). altColor="#f4f4f4" # Table background alternates row colors between white and this color (default is light gray, #f4f4f4). whtColor="#ffffff" # Hex for White background. ovrdColor="#ffffe4" # Hex code for Override Yellow. blueColor="#87ceeb" # Hex code for Sky Blue, used for the SCRUB/SMART Test In Progress/background. yellowColor="#f1ffad" # Hex code for pale yellow. ########################## ########################## ### ### ### STOP EDITING THE ### ### SCRIPT HERE ### ### ### ########################## ########################## ###### Auto-generated Parameters softver=$(uname -s) #host=$(hostname -s) truenas_ver=$(cat /etc/version) testdata_path="data" Dump_Loop="0" re='^[0-9]+$' runfilename="multi_report.sh" ### temp files have been converted to variable stored, not stored in /tmp/ as a file. ### tempfilepath=$(( 10 + $RANDOM % 1000 )) logfile="/tmp/${tempfilepath}smart_report_body.tmp" logfile_header="/tmp/${tempfilepath}smart_report_header.tmp" logfile_temp="/tmp/${tempfilepath}smart_report_temp.tmp" boundary="gc0p4Jq0M2Yt08jU534c0p" CurrentFilename="multi_report_v3.0_2024_03_30.txt" valid_config_version_date="2024-03-30" # Configuration file valid date progverdate="$(echo $CurrentFilename | cut -d '_' -f4,5,6 | cut -d '.' -f1 | sed -r 's/[_]+/-/g')" progname="Multi-Report "$(echo $CurrentFilename | cut -d '_' -f3)" dtd:" if [[ $softver != "Linux" ]]; then if [[ "$(cat /etc/version | grep "FreeNAS")" ]]; then programver=progname$progverdate" (FreeNAS "$(cat /etc/version | cut -d " " -f1 | sed 's/FreeNAS-//')")" programver2="$(cat /etc/version | cut -d"-" -f1)" programver3="$(cat /etc/version | cut -d " " -f1 | sed 's/FreeNAS-//')" else programver=$progname$progverdate" (TrueNAS Core "$(cat /etc/version | cut -d " " -f1 | sed 's/TrueNAS-//')")" programver2="$(cat /etc/version | cut -d"-" -f1)_Core" programver3="$(cat /etc/version | cut -d " " -f1 | sed 's/TrueNAS-//')" fi else programver=$progname$progverdate" (TrueNAS Scale "$(cat /etc/version)")" programver2="TrueNAS_Scale_$(cat /etc/version | cut -d" " -f1)" programver3="$(cat /etc/version)" fi declare -a testfilenames declare -a testfilenamesHDD declare -a smartdrives ########################## ########################## ### ### ### PROGRAMMING / ### ### TROUBLESHOOTING ### ### HACKS ### ### ### ########################## ########################## #Unique programming hacks to properly emulate other hardware that is not actually on the system. VMWareNVME="off" # Set to "off" normally, "on" to assist in incorrect VMWare reporting. Joes_System="false" # Custom settings for my system and to remove these from your system. Sample_Test="false" # Setup static test values for testing. Develop="false" # Set to 'true' for development output. GitHubSimulate="false" # Use test section of GitHub. ########################## ########################## ### ### ### DEFINE FUNCTIONS ### ### ### ########################## ########################## ########## EXIT IF MULTIPLE INSTANCES ARE RUNNING ########## # Let's stop a second instance from running. mefull=`basename "$0"` if [[ "$1" == "-ignore_lock" || "$2" == "-ignore_lock" || "$3" == "-ignore_lock" || "$4" == "-ignore_lock" ]]; then printf "Ignoring Multiple Instance Check\n" >&2 else if ! mkdir /tmp/multi_report.lock; then printf "Script is already Running... Exiting\n" >&2 printf "If this message is in error, remove '/tmp/multi_report.lock' directory or just reboot TrueNAS to recover.\n" >&2 exit 1 fi fi trap 'rm -rf /tmp/multi_report.lock' EXIT # Remove the lock directory on exit #################### FUNCTIONS #################### ########## SPENCER INTEGRATION - CHECK MESSAGES FILE FOR ERROR MESSAGES ########## # We will check the /var/log/messages file for any iscsi, cam, ctl, or cdb error messages # # Use the CSV that is used in the data recording to record each instance of an alarm. # Search each instance for an exact match for repeat offenders. # Remove entries if they no longer exits in the messages log file. # # Count up the duplicate errors and put them as "6x error message" for example. # Create a Message file to add to the Warning list so we can punt this out. # spencer () { # Check status of "/tmp/spencer_report.txt" file, then if we have errors process it. if [ -f "/tmp/spencer_report.txt" ]; then if [ $(grep -wic "New Error Messages" "/tmp/spencer_report.txt") -gt 0 ]; then # Error Levels are: None, Warning, Critical if [[ $spencer_new_warning_level == "Warning" ]]; then logfile_warning=$logfile_warning"$(printf "Spencer New Error Data - See Attachment")" elif [[ $spencer_new_warning_level == "Critical" ]]; then logfile_critical=$logfile_critical"$(printf "Spencer New Error Data - See Attachment")" fi fi if [ $(grep -wic "Previous" "/tmp/spencer_report.txt") -gt 0 ]; then # Error Levels are: None, Warning, Critical if [[ $spencer_existing_warning_level == "Warning" ]]; then logfile_warning=$logfile_warning"$(printf "Spencer Existing Error Data - See Attachment")" elif [[ $spencer_existing_warning_level == "Critical" ]]; then logfile_critical=$logfile_critical"$(printf "Spencer Existing Error Data - See Attachment")" fi fi fi } ########## RUN NVMe SELFTEST ########## nvm_selftest_short () { # Smartmontools is less than 7.4 or Overridden if [[ $smart_ver < "7.4" ]]; then echo "TrueNAS using smartmontools v$smart_ver" >> /tmp/selftestlog; echo "TrueNAS using smartmontools v$smart_ver"; fi for drive in $smartdrivesNVM; do if [ $softver != "Linux" ]; then # FreeBSD Commands check_nvme_selftest=$(nvmecontrol identify $drive | grep -i "self-test") shopt -s nocasematch # Make test not case sensitive if [[ $check_nvme_selftest != *"Not"* ]]; then echo "Running Short Self-test for $drive in background" echo "Running Short Self-test for $drive in background" >> /tmp/selftestlog nvmecontrol selftest -c 1 $drive else echo "$drive does not support Self-test" echo "$drive does not support Self-test" >> /tmp/selftestlog fi shopt -u nocasematch else # Debian Commands check_nvme_selftest=$(nvme id-ctrl /dev/$drive -H | grep -i "self-test") if [[ $check_nvme_selftest != *"Not"* ]]; then echo "Running Short Self-test for $drive in background" echo "Running Short Self-test for $drive in background" >> /tmp/selftestlog nvme device-self-test /dev/$drive -s 1 else echo "$drive does not support Self-test" echo "$drive does not support Self-test" >> /tmp/selftestlog fi fi done } nvm_selftest_long () { # Smartmontools is less than 7.4 or Overridden if [[ $smart_ver < "7.4" ]]; then echo "TrueNAS using smartmontools v$smart_ver" >> /tmp/selftestlog; echo "TrueNAS using smartmontools v$smart_ver"; fi for drive in $smartdrivesNVM; do if [ $softver != "Linux" ]; then # FreeBSD Commands check_nvme_selftest=$(nvmecontrol identify $drive | grep -i "self-test") shopt -s nocasematch # Make test not case sensitive if [[ $check_nvme_selftest != *"Not"* ]]; then echo "Running Long Self-test for $drive in background" echo "Running Long Self-test for $drive in background" >> /tmp/selftestlog nvmecontrol selftest -c 2 $drive else echo "$drive does not support Self-test" echo "$drive does not support Self-test" >> /tmp/selftestlog fi shopt -u nocasematch else # Debian Commands check_nvme_selftest=$(nvme id-ctrl /dev/$drive -H | grep -i "self-test") if [[ $check_nvme_selftest != *"Not"* ]]; then echo "Running Long Self-test for $drive in background" echo "Running Long Self-test for $drive in background" >> /tmp/selftestlog nvme device-self-test /dev/$drive -s 2 else echo "$drive does not support Self-test" echo "$drive does not support Self-test" >> /tmp/selftestlog fi fi done } ########## SET NVMe TO LOWEST POWER SETTING ########## nvm_power () { for drive in $smartdrivesNVM; do current_power_state=$(($(nvmecontrol power $drive | rev | cut -c1-2 | rev))) lowest_power_state=$(($(nvmecontrol power -l $drive | tail -1 | cut -c1-2))) if [[ $current_power_state -ne $lowest_power_state ]]; then echo "Changing "$drive" power state from "$current_power_state" to "$lowest_power_state"." fi sleep 2 nvmecontrol power -p $lowest_power_state $drive sleep 1.5 echo "Checking to see if we remained at power state "$lowest_power_state current_power_state=$(($(nvmecontrol power $drive | rev | cut -c1-2 | rev))) echo "Power state is "$current_power_state done } ########## SEE IF 7ZIP IS LOADED AND IF NOT, INSTALL IT ########## # NOTE: We check for 7zip installed and if not install it. check_7zip () { if type "7z" &> /dev/null; then return; fi # No 7zip is installed, lets go get it. echo "Installing 7-Zip..." wget https://www.7-zip.org/a/7z2201-linux-x64.tar.xz > /dev/null 2>&1 # Get 7-zip tar xf 7z*-linux-x64.tar.xz 7zzs > /dev/null 2>&1 # Extract the executable file only # Check if /usr is readonly (SCALE 24.x started this) if grep "[[:space:]]ro[[:space:],]" /proc/mounts | grep -q "/usr"; then mount -o remount,rw '/usr' romount="true" fi cp 7zzs /usr/local/bin # Copy to /bin ln -s /usr/local/bin/7zzs /usr/local/bin/7z # Symlink it to "7z" echo "7-Zip Installed" # Cleanup if test -e "7z2201-linux-x64.tar.xz"; then rm "7z2201-linux-x64.tar.xz"; fi if test -e "7zzs"; then rm "7zzs"; fi # If /usr was originally 'ro', change it back. if [[ romount == "true" ]]; then mount -o remount,ro '/usr' fi } ########## AUTOMATIC SCRIPT UPDATE ########## # This will update the script with any newer script on GitHub and is manually evoked [-update] # # How it works or might work # Check GitHub for Date on script, if that is possible. # If date/time is more current than this current script is, download the new script. # Next write the new file over the current file name. # Edit line #5 with a password if one was present. update_script () { echo "Update Script Routine" echo "Removing Old Script Source if it exists" if test -e "/tmp/Multi-Report"; then rm -R "/tmp/Multi-Report"; fi if test -e "/tmp/multi_report_update.txt"; then rm "/tmp/multi_report_update.txt"; fi echo "Downloading new script files" if [[ "$(curl -is https://github.com | head -n 1)" ]]; then # Go git the file ( cd /tmp git clone -q https://github.com/JoeSchmuck/Multi-Report.git ) # For the Public distribution GitVersion="$(cat "/tmp/Multi-Report/current_script")" validate="$(cat "/tmp/Multi-Report/current_cksum")" checksum="$(cat "/tmp/Multi-Report/current_script")" # For the Development distribution if [[ $GitHubSimulate == "true" ]]; then GitVersion="$(cat "/tmp/Multi-Report/current_script1")" validate="$(cat "/tmp/Multi-Report/current_cksum1")" checksum="$(cat "/tmp/Multi-Report/current_script1")" fi echo " " echo "Your current version is: "$CurrentFilename echo " The new version is: "$GitVersion echo " " if [ $softver != "Linux" ]; then checksum="$(md5 /tmp/Multi-Report/$checksum | cut -d '=' -f 2 | cut -d ' ' -f 2)" else checksum="$(md5sum --tag /tmp/Multi-Report/$checksum | cut -d '=' -f 2 | cut -d ' ' -f 2)" fi if [[ $checksum == $validate ]]; then echo "File is valid" else echo "Downloaded file is corrupt, does not match CRC." if [[ $AutomaticUpdate == "true" ]]; then AutomaticUpdate="false" return fi fi if [[ $AutomaticUpdate != "true" ]]; then echo "Enter 'y' to commit or any other key to abort." read Keyboard_yn else echo "Automatic Update is Enabled..." Keyboard_yn="y" fi if [[ $Keyboard_yn == "y" ]] || [[ $Keyboard_yn == "Y" ]]; then echo "Updating Script..." echo "3" sed '5s/.*/TrueNASConfigEmailEncryption="'$TrueNASConfigEmailEncryption'" # Set this to "" for no encryption, MUST REMAIN ON LINE 5./' /tmp/Multi-Report/$GitVersion > /tmp/multi_report_update.txt sleep .5 # Generate the full file name minus the .txt VersionFilename="$(echo $CurrentFilename | rev | cut -d '.' -f 2,3,4,5 | rev)" echo "2" sleep .5 # Backup the original multi_report.sh file and config file to save if test -e $SCRIPT_DIR"/multi_report.sh"; then cp $SCRIPT_DIR"/multi_report.sh" $SCRIPT_DIR"/"$CurrentFilename; fi cp $SCRIPT_DIR"/multi_report_config.txt" $SCRIPT_DIR"/"$VersionFilename"_config.txt" # Check if old multi_report.sh file exists and delete if test -e $SCRIPT_DIR"/"$runfilename; then rm $SCRIPT_DIR"/"${runfilename} # Remove actual script fi # Copy the new multi_report.sh file and set permissions cp /tmp/multi_report_update.txt $SCRIPT_DIR"/"$runfilename chmod 755 $SCRIPT_DIR"/"$runfilename # Copy the User Guide echo "1" sleep .5 cp "/tmp/Multi-Report/Multi_Report_User_Guide.pdf" $SCRIPT_DIR"/." cp "/tmp/Multi-Report/changelog.txt" $SCRIPT_DIR"/." echo " " echo "Your script has been updated and a new copy of the User Guide and changelog is in your directory." sleep 1 # Cleanup Leftover Files if test -e "/tmp/Multi-Report"; then rm -R "/tmp/Multi-Report"; fi if test -e "/tmp/multi_report_update.txt"; then rm "/tmp/multi_report_update.txt"; fi else echo "Aborted" if test -e "/tmp/Multi-Report"; then rm -R "/tmp/Multi-Report"; fi exit 0 fi else echo "GitHub is Not Available" fi if [[ $AutomaticUpdate != "true" ]]; then echo "Exiting..." exit 0 fi } ########## CHECK FOR NEWER SCRIPT ########## # This will check GitHub to find out if the script is newer. # If it is newer then it will send a message to the user via email. # It's up to the user if they want the update or not. checkforupdate () { echo "Checking for Updates" if test -e "/tmp/Multi-Report"; then rm -R "/tmp/Multi-Report"; fi if [[ "$(curl -is https://github.com | head -n 1)" ]]; then # Go git the file ( cd /tmp git clone -q https://github.com/JoeSchmuck/Multi-Report.git ) # Examine the file name in 'current_script' for version and date. if test -e "/tmp/Multi-Report/current_script"; then GitVersion="$(cat "/tmp/Multi-Report/current_script" | cut -d 'v' -f 2 | cut -d '_' -f 1)" # For the Development distribution if [[ $GitHubSimulate == "true" ]]; then if test -e "/tmp/Multi-Report/current_script1"; then GitVersion="$(cat "/tmp/Multi-Report/current_script1" | cut -d 'v' -f 2 | cut -d '_' -f 1)" fi fi VersionFilename="$(echo $CurrentFilename | cut -d 'v' -f 2 | cut -d '_' -f 1)" echo "Current Version "$VersionFilename" -- GitHub Version "$GitVersion if [[ $GitVersion > $VersionFilename ]] && [[ $VersionFilename != "" ]]; then UpdateAvailable="true" echo "Update Available -- Use the '-update' switch to update the script." else echo "No Update Required" fi else echo "No GitHub Version File Available" fi else echo "GitHub.com Not Available" fi if test -e "/tmp/Multi-Report/messages.txt"; then Messages="$(cat "/tmp/Multi-Report/messages.txt")" if [[ $Messages != "" ]]; then echo "Message from the Creator"; fi else echo "No GitHub Message Available" fi } ########## LOAD EXTERNAL CONFIGURATION FILE ########## load_config () { if test -e "$Config_File_Name"; then . "$Config_File_Name" # Lets test if the config file needs to be updated first. config_version_date="$(cat "$Config_File_Name" | grep "dtd" | cut -d ':' -f 2 | cut -d ' ' -f 1 )" if [[ $config_version_date < $valid_config_version_date ]]; then echo "Found Old Configuration File"; echo "Automatically updating configuration file..."; update_config_file; echo "Continuing to run script"; fi . "$Config_File_Name" else echo " " echo " No Config File Exists --- Checking for a valid email within the script..." if [[ $Email == "YourEmail@Address.com" ]]; then echo " " echo " No Valid Email Address..." echo " Recommend running script with the '-config' switch and selecting" echo " the N)ew Configuration option." echo " " echo "... Exiting" echo " " exit 1 else echo "Valid Email within the script = "$Email", using script parameters..." echo " " External_Config="no" return fi fi } ########## CLEAR VARIABLES ########## # Setup variables for each drive pass. clear_variables () { altlastTestHours="" altlastTestType="" capacity="" chkreadfailure="" crcErrors="" crcErrorsOrig="" devicetype="" Helium="" He2="" lastTestHours="" lastTestType="" loadCycle="" modelnumber="" multiZone="" multiZoneOrig="" nvm_power_level="" NVMcriticalWarning="" NVMelastTestHoursFake="" offlineUnc="" onHours="" onTime="" pending="" rawReadErrorRate="" rawReadErrorRate2="" reAlloc="" reAllocEvent="" reAllocEventOrig="" reAllocOrig="" rotation="" seek="" seekErrorHealth="" seekErrorHealth2="" seekErrorRate="" SER="" serial="" serial1="" smartStatus="" smarttesting="" spinRetry="" startStop="" tdw="" temp="" temp_max="" temp_min="" testAge="" test_ata_error="" WarrantyClock="" warrantytemp="" wearLevel="" wearLevelAdj="" # And Reset bgColors if [[ "$bgColor" == "$altColor" ]]; then bgColor="#ffffff"; else bgColor="$altColor"; fi crcErrorsColor=$bgColor deviceStatusColor=$bgColor HeliumColor=$bgColor lastTestTypeColor=$bgColor mediaErrorsColor=$bgColor multiZoneColor=$bgColor NVMcriticalWarningColor=$bgColor offlineUncColor=$bgColor onTimeColor=$bgColor pendingColor=$bgColor rawReadErrorRateColor=$bgColor reAllocColor=$bgColor reAllocEventColor=$bgColor seekErrorHealthColor=$bgColor smartStatusColor=$bgColor spinRetryColor=$bgColor tempColor=$bgColor temp_maxColor=$bgColor testAgeColor=$bgColor WarrantyBackgroundColor=$bgColor WarrantyBoxColor="black" wearLevelColor=$bgColor } ########## CONVERT TO DECIMAL ########## # Convert any number into decimal format convert_to_decimal () { if [[ "$1" == "" ]]; then return; fi Converting_Value=${1#0} Converting_Value="${Converting_Value//,}" Return_Value=$Converting_Value if [[ $1 == "0" ]]; then Return_Value=0; fi } ########## SORT DRIVES ROUTINE for CORE ########## # Sort drives into alphabetical order. sort_drives () { sort_list=$(for i in `echo $sort_list`; do echo "$i" done | sort -V) } ########## SORT DRIVES ROUTINE for SCALE ########## # Sort drives into alphabetical order. sort_drives_scale () { sort_list_short=$(for i in `echo $sort_list`; do echo "$i" done | awk 'length<4' | sort -V) sort_list_long=$(for i in `echo $sort_list`; do echo "$i" done | awk 'length>3' | sort -V) sort_list=$sort_list_short" "$sort_list_long sort_list="$(echo $sort_list | tr -s " ")" } ########## REMOVE DUPLICATE DRIVE SERIAL NUMBERS - MULTIPATH FIX ########## remove_duplicate () { # Passed Variable is 'duplicate_list' good_serial="" good_drive="" bad_drive="" bad_serial="" good_gptid="" bad_gptid="" s_serial="" d_serial="" u_serial="" # Get drive serial number for i in ${duplicate_list}; do # Drive ID's smartdata5="$(smartctl -x --json=u /dev/"$i")" # Need to put simulated smartdata5 here. dup_test="$(echo "${smartdata5}" | jq -Mre '.serial_number | values')" # Drive Serial Number # Get the gptid if [ $softver != "Linux" ]; then # For FreeBSD gptid_test="$(glabel status | tail -n +2 | grep " $i" | cut -d '/' -f2 | cut -d ' ' -f1)" else # For Debian gptid_test="$(lsblk -o +PARTUUID,NAME,LABEL | grep "$i" | egrep -v "swap" | grep "part" | awk '{print $7}' | tail -n1)" fi # Save some data for Multipath='serial' s_serial=$s_serial" "$dup_test d_serial=$d_serial" "$i ### Remove this after working script #if [[ $i == ada0 ]]; then dup_test="12351"; gptid_test="01"; fi #if [[ $i == ada1 ]]; then dup_test="12345"; gptid_test="04"; fi #if [[ $i == ada2 ]]; then dup_test="12342"; gptid_test="02"; fi #if [[ $i == ada3 ]]; then dup_test="12345"; gptid_test="04"; fi # 'normal' runs well here. Flag duplicate serial numbers and record those and gptid if [[ "$Multipath" != "serial" ]]; then if [[ ! "$good_serial" == *"$dup_test"* ]]; then #good_serial=$good_serial" "$dup_test if [[ $good_drive == "" ]]; then good_drive=$i good_gptid=$gptid_test good_serial=$dup_test else good_drive=$good_drive" "$i good_gptid=$good_gptid" "$gptid_test good_serial=$good_serial" "$dup_test fi else bad_serial=$bad_serial" "$dup_test bad_gptid=$bad_gptid" "$gptid_test # Check against good_gptid if [[ "$Multipath" != "off" && "$Multipath" != "normal" ]]; then # Do the script if we are ensuring no duplicate gptid. if [[ "$good_gptid" == *"$gptid_test"* ]]; then # This gptid exists in the good_gptid table, we can ignore this one. if [[ $bad_drive == "" ]]; then bad_drive=$i bad_gptid=$gptid_test else bad_drive=$bad_drive" "$i bad_gptid=$bad_gptid" "$gptid_test fi else # ELSE this is a second gptid on the same drive, lets keep it. good_drive=$good_drive" "$i good_gptid=$good_gptid" "$gptid_test fi else # ELSE No duplicate serial numbers, no checking of gptid at all. if [[ $bad_drive == "" ]]; then bad_drive=$i bad_gptid=$gptid_test else bad_drive=$bad_drive" "$i bad_gptid=$bad_gptid" "$gptid_test fi fi fi else good_serial=$good_serial" "$dup_test fi done if [[ "$Multipath" == "serial" ]]; then # Sort by serial number u_serial=$(echo $s_serial | tr -s " ") sort_list=$s_serial if [ $softver != "Linux" ]; then sort_drives else sort_drives_scale fi s_serial=$sort_list d_serial=$(echo $d_serial | tr -s " ") # We have serial numbers sorted. Now look up the drives by serial number for k in ${s_serial}; do # Not used except to loop the number of times for serial numbers. st="1" good_test="" for j in ${u_serial}; do # Rotate through all the sorted serial numbers, the order they are displayed good_test="$(echo $u_serial | awk '{print $var}' var="${st}")" if [[ "$j" == "$k" ]]; then good_test="$(echo $d_serial | awk '{print $var}' var="${st}")" good_drive=$good_drive" "$good_test continue else st=$((${st} + 1)) fi done done fi if [[ "$Develop" == "true" ]]; then echo "----" echo "good_drive="$good_drive echo "good_serial="$good_serial echo "good_gptid="$good_gptid echo "---" echo "bad_drive="$bad_drive echo "bad_serial="$bad_serial echo "bad_gptid="$bad_gptid fi duplicate_list=$good_drive } ########## CONVERT TEST DRIVE NUMBERS INTO SCALE DRIVE NAMES ########## # This will convert ada"1" into sd"a", ada"26" into sd"z", etc. # Incoming and outgoing variable is "numbertoletters" number_to_letters () { declare -i j=$numbertoletters+1 if [[ $j -gt 52 && $j -lt 79 ]]; then # First Character 'b' declare -i scaleid=$(($j-52)) firstletter="b" declare -i k=$[ 96+${scaleid} ] secondletter=$(printf \\$(printf '%03o' $k)) fi if [[ $j -gt 26 && $j -lt 53 ]]; then # First Character 'a' declare -i scaleid=$(($j-26)) firstletter="a" declare -i k=$[ 96+${scaleid} ] secondletter=$(printf \\$(printf '%03o' $k)) fi numbertoletters=$firstletter$secondletter if [[ $j -lt 27 ]]; then declare -i k=$[ 96+${j} ] firstletter=$(printf \\$(printf '%03o' $k)) numbertoletters=$firstletter fi } ########## PURGE OLD DATA FROM CSV FILE ########## # This routine will purge the "statistical_data_file" of data older then "SDF_DataPurgeDays". purge_exportdata () { # Delete temp file if it exists if test -e "/tmp/temp_purge_file.csv"; then rm "/tmp/temp_purge_file.csv"; fi if [ $softver != "Linux" ]; then expireDate=$(date -v -"$SDF_DataPurgeDays"d +%Y/%m/%d) else expireDate=$(date -d "$SDF_DataPurgeDays days ago" +%Y/%m/%d) fi awk -v expireDate="$expireDate" -F, '{ if($1 >= expireDate) print $0;}' "$statistical_data_file" > "/tmp/temp_purge_file.csv" cp -R "/tmp/temp_purge_file.csv" "$statistical_data_file" } ########## PURGE TEST DATA FROM CSV FILE ########## # This routine will purge the "statistical_data_file" of test data matching "TEST". purge_testdata () { echo "Purging Statistical Database of Test Data" # Delete temp file if it exists if test -e "/tmp/temp_purge_file.csv"; then rm "/tmp/temp_purge_file.csv"; fi awk -F, '{ if($3 != "TEST") print $0;}' "$statistical_data_file" > "/tmp/temp_purge_file.csv" cp -R "/tmp/temp_purge_file.csv" "$statistical_data_file" } ########## CLEAN UP TEMPORARY FILES ########## cleanup_files () { ### Clean up our temporary files if test -e "/tmp/temp_purge_file.csv"; then rm "/tmp/temp_purge_file.csv"; fi rm /tmp/${tempfilepath}* > /dev/null 2>&1 ### Clean up multiple drive data files f=(/tmp/*_a.txt) if [[ -f "${f[0]}" ]]; then rm /tmp/*_a.txt; fi f=(/tmp/*_x.txt) if [[ -f "${f[0]}" ]]; then rm /tmp/*_x.txt; fi f=(/tmp/*.json) if [[ -f "${f[0]}" ]]; then rm /tmp/*.json; fi if [[ -f "/tmp/selftestlog" ]]; then rm /tmp/selftestlog; fi ### Clean up individual files if test -e "/tmp/${Config_Name}.db"; then rm "/tmp/${Config_Name}.db"; fi if test -e /tmp/config_backup.md5; then rm /tmp/config_backup.md5; fi if test -e /tmp/config_backup.sha256; then rm /tmp/config_backup.sha256; fi if test -e "/tmp/${Config_Name}.zip"; then rm "/tmp/$Config_Name.zip"; fi if test -e "/tmp/${Config_Name}.tar"; then rm "/tmp/$Config_Name.tar"; fi if test -e "/tmp/freenas-v1.db"; then rm "/tmp/freenas-v1.db"; fi if test -e "/tmp/freenas-v1.md5"; then rm "/tmp/freenas-v1.md5"; fi if test -e "/tmp/freenas-v1.sha256"; then rm "/tmp/freenas-v1.sha256"; fi if test -e "/tmp/pwenc_secret"; then rm "/tmp/pwenc_secret"; fi if test -e "/tmp/pwenc_secret.md5"; then rm "/tmp/pwenc_secret.md5"; fi if test -e "/tmp/pwenc_secret.sha256"; then rm "/tmp/pwenc_secret.sha256"; fi ### Clean up Spencer File if test -e "/tmp/spencer_report.txt"; then rm "/tmp/spencer_report.txt"; fi ### Clean up complete! } #################### EMAIL FUNCTIONS #################### ########## EMAIL EXPORT DATA CVS FILE ########## # Attach statistical data file Email_datafile () { if [ "$SDF_DataEmail" == "true" ]; then Now=$(date +"%a") doit="false" case $SDF_DataEmailDay in All) doit="true" ;; Mon|Tue|Wed|Thu|Fri|Sat|Sun) if [[ "$SDF_DataEmailDay" == "$Now" ]]; then doit="true"; fi ;; Month) if [[ $(date +"%d") == "01" ]]; then doit="true"; fi ;; *) ;; esac if [[ "$doit" == "true" ]]; then Attach_Statistical_File="true"; Attach_Files1="true"; else Attach_Statistical_File="false"; fi doit="" fi if [[ "$MRConfigEmailEnable" == "true" ]]; then if [[ "$MR_Attach_Config" == "1" ]]; then doit="true"; fi Now=$(date +"%a") case $MRConfigEmailDay in All) doit="true" ;; Mon|Tue|Wed|Thu|Fri|Sat|Sun) if [[ "$MRConfigEmailDay" == "$Now" ]]; then doit="true"; fi ;; Month) if [[ $(date +"%d") == "01" ]]; then doit="true"; fi ;; Never) ;; *) ;; esac if [[ "$doit" == "true" ]] || [[ "$dump_all" != "0" ]]; then Attach_Multi_Report="true"; Attach_Files1="true"; else Attach_Multi_Report="false"; fi fi } ########## COMBINE ALL DATA INTO A FORMAL EMAIL MESSAGE AND SEND IT ########## # Create Email Header and Send Email create_Email () { # Test if there is a Warning Message and Setup Subject Line if [[ $logfile_critical != "" ]]; then subject=$Subject_Line_Critical elif [[ $logfile_warning != "" ]]; then subject=$Subject_Line_Warning elif [[ $DisableWarranty == "false" ]]; then if [[ ! $logfile_warranty == "" ]]; then subject=${host}" *Drive Warranty Expired* - SMART Testing Results" else subject=$Subject_Line_Normal fi else subject=$Subject_Line_Normal fi if [[ $Monitor == "true" ]]; then Email=$AlertEmail subject="${host} -> ALERT" fi ### Set email headers ### ( echo "MIME-Version: 1.0" echo "Content-Type: multipart/mixed; boundary=${boundary}" echo "From: ${From}" echo "To: ${Email}" echo "Subject: ${subject}" echo "--${boundary}" echo "Content-Type: text/html" if [[ $Monitor == "false" ]]; then if [[ "$(echo $programver | grep -i "beta")" ]] || [[ $UpdateAvailable == "true" ]]; then echo ""; fi echo $programver"
Report Run "$(date +%d-%b-%Y)" @ "$timestamp"
" duration=$SECONDS if [[ $duration < "60" ]]; then echo "Execution Time: $(($duration % 60)) Seconds" else echo "Execution Time: $(($duration / 60)) Minutes : $(($duration % 60)) Seconds" fi if [[ $UpdateAvailable == "true" ]]; then echo "
UPDATE AVAILABLE --> "$GitVersion; fi if [[ $Messages != "" ]]; then echo "
Message from the Creator: "$Messages"
"; fi if [[ "$(echo $programver | grep -i "beta")" ]] || [[ $UpdateAvailable == "true" ]]; then echo ""; fi if [[ $TrueNASConfigEmailEncryption != "" ]] && [[ $Config_Encrypted == "true" ]]; then echo "
Attached zip file is encrypted."; fi echo "

" # Keyboard_Message comes from the '-dump email' command. if [[ $Keyboard_Message != "" ]]; then echo ""$Keyboard_Message"

" fi fi ) > ${logfile_header} cat $logfile >> $logfile_header ### Send report sendmail -t -oi < "$logfile_header" } ########## TRUENAS CONFIGURATION BACKUP ########## # TrueNAS Configuration Backup (if enabled) config_backup () { if [[ "$Monitor" == "true" ]]; then return; fi ###### Config backup (if enabled) if [ "$TrueNASConfigEmailEnable" == "true" ]; then Now=$(date +"%a") doit="false" case $TrueNASConfigEmailDay in All) doit="true" ;; Mon|Tue|Wed|Thu|Fri|Sat|Sun) if [[ "$TrueNASConfigEmailDay" == "$Now" ]]; then doit="true" fi ;; Month) if [[ $(date +"%d") == "01" ]]; then doit="true" fi ;; *) ;; esac if [[ "$doit" == "true" ]]; then # Set up file names, etc for later tarfile="/tmp/config_backup.zip" ### Test config integrity if ! [ "$(sqlite3 /data/freenas-v1.db "pragma integrity_check;")" == "ok" ]; then # Config integrity check failed, set MIME content type to html and print warning ( echo "--${boundary}" echo "Content-Type: text/html" echo "Automatic backup of FreeNAS configuration has failed! The configuration file is corrupted!" echo "You should correct this problem as soon as possible!" echo "
" ) >> "$logfile" else Attach_TrueNAS_Config="true" Attach_Files1="true" # Config integrity check passed; copy config db, generate checksums, make .zip archive cp /data/freenas-v1.db "/tmp/freenas-v1.db" cp /data/pwenc_secret "/tmp/pwenc_secret" if [ $softver != "Linux" ]; then md5 "/tmp/freenas-v1.db" > /tmp/freenas-v1.md5 sha256 "/tmp/freenas-v1.db" > /tmp/freenas-v1.sha256 md5 "/tmp/pwenc_secret" > /tmp/pwenc_secret.md5 sha256 "/tmp/pwenc_secret" > /tmp/pwenc_secret.sha256 else md5sum "/tmp/freenas-v1.db" > /tmp/freenas-v1.md5 sha256sum "/tmp/freenas-v1.db" > /tmp/freenas-v1.sha256 md5sum "/tmp/pwenc_secret" > /tmp/pwenc_secret.md5 sha256sum "/tmp/pwenc_secret" > /tmp/pwenc_secret.sha256 fi if [[ $TrueNASConfigEmailEncryption != "" ]]; then Config_Name="$(date "+"$host"_Config_"$programver3"_%Y-%m-%d.zip")" Config_Encrypted="true" else Config_Name="$(date "+"$host"_Config_"$programver3"_%Y-%m-%d.tar")" fi ( cd "/tmp/" || exit; if [[ $TrueNASConfigEmailEncryption != "" ]]; then if [ $softver != "Linux" ]; then # Core - Encrypt 7z a "${Config_Name}" ./freenas-v1.db ./pwenc_secret -tzip -mem=AES256 -mx1 -p${TrueNASConfigEmailEncryption} > /dev/null 2<&1 else # Scale - Encrypt (using local copy) 7z a "${Config_Name}" ./freenas-v1.db ./pwenc_secret -tzip -mem=AES256 -mx1 -p${TrueNASConfigEmailEncryption} > /dev/null 2<&1 fi else if [ $softver != "Linux" ]; then # Core - UnEncrypted tar -czf "${Config_Name}" ./freenas-v1.db ./pwenc_secret > /dev/null 2<&1 else # Scale - UnEncrypted tar -czf "${Config_Name}" ./freenas-v1.db ./pwenc_secret > /dev/null 2<&1 fi fi ) # If logfile saving is enabled, copy .zip file to specified location before it (and everything else) is removed below if [ "$TrueNASConfigBackupSave" == "true" ]; then TrueNASConfigBackupLocation="$(echo $TrueNASConfigBackupLocation | sed 's:/*$::')" cp "/tmp/${Config_Name}" "${TrueNASConfigBackupLocation}/${Config_Name}" fi fi fi fi } ########## DUMP DRIVE DATA ########## # This routine will dump the selected drive data into individual files for troubleshooting. dump_drive_data () { if [[ "$drive" != "cd0" ]]; then tempjson="$(smartctl -x --json /dev/${drive})" serial1="$(echo "${tempjson}" | jq -Mre '.serial_number | values')" rpm="$(echo "${tempjson}" | jq -Mre '.rotation_rate | values')" if [[ "$rpm" == "0" ]]; then drivetype="SSD"; else drivetype="HDD"; fi if [[ "$(echo $tempjson | grep -i "nvm")" ]]; then drivetype="NVM"; fi ( if [[ $json_error_log != "" ]]; then if test -e "/tmp/${tempfilepath}${drive}_${serial1}_a.txt"; then echo "--${boundary}" echo "Content-Type: text/html" echo "Content-Transfer-Encoding: base64" echo "Content-Disposition: attachment; filename=${drive}_${serial1}_a.txt" base64 "/tmp/${tempfilepath}${drive}_${serial1}_a.txt" fi if test -e "/tmp/${tempfilepath}${drive}_${serial1}_x.txt"; then echo "--${boundary}" echo "Content-Type: text/html" echo "Content-Transfer-Encoding: base64" echo "Content-Disposition: attachment; filename=${drive}_${serial1}_x.txt" base64 "/tmp/${tempfilepath}${drive}_${serial1}_x.txt" fi fi if test -e "/tmp/${tempfilepath}${drivetype}_${drive}_${serial1}.json"; then echo "--${boundary}" echo "Content-Type: text/html" echo "Content-Transfer-Encoding: base64" echo "Content-Disposition: attachment; filename=${drivetype}_${drive}_${serial1}.json" base64 "/tmp/${tempfilepath}${drivetype}_${drive}_${serial1}.json" fi ) >> "$logfile" fi } #################### ZPOOL FUNCTIONS #################### ########## GENERATE ZPOOL REPORT ########## # Report Summary Section (html tables) zpool_report () { if [[ "$Monitor" == "false" ]]; then ( echo "" if [[ $Pool_Capacity_Type == "zfs" ]]; then echo "" else echo "" fi echo "" echo " " echo " " echo " " echo " " echo " " echo " " echo " " echo " " echo " " echo " " echo " " echo " " echo " " echo "" ) > "$logfile" fi # Let's find out the name of the pools and sort them. if [[ "$testdata" == "-t" ]]; then # Use IMPORTED test data echo "Zpool Does not Function Yet. Next Revision." ### Until is does function, use the system at hand for the data. pools=$(zpool list -H -o name) else pools=$(zpool list -H -o name) fi sort_list=$pools sort_drives pools=$sort_list poolNum=0 # Alternate background in pools table for pool in $pools; do # zpool health summary status="$(zpool list -H -o health "$pool")" # Total all read, write, and checksum errors per pool errors="$(zpool status "$pool" | grep -E "(ONLINE|DEGRADED|FAULTED|UNAVAIL|REMOVED)[ \\t]+[0-9]+")" readErrors=0 for err in $(echo "$errors" | awk '{print $3}'); do if echo "$err" | grep -E -q "[^0-9]+"; then readErrors=1000 break fi readErrors=$((readErrors + err)) done writeErrors=0 for err in $(echo "$errors" | awk '{print $4}'); do if echo "$err" | grep -E -q "[^0-9]+"; then writeErrors=1000 break fi writeErrors=$((writeErrors + err)) done cksumErrors=0 for err in $(echo "$errors" | awk '{print $5}'); do if echo "$err" | grep -E -q "[^0-9]+"; then cksumErrors=1000 break fi cksumErrors=$((cksumErrors + err)) done if [ "$readErrors" -gt 999 ]; then readErrors=">1K"; fi if [ "$writeErrors" -gt 999 ]; then writeErrors=">1K"; fi if [ "$cksumErrors" -gt 999 ]; then cksumErrors=">1K"; fi # Get ZFS capacity (the real capacity) zfs_pool_used="$(zfs list $pool | awk '{print $2}' | sed -e '/USED/d')" zfs_pool_avail="$(zfs list $pool | awk '{print $3}' | sed -e '/AVAIL/d')" if [[ $zfs_pool_used == *"T"* ]]; then zfs_pool_used1="$(awk -v a="$zfs_pool_used" 'BEGIN { printf a*1000 }' ")"; else statusColor="$bgColor"; fi if [ "$readErrors" != "0" ]; then readErrorsColor="$warnColor"; logfile_warning=$logfile_warning"$(echo "$pool - Scrub Read Errors
")"; else readErrorsColor="$bgColor"; fi if [ "$writeErrors" != "0" ]; then writeErrorsColor="$warnColor"; logfile_warning=$logfile_warning"$(echo "$pool - Scrub Write Errors
")"; else writeErrorsColor="$bgColor"; fi if [ "$cksumErrors" != "0" ]; then cksumErrorsColor="$warnColor"; logfile_warning=$logfile_warning"$(echo "$pool - Scrub Cksum Errors
")"; else cksumErrorsColor="$bgColor"; fi if [ "$used" -ge "$PoolUsedWarn" ]; then usedColor="$warnColor"; logfile_warning=$logfile_warning"$(echo "$pool - Scrub Used
")"; else usedColor="$bgColor"; fi if [ "$scrubRepBytes" != "$Non_Exist_Value" ] && [ "$scrubRepBytes" != "0" ] && ! [ "$(echo "$scrubRepBytesfull" | grep "Resilvered")" ]; then scrubRepBytesColor="$warnColor"; logfile_warning=$logfile_warning"$(echo "$pool - Scrub Rep Bytes
")"; else scrubRepBytesColor="$bgColor"; fi if [ "$scrubErrors" != "$Non_Exist_Value" ] && [ "$scrubErrors" != "0" ]; then scrubErrorsColor="$warnColor"; logfile_critical=$logfile_critical"$(echo "$pool - Scrub Errors
")"; else scrubErrorsColor="$bgColor"; fi if [ "$(echo "$scrubAge" | awk '{print int($1)}')" -ge "$ScrubAgeWarn" ]; then scrubAgeColor="$warnColor"; logfile_warning=$logfile_warning"$(echo "$pool - Scrub Age
")"; else scrubAgeColor="$bgColor"; fi if [ "$scrubAge" == "In Progress" ]; then scrubAgeColor="$blueColor"; fi if [ "$frag" != "$Non_Exist_Value" ]; then if [ "$frag" -ge "$ZpoolFragWarn" ]; then fragColor="$warnColor"; logfile_warning=$logfile_warning"$(echo "$pool - Fragmentation above Threshold - $frag%
")"; else fragColor="$bgColor"; fi frag=" "$frag"%" fi if [[ $Pool_Capacity_Type == "zfs" ]]; then pool_size=$zfs_pool_size pool_free=$zfs_pool_avail used=$zfs_pool_used" ("$used"%)" else used=$pool_used" ("$used"%)" fi if [[ "$Monitor" == "false" ]]; then ( # Use the information gathered above to write the date to the current table row printf "\\n" "$bgColor" "$pool" "$statusColor" "$status" "$pool_size" "$pool_free" "$usedColor" "$used" "$fragColor" "$frag" "$readErrorsColor" "$readErrors" "$writeErrorsColor" "$writeErrors" "$cksumErrorsColor" \ "$cksumErrors" "$scrubRepBytesColor" "$scrubRepBytesfull" "$scrubErrorsColor" "$scrubErrors" "$scrubAgeColor" "$scrubAge" "$scrubTime" ) >> "$logfile" fi done # End of zpool status table if [[ "$Monitor" == "false" ]]; then echo "
*ZPool/ZFS Status Report Summary
*ZPool Status Report Summary
"$Zpool_Pool_Name_Title""$Zpool_Status_Title""$Zpool_Pool_Size_Title""$Zpool_Free_Space_Title""$Zpool_Used_Space_Title""$Zpool_Frag_Title""$Zpool_Read_Errors_Title""$Zpool_Write_Errors_Title""$Zpool_Checksum_Errors_Title""$Zpool_Scrub_Repaired_Title""$Zpool_Scrub_Errors_Title""$Zpool_Scrub_Age_Title""$Zpool_Scrub_Duration_Title"
%s %s %s %s %s %s %s %s %s %s %s %s %s
" >> "$logfile" if [[ $Pool_Capacity_Type == "zfs" ]]; then echo "
*Data obtained from zpool and zfs commands." >> "$logfile" else echo "
*Data obtained from zpool command. Capacities include Parity Data." >> "$logfile" fi fi } #################### DRIVE DATA FUNCTIONS #################### ########## IGNORE DRIVES ROUTINE ########## # Examine the $Ignore_Drives_List variable and remove any matches from the $drive variable. process_ignore_drives () { targument="$(smartctl -i /dev/"${drive}" | grep "Serial Number:" | awk '{print $3}')"; s="0" IFS=',' read -ra ADDR <<< "$Ignore_Drives_List" for i in "${ADDR[@]}"; do if [[ $i == $targument ]]; then s="1"; continue; fi done if [[ $s == "0" ]]; then printf "%s " "${drive}"; fi } ########## GET SMART HARD DRIVES ########## # Get listing of the SMART HDD's and put into $smartdrives variable - example: 'ada0 ada1 ada2 ada3' OR 'sda sdb sdc sdd', in the order desired. # The output is smartdrives sorted however we desire. get_smartHDD_listings () { if [ $softver != "Linux" ]; then smartdrives=$(for drive in $(sysctl -n kern.disks); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') else smartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-15 | cut -d ":" -f1 | tr '\n' ' '); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && ! [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') fi > /dev/null 2<&1 # Sort drive idents sort_list=$smartdrives if [ $softver != "Linux" ]; then sort_drives else sort_drives_scale fi smartdrives=$sort_list # Call Sort Routine with the drive string. if [[ "$smartdrives" != "" ]]; then if [[ "$Multipath" != "off" ]]; then # 'off' means skip the remove_duplicate code duplicate_list=$smartdrives remove_duplicate # Remove duplicate serial numbers for MultiPath smartdrives=$duplicate_list fi fi } ########## GET SMART SOLID DISK DRIVES ########## # Get listing of the SMART SSD's and put into $smartdrivesSSD variable # The output is smartdrivesSSD sorted however we desire. get_smartSSD_listings () { if [ $softver != "Linux" ]; then smartdrivesSSD=$(for drive in $(sysctl -n kern.disks); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') else smartdrivesSSD=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-15 | cut -d ":" -f1 | tr '\n' ' '); do if [ "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ "$(smartctl -i /dev/"${drive}" | grep "Solid State Device")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') fi > /dev/null 2<&1 # Sort Drive Idents sort_list=$smartdrivesSSD if [ $softver != "Linux" ]; then sort_drives else sort_drives_scale fi smartdrivesSSD=$sort_list # Call Sort Routine with the drive string. if [[ "$smartdrivesSSD" != "" ]]; then if [[ "$Multipath" != "off" ]]; then duplicate_list=$smartdrivesSSD remove_duplicate # Remove duplicate serial numbers smartdrivesSSD=$duplicate_list fi fi } ########## GET NVMe DRIVES ########## # Get listing of the SMART NVMe's and put into $smartdrivesNVM variable # The output is smartdrivesNVM sorted however we desire. get_smartNVM_listings () { if [ $softver != "Linux" ]; then smartdrivesNVM=$(for drive in $(sysctl -n kern.disks); do if [ "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') ### Convert nvdx to nvmexx in smartdrivesNVM ### else smartdrivesNVM=$(for drive in $(fdisk -l | grep "Disk /dev/nvm" | cut -d ':' -f 1 | cut -d '/' -f 3 | tr '\n' ' '); do if [ "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') ### Convert nvdx to nvmexx in smartdrivesNVM ### fi > /dev/null 2<&1 ### Convert nvdx to nvmexx in smartdrivesNVM ### smartdrivesNVM=$( echo "$smartdrivesNVM" | sed 's/nvd/nvme/g' ) ### Convert nvme0n1 to nvme0, or nvme34n1 to nvme34 to make compatible with Smartmontools 7.4 ### Maybe this can go away with version 7.5? for smartdrivesnvme in $smartdrivesNVM; do nvme_drive=$nvme_drive$(echo "nvme"$(echo $smartdrivesnvme | sed -r 's#^nvme##' | cut -d 'n' -f 1)" ") done smartdrivesNVM=$nvme_drive # Sort Drive Idents sort_list=$smartdrivesNVM if [ $softver != "Linux" ]; then sort_drives else sort_drives_scale fi smartdrivesNVM=$sort_list # Call Sort Routine with the drive string. if [[ "$smartdrivesNVM" != "" ]]; then if [[ "$Multipath" != "off" ]]; then duplicate_list=$smartdrivesNVM remove_duplicate # Remove duplicate serial numbers smartdrivesNVM=$duplicate_list fi fi } ########## GET OTHER SMART DEVICES ########## # Get listing of the OTHER-SMART devices and put into $non-smartdrives variable get_smartOther_listings () { ### Get the non-SSD listing - MUST support SMART # variable nonsmartdrives if [ $softver != "Linux" ]; then nonsmartdrives=$(for drive in $(sysctl -n kern.disks); do if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is:.\s*Enabled")" ] && [ ! "$(smartctl -i /dev/"${drive}" | grep "NVM")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') else nonsmartdrives=$(for drive in $(fdisk -l | grep "Disk /dev/sd" | cut -c 11-13 | tr '\n' ' '); do if [ ! "$(smartctl -i /dev/"${drive}" | grep "SMART support is: Enabled")" ]; then process_ignore_drives; fi done | awk '{for (i=NF; i!=0 ; i--) print $i }' | tr ' ' '\n' | sort | tr '\n' ' ') fi > /dev/null 2<&1 # Sort Drive Idents sort_list=$nonsmartdrives if [ $softver != "Linux" ]; then sort_drives else sort_drives_scale fi nonsmartdrives=$sort_list # Call Sort Routine with the drive string. if [[ "$nonsmartdrives" != "" ]]; then if [[ "$Multipath" != "false" ]]; then duplicate_list=$nonsmartdrives remove_duplicate # Remove duplicate serial numbers nonsmartdrives=$duplicate_list fi fi } ########## SAVE JSON DATA ########## # Rename function to "save_json_data" get_json_data () { # Save drive JSON data if -dump command used # Uses $drive parameter. if [[ "$drive" != "cd0" ]]; then if [[ "$dump_all" != "0" ]]; then # Pull a fresh JSON listing tempjson="$(smartctl -x --json=u /dev/${drive})" serial1="$(echo "${tempjson}" | jq -Mre '.serial_number | values')" rpm="$(echo "${tempjson}" | jq -Mre '.rotation_rate | values')" if [[ "$rpm" == "0" ]]; then drivetype="SSD"; else drivetype="HDD"; fi if [[ "$(echo $tempjson | grep -i "nvm")" ]]; then drivetype="NVM"; fi echo "$tempjson" > "/tmp/${tempfilepath}${drivetype}_${drive}_${serial1}.json" fi fi } ########## GET DRIVE DATA ########## # Gets drive data for HDD/SSD/NVM drives. # $1 = HDD/SSD/NVM, $smartdrives* must have drive ident data as well. get_drive_data () { # File /tmp/json_errors.txt is used to write what data did not come from a json file. if [[ $Develop == "true" ]]; then echo " " echo "Looking for "$drive echo " " fi if [[ "$testfilepath" != "" ]]; then # Import simulated drive data. # Find matching "$drive" to file in $smartdrives ${testfilenames[i]} smartdata5="" if [[ "$1" == "HDD" ]]; then smartdriveswc="$(echo ${smartdrives[*]} | wc -w)" if [[ $Develop == "true" ]]; then echo "Number of words in this run to scan = "$smartdriveswc; fi for i in $(seq 0 $smartdriveswc); do if [[ "${smartdrives[$i]}" == "${drive}" ]]; then if [[ $Develop == "true" ]]; then echo "We have a match"; fi smartdata5="$(cat ${testfilenamesHDD[$i]})" echo "/dev/$drive -- ${testfilenamesHDD[i]}" break fi done fi if [[ "$1" == "SSD" ]]; then smartdriveswc="$(echo ${smartdrivesSSD[*]} | wc -w)" if [[ $Develop == "true" ]]; then echo "Number of words in this run to scan = "$smartdriveswc; fi for i in $(seq 0 $smartdriveswc); do if [[ "${smartdrivesSSD[$i]}" == "${drive}" ]]; then if [[ $Develop == "true" ]]; then echo "We have a match"; fi smartdata5="$(cat ${testfilenamesSSD[$i]})" echo "/dev/$drive -- ${testfilenamesSSD[i]}" break fi done fi if [[ "$1" == "NVM" ]]; then smartdriveswc="$(echo ${smartdrivesNVM[*]} | wc -w)" if [[ $Develop == "true" ]]; then echo "Number of words in this run to scan = "$smartdriveswc; fi for i in $(seq 0 $smartdriveswc); do if [[ "${smartdrivesNVM[$i]}" == "${drive}" ]]; then if [[ $Develop == "true" ]]; then echo "We have a match"; fi smartdata5="$(cat ${testfilenamesNVM[$i]})" echo "/dev/$drive -- ${testfilenamesNVM[i]}" break fi done fi ######################################## FIX ME ############################################################## # NON Option - This needs to be changed to the non-smart drive list. if [[ "$1" == "NON" ]]; then smartdriveswc="$(echo ${smartdrivesNVM[*]} | wc -w)" if [[ $Develop == "true" ]]; then echo "Number of words in this run to scan = "$smartdriveswc; fi for i in $(seq 0 $smartdriveswc); do if [[ "${smartdrivesNVM[$i]}" == "${drive}" ]]; then if [[ $Develop == "true" ]]; then echo "We have a match"; fi smartdata5="$(cat ${testfilenamesNVM[$i]})" echo "/dev/$drive -- ${testfilenamesNVM[i]}" break fi done fi ############################################################################################################### if [[ "$testfile2" != "" ]]; then smartdata="$(cat "$testfile2")" else smartdata="" fi else # Else let's read the Real Drive Data if [[ $Develop == "true" ]]; then echo "Looking at Real Drive Data, Not Simulated"; fi # Get drive data for json processing smartdata5="$(smartctl -x --json=u /dev/"$drive")" # Get drive data old school for what isn't in json yet smartdata="$(smartctl -a /dev/"$drive")" # Save data for -dump routine fi # Get Serial Number if [[ "$(echo "${smartdata5}" | jq -Mre '.serial_number | values')" ]]; then serial="$(echo "${smartdata5}" | jq -Mre '.serial_number | values')" else if [[ "$(echo "${smartdata5}" | grep -i "vmware")" ]]; then serial="VMWare"; fi json_error_log=$json_error_log"\n$(echo $drive" .serial_number not in json")" > /dev/null 2<&1 fi > /dev/null 2<&1 # Get Model Number if [[ "$(echo "${smartdata5}" | jq -Mre '.model_name | values')" ]]; then modelnumber="$(echo "${smartdata5}" | jq -Mre '.model_name | values')"; elif [[ "$(echo "${smartdata5}" | jq -Mre '.scsi_model_name | values')" ]]; then modelnumber="$(echo "${smartdata5}" | jq -Mre '.scsi_model_name | values')"; else json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .model_name not in json")"; fi > /dev/null 2<&1 # Get Capacity if [[ "$(echo "${smartdata5}" | jq -Mre '.logical_block_size | values')" ]]; then capacity1="$(echo "${smartdata5}" | jq -Mre '.logical_block_size | values')"; else json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .logical_block_size not in json")"; fi > /dev/null 2<&1 if [[ "$(echo "${smartdata5}" | jq -Mre '.user_capacity.blocks | values')" ]]; then capacity2="$(echo "${smartdata5}" | jq -Mre '.user_capacity.blocks | values')"; else json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .user_capacity.blocks not in json")"; fi > /dev/null 2<&1 # Calculate Capacity and adjust for GB/TB capacity=$(( ${capacity1} * ${capacity2} )) capacity="$(( capacity/1000000000 ))" if [[ $capacity -gt 999 ]]; then capacity="$(awk -v a="$capacity" 'BEGIN { printf "%.2f", a/1000 }' /dev/null 2<&1 # Get SMART Status if [[ "$(echo "${smartdata5}" | jq -Mre '.smart_status.passed | values')" ]]; then if [ "$(echo "${smartdata5}" | jq -Mre '.smart_status.passed | values')" = "true" ]; then smartStatus="PASSED" else smartStatus="FAILED" fi else json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .smart_status.passed not in json")" fi > /dev/null 2<&1 # Get Temperature - Normal if [[ "$(echo "${smartdata5}" | jq -Mre '.temperature.current | values')" ]]; then temp="$(echo "${smartdata5}" | jq -Mre '.temperature.current | values')" fi # Check if invalid json temp value - If yes then check ID 194 on json data. if [[ $temp < 1 ]]; then if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 194) | .raw.value | values')" ]]; then temp="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 194) | .raw.value | values')" # If Temp is still not there, look at the raw -x data and hope it shows up. else json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .temperature.current not in json")" if [[ "$(smartctl -x /dev/"$drive" | grep "Current Temperature:" | awk '{print $3}' | cut -d '/' -f2)" != "?" ]]; then temp=$(smartctl -x /dev/"$drive" | grep "Current Temperature:" | awk '{print $3}' | cut -d '/' -f2) else temp=$Non_Exist_Value fi fi > /dev/null 2<&1 fi # Get Temperature - Minimum for Power Cycle if [[ "$(echo "${smartdata5}" | jq -Mre '.temperature.power_cycle_min | values')" ]]; then temp_min="$(echo "${smartdata5}" | jq -Mre '.temperature.power_cycle_min | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 194) | .raw.string | values')" ]]; then tempstring="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 194) | .raw.string | values')" temp_min=$(echo "${tempstring}" | awk '{print $3}' | cut -d '/' -f1) temp_max=$(echo "${tempstring}" | awk '{print $3}' | cut -d '/' -f2 | tr -d ')') else temp_min=$Non_Exist_Value if [[ $1 == "HDD" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .temperature.power_cycle_min not in json")"; fi > /dev/null 2<&1 fi > /dev/null 2<&1 # Get Temperature - Maximum for Power Cycle if [[ "$(echo "${smartdata5}" | jq -Mre '.temperature.power_cycle_max | values')" ]]; then temp_max="$(echo "${smartdata5}" | jq -Mre '.temperature.power_cycle_max | values')" fi if [[ $temp_max == "" ]]; then temp_max=$Non_Exist_Value if [[ $1 == "HDD" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .temperature.power_cycle_max not in json")"; fi > /dev/null 2<&1 fi ### SETUP FOR NO VALUE OBTAINED if [[ $TempDisplaytemp != "" ]]; then TempDisplay=$TempDisplaytemp; fi TempDisplaymin=$TempDisplay TempDisplaymax=$TempDisplay if ! [[ $temp =~ $re ]]; then temp=$Non_Exist_Value; TempDisplaytemp=$TempDisplay; TempDisplay=""; fi if ! [[ $temp_min =~ $re ]]; then temp_min=$Non_Exist_Value; TempDisplaymin=""; fi if ! [[ $temp_max =~ $re ]]; then temp_max=$Non_Exist_Value; TempDisplaymax=""; fi # Get Power On Hours if [[ "$(echo "${smartdata5}" | jq -Mre '.power_on_time.hours | values')" ]]; then onHours="$(echo "${smartdata5}" | jq -Mre '.power_on_time.hours | values')" logfile_caution=$logfile_caution"$(printf "Drive "$serial" actual Power On Hours = "$onHours"
")" > /dev/null 2<&1 else json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .power_on_time.hours not in json")"; fi > /dev/null 2<&1 ####### Quick Fix for SCSI/SAS Drives Not reporting Hours ########################### ##################################################################################### #if [[ $drive == "sdc" ]]; then onHours=""; fi # Null the value for testing if [[ $onHours == "" ]]; then # We have to start the test now. In the future I want to collect a list of drives reporting NULL and then test all at once. maximum_loop_time=36 echo "Drive '"$drive"' does not report Power_On_Hours, using alternate method which can take up to "$(( maximum_loop_time * 5 ))" seconds." onHour_Zero_count=1 keep_testing="RUN" while [ $keep_testing == "RUN" ] do if [[ $onHour_Zero_count -gt 0 ]]; then smartctl -t short /dev/$drive > /dev/null 2<&1 onHour_Zero_count=0 fi if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.extended.table | values')" ]]; then start_test_value="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.extended.table[0].lifetime_hours | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.standard.table | values')" ]]; then start_test_value="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.standard.table[0].lifetime_hours | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_selective_self_test_log.table | values')" ]]; then start_test_value="$(echo "${smartdata5}" | jq -Mre '.ata_smart_selective_self_test_log.table[0].lifetime_hours | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table | values')" ]]; then start_test_value="$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table[0].power_on_hours | values')" fi if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.extended.table | values')" ]]; then end_test_value="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.extended.table[0].lifetime_hours | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.standard.table | values')" ]]; then end_test_value="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.standard.table[0].lifetime_hours | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_selective_self_test_log.table | values')" ]]; then end_test_value="$(echo "${smartdata5}" | jq -Mre '.ata_smart_selective_self_test_log.table[0].lifetime_hours | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table | values')" ]]; then end_test_value="$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table[0].power_on_hours | values')" fi if [[ $end_test_value != "NOW" ]] && [[ $end_test_value -gt $start_test_value ]]; then keep_testing="STOP"; fi maximum_loop_time=$(( $maximum_loop_time - 1 )) if [[ $maximum_loop_time -le 0 ]]; then break; fi done onHours=$end_test_value fi ####### END SAS/SCSI SECTION ###### # Setup for a Zero Hour Report, Yes in case someone has a zero hour value, the division will fail. if [[ $onHours == "" ]]; then onHours=0; fi if [[ $onHours == "0" ]]; then onHours=1; json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .power_on_time.hours = 0")"; logfile_caution=$logfile_caution"$(printf "Drive "$serial" actual Power On Hours = 0, added 1 hour so the script doesn't fail, if this persists it may be a script failure or unrecognized drive.
")"; fi > /dev/null 2<&1 # Get Start Stop Count if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 4) | .raw.value | values')" ]]; then startStop="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 4) | .raw.value | values')"; fi > /dev/null 2<&1 if [[ $startStop == "" ]] && [[ $1 == "HDD" ]]; then startStop="$(echo "${smartdata5}" | grep -i "Accumulated start-stop cycles:" | cut -d ':' -f 3 | tr -d ' ",')" json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 4 Start_Stop_Count not in json")"; fi > /dev/null 2<&1 # Get Load Cycle Count if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 193) | .raw.value | values')" ]]; then loadCycle="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 193) | .raw.value | values')"; fi > /dev/null 2<&1 if [[ $loadCycle == "" ]] && [[ $1 == "HDD" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 193 Load_Cycle_Count not in json")"; fi > /dev/null 2<&1 # Get Spin Retry Count if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 10) | .raw.value | values')" ]]; then spinRetry="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 10) | .raw.value | values')"; fi > /dev/null 2<&1 if [[ $spinRetry == "" ]] && [[ $1 == "HDD" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 10 Spin_Retry_Count not in json")"; fi > /dev/null 2<&1 # Get Reallocated Sectors Count if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 5) | .raw.value | values')" ]]; then reAlloc="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 5) | .raw.value | values')"; fi > /dev/null 2<&1 if [[ $reAlloc == "" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 5 Reallocated_Sector_Ct not in json")"; fi > /dev/null 2<&1 # Get Reallocated Sector Events Count if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 196) | .raw.value | values')" ]]; then reAllocEvent="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 196) | .raw.value | values')"; fi > /dev/null 2<&1 if [[ $reAllocEvent == "" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 196 Reallocated_Event_Count not in json")"; fi > /dev/null 2<&1 # Get Current Pending Sectors Count if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 197) | .raw.value | values')" ]]; then pending="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 197) | .raw.value | values')"; fi > /dev/null 2<&1 if [[ $pending == "" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 197 Current_Pending_Sector not in json")"; fi > /dev/null 2<&1 # Get Offline Uncorrectable Errors if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 198) | .raw.value | values')" ]]; then offlineUnc="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 198) | .raw.value | values')"; fi > /dev/null 2<&1 if [[ $offlineUnc == "" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 198 Offline_Uncorrectable not in json")"; fi > /dev/null 2<&1 if [[ "$(echo "${smartdata5}" | jq -Mre '.scsi_error_counter_log.write.total_uncorrected_errors | values')" ]]; then offlineUnc1="$(echo "${smartdata5}" | jq -Mre '.scsi_error_counter_log.write.total_uncorrected_errors | values')"; offlineUnc2="$(echo "${smartdata5}" | jq -Mre '.scsi_error_counter_log.read.total_uncorrected_errors | values')"; offlineUnc=$((( $offlineUnc1 + $offlineUnc2 ))); fi > /dev/null 2<&1 # Get UDMA CRC Errors if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 199) | .raw.value | values')" ]]; then crcErrors="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 199) | .raw.value | values')"; fi > /dev/null 2<&1 if [[ $crcErrors == "" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 199 UDMA_CRC_Error_Count not in json")"; fi > /dev/null 2<&1 # Get Read Error Rate if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 1) | .raw.value | values')" ]]; then rawReadErrorRate="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 1) | .raw.value | values')"; fi > /dev/null 2<&1 if [[ $rawReadErrorRate == "" ]] && [[ $1 == "HDD" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 1 Raw_Read_Error_Rate not in json")"; fi > /dev/null 2<&1 # Get Seek Error Rate if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 7) | .raw.value | values')" ]]; then seekErrorHealth="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 7) | .raw.value | values')"; fi > /dev/null 2<&1 if [[ $seekErrorHealth == "" ]] && [[ $1 == "HDD" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 7 Seek_Error_Rate not in json")"; fi > /dev/null 2<&1 # Get MultiZone Error Rate if [[ $1 == "HDD" ]]; then # Pass HDD Only, Ignore SSD & NVMe drives not on these drives, yet. if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 200) | .raw.value | values')" ]]; then multiZone="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 200) | .raw.value | values')"; fi > /dev/null 2<&1 if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 200) | .name | values')" == "Pressure_Limit" ]]; then multiZone=""; fi > /dev/null 2<&1 fi if [[ $multiZone == "" ]]; then if [[ "$(echo "${smartdata5}" | grep "Multi_Zone_Error_Rate")" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 200 Multi_Zone_Error not decoded properly")" fi fi > /dev/null 2<&1 # Get Helium Level he2="" # Are these two variables needed here? lastTestHours="" he2="$(echo "${smartdata5}" | grep -i "helium" | awk '{print $2}' | tr -d '",' | head -1)" if [[ "$he2" != "" ]]; then Helium="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "'${he2}'") | .raw.value | values')" heliumthresh="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "'${he2}'") | .thresh | values')" # Normalize Helium to 100. if [[ $heliumthresh > 50 ]]; then Helium=$(( 100 - $Helium )) # Helium is now Normalized where "100" is good. fi else if [[ "$(echo "${smartdata5}" | grep -i "helium")" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" Helium_Level not decoded properly")" #> /dev/null 2<&1 fi fi > /dev/null 2<&1 # Get SSD Total Data Written ssdtdw="" ssdblocksize="" ID241="" if [[ "$(echo "${smartdata5}" | jq -Mre '.logical_block_size | values')" ]]; then ssdblocksize="$(echo "${smartdata5}" | jq -Mre '.logical_block_size | values')" fi > /dev/null 2<&1 if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_device_statistics.pages[].table[] | select(.name == "Logical Sectors Written") | .value | values')" ]]; then ssdtdw="$(echo "${smartdata5}" | jq -Mre '.ata_device_statistics.pages[0].table[] | select(.name == "Logical Sectors Written") | .value | values')" fi > /dev/null 2<&1 if [[ $ssdtdw == "" ]]; then # If page 0 above fails, try no page ident if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_device_statistics.pages[].table[] | select(.name == "Logical Sectors Written") | .value | values')" ]]; then ssdtdw="$(echo "${smartdata5}" | jq -Mre '.ata_device_statistics.pages[].table[] | select(.name == "Logical Sectors Written") | .value | values')" fi > /dev/null 2<&1 fi if [[ $ssdtdw == "" ]]; then if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Total_Writes_GiB") | .raw.value | values')" ]]; then ID241="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Total_Writes_GiB") | .raw.value | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 241) | .raw.value | values')" ]]; then ssdtdw="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 241) | .raw.value | values')" fi > /dev/null 2<&1 fi # Get Wear Level wearLevelThreshold="" wearLevel="" if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_device_statistics.pages[].table[] | select(.name == "Percentage Used Endurance Indicator") | .value | values')" ]]; then wearLevel="$(echo "${smartdata5}" | jq -Mre '.ata_device_statistics.pages[].table[] | select(.name == "Percentage Used Endurance Indicator") | .value | values')" wearLevelThreshold=100 if [[ $wearLevel -gt 100 ]]; then wearLevel=""; fi fi > /dev/null 2<&1 if [[ $wearLevel == "" ]]; then if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 231) | .value | values')" ]]; then wearLevel="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 231) | .value | values')" wearLevelThreshold="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.id == 231) | .thresh | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Percent_Lifetime_Remain") | .value | values')" ]]; then wearLevel="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Percent_Lifetime_Remain") | .value | values')" wearLevelThreshold="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Percent_Lifetime_Remain") | .thresh | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Media_Wearout_Indicator") | .value | values')" ]]; then wearLevel="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Media_Wearout_Indicator") | .value | values')" wearLevelThreshold="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Media_Wearout_Indicator") | .thresh | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Wear_Leveling_Count") | .value | values')" ]]; then wearLevel="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Wear_Leveling_Count") | .value | values')" wearLevelThreshold="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Wear_Leveling_Count") | .thresh | values')" fi > /dev/null 2<&1 if [[ $wearLevel == "" ]] && [[ $1 != "HDD" ]]; then if [[ "$(echo ${smartdata5} | grep -i "wear")" ]] || [[ "$(echo ${smartdata5} | grep -i "life")" ]]; then json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .id == 231 SSD_Life_Left not properly decoded")"; fi fi > /dev/null 2<&1 fi > /dev/null 2<&1 # Get Normalized Wear Level for Custom Drive Selection if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Wear_Leveling_Count") | .value | values')" ]]; then wearLevelNormalized="$(echo "${smartdata5}" | jq -Mre '.ata_smart_attributes.table[] | select(.name == "Wear_Leveling_Count") | .value | values')" fi > /dev/null 2<&1 # Adjust wearLevel for negative values. if [[ $wearLevelThreshold != "" ]] && [[ $wearLevel != "" ]]; then if [[ $wearLevelThreshold -gt 50 ]]; then wearLevel=$((100 - $wearLevel)) fi fi # Check SSD/NVMe Wear Level - THIS WORKS for NVMe Where 100 = 100 if [[ $wearLevel == "" ]]; then if [[ "$(echo "${smartdata5}" | jq -Mre '.nvme_smart_health_information_log.percentage_used | values')" ]]; then wearLevel="$(echo "${smartdata5}" | jq -Mre '.nvme_smart_health_information_log.percentage_used | values')" wearLevel=$((100 - $wearLevel)) elif [[ "$(echo "${smartdata5}" | jq -Mre '.nvme_smart_health_information_log.available_spare | values')" ]]; then wearLevel="$(echo "${smartdata5}" | jq -Mre '.nvme_smart_health_information_log.available_spare | values')" fi > /dev/null 2<&1 fi # WORKS FOR SCSI Where 0 = 100 if [[ "$(echo "${smartdata5}" | jq -Mre '.scsi_percentage_used_endurance_indicator | values')" ]]; then wearLevel=$((100 - "$(echo "${smartdata5}" | jq -Mre '.scsi_percentage_used_endurance_indicator | values')")) fi > /dev/null 2<&1 # Get Device Type - SCSI if [[ "$(echo "${smartdata5}" | jq -Mre '.device.type | values')" ]]; then devicetype="$(echo "${smartdata5}" | jq -Mre '.device.type | values')" else json_error_log=$json_error_log$'\n'"$(echo $1 " "$serial" .device.type not in json")" > /dev/null 2<&1 fi > /dev/null 2<&1 if [[ $devicetype == "scsi" ]]; then modelnumber=$modelnumber" (SCSI)" fi # Get NVMe Critical Warning Value if [[ "$(echo "${smartdata5}" | jq -Mre '.nvme_smart_health_information_log.critical_warning | values')" ]]; then NVMcriticalWarning="$(echo "${smartdata5}" | jq -Mre '.nvme_smart_health_information_log.critical_warning | values')" fi > /dev/null 2<&1 # Get NVMe Media Errors if [[ "$(echo "${smartdata5}" | jq -Mre '.nvme_smart_health_information_log.media_errors | values')" ]]; then mediaErrors="$(echo "${smartdata5}" | jq -Mre '.nvme_smart_health_information_log.media_errors | values')" fi > /dev/null 2<&1 # Get NVMe Power State if [[ "$testfilepath" == "" ]]; then if [[ $smartdrivesNVM != "" ]]; then if [[ $softver != "Linux" ]]; then nvm_power_level=$(nvmecontrol power "$drive" | rev | cut -c1-2 | rev) if [[ "$nvm_power_level" != "" ]] && [[ "$nvm_power_level" != "0" ]]; then convert_to_decimal $nvm_power_level; nvm_power_level=$Return_Value; fi nvm_power_watts=$(nvmecontrol power -l "$drive" | grep $nvm_power_level":" | grep "W" | cut -d ':' -f 2 | cut -d 'W' -f 1)"W" nvm_power_level="PS-"$nvm_power_level"
"$nvm_power_watts else nvm_power_level_temp2=$(nvme get-feature /dev/"$drive" -f 2 | rev | cut -c1) nvm_power_level_temp1=$(nvme get-feature /dev/"$drive" -f 2 | rev | cut -c2) nvm_power_level=$nvm_power_level_temp1$nvm_power_level_temp2 if [[ "$nvm_power_level" != "" ]] && [[ "$nvm_power_level" != "0" ]]; then convert_to_decimal $nvm_power_level; nvm_power_level=$Return_Value; fi nvm_power_watts=$(nvme -id-ctrl /dev/"$drive" | grep $nvm_power_level" :" | grep "W" | cut -d ':' -f 3 | cut -d 'W' -f 1)"W" nvm_power_level="PS-"$nvm_power_level$"
"$nvm_power_watts fi > /dev/null 2<&1 fi fi # Get NVMe Total Data Written if [[ "$(echo "${smartdata5}" | jq -Mre '.nvme_smart_health_information_log.data_units_written | values')" ]]; then tdw="$(echo "${smartdata5}" | jq -Mre '.nvme_smart_health_information_log.data_units_written | values')" fi > /dev/null 2<&1 # Get SCSI Total Data Written if [[ $tdw == "" ]]; then if [[ "$(echo "${smartdata5}" | jq -Mre '.scsi_error_counter_log.write.gigabytes_processed | values')" ]]; then tdw="$(echo "${smartdata5}" | jq -Mre '.scsi_error_counter_log.write.gigabytes_processed | values')" fi > /dev/null 2<&1 fi sas=0 if [[ "$(echo "$smartdata5" | grep "SAS")" ]]; then sas=1; fi if [[ "$(echo "${smartdata5}" | grep ".ata_smart_attributes")" ]]; then if [[ $Develop == "true" ]]; then echo "SMART Attributes are Present Look Up By Number Add common ID's here."; fi else json_error_log=$json_error_log$'\n'"$(echo $1" "$serial" .ata_smart_attributes not in json")" > /dev/null 2<&1 # SCSI Unique if [[ $Develop == "true" ]]; then echo "++++++++ SMART Attributes are NOT present ++++++++"; fi loadCycle="$(echo "${smartdata5}" | grep -i "Accumulated load-unload cycles:" | cut -d ':' -f 3 | tr -d ' ",')" reAllocEvent="$(echo "${smartdata5}" | jq -Mre '.scsi_grown_defect_list | values')" startStop="$(echo "${smartdata5}" | grep -i "Accumulated start-stop cycles:" | cut -d ':' -f 3 | tr -d ' ",')" # Data from 'smartctl -x' if [[ "$(echo "$smartdata" | grep "Spin_Retry_Count" | awk '{print $10}')" ]]; then spinRetry="$(echo "$smartdata" | grep "Spin_Retry_Count" | awk '{print $10 + 0}')"; fi if [[ "$(echo "$smartdata" | grep "Current_Pending_Sector" | awk '{print $10}')" ]]; then pending="$(echo "$smartdata" | grep "Current_Pending_Sector" | awk '{print $10}')"; fi if [[ "$(echo "$smartdata" | grep "Offline_Uncorrectable" | awk '{print $10}')" ]]; then offlineUnc="$(echo "$smartdata" | grep "Offline_Uncorrectable" | awk '{print $10}')"; fi if [[ "$(echo "$smartdata" | grep "Uncorrectable_Error_Cnt" | awk '{print $10}')" ]]; then offlineUnc="$(echo "$smartdata" | grep "Uncorrectable_Error_Cnt" | awk '{print $10}')"; fi if [[ "$(echo "$smartdata" | grep "UDMA_CRC_Error_Count" | awk '{print $10}')" ]]; then crcErrors="$(echo "$smartdata" | grep "UDMA_CRC_Error_Count" | awk '{print $10 + 0}')"; fi if [[ "$(echo "$smartdata" | grep "CRC_Error_Count" | awk '{print $10}')" ]]; then crcErrors="$(echo "$smartdata" | grep "CRC_Error_Count" | awk '{print $10}')"; fi if [[ "$(echo "$smartdata" | grep "Multi_Zone_Error_Rate" | awk '{print $10}')" ]]; then multiZone="$(echo "$smartdata" | grep "Multi_Zone_Error_Rate" | awk '{print $10 + 0}')"; fi # May not need anymore because of step above... if [[ "$(echo "$smartdata" | grep "Reallocated_Event_Count" | awk '{print $10}')" ]]; then # reAllocEvent="$(echo "$smartdata" | grep "Reallocated_Event_Count" | awk '{print $10}')"; fi fi if [[ $crcErrors == "" ]]; then if [[ "$(echo "$smartdata" | grep "Interface CRC Errors" | awk '{print $4}')" ]]; then crcErrors="$(echo "$smartdata" | grep "Interface CRC Errors" | awk '{print $4}')"; fi fi if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_data.self_test.status.remaining_percent | values')" ]]; then smarttesting="$(echo "${smartdata5}" | jq -Mre '.ata_smart_data.self_test.status.remaining_percent | values')" fi lastTestStatusPass="true" # Setup default, a failure can change it. if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.extended.table | values')" ]]; then lastTestHours="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.extended.table[0].lifetime_hours | values')" lastTestType="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.extended.table[0].type.string | values')" lastTestStatus="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.extended.table[0].status.string | values')" lastTestStatusPass="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.extended.table[0].status.passed | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.standard.table | values')" ]]; then lastTestHours="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.standard.table[0].lifetime_hours | values')" lastTestType="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.standard.table[0].type.string | values')" lastTestStatus="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.standard.table[0].status.string | values')" lastTestStatusPass="$(echo "${smartdata5}" | jq -Mre '.ata_smart_self_test_log.standard.table[0].status.passed | values')" elif [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_selective_self_test_log.table | values')" ]]; then lastTestHours="$(echo "${smartdata5}" | jq -Mre '.ata_smart_selective_self_test_log.table[0].lifetime_hours | values')" lastTestType="$(echo "${smartdata5}" | jq -Mre '.ata_smart_selective_self_test_log.table[0].status.string | values')" lastTestStatus="$(echo "${smartdata5}" | jq -Mre '.ata_smart_selective_self_test_log.table[0].status.string | values')" lastTestStatusPass="$(echo "${smartdata5}" | jq -Mre '.ata_smart_selective_self_test_log.table[0].status.passed | values')" # New NVMe to hopefully capture testing % in progress elif [[ "$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log | values')" ]]; then lastTestHours="$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table[0].power_on_hours | values')" lastTestType="$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.current_self_test_operation.string | values')" lastTestStatus="$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.current_self_test_completion_percent | values')" if [[ "$lastTestType" == *"No self-test"* ]]; then lastTestType="$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table[0].self_test_code.string | values')" else lastTestType=$lastTestType" "$((100-$lastTestStatus))"% Remaining" fi lastTestStatusPass="$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table[0].self_test_result.value | values')" if [[ $lastTestStatusPass == 0 ]]; then lastTestStatusPass="true"; fi elif [[ "$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table | values')" ]]; then lastTestHours="$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table[0].power_on_hours | values')" lastTestType="$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table[0].self_test_code.string | values')" lastTestStatus="$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table[0].self_test_result.string | values')" lastTestStatusPass="$(echo "${smartdata5}" | jq -Mre '.nvme_self_test_log.table[0].self_test_result.value | values')" if [[ $lastTestStatusPass == 0 ]]; then lastTestStatusPass="true"; fi ################ REMOVE THIS SECTION WHEN SMARTMONTOOLS 7.4 IS INCORPORATED INTO TRUENAS ######################## else if [[ $softver != "Linux" ]]; then if [[ "$(nvmecontrol logpage -p 0x06 ${drive})" ]]; then lastTestType="$(nvmecontrol logpage -p 0x06 ${drive} | grep -m 1 "0]" | cut -d ' ' -f3)" lastTestStatus="$(nvmecontrol logpage -p 0x06 ${drive} | grep -m 1 "0]" | cut -d ' ' -f5-10)" temptest="$(echo $lastTestStatus | grep "without")" if [[ "$temptest" == "" ]]; then lastTestStatusPass="false" else lastTestStatusPass="true" fi LTH_HEX="0x$(nvmecontrol logpage -p 0x06 ${drive} -x | cut -d " " -f 4 | head -1)xFF" lastTestHours=$(printf "%d\n" $LTH_HEX) fi > /dev/null 2<&1 else if [[ "$(nvme self-test-log --output-format=json /dev/${drive} | grep -m 1 "Self test result")" ]]; then # Get Last Self Test Type lastTestType="$(nvme self-test-log --output-format=json /dev/${drive} | grep -m 1 "Self test code" | cut -d ':' -f2 | cut -d ',' -f1)" if [[ $lastTestType == 1 ]]; then lastTestType="Short" fi if [[ $lastTestType == 2 ]]; then lastTestType="Extended" fi if [[ $lastTestType == 0 ]]; then lastTestType="" fi # Get Last Self Test Result lastTestStatus="$(nvme self-test-log --output-format=json /dev/${drive} | grep -m 1 "Self test result" | cut -d ':' -f2 | cut -d ',' -f1)" if [[ $lastTestStatus == 0 ]]; then lastTestStatus="Completed without error" lastTestStatusPass="true" else lastTestStatus="Failure or Abort" lastTestStatusPass="false" fi # Get Last Self Test Hours lastTestHours="$(nvme self-test-log --output-format=json /dev/${drive} | grep -m 1 "Power on hours" | cut -d ':' -f2 | cut -d ',' -f1)" fi > /dev/null 2<&1 fi ####################################################### NVME SMARTMONTOOLS ALTERNATE END OF CODE ################################################# fi if [[ $lastTestStatus == "" ]]; then if [[ "$(echo "${smartdata5}" | jq -Mre '.ata_smart_data.self_test.status.string | values')" ]]; then lastTestStatus="$(echo "${smartdata5}" | jq -Mre '.ata_smart_data.self_test.status.string | values')" fi fi if [[ $lastTestHours == "" ]]; then if [[ "$(echo "${smartdata5}" | grep "NOW")" ]]; then lastTestHours="$(echo "${smartdata5}" | grep "# 2" | awk '{print $8}' )" lastTestStatus="$(echo "${smartdata5}" | grep "# 1" | awk '{print $6" "$7" "$8" "$9}' )" smarttesting="$(echo "${smartdata5}" | grep -i "Self-test execution status:" | cut -d ':' -f 3 | tr -d '[a-z]% ,\\"')" # Alternate? smarttesting="$(echo "${smartdata5}" | grep -i "remaining" | cut -d% -f1 | rev | cut -b -2 | rev)" else lastTestHours="$(echo "${smartdata5}" | grep "# 1" | awk '{print $8}' )" lastTestStatus="$(echo "${smartdata5}" | grep "# 1" | awk '{print $6}' )" smarttesting="$(echo "${smartdata5}" | grep -i "Self-test execution status:" | cut -d ':' -f 3 | tr -d '[a-z]% ,\\"')" fi lastTestType="$(echo "${smartdata5}" | grep "# 1" | awk '{print $4" "$5}' )" fi # Adjust for a drive that does report a passed SMART Test but not the hour. if [[ $lastTestHours == "0" ]] && [[ $lastTestStatusPass == "true" ]]; then lastTestHours=$onHours fi if [[ $onHours == "1" ]]; then lastTestHours=$onHours; fi if [[ $lastTestStatusPass == "" ]]; then lastTestStatusPass="true"; fi if [[ "$(echo "${smartdata}" | grep "# 1" | awk '{print $5}')" ]]; then chkreadfailure="$(echo "${smartdata}" | grep "# 1" | awk '{print $5}')" fi if [[ $lastTestType == "" ]]; then lastTestType=$Non_Exist_Value; fi ### Try looking for Custom_Drive data here. Need it for wearLevelAdj. IFS=',' read -ra ADDR <<< "$Custom_Drives_List" for i in "${ADDR[@]}"; do cdrivesn1="$(echo $i | cut -d':' -f 1)" if [[ $cdrivesn1 == $serial ]]; then if [[ "$(echo $i | cut -d':' -f 16)" == "d" || "$(echo $i | cut -d':' -f 16)" == "r" || "$(echo $i | cut -d':' -f 16)" == "n" ]] ; then wearLevelAdj="$(echo $i | cut -d':' -f 16)"; fi fi done if [[ "$rotation" > 0 ]]; then Custom_Drives_ListDrive="HDD" fi if [[ "$rotation" == "0" ]]; then Custom_Drives_ListDrive="SSD" if [[ $wearLevelAdj == "r" ]]; then wearLevel=$(( 100 - $wearLevel )) fi if [[ $wearLevelAdj == "n" ]]; then wearLevel=$wearLevelNormalized fi fi if [[ "$rotation" == "" ]] && [[ $1 == "HDD" ]]; then Custom_Drives_ListDrive="HDD" fi if [[ "$(echo "$smartdata5" | grep "NVM")" ]]; then Custom_Drives_ListDrive="NVM" fi if [[ $Develop == "true" ]]; then echo " " echo "Drive Type: "$1" | Serial Number: "$serial" | Model: "$modelnumber echo "Capacity: "$capacity" | RPM: "$rotation" | SMART Status: "$smartStatus echo "Curr Temp: "$temp" | Temp Min: "$temp_min" | Temp Max: "$temp_max echo "onHours="$onHours" | Start Stop Count: "$startStop" | Load Cycle: "$loadCycle echo "Spin Retry: "$spinRetry" | Reallocated Sectors: "$reAlloc" | Reallocated Events: "$reAllocEvent echo "Curr Pending Sectors: "$pending" | Offline uncorrectable: "$offlineUnc" | UDMA CRC Errors: "$crcErrors echo "Read Error Rate: "$rawReadErrorRate" | Seek Error Rate: "$seekErrorHealth" | MultiZone: "$multiZone echo "Helium: "$Helium" | HeliumThreshold="$heliumthresh" | Wear Level: "$wearLevel echo "Last Test Age: "$lastTestHours" | Last Test Type: "$lastTestType" | lastTestStatus="$lastTestStatus echo "testStatus="$smarttesting" | lastTestStatusPass="$lastTestStatusPass echo #echo "SAS="$sas #echo "lastTestHours="$lastTestHours", altlastTestHours="$altlastTestHours echo "==========================================" fi # Definitions of variables: # # $1 = Drive Type (HDD/SSD/NVM) # $lastTestType = The last/current test (background short/long, Short/Long offline, Conveyance) # $lastTestStatus = "Completed without error", "Completed", "Self test in progress", etc... # $testStatus = Percent completed of a test in progress "35%" with "%" trimmed. # $lastTestStatusPass = json passed = "true" or "false". if [[ $Sample_Test == "true" ]]; then echo "In Testing Mode" # Change any value below to override the actual drive values. # These are critical monitored values #temp_min=10 #temp_max=50 #temp=44 #spinRetry=0 reAlloc=1 reAllocEvent=1 pending=1 #offlineUnc=10 crcErrors=1 multiZone=1 #Helium=91 #wearLevel=16 #mediaErrors=0 # Below here are non-critical (No alarm generated) #seekErrorHealth=6 #seekErrorRate=6 #rawReadErrorRate=5 #startStop=490 #loadCycle=500 #onHours=50026 #lastTestHours=50000 #tdw=175234 fi ########## CALL CONVERT VARIABLES TO DECIMAL ########## if [[ "$temp_min" != "" ]] && [[ "$temp_min" != "0" ]] && [[ "$temp_min" != "$Non_Exist_Value" ]]; then convert_to_decimal $temp_min; temp_min=$Return_Value; fi if [[ "$temp_max" != "" ]] && [[ "$temp_max" != "0" ]] && [[ "$temp_max" != "$Non_Exist_Value" ]]; then convert_to_decimal $temp_max; temp_max=$Return_Value; fi if [[ "$temp" != "" ]] && [[ "$temp" != "0" ]] && [[ "$temp" != "$Non_Exist_Value" ]]; then convert_to_decimal $temp; temp=$Return_Value; fi if [[ "$spinRetry" != "" ]] && [[ "$spinRetry" != "0" ]]; then convert_to_decimal $spinRetry; spinRetry=$Return_Value; fi if [[ "$reAllocEvent" != "" ]] && [[ "$reAllocEvent" != "0" ]]; then convert_to_decimal $reAllocEvent; reAllocEvent=$Return_Value; fi if [[ "$pending" != "" ]] && [[ "$pending" != "0" ]]; then convert_to_decimal $pending; pending=$Return_Value; fi if [[ "$offlineUnc" != "" ]] && [[ "$offlineUnc" != "0" ]]; then convert_to_decimal $offlineUnc; offlineUnc=$Return_Value; fi if [[ "$crcErrors" != "" ]] && [[ "$crcErrors" != "0" ]]; then convert_to_decimal $crcErrors; crcErrors=$Return_Value; fi if [[ "$seekErrorHealth2" != "" ]] && [[ "$seekErrorHealth2" != "0" ]]; then convert_to_decimal $seekErrorHealth2; seekErrorHealth2=$Return_Value; fi if [[ "$seekErrorHealth" != "" ]] && [[ "$seekErrorHealth" != "0" ]]; then convert_to_decimal $seekErrorHealth; seekErrorHealth=$Return_Value; fi if [[ "$rawReadErrorRate2" != "" ]] && [[ "$rawReadErrorRate2" != "0" ]]; then convert_to_decimal $rawReadErrorRate2; rawReadErrorRate2=$Return_Value; fi if [[ "$rawReadErrorRate" != "" ]] && [[ "$rawReadErrorRate" != "0" ]]; then convert_to_decimal $rawReadErrorRate; rawReadErrorRate=$Return_Value; fi if [[ "$multiZone" != "" ]] && [[ "$multiZone" != "0" ]]; then convert_to_decimal $multiZone; multiZone=$Return_Value; fi if [[ "$wearLevel" != "" ]] && [[ "$wearLevel" != "0" ]]; then convert_to_decimal $wearLevel; wearLevel=$Return_Value; fi if [[ "$mediaErrors" != "" ]] && [[ "$mediaErrors" != "0" ]] && [[ "$mediaErrors" != "$Non_Exist_Value" ]]; then convert_to_decimal $mediaErrors; mediaErrors=$Return_Value; fi if [[ "$tdw" != "" ]] && [[ "$tdw" != "0" ]] && [[ "$tdw" != "$Non_Exist_Value" ]]; then convert_to_decimal $tdw; tdw=$Return_Value; fi if [[ "$temp" != "" ]] && [[ "$temp" != "0" ]]; then convert_to_decimal $temp; temp=$Return_Value; fi if [[ "$startStop" != "" ]] && [[ "$startStop" != "0" ]]; then convert_to_decimal $startStop; startStop=$Return_Value; fi if [[ "$loadCycle" != "" ]] && [[ "$loadCycle" != "0" ]]; then convert_to_decimal $loadCycle; loadCycle=$Return_Value; fi if [[ "$reAlloc" != "" ]] && [[ "$reAlloc" != "0" ]]; then convert_to_decimal $reAlloc; reAlloc=$Return_Value; fi if [[ "$onHours" != "" ]] && [[ "$onHours" != "0" ]]; then convert_to_decimal $onHours; onHours=$Return_Value; fi if [[ "$Helium" != "" ]] && [[ "$Helium" != "0" ]]; then convert_to_decimal $Helium; Helium=$Return_Value; fi lastTestHours="$(echo $lastTestHours | tr -d "()%/")" if [[ "$lastTestHours" != "" ]] && [[ "$lastTestHours" != "0" ]]; then convert_to_decimal $lastTestHours; lastTestHours=$Return_Value; fi altlastTestHours="$(echo $altlastTestHours | tr -d "()%/")" if [[ "$altlastTestHours" -gt "0" ]]; then convert_to_decimal $altlastTestHours; altlastTestHours=$Return_Value; fi # Some drives do not report test age after 65536 hours. if [[ $onHours -gt "65536" ]] && [[ $lastTestHours -gt "0" && $lastTestHours -lt "65536" ]]; then lastTestHours=$(($lastTestHours + 65536)); fi ######## VMWare Hack to fix NVMe bad variables ##### if [[ "$VMWareNVME" == "on" ]]; then if [[ "$serial" == "VMWare" ]]; then onHours="17,200" wearLevel="98" temp="38" temp_min="20" temp_max="43" fi fi # Save data for -dump routine if [[ "$dump_all" != "0" ]]; then get_json_data if [[ "$dump_all" == "4" ]]; then # Dump '-dump emailextra' "$(echo "$smartdata" > /tmp/${tempfilepath}${drive}_${serial}_a.txt)" 2> /dev/null "$(smartctl -x /dev/"$drive" > /tmp/${tempfilepath}${drive}_${serial}_x.txt)" 2> /dev/null fi fi # Add a line return between drives json_error_log=$json_error_log$'\n' } ########## GENERATE TABLE ########## # Call function with generate_table "HDD|SSD|NVM" generate_table () { if [[ "$Monitor" == "true" ]]; then return; fi detail_level="$1" # Lets add up how many columns we will need. if [[ "$Drive_Warranty_List" == "none" || "$Drive_Warranty_List" == "" ]]; then HDD_Warranty="false" SSD_Warranty="false" NVM_Warranty="false" fi Columns=0; if [[ "$1" == "HDD" ]] && [[ "$HDD_Device_ID" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Serial_Number" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Model_Number" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Capacity" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Rotational_Rate" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_SMART_Status" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Warranty" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Drive_Temp" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Drive_Temp_Min" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Drive_Temp_Max" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Power_On_Hours" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Start_Stop_Count" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Load_Cycle" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Spin_Retry" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Reallocated_Sectors" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Reallocated_Events" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Pending_Sectors" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Offline_Uncorrectable" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_UDMA_CRC_Errors_List" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Raw_Read_Error_Rate" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Seek_Error_Rate" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_MultiZone_Errors" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Helium_Level" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Total_Data_Written" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Last_Test_Age" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "HDD" ]] && [[ "$HDD_Last_Test_Type" == "true" ]]; then ((Columns=Columns+1)); fi; # Count for SSD if [[ "$1" == "SSD" ]] && [[ "$SSD_Device_ID" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Serial_Number" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Model_Number" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Capacity" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_SMART_Status" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Warranty" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Drive_Temp" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Drive_Temp_Min" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Drive_Temp_Max" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Power_On_Hours" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Wear_Level" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Reallocated_Sectors" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Reallocated_Events" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Pending_Sectors" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Offline_Uncorrectable" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_UDMA_CRC_Errors_List" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Total_Data_Written" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Last_Test_Age" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "SSD" ]] && [[ "$SSD_Last_Test_Type" == "true" ]]; then ((Columns=Columns+1)); fi; # Count for NVMe if [[ "$1" == "NVM" ]] && [[ "$NVM_Device_ID" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Serial_Number" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Model_Number" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Capacity" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_SMART_Status" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Warranty" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Critical_Warning" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Drive_Temp" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Drive_Temp_Min" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Drive_Temp_Max" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Power_On_Hours" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Power_Level" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Wear_Level" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Media_Error" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Last_Test_Age" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Last_Test_Type" == "true" ]]; then ((Columns=Columns+1)); fi; if [[ "$1" == "NVM" ]] && [[ "$NVM_Total_Data_Written" == "true" ]]; then ((Columns=Columns+1)); fi; ( # Write HTML table headers to log file echo "

" echo "" if [[ "$1" == "HDD" ]]; then echo ""; fi if [[ "$1" == "SSD" ]]; then echo ""; fi if [[ "$1" == "NVM" ]]; then echo ""; fi echo "" if [[ "$1" == "HDD" ]] && [[ "$HDD_Device_ID" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Device_ID" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Device_ID" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Serial_Number" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Serial_Number" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Serial_Number" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Model_Number" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Model_Number" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Model_Number" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Capacity" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Capacity" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Capacity" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Rotational_Rate" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_SMART_Status" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_SMART_Status" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_SMART_Status" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Warranty" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Warranty" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Warranty" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Critical_Warning" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Drive_Temp" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Drive_Temp" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Drive_Temp" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Drive_Temp_Min" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Drive_Temp_Min" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Drive_Temp_Min" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Drive_Temp_Max" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Drive_Temp_Max" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Drive_Temp_Max" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Power_Level" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Power_On_Hours" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Power_On_Hours" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Power_On_Hours" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Wear_Level" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Wear_Level" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Start_Stop_Count" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Load_Cycle" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Spin_Retry" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Reallocated_Sectors" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Reallocated_Sectors" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Reallocated_Events" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Reallocated_Events" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Pending_Sectors" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Pending_Sectors" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Offline_Uncorrectable" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Offline_Uncorrectable" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_UDMA_CRC_Errors_List" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_UDMA_CRC_Errors_List" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Raw_Read_Error_Rate" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Seek_Error_Rate" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_MultiZone_Errors" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Helium_Level" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Media_Error" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Last_Test_Age" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Last_Test_Age" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Last_Test_Age" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Last_Test_Type" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Last_Test_Type" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Last_Test_Type" == "true" ]]; then echo " "; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Total_Data_Written" == "true" ]]; then echo " "; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Total_Data_Written" == "true" ]]; then echo " "; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Total_Data_Written" == "true" ]]; then echo " "; fi echo "" ) >> "$logfile" } ########## WRITE TABLE ########## # Call function with end_table "HDD|SSD|NVM" write_table () { if [[ "$Monitor" == "true" ]]; then return; fi ( printf "\n" $bgColor; # DRIVE ID - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Device_ID" == "true" ]]; then printf "\n" "$deviceStatusColor" "$drive"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Device_ID" == "true" ]]; then printf "\n" "$deviceStatusColor" "$drive"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Device_ID" == "true" ]]; then printf "\n" "$deviceStatusColor" "$drive"; fi # SERIAL NUMBER - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Serial_Number" == "true" ]]; then printf "\n" "$bgColor" "$serial"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Serial_Number" == "true" ]]; then printf "\n" "$bgColor" "$serial"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Serial_Number" == "true" ]]; then printf "\n" "$bgColor" "$serial"; fi # MODEL NUMBER - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Model_Number" == "true" ]]; then printf "\n" "$modelnumber"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Model_Number" == "true" ]]; then printf "\n" "$modelnumber"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Model_Number" == "true" ]]; then printf "\n" "$modelnumber"; fi # CAPACITY - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Capacity" == "true" ]]; then printf "\n" "$capacity"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Capacity" == "true" ]]; then printf "\n" "$capacity"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Capacity" == "true" ]]; then printf "\n" "$capacity"; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Rotational_Rate" == "true" ]]; then printf "\n" "$rotation"; fi # SMART STATUS - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_SMART_Status" == "true" ]]; then printf "\n" "$smartStatusColor" "$smartStatus"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_SMART_Status" == "true" ]]; then printf "\n" "$smartStatusColor" "$smartStatus"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_SMART_Status" == "true" ]]; then printf "\n" "$smartStatusColor" "$smartStatus"; fi # WARRANTY - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Warranty" == "true" ]] && [[ "$WarrantyBoxColor" == "$expiredWarrantyBoxColor" ]]; then printf "\n" "$WarrantyBackgroundColor" "$WarrantyBoxColor" "$WarrantyClock"; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Warranty" == "true" ]] && [[ "$WarrantyBoxColor" != "$expiredWarrantyBoxColor" ]]; then printf "\n" "$WarrantyBackgroundColor" "$WarrantyBoxColor" "$WarrantyClock"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Warranty" == "true" ]] && [[ "$WarrantyBoxColor" == "$expiredWarrantyBoxColor" ]]; then printf "\n" "$WarrantyBackgroundColor" "$WarrantyBoxColor" "$WarrantyClock"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Warranty" == "true" ]] && [[ "$WarrantyBoxColor" != "$expiredWarrantyBoxColor" ]]; then printf "\n" "$WarrantyBackgroundColor" "$WarrantyBoxColor" "$WarrantyClock"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Warranty" == "true" ]] && [[ "$WarrantyBoxColor" == "$expiredWarrantyBoxColor" ]]; then printf "\n" "$WarrantyBackgroundColor" "$WarrantyBoxColor" "$WarrantyClock"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Warranty" == "true" ]] && [[ "$WarrantyBoxColor" != "$expiredWarrantyBoxColor" ]]; then printf "\n" "$WarrantyBackgroundColor" "$WarrantyBoxColor" "$WarrantyClock"; fi # CRITICAL WARNING - CHART if [[ "$1" == "NVM" ]] && [[ "$NVM_Critical_Warning" == "true" ]]; then printf "\n" "$NVMcriticalWarningColor" "$NVMcriticalWarning"; fi # DRIVE TEMPERATURE - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Drive_Temp" == "true" ]]; then printf "\n" "$tempColor" "$temp"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Drive_Temp" == "true" ]]; then printf "\n" "$tempColor" "$temp"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Drive_Temp" == "true" ]]; then printf "\n" "$tempColor" "$temp"; fi # DRIVE TEMPERATURE MIN - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Drive_Temp_Min" == "true" ]]; then printf "\n" "$temp_min"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Drive_Temp_Min" == "true" ]]; then printf "\n" "$temp_min"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Drive_Temp_Min" == "true" ]]; then printf "\n" "$temp_min"; fi # DRIVE TEMPERATURE MAX - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Drive_Temp_Max" == "true" ]]; then printf "\n" "$temp_maxColor" "$temp_max"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Drive_Temp_Max" == "true" ]]; then printf "\n" "$temp_maxColor" "$temp_max"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Drive_Temp_Max" == "true" ]]; then printf "\n" "$temp_maxColor" "$temp_max"; fi # NVME POWER LEVEL - CHART if [[ "$1" == "NVM" ]] && [[ "$NVM_Power_Level" == "true" ]]; then printf "\n" "$bgColor" "$nvm_power_level"; fi # POWER ON HOURS - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Power_On_Hours" == "true" ]]; then printf "\n" "$onTimeColor" "$onTime"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Power_On_Hours" == "true" ]]; then printf "\n" "$onTimeColor" "$onTime"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Power_On_Hours" == "true" ]]; then printf "\n" "$onTimeColor" "$onTime"; fi # WEAR LEVEL - CHART if [[ "$1" == "SSD" ]] && [[ "$SSD_Wear_Level" == "true" ]]; then printf "\n" "$wearLevelColor" "$wearLevel"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Wear_Level" == "true" ]]; then printf "\n" "$wearLevelColor" "$wearLevel"; fi # NVME MEDIA ERROR - CHART if [[ "$1" == "NVM" ]] && [[ "$NVM_Media_Error" == "true" ]]; then printf "\n" "$mediaErrorsColor" "$mediaErrors"; fi # HDD START STOP COUNT - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Start_Stop_Count" == "true" ]]; then printf "\n" "$startStop"; fi # HDD LOAD CYCLE - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Load_Cycle" == "true" ]]; then printf "\n" "$loadCycle"; fi # HDD SPIN RETRY - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Spin_Retry" == "true" ]]; then printf "\n" "$spinRetryColor" "$spinRetry"; fi # REALLOCATED SECTORS - CHART if [[ $reAllocColor != $ovrdColor ]]; then if [[ "$1" == "HDD" ]] && [[ "$HDD_Reallocated_Sectors" == "true" ]]; then printf "\n" "$reAllocColor" "$reAlloc"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Reallocated_Sectors" == "true" ]]; then printf "\n" "$reAllocColor" "$reAlloc"; fi else if [[ "$1" == "HDD" ]] && [[ "$HDD_Reallocated_Sectors" == "true" ]]; then printf "\n" "$reAllocColor" "$reAlloc" "$reAllocOrig"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Reallocated_Sectors" == "true" ]]; then printf "\n" "$reAllocColor" "$reAlloc" "$reAllocOrig"; fi fi # REALLOCATED EVENTS - CHART if [[ $reAllocEventColor != $ovrdColor ]]; then if [[ "$1" == "HDD" ]] && [[ "$HDD_Reallocated_Events" == "true" ]]; then printf "\n" "$reAllocEventColor" "$reAllocEvent"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Reallocated_Events" == "true" ]]; then printf "\n" "$reAllocEventColor" "$reAllocEvent"; fi else if [[ "$1" == "HDD" ]] && [[ "$HDD_Reallocated_Events" == "true" ]]; then printf "\n" "$reAllocEventColor" "$reAllocEvent" "$reAllocEventOrig"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Reallocated_Events" == "true" ]]; then printf "\n" "$reAllocEventColor" "$reAllocEvent" "$reAllocEventOrig"; fi fi # PENDING SECTORS - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Pending_Sectors" == "true" ]]; then printf "\n" "$pendingColor" "$pending"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Pending_Sectors" == "true" ]]; then printf "\n" "$pendingColor" "$pending"; fi # OFFLINE UNCORRECTABLE SECTORS - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Offline_Uncorrectable" == "true" ]]; then printf "\n" "$offlineUncColor" "$offlineUnc"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Offline_Uncorrectable" == "true" ]]; then printf "\n" "$offlineUncColor" "$offlineUnc"; fi # UDMA CRC ERRORS - CHART if [[ $crcErrorsColor != $ovrdColor ]]; then if [[ "$1" == "HDD" ]] && [[ "$HDD_UDMA_CRC_Errors_List" == "true" ]]; then printf "\n" "$crcErrorsColor" "$crcErrors"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_UDMA_CRC_Errors_List" == "true" ]]; then printf "\n" "$crcErrorsColor" "$crcErrors"; fi else if [[ "$1" == "HDD" ]] && [[ "$HDD_UDMA_CRC_Errors_List" == "true" ]]; then printf "\n" "$crcErrorsColor" "$crcErrors" "$crcErrorsOrig"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_UDMA_CRC_Errors_List" == "true" ]]; then printf "\n" "$crcErrorsColor" "$crcErrors" "$crcErrorsOrig"; fi fi # RAW READ ERROR RATE - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Raw_Read_Error_Rate" == "true" ]]; then printf "\n" "$rawReadErrorRateColor" "$rawReadErrorRate"; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Seek_Error_Rate" == "true" ]]; then printf "\n" "$seekErrorHealthColor" "$seekErrorHealth"; fi # MULTIZONE - CHART if [[ $multiZoneColor != $ovrdColor ]]; then if [[ "$1" == "HDD" ]] && [[ "$HDD_MultiZone_Errors" == "true" ]]; then printf "\n" "$multiZoneColor" "$multiZone"; fi else if [[ "$1" == "HDD" ]] && [[ "$HDD_MultiZone_Errors" == "true" ]]; then printf "\n" "$multiZoneColor" "$multiZone" "$multiZoneOrig"; fi fi # HELIUM LEVEL - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Helium_Level" == "true" ]]; then printf "\n" "$HeliumColor" "$Helium"; fi # LAST TEST AGE - CHART if [[ "$1" == "HDD" ]] && [[ "$HDD_Last_Test_Age" == "true" ]]; then printf "\n" "$testAgeColor" "$testAge"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Last_Test_Age" == "true" ]]; then printf "\n" "$testAgeColor" "$testAge"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Last_Test_Age" == "true" ]]; then printf "\n" "$testAgeColor" "$testAge"; fi # LAST TEST TYPE - CHART if [[ $lastTestTypeColor == $bgColor ]] && [[ "$(echo $lastTestStatus | grep -i "progress" )" ]]; then lastTestTypeColor=$blueColor; fi if [[ "$1" == "HDD" ]] && [[ "$HDD_Last_Test_Type" == "true" ]]; then printf "\n" "$lastTestTypeColor" "$lastTestType"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Last_Test_Type" == "true" ]]; then printf "\n" "$lastTestTypeColor" "$lastTestType"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Last_Test_Type" == "true" ]]; then printf "\n" "$lastTestTypeColor" "$lastTestType"; fi # TOTAL DATA WRITTEN if [[ "$1" == "HDD" ]] && [[ "$HDD_Total_Data_Written" == "true" ]]; then printf "\n" "$bgColor" "$tdw"; fi if [[ "$1" == "SSD" ]] && [[ "$SSD_Total_Data_Written" == "true" ]]; then printf "\n" "$bgColor" "$tdw"; fi if [[ "$1" == "NVM" ]] && [[ "$NVM_Total_Data_Written" == "true" ]]; then printf "\n" "$bgColor" "$tdw"; fi echo "" ) | tr -d "[]" >> "$logfile" } ########## END THE TABLE ########## end_table () { if [[ "$Monitor" == "true" ]]; then return; fi ( echo "" echo "
"$HDDreportTitle"
"$SSDreportTitle"
"$NVMreportTitle"
"$HDD_Device_ID_Title""$SSD_Device_ID_Title""$NVM_Device_ID_Title""$HDD_Serial_Number_Title""$SSD_Serial_Number_Title""$NVM_Serial_Number_Title""$HDD_Model_Number_Title""$SSD_Model_Number_Title""$NVM_Model_Number_Title""$HDD_Capacity_Title""$SSD_Capacity_Title""$NVM_Capacity_Title""$HDD_Rotational_Rate_Title""$HDD_SMART_Status_Title""$SSD_SMART_Status_Title""$NVM_SMART_Status_Title""$HDD_Warranty_Title""$SSD_Warranty_Title""$NVM_Warranty_Title""$NVM_Critical_Warning_Title""$HDD_Drive_Temp_Title""$SSD_Drive_Temp_Title""$NVM_Drive_Temp_Title""$HDD_Drive_Temp_Min_Title""$SSD_Drive_Temp_Min_Title""$NVM_Drive_Temp_Min_Title""$HDD_Drive_Temp_Max_Title""$SSD_Drive_Temp_Max_Title""$NVM_Drive_Temp_Max_Title""$NVM_Power_Level_Title""$HDD_Power_On_Hours_Title""$SSD_Power_On_Hours_Title""$NVM_Power_On_Hours_Title""$SSD_Wear_Level_Title""$NVM_Wear_Level_Title""$HDD_Start_Stop_Count_Title""$HDD_Load_Cycle_Title""$HDD_Spin_Retry_Title""$HDD_Reallocated_Sectors_Title""$SSD_Reallocated_Sectors_Title""$HDD_Reallocated_Events_Title""$SSD_Reallocated_Events_Title""$HDD_Pending_Sectors_Title""$SSD_Pending_Sectors_Title""$HDD_Offline_Uncorrectable_Title""$SSD_Offline_Uncorrectable_Title""$HDD_UDMA_CRC_Errors_List_Title""$SSD_UDMA_CRC_Errors_List_Title""$HDD_Raw_Read_Error_Rate_Title""$HDD_Seek_Error_Rate_Title""$HDD_MultiZone_Errors_Title""$HDD_Helium_Level_Title""$NVM_Media_Error_Title""$HDD_Last_Test_Age_Title""$SSD_Last_Test_Age_Title""$NVM_Last_Test_Age_Title""$HDD_Last_Test_Type_Title""$SSD_Last_Test_Type_Title""$NVM_Last_Test_Type_Title""$HDD_Total_Data_Written_Title""$SSD_Total_Data_Written_Title""$NVM_Total_Data_Written_Title"
/dev/%s/dev/%s/dev/%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s$TempDisplay%s$TempDisplay%s$TempDisplay%s$TempDisplaymin%s$TempDisplaymin%s$TempDisplaymin%s$TempDisplaymax%s$TempDisplaymax%s$TempDisplaymax%s%s%s%s%s%s%s%s%s%s%s%s%s(%s)%s(%s)%s%s%s(%s)%s(%s)%s%s%s%s%s%s%s(%s)%s(%s)$SER%s$SER%s%s%s(%s)%s%s%s%s%s%s%s%s%s%s
" if [[ "$SER1" == "1" ]]; then echo "* = Seek Error Rate is Normalized. Higher is better."; fi echo "
" ) >> "$logfile" } ########## COMPILE DETAILED REPORT ########## # Detailed Report Section (monospace text) detailed_report () { testfile=$1 if [[ $Monitor == "false" ]]; then ( echo "
"
			echo "Multi-Report Text Section"
			echo "
"
		) >> "$logfile"

		if test -e "$Config_File_Name"; then
			echo "External Configuration File in use dtd:$config_version_date " >> "$logfile"
		else
			echo "No External Configuration File Exists" >> "$logfile"
		fi

		if [[ "$dump_all" == "3" ]]; then
			echo "YOU have requested '-dump email' and thus an email was sent to Joe Schmuck for analysis.
He will try to contact you on the sending email address.  If this is an invalid
email address then please send a followup email to joeschmuck2023@hotmail.com
Your personal information will not be shared.
" >> "$logfile"
		fi

		if [[ $TrueNASConfigEmailEnable == "true" ]]; then echo "TrueNAS Configuration File Emailed every: $TrueNASConfigEmailDay" >> "$logfile"; fi
		if [[ $MRConfigEmailEnable == "true" ]]; then echo "Multi Report Configuration File Emailed every: $MRConfigEmailDay" >> "$logfile"; fi
		if [[ $SDF_DataRecordEnable == "true" ]]; then
			if [[ "$(echo $statistical_data_file | grep "/tmp/")" ]]; then
				echo "The Statistical Data File is located in the /tmp directory and is not permanent.
Recommend changing to a proper dataset." >> "$logfile"
			fi

			if [[ $statistical_data_file_created == "1" ]]; then echo "Statistical Data File Created.
" >> "$logfile"; fi

			if [[ $SDF_DataEmail == "true" ]]; then
				(
				echo "Statistical Log Sent on: $SDF_DataEmailDay"
				echo "Statistical Export Log Located: $statistical_data_file"				
				) >> "$logfile"
			else
				echo "Statistical Export Log Located at: $statistical_data_file" >> "$logfile"
			fi
		fi
		echo "Multipath set to: $Multipath
" >> "$logfile"

		if [ -f "/tmp/selftestlog" ]; then	# If there is a selftestlog file, then we did some selftests
			cat /tmp/selftestlog >> "$logfile"
			echo "
" >> "$logfile"
		fi

	fi
	### Lets write out the error messages if there are any, Critical first followed by Warning

	Monitor_Send_Email="false"

	if [[ $AlertOnCriticalError == "true" ]] && [[ $Monitor == "true" ]]; then

		if [[ $logfile_critical != "" ]]; then
			Monitor_Send_Email="true"
		fi
	fi

	if [[ $AlertOnWarningTemp == "true" ]] && [[ $Monitor == "true" ]]; then
		if [[ $Temperature_Log != "" ]]; then
			Monitor_Send_Email="true"
		fi
	fi

	(
		if [[ $Monitor == "false" ]]; then
			if [[ ! $logfile_messages == "" ]]; then
				echo "MESSAGES LOG FILE"
				echo $logfile_messages
				echo "
END
"
			fi

			if [[ $logfile_critical != "" ]]; then
				echo "CRITICAL LOG FILE"
				echo $logfile_critical
				echo "
END
"
			fi

			if [[ $logfile_warning != "" ]]; then
				echo "WARNING LOG FILE"
				echo $logfile_warning
				echo "
END
"
			fi

			if [[ ! $Ignore_Drives_List == "" ]]; then
				echo "Ignored Drives = "$Ignore_Drives_List
				echo "
END
"
			fi

			if [[ ! $logfile_warranty == "" ]]; then 
				echo $logfile_warranty
				echo "
"
			fi
		else
			if [[ $logfile_critical != "" ]]; then
				echo $logfile_critical
			fi

			if [[ $Temperature_Log != "" ]]; then
				echo $Temperature_Log
			fi
		fi
	) >> "$logfile"
	if [[ $Monitor == "false" ]]; then

		if [[ $DisableRAWdata != "true" ]]; then

			### zpool status for each pool
			for pool in $pools; do
				###### FREEBSD VERSION OF THIS SECTION
				if [ $softver != "Linux" ]; then
					drives_in_gptid=$(zpool status "$pool" | grep "gptid" | awk '{print $1}')  # HDD and SSD
					if [[ $(zpool status "$pool" | grep "nvd") ]]; then
						drives_in_gptid=$drives_in_gptid$(zpool status "$pool" | grep "nvd" | awk '{print $1}' | cut -d " " -f1 | cut -d "p" -f1)  # NVM
					fi
					if [[ $(zpool status "$pool" | grep -v "data" | grep " da") ]]; then
						drive_da=$(zpool status "$pool" | grep -v "data" | grep " da0" | awk '{print $1}' | cut -d " " -f1 | cut -d "p" -f1)  # daX
						drive_da=$(glabel status | tail -n +2 | grep " $drive_da" | awk '{print $1}')
						drives_in_gptid=$drives_in_gptid$drive_da
					fi

				driveit=0
	 			(
					# Create a simple header and drop the output of zpool status -v
					echo "########## ZPool status report for ${pool} ##########"
					zpool status -v "$pool"
					zpool_list=$(zpool status -v "$pool")

					for longgptid in $drives_in_gptid; do
							drive_ident=$(glabel status | tail -n +2 | grep "$longgptid" | awk '{print $1 " -> " $3}' | cut -d '/' -f2 | cut -d 'p' -f1)
							drive_name=$(glabel status | tail -n +2 | grep "$longgptid" | awk '{print $3}' | cut -d '/' -f2 | cut -d 'p' -f1)

							virtual_drive_test=$(smartctl -a "/dev/"$drive_name | grep -i "virtual")
							if [[ $virtual_drive_test != "" ]]; then
							   drive_ident="Virtual Drive "$drive_ident
							fi

						if [[ $drive_ident != "" ]]; then
							if [[ $driveit == "0" ]]; then echo "
Drives for this pool are listed in order:"; driveit="1"; fi
							echo $drive_ident
						fi
					done
					echo "
"
				) >> "$logfile"
				fi

				#### LINUX VERSION OF THIS SECTION
				if [ $softver == "Linux" ]; then
                     			drives_in_gptid=$(lsblk -o +PARTUUID,NAME,LABEL | grep -w "$pool" | awk -F" " '{print $7}')

				driveit=0
	 			(
					# Create a simple header and drop the output of zpool status -v
					echo "########## ZPool status report for ${pool} ##########"
					zpool status -v "$pool"
					zpool_list=$(zpool status -v "$pool")
					for longgptid in $drives_in_gptid; do
						
						drive_ident=$(lsblk -o +PARTUUID,NAME,LABEL | grep -w "$longgptid" | awk -F" " '{print $7 " -> " $8}')
						drive_name=$(lsblk -o +PARTUUID,NAME,LABEL | grep -w "$longgptid" | awk -F" " '{print $8}')
						virtual_drive_test=$(smartctl -a "/dev/"$drive_name | grep -i "virtual")
						if [[ $virtual_drive_test != "" ]]; then
						   drive_ident="Virtual Drive "$drive_ident
						fi

						if [[ $drive_ident != "" ]]; then
							if [[ $driveit == "0" ]]; then echo "
Drives for this pool are listed below:"; driveit="1"; fi
							echo $drive_ident
							drive_ident=""
						fi
					done
					echo "
"
				) >> "$logfile"
				fi
			done

			drives="${smartdrives} ${smartdrivesSSD} ${smartdrivesNVM}"

			### SMART status for each drive - SMART Enabled
			write_ATA_Errors="0"

			for drive in $drives; do
				if [[ $drive == "TEST" ]] ; then
					testfileX="$( cat "${testfile}" )"
					modelnumber="$(echo "${testfileX}" | jq -Mre '.model_name | values')"
					serial="$(echo "${testfileX}" | jq -Mre '.serial_number | values')"

					(
						echo "
########## FULL TESTFILE -- SMART status report for ${drive} drive (${modelnumber}: ${serial}) ##########" 
						echo "
Data not available in test mode"
					) >> "$logfile"
				else
					# Gather brand and serial number of each drive
					smartdata="$(smartctl -a /dev/"$drive")"
					smartdata5="$(smartctl -x --json=u /dev/"$drive")"

					modelnumber="$(echo "${smartdata5}" | jq -Mre '.model_name | values')"

					serial="$(echo "${smartdata5}" | jq -Mre '.serial_number | values')"

					test_ata_error="$(smartctl -H -A -l error /dev/"$drive" | grep "ATA Error Count" | awk '{print $4}')" 

					modelnumber="$(echo "${modelnumber}" | sed -e 's/\ *$//g')"

					if [[ $serial == "" ]]; then serial="N/A"; fi
					if [[ $modelnumber == "" ]]; then modelnumber="N/A"; fi

					# If no data in ATA_Errors_List then lets gather data if needed.
					### ATA ERROR LOG ### Let's find a match to string ATA_Errors_List
					IFS=',' read -ra ADDR <<< "$ATA_Errors_List"
					for i in "${ADDR[@]}"; do
						ataerrorssn1="$(echo $i | cut -d':' -f 1)"
						ataerrorsdt1="$(echo $i | cut -d':' -f 2)"
						if [[ $ataerrorssn1 == "" ]]; then
							if [[ ! $test_ata_error == "" ]]; then
								if [[ $ATA_Auto_Enable == "true" ]]; then
									temp_ATA_Errors=$temp_ATA_Errors$serial:$test_ata_error","
									write_ATA_Errors="1"
								fi
							fi
						fi

						if [[ "$ataerrorssn1" == "$serial" ]]; then
							ataerrors=$ataerrorsdt1
							if [[ $ATA_Errors_List == "" ]]; then
								write_ATA_Errors="1"
							fi
							temp_ATA_Errors=$temp_ATA_Errors$serial:$test_ata_error","
							if [[ $test_ata_error -gt $ataerrors ]]; then
								write_ATA_Errors="1"
								logfile_warning=$logfile_warning"$(printf "Drive "$serial" ATA Error Count: "$test_ata_error" - Value Increased 
")"
							fi
						fi
						continue
					done
					(
						# Create a simple header and drop the output of some basic smartctl commands
						echo "########## SMART status report for ${drive} drive (${modelnumber} : ${serial}) ##########"

						if [[ $test_ata_error -gt "0" ]]; then
							if [[ $test_ata_error -gt $ataerrors ]]; then 
								if [[ $NVMe_Ignore_Invalid_Errors != "enable" ]]; then
									smartctl -H -A -l error /dev/"$drive"    # Lets allow all error messages
								else
									smartctl -H -A -l error /dev/"$drive" | grep -v -i 'Invalid Field in Command'  # Erase those error messages
								fi
							else
								if [[ $NVMe_Ignore_Invalid_Errors != "enable" ]]; then
									smartctl -H -A -l error /dev/"$drive"    # Lets allow all error messages
								else
									smartctl -H -A -l error /dev/"$drive" | grep -v -i 'Invalid Field in Command'   # Lets allow all error messages
								fi
								echo "ATA Error Count: "$test_ata_error
								echo " "
							fi
						else
							if [[ $NVMe_Ignore_Invalid_Errors != "enable" ]]; then
								smartctl -H -A -l error /dev/"$drive"
							else
								smartctl -H -A -l error /dev/"$drive" | grep -v -i 'Invalid Field in Command'   # Lets allow all error messages
							fi
						fi

						# Create Recent Tests Report
						echo "Most recent Short & Extended Tests - Listed by test number"

					if [[ $(smartctl -x /dev/"$drive" | egrep "# 1") ]]; then		# Adjustment for Smartctl 7.4
						lasttest1="$(smartctl -x /dev/"$drive" | egrep "# 1")"
						if [[ $lasttest1 == *"Short offline"* ]]; then lastfind="Extended offline"; fi;
						if [[ $lasttest1 == *"Extended offline"* ]]; then lastfind="Short offline"; fi;
					else
						lasttest1="$(smartctl -x /dev/"$drive" | egrep -E ' (Short|Extended) ' | head -1)"
						if [[ $lasttest1 == *"Short"* ]]; then lastfind="Extended"; fi;
						if [[ $lasttest1 == *"Extended"* ]]; then lastfind="Short"; fi;
					fi
						echo $lasttest1

						lasttest2="$(smartctl -x /dev/"$drive" | egrep "$lastfind" | head -1)"
						echo $lasttest2
						echo "
"
					) >> "$logfile"

	 				# SCT Error Recovery Control Report
					scterc="$(smartctl -l scterc /dev/"$drive" | tail -3 | head -2)"
					(
						echo "SCT Error Recovery Control: "$scterc
						echo "
"
					) >> "$logfile"
				fi
			done
		fi
	fi
		}

########## COMPILE NON-SMART REPORT ##########
### NON-SMART status report section
# I don't particularly use this but some folks might find it useful.
# To activate it, in the variables set ReportNonSMART=true.
# It will list all drives where Non-SMART is true and remove devices starting with "cd", for example "cd0"

non_smart_report () {
if [[ "$Monitor" == "true" ]]; then return; fi

	drives=$nonsmartdrives
	if [ $ReportNonSMART == "true" ]; then 
		for drive in $drives; do
			if [ ! "$(echo "$drive" | grep "cd")" ]; then
				# Gather model number and serial number of each drive
				modelnumber=""
				serial=""   
				smartdata="$(smartctl -a /dev/"$drive")"
				smartdata5="$(smartctl -x --json=u /dev/"$drive")"
				serial="$(echo "${smartdata5}" | jq -Mre '.serial_number | values')"
				if [[ $serial == "" ]]; then
					serial="$(echo "$smartdata" | grep "Serial Number" | awk '{print $3}')"
				fi
				modelnumber="$(echo "${smartdata5}" | jq -Mre '.model_name | values')"
				modelnumber="$(echo "${modelnumber}" | sed -e 's/\ *$//g')"
				if [[ $serial == "" ]]; then serial="N/A"; fi
				if [[ $modelnumber == "" ]]; then modelnumber="N/A"; fi
				(
					echo "
"
					echo "########## NON-SMART status report for ${drive} drive (${modelnumber} : ${serial}) ##########"
					# And we will dump everything since it's not a standard SMART device.
					echo "SMARTCTL DATA"
					smartctl -a /dev/"$drive"
					echo "
"
					if [ $softver == "Linux" ]; then
						echo "FDISK DATA"
						fdisk -l /dev/"$drive"
					fi > /dev/null 2<&1
				) >> "$logfile"
			fi
		done
	fi
	}

########## REMOVE UN-NEEDED JUNK AND FINALIZE EMAIL MESSAGE END ##########

remove_junk_report () {
 
	### Remove some un-needed junk from the output
	sed -i -e '/smartctl/d' "$logfile"
	sed -i -e '/Copyright/d' "$logfile"
	sed -i -e '/=== START OF READ/d' "$logfile"
	sed -i -e '/SMART Attributes Data/d' "$logfile"
	sed -i -e '/Vendor Specific SMART/d' "$logfile"
	sed -i -e '/SMART Error Log Version/d' "$logfile"
	}

attach_files () {

#### CAN THE REST OF THIS FUNCTION THIS BE MOVED INTO A SINGLE SECTION FOR ATTACHMENTS ?????   ######
	doit="false"

	if [[ "$dump_all" != "0" ]]; then
	output_x=0
	# Now attach all the attachments.
	(
	#	echo "MIME-Version: 1.0"	
	#	echo "Content-Type: multipart/mixed; boundary=${boundary}"

	#	echo "From: ${From}"		
	#	echo "To: ${Email}"
	#	echo "Subject: ${subject}"

	#	echo "--${boundary}"
	#	echo "Content-Type: text/html"
	#	if [[ $Monitor == "false" ]]; then
			if [[ "$(echo $programver | grep -i "beta")" ]] || [[ $UpdateAvailable == "true" ]]; then echo ""; fi
				echo $programver"
Report Run "$(date +%d-%b-%Y)" @ "$timestamp"
"
				duration=$SECONDS
				if [[ $duration < "60" ]]; then
					echo "Execution Time: $(($duration % 60)) Seconds"
				else
					echo "Execution Time: $(($duration / 60)) Minutes : $(($duration % 60)) Seconds"
				fi
				if [[ $UpdateAvailable == "true" ]]; then echo "
UPDATE AVAILABLE --> "$GitVersion; fi
				if [[ $Messages != "" ]]; then echo "
Message from the Creator: "$Messages"
"; fi
			if [[ "$(echo $programver | grep -i "beta")" ]] || [[ $UpdateAvailable == "true" ]]; then echo ""; fi
			if [[ $TrueNASConfigEmailEncryption != "" ]] && [[ $Config_Encrypted == "true" ]]; then echo "
Attached zip file is encrypted."; fi
			echo "

"
			# Keyboard_Message comes from the '-dump email' command.
			if [[ $Keyboard_Message != "" ]]; then
				echo ""$Keyboard_Message"

"
			fi
	#	fi
	) > ${logfile_header}
		output_html="$(cat $logfile_header)$(cat $logfile)"
		output_x=1
		(
			# Write MIME section header for file attachment (encoded with base64)
			if [[ $output_html != "" ]]; then
				echo "--${boundary}"
				echo "Content-Type: text/html; charset=utf-8"
				echo "Content-Transfer-Encoding: base64"
				echo "Content-Disposition: attachment; filename=email_body.html"
				base64 <<< $output_html
			fi
		) >> "$logfile"

	# Attach dump files first

		for drive in $smartdrives; do
			dump_drive_data
		done
		for drive in $smartdrivesSSD; do
			dump_drive_data
		done
		for drive in $smartdrivesNVM; do
			dump_drive_data
		done
		for drive in $nonsmartdrives; do
			dump_drive_data
		done
		doit="true"
	fi

	if [[ $Attach_Files1 == "true" ]]; then
		(
			if [[ $Attach_Multi_Report == "true" ]]; then
				if test -e "/tmp/Old_multi_report_config.txt"; then
					# Write MIME section header for file attachment (encoded with base64)
					echo "--${boundary}"
					echo "Content-Type: text/html"
					echo "Content-Transfer-Encoding: base64"
					echo "Content-Disposition: attachment; filename=Old_multi_report_config.txt"
					base64 "/tmp/Old_multi_report_config.txt"
					rm /tmp/Old_multi_report_config.txt
				fi

				if test -e "$Config_File_Name"; then
					# Write MIME section header for file attachment (encoded with base64)
					echo "--${boundary}"
					echo "Content-Type: text/html"
					echo "Content-Transfer-Encoding: base64"
					echo "Content-Disposition: attachment; filename=multi_report_config.txt"
					base64 "$Config_File_Name"
				fi
			fi

			if [[ $Attach_Statistical_File == "true" ]]; then
				if test -e $statistical_data_file; then
					# Write MIME section header for file attachment (encoded with base64)
					echo "--${boundary}"
					echo "Content-Type: text/csv"
					echo "Content-Transfer-Encoding: base64"
					echo "Content-Disposition: attachment; filename=Statistical_Data.csv"
					base64 "$statistical_data_file"
				fi
			fi

			if [[ $Attach_TrueNAS_Config == "true" ]]; then
				if test -e /tmp/$Config_Name; then
					echo "--${boundary}"
					echo "Content-Type: application/zip"
					echo "Content-Transfer-Encoding: base64"
					echo "Content-Disposition: attachment; filename=${Config_Name}"
					base64 /tmp/$Config_Name
					rm /tmp/$Config_Name
				fi
			fi

			if [[ $dump_all != "0" ]]; then
				$(zpool list > /tmp/zpoollist.txt)
				if test -e "/tmp/zpoollist.txt"; then
					echo "--${boundary}"
					echo "Content-Type: text/html"
					echo "Content-Transfer-Encoding: base64"
					echo "Content-Disposition: attachment; filename=zpoollist.txt"
					base64 "/tmp/zpoollist.txt"
					rm /tmp/zpoollist.txt
				fi

				if test -e "/tmp/zpoolstatus.txt"; then
				$(zpool status -v > /tmp/zpoolstatus.txt)
					echo "--${boundary}"
					echo "Content-Type: text/html"
					echo "Content-Transfer-Encoding: base64"
					echo "Content-Disposition: attachment; filename=zpoolstatus.txt"
					base64 "/tmp/zpoolstatus.txt"
					rm /tmp/zpoolstatus.txt
				fi

				if test -e "/tmp/zfslist.txt"; then
				$(zfs list > /tmp/zfslist.txt)
					echo "--${boundary}"
					echo "Content-Type: text/html"
					echo "Content-Transfer-Encoding: base64"
					echo "Content-Disposition: attachment; filename=zfslist.txt"
					base64 "/tmp/zfslist.txt"
					rm /tmp/zfslist.txt
				fi

				if [[ $json_error_log != "" ]]; then
					echo "--${boundary}"
					echo "Content-Type: text/plain"
					echo "Content-Transfer-Encoding: base64"
					echo "Content-Disposition: attachment; filename=json_error_log.txt"
					base64 <<< $json_error_log
					
				fi
			fi
		) >> "$logfile"
	fi

	if test -e "/tmp/spencer_report.txt"; then
		(
		if [ $(grep -wic "New Error Messages" "/tmp/spencer_report.txt") -gt 0 ] || [ $(grep -wic "Previous" "/tmp/spencer_report.txt") -gt 0 ]; then
			echo "--${boundary}"
			echo "Content-Type: text/html"
			echo "Content-Transfer-Encoding: base64"
			echo "Content-Disposition: attachment; filename=spencer_report.txt"
			base64 "/tmp/spencer_report.txt"
		fi
		) >> "$logfile"
	fi
	}

########## CRUNCH THE NUMBERS and FORMAT MESSAGES and COLORS ##########
##### Call with HDD|SSD|NVM ##############

crunch_numbers () {
	detail_level=$1
	### Lets adjust for all the Media Alarms, Temp Alarms, and Wear Level for the new Custom_Drives_List
	# We need to change the values in the running script to use slight different variables
	# for example SectorsWarn will now be SectorsWarnx
	# Do this for all the pertinent variables and add a section to scan the Custom_Drives_List variable
	# and if a serial number matches, then use the variables there vs the defaults.

	### Order of data -- $serial":"$tempwarn":"$tempcrit":"$sectorswarn":"$sectorscrit":"$reallocwarn":"$multizonewarn":"$multizonecrit":"$rawreadwarn":"$rawreadcrit":"$seekerrorswarn":"$seekerrorscrit":"$testage":"$testAgeOvrd":"$heliummin":"$wearleveladj"

	# Predefine default variables
	if [[ $Custom_Drives_ListDrive == "HDD" ]]; then
		HDDtempWarnx=$HDDtempWarn; HDDtempCritx=$HDDtempCrit
	fi
	if [[ $Custom_Drives_ListDrive == "SSD" ]]; then
		SSDtempWarnx=$SSDtempWarn; SSDtempCritx=$SSDtempCrit
	fi
	if [[ $Custom_Drives_ListDrive == "NVM" ]]; then
		NVMtempWarnx=$NVMtempWarn; NVMtempCritx=$NVMtempCrit
	fi 

	SectorsWarnx=$SectorsWarn
	SectorsCritx=$SectorsCrit
	ReAllocWarnx=$ReAllocWarn
	MultiZoneWarnx=$MultiZoneWarn
	MultiZoneCritx=$MultiZoneCrit
	RawReadWarnx=$RawReadWarn
	RawReadCritx=$RawReadCrit
	SeekErrorsWarnx=$SeekErrorsWarn
	SeekErrorsCritx=$SeekErrorsCrit
	TestWarnAgex=$TestWarnAge
	testAgeOvrd="0"
	HeliumMinx=$HeliumMin

	### 'd' = Default Values

	IFS=',' read -ra ADDR <<< "$Custom_Drives_List"
	for i in "${ADDR[@]}"; do
		cdrivesn1="$(echo $i | cut -d':' -f 1)"
		if [[ $cdrivesn1 == $serial ]]; then
			if [[ $Custom_Drives_ListDrive == "HDD" ]]; then
				if [[ "$(echo $i | cut -d':' -f 2)" != "d" ]]; then
					HDDtempWarnx="$(echo $i | cut -d':' -f 2)"
				fi
				if [[ "$(echo $i | cut -d':' -f 3)" != "d" ]]; then
					HDDtempCritx="$(echo $i | cut -d':' -f 3)"
				fi
			fi
			if [[ $Custom_Drives_ListDrive == "SSD" ]]; then
				if [[ "$(echo $i | cut -d':' -f 2)" != "d" ]]; then
					SSDtempWarnx="$(echo $i | cut -d':' -f 2)"
				fi
				if [[ "$(echo $i | cut -d':' -f 3)" != "d" ]]; then
					SSDtempCritx="$(echo $i | cut -d':' -f 3)"
				fi
			fi
			if [[ $Custom_Drives_ListDrive == "NVM" ]]; then
				if [[ "$(echo $i | cut -d':' -f 2)" != "d" ]]; then
					NVMtempWarnx="$(echo $i | cut -d':' -f 2)"
				fi
				if [[ "$(echo $i | cut -d':' -f 3)" != "d" ]]; then
					NVMtempCritx="$(echo $i | cut -d':' -f 3)"
				fi
			fi

			if [[ "$(echo $i | cut -d':' -f 4)" != "d" ]]; then SectorsWarnx="$(echo $i | cut -d':' -f 4)"; fi
			if [[ "$(echo $i | cut -d':' -f 5)" != "d" ]]; then SectorsCritx="$(echo $i | cut -d':' -f 5)"; fi
			if [[ "$(echo $i | cut -d':' -f 6)" != "d" ]]; then ReAllocWarnx="$(echo $i | cut -d':' -f 6)"; fi
			if [[ "$(echo $i | cut -d':' -f 7)" != "d" ]]; then MultiZoneWarnx="$(echo $i | cut -d':' -f 7)"; fi
			if [[ "$(echo $i | cut -d':' -f 8)" != "d" ]]; then MultiZoneCritx="$(echo $i | cut -d':' -f 8)"; fi
			if [[ "$(echo $i | cut -d':' -f 9)" != "d" ]]; then RawReadWarnx="$(echo $i | cut -d':' -f 9)"; fi
			if [[ "$(echo $i | cut -d':' -f 10)" != "d" ]]; then RawReadCritx="$(echo $i | cut -d':' -f 10)"; fi
			if [[ "$(echo $i | cut -d':' -f 11)" != "d" ]]; then SeekErrorsWarnx="$(echo $i | cut -d':' -f 11)"; fi
			if [[ "$(echo $i | cut -d':' -f 12)" != "d" ]]; then SeekErrorsCritx="$(echo $i | cut -d':' -f 12)"; fi
			if [[ "$(echo $i | cut -d':' -f 13)" != "d" ]]; then TestWarnAgex="$(echo $i | cut -d':' -f 13)"; fi

			testAgeOvrd="$(echo $i | cut -d':' -f 14)"
			if [[ "$(echo $i | cut -d':' -f 15)" != "d" ]]; then HeliumMinx="$(echo $i | cut -d':' -f 15)"; fi
			if [[ "$(echo $i | cut -d':' -f 16)" != "d" ]]; then wearLevelAdj="$(echo $i | cut -d':' -f 16)"; fi

			### Remove this check in future version
			if [[ "$(echo $i | cut -d':' -f 16)" == "" ]] || [[ "$(echo $i | cut -d':' -f 16)" == "," ]]; then
				wearLevelAdj="d"
				echo "Illegal Custom Drive Configuration Data...  Delete and recreate the Custom Drive configuration data."
				echo "Temporary patch applied for Wear Level Adjustment."
			else
				wearLevelAdj="$(echo $i | cut -d':' -f 16)"
			fi
		fi
	done

	### Remove Leading Zeros from all variables
	# This is important because double square brackets interpret a leading zero as Octal number
	# This only works for positive numbers, not negative.  Thankfully I should not have negative
	# numbers in this script.

	# Make onHours a base 10 number and remove any commas
	onHours=${onHours#0}    # DO I STILL NEED THIS - REDUNDANT?
	onHours="${onHours//,}"

	### Convert onHours to onTime
	if [[ $onHours -gt "262800" ]]; then onHours=0; fi  # 30 years
	if [[ $onHours != 0 ]]; then
		if [[ $lastTestHours != "" ]]; then let testAge=$(((($onHours - $lastTestHours) / 24))); fi
		let yrs=$((($onHours / 8760)))
		let mos=$(((($onHours % 8760) / 730)))
		let dys=$((((($onHours % 8760) % 730) / 24)))
		let hrs=$(((($onHours % 8760) % 730) % 24))
		if [[ $PowerTimeFormat == "ymdh" ]]; then onTime="${yrs}y ${mos}m ${dys}d ${hrs}h";
			elif [[ $PowerTimeFormat == "ymd" ]]; then onTime="${yrs}y ${mos}m ${dys}d";
			elif [[ $PowerTimeFormat == "ym" ]]; then onTime="${yrs}y ${mos}m";
			elif [[ $PowerTimeFormat == "y" ]]; then onTime="${yrs}y";
			elif [[ $PowerTimeFormat == "h" ]]; then onTime=$onHours;
			else onTime=$onHours;
		fi
	else
		lastTestHours=$onHours
	fi

	##### CALCULATE TOTAL DATA WRITTEN ##########
	# Display in TiB Written or GiB Written

	if [[ "$1" == "NVM" ]] && [[ $tdw != "" ]]; then tdwbytes=$((( $tdw * 512000 ))); else tdwbytes=$((( $ssdblocksize * ssdtdw ))); fi
	if [[ $ID241 != "" ]]; then tdwbytes=$((( $ID241 * 1074000000 ))); fi

	if [[ $tdw != "" ]] && [[ "$1" != "NVM" ]]; then   # Assume SCSI Drive
		tdwbytes1=$( echo $tdw | cut -d'.' -f 1)
		tdwbytes2=$( echo $tdw | cut -d'.' -f 2)
		tdw=$tdwbytes1$tdwbytes2
		tdwbytes=$((( $tdw * 1000000)))
	fi

	if [[ $tdwbytes != "" ]]; then
		if [[ $tdwbytes -ge 1000000000000000 ]]; then
			tdwbytes=$((( $tdwbytes / 10000000000000 )))
			tdwbytes1="$((( $tdwbytes / 100 )))"
			tdwbytes2="$(echo $tdwbytes | rev | cut -c 1,2 | rev)"
			tdw=$tdwbytes1"."$tdwbytes2" PB"

		elif [[ $tdwbytes -ge 1000000000000 ]]; then
			tdwbytes=$((( $tdwbytes / 10000000000 )))
			tdwbytes1="$((( $tdwbytes / 100 )))"
			tdwbytes2="$(echo $tdwbytes | rev | cut -c 1,2 | rev)"
			tdw=$tdwbytes1"."$tdwbytes2" TB"

		elif [[ $tdwbytes -ge 1000000000 ]]; then
			tdwbytes=$((( $tdwbytes / 10000000 )))
			tdwbytes1="$((( $tdwbytes / 100 )))"
			tdwbytes2="$(echo $tdwbytes | rev | cut -c 1,2 | rev)"
			tdw=$tdwbytes1"."$tdwbytes2" GB"

		elif [[ $tdwbytes -ge 1000000 ]]; then
			tdwbytes=$((( $tdwbytes / 100000 )))
			tdwbytes1="$((( $tdwbytes / 10 )))"
			tdwbytes2="$(echo $tdwbytes | rev | cut -c 1)"
			tdw=$tdwbytes1"."$tdwbytes2" MB"

		elif [[ $tdwbytes -ge 1000 ]]; then
			tdwbytes=$((( $tdwbytes / 100 )))
			tdwbytes1="$((( $tdwbytes / 10 )))"
			tdwbytes2="$(echo $tdwbytes | rev | cut -c 1)"
			tdw=$tdwbytes1"."$tdwbytes2" KB"
		elif [[ $tdwbytes -ge 1 ]]; then
			tdw="<1 KB"
		else
			tdw=$Non_Exist_Value
		fi
	else
		tdw=$Non_Exist_Value
	fi

	##### CUSTOM DRIVE HACK Section #####
	# This will set the testAge value to 1 so it passes the math portion of the quality checks.

	if [[ $testAgeOvrd == "1" ]]; then testAge=1; fi

	### WARRANTY DATE
	# Use Format: DriveWarranty="DriveSerialNumber YYYY-MM-DD,"

	s="0"
	IFS=',' read -ra ADDR <<< "$Drive_Warranty_List"
	for i in "${ADDR[@]}"; do
		drivesn1="$(echo $i | cut -d':' -f 1)"
		drivedt1="$(echo $i | cut -d':' -f 2)"
		if [[ $drivesn1 == $serial ]]; then
			warrantyyear="$(echo $drivedt1 | cut -d '-' -f1)"
			warrantymonth="$(echo $drivedt1 | cut -d '-' -f2)"
			warrantyday="$(echo $drivedt1 | cut -d '-' -f3)"
			tempnow="$((`date +%s`))"

			if [[ $softver != "Linux" ]]; then 
				warrantytemp="$((`date -j -v"$warrantyyear"y -v"$warrantymonth"m -v"$warrantyday"d +%s`))"
			else
				# Debian Date in seconds
				warrantytemp="$((`date -d "$drivedt1" +%s`))"
			fi

			warrantytemp="$((("$tempnow" - "$warrantytemp")/3600))"
			let waryrs=$((($warrantytemp / 8760)))
			let warmos=$(((($warrantytemp % 8760) / 730)))
			let wardys=$((((($warrantytemp % 8760) % 730) / 24)))
			let warhrs=$(((($warrantytemp % 8760) % 730) % 24))
			let wardays=$((($warrantytemp / 24)))
			wartemp2=${wardays#-}
			wartemp3=${waryrs#-}

			if [[ $wartemp2 -gt 31 ]]; then
				if [[ $wartemp3 -gt 0 ]]; then wartext="${waryrs#-}y ${warmos#-}m ${wardys#-}d"
				else wartext="${warmos#-}m ${wardys#-}d"
				fi
			else
				wartext="${wardays#-}d"
			fi

			if [[ "$warrantytemp" > 0 ]]; then
				WarrantyClock=$wartext
			else
				WarrantyClock=${wartext#-}
			fi

			if [[ "$datestamp2" > "$drivedt1" ]]; then
				s="1"
				drivesn2=$drivesn1
				drivedt2=$drivedt1
				continue
			fi
		fi
	done
	if [[ "$WarrantyClock" == "" ]]; then
		WarrantyClock=$Non_Exist_Value
	fi
	if [[ $s != "0" ]]; then
		if [[ $WarrantyBackgndColor != "none" ]]; then WarrantyBackgroundColor=$WarrantyBackgndColor; fi
		WarrantyBoxColor=$expiredWarrantyBoxColor
		logfile_warranty=$logfile_warranty"Drive "$drivesn2" Warranty Expired on "$drivedt2"
"
	fi

	### SMART STATUS
	if [[ $smartStatus == "" || $smartStatus == "PASSED" || $smartStatus == "OK" ]]; then smartStatusColor=$okColor; else smartStatusColor=$critColor; fi
	if [[ $smartStatus == "" || $smartStatus == "PASSED" || $smartStatus == "OK" ]]; then a=1; else logfile_critical=$logfile_critical"$(printf "Drive "$device " - Check Smart Status
")"; fi
	if [[ $smartStatus == "" ]]; then smartStatus="$Non_Exist_Value"; fi

	### BAD SECTORS
	s="0"
	IFS=',' read -ra ADDR <<< "$ReAllocated_Sector_List"
	for i in "${ADDR[@]}"; do
		badsectsn1="$(echo $i | cut -d':' -f 1)"
		badsectdt1="$(echo $i | cut -d':' -f 2)"
		if [[ $badsectsn1 == $serial ]]; then
			s="1"
			badsectsn2=$badsectsn1
			badsectdt2=$badsectdt1
			continue
		fi
	done
	if [[ $s != "0" ]]; then
		reAllocColor=$ovrdColor
		reAllocOrig=$reAlloc
		reAlloc=$(($reAlloc-$badsectdt2))
	fi

	### BAD SECTORS2
	s="0"
	IFS=',' read -ra ADDR <<< "$ReAllocated_Sector_Events_List"
	for i in "${ADDR[@]}"; do
		badsectsn3="$(echo $i | cut -d':' -f 1)"
		badsectdt3="$(echo $i | cut -d':' -f 2)"
		if [[ $badsectsn3 == $serial ]]; then
			s="1"
			badsectsn4=$badsectsn3
			badsectdt4=$badsectdt3
			continue
		fi
	done
	if [[ $s != "0" ]]; then
		reAllocEventColor=$ovrdColor
		reAllocEventOrig=$reAllocEvent
		reAllocEvent=$(($reAllocEvent-$badsectdt4))
	fi

	########## TEMPERATURE SECTION ##########
	# LETS ZERO OUT BOGUS HIGH TEMPS and LOW TEMPS
	if [[ $temp != $Non_Exist_Value ]]; then
		if [[ $temp -gt 150 ]]; then temp="$Non_Exist_Value"; fi
		if [[ $temp -lt -60 ]]; then temp="$Non_Exist_Value"; fi
	fi

	### TEMP for HDD
	if [[ $detail_level == "HDD" ]]; then
		if [[ $temp != "$Non_Exist_Value" ]]; then if [[ $temp -ge $HDDtempCritx ]]; then tempColor=$critColor; else if [[ $temp -ge $HDDtempWarnx ]]; then tempColor=$warnColor; fi; fi; fi
		if [[ $temp != "$Non_Exist_Value" ]]; then if [[ $temp -ge $HDDtempCritx ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" Critical Drive Temp "$temp" - Threshold = "$HDDtempCritx"
")";
		else if [[ $temp -ge $HDDtempWarnx ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" High Drive Temp "$temp" - Threshold set at "$HDDtempWarnx"
")"; Temperature_Log=$Temperature_Log$"$(printf "Drive "$serial" High Temp "$temp"
")"; fi; fi; fi

	if [[ $HDD_Cur_Pwr_Max_Temp_Ovrd != "true" ]]; then
		if [[ $temp_max != "$Non_Exist_Value" ]]; then if [[ $temp_max -ge $HDDtempCritx ]]; then temp_maxColor=$critColor; else if [[ $temp_max -ge $HDDtempWarnx ]]; then temp_maxColor=$warnColor; fi; fi; fi
		if [[ $temp_max != "$Non_Exist_Value" ]]; then if [[ $temp_max -ge $HDDtempCritx ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" Critical Drive Temp "$temp_max" - Temp Max Threshold = "$HDDtempCritx"
")";
		else if [[ $temp_max -ge $HDDtempWarnx ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" High Drive Temp "$temp_max" - Temp Max Threshold set at "$HDDtempWarnx"
")"; fi; fi; fi
		fi
	fi

	### TEMP for SSD
	if [[ $detail_level == "SSD" ]]; then
		if [[ $temp != "$Non_Exist_Value" ]]; then if [[ $temp -ge $SSDtempCritx ]]; then tempColor=$critColor; else if [[ $temp -ge $SSDtempWarnx ]]; then tempColor=$warnColor; fi; fi; fi
		if [[ $temp != "$Non_Exist_Value" ]]; then if [[ $temp -ge $SSDtempCritx ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" Critical Drive Temp "$temp" - Threshold = "$SSDtempCritx"
")";
			else if [[ $temp -ge $SSDtempWarnx ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" High Drive Temp "$temp" - Threshold set at "$SSDtempWarnx"
")"; Temperature_Log=$Temperature_Log"$(printf "Drive "$serial" High Temp "$temp"
")"; fi; fi; fi

	if [[ $SSD_Cur_Pwr_Max_Temp_Ovrd != "true" ]]; then
		if [[ $temp_max != "$Non_Exist_Value" ]]; then if [[ $temp_max -ge $SSDtempCritx ]]; then temp_maxColor=$critColor; else if [[ $temp_max -ge $SSDtempWarnx ]]; then temp_maxColor=$warnColor; fi; fi; fi
		if [[ $temp_max != "$Non_Exist_Value" ]]; then if [[ $temp_max -ge $SSDtempCritx ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" Critical Drive Temp "$temp_max" - Threshold = "$SSDtempCritx"
")";
			else if [[ $temp_max -ge $SSDtempWarnx ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" High Drive Temp "$temp_max" - Temp Max Threshold set at "$SSDtempWarnx"
")"; fi; fi; fi
		fi
	fi

	### TEMP for NVM
	if [[ $detail_level == "NVM" ]]; then
		if [[ $temp != "$Non_Exist_Value" ]]; then if [[ $temp -ge $NVMtempCritx ]]; then tempColor=$critColor; else if [[ $temp -ge $NVMtempWarnx ]]; then tempColor=$warnColor; fi; fi; fi
		if [[ $temp != "$Non_Exist_Value" ]]; then if [[ $temp -ge $NVMtempCritx ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" Critical Drive Temp "$temp" - Threshold = "$NVMtempCritx"
")";
			else if [[ $temp -ge $NVMtempWarnx ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" High Drive Temp "$temp" - Threshold set at "$NVMtempWarnx"
")"; Temperature_Log=$Temperature_Log"$(printf "Drive "$serial" High Temp "$temp"
")"; fi; fi; fi

	### TEMP_MAX for NVM
	if [[ $NVM_Cur_Pwr_Max_Temp_Ovrd != "true" ]]; then
		if [[ $temp_max != "$Non_Exist_Value" ]]; then if [[ $temp_max -ge $NVMtempCritx ]]; then temp_maxColor=$critColor; else if [[ $temp_max -ge $NVMtempWarnx ]]; then temp_maxColor=$warnColor; fi; fi; fi
		if [[ $temp_max != "$Non_Exist_Value" ]]; then if [[ $temp_max -ge $NVMtempCritx ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" Critical Drive Temp "$temp_max" - Threshold = "$NVMtempCritx"
")";
			else if [[ $temp_max -ge $NVMtempWarnx ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" High Drive Temp "$temp_max" - Threshold set at "$NVMtempWarnx"
")"; fi; fi; fi
		fi
	fi

	# NVM CRITICAL WARNING
	if [[ $detail_level == "NVM" ]]; then
		NVMcriticalWarningColor="$okColor"
		if [[ $NVMcriticalWarning != "0" ]]; then NVMcriticalWarning="0"; fi
		if [[ $NVMcriticalWarning != "" ]]; then if [[ $NVMcriticalWarning != "0" ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" NVM Critical Warning "$NVMcriticalWarning"
")"; fi; fi
		if [[ $NVMcriticalWarning != "" ]]; then if [[ $NVMcriticalWarning != "0" ]]; then NVMcriticalWarningColor=$critColor; fi; fi
		if [[ $NVMcriticalWarning != "" ]]; then if [[ $NVMcriticalWarning != "0" ]]; then NVMcriticalWarning="CRITICAL FAILURE"; fi; fi
		if [[ $NVMcriticalWarning == "0" ]]; then NVMcriticalWarning="GOOD"; fi
		if [[ $NVMcriticalWarning == "" ]]; then NVMcriticalWarning="$Non_Exist_Value"; fi
		if [[ $mediaErrors != "" ]]; then
			if [[ $mediaErrors -ge $NVM_Media_Errors ]]; then
				mediaErrorsColor=$critColor
				logfile_critical=$logfile_critical"$(printf "Drive "$serial" - Media Errors
")"
			fi
		fi
	fi

	if [[ $detail_level == "HDD" ]]; then
		# Helium Critical Warning
		if [[ $Helium == "" ]]; then Helium="$Non_Exist_Value"; HeliumColor="$bgColor"; fi
		if [[ $Helium -ge $HeliumMinx ]] || [[ "$Helium" == "$Non_Exist_Value" ]]; then
			HeliumColor="$bgColor"
		else
			if [[ $HeliumAlarm == "true" ]]; then
			HeliumColor="$critColor"
			logfile_critical=$logfile_critical"$(printf "Drive "$serial" Helium Critical Warning - Value "$Helium"
")"
			fi
		fi > /dev/null 2>&1
	fi

	if [[ $rotation == "" ]]; then rotation="$Non_Exist_Value"; fi
	if [[ $capacity == "" ]]; then capacity="$Non_Exist_Value"; fi

	########## PROCESSING THAT AFFECTS EVERYTHING ##########
	### SPINRETRY
	if [[ $spinRetry != "" ]]; then if [[ $spinRetry != "0" ]]; then spinRetryColor=$critColor; fi; fi
	if [[ $spinRetry != "" ]]; then if [[ $spinRetry != "0" ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" Spin Retry "$spinRetry" - Threshold = 0 
")"; fi; fi
	if [[ $spinRetry == "" ]]; then spinRetry="$Non_Exist_Value"; fi

	### REALLOC and REALLOCEVENT
	if [[ $reAlloc != "" ]]; then if [[ $(($reAlloc + 0)) -gt $SectorsCritx ]]; then reAllocColor=$critColor; else if [[ $(($reAlloc + 0)) -gt $SectorsWarnx ]]; then reAllocColor=$warnColor; fi; fi; fi
	if [[ $reAlloc != "" ]]; then if [[ $(($reAlloc + 0)) -gt $SectorsCritx ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" Critical Reallocated Sectors "$reAlloc" - Threshold = "$SectorsCritx"
")";
	else if [[ $(($reAlloc + 0)) -gt $SectorsWarnx ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" Warning Reallocated Sectors "$reAlloc" - Threshold = "$SectorsWarnx"
")"; fi; fi; fi
	if [[ $reAlloc == "" ]]; then reAlloc="$Non_Exist_Value"; fi

	if [[ $reAllocEvent != "" ]]; then if [[ $(($reAllocEvent + 0)) -gt $SectorsCritx ]]; then reAllocEventColor=$critColor; else if [[ $(($reAllocEvent + 0)) -gt $SectorsWarnx ]]; then reAllocEventColor=$warnColor; fi; fi; fi
	if [[ $reAllocEvent != "" ]]; then if [[ $(($reAllocEvent + 0)) -gt $SectorsCritx ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" Critical Reallocated Sectors Events "$reAllocEvent" - Threshold = "$SectorsCritx"
")";
	else if [[ $(($reAllocEvent + 0)) -gt $SectorsWarnx ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" Warning Reallocated Sectors Events "$reAllocEvent" - Threshold = "$SectorsWarnx"
")"; fi; fi; fi
	if [[ $reAllocEvent == "" ]]; then reAllocEvent="$Non_Exist_Value"; fi

	### PENDING SECTORS
	if [[ $pending != "" ]]; then if [[ $(($pending + 0)) -gt $SectorsCritx ]]; then pendingColor=$critColor; else if [[ $(($pending + 0)) -gt $SectorsWarnx ]]; then pendingColor=$warnColor; fi; fi; fi
	if [[ $pending != "" ]]; then if [[ $(($pending + 0)) -gt $SectorsCritx ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" Pending Sector Errors "$pending" - Threshold = "$SectorsCritx"
")";
	else if [[ $(($pending + 0)) -gt $SectorsWarnx ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" Pending Sector Errors "$pending" - Threshold = "$SectorsWarnx"
")"; fi; fi; fi
	if [[ $pending == "" ]]; then pending="$Non_Exist_Value"; fi

	### OFFLINE UNCORRECTABLE SECTORS
	if [[ $offlineUnc != "" ]]; then if [[ $(($offlineUnc + 0)) > $SectorsCritx ]]; then offlineUncColor=$critColor; else if [[ $offlineUnc != 0 ]]; then offlineUncColor=$warnColor; fi; fi; fi
	if [[ $offlineUnc != "" ]]; then if [[ $(($offlineUnc + 0)) > $SectorsCritx ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" Uncorrectable Sector Errors "$offlineUnc"
")";
	else if [[ $(($offlineUnc + 0)) -gt $SectorsWarnx ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" Uncorrectable Sector Errors "$offlineUnc" - Threshold = "$SectorsWarnx"
")";fi; fi; fi
	if [[ $offlineUnc == "" ]]; then offlineUnc="$Non_Exist_Value"; fi

	### CRC ERRORS
	if [[ $crcErrors != "" ]]; then if [[ $crcErrors != "0" ]]; then crcErrorsColor=$critColor; fi; fi
	if [[ $crcErrors == "" ]]; then crcErrors="$Non_Exist_Value"; fi

	### SMARTTESTING
	if [[ $smarttesting -gt 0 ]]; then lastTestType="$smarttesting% Remaining"; fi

	### CHKREADFAILURE
	if [[ $lastTestStatusPass != "true" ]]; then lastTestType=$lastTestStatus; lastTestTypeColor=$critColor; logfile_critical=$logfile_critical"$(printf "Drive "$serial" Read Failure "$chkreadfailure"
")"; else lastTestTypeColor=$bgColor; fi 

	### SEEK ERRORS
	# If seekErrorHealth RAW_VALUE is some crazy number, use the VALUE column data.
	# Seek Error Rate fix for Seagate Drives
	# We use seekErrorHealth for the Seagate Rate, and seekErrorHealth2 for the Normalized Rate if we must.

	if [[ $seekErrorHealth -gt 0 ]]; then

		# Lets see if this is a NORMAL drive, not reporting crazy ass numbers.
		if [[ $seekErrorHealth -lt $SeekErrorsWarnx ]] && [[ $seekErrorHealth -le 4294967295 ]]; then seek="done"; fi

		if [[ $seekErrorHealth -ge $SeekErrorsWarnx ]] && [[ $seekErrorHealth -lt $SeekErrorsCritx ]]; then
			seekErrorHealthColor=$warnColor
			seek="done"
		fi
		if [[ $seekErrorHealth -ge $SeekErrorsCritx ]] && [[ $seekErrorHealth -le 500 ]]; then
			seekErrorHealthColor=$critColor
			seek="done"
		fi

		# If the count is above the Seagate FFFFFFFF value, divide it out or if below FFFFFFFF then make value zero.

		if [[ $seekErrorHealth -lt 4294967295 ]] && [[ $seek != "done" ]]; then seekErrorHealth=0; fi
		if [[ $seekErrorHealth -ge 4294967295 ]]; then seekErrorHealth=$(($seekErrorHealth / 4294967295)); fi

		if [[ $IgnoreSeekError != "true" ]]; then
			if [[ $(($seekErrorHealth + 0)) -gt $SeekErrorsCritx ]]; then
				seekErrorHealthColor=$critColor
				logfile_critical=$logfile_critical"$(printf "Drive "$serial" Seek Errors "$seekErrorHealth" - Threshold = "$SeekErrorsCritx"
")"
			else
				if [[ $(($seekErrorHealth + 0)) -gt $SeekErrorsWarnx ]]; then
					seekErrorHealthColor=$warnColor
					logfile_warning=$logfile_warning"$(printf "Drive "$serial" Seek Errors "$seekErrorHealth" - Threshold = "$SeekErrorsWarnx"
")"
				fi
			fi
		fi
	fi
	seek=""

	### Raw Read Error Rate
	# If seekErrorHealth RAW_VALUE is some crazy number, use the VALUE column data.
	# Raw Read Error Rate fix for Seagate Drives
	# We use rawReadErrorRate for the Seagate Rate, and rawReadErrorRate2 for the Normalized Rate if we must.

	if [[ $rawReadErrorRate -gt 0 ]] && [[ $rotation == "0" ]]; then rawReadErrorRate=""; fi

	if [[ $rawReadErrorRate -gt 0 ]]; then
		# Lets see if this is a NORMAL drive, not reporting crazy ass numbers.
		if [[ $rawReadErrorRate -lt $RawReadWarnx ]] && [[ $rawReadErrorRate -le 4294967295 ]]; then seek="done"; fi

		if [[ $rawReadErrorRate -ge $RawReadWarnx ]] && [[ $rawReadErrorRate -lt $RawReadCritx ]]; then
			rawReadErrorRateColor=$warnColor
			seek="done"
		fi
		if [[ $rawReadErrorRate -ge $RawReadCritx ]] && [[ $rawReadErrorRate -le 500 ]]; then
			rawReadErrorRateColor=$critColor
			seek="done"
		fi

		# If the count is above the Seagate FFFFFFFF value, divide it out or if below FFFFFFFF then make value zero

		if [[ $rawReadErrorRate -lt 4294967295 ]] && [[ $seek != "done" ]]; then rawReadErrorRate=0; fi
		if [[ $rawReadErrorRate -ge 4294967295 ]]; then rawReadErrorRate=$(($rawReadErrorRate / 4294967295)); fi

		if [[ $IgnoreReadError != "true" ]]; then
			if [[ $(($rawReadErrorRate + 0)) -ge $RawReadCritx ]]; then
				rawReadErrorRateColor=$critColor
				logfile_critical=$logfile_critical"$(printf "Drive "$serial" Raw Read Error Rate "$rawReadErrorRate" - Threshold = "$RawReadCritx"
")"
			else
				if [[ $(($rawReadErrorRate + 0)) -ge $RawReadWarnx ]]; then
					rawReadErrorRateColor=$warnColor
					logfile_warning=$logfile_warning"$(printf "Drive "$serial" Raw Read Error Rate "$rawReadErrorRate" - Threshold = "$RawReadWarnx"
")"
				fi
			fi
		fi
	fi
	seek=""

	wearLevelColor=$bgColor
	if [[ $multiZone == "" ]]; then multiZone="$Non_Exist_Value"; fi
	if [[ $wearLevelAdj == "i" ]]; then wearLevel=""; fi
	if [[ $wearLevel == "" || $wearLevel == "0" ]]; then wearLevel="$Non_Exist_Value"; else wearLevel=$(($wearLevel + 0)); fi
	if [[ $wearLevel != "$Non_Exist_Value" ]]; then if [[ $wearLevel -lt $WearLevelCrit ]]; then wearLevelColor=$warnColor; logfile_warning=$logfile_warning"$(printf "Drive: "$serial" - Wear Level = "$wearLevel"%%
")"; fi; fi
	if [[ $modelnumber == "" ]]; then modelnumber="$Non_Exist_Value"; fi
	if [[ $startStop == "" ]]; then startStop="$Non_Exist_Value"; fi
	if [[ $loadCycle == "" ]]; then loadCycle="$Non_Exist_Value"; fi
	if [[ $seekErrorHealth == "" ]]; then seekErrorHealth="$Non_Exist_Value"; fi
	if [[ $rawReadErrorRate == "" ]]; then rawReadErrorRate="$Non_Exist_Value"; fi
	if [[ $Helium == "" ]]; then Helium="$Non_Exist_Value"; fi
	if [[ $mediaErrors == "" ]]; then mediaErrors="$Non_Exist_Value"; fi

	########## WRITE STATISTICAL DATA ##########
	# Save Statistical Data before we make any changes to it.

	if [[ $SDF_DataRecordEnable == "true" && $writing_data != "1" ]]; then
		writing_data=1
	fi

	tempdrive=$drive
	# Replace Device ID in spreadsheet with TEST so we can identify test data and use the Purge routine.
	if [[ $testfilepath != "" ]]; then drive="TEST"; fi
	if [[ $SDF_DataRecordEnable == "true" ]] && [[ $Write_Statistics == "true" ]]; then printf $datestamp","$timestamp","$drive","$detail_level","$serial","$smartStatus","$temp","$onHours","$wearLevel","$startStop","$loadCycle","$spinRetry","$reAlloc","$reAllocEvent","$pending","$offlineUnc","$crcErrors","$seekErrorHealth","$multiZone","$rawReadErrorRate","$Helium",\n" >> "$statistical_data_file";fi
	drive="$tempdrive"

	### Routine to zero out the UDMA CRC Error Count and Highlights it Yellow.

	s="0"
	IFS=',' read -ra ADDR <<< "$CRC_Errors_List"
	for i in "${ADDR[@]}"; do
		crc_errsn1="$(echo $i | cut -d':' -f 1)"
		crc_errst1="$(echo $i | cut -d':' -f 2)"
		if [[ $crc_errsn1 == $serial ]]; then
			s="1"
			crc_errsn2=$crc_errsn1
			crc_errst2=$crc_errst1
			continue
		fi
	done
	if [[ $s != "0" ]]; then
		crcErrorsColor=$ovrdColor
		crcErrorsOrig=$crcErrors
		crcErrors=$(($crcErrors-$crc_errst2))
	fi

	s="0"
	IFS=',' read -ra ADDR <<< "$MultiZone_List"
	for i in "${ADDR[@]}"; do
		badsectsn1="$(echo $i | cut -d':' -f 1)"
		badsectdt1="$(echo $i | cut -d':' -f 2)"
		if [[ $badsectsn1 == $serial ]]; then
			s="1"
			badsectsn2=$badsectsn1
			badsectdt2=$badsectdt1
			continue
		fi
	done
	if [[ $s != "0" ]]; then
		multiZoneColor=$ovrdColor
		multiZoneOrig=$multiZone
		multiZone=$(($multiZone-$badsectdt2))
	fi

	if [[ $IgnoreMultiZone != "true" ]]; then if [[ $multiZone != "$Non_Exist_Value" ]]; then if [[ $multiZone -gt $MultiZoneWarnx ]]; then multiZoneColor=$warnColor; logfile_warning=$logfile_warning"$(printf "Drive: "$serial" - MultiZone Errors = "$multiZone"
")"; fi; fi; fi
	if [[ $IgnoreMultiZone != "true" ]]; then if [[ $multiZone != "$Non_Exist_Value" ]]; then if [[ $multiZone -gt $MultiZoneCritx ]]; then multiZoneColor=$critColor; logfile_critical=$logfile_critical"$(printf "Drive: "$serial" - MultiZone Errors = "$multiZone"
")";fi ;fi ;fi
	if [[ $IgnoreUDMA != "true" ]]; then if [[ $crcErrors != "$Non_Exist_Value" ]]; then if [[ $crcErrors != "0" ]]; then logfile_critical=$logfile_critical"$(printf "Drive "$serial" CRC Errors "$crcErrors"
")";fi; fi; fi

	if [[ $testAge -ge $TestWarnAgex ]]; then testAgeColor=$warnColor; else testAgeColor=$bgColor; fi
	if [[ $testAge -ge $TestWarnAgex ]]; then logfile_warning=$logfile_warning"$(printf "Drive: "$serial" - Test Age = "$testAge" Days
")"; fi
	if [[ $lastTestHours == "0" || $lastTestHours == "" ]]; then testAge=$Non_Exist_Value; fi
# REMOVE THE NEXT LINE ONCE SMARTMONTOOLS 7.4 IS ADDED
if [[ $NVMelastTestHoursFake == "true" ]]; then testAge=$Non_Exist_Value; fi

	########## DEVICE ID WARNING COLOR ##########
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$HeliumColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$reAllocColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$reAllocEventColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$crcErrorsColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$pendingColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$offlineUncColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$seekErrorHealthColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$rawReadErrorRateColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$testAgeColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$multiZoneColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$wearLevelColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$spinRetryColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$temp_maxColor; fi; fi; fi
	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$tempColor; fi; fi; fi
	if [[ $smartStatusColor != $okColor ]]; then if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$smartStatusColor; fi; fi; fi; fi

	if [[ $DeviceRedFlag == "true" ]]; then if [[ $deviceStatusColor != $critColor ]]; then if [[ $deviceStatusColor != $warnColor ]]; then deviceStatusColor=$mediaErrorsColor; fi; fi; fi

	# SCT Error Recovery Control Report
	scterc="$(smartctl -l scterc /dev/"$drive" | tail -3 | head -2)"

	if [[ $SCT_Warning_Level == "TLER" ]]; then
		# Warning Level TLER = Ignore Drives that do not report "seconds" or "Disable"
		# Warning Level TLER_No_Msg = same as above but will not report TLER disabled message until after trying to set TLER fails.
		if [[ $scterc =~ "Disabled" ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" TLER is Disabled
")"; fi
	fi

	if [[ $SCT_Warning_Level == "TLER_No_Msg" && $SCT_Enable == "true" ]]; then
		if [[ $scterc =~ "Disabled" ]]; then
			# Now we set the TLER ONLY for Disabled Drives because we do not know how it will affect other drives.
			smartctl -l scterc,"$SCT_Read_Timeout","$SCT_Write_Timeout" /dev/"$drive" > /dev/null 2>&1
			scterc="$(smartctl -l scterc /dev/"$drive" | tail -3 | head -2)"
			if [[ $scterc =~ "seconds" ]]; then logfile_messages=$logfile_messages"$(printf "Drive "$serial" TLER is NOW ENABLED !
")"; fi
			if [[ $scterc =~ "Disabled" ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" TLER is Disabled and failed to set.
")"; fi
		fi
	fi

	if [[ $SCT_Warning_Level == "all" ]]; then 
		if [[ $scterc =~ "Disabled" ]]; then
			logfile_warning=$logfile_warning"$(printf "Drive "$serial" TLER is Disabled
")"
		else
			if [[ ! $scterc =~ "seconds" ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" TLER is Unsupported
")"; fi
		fi
	fi

	if [[ $SCT_Enable == "true" ]]; then
		if [[ $scterc =~ "Disabled" ]]; then
			# Now we set the TLER ONLY for Disabled Drives because we do not know how it will affect other drives.
			smartctl -l scterc,"$SCT_Read_Timeout","$SCT_Write_Timeout" /dev/"$drive" > /dev/null 2>&1
			scterc="$(smartctl -l scterc /dev/"$drive" | tail -3 | head -2)"
			if [[ $SCT_Warning_Level == "all" || $SCT_Warning_Level == "TLER" ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" TLER is NOW ENABLED !
")"; fi
			if [[ $scterc =~ "Disabled" ]]; then
				if [[ $SCT_Warning_Level == "all" || SCT_Warning_Level == "TLER" ]]; then logfile_warning=$logfile_warning"$(printf "Drive "$serial" TLER is Disabled and failed to set.
")"; fi
			fi
		fi
	fi

	# This section will change the testAge value to the Non_Exist_Value for proper display in the chart.  Displaying a bogus "1" is misleading.

	if [[ $testAgeOvrd == "1" ]]; then testAge=$Non_Exist_Value; fi

	}

########## UPDATE CONFIG FILE ##########
### This will generate or update a configuration file with any new changes.
### The customizations will remain until the end of a major version number and then
###   be removed in order to support the next major version number.
### Current updates incorporated to support version 2.1.x to 3.0.x

update_config_file () {

		# Lets backup the current config file and send to the user.
		cp "$Config_File_Name" /tmp/Old_multi_report_config.txt
	(
		# Lets convert any old variables into new variables from version 2.4 to 3.0

		# version 3.0
		if [[ "$Ignore_Drives_List" == "none" ]]; then Ignore_Drives_List=""; fi
		if [[ "$Multipath" == "true" ]]; then Multipath="normal"; fi
		if [[ "$Multipath" == "false" ]]; then Multipath="off"; fi

		# Lets write the configuration file.
		echo "#" $programver
		echo "#"
		echo "# This file is used exclusively to configure the multi_report version 2.4 or later."
		echo "#"
		echo "# The configuration file will be created in the same directory as the script."
		echo "#"
		echo "# The configuration file will override the default values coded into the script."
		echo " "
		echo "###### Email Address"
		echo "### Enter your Email address to send the report to.  The from address does not need to be changed unless you experience"
		echo "### an error sending the email.  Some email servers only use the email address associated with the email server."
		echo " "
		echo 'Email="'$Email'"		# Send normal emails to this address'
		echo 'From="'$From'"			# From address (default works for many)'
		echo " "
		echo "###### Alert Email Configuration"
		echo "### You must use the '-m' switch"
		echo 'AlertEmail="'$AlertEmail'"		# Alert email address used with the '-m' switch.'
		echo 'AlertOnWarningTemp="'$AlertOnWarningTemp'"			# Send alert on Warning Temp. Default="true"'
		echo 'AlertOnCriticalError="'$AlertOnCriticalError'"			# Send alert on Critical Error. Default="true"'
		echo " "
		echo "###### EMAIL ON ALARM ONLY"
		echo 'Email_On_Alarm_Only="'$Email_On_Alarm_Only'"			# When true, an email will only be sent if an alarm condition exists.  Default = "false"'
		echo 'Email_On_Alarm_Only_And_Attachments="'$Email_On_Alarm_Only_And_Attachments'"	# When "true", email attachments will be sent even when no alarm condition exists.  Default = "true"'
		echo " "
		echo "########## Script Update"
		echo "### Ensure you understand these options."
		echo 'CheckForUpdates="'$CheckForUpdates'"	# Will check GitHub for updates and include message in next email.'
		echo 'AutomaticUpdate="'$AutomaticUpdate'"	# WARNING !!!  This option will automatically update the script if a newer version exists on GitHub.'
		echo " "
		echo "###### Spencer Integration"
		echo "### Warning Levels are: None, Warning, Critical -- This only affects the Email Subject Line, if any errors are present, an attachment will occur."
		echo 'spencer_new_warning_level="'$spencer_new_warning_level'"	# What to do if a "new" error occurs.'
		echo 'spencer_existing_warning_level="'$spencer_existing_warning_level'"	# What to do for an existing error.'
		echo 'spencer_enable="'$spencer_enable'"			# To call the Spencer.py script if "true" or "false" to not run the Spencer.py script.'
		echo 'spencer_script_name="'$spencer_script_name'"	# The default is "spencer.py" located in the default script directory.'
		echo " "
		echo "##### Multipath Enable/Disable"
		echo "### When active this will (read below options):"
		echo 'Multipath="'$Multipath'"		# "off" = No processing of serial numbers and is the default value.'
		echo '			# "normal" = Automatically remove duplicate serial numbers.'
		echo '			# "Exos2x" = Remove duplicate serial numbers ONLY IF the gptid matches.'
		echo '			# "serial" = Sort by serial numbers.'
		echo " "
		echo "###### Zpool Status Summary Table Settings"
		echo 'PoolUsedWarn='$PoolUsedWarn'		# Pool used percentage for CRITICAL color to be used.'
		echo 'ScrubAgeWarn='$ScrubAgeWarn'		# Maximum age (in days) of last pool scrub before CRITICAL error (30 + 7 days for day of week). Default=37.'
		echo " "
		echo "###### Temperature Settings"
		echo "HDDtempWarn=$HDDtempWarn		# HDD Drive temp (in C) when WARNING message will be used."
		echo "HDDtempCrit=$HDDtempCrit		# HDD Drive temp (in C) when CRITICAL message will be used."
		echo "SSDtempWarn=$SSDtempWarn		# SSD Drive temp (in C) when WARNING message will be used."
		echo "SSDtempCrit=$SSDtempCrit		# SSD Drive temp (in C) when CRITICAL message will be used."
		echo "NVMtempWarn=$NVMtempWarn		# NVM Drive temp (in C) when WARNING message will be used."
		echo "NVMtempCrit=$NVMtempCrit		# NVM Drive temp (in C) when CRITICAL message will be used."
		echo "			# --- NOTE: NVMe drives currently do not report Min/Max temperatures so this is a future feature."
		echo " "
		echo "###### Current Power Cycle Maximum Temperature Override"
		echo 'HDD_Cur_Pwr_Max_Temp_Ovrd="'$HDD_Cur_Pwr_Max_Temp_Ovrd'"	# HDD Max Drive Temp Override. This value when "true" will not alarm on any Current Power Cycle Max Temperature Limit.'
		echo 'SSD_Cur_Pwr_Max_Temp_Ovrd="'$SSD_Cur_Pwr_Max_Temp_Ovrd'"	# SSD Max Drive Temp Override. This value when "true" will not alarm on any Current Power Cycle Max Temperature Limit.'
		echo 'NVM_Cur_Pwr_Max_Temp_Ovrd="'$NVM_Cur_Pwr_Max_Temp_Ovrd'"	# NVM Max Drive Temp Override. This value when "true" will not alarm on any Current Power Cycle Max Temperature Limit.'
		echo " "
		echo "###### SSD/NVMe Specific Settings"
		echo "WearLevelCrit=$WearLevelCrit				# Wear Level Alarm Setpoint for WARNING message, 9% is the default."
		echo 'NVM_Low_Power="'$NVM_Low_Power'"			# Set the NVMe power level to the minimum setting. This does not mean the NVMe will remain at this power level.'
		echo 'NVM_Daily_Short_Selftest="'$NVM_Daily_Short_Selftest'"		# This will run a short test on all NVMe drives everyday of the week, IF smartmontools is not 7.4 or greater.'
		echo 'NVM_Weekly_Long_Selftest="'$NVM_Weekly_Long_Selftest'"		# This will run a long test on all NVMe drives on Sunday and ignore the short test run if set, IF smartmontools is not 7.4 or greater.'
		echo 'NVM_Weekly_Long_Selftest_Day="'$NVM_Weekly_Long_Selftest_Day'"	# Value must be "Sun, Mon, Tue, Wed, Thu, Fri, Sat".  Default is Sun.'
		echo 'NVM_Smartmontools_74_Override="'$NVM_Smartmontools_74_Override'"	# Set to "enable" to override the check for smartmontools 7.4 and force the script to run the SMART tests. Default "disable".'
		echo 'Wait_For_SMART="'$Wait_For_SMART'"			# "true" will wait for the SMART Self-test timer to complete, the goal being the test completes before the report is generated. Default = "true"'
		echo 'Wait_For_SMART_Short_Duration='$Wait_For_SMART_Short_Duration'	# Set how long to wait for an NVMe SMART Short Test to pause the script, waiting on the test to complete. Default = 2 minutes 10 seconds (130 seconds)'
		echo 'Wait_For_SMART_Long_Duration='$Wait_For_SMART_Long_Duration'	# Set how long to wait for an NVMe SMART Long Test to pause the script, waiting on the test to complete. Default = 20 minutes 10 seconds (1200 seconds)'
		echo 'NVMe_Ignore_Invalid_Errors='$NVMe_Ignore_Invalid_Errors'	# Set to "enable" to ignore "Invalid Field in Command" messages.  Google this message to see if you are comfortable ignoring it.'
		echo " "
		echo "###### General Settings"
		echo "### Output Formats"
		echo 'PowerTimeFormat="'$PowerTimeFormat'"		# Format for power-on hours string, valid options are "ymdh", "ymd", "ym", "y", or "h" (year month day hour).'
		echo 'TempDisplay="'$TempDisplay'"		# The format you desire the temperature to be displayed in. Common formats are: "*C", "^C", or "^c". Choose your own.'
		echo 'Non_Exist_Value="'$Non_Exist_Value'"		# How do you desire non-existent data to be displayed.  The Default is "---", popular options are "N/A" or " ".'
		echo 'Pool_Capacity_Type="'$Pool_Capacity_Type'"	# Select "zfs" or "zpool" for Zpool Status Report - Pool Size and Free Space capacities. "zfs" is default.'
		echo " "
		echo "###### Ignore Alarms"
		echo 'IgnoreUDMA="'$IgnoreUDMA'"        # Set to "true" to ignore all UltraDMA CRC Errors for the summary alarm (Email Header) only, errors will still appear in the graphical chart. Default is false.'
		echo 'IgnoreSeekError="'$IgnoreSeekError'"    # Set to "true" to ignore all Seek Error Rate/Health errors.  Default is true.'
		echo 'IgnoreReadError="'$IgnoreReadError'"    # Set to "true" to ignore all Raw Read Error Rate/Health errors.  Default is true.'
		echo 'IgnoreMultiZone="'$IgnoreMultiZone'"   # Set to "true" to ignore all MultiZone Errors. Default is false.'
		echo 'DisableWarranty="'$DisableWarranty'"    # Set to "true to disable Email Subject line alerts for any expired warranty alert. The email body will still report the alert. Default is true.'
		echo " "
		echo "###### Disable or Activate Input/Output File Settings"
		echo 'ReportNonSMART="'$ReportNonSMART'"	# Will force even non-SMART devices to be reported, "true" = normal operation to report non-SMART devices.'
		echo 'DisableRAWdata="'$DisableRAWdata'"	# Set to "true" to remove the 'smartctl -a' data and non-smart data appended to the normal report.  Default is false.'
		echo 'ATA_Auto_Enable="'$ATA_Auto_Enable'"	# Set to "true" to automatically update Log Error count to only display a log error when a new one occurs.'
		echo " "
		echo "###### Media Alarms"
		echo "SectorsWarn=$SectorsWarn		# Number of sectors per drive to allow with errors before WARNING color/message will be used, this value should be less than SectorsCrit."
		echo "SectorsCrit=$SectorsCrit		# Number of sectors per drive with errors before CRITICAL color/message will be used."
		echo "ReAllocWarn=$ReAllocWarn		# Number of Reallocated sector events allowed.  Over this amount is an alarm condition."
		echo "MultiZoneWarn=$MultiZoneWarn		# Number of MultiZone Errors to allow before a Warning color/message will be used.  Default is 0."
		echo "MultiZoneCrit=$MultiZoneCrit		# Number of MultiZone Errors to allow before a Warning color/message will be used.  Default is 5."
		echo 'DeviceRedFlag="'$DeviceRedFlag'"	# Set to "true" to have the Device Column indicate RED for ANY alarm condition.  Default is true.'
		echo 'HeliumAlarm="'$HeliumAlarm'"	# Set to "true" to set for a critical alarm any He value below "HeliumMin" value.  Default is true.'
		echo "HeliumMin=$HeliumMin		# Set to 100 for a zero leak helium result.  An alert will occur below this value."
		echo "RawReadWarn=$RawReadWarn		# Number of read errors to allow before WARNING color/message will be used, this value should be less than RawReadCrit."
		echo "RawReadCrit=$RawReadCrit		# Number of read errors to allow before CRITICAL color/message will be used."
		echo "SeekErrorsWarn=$SeekErrorsWarn	# Number of seek errors to allow before WARNING color/message will be used, this value should be less than SeekErrorsCrit."
		echo "SeekErrorsCrit=$SeekErrorsCrit	# Number of seek errors to allow before CRITICAL color/message will be used."
		echo "NVM_Media_Errors=$NVM_Media_Errors	# Number of NVM Media Errors will cause a CRITICAL alarm.  Default is 1."
		echo " "
		echo "###### Time-Limited Error Recovery (TLER)"
		echo 'SCT_Enable="'$SCT_Enable'"		# Set to "true" to send a command to enable SCT on your drives for user defined timeout.'
		echo 'SCT_Warning_Level="'$SCT_Warning_Level'"	# Set to "all" will generate a Warning Message for all devices not reporting SCT enabled. "TLER" reports only drive which support TLER.'
		echo '				# "TLER_No_Msg" will only report for TLER drives and not report a Warning Message if the drive can set TLER on.'
		echo '###### Read/Write Timeout'
		echo "SCT_Read_Timeout=$SCT_Read_Timeout	# Set to the read threshold. Default = 70 = 7.0 seconds."
		echo "SCT_Write_Timeout=$SCT_Write_Timeout	# Set to the write threshold. Default = 70 = 7.0 seconds."
		echo " "
		echo "###### SMART Testing Alarm"
		echo "TestWarnAge=$TestWarnAge		# Maximum age (in days) of last SMART test before CRITICAL color/message will be used."
		echo " "
		echo "###### Zpool Fragmentation Alarm"
		echo "ZpoolFragWarn=$ZpoolFragWarn	# Percent of fragmentation before a Warning message occurs."
		echo " "
		echo '######## Enable-Disable Text Portion ########'
		echo 'Enable_Text_Section="'$Enable_Text_Section'"	# This will display the Text Section  below the CHART when = "true".  Default="true"'
		echo " "
		echo "###### Statistical Data File (SDF)"
		echo 'statistical_data_file="'$statistical_data_file'"	# Default location is where the script is located.'
		echo 'SDF_DataRecordEnable="'$SDF_DataRecordEnable'"	# Set to "true" will save all drive data into a CSV file defined by "statistical_data_file" below.'
		echo "SDF_DataPurgeDays=$SDF_DataPurgeDays		# Set to the number of day you wish to keep in the data.  Older data will be purged. Default is 730 days (2 years). 0=Disable."
		echo 'SDF_DataEmail="'$SDF_DataEmail'"		# Set to "true" to have an attachment of the file emailed to you. Default is true.'
		echo 'SDF_DataEmailDay="'$SDF_DataEmailDay'"		# Set to the day of the week the statistical report is emailed.  (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month)'
		echo " "
		echo "###### TrueNAS config backup settings"
		echo 'TrueNASConfigEmailEnable="'$TrueNASConfigEmailEnable'"		# Set to "true" to save config backup (which renders next two options operational); "false" to keep disable config backups.'
		echo 'TrueNASConfigEmailDay="'$TrueNASConfigEmailDay'"		# Set to the day of the week the config is emailed.  (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month)'
		echo 'TrueNASConfigBackupSave="'$TrueNASConfigBackupSave'"		# Set to "false" to delete TrueNAS config backup after mail is sent; "true" to keep it in dir below.'
		echo 'TrueNASConfigBackupLocation="'$TrueNASConfigBackupLocation'"	# Directory in which to store the backup FreeNAS config files.'
		echo " "
		echo "###### Attach multi_report_config.txt to Email ######"
		echo 'MRConfigEmailEnable="'$MRConfigEmailEnable'"	# Set to "true" to enable periodic email (which renders next two options operational).'
		echo 'MRChangedEmailSend="'$MRChangedEmailSend'"	# If "true" it will attach the updated/changed file to the email.'
		echo 'MRConfigEmailDay="'$MRConfigEmailDay'"		# Set to the day of the week the multi_report_config.txt is emailed.  (All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month, Never)'
		echo " "
		echo "###### REPORT CHART CONFIGURATION"
		echo " "
		echo "###### CUSTOM SUBJECT LINE"
		echo "### The host name will precede the subject line message."
		echo " "
		echo 'Subject_Line_Critical="'$Subject_Line_Critical'"'
		echo 'Subject_Line_Warning="'$Subject_Line_Warning'"'
		echo 'Subject_Line_Normal="'$Subject_Line_Normal'"'
		echo " "
		echo "###### REPORT HEADER TITLE"
		echo 'HDDreportTitle="'$HDDreportTitle'"	# This is the title of the HDD report, change as you desire.'
		echo 'SSDreportTitle="'$SSDreportTitle'"		# This is the title of the SSD report, change as you desire.'
		echo 'NVMreportTitle="'$NVMreportTitle'"		# This is the title of the NVMe report, change as you desire.'
		echo " "
		echo "###### CUSTOM REPORT CONFIGURATION"
		echo "### By default most items are selected. Change the item to "false" to have it not displayed in the graph, "true" to have it displayed."
		echo "### NOTE: Alarm setpoints are not affected by these settings, this is only what columns of data are to be displayed on the graph."
		echo "### I would recommend that you remove columns of data that you don't really care about to make the graph less busy."
		echo " "
		echo "# For Zpool Status Summary"
		echo 'Zpool_Pool_Name_Title="'$Zpool_Pool_Name_Title'"'
		echo 'Zpool_Status_Title="'$Zpool_Status_Title'"'
		echo 'Zpool_Pool_Size_Title="'$Zpool_Pool_Size_Title'"'
		echo 'Zpool_Free_Space_Title="'$Zpool_Free_Space_Title'"'
		echo 'Zpool_Used_Space_Title="'$Zpool_Used_Space_Title'"'
		echo 'Zfs_Pool_Size_Title="'$Zfs_Pool_Size_Title'"'
		echo 'Zfs_Free_Space_Title="'$Zfs_Free_Space_Title'"'
		echo 'Zfs_Used_Space_Title="'$Zfs_Used_Space_Title'"'
		echo 'Zpool_Frag_Title="'$Zpool_Frag_Title'"'
		echo 'Zpool_Read_Errors_Title="'$Zpool_Read_Errors_Title'"'
		echo 'Zpool_Write_Errors_Title="'$Zpool_Write_Errors_Title'"'
		echo 'Zpool_Checksum_Errors_Title="'$Zpool_Checksum_Errors_Title'"'
		echo 'Zpool_Scrub_Repaired_Title="'$Zpool_Scrub_Repaired_Title'"'
		echo 'Zpool_Scrub_Errors_Title="'$Zpool_Scrub_Errors_Title'"'
		echo 'Zpool_Scrub_Age_Title="'$Zpool_Scrub_Age_Title'"'
		echo 'Zpool_Scrub_Duration_Title="'$Zpool_Scrub_Duration_Title'"'
		echo " "
		echo "# For Hard Drive Section"
		echo 'HDD_Device_ID="'$HDD_Device_ID'"'
		echo 'HDD_Device_ID_Title="'$HDD_Device_ID_Title'"'
		echo 'HDD_Serial_Number="'$HDD_Serial_Number'"'
		echo 'HDD_Serial_Number_Title="'$HDD_Serial_Number_Title'"'
		echo 'HDD_Model_Number="'$HDD_Model_Number'"'
		echo 'HDD_Model_Number_Title="'$HDD_Model_Number_Title'"'
		echo 'HDD_Capacity="'$HDD_Capacity'"'
		echo 'HDD_Capacity_Title="'$HDD_Capacity_Title'"'
		echo 'HDD_Rotational_Rate="'$HDD_Rotational_Rate'"'
		echo 'HDD_Rotational_Rate_Title="'$HDD_Rotational_Rate_Title'"'
		echo 'HDD_SMART_Status="'$HDD_SMART_Status'"'
		echo 'HDD_SMART_Status_Title="'$HDD_SMART_Status_Title'"'
		echo 'HDD_Warranty="'$HDD_Warranty'"'
		echo 'HDD_Warranty_Title="'$HDD_Warranty_Title'"'
		echo 'HDD_Raw_Read_Error_Rate="'$HDD_Raw_Read_Error_Rate'"'
		echo 'HDD_Raw_Read_Error_Rate_Title="'$HDD_Raw_Read_Error_Rate_Title'"'
		echo 'HDD_Drive_Temp="'$HDD_Drive_Temp'"'
		echo 'HDD_Drive_Temp_Title="'$HDD_Drive_Temp_Title'"'
		echo 'HDD_Drive_Temp_Min="'$HDD_Drive_Temp_Min'"'
		echo 'HDD_Drive_Temp_Min_Title="'$HDD_Drive_Temp_Min_Title'"'
		echo 'HDD_Drive_Temp_Max="'$HDD_Drive_Temp_Max'"'
		echo 'HDD_Drive_Temp_Max_Title="'$HDD_Drive_Temp_Max_Title'"'
		echo 'HDD_Power_On_Hours="'$HDD_Power_On_Hours'"'
		echo 'HDD_Power_On_Hours_Title="'$HDD_Power_On_Hours_Title'"'
		echo 'HDD_Start_Stop_Count="'$HDD_Start_Stop_Count'"'
		echo 'HDD_Start_Stop_Count_Title="'$HDD_Start_Stop_Count_Title'"'
		echo 'HDD_Load_Cycle="'$HDD_Load_Cycle'"'
		echo 'HDD_Load_Cycle_Title="'$HDD_Load_Cycle_Title'"'
		echo 'HDD_Spin_Retry="'$HDD_Spin_Retry'"'
		echo 'HDD_Spin_Retry_Title="'$HDD_Spin_Retry_Title'"'
		echo 'HDD_Reallocated_Sectors="'$HDD_Reallocated_Sectors'"'
		echo 'HDD_Reallocated_Sectors_Title="'$HDD_Reallocated_Sectors_Title'"'
		echo 'HDD_Reallocated_Events="'$HDD_Reallocated_Events'"'
		echo 'HDD_Reallocated_Events_Title="'$HDD_Reallocated_Events_Title'"'
		echo 'HDD_Pending_Sectors="'$HDD_Pending_Sectors'"'
		echo 'HDD_Pending_Sectors_Title="'$HDD_Pending_Sectors_Title'"'
		echo 'HDD_Offline_Uncorrectable="'$HDD_Offline_Uncorrectable'"'
		echo 'HDD_Offline_Uncorrectable_Title="'$HDD_Offline_Uncorrectable_Title'"'
		echo 'HDD_UDMA_CRC_Errors_List="'$HDD_UDMA_CRC_Errors_List'"'
		echo 'HDD_UDMA_CRC_Errors_List_Title="'$HDD_UDMA_CRC_Errors_List_Title'"'
		echo 'HDD_Seek_Error_Rate="'$HDD_Seek_Error_Rate'"'
		echo 'HDD_Seek_Error_Rate_Title="'$HDD_Seek_Error_Rate_Title'"'
		echo 'HDD_MultiZone_Errors="'$HDD_MultiZone_Errors'"'
		echo 'HDD_MultiZone_Errors_Title="'$HDD_MultiZone_Errors_Title'"'
		echo 'HDD_Helium_Level="'$HDD_Helium_Level'"'
		echo 'HDD_Helium_Level_Title="'$HDD_Helium_Level_Title'"'
		echo 'HDD_Last_Test_Age="'$HDD_Last_Test_Age'"'
		echo 'HDD_Last_Test_Age_Title="'$HDD_Last_Test_Age_Title'"'
		echo 'HDD_Last_Test_Type="'$HDD_Last_Test_Type'"'
		echo 'HDD_Last_Test_Type_Title="'$HDD_Last_Test_Type_Title'"'
		echo 'HDD_Total_Data_Written="'$HDD_Total_Data_Written'"'
		echo 'HDD_Total_Data_Written_Title="'$HDD_Total_Data_Written_Title'"'
		echo " "
		echo "# For Solid State Drive Section"
		echo 'SSD_Device_ID="'$SSD_Device_ID'"'
		echo 'SSD_Device_ID_Title="'$SSD_Device_ID_Title'"'
		echo 'SSD_Serial_Number="'$SSD_Serial_Number'"'
		echo 'SSD_Serial_Number_Title="'$SSD_Serial_Number_Title'"'
		echo 'SSD_Model_Number="'$SSD_Model_Number'"'
		echo 'SSD_Model_Number_Title="'$SSD_Model_Number_Title'"'
		echo 'SSD_Capacity="'$SSD_Capacity'"'
		echo 'SSD_Capacity_Title="'$SSD_Capacity_Title'"'
		echo 'SSD_SMART_Status="'$SSD_SMART_Status'"'
		echo 'SSD_SMART_Status_Title="'$SSD_SMART_Status_Title'"'
		echo 'SSD_Warranty="'$SSD_Warranty'"'
		echo 'SSD_Warranty_Title="'$SSD_Warranty_Title'"'
		echo 'SSD_Drive_Temp="'$SSD_Drive_Temp'"'
		echo 'SSD_Drive_Temp_Title="'$SSD_Drive_Temp_Title'"'
		echo 'SSD_Drive_Temp_Min="'$SSD_Drive_Temp_Min'"'
		echo 'SSD_Drive_Temp_Min_Title="'$SSD_Drive_Temp_Min_Title'"'
		echo 'SSD_Drive_Temp_Max="'$SSD_Drive_Temp_Max'"'
		echo 'SSD_Drive_Temp_Max_Title="'$SSD_Drive_Temp_Max_Title'"'
		echo 'SSD_Power_On_Hours="'$SSD_Power_On_Hours'"'
		echo 'SSD_Power_On_Hours_Title="'$SSD_Power_On_Hours_Title'"'
		echo 'SSD_Wear_Level="'$SSD_Wear_Level'"'
		echo 'SSD_Wear_Level_Title="'$SSD_Wear_Level_Title'"'
		echo 'SSD_Reallocated_Sectors="'$SSD_Reallocated_Sectors'"'
		echo 'SSD_Reallocated_Sectors_Title="'$SSD_Reallocated_Sectors_Title'"'
		echo 'SSD_Reallocated_Events="'$SSD_Reallocated_Events'"'
		echo 'SSD_Reallocated_Events_Title="'$SSD_Reallocated_Events_Title'"'
		echo 'SSD_Pending_Sectors="'$SSD_Pending_Sectors'"'
		echo 'SSD_Pending_Sectors_Title="'$SSD_Pending_Sectors_Title'"'
		echo 'SSD_Offline_Uncorrectable="'$SSD_Offline_Uncorrectable'"'
		echo 'SSD_Offline_Uncorrectable_Title="'$SSD_Offline_Uncorrectable_Title'"'
		echo 'SSD_UDMA_CRC_Errors_List="'$SSD_UDMA_CRC_Errors_List'"'
		echo 'SSD_UDMA_CRC_Errors_List_Title="'$SSD_UDMA_CRC_Errors_List_Title'"'
		echo 'SSD_Last_Test_Age="'$SSD_Last_Test_Age'"'
		echo 'SSD_Last_Test_Age_Title="'$SSD_Last_Test_Age_Title'"'
		echo 'SSD_Last_Test_Type="'$SSD_Last_Test_Type'"'
		echo 'SSD_Last_Test_Type_Title="'$SSD_Last_Test_Type_Title'"'
		echo 'SSD_Total_Data_Written="'$SSD_Total_Data_Written'"'
		echo 'SSD_Total_Data_Written_Title="'$SSD_Total_Data_Written_Title'"'
		echo " "
		echo "# For NVMe Drive Section"
		echo 'NVM_Device_ID="'$NVM_Device_ID'"'
		echo 'NVM_Device_ID_Title="'$NVM_Device_ID_Title'"'
		echo 'NVM_Serial_Number="'$NVM_Serial_Number'"'
		echo 'NVM_Serial_Number_Title="'$NVM_Serial_Number_Title'"'
		echo 'NVM_Model_Number="'$NVM_Model_Number'"'
		echo 'NVM_Model_Number_Title="'$NVM_Model_Number_Title'"'
		echo 'NVM_Capacity="'$NVM_Capacity'"'
		echo 'NVM_Capacity_Title="'$NVM_Capacity_Title'"'
		echo 'NVM_SMART_Status="'$NVM_SMART_Status'"'
		echo 'NVM_SMART_Status_Title="'$NVM_SMART_Status_Title'"'
		echo 'NVM_Warranty="'$NVM_Warranty'"'
		echo 'NVM_Warranty_Title="'$NVM_Warranty_Title'"'
		echo 'NVM_Critical_Warning="'$NVM_Critical_Warning'"'
		echo 'NVM_Critical_Warning_Title="'$NVM_Critical_Warning_Title'"'
		echo 'NVM_Drive_Temp="'$NVM_Drive_Temp'"'
		echo 'NVM_Drive_Temp_Title="'$NVM_Drive_Temp_Title'"'
		echo 'NVM_Drive_Temp_Min="'$NVM_Drive_Temp_Min'"               # NVMe does not support yet'
		echo 'NVM_Drive_Temp_Min_Title="'$NVM_Drive_Temp_Min_Title'"'
		echo 'NVM_Drive_Temp_Max="'$NVM_Drive_Temp_Max'"               # NVMe does not support yet'
		echo 'NVM_Drive_Temp_Max_Title="'$NVM_Drive_Temp_Max_Title'"'
		echo 'NVM_Power_Level="'$NVM_Power_Level'"'
		echo 'NVM_Power_Level_Title="'$NVM_Power_Level_Title'"'
		echo 'NVM_Power_On_Hours="'$NVM_Power_On_Hours'"'
		echo 'NVM_Power_On_Hours_Title="'$NVM_Power_On_Hours_Title'"'
		echo 'NVM_Wear_Level="'$NVM_Wear_Level'"'
		echo 'NVM_Wear_Level_Title="'$NVM_Wear_Level_Title'"'
		echo 'NVM_Media_Error="'$NVM_Media_Error'"'
		echo 'NVM_Media_Error_Title="'$NVM_Media_Error_Title'"'
		echo 'NVM_Last_Test_Age="'$NVM_Last_Test_Age'"'
		echo 'NVM_Last_Test_Age_Title="'$NVM_Last_Test_Age_Title'"'
		echo 'NVM_Last_Test_Type="'$NVM_Last_Test_Type'"'
		echo 'NVM_Last_Test_Type_Title="'$NVM_Last_Test_Type_Title'"'
		echo 'NVM_Total_Data_Written="'$NVM_Total_Data_Written'"'
		echo 'NVM_Total_Data_Written_Title="'$NVM_Total_Data_Written_Title'"'
		echo " "
		echo "###### Drive Ignore List"
		echo "# What does it do:"
		echo '#  Use this to list any drives to ignore and remove from the report.  This is'
		echo '#  very useful for ignoring USB Flash Drives or other drives for which good'
		echo '#  data is not able to be collected (non-standard).'
		echo "#"
		echo '# How to use it:'
		echo '#  We are using a comma delimited file to identify the drive serial numbers.'
		echo '#  You MUST use the exact and full serial number smartctl reports, if there'
		echo '#  is no identical match then it will not ignore the drive.'
		echo "#"
		echo '# Format: Ignore_Drives_List="serial_number,serial_number,serial_number"'
		echo '# Example: Ignore_Drives_List="VMWare,1JUMLBD,21HNSAFC21410E"'
		echo " "
		echo 'Ignore_Drives_List="'$Ignore_Drives_List'"'
		echo " "
		echo '###### Drive ATA Errors, UDMA CRC Error Count List, MultiZone List Errors List,'
		echo '###### Reallocated Sectors Exceptions, Reallocated Sectors Events Exceptions'
		echo '#'
		echo '# What does it do:'
		echo '#  If you have a drive which has one of the above errors not a 0 (zero) value,'
		echo '#  this setting will offset the value back to zero for the considerations of'
		echo '#  monitoring future increases of this specific error.  This will subtract'
		echo '#  the value by a correction value in order to 0 (zero) the value and'
		echo '#  highlight it in yellow to denote it was overridden.  The Warning Title'
		echo "#  will not be flagged if this is zero'd out in this manner."
		echo '#'
		echo '# How to use it:'
		echo '#  List each drive by serial number and include the current error count value.'
		echo '#  The format is very specific and will not work if you wing it, use the Example.'
		echo '#'
		echo '# Format: CRC_Errors_List="serial_number:current_udma_error_count,serial_number:current_udma_error_count"'
		echo '# Example: CRC_Errors_List="WD-WMC4N2578099:1,S2X1J90CA48799:2,P02618119268:1"'
		echo " "
		echo 'ATA_Errors_List="'$ATA_Errors_List'"'
		echo 'CRC_Errors_List="'$CRC_Errors_List'"'
		echo 'MultiZone_List="'$MultiZone_List'"'
		echo 'ReAllocated_Sector_List="'$ReAllocated_Sector_List'"'
		echo 'ReAllocated_Sector_Events_List="'$ReAllocated_Sector_Events_List'"'
		echo " "
		echo '###### Custom Drive Configuration'
		echo '# Used to define specific alarm values for specific drives by serial number.'
		echo '# This should only be used for drives where the default alarm settings'
		echo '# are not proper or you need to reverse some values where they may be listed'
		echo '# opposite, for example WearLevel may be listed as 0% vice 100%.'
		echo '# Up to 24 unique drive values may be stored (tested).'
		echo '#'
		echo '# Use -config to set these values.'
		echo '#'
		echo '# THE BREAKDOWN OF THIS LINE FORMAT (entire list broken to two lines for viewing, separated using a comma)'
		echo '# serial:tempwarn:tempcrit:sectorswarn:sectorscrit:reallocwarn:multizonewarn:multizonecrit:rawreadwarn:'
		echo '# rawreadcrit:seekerrorswarn:seekerrorscrit:testage:testAgeOvrd:heliummin:wearleveladj'
		echo '# (testAgeOvrd '0'=Default, '1'=Ignore, wearleveladj 'd'=Default 'r'=Reverse value 'n'=Normalized 'i'=Ignore)'
		echo " "
		echo 'Custom_Drives_List="'$Custom_Drives_List'"'
		echo " "
		echo '####### Warranty Expiration Date'
		echo '# What does it do:'
		echo '#  This section is used to add warranty expiration dates for designated drives'
		echo '#  and to create an alert when they expire. This is good to give you a'
		echo '#  heads-up on when you might need to start looking for a replacement drive.'
		echo '#'
		echo '# How to use it:'
		echo '#   Format: Drive_Warranty_List="serial_number:YYYY-MM-DD,serial_number:YYYY-MM-DD"'
		echo '#   Example: Drive_Warranty_List="K1JUMLBD:2020-09-30,K1JRSWLD:2020-09-30,K1JUMW4D:2020-09-30,K1GVD84B:2020-10-12"'
		echo " "
		echo 'Drive_Warranty_List="'$Drive_Warranty_List'"'
		echo " "
		echo '######## Expired Drive Warranty Setup'
		echo 'expiredWarrantyBoxColor="'$expiredWarrantyBoxColor'"	# "black" = normal box perimeter color.'
		echo 'WarrantyBackgndColor="'$WarrantyBackgndColor'"		# Background color for expired drives. "none" = normal background.'
		echo " "
		echo "###### Global table of colors"
		echo "# The colors selected you can change but you will need to look up the proper HEX code for a color."
		echo " "
		# Reset the colors option in '-config'
		if [[ $Update_Colors != "1" ]]; then
			echo 'okColor="'$okColor'"	# Hex code for color to use in SMART Status column if drives pass (default is darker light green, #b5fcb9).'
			echo 'warnColor="'$warnColor'"	# Hex code for WARN color (default is orange, #F38B16).'
			echo 'critColor="'$critColor'"	# Hex code for CRITICAL color (default is red, #f44336).'
			echo 'altColor="'$altColor'"	# Table background alternates row colors between white and this color (default is light gray, #f4f4f4).'
			echo 'whtColor="'$whtColor'"	# Hex for White background.'
			echo 'ovrdColor="'$ovrdColor'"	# Hex code for Override Yellow.'
			echo 'blueColor="'$blueColor'"	# Hex code for Sky Blue, used for the SCRUB/SMART Test In Progress/background.'
			echo 'yellowColor="'$yellowColor'"	# Hex code for pale yellow.'
		else
			echo 'okColor="#b5fcb9"	# Hex code for color to use in SMART Status column if drives pass (default is darker light green, #b5fcb9).'
			echo 'warnColor="#F38B16"	# Hex code for WARN color (default is orange, #F38B16).'
			echo 'critColor="#f44336"	# Hex code for CRITICAL color (default is red, #f44336).'
			echo 'altColor="#f4f4f4"	# Table background alternates row colors between white and this color (default is light gray, #f4f4f4).'
			echo 'whtColor="#ffffff"	# Hex for White background.'
			echo 'ovrdColor="#ffffe4"	# Hex code for Override Yellow.'
			echo 'blueColor="#87ceeb"	# Hex code for Sky Blue, used for the SCRUB In Progress background.'
			echo 'yellowColor="#f1ffad"	# Hex code for pale yellow.'
		fi
	) > "$Config_File_Name"
	if [[ $MRChangedEmailSend == "true" ]]; then MR_Attach_Config="1"; fi
	}

########## GENERATE CONFIG FILE ##########
#
# Update configuration file is retired now that the config file is automatically updated.

generate_config_file () {
	SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
	Config_File_Name="$SCRIPT_DIR/multi_report_config.txt"
	for (( z=1; z<=50; z++ )); do
		clear
		echo $programver
		echo " "
		echo "                Configuration File Management"
		echo " "
		if test -e "$Config_File_Name"; then echo " *** WARNING - A CONFIGURATION CURRENTLY FILE EXISTS ***";	fi
		echo " "
		echo "      N)ew configuration file (creates a new clean external configuration file)"
	#	echo "      U)pdate configuration file (updates select static variables to default)"
		echo "      A)dvanced configuration (must have a configuration file already present)"
		echo "      S)pencer Integration (configure Spencer add-on)"
		echo "      H)ow to use this configuration tool (general instructions)"
		echo "      X) Exit"
		echo " "
		echo "NOTE: In using this configuration script when the value is:"
		echo "  Number or Text: The current value will be displayed. You have the option to"
		echo "  just press Enter/Return to accept the current value or you may enter a"
		echo "  different value."
		echo " "
		echo "  True or False: The current value will be displayed. You have the option to"
		echo "  press Enter/Return to accept the current value or you may press 't' for true"
		echo "  or 'f' for false."
		echo " "
		echo " "
		echo -n "   Make your selection: "
		read -s -n 1 Keyboard_var
		shopt -s nocasematch
		case $Keyboard_var in
			# First Level Start
			A)
				clear
				echo " "
				echo "            Advanced Configuration Settings"
				echo " "
				echo " Loading Configuration File Data..."
				echo " "
			      if [[ ! -f "$Config_File_Name" ]]; then
					echo "You do not have an external configuration file yet."
					echo "Please create an external configuration file."
					echo " "
					exit 1
				fi
				load_config
				echo "This is not a complete configuration setup, it is just the most common settings"
				echo "that a user would typically require for a normal setup.  You may directly edit"
				echo "the config text file with any text editor to take full advantage of the options."
				echo " "
				echo "The config text file is located here: "$Config_File_Name
				echo " "
				for (( x=1; x<=50; x++ )); do
					clear
					echo "            Advanced Configuration Settings"
					echo " "
					echo "   A) Alarm Setpoints (Temp, Zpool, Media, Activate In/Out, Ignore, Email On Alert Only)" 
					echo "   B) Config-Backup (Edit Config-Backup & Multi-Report_Config Settings)"
					echo "   C) Email Address (Edit Email address and Encryption)" 
					echo "   D) HDD Column Selection (Select columns to display/hide)"
					echo "   E) SSD Column Selection (Select columns to display/hide)"
					echo "   F) NVMe Column Selection (Select columns to display/hide)"
					echo "   G) Output Formats (Hours, Temp, Non-Existent, Pool Capacity)"
					echo "   H) Report Header Titles (Edit Header Titles, Add/Remove Text Section)" 
					echo "   I) Statistical Data File Setup"
					echo "   J) TLER / SCT (Setup if TLER is active)"
					echo "   K) Drive Errors (Ignore Drives, UDMA CRC, MultiZone,"
					echo "            Reallocated Sectors, ATA Errors, Warranty Expiration)"
					echo "   L) Subject Line Custom Settings"
					echo "   M) Multipath Setting"
					echo "   N) NVMe Custom Settings (Set Low Power State, SMART Test Options)"
					echo "   S) Custom Drive Configuration"
					echo "   U) Update Script - Automatic or Manual Internet (Github) Updates"
					echo "   W) Write Configuration File (Save your changes)"
					echo "   X) Exit - Will not automatically save changes"
					echo " "
					echo -n "   Make your selection: "
					read -s -n 1 Keyboard_var2
					echo " "
					shopt -s nocasematch
					case $Keyboard_var2 in
						# Advanced Configuration Level Start
						A)
							for (( y=1; y<=50; y++ )); do
							clear
							echo "            Alarm Configuration Settings"
							echo " "
							echo "These setting affect all drives unless they are overridden"
							echo "      using the Custom Drive Configuration option."
							echo " "
							echo "   A) Temperature Settings (Various Temperature Settings)" 
							echo "   B) Zpool Settings (Scrub Age, Pool % Avail, and Frag % Alarms)"
							echo "   C) Media Alarm Settings (Sectors and CRC Type Alarms)"
							echo "   D) Activate Input/Output Settings (Enable SSD/NVMe/Non-SMART)" 
							echo "   E) Ignore Alarms (Ignore CRC/MultiZone/Seek Type Errors/Email on Alert Only)"
							echo "   F) Monitor Email Settings"
							echo "   X) Exit - Return to previous menu"
							echo " "
							echo -n "   Make your selection: "
							read -s -n 1 Keyboard_var3
							echo " "
							shopt -s nocasematch
							case $Keyboard_var3 in
								A)
									clear
									echo "Temperature Settings"
									echo " "
									echo "The current value will be in parentheses."
									echo "Enter a new value or press Return to keep current value."
									echo " "
									echo "HDD Warning Temperature ("$HDDtempWarn") "
									echo -n "(1-256) "
									read Keyboard_yn
									if [[ $Keyboard_yn -gt 85 ]]; then echo "Ouch! that is HOT!, but you are the boss."; fi
									if [[ ! $Keyboard_yn == "" ]]; then HDDtempWarn=$Keyboard_yn; fi
									echo "Set Value: ("$HDDtempWarn")"
									echo " "
									echo "HDD Critical Temperature ("$HDDtempCrit") "
									echo -n "(1-256) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then HDDtempCrit=$Keyboard_yn; fi
									echo "Set Value: ("$HDDtempCrit")"
									echo " "
									echo "HDD Max Temperature Override for Power Cycle Enabled ("$HDD_Cur_Pwr_Max_Temp_Ovrd") "
									echo 'When "true" will not alarm on any Current Power Cycle Max Temperature Limit.'
									echo -n "(t = true, f = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then HDD_Cur_Pwr_Max_Temp_Ovrd="true"; else HDD_Cur_Pwr_Max_Temp_Ovrd="false"; fi
									fi
									echo "Set Value: ("$HDD_Cur_Pwr_Max_Temp_Ovrd")"
									echo " "
									echo "SSD Warning Temperature ("$SSDtempWarn") "
									echo -n "(1-256) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then SSDtempWarn=$Keyboard_yn; fi
									echo "Set Value: ("$SSDtempWarn")"
									echo " "
									echo "SSD Critical Temperature ("$SSDtempCrit") "
									echo -n "(1-256) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then SSDtempCrit=$Keyboard_yn; fi
									echo "Set Value: ("$SSDtempCrit")"
									echo " "
									echo "SSD Max Temperature Override for Power Cycle Enabled ("$SSD_Cur_Pwr_Max_Temp_Ovrd") "
									echo 'When "true" will not alarm on any Current Power Cycle Max Temperature Limit.'
									echo -n "(t = true, f = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then SSD_Cur_Pwr_Max_Temp_Ovrd="true"; else SSD_Cur_Pwr_Max_Temp_Ovrd="false"; fi
									fi
									echo "Set Value: ("$SSD_Cur_Pwr_Max_Temp_Ovrd")"
									echo " "
									echo "NVMe Warning Temperature ("$NVMtempWarn") "
									echo -n "(1-256) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then NVMtempWarn=$Keyboard_yn; fi
									echo "Set Value: ("$NVMtempWarn")"
									echo " "
									echo "NVMe Critical Temperature ("$NVMtempCrit") "
									echo -n "(1-256) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then NVMtempCrit=$Keyboard_yn; fi
									echo "Set Value: ("$NVMtempCrit")"
									echo " "
									echo "returning..."
									sleep 2
								;;

								B)
									clear
									echo "Zpool Settings"
									echo " "
									echo "Maximum age (in days) since last pool scrub before CRITICAL color will be used."
									echo "Scrub maximum days since last completion ("$ScrubAgeWarn") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then ScrubAgeWarn=$Keyboard_yn; fi
									echo "Set Value: ("$ScrubAgeWarn")"
									echo " "
									echo "Pool used percentage for CRITICAL color to be used."
									echo "Pool Space Used Alert ("$PoolUsedWarn") "
									echo -n "(1-100) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then PoolUsedWarn=$Keyboard_yn; fi
									echo "Set Value: ("$PoolUsedWarn")"
									echo " "
									echo "Pool Fragmentation percentage for WARNING color to be used."
									echo "Pool Frag Alert ("$ZpoolFragWarn") "
									echo -n "(1-100) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then ZpoolFragWarn=$Keyboard_yn; fi
									echo "Set Value: ("$ZpoolFragWarn")"
									echo "returning..."
									sleep 2
								;;

								C)
									clear
									echo "Media Alarm Settings"
									echo " "
									echo "SSD/NVMe Wear Level lower limit ("$WearLevelCrit") "
									echo -n "(1-100) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then WearLevelCrit=$Keyboard_yn; fi
									echo "Set Value: ("$WearLevelCrit")"
									echo " "
									echo "Sector Errors Warning ("$SectorsWarn") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then SectorsWarn=$Keyboard_yn; fi
									echo "Set Value: ("$SectorsWarn")"
									echo " "
									echo "Sector Errors Critical ("$SectorsCrit") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then SectorsCrit=$Keyboard_yn; fi
									echo "Set Value: ("$SectorsCrit")"
									echo " "
									echo "Reallocated Sectors Warning ("$ReAllocWarn") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then ReAllocWarn=$Keyboard_yn; fi
									echo "Set Value: ("$ReAllocWarn")"
									echo " "
									echo "Raw Read Errors Warning ("$RawReadWarn") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then RawReadWarn=$Keyboard_yn; fi
									echo "Set Value: ("$RawReadWarn")"
									echo " "
									echo "Raw Read Errors Critical ("$RawReadCrit") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then RawReadCrit=$Keyboard_yn; fi
									echo "Set Value: ("$RawReadCrit")"
									echo " "
									echo "Seek Errors Warning ("$SeekErrorsWarn") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then SeekErrorsWarn=$Keyboard_yn; fi
									echo "Set Value: ("$SeekErrorsWarn")"
									echo " "
									echo "Seek Errors Critical ("$SeekErrorsCrit") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then SeekErrorsCrit=$Keyboard_yn; fi
									echo "Set Value: ("$SeekErrorsCrit")"
									echo " "
									echo "MultiZone Errors Warning ("$MultiZoneWarn") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then MultiZoneWarn=$Keyboard_yn; fi
									echo "Set Value: ("$MultiZoneWarn")"
									echo " "
									echo "MultiZone Errors Critical ("$MultiZoneCrit") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then MultiZoneCrit=$Keyboard_yn; fi
									echo "Set Value: ("$MultiZoneCrit")"
									echo " "
									echo "Helium Minimum Level ("$HeliumMin") "
									echo -n "(1-100) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then HeliumMin=$Keyboard_yn; fi
									echo "Set Value: ("$HeliumMin")"
									echo " "
									echo "Helium Critical Alert Message ("$HeliumAlarm") "
									echo 'A "true" value will generate an email subject line alert for a error.'
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then HeliumAlarm="true"; else HeliumAlarm="false"; fi
									fi
									echo "Set Value: ("$HeliumAlarm")"
									echo " "
									echo "S.M.A.R.T. Test Age Warning ("$TestWarnAge") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then TestWarnAge=$Keyboard_yn; fi
									echo "Set Value: ("$TestWarnAge")"
									echo " "
									echo "NVMe Media Errors ("$NVM_Media_Errors") "
									echo -n "(1-10000) "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then NVM_Media_Errors=$Keyboard_yn; fi
									echo "Set Value: ("$NVM_Media_Errors")"
									echo " "
									echo "Flag Device ID RED on Error ("$DeviceRedFlag") "
									echo -n "('t' = true. 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then DeviceRedFlag="true"; else DeviceRedFlag="false"; fi
									fi
									echo "Set Value: ("$DeviceRedFlag")"
									echo " "
									echo "returning..."
									sleep 2
								;;

								D)
									clear
									echo "Activate/Disable Input/Output Settings"
									echo " "
									echo "Force non-SMART devices to be reported"
									echo "Report Non-SMART Devices ("$ReportNonSMART") "
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then ReportNonSMART="true"; else ReportNonSMART="false"; fi
									fi
									echo "Set Value: ("$ReportNonSMART")"
									echo " "
									echo 'Set to "true" to remove the smartctl -a data and non-smart data appended to the normal report.'
									echo "Remove Non-SMART Data from the emailed report ("$DisableRAWdata") "
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then DisableRAWdata="true"; else DisableRAWdata="false"; fi
									fi
									echo "Set Value: ("$DisableRAWdata")"
									echo " "
									echo "returning..."
									sleep 2
								;;

								E)
									clear
									echo "Ignore Alarm Settings"
									echo " "
									echo "Email Only On Alarm ("$Email_On_Alarm_Only")"
									echo "This will Only send an email if an alarm is present."
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then Email_On_Alarm_Only="true"; else Email_On_Alarm_Only="false"; fi
									fi
									echo "Set Value: ("$Email_On_Alarm_Only")"

									echo " "
									echo "Email Only On Alarm And Attachments ("$Email_On_Alarm_Only_And_Attachments")"
									echo "This will Only send an email if an alarm is present OR Attachments"
									echo "are ready such as TrueNAS Config or Multi-Report Config."
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then Email_On_Alarm_Only_And_Attachments="true"; else Email_On_Alarm_Only_And_Attachments="false"; fi
									fi
									echo "Set Value: ("$Email_On_Alarm_Only_And_Attachments")"

									echo " "
									echo "Ignore UDMA CRC Errors ("$IgnoreUDMA") "
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then IgnoreUDMA="true"; else IgnoreUDMA="false"; fi
									fi
									echo "Set Value: ("$IgnoreUDMA")"
									echo " "
									echo "Ignore Raw Read Errors ("$IgnoreReadError") "
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then IgnoreReadError="true"; else IgnoreReadError="false"; fi
									fi
									echo "Set Value: ("$IgnoreReadError")"
									echo " "
									echo "Ignore Seek Errors ("$IgnoreSeekError") "
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then IgnoreSeekError="true"; else IgnoreSeekError="false"; fi
									fi
									echo "Set Value: ("$IgnoreSeekError")"
									echo " "
									echo "Ignore MultiZone Errors ("$IgnoreMultiZone") "
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then IgnoreMultiZone="true"; else IgnoreMultiZone="false"; fi
									fi
									echo "Set Value: ("$IgnoreMultiZone")"
									echo " "
									echo "Disable Warranty Email Header Warning ("$DisableWarranty") "
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then DisableWarranty="true"; else DisableWarranty="false"; fi
									fi
									echo "Set Value: ("$DisableWarranty")"
									echo " "
									echo "ATA Auto Enable ("$ATA_Auto_Enable") "
									echo 'Set to "true" to automatically update Log Error count to ONLY display a log'
									echo "error when a new one occurs."
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then ATA_Auto_Enable="true"; else ATA_Auto_Enable="false"; fi
									fi
									echo "Set Value: ("$ATA_Auto_Enable")"
									echo " "
									echo "returning..."
									sleep 2
								;;
								F)	clear
									echo "Monitor Email Settings"
									echo " "
									echo "AlertOnWarningTemp ("$AlertOnWarningTemp") "
									echo 'Set to "true" will send Temperature Warnings to the Monitor Email address'
									echo 'when using the "-m" switch.'
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then AlertOnWarningTemp="true"; else AlertOnWarningTemp="false"; fi
									fi
									echo "Set Value: ("$AlertOnWarningTemp")"
									echo " "
									echo "AlertOnCriticalError ("$AlertOnCriticalError") "
									echo 'Set to "true" will send Critical Alarms to the Monitor Email address'
									echo 'when using the "-m" switch.'
									echo -n "('t' = true, 'f' = false) "
									read -s -n 1 Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then
										if [[ $Keyboard_yn == "t" ]]; then AlertOnCriticalError="true"; else AlertOnCriticalError="false"; fi
									fi
									echo "Set Value: ("$AlertOnCriticalError")"
									echo " "
								;;

								X)
									clear
									echo "Returning to the previous menu..."
									sleep 2
									y=100
								;;

								*)
									echo "Invalid Option"
									sleep 2
								;;
							esac
						done
						;;

						B)
							clear
							echo "TrueNAS Configuration Backup Setup"
							echo " "
							echo "Configuration Backup Enabled ("$TrueNASConfigEmailEnable") "
							echo -n "('t' = true, 'f' = false) "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then TrueNASConfigEmailEnable="true"; else TrueNASConfigEmailEnable="false"; fi
							fi
							echo "Set Value: ("$TrueNASConfigEmailEnable")"
							echo " "
							if [[ $TrueNASConfigEmailEnable == "true" ]]; then
								echo "Save a local copy of the config-backup file ("$TrueNASConfigBackupSave") "
								echo -n "('t' = true, 'f' = false) "
								read -s -n 1 Keyboard_yn
								if [[ ! $Keyboard_yn == "" ]]; then
									if [[ $Keyboard_yn == "t" ]]; then TrueNASConfigBackupSave="true"; else TrueNASConfigBackupSave="false"; fi
								fi
								echo "Set Value: ("$TrueNASConfigBackupSave")"
								if [[ $TrueNASConfigBackupSave == "true" ]]; then
									echo " "
									echo "TrueNAS Backup Configuration file location ("$TrueNASConfigBackupLocation")"
									echo "Enter new location or press Enter/Return to accept current value"
									echo "NOTE: Do not use any spaces."
									echo -n ": "
									read Keyboard_yn
									if [[ ! $Keyboard_yn == "" ]]; then TrueNASConfigBackupLocation=$Keyboard_yn; fi
									echo "Set Value: ("$TrueNASConfigBackupLocation")"
								fi
								echo " "
								echo "What day of the week would you like the file attached?"
								echo "Current Value: "$TrueNASConfigEmailDay
								echo -n "(All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month) "
								read Keyboard_HDD
								if [[ ! $Keyboard_HDD == "" ]]; then TrueNASConfigEmailDay=$Keyboard_HDD; fi
								echo "Set Value: ("$TrueNASConfigEmailDay")"
							fi
							echo " "
							echo '"multi_report_config.txt" Backup Setup'
							echo " "
							echo "Enable sending multi_report_config.txt file"
							echo "(will enable next two options if true) ("$MRConfigEmailEnable") "
							echo -n "('t' = true, 'f' = false) "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then MRConfigEmailEnable="true"; else MRConfigEmailEnable="false"; fi
							fi
							echo "Set Value: ("$MRConfigEmailEnable")"
							if [[ $MRConfigEmailEnable == "true" ]]; then
								echo " "
								echo "What day of the week would you like the file attached?"
								echo "Current Value: "$MRConfigEmailDay
								echo -n "(All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month, Never) "
								read Keyboard_HDD
								if [[ ! $Keyboard_HDD == "" ]]; then MRConfigEmailDay=$Keyboard_HDD; fi
								echo "Set Value: ("$MRConfigEmailDay")"
								echo " "
								echo " "
								echo "Send email of multi_report_config.txt file for any change ("$MRChangedEmailSend") "
								echo -n "('t' = true, 'f' = false) "
								read -s -n 1 Keyboard_yn
								if [[ ! $Keyboard_yn == "" ]]; then
									if [[ $Keyboard_yn == "t" ]]; then MRChangedEmailSend="true"; else MRChangedEmailSend="false"; fi
								fi
								echo "Set Value: ("$MRChangedEmailSend")"
							fi
							echo " "
							echo "returning..."
							sleep 2
						;;

						C)
							clear
							echo "Email Settings"
							echo " "
							echo "Current Email address(s): "$Email" "
							echo " "
							echo "Separate multiple email addresses with a comma "
							echo 'Enter nothing to accept the default or change it to something new.'
							echo -n ": "
							read Keyboard_Email
							if [[ ! $Keyboard_Email == "" ]]; then Email=$Keyboard_Email; fi
							echo "Set Value: "$Email
							echo "------------------------------------------------------------"
							echo " "
							echo "Current Email address(s): "$AlertEmail" "
							echo " "
							echo "Enter your Drive temperature monitoring Email address (used with -m switch only)."
							echo "This is the email or series of emails (comma separated) address(s) to send an"
							echo "alert to. Blank = Use Normal Email Address."
							echo "Press Return to use your normal email address"
							echo -n ": "
							read Keyboard_Email
							if [[ $Keyboard_Email == "" ]]; then
								echo "Using normal Email address..."
								AlertEmail=$Email
							fi
							if [[ ! $Keyboard_Email == "" ]]; then AlertEmail=$Keyboard_Email; fi
							echo "Set Value: "$AlertEmail
							echo "-------------------------------------------------------------"
							echo " "
							echo "Current from address: "$From" "
							echo 'While most people are able to use the default "from" address,'
							echo 'Some email servers will not work unless you use the email address'
							echo 'the email address the server is associated with.'
							echo 'Enter nothing to accept the default or change it.'
							echo -n ": "
							read Keyboard_Email
							if [[ ! $Keyboard_Email == "" ]]; then From=$Keyboard_Email; fi
							echo "Set Value: "$From
							echo "--------------------------------------------------------------"
							echo " "
							echo "Current TrueNAS Configuration File passphrase: "$TrueNASConfigEmailEncryption
							echo " "
							echo 'You have the option to Encrypt your TrueNAS Configuration Backup File.'
							echo 'This is just for the sake of the email, the actual file is already encrypted.'
							echo 'However by request I have added it.'
							echo 'Use 7-zip, PKZIP, WinZip or similar to decrypt the file.'
							echo ' '
							echo 'Press Enter/Return to accept the current passphrase, or'
							echo 'enter "disable" to remove any passphrase, or enter a new passphrase.'
							echo -n ": "
							read Keyboard_Email
							if [[ ! $Keyboard_Email == "" ]]; then TrueNASConfigEmailEncryption=$Keyboard_Email; fi
							if [[ $Keyboard_Email == "disable" ]]; then TrueNASConfigEmailEncryption=""; fi
							echo "Set Value: "$TrueNASConfigEmailEncryption
							echo " "
							echo " "
							if [[ $Keyboard_Email == "" ]]; then
								echo "No change"
							else
								echo "Writing Passphrase into Script Now..."
								sed '5s/.*/TrueNASConfigEmailEncryption="'$TrueNASConfigEmailEncryption'"	 # Set this to "" for no encryption or enter some text as your passphrase./' "$mefull" > /tmp/multi_report_update.txt
								cp "/tmp/multi_report_update.txt" "$mefull"
								rm "/tmp/multi_report_update.txt"
							fi
							echo "You still need to write any other changes made such as an email address."
							echo "returning..."
							sleep 2
						;;

						D)
							clear
							echo "HDD Column Selection"
							echo " "
							echo "This does not disable/enable any alarm settings."
							echo " "
							echo -n "Device ID ("$HDD_Device_ID") "  
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Device_ID="true"; else HDD_Device_ID="false"; fi
							fi
							echo "Set Value: ("$HDD_Device_ID")"
							echo " "
							echo -n "Serial Number ("$HDD_Serial_Number") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Serial_Number="true"; else HDD_Serial_Number="false"; fi
							fi
							echo "Set Value: ("$HDD_Serial_Number")"
							echo " "
							echo -n "Model Number ("$HDD_Model_Number") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Model_Number="true"; else HDD_Model_Number="false"; fi
							fi
							echo "Set Value: ("$HDD_Model_Number")"
							echo " "
							echo -n "Capacity ("$HDD_Capacity") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Capacity="true"; else HDD_Capacity="false"; fi
							fi
							echo "Set Value: ("$HDD_Capacity")"
							echo " "
							echo -n "Rotational Rate ("$HDD_Rotational_Rate") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Rotational_Rate="true"; else HDD_Rotational_Rate="false"; fi
							fi
							echo "Set Value: ("$HDD_Rotational_Rate")"
							echo " "
							echo -n "SMART Status ("$HDD_SMART_Status") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_SMART_Status="true"; else HDD_SMART_Status="false"; fi
							fi
							echo "Set Value: ("$HDD_SMART_Status")"
							echo " "
							echo -n "Warranty ("$HDD_Warranty") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Warranty="true"; else HDD_Warranty="false"; fi
							fi
							echo "Set Value: ("$HDD_Warranty")"
							echo " "
							echo -n "Drive Temp ("$HDD_Drive_Temp") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Drive_Temp="true"; else HDD_Drive_Temp="false"; fi
							fi
							echo "Set Value: ("$HDD_Drive_Temp")"
							echo " "
							echo -n "Drive Temp Minimum for power cycle ("$HDD_Drive_Temp_Min") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Drive_Temp_Min="true"; else HDD_Drive_Temp_Min="false"; fi
							fi
							echo "Set Value: ("$HDD_Drive_Temp_Min")"
							echo " "
							echo -n "Drive Temp Maximum for power cycle ("$HDD_Drive_Temp_Max") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Drive_Temp_Max="true"; else HDD_Drive_Temp_Max="false"; fi
							fi
							echo "Set Value: ("$HDD_Drive_Temp_Max")"
							echo " "
							echo -n "Power On Hours ("$HDD_Power_On_Hours") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Power_On_Hours="true"; else HDD_Power_On_Hours="false"; fi
							fi
							echo "Set Value: ("$HDD_Power_On_Hours")"
							echo " "
							echo -n "Start / Stop Count ("$HDD_Start_Stop_Count") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Start_Stop_Count="true"; else HDD_Start_Stop_Count="false"; fi
							fi
							echo "Set Value: ("$HDD_Start_Stop_Count")"
							echo " "
							echo -n "Load Cycle Count ("$HDD_Load_Cycle") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Load_Cycle="true"; else HDD_Load_Cycle="false"; fi
							fi
							echo "Set Value: ("$HDD_Load_Cycle")"
							echo " "
							echo -n "Spin Retry Count ("$HDD_Spin_Retry") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Spin_Retry="true"; else HDD_Spin_Retry="false"; fi
							fi
							echo "Set Value: ("$HDD_Spin_Retry")"
							echo " "
							echo -n "Reallocated Sectors ("$HDD_Reallocated_Sectors") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Reallocated_Sectors="true"; else HDD_Reallocated_Sectors="false"; fi
							fi
							echo "Set Value: ("$HDD_Reallocated_Sectors")"
							echo " "
							echo -n "Reallocated Events ("$HDD_Reallocated_Events") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Reallocated_Events="true"; else HDD_Reallocated_Events="false"; fi
							fi
							echo "Set Value: ("$HDD_Reallocated_Events")"
							echo " "
							echo -n "Pending Sectors ("$HDD_Pending_Sectors") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Pending_Sectors="true"; else HDD_Pending_Sectors="false"; fi
							fi
							echo "Set Value: ("$HDD_Pending_Sectors")"
							echo " "
							echo -n "Offline Uncorrectable Errors ("$HDD_Offline_Uncorrectable") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Offline_Uncorrectable="true"; else HDD_Offline_Uncorrectable="false"; fi
							fi
							echo "Set Value: ("$HDD_Offline_Uncorrectable")"
							echo " "
							echo -n "UDMA CRC Errors ("$HDD_UDMA_CRC_Errors_List") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_UDMA_CRC_Errors_List="true"; else HDD_UDMA_CRC_Errors_List="false"; fi
							fi
							echo "Set Value: ("$HDD_UDMA_CRC_Errors_List")"
							echo " "
							echo -n "Raw Read Error Rate ("$HDD_Raw_Read_Error_Rate") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Raw_Read_Error_Rate="true"; else HDD_Raw_Read_Error_Rate="false"; fi
							fi
							echo "Set Value: ("$HDD_Raw_Read_Error_Rate")"
							echo " "
							echo -n "Seek Error Rate ("$HDD_Seek_Error_Rate") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Seek_Error_Rate="true"; else HDD_Seek_Error_Rate="false"; fi
							fi
							echo "Set Value: ("$HDD_Seek_Error_Rate")"
							echo " "
							echo -n "MultiZone Errors ("$HDD_MultiZone_Errors") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_MultiZone_Errors="true"; else HDD_MultiZone_Errors="false"; fi
							fi
							echo "Set Value: ("$HDD_MultiZone_Errors")"
							echo " "
							echo -n "Helium Level ("$HDD_Helium_Level") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Helium_Level="true"; else HDD_Helium_Level="false"; fi
							fi
							echo "Set Value: ("$HDD_Helium_Level")"
							echo " "
							echo -n "Last Test Age ("$HDD_Last_Test_Age") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Last_Test_Age="true"; else HDD_Last_Test_Age="false"; fi
							fi
							echo "Set Value: ("$HDD_Last_Test_Age")"
							echo " "
							echo -n "Last Test Type ("$HDD_Last_Test_Type") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Last_Test_Type="true"; else HDD_Last_Test_Type="false"; fi
							fi
							echo "Set Value: ("$HDD_Last_Test_Type")"
							echo " "
							echo -n "Total Data Written ("$HDD_Total_Data_Written") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then HDD_Total_Data_Written="true"; else HDD_Total_Data_Written="false"; fi
							fi
							echo "Set Value: ("$HDD_Total_Data_Written")"
							echo " "
							echo " "
							echo "returning..."
							sleep 2
						;;

						E)
							clear
							echo "SSD Column Selection"
							echo " "
							echo "This does not disable/enable any alarm settings."
							echo " "
							echo -n "Device ID ("$SSD_Device_ID") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Device_ID="true"; else SSD_Device_ID="false"; fi
							fi
							echo "Set Value: ("$SSD_Device_ID")"
							echo " "
							echo -n "Serial Number ("$SSD_Serial_Number") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Serial_Number="true"; else SSD_Serial_Number="false"; fi
							fi
							echo "Set Value: ("$SSD_Serial_Number")"
							echo " "
							echo -n "Model Number ("$SSD_Model_Number") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Model_Number="true"; else SSD_Model_Number="false"; fi
							fi
							echo "Set Value: ("$SSD_Model_Number")"
							echo " "
							echo -n "Capacity ("$SSD_Capacity") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Capacity="true"; else SSD_Capacity="false"; fi
							fi
							echo "Set Value: ("$SSD_Capacity")"
							echo " "
							echo -n "SMART Status ("$SSD_SMART_Status") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_SMART_Status="true"; else SSD_SMART_Status="false"; fi
							fi
							echo "Set Value: ("$SSD_SMART_Status")"
							echo " " 
							echo -n "Drive Temp ("$SSD_Drive_Temp") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Drive_Temp="true"; else SSD_Drive_Temp="false"; fi
							fi
							echo "Set Value: ("$SSD_Drive_Temp")"
							echo " "
							echo -n "Warranty ("$SSD_Warranty") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Warranty="true"; else SSD_Warranty="false"; fi
							fi
							echo "Set Value: ("$SSD_Warranty")"
							echo " "
							echo -n "Drive Temp Minimum for power cycle ("$SSD_Drive_Temp_Min") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Drive_Temp_Min="true"; else SSD_Drive_Temp_Min="false"; fi
							fi
							echo "Set Value: ("$SSD_Drive_Temp_Min")"
							echo " "
							echo -n "Drive Temp Maximum for power cycle ("$SSD_Drive_Temp_Max") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Drive_Temp_Max="true"; else SSD_Drive_Temp_Max="false"; fi
							fi
							echo "Set Value: ("$SSD_Drive_Temp_Max")"
							echo " "
							echo -n "Power On Hours ("$SSD_Power_On_Hours") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Power_On_Hours="true"; else SSD_Power_On_Hours="false"; fi
							fi
							echo "Set Value: ("$SSD_Power_On_Hours")"
							echo " "
							echo -n "Wear Level ("$SSD_Wear_Level") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Wear_Level="true"; else SSD_Wear_Level="false"; fi
							fi
							echo "Set Value: ("$SSD_Wear_Level")"
							echo " "
							echo -n "Reallocated Sectors ("$SSD_Reallocated_Sectors") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Reallocated_Sectors="true"; else SSD_Reallocated_Sectors="false"; fi
							fi
							echo "Set Value: ("$SSD_Reallocated_Sectors")"
							echo " "
							echo -n "Reallocated Events ("$SSD_Reallocated_Events") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Reallocated_Events="true"; else SSD_Reallocated_Events="false"; fi
							fi
							echo "Set Value: ("$SSD_Reallocated_Events")"
							echo " "
							echo -n "Pending Sectors ("$SSD_Pending_Sectors") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Pending_Sectors="true"; else SSD_Pending_Sectors="false"; fi
							fi
							echo "Set Value: ("$SSD_Pending_Sectors")"
							echo " "
							echo -n "Offline Uncorrectable Errors ("$SSD_Offline_Uncorrectable") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Offline_Uncorrectable="true"; else SSD_Offline_Uncorrectable="false"; fi
							fi
							echo "Set Value: ("$SSD_Offline_Uncorrectable")"
							echo " "
							echo -n "UDMA CRC Errors ("$SSD_UDMA_CRC_Errors_List") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_UDMA_CRC_Errors_List="true"; else SSD_UDMA_CRC_Errors_List="false"; fi
							fi
							echo "Set Value: ("$SSD_UDMA_CRC_Errors_List")"
							echo " "
							echo -n "Last Test Age ("$SSD_Last_Test_Age") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Last_Test_Age="true"; else SSD_Last_Test_Age="false"; fi
							fi
							echo "Set Value: ("$SSD_Last_Test_Age")"
							echo " "
							echo -n "Last Test Type ("$SSD_Last_Test_Type") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Last_Test_Type="true"; else SSD_Last_Test_Type="false"; fi
							fi
							echo "Set Value: ("$SSD_Last_Test_Type")"
							echo " "
							echo -n "Total Data Written ("$SSD_Total_Data_Written") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SSD_Total_Data_Written="true"; else SSD_Total_Data_Written="false"; fi
							fi
							echo "Set Value: ("$SSD_Total_Data_Written")"
							echo " "
							echo " "
							echo "returning..."
							sleep 2
						;;

						F)
							clear
							echo "NVMe Column Selection"
							echo " "
							echo "This does not disable/enable any alarm settings."
							echo " "
							echo -n "Device ID ("$NVM_Device_ID") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Device_ID="true"; else NVM_Device_ID="false"; fi
							fi
							echo "Set Value: ("$NVM_Device_ID")"
							echo " "
							echo -n "Serial Number ("$NVM_Serial_Number") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Serial_Number="true"; else NVM_Serial_Number="false"; fi
							fi
							echo "Set Value: ("$NVM_Serial_Number")"
							echo " "
							echo -n "Model Number ("$NVM_Model_Number") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Model_Number="true"; else NVM_Model_Number="false"; fi
							fi
							echo "Set Value: ("$NVM_Model_Number")"
							echo " "
							echo -n "Capacity ("$NVM_Capacity") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Capacity="true"; else NVM_Capacity="false"; fi
							fi
							echo "Set Value: ("$NVM_Capacity")"
							echo " "
							echo -n "SMART Status ("$NVM_SMART_Status") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_SMART_Status="true"; else NVM_SMART_Status="false"; fi
							fi
							echo "Set Value: ("$NVM_SMART_Status")"
							echo " "
							echo -n "Warranty ("$NVM_Warranty") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Warranty="true"; else NVM_Warranty="false"; fi
							fi
							echo "Set Value: ("$NVM_Warranty")"
							echo " "
							echo -n "Critical Warning Status ("$NVM_Critical_Warning") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Critical_Warning="true"; else NVM_Critical_Warning="false"; fi
							fi
							echo "Set Value: ("$NVM_Critical_Warning")"
							echo " "
							echo -n "Drive Temp ("$NVM_Drive_Temp") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Drive_Temp="true"; else NVM_Drive_Temp="false"; fi
							fi
							echo "Set Value: ("$NVM_Drive_Temp")"
							echo " "
							echo -n "Drive Temp Minimum for power cycle ("$NVM_Drive_Temp_Min") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Drive_Temp_Min="true"; else NVM_Drive_Temp_Min="false"; fi
							fi
							echo "Set Value: ("$NVM_Drive_Temp_Min")"
							echo " "
							echo -n "Drive Temp Maximum for power cycle ("$NVM_Drive_Temp_Max") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Drive_Temp_Max="true"; else NVM_Drive_Temp_Max="false"; fi
							fi
							echo "Set Value: ("$NVM_Drive_Temp_Max")"
							echo " "

							echo -n "Power State ("$NVM_Power_Level") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Power_Level="true"; else NVM_Power_Level="false"; fi
							fi
							echo "Set Value: ("$NVM_Power_Level")"
							echo " "

							echo -n "Power On Hours ("$NVM_Power_On_Hours") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Power_On_Hours="true"; else NVM_Power_On_Hours="false"; fi
							fi
							echo "Set Value: ("$NVM_Power_On_Hours")"
							echo " "
							echo -n "Wear Level ("$NVM_Wear_Level") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Wear_Level="true"; else NVM_Wear_Level="false"; fi
							fi
							echo "Set Value: ("$NVM_Wear_Level")"
							echo " "
							echo -n "Media Errors ("$NVM_Media_Error") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Media_Error="true"; else NVM_Media_Error="false"; fi
							fi
							echo "Set Value: ("$NVM_Media_Error")"
							echo " "

							echo -n "Last Test Age ("$NVM_Last_Test_Age") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Last_Test_Age="true"; else NVM_Last_Test_Age="false"; fi
							fi
							echo "Set Value: ("$NVM_Last_Test_Age")"
							echo " "

							echo -n "Last Test Type ("$NVM_Last_Test_Type") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Last_Test_Type="true"; else NVM_Last_Test_Type="false"; fi
							fi
							echo "Set Value: ("$NVM_Last_Test_Type")"
							echo " "
							echo -n "Total Data Written ("$NVM_Total_Data_Written") "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then NVM_Total_Data_Written="true"; else NVM_Total_Data_Written="false"; fi
							fi
							echo "Set Value: ("$NVM_Total_Data_Written")"
							echo " "
							echo " "
							echo "returning..."
							sleep 2
						;;

						G)
							clear
							echo "Output Formats"
							echo " "
							echo "Power On Hours Time Format ("$PowerTimeFormat") "
							echo -n "valid options are "ymdh", "ymd", "ym", "y", or "h" (year month day hour): "
							read Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then PowerTimeFormat=$Keyboard_yn; fi
							echo "Set Value: "$PowerTimeFormat
							echo " "
							echo "Temperature Display ("$TempDisplay") "
							echo -n "You may use anything you desire. Common formats are: *C, ^C, or ^c: "
							read Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then TempDisplay=$Keyboard_yn; fi
							echo "Set Value: "$TempDisplay
							echo " "
							echo "Non-existent Value ("$Non_Exist_Value") "
							echo "You may use anything you desire. Common formats are: ---, N/A, or a"
							echo -n "space character: "
							read Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then Non_Exist_Value=$Keyboard_yn; fi
							echo "Set Value: "$Non_Exist_Value
							echo " "
							echo "Pool Size and Free Space"
							echo "ZFS is the most accurate and conforms to the GUI values"
							echo "for RAIDZ's, however MIRRORs you might try 'zpool'."
							echo "Current Value:  ("$Pool_Capacity_Type") "
							echo -n "Enter 'zfs' or 'zpool' or Enter/Return for unchanged: "
							read Keyboard_yn
							if [[ $Keyboard_yn != "" ]] && [[ $Keyboard_yn != "zfs" ]] && [[ $Keyboard_yn != "zpool" ]]; then
								echo "INCORRECT VALUE!: "$Keyboard_yn
								echo "Setting default value: 'zfs'"
								Keyboard_yn="zfs"
							fi
							if [[ ! $Keyboard_yn == "" ]]; then Pool_Capacity_Type=$Keyboard_yn; fi
							echo "Set Value: "$Pool_Capacity_Type
							echo " "
							echo " "
							echo "returning..."
							sleep 2
						;;

						H)
							clear
							echo "Report Header Titles"
							echo "The titles for each CHART section."
							echo " "
							echo 'Current HDD Report Header: "'$HDDreportTitle'" '
							echo -n 'Enter new value or Return to accept current value: '
							read Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then HDDreportTitle=$Keyboard_yn; fi
							echo 'Set Value: "'$HDDreportTitle'"'
							echo " "
							echo 'Current SSD Report Header: "'$SSDreportTitle'" '
							echo -n 'Enter new value or Return to accept current value: '
							read Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then SSDreportTitle=$Keyboard_yn; fi
							echo 'Set Value: "'$SSDreportTitle'"'
							echo " "
							echo 'Current NVM Report Header: "'$NVMreportTitle'" '
							echo -n 'Enter new value or Return to accept current value: '
							read Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then NVMreportTitle=$Keyboard_yn; fi
							echo 'Set Value: "'$NVMreportTitle'"'
							echo " "
							echo "Text Section - Enable/Disable"
							echo "This will display (true) or remove (false) the Text Section"
							echo "of the email report."
							echo 'Current value: "'$Enable_Text_Section'" '
							echo 'Enter new value or Return to accept current value: '
							echo -n "('t' = true, 'f' = false) "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then Enable_Text_Section="true"; else Enable_Text_Section="false"; fi
							fi
							echo 'Set Value: "'$Enable_Text_Section'"'
							echo " "
							echo "returning..."
							sleep 2
						;;

						I)
							clear
							echo "Statistical Data Setup"
							echo " "

							echo " "
							echo "Statistical Data Recording Enabled ("$SDF_DataRecordEnable") "
							echo -n "('t' - true, 'f' = false) "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SDF_DataRecordEnable="true"; else SDF_DataRecordEnable="false"; fi
							fi
							echo "Set Value: ("$SDF_DataRecordEnable")"
							if [[ $SDF_DataRecordEnable == "true" ]]; then
								echo " "
								echo "Statistical Data Email Enabled ("$SDF_DataEmail") "
								echo -n "('t' - true, 'f' = false) "
								read -s -n 1 Keyboard_yn
								if [[ ! $Keyboard_yn == "" ]]; then
									if [[ $Keyboard_yn == "t" ]]; then SDF_DataEmail="true"; else SDF_DataEmail="false"; fi
								fi
								echo "Set Value: ("$SDF_DataEmail")"
								echo " "
								echo "Statistical file : ("$statistical_data_file")"
								echo "Enter new location and file name or press Enter/Return"
								echo -n "to accept current value: "
								read Keyboard_yn
								if [[ ! $Keyboard_yn == "" ]]; then statistical_data_file=$Keyboard_yn; fi
								echo "Set Value: ("$statistical_data_file")"
								echo " "
								echo "Statistical Data Purge Days ("$SDF_DataPurgeDays") "
								echo -n "(1-10000) "
								read Keyboard_HDD
								if [[ ! $Keyboard_HDD == "" ]]; then SDF_DataPurgeDays=$Keyboard_HDD; fi
								echo "Set Value: ("$SDF_DataPurgeDays")"
								echo " "
								echo "What day of the week would you like the file attached?"
								echo "Current Value: "$SDF_DataEmailDay
								echo -n "(All, Mon, Tue, Wed, Thu, Fri, Sat, Sun, Month) "
								read Keyboard_SDF_DataEmailDay
								if [[ ! $Keyboard_SDF_DataEmailDay == "" ]]; then SDF_DataEmailDay=$Keyboard_SDF_DataEmailDay; fi
								echo "Set Value: ("$SDF_DataEmailDay")"
							fi
							echo " "
							echo "returning..."
							sleep 2
						;;

						J)
							clear
							echo "Activate TLER (Time Limited Error Recovery)"
							echo " "
							echo "Hardware RAID controllers have a timeout period, where if a drive becomes"
							echo "unresponsive, the controller gives the drive a chance to respond again."
							echo "If the drive doesn’t respond within that time frame, it is removed from"
							echo "the array. This puts the array in a degraded state, which creates a greater"
							echo "risk of losing the entire array during the rebuilding process."
							echo " "
							echo "This option will turn on TLER if the drive supports it."
							echo " "
							echo "Activate TLER ("$SCT_Enable") "
							echo -n "('t' = true, 'f' = false) "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then SCT_Enable="true"; else SCT_Enable="false"; fi
							fi
							echo "Set Value: ("$SCT_Enable")"
							if [[ $SCT_Enable == "true" ]]; then
								echo " "
								echo "TLER Warning Level: ("$SCT_Warning_Level") "
								echo " 1) TLER_No_Msg = Only generate an error message if TLER cannot be turned on for"
								echo "    a supported drive."
								echo " 2) TLER = Report error messages in WARNING Section and email header."
								echo " 3) all = Report drives which also do not support TLER."
								echo " "
								echo "Note: The default 'TLER_No_Msg' option 1 is recommended." 
								echo -n "(1, 2, 3) "
								read -s -n 1 Keyboard_SCT_Warning_Level
								if [[ $Keyboard_SCT_Warning_Level == "1" ]]; then SCT_Warning_Level="TLER_No_Msg"; fi
								if [[ $Keyboard_SCT_Warning_Level == "2" ]]; then SCT_Warning_Level="TLER"; fi
								if [[ $Keyboard_SCT_Warning_Level == "3" ]]; then SCT_Warning_Level="all"; fi
								echo "Set Value: ("$SCT_Warning_Level")"
								echo " "
								echo "SCT Read Timeout Setting ("$SCT_Read_Timeout") "
								echo -n "(0-200, 70 is typical = 7.0 seconds) "
								read Keyboard_yn
								if [[ ! $Keyboard_yn == "" ]]; then SCT_Read_Timeout=$Keyboard_yn; fi
								echo "Set Value: ("$SCT_Read_Timeout")"
								echo " "
								echo "SCT Write Timeout Setting ("$SCT_Write_Timeout") "
								echo -n "(0-200, 70 is typical = 7.0 seconds) "
								read Keyboard_yn
								if [[ ! $Keyboard_yn == "" ]]; then SCT_Write_Timeout=$Keyboard_yn; fi
								echo "Set Value: ("$SCT_Write_Timeout")"
							fi
							echo " "
							echo "returning..."
							sleep 2
						;;

						K)
							clear
							echo "Drive Errors"
							echo " "
							echo "Collecting data, Please wait..."
							# Lets go ahead and grab all the drive data we will need for the entire K section.
							get_smartHDD_listings
							get_smartSSD_listings
							get_smartNVM_listings
							smartdrivesall="$smartdrives $smartdrivesSSD $smartdrivesNVM"
							clear
							echo "Drive Errors"
							echo " "
							echo "Ignore Drives"
							echo " "
							echo "Drives listed will be ignored from the entire report and are"
							echo "listed by serial number."
							echo " "
							if [[ $Ignore_Drives_List == "" ]]; then Ignore_Drives_List2="EMPTY"; else Ignore_Drives_List2=$Ignore_Drives_List; fi
							echo "Current: "$Ignore_Drives_List2
							echo " "
							echo "Press 'e' to Edit/Add, 'd' to Delete, or any other key to Accept "
							echo -n "('e' = Edit, 'd' = Delete, 'Return' = Accept) "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then Ignore_Drives_List=$Keyboard_yn; fi
							if [[ $Keyboard_yn == "d" ]]; then Ignore_Drives_List="none"; fi
							if [[ $Keyboard_yn == "e" ]]; then
								# Let's list each drive and ask to keep or reject
								for drive in $smartdrivesall; do
									clear_variables
									get_drive_data
									echo " "
									echo "Do you want to ignore this drive (y/n): Drive ID: "$drive" Serial Number: "$serial
									read -s -n 1 Keyboard_yn
									if [[ $Keyboard_yn == "y" ]]; then ignoredriveslist=$ignoredriveslist$serial","; fi
									echo "Updated Value="$ignoredriveslist
								done
								if [[ ! $ignoredriveslist == "" ]]; then Ignore_Drives_List="$(echo "$ignoredriveslist" | sed 's/.$//')"; else Ignore_Drives_List="none"; fi
							fi
							if [[ $Ignore_Drives_List == "" ]]; then Ignore_Drives_List2="EMPTY"; else Ignore_Drives_List2=$Ignore_Drives_List; fi
							echo "Set Value: "$Ignore_Drives_List2
							echo " "
							echo "Press any key to continue"
							read -s -n 1 Keyboard_yn
							clear
							echo "Drive Errors"
							echo " "
							echo "AUTOMATIC DRIVE COMPENSATION - UDMA_CRC, MultiZone, and Reallocated Sectors"
							echo " "
							echo "You have the option to automatically setup offset values for UDMA_CRC,"
							echo "MultiZone, and Bad Sectors that are permanently recorded on the drives."
							echo " "
							echo "This will create an offset to be displayed in the charts resulting in a"
							echo "zero value and the value will be colored in light yellow by default."
							echo "You will be able to see any future changes/failures which occur."
							echo " "
							echo "If you select No, you will be allowed to manually enter each offset."
							echo " "
							echo "Enter 'y' for yes or 'n' for no, or Return to skip this section."
							read -s -n 1 Keyboard_yn

							if [[ $Keyboard_yn == "y" ]]; then
								autoselect=1
								echo "Automatic Configuration selected..."
								echo "This may take a minute while scanning the drives..."
								echo " "
								for drive in $smartdrivesall; do
									echo "Scanning drive "$drive
									clear_variables
									get_drive_data
									if [[ ! $crcErrors == "0" ]] && [[ ! $crcErrors == "" ]]; then listofdrivescrc="$listofdrivescrc$serial":"$crcErrors,"; fi
									if [[ ! $multiZone == "0" ]] && [[ ! $multiZone == "" ]]; then listofdrivesmulti="$listofdrivesmulti$serial":"$multiZone,"; fi
									if [[ ! $reAlloc == "0" ]] && [[ ! $reAlloc == "" ]]; then listofdrivesbad="$listofdrivesbad$serial":"$reAlloc,"; fi
									if [[ ! $reAllocEvent == "0" ]] && [[ ! $reAllocEvent == "" ]]; then listofdrivesbad2="$listofdrivesbad2$serial":"$reAllocEvent,"; fi
								done
								echo " "
								echo "Scanning Results:"
								if [[ ! $listofdrivescrc == "" ]]; then CRC_Errors_List="$(echo "$listofdrivescrc" | sed 's/.$//')"; echo "UDMA_CRC Errors detected"; else CRC_Errors_List=""; echo "No UDMA_CRC Errors"; fi
								if [[ ! $listofdrivesmulti == "" ]]; then MultiZone_List="$(echo "$listofdrivesmulti" | sed 's/.$//')"; echo "MultiZone Errors Detected"; else MultiZone_List=""; echo "No MultiZone Errors"; fi
								if [[ ! $listofdrivesbad == "" ]]; then ReAllocated_Sector_List="$(echo "$listofdrivesbad" | sed 's/.$//')"; echo "Bad Sectors Detected"; else ReAllocated_Sector_List=""; echo "No Reallocated Sectors"; fi
								if [[ ! $listofdrivesbad2 == "" ]]; then ReAllocated_Sector_Events_List="$(echo "$listofdrivesbad2" | sed 's/.$//')"; echo "Bad Sectors Detected"; else ReAllocated_Sector_Events_List=""; echo "No Reallocated Sectors"; fi
								echo " "
								echo "Values Set:"
								echo "CRC_Errors_List: "$CRC_Errors_List
								echo "MultiZone_List_Errors: "$MultiZone_List
								echo "Reallocated_Sectors: "$ReAllocated_Sector_List
								echo "Reallocated_Sectors_Events: "$ReAllocated_Sector_Events_List
								echo " "
							fi
							if [[ ! $autoselect == "1" ]] && [[ $Keyboard_yn == "n" ]]; then
								clear
								echo "Drive Errors"
								echo " "
								echo "Offset UDMA CRC Errors"
								echo "Press 'd' to delete, 'e' to edit, or Enter/Return to accept."
								echo "Current List: "$CRC_Errors_List
								read -s -n 1 Keyboard_yn
								if [[ ! $Keyboard_yn == "" ]]; then CRC_Errors_List=$Keyboard_yn; fi
								if [[ $Keyboard_yn == "d" ]]; then CRC_Errors_List=""; Keyboard_yn=""; fi
								if [[ $Keyboard_yn == "e" ]]; then
									# Let's list each drive and ask to keep or reject
									drive_select=""
									for drive in $smartdrivesall; do
										clear_variables
										get_drive_data
										echo " "
										echo "Do you want to add this drive (y/n): Drive ID: "$drive" Serial Number: "$serial
										read -s -n 1 Keyboard_yn
										if [[ $Keyboard_yn == "y" ]]; then drive_select=$drive_select$serial":"
											echo "Enter the sector count offset you desire: "
											read Keyboard_yn
											drive_select=$drive_select$Keyboard_yn","
											echo "drive_select="$drive_select
										fi
      								done
									if [[ ! $drive_select == "" ]]; then CRC_Errors_List="$(echo "$drive_select" | sed 's/.$//')"; else CRC_Errors_List="none"; fi
								fi
								echo "Set Value: "$CRC_Errors_List
								echo " "
								echo "Press any key to continue"
								read -s -n 1 Keyboard_yn
								clear
								echo "Drive Errors"
								echo " "
								echo "Offset MultiZone Errors"
								echo "Press 'd' to delete, 'e' to edit, or Enter/Return to accept."
								echo "Current: "$MultiZone_List
								read -s -n 1 Keyboard_yn
								if [[ ! $Keyboard_yn == "" ]]; then MultiZone_List=$Keyboard_yn; fi
								if [[ $Keyboard_yn == "d" ]]; then MultiZone_List=""; fi
								if [[ $Keyboard_yn == "e" ]]; then
									# Let's list each drive and ask to keep or reject
									drive_select=""
									for drive in $smartdrivesall; do
										clear_variables
										get_drive_data
										echo " "
										echo "Do you want to add this drive (y/n): Drive ID: "$drive" Serial Number: "$serial
										read -s -n 1 Keyboard_yn
										if [[ $Keyboard_yn == "y" ]]; then drive_select=$drive_select$serial":"
											echo "Enter the MultiZone_List count offset you desire: "
											read Keyboard_yn
											drive_select=$drive_select$Keyboard_yn","
											echo "drive_select="$drive_select
										fi
									done
									if [[ ! $drive_select == "" ]]; then MultiZone_List="$(echo "$drive_select" | sed 's/.$//')"; else MultiZone_List="none"; fi
								fi
								echo "Set Value: "$MultiZone_List
								echo " "
								echo "Press any key to continue"
								read -s -n 1 Keyboard_yn
								clear
								echo "Drive Errors"
								echo " "
								echo "Offset Bad Sector Errors"
								echo "Press 'd' to delete, 'e' to edit, or Enter/Return to accept."
								echo "Current: "$ReAllocated_Sector_List
								read -s -n 1 Keyboard_yn
								if [[ ! $Keyboard_yn == "" ]]; then ReAllocated_Sector_List=$Keyboard_yn; fi
								if [[ $Keyboard_yn == "d" ]]; then ReAllocated_Sector_List=""; fi
								if [[ $Keyboard_yn == "e" ]]; then
									# Let's list each drive and ask to keep or reject
									drive_select=""
									for drive in $smartdrivesall; do
										clear_variables
										get_drive_data
										echo " "
										echo "Do you want to add this drive (y/n): Drive ID: "$drive" Serial Number: "$serial
										read -s -n 1 Keyboard_yn
										if [[ $Keyboard_yn == "y" ]]; then drive_select=$drive_select$serial":"
											echo "Enter the Bad Sector count offset you desire: "
											read Keyboard_yn
											drive_select=$drive_select$Keyboard_yn","
											echo "drive_select="$drive_select
										fi
									done
									if [[ ! $drive_select == "" ]]; then ReAllocated_Sector_List="$(echo "$drive_select" | sed 's/.$//')"; else ReAllocated_Sector_List="none"; fi
								fi
								echo "Set Value: "$ReAllocated_Sector_List
								echo " "
								echo "Press any key to continue"
								read -s -n 1 Keyboard_yn
								clear
								echo "Drive Errors"
								echo " "
								echo "Offset Bad Sector Event Errors"
								echo "Press 'd' to delete, 'e' to edit, or Enter/Return to accept."
								echo "Current: "$ReAllocated_Sector_Events_List
								read -s -n 1 Keyboard_yn
								if [[ ! $Keyboard_yn == "" ]]; then ReAllocated_Sector_Events_List=$Keyboard_yn; fi
								if [[ $Keyboard_yn == "d" ]]; then ReAllocated_Sector_Events_List=""; fi
								if [[ $Keyboard_yn == "e" ]]; then
									# Let's list each drive and ask to keep or reject
									drive_select=""
									for drive in $smartdrivesall; do
										clear_variables
										get_drive_data
										echo " "
										echo "Do you want to add this drive (y/n): Drive ID: "$drive" Serial Number: "$serial
										read -s -n 1 Keyboard_yn
										if [[ $Keyboard_yn == "y" ]]; then drive_select=$drive_select$serial":"
											echo "Enter the Bad Sector count offset you desire: "
											read Keyboard_yn
											drive_select=$drive_select$Keyboard_yn","
											echo "drive_select="$drive_select
										fi
									done
									if [[ ! $drive_select == "" ]]; then ReAllocated_Sector_Events_List="$(echo "$drive_select" | sed 's/.$//')"; else ReAllocated_Sector_Events_List="none"; fi
								fi
								echo "Set Value: "$ReAllocated_Sector_Events_List
							fi
							echo " "
							echo "Press any key to continue"
							read -s -n 1 Keyboard_yn
							clear
							echo "Drive Errors"
							echo " "
							echo "Automatic ATA Error Count Updates"
							echo " "
							echo "This will automatically let the script update the multi_report_config.txt"
							echo "file with the current Error Log count."
							echo "This might be desirable if you have a drive that keeps throwing minor errors."
							echo "Enter/Return to keep current value." 
							echo "Current: "$ATA_Auto_Enable
							echo -n "('t' = true, 'f' = false) "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then if [[ $Keyboard_yn == "t" ]]; then ATA_Auto_Enable="true"; else ATA_Auto_Enable="false"; fi; fi
							echo "Set Value: "$ATA_Auto_Enable
							echo " "
							echo "Press any key to continue"
							read -s -n 1 Keyboard_yn
							clear
							echo "Drive Errors"
							echo " "
							echo "ATA Error Count - This will ignore any drive with an error count less than"
							echo "the number provided.  When the drive errors exceed this value then the"
							echo "Error Log will be displayed."
							echo " "
							if [[ $ATA_Errors_List == "" ]]; then ATA_Errors_List2="EMPTY"; else ATA_Errors_List2=$ATA_Errors_List; fi
							echo "Current: "$ATA_Errors_List2
							echo -n "('e' = Edit/Add, 'd' = Delete, Any key to Accept) "
							read -s -n 1 Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then ATA_Errors_List=$Keyboard_yn; fi
							if [[ $Keyboard_yn == "d" ]]; then ATA_Errors_List=""; fi
							if [[ $Keyboard_yn == "e" ]]; then
								ATA_Errors_List=""
								for drive in $smartdrivesall; do
									clear_variables
									get_drive_data
									echo " "
									echo "Do you want to add this drive (y/n): Drive ID: "$drive" Serial Number: "$serial
									read -s -n 1 Keyboard_yn
									if [[ $Keyboard_yn == "y" ]]; then
										echo "Enter the Error Log threshold: "
										echo -n "(0-10000) "
										read Keyboard_yn
										ATA_Errors_List=$ATA_Errors_List$serial":"$Keyboard_yn","
									fi
									echo "ATA_Errors_List="$ATA_Errors_List
								done
								if [[ ! $ATA_Errors_List == "" ]]; then ATA_Errors_List="$(echo "$ATA_Errors_List" | sed 's/.$//')"; fi
							fi
							if [[ $ATA_Errors_List == "" ]]; then ATA_Errors_List2="EMPTY"; else ATA_Errors_List2=$ATA_Errors_List; fi
							echo "Set Value: "$ATA_Errors_List2
							echo " "
							echo "Press any key to continue"
							read -s -n 1 Keyboard_yn
							clear
							echo "Drive Errors"
							echo " "
							echo "Drive Warranty Expiration Date Warning"
							echo "This will provide a yellow background and a text message when"
							echo "the warranty date occurs."
							echo " "
							echo "The format is: drive_serial_number:yyyy-mm-dd and comma separated."
							echo "Enter 'd' to delete, 'e' to edit, 'a' to add, Enter/Return for no change."
							if [[ $Drive_Warranty_List == "" ]]; then Drive_Warranty_List2="EMPTY"; else Drive_Warranty_List2=$Drive_Warranty_List; fi
							echo "Current: "$Drive_Warranty_List2
							echo -n "('a' = Add Drive, 'e' = Edit Drive, 'd' = Delete All, Any key = Accept) "
							read -s -n 1 Keyboard_yn
							if [[ $Keyboard_yn == "d" ]]; then
								Drive_Warranty_List=""
								echo "Working..."
							fi

							if [[ $Keyboard_yn == "e" ]]; then
								echo "Working..."
								for drive in $smartdrivesall; do
									clear_variables
									get_drive_data
									echo " "
									# Lets compare the $serial to the Drive_Warranty_List

									IFS=',' read -ra ADDR <<< "$Drive_Warranty_List"
									for i in "${ADDR[@]}"; do
										drivesn1="$(echo $i | cut -d':' -f 1)"
										drivedt1="$(echo $i | cut -d':' -f 2)"
										if [[ $drivesn1 == $serial ]]; then
											warrantyyear="$(echo $drivedt1 | cut -d '-' -f1)"
											warrantymonth="$(echo $drivedt1 | cut -d '-' -f2)"
											warrantyday="$(echo $drivedt1 | cut -d '-' -f3)"

											# Display the results and ask to edit, keep, or delete.

											echo "Drive S/N: "$serial" Warranty Date is "$warrantyyear"-"$warrantymonth"-"$warrantyday
											echo "Do you want to (k)eep, (e)dit, or (d)elete this entry? "

											read -s -n 1 Keyboard_yn
											if [[ $Keyboard_yn == "e" ]]; then
												# Edit the drive string
												echo "Enter new warranty date in format yyyy-mm-dd"
												# Need to ask for the date to be entered again
												read Keyboard_yn
												# Save serial and date to $warrantydrivelist
												if [[ $warrantydrivelist == "" ]]; then
													warrantydrivelist=$serial":"$Keyboard_yn
												else
													warrantydrivelist=$warrantydrivelist","$serial":"$Keyboard_yn
												fi
											elif [[ $Keyboard_yn == "d" ]]; then
												# Delete the entry
												echo "Deleting "$serial" from the warranty list"
											else
												# Leave the drive warranty data unaltered.
												echo "Leaving drive "$serial" untouched."
												if [[ $warrantydrivelist == "" ]]; then
													warrantydrivelist=$serial":"$warrantyyear"-"$warrantymonth"-"$warrantyday
												else
													warrantydrivelist=$warrantydrivelist","$serial":"$warrantyyear"-"$warrantymonth"-"$warrantyday
												fi	
											fi
											echo "Edited Value is: "$warrantydrivelist	
										else
											addserial="1"
										fi
									done
								done
								if [[ ! $warrantydrivelist == "" ]]; then Drive_Warranty_List="$(echo "$warrantydrivelist" | sed 's/.$//')"; else Drive_Warranty_List=""; fi
							fi

							if [[ $Keyboard_yn == "a" ]]; then
								echo "Working..."
								echo " "
								echo "Searching for drives not in the Warranty List..."
								echo " "
								for drive in $smartdrivesall; do
									clear_variables
									addserial=""
									get_drive_data
									if [[ $Drive_Warranty_List == "" ]] || [[ $Drive_Warranty_List == "none" ]]; then addserial=$serial; fi
										# Lets compare the $serial to the Drive_Warranty_List
										IFS=',' read -ra ADDR <<< "$Drive_Warranty_List"
										for i in "${ADDR[@]}"; do
											drivesn1="$(echo $i | cut -d':' -f 1)"
											drivedt1="$(echo $i | cut -d':' -f 2)"
											if [[ $drivesn1 == $serial ]]; then
												addserial=""
												break
											else
												addserial=$serial
											fi
										done
									if [[ $addserial != "" ]]; then
										# Display the results and ask to edit, keep, or delete.
										newdrivesfound="1"
										echo "Do you want to add this drive "$serial" (y/n)?"
										read -s -n 1 Keyboard_yn
										if [[ $Keyboard_yn == "y" ]]; then
											# Keep the drive from the string
											echo " "
											echo "Enter the date the drive expires in the following format: yyyy-mm-dd"
											read Keyboard_yn
											if [[ $Drive_Warranty_List == "" ]] || [[ $Drive_Warranty_List == "none" ]]; then
												Drive_Warranty_List=$serial":"$Keyboard_yn
											else
												Drive_Warranty_List=$Drive_Warranty_List","$serial":"$Keyboard_yn
											fi
										else
											echo "Not adding the drive to the Drive Warranty List"
										fi
									fi
								done
								if [[ $newdrivesfound != "1" ]]; then echo "No new drives were found."; fi
							fi
							if [[ $Drive_Warranty_List == "" ]]; then Drive_Warranty_List2="EMPTY"; else Drive_Warranty_List2=$Drive_Warranty_List; fi
							echo " "
							echo "Set Value: "$Drive_Warranty_List2
							echo " "
							echo "Press any key to continue"
							read -s -n 1 Keyboard_yn
							clear
							echo "Drive Errors"
							echo " "
							echo "Drive Warranty Expiration Chart Box Pixel Color"
							echo " "
							echo "Enter/Return = no change, or enter Hex Color Code (Google it)"
							echo "Examples: black=#000000, red=#FF0000, lightblue=#add8e6"
							echo "Current: "$expiredWarrantyBoxColor
							echo " "
							echo -n "(Press Enter to Accept or enter a Hex Color Code) "
							read Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then expiredWarrantyBoxColor=$Keyboard_yn; fi
							echo "Set Value: "$expiredWarrantyBoxColor
							echo " "
							echo "Press any key to continue"
							read -s -n 1 Keyboard_yn
							clear
							echo "Drive Errors"
							echo " "
							echo "Drive Warranty Expiration Chart Box Background Color"
							echo " "
							echo "Enter/Return = no change, or enter Hex Color Code (Google it)"
							echo "Examples: black=#000000, red=#FF0000, lightblue=#add8e6"
							echo "yellow=#f1ffad"
							echo " "
							echo 'You may also enter "none" to use the default background.'
							echo "Current: "$WarrantyBackgndColor
							echo " "
							echo -n "(Press Enter to Accept or enter a Hex Color Code) "
							read Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then WarrantyBackgndColor=$Keyboard_yn; fi
							echo "Set Value: "$WarrantyBackgndColor
							echo " "
							echo "returning..."
							sleep 2
						;;

						L )
							clear
							echo "Subject Line Customization"
							echo " "
							echo "The default subject line is typically sufficent however there have been"
							echo "requests to alter it.  This is the option to alter it."
							echo " "
							echo "If you want to add the 'host' name, enter into your string using ${host}"
							echo "where you want the information displayed."
							echo " "
							echo "Example: Yo! Server ${host} is having a problem, better look at it!"
							echo "The text '${host}' will not be displayed but reather the host name will"
							echo "replace it when this program saves the data."
							echo " "
							echo "A) Normal Subject Line"
							echo "B) Warning Subject Line"
							echo "C) Critical Subject Line"
							echo " "
							echo "D) Set all back to default"
							echo " "
							echo "   Make your selection: "
							echo -n "('a', 'b', 'c', 'd', or Any other key to continue) "
							read -s -n 1 Keyboard_var4
							echo " "
							shopt -s nocasematch
							case $Keyboard_var4 in							
								A)
									clear
									echo "Normal Subject Line"
									echo " "
									echo "Current :"$Subject_Line_Normal
									echo "Enter new value or Press Return/Enter to leave the same."
									read Keyboard
									if [[ ! $Keyboard == "" ]]; then
										Subject_Line_Normal=$Keyboard
									fi
									echo " "
									echo "Press any key to continue"
									read -s -n 1 key 
								;;

								B)
									clear
									echo "Warning Subject Line"
									echo " "
									echo "Current :"$Subject_Line_Warning
									echo "Enter new value or Press Return/Enter to leave the same."
									read Keyboard
									if [[ ! $Keyboard == "" ]]; then
										Subject_Line_Warning=$Keyboard
									fi
									echo " "
									echo "Press any key to continue"
									read -s -n 1 key 
								;;

								C)
									clear
									echo "Critical Subject Line"
									echo " "
									echo "Current :"$Subject_Line_Critical
									echo "Enter new value or Press Return/Enter to leave the same."
									read Keyboard
									if [[ ! $Keyboard == "" ]]; then
										Subject_Line_Critical=$Keyboard
									fi
									echo " "
									echo "Press any key to continue"
									read -s -n 1 key 
								;;

								D)
									clear
									echo "Return to Default Subject Lines"
									echo " "
									echo "Current Normal: "$Subject_Line_Normal
									echo "Current Warning: "$Subject_Line_Warning
									echo "Current Critical: "$Subject_Line_Critical
									echo " "
									echo "Change back to defaults (y/n) or Press Return/Enter to leave the same."
									read -n 1 Keyboard
									if [[ $Keyboard == "y" ]]; then
										Subject_Line_Critical='*CRITICAL ERROR*  SMART Testing Results for ${host}  *CRITICAL ERROR*'
										Subject_Line_Warning='*WARNING*  SMART Testing Results for ${host}  *WARNING*'
										Subject_Line_Normal='SMART Testing Results for ${host} - All is Good'
									fi
									echo " "
									echo "Current Normal:"$Subject_Line_Normal
									echo "Current Warning:"$Subject_Line_Warning
									echo "Current Critical:"$Subject_Line_Critical
									echo " "
									echo "Press any key to continue"
									read -s -n 1 key 
								;;
							esac
						;;

						M)
							clear
							echo "Multipath Configuration"
							echo " "
							echo "You will be able to select one of four Multipath options:"
							echo " "
							echo "'off' = No processing of serial numbers and is the default value."
							echo "'normal' = Automatically remove duplicate serial numbers."
							echo "'Exos2x' = Remove duplicate serial numbers ONLY IF the gptid matches."
							echo "'serial' = Sort by serial numbers."
							echo " "
							echo "Multipath is currently: "$Multipath
							echo " "
							echo "Your options are: '1'=Off, '2'=Normal, '3'=Exos2x, or '4'=Serial"
							read -s -n 1 Keyboard_yn
							if [[ $Keyboard_yn == "1" ]]; then Multipath="off"; fi
							if [[ $Keyboard_yn == "2" ]]; then Multipath="normal"; fi
							if [[ $Keyboard_yn == "3" ]]; then Multipath="Exos2x"; fi
							if [[ $Keyboard_yn == "4" ]]; then Multipath="serial"; fi
							if [[ $Keyboard_yn != "1" && $Keyboard_yn != "2" && $Keyboard_yn != "3" && $Keyboard_yn != "4" ]]; then	echo "No change"; fi
							echo " "
							echo "Set Value: "$Multipath
							echo " "
							echo "Press any key to continue"
							read -s -n 1 key
						;;

						N)
							clear
							echo "NVMe Custom Selections"
							echo " "
							echo "NVMe support is lacking in both TrueNAS CORE 13.0-U6.1 and"
							echo "TrueNAS SCALE 23.10.2.  These settings will cope with the"
							echo "descrepancies."
							echo " "
							echo "TrueNAS CORE (FreeBSD) does not adjust NVMe power to lower"
							echo "levels as TrueNAS SCALE (Debian)."
							echo "Under CORE the NVMe's will run at maximum power and you have"
							echo "the option here to attempt to set it to it's lowest state."
							echo " "
							echo "The Rub: The NVMe can and will automatically switch to a"
							echo "higher power state if the NVMe needs to, however it"
							echo "will not automaticaly return to the lower state."
							echo " "
							echo "Current: "$NVM_Low_Power
							echo " "
							echo "Would you like to set the lowest power state (y/n)?"
							read -s -n 1 Keyboard_yn
							if [[ $Keyboard_yn == "y" || $Keyboard_yn == "n" ]]; then
								if [[ $Keyboard_yn == "y" ]]; then
									NVM_Low_Power="true"
								else
									NVM_Low_Power="false"
								fi
							fi
							echo "Set Value: "$NVM_Low_Power
							clear
							echo "NVMe Custom Selections"
							echo " "
							echo "Prior to smartmontools v7.4 the NVMe SMART tests were not"
							echo "available on 'smartctl' and TrueNAS CORE 13.0-U6.1 currently"
							echo "supports version 7.2.  TrueNAS SCALE (23.10.2) does support"
							echo "version 7.4, however TrueNAS SCALE GUI does not allow the"
							echo "option to configure running a S.M.A.R.T. test."
							echo " "
							echo "This sction compensates for these issues until they are"
							echo "corrected in TrueNAS."
							echo " "
							echo "Run NVME SMART Test if smartmontools 7.4 is installed"
							echo "Current: "$NVM_Smartmontools_74_Override
							echo -n "('e' = enable, 'd' = disable, Any key to accept) "
							read -s -n 1 Keyboard_yn
							if [[ $Keyboard_yn == "e" || $Keyboard_yn == "d" ]]; then
								if [[ $Keyboard_yn == "e" ]]; then
									NVM_Smartmontools_74_Override="enable"
								else
									NVM_Smartmontools_74_Override="disable"
								fi
							fi
							echo " "
							echo "Set Value: "$NVM_Smartmontools_74_Override
							echo " "
							echo "Press any key to continue"
							read -s -n 1 key 
							clear
							echo "NVMe Custom Selections"
							echo " "
							echo "NVMe SMART Short Daily Test"
							echo "This will run a daily SMART Short test on all days except"
							echo "the day the Long test is scheduled."
							echo " "
							echo "Run a SMART Short Daily Self-test"
							echo "Current: "$NVM_Daily_Short_Selftest
							echo -n "('t' = true, 'f' = false, Any key to accept) "
							read -s -n 1 Keyboard_yn
							if [[ $Keyboard_yn == "t" || $Keyboard_yn == "f" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then
									NVM_Daily_Short_Selftest="true"
								else
									NVM_Daily_Short_Selftest="false"
								fi
							fi
							echo "Set Value: "$NVM_Daily_Short_Selftest
							echo " "
							echo "Press any key to continue"
							read -s -n 1 key 
							clear
							echo "NVMe Custom Selections"
							echo " "
							echo "NVMe SMART Long Weekly Test Day"
							echo " "
							echo "Run a SMART Long Self-test on "$SMART_Weekly_Long_Selftest_Day
							echo "A Short test will not be run on this day."
							echo " "
							echo "Current: "$NVM_Weekly_Long_Selftest_Day
							echo " "
							echo "What would you like to run a SMART Long Weekly Self-test on?"
							echo -n "(Sun, Mon, Tue, Wed, Thu, Fri, Sat) "
							read Keyboard_yn
							if [[ $Keyboard_yn != "" ]]; then NVM_Weekly_Long_Selftest_Day=$Keyboard_yn; fi
							echo "Set Value: "$NVM_Weekly_Long_Selftest_Day
							echo " "
							echo "Press any key to continue"
							read -s -n 1 key 
							clear
							echo "NVMe Custom Selections"
							echo " "
							echo "Another option is to delay the script email until after"
							echo "the SMART Self-test has completed.  This gives you the"
							echo "benefit of having the most recent test in the Multi-Report"
							echo "report.  Note: you can disable the SMART Long Self-test"
							echo "if you choose to not wait for it."
							echo " "
							echo "Wait For SMART Self-test to complete?"
							echo "Current: "$Wait_For_SMART
							echo -n "('t' = true, 'f' = false, Any key to accept) "
							read -s -n 1 Keyboard_yn
							if [[ $Keyboard_yn == "t" || $Keyboard_yn == "f" ]]; then
								if [[ $Keyboard_yn == "t" ]]; then
									Wait_For_SMART="true"
								else
									Wait_For_SMART="false"
								fi
							fi
							echo "Set Value: "$Wait_For_SMART
							echo " "
							echo "Press any key to continue"
							read -s -n 1 key 
							clear
							echo "NVMe Custom Selections"
							echo " "
							echo "Wait For SMART Self-test delay"
							echo " "
							echo "If you selected to Wait For SMART Self-test (previous question)"
							echo "and do not want to wait for the Daily Short Self-test to complete,"
							echo "enter '1' second.   Remember, this is not a delay for normal HDD/SSD,"
							echo "this is for NVMe ONLY."
							echo " "
							echo "NVMe Short Self-test SECONDS to wait"
							echo "Current: "$Wait_For_SMART_Short_Duration
							echo -n "(1-82800) "
							read Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								Wait_For_SMART_Short_Duration=$Keyboard_yn
							fi
							echo " "
							echo "Set Value: "$Wait_For_SMART_Short_Duration
							echo " "
							echo "Press any key to continue"
							read -s -n 1 key 
							clear
							echo "NVMe Custom Selections"
							echo " "
							echo "Wait for SMART Self-test delay"
							echo " "
							echo "If you selected to Wait For SMART Self-test and do not"
							echo "want to wait for the Weekly Long Self-test to complete,"
							echo "enter '1' second for the weekly below. Remember, this is"
							echo "not a delay for normal HDD/SSD, this is NVMe ONLY."
							echo " "
							echo "NVMe Long Self-test SECONDS to wait"
							echo "Current: "$Wait_For_SMART_Long_Duration
							echo -n "(1-82800) "
							read Keyboard_yn
							if [[ ! $Keyboard_yn == "" ]]; then
								Wait_For_SMART_Long_Duration=$Keyboard_yn
							fi
							echo " "
							echo "Set Value: "$Wait_For_SMART_Long_Duration
							echo " "
							echo " "
							echo " "
							echo "End of NVMe section, Press any key to continue"
							read -s -n 1 key 
						;;	

						S)
							clear
							echo "Custom Drive Configuration Mode"
							echo " "
							echo "This series of questions will allow you to customize each alarm setting"
							echo "for up to at least 24 drives on your system.  It is suggested that only"
							echo "drives which need customization be included here."
							echo " "
							echo "If you choose to customize a drive you will be presented with the"
							echo 'Drive ID, Drive Serial Number, and the "system default" alarm setting.'
							echo " "
							echo 'Press Return/Enter to accept the "system default" value.  This means that if'
							echo "the system default value changes in the future, it will mirror that value."
							echo "Or you may enter a numeric value and this value will be hardcoded for this"
							echo "one drive by serial number."
							echo " "
							echo "IMPORTANT: Do not enter a value unless you want to hardcode the value."
							echo "Example: Warning Temperature defaults to (45).  If you press Return/Enter"
							echo "to accept the default then if you later change the configuration file"
							echo "Warning Temperature to (50), this drive will use (50)."
							echo "However, if you entered '45' and pressed Return/Enter, this specific drive"
							echo "is set to (45), even if you changed the configuration file default value."
							echo " "
							echo "Press any key to continue"
							read -s -n 1 key
							clear
							echo "Custom Drive Configuration Mode"
							echo " "
							echo "Two additional setpoints:"
							echo "Disable "Last Test Age". This is useful for some older drives"
							echo "which may generate an alarm."
							echo " "
							echo "Reverse the Wear Level value. Unfortunately sometimes the value starts at 100"
							echo "and counts down, sometimes it starts at 0 and counts up, and sometimes we"
							echo "need to use the Normalized value, or worst case, Ignore it all together."
							echo " "
							echo " "
							echo "Follow the prompts."
							echo " "
							echo "Press any key to continue"
							read -s -n 1 key
							clear
							echo "Custom Drive Configuration Mode"
							echo " "
							echo "Collecting list of drives, Please wait..."
							# Lets go ahead and grab all the drive data we will need for the entire section.
							get_smartHDD_listings
							get_smartSSD_listings
							get_smartNVM_listings
							smartdrivesall="$smartdrives $smartdrivesSSD $smartdrivesNVM"
							# So we have all the drives listed now.
							# We will step through each drive and then compare the S/N's to Custom_Drives_List, if there
							# is a match then we display the values.  If no match then display default values.
							if [[ $Custom_Drives_List != "" ]]; then
								echo "Would you like to delete all Custom Configuration Data (y/n)?"
								read -s -n 1 Keyboard_yn
								if [[ $Keyboard_yn == "y" ]]; then
									echo " "
									Custom_Drives_List=""
									echo "Data Deleted"
								fi
							else
								echo "No Custom Drive Configuration Exists."
							fi
							echo " "
							echo "Would you like to continue to customize (y/n)?"
							echo "Anything other than (y) will exit this section."
							read -s -n 1 Keyboard_exit
							echo " "
							if [[ $Keyboard_exit != "y" ]]; then
								clear
								echo "If you deleted the custom configuration data,"
								echo "Make sure you write your changes."
								echo " "
								echo "Press any key to continue"
								read -s -n 1 key 
								continue
							fi
							clear
							echo "Custom Drive Configuration Mode"
							echo " "
							echo "Collecting Individual Drive Data..."
							echo " "
							for drive in $smartdrivesall; do
								clear_variables
								get_drive_data
								echo "Drive ID: "$drive
								echo "Drive Serial Number: "$serial
								echo " "

								# Check to see if the drive is listed in the Custom_Drives_List file, if yes then list the alarm setpoints.
								# If the drive is not listed then ask to add it to the list.  Next list the set/default setpoint values.
								if [[ "$(echo $Custom_Drives_List | grep $serial)" ]]; then
									echo "The drive serial number "$serial" is already listed as Custom."
									echo " "
									echo "Options are: Delete the entry for this drive, then you may"
									echo "re-add and edit it immediately."
									IFS=',' read -ra ADDR <<< "$Custom_Drives_List"
									for i in "${ADDR[@]}"; do
										cdrivesn1="$(echo $i | cut -d':' -f 1)"
										if [[ $cdrivesn1 == $serial ]]; then
											tempWarnx="$(echo $i | cut -d':' -f 2)"
											tempCritx="$(echo $i | cut -d':' -f 3)"
											SectorsWarnx="$(echo $i | cut -d':' -f 4)"
											SectorsCritx="$(echo $i | cut -d':' -f 5)"
											ReAllocWarnx="$(echo $i | cut -d':' -f 6)"
											MultiZoneWarnx="$(echo $i | cut -d':' -f 7)"
											MultiZoneCritx="$(echo $i | cut -d':' -f 8)"
											RawReadWarnx="$(echo $i | cut -d':' -f 9)"
											RawReadCritx="$(echo $i | cut -d':' -f 10)"
											SeekErrorsWarnx="$(echo $i | cut -d':' -f 11)"
											SeekErrorsCritx="$(echo $i | cut -d':' -f 12)"
											TestWarnAgex="$(echo $i | cut -d':' -f 13)"
											testAgeOvrd="$(echo $i | cut -d':' -f 14)"
											HeliumMinx="$(echo $i | cut -d':' -f 15)"
											wearLevelAdj="$(echo $i | cut -d":" -f 16)"
										fi
									done
									echo " "
									echo "The current alarm setpoints are:"
									echo "Temperature Warning=("$tempWarnx")  Temperature Critical=("$tempCritx")"
									echo "Sectors Warning=("$SectorsWarnx")  Sectors Critical=("$SectorsCritx")  ReAllocated Sectors Warning=("$ReAllocWarnx")"
									echo "MultiZone Warning=("$MultiZoneWarnx")  MultiZone Critical=("$MultiZoneCritx")"
									echo "Raw Read Error Rate Warning=("$RawReadWarnx")  Raw Read Error Rate Critical=("$RawReadCritx")"
									echo "Seek Error Rate Warning=("$SeekErrorsWarnx")  Seek Error Rate Critical=("$SeekErrorsCritx")"
									echo "Test Age=("$TestWarnAgex")  Ignore Test Age=("$testAgeOvrd")"
									echo "Helium Minimum Level=("$HeliumMinx")"
									echo "Wear Level Adjustment=("$wearLevelAdj")"
									echo " "
									echo "  ('d' System default, 'r' Reverse, 'n' Normalized, 'i' Ignore)"
									echo " "
									echo "Do you want to delete this drive from the custom configuration (y/n)?"
									read -s -n 1 Keyboard_yn
									if [[ $Keyboard_yn == "y" ]]; then
										echo "Custom Drive Configuration for "$serial" -- Deleting."
										echo " "
										echo " "														    
										### Roll through the Custom_Drives_List data until a match for $serial occurs, then copy all but that data back.
										tempstring=""
										for (( i=1; i<=32; i++ )); do
											tempvar="$(echo $Custom_Drives_List | cut -d',' -f $i)"
											if [[ ! $tempvar == "" ]]; then
												tempsn="$(echo $tempvar | cut -d":" -f 1)"
												if [[ $tempsn == $serial ]]; then
													bogusvalue=1
												else
													if [[ $tempstring == "" ]]; then
														tempstring=$tempvar																												
													else
														tempstring=$tempstring","$tempvar
													fi
												fi
											fi
										done
										Custom_Drives_List=$tempstring
										sleep 1
										clear
									fi
								fi

								# Lets assign the local variables with the default values.  They will be changed
								# later if the drive is in the Custom_Drives_List variable.

								sectorswarn=$SectorsWarn
								sectorscrit=$SectorsCrit
								reallocwarn=$ReAllocWarn
								multizonewarn=$MultiZoneWarn
								multizonecrit=$MultiZoneCrit
								rawreadwarn=$RawReadWarn
								rawreadcrit=$RawReadCrit
								seekerrorswarn=$SeekErrorsWarn
								seekerrorscrit=$SeekErrorsCrit
								testage=$TestWarnAge
								testAgeOvrd="0"
								heliummin=$HeliumMin
								wearleveladj="d"

								if [[ ! "$(echo $Custom_Drives_List | grep $serial)" ]]; then
									echo "The drive "$serial" is not in the Custom Drive Config database."
									echo " "
									echo "Displaying Default Values"
									if [[ $Custom_Drives_ListDrive == "HDD" ]]; then
										tempwarn=$HDDtempWarn; tempcrit=$HDDtempCrit
									fi
									if [[ $Custom_Drives_ListDrive == "SSD" ]]; then
										tempwarn=$SSDtempWarn; tempcrit=$SSDtempCrit
									fi
									if [[ $Custom_Drives_ListDrive == "NVM" ]]; then
										tempwarn=$NVMtempWarn; tempcrit=$NVMtempCrit
									fi
									echo " "
									echo "The current alarm setpoints are:"
									echo "Temperature Warning=("$tempwarn")  Temperature Critical=("$tempcrit")"
									echo "Sectors Warning=("$sectorswarn")  Sectors Critical=("$sectorscrit")  ReAllocated Sectors Warning=("$reallocwarn")"
									echo "MultiZone Warning=("$multizonewarn")  MultiZone Critical=("$multizonecrit")"
									echo "Raw Read Error Rate Warning=("$rawreadwarn")  Raw Read Error Rate Critical=("$rawreadcrit")"
									echo "Seek Error Rate Warning=("$seekerrorswarn")  Seek Error Rate Critical=("$seekerrorscrit")"
									echo "Test Age=("$testage")  Ignore Test Age=("$testAgeOvrd")"
									echo "Helium Minimum Level=("$heliummin")"
									echo "Wear Level Adjustment=("$wearleveladj")"
									echo " "
									echo "  ('d' System default, 'r' Reverse, 'n' Normalized, 'i' Ignore)"
									echo " "
									echo "Would you like to customize an Alarm Setpoint for this drive?"
									echo "Return accepts current setting, 'y' to modify. "
									read -s -n 1 Keyboard_yn

									if [[ $Keyboard_yn == "y" ]]; then
										echo " "
										echo "Let's modify some values..."
										echo " "
										echo "You have the following two options..."
										echo ' Return to accept the "system default" value, or'
										echo " Enter a numeric value to override the system default value."
										echo " "
								  		echo "Temperature Warning=("$tempwarn") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											tempwarn="d"
										else
											tempwarn=$Keyboard_yn
										fi

										echo "Temperature Critical=("$tempcrit") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											tempcrit="d"
										else
											tempcrit=$Keyboard_yn
										fi

										echo "Sectors Warning=("$sectorswarn") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											sectorswarn="d"
										else
											sectorswarn=$Keyboard_yn
										fi
										echo "Sectors Critical=("$sectorscrit") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											sectorscrit="d"
										else
											sectorscrit=$Keyboard_yn
										fi

										echo "Reallocated Sectors Warning=("$reallocwarn") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											reallocwarn="d"
										else
											reallocwarn=$Keyboard_yn
										fi

										echo "MultiZone Warning=("$multizonewarn") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											multizonewarn="d"
										else
											multizonewarn=$Keyboard_yn
										fi
										echo "MultiZone Critical=("$multizonecrit") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											multizonecrit="d"
										else
											multizonecrit=$Keyboard_yn
										fi

										echo "Raw Read Rate Warning=("$rawreadwarn") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											rawreadwarn="d"
										else
											rawreadwarn=$Keyboard_yn
										fi

										echo "Raw Read Rate Critical=("$rawreadcrit") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											rawreadcrit="d"
										else
											rawreadcrit=$Keyboard_yn
										fi
										echo "Seek Errors Warning=("$seekerrorswarn") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											seekerrorswarn="d"
										else
											seekerrorswarn=$Keyboard_yn
										fi

										echo "Seek Errors Critical=("$seekerrorscrit") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											seekerrorscrit="d"
										else
											seekerrorscrit=$Keyboard_yn
										fi

										echo "Test Age=("$testage") "
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											testage="d"
										else
											testage=$Keyboard_yn
										fi

										echo "Ignore Test Age=("$testAgeOvrd") (0=No, 1=Yes)"
										read Keyboard_yn
										if [[ $Keyboard_yn != $testAgeOvrd && $Keyboard_yn != "" ]]; then
											testAgeOvrd=$Keyboard_yn
										fi
										echo "Helium Warning Level=("$heliummin") "
										echo "NOTE: Helium may not be used by this drive so enter the default."
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											heliummin="d"
										else
											heliummin=$Keyboard_yn
										fi
										echo "Wear Level Adjustment=("$wearleveladj") "
										echo "NOTE: Wear Level Adjustment is set to: 'd' for Default, 'r' for Reverse value,"
										echo "      'n' for Normalized, or 'i' for Ignore."
										read Keyboard_yn
										if [[ $Keyboard_yn == "" ]]; then
											wearleveladj="d"
										else
											wearleveladj=$Keyboard_yn
										fi
										if [[ $wearleveladj == "d" ]] || [[ $wearleveladj == "r" ]] || [[ $wearleveladj == "i" ]] || [[ $wearleveladj == "n" ]] || [[ $wearleveladj == "" ]]; then
											if [[ $Keyboard_yn == "" ]]; then
												wearleveladj="d"
											else
												wearleveladj=$Keyboard_yn
											fi
										else   
											echo "ERROR, Incorrect Value!  Must be 'Enter/Return', 'd', 'r', 'i', or 'n'"
											echo -n "Enter new value :"
											read Keyboard_yn
											if [[ $Keyboard_yn == "" ]]; then
												wearleveladj="d"
											else
												wearleveladj=$Keyboard_yn
											fi											     
										fi 
										echo " "
										echo "The current alarm setpoints are ('d' = system default):"
										echo "Temperature Warning=("$tempwarn")  Temperature Critical=("$tempcrit")"
										echo "Sectors Warning=("$sectorswarn")  Sectors Critical=("$sectorscrit")  ReAllocated Sectors Warning=("$reallocwarn")"
										echo "MultiZone Warning=("$multizonewarn")  MultiZone Critical=("$multizonecrit")"
										echo "Raw Read Error Rate Warning=("$rawreadwarn")  Raw Read Error Rate Critical=("$rawreadcrit")"
										echo "Seek Error Rate Warning=("$seekerrorswarn")  Seek Error Rate Critical=("$seekerrorscrit")"
										echo "Test Age=("$testage")  Ignore Test Age=("$testAgeOvrd")"
										echo "Helium Minimum Level=("$heliummin")"
										echo "Wear Level Adjustment=("$wearleveladj")"
										echo " "
										echo "Adding "$drive" Serial Number: "$serial" to the custom configuration"
										echo "variable."
										echo " "
										# Add all the current values for this $serial to the Custom_Drives_List variable.
		   
										if [[ $Custom_Drives_List == "" ]]; then
											Custom_Drives_List=$serial":"$tempwarn":"$tempcrit":"$sectorswarn":"$sectorscrit":"$reallocwarn":"$multizonewarn":"$multizonecrit":"$rawreadwarn":"$rawreadcrit":"$seekerrorswarn":"$seekerrorscrit":"$testage":"$testAgeOvrd":"$heliummin":"$wearleveladj
										else
											Custom_Drives_List=$Custom_Drives_List","$serial":"$tempwarn":"$tempcrit":"$sectorswarn":"$sectorscrit":"$reallocwarn":"$multizonewarn":"$multizonecrit":"$rawreadwarn":"$rawreadcrit":"$seekerrorswarn":"$seekerrorscrit":"$testage":"$testAgeOvrd":"$heliummin":"$wearleveladj
										fi
										echo " "
									else
										echo "Drive skipped."
									fi
								fi
								echo " "
								echo "Press any key to continue"
								read -s -n 1 key
								clear
							done
							echo "Make sure you write your changes."
							echo "Press any key to continue"
							read -s -n 1 key       
							sleep .5
						;;

						U)
							clear
							echo "Update Script - Automatic or Manual Internet (Github) Updates"
							echo " "
							echo "Automatic Script Update"
							echo " "
							echo "WARNING: Fully Automatic Updates takes you out of control."
							echo "Use at your own peral.  I personnaly do not recommend this"
							echo "option but it was requested so I created it."
							echo " "
							echo "You may select normal Manual Updates or Fuly Automatic Updates."
							echo " "
							echo "Manual Updates requires the user to run the script using the"
							echo "'-update' switch and answering a question to update or not."
							echo " "
							echo "Automatic Updates is a fully automatic update and requires"
							echo "no user interaction.  This happens completely automatically"
							echo "and if Github has an update on it, it will be downloaded"
							echo "and installed immediately."
							echo " "
							if [[ $AutomaticUpdate != "true" ]]; then
								echo "You are currently using Manual Updates (system default)."
							else
								echo "You are curently using Fully Automatic Updates."
							fi
							echo " "
							echo "Press 'a' for Automatic, or 'm' for Manual Updates,"
							echo "or any other key to keep the current setting."
							read -s -n 1 Keyboard_yn
							if [[ $Keyboard_yn == "" ]]; then
								echo "Keeping the current setting"
							elif [[ $Keyboard_yn == "a" ]]; then 
								echo "You selected Fully Automatic Updates."
								AutomaticUpdate="true"
							elif [[ $Keyboard_yn == "m" ]]; then
								echo "You selected the safer Manual Updates."
								AutomaticUpdate="false"
							fi
							echo " "
							echo "AutomaticUpdate ("$AutomaticUpdate")"
							echo " "
							echo "Make sure you write your changes."
							echo "Press any key to continue"
							read -s -n 1 key       
						;;

						W)
							echo " "
							echo "Writing Configuration File"
							echo " "
							echo " "
							sleep 1
							update_config_file
							echo "File updated."
							echo " "
							x=100
							sleep 1
						;;

						X)
							echo "Exiting, Not Saving"
							sleep 1
							x=100
						;;


						*)
							echo "Invalid Option"
							sleep 2
						;;
					esac
					done
					;;
					# Advanced Configuration Level End

				H)
					clear
					echo "HOW TO USE THIS CONFIGURATION TOOL"
					echo " "
					echo "This tool has many options and you should be able to perform a complete"
					echo "configuration using this tool."
					echo " "
					echo "In order to use the advanced options you will need to have created an external"
					echo "configuration file then the tool will be able to read and write to this file."
					echo " "
					echo "Throughout this process you will be asked questions that require different"
					echo "responses:"
					echo " "
					echo " 1) String content: Where you will either enter a new string followed by the"
					echo "    Enter/Return key, or just press Enter/Return to accept the current value."
					echo " "
					echo " 2) Numeric content: Where you will either enter a new number followed by the"
					echo "    Enter/Return key, or just press Enter/Return to accept the current value."
					echo " "
					echo " 3) True/False content: Where you will either enter 't' or 'f' followed by the"
					echo "    Enter/Return key, or just press Enter/Return to accept the current value."
					echo " "
					echo " 4) Enable/Disable content: Where you will either enter 'e' or 'd' followed by the"
					echo "    Enter/Return key, or just press Enter/Return to accept the current value."
					echo " "
					echo " 5) Other possible options: 'd' = delete or default, 'r' = reverse, 'i' = ignore,"
					echo "   'n' = normalized, 'e' to edit."
					echo " "
					echo " "
					echo -n "Press any key to continue"
					read -s -n 1 key
					clear
					echo "HOW TO USE THIS CONFIGURATION TOOL"
					echo " "
					echo "Just to re-iterate: Press the Enter/Return key to accept the current value."
					echo "Press 't' or 'f' to change to 'true' or 'false'.  Enter a number or string"
					echo "followed by the Enter/Return key to change a value."
					echo " "
					echo "For a listing of switches, run the program with the '-h' switch."
					echo "For more detailed Help information, run the program with the '-help' switch."
					echo "And do not forget to read the User Guide."
					echo " "
					echo -n "Press any key to continue"
					read -s -n 1 key
					echo " "
					sleep 1
				;;


				N) 
					clear
					echo "Creating a new configuration file.  This will overwrite an existing file."
					echo " "
					echo "Enter your email address to send the report to, or press Return to abort"
					echo -n ": "
					read Keyboard_Email
					if [[ $Keyboard_Email == "" ]]; then
						echo "Aborting"
						sleep 2
						continue
					fi
					if [[ ! $Keyboard_Email == "" ]]; then Email=$Keyboard_Email; fi
					echo "Set Value: "$Email
					echo " "
					echo 'Enter the email for Drive temperature monitoring (used with -m switch only).'
					echo "to send an alert to."
					echo " "
					echo -n "Press Return to use your normal email address:"
					read Keyboard_Email
					if [[ $Keyboard_Email == "" ]]; then
						echo "Using normal Email address..."
						AlertEmail=$Email
					fi
					if [[ ! $Keyboard_Email == "" ]]; then AlertEmail=$Keyboard_Email; fi
					echo "Set Value: "$AlertEmail
					echo " "
					echo "Current from address: "$From" "
					echo 'While most people are able to use the default "From" address, some email'
					echo 'servers will not work unless you use the email address that the email'
					echo 'server is associated with.'
					echo -n "Enter your from Email address: "
					read Keyboard_Email
					if [[ ! $Keyboard_Email == "" ]]; then From=$Keyboard_Email; fi
					echo "Set Value: "$From
					echo " "
					echo "Enter path and name of statistics file or just hit Enter to use the"
					echo 'default (recommended): '
					echo 'Default is '$SCRIPT_DIR'/statistical_data_file.csv'
					read Keyboard_statistics
					if [[ $Keyboard_statistics == "" ]]; then echo "Default Selected"; Keyboard_statistics="$SCRIPT_DIR/statisticalsmartdata.csv"; fi
					echo "Set Value: "$Keyboard_statistics
					echo " "
					echo "Automatic Drive Offsets will effectively zero out the following values"
					echo "if they have any current errors - UDMA CRC Errors, MultiZone, and"
					echo "Reallocated Sectors. This is useful for tracking additional errors."
					echo " "
					echo 'Would you like to scan the drives and setup these offsets (y/n): '
					read -s -n 1 Keyboard_yn
					if [[ $Keyboard_yn == "y" ]]; then
						echo " "
						echo "Collecting data, Please wait..."
						get_smartHDD_listings
						get_smartSSD_listings
						get_smartNVM_listings
						smartdrivesall="$smartdrives $smartdrivesSSD $smartdrivesNVM"
						echo " "
						echo "AUTOMATIC DRIVE COMPENSATION - UDMA_CRC, MultiZone, and Reallocated Sectors"
						echo " "
						for drive in $smartdrivesall; do
							clear_variables
							get_drive_data
							if [[ ! $crcErrors == "0" ]] && [[ ! $crcErrors == "" ]]; then listofdrivescrc="$listofdrivescrc$serial":"$crcErrors,"; fi
							if [[ ! $multiZone == "0" ]] && [[ ! $multiZone == "" ]]; then listofdrivesmulti="$listofdrivesmulti$serial":"$multiZone,"; fi
							if [[ ! $reAlloc == "0" ]] && [[ ! $reAlloc == "" ]]; then listofdrivesbad="$listofdrivesbad$serial":"$reAlloc,"; fi
							if [[ ! $reAllocEvent == "0" ]] && [[ ! $reAllocEvent == "" ]]; then listofdrivesbad2="$listofdrivesbad2$serial":"$reAllocEvent,"; fi
						done
						echo "Scanning Results:"
						if [[ ! $listofdrivescrc == "" ]]; then CRC_Errors_List="$(echo "$listofdrivescrc" | sed 's/.$//')"; echo "UDMA_CRC Errors detected"; else CRC_Errors_List=""; echo "No UDMA_CRC Errors"; fi
						if [[ ! $listofdrivesmulti == "" ]]; then MultiZone_List="$(echo "$listofdrivesmulti" | sed 's/.$//')"; echo "MultiZone Errors Detected"; else MultiZone_List=""; echo "No MultiZone Errors"; fi
						if [[ ! $listofdrivesbad == "" ]]; then ReAllocated_Sector_List="$(echo "$listofdrivesbad" | sed 's/.$//')"; echo "Bad Sectors Detected"; else ReAllocated_Sector_List=""; echo "No Reallocated Sectors"; fi
						if [[ ! $listofdrivesbad2 == "" ]]; then ReAllocated_Sector_Events_List="$(echo "$listofdrivesbad2" | sed 's/.$//')"; echo "Bad Sectors Detected"; else ReAllocated_Sector_Events_List=""; echo "No Reallocated Sectors"; fi
						echo " "
						echo "Values Set:"
						echo "CRC_Errors_List: "$CRC_Errors_List
						echo "MultiZone_List_Errors: "$MultiZone_List
						echo "Reallocated_Sectors: "$ReAllocated_Sector_List
						echo "Reallocated_Sectors_Events: "$ReAllocated_Sector_Events_List
						echo " "
					fi
					#The default values at the beginning of the script are written.
					echo "Creating the new file..."
					update_config_file
					echo " "
					echo "Success! --- New clean configuration written."
					echo " "
					echo "Path and Name if the configuration file: "$Config_File_Name
					echo " "
					echo "If you desire more customization, rerun the -config and select Advanced options."
					echo " "
					exit 0
				;;

				S)
					clear
					load_config
					echo "Spencer Add-On Script by NickF"
					echo " "
					echo "Spencer Enabled is currently: "$spencer_enable
					echo 'Do you want to change this (y/n or Enter/Return to remain the same)'
					read -s -n 1 Keyboard_yn
					if [[ $Keyboard_yn == "y" ]]; then
						if [[ $spencer_enable == "true" ]]; then spencer_enable="false"; else spencer_enable="true"; fi
					fi
					echo "New Value: "$spencer_enable

					echo " "
					echo "It is recommended that spencer.py be located in the same directory as the multi_report.sh script."
					echo "You may change the path and file name if desired."
					echo " "
					echo "Current path and name: "$spencer_script_name
					echo "Enter a new path and name, Enter/Return to keep current value, or 'd' to reset to the default."
					read Keyboard_yn
					if [[ $Keyboard_yn != "" ]]; then
						if [[ $Keyboard_yn == "d" ]]; then
							spencer_script_name="$SCRIPT_DIR/spencer.py"
						else
							spencer_script_name=$Keyboard_yn
						fi
						echo "New Spencer script full path is: "$spencer_script_name
					else
						echo "Location not changed"
					fi

					if [ ! -e $spencer_script_name ]; then
						echo " "
						echo "Spencer is not found!"
						echo "Please ensure you install Spencer in the path you provided or reset to default."
						sleep 2
					fi
					echo " "
					echo "Set Warning Levels"
					echo 'New "Not previously reported" Errors Warning Level'
					echo " "
					echo "Valid Error Levels"
					echo "1) None = No Email Subject Error Message"
					echo "2) Warning = New Errors are reported as a Warning Message (default)"
					echo "3) Critical = New Errors are reported as a Critical Message"
					echo " "
					echo "Current Value: "$spencer_new_warning_level
					echo "Enter new level or Enter/Return to leave unchanged (1/2/3)"
					read -n 1 Keyboard_yn
					if [[ $Keyboard_yn == "1" ]]; then spencer_new_warning_level="None"
					elif [[ $Keyboard_yn == "2" ]]; then spencer_new_warning_level="Warning"
					elif [[ $Keyboard_yn == "3" ]]; then spencer_new_warning_level="Critical"
					elif [[ $Keyboard_yn == "" ]]; then a=$a
					else
						echo "Invalid Entry"
						sleep 3
					fi
					echo "New Value: "$spencer_new_warning_level
					echo " "
					echo 'Existing "Previously reported" Errors Warning Level'
					echo " "
					echo "Valid Error Levels"
					echo "1) None = No Email Subject Error Message (default)"
					echo "2) Warning = Existing Errors are reported as a Warning Message"
					echo "3) Critical = Existing Errors are reported as a Critical Message"
					echo " "
					echo "Current Value: "$spencer_existing_warning_level
					echo "Enter new level or Enter/Return to leave unchanged (1/2/3)"
					read -n 1 Keyboard_yn
					if [[ $Keyboard_yn == "1" ]]; then spencer_existing_warning_level="None"
					elif [[ $Keyboard_yn == "2" ]]; then spencer_existing_warning_level="Warning"
					elif [[ $Keyboard_yn == "3" ]]; then spencer_existing_warning_level="Critical"
					elif [[ $Keyboard_yn == "" ]]; then a=$a
					else
						echo "Invalid Entry"
						sleep 3
					fi
						echo "New Value: "$spencer_existing_warning_level
					echo " "
					echo "Write the changes for Spencer (y/n)?"
					read -s -n 1 Keyboard_yn
					if [[ $Keyboard_yn == "y" ]]; then
						echo "Writing File"
						update_config_file					
					else
						echo "Aborting"
					fi
					sleep 3					
				;;

				X)
					echo " "
					echo " "
					echo "Exiting..."
					echo " "
					echo " "
					z=50
					exit 1
				;;

				*)
					echo " "
					echo " "
					echo "GREETINGS PROFESSOR FALKEN."
					echo "SHALE WE PLAY A GAME?"
					echo " "
					echo " "
					sleep 6
				;;
			# End First Level
		esac
	done
	shopt -u nocasematch
	}

########## HELP INSTRUCTIONS ##########

display_help () {
	clear
	echo "NAME"
	echo "      Multi Report - System status reporting for TrueNAS Core and Scale"
	echo " "
	echo "SYNOPSIS"
	echo "      multi_report.sh [options]"
	echo " "
	echo "COPYRIGHT AND LICENSE"
	echo "      Multi Report is Copyright (C) by its authors and licensed under the"
	echo "      GNU General Public License v3.0"
	echo " "
	echo "DESCRIPTION"
	echo "      Multi Report generates an email containing a summary chart of your"
	echo "      media and their health. Directly after the chart is a Text Section which"
	echo "      may immediately contain failure information followed by Zpool data,"
	echo "      key SMART data for each drive, and raw data for drives that do not report"
	echo "      SMART.  In addition if configured, statistical data is collected for"
	echo "      long-term monitoring. This script currently runs on both CORE (FreeBSD)"
	echo "      and SCALE (Debian Linux) versions."
	echo " "
	echo "OPTIONS"
	echo "      -help         This message."
	echo " "
	echo "      -h            List the most common options."
	echo " "
	echo "      -s [-m]       Record drive statistics only, do not generate a"
	echo '                    corresponding email, unless used with "-m"'
	echo " "
	echo "      -config       Generate or edit a configuration file in the directory the"
	echo "                    script is run from."
	echo "      -delete       Deletes the statistical data file if the file exists."
	echo " "
	echo "      -dump [all]         Generates an email with attachments of all drive data"
	echo "            [email]       and the multi_report_config.txt additionally it also"
	echo "            [emailextra]  suppress the config_backup file and statistics file"
	echo "                          from being attached to the email unless you use"
	echo "                    the [all] option, then the config_backup and statistics files"
	echo "                    will be appended. The [email] option runs the normal -dump"
	echo "                    command but also will send the email to"
	echo "                    joeschmuck2023@hotmail.com for further analysis or just to"
	echo "                    provide drive data information to Joe to help make improvments"
	echo "                    to the script.  If you use the [-dump email] option, you will"
	echo "                    be asked to confirm sending of the email. So you cannot run"
	echo "                    this parameter from a script, it must be CLI.  You will also"
	echo "                    be asked to enter a comment to aid Joe in your problem."
	echo "                    [-dump emailextra] option is the same as '-dump email'"
	echo "                    except it also includes 'smartctl -a' and smartctl -x' text file"
	echo "                    data for each drive.  This should normally not be required."
	echo " "
	echo "      -m [-s]       Monitor Drive Temperature Alarms. This will generate"
	echo '                    an email to "AlertEmail" that contains a'
	echo "                    notification when a drive reaches the Warning Temp Limit."
	echo "                    Great for Smartphone Text Messages. By default it will not"
	echo '                    write to the Statistical Data File, appending "-s"'
	echo "                    will update the statistics as well."
	echo " "
	echo "      -purge	This will purge all test data from the statistical data file."
	echo " "
	echo "      -t [path] [-dump]	Use strickly for test files (.json format)"
	echo "	   			This is strickly for development."
	echo " "
	echo " 	    -u7zip	This will remove 7-Zip from Scale if it was installed."
	echo " "
	echo "      -update	Manually check GitHub for an update."
	echo " "
	echo "      -ignore_lock  This will ignore multiple instances running."
	echo "			  This option should be last on the command line."
	echo " "
	echo "      Running the script without any switches will collect statistical data"
	echo "      and generate an email report."
	echo " "
	echo "CONFIGURATION"
	echo "      The script has become quite complex over time and with added features"
	echo "      ultimately required an external configuration file with version 1.6c"
	echo "      to simplify upgrades to the end user."
	echo " "
	echo "      If the external configuration file does not exist, the script will use"
	echo "      the values hard code into the script (just like versions 1.6b and"
	echo "      earlier), however there is now an email address check to ensure you have"
	echo "      changed the email address within the script."
	echo " "
	echo "      In order to generate an external configuration file you must use the"
	echo "      [-config] parameter when running the script which is the preferred"
	echo "      method to configure your script.  Five options will be available:"
	echo " "
	echo "          N)ew configuration file"
	echo "          A)dvanced configuration"
	echo "          S)pencer Integration"
	echo "          H)ow to use this configuration tool"
	echo "          X) Exit"
	echo " "
	echo "      N)ew configuration file will create/overwrite a new configuration file."
	echo "      This is the minimal setup before running the script without parameters."
	echo "      The configuration file will be created in the directory the script is"
	echo "      located in using default parameters."
	echo " "
	echo "      The default settings should be sufficient to test this script."
	echo " "
	echo "      A)dvanced configuration allows you to customize the Multi-Report script,"
	echo "      changing the default parameters as desired.  Before exiting the"
	echo "      advanced configuration screen, ensure you Write and changes you wish"
	echo "      to retain.  Not writing will abort the changes."
	echo " "
	echo "      S)pencer Integrations allows you to enable/disable the Spencer external"
	echo "      script and adjust some minimal settings.  Out of the box, Spencer is"
	echo "      enabled if the script is present."
	echo " "
	echo "STATISTICAL DATA"
	echo "      Besides the emailed chart, the script can also email you attachments with"
	echo "      your FreeNAS/TrueNAS configuration file and a Statistical Data file."
	echo " "
	echo "      The statistical data file is a comma delimited collection of drive data"
	echo "      that can be opened in any spreadsheet program such as Microsoft Excel."
	echo "      This data could prove to be useful in diagnosing and troubleshooting"
	echo "      drive or system problems."
	echo " "
	echo "EMAIL CONTENT"
	echo "      Normal operation of this script will produce an output email and it may"
	echo "      or may not have attachments per your configuration.  The email will"
	echo "      contain the following information:"
	echo " "
	echo "      - Subject with Summary Result of 'All is Good', '*WARNING*', or"
	echo "        '*CRITICAL ERROR*'"  
	echo "        The summary result is based on your settings of the warning and critical"
	echo "        settings, where:"
	echo "        All is Good = No Alarm indications."
	echo "        *WARNING* = A warning threshold has been crossed and it means you should"
	echo "           investigate and take action."
	echo "        *CRITICAL* = Something significant has occurred and your data could be"
	echo "           at risk, take immediate action."
	echo " "
	echo "      - The version of the script and the version of TrueNAS you are running."
	echo "      - The date and time the script was run and how long it took to generate."
	echo "      - Zpool Status Report Summary: (Pool Name/Status/Size/Errors/Scrub Info)"
	echo "      - HDD Summary Report Chart: (Drive ID/Serial Number/other data)"
	echo "      - SSD Summary Report Chart: (Same as the HDD report)"
	echo "      - NVMe Summary Report Chart: (Same as the SSD report)"
	echo "      - Text Section:  The Text section contains the text version of most of"
	echo "        the previously displayed data.  It will tell you:"
	echo "      -- if using an external configuration file"
	echo "      -- if saving statistical information"
	echo "      -- if a drive warranty has expired"
	echo "      -- Zpool native report - in which the gptid's are listed '*followed by a" 
	echo "         listing of the drives that make up the pool and the drives are listed"
	echo "         in the order the gptid numbers are listed.*'"
	echo "      -- Drive relevant SMART data followed by if TLER/SCT is enabled/disabled"
	echo "         or available."
#	echo "      -- NON-SMART data will be listed for drives that do not support SMART."
	echo " "
	echo "      While all this data is nice to have, there is no substitute for having"
	echo "      due diligence in examining your hardware and ensuring it is working"
	echo "      correctly.  This means you may need to examine your SMART data closer."
	echo " "
	echo "USAGE"
	echo "      This script was designed to be run from the CRON service, generally once"
	echo "      a day in order to produce an email output to notify the user of any"
	echo "      problems or trends. To identify trends the script also collects"
	echo "      statistical data for analysis."
	echo " "
	echo "      A good starting point is to set up a CRON job to run this script once a"
	echo "      day at e.g. 2:00AM using no switches. This will produce an email snapshot"
	echo "      once a day."
	echo " "
	echo "      In addition if you are trying to troubleshoot drive problems where SMART"
	echo "      data would benefit you, I would recommend you setup an additional CRON"
	echo "      job using the [-s] switch for collecting statistical data only (ie. no"
	echo "      email report). This statistics cron job should run more frequently."
	echo "      The corresponding cron job should be scheduled to not overlap with the"
	echo "      daily report email."
	echo "      --If you sleep your drives then this option may not be desirable.--"
	echo " "
	echo "CUSTOMIZATIONS AND FEATURES"
	echo "      There are quite a few built in features in the external configuration file"
	echo "      and these are a few:"
	echo " "
	echo "      Custom Chart Titles: Change the name of any or all headers and columns."
	echo " "
	echo "      Selectable Columns: Display or remove as many or few columns of data as"
	echo "          desired. This is great for removing a column with no relevant data."
	echo " "
	echo "      Alarm Setpoints: Practically everything has an alarm setpoint, from pool"
	echo "          capacity to Scrub Age, to temperature Warnings and Critical Warnings,"
	echo "          and a plethora of options."
	echo " "
	echo "      Custom Drive Configuration: Allows unique customizing of alarms where"
	echo "          the general settings would not work properly for a drive,"
	echo "          including inverting the Wear Level value for SSD/NVM drives."
	echo " "
	echo "      TLER: You can monitor and even have TLER automatically set if required,"
	echo "          for drives which support it. The default is to not automatically set"
	echo "          TLER on, I believe the user should make that decision."
	echo " "
	echo "      Statistical Data: Modify the location of your data, if you want it emailed"
	echo "          and when to email, and we include an automatic purge to ensure the"
	echo "          data file doesn't get too large (default 2 years)."
	echo " "
	echo "      UDMA CRC Error Corrections: Have you ever had a hard drive UDMA_CRC_Error?"
	echo "          Well they often are caused by a poor/loose data cable but the error"
	echo "          will be recorded forever in the drive electronics. This option lets"
	echo "          you zero out the value in the script. This feature is usable for"
	echo "          Bad Sectors and Multi Zone Errors as well."
	echo " "
	echo "      Ignore Drive: Every wished you could just ignore a USB Flash Drive or any"
	echo "          drive just giving you problems? With this feature the drive will be"
	echo "          completely ignored in the script."
	echo " "
	echo "      Warranty Expiring Warning: You can configure the configuration file to"
	echo "          provide a warning message for a drive on a certain date.  This a great"
	echo "          tool to keep track on when it might be time to consider buying some"
	echo "          replacement drives."
	echo " "
	echo "      Custom Colors: Use the HEX color codes to change the color scheme of the"
	echo "          background and alerting colors on the charts.  This is ONLY"
	echo "          changeable manually, you have to edit the config file."
	echo " "
	echo "      NVMe drive SMART Self-test (for TrueNAS which does not support"
	echo "          smartmontools v7.4 and/or TrueNAS GUI does not support configuring"
	echo "          NVMe drives for automatic SMART Self-tests."
	echo " "
	echo "      It is very important that if you edit the configuration file or the script"
	echo "      that you need to maintain the proper formatting of the text or you will"
	echo "      throw a wrench into things."
	echo " "
	echo "HOW TO HANDLE ERRORS"
	echo "      If you run across errors running the script, odd are it is because a drive"
	echo "      was not recognized properly.  I recommend you post your error to the forum"
	echo "      to as for assistance.  It is possible you will be asked for a [-dump]"
	echo "      of your data in order to let the developer assist you and correct the"
	echo "      problem.  There are varying levels of dump data (indicated above)."
	echo " "
	echo "      Please note that a [-dump] file is not the same as a cut/paste of a"
	echo "      terminal/SSH window.  Critical formatting data is lost that is required."
	echo " "
	echo "Recommendation:  When troubleshooting a problem you may be asked to provide"
	echo "dump data to assist troubleshooting. Use the [-dump email] or [-dump emailextra]"
	echo "to include all relevant data (drive data and configuration file) which sends an"
	echo "email to joeschmuck2023@hotmail.com, your email address will not be shared!."
	echo "If you prefer you can use [-dump] or [-dump all] and then open up a dialog"
	echo "with Joe Schmuck and attach the files in a message on the TrueNAS forum."
	echo " "
	}

########## HELP COMMANDS ##########

display_help_commands () {
	clear
	echo "NAME"
	echo "      Multi Report - System status reporting for TrueNAS Core and Scale"
	echo " "
	echo "SYNOPSIS"
	echo "      multi_report.sh [options]"
	echo " "
	echo "COPYRIGHT AND LICENSE"
	echo "      Multi Report is Copyright (C) by its authors and licensed under the"
	echo "      GNU General Public License v3.0"
	echo " "
	echo "COMMON OPTIONS"
	echo "      -h            This message."
	echo " "
	echo "      -help         Full Help message."
	echo " "
	echo "      -s [-m]       Record drive statistics only, do not generate a"
	echo '                    corresponding email, unless used with "-m".'
	echo " "
	echo "      -m [-s]       Monitor Drive Temperature Alarms, generate special email"
	echo "                    notification when a drive reaches the Warning Temp Limit."
	echo " "
	echo "      -config       Generate or edit a configuration file in the directory the"
	echo "                    script is run from."
	echo " "
	echo "      -dump [all]         Generates an email with attachments of all drive data"
	echo "            [email]       and the multi_report_config.txt additionally it also"
	echo "            [emailextra]  suppress the config_backup file and statistics file"
	echo "                          from being attached to the email unless you use"
	echo "                    the [all] option, then the config_backup and statistics files"
	echo "                    will be appended. The [email] option runs the normal -dump"
	echo "                    command but also will send the email to"
	echo "                    joeschmuck2023@hotmail.com for further analysis or just to"
	echo "                    provide drive data information to Joe to help make improvments"
	echo "                    to the script.  If you use the [-dump email] option, you will"
	echo "                    be asked to confirm sending of the email. So you cannot run"
	echo "                    this parameter from a script, it must be CLI.  You will also"
	echo "                    be asked to enter a comment to aid Joe in your problem."
	echo "                    [-dump emailextra] option is the same as '-dump email'"
	echo "                    except it also includes 'smartctl -a' and smartctl -x' text file"
	echo "                    data for each drive.  This should normally not be required."
	echo " "
	echo "      -update       Manually check GitHub for an update."
	echo " "
	echo "      -ignore_lock  This will ignore multiple instances running."
	echo "			  This option should be last on the command line."
	echo " "
	echo "Recommendation:  When troubleshooting a problem you may be asked to provide"
	echo "dump data to assist troubleshooting. Use the [-dump email] or [-dump emailextra]"
	echo "to include all relevant data (drive data and configuration file) which sends an"
	echo "email to joeschmuck2023@hotmail.com, your email address will not be shared!."
	echo "If you prefer you can use [-dump] or [-dump all] and then open up a dialog"
	echo "with Joe Schmuck and attach the files in a message on the TrueNAS forum."
	echo " "
	}

### DEFINE FUNCTIONS END ###

#######################
#######################
###                 ###
###  PROGRAM START  ###
###                 ###
#######################
#######################

# The order in which these processed occur is unfortunately dependent.
# The -s switch will just collect statistical data.
# The -t switch will allow a raw text file for smartctl -a, -x, and .json, followed by the input path.
SECONDS=0
echo $programver
smartdata=""  # Don't think we need this here.

# Verify all the commands are available. Thanks @dak180
# If not, generate a message.
if [[ $softver != "Linux" ]]; then
	# FreeBSD
	commands=(	# FreeBSD Unique
	md5
	sha256	
	sysctl
	glabel
	nvmecontrol
	)
else
	# Debian
	commands=(	# Debian Unique
	fdisk
	lsblk
	md5sum
	sha256sum
	nvme
	)
fi

commands+=(	#Common Commands
awk
base64
case
chmod
clear
curl
cut
date
egrep
git
grep
head
hostname
jq
mount
printf
read
rev
sed
sendmail
shopt
sleep
smartctl
sort
sqlite3
tail
tar
test
trap
uname
wc
wget
zfs
zpool
)

# Test if the commands exist
for command in "${commands[@]}"; do
	if ! type "${command}" &> /dev/null; then
		echo "'${command}' is missing, Contact Joe referencing this message." >&2
		exit 1
	fi
done

if ! [[ "$1" == "-config" || "$1" == "-help" || "$1" == "-delete" || "$1" == "-s" || "$1" == "" || "$1" == "-dump" || "$1" == "-purge" || "$1" == "-h" || "$1" == "-t" || "$1" == "-m" || "$1" == "-u7zip" || $1 == "-update" || "-ignore_lock" ]]; then
	echo '"'$1'" is not a valid option.'
	echo "Use -h for help and look under OPTIONS."
	echo " "
	exit 1
fi

# Set the default
Write_Statistics="true"

# If -m is used, turn off Statistics by default.
if [[ "$1" == "-m" || "$2" == "-m" ]]; then Monitor="true"; Write_Statistics="false"; echo "Monitoring for Critical & Temperature Alarms"; else Monitor="false"; fi

# If -s is in either position, turn on Statistics if we turned it off above.
if [[ "$1" == "-s" || "$2" == "-s" ]]; then Write_Statistics="true"; fi

if [[ "$1" == "-config" ]]; then
	generate_config_file
	exit 0
fi

if [[ "$1" == "-help" ]]; then
	display_help | more
	exit 0
fi

if [[ "$1" == "-h" ]]; then
	display_help_commands | more
	exit 0
fi

# if -dump then interactive user selected dumping, if "all" then automatic dumping of everything.
# Use dump_all=1 during the running routine to gather all the drive data and dump to drive ID files.
# if -dump all is used, then include config and statistical attachments (dump_all=2).
# if -dump email is used, send "-dump" data to Joe Schmuck for analysis (dump_all=3).
# if -dump emailextra is used, send "-dump" plus smart _a.txt & -x.txt files (dump_all=4).
# Dump the files into /tmp/ and then email them.

if [[ "$1" == "-dump" || "$3" == "-dump" ]]; then
	Attach_Files1="true"
	zpool list > /tmp/zpoollist.txt
	zpool status -v > /tmp/zpoolstatus.txt
	zfs list -d 0 > /tmp/zfslist.txt
	if [[ "$2" == "all" ]]; then dump_all="2"; echo "Attaching Drive Data, Multi-Report Configuration, Statistics, and TrueNAS Configuration files."; fi
	if [[ "$2" == "" || "$3" == "-dump" ]]; then dump_all="1"; echo "Attaching Drive Data and Multi-Report Configuration files."; fi
	if [[ "$2" == "email" || "$2" == "emailextra" ]]; then echo "emailing to Joe Schmuck & Attaching Drive Data and Multi-Report Configuration."
		if [[ "$2" == "emailextra" ]]; then
			dump_all="4"
		else
			dump_all="3"
		fi
		echo " "
		echo "Processing..."
		echo "Collecting the data and sending to Joe Schmuck for analysis."
		echo " "
		echo " "
		echo "Please enter a comment to Joe or just press Return to continue."
		echo " "
		echo 'Example: "Hey Joe, ada2 last test age should be zero value. Could you'
		echo 'please take a look at it and let me know if it is okay? Thanks, Jimmy"'
		echo " "
		echo -n "Enter your message: "
           	read Keyboard_Message
		if [[ $Keyboard_Message == "" ]]; then echo "No Message Included"; Keyboard_Message="No Message Included"; else Keyboard_Message='"'$Keyboard_Message'"'; fi
		echo " "
		echo "Working..."
	fi
else
	dump_all="0"
fi

if [[ "$1" == "-delete" ]]; then
	echo "Preparing to Delete Statistical Data File"
	echo " "
	read -p "Press Enter/Return to Continue or CTRL-C to Abort Script"
	rm "$statistical_data_file"
	echo " "
	echo "File Obliterated !!!"
	echo " "
	exit 0
fi

if [[ "$1" == "-purge" ]]; then
	purge_testdata
	exit 0
fi

if [[ "$1" == "-s" || "$2" == "-s" ]]; then echo "Commencing Statistical Data Collection"; fi


# Remove 7-Zip and Exit
if [[ "$1" == "-u7zip" ]]; then
	if test -e "/usr/local/bin/7zzs"; then

		# Check if /usr is readonly (SCALE 24.x started this)
		romounttest=""
		romounttest=$(grep "[[:space:]]ro[[:space:],]" /proc/mounts | grep "/usr")
		if [[ $romounttest  != "" ]]; then
			mount -o remount,rw '/usr'
			romount="true"
		fi

		rm "/usr/local/bin/7zzs"	# Remove 7zzs Standalone Program
		rm "/usr/local/bin/7z"		# Remove Symbolic Link

		# Check if /usr is readonly (SCALE 24.x started this)
		if [[ romount == "true" ]]; then
			mount -o remount,ro '/usr'
		fi
	fi
echo " "
echo "7-Zip has been removed."
echo " "
exit 0
fi

if [[ "$1" == "-update" ]]; then
	update_script
	echo "The script has been updated.  Exiting."
	sleep 1
	exit 0
fi

load_config
if [[ $TrueNASConfigEmailEncryption != "" ]]; then check_7zip; fi

# Cleanup previous run files if anything is left (there shoudn't be).
cleanup_files

# Let's check for a script update.
if [[ $CheckForUpdates == "true" ]] && [[ "$1" != "-t" && "$1" != "-m" && "$1" != "-dump" ]]; then
	checkforupdate
fi

# Let's Update Automatically if allowed.
if [[ $UpdateAvailable == "true" ]] && [[ $AutomaticUpdate == "true" ]]; then
	# We are updating automatically
	update_script
	if [[ $AutomaticUpdate == "true" ]]; then
		echo "Running script normally."
	# How do we run the script when the script is the same name?  Temporarily copy the new script to a new name and run that name.
		(
		cd $SCRIPT_DIR"/"
		cp $SCRIPT_DIR"/"$runfilename $SCRIPT_DIR"/"temprunfile.sh ; chmod 755 $SCRIPT_DIR"/"temprunfile.sh
		./temprunfile.sh $1
		sleep 3
		rm $SCRIPT_DIR"/"temprunfile.sh
		)
		exit 0
	else
		echo "Script Not Updated"
		echo "Running script normally..."
	fi
fi

# 1 = -dump : Normal Dump, No TrueNAS configuration and Statistical Data File.
# 2 = -dump all : Attach Everything.
# 3 = -dump email : Add Joe's email address and SAME data as #1.

if [[ "$dump_all" == "1" ]]; then TrueNASConfigEmailEnable="false"; SDF_DataEmail="false"; fi
if [[ "$dump_all" == "2" ]]; then TrueNASConfigEmailEnable="true"; SDF_DataEmail="true"; TrueNASConfigEmailDay="All"; SDF_DataEmailDay="All"; fi
if [[ "$dump_all" > "2" ]]; then Email=$Email",joeschmuck2023@hotmail.com"; TrueNASConfigEmailEnable="false"; SDF_DataEmail="false"; fi

testfilepath=""

datestamp2=$(date +%Y-%m-%d)
datestamp=$(date +%Y/%m/%d)
# FreeBSD gets second resolution
if [[ $softver != "Linux" ]]; then
	timestamp=$(date +%T)
else
# Linux gets second resolution as well
	timestamp=$(date +"%T")
fi

#### SIMULATION SECTION START - NOT FOR REAL DRIVES

# Command Format: -t /path/to/files [-dump|-config|-m]

if [[ "$1" == "-t" ]]; then
	# $2 is the path for the test files, it must be present.
	if [[ "$2" == "" ]]; then
		echo "Invalid Test File Path"
		exit 1
	fi
	# Ensure the path does not have the following three as these are used to identify the drive type by file name.
	if [[ "$2" =~ "HDD" ]] || [[ "$2" =~ "SSD" ]] || [[ "$2" =~ "NVM" ]]; then
		echo 'Invalid Path - Cannot contain "HDD, SSD, or NVM"'
		exit 1
	fi

	# Set testfilepath
	testfilepath=$2

	# How many test files are there?
	testfilecount="$(ls $2/*.json | wc -l)"

	echo "testfilecount="$testfilecount

	# Collect all the test file names.
	testfilenames=(
		"$testfilepath"/*.json
		)

	# Separate into HDD, SSD, NVM

	for (( i=0; i<=$testfilecount; i++ )); do

		if echo "${testfilenames[i]}" | grep -q "HDD"; then
			testfilenamesHDD+=("${testfilenames[$i]}")

########## TEST DRIVES CORE OR SCALE NAMING CONVENTION

			if [[ $softver != "Linux" ]]; then 
				smartdrives+=("ada${i}")
			else
				numbertoletters=$i
				number_to_letters
				smartdrives+=("sd${numbertoletters}")
			fi
		fi

		if echo "${testfilenames[i]}" | grep -q "SSD"; then
			testfilenamesSSD+=("${testfilenames[$i]}")
			if [[ $softver != "Linux" ]]; then 
				smartdrivesSSD+=("ada${i}")
			else
				numbertoletters=$i
				number_to_letters
				smartdrivesSSD+=("sd${numbertoletters}")
			fi
		fi

		if echo "${testfilenames[i]}" | grep -q "NVM"; then
			testfilenamesNVM+=("${testfilenames[$i]}")
			if [[ $softver != "Linux" ]]; then 
				smartdrivesNVM+=("ada${i}")
			else
				numbertoletters=$i
				number_to_letters
				smartdrivesNVM+=("sd${numbertoletters}")
			fi
		fi
	done

#  THE OUTPUT HERE IS ONLY ONE DRIVE IDENT FOR HDD/SSD/NVM IS A DRIVE OR MANY DRIVES EXIST IN THE CATEGORY
#  NOT EXACTLY HOW I THOUGHT IT WAS GOING TO BE BUT IT WORKS TO TRIGGER HDD/SSD/NVM ACTIONS LATER
#  WE NEED TO USE AN IN-TEST VARIABLE AND SORT LATER IN THE SCRIPT. BUT I THINK WE LOOK AT THE ACTUAL DRIVE LATER, COULD BE A PROBLEM.

	#if [[ "$smartdrives" != "" || "$smartdrivesSSD" != "" || "$smartcrivesNVM" != "" ]]; then
	#	if [[ $Multipath != "off" ]]; then
	#		duplicate_list=$smartdrives
	#		remove_duplicate		# Remove duplicate serial numbers for MultiPath
	#		smartdrives=$duplicate_list
	#	fi
#  THIS SORT SHOULD BE SKIPPED IF USING serial
#  ACTUALLY THERE IS NO SORTING SINCE WE DO NOT HAVE A LIST OF DRIVE IDENTS
	#	if [[ "$Multipath" != "serial" ]]; then
	#		sort_list=$smartdrives
	#		if [ $softver != "Linux" ]; then
	#			sort_drives
	#		else
	#			sort_drives_scale
	#		fi
	#		smartdrives=$sort_list
	#	fi
	#fi

# AT THIS POINT I DO NOT HAVE A LISTING OF SMARTDRIVES IN THE SIMULATOR AS AN OUTPUT.  IS IT NEEDED?  SEEMS TO WORK WITHOUT IT.

fi

# Generate a config file using test mode and test json files to perform automatic compensation.
# Does not work yet. 
if [[ "$3" == "-config" ]] && [[ "$1" == "-t" ]]; then
	generate_config_file
	exit 0
fi

# Do not collect real drive data if we are using test data.
if [[ "$testfilepath" == "" ]]; then
	get_smartHDD_listings
fi

# Do not collect real drive data if we are using test data.
if [[ "$testfilepath" == "" ]]; then
	get_smartSSD_listings
fi

# Do not collect real drive data if we are using test data.
if [[ "$testfilepath" == "" ]]; then
	get_smartNVM_listings
fi

get_smartOther_listings

zpool_report

if [[ $SDF_DataRecordEnable == "true" ]]; then
	if test -e "$statistical_data_file"; then
		# Purge items over SDF_DataPurgeDays days
		statistical_data_file_created=0
	else
		# The file does not exist, create it.
		printf "Date,Time,Device ID,Drive Type,Serial Number,SMART Status,Temp,Power On Hours,Wear Level,Start Stop Count,Load Cycle,Spin Retry,Reallocated Sectors,\
		Reallocated Sector Events,Pending Sectors,Offline Uncorrectable,UDMA CRC Errors,Seek Error Rate,Multi Zone Errors,Read Error Rate,Helium Level\n" > "$statistical_data_file"
		# And set flag the file was created.
		statistical_data_file_created=1
	fi
fi

# Check if smartctl is less than version 7.4
Now=$(date +"%a")
smart_ver=$(smartctl | grep "7." | cut -d " " -f 2)
if [[ "$smart_ver" < "7.4" || $NVM_Smartmontools_74_Override == "enable" ]] && [[ "$NVM_Daily_Short_Selftest" == "true" || "$NVM_Weekly_Long_Selftest" == "true" ]] && [[ "$dump_all" == "0" ]]; then
	if [[ "$Wait_For_SMART" == "true" ]]; then
		if [[ "$Now" == "$NVM_Weekly_Long_Selftest_Day" ]] && [[ "$NVM_Weekly_Long_Selftest" == "true" ]]; then
			nvm_selftest_long
			if [[ $Wait_For_SMART_Long_Duration -lt 120 ]]; then
				echo "Sleeping for "$Wait_For_SMART_Long_Duration" seconds to let S.M.A.R.T. Long Self-test running..."
			#	echo "Sleeping for "$Wait_For_SMART_Long_Duration" seconds to let S.M.A.R.T. Long Self-test running..." >> /tmp/selftestlog
				sleep $Wait_For_SMART_Long_Duration
				echo "Not waiting for the S.M.A.R.T. test to complete, continuing."
				echo "Did not wait for the S.M.A.R.T. test to complete, continuing." >> /tmp/selftestlog
			else
				echo "Sleeping for "$Wait_For_SMART_Long_Duration" seconds to let S.M.A.R.T. Long Self-test finish..."
				echo "Sleeping for "$Wait_For_SMART_Long_Duration" seconds to let S.M.A.R.T. Long Self-test finish..." >> /tmp/selftestlog
				sleep $Wait_For_SMART_Long_Duration
				echo "S.M.A.R.T. Long Self-test completed. Results in chart."
				echo "S.M.A.R.T. Long Self-test completed. Results in chart." >> /tmp/selftestlog
			fi

		elif [[ "$NVM_Daily_Short_Selftest" == "true" ]]; then
			nvm_selftest_short
			if [[ $Wait_For_SMART_Short_Duration -lt 100 ]]; then
				echo "Sleeping for "$Wait_For_SMART_Short_Duration" seconds to let S.M.A.R.T. Short Self-test running..."
			#	echo "Sleeping for "$Wait_For_SMART_Short_Duration" seconds to let S.M.A.R.T. Short Self-test running..." >> /tmp/selftestlog
				sleep $Wait_For_SMART_Short_Duration
				echo "Not waiting for the S.M.A.R.T. test to complete, continuing."
				echo "Did not wait for the S.M.A.R.T. test to complete, continuing." >> /tmp/selftestlog
			else
				echo "Sleeping for "$Wait_For_SMART_Short_Duration" seconds to let S.M.A.R.T. Short Self-test finish..."
				echo "Sleeping for "$Wait_For_SMART_Short_Duration" seconds to let S.M.A.R.T. Short Self-test finish..." >> /tmp/selftestlog
				sleep $Wait_For_SMART_Short_Duration
				echo "S.M.A.R.T. Short Self-test completed. Results in chart."
				echo "S.M.A.R.T. Short Self-test completed. Results in chart." >> /tmp/selftestlog
			fi
		fi
	fi

	if [[ "$Wait_For_SMART" != "true" ]]; then
		if [[ "$Now" == "$NVM_Weekly_Long_Selftest_Day" ]] && [[ "$NVM_Weekly_Long_Selftest" == "true" ]]; then
			nvm_selftest_long
		elif [[ "$NVM_Daily_Short_Selftest" == "true" ]]; then
			nvm_selftest_short
		fi
	fi
fi

# Lets start processing the drives baby!
if [[ $Develop == "true" ]]; then
	echo " "
	echo " "
	echo "smartdrives="${smartdrives[*]}
	echo " "
fi
# Generate SMART HDD Report
SER1=""

if [[ "$smartdrives" != "" ]]; then
	generate_table "HDD"
	for drive in ${smartdrives[@]}; do
		clear_variables
		get_drive_data "HDD"
		crunch_numbers "HDD"
		write_table "HDD"
	done
	end_table "HDD"
fi

if [[ $Develop == "true" ]]; then
	echo "End of HDD Section"
	duration=$SECONDS
	echo "$(($duration / 60)) minutes and $(($duration % 60)) seconds elapsed."
	echo " "
	echo " "
	echo "smartdrivesSSD="${smartdrivesSSD[*]}
	echo " "
fi

# Generate SSD Report
SER1=""

if [[ $smartdrivesSSD != "" ]]; then
	generate_table "SSD"
	for drive in ${smartdrivesSSD[@]}; do
		clear_variables
		get_drive_data "SSD"
		crunch_numbers "SSD"
		write_table "SSD"
	done
	end_table "SSD"
fi

if [[ $Develop == "true" ]]; then
	echo "End of SSD Section"
	duration=$SECONDS
	echo "$(($duration / 60)) minutes and $(($duration % 60)) seconds elapsed."
	echo " "
	echo " "
	echo "smartdrivesNVM="${smartdrivesNVM[*]}
	echo " "
fi

# Generate NVMe Report
SER1=""
if [[ $smartdrivesNVM != "" ]]; then
	generate_table "NVM"
	for drive in ${smartdrivesNVM[@]}; do
		clear_variables
		get_drive_data "NVM"
		crunch_numbers "NVM"
		write_table "NVM"
	done
	end_table "NVM"
fi

if [[ $Develop == "true" ]]; then echo "End of NVMe Section"; fi

#if [[ "$testfilepath" == "" ]]; then
#	for drive in $nonsmartdrives; do
#		clear_variables
#
###FIXME # routine does not have a "NON" option.  Fix this!
#
#		get_drive_data "NON" 
#	done
#fi

if [[ $Develop == "true" ]]; then echo "$(echo "${json_error_log}")"; fi

# This purge happens here directly after data collection.
# A zero (0) value = Disable Purging (Keep all data)
if [[ $SDF_DataPurgeDays != 0 ]]; then
	purge_exportdata
fi

write_ATA_Errors="0"

if [[ "$Enable_Text_Section" == "true" ]]; then
	if [[ $spencer_enable == "true" ]]; then
		if test -e "$spencer_script_name"; then python3 "$spencer_script_name" multi_report; fi
		spencer
	fi

	detailed_report $2
	if [[ $ReportNonSMART == "true" ]]; then
		if [[ $DisableRAWdata != "true" ]]; then
			non_smart_report
		fi
	fi
fi

if [[ $Monitor == "false" ]]; then
	echo "End of data section" >> "$logfile"
fi

# Update multi_report_config.txt file if required.
	if [[ $write_ATA_Errors == "1" ]]; then
		ATA_Errors_List="$(echo "$temp_ATA_Errors" | sed 's/.$//')"
		update_config_file
	fi


if [[ "$1" == "-s" || "$2" == "-s" ]] && [[ "$Monitor" == "false" ]]; then
	echo "Statistical Data Collection Complete"
else
	Email_datafile
	remove_junk_report
	config_backup
	attach_files
fi


if [[ $logfile_critical != "" ]] || [[ $logfile_warning != "" ]]; then
	Send_On_Alarm="true"
else
	Send_On_Alarm="false"
fi

Email_Sent="false"
if [[ "$1" != "-s" && "$2" != "-s" ]] && [[ $Monitor == "false" ]]; then
	if [[ "$Email_On_Alarm_Only" == "false" ]] || [[ "$Send_On_Alarm" == "true" ]]; then
		if [[ $Develop == "true" ]]; then echo "Create Normal Email"; fi
		create_Email
		Email_Sent="true"
	fi
elif [[ $Monitor_Send_Email == "true" ]]; then
	if [[ $Develop == "true" ]]; then echo "Create Monitor Email"; fi
	if [[ $Email_On_Alarm_Only == "false" ]] || [[ $Send_On_Alarm="true" ]]; then
		create_Email
		Email_Sent="true"
	fi
fi

if [[ $Email_On_Alarm_Only_And_Attachments == "true" ]] && [[ $Attach_Files1 == "true" ]] && [[ $Email_Sent == "false" ]]; then
		create_Email
fi

if [[ $NVM_Low_Power == "true" ]]; then
	if [[ "$testfilepath" == "" ]]; then
		if [[ $softver != "Linux" ]]; then
			nvm_power
		fi
	fi
fi

cleanup_files
duration=$SECONDS
if [[ $Develop == "true" ]]; then echo "$(($duration / 60)) minutes and $(($duration % 60)) seconds elapsed."; fi

# All reporting files are left in the /tmp/ directory for troubleshooting and cleaned up when the script is initial run.