Editing Introduction to a Self Managed Life: a 13 hour & 28 minute presentation by FUTO software (section)

==== 5.1 Creating the alert script ====

I’m not a programmer, so bear with me. This script is for my personal use, but I’m sharing it because it works. Here’s what you need to do:

# '''Edit Email Addresses''': You’ll need to change the email addresses in the script. This includes:
#* The recipient email
#* The sender email
#* The reply-to address
#* The return path for bounced emails
# '''Script Location''': Save the script at <code>root/mdadm_alert.sh</code>

<pre>sudo -u root nano -w /root/mdadm_alert.sh</pre>
Enter the following:

<pre>#!/bin/bash

# thank you to stack overflow for giving me the courage to wade through 100s of posts and hack together something that looks like it works. 

# stricter error handling
set -euo pipefail  # ‘set -e’ exits on errors, ‘u’ throws errors on unset variables, &amp; ‘pipefail’ exits if any part of a pipeline fails
IFS=$'\n\t'  # Set IFS (Internal Field Separator) to newline &amp; tab to avoid issues with spaces and other weird characters in filenames

# Configuration variables (where settings are stored)
EMAIL=&quot;l.a.rossmann@gmail.com&quot;  # Email to send alerts to - EDIT THIS 
HOSTNAME=$(hostname)  # Pull the system's hostname dynamically and save it here
LOG_DIR=&quot;/var/log/mdadm-monitor&quot;  # Directory path for where logs go
LOG_FILE=&quot;${LOG_DIR}/raid_health_check.log&quot;  # Full path to the specific log file for RAID checks
LOG_MAX_SIZE=$((10 * 1024 * 1024))  # Maximum log file size in bytes (10 MB here)

# Email configuration for the alert message
FROM_EMAIL=&quot;yourdriveisdead@stevesavers.com&quot;  # The email address that will appear as the sender - EDIT THIS
FROM_NAME=&quot;Steve&quot;  # name of the sender, EDIT THIS
REPLY_TO=&quot;Steve &lt;steve@stevesavers.com&gt;&quot;  # Reply-to email address, EDIT THIS
RETURN_PATH=&quot;bounce@stevesavers.com&quot;  # Return path for bounced emails when email fails EDIT THIS

# make empty variables &amp; associated arrays 
errors=&quot;&quot;  # Empty variable to collect error messages
drive_health_report=&quot;&quot;  # Another empty variable to store drive health details
declare -A RAID_ARRAYS  # array to keep track of RAID arrays we find, indexed by  name like &quot;boot&quot;
declare -A SMART_SCORES  # array to store SMART scores for drives, indexed by rive path

# Set up log directory and ensure permissions are correct
setup_logging() {
    # Make the log directory if it doesn’t already exist
    mkdir -p &quot;$LOG_DIR&quot; || { echo &quot;ERROR: Cannot create log directory $LOG_DIR&quot;; exit 1; }  # Exit with error if I can’t make the directory
    chmod 750 &quot;$LOG_DIR&quot;  # Set directory permissions to allow owner &amp; group access but not others

    # Check if the log file exists and exceeds the max size limit
    if [ -f &quot;$LOG_FILE&quot; ] &amp;&amp; [ &quot;$(stat -c%s &quot;$LOG_FILE&quot;)&quot; -gt &quot;$LOG_MAX_SIZE&quot; ]; then  # ‘stat -c%s’ gives the size in bytes
        mv &quot;$LOG_FILE&quot; &quot;$LOG_FILE.old&quot;  # Archive the old log file by renaming it
    fi
    touch &quot;$LOG_FILE&quot;  # Create an empty log file if it doesn’t exist
    chmod 640 &quot;$LOG_FILE&quot;  # Set permissions on the log file (read/write for owner, read for group)
}

# Function for logging messages w/ timestamps
log_message() {
    local timestamp  # Make local variable for this
    timestamp=$(date '+%Y-%m-%d %H:%M:%S')  # Generate a timestamp in this specific format
    echo &quot;[$timestamp] $1&quot; | tee -a &quot;$LOG_FILE&quot;  # Output the message with the timestamp to both console &amp; log file
}

# Function for logging errors (adds them to the error string and logs them as &quot;ERROR&quot;)
log_error() {
    local message=&quot;$1&quot;  # Message passed to this function
    errors=&quot;${errors}\n$message&quot;  # Append this message to the errors variable
    log_message &quot;ERROR: $message&quot;  # Log the error with a timestamp
}

# Check that required (commands) are installed on the system
check_dependencies() {
    log_message &quot;Checking required dependencies...&quot;  # Announce the check in the log
    local missing_deps=()  # Initialize an empty array for any missing commands

    # Loop through each command we need, checking if it’s available
    for dep in mdadm smartctl lsblk findmnt awk grep dmsetup; do
        if ! command -v &quot;$dep&quot; &amp;&gt;/dev/null; then  # If the command is missing, add it to the array
            missing_deps+=(&quot;$dep&quot;)
        fi
    done

    # If the array of missing dependencies isn’t empty, log an error and exit
    if [ ${#missing_deps[@]} -ne 0 ]; then
        log_error &quot;Missing required dependencies: ${missing_deps[*]}&quot;  # Log missing commands
        log_error &quot;Install them with: sudo apt-get install mdadm smartmontools util-linux findutils gawk grep dmsetup&quot;
        exit 1  # Exit with error because we’re missing something we need(find what you need if you're getting this)
    fi
}

# Find &amp; detect RAID arrays on this system
detect_raid_arrays() {
    log_message &quot;Detecting RAID arrays...&quot;  # Log that we’re looking for RAID arrays

    # Find all block devices with names like /dev/md0, /dev/md1 (these are RAID arrays like the one you made for the OS &amp; boot)
    local md_devices
    md_devices=$(find /dev -name 'md[0-9]*' -type b)  # Save this list to the md_devices variable

    # Loop through each RAID array found and log its details
    for md_dev in $md_devices; do
        local array_detail  # Temporary variable for array details
        array_detail=$(mdadm --detail &quot;$md_dev&quot; 2&gt;/dev/null) || continue  # Get RAID details; skip if it fails

        # Extract the RAID array name from the details
        local array_name
        array_name=$(echo &quot;$array_detail&quot; | grep &quot;Name&quot; | awk '{print $NF}')  # Last word on the &quot;Name&quot; line is the array name

        # Use the name to decide if this array is for boot or root, then add it to RAID_ARRAYS
        if [[ &quot;$array_name&quot; == *&quot;bootraid&quot;* ]]; then  # Array name contains &quot;bootraid&quot;
            RAID_ARRAYS[&quot;boot&quot;]=&quot;$md_dev&quot;  # Save the device path with the key &quot;boot&quot;
            log_message &quot;Found boot array: $md_dev ($array_name)&quot;  # Log the found boot array
        elif [[ &quot;$array_name&quot; == *&quot;osdriveraid&quot;* ]]; then  # Array name contains &quot;osdriveraid&quot;
            RAID_ARRAYS[&quot;root&quot;]=&quot;$md_dev&quot;  # Save the device path with the key &quot;root&quot;
            log_message &quot;Found root array: $md_dev ($array_name)&quot;  # Log the found root array
        fi
    done

    # Check if we actually found both root and boot arrays, and log an error if any are missing
    if [ -z &quot;${RAID_ARRAYS[&quot;boot&quot;]:-}&quot; ] || [ -z &quot;${RAID_ARRAYS[&quot;root&quot;]:-}&quot; ]; then  # If either key is empty
        log_error &quot;Failed to detect both boot and root RAID arrays&quot;  # Log a general error
        [ -z &quot;${RAID_ARRAYS[&quot;boot&quot;]:-}&quot; ] &amp;&amp; log_error &quot;Boot array not found&quot;  # Specific message if boot is missing
        [ -z &quot;${RAID_ARRAYS[&quot;root&quot;]:-}&quot; ] &amp;&amp; log_error &quot;Root array not found&quot;  # Specific message if root is missing
        return 1  # Return an error code
    fi

    # Print out a summary of all arrays found
    log_message &quot;Detected arrays:&quot;
    for purpose in &quot;${!RAID_ARRAYS[@]}&quot;; do
        log_message &quot;  $purpose: ${RAID_ARRAYS[$purpose]}&quot;
    done
}

# Check the health of a specific RAID array
check_array_status() {
    local array=&quot;$1&quot;  # The path of the array device
    local purpose=&quot;$2&quot;  # Either &quot;boot&quot; or &quot;root&quot; to clarify which array this is

    # Verify that the array actually exists as a block device
    if [ ! -b &quot;$array&quot; ]; then
        log_error &quot;$purpose array device $array does not exist&quot;  # Log the missing device
        return 1  # Return error because we can’t check a nonexistent device
    fi

    # Get details about the RAID array and store it in the detail variable
    local detail
    detail=$(mdadm --detail &quot;$array&quot; 2&gt;&amp;1) || {  # ‘2&gt;&amp;1’ captures error output in case of issues
        log_error &quot;Failed to get details for $purpose array ($array)&quot;
        return 1  # Exit with an error code if it failed
    }

    # Extract the state of the array (like &quot;clean&quot; or &quot;active&quot;) and log it
    local state
    state=$(echo &quot;$detail&quot; | grep &quot;State :&quot; | awk '{print $3,$4}')  # Get the words after &quot;State :&quot; from the details
    log_message &quot;$purpose array status: $state&quot;

    # If the array is in an undesirable state, log a warning
if [[ &quot;$state&quot; =~ degraded|DEGRADED|failed|FAILED|inactive|INACTIVE ]]; then
    log_error &quot;$purpose array ($array) is in concerning state: $state&quot;
fi

    # Detect failed devices within the array
    local failed_devices
    failed_devices=$(echo &quot;$detail&quot; | grep &quot;Failed Devices&quot; | awk '{print $4}')  # Pull the failed devices count
    if [ &quot;$failed_devices&quot; -gt 0 ]; then  # If there are failed devices, go through each one
        while read -r line; do
            if [[ &quot;$line&quot; =~ &quot;faulty&quot; ]]; then  # If the line mentions &quot;faulty&quot;
                local failed_dev
                failed_dev=$(echo &quot;$line&quot; | awk '{print $7}')  # Get the 7th word (the device name)
                log_error &quot;$purpose array ($array) has failed device: $failed_dev&quot;  # Log which device failed
            fi
        done &lt; &lt;(echo &quot;$detail&quot; | grep -A20 &quot;Number&quot; | grep &quot;faulty&quot;)  # Look up to 20 lines after &quot;Number&quot; to find &quot;faulty&quot;
    fi

    # Check if any devices are rebuilding, and log it if they are
    if echo &quot;$detail&quot; | grep -q &quot;rebuilding&quot;; then
        while read -r line; do
            if [[ &quot;$line&quot; =~ &quot;rebuilding&quot; ]]; then  # Check for &quot;rebuilding&quot; in the line
                local rebuilding_dev
                rebuilding_dev=$(echo &quot;$line&quot; | awk '{print $7}')  # Get the device name being rebuilt
                log_error &quot;$purpose array ($array) is rebuilding device: $rebuilding_dev&quot;  # Log the rebuilding device
            fi
        done &lt; &lt;(echo &quot;$detail&quot; | grep -A20 &quot;Number&quot; | grep &quot;rebuilding&quot;)  # Again, look ahead 20 lines for any &quot;rebuilding&quot; mention
    fi
}

# Function to check the health of each drive within a RAID array
check_drive_health() {
    local drive=&quot;$1&quot;  # The drive device to check (e.g., /dev/sda)
    local health_score=100  # Initialize health score to 100 (a perfect score)
    local issues=&quot;&quot;

    # Skip the check if it’s not a valid block device
    if [ ! -b &quot;$drive&quot; ]; then
        log_error &quot;Device $drive is not a block device&quot;  # Log the invalid device
        return 1  # Exit with an error code
    fi

    log_message &quot;Checking health of drive $drive...&quot;  # Announce which drive we’re checking

    # Run SMART health check and reduce health score if it fails
    if ! smartctl -H &quot;$drive&quot; | grep -q &quot;PASSED&quot;; then  # If it does NOT say &quot;PASSED&quot;
        health_score=$((health_score - 50))  # Drop score by 50 points if it fails
        issues+=&quot;\n- Overall health check failed&quot;  # Log this specific issue
    fi

    # Collect SMART attributes for further checks
    local smart_attrs
    smart_attrs=$(smartctl -A &quot;$drive&quot; 2&gt;/dev/null) || true  # Redirect error to /dev/null

    # Check for reallocated sectors (sign of drive wear and tear)
    local reallocated
    reallocated=$(echo &quot;$smart_attrs&quot; | awk '/^  5/ {print $10}')  # Look for attribute ID 5 in SMART data
    if [ -n &quot;$reallocated&quot; ] &amp;&amp; [ &quot;$reallocated&quot; -gt 0 ]; then
        health_score=$((health_score - 10))  # Drop health score by 10 if we have reallocated sectors
        issues+=&quot;\n- Reallocated sectors: $reallocated&quot;  # Add to issues list
    fi

    # Check for pending sectors (could cause read/write errors)
    local pending
    pending=$(echo &quot;$smart_attrs&quot; | awk '/^197/ {print $10}')  # Look for attribute ID 197 in SMART data
    if [ -n &quot;$pending&quot; ] &amp;&amp; [ &quot;$pending&quot; -gt 0 ]; then
        health_score=$((health_score - 10))  # Drop health score by 10 if pending sectors are present
        issues+=&quot;\n- Pending sectors: $pending&quot;  # Add to issues list
    fi

    SMART_SCORES[&quot;$drive&quot;]=$health_score  # Save the final score in SMART_SCORES array
    if [ &quot;$health_score&quot; -lt 100 ]; then
        drive_health_report+=&quot;\nDrive: $drive\nHealth Score: $health_score/100\nIssues:$issues&quot;  # Append issues to report if any were found
    fi
}

# Send email if any errors or health issues were found
send_email() {
    local subject=&quot;RAID Alert: Issues Detected on ${HOSTNAME}&quot;  # Set email subject line
    local content=&quot;RAID Health Monitor Report from ${HOSTNAME}\nTime: $(date '+%Y-%m-%d %H:%M:%S')\n&quot;
    [ -n &quot;$errors&quot; ] &amp;&amp; content+=&quot;\nRAID Issues:${errors}&quot;  # Append RAID issues to the email content if any
    [ -n &quot;$drive_health_report&quot; ] &amp;&amp; content+=&quot;\nDrive Health Report:${drive_health_report}&quot;  # Append drive health report if any issues were found

    # Build the email using sendmail syntax
    {
        echo &quot;Subject: $subject&quot;
        echo &quot;To: ${EMAIL}&quot;
        echo &quot;From: ${FROM_NAME} &lt;${FROM_EMAIL}&gt;&quot;
        echo &quot;Reply-To: ${REPLY_TO}&quot;
        echo &quot;Return-Path: ${RETURN_PATH}&quot;
        echo &quot;Content-Type: text/plain; charset=UTF-8&quot;  # Text format for readability
        echo
        echo -e &quot;$content&quot;  # Use ‘-e’ to allow newline characters
    } | sendmail -t  # Pipe the entire email message to sendmail for delivery
}

# Main function to execute checks and send email if needed
main() {
    # Make sure script is run as root for necessary permissions
    [ &quot;$(id -u)&quot; -ne 0 ] &amp;&amp; { echo &quot;ERROR: This script must be run as root&quot;; exit 1; }
    setup_logging  # Call function to initialize logging setup
    log_message &quot;Starting RAID health check&quot;  # Announce the start of the health check
    check_dependencies  # Verify dependencies are available
    detect_raid_arrays  # Detect RAID arrays

    # Loop through each RAID array and check its status, then check each drive in the array
    for purpose in &quot;${!RAID_ARRAYS[@]}&quot;; do
        array=&quot;${RAID_ARRAYS[$purpose]}&quot;
        check_array_status &quot;$array&quot; &quot;$purpose&quot;

        # For each device in the RAID array, check health
        while read -r device; do
            if [[ &quot;$device&quot; =~ ^/dev/ ]]; then
                check_drive_health &quot;$device&quot;
            fi
        done &lt; &lt;(mdadm --detail &quot;$array&quot; | grep &quot;active sync&quot; | awk '{print $NF}')
    done

    # Send an email if errors or health issues were found; otherwise, log a success message
    [ -n &quot;$errors&quot; ] || [ -n &quot;$drive_health_report&quot; ] &amp;&amp; send_email || log_message &quot;All checks passed successfully&quot;
}

# Execute the main function to start everything
main  # Calls the main function, running all the checks</pre>
Set permissions properly so it can run:

<pre>sudo -u root chmod +x /root/mdadm_alert.sh</pre>
<span id="setting-up-the-cron-job"></span>