timestamp=$(date +"%Y%m%d_%H%M%S")

out_dir="/tmp/health-check_$timestamp"

mkdir -p "$out_dir"

mkdir -p "$out_dir/ceph"

mkdir -p "$out_dir/ceph/device_health"

ctdb_dir="$out_dir/ctdb"

mkdir -p "$ctdb_dir"

logfile="$out_dir/ctdb/report.log"

output_file="config_summary_$(date +'%Y%m%d_%H%M%S').txt"

warnings=""

filename="report_$timestamp.json"

if [ -f /etc/os-release ]; then

. /etc/os-release

platform=$PRETTY_NAME

os_id=$ID

platform=$(uname -a)

os_id="unknown"

start_time=$(date +"%Y-%m-%dT%H:%M:%S%:z")

echo "Starting health check script at $start_time for platform: $platform" | tee -a "$logfile"

echo "The Health Check Report has been saved in tmp/ folder." | tee -a "$logfile"

remote_hosts=$(awk '$1 ~ /^[0-9]+(\.[0-9]+){3}$/ && $2 !~ /localhost/ {print $2}' /etc/hosts | sort -u)

if [ -z "$remote_hosts" ]; then

remote_hosts=$(hostname)

collect_from_all_hosts() {

}

collect_from_all_hosts '

' "tuned"

collect_from_all_hosts '

' "selinux"

collect_from_all_hosts "free -h" "memory"

collect_from_all_hosts "free -m && echo && used_swap=\$(free -m | awk '/Swap:/ {print \$3}') && [ \"\$used_swap\" -gt 500 ] && echo WARNING: High swap usage detected" "swap"

collect_from_all_hosts '

' "smartctl"

collect_from_all_hosts '

' "drive_age"

collect_from_all_hosts '

' "zfs_dataset_snapshots"

collect_from_all_hosts '

' "zfs_snapshots"

collect_from_all_hosts '

' "packet_errors"

collect_from_all_hosts '

' "zfs_summary"

collect_from_all_hosts '

' "zfs_pool_errors"

collect_from_all_hosts "uptime" "uptime"

collect_from_all_hosts "uname -a" "kernel_version"

collect_from_all_hosts "lspci -nnk" "pci_devices"

collect_from_all_hosts "ss -tuln" "open_ports"

collect_from_all_hosts "systemctl --failed" "failed_units"

collect_from_all_hosts "systemd-analyze" "boot_time"

collect_from_all_hosts "ip route show default" "default_route"

collect_from_all_hosts "cat /etc/os-release" "linux_distribution"

collect_from_all_hosts "last reboot" "reboot_history"

collect_from_all_hosts "systemctl status winbind" "winbind_status"

if [[ -f /etc/debian_version ]]; then

collect_from_all_hosts "apt list --upgradable" "updates"

ceph -s > "$out_dir/ceph_status.txt" 2>/dev/null

collect_from_all_hosts '

' "alertmanager_status"

collect_from_all_hosts "cat /etc/samba/smb.conf 2>/dev/null || echo '/etc/samba/smb.conf not found'" "samba_conf"

collect_from_all_hosts "cat /etc/exports.d/cockpit-file-sharing.exports 2>/dev/null || echo '/etc/exports.d/cockpit-file-sharing.exports not found'" "nfs_exports"

collect_from_all_hosts "cat /etc/scst.conf 2>/dev/null || echo '/etc/scst.conf not found'" "iscsi_conf"

zfs list > "$out_dir/zfs_usage.txt" 2>/dev/null || echo "zfs list failed" > "$out_dir/zfs_usage.txt"

if [ "$os_id" == "rocky" ] || grep -qi "rocky" /etc/os-release; then

rpm -qa --queryformat '%{NAME} %{VERSION}-%{RELEASE}\n' > "$out_dir/rocky_packages.txt"

elif [ "$os_id" == "ubuntu" ] || grep -qi "ubuntu" /etc/os-release; then

dpkg-query -W -f='${binary:Package} ${Version}\n' > "$out_dir/ubuntu_packages.txt"

echo "Unsupported OS for package query. (No Rocky or Ubuntu detected)" > "$out_dir/package_versions.txt"

cat <<EOF > "$out_dir/$filename"

ceph status > "$out_dir/ceph/status" 2>/dev/null

ceph -v > "$out_dir/ceph/version" 2>/dev/null

ceph versions > "$out_dir/ceph/versions" 2>/dev/null

ceph features > "$out_dir/ceph/features" 2>/dev/null

ceph fsid > "$out_dir/ceph/fsid" 2>/dev/null

collect_from_all_hosts "cat /etc/ceph/ceph.conf 2>/dev/null || echo '/etc/ceph/ceph.conf not found'" "ceph_conf"

ceph config dump > "$out_dir/ceph/config" 2>/dev/null

ceph health > "$out_dir/ceph/health_summary" 2>/dev/null

ceph health detail > "$out_dir/ceph/health_detail" 2>/dev/null

ceph report > "$out_dir/ceph/health_report" 2>/dev/null

ceph df > "$out_dir/ceph/health_df" 2>/dev/null

if grep -qE 'HEALTH_WARN|HEALTH_ERR' "$out_dir/ceph/health_detail"; then

echo "Ceph cluster has warnings or errors:"

cat "$out_dir/ceph/health_detail"

if command -v lsb_release &> /dev/null; then

collect_from_all_hosts "lsb_release -a" "lsb_release"

ceph mon stat > "$out_dir/ceph/mon_stat" 2>/dev/null

ceph mon dump > "$out_dir/ceph/mon_dump" 2>/dev/null

ceph mon getmap -o "$out_dir/ceph/mon_map" 2>/dev/null

ceph mon metadata > "$out_dir/ceph/mon_metadata" 2>/dev/null

ceph osd tree > "$out_dir/ceph/osd_tree" 2>/dev/null

ceph osd df > "$out_dir/ceph/osd_df" 2>/dev/null

ceph osd dump > "$out_dir/ceph/osd_dump" 2>/dev/null

ceph osd stat > "$out_dir/ceph/osd_stat" 2>/dev/null

ceph osd getcrushmap -o "$out_dir/ceph/osd_crushmap" 2>/dev/null

ceph osd getmap -o "$out_dir/ceph/osd_map" 2>/dev/null

ceph osd metadata > "$out_dir/ceph/osd_metadata" 2>/dev/null

ceph osd perf > "$out_dir/ceph/osd_perf" 2>/dev/null

ceph pg stat > "$out_dir/ceph/pg_stat" 2>/dev/null

ceph pg dump > "$out_dir/ceph/pg_dump" 2>/dev/null

ceph pg dump_stuck > "$out_dir/ceph/pg_dump_stuck" 2>/dev/null

ceph mds metadata > "$out_dir/ceph/mds_metadata" 2>/dev/null

ceph mds dump > "$out_dir/ceph/mds_dump" 2>/dev/null

ceph mds stat > "$out_dir/ceph/mds_stat" 2>/dev/null

ceph fs dump > "$out_dir/ceph/fs_dump" 2>/dev/null

ceph fs status > "$out_dir/ceph/fs_status" 2>/dev/null

ceph device check-health > "$out_dir/ceph/check-health" 2>/dev/null

ceph device ls > "$out_dir/ceph/device_ls" 2>/dev/null

if command -v ceph &> /dev/null && ceph device ls &> /dev/null; then

for dev in $(ceph device ls 2>/dev/null | awk '{print $1}' | grep -v NAME); do

osd_id=$(ceph osd metadata -f json 2>/dev/null | jq -r ".[] | select(.device_ids[]? == \"$dev\") | .id" | head -n 1)

if [[ -n "$osd_id" && "$osd_id" != "null" ]]; then

filename="osd.$osd_id.txt"

else

safe_dev=$(echo "$dev" | tr '/ ' '__')

filename="device_health_$safe_dev.txt"

fi

ceph device get-health-metrics "$dev" > "$out_dir/ceph/device_health/$filename" 2>/dev/null

done

for host in $remote_hosts; do

echo "[$host]" | tee -a "$logfile"

ssh -o ConnectTimeout=5 -o BatchMode=yes "$host" 'systemctl is-active --quiet ctdb' 2>/dev/null

if [ $? -eq 0 ]; then

echo "CTDB is running on $host. Gathering status..." | tee -a "$logfile"

# Save CTDB output to file

{

echo "===== ctdb status ($host) ====="

ssh "$host" 'ctdb status' 2>&1

echo

echo "===== ctdb ip ($host) ====="

ssh "$host" 'ctdb ip' 2>&1

} > "$ctdb_dir/$host.txt"

else

echo "CTDB not running on $host." | tee -a "$logfile"

fi

if tar -czf "$out_dir.tar.gz" -C "$(dirname "$out_dir")" "$(basename "$out_dir")"; then

rm -rf "$out_dir"

echo "=== SMART Health Warnings $(date) ==="

for dev in $(ls /dev | grep -E "^sd[a-z]$"); do

device="/dev/$dev"

# Collect info

serial=$(smartctl -i "$device" 2>/dev/null | grep -i "Serial Number" | awk -F: '{print $2}' | xargs)

smartout=$(smartctl -a "$device" 2>/dev/null)

# Quick health assessment

health=$(echo "$smartout" | grep -i "SMART overall-health" | awk -F: '{print $2}' | xargs)

# Extract key values

realloc=$(echo "$smartout" | grep -i "Reallocated_Sector_Ct" | awk '{print $10}')

pending=$(echo "$smartout" | grep -i "Current_Pending_Sector" | awk '{print $10}')

offline_uncorr=$(echo "$smartout" | grep -i "Offline_Uncorrectable" | awk '{print $10}')

crc=$(echo "$smartout" | grep -i "UDMA_CRC_Error_Count" | awk '{print $10}')

reserved=$(echo "$smartout" | grep -i "Available_Reservd_Space" | awk '{print $10}')

# Print warnings only

if [[ "$health" == "FAILED" ]]; then

echo "[$device | $serial] SMART overall health test FAILED!"

fi

if [[ -n "$realloc" && "$realloc" -gt 0 ]]; then

echo "[$device | $serial] Reallocated sectors detected: $realloc"

fi

if [[ -n "$pending" && "$pending" -gt 0 ]]; then

echo "[$device | $serial] Pending sectors detected: $pending"

fi

if [[ -n "$offline_uncorr" && "$offline_uncorr" -gt 0 ]]; then

echo "[$device | $serial] Offline uncorrectable sectors detected: $offline_uncorr"

fi

if [[ -n "$crc" && "$crc" -gt 0 ]]; then

echo "[$device | $serial] CRC interface errors detected: $crc"

fi

if [[ -n "$reserved" && "$reserved" -lt 100 ]]; then

echo "[$device | $serial] SSD spare blocks low: $reserved%"

fi

keywords=("I/O error" "failed" "hardware error" "buffer I/O" "uncorrectable" "CRC error" "sense key" "end_request" "link reset" "error handler" "device offlined" "S.M.A.R.T. error")

dmesg_output=$(dmesg --ctime --color=never 2>/dev/null)

found_issue=false

for kw in "${keywords[@]}"; do

if echo "$dmesg_output" | grep -i -q "$kw"; then

echo "⚠️ Possible hardware issue detected: \"$kw\" found in dmesg logs"

found_issue=true

fi

if [ "$found_issue" = false ]; then

echo "✅ No hardware-related problems detected in dmesg output."

echo "🔍 Review 'dmesg' output above for details. Recommend checking SMART data or system health."

if systemctl is-active --quiet zfs.target; then

zpool_parse_script="zpool_parse.py"

if [ ! -f "$zpool_parse_script" ]; then

echo "zpool_parse.py not found. Attempting to download..."

curl -o "$zpool_parse_script" https://raw.githubusercontent.com/45Drives/scripts/main/parse_zpool.py

if [ ! -f "$zpool_parse_script" ]; then

echo "Error: Unable to download zpool_parse.py. The script cannot continue."

echo "Please download zpool_parse.py manually from https://raw.githubusercontent.com/45Drives/scripts/main/parse_zpool.py and place it in the working directory."

exit 1

fi

fi

# Function to get raw ZFS pool status

get_raw_zpool_status() {

}

# Function to get ZFS version

get_zfs_version() {

}

# Function to get ZFS ARC stats

get_zfs_arc_stats() {

}

# Function to check dataset configurations

get_datasets_info() {

}

# Function to check for snapshots

check_snapshots() {

}

# ZFS VDEV Checking Functions

get_zpool_status() {

}

# Run functions and output to screen and file

get_raw_zpool_status

get_zpool_status

get_zfs_version

get_zfs_arc_stats

get_datasets_info

check_snapshots

echo "zfs.target not running on this host."

get_bond_info() {

}

get_interface_info() {

}

get_tuned_profile() {

}

get_ram_info() {

}

get_selinux_status() {

}

run_lsdev() {

}

output_server_info() {

}

echo "Network Configuration:" >> $output_file

get_bond_info

get_interface_info

echo "System Configuration:" >> $output_file

get_tuned_profile

get_ram_info

if [ -f /etc/redhat-release ]; then

get_selinux_status

run_lsdev

output_server_info

echo -e "\nWarnings Summary:" >> $output_file

echo -e "$warnings" >> $output_file

cat $output_file