HP OA, Netapp, and Vmware environment monitoring with #Powershell
Here is a custom script I made to check HP Onboard Administrators, Netapp Controllers, and vCenter servers for health issues in Powershell. Cheers!
This relies on a few things first:
VMWare PowerCLI 5.5+
Netapp DataOnTap 4.0 modules
HP OA Powershell Cmdlets to function properly.
If you want to capture VM Snapshot alarms you must create an alarm in your vCenters called “VMSnapshot Running” . I personally set mine to if the snapshot is 2GB or larger, trigger the alarm.
You must export any credentials that are not fully integrated into your AD domain. I export $nacred, and $oacred with export-clixml, and then import those creds into the function upon load time. You would need to add a vCenter cred, and use -Credential $vccred (or whatever your variable is called) on the vmware piece if that is what your environment requires.
function check-env {
[System.Reflection.Assembly]::LoadWithPartialName("System.Diagnostics")
$sw = new-object system.diagnostics.stopwatch
$sw.Start()
############################################################
#### Set all static variables.
$oas = "oa1" , "oa2", "oa3", "oa4", "oa5"
$controllers = "filer1", "filer2", "filer3", "filer4"
$viservers = "vc1", "vc2"
$nacred = Import-clixml C:\users\user1\Documents\NACred.xml
$oacred = import-clixml C:\users\user1\Documents\OACred.xml
$nothing = ''
$vms = $null
#### Check all HP Enclosures, Fans, OAs, Interconnects, Power, and Blade health.
write-host "----- Checking all HP Onboard Administrators for alarms -----" -foregroundcolor "magenta" -backgroundcolor "black"
foreach ($oa in $oas){
write-progress "Checking HP OnBoard Administrators:"
; ""
; ""
$con = Connect-HPOA -OA $oa -Credential $oacred
$health = Get-HPOAHealth $con
$bladehealth = $health.bladehealth
$fanhealth = $health.FanHealth
$interconnecthealth = $health.InterconnectHealth
$Powerhealth = $health.PowerSupplyHealth
$OAhealth = $health.OnboardAdministratorHealth
$messages = "Absent", "OK"
### Check OA Blade Health
foreach ($item in $bladehealth) {
if ($item.Status -notin $messages) {
write-host "$oa has not OK Blade status on Bay:" $item.bay -foregroundcolor "red" -backgroundcolor "black"
$item.Status
$item.CorrectiveAction ; "" ; ""
$nothing = "something"
} else {
$nothing = ''
}
}
if ($nothing -eq $null -or $nothing -eq '') {
write-host "$oa has no active BLADE alarms or problems." -foregroundcolor "green" -backgroundcolor "black"
} else {
$whatever = "whatever"
}
### Check OA Fan Health
foreach ($item in $fanhealth) {
if ($item.Status -notin $messages) {
write-host "$oa has not OK FAN status on Bay:" $item.bay -foregroundcolor "red" -backgroundcolor "black"
$item.Status
$item.CorrectiveAction ; "" ; ""
$nothing = "something"
} else {
$nothing = ''
}
}
if ($nothing -eq $null -or $nothing -eq '') {
write-host "$oa has no active FAN alarms or problems." -foregroundcolor "green" -backgroundcolor "black"
} else {
$whatever = "whatever"
}
#### Check OA Interconnect Bay Health
foreach ($item in $interconnecthealth) {
if ($item.Status -notin $messages) {
write-host "$oa has not OK Interconnect status on Bay:" $item.bay -foregroundcolor "red" -backgroundcolor "black"
$item.Status
$item.CorrectiveAction ; "" ; ""
$nothing = "something"
} else {
$nothing = ''
}
}
if ($nothing -eq $null -or $nothing -eq '') {
write-host "$oa has no active INTERCONNECT BAY alarms or problems." -foregroundcolor "green" -backgroundcolor "black"
} else {
$whatever = "whatever"
}
### Check OA Power Supply Health
foreach ($item in $powerhealth) {
if ($item.Status -notin $messages) {
write-host "$oa has not OK Power Supply status on Bay:" $item.bay -foregroundcolor "red" -backgroundcolor "black"
$item.Status
$item.CorrectiveAction ; "" ; ""
$nothing = "something"
} else {
$nothing = ''
}
}
if ($nothing -eq $null -or $nothing -eq '') {
write-host "$oa has no active POWER SUPPLY alarms or problems." -foregroundcolor "green" -backgroundcolor "black"
} else {
$whatever = "whatever"
}
### Check Onboard Administrator Health
foreach ($item in $OAhealth) {
if ($item.Status -notin $messages) {
write-host "$oa has NOT OK OA status on Bay:" $item.bay -foregroundcolor "red" -backgroundcolor "black"
$item.Status
$item.CorrectiveAction ; "" ; ""
$nothing = "something"
} else {
$nothing = ''
}
}
if ($nothing -eq $null -or $nothing -eq '') {
write-host "$oa has no active OA bay alarms or problems." -foregroundcolor "green" -backgroundcolor "black"
} else {
$whatever = "whatever"
}
}
"";
"";
#### Check NETAPP Controllers TDKP and KNMD for Failed Disks, disconnected fiber connections, and channel failures.
write-host "----- Checking all Netapp Filers for Failed Disks, Channel Failures, failed aggregates, and offline luns or volumes -----" -foregroundcolor "magenta" -backgroundcolor "black"
; ""
; ""
foreach ($controller in $controllers) {
$nothing = ''
write-progress "Checking NetAPP Controllers for Failed Disks, Channel Failures, failed aggregates, and offline luns or volumes: "
Connect-NaController -Name $controller -Credential $nacred | out-null
### Check for Failed Disks
$disk = Get-NaDiskOwner | ? {$_.Failed -eq "True"} | ? {$_.Owner -eq $controller}
$shelfstatus = Get-NaShelf | Get-NaShelfEnvironment | where-object {$_.IsShelfChannelFailure -eq 1}
if ($disk -eq $null) {
write-host $controller "Has No Failed Disks." -foregroundcolor "green" -backgroundcolor "black"
} else {
write-host "The following controller $($controller) has failed disks:" -foregroundcolor "red" -backgroundcolor "black"
$disk | Select-Object -Property Name, SerialNumber, Owner, OwnerId, Pool, Failed | Format-Table -Wrap -Autosize
$diskdata = get-nadisk $disk.Name
$diskdata | Select-Object -Property Name, Shelf, Bay, Status, PhysSpace, RPM, FW, Model, Pool, Aggregate | Format-Table -Wrap -Autosize
$drivesize = $diskdata.PhysSpace
$converted = $drivesize/1TB
write-host "Failed Drive:" $disk.Name "Size is:" -foregroundcolor "red" -backgroundcolor "black"
$rounded = [math]::round($converted,2)
write-host $rounded"TB" -foregroundcolor "red" -backgroundcolor "black"
; ""
}
### Check for Shelf Channel Failures
if ($shelfstatus -eq $null) {
write-host "$controller has no Shelf Channel failures." -foregroundcolor "green" -backgroundcolor "black"
} else {
write-host "$controller has the following Shelf Channel failures:" -foregroundcolor "red" -backgroundcolor "black"
$shelfstatus
}
### Check if cluster partnering is enabled.
$cfstatus = get-nacluster
if ($cfstatus.State -ne 'CONNECTED' -and $cfstatus.IsEnabled -ne $true){
write-host "!!!!!!!!!!!!!!!!! Failover is not enabled on $($controller) and does not have a connected partner. !!!!!!!!!!!!!!!!!" -foregroundcolor "red" -backgroundcolor "black"
}
### Check for Failed aggregates, offline Volumes and Luns.
$aggs = get-naaggr
$vols = get-navol
$luns = get-nalun
foreach ($agg in $aggs){
if ($agg.State -ne 'Online'){
write-host "$($controller) has the following aggreates offline:"
write-host "!!!!!!!!!!!!!!!!! $($agg.Name) IS OFFLINE !!!!!!!!!!!!!!!!!" -foregroundcolor "red" -backgroundcolor "black"
}
}
foreach ($vol in $vols){
if ($vol.State -ne 'Online'){
write-host "$($controller) has the following Volumes offline:"
write-host "!!!!!!!!!!!!!!!!! $($vol.Name) IS OFFLINE !!!!!!!!!!!!!!!!!" -foregroundcolor "red" -backgroundcolor "black"
}
}
foreach ($lun in $luns){
if ($lun.Online -ne $true){
write-host "$($controller) has the following LUNs offline:"
write-host "!!!!!!!!!!!!!!!!! $($Lun.Path) IS OFFLINE !!!!!!!!!!!!!!!!!" -foregroundcolor "red" -backgroundcolor "black"
}
}
;""
}
; ""
; ""
##############################################################
#### Check VMWare Clusters, Hosts, Datastores, and VM's for triggered Alarms and high value settings.
Add-PSSnapin Vmware.VIMAutomation.Core | Out-Null
set-PowerCLIConfiguration -invalidCertificateAction "ignore" -confirm:$false | out-null
write-host "----- Checking VMWare Hosts, Datastores, and VM alarms -----" -foregroundcolor "magenta" -backgroundcolor "black"
foreach ($viserver in $viservers) {
$vms = ''
$vmwarehosts = ''
$datastores = ''
write-progress "Checking VMWare Clusters, Hosts, Datastores and VMs for Triggered Alarms: "
; ""
; ""
connect-viserver $viserver | out-null
#### Checking Cluster Settings HA/DRS.
$clusters = ''
$cluster = ''
$clusters = get-cluster
foreach ($cluster in $clusters){
if ($cluster.HAEnabled -eq $false){
write-host "!!!! $($viserver) - $($cluster.Name) does not have HA enabled !!!!" -foregroundcolor "red" -backgroundcolor "black"
} else {
write-host "$($viserver) - $($cluster.Name) has HA enabled" -foregroundcolor "green" -backgroundcolor "black"
}
if ($cluster.DRSAutomationLevel -notlike "*FullyAutomated*"){
write-host "!!!! $($viserver) - $($cluster.Name) DRS is not fully automated !!!!" -foregroundcolor "red" -backgroundcolor "black"
} else {
write-host "$($viserver) - $($cluster.Name) DRS is fully automated" -foregroundcolor "green" -backgroundcolor "black"
}
}
#### Checking Host alarms.
$vmwarehosts = get-vmhost | get-view
$alarm = ''
$definition = ''
foreach ($box in $vmwarehosts) {
if ($box.TriggeredAlarmState -ne $null -or $box.TriggeredAlarmState -ne '') {
$alarm = $box.TriggeredAlarmState.Alarm
$definition = Get-AlarmDefinition -Id $alarm
Write-host "$($box.Name) Has the following Host Alarms triggered:" -foregroundcolor "red" -backgroundcolor "black"
Write-host $definition.Name -backgroundcolor "black"
}
}
if ($vmwarehosts.TriggeredAlarmState -eq $null -or $vmwarehosts.TriggeredAlarmState -eq '') {
write-host "There are no active HOST alarms on:" $viserver -foregroundcolor "green" -backgroundcolor "black"
}
#### Checking Datastore alarms.
$datastores = get-datastore | get-view
$alarm = ''
$definition = ''
foreach ($store in $datastores) {
if ($store.TriggeredAlarmState -ne $null -or $store.TriggeredAlarmState -ne '') {
$alarm = $store.TriggeredAlarmState.Alarm
$definition = Get-AlarmDefinition -Id $alarm
Write-host "$($store.Name) Has the following Storage Alarms triggered:" -foregroundcolor "red" -backgroundcolor "black"
Write-host $definition.Name -backgroundcolor "black"
}
}
if ($datastores.TriggeredAlarmState -eq $null -or $datastores.TriggeredAlarmState -eq '') {
write-host "There are no active DATASTORE alarms on:" $viserver -foregroundcolor "green" -backgroundcolor "black"
}
#### Checking VM alarms and Snapshot dates.
$vms = get-vm | get-view
$alarm = ''
$definition = ''
$snapdate = ''
foreach ($vm in $vms) {
if ($vm.TriggeredAlarmState -ne $null -or $vm.TriggeredAlarmState -ne '') {
$alarm = $vm.TriggeredAlarmState.Alarm
$definition = Get-AlarmDefinition -Id $alarm
Write-host "$($vm.Name) Has the following VM alarms triggered:" -foregroundcolor "red" -backgroundcolor "black"
Write-host $definition.Name -backgroundcolor "black"
#### If alarm is Snapshot, show Snapshot name and Creation Date.
if ($definition.Name -eq "VMSnapshot Running") {
$snapdate = get-snapshot -VM $vm.Name
write-host "$($snapdate.Name) was created on:" $snapdate.Created -backgroundcolor "black"
write-host "Snapshot is the following size in GB:" $snapdate.SizeGB -backgroundcolor "black"
;""
}
}
}
if ($vms.TriggeredAlarmState -eq $null -or $vmview.TriggeredAlarmState -eq '') {
write-host "There are no active VM alarms on" $viserver -foregroundcolor "green" -backgroundcolor "black"
}
$vms = $null
$vms = get-vm | where {$_.Name -like "*_old*" -or $_.Name -like "*_old*"} | out-null
if ($vms -ne $null){
write-host "The following VM's have old in their names:" -foregroundcolor "red" -backgroundcolor "black"
$vms.Name
;""
}
$vms = $null
$vms = Get-VM | Where-Object {$_.Extensiondata.Runtime.ConsolidationNeeded}
if ($vms -ne $null){
write-host "Consolidating any triggered VMs" -foregroundcolor "green" -backgroundcolor "black"
foreach ($vm in $vms){
(Get-VM -Name $vm.Name).ExtensionData.ConsolidateVMDisks_Task()
write-host "Task sent for consolidation of the following VM: $($vm.Name) sent to vCenter"
}
}
disconnect-viserver $viserver -confirm:$false | out-null
;""
;""
}
;""
$sw.stop()
write-host "All of your sweet checks took this much time to run:" -foregroundcolor "green" -backgroundcolor "black"
$sw.Elapsed
}