<# .EXTERNALHELP CheckDatabaseRedundancy-help.xml #> # Copyright (c) Microsoft Corporation. All rights reserved. # # Checks the redundancy of databases by validating that they have at least N # configured and "healthy" copies. Active and passive copies are both counted. # # To use this script you need to provide either $MailboxDatabaseName or $MailboxServerName. # To generate events for Monitoring, you need to provide -MonitoringContext switch. [CmdletBinding(DefaultParametersetName="Server")] param( [Parameter(ParameterSetName="Database",Mandatory=$true,Position=0)] [string] $MailboxDatabaseName, # By default, check against the local server [Parameter(ParameterSetName="Server",Position=0)] [string] $MailboxServerName = $env:COMPUTERNAME, # Skip checking the "default" mailbox databases. eg: Mailbox Database 0017891750 # Specify $null (or an empty string) if you don't want to skip any databases. # # NOTE: When modifying this default value, remember to also modify it in the # function: Check-DatabaseRedundancyForCopyRemoval [Parameter(ParameterSetName="Server")] [string] $SkipDatabasesRegex = "^Mailbox Database \d{10}$", [Parameter(ParameterSetName="Monitoring",Mandatory=$true)] [Parameter(ParameterSetName="Database")] [Parameter(ParameterSetName="Server")] [switch] $MonitoringContext = $false, [Parameter(ParameterSetName="Monitoring")] [Parameter(ParameterSetName="Database")] [Parameter(ParameterSetName="Server")] [UInt32] $SleepDurationBetweenIterationsSecs = 60, [Parameter(ParameterSetName="Monitoring")] [Parameter(ParameterSetName="Database")] [Parameter(ParameterSetName="Server")] [Int32] $TerminateAfterDurationSecs = 3480, # 58 minutes; -1,0 are "Infinite" [Parameter(ParameterSetName="Monitoring")] [Parameter(ParameterSetName="Database")] [Parameter(ParameterSetName="Server")] [UInt32] $SuppressGreenEventForSecs = 600, # 10 minutes # If the total duration of being "red" exceeds this amount, raise the Red event [Parameter(ParameterSetName="Monitoring")] [Parameter(ParameterSetName="Database")] [Parameter(ParameterSetName="Server")] [UInt32] $ReportRedEventAfterDurationSecs = 1200, # 20 minutes # Once we raise a red event, report it periodically every $ReportRedEventIntervalSecs seconds. [Parameter(ParameterSetName="Monitoring")] [Parameter(ParameterSetName="Database")] [Parameter(ParameterSetName="Server")] [UInt32] $ReportRedEventIntervalSecs = 900, # 15 minutes [Parameter(ParameterSetName="Monitoring")] [Parameter(ParameterSetName="Database")] [Parameter(ParameterSetName="Server")] [switch] $SkipEventLogging = $false, [UInt32] $AtLeastNCopies = 2, # If false, detailed summary status is left out of the events/objects reported [switch] $ShowDetailedErrors = $false, # The email FROM address to use for the summary report [string] $SummaryMailFrom = $null, # Send a summary report email to the following addresses [string[]] $SendSummaryMailTos = $null, # Useful to "dot-source" this script as a library - call the script as such: # PS D:\Exchange Mailbox\v15\Scripts> . .\CheckDatabaseRedundancy.ps1 -DotSourceMode [Parameter(ParameterSetName="DotSourceMode",Mandatory=$true)] [switch] $DotSourceMode = $false ) Set-StrictMode -Version 2.0 Import-LocalizedData -BindingVariable CheckDatabaseRedundancy_LocalizedStrings -FileName CheckDatabaseRedundancy.strings.psd1 function LoadExchangeSnapin { if (! (Get-PSSnapin Microsoft.Exchange.Management.PowerShell.E2010 -ErrorAction:SilentlyContinue) ) { Add-PSSnapin Microsoft.Exchange.Management.PowerShell.E2010 } } LoadExchangeSnapin #--------------------------------------- # Aliases for commonly used enum types # #--------------------------------------- $CopyStatusType = [Microsoft.Exchange.Management.SystemConfigurationTasks.CopyStatus] $ReplicationTypeType = [Microsoft.Exchange.Data.Directory.SystemConfiguration.ReplicationType] $MountDialType = [Microsoft.Exchange.Data.Directory.SystemConfiguration.AutoDatabaseMountDial] #------------ # Constants # #------------ # This is the maximum copy queue length considered "healthy" for a passive copy. # Currently, this value is higher than BestAvailability to prevent transient alerts on spikes. $CopyQueueLengthThreshold = 100 $InspectorQueueLengthWarningThreshold = $CopyQueueLengthThreshold $InspectorQueueLengthFailedThreshold = 1000 #------------------- # Script variables # #------------------- [System.Diagnostics.Stopwatch] $script:copyStatusStopwatch = New-Object -TypeName System.Diagnostics.Stopwatch [System.Diagnostics.Stopwatch] $script:copyStatusAllStopwatch = New-Object -TypeName System.Diagnostics.Stopwatch [System.Diagnostics.Stopwatch] $script:clusterNodeStopwatch = New-Object -TypeName System.Diagnostics.Stopwatch [System.Diagnostics.Stopwatch] $script:clusterNodeOverallStopwatch = New-Object -TypeName System.Diagnostics.Stopwatch [System.Diagnostics.Stopwatch] $script:oneIterationStopwatch = New-Object -TypeName System.Diagnostics.Stopwatch $script:databaseToStatusTable = @{} # Hashtable indexed by DatabaseName, value of Collection $script:databasesToCheckTable = @{} # Hashtable for the databases that we want to check, indexed by DatabaseName $script:databaseStateTable = @{} # Hashtable indexed by DatabaseName, that holds the redundancy state of each DB $script:clusterNodeStateTable = @{} #Hashtable indexed by server name, that holds a boolean for whether the node is online $script:serverToSiteTable = @{} # Hashtable indexed by server name, that holds the site name $script:outputObjects = @() # List of objects to send to the output pipeline [Microsoft.Exchange.Data.Directory.Management.MailboxServer] $script:mailboxServer = $null [UInt64] $script:iteration = 0 $script:clusterOutput = $null [string]$script:dagName = $null [string[]]$script:dagSites = $null [System.Text.StringBuilder]$script:report = $null $script:IsDataCenterLibraryPresent = $false $script:ExitOut = $false function Is-DatabaseReplicated ([Microsoft.Exchange.Data.Directory.SystemConfiguration.MailboxDatabase] $mdb) { if ($mdb.ReplicationType -eq $ReplicationTypeType::Remote) { return $true; } return $false; } # Filters out recovery DBs and DBs that are excluded from monitoring filter Select-DatabasesToMonitor { if ($SkipDatabasesRegex) { # filter out the DBs matching the regex specified, or that are excluded from monitoring Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0067 -f $SkipDatabasesRegex) } else { # no database name filter specified, so check against all that are not excluded from monitoring Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0087) } if ( !$_.Recovery -and !$_.AutoDagExcludeFromMonitoring ) { if ($SkipDatabasesRegex) { if ($_.Name -inotmatch $SkipDatabasesRegex) { $_ } } else { $_ } } } function Is-DagServerOnline ([string] $serverName) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0000 -f $serverName,"Is-DagServerOnline") [bool]$isOnline = $false if ($script:clusterNodeStateTable.Contains($serverName)) { $isOnline = $script:clusterNodeStateTable[$serverName] } Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0001 -f $isOnline,"Is-DagServerOnline") return $isOnline } # Checks if the server is DatabaseCopyAutoActivationBlocked function Is-ServerActivationBlocked ([string] $serverName) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0089 -f $serverName,"Is-ServerActivationBlocked") [bool]$isBlocked = $false $server = Get-MailboxServer $serverName if ($server -and ($server.DatabaseCopyAutoActivationPolicy -eq 'Blocked')) { $isBlocked = $true } Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0090 -f $serverName, $isBlocked, "Is-ServerActivationBlocked") return $isBlocked } # The following states are possibly healthy for a passive copy: # Healthy, DisconnectedAndHealthy, SeedingSource function Is-PassiveCopyPossiblyHealthy ([Microsoft.Exchange.Management.SystemConfigurationTasks.DatabaseCopyStatusEntry] $status) { $healthy = $false; switch ($status.Status) { $CopyStatusType::Healthy { $healthy = $true } $CopyStatusType::DisconnectedAndHealthy { $healthy = $true } $CopyStatusType::SeedingSource { $healthy = $true } default { } } return $healthy } function Is-ActiveReplayServiceDown ([string] $databaseName, [string] $serverName) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0002 -f $databaseName,$serverName,"Is-ActiveReplayServiceDown") [Microsoft.Exchange.Management.SystemConfigurationTasks.DatabaseCopyStatusEntry[]] $statuses = @() $statuses = $script:databaseToStatusTable[$databaseName] $activeStatus = $statuses | where { $_.Name -eq "$databaseName\$serverName" } if (!$activeStatus) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0003 -f $databaseName,$serverName,$databaseName,$serverName,"Is-ActiveReplayServiceDown") return $true } if (!$activeStatus.ActiveCopy) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0004 -f $databaseName,$serverName,$databaseName,$serverName,"Is-ActiveReplayServiceDown") return $true } if ($activeStatus.Status -eq $CopyStatusType::ServiceDown) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0005 -f $activeStatus.Name,$databaseName,$serverName,"Is-ActiveReplayServiceDown") return $true } Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0006 -f $databaseName,$serverName,"Is-ActiveReplayServiceDown") return $false } function Populate-DatabasesTable ([Microsoft.Exchange.Data.Directory.SystemConfiguration.MailboxDatabase[]] $databases) { $script:databasesToCheckTable.Clear() Foreach ($database in $databases) { $script:databasesToCheckTable[$database.Name] = $database } } # Queries the DAG object for the cluster node states. Populates a hashtable indexed by server name, # with booleans for whether or not the node is online. function Populate-ClusterNodeStatus { $script:dagName = $null $script:clusterNodeStateTable.Clear() # Find the DAG name first. if ($script:mailboxServer) { $script:dagName = $script:mailboxServer.DatabaseAvailabilityGroup.Name } else { # running in DB mode, which means there should only be one DB in this table $db = $script:databasesToCheckTable.Values | select -First 1 $script:dagName = $db.MasterServerOrAvailabilityGroup.Name } Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0013 -f $script:dagName,"Populate-ClusterNodeStatus") $script:clusterNodeOverallStopwatch.Reset(); $script:clusterNodeOverallStopwatch.Start(); $dag = Get-DatabaseAvailabilityGroup $script:dagName -status if ($dag) { if (!$dag.Servers -or ` ($dag.Servers.Count -eq 0)) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0015 -f $script:dagName,"Populate-ClusterNodeStatus") } else { foreach ($server in $dag.Servers) { # Server is online if it is in the operational servers list $IsOnline = [bool] ($dag.OperationalServers | where {$_ -eq $server}) $script:clusterNodeStateTable.Add($server.Name, $IsOnline) } } } $script:clusterNodeOverallStopwatch.Stop() Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0018 -f $script:clusterNodeOverallStopwatch.Elapsed.TotalMilliseconds,"Populate-ClusterNodeStatus") } function Populate-ServerToSiteTable { $script:serverToSiteTable.Clear() $adSiteTable = @{} $adSites = @( Get-ADSite ) Foreach ($adSite in $adSites) { $adSiteTable[$adSite.Identity] = $adSite.Name } $servers = @( Get-ExchangeServer | where { $_.ServerRole -like "*Mailbox*" } ) Foreach ($server in $servers ) { if ((($server | Get-Member Site) -eq $null) -or ($server.Site -eq $null)) { $script:serverToSiteTable[$server.Name] = "#invalid-site#" } else { $script:serverToSiteTable[$server.Name] = $adSiteTable[$server.Site] } } } # This function returns an array of AD sites that every database in this database availability group should have copies on function Get-DatabaseAvailabilityGroupSites([string] $databaseAvailabilityGroupName) { $adSiteNames = @{} if ( $script:serverToSiteTable.Count -eq 0 ) { Populate-ServerToSiteTable } $dag = Get-DatabaseAvailabilityGroup $databaseAvailabilityGroupName -status if ($dag) { if (!$dag.Servers -or ` ($dag.Servers.Count -eq 0)) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0015 -f $databaseAvailabilityGroupName,"Get-DatabaseAvailabilityGroupSites") } else { foreach ($server in $dag.Servers) { if ( -not ($dag.StoppedMailboxServers | where {$_ -like "$server.*"}) ) { [string] $serverSite = $script:serverToSiteTable[$server.Name] if ($serverSite -and (-not $adSiteNames.Contains($serverSite))) { $adSiteNames[$serverSite] = $null } } } } } return @( $adSiteNames.Keys ) } function Check-Databases ([Microsoft.Exchange.Data.Directory.SystemConfiguration.MailboxDatabase[]] $databases, [string] $ParameterSetName) { Populate-DatabasesTable $databases # find the servers to check copy statuses on [String[]]$servers = Get-ServersForDatabases $databases if ($servers.Length -lt 2) { # Normally we should not get here, since we're only checking replicated DBs, which means # we should have at least 2 distinct servers. However, this can happen if copies are # removed while this script is running. Log-Warning ($CheckDatabaseRedundancy_LocalizedStrings.res_0019 -f $servers.Length,"Get-ServersForDatabases") } # get the status results and index them by database name $script:databaseToStatusTable.Clear() $script:databaseToStatusTable = Get-CopyStatusFromAllServers $servers $ParameterSetName | ` Group-Object -AsHashTable -Property DatabaseName # look up the cluster node status for the DAG Populate-ClusterNodeStatus Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0020 -f "Check-Databases") # Filter out the databases we are not going to check, and then perform the redundancy check. $script:databaseToStatusTable.Keys | ` where { $script:databasesToCheckTable.Contains( $_ ) } | ` foreach { Check-DatabaseRedundancy -dbName $_ } # NOTE: # If the DB is completely removed from AD while this script is running, we may keep reporting # a Red alert for it. If need be, we can log a green event for the DB in this case... } # This object represents a DB's redundancy state. It is initialized once at script startup and # is subsequently maintained over multiple passes of Check-Databases. function CreateEmptyDatabaseRedundancyEntry { [CheckHADatabaseRedundancy.DatabaseRedundancyEntry]$entry = New-Object -TypeName "CheckHADatabaseRedundancy.DatabaseRedundancyEntry" return $entry } function Initialize-DatabaseRedundancyEntry ([CheckHADatabaseRedundancy.DatabaseRedundancyEntry] $dbRedundancy) { $dbRedundancy.LastRedundancyCount = $dbRedundancy.CurrentRedundancyCount $dbRedundancy.LastState = $dbRedundancy.CurrentState $dbRedundancy.CurrentRedundancyCount = 0 $dbRedundancy.CurrentState = "Unknown" $dbRedundancy.CurrentErrorMessages = $null $dbRedundancy.ResetCurrentRedundancyCountPerSite() } function Get-SummaryCopyStatusString( [Parameter(Mandatory=$true,ValueFromPipeline=$true)] [Microsoft.Exchange.Management.SystemConfigurationTasks.DatabaseCopyStatusEntry] $status) { Begin { $statusOutputs = @() } Process { $statusOutput = $status | Select-Object *,` @{Name="RealCopyQueue"; Expression={ [Math]::Max($_.LastLogGenerated - $_.LastLogCopied, 0) }}, ` @{Name="InspectorQueue"; Expression={ [Math]::Max($_.LastLogCopied - $_.LastLogInspected, 0) }}, ` @{Name="ReplayQueue"; Expression={ $_.ReplayQueueLength }}, ` @{Name="CIState"; Expression={ $_.ContentIndexState }} $statusOutputs += $statusOutput } End { [string]$statusStr = ($statusOutputs | ft -Wrap Name,Status,RealCopyQueue,InspectorQueue,ReplayQueue,CIState | Out-String) $statusStr = $statusStr -replace "\s+$" # trim the white space at the end Write-Output $statusStr } } # Logic to decide if a DB has insufficient redundancy function Check-DatabaseRedundancy ([string] $dbName, [string] $skipCopyOnServer = [string]::Empty, [bool] $checkOneDatacenter = $true) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0021 -f $dbName,"Check-DatabaseRedundancy") # Populate server to site table, if not already done if ( $script:serverToSiteTable.Count -eq 0 ) { Populate-ServerToSiteTable } # Populate datbase availability group site table, if not already done if ( -not $script:dagSites ) { $script:dagSites = @( Get-DatabaseAvailabilityGroupSites -databaseAvailabilityGroupName $script:dagName) } [bool] $performDatacenterRedundancyCheck = $false if ( $checkOneDatacenter ) { $performDatacenterRedundancyCheck = $script:dagSites.Count -gt 1 } # Initialize the DB redundancy state if necessary if (!$script:databaseStateTable.Contains($dbName)) { $dbState = CreateEmptyDatabaseRedundancyEntry $dbState.DatabaseName = $dbName $script:databaseStateTable.Add($dbName, $dbState) Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0022 -f $dbName,"Check-DatabaseRedundancy") } # Retrieve the redundancy state object, and initialize states $dbRedundancy = $script:databaseStateTable[$dbName] Initialize-DatabaseRedundancyEntry $dbRedundancy # Get the list of copy status entries from the hashtable [Microsoft.Exchange.Management.SystemConfigurationTasks.DatabaseCopyStatusEntry[]] $statuses = @() $statuses = $script:databaseToStatusTable[$dbName] [string[]] $tmpErrMessages = @() [string] $errMsg = $null [string] $summaryStatusStr = $null [string] $adSiteName = $null [CheckHADatabaseRedundancy.CopyCheckState]$checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Unknown # In case there's only one configured copy, let's report that as an error if ($statuses.Count -lt $AtLeastNCopies) { $tmpErrMessages += "The number of configured copies for database '$dbName' ($($statuses.Count)) is less than the required redundancy count ($AtLeastNCopies)." } Foreach ($status in $statuses) { $adSiteName = $script:serverToSiteTable[$status.MailboxServer] # Check the health of the active or passive copy ($errMsg,$checkState) = Get-DatabaseCopyHealth $status if ($checkState -eq [CheckHADatabaseRedundancy.CopyCheckState]::Passed) { if ( $skipCopyOnServer -and ([string]::Equals($status.MailboxServer,$skipCopyOnServer,[StringComparison]::OrdinalIgnoreCase))) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0086 -f $skipCopyOnServer,"Check-DatabaseRedundancy") } else { $dbRedundancy.CurrentRedundancyCount++ $dbRedundancy.IncrementRedundancyCountInSite($adSiteName) if ( $status.ContentIndexState -eq "Healthy" ) { $dbRedundancy.IncrementSearchRedundancyCountInSite($adSiteName) } } } elseif ($checkState -eq [CheckHADatabaseRedundancy.CopyCheckState]::Warning) { if ( $skipCopyOnServer -and ([string]::Equals($status.MailboxServer,$skipCopyOnServer,[StringComparison]::OrdinalIgnoreCase))) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0086 -f $skipCopyOnServer,"Check-DatabaseRedundancy") } else { $dbRedundancy.CurrentRedundancyCount++ $dbRedundancy.IncrementRedundancyCountInSite($adSiteName) if ( $status.ContentIndexState -eq "Healthy" ) { $dbRedundancy.IncrementSearchRedundancyCountInSite($adSiteName) } $tmpErrMessages += $errMsg } } else { # This copy has failed the check, so let's record the reason why $tmpErrMessages += $errMsg } } # If we've got some errors, remember them for emailing purposes if ($tmpErrMessages.Length -gt 0) { # Append the summary status $summaryStatusStr = ($statuses | Get-SummaryCopyStatusString) $tmpErrMessages += "$summaryStatusStr" # Add the overall errors to the history for this DB $dbRedundancy.AddErrorRecordToHistory( [DateTime]::UtcNow, $tmpErrMessages ) } if ($ShowDetailedErrors) { # Additionally, log the copy status output into the event $statusStr = $summaryStatusStr if (!$statusStr) { $statusStr = ($statuses | Get-SummaryCopyStatusString) if ($statusStr) { $tmpErrMessages += "`n`n================`n Summary Status `n================`n`n$statusStr" } } $statusStr = ($statuses | fl | Out-String) if ($statusStr) { $tmpErrMessages += "`n`n===============`n Full Status `n===============`n`n$statusStr" } } if ($tmpErrMessages.Length -gt 0) { $dbRedundancy.CurrentErrorMessages = $tmpErrMessages } Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0023 -f $dbRedundancy.CurrentRedundancyCount,$dbRedundancy.LastRedundancyCount,$dbName,"Check-DatabaseRedundancy") # Decide if the state is Red or Green if ($dbRedundancy.CurrentRedundancyCount -lt $AtLeastNCopies) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0024 -f $dbName,$AtLeastNCopies,"Check-DatabaseRedundancy") $dbRedundancy.CurrentState = [CheckHADatabaseRedundancy.AlertState]::Red } else { if ($performDatacenterRedundancyCheck -and ($dbRedundancy.CheckIfInOneDatacenter())) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0085 -f $dbName,"Check-DatabaseRedundancy") $dbRedundancy.CurrentState = [CheckHADatabaseRedundancy.AlertState]::Red } else { $dbRedundancy.CurrentState = [CheckHADatabaseRedundancy.AlertState]::Green } } # record the state transition times [datetime]$nowUtc = [DateTime]::UtcNow if ($dbRedundancy.IsTransitioningState) { $dbRedundancy.LastStateTransitionUtc = $nowUtc if ($dbRedundancy.CurrentState -eq [CheckHADatabaseRedundancy.AlertState]::Green) { $dbRedundancy.LastGreenTransitionUtc = $nowUtc if ($dbRedundancy.LastRedTransitionUtc) { [TimeSpan]$prevTimeInRed = $dbRedundancy.LastGreenTransitionUtc.Subtract( $dbRedundancy.LastRedTransitionUtc ) $dbRedundancy.PreviousTotalRedDuration = $dbRedundancy.PreviousTotalRedDuration.Add( $prevTimeInRed ) } } elseif ($dbRedundancy.CurrentState -eq [CheckHADatabaseRedundancy.AlertState]::Red) { $dbRedundancy.LastRedTransitionUtc = $nowUtc } } # Report a red/green event if necessary (suppression may occur if MonitoringContext is specified) PossiblyReport-RedGreenStatus $dbRedundancy } # Reports Red/Green status via mail/event etc, taking into account whether or not # we are running in the MonitoringContext (which affects suppression) function PossiblyReport-RedGreenStatus ( [CheckHADatabaseRedundancy.DatabaseRedundancyEntry] $dbRedundancy ) { if ($MonitoringContext) { # In the monitoring context, we should run the suppression logic if ($dbRedundancy.CurrentState -eq [CheckHADatabaseRedundancy.AlertState]::Green) { [int]$timeInGreenSecs = Get-ElapsedTimeInSeconds $dbRedundancy.LastGreenTransitionUtc Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0025 -f $dbRedundancy.DatabaseName,$timeInGreenSecs,"PossiblyReport-RedGreenStatus") if ($SuppressGreenEventForSecs -le 0 -or ` (($timeInGreenSecs -gt $SuppressGreenEventForSecs) -and ` (!$dbRedundancy.HasReportedGreenEvent))) { # Only log a green event once, or if it transitions into Green again Report-GreenStatus $dbRedundancy } } elseif ($dbRedundancy.CurrentState -eq [CheckHADatabaseRedundancy.AlertState]::Red) { # We need to log an event if the total duration of being in Red (including flickering states) # is larger than $ReportRedEventAfterDurationSecs. $totalRedSecs = $dbRedundancy.TotalRedDuration.TotalSeconds Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0026 -f $dbRedundancy.DatabaseName,$totalRedSecs,"PossiblyReport-RedGreenStatus") if ($ReportRedEventAfterDurationSecs -le 0 -or ` $totalRedSecs -gt $ReportRedEventAfterDurationSecs) { # Reporting a red event for the first time if ($dbRedundancy.LastRedReportedUtc -eq $null) { Report-RedStatus $dbRedundancy } else { # Additionally, we need to log an event every $ReportRedEventIntervalSecs seconds # while the DB is in "Red". [int]$timeSinceLastRedEventSecs = Get-ElapsedTimeInSeconds $dbRedundancy.LastRedReportedUtc if ($timeSinceLastRedEventSecs -gt $ReportRedEventIntervalSecs) { Report-RedStatus $dbRedundancy } } } } } else { # No monitoring context, so no suppression if ($dbRedundancy.CurrentState -eq [CheckHADatabaseRedundancy.AlertState]::Green) { Report-GreenStatus $dbRedundancy } elseif ($dbRedundancy.CurrentState -eq [CheckHADatabaseRedundancy.AlertState]::Red) { Report-RedStatus $dbRedundancy } } } # Reports Green status via mail/event as appropriate [no suppression]. function Report-GreenStatus ( [CheckHADatabaseRedundancy.DatabaseRedundancyEntry] $dbRedundancy ) { $dbRedundancy.LastGreenReportedUtc = [DateTime]::UtcNow $dbRedundancy.HasReportedGreenEvent = $true [string]$dbCopyForAlerting = Get-DatabaseCopyForAlerting $dbRedundancy.DatabaseName [bool]$writeOutput = $true Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0027 -f $dbCopyForAlerting,"Report-GreenStatus") if ($MonitoringContext) { if (!$SkipEventLogging) { # Write green event log into Application log $writeOutput = $false # MonitoringDatabaseRedundancyCheckPassed - EventId 4114 Write-HAAppLogInformationEvent "40041012" 1 @($dbCopyForAlerting, $dbRedundancy.CurrentRedundancyCount, $dbRedundancy.GetErrorStringForAlerting()) if ($script:dagSites.Count -gt 1) { # MonitoringDatabaseOneDatacenterCheckSuccess - EventId 4334 Write-HAAppLogInformationEvent "400410EE" 1 @($dbCopyForAlerting, $dbRedundancy.CurrentRedundancyCount, $dbRedundancy.GetErrorStringForAlerting()) } } } if ($writeOutput) { Write-Output $dbRedundancy } } # Reports Red status via mail/event as appropriate [no suppression]. function Report-RedStatus ( [CheckHADatabaseRedundancy.DatabaseRedundancyEntry] $dbRedundancy ) { $dbRedundancy.LastRedReportedUtc = [DateTime]::UtcNow $dbRedundancy.HasReportedGreenEvent = $false [string]$dbCopyForAlerting = Get-DatabaseCopyForAlerting $dbRedundancy.DatabaseName [bool]$writeOutput = $true Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0028 -f $dbCopyForAlerting,"Report-RedStatus") if ($MonitoringContext) { if (!$SkipEventLogging) { # Write red event log into Application log $writeOutput = $false if ( $dbRedundancy.IsOneDatacenter ) { # This is not one-copy, but one-datacenter event # MonitoringDatabaseRedundancyCheckPassed - EventId 4114 Write-HAAppLogInformationEvent "40041012" 1 @($dbCopyForAlerting, $dbRedundancy.CurrentRedundancyCount, $dbRedundancy.GetErrorStringForAlerting()) # MonitoringDatabaseOneDatacenterCheckFailed - EventId 4133 Write-HAAppLogErrorEvent "C0041025" 1 @($dbCopyForAlerting, $dbRedundancy.CurrentRedundancyCount, $dbRedundancy.GetErrorStringForAlerting()) } else { # MonitoringDatabaseRedundancyCheckFailed - EventId 4113 Write-HAAppLogErrorEvent "C0041011" 1 @($dbCopyForAlerting, $dbRedundancy.CurrentRedundancyCount, $dbRedundancy.GetErrorStringForAlerting()) } } } if ($writeOutput) { Write-Output $dbRedundancy } } function Get-DatabaseCopyForAlerting ( [string] $dbName ) { # We need to report the Database name (not the DBCopy name) return $dbName } function Get-ElapsedTimeInSeconds( [DateTime] $startTimeUtc ) { [TimeSpan]$elapsedTime = [DateTime]::UtcNow.Subtract( $startTimeUtc ) [int]$elapsedSeconds = [int][System.Math]::Floor($elapsedTime.TotalSeconds) return $elapsedSeconds } # Returns the string describing why the copy was not healthy. $null if the copy is healthy. function Get-DatabaseCopyHealth ([Microsoft.Exchange.Management.SystemConfigurationTasks.DatabaseCopyStatusEntry] $copyStatus) { if ($copyStatus.ActiveCopy) { return Get-ActiveDatabaseCopyHealth $copyStatus } else { return Get-PassiveDatabaseCopyHealth $copyStatus } } # Returns the string describing why the copy was not healthy. $null if the copy is healthy. function Get-ActiveDatabaseCopyHealth ( ` [Microsoft.Exchange.Management.SystemConfigurationTasks.DatabaseCopyStatusEntry] $copyStatus) { $dbCopy = $copyStatus.Name $dbName = $copyStatus.DatabaseName $server = $copyStatus.MailboxServer [string]$errMsg = $null [CheckHADatabaseRedundancy.CopyCheckState]$checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Unknown Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0029 -f $copyStatus.Status,$copyStatus.ErrorEventId,$copyStatus.ErrorMessage,$copyStatus.SuspendComment,$dbCopy,"Get-ActiveDatabaseCopyHealth") # Log that the DB is not replicated if ($VerbosePreference -ne "SilentlyContinue") { if ( !(Is-DatabaseReplicated ($script:databasesToCheckTable[$dbName])) ) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0030 -f $dbName,"Get-ActiveDatabaseCopyHealth") } } # First, we need the cluster node status to be Up if (!(Is-DagServerOnline $server)) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "Active copy '$dbCopy' is not UP according to clustering." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-ActiveDatabaseCopyHealth") return $errMsg,$checkState } # Check if the server is activation blocked if (Is-ServerActivationBlocked $server) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "$server has DatabaseCopyAutoActivationPolicy Blocked!." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-ActiveDatabaseCopyHealth") return $errMsg,$checkState } # Check if copy is activation suspended if ($copyStatus.ActivationSuspended -eq $true) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "$dbCopy is ActivationSuspended." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-ActiveDatabaseCopyHealth") return $errMsg,$checkState } # If replay service is down, we'll just assume that this copy isn't healthy since # all the passives will be "stalled", which means we're anyway down to at most 1 copy. if ($copyStatus.Status -eq $CopyStatusType::ServiceDown) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "Active copy '$dbCopy' has replay service down. Assuming the copy is unhealthy." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-ActiveDatabaseCopyHealth") return $errMsg,$checkState } if ( ($copyStatus.Status -eq $CopyStatusType::Dismounted) ` -or ($copyStatus.Status -eq $CopyStatusType::Dismounting) ) { # There may have been a permanent failure (such as a DB corruption) that is preventing # the DB from mounting. If so, there will be an error message recorded. if ($copyStatus.ErrorMessage) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "Active copy '$dbCopy' is dismounted with an error. Error: $($copyStatus.ErrorMessage)." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-ActiveDatabaseCopyHealth") return $errMsg,$checkState } else { # A dismounted copy is fine $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Passed return $errMsg,$checkState } } if ($copyStatus.Status -eq $CopyStatusType::Mounted) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Passed return $errMsg,$checkState } elseif ($copyStatus.Status -eq $CopyStatusType::Mounting) { # NOTE: This is only a warning and doesn't cause RED alert to go off! $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Warning $errMsg = "Active copy '$dbCopy' is in 'Mounting' state." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0032 -f $errMsg,"Get-ActiveDatabaseCopyHealth") return $errMsg,$checkState } # Any other state, assume the worst $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "Active copy '$dbCopy' has some unknown/unhealthy state. Status: $($copyStatus.Status)." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-ActiveDatabaseCopyHealth") return $errMsg,$checkState } # Returns the string describing why the copy was not healthy. $null if the copy is healthy. function Get-PassiveDatabaseCopyHealth ( ` [Microsoft.Exchange.Management.SystemConfigurationTasks.DatabaseCopyStatusEntry] $copyStatus) { $dbCopy = $copyStatus.Name $dbName = $copyStatus.DatabaseName $server = $copyStatus.MailboxServer $activeServer = $copyStatus.ActiveDatabaseCopy [string]$errMsg = $null [CheckHADatabaseRedundancy.CopyCheckState]$checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Unknown Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0033 -f $copyStatus.Status,$copyStatus.CopyQueueLength,$copyStatus.ReplayQueueLength,$copyStatus.ErrorEventId,$copyStatus.ErrorMessage,$copyStatus.SuspendComment,$dbCopy,$dbName,"Get-PassiveDatabaseCopyHealth") # First, we need the cluster node status to be Up if (!(Is-DagServerOnline $server)) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "Passive copy '$dbCopy' is not UP according to clustering." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-PassiveDatabaseCopyHealth") return $errMsg,$checkState } # Check if the server is activation blocked if (Is-ServerActivationBlocked $server) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "$server has DatabaseCopyAutoActivationPolicy Blocked!." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-ActiveDatabaseCopyHealth") return $errMsg,$checkState } # Check if copy is activation suspended if ($copyStatus.ActivationSuspended -eq $true) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "$dbCopy is ActivationSuspended." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-ActiveDatabaseCopyHealth") return $errMsg,$checkState } # Rule out the obviously unhealthy cases first if (!(Is-PassiveCopyPossiblyHealthy $copyStatus)) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "Passive copy '$dbCopy' is not in a good state. Status: $($copyStatus.Status)." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-PassiveDatabaseCopyHealth") return $errMsg,$checkState } # Check the *real* copy queue length first (i.e. not including the inspector queue) $realCopyQ = [Math]::Max($copyStatus.LastLogGenerated - $copyStatus.LastLogCopied, 0) if ($realCopyQ -gt $CopyQueueLengthThreshold) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "Passive copy '$dbCopy' has actual log copy queue higher than the threshold of '$CopyQueueLengthThreshold'. Copy queue: $realCopyQ." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-PassiveDatabaseCopyHealth") return $errMsg,$checkState } # Check the inspector queue length $inspectorQ = [Math]::Max($copyStatus.LastLogCopied - $copyStatus.LastLogInspected, 0) if ($inspectorQ -gt $InspectorQueueLengthFailedThreshold) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "Passive copy '$dbCopy' has an inspector queue higher than the failure threshold of '$InspectorQueueLengthFailedThreshold'. Inspector queue: $inspectorQ." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-PassiveDatabaseCopyHealth") return $errMsg,$checkState } # Even if the copy queue is small, we can't trust it because the active replay service might # be down, in which case the queues will be stale. (E14# 138911) # So, if the active status is "ServiceDown", but the node is up, we can be fairly certain # that we shouldn't trust the queues. if ((Is-ActiveReplayServiceDown $dbName $activeServer) -and ` (Is-DagServerOnline $activeServer)) { $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Failed $errMsg = "Passive copy '$dbCopy' has a small copy queue length, but it could be stale. The active replay service on server '$activeServer' appears to be down. Copy queue: $($copyStatus.CopyQueueLength)." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0031 -f $errMsg,"Get-PassiveDatabaseCopyHealth") return $errMsg,$checkState } if ($inspectorQ -gt $InspectorQueueLengthWarningThreshold) { # NOTE: This is only a warning and doesn't cause RED alert to go off! $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Warning $errMsg = "Passive copy '$dbCopy' has an inspector queue higher than the warning threshold of '$InspectorQueueLengthWarningThreshold'. Inspector queue: $inspectorQ." Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0032 -f $errMsg,"Get-PassiveDatabaseCopyHealth") return $errMsg,$checkState } # The copy is good in the time alotted... $checkState = [CheckHADatabaseRedundancy.CopyCheckState]::Passed return $errMsg,$checkState } # Given a list of databases, find the set of unique servers hosting copies of all of them. # Returns an array with all the server names. function Get-ServersForDatabases ( ` [Microsoft.Exchange.Data.Directory.SystemConfiguration.MailboxDatabase[]] $databases) { $servers = @{} Foreach ($db in $databases) { Foreach ($dbCopy in $db.AllDatabaseCopies) { if (!$servers.Contains($dbCopy.HostServerName)) { $servers.Add($dbCopy.HostServerName, 1); } } } # convert the hashtable into an array [String[]]$serversList = @() $servers.Keys | % { $serversList += $_ } Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0034 -f $serversList.Length,"Get-ServersForDatabases") return $serversList; } # Runs get-mailboxdatabasecopystatus against copy(ies) on a server and returns an array of status results. # Return type: Microsoft.Exchange.Management.SystemConfigurationTasks.DatabaseCopyStatusEntry[] function Get-CopyStatusFromServer ([string] $server, [string] $ParameterSetName) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0035 -f $server,"Get-CopyStatusFromServer") $script:copyStatusStopwatch.Reset(); $script:copyStatusStopwatch.Start(); [Microsoft.Exchange.Management.SystemConfigurationTasks.DatabaseCopyStatusEntry[]] $statuses = @() if ($ParameterSetName -eq "Database" ) { $statuses = @( Get-MailboxDatabaseCopyStatus "$MailboxDatabaseName\$server" -UseServerCache ) } else { $statuses = Get-MailboxDatabaseCopyStatus -Server $server -UseServerCache } $script:copyStatusStopwatch.Stop(); Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0036 -f $script:copyStatusStopwatch.Elapsed.TotalMilliseconds,$server,"Get-CopyStatusFromServer") return $statuses } # Synchronously executes get-mdbcs against all the specified servers and returns an # array of type DatabaseCopyStatusEntry, which holds all the statuses returned. function Get-CopyStatusFromAllServers ([String[]] $servers, [string] $ParameterSetName) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0037 -f "Get-CopyStatusFromAllServers") $script:copyStatusAllStopwatch.Reset(); $script:copyStatusAllStopwatch.Start(); [Microsoft.Exchange.Management.SystemConfigurationTasks.DatabaseCopyStatusEntry[]] $allStatuses = @() Foreach ($server in $servers) { [Microsoft.Exchange.Management.SystemConfigurationTasks.DatabaseCopyStatusEntry[]] $statuses = @() $statuses = Get-CopyStatusFromServer $server $ParameterSetName $allStatuses += $statuses } $script:copyStatusAllStopwatch.Stop(); Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0038 -f $script:copyStatusAllStopwatch.Elapsed.TotalMilliseconds,"Get-CopyStatusFromAllServers") return $allStatuses } ####################################################################### # Dynamic code compiler logic ####################################################################### function ConstructReferences([Array]$References) { # # Build up a compiler params object... $refs = @() $refs.AddRange( @("${framework}\System.dll", "${framework}\system.windows.forms.dll", "${framework}\System.data.dll", "${framework}\System.Drawing.dll", "${framework}\System.Xml.dll")) if (($References -ne $null) -and ($References.Count -ge 1)) { foreach ($refAssembly in $References) { [string] $refTmp = $refAssembly if ($refTmp.IndexOf("\") -eq -1) { $refTmp = "${framework}\$refTmp" } $refs.Add($refTmp); } } return $refs } # Compile the types to be used for tracking the Database Redundancy state. # The compilation is only performed once per runspace and is entirely in memory. function Prepare-DatabaseRedundancyEntryDefinition { $code = ' using System; using System.Collections.Generic; using System.Diagnostics; namespace CheckHADatabaseRedundancy { public enum AlertState : int { Unknown = 0, Green, Red } // Enum describing the state of an individual database copy. At the moment, // both Passed and Warning are treated as having passed the redundancy check // and hence CurrentRedundancyCount is incremented. public enum CopyCheckState : int { Unknown = 0, Passed, Warning, Failed } public class DatabaseRedundancyEntry { public class ErrorRecord { public DateTime ErrorTime { get; set; } public string[] ErrorMessages { get; set; } public string GetErrorStringForAlerting() { if (ErrorMessages == null || ErrorMessages.Length == 0) { return String.Empty; } return String.Join("\n", ErrorMessages); } } public string DatabaseName { get; set; } public int LastRedundancyCount { get; set; } public int CurrentRedundancyCount { get; set; } public AlertState LastState { get; set; } public AlertState CurrentState { get; set; } public DateTime? LastStateTransitionUtc { get; set; } public DateTime? LastGreenTransitionUtc { get; set; } public DateTime? LastRedTransitionUtc { get; set; } public DateTime? LastGreenReportedUtc { get; set; } public bool HasReportedGreenEvent { get; set; } public DateTime? LastRedReportedUtc { get; set; } // the previous total red duration (not counting the current stretch of reds) public TimeSpan PreviousTotalRedDuration { get; set; } public TimeSpan TotalRedDuration { get { if (this.CurrentState == AlertState.Red) { // count the current duration of reds return this.PreviousTotalRedDuration + (DateTime.UtcNow - this.LastRedTransitionUtc.Value); } else { return this.PreviousTotalRedDuration; } } } public bool IsTransitioningState { get { return LastState != CurrentState; } } public bool HasErrorsInHistory { get { if (this.ErrorHistory == null || this.ErrorHistory.Count == 0) { return false; } return true; } } public string[] CurrentErrorMessages { get; set; } public List ErrorHistory { get; private set; } public string GetErrorStringForAlerting() { if (CurrentErrorMessages == null || CurrentErrorMessages.Length == 0) { return String.Empty; } return String.Join("\n", CurrentErrorMessages); } // Create a copy of the errorMessages array and then add the record to the history public void AddErrorRecordToHistory(DateTime errorTime, string[] errorMessages) { string[] tmpMessages = new string[errorMessages.Length]; errorMessages.CopyTo(tmpMessages, 0); ErrorRecord er = new ErrorRecord(); er.ErrorTime = errorTime; er.ErrorMessages = tmpMessages; if (this.ErrorHistory == null) { this.ErrorHistory = new List(15); } this.ErrorHistory.Add(er); } public bool CheckIfInOneDatacenter() { bool f_oneDatacenter = false; if ( this.CurrentRedundancyCountPerSite == null | this.CurrentRedundancyCountPerSite.Count == 0 ) { f_oneDatacenter = true; } else { int numberOfRedundantSites = 0; foreach( KeyValuePair redundancyCount in this.CurrentRedundancyCountPerSite ) { if ( redundancyCount.Value >= 1 ) { numberOfRedundantSites++; } if ( numberOfRedundantSites >= 2 ) { break; } } f_oneDatacenter = ( numberOfRedundantSites < 2 ); } this.IsOneDatacenter = f_oneDatacenter; return f_oneDatacenter; } private bool m_OneDatacenter = false; public bool IsOneDatacenter { get { return this.m_OneDatacenter; } private set { this.m_OneDatacenter = value; } } public Dictionary CurrentRedundancyCountPerSite { get; private set; } public Dictionary CurrentSearchRedundancyCountPerSite { get; private set; } public void ResetCurrentRedundancyCountPerSite() { if ( this.CurrentRedundancyCountPerSite != null ) { this.CurrentRedundancyCountPerSite.Clear(); this.CurrentRedundancyCountPerSite = null; } if ( this.CurrentSearchRedundancyCountPerSite != null ) { this.CurrentSearchRedundancyCountPerSite.Clear(); this.CurrentSearchRedundancyCountPerSite = null; } this.IsOneDatacenter = false; } public void IncrementRedundancyCountInSite(string adSiteName) { if ( this.CurrentRedundancyCountPerSite == null ) { this.CurrentRedundancyCountPerSite = new Dictionary(2); } int currentValue = 0; if (this.CurrentRedundancyCountPerSite.TryGetValue(adSiteName, out currentValue)) { this.CurrentRedundancyCountPerSite[adSiteName] = currentValue + 1; } else { this.CurrentRedundancyCountPerSite[adSiteName] = 1; } } public int GetRedundancyCountInSite(string adSiteName) { if ( this.CurrentRedundancyCountPerSite == null ) { return 0; } else { int currentValue = 0; if (this.CurrentRedundancyCountPerSite.TryGetValue(adSiteName, out currentValue)) { return currentValue; } else { return 0; } } } public void IncrementSearchRedundancyCountInSite(string adSiteName) { if ( this.CurrentSearchRedundancyCountPerSite == null ) { this.CurrentSearchRedundancyCountPerSite = new Dictionary(2); } int currentValue = 0; if (this.CurrentSearchRedundancyCountPerSite.TryGetValue(adSiteName, out currentValue)) { this.CurrentSearchRedundancyCountPerSite[adSiteName] = currentValue + 1; } else { this.CurrentSearchRedundancyCountPerSite[adSiteName] = 1; } } public int GetSearchRedundancyCountInSite(string adSiteName) { if ( this.CurrentSearchRedundancyCountPerSite == null ) { return 0; } else { int currentValue = 0; if (this.CurrentSearchRedundancyCountPerSite.TryGetValue(adSiteName, out currentValue)) { return currentValue; } else { return 0; } } } } public static class EventLogger { public static void WriteLocalizedEvent( string logName, // eg: Application string sourceName, // eg: MSExchangeRepl long eventId, // Message resource ID: eg: (long)0xC0041011 int categoryId, // category of the event EventLogEntryType entryType, // error, information, warning byte[] data, params object[] messageArgs) { EventLog eventLog = new EventLog(logName, Environment.MachineName, sourceName); EventInstance instance = new EventInstance( eventId, categoryId, entryType); eventLog.WriteEvent(instance, data, messageArgs); } } }' $checkCompiledCmd = { # Check if the type is loaded. If not, a RuntimeException is thrown. [CheckHADatabaseRedundancy.AlertState]; } [bool]$isCompiled = TryExecute-ScriptBlock -runCommand:$checkCompiledCmd -silentOnErrors:$true if (!$isCompiled) { ################################################################## # So now we compile the code and use .NET object access to run it. ################################################################## Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0042 ) Add-Type -TypeDefinition $code Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0043 ) } } function Write-HAAppLogInformationEvent( [Parameter(Mandatory=$true)] [string] $eventId, # eg: "C0041011" [Parameter(Mandatory=$true)] [int] $categoryId, [Object[]] $messageArgs) { Write-LocalizedEventLog "Application" "MSExchangeRepl" $eventId $categoryId ` "Information" $null $messageArgs } function Write-HAAppLogWarningEvent( [Parameter(Mandatory=$true)] [string] $eventId, # eg: "C0041011" [Parameter(Mandatory=$true)] [int] $categoryId, [Object[]] $messageArgs) { Write-LocalizedEventLog "Application" "MSExchangeRepl" $eventId $categoryId ` "Warning" $null $messageArgs } function Write-HAAppLogErrorEvent( [Parameter(Mandatory=$true)] [string] $eventId, # eg: "C0041011" [Parameter(Mandatory=$true)] [int] $categoryId, [Object[]] $messageArgs) { Write-LocalizedEventLog "Application" "MSExchangeRepl" $eventId $categoryId ` "Error" $null $messageArgs } function Write-LocalizedEventLog( [Parameter(Mandatory=$true)] [string] $logName, [Parameter(Mandatory=$true)] [string] $sourceName, [Parameter(Mandatory=$true)] [string] $eventId, # eg: "C0041011" [Parameter(Mandatory=$true)] [int] $categoryId, [Parameter(Mandatory=$true)] [System.Diagnostics.EventLogEntryType] $entryType, [Byte[]] $data, [Object[]] $messageArgs) { # parse the eventId into an Int64 first. [Int64]$id = [Int64]::Parse($eventId, [System.Globalization.NumberStyles]::HexNumber) [CheckHADatabaseRedundancy.EventLogger]::WriteLocalizedEvent( ` $logName, $sourceName, $id , $categoryId, $entryType, $data, $messageArgs) } # Common function to run a scriptblock, log any error that occurred, and return # a boolean to indicate whether it was successful or not. # NOTE: ErrorActionPreference of "Stop" is used to catch all errors. # # Optional parameters: # # cleanupCommand # This scriptblock will be executed with ErrorActionPreference of "Continue", # if an error occurred while running $runCommand. # # throwOnError # If true, the error from $runCommand will be rethrown. Otherwise 'false' is returned on error. # # silentOnErrors # If true, the error from $runCommand will not be logged via Log-ErrorRecord (i.e. Write-Error) function TryExecute-ScriptBlock ([ScriptBlock]$runCommand, [ScriptBlock]$cleanupCommand={}, [bool]$throwOnError=$false, [bool]$silentOnErrors=$false) { # Run the following in a separate script block so that we can change # ErrorActionPerefence without affecting the rest of the script. &{ $ErrorActionPreference = "Stop" [bool]$success = $false; try { $ignoredObjects = @(&$runCommand) $success = $true; } catch { # Any error will end up in this catch block # For some reason, PS does not write out any errors unless I use this # scriptblock with "Continue" ErrorActionPreference. &{ $ErrorActionPreference = "Continue" if (!$silentOnErrors) { Log-ErrorRecord $_ } # Run the cleanup scriptblock $ignoredObjects = @(&$cleanupCommand) } if ($throwOnError) { throw } } # Curious PS behavior: It appears that 'return' trumps 'throw', so don't return... if (!$throwOnError -or $success) { return $success } } } # Sleep for the specified duration (in seconds) function Sleep-ForSeconds ( [int]$sleepSecs ) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0044 -f $sleepSecs) Start-Sleep $sleepSecs } # Common function to retrieve the current UTC time string function Get-CurrentTimeString { return [DateTime]::UtcNow.ToString("[HH:mm:ss.fff UTC]") } # Common function for verbose logging function Log-Verbose ( [string]$msg ) { $timeStamp = Get-CurrentTimeString Write-Verbose "$timeStamp $msg" } # Common function for warning logging function Log-Warning ( [string]$msg ) { $timeStamp = Get-CurrentTimeString Write-Warning "$timeStamp $msg" } # Common function for error logging function Log-Error ( [string]$msg, [switch]$Stop) { $timeStamp = Get-CurrentTimeString if (!$Stop) { Write-Error "$timeStamp $msg" } else { Write-Error "$timeStamp $msg" -ErrorAction:Stop } } # Common function for logging an error, given an ErrorRecord function Log-ErrorRecord( [System.Management.Automation.ErrorRecord] $errRecord, [switch]$Stop ) { # Trim the message so it will not display the "ErrorActionPreference is set to Stop" message # $failedMessage = $errRecord.ToString() if ($failedMessage.IndexOf("ErrorActionPreference") -ne -1) { $failedMessage = $failedMessage.Substring($failedMessage.IndexOf("set to Stop: ") + 13) } $failedMessage = $failedMessage -replace "`r" $failedMessage = $failedMessage -replace "`n" $failedCommand = $errRecord.InvocationInfo.MyCommand Log-Error ($CheckDatabaseRedundancy_LocalizedStrings.res_0045 -f $failedCommand,$failedMessage) -Stop:$Stop } # Shuffles objects coming from the input pipeline. # NOTE: This method only works when invoked via a pipeline. function Shuffle-Objects( [Parameter(Mandatory=$true,ValueFromPipeline=$true)] $inputData ) { Begin { $inputDataList = @() } Process { # build the input list first $inputDataList += $inputData } End { # now shuffle the contents of the input list $len = $inputDataList.Length for ([int] $i = 0; $i -lt $len; $i++) { # pick the next random number [int]$randomIndex = Get-Random -Minimum:$i -Maximum:$len # swap the values $temp = $inputDataList[$i] $inputDataList[$i] = $inputDataList[$randomIndex] $inputDataList[$randomIndex] = $temp } foreach ($element in $inputDataList) { # send each element to the output pipeline $element } } } # This will send a mail message to the specified recipients. # # Based on Send-Mail function from DatacenterHealthCommonLibrary.ps1 (service engineering scripts). # # Here we will create two SMTP clients: (1) datacenter client is configured to send mails from # production datacenter environment (based on Send-Mail function from DatacenterHealthCommonLibrary.ps1), # (2) CorpNet client is configured to send mails from Topobuilder machines inside a CORPNET. # # This function is also used by some other scripts and workflows. # Use $usemscorp only for testing. Production scripts should use the # 'send-mail' implementation (which does not work on test topologies). # function Send-HANotificationMail( [string]$title, [string]$body, [string[]]$attachments, [string]$from, [string[]]$tos, [string[]]$ccs, [string]$pri = "Normal", [int]$maxRetryAttempts = 2, [switch] $usemscorp ) { if ($script:IsDataCenterLibraryPresent -and ! $usemscorp ) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0046 -f "send-mail") Set-StrictMode -Off send-mail -title:$title -body:$body -from:$from -tos:$tos -ccs:$ccs -attachments:$attachments -pri:$pri Set-StrictMode -Version 2.0 return } else { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0047 -f "Send-HANotificationMailCorpHub") [bool]$sent = Send-HANotificationMailCorpHub -title:$title -body:$body -attachments:$attachments -from:$from ` -tos:$tos -ccs:$ccs -pri:$pri -maxRetryAttempts:$maxRetryAttempts if (!$sent){ Log-Error ($CheckDatabaseRedundancy_LocalizedStrings.res_0048 ) } } } function Get-HubServers { Get-ExchangeServer | where { $_.IsHubTransportServer } } # Build a list of SMTP clients that can send mail to a local hub server. # function Build-HubSmtpClients { #FUTURE: We should return a list of hostnames and port and let the caller iterate over them if failure... # Also, try to choose a server in the same site as we are running # In production we will use the "send-mail" function provided by the svc engineering team $hubServers = (Get-HubServers | Shuffle-Objects ) if (!$hubServers) { Log-Error ($CheckDatabaseRedundancy_LocalizedStrings.res_0049 ) return } foreach ($hubServer in @($hubServers)) { $smtpClient = New-Object System.Net.Mail.SmtpClient($hubServer.Fqdn) $smtpClient.UseDefaultCredentials = $true Write-Output $smtpClient } } function Get-SmtpClients () { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0050 -f "Get-SmtpClients") $clients = @() $clients += Build-HubSmtpClients return $clients } # Build a Mail message # function Build-MailMsg( [string]$title, [string]$body, [string[]]$attachments, [string]$from, [string[]]$tos, [string[]]$ccs, [string]$pri = "Normal") { $mailMessage = New-Object System.Net.Mail.MailMessage $mailMessage.Body = $body $mailMessage.Priority = $pri $mailMessage.Subject = $title $mailMessage.From = New-Object System.Net.Mail.MailAddress($from); # Add attachments if ($attachments) { foreach ($attachment in @($attachments)) { if ( Test-Path $attachment ) { $data = New-Object System.Net.Mail.Attachment -ArgumentList $attachment, 'Application/Octet' [void]$mailMessage.Attachments.Add($data); } } } foreach ($to in @($tos)) { [void]$mailMessage.To.Add($to) } if ($ccs) { foreach ($cc in @($ccs)) { [void]$mailMessage.CC.Add($cc) } } return $mailMessage } # Send an email message # Return $true if an SMTP host was contacted and the mail transmitted. # There is no guarantee that the mail will get through. # This is cloned from Send-NotificationMail in DatacenterSvcEngCommonLibrary.ps1, but made simpler # so I could add function and simplify it at the same time. # function Send-HANotificationMailCorpHub( [string]$title, [string]$body, [string[]]$attachments, [string]$from, [string[]]$tos, [string[]]$ccs, [string]$pri = "Normal", [int]$maxRetryAttempts = 2) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0051 -f $from,$pri,"Send-HANotificationMailCorpHub") $clients = @(Get-SmtpClients) if (!$clients) { Log-Error ($CheckDatabaseRedundancy_LocalizedStrings.res_0052 -f "Get-SmtpClients") return $false } $mailMessage = Build-MailMsg -title $title -body $body -attachments $attachments ` -from $from -tos $tos -ccs $ccs -pri $pri if (!$mailMessage) { Log-Error ($CheckDatabaseRedundancy_LocalizedStrings.res_0052 -f "Build-MailMsg") return $false } try { foreach ($smtpClient in $clients) { # Change the timeout for synchronous Send() call to 30 secs $smtpClient.Timeout = 30000 $retries = 0 Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0053 -f [string]::Join(';',$tos)) Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0054 -f $smtpClient.Host,$smtpClient.Port) Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0055 -f $from) do { try { $success = $true $smtpClient.Send($mailMessage) Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0056 ) return $true } catch { $success = $false $retries++ if ($retries -eq $maxRetryAttempts) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0057 -f $maxRetryAttempts,$tos) } else { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0058 -f $tos) } } } while ((-not $success) -and $retries -lt $maxRetryAttempts) Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0059 ) } } finally { $mailMessage.Dispose() } return $success } function Append-RedundancyInformation( [Parameter(Mandatory=$true,ValueFromPipeline=$true)] [CheckHADatabaseRedundancy.DatabaseRedundancyEntry] $dbState) { Process { $totalMins = $dbState.TotalRedDuration.TotalMinutes.ToString("F2") $msg = " Database : $($dbState.DatabaseName) Redundancy Count : $($dbState.CurrentRedundancyCount) Total Red Minutes : $totalMins" $script:report.AppendLine($msg) | Out-Null foreach ($errRecord in @($dbState.ErrorHistory)) { $timeStr = $errRecord.ErrorTime.ToString("HH:mm:ss.fff UTC") $msg = " $timeStr : $($errRecord.GetErrorStringForAlerting()) " $script:report.AppendLine($msg) | Out-Null } } } function Send-SummaryEmail { if (!$SendSummaryMailTos) { return } [System.Text.StringBuilder]$script:report = New-Object -TypeName System.Text.StringBuilder -ArgumentList 2048 $states = $script:databaseStateTable.Values $dbsWithOneCopy = $states | where { ` ($_.CurrentState -eq [CheckHADatabaseRedundancy.AlertState]::Red) -and ` ($_.TotalRedDuration.TotalSeconds -gt $ReportRedEventAfterDurationSecs) ` } | sort -Property DatabaseName $dbsWithErrors = $states | where { ` $_.HasErrorsInHistory -and ` ( ($_.CurrentState -ne [CheckHADatabaseRedundancy.AlertState]::Red) -or ` ($_.TotalRedDuration.TotalSeconds -le $ReportRedEventAfterDurationSecs) ` )} | sort -Property DatabaseName [int]$databasesCount = ($states | Measure-Object).Count [int]$dbsWithErrorsCount = ($dbsWithErrors | Measure-Object).Count [int]$dbsOneCopyCount = ($dbsWithOneCopy | Measure-Object).Count if ( ($dbsWithErrorsCount -eq 0) -and ($dbsOneCopyCount -eq 0) ) { # No need to send an email to report that everything is healthy Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0060 ) return } [string]$priority = "Normal" $dbsWithOneCopyNames = ( $dbsWithOneCopy | select -ExpandProperty DatabaseName ) $dbsWithErrorsNames = ( $dbsWithErrors | select -ExpandProperty DatabaseName ) [string]$dbsWithOneCopyNamesStr = $null [string]$dbsWithErrorsNamesStr = $null if ($dbsWithOneCopyNames) { $dbsWithOneCopyNamesStr = [string]::Join(", ", $dbsWithOneCopyNames) } if ($dbsWithErrorsNames) { $dbsWithErrorsNamesStr = [string]::Join(", ", $dbsWithErrorsNames) } [string] $msg = " *************************************** Database Redundancy Report $((Get-Date).DateTime) ***************************************" $script:report.AppendLine($msg) | Out-Null if ($PSCmdlet.ParameterSetName -eq "Database" ) { $msg = "Database : $MailboxDatabaseName" $script:report.AppendLine($msg) | Out-Null } elseif ($PSCmdlet.ParameterSetName -eq "Server" ) { $msg = "Server : $MailboxServerName" $script:report.AppendLine($msg) | Out-Null } $msg = ` "DatabaseCount : $databasesCount DbsCountWithLowRedundancy : $dbsOneCopyCount DbsCountWithErrors : $dbsWithErrorsCount DatabasesWithLowRedundancy : $dbsWithOneCopyNamesStr DatabasesWithErrors : $dbsWithErrorsNamesStr" $script:report.AppendLine($msg) | Out-Null if ($dbsOneCopyCount -gt 0) { # mark email as urgent $priority = "High" $msg = " ================================================================= Databases with low redundancy ( < $AtLeastNCopies copies) =================================================================" $script:report.AppendLine($msg) | Out-Null @($dbsWithOneCopy) | Append-RedundancyInformation } if ($dbsWithErrorsCount -gt 0) { $msg = " ================================================================= Databases with errors ( >= $AtLeastNCopies copies) =================================================================" $script:report.AppendLine($msg) | Out-Null @($dbsWithErrors) | Append-RedundancyInformation } # Create the email subject [string]$title = "DB Redundancy: " if ($script:dagName) { $title += "$($script:dagName): " } if ($PSCmdlet.ParameterSetName -eq "Database" ) { $title += "$MailboxDatabaseName - " } elseif ($PSCmdlet.ParameterSetName -eq "Server" ) { $title += "$MailboxServerName - " } if ($dbsOneCopyCount -gt 0) { $durationMins = [TimeSpan]::FromSeconds($ReportRedEventAfterDurationSecs).TotalMinutes if ($dbsOneCopyCount -eq 1) { $title += "1 DB has less than $AtLeastNCopies copies for more than $durationMins mins" } else { $title += "$dbsOneCopyCount DBs have less than $AtLeastNCopies copies for more than $durationMins mins" } } elseif ($dbsWithErrorsCount -gt 0) { if ($dbsWithErrorsCount -eq 1) { $title += "1 DB has had errors in the past hour" } else { $title += "$dbsWithErrorsCount DBs have had errors in the past hour" } } else { $title += "All DBs have been sufficiently redundant for the past hour" } # send the email Send-HANotificationMail -title:$title -body:($script:report.ToString()) -from:$SummaryMailFrom -tos:$SendSummaryMailTos -pri:$priority } ################################################################### ### Entry point for the script itself ################################################################### function RunOnce { $script:outputObjects = $null # Run each iteration of the check in a separate script block so that any errors # can be trapped and the entire script block exits. $checkCmd = { $script:outputObjects = RunOnceInternal } [bool]$success = TryExecute-ScriptBlock -runCommand $checkCmd if ($success) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0061 -f $script:iteration) } else { Log-Error ($CheckDatabaseRedundancy_LocalizedStrings.res_0062 -f $script:iteration) } # send to the output pipeline $script:outputObjects if ($script:ExitOut) { Log-Verbose "Script is now exiting..." Exit } } function RunOnceInternal { $script:oneIterationStopwatch.Reset() $script:oneIterationStopwatch.Start() $script:iteration++ Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0063 -f $script:iteration) # The databases being monitored [Microsoft.Exchange.Data.Directory.SystemConfiguration.MailboxDatabase[]] $mdbs = @() # Check if the node is clustered (i.e. participating member of a DAG) if ($MonitoringContext -and ![Microsoft.Exchange.Cluster.Replay.DagTaskHelperPublic]::IsLocalNodeClustered()) { Log-Verbose "The local node is not a participating member of a database availability group (DAG). Skipping running the monitoring checks." $script:ExitOut = $true Exit } # Lookup the specified database if ($PSCmdlet.ParameterSetName -eq "Database" ) { $mdb = Get-MailboxDatabase $MailboxDatabaseName if ($mdb) { if ($mdb.Recovery) { Log-Error ($CheckDatabaseRedundancy_LocalizedStrings.res_0064 -f $mdb,"non-recovery") -Stop return } # We will check databases even if they only have 1 configured copy $mdbs += $mdb } else { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0065 -f $MailboxDatabaseName) } } # Lookup all databases on the specified server elseif ($PSCmdlet.ParameterSetName -eq "Server" ) { $server = Get-MailboxServer $MailboxServerName if ($server) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0066 -f $MailboxServerName) $script:mailboxServer = $server $allMdbs = @( Get-MailboxDatabase -Server $server ) $mdbs = @( $allMdbs | Select-DatabasesToMonitor ) } else { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0068 -f $MailboxServerName) } } Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0069 -f $mdbs.Length) # perform the check if ($mdbs.Length -gt 0) { Check-Databases $mdbs $PSCmdlet.ParameterSetName } else { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0070 -f "Check-Databases") } $script:oneIterationStopwatch.Stop() Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0071 -f $script:iteration,$script:oneIterationStopwatch.Elapsed.TotalMilliseconds) } # This function returns database redundancy object function Get-DatabaseRedundancy( [string] $databaseName = $(throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0072 -f "Get-DatabaseRedundancy","-databaseName"))) { if ( -not $databaseName ) { throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0073 -f "Get-DatabaseRedundancy","-databaseName") } $MailboxDatabaseName = $databaseName $SkipEventLogging = $true $MonitoringContext = $false $databases = @( Get-MailboxDatabase $MailboxDatabaseName ) Populate-DatabasesTable $databases # find the servers to check copy statuses on [String[]]$servers = Get-ServersForDatabases $databases if ($servers.Length -lt 2) { # Normally we should not get here, since we're only checking replicated DBs, which means # we should have at least 2 distinct servers. However, this can happen if copies are # removed while this script is running. Log-Warning ($CheckDatabaseRedundancy_LocalizedStrings.res_0074 -f $servers.Length,"Check-DatabaseRedundancyInSite","Get-ServersForDatabases") } # get the status results and index them by database name $script:databaseToStatusTable.Clear() $script:databaseToStatusTable = Get-CopyStatusFromAllServers $servers "Database" | ` Group-Object -AsHashTable -Property DatabaseName # look up the cluster node status for the DAG Populate-ClusterNodeStatus [CheckHADatabaseRedundancy.DatabaseRedundancyEntry]$dbRedundancy = Check-DatabaseRedundancy $MailboxDatabaseName return $dbRedundancy } # Returns number of heatlhy copies in a given site function Check-DatabaseRedundancyInSite( [string] $databaseName = $(throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0072 -f "Check-DatabaseRedundancyInSite","-databaseName")), [string] $adSiteName = $(throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0072 -f "Check-DatabaseRedundancyInSite","-adSiteName"))) { if ( -not $databaseName ) { throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0073 -f "Check-DatabaseRedundancyInSite","-databaseName") } if ( -not $adSiteName ) { throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0073 -f "Check-DatabaseRedundancyInSite","-adSiteName") } [CheckHADatabaseRedundancy.DatabaseRedundancyEntry]$dbRedundancy = Get-DatabaseRedundancy -databaseName $databaseName return $dbRedundancy.GetRedundancyCountInSite($adSiteName) } # Returns number of heatlhy copies with heatlhy search catalog in a given site function Check-DatabaseSearchRedundancyInSite( [string] $databaseName = $(throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0072 -f "Check-DatabaseSearchRedundancyInSite","-databaseName")), [string] $adSiteName = $(throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0072 -f "Check-DatabaseSearchRedundancyInSite","-adSiteName"))) { if ( -not $databaseName ) { throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0073 -f "Check-DatabaseSearchRedundancyInSite","-databaseName") } if ( -not $adSiteName ) { throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0073 -f "Check-DatabaseSearchRedundancyInSite","-adSiteName") } [CheckHADatabaseRedundancy.DatabaseRedundancyEntry]$dbRedundancy = Get-DatabaseRedundancy -databaseName $databaseName return $dbRedundancy.GetSearchRedundancyCountInSite($adSiteName) } # This function returns true if you can remove the copy without losing redundancy and false otherwise function Check-DatabaseRedundancyForCopyRemoval( [string] $databaseName = $(throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0072 -f "Check-DatabaseRedundancyForCopyRemoval","-databaseName")), [string] $serverName = $(throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0072 -f "Check-DatabaseRedundancyForCopyRemoval","-serverName"))) { if ( -not $databaseName ) { throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0073 -f "Check-DatabaseRedundancyForCopyRemoval","-databaseName") } if ( -not $serverName ) { throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0073 -f "Check-DatabaseRedundancyForCopyRemoval","-serverName") } $MailboxDatabaseName = $databaseName $SkipEventLogging = $true $MonitoringContext = $false $SkipDatabasesRegex = "^Mailbox Database \d{10}$" $databases = @( Get-MailboxDatabase $MailboxDatabaseName | Select-DatabasesToMonitor ) if (!$databases) { # Since this database has been skipped from the check, return success Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0088 -f $MailboxDatabaseName) return $true } Populate-DatabasesTable $databases # find the servers to check copy statuses on [String[]]$servers = Get-ServersForDatabases $databases if ($servers.Length -lt 2) { # Normally we should not get here, since we're only checking replicated DBs, which means # we should have at least 2 distinct servers. However, this can happen if copies are # removed while this script is running. Log-Warning ($CheckDatabaseRedundancy_LocalizedStrings.res_0074 -f $servers.Length,"Check-DatabaseRedundancyForCopyRemoval","Get-ServersForDatabases") } # get the status results and index them by database name $script:databaseToStatusTable.Clear() $script:databaseToStatusTable = Get-CopyStatusFromAllServers $servers "Database" | ` Group-Object -AsHashTable -Property DatabaseName # look up the cluster node status for the DAG # since it is a very expensive operation, we only need to do it once here if ($script:clusterNodeStateTable.Count -eq 0) { Populate-ClusterNodeStatus } # Simulate copy removal [bool] $foundCopy = $false for ($i = 0; $i -lt $script:databaseToStatusTable[$MailboxDatabaseName].Count; $i++) { if ( $script:databaseToStatusTable[$MailboxDatabaseName][$i].Name -eq "$MailboxDatabaseName\$serverName" ) { if ( $script:databaseToStatusTable[$MailboxDatabaseName][$i].ActiveCopy ) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0075 -f $MailboxDatabaseName,$serverName,"Check-DatabaseRedundancyForCopyRemoval") } else { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0076 -f $MailboxDatabaseName,$serverName,"Check-DatabaseRedundancyForCopyRemoval") } $foundCopy = $true break } } if ( -not $foundCopy ) { throw ($CheckDatabaseRedundancy_LocalizedStrings.res_0077 -f $MailboxDatabaseName,$serverName,"Check-DatabaseRedundancyForCopyRemoval") } $status = Check-DatabaseRedundancy -dbName $MailboxDatabaseName -skipCopyOnServer $serverName -checkOneDatacenter $false [bool] $result = $status.CurrentState -eq "Green" return $result } function Main { if ($PSCmdlet.ParameterSetName -eq "DotSourceMode") { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0078 -f "-DotSourceMode") return } # Ensure this table is cleared at script startup # Other hashtables get cleared every iteration of RunOnce. $script:databaseStateTable.Clear() # Validate the email parameters if ($SendSummaryMailTos -and !$SummaryMailFrom) { Log-Error ($CheckDatabaseRedundancy_LocalizedStrings.res_0079 -f "-SummaryMailFrom","-SendSummaryMailTos") # Let monitoring continue anyway... } # Check if the node is clustered (i.e. participating member of a DAG) if ($MonitoringContext -and ![Microsoft.Exchange.Cluster.Replay.DagTaskHelperPublic]::IsLocalNodeClustered()) { Log-Verbose "The local node is not a participating member of a database availability group (DAG). Skipping running the monitoring checks." return } if (!$MonitoringContext) { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0080 ) RunOnce Send-SummaryEmail return } # We are in the MonitoringContext [bool] $keepRunning = $true [System.Diagnostics.Stopwatch] $overallScriptStopwatch = New-Object -TypeName System.Diagnostics.Stopwatch $overallScriptStopwatch.Reset() $overallScriptStopwatch.Start() Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0081 ) while ($keepRunning) { RunOnce # decide if we should run the next iteration if (($TerminateAfterDurationSecs -eq -1) -or ` ($TerminateAfterDurationSecs -eq 0)) { # infinite duration specified $keepRunning = $true } else { [double]$lastIterationMsecs = $script:oneIterationStopwatch.Elapsed.TotalMilliseconds [double]$timeLeftMsecs = [double]($TerminateAfterDurationSecs * 1000) - $overallScriptStopwatch.Elapsed.TotalMilliseconds # Is there enough time left for a (sleep + RunOnce) ? if ( ([double]($SleepDurationBetweenIterationsSecs * 1000) + $lastIterationMsecs) -lt $timeLeftMsecs ) { $keepRunning = $true } else { $keepRunning = $false break } } Sleep-ForSeconds $SleepDurationBetweenIterationsSecs } Send-SummaryEmail } $Command = $MyInvocation.MyCommand Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0082 -f $Command.Path) # The command below is useful to see what parameters are defined in this script cmdlet. # $Command | fl Path, CommandType, Parameters, ParameterSets # Get the code compilation out of the way Prepare-DatabaseRedundancyEntryDefinition LoadExchangeSnapin [string]$version = [Microsoft.Exchange.Diagnostics.BuildVersionConstants]::RegistryVersionSubKey # In datacenter configurations we can use libraries provided by service engineering $InstallPath = (Get-ItemProperty -path "HKLM:SOFTWARE\Microsoft\ExchangeServer\$version\Setup").MsiInstallPath.Trim().TrimEnd("\") $DatacenterLibraryPath = "$InstallPath\DataCenter" $SvcLibaryFileName = "DatacenterHealthCommonLibrary.ps1" $ServiceCommonLib = "$DatacenterLibraryPath\$SvcLibaryFileName" $script:IsDataCenterLibraryPresent = Test-Path $ServiceCommonLib if ($script:IsDataCenterLibraryPresent) { # Get a send-mail function Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0083 -f $ServiceCommonLib) # The common lib doesn't use clean practices so we have to avoid strict mode Set-StrictMode -Off . $ServiceCommonLib Set-StrictMode -Version 2.0 } else { Log-Verbose ($CheckDatabaseRedundancy_LocalizedStrings.res_0084 -f $DatacenterLibraryPath,$SvcLibaryFileName) } Main # SIG # Begin signature block # MIIaxwYJKoZIhvcNAQcCoIIauDCCGrQCAQExCzAJBgUrDgMCGgUAMGkGCisGAQQB # gjcCAQSgWzBZMDQGCisGAQQBgjcCAR4wJgIDAQAABBAfzDtgWUsITrck0sYpfvNR # AgEAAgEAAgEAAgEAAgEAMCEwCQYFKw4DAhoFAAQUIsyEn3pZ7hVALsEpoGPBA5qh # UyOgghV6MIIEuzCCA6OgAwIBAgITMwAAAFnWc81RjvAixQAAAAAAWTANBgkqhkiG # 9w0BAQUFADB3MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4G # A1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSEw # HwYDVQQDExhNaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EwHhcNMTQwNTIzMTcxMzE1 # WhcNMTUwODIzMTcxMzE1WjCBqzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAldBMRAw # DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # DTALBgNVBAsTBE1PUFIxJzAlBgNVBAsTHm5DaXBoZXIgRFNFIEVTTjpGNTI4LTM3 # NzctOEE3NjElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC # ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMZsTs9oU/3vgN7oi8Sx8H4H # zh487AyMNYdM6VE6vLawlndC+v88z+Ha4on6bkIAmVsW3QlkOOJS+9+O+pOjPbuH # j264h8nQYE/PnIKRbZEbchCz2EN8WUpgXcawVdAn2/L2vfIgxiIsnmuLLWzqeATJ # S8FwCee2Ha+ajAY/eHD6du7SJBR2sq4gKIMcqfBIkj+ihfeDysVR0JUgA3nSV7wT # tU64tGxWH1MeFbvPMD/9OwHNX3Jo98rzmWYzqF0ijx1uytpl0iscJKyffKkQioXi # bS5cSv1JuXtAsVPG30e5syNOIkcc08G5SXZCcs6Qhg4k9cI8uQk2P6hTXFb+X2EC # AwEAAaOCAQkwggEFMB0GA1UdDgQWBBRbKBqzzXUNYz39mfWbFQJIGsumrDAfBgNV # HSMEGDAWgBQjNPjZUkZwCu1A+3b7syuwwzWzDzBUBgNVHR8ETTBLMEmgR6BFhkNo # dHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNyb3Nv # ZnRUaW1lU3RhbXBQQ0EuY3JsMFgGCCsGAQUFBwEBBEwwSjBIBggrBgEFBQcwAoY8 # aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNyb3NvZnRUaW1l # U3RhbXBQQ0EuY3J0MBMGA1UdJQQMMAoGCCsGAQUFBwMIMA0GCSqGSIb3DQEBBQUA # A4IBAQB68A30RWw0lg538OLAQgVh94jTev2I1af193/yCPbV/cvKdHzbCanf1hUH # mb/QPoeEYnvCBo7Ki2jiPd+eWsWMsqlc/lliJvXX+Xi2brQKkGVm6VEI8XzJo7cE # N0bF54I+KFzvT3Gk57ElWuVDVDMIf6SwVS3RgnBIESANJoEO7wYldKuFw8OM4hRf # 6AVUj7qGiaqWrpRiJfmvaYgKDLFRxAnvuIB8U5B5u+mP0EjwYsiZ8WU0O/fOtftm # mLmiWZldPpWfFL81tPuYciQpDPO6BHqCOftGzfHgsha8fSD4nDkVJaEmLdaLgb3G # vbCdVP5HC18tTir0h+q1D7W37ZIpMIIE7DCCA9SgAwIBAgITMwAAAMps1TISNcTh # VQABAAAAyjANBgkqhkiG9w0BAQUFADB5MQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMSMwIQYDVQQDExpNaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBD # QTAeFw0xNDA0MjIxNzM5MDBaFw0xNTA3MjIxNzM5MDBaMIGDMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMQ0wCwYDVQQLEwRNT1BSMR4wHAYDVQQD # ExVNaWNyb3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw # ggEKAoIBAQCWcV3tBkb6hMudW7dGx7DhtBE5A62xFXNgnOuntm4aPD//ZeM08aal # IV5WmWxY5JKhClzC09xSLwxlmiBhQFMxnGyPIX26+f4TUFJglTpbuVildGFBqZTg # rSZOTKGXcEknXnxnyk8ecYRGvB1LtuIPxcYnyQfmegqlFwAZTHBFOC2BtFCqxWfR # +nm8xcyhcpv0JTSY+FTfEjk4Ei+ka6Wafsdi0dzP7T00+LnfNTC67HkyqeGprFVN # TH9MVsMTC3bxB/nMR6z7iNVSpR4o+j0tz8+EmIZxZRHPhckJRIbhb+ex/KxARKWp # iyM/gkmd1ZZZUBNZGHP/QwytK9R/MEBnAgMBAAGjggFgMIIBXDATBgNVHSUEDDAK # BggrBgEFBQcDAzAdBgNVHQ4EFgQUH17iXVCNVoa+SjzPBOinh7XLv4MwUQYDVR0R # BEowSKRGMEQxDTALBgNVBAsTBE1PUFIxMzAxBgNVBAUTKjMxNTk1K2I0MjE4ZjEz # LTZmY2EtNDkwZi05YzQ3LTNmYzU1N2RmYzQ0MDAfBgNVHSMEGDAWgBTLEejK0rQW # WAHJNy4zFha5TJoKHzBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNDb2RTaWdQQ0FfMDgtMzEtMjAx # MC5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1p # Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY0NvZFNpZ1BDQV8wOC0zMS0yMDEwLmNy # dDANBgkqhkiG9w0BAQUFAAOCAQEAd1zr15E9zb17g9mFqbBDnXN8F8kP7Tbbx7Us # G177VAU6g3FAgQmit3EmXtZ9tmw7yapfXQMYKh0nfgfpxWUftc8Nt1THKDhaiOd7 # wRm2VjK64szLk9uvbg9dRPXUsO8b1U7Brw7vIJvy4f4nXejF/2H2GdIoCiKd381w # gp4YctgjzHosQ+7/6sDg5h2qnpczAFJvB7jTiGzepAY1p8JThmURdwmPNVm52Iao # AP74MX0s9IwFncDB1XdybOlNWSaD8cKyiFeTNQB8UCu8Wfz+HCk4gtPeUpdFKRhO # lludul8bo/EnUOoHlehtNA04V9w3KDWVOjic1O1qhV0OIhFeezCCBbwwggOkoAMC # AQICCmEzJhoAAAAAADEwDQYJKoZIhvcNAQEFBQAwXzETMBEGCgmSJomT8ixkARkW # A2NvbTEZMBcGCgmSJomT8ixkARkWCW1pY3Jvc29mdDEtMCsGA1UEAxMkTWljcm9z # b2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MB4XDTEwMDgzMTIyMTkzMloX # DTIwMDgzMTIyMjkzMloweTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0 # b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3Jh # dGlvbjEjMCEGA1UEAxMaTWljcm9zb2Z0IENvZGUgU2lnbmluZyBQQ0EwggEiMA0G # CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCycllcGTBkvx2aYCAgQpl2U2w+G9Zv # zMvx6mv+lxYQ4N86dIMaty+gMuz/3sJCTiPVcgDbNVcKicquIEn08GisTUuNpb15 # S3GbRwfa/SXfnXWIz6pzRH/XgdvzvfI2pMlcRdyvrT3gKGiXGqelcnNW8ReU5P01 # lHKg1nZfHndFg4U4FtBzWwW6Z1KNpbJpL9oZC/6SdCnidi9U3RQwWfjSjWL9y8lf # RjFQuScT5EAwz3IpECgixzdOPaAyPZDNoTgGhVxOVoIoKgUyt0vXT2Pn0i1i8UU9 # 56wIAPZGoZ7RW4wmU+h6qkryRs83PDietHdcpReejcsRj1Y8wawJXwPTAgMBAAGj # ggFeMIIBWjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTLEejK0rQWWAHJNy4z # Fha5TJoKHzALBgNVHQ8EBAMCAYYwEgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEE # AYI3FQIEFgQU/dExTtMmipXhmGA7qDFvpjy82C0wGQYJKwYBBAGCNxQCBAweCgBT # AHUAYgBDAEEwHwYDVR0jBBgwFoAUDqyCYEBWJ5flJRP8KuEKU5VZ5KQwUAYDVR0f # BEkwRzBFoEOgQYY/aHR0cDovL2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJv # ZHVjdHMvbWljcm9zb2Z0cm9vdGNlcnQuY3JsMFQGCCsGAQUFBwEBBEgwRjBEBggr # BgEFBQcwAoY4aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNy # b3NvZnRSb290Q2VydC5jcnQwDQYJKoZIhvcNAQEFBQADggIBAFk5Pn8mRq/rb0Cx # MrVq6w4vbqhJ9+tfde1MOy3XQ60L/svpLTGjI8x8UJiAIV2sPS9MuqKoVpzjcLu4 # tPh5tUly9z7qQX/K4QwXaculnCAt+gtQxFbNLeNK0rxw56gNogOlVuC4iktX8pVC # nPHz7+7jhh80PLhWmvBTI4UqpIIck+KUBx3y4k74jKHK6BOlkU7IG9KPcpUqcW2b # Gvgc8FPWZ8wi/1wdzaKMvSeyeWNWRKJRzfnpo1hW3ZsCRUQvX/TartSCMm78pJUT # 5Otp56miLL7IKxAOZY6Z2/Wi+hImCWU4lPF6H0q70eFW6NB4lhhcyTUWX92THUmO # Lb6tNEQc7hAVGgBd3TVbIc6YxwnuhQ6MT20OE049fClInHLR82zKwexwo1eSV32U # jaAbSANa98+jZwp0pTbtLS8XyOZyNxL0b7E8Z4L5UrKNMxZlHg6K3RDeZPRvzkbU # 0xfpecQEtNP7LN8fip6sCvsTJ0Ct5PnhqX9GuwdgR2VgQE6wQuxO7bN2edgKNAlt # HIAxH+IOVN3lofvlRxCtZJj/UBYufL8FIXrilUEnacOTj5XJjdibIa4NXJzwoq6G # aIMMai27dmsAHZat8hZ79haDJLmIz2qoRzEvmtzjcT3XAH5iR9HOiMm4GPoOco3B # oz2vAkBq/2mbluIQqBC0N1AI1sM9MIIGBzCCA++gAwIBAgIKYRZoNAAAAAAAHDAN # BgkqhkiG9w0BAQUFADBfMRMwEQYKCZImiZPyLGQBGRYDY29tMRkwFwYKCZImiZPy # LGQBGRYJbWljcm9zb2Z0MS0wKwYDVQQDEyRNaWNyb3NvZnQgUm9vdCBDZXJ0aWZp # Y2F0ZSBBdXRob3JpdHkwHhcNMDcwNDAzMTI1MzA5WhcNMjEwNDAzMTMwMzA5WjB3 # MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVk # bW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSEwHwYDVQQDExhN # aWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw # ggEKAoIBAQCfoWyx39tIkip8ay4Z4b3i48WZUSNQrc7dGE4kD+7Rp9FMrXQwIBHr # B9VUlRVJlBtCkq6YXDAm2gBr6Hu97IkHD/cOBJjwicwfyzMkh53y9GccLPx754gd # 6udOo6HBI1PKjfpFzwnQXq/QsEIEovmmbJNn1yjcRlOwhtDlKEYuJ6yGT1VSDOQD # LPtqkJAwbofzWTCd+n7Wl7PoIZd++NIT8wi3U21StEWQn0gASkdmEScpZqiX5NMG # gUqi+YSnEUcUCYKfhO1VeP4Bmh1QCIUAEDBG7bfeI0a7xC1Un68eeEExd8yb3zuD # k6FhArUdDbH895uyAc4iS1T/+QXDwiALAgMBAAGjggGrMIIBpzAPBgNVHRMBAf8E # BTADAQH/MB0GA1UdDgQWBBQjNPjZUkZwCu1A+3b7syuwwzWzDzALBgNVHQ8EBAMC # AYYwEAYJKwYBBAGCNxUBBAMCAQAwgZgGA1UdIwSBkDCBjYAUDqyCYEBWJ5flJRP8 # KuEKU5VZ5KShY6RhMF8xEzARBgoJkiaJk/IsZAEZFgNjb20xGTAXBgoJkiaJk/Is # ZAEZFgltaWNyb3NvZnQxLTArBgNVBAMTJE1pY3Jvc29mdCBSb290IENlcnRpZmlj # YXRlIEF1dGhvcml0eYIQea0WoUqgpa1Mc1j0BxMuZTBQBgNVHR8ESTBHMEWgQ6BB # hj9odHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9taWNy # b3NvZnRyb290Y2VydC5jcmwwVAYIKwYBBQUHAQEESDBGMEQGCCsGAQUFBzAChjho # dHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY3Jvc29mdFJvb3RD # ZXJ0LmNydDATBgNVHSUEDDAKBggrBgEFBQcDCDANBgkqhkiG9w0BAQUFAAOCAgEA # EJeKw1wDRDbd6bStd9vOeVFNAbEudHFbbQwTq86+e4+4LtQSooxtYrhXAstOIBNQ # md16QOJXu69YmhzhHQGGrLt48ovQ7DsB7uK+jwoFyI1I4vBTFd1Pq5Lk541q1YDB # 5pTyBi+FA+mRKiQicPv2/OR4mS4N9wficLwYTp2OawpylbihOZxnLcVRDupiXD8W # mIsgP+IHGjL5zDFKdjE9K3ILyOpwPf+FChPfwgphjvDXuBfrTot/xTUrXqO/67x9 # C0J71FNyIe4wyrt4ZVxbARcKFA7S2hSY9Ty5ZlizLS/n+YWGzFFW6J1wlGysOUzU # 9nm/qhh6YinvopspNAZ3GmLJPR5tH4LwC8csu89Ds+X57H2146SodDW4TsVxIxIm # dgs8UoxxWkZDFLyzs7BNZ8ifQv+AeSGAnhUwZuhCEl4ayJ4iIdBD6Svpu/RIzCzU # 2DKATCYqSCRfWupW76bemZ3KOm+9gSd0BhHudiG/m4LBJ1S2sWo9iaF2YbRuoROm # v6pH8BJv/YoybLL+31HIjCPJZr2dHYcSZAI9La9Zj7jkIeW1sMpjtHhUBdRBLlCs # lLCleKuzoJZ1GtmShxN1Ii8yqAhuoFuMJb+g74TKIdbrHk/Jmu5J4PcBZW+JC33I # acjmbuqnl84xKf8OxVtc2E0bodj6L54/LlUWa8kTo/0xggS3MIIEswIBATCBkDB5 # MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVk # bW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSMwIQYDVQQDExpN # aWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQQITMwAAAMps1TISNcThVQABAAAAyjAJ # BgUrDgMCGgUAoIHQMBkGCSqGSIb3DQEJAzEMBgorBgEEAYI3AgEEMBwGCisGAQQB # gjcCAQsxDjAMBgorBgEEAYI3AgEVMCMGCSqGSIb3DQEJBDEWBBTl91Sgp5JWyQyC # V+/Vrej8AOYNDjBwBgorBgEEAYI3AgEMMWIwYKA4gDYAQwBoAGUAYwBrAEQAYQB0 # AGEAYgBhAHMAZQBSAGUAZAB1AG4AZABhAG4AYwB5AC4AcABzADGhJIAiaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL2V4Y2hhbmdlIDANBgkqhkiG9w0BAQEFAASCAQBi # iwAUEDmW3mmrrnky3JpYoRCVceQA5NchfaBiGwfw6k/LAlbEWaT/niWc+oXLVt++ # 3merswa4/wVsdpzqViOQtO5CP52meW0OF5XepVGC2RyoHNFkw2lTu6ha/qruVwZx # degWx1FyyFsnuDggwwGu8io+iYA/ldoelnvJ9Vve41HbTW6I5yp9R31y1zUSjLSP # Py9ziKZt7MSnA0i13u0rROKL8Ul7XRF78qtyMSytfyhbj4iQNHuHWGcs4OqlOyLF # /Mha8tdzI5rfbz3Ss51Nub6v+ZLoH8VG6HLRgc4hBDyeQMNPjU9HNMRgbdqpXmZG # 6qaIFBmWIMx+Bda1/yFNoYICKDCCAiQGCSqGSIb3DQEJBjGCAhUwggIRAgEBMIGO # MHcxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xITAfBgNVBAMT # GE1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQQITMwAAAFnWc81RjvAixQAAAAAAWTAJ # BgUrDgMCGgUAoF0wGAYJKoZIhvcNAQkDMQsGCSqGSIb3DQEHATAcBgkqhkiG9w0B # CQUxDxcNMTQwNzExMDUwNTM5WjAjBgkqhkiG9w0BCQQxFgQUWQzZOe+Lt2BbXRb+ # k4DaGZFn2eowDQYJKoZIhvcNAQEFBQAEggEAEZpkRtBd7wxZEz0Jyh6En4SXugSR # YCz+Sz8FSy/wVjhT45r4C/2E9kw6CORkigzd0ErktW37DOQMg/AiRYY6S9Lfu89b # hkj2nPRW3hMZbhBXSJLIJ3y/SlFzr8zi1lTvw6Enfeuwg8HKbR/r1msyRqiUd2Ov # +KHtGlvB+XECR1aPbwJX9xd8a7y0aKsC8Utm7cGGJNcq8/Jp4Ac4m6F3gUqJaQY8 # 6THFomEqq9uEcPQjNpY4j7sRh4MuLYk4VSW9UFuPEDWKr4ITkwXQG29eGmIOXxAp # G7O6lmfZXCW9smBmeXV60RIXDzdJl9OG+0g/LbLTt4Zb1hpKrLTDLj4GIg== # SIG # End signature block