# Copyright (c) Microsoft Corporation. All rights reserved. # # DagCommonLibrary.ps1 # A collection of DAG-related functions for use by other scripts. Import-LocalizedData -BindingVariable DagCommonLibrary_LocalizedStrings -FileName DagCommonLibrary.strings.psd1 # Checks for RSAT-Clustering function Test-RsatClusteringInstalled { if ( test-path $( [System.IO.Path]::Combine( $env:windir, "system32\cluster.exe" ) ) ) { log-verbose ($DagCommonLibrary_LocalizedStrings.res_0000 -f "RSAT-Clustering"); } else { log-error ($DagCommonLibrary_LocalizedStrings.res_0001 -f "RSAT-Clustering"); } } # Returns the objects that are hosted on the specified mailbox server. # The checks that are done include: # -No replicated mailboxes are sources. # -Any non-replicated mailbox contains no mailboxes (regular or arbitration). # -Not the Primary Active Manager (PAM). # -No active mailbox databases ('sources'). # # If the server is NOT being monitored, then the following is also checked: # -Any target replicated mailbox databases are suspended or Misconfigured. # # Only the objects are returned, not the reason that they're returned. # # NOTE: There is no 'PAM' object, so if the PAM is owned by the # server, a string 'Primary Active Manager=' is returned instead. function GetCriticalMailboxResources( [string] $serverName, [bool] $serverIsMonitored = $false, [UInt32] $AtLeastNCriticalCopies ) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0002 -f $serverName,$serverIsMonitored) # Look for the mailbox server so that we can find out if it belongs to a DAG $mbxServer = get-mailboxserver $serverName; $clusterService = get-service clussvc -ErrorAction:SilentlyContinue [bool] $clusterServiceDown = $false if ( ( $clusterService -eq $null ) -or ( $clusterService.Status -ne 'Running' ) ) { $clusterServiceDown = $true Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0003 -f $serverName,"Get-CriticalMailboxResources") } if ( $mbxServer.DatabaseAvailabilityGroup ) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0004 -f $mbxServer.DatabaseAvailabilityGroup,$serverName) $dag = Get-DatabaseAvailabilityGroup $mbxServer.DatabaseAvailabilityGroup; if (!$dag) { Log-Error ($DagCommonLibrary_LocalizedStrings.res_0005 -f $serverName) -Stop } $nodeCount = Get-NodeCountOfDag $dag; if ( $nodeCount -lt 2 ) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0006 -f $nodeCount); } else { $dag = Get-DatabaseAvailabilityGroup $mbxServer.DatabaseAvailabilityGroup -status; $pamServerName = $dag.PrimaryActiveManager Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0012 -f $pamServerName) if ( $serverName -ieq $pamServerName ) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0013 -f $serverName); write-output "(Primary Active Manager=$serverName)" } } } else { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0014 -f $serverName) } # Check for replicated databases. Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0015 -f $serverName,"non-replicated") $databases = get-mailboxdatabase -server $serverName $numDatabases = ($databases | Measure-Object).Count Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0016 -f $numDatabases,$serverName) if ( $numDatabases -gt 0 ) { foreach ( $database in $databases ) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0111 -f $database.Name,"Get-CriticalMailboxResources",$database.AutoDagExcludeFromMonitoring); if ( $database.ReplicationType -eq 'Remote' ) { if ($database.AutoDagExcludeFromMonitoring -eq $true) { Continue } # Make sure replicated databases aren't hosted on the server. # And if they aren't hosted, they need to be suspended. $status = get-mailboxdatabasecopystatus $database\$serverName Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0018 -f $status.ActiveCopy,$status.Status,$status.ActivationSuspended,$database,"Get-CriticalMailboxResources"); if ( $status.ActiveCopy ) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0019 -f $status.ActiveCopy,$status.Status,$status.ActivationSuspended,$database,"Get-CriticalMailboxResources"); write-output "(Database='$($database.Name)', Reason='Copy is active')"; } else { # Check if the active is copy is mounted $activeServerName = $status.ActiveDatabaseCopy if ($activeServerName) { $activeStatus = Get-MailboxDatabaseCopyStatus $database\$activeServerName if ($activeStatus.Status -ine 'Mounted') { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0110 -f $database,$activeServerName,"Get-CriticalMailboxResources"); write-output "(Database='$($database.Name)', Reason='Active is dismounted on $activeServerName')"; } } } if ( $status.Status -eq 'ServiceDown' ) { # Check if the copy is not active $statuses = @( get-mailboxdatabasecopystatus $database ) [bool] $foundAnotherActiveCopy = $false foreach ( $copyStatus in $statuses ) { if ( $copyStatus.ActiveCopy -and ( $copyStatus.MailboxServer.ToUpper() -ne $serverName.ToUpper() ) ) { $foundAnotherActiveCopy = $true Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0020 -f $copyStatus.MailboxServer,$database,"Get-CriticalMailboxResources"); break; } } if ( -not $foundAnotherActiveCopy ) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0019 -f $status.ActiveCopy,$status.Status,$status.ActivationSuspended,$database,"Get-CriticalMailboxResources"); write-output "(Database='$($database.Name)', Reason='Copy is potentially active while replay service is down')"; } elseif ( $mbxServer.DatabaseCopyAutoActivationPolicy -ne 'Blocked' ) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0021 -f $status.ActiveCopy,$status.Status,$status.ActivationSuspended,$database,$serverName,"Get-CriticalMailboxResources"); write-output "(Database='$($database.Name)', Reason='Service is not running on $serverName and database copy auto activation policy is not Blocked. Database copy auto activation policy is $($mbxServer.DatabaseCopyAutoActivationPolicy)')"; } } # O15 372945: If we add single-copy check for non-replicated databases, then we break new DAG deployment # scenario. Consider: If we have only 1 copy of a DB and need to reinstall, we should not block due to # this check. It would be great if deployment process could mark up the DAG state as "not-yet-fully-deployed" # so that scripts like this could be smarter and safer. # else { $result = IsDatabaseReadyForCopyRemoval -DatabaseGuid $database.Guid -AtleastNCopies $AtLeastNCriticalCopies -Target $serverName if (!$result.Success) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0023 -f $database,$serverName,$database,$serverName) write-output "(Database='$($database.Name)', Reason='$($result.Message)')"; } } } else { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0024 -f $database); # For non-replicated databases, check for mailboxes. $tempMailboxes = Get-Mailbox -Database $database if ( $tempMailboxes ) { $numMailboxes = ($tempMailboxes | Measure-Object).Count Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0025 -f $database,$numMailboxes); foreach ( $mbx in $tempMailboxes ) { write-output "(Mailbox='$mbx', Reason='Mailbox is hosted on '$($database.Name)', which is not a replicated database. )"; } } $tempMailboxes = Get-Mailbox -Database $database -Arbitration if ( $tempMailboxes ) { $numMailboxes = ($tempMailboxes | Measure-Object).Count Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0026 -f $database,$numMailboxes); foreach ( $mbx in $tempMailboxes ) { write-output "(Mailbox='$mbx', Reason='Arbitration Mailbox is hosted on '$($database.Name)', which is not a replicated database. )"; } } } } } Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0027 ) } function IsDatabaseReadyForCopyRemoval ([Guid] $databaseGuid, [int] $AtLeastNCopies, [string] $Target) { [string]$version = [Microsoft.Exchange.Diagnostics.BuildVersionConstants]::RegistryVersionSubKey; $setupRegistryPath = Get-ItemProperty -path "HKLM:SOFTWARE\Microsoft\ExchangeServer\$version\Setup" $replPath = Join-Path $setupRegistryPath.MsiInstallPath "Bin\Microsoft.Exchange.Cluster.Replay.dll" [System.Reflection.Assembly]::LoadFrom($replPath) | Out-Null $dbHealthValidater = New-Object -TypeName Microsoft.Exchange.Cluster.Replay.DatabaseHealthValidationRunner -ArgumentList $Target $dbHealthValidater.Initialize() $validaterType = $dbHealthValidater.GetType() $sickDatabases = $null [hashtable]$result = @{} $result.Success = $true $result.Message = "" # Check if the new parameters is available (checking one of the new is enough since they were deployed the same time). $parameters = ($validaterType.GetMember("RunDatabaseRedundancyChecksForCopyRemoval")).GetParameters() $param = $parameters | where {$_.Name -like "atLeastNCopies"} # New parameters aren't deployed yet so call copyremoval with old parameters # Note: param won't exist only in the rare chance that deployment decides to use new scripts with old binaries to put the server in Maintenance if (!$param) { $sickDatabases = $dbHealthValidater.RunDatabaseRedundancyChecksForCopyRemoval($false, $databaseGuid) } else { if ($AtLeastNCopies) { $sickDatabases = $dbHealthValidater.RunDatabaseRedundancyChecksForCopyRemoval($false, $databaseGuid, $false, $true, $AtLeastNCopies) } else { $sickDatabases = $dbHealthValidater.RunDatabaseRedundancyChecksForCopyRemoval($false, $databaseGuid, $false, $true) } } foreach ($sickDatabase in $sickDatabases) { if ($sickDatabase.IsValidationSuccessful -eq $false) { $result.Success = $false $result.Message = $sickDatabase.ErrorMessageWithoutFullStatus break } } return $result } # Gets the count of the servers in the DAG. function Get-NodeCountOfDag( [object] $dag ) { $nodeCount = $dag.Servers.Count; Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0028 -f $nodeCount,"Get-NodeCountOfDag"); return $nodeCount; } # Prints the output of GetCriticalMailboxResources function PrintCriticalMailboxResourcesOutput( [object[]]$resources ) { [string]$resString = ( $resources | % { ` if ($_.GetType() -eq [string]) { $_ } else { # every Exchange presentation object should have a Name property $_.Name } } ) $resString; } # This function logs critical resources detailed information function Log-CriticalResource([object[]] $resources) { foreach ( $resource in $resources ) { switch ($resource.GetType().Name) { "String" { log-verbose ($DagCommonLibrary_LocalizedStrings.res_0029 -f $resource) } "Mailbox" { log-verbose ($DagCommonLibrary_LocalizedStrings.res_0030 -f $resource.Name,$resource.Database.Name) } "DatabaseCopyStatusEntry" { log-verbose ($DagCommonLibrary_LocalizedStrings.res_0031 -f $resource.Name,$resource.Status) } "MailboxDatabase" { log-verbose ($DagCommonLibrary_LocalizedStrings.res_0032 -f $resource.Name) } default { log-verbose ($DagCommonLibrary_LocalizedStrings.res_0029 -f $resource.Name) } } } } # Moves the PAM from the given server if necessary # Supports $DagScriptTesting for whatif processing. function Move-DagActiveManager([string]$Server) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0033 -f $Server,"Move-DagActiveManager") # Move PAM if the target is a PAM. $mbxServer = get-mailboxserver $server -erroraction:silentlycontinue if ( $mbxServer -and $mbxServer.DatabaseAvailabilityGroup ) { $dag = Get-DatabaseAvailabilityGroup $mbxServer.DatabaseAvailabilityGroup -status; } if (!$dag) { Log-Error ($DagCommonLibrary_LocalizedStrings.res_0034 -f $Server,"Move-DagActiveManager") -Stop } $nodeCount = Get-NodeCountOfDag $dag; if ( $nodeCount -lt 2 ) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0035 -f $nodeCount,"Move-DagActiveManager"); return; } if ( $Server -ieq $dag.PrimaryActiveManager ) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0038 -f $Server,"Move-DagActiveManager") # Pick another target. $targetServer = $null $dag.Servers | where { $_.Name -ne $Server } | foreach { $targetServer = $_ } Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0039 -f $dag.Name,$targetServer,"Move-DagActiveManager") if ( $DagScriptTesting ) { write-host ($DagCommonLibrary_LocalizedStrings.res_0040 ) } else { if (![Microsoft.Exchange.Cluster.Replay.DagTaskHelperPublic]::MovePrimaryActiveManagerRole($server)) { Log-Error -Stop ($DagCommonLibrary_LocalizedStrings.res_0109 -f "Move-DagActiveManager","couldn't move the cluster group") } } } else { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0041 -f $dag.PrimaryActiveManager,$Server,"Move-DagActiveManager") } Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0042 -f "Move-DagActiveManager") } # Move the critical resources off the specified server. # This includes Active Databases, and the Primary Active Manager. function Move-CriticalMailboxResources([string]$Server, [string] $MoveComment = "DagCommonLibrary:Move-CriticalMailboxResources") { $msg = "" $moveActCmd = { # 1. Move any mailbox database masters to other servers. $Error.Clear() Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0043 -f $Server,"Move-CriticalMailboxResources") Move-ActiveMailboxDatabase -Server $Server -Confirm:$false -MoveComment $MoveComment foreach ($er in $Error) { Log-Verbose "[Error] $er" } } $msg += TryExecute-ScriptBlockReturnErrorMessage -runCommand $moveActCmd -throwOnError $false -silentonerrors $true $moveResCmd = { # 2. Move PAM if the server is a PAM. Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0044 -f $Server,"Move-CriticalMailboxResources") Move-DagActiveManager -Server $Server Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0045 -f $Server,"Move-CriticalMailboxResources") } $errorCmd = { Log-Error ($DagCommonLibrary_LocalizedStrings.res_0046 -f $Server,"Move-CriticalMailboxResources") } $msg += TryExecute-ScriptBlockReturnErrorMessage -runCommand $moveResCmd -cleanupCommand $errorCmd -throwOnError $true Return $msg } # Entry point of the lossless switchover function # If both $Database and $MailboxServer are provided, it moves only that database # If only $MailboxServer is provided, it moves all the active databases on the server # function Move-DagActiveCopy([string]$MailboxServer, [string]$Database) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0047 -f $MailboxServer,$Database,"Move-DagActiveCopy") $successdb = @() $activedb = @() $sourceServer = Get-ExchangeServer | where {$_.Name -eq $MailboxServer} if (($Database) -and ($sourceServer)) { $sourcedb = Get-MailboxDatabase -Identity $Database -Status Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0048 -f $sourcedb.MountedOnServer,$Database,"Move-DagActiveCopy") if ($sourcedb.MountedOnServer -eq $sourceServer.Fqdn) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0049 -f $sourcedb,$sourceServer,"Move-DagActiveCopy") if (Move-DagMasterCopy -db $sourcedb -srcServer $sourceServer) { if ( ! $DagScriptTesting ) { # REVIEW: Is this 60 seconds still needed?!? Sleep-ForSeconds 60 } Verify-DagMasterCopy -db $sourcedb -srcServer $sourceServer -StopOnError } else { Log-Error ($DagCommonLibrary_LocalizedStrings.res_0050 -f $Database,$MailboxServer,"Move-DagActiveCopy") } } else { Log-Error ($DagCommonLibrary_LocalizedStrings.res_0051 -f $Database,$MailboxServer,"Move-DagActiveCopy") } } elseif ($sourceServer) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0052 -f $sourceServer,"Move-DagActiveCopy") $activedb = Get-MailboxDatabase -Server $sourceServer.Name -Status | where { ( $_.MountedOnServer -eq $sourceServer.Fqdn ) -and ( $_.ReplicationType -eq 'Remote' )} if ($activedb) { foreach ($sourcedb in $activedb) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0053 -f $sourcedb,$sourceServer,"Move-DagActiveCopy") if (Move-DagMasterCopy -db $sourcedb -srcServer $sourceServer) { $successdb += $sourcedb } else { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0054 -f $sourcedb,$sourceServer,"Move-DagActiveCopy") } } if ($successdb) { if ( ! $DagScriptTesting ) { # REVIEW: Is this 60 seconds still needed?!? Sleep-ForSeconds 60 } $successdb | foreach { Verify-DagMasterCopy -db $_ -srcServer $sourceServer -StopOnError } } } else { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0055 -f $MailboxServer,"Move-DagActiveCopy") } } else { Log-Error ($DagCommonLibrary_LocalizedStrings.res_0056 -f "Move-DagActiveCopy") } Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0042 -f "Move-DagActiveCopy") } # This function picks up the best database copy based on the switchover criteria and moves the master to it. # Currently, it picks up the server closest to the front on the servers property as the # most preferrable copy as long as it meets the criteria. # If the PreferredTarget parameter is specified, it attempts to move the database only to the specified server # and it does not try other servers. # If the move fails, this function will attempt to "rollback" the DB to the original active server to prevent # an extended DB outage. function Move-DagMasterCopy ([object] $db, [object] $srcServer, [string] $preferredTarget) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0057 -f $db,$srcServer,$preferredTarget,"Move-DagMasterCopy") [bool]$moveSuccessful = $true; $targetdbs = @() if ($preferredTarget) { # $db\server does not accept a FQDN for the server. $shortName = $preferredTarget -replace "\..*$"; $dbCopyStatus = Get-MailboxDatabaseCopyStatus $db\$shortName; if (Test-DagTargetCopy -copyStatus $dbCopyStatus -Lossless) {$targetdbs += $dbCopyStatus} } else { foreach ($Server in $db.Servers) { if ($Server.Name -ne $srcServer.Name) { $dbCopyStatus = Get-MailboxDatabaseCopyStatus $db\$Server if (Test-DagTargetCopy -copyStatus $dbCopyStatus -Lossless) {$targetdbs += $dbCopyStatus} } } } $numTargetDbs = ($targetdbs | Measure-Object).Count Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0058 -f $db.Servers.Count,$numTargetDbs,$db,"Move-DagMasterCopy") $Error.Clear() try { if ($targetdbs) { foreach ($targetdb in $targetdbs) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0059 -f $targetdb.MailboxServer,$db,$false,"Move-DagMasterCopy","Move-ActiveMailboxDatabase","-Identity","-ActivateOnServer","-SkipClientExperienceChecks","-Confirm") if ( $DagScriptTesting ) { $moveCmd = { write-host ($DagCommonLibrary_LocalizedStrings.res_0060 -f $db,$targetdb,$false,"Move-DagMasterCopy","Move-ActiveMailboxDatabase","-Identity","-ActivateOnServer","-SkipClientExperienceChecks","-Confirm") } } else { $moveCmd = { Move-ActiveMailboxDatabase -Identity $db -ActivateOnServer $targetdb.MailboxServer -SkipClientExperienceChecks -Confirm:$false } } if (TryExecute-ScriptBlock $moveCmd) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0061 -f $targetdb.MailboxServer,$db,$false,"Move-DagMasterCopy","Move-ActiveMailboxDatabase","-Identity","-ActivateOnServer","-SkipClientExperienceChecks","-Confirm") break; } else { # Log-Verbose instead of Log-Error so that we can try another copy to move to. # In any case, we will verify the move using Verify-DagMasterCopy after this function returns. $moveSuccessful = $false; Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0062 -f $targetdb.MailboxServer,$db,$false,"Move-DagMasterCopy","Move-ActiveMailboxDatabase","-Identity","-ActivateOnServer","-SkipClientExperienceChecks","-Confirm") } } } else { # When moving all the databases on a server, the script continues even with this error to move # other "good" databases. When moving a single database, it ends up with no ops. $moveSuccessful = $false; Log-Error ($DagCommonLibrary_LocalizedStrings.res_0063 -f $db,"Move-DagMasterCopy") -Stop } } catch { # We need to log all new error messages if ($Error.Count -gt 0 ) { [string] $error_message = "" foreach ( $err in $Error ) { $error_message += $err.ToString() + "`n" } $Error.Clear() write-Error $error_message -ErrorAction:Continue } } finally { if (!$moveSuccessful) { &{ # Cleanup code is run with "Continue" ErrorActionPreference $ErrorActionPreference = "Continue" Log-Error ($DagCommonLibrary_LocalizedStrings.res_0064 -f $db,"Move-DagMasterCopy") [string]$originalActive = $db.MountedOnServer -replace "\..*$" [bool]$originalMountState = $db.Mounted Rollback-DatabaseOnFailedMove -desiredActiveServer $originalActive -shouldBeMounted $originalMountState -db $db Log-Error ($DagCommonLibrary_LocalizedStrings.res_0065 -f $db,"Move-DagMasterCopy") -Stop } } Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0066 -f $moveSuccessful,"Move-DagMasterCopy") } return $moveSuccessful } #-------------------------------------------------------------------------------- # Check whether the specific database copy meets the criteria for a switchover or failover # This function is called by the Move-DagActiveCopy and Execute-LossyDatabaseFailOver functions # # CI status is ignored for lossy moves. The calling function will need to decide if that is suitable. # function Test-DagTargetCopy ([object]$copyStatus, [switch]$Lossless, [switch]$CICheck=$false) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0067 -f $copyStatus.Name,$Lossless,$CICheck,"Test-DagTargetCopy") Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0068 -f $copyStatus.Name,$copyStatus.Status,$copyStatus.ContentIndexState,$copyStatus.CopyQueueLength,$copyStatus.ReplayQueueLength,"Test-DagTargetCopy") [bool]$healthy = $false if ($Lossless) {$allowedCopyQueueLength = 10} else {$allowedCopyQueueLength = 50} if ( ($copyStatus.Status -eq 'Healthy') ` -and ($copyStatus.CopyQueueLength -le $allowedCopyQueueLength) ` -and ($copyStatus.ReplayQueueLength -le 100)) { if ( !$CICheck ) { $healthy = $true; } else { if (($copyStatus.ContentIndexState -eq 'Healthy') -or !$Lossless) { $healthy = $true } } } Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0066 -f $healthy,"Test-DagTargetCopy") return $healthy } # Verify that the new active is mounted # This function can be called with or without the srcServer property. # If it is called with srcServer, it assumes that the database is mounted on another server. # If it is not, it assumes that the database may be mounted on the original server. # # Supports $DagScriptTesting for -whatif function Verify-DagMasterCopy ([object] $db, [object] $srcServer, [switch]$StopOnError) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0069 -f $db,$srcServer,"Verify-DagMasterCopy") $moveddb = Get-MailboxDatabase -Identity $db -Status $srcServerName = $srcServer.Name $targetServerName = $moveddb.MountedOnServer -replace "\..*$" $originallyMounted = $db.Mounted if ($originallyMounted -eq $true) { if ($moveddb.Mounted -eq $true) { if ($srcServerName) { if ($srcServerName -ne $targetServerName) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0070 -f $db,$srcServerName,$targetServerName,"Verify-DagMasterCopy") } else { if ( $DagScriptTesting ) { write-host ($DagCommonLibrary_LocalizedStrings.res_0071 -f $db,$srcServerName,"Verify-DagMasterCopy") } else { Log-Error ($DagCommonLibrary_LocalizedStrings.res_0072 -f $db,$srcServerName,"Verify-DagMasterCopy") -Stop:$StopOnError } } } else { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0073 -f $db,$targetServerName,"Verify-DagMasterCopy") } } else { if ( $DagScriptTesting ) { write-host ($DagCommonLibrary_LocalizedStrings.res_0074 -f $db,$targetServerName,"Verify-DagMasterCopy","Move-ActiveMailboxDatabase") } else { Log-Error ($DagCommonLibrary_LocalizedStrings.res_0075 -f $db,$targetServerName,"Verify-DagMasterCopy","Move-ActiveMailboxDatabase") -Stop:$StopOnError } } } elseif ($originallyMounted -eq $false) { if ($moveddb.Mounted -eq $true) { if ( $DagScriptTesting ) { write-host ($DagCommonLibrary_LocalizedStrings.res_0076 -f $db,$targetServerName,"Verify-DagMasterCopy","Move-ActiveMailboxDatabase") } else { Log-Error ($DagCommonLibrary_LocalizedStrings.res_0077 -f $db,$targetServerName,"Verify-DagMasterCopy","Move-ActiveMailboxDatabase") -Stop:$StopOnError } } else { if ($srcServerName) { if ($srcServerName -ne $targetServerName) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0078 -f $db,$srcServerName,$targetServerName,"Verify-DagMasterCopy") } else { if ( $DagScriptTesting ) { write-host ($DagCommonLibrary_LocalizedStrings.res_0079 -f $db,$srcServerName,"Verify-DagMasterCopy") } else { Log-Error ($DagCommonLibrary_LocalizedStrings.res_0080 -f $db,$srcServerName,"Verify-DagMasterCopy") -Stop:$StopOnError } } } else { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0081 -f $db,$targetServerName,"Verify-DagMasterCopy") } } } else { if ( $DagScriptTesting ) { write-host ($DagCommonLibrary_LocalizedStrings.res_0082 -f $db,"Verify-DagMasterCopy","Verify-DagMasterCopy","get-mailboxdatabase","-status") } else { Log-Error ($DagCommonLibrary_LocalizedStrings.res_0083 -f $db,"Verify-DagMasterCopy","Verify-DagMasterCopy","get-mailboxdatabase","-status") -Stop:$StopOnError } } Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0042 -f "Verify-DagMasterCopy") } # This function tries to move the DB back to the specified server. # It is only meant to be called as part of "rollback" of another move that failed. # Any other usage of this method should be carefully evaluated. # Typically, this should be called in a 'catch' or 'finally' block as part of error handling. function Rollback-DatabaseOnFailedMove([string]$desiredActiveServer, [bool]$shouldBeMounted, [Object]$db) { $currentDb = Get-MailboxDatabase -Identity $db -Status $currentActive = $currentDb.MountedOnServer -replace "\..*$" # Case 1: The active server hasn't changed so we simply need to mount the DB if ($desiredActiveServer -eq $currentActive) { if ($shouldBeMounted) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0084 -f $db.Name,$currentActive,"Rollback-DatabaseOnFailedMove") if ( $DagScriptTesting ) { $mountCmd = { write-host ($DagCommonLibrary_LocalizedStrings.res_0085 -f $currentDb,"Rollback-DatabaseOnFailedMove","Mount-Database","-Identity") } } else { $mountCmd = { Mount-Database -Identity $currentDb } } if (TryExecute-ScriptBlock $mountCmd) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0086 -f $db.Name,$currentActive,"Rollback-DatabaseOnFailedMove") return } Log-Error ($DagCommonLibrary_LocalizedStrings.res_0087 -f $db.Name,$currentActive,"Rollback-DatabaseOnFailedMove") return } else { # The DB needs to be left dismounted, but we won't explicitly dismount it in the script. Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0088 -f $db.Name,$currentActive,$shouldBeMounted,$false,"Rollback-DatabaseOnFailedMove") return } } # Case 2: The active server has changed so we need to move the DB back else { # We'll try the most basic move first Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0089 -f $db.Name,$currentActive,$currentDb,$desiredActiveServer,$false,"Rollback-DatabaseOnFailedMove","Move-ActiveMailboxDatabase","-Identity","-ActivateOnServer","-Confirm") if ( $DagScriptTesting ) { $mountCmd = { write-host ($DagCommonLibrary_LocalizedStrings.res_0090 -f $currentDb,$desiredActiveServer,$false,"Rollback-DatabaseOnFailedMove","Move-ActiveMailboxDatabase","-Identity","-ActivateOnServer","-Confirm") } } else { $moveCmd = { Move-ActiveMailboxDatabase -Identity $currentDb -ActivateOnServer $desiredActiveServer -Confirm:$false } } if (TryExecute-ScriptBlock $moveCmd) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0091 -f $db.Name,$currentActive,"Rollback-DatabaseOnFailedMove") return } Log-Error ($DagCommonLibrary_LocalizedStrings.res_0092 -f $db.Name,$currentActive,"Rollback-DatabaseOnFailedMove") # Now, try the move with -SkipClientExperienceChecks in case ContentIndexing was preventing the move Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0093 -f $db.Name,$currentActive,$currentDb,$desiredActiveServer,$false,"Rollback-DatabaseOnFailedMove","Move-ActiveMailboxDatabase","-Identity","-ActivateOnServer","-SkipClientExperienceChecks","-Confirm") if ( $DagScriptTesting ) { $mountCmd = { write-host ($DagCommonLibrary_LocalizedStrings.res_0094 -f $currentDb,$desiredActiveServer,$false,"Rollback-DatabaseOnFailedMove","Move-ActiveMailboxDatabase","-Identity","-ActivateOnServer","-SkipClientExperienceChecks","-Confirm") } } else { $moveCmd = { Move-ActiveMailboxDatabase -Identity $currentDb -ActivateOnServer $desiredActiveServer -SkipClientExperienceChecks -Confirm:$false } } if (TryExecute-ScriptBlock $moveCmd) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0095 -f $db.Name,$currentActive,"Rollback-DatabaseOnFailedMove") return } Log-Error ($DagCommonLibrary_LocalizedStrings.res_0096 -f $db.Name,$currentActive,"Rollback-DatabaseOnFailedMove") } } # Common function to call cluster.exe # Will try to connect to $dagName or $serverName to execute the command. If it # fails for certain reasons (such as the server being unavailable) it will # retry on the other machines in the cluster. # # Parameters: # $dagName. Name of the DAG. Optional. # $serverName. Name of the server to use. Optional. # One of $dagName or $serverName MUST be supplied. # $clusterCommand. The command to execute. # # Returns: # @( $errorCode, $textOutput ) function Call-ClusterExe( [string] $dagName, [string] $serverName, [string] $clusterCommand ) { log-verbose ($DagCommonLibrary_LocalizedStrings.res_0097 -f $dagName,$serverName,$clusterCommand,"Call-ClusterExe") $script:exitCode = -1; $script:textOutput = $null; $exitCode = -1; $textOutput = $null; if ( ( ! $dagName ) -and ( ! $serverName ) ) { log-error -stop "Call-ClusterExe was called with neither a dag name nor a server name!" } $namesToTry = @( Get-ClusterNames -dagName $dagName -serverName $serverName ) foreach ( $nodeName in $namesToTry ) { # Start a script block so that $error.Clear doesn't affect # callers, and ErrorActionPreference is unchanged. &{ # Simply specifying -erroraction is not enough. $ErrorActionPreference = [System.Management.Automation.ActionPreference]::SilentlyContinue # Method B $clusCommand = "cluster.exe /cluster:$nodeName $clusterCommand 2>&1" log-verbose ($DagCommonLibrary_LocalizedStrings.res_0098 -f $clusCommand,$nodeName); # Run the command. $script:textOutput = invoke-expression $clusCommand -erroraction:silentlycontinue # We're handling the errors with the exit codes. $error.Clear() # Save the exit code. $script:exitCode = $LastExitCode # Superfluous verbose logging. log-verbose ($DagCommonLibrary_LocalizedStrings.res_0099 -f $script:exitCode,$script:textOutput,"Call-ClusterExe") } # Convert from $script scope to locals. $exitCode=$script:exitCode $textOutput=$script:textOutput log-verbose ($DagCommonLibrary_LocalizedStrings.res_0100 -f $exitCode,$textOutput,"Call-ClusterExe") if ( $LastExitCode -eq 1722 ) { # 1722 is RPC_S_SERVER_UNAVAILABLE log-verbose ($DagCommonLibrary_LocalizedStrings.res_0101 -f $serverName,"Call-ClusterExe") continue; } elseif ( $LastExitCode -eq 1753 ) { # 1753 is EPT_S_NOT_REGISTERED log-verbose ($DagCommonLibrary_LocalizedStrings.res_0102 -f $serverName,"Call-ClusterExe") continue; } elseif ( $LastExitCode -eq 1825 ) { # 1825 is RPC_S_SEC_PKG_ERROR log-verbose ($DagCommonLibrary_LocalizedStrings.res_0103 -f $serverName,"Call-ClusterExe") continue; } elseif ( $LastExitCode -ne 0 ) { Log-warning ($DagCommonLibrary_LocalizedStrings.res_0104 -f $LastExitCode,"Call-ClusterExe","retry-able") break; } else { # It returned 0. break; } } return @( $exitCode, $textOutput ); } # Looks up the DAG and returns an array of servers to contact. # If serverName is specified, list it first. # If dagName is specified, then add the dag netname first, # followed by all of the servers in the dag (excluding serverName) function Get-ClusterNames( [string] $dagName, [string] $serverName ) { $serversToTry = @( ); # If they specified a server, use it first. if ( $serverName ) { $serversToTry += $serverName; } if ( $dagName ) { $dag = @( get-databaseavailabilitygroup $dagName -erroraction silentlycontinue ); if ( ( ! $dag ) -or ( $dag.Length -eq 0 ) ) { log-warning ($DagCommonLibrary_LocalizedStrings.res_0105 -f $dagName,"Get-ClusterNames","get-dag") } elseif ( $dag.Length -ne 1 ) { log-warning ($DagCommonLibrary_LocalizedStrings.res_0106 -f $dag.Length,$dagName,"Get-ClusterNames","get-dag") } else { # If they specified a valid DAG, try the netname before any of # the member servers. $serversToTry += $dag[0].Name foreach ( $server in $dag[0].Servers ) { # Don't add it a second time! if ( $server.Name -ne $serverName ) { $serversToTry += $server.Name } } } } log-verbose ($DagCommonLibrary_LocalizedStrings.res_0107 -f $dagName,$serverName,$serversToTry,"Get-ClusterNames") return $serversToTry; } # Common function to run a scriptblock, log any error that occurred, and return # a boolean to indicate whether it was successful or not. # NOTE: ErrorActionPreference of "Stop" is used to catch all errors. # # Optional parameters: # # cleanupCommand # This scriptblock will be executed with ErrorActionPreference of "Continue", # if an error occurred while running $runCommand. # # throwOnError # If true, the error from $runCommand will be rethrown. Otherwise 'false' is returned on error. # # silentOnErrors # If true, the error from $runCommand will not be logged via Log-ErrorRecord (i.e. Write-Error) function TryExecute-ScriptBlock ([ScriptBlock]$runCommand, [ScriptBlock]$cleanupCommand={}, [bool]$throwOnError=$false, [bool]$silentOnErrors=$false) { # Run the following in a separate script block so that we can change # ErrorActionPerefence without affecting the rest of the script. &{ $ErrorActionPreference = "Stop" [bool]$success = $false; try { $ignoredObjects = @(&$runCommand) $success = $true; } catch { # Any error will end up in this catch block # For some reason, PS does not write out any errors unless I use this # scriptblock with "Continue" ErrorActionPreference. &{ $ErrorActionPreference = "Continue" if (!$silentOnErrors) { Log-ErrorRecord $_ } else { Log-Verbose $_ } # Run the cleanup scriptblock $ignoredObjects = @(&$cleanupCommand) } if ($throwOnError) { throw } } # Curious PS behavior: It appears that 'return' trumps 'throw', so don't return... if (!$throwOnError -or $success) { return $success } } } # Common function to run a scriptblock, log any error that occurred, and return # a boolean to indicate whether it was successful or not. # NOTE: ErrorActionPreference of "Stop" is used to catch all errors. # # Optional parameters: # # cleanupCommand # This scriptblock will be executed with ErrorActionPreference of "Continue", # if an error occurred while running $runCommand. # # throwOnError # If true, the error from $runCommand will be rethrown. Otherwise 'false' is returned on error. # # silentOnErrors # If true, the error from $runCommand will not be logged via Log-ErrorRecord (i.e. Write-Error) function TryExecute-ScriptBlockReturnErrorMessage ([ScriptBlock]$runCommand, [ScriptBlock]$cleanupCommand={}, [bool]$throwOnError=$false, [bool]$silentOnErrors=$false) { # Run the following in a separate script block so that we can change # ErrorActionPerefence without affecting the rest of the script. &{ $ErrorActionPreference = "Stop" [bool]$success = $false; try { $ignoredObjects = @(&$runCommand) $success = $true; } catch { # Any error will end up in this catch block # For some reason, PS does not write out any errors unless I use this # scriptblock with "Continue" ErrorActionPreference. &{ $ErrorActionPreference = "Continue" if (!$silentOnErrors) { Log-ErrorRecord $_ } else { Log-Verbose $_ Write-Output $_ } # Run the cleanup scriptblock $ignoredObjects = @(&$cleanupCommand) } if ($throwOnError) { throw } } } } # Sleep for the specified duration (in seconds) function Sleep-ForSeconds ( [int]$sleepSecs ) { Log-Verbose ($DagCommonLibrary_LocalizedStrings.res_0108 -f $sleepSecs) Start-Sleep $sleepSecs } # Common function to retrieve the current UTC time string function Get-CurrentTimeString { return [DateTime]::UtcNow.ToString("[HH:mm:ss.fff UTC]") } # Common function for verbose logging function Log-Verbose ( [string]$msg ) { $timeStamp = Get-CurrentTimeString Write-Debug "$timeStamp $msg" } # Common function for warning logging function Log-Warning ( [string]$msg ) { $timeStamp = Get-CurrentTimeString Write-Warning "$timeStamp $msg" } # Common function for error logging function Log-Error ( [string]$msg, [switch]$Stop) { $timeStamp = Get-CurrentTimeString if (!$Stop) { Write-Error "$timeStamp $msg" } else { Write-Error "$timeStamp $msg" -ErrorAction:Stop } } # Common function for logging an error, given an ErrorRecord function Log-ErrorRecord( [System.Management.Automation.ErrorRecord] $errRecord, [switch]$Stop ) { # Trim the message so it will not display the "ErrorActionPreference is set to Stop" message # $failedMessage = $errRecord.ToString() if ($failedMessage.IndexOf("ErrorActionPreference") -ne -1) { $failedMessage = $failedMessage.Substring($failedMessage.IndexOf("set to Stop: ") + 13) } $failedMessage = $failedMessage -replace "`r" $failedMessage = $failedMessage -replace "`n" $failedCommand = $errRecord.InvocationInfo.MyCommand Log-Error ($DagCommonLibrary_LocalizedStrings.res_0109 -f $failedCommand,$failedMessage) -Stop:$Stop } function Is-GlobalVariableDefined( [string] $varName ) { return get-variable -scope global -name $varName -ErrorAction SilentlyContinue; } function Is-VariableDefined( [string] $varName ) { test-path variable:$varName; } function Get-DatacenterScriptDirectory { #[string]$filePath = $RoleDatacenterPath [string]$filePath = $null if ( Is-GlobalVariableDefined "RoleDatacenterPath") { $filePath = $RoleDatacenterPath } if (!$filePath) { # For testing scenarios, $RoleDatacenterPath is not defined # Try to find the Datacenter directory [string]$version = [Microsoft.Exchange.Diagnostics.BuildVersionConstants]::RegistryVersionSubKey $installPath = (Get-ItemProperty -path "HKLM:SOFTWARE\Microsoft\ExchangeServer\$version\Setup").MsiInstallPath.TrimEnd("\") $datacenterPath = $installPath + '\Datacenter' if (Test-Path $datacenterPath) { $filePath = $datacenterPath } else { # Get the current script name. The method is different if the script is # executed or if it is dot-sourced, so do both. $thisScriptName = $myinvocation.scriptname if ( ! $thisScriptName ) { $thisScriptName = $myinvocation.MyCommand.Path } $filePath = Split-Path -Parent $thisScriptName; } } return $filePath } function Initialize-RoleDatacenterPathVariable { if (!(Is-GlobalVariableDefined "RoleDatacenterPath")) { $global:RoleDatacenterPath = Get-DatacenterScriptDirectory } } Initialize-RoleDatacenterPathVariable # If DatacenterLogHelper is available dot source the file if (Test-Path "$RoledatacenterPath\DatacenterLogHelper.ps1") { . "$RoledatacenterPath\DatacenterLogHelper.ps1" } elseif (Test-Path ".\DatacenterLogHelper.ps1") { . ".\DatacenterLogHelper.ps1" } # SIG # Begin signature block # MIIauQYJKoZIhvcNAQcCoIIaqjCCGqYCAQExCzAJBgUrDgMCGgUAMGkGCisGAQQB # gjcCAQSgWzBZMDQGCisGAQQBgjcCAR4wJgIDAQAABBAfzDtgWUsITrck0sYpfvNR # AgEAAgEAAgEAAgEAAgEAMCEwCQYFKw4DAhoFAAQUWZrD8cF5IJq3/0l0ZE/tXC1f # FvygghV6MIIEuzCCA6OgAwIBAgITMwAAAFrtL/TkIJk/OgAAAAAAWjANBgkqhkiG # 9w0BAQUFADB3MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4G # A1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSEw # HwYDVQQDExhNaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EwHhcNMTQwNTIzMTcxMzE1 # WhcNMTUwODIzMTcxMzE1WjCBqzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAldBMRAw # DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # DTALBgNVBAsTBE1PUFIxJzAlBgNVBAsTHm5DaXBoZXIgRFNFIEVTTjpCOEVDLTMw # QTQtNzE0NDElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC # ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALMhIt9q0L/7KcnVbHqJqY0T # vJS16X0pZdp/9B+rDHlhZlRhlgfw1GBLMZsJr30obdCle4dfdqHSxinHljqjXxeM # duC3lgcPx2JhtLaq9kYUKQMuJrAdSgjgfdNcMBKmm/a5Dj1TFmmdu2UnQsHoMjUO # 9yn/3lsgTLsvaIQkD6uRxPPOKl5YRu2pRbRptlQmkRJi/W8O5M/53D/aKWkfSq7u # wIJC64Jz6VFTEb/dqx1vsgpQeAuD7xsIsxtnb9MFfaEJn8J3iKCjWMFP/2fz3uzH # 9TPcikUOlkYUKIccYLf1qlpATHC1acBGyNTo4sWQ3gtlNdRUgNLpnSBWr9TfzbkC # AwEAAaOCAQkwggEFMB0GA1UdDgQWBBS+Z+AuAhuvCnINOh1/jJ1rImYR9zAfBgNV # HSMEGDAWgBQjNPjZUkZwCu1A+3b7syuwwzWzDzBUBgNVHR8ETTBLMEmgR6BFhkNo # dHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNyb3Nv # ZnRUaW1lU3RhbXBQQ0EuY3JsMFgGCCsGAQUFBwEBBEwwSjBIBggrBgEFBQcwAoY8 # aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNyb3NvZnRUaW1l # U3RhbXBQQ0EuY3J0MBMGA1UdJQQMMAoGCCsGAQUFBwMIMA0GCSqGSIb3DQEBBQUA # A4IBAQAgU4KQrqZNTn4zScizrcTDfhXQEvIPJ4p/W78+VOpB6VQDKym63VSIu7n3 # 2c5T7RAWPclGcLQA0fI0XaejIiyqIuFrob8PDYfQHgIb73i2iSDQLKsLdDguphD/ # 2pGrLEA8JhWqrN7Cz0qTA81r4qSymRpdR0Tx3IIf5ki0pmmZwS7phyPqCNJp5mLf # cfHrI78hZfmkV8STLdsWeBWqPqLkhfwXvsBPFduq8Ki6ESus+is1Fm5bc/4w0Pur # k6DezULaNj+R9+A3jNkHrTsnu/9UIHfG/RHpGuZpsjMnqwWuWI+mqX9dEhFoDCyj # MRYNviGrnPCuGnxA1daDFhXYKPvlMIIE7DCCA9SgAwIBAgITMwAAAMps1TISNcTh # VQABAAAAyjANBgkqhkiG9w0BAQUFADB5MQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMSMwIQYDVQQDExpNaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBD # QTAeFw0xNDA0MjIxNzM5MDBaFw0xNTA3MjIxNzM5MDBaMIGDMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMQ0wCwYDVQQLEwRNT1BSMR4wHAYDVQQD # ExVNaWNyb3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw # ggEKAoIBAQCWcV3tBkb6hMudW7dGx7DhtBE5A62xFXNgnOuntm4aPD//ZeM08aal # IV5WmWxY5JKhClzC09xSLwxlmiBhQFMxnGyPIX26+f4TUFJglTpbuVildGFBqZTg # rSZOTKGXcEknXnxnyk8ecYRGvB1LtuIPxcYnyQfmegqlFwAZTHBFOC2BtFCqxWfR # +nm8xcyhcpv0JTSY+FTfEjk4Ei+ka6Wafsdi0dzP7T00+LnfNTC67HkyqeGprFVN # TH9MVsMTC3bxB/nMR6z7iNVSpR4o+j0tz8+EmIZxZRHPhckJRIbhb+ex/KxARKWp # iyM/gkmd1ZZZUBNZGHP/QwytK9R/MEBnAgMBAAGjggFgMIIBXDATBgNVHSUEDDAK # BggrBgEFBQcDAzAdBgNVHQ4EFgQUH17iXVCNVoa+SjzPBOinh7XLv4MwUQYDVR0R # BEowSKRGMEQxDTALBgNVBAsTBE1PUFIxMzAxBgNVBAUTKjMxNTk1K2I0MjE4ZjEz # LTZmY2EtNDkwZi05YzQ3LTNmYzU1N2RmYzQ0MDAfBgNVHSMEGDAWgBTLEejK0rQW # WAHJNy4zFha5TJoKHzBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNDb2RTaWdQQ0FfMDgtMzEtMjAx # MC5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1p # Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY0NvZFNpZ1BDQV8wOC0zMS0yMDEwLmNy # dDANBgkqhkiG9w0BAQUFAAOCAQEAd1zr15E9zb17g9mFqbBDnXN8F8kP7Tbbx7Us # G177VAU6g3FAgQmit3EmXtZ9tmw7yapfXQMYKh0nfgfpxWUftc8Nt1THKDhaiOd7 # wRm2VjK64szLk9uvbg9dRPXUsO8b1U7Brw7vIJvy4f4nXejF/2H2GdIoCiKd381w # gp4YctgjzHosQ+7/6sDg5h2qnpczAFJvB7jTiGzepAY1p8JThmURdwmPNVm52Iao # AP74MX0s9IwFncDB1XdybOlNWSaD8cKyiFeTNQB8UCu8Wfz+HCk4gtPeUpdFKRhO # lludul8bo/EnUOoHlehtNA04V9w3KDWVOjic1O1qhV0OIhFeezCCBbwwggOkoAMC # AQICCmEzJhoAAAAAADEwDQYJKoZIhvcNAQEFBQAwXzETMBEGCgmSJomT8ixkARkW # A2NvbTEZMBcGCgmSJomT8ixkARkWCW1pY3Jvc29mdDEtMCsGA1UEAxMkTWljcm9z # b2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MB4XDTEwMDgzMTIyMTkzMloX # DTIwMDgzMTIyMjkzMloweTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0 # b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3Jh # dGlvbjEjMCEGA1UEAxMaTWljcm9zb2Z0IENvZGUgU2lnbmluZyBQQ0EwggEiMA0G # CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCycllcGTBkvx2aYCAgQpl2U2w+G9Zv # zMvx6mv+lxYQ4N86dIMaty+gMuz/3sJCTiPVcgDbNVcKicquIEn08GisTUuNpb15 # S3GbRwfa/SXfnXWIz6pzRH/XgdvzvfI2pMlcRdyvrT3gKGiXGqelcnNW8ReU5P01 # lHKg1nZfHndFg4U4FtBzWwW6Z1KNpbJpL9oZC/6SdCnidi9U3RQwWfjSjWL9y8lf # RjFQuScT5EAwz3IpECgixzdOPaAyPZDNoTgGhVxOVoIoKgUyt0vXT2Pn0i1i8UU9 # 56wIAPZGoZ7RW4wmU+h6qkryRs83PDietHdcpReejcsRj1Y8wawJXwPTAgMBAAGj # ggFeMIIBWjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTLEejK0rQWWAHJNy4z # Fha5TJoKHzALBgNVHQ8EBAMCAYYwEgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEE # AYI3FQIEFgQU/dExTtMmipXhmGA7qDFvpjy82C0wGQYJKwYBBAGCNxQCBAweCgBT # AHUAYgBDAEEwHwYDVR0jBBgwFoAUDqyCYEBWJ5flJRP8KuEKU5VZ5KQwUAYDVR0f # BEkwRzBFoEOgQYY/aHR0cDovL2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJv # ZHVjdHMvbWljcm9zb2Z0cm9vdGNlcnQuY3JsMFQGCCsGAQUFBwEBBEgwRjBEBggr # BgEFBQcwAoY4aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNy # b3NvZnRSb290Q2VydC5jcnQwDQYJKoZIhvcNAQEFBQADggIBAFk5Pn8mRq/rb0Cx # MrVq6w4vbqhJ9+tfde1MOy3XQ60L/svpLTGjI8x8UJiAIV2sPS9MuqKoVpzjcLu4 # tPh5tUly9z7qQX/K4QwXaculnCAt+gtQxFbNLeNK0rxw56gNogOlVuC4iktX8pVC # nPHz7+7jhh80PLhWmvBTI4UqpIIck+KUBx3y4k74jKHK6BOlkU7IG9KPcpUqcW2b # Gvgc8FPWZ8wi/1wdzaKMvSeyeWNWRKJRzfnpo1hW3ZsCRUQvX/TartSCMm78pJUT # 5Otp56miLL7IKxAOZY6Z2/Wi+hImCWU4lPF6H0q70eFW6NB4lhhcyTUWX92THUmO # Lb6tNEQc7hAVGgBd3TVbIc6YxwnuhQ6MT20OE049fClInHLR82zKwexwo1eSV32U # jaAbSANa98+jZwp0pTbtLS8XyOZyNxL0b7E8Z4L5UrKNMxZlHg6K3RDeZPRvzkbU # 0xfpecQEtNP7LN8fip6sCvsTJ0Ct5PnhqX9GuwdgR2VgQE6wQuxO7bN2edgKNAlt # HIAxH+IOVN3lofvlRxCtZJj/UBYufL8FIXrilUEnacOTj5XJjdibIa4NXJzwoq6G # aIMMai27dmsAHZat8hZ79haDJLmIz2qoRzEvmtzjcT3XAH5iR9HOiMm4GPoOco3B # oz2vAkBq/2mbluIQqBC0N1AI1sM9MIIGBzCCA++gAwIBAgIKYRZoNAAAAAAAHDAN # BgkqhkiG9w0BAQUFADBfMRMwEQYKCZImiZPyLGQBGRYDY29tMRkwFwYKCZImiZPy # LGQBGRYJbWljcm9zb2Z0MS0wKwYDVQQDEyRNaWNyb3NvZnQgUm9vdCBDZXJ0aWZp # Y2F0ZSBBdXRob3JpdHkwHhcNMDcwNDAzMTI1MzA5WhcNMjEwNDAzMTMwMzA5WjB3 # MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVk # bW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSEwHwYDVQQDExhN # aWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw # ggEKAoIBAQCfoWyx39tIkip8ay4Z4b3i48WZUSNQrc7dGE4kD+7Rp9FMrXQwIBHr # B9VUlRVJlBtCkq6YXDAm2gBr6Hu97IkHD/cOBJjwicwfyzMkh53y9GccLPx754gd # 6udOo6HBI1PKjfpFzwnQXq/QsEIEovmmbJNn1yjcRlOwhtDlKEYuJ6yGT1VSDOQD # LPtqkJAwbofzWTCd+n7Wl7PoIZd++NIT8wi3U21StEWQn0gASkdmEScpZqiX5NMG # gUqi+YSnEUcUCYKfhO1VeP4Bmh1QCIUAEDBG7bfeI0a7xC1Un68eeEExd8yb3zuD # k6FhArUdDbH895uyAc4iS1T/+QXDwiALAgMBAAGjggGrMIIBpzAPBgNVHRMBAf8E # BTADAQH/MB0GA1UdDgQWBBQjNPjZUkZwCu1A+3b7syuwwzWzDzALBgNVHQ8EBAMC # AYYwEAYJKwYBBAGCNxUBBAMCAQAwgZgGA1UdIwSBkDCBjYAUDqyCYEBWJ5flJRP8 # KuEKU5VZ5KShY6RhMF8xEzARBgoJkiaJk/IsZAEZFgNjb20xGTAXBgoJkiaJk/Is # ZAEZFgltaWNyb3NvZnQxLTArBgNVBAMTJE1pY3Jvc29mdCBSb290IENlcnRpZmlj # YXRlIEF1dGhvcml0eYIQea0WoUqgpa1Mc1j0BxMuZTBQBgNVHR8ESTBHMEWgQ6BB # hj9odHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9taWNy # b3NvZnRyb290Y2VydC5jcmwwVAYIKwYBBQUHAQEESDBGMEQGCCsGAQUFBzAChjho # dHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY3Jvc29mdFJvb3RD # ZXJ0LmNydDATBgNVHSUEDDAKBggrBgEFBQcDCDANBgkqhkiG9w0BAQUFAAOCAgEA # EJeKw1wDRDbd6bStd9vOeVFNAbEudHFbbQwTq86+e4+4LtQSooxtYrhXAstOIBNQ # md16QOJXu69YmhzhHQGGrLt48ovQ7DsB7uK+jwoFyI1I4vBTFd1Pq5Lk541q1YDB # 5pTyBi+FA+mRKiQicPv2/OR4mS4N9wficLwYTp2OawpylbihOZxnLcVRDupiXD8W # mIsgP+IHGjL5zDFKdjE9K3ILyOpwPf+FChPfwgphjvDXuBfrTot/xTUrXqO/67x9 # C0J71FNyIe4wyrt4ZVxbARcKFA7S2hSY9Ty5ZlizLS/n+YWGzFFW6J1wlGysOUzU # 9nm/qhh6YinvopspNAZ3GmLJPR5tH4LwC8csu89Ds+X57H2146SodDW4TsVxIxIm # dgs8UoxxWkZDFLyzs7BNZ8ifQv+AeSGAnhUwZuhCEl4ayJ4iIdBD6Svpu/RIzCzU # 2DKATCYqSCRfWupW76bemZ3KOm+9gSd0BhHudiG/m4LBJ1S2sWo9iaF2YbRuoROm # v6pH8BJv/YoybLL+31HIjCPJZr2dHYcSZAI9La9Zj7jkIeW1sMpjtHhUBdRBLlCs # lLCleKuzoJZ1GtmShxN1Ii8yqAhuoFuMJb+g74TKIdbrHk/Jmu5J4PcBZW+JC33I # acjmbuqnl84xKf8OxVtc2E0bodj6L54/LlUWa8kTo/0xggSpMIIEpQIBATCBkDB5 # MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVk # bW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSMwIQYDVQQDExpN # aWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQQITMwAAAMps1TISNcThVQABAAAAyjAJ # BgUrDgMCGgUAoIHCMBkGCSqGSIb3DQEJAzEMBgorBgEEAYI3AgEEMBwGCisGAQQB # gjcCAQsxDjAMBgorBgEEAYI3AgEVMCMGCSqGSIb3DQEJBDEWBBQFTqZEDyFhXsum # faGc1RSkWjFJkjBiBgorBgEEAYI3AgEMMVQwUqAqgCgARABhAGcAQwBvAG0AbQBv # AG4ATABpAGIAcgBhAHIAeQAuAHAAcwAxoSSAImh0dHA6Ly93d3cubWljcm9zb2Z0 # LmNvbS9leGNoYW5nZSAwDQYJKoZIhvcNAQEBBQAEggEAZZnb1QzbaHOT2+ZUt1FQ # r/ees+PY5+oAkrqZlUxXIvkrzMD3HPp5dfu/vR7KjteSJUiUN3IanS1TQhO3KmsF # z7QrAEacuYHHFnO8Bi11pUaA4pvdvZA/xeNKSShAZ5gX66zKjBrNIz/mQ9WTTAf7 # apDoeza3kihSgZ7gz44iaKwEAKvwEuDaf1Z2/HyLJLA9G+0CxY1TxBJuLAm8iJCn # XlrKCfwzltqznEUMhldCh3gl7L4IphissHSJazhIXLCOb6zZ2xeB7YGRiYw+5Rw5 # ytKrhA2orD+A/Z8PdAjUQNgAHBkhkGxguiyoH7WQmpALIjCjvFX3UvRYdTC9Tjjq # YaGCAigwggIkBgkqhkiG9w0BCQYxggIVMIICEQIBATCBjjB3MQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSEwHwYDVQQDExhNaWNyb3NvZnQgVGlt # ZS1TdGFtcCBQQ0ECEzMAAABa7S/05CCZPzoAAAAAAFowCQYFKw4DAhoFAKBdMBgG # CSqGSIb3DQEJAzELBgkqhkiG9w0BBwEwHAYJKoZIhvcNAQkFMQ8XDTE0MDcxMTA1 # MDU0N1owIwYJKoZIhvcNAQkEMRYEFPr0DPyqFvtsy/Ym7zK9cMUJ0JOVMA0GCSqG # SIb3DQEBBQUABIIBACEFr5HF7w2yDSYtFC5t3TGh7jno0dz+186UhJZ+Dg50TNG7 # A1EQ/hIOinWKlmqJ5pIDx+IqnrF8DUST04O/kmgojgidgFqtwyuqGKqgJAYnCJdU # 0Lc86/5mdh1DSO3qsbbnV3VPSl9+rS8pot/sCWjcCs++zQ5eKVaAfeWuiXglRgYj # h0QtHszrWCbS5A+CSsQqb5aF3aoxMRmRe3fFMEnPBrab8gsa0SaJxUOBWt+BzYu8 # cdgIY2hEdp+SJ3UjvP97axC08T4ge2P0eAWV1wPQgtg+tbYoJTGh0vkP7VKxqOV4 # NCM4lXwLtlQmJ5rp+dz+ZANWO9Ae3cejaYH0zoo= # SIG # End signature block