# Copyright (c) 2010 Microsoft Corporation. All rights reserved. # # THIS CODE IS MADE AVAILABLE AS IS, WITHOUT WARRANTY OF ANY KIND. THE ENTIRE RISK # OF THE USE OR THE RESULTS FROM THE USE OF THIS CODE REMAINS WITH THE USER. # Requires -Version 2 [CmdletBinding(DefaultParametersetName="Default")] PARAM( [parameter( ParameterSetName = "Default", Mandatory = $true, HelpMessage = "The database to troubleshoot.")] [String] [ValidateNotNullOrEmpty()] $MailboxDatabaseName, [parameter( ParameterSetName = "Default", Mandatory = $true, HelpMessage = "The maximum RPC average latency the server should be experiencing. (Typical thresholds might be 70 or 150.)")] [int] [ValidateRange(1, 200)] [ValidateNotNullOrEmpty()] $LatencyThreshold, [parameter( ParameterSetName = "NoOp", Mandatory = $true, HelpMessage = "Loads helper libraries, then quits. Used to verify proper deployment/installation.")] [switch] $Deploy, [int] [ValidateRange(1,600000)] [ValidateNotNullOrEmpty()] $TimeInServerThreshold, [int] [ValidateRange(1,500)] [ValidateNotNullOrEmpty()] $OperationPerSecondThreshold, [parameter( Mandatory = $false, HelpMessage = "Minimum number of samples of StoreUsageStatistics a user should have to be able to positively identify a single user problem.")] [int] [ValidateRange(1,10)] [ValidateNotNullOrEmpty()] $MinimumUserSampleCount, [parameter( Mandatory = $false, HelpMessage = "Minimum number of samples of StoreUsageStatistics in the set that was collected.")] [int] [ValidateRange(1,250)] [ValidateNotNullOrEmpty()] $MinimumStoreUsageStatisticsSampleCount, [parameter( Mandatory = $false, HelpMessage = "Percent of samples below rop latency threshold at which we want to investigate further.")] [int] [ValidateRange(1,100)] [ValidateNotNullOrEmpty()] $PercentSampleBelowThresholdToAlert, [parameter( Mandatory = $false, HelpMessage = "Whether or not to quarantine heavy users.")] [switch] $Quarantine, [parameter( Mandatory = $false, HelpMessage = "Whether or not we're running under the monitoring context.")] [switch] $MonitoringContext, [parameter( Mandatory = $false, HelpMessage = "Whether or not to quarantine heavy users.")] [String] $QuarantineString, [parameter( Mandatory = $false, HelpMessage = "Whether or not we're running under the monitoring context.")] [String] $MonitoringContextString, [parameter( Mandatory = $false, HelpMessage = "Alert Guid if executed in response to an alert.")] [String] $AlertGUID ) ################################################################################################################################### # # # Script Body # # # ################################################################################################################################### Set-StrictMode -Version Latest if ($QuarantineString -eq "true") { $Quarantine = $true; } if ($MonitoringContextString -eq "true") { $MonitoringContext = $true; } $scriptPath = [System.IO.Path]::GetDirectoryName($MyInvocation.MyCommand.Path) . $scriptPath\CITSLibrary.ps1 . $scriptPath\StoreTSLibrary.ps1 . $scriptPath\StoreTSConstants.ps1 . $scriptPath\DiagnosticScriptCommonLibrary.ps1 Load-ExchangeSnapin # Since we're in strict mode we must declare all variables we use $script:monitoringEvents = $null if ($PSCmdlet.ParameterSetName -eq "NoOp") { if ($MonitoringContext) { #E14:303295 Add a dummy monitoring event to supress SCOM failure alerts Add-MonitoringEvent -Id $StoreLogEntries.DatabaseSpaceTroubleShooterStarted[0] -Type $EVENT_TYPE_INFORMATION -Message "Latency TS in No-op mode" Write-MonitoringEvents } return } $database = Get-MailboxDatabase $MailboxDatabaseName -Status if ($database -eq $null) { $argError = new-object System.ArgumentException ($error[0].Exception.ToString()) throw $argError } if (!$MyInvocation.BoundParameters.ContainsKey("TimeInServerThreshold")) { $TimeInServerThreshold = $TimeInServerDefaultThreshold } # Event log source name for application log $appLogSourceName = "Database Latency Troubleshooter" # Event log source name for crimson log $crimsonLogSourceName = "Database Latency" # The Arguments object is needed for logging # events. $Arguments = new-object -typename Arguments $Arguments.Server = $database.MountedOnServer $Arguments.Database = $database $Arguments.MonitoringContext = $MonitoringContext # Since this TS doesn't run in SCOM we need to write App event logs # for alerts to fire $Arguments.WriteApplicationEvent = $MonitoringContext Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterStarted ` -Parameters @($database) $rpcLatencyCounterName = "\MSExchangeIS Mailbox($database)\rpc average latency" $rpcOpsPerSecondCounterName = "\MSExchangeIS Mailbox($database)\RPC Operations/sec" $readLatencyCounterName = "\MSExchange Database ==> Instances($database)\I/O Database Reads Average Latency" $readRateCounterName = "\MSExchange Database ==> Instances($database)\I/O Database Reads/sec" $writeLatencyCounterName = "\MSExchange Database ==> Instances($database)\I/O Database Writes Average Latency" $writeRateCounterName = "\MSExchange Database ==> Instances($database)\I/O Database Writes/sec" $dsaccessLatencyCounterName = "\MSExchangeIS\dsaccess average latency" $dsaccessCallsCounterName = "\MSExchangeIS\dsaccess active call count" $counterNames = @($rpcLatencyCounterName, $rpcOpsPerSecondCounterName, $readLatencyCounterName, $readRateCounterName, $writeLatencyCounterName, $writeRateCounterName, $dsaccessLatencyCounterName, $dsaccessCallsCounterName) $retries = 0 do { $success = $true $error.Clear() $counterValues = get-counter -ComputerName $database.MountedOnServer -Counter $counterNames -MaxSamples 10 -ErrorAction SilentlyContinue #Verify there were no errors when trying to get the perf counters #Even if we are unsuccessful getting the perf counters and come #out of this loop, it will be handled properly when we try to get #the average results for the counters. This way, we handle the case #where we at least got some counters back and try to go as far into #the TS as possible if ($error.Count -gt 0) { #Check to see if the error was related to the database being failed over, #in this case, we should get CounterPathIsInvalid (E14 Bug: 468424) if ($error[0].FullyQualifiedErrorId -match "CounterPathIsInvalid") { #Get the computername database is currently on to verify that the #database indeed moved $currentDatabase = Get-MailboxDatabase $MailboxDatabaseName -Status if ($currentDatabase.MountedOnServer -ne $database.MountedOnServer) { #Log an event for TS not able to run due to database moved $failureReason = ($StoreLocStrings.DatabaseMoved + $error[0]) Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterNotRunDatabase ` -Parameters @($database, $failureReason) if ($MonitoringContext) { Write-MonitoringEvents } #Clear the error so that it does not get escalated $error.Clear() return } } elseif ($error[0].FullyQualifiedErrorId -match "CounterApiError") { #This could be due to stale powershell session or some other transient issue #Let's try a few more times to get the counters after sleeping for a few #seconds between each retries, with a total sleep time of 30 seconds $retries++ $success = $false if ($retries -lt 5) { $sleepSeconds = $retries * 3 write-verbose ("Unable to get perf counters after {0} retries, Re-trying after sleeping for {1} seconds." -f $retries, $sleepSeconds) Start-Sleep -Seconds $sleepSeconds } } else { #Not the expected error, throw so recovery workflow will escalate if ($MonitoringContext) { Write-MonitoringEvents } throw $error[0] } } } while ((-not $success) -and $retries -lt 5) #Check to see if we got any perf counters back #If we didn't, does not make sense to continue #executing the troubleshooter if (!$counterValues) { if ($MonitoringContext) { Write-MonitoringEvents } $failureReason = "{0} Error: {1}" -f $StoreLocStrings.UnableToGetAnyCounters, $error[0] Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterNotRunDatabase ` -Parameters @($database, $failureReason) throw $failureReason } #Check that latencies are still high $rpcLatency = Get-AverageResultForCounter -results $counterValues -counter $rpcLatencyCounterName if ($rpcLatency -lt $LatencyThreshold) { Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterNoLatency ` -Parameters @($rpcLatency, $database, $LatencyThreshold) if ($MonitoringContext) { Write-MonitoringEvents } return } # Check that Rpc operations/sec are high enough to monitor. if (!$MyInvocation.BoundParameters.ContainsKey("OperationPerSecondThreshold")) { $OperationPerSecondThreshold = $OperationPerSecondDefaultThreshold } $rpcOpsPerSecond = Get-AverageResultForCounter -results $counterValues -counter $rpcOpsPerSecondCounterName if ($rpcOpsPerSecond -lt $OperationPerSecondThreshold) { Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterLowOps ` -Parameters @($database, $rpcLatency, $LatencyThreshold, $rpcOpsPerSecond, $OperationPerSecondThreshold) if ($MonitoringContext) { Write-MonitoringEvents } return } # check if disk transfers/sec < X and disk secs/transfer > Y... if yes, disk is bad. $readLatency = Get-AverageResultForCounter -results $counterValues -counter $readLatencyCounterName $readRate = Get-AverageResultForCounter -results $counterValues -counter $readRateCounterName $writeLatency = Get-AverageResultForCounter -results $counterValues -counter $writeLatencyCounterName $writeRate = Get-AverageResultForCounter -results $counterValues -counter $writeRateCounterName # We don't want to report a bad disk if the disk is being overloaded # so check to see if we have a lot of read/write requests to the disk # if we don't have many requests and either latency (read or write) # is above our thresholds then we have a bad disk. if ((($readRate -lt $DiskReadRateThreshold) -and ($writeRate -lt $DiskWriteRateThreshold)) ` -and (($readLatency -gt $DiskReadLatencyThreshold) -or ($writeLatency -gt $DiskWriteLatencyThreshold))) { # DatabaseLatencyTroubleShooterBadDiskLatencies is only reporting read rate # and read latency, it should be really report write rate and latencies as well Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterBadDiskLatencies ` -Parameters @($database, $readLatency, $readRate, $rpcLatency) if ($MonitoringContext) { Write-MonitoringEvents } return } # Look for high AD latencies and/or call count. $dsaccessLatency = Get-AverageResultForCounter -results $counterValues -counter $dsaccessLatencyCounterName $dsaccessCalls = Get-AverageResultForCounter -results $counterValues -counter $dsaccessCallsCounterName # Report a very-high call count, or a medium-high call count in combination with a medium-high latency. if (($dsaccessCalls -gt $DSAccessCallsStandaloneThreshold) -or (($dsaccessCalls -gt $DSAccessCallsCombinedThreshold) -and ($dsaccessLatency -gt $DSAccessLatencyCombinedThreshold))) { Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterBadDSAccessActiveCallCount ` -Parameters @($database, $dsaccessLatency, $dsaccessCalls, $rpcLatency) if ($MonitoringContext) { Write-MonitoringEvents } return } # Report a very-high latency, even without a high call count. This test must come after the combined test. if ($dsaccessLatency -gt $DSAccessLatencyStandaloneThreshold) { Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterBadDSAccessAverageLatency ` -Parameters @($database, $dsaccessLatency, $dsaccessCalls, $rpcLatency) if ($MonitoringContext) { Write-MonitoringEvents } return } # Run get-storeusagestatistics to find the user who is not already quarantined, has TimeInServer > Threshold # and RopLatency (TimeInServer/RopCount) > Threshold when averaged across all samples and is the top user if (!$MyInvocation.BoundParameters.ContainsKey("PercentSampleBelowThresholdToAlert")) { $PercentSampleBelowThresholdToAlert = $PercentSampleBelowThresholdToAlertDefault } if (!$MyInvocation.BoundParameters.ContainsKey("MinimumStoreUsageStatisticsSampleCount")) { $MinimumStoreUsageStatisticsSampleCount = $MinimumStoreUsageStatisticsSampleCountDefault } $topCpuUsers = @(Get-TopCpuUsers $database -TimeInServerThreshold $TimeInServerThreshold -ROPLatencyThreshold $ROPLatencyThreshold -PercentSampleBelowThresholdToAlert $PercentSampleBelowThresholdToAlert -MinimumStoreUsageStatisticsSampleCount $MinimumStoreUsageStatisticsSampleCount) # See if there is one or more user with TimeInServer and ROP Latency greater than the threshold # If no user in the list, there is no impact to anybody even with high latency # It could likely be SystemMailbox causing this. In any case, we don't care so # just suppress if ($topCpuUsers.Length -eq 0) { # Log an event and return Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterNoMailbox ` -Parameters @($database, $rpcLatency) if ($MonitoringContext) { Write-MonitoringEvents } return } else { # If we have a top user and we are down here, the only action left to take is quarantine *only that user*. # Fire an event indicating quarantine. Exit # In production, we have quarantine set to false. So if we get this far, we are going to get a paging # or non-paging alert (based on RPC latency) with a pointer to saved SUS data. if ($Quarantine -eq $true) { # Quarantine the mailbox write-verbose ("Quarantining: " + $topCpuUsers[0].MailboxGuid + " due to following:") write-verbose ("TotalTimeInServer: " + $topCpuUsers[0].TotalTimeInServer) write-verbose ("AverageTimeInServer: " + $topCpuUsers[0].AverageTimeInServer) write-verbose ("TimeInServerThreshold: " + $TimeInServerThreshold) write-verbose ("RPC Operations per sec: " + $rpcOpsPerSecond) Enable-MailboxQuarantine -Identity $topCpuUsers[0].MailboxGuid.ToString() -Confirm:$false -ErrorAction Stop Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterQuarantineUser ` -Parameters @($topCpuUsers[0].MailboxGuid, $database, $topCpuUsers[0].AverageTimeInServer, $rpcLatency, $rpcOpsPerSecond) if ($MonitoringContext) { Write-MonitoringEvents } return } else { Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterNoQuarantine ` -Parameters @($topCpuUsers[0].MailboxGuid, $database, $topCpuUsers[0].AverageTimeInServer, $rpcLatency, $rpcOpsPerSecond) } } # If we are down here, let's save store usage statistics and send mail to on-calls # so that it can be analysed irrespective of whether we quarantine the mailbox or not. # We only want to do this in prod. Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseLatencyTroubleShooterIneffective ` -Parameters @($database, $rpcLatency, $rpcOpsPerSecond) if ($MonitoringContext -and $global:StoreUsageStatsTS) { $exchangeInstallPath = (get-item HKLM:\SOFTWARE\Microsoft\ExchangeServer\V14\Setup).GetValue("MsiInstallPath") $StoreLibraryPath = Join-Path $exchangeInstallPath "Datacenter\StoreCommonLibrary.ps1" if (Test-Path $StoreLibraryPath) { # Dot-source common libraries to populate alert details. . ($StoreLibraryPath) # Get store usage statistics and send mail $statsPath = Output-StoreUsageStatistics $global:StoreUsageStatsTS $alertGUID # Because of E14:497368, we are going to just throw here so that proper alert mail gets sent # to the on-calls instead of us trying to populate alert details. $failureReason = "{0} {1}" -f $StoreLocStrings.UnableToAnalyze, $statsPath Write-MonitoringEvents throw $failureReason } } if ($MonitoringContext) { Write-MonitoringEvents # Throw an exception to make the workflow that invoked us go to its failure state. # This should result in an escalation - either urgent or non-urgent, depending on what # the health manifest specified in the RecoveryAction. throw "Troubleshoot-DatabaseLatency failed. See the event log for further details." } # SIG # Begin signature block # MIIa0QYJKoZIhvcNAQcCoIIawjCCGr4CAQExCzAJBgUrDgMCGgUAMGkGCisGAQQB # gjcCAQSgWzBZMDQGCisGAQQBgjcCAR4wJgIDAQAABBAfzDtgWUsITrck0sYpfvNR # AgEAAgEAAgEAAgEAAgEAMCEwCQYFKw4DAhoFAAQUpW6nGzvZZwYSous7ihM4fagt # kU+gghV6MIIEuzCCA6OgAwIBAgITMwAAAFrtL/TkIJk/OgAAAAAAWjANBgkqhkiG # 9w0BAQUFADB3MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4G # A1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSEw # HwYDVQQDExhNaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EwHhcNMTQwNTIzMTcxMzE1 # WhcNMTUwODIzMTcxMzE1WjCBqzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAldBMRAw # DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # DTALBgNVBAsTBE1PUFIxJzAlBgNVBAsTHm5DaXBoZXIgRFNFIEVTTjpCOEVDLTMw # QTQtNzE0NDElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC # ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALMhIt9q0L/7KcnVbHqJqY0T # vJS16X0pZdp/9B+rDHlhZlRhlgfw1GBLMZsJr30obdCle4dfdqHSxinHljqjXxeM # duC3lgcPx2JhtLaq9kYUKQMuJrAdSgjgfdNcMBKmm/a5Dj1TFmmdu2UnQsHoMjUO # 9yn/3lsgTLsvaIQkD6uRxPPOKl5YRu2pRbRptlQmkRJi/W8O5M/53D/aKWkfSq7u # wIJC64Jz6VFTEb/dqx1vsgpQeAuD7xsIsxtnb9MFfaEJn8J3iKCjWMFP/2fz3uzH # 9TPcikUOlkYUKIccYLf1qlpATHC1acBGyNTo4sWQ3gtlNdRUgNLpnSBWr9TfzbkC # AwEAAaOCAQkwggEFMB0GA1UdDgQWBBS+Z+AuAhuvCnINOh1/jJ1rImYR9zAfBgNV # HSMEGDAWgBQjNPjZUkZwCu1A+3b7syuwwzWzDzBUBgNVHR8ETTBLMEmgR6BFhkNo # dHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNyb3Nv # ZnRUaW1lU3RhbXBQQ0EuY3JsMFgGCCsGAQUFBwEBBEwwSjBIBggrBgEFBQcwAoY8 # aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNyb3NvZnRUaW1l # U3RhbXBQQ0EuY3J0MBMGA1UdJQQMMAoGCCsGAQUFBwMIMA0GCSqGSIb3DQEBBQUA # A4IBAQAgU4KQrqZNTn4zScizrcTDfhXQEvIPJ4p/W78+VOpB6VQDKym63VSIu7n3 # 2c5T7RAWPclGcLQA0fI0XaejIiyqIuFrob8PDYfQHgIb73i2iSDQLKsLdDguphD/ # 2pGrLEA8JhWqrN7Cz0qTA81r4qSymRpdR0Tx3IIf5ki0pmmZwS7phyPqCNJp5mLf # cfHrI78hZfmkV8STLdsWeBWqPqLkhfwXvsBPFduq8Ki6ESus+is1Fm5bc/4w0Pur # k6DezULaNj+R9+A3jNkHrTsnu/9UIHfG/RHpGuZpsjMnqwWuWI+mqX9dEhFoDCyj # MRYNviGrnPCuGnxA1daDFhXYKPvlMIIE7DCCA9SgAwIBAgITMwAAAMps1TISNcTh # VQABAAAAyjANBgkqhkiG9w0BAQUFADB5MQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMSMwIQYDVQQDExpNaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBD # QTAeFw0xNDA0MjIxNzM5MDBaFw0xNTA3MjIxNzM5MDBaMIGDMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMQ0wCwYDVQQLEwRNT1BSMR4wHAYDVQQD # ExVNaWNyb3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw # ggEKAoIBAQCWcV3tBkb6hMudW7dGx7DhtBE5A62xFXNgnOuntm4aPD//ZeM08aal # IV5WmWxY5JKhClzC09xSLwxlmiBhQFMxnGyPIX26+f4TUFJglTpbuVildGFBqZTg # rSZOTKGXcEknXnxnyk8ecYRGvB1LtuIPxcYnyQfmegqlFwAZTHBFOC2BtFCqxWfR # +nm8xcyhcpv0JTSY+FTfEjk4Ei+ka6Wafsdi0dzP7T00+LnfNTC67HkyqeGprFVN # TH9MVsMTC3bxB/nMR6z7iNVSpR4o+j0tz8+EmIZxZRHPhckJRIbhb+ex/KxARKWp # iyM/gkmd1ZZZUBNZGHP/QwytK9R/MEBnAgMBAAGjggFgMIIBXDATBgNVHSUEDDAK # BggrBgEFBQcDAzAdBgNVHQ4EFgQUH17iXVCNVoa+SjzPBOinh7XLv4MwUQYDVR0R # BEowSKRGMEQxDTALBgNVBAsTBE1PUFIxMzAxBgNVBAUTKjMxNTk1K2I0MjE4ZjEz # LTZmY2EtNDkwZi05YzQ3LTNmYzU1N2RmYzQ0MDAfBgNVHSMEGDAWgBTLEejK0rQW # WAHJNy4zFha5TJoKHzBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNDb2RTaWdQQ0FfMDgtMzEtMjAx # MC5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1p # Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY0NvZFNpZ1BDQV8wOC0zMS0yMDEwLmNy # dDANBgkqhkiG9w0BAQUFAAOCAQEAd1zr15E9zb17g9mFqbBDnXN8F8kP7Tbbx7Us # G177VAU6g3FAgQmit3EmXtZ9tmw7yapfXQMYKh0nfgfpxWUftc8Nt1THKDhaiOd7 # wRm2VjK64szLk9uvbg9dRPXUsO8b1U7Brw7vIJvy4f4nXejF/2H2GdIoCiKd381w # gp4YctgjzHosQ+7/6sDg5h2qnpczAFJvB7jTiGzepAY1p8JThmURdwmPNVm52Iao # AP74MX0s9IwFncDB1XdybOlNWSaD8cKyiFeTNQB8UCu8Wfz+HCk4gtPeUpdFKRhO # lludul8bo/EnUOoHlehtNA04V9w3KDWVOjic1O1qhV0OIhFeezCCBbwwggOkoAMC # AQICCmEzJhoAAAAAADEwDQYJKoZIhvcNAQEFBQAwXzETMBEGCgmSJomT8ixkARkW # A2NvbTEZMBcGCgmSJomT8ixkARkWCW1pY3Jvc29mdDEtMCsGA1UEAxMkTWljcm9z # b2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MB4XDTEwMDgzMTIyMTkzMloX # DTIwMDgzMTIyMjkzMloweTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0 # b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3Jh # dGlvbjEjMCEGA1UEAxMaTWljcm9zb2Z0IENvZGUgU2lnbmluZyBQQ0EwggEiMA0G # CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCycllcGTBkvx2aYCAgQpl2U2w+G9Zv # zMvx6mv+lxYQ4N86dIMaty+gMuz/3sJCTiPVcgDbNVcKicquIEn08GisTUuNpb15 # S3GbRwfa/SXfnXWIz6pzRH/XgdvzvfI2pMlcRdyvrT3gKGiXGqelcnNW8ReU5P01 # lHKg1nZfHndFg4U4FtBzWwW6Z1KNpbJpL9oZC/6SdCnidi9U3RQwWfjSjWL9y8lf # RjFQuScT5EAwz3IpECgixzdOPaAyPZDNoTgGhVxOVoIoKgUyt0vXT2Pn0i1i8UU9 # 56wIAPZGoZ7RW4wmU+h6qkryRs83PDietHdcpReejcsRj1Y8wawJXwPTAgMBAAGj # ggFeMIIBWjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTLEejK0rQWWAHJNy4z # Fha5TJoKHzALBgNVHQ8EBAMCAYYwEgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEE # AYI3FQIEFgQU/dExTtMmipXhmGA7qDFvpjy82C0wGQYJKwYBBAGCNxQCBAweCgBT # AHUAYgBDAEEwHwYDVR0jBBgwFoAUDqyCYEBWJ5flJRP8KuEKU5VZ5KQwUAYDVR0f # BEkwRzBFoEOgQYY/aHR0cDovL2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJv # ZHVjdHMvbWljcm9zb2Z0cm9vdGNlcnQuY3JsMFQGCCsGAQUFBwEBBEgwRjBEBggr # BgEFBQcwAoY4aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNy # b3NvZnRSb290Q2VydC5jcnQwDQYJKoZIhvcNAQEFBQADggIBAFk5Pn8mRq/rb0Cx # MrVq6w4vbqhJ9+tfde1MOy3XQ60L/svpLTGjI8x8UJiAIV2sPS9MuqKoVpzjcLu4 # tPh5tUly9z7qQX/K4QwXaculnCAt+gtQxFbNLeNK0rxw56gNogOlVuC4iktX8pVC # nPHz7+7jhh80PLhWmvBTI4UqpIIck+KUBx3y4k74jKHK6BOlkU7IG9KPcpUqcW2b # Gvgc8FPWZ8wi/1wdzaKMvSeyeWNWRKJRzfnpo1hW3ZsCRUQvX/TartSCMm78pJUT # 5Otp56miLL7IKxAOZY6Z2/Wi+hImCWU4lPF6H0q70eFW6NB4lhhcyTUWX92THUmO # Lb6tNEQc7hAVGgBd3TVbIc6YxwnuhQ6MT20OE049fClInHLR82zKwexwo1eSV32U # jaAbSANa98+jZwp0pTbtLS8XyOZyNxL0b7E8Z4L5UrKNMxZlHg6K3RDeZPRvzkbU # 0xfpecQEtNP7LN8fip6sCvsTJ0Ct5PnhqX9GuwdgR2VgQE6wQuxO7bN2edgKNAlt # HIAxH+IOVN3lofvlRxCtZJj/UBYufL8FIXrilUEnacOTj5XJjdibIa4NXJzwoq6G # aIMMai27dmsAHZat8hZ79haDJLmIz2qoRzEvmtzjcT3XAH5iR9HOiMm4GPoOco3B # oz2vAkBq/2mbluIQqBC0N1AI1sM9MIIGBzCCA++gAwIBAgIKYRZoNAAAAAAAHDAN # BgkqhkiG9w0BAQUFADBfMRMwEQYKCZImiZPyLGQBGRYDY29tMRkwFwYKCZImiZPy # LGQBGRYJbWljcm9zb2Z0MS0wKwYDVQQDEyRNaWNyb3NvZnQgUm9vdCBDZXJ0aWZp # Y2F0ZSBBdXRob3JpdHkwHhcNMDcwNDAzMTI1MzA5WhcNMjEwNDAzMTMwMzA5WjB3 # MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVk # bW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSEwHwYDVQQDExhN # aWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw # ggEKAoIBAQCfoWyx39tIkip8ay4Z4b3i48WZUSNQrc7dGE4kD+7Rp9FMrXQwIBHr # B9VUlRVJlBtCkq6YXDAm2gBr6Hu97IkHD/cOBJjwicwfyzMkh53y9GccLPx754gd # 6udOo6HBI1PKjfpFzwnQXq/QsEIEovmmbJNn1yjcRlOwhtDlKEYuJ6yGT1VSDOQD # LPtqkJAwbofzWTCd+n7Wl7PoIZd++NIT8wi3U21StEWQn0gASkdmEScpZqiX5NMG # gUqi+YSnEUcUCYKfhO1VeP4Bmh1QCIUAEDBG7bfeI0a7xC1Un68eeEExd8yb3zuD # k6FhArUdDbH895uyAc4iS1T/+QXDwiALAgMBAAGjggGrMIIBpzAPBgNVHRMBAf8E # BTADAQH/MB0GA1UdDgQWBBQjNPjZUkZwCu1A+3b7syuwwzWzDzALBgNVHQ8EBAMC # AYYwEAYJKwYBBAGCNxUBBAMCAQAwgZgGA1UdIwSBkDCBjYAUDqyCYEBWJ5flJRP8 # KuEKU5VZ5KShY6RhMF8xEzARBgoJkiaJk/IsZAEZFgNjb20xGTAXBgoJkiaJk/Is # ZAEZFgltaWNyb3NvZnQxLTArBgNVBAMTJE1pY3Jvc29mdCBSb290IENlcnRpZmlj # YXRlIEF1dGhvcml0eYIQea0WoUqgpa1Mc1j0BxMuZTBQBgNVHR8ESTBHMEWgQ6BB # hj9odHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9taWNy # b3NvZnRyb290Y2VydC5jcmwwVAYIKwYBBQUHAQEESDBGMEQGCCsGAQUFBzAChjho # dHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY3Jvc29mdFJvb3RD # ZXJ0LmNydDATBgNVHSUEDDAKBggrBgEFBQcDCDANBgkqhkiG9w0BAQUFAAOCAgEA # EJeKw1wDRDbd6bStd9vOeVFNAbEudHFbbQwTq86+e4+4LtQSooxtYrhXAstOIBNQ # md16QOJXu69YmhzhHQGGrLt48ovQ7DsB7uK+jwoFyI1I4vBTFd1Pq5Lk541q1YDB # 5pTyBi+FA+mRKiQicPv2/OR4mS4N9wficLwYTp2OawpylbihOZxnLcVRDupiXD8W # mIsgP+IHGjL5zDFKdjE9K3ILyOpwPf+FChPfwgphjvDXuBfrTot/xTUrXqO/67x9 # C0J71FNyIe4wyrt4ZVxbARcKFA7S2hSY9Ty5ZlizLS/n+YWGzFFW6J1wlGysOUzU # 9nm/qhh6YinvopspNAZ3GmLJPR5tH4LwC8csu89Ds+X57H2146SodDW4TsVxIxIm # dgs8UoxxWkZDFLyzs7BNZ8ifQv+AeSGAnhUwZuhCEl4ayJ4iIdBD6Svpu/RIzCzU # 2DKATCYqSCRfWupW76bemZ3KOm+9gSd0BhHudiG/m4LBJ1S2sWo9iaF2YbRuoROm # v6pH8BJv/YoybLL+31HIjCPJZr2dHYcSZAI9La9Zj7jkIeW1sMpjtHhUBdRBLlCs # lLCleKuzoJZ1GtmShxN1Ii8yqAhuoFuMJb+g74TKIdbrHk/Jmu5J4PcBZW+JC33I # acjmbuqnl84xKf8OxVtc2E0bodj6L54/LlUWa8kTo/0xggTBMIIEvQIBATCBkDB5 # MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVk # bW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSMwIQYDVQQDExpN # aWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQQITMwAAAMps1TISNcThVQABAAAAyjAJ # BgUrDgMCGgUAoIHaMBkGCSqGSIb3DQEJAzEMBgorBgEEAYI3AgEEMBwGCisGAQQB # gjcCAQsxDjAMBgorBgEEAYI3AgEVMCMGCSqGSIb3DQEJBDEWBBTML2CjzbNoTuls # EzvXUM2RlFdnOjB6BgorBgEEAYI3AgEMMWwwaqBCgEAAVAByAG8AdQBiAGwAZQBz # AGgAbwBvAHQALQBEAGEAdABhAGIAYQBzAGUATABhAHQAZQBuAGMAeQAuAHAAcwAx # oSSAImh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9leGNoYW5nZSAwDQYJKoZIhvcN # AQEBBQAEggEAENsOtdSqy5rNxJ1KTCXRj9AP0Pk+nV0s/FdeyoBDXkUFyI4L9jdM # IAdnVHDm7iYhkZbXaRkmkzE3aPjSKPz5k3WP/v9e/wSGyIqQNn5KoTICfEgSZJ9+ # oJrVyiOWXwmB8QBcEJ2zmUVd6Zs3Y4fydtJbafdgmW4Hxp8I8xxuZPTsuvPmjdbh # 0dDwoiu/sgCx1cMzsnXM6F96vvgQTL86I1/i+imZ5+KAQtyp1XnO8yD4YSxwIP2P # AUBfdA4H4R+CV7WzFWw+aG7/wQ46sV4p7y4PRExwwchyW34zry2tsWWooHiq8GJP # M7M1nAXcsAVsI42NpK9dpjPm2UEypRfu86GCAigwggIkBgkqhkiG9w0BCQYxggIV # MIICEQIBATCBjjB3MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQ # MA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9u # MSEwHwYDVQQDExhNaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0ECEzMAAABa7S/05CCZ # PzoAAAAAAFowCQYFKw4DAhoFAKBdMBgGCSqGSIb3DQEJAzELBgkqhkiG9w0BBwEw # HAYJKoZIhvcNAQkFMQ8XDTE0MDgxNzA1MTIyNVowIwYJKoZIhvcNAQkEMRYEFE3Z # c0gMPghbmiN28LmwY0rahPv6MA0GCSqGSIb3DQEBBQUABIIBAHeIkYCsYSQcFmPc # nKMpZ5ckydhBo+RlUB09tL3ZfMjeWEYD3VQSCb86eCKW20UTNyyX9Hgf7ekKJHL/ # UyfP5ddsk9UUfBn/bfsGNJEYEWah51cWIZTeV8xR8MBGaAdqW9DffnCSN7JN4fo+ # kU0WqSQWOBdZ9ce+dq7M5YzvxtUvPFUTX6zA0ZcJI3S2K+1nwQbQkWLO2Ktn2ibT # eop+zGbx8/U+b5dv+mA/HVQVN2x4dVQHOfTdYB8Nz7zsoAZ9k3ViLjLw1IXFErUu # MDsUSoioZtakNUPNcYuY4VZBJkZGcw34WdZEn/zgR8qEn2gFYuryxxANbBaOuRQi # MeOA6tE= # SIG # End signature block