# Copyright (c) 2010 Microsoft Corporation. All rights reserved. # # THIS CODE IS MADE AVAILABLE AS IS, WITHOUT WARRANTY OF ANY KIND. THE ENTIRE RISK # OF THE USE OR THE RESULTS FROM THE USE OF THIS CODE REMAINS WITH THE USER. # Requires -Version 2 PARAM( [parameter( ParameterSetName = "Default", Mandatory = $true, HelpMessage = "The database to troubleshoot.")] [String] [ValidateNotNullOrEmpty()] $MailboxDatabaseName, [parameter( ParameterSetName = "Server", Mandatory = $true, HelpMessage = "The mailbox server to troubleshoot.")] [String] [ValidateNotNullOrEmpty()] $Server, [parameter( Mandatory = $false, HelpMessage = "The percentage of disk space for the EDB file at which we should start quarantining users.")] [int] [ValidateRange(1, 99)] [ValidateNotNullOrEmpty()] $PercentEdbFreeSpaceThreshold, [parameter( Mandatory = $false, HelpMessage = "The percentage of disk space for the logs at which we should start quarantining users.")] [int] [ValidateRange(1, 99)] [ValidateNotNullOrEmpty()] $PercentLogFreeSpaceThreshold, [parameter( Mandatory = $false, HelpMessage = "The number of hours we can wait before running out of space.")] [int] [ValidateRange(1, 1000000000)] [ValidateNotNullOrEmpty()] $HourThreshold, [parameter( Mandatory = $false, HelpMessage = "Whether or not to quarantine heavy users.")] [switch] $Quarantine, [parameter( Mandatory = $false, HelpMessage = "Whether or not we're running under the monitoring context.")] [switch] $MonitoringContext ) # # Check a single database for free space # function Troubleshoot-DatabaseCopySpace([Microsoft.Exchange.Data.Directory.SystemConfiguration.MailboxDatabase] $database, [int] $PercentEdbFreeSpaceThreshold, [int] $PercentLogFreeSpaceThreshold, [int] $HourThreshold, [bool] $Quarantine, [bool] $MonitoringContext) { if ($database -eq $null) { $argError = new-object System.ArgumentException ($error[0].Exception.ToString()) throw $argError } # Event log source name for application log $appLogSourceName = "Database Space Troubleshooter" # Event log source name for crimson log $crimsonLogSourceName = "Database Space" # The Arguments object is needed for logging # events. $Arguments = new-object -typename Arguments $Arguments.Server = $database.MountedOnServer $Arguments.Database = $database $Arguments.MonitoringContext = $MonitoringContext $Arguments.WriteApplicationEvent = $false # Check the Edb drive for space $edbGrowthRateThreshold = -1 $edbVolume = Get-WmiVolumeFromPath $database.EdbFilePath $database.MountedOnServer $Arguments.InstanceName = $edbVolume.Name Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseSpaceTroubleShooterStarted ` -Parameters @($edbVolume.Name, $database) $edbVolumeFreeSpace = $edbVolume.FreeSpace + $database.AvailableNewMailboxSpace $edbPercentFreeSpace = ($edbVolumeFreeSpace * 100) / $edbVolume.Capacity # Check the Log drive for space $logGrowthRateThreshold = -1 $logVolume = Get-WmiVolumeFromPath $database.LogFolderPath $database.MountedOnServer $logVolumeFreeSpace = $logVolume.FreeSpace $logPercentFreeSpace = ($logVolumeFreeSpace * 100) / $logVolume.Capacity if ($logPercentFreeSpace -lt $PercentLogFreeSpaceThreshold) { $logGrowthRateThreshold = $logVolume.FreeSpace / $HourThreshold } $growthRateThreshold = -1 # Figure out which of the 2 thresholds is lower # to determine whether or not we need to quarantine # some users if ($logGrowthRateThreshold -ne -1) { $growthRateThreshold = $logGrowthRateThreshold } if (($edbGrowthRateThreshold -ne -1) -and ($edbGrowthRateThreshold -lt $logGrowthRateThreshold)) { $growthRateThreshold = $edbGrowthRateThreshold } $counterValue = 0 $counterName = "\MSExchangeIS Mailbox($database)\JET Log Bytes Generated/hour" $logBytesCounter = get-counter -ComputerName $database.MountedOnServer -Counter $counterName -MaxSamples 10 $countervalue = Get-AverageResultForCounter -results $logBytesCounter -counter $counterName if ($null -eq $countervalue) { $countervalue = 0 } write-verbose ("Current Growth Rate: " + $counterValue) write-verbose ("Growth Rate Threshold: " + $growthRateThreshold) # Check if we are low on space compared to any of the thresholds $problemDetected = $false if ($edbPercentFreeSpace -lt $PercentEdbFreeSpaceCriticalThreshold) { # Log an event that will trigger paging alert for critical space issue Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseSpaceTroubleDetectedCriticalSpaceIssue ` -Parameters @($edbVolume.Name, $database, $PercentEdbFreeSpaceCriticalThreshold) $problemDetected = $true } elseif ($edbPercentFreeSpace -lt $PercentEdbFreeSpaceAlertThreshold) { # Log an event that will trigger non-paging alert for low space issue Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseSpaceTroubleDetectedAlertSpaceIssue ` -Parameters @($edbVolume.Name, $database, $PercentEdbFreeSpaceAlertThreshold) $problemDetected = $true } elseif ($edbPercentFreeSpace -lt $PercentEdbFreeSpaceThreshold) { $edbGrowthRateThreshold = $edbVolumeFreeSpace / $HourThreshold # Log a warning event for low space issue Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseSpaceTroubleDetectedWarningSpaceIssue ` -Parameters @($edbVolume.Name, $database, $PercentEdbFreeSpaceThreshold) $problemDetected = $true } # Format to show only 2 decimal places in event log $edbPercentFreeSpace = "{0:N2}" -f $edbPercentFreeSpace $logPercentFreeSpace = "{0:N2}" -f $logPercentFreeSpace $edbVolumeFreeSpaceInGB = "{0:N2}" -f ($edbVolumeFreeSpace/1GB) $logVolumeFreeSpaceInGB = "{0:N2}" -f ($logVolumeFreeSpace/1GB) if ($problemDetected) { # Exclude the database from provisioning as we are below one of the space thresholds Set-MailboxDatabase $database -IsExcludedFromProvisioning $true } else { # We have enough free space that we are not going to even bother looking for outliers causing rapid growth to try to qurantine. if ($growthRateThreshold -eq -1) { Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseSpaceTroubleShooterNoProblemDetected ` -Parameters @($edbVolume.Name, $database, $edbVolumeFreeSpaceInGB, $edbPercentFreeSpace, $logVolumeFreeSpaceInGB, $logPercentFreeSpace, $PercentEdbFreeSpaceThreshold, $PercentLogFreeSpaceThreshold, $HourThreshold, $counterValue, $PercentEdbFreeSpaceCriticalThreshold, $PercentEdbFreeSpaceAlertThreshold, $QuarantineValue) return } } $cUsersQuarantined = 0 $currentGrowthRate = $counterValue # Do we need to quarantine users? if (($currentGrowthRate -lt $growthRateThreshold)) { Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseSpaceTroubleShooterFoundLowSpaceNoQuarantine ` -Parameters @($edbVolume.Name, $database, $edbVolumeFreeSpaceInGB, $edbPercentFreeSpace, $logVolumeFreeSpaceInGB, $logPercentFreeSpace, $PercentEdbFreeSpaceThreshold, $PercentLogFreeSpaceThreshold, $HourThreshold, $growthRateThreshold, $counterValue, $currentGrowthRate, $PercentEdbFreeSpaceCriticalThreshold, $PercentEdbFreeSpaceAlertThreshold, $QuarantineValue) return } # We have to quarantine some users now, so # get the least of the most active users # and figure out which ones we need to quarantine # so that the current log generation rate drops # below our maximum calculated growth rate $topLogGenerators = @(Get-TopLogGenerators $database) $iNextUserToQuarantine = 0 #If we are down here, let's save store usage statistics and send mail to on-calls #so that it can be analysed irrespective of whether we quarantine the mailbox or not. #We only want to do this in prod. if ($MonitoringContext -and $global:StoreUsageStatsLB) { $exchangeInstallPath = (get-item HKLM:\SOFTWARE\Microsoft\ExchangeServer\V14\Setup).GetValue("MsiInstallPath") $StoreLibraryPath = Join-Path $exchangeInstallPath "Datacenter\StoreCommonLibrary.ps1" if (Test-Path $StoreLibraryPath) { # Dot-source common libraries to populate alert details. . ($StoreLibraryPath) #Get store usage statistics and send mail #we are passing a null alertGuid since the space TS does not run in #response to an alert hence there is no alertGuid. $statsPath = Output-StoreUsageStatistics $global:StoreUsageStatsLB $null Send-DataCollectionMail $null $statsPath } } if ($Quarantine) { for($iLogGenerator = 0; $iLogGenerator -lt $topLogGenerators.Length; $iLogGenerator++) { write-verbose ("Current Growth Rate: " + $currentGrowthRate) write-verbose ("Growth Rate Threshold: " + $growthRateThreshold) write-verbose ("Top user: " + $topLogGenerators[$iLogGenerator].MailboxGuid) write-verbose ("Top user logs: " + $topLogGenerators[$iLogGenerator].TotalLogBytes) if ($currentGrowthRate -lt $growthRateThreshold) { Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseSpaceTroubleShooterFoundLowSpace ` -Parameters @($edbVolume.Name, $database, $edbVolumeFreeSpaceInGB, $edbPercentFreeSpace, $logVolumeFreeSpaceInGB, $logPercentFreeSpace, $PercentEdbFreeSpaceThreshold, $PercentLogFreeSpaceThreshold, $HourThreshold, $growthRateThreshold, $counterValue, $currentGrowthRate, $cUsersQuarantined, $PercentEdbFreeSpaceCriticalThreshold, $PercentEdbFreeSpaceAlertThreshold, $QuarantineValue) return } #Quarantine the mailbox write-verbose ("Quarantining: " + $topLogGenerators[$iNextUserToQuarantine].MailboxGuid) Enable-MailboxQuarantine -Identity $topLogGenerators[$iNextUserToQuarantine].MailboxGuid.ToString() -Confirm:$false -ErrorAction Stop Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseSpaceTroubleShooterQuarantineUser ` -Parameters @($topLogGenerators[$iNextUserToQuarantine].MailboxGuid, $database) $currentGrowthRate -= $topLogGenerators[$iNextUserToQuarantine].TotalLogBytes $iNextUserToQuarantine++ $cUsersQuarantined++ } } Log-Event ` -Arguments $Arguments ` -EventInfo $StoreLogEntries.DatabaseSpaceTroubleShooterFinishedInsufficient ` -Parameters @($edbVolume.Name, $database, $edbVolumeFreeSpaceInGB, $edbPercentFreeSpace, $logVolumeFreeSpaceInGB, $logPercentFreeSpace, $PercentEdbFreeSpaceThreshold, $PercentLogFreeSpaceThreshold, $HourThreshold, $growthRateThreshold, $counterValue, $currentGrowthRate, $cUsersQuarantined, $PercentEdbFreeSpaceCriticalThreshold, $PercentEdbFreeSpaceAlertThreshold, $QuarantineValue) } ################################################################################################################################### # # # Script Body # # # ################################################################################################################################### Set-StrictMode -Version Latest $scriptPath = [System.IO.Path]::GetDirectoryName($MyInvocation.MyCommand.Path) . $scriptPath\CITSLibrary.ps1 . $scriptPath\StoreTSLibrary.ps1 . $scriptPath\StoreTSConstants.ps1 . $scriptPath\DiagnosticScriptCommonLibrary.ps1 Load-ExchangeSnapin $QuarantineValue = $false $MonitoringContextValue = $false $MailboxServer = $null $InvocationGuid = $null # Since we're in strict mode we must declare all variables we use $script:monitoringEvents = $null if (!$MyInvocation.BoundParameters.ContainsKey("PercentEdbFreeSpaceThreshold")) { $PercentEdbFreeSpaceThreshold = $PercentEdbFreeSpaceDefaultThreshold } if (!$MyInvocation.BoundParameters.ContainsKey("PercentLogFreeSpaceThreshold")) { $PercentLogFreeSpaceThreshold = $PercentLogFreeSpaceDefaultThreshold } if (!$MyInvocation.BoundParameters.ContainsKey("HourThreshold")) { $HourThreshold = $HourDefaultThreshold } if ($Quarantine) { $QuarantineValue = $true } if ($MonitoringContext) { $MonitoringContextValue = $true #E14:303295 Add a monitoring event to suppress SCOM failure alerts #Use a guid for each invocation so that we can identify #start and finish events for each unique invocation $InvocationGuid = [System.Guid]::NewGuid().ToString() $messageStart = "Database Space TS started successfully for Invocation Guid {0}" -f $InvocationGuid Add-MonitoringEvent -Id $StoreLogEntries.DatabaseSpaceTroubleShooterStarted[0] -Type $EVENT_TYPE_INFORMATION -Message $messageStart } if ($PSCmdlet.ParameterSetName -eq "Server") { $MailboxServer = $Server $databases = @(Get-MailboxDatabase -Server $MailboxServer) } else { $databases = @(Get-MailboxDatabase $MailboxDatabaseName) } foreach($database in $databases) { #Get the status right before invoking the function #so we can narrow down failover related issues $database = Get-MailboxDatabase $database -Status #Run only if the database is specified or in case of server, #on the active copy to avoid running it multiple times #against the same database if (($MailboxServer -eq $null) -or ($database.MountedOnServer -match $MailboxServer)) { Troubleshoot-DatabaseCopySpace ` -database $database ` -PercentEdbFreeSpaceThreshold $PercentEdbFreeSpaceThreshold ` -PercentLogFreeSpaceThreshold $PercentLogFreeSpaceThreshold ` -HourThreshold $HourThreshold ` -Quarantine $QuarantineValue ` -MonitoringContext $MonitoringContextValue } } if ($MonitoringContext) { #Monitoring event to suppress SCOM failure alerts #Also lets us know that TS did finish successfully $messageFinish = "Database Space TS finished successfully for Invocation Guid {0}" -f $InvocationGuid Add-MonitoringEvent -Id 5102 -Type $EVENT_TYPE_INFORMATION -Message $messageFinish Write-MonitoringEvents } # SIG # Begin signature block # MIIa1QYJKoZIhvcNAQcCoIIaxjCCGsICAQExCzAJBgUrDgMCGgUAMGkGCisGAQQB # gjcCAQSgWzBZMDQGCisGAQQBgjcCAR4wJgIDAQAABBAfzDtgWUsITrck0sYpfvNR # AgEAAgEAAgEAAgEAAgEAMCEwCQYFKw4DAhoFAAQUNXI/XtD1kNF5Gip8hvLtLiei # RcygghWCMIIEwzCCA6ugAwIBAgITMwAAAEyh6E3MtHR7OwAAAAAATDANBgkqhkiG # 9w0BAQUFADB3MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4G # A1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSEw # HwYDVQQDExhNaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EwHhcNMTMxMTExMjIxMTMx # WhcNMTUwMjExMjIxMTMxWjCBszELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp # bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw # b3JhdGlvbjENMAsGA1UECxMETU9QUjEnMCUGA1UECxMebkNpcGhlciBEU0UgRVNO # OkMwRjQtMzA4Ni1ERUY4MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBT # ZXJ2aWNlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsdj6GwYrd6jk # lF18D+Z6ppLuilQdpPmEdYWXzMtcltDXdS3ZCPtb0u4tJcY3PvWrfhpT5Ve+a+i/ # ypYK3EbxWh4+AtKy4CaOAGR7vjyT+FgyeYfSGl0jvJxRxA8Q+gRYtRZ2buy8xuW+ # /K2swUHbqs559RyymUGneiUr/6t4DVg6sV5Q3mRM4MoVKt+m6f6kZi9bEAkJJiHU # Pw0vbdL4d5ADbN4UEqWM5zYf9IelsEEXb+NNdGbC/aJxRjVRzGsXUWP6FZSSml9L # KLrmFkVJ6Sy1/ouHr/ylbUPcpjD6KSjvmw0sXIPeEo1qtNtx71wUWiojKP+BcFfx # jAeaE9gqUwIDAQABo4IBCTCCAQUwHQYDVR0OBBYEFLkNrbNN9NqfGrInJlUNIETY # mOL0MB8GA1UdIwQYMBaAFCM0+NlSRnAK7UD7dvuzK7DDNbMPMFQGA1UdHwRNMEsw # SaBHoEWGQ2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3Rz # L01pY3Jvc29mdFRpbWVTdGFtcFBDQS5jcmwwWAYIKwYBBQUHAQEETDBKMEgGCCsG # AQUFBzAChjxodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY3Jv # c29mdFRpbWVTdGFtcFBDQS5jcnQwEwYDVR0lBAwwCgYIKwYBBQUHAwgwDQYJKoZI # hvcNAQEFBQADggEBAAmKTgav6O2Czx0HftcqpyQLLa+aWyR/lHEMVYgkGlIVY+KQ # TQVKmEqc++GnbWhVgrkp6mmpstXjDNrR1nolN3hnHAz72ylaGpc4KjlWRvs1gbnk # PUZajuT8dTdYWUmLTts8FZ1zUkvreww6wi3Bs5tSLeA1xbnBV7PoPaE8RPIjFh4K # qlk3J9CVUl6ofz9U8IHh3Jq9ZdV49vdMObvd4NY3DpGah4xz53FkUvc+A9jGzXK4 # NDSYW4zT9Qim63jGUaANDm/0azxAGmAWLKkGUp0cE5DObwIe6nucs/b4l2DyZdHR # H4c6wXXwQo167Yxysnv7LIq0kUdU4i5pzBZUGlkwggTsMIID1KADAgECAhMzAAAA # ymzVMhI1xOFVAAEAAADKMA0GCSqGSIb3DQEBBQUAMHkxCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xIzAhBgNVBAMTGk1pY3Jvc29mdCBDb2RlIFNp # Z25pbmcgUENBMB4XDTE0MDQyMjE3MzkwMFoXDTE1MDcyMjE3MzkwMFowgYMxCzAJ # BgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25k # MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xDTALBgNVBAsTBE1PUFIx # HjAcBgNVBAMTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjCCASIwDQYJKoZIhvcNAQEB # BQADggEPADCCAQoCggEBAJZxXe0GRvqEy51bt0bHsOG0ETkDrbEVc2Cc66e2bho8 # P/9l4zTxpqUhXlaZbFjkkqEKXMLT3FIvDGWaIGFAUzGcbI8hfbr5/hNQUmCVOlu5 # WKV0YUGplOCtJk5MoZdwSSdefGfKTx5xhEa8HUu24g/FxifJB+Z6CqUXABlMcEU4 # LYG0UKrFZ9H6ebzFzKFym/QlNJj4VN8SOTgSL6RrpZp+x2LR3M/tPTT4ud81MLrs # eTKp4amsVU1Mf0xWwxMLdvEH+cxHrPuI1VKlHij6PS3Pz4SYhnFlEc+FyQlEhuFv # 57H8rEBEpamLIz+CSZ3VlllQE1kYc/9DDK0r1H8wQGcCAwEAAaOCAWAwggFcMBMG # A1UdJQQMMAoGCCsGAQUFBwMDMB0GA1UdDgQWBBQfXuJdUI1Whr5KPM8E6KeHtcu/ # gzBRBgNVHREESjBIpEYwRDENMAsGA1UECxMETU9QUjEzMDEGA1UEBRMqMzE1OTUr # YjQyMThmMTMtNmZjYS00OTBmLTljNDctM2ZjNTU3ZGZjNDQwMB8GA1UdIwQYMBaA # FMsR6MrStBZYAck3LjMWFrlMmgofMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9j # cmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY0NvZFNpZ1BDQV8w # OC0zMS0yMDEwLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6 # Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljQ29kU2lnUENBXzA4LTMx # LTIwMTAuY3J0MA0GCSqGSIb3DQEBBQUAA4IBAQB3XOvXkT3NvXuD2YWpsEOdc3wX # yQ/tNtvHtSwbXvtUBTqDcUCBCaK3cSZe1n22bDvJql9dAxgqHSd+B+nFZR+1zw23 # VMcoOFqI53vBGbZWMrrizMuT269uD11E9dSw7xvVTsGvDu8gm/Lh/idd6MX/YfYZ # 0igKIp3fzXCCnhhy2CPMeixD7v/qwODmHaqelzMAUm8HuNOIbN6kBjWnwlOGZRF3 # CY81WbnYhqgA/vgxfSz0jAWdwMHVd3Js6U1ZJoPxwrKIV5M1AHxQK7xZ/P4cKTiC # 095Sl0UpGE6WW526Xxuj8SdQ6geV6G00DThX3DcoNZU6OJzU7WqFXQ4iEV57MIIF # vDCCA6SgAwIBAgIKYTMmGgAAAAAAMTANBgkqhkiG9w0BAQUFADBfMRMwEQYKCZIm # iZPyLGQBGRYDY29tMRkwFwYKCZImiZPyLGQBGRYJbWljcm9zb2Z0MS0wKwYDVQQD # EyRNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwHhcNMTAwODMx # MjIxOTMyWhcNMjAwODMxMjIyOTMyWjB5MQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMSMwIQYDVQQDExpNaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBD # QTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJyWVwZMGS/HZpgICBC # mXZTbD4b1m/My/Hqa/6XFhDg3zp0gxq3L6Ay7P/ewkJOI9VyANs1VwqJyq4gSfTw # aKxNS42lvXlLcZtHB9r9Jd+ddYjPqnNEf9eB2/O98jakyVxF3K+tPeAoaJcap6Vy # c1bxF5Tk/TWUcqDWdl8ed0WDhTgW0HNbBbpnUo2lsmkv2hkL/pJ0KeJ2L1TdFDBZ # +NKNYv3LyV9GMVC5JxPkQDDPcikQKCLHN049oDI9kM2hOAaFXE5WgigqBTK3S9dP # Y+fSLWLxRT3nrAgA9kahntFbjCZT6HqqSvJGzzc8OJ60d1ylF56NyxGPVjzBrAlf # A9MCAwEAAaOCAV4wggFaMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMsR6MrS # tBZYAck3LjMWFrlMmgofMAsGA1UdDwQEAwIBhjASBgkrBgEEAYI3FQEEBQIDAQAB # MCMGCSsGAQQBgjcVAgQWBBT90TFO0yaKleGYYDuoMW+mPLzYLTAZBgkrBgEEAYI3 # FAIEDB4KAFMAdQBiAEMAQTAfBgNVHSMEGDAWgBQOrIJgQFYnl+UlE/wq4QpTlVnk # pDBQBgNVHR8ESTBHMEWgQ6BBhj9odHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtp # L2NybC9wcm9kdWN0cy9taWNyb3NvZnRyb290Y2VydC5jcmwwVAYIKwYBBQUHAQEE # SDBGMEQGCCsGAQUFBzAChjhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpL2Nl # cnRzL01pY3Jvc29mdFJvb3RDZXJ0LmNydDANBgkqhkiG9w0BAQUFAAOCAgEAWTk+ # fyZGr+tvQLEytWrrDi9uqEn361917Uw7LddDrQv+y+ktMaMjzHxQmIAhXaw9L0y6 # oqhWnONwu7i0+Hm1SXL3PupBf8rhDBdpy6WcIC36C1DEVs0t40rSvHDnqA2iA6VW # 4LiKS1fylUKc8fPv7uOGHzQ8uFaa8FMjhSqkghyT4pQHHfLiTviMocroE6WRTsgb # 0o9ylSpxbZsa+BzwU9ZnzCL/XB3Nooy9J7J5Y1ZEolHN+emjWFbdmwJFRC9f9Nqu # 1IIybvyklRPk62nnqaIsvsgrEA5ljpnb9aL6EiYJZTiU8XofSrvR4Vbo0HiWGFzJ # NRZf3ZMdSY4tvq00RBzuEBUaAF3dNVshzpjHCe6FDoxPbQ4TTj18KUicctHzbMrB # 7HCjV5JXfZSNoBtIA1r3z6NnCnSlNu0tLxfI5nI3EvRvsTxngvlSso0zFmUeDord # EN5k9G/ORtTTF+l5xAS00/ss3x+KnqwK+xMnQK3k+eGpf0a7B2BHZWBATrBC7E7t # s3Z52Ao0CW0cgDEf4g5U3eWh++VHEK1kmP9QFi58vwUheuKVQSdpw5OPlcmN2Jsh # rg1cnPCiroZogwxqLbt2awAdlq3yFnv2FoMkuYjPaqhHMS+a3ONxPdcAfmJH0c6I # ybgY+g5yjcGjPa8CQGr/aZuW4hCoELQ3UAjWwz0wggYHMIID76ADAgECAgphFmg0 # AAAAAAAcMA0GCSqGSIb3DQEBBQUAMF8xEzARBgoJkiaJk/IsZAEZFgNjb20xGTAX # BgoJkiaJk/IsZAEZFgltaWNyb3NvZnQxLTArBgNVBAMTJE1pY3Jvc29mdCBSb290 # IENlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0wNzA0MDMxMjUzMDlaFw0yMTA0MDMx # MzAzMDlaMHcxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYD # VQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xITAf # BgNVBAMTGE1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQTCCASIwDQYJKoZIhvcNAQEB # BQADggEPADCCAQoCggEBAJ+hbLHf20iSKnxrLhnhveLjxZlRI1Ctzt0YTiQP7tGn # 0UytdDAgEesH1VSVFUmUG0KSrphcMCbaAGvoe73siQcP9w4EmPCJzB/LMySHnfL0 # Zxws/HvniB3q506jocEjU8qN+kXPCdBer9CwQgSi+aZsk2fXKNxGU7CG0OUoRi4n # rIZPVVIM5AMs+2qQkDBuh/NZMJ36ftaXs+ghl3740hPzCLdTbVK0RZCfSABKR2YR # JylmqJfk0waBSqL5hKcRRxQJgp+E7VV4/gGaHVAIhQAQMEbtt94jRrvELVSfrx54 # QTF3zJvfO4OToWECtR0Nsfz3m7IBziJLVP/5BcPCIAsCAwEAAaOCAaswggGnMA8G # A1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCM0+NlSRnAK7UD7dvuzK7DDNbMPMAsG # A1UdDwQEAwIBhjAQBgkrBgEEAYI3FQEEAwIBADCBmAYDVR0jBIGQMIGNgBQOrIJg # QFYnl+UlE/wq4QpTlVnkpKFjpGEwXzETMBEGCgmSJomT8ixkARkWA2NvbTEZMBcG # CgmSJomT8ixkARkWCW1pY3Jvc29mdDEtMCsGA1UEAxMkTWljcm9zb2Z0IFJvb3Qg # Q2VydGlmaWNhdGUgQXV0aG9yaXR5ghB5rRahSqClrUxzWPQHEy5lMFAGA1UdHwRJ # MEcwRaBDoEGGP2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1 # Y3RzL21pY3Jvc29mdHJvb3RjZXJ0LmNybDBUBggrBgEFBQcBAQRIMEYwRAYIKwYB # BQUHMAKGOGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljcm9z # b2Z0Um9vdENlcnQuY3J0MBMGA1UdJQQMMAoGCCsGAQUFBwMIMA0GCSqGSIb3DQEB # BQUAA4ICAQAQl4rDXANENt3ptK132855UU0BsS50cVttDBOrzr57j7gu1BKijG1i # uFcCy04gE1CZ3XpA4le7r1iaHOEdAYasu3jyi9DsOwHu4r6PCgXIjUji8FMV3U+r # kuTnjWrVgMHmlPIGL4UD6ZEqJCJw+/b85HiZLg33B+JwvBhOnY5rCnKVuKE5nGct # xVEO6mJcPxaYiyA/4gcaMvnMMUp2MT0rcgvI6nA9/4UKE9/CCmGO8Ne4F+tOi3/F # NSteo7/rvH0LQnvUU3Ih7jDKu3hlXFsBFwoUDtLaFJj1PLlmWLMtL+f5hYbMUVbo # nXCUbKw5TNT2eb+qGHpiKe+imyk0BncaYsk9Hm0fgvALxyy7z0Oz5fnsfbXjpKh0 # NbhOxXEjEiZ2CzxSjHFaRkMUvLOzsE1nyJ9C/4B5IYCeFTBm6EISXhrIniIh0EPp # K+m79EjMLNTYMoBMJipIJF9a6lbvpt6Znco6b72BJ3QGEe52Ib+bgsEnVLaxaj2J # oXZhtG6hE6a/qkfwEm/9ijJssv7fUciMI8lmvZ0dhxJkAj0tr1mPuOQh5bWwymO0 # eFQF1EEuUKyUsKV4q7OglnUa2ZKHE3UiLzKoCG6gW4wlv6DvhMoh1useT8ma7kng # 9wFlb4kLfchpyOZu6qeXzjEp/w7FW1zYTRuh2Povnj8uVRZryROj/TGCBL0wggS5 # AgEBMIGQMHkxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYD # VQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xIzAh # BgNVBAMTGk1pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBAhMzAAAAymzVMhI1xOFV # AAEAAADKMAkGBSsOAwIaBQCggdYwGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw # HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwIwYJKoZIhvcNAQkEMRYEFL+u # 5jpC/moixqqM5TQf2hreaemdMHYGCisGAQQBgjcCAQwxaDBmoD6APABUAHIAbwB1 # AGIAbABlAHMAaABvAG8AdAAtAEQAYQB0AGEAYgBhAHMAZQBTAHAAYQBjAGUALgBw # AHMAMaEkgCJodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vZXhjaGFuZ2UgMA0GCSqG # SIb3DQEBAQUABIIBABPnamYXGyskXq4CdL6N89dDZ8ei6xMWVgZLhjE9K91lDcao # eqRxagcPVWgqrB2zQWZS+OAaydBljHt3tHoKayskT1vyBH3QqctRpPoX+kkBXK51 # BihpxZGHzlw5fz6m45507rSBwrU3i0tw8g1YqZWTL73iCK00CCJqIxaZ8cshQ50+ # spEWjMhNF12nks99mwUUmn+1MaCK2ELd29tk/BKg0UUKjTGbeiWqTHTvvC4OXBBw # JASDkRfOz1BhP9YpUlicnOlGSkjdyRNx2rEjvOyf1atm6Gj7k73f0wvX35tmdhhp # 6OFwYNqdfKBX1CRhUtntalNMrqCZK9jwj+caR+qhggIoMIICJAYJKoZIhvcNAQkG # MYICFTCCAhECAQEwgY4wdzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0 # b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3Jh # dGlvbjEhMB8GA1UEAxMYTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBAhMzAAAATKHo # Tcy0dHs7AAAAAABMMAkGBSsOAwIaBQCgXTAYBgkqhkiG9w0BCQMxCwYJKoZIhvcN # AQcBMBwGCSqGSIb3DQEJBTEPFw0xNDA4MTcwNTEyMjVaMCMGCSqGSIb3DQEJBDEW # BBQejihLMTkCc7YO7Z4hWtNyt8VkIzANBgkqhkiG9w0BAQUFAASCAQBCakwgQIdN # SSRjRJyG6m3wHHCNPrNcUYNN7D0Mp3Vg87vkyySPDlKcJJfoWk0lGo980WqlAW9N # lQGRRpNVGPYJa+iSw3YfLtfpTygPdnBmOEt3Yciigmk14awviZS6F8ku/c8T79u4 # /dOjwrgkD4lxwSyifwI+uJxLrt+8Rr0O7Dkld5fokIIS16FNm5W9Ugspre71Y8D6 # L5joIVMYsAjVsGNrDgCKzMRfi60PIZrwp3Q1T26qMET2DqNFR3gZ1BTHR6oVm8pl # +x2ir72oztpgvwxclyUbAsPXRloZ6F8CLH8VvcisNfP42dQ/ygNE9tBhKIg37DYB # IhDGKAl4Yz62 # SIG # End signature block