<# .SYNOPSIS Downloads all files from a web directory recursively, preserving the folder structure. .DESCRIPTION Downloads all files from a specified web directory URL, including subdirectories, and saves them to a local directory while maintaining the original folder structure. It uses parallel processing to speed up downloads. .PARAMETER Url The root URL of the web directory to scan and download files from. Default is "http://rax.mirror.rackspace.com/dcm-migration-scripts". Example: -Url specifies the website to download. .PARAMETER Destination The local directory where downloaded files will be saved. Default is "C:\rs-pkgs\dcm-migration-scripts". Example: -Destination specifies the local directory to save files. .EXAMPLE PS C:\> .\Get-Site.ps1 Downloads the default URL to the default directory. .EXAMPLE PS C:\> .\Get-Site.ps1 -Destination "C:\Logs" Downloads the default URL to the specified directory. .EXAMPLE PS C:\> .\Get-Site.ps1 -Url "http://rax.mirror.rackspace.com/dcm-migration-scripts/Logs/" -Destination "C:\Logs" Downloads the specified URL to the specified directory. .NOTES Author: Steve Pye Date: 2025-09-30 Version: 1.0 #> param( [string]$Url = "http://rax.mirror.rackspace.com/dcm-migration-scripts/", # e.g. http://example.com/files/ [string]$Destination = "C:\rs-pkgs\dcm-migration-scripts\", # e.g. C:\Downloads [int]$MaxThreads = 6 ) Add-Type -AssemblyName System.Web Add-Type -TypeDefinition @" using System; using System.Text.RegularExpressions; public class HtmlLinkExtractor { public static MatchCollection ExtractLinks(string html) { string pattern = @"]+href=['""]([^'"">]+)['""][^>]*>([^<]*)"; return Regex.Matches(html, pattern); } } "@ if (-not (Test-Path -LiteralPath $Destination)) { New-Item -ItemType Directory -Path $Destination | Out-Null } $Queue = New-Object System.Collections.ArrayList $global:Completed = New-Object System.Collections.ArrayList $global:TotalFiles = 0 $global:ScannedCount = 0 $global:Visited = New-Object System.Collections.Generic.HashSet[string] function Stream-FilesRecursively { param ( [string]$BaseUrl, [string]$BasePath ) try { $html = (Invoke-WebRequest -Uri $BaseUrl -UseBasicParsing).Content } catch { return } $linkMatches = [HtmlLinkExtractor]::ExtractLinks($html) if ($linkMatches.Count -eq 0) { return } foreach ($m in $linkMatches) { $href = $m.Groups[1].Value $linkText = $m.Groups[2].Value if ($href -eq '../' -or $linkText -match 'Parent Directory') { continue } $decodedName = [System.Web.HttpUtility]::UrlDecode($href.TrimEnd('/')) $localPath = Join-Path $BasePath $decodedName if ($BaseUrl.EndsWith('/')) { $targetUrl = $BaseUrl + $href } else { $targetUrl = $BaseUrl + '/' + $href } $targetUrl = (New-Object System.Uri $targetUrl).AbsoluteUri $isFolder = ($href.EndsWith('/') -or $linkText.EndsWith('/')) if ($isFolder) { if (-not $targetUrl.EndsWith('/')) { $targetUrl += '/' } if ($global:Visited.Contains($targetUrl)) { continue } [void]$global:Visited.Add($targetUrl) if (-not (Test-Path -LiteralPath $localPath)) { New-Item -ItemType Directory -Path $localPath | Out-Null } $global:ScannedCount++ Write-Progress -Activity "Scanning directories" -Status ("$global:ScannedCount links scanned") -PercentComplete -1 Stream-FilesRecursively -BaseUrl $targetUrl -BasePath $localPath } else { $null = $Queue.Add(@{ Url = $targetUrl; Path = $localPath }) $global:TotalFiles++ $global:ScannedCount++ Write-Progress -Activity "Scanning directories" -Status ("$global:ScannedCount links scanned") -PercentComplete -1 } } } Stream-FilesRecursively -BaseUrl $Url -BasePath $Destination Write-Progress -Activity "Scanning directories" -Completed $sync = [object]::new() $jobs = @() for ($i = 1; $i -le $MaxThreads; $i++) { $job = Start-Job -ScriptBlock { param($syncObj, $sharedQueue) while ($true) { $item = $null [System.Threading.Monitor]::Enter($syncObj) try { if ($sharedQueue.Count -gt 0) { $item = $sharedQueue[0] $sharedQueue.RemoveAt(0) } } finally { [System.Threading.Monitor]::Exit($syncObj) } if ($null -eq $item) { Start-Sleep -Milliseconds 500 continue } try { $wc = New-Object System.Net.WebClient $wc.DownloadFile($item.Url, $item.Path) Write-Output $item.Url } catch { # Optional: handle failed downloads } } } -ArgumentList $sync, $Queue $null = $jobs += $job } do { foreach ($job in $jobs) { $results = Receive-Job $job foreach ($r in $results) { if (-not $global:Completed.Contains($r)) { $null = $global:Completed.Add($r) } } } $count = $global:Completed.Count $total = $global:TotalFiles if ($total -gt 0) { $percent = [math]::Min([math]::Round(($count / $total) * 100), 100) Write-Progress -Activity "Downloading files" -Status ("$count of $total complete") -PercentComplete $percent } Start-Sleep -Milliseconds 500 } while ($global:Completed.Count -lt $global:TotalFiles) Start-Sleep -Seconds 1 Write-Progress -Activity "Downloading files" -Completed $jobs | ForEach-Object { Stop-Job $_ -Confirm:$false Wait-Job $_ | Out-Null Remove-Job $_ }