<# .SYNOPSIS Requires Powershell 7! Downloads all files from a web directory recursively, preserving the folder structure. .DESCRIPTION Requires Powershell 7! Downloads all files from a specified web directory URL, including subdirectories, and saves them to a local directory while maintaining the original folder structure. It uses parallel processing to speed up downloads. .PARAMETER Url The root URL of the web directory to scan and download files from. Default is "http://rax.mirror.rackspace.com/dcm-migration-scripts". Example: -Url specifies the website to download. .PARAMETER Destination The local directory where downloaded files will be saved. Default is "C:\rs-pkgs\dcm-migration-scripts". Example: -Destination specifies the local directory to save files. .EXAMPLE PS C:\> .\Get-Site.ps1 Downloads the default URL to the default directory. .EXAMPLE PS C:\> .\Get-Site.ps1 -Destination "C:\Logs" Downloads the default URL to the specified directory. .EXAMPLE PS C:\> .\Get-Site.ps1 -Url "http://rax.mirror.rackspace.com/dcm-migration-scripts/Logs/" -Destination "C:\Logs" Downloads the specified URL to the specified directory. .NOTES Author: Steve Pye Date: 2025-09-30 Version: 1.0 #> param( [string]$Url = "http://rax.mirror.rackspace.com/dcm-migration-scripts/", # e.g. http://example.com/files/ [string]$Destination = "C:\rs-pkgs\dcm-migration-scripts/", # e.g. C:\Downloads [int]$TimeoutSeconds = 300, [int]$MaxThreads = 6 ) if (-not (Test-Path -LiteralPath $Destination)) { New-Item -ItemType Directory -Path $Destination | Out-Null } $handler = [System.Net.Http.HttpClientHandler]::new() $handler.AutomaticDecompression = [System.Net.DecompressionMethods]::GZip -bor [System.Net.DecompressionMethods]::Deflate $HttpClient = [System.Net.Http.HttpClient]::new($handler) $HttpClient.Timeout = [TimeSpan]::FromSeconds($TimeoutSeconds) $shared = @{ queue = [System.Collections.Concurrent.BlockingCollection[object]]::new() completed = [System.Collections.Concurrent.ConcurrentBag[string]]::new() } $refShared = [ref]$shared $global:TotalFiles = 0 $script:ScannedCount = 0 # ✅ track scan progress $visited = [System.Collections.Generic.HashSet[string]]::new() Write-Host "🔍 Starting scan of: $Url" $downloadJobs = @() for ($i = 1; $i -le $MaxThreads; $i++) { $job = Start-ThreadJob -ScriptBlock { param($sharedRef, $client) $queue = $sharedRef.Value.queue $completed = $sharedRef.Value.completed while (-not $queue.IsCompleted) { $item = $null if ($queue.TryTake([ref]$item, 500)) { try { $response = $client.GetAsync($item.Url).Result $bytes = $response.Content.ReadAsByteArrayAsync().Result [System.IO.File]::WriteAllBytes($item.Path, $bytes) $completed.Add($item.Url) | Out-Null } catch { $completed.Add("Failed: $($item.Url)") | Out-Null } } } $null } -ArgumentList $refShared, $HttpClient $null = ($downloadJobs += $job) } function Stream-FilesRecursively { param($BaseUrl, $BasePath, $Client, $Queue, $Visited) try { $html = $Client.GetStringAsync($BaseUrl).Result } catch { return } $matches = [regex]::Matches($html, ']+href=["'']([^"''>]+)["''][^>]*>([^<]*)') if ($matches.Count -eq 0) { return } foreach ($m in $matches) { $href = $m.Groups[1].Value $linkText = $m.Groups[2].Value if ($href -eq '../' -or $linkText -match 'Parent Directory') { continue } $decodedName = [System.Web.HttpUtility]::UrlDecode($href.TrimEnd('/')) $localPath = Join-Path $BasePath $decodedName if ($BaseUrl.EndsWith('/')) { $targetUrl = $BaseUrl + $href } else { $targetUrl = $BaseUrl + '/' + $href } $targetUrl = [System.Uri]::new($targetUrl).AbsoluteUri $isFolder = ($href.EndsWith('/') -or $linkText.EndsWith('/')) if ($isFolder) { if (-not $targetUrl.EndsWith('/')) { $targetUrl += '/' } if ($Visited.Contains($targetUrl)) { continue } [void]$Visited.Add($targetUrl) if (-not (Test-Path -LiteralPath $localPath)) { New-Item -ItemType Directory -Path $localPath | Out-Null } $script:ScannedCount++ Write-Progress -Activity "Scanning directories" -Status "$script:ScannedCount links scanned" -PercentComplete -1 Stream-FilesRecursively -BaseUrl $targetUrl -BasePath $localPath -Client $Client -Queue $Queue -Visited $Visited } else { $Queue.Add([PSCustomObject]@{ Url = $targetUrl; Path = $localPath }) | Out-Null $global:TotalFiles++ $script:ScannedCount++ Write-Progress -Activity "Scanning directories" -Status "$script:ScannedCount links scanned" -PercentComplete -1 } } } Stream-FilesRecursively -BaseUrl $Url -BasePath $Destination -Client $HttpClient -Queue $refShared.Value.queue -Visited $visited $refShared.Value.queue.CompleteAdding() Write-Progress -Activity "Scanning directories" -Completed Write-Host "📁 Finished scanning. Total files found: $global:TotalFiles" do { $count = $refShared.Value.completed.Count $total = $global:TotalFiles if ($total -gt 0) { $percent = [math]::Round(($count / $total) * 100) Write-Progress -Activity "Downloading files" -Status "$count of $total complete" -PercentComplete $percent } Start-Sleep -Milliseconds 500 } while ($count -lt $total) Write-Progress -Activity "Downloading files" -Completed Write-Host "✅ All downloads complete: $count files processed." $downloadJobs | ForEach-Object { Wait-Job $_ | Out-Null Receive-Job $_ | Out-Null Remove-Job $_ }