KHDB rework
This commit is contained in:
685
Update-KHDB.ps1
685
Update-KHDB.ps1
@@ -7,32 +7,29 @@
|
||||
##################################################
|
||||
## Project: Elysium ##
|
||||
## File: Update-KHDB.ps1 ##
|
||||
## Version: 1.1.0 ##
|
||||
## Version: 2.0.0 ##
|
||||
## Support: support@cqre.net ##
|
||||
##################################################
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Known hashes database update script for the Elysium AD password testing tool.
|
||||
Known-hashes database updater for the Elysium AD password testing tool.
|
||||
|
||||
.DESCRIPTION
|
||||
This script downloads khdb.txt.zip from the designated storage (Azure Blob or S3-compatible), validates and decompresses it, and atomically updates the current version with backup and logging.
|
||||
Downloads a sharded KHDB manifest, performs incremental shard updates, validates
|
||||
checksums, and atomically refreshes the merged khdb.txt for downstream scripts.
|
||||
Supports Azure Blob Storage (via SAS) and S3-compatible endpoints (SigV4).
|
||||
#>
|
||||
|
||||
# safer defaults
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
# ensure TLS 1.2
|
||||
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor [System.Net.SecurityProtocolType]::Tls12
|
||||
|
||||
# Resolve paths
|
||||
$scriptRoot = $PSScriptRoot
|
||||
|
||||
function Start-UpdateTranscript {
|
||||
param(
|
||||
[string]$BasePath
|
||||
)
|
||||
param([string]$BasePath)
|
||||
try {
|
||||
$logsDir = Join-Path -Path $BasePath -ChildPath 'Reports/logs'
|
||||
if (-not (Test-Path $logsDir)) { New-Item -Path $logsDir -ItemType Directory -Force | Out-Null }
|
||||
@@ -55,7 +52,9 @@ function Read-ElysiumSettings {
|
||||
Get-Content $settingsPath | ForEach-Object {
|
||||
if ($_ -and -not $_.Trim().StartsWith('#')) {
|
||||
$kv = $_ -split '=', 2
|
||||
if ($kv.Count -eq 2) { $settings[$kv[0].Trim()] = $kv[1].Trim().Trim("'") }
|
||||
if ($kv.Count -eq 2) {
|
||||
$settings[$kv[0].Trim()] = $kv[1].Trim().Trim("'")
|
||||
}
|
||||
}
|
||||
}
|
||||
return $settings
|
||||
@@ -69,22 +68,32 @@ function Get-InstallationPath([hashtable]$settings) {
|
||||
}
|
||||
|
||||
function New-HttpClient {
|
||||
Add-Type -AssemblyName System.Net.Http
|
||||
Add-Type -AssemblyName System.Net.Http -ErrorAction SilentlyContinue
|
||||
$client = [System.Net.Http.HttpClient]::new()
|
||||
$client.Timeout = [TimeSpan]::FromSeconds(600)
|
||||
$client.DefaultRequestHeaders.UserAgent.ParseAdd('Elysium/1.1 (+Update-KHDB)')
|
||||
$client.DefaultRequestHeaders.UserAgent.ParseAdd('Elysium/2.0 (+Update-KHDB)')
|
||||
return $client
|
||||
}
|
||||
|
||||
function Build-BlobUri([string]$account, [string]$container, [string]$sas) {
|
||||
if ([string]::IsNullOrWhiteSpace($account)) { throw 'storageAccountName is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($container)) { throw 'containerName is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($sas)) { throw 'sasToken is missing or empty.' }
|
||||
$sas = $sas.Trim()
|
||||
function Build-BlobUri {
|
||||
param(
|
||||
[string]$Account,
|
||||
[string]$Container,
|
||||
[string]$Sas,
|
||||
[string]$BlobName
|
||||
)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Account)) { throw 'storageAccountName is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($Container)) { throw 'containerName is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($Sas)) { throw 'sasToken is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($BlobName)) { throw 'BlobName cannot be empty.' }
|
||||
|
||||
$sas = $Sas.Trim()
|
||||
if (-not $sas.StartsWith('?')) { $sas = '?' + $sas }
|
||||
$ub = [System.UriBuilder]::new("https://$account.blob.core.windows.net/$container/khdb.txt.zip")
|
||||
$ub.Query = $sas.TrimStart('?')
|
||||
return $ub.Uri.AbsoluteUri
|
||||
$normalizedBlob = $BlobName.Replace('\', '/').TrimStart('/')
|
||||
$uriBuilder = [System.UriBuilder]::new("https://$Account.blob.core.windows.net/$Container/$normalizedBlob")
|
||||
$uriBuilder.Query = $sas.TrimStart('?')
|
||||
return $uriBuilder.Uri.AbsoluteUri
|
||||
}
|
||||
|
||||
function Ensure-AWSS3Module {
|
||||
@@ -102,6 +111,7 @@ function New-S3Client {
|
||||
[string]$SecretAccessKey,
|
||||
[bool]$ForcePathStyle = $true
|
||||
)
|
||||
|
||||
Ensure-AWSS3Module
|
||||
$creds = New-Object Amazon.Runtime.BasicAWSCredentials($AccessKeyId, $SecretAccessKey)
|
||||
$cfg = New-Object Amazon.S3.AmazonS3Config
|
||||
@@ -111,10 +121,8 @@ function New-S3Client {
|
||||
return (New-Object Amazon.S3.AmazonS3Client($creds, $cfg))
|
||||
}
|
||||
|
||||
# Native S3 SigV4 (no AWS Tools) helpers
|
||||
function Get-Bytes([string]$s) { return [System.Text.Encoding]::UTF8.GetBytes($s) }
|
||||
function Get-HashHex([byte[]]$bytes) {
|
||||
# Use stream overload to avoid ambiguous resolution and property access
|
||||
if ($null -eq $bytes) { $bytes = [byte[]]@() }
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
@@ -126,7 +134,6 @@ function Get-HashHex([byte[]]$bytes) {
|
||||
} finally { $sha.Dispose() }
|
||||
}
|
||||
function HmacSha256([byte[]]$key, [string]$data) {
|
||||
# Use stream overload to avoid ambiguous resolution and property access
|
||||
$h = [System.Security.Cryptography.HMACSHA256]::new($key)
|
||||
try {
|
||||
$b = [System.Text.Encoding]::UTF8.GetBytes($data)
|
||||
@@ -142,86 +149,156 @@ function GetSignatureKey([string]$secret, [string]$dateStamp, [string]$regionNam
|
||||
$kService = HmacSha256 $kRegion $serviceName
|
||||
HmacSha256 $kService 'aws4_request'
|
||||
}
|
||||
function UriEncode([string]$data, [bool]$encodeSlash) { $enc=[System.Uri]::EscapeDataString($data); if (-not $encodeSlash) { $enc = $enc -replace '%2F','/' }; $enc }
|
||||
function BuildCanonicalPath([System.Uri]$uri) { $segments=$uri.AbsolutePath.Split('/'); $encoded=@(); foreach($s in $segments){ $encoded += (UriEncode $s $false) }; $p=($encoded -join '/'); if (-not $p.StartsWith('/')){ $p='/' + $p }; $p }
|
||||
function UriEncode([string]$data, [bool]$encodeSlash) {
|
||||
$enc = [System.Uri]::EscapeDataString($data)
|
||||
if (-not $encodeSlash) { $enc = $enc -replace '%2F', '/' }
|
||||
return $enc
|
||||
}
|
||||
function BuildCanonicalPath([System.Uri]$uri) {
|
||||
$segments = $uri.AbsolutePath.Split('/')
|
||||
$encoded = @()
|
||||
foreach ($s in $segments) { $encoded += (UriEncode $s $false) }
|
||||
$path = ($encoded -join '/')
|
||||
if (-not $path.StartsWith('/')) { $path = '/' + $path }
|
||||
return $path
|
||||
}
|
||||
function ToHex([byte[]]$b) { ([BitConverter]::ToString($b)).Replace('-', '').ToLowerInvariant() }
|
||||
function BuildAuthHeaders($method, [System.Uri]$uri, [string]$region, [string]$accessKey, [string]$secretKey, [string]$payloadHash) {
|
||||
$algorithm = 'AWS4-HMAC-SHA256'
|
||||
$amzdate = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssZ')
|
||||
$datestamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd')
|
||||
$hostHeader = $uri.Host; if (-not $uri.IsDefaultPort) { $hostHeader = "${hostHeader}:$($uri.Port)" }
|
||||
$timestamp = (Get-Date).ToUniversalTime()
|
||||
$amzDate = $timestamp.ToString('yyyyMMddTHHmmssZ')
|
||||
$dateStamp = $timestamp.ToString('yyyyMMdd')
|
||||
$hostHeader = $uri.Host
|
||||
if (-not $uri.IsDefaultPort) { $hostHeader = "${hostHeader}:$($uri.Port)" }
|
||||
$canonicalUri = BuildCanonicalPath $uri
|
||||
$canonicalQueryString = ''
|
||||
$canonicalHeaders = "host:$hostHeader`n" + "x-amz-content-sha256:$payloadHash`n" + "x-amz-date:$amzdate`n"
|
||||
$canonicalHeaders = "host:$hostHeader`n" + "x-amz-content-sha256:$payloadHash`n" + "x-amz-date:$amzDate`n"
|
||||
$signedHeaders = 'host;x-amz-content-sha256;x-amz-date'
|
||||
$canonicalRequest = "$method`n$canonicalUri`n$canonicalQueryString`n$canonicalHeaders`n$signedHeaders`n$payloadHash"
|
||||
$credentialScope = "$datestamp/$region/s3/aws4_request"
|
||||
$stringToSign = "$algorithm`n$amzdate`n$credentialScope`n$((Get-HashHex (Get-Bytes $canonicalRequest)))"
|
||||
$signingKey = GetSignatureKey $secretKey $datestamp $region 's3'
|
||||
$credentialScope = "$dateStamp/$region/s3/aws4_request"
|
||||
$stringToSign = "$algorithm`n$amzDate`n$credentialScope`n$((Get-HashHex (Get-Bytes $canonicalRequest)))"
|
||||
$signingKey = GetSignatureKey $secretKey $dateStamp $region 's3'
|
||||
$signature = ToHex (HmacSha256 $signingKey $stringToSign)
|
||||
$authHeader = "$algorithm Credential=$accessKey/$credentialScope, SignedHeaders=$signedHeaders, Signature=$signature"
|
||||
@{ 'x-amz-date' = $amzdate; 'x-amz-content-sha256' = $payloadHash; 'Authorization' = $authHeader }
|
||||
@{
|
||||
'x-amz-date' = $amzDate
|
||||
'x-amz-content-sha256' = $payloadHash
|
||||
'Authorization' = $authHeader
|
||||
}
|
||||
}
|
||||
function BuildS3Uri([string]$endpointUrl, [string]$bucket, [string]$key, [bool]$forcePathStyle) {
|
||||
$base=[System.Uri]$endpointUrl; $ub=[System.UriBuilder]::new($base)
|
||||
if ($forcePathStyle) { $p=$ub.Path.TrimEnd('/'); if ([string]::IsNullOrEmpty($p)){ $p='/' }; $ub.Path = ($p.TrimEnd('/') + '/' + $bucket + '/' + $key) }
|
||||
else { $ub.Host = "$bucket." + $ub.Host; $p=$ub.Path.TrimEnd('/'); if ([string]::IsNullOrEmpty($p)){ $p='/' }; $ub.Path = ($p.TrimEnd('/') + '/' + $key) }
|
||||
$ub.Uri
|
||||
$base = [System.Uri]$endpointUrl
|
||||
$builder = [System.UriBuilder]::new($base)
|
||||
$normalizedKey = $key.Replace('\', '/').TrimStart('/')
|
||||
if ($forcePathStyle) {
|
||||
$path = $builder.Path.TrimEnd('/')
|
||||
if ([string]::IsNullOrEmpty($path)) { $path = '/' }
|
||||
$builder.Path = ($path.TrimEnd('/') + '/' + $bucket + '/' + $normalizedKey)
|
||||
} else {
|
||||
$builder.Host = "$bucket." + $builder.Host
|
||||
$path = $builder.Path.TrimEnd('/')
|
||||
if ([string]::IsNullOrEmpty($path)) { $path = '/' }
|
||||
$builder.Path = ($path.TrimEnd('/') + '/' + $normalizedKey)
|
||||
}
|
||||
return $builder.Uri
|
||||
}
|
||||
function Invoke-S3HttpDownloadWithRetry([string]$endpointUrl, [string]$bucket, [string]$key, [string]$targetPath, [string]$region, [string]$ak, [string]$sk, [bool]$forcePathStyle) {
|
||||
|
||||
function Invoke-S3HttpDownloadWithRetry {
|
||||
param(
|
||||
[string]$EndpointUrl,
|
||||
[string]$Bucket,
|
||||
[string]$Key,
|
||||
[string]$TargetPath,
|
||||
[string]$Region,
|
||||
[string]$AccessKeyId,
|
||||
[string]$SecretAccessKey,
|
||||
[bool]$ForcePathStyle,
|
||||
[string]$Activity
|
||||
)
|
||||
|
||||
Add-Type -AssemblyName System.Net.Http -ErrorAction SilentlyContinue
|
||||
[System.Net.Http.HttpClient]$client = [System.Net.Http.HttpClient]::new()
|
||||
$retries=5; $delay=2
|
||||
$retries = 5
|
||||
$delay = 2
|
||||
try {
|
||||
for($i=0;$i -lt $retries;$i++){
|
||||
for ($attempt = 0; $attempt -lt $retries; $attempt++) {
|
||||
$request = $null
|
||||
try {
|
||||
# Initialize here to satisfy StrictMode even if exceptions occur before assignment
|
||||
$req = $null
|
||||
$uri = BuildS3Uri -endpointUrl $endpointUrl -bucket $bucket -key $key -forcePathStyle $forcePathStyle
|
||||
$uri = BuildS3Uri -endpointUrl $EndpointUrl -bucket $Bucket -key $Key -forcePathStyle $ForcePathStyle
|
||||
$payloadHash = (Get-HashHex (Get-Bytes ''))
|
||||
[System.Net.Http.HttpRequestMessage]$req = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Get, $uri)
|
||||
$hdrs = BuildAuthHeaders -method 'GET' -uri $uri -region $region -accessKey $ak -secretKey $sk -payloadHash $payloadHash
|
||||
$req.Headers.TryAddWithoutValidation('x-amz-date', $hdrs['x-amz-date']) | Out-Null
|
||||
$req.Headers.TryAddWithoutValidation('Authorization', $hdrs['Authorization']) | Out-Null
|
||||
$req.Headers.TryAddWithoutValidation('x-amz-content-sha256', $hdrs['x-amz-content-sha256']) | Out-Null
|
||||
[System.Net.Http.HttpResponseMessage]$resp = $client.SendAsync($req, [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead).GetAwaiter().GetResult()
|
||||
$null = $resp.EnsureSuccessStatusCode()
|
||||
$totalBytes = $resp.Content.Headers.ContentLength
|
||||
$stream = $resp.Content.ReadAsStreamAsync().Result
|
||||
$fs = [System.IO.File]::Create($targetPath)
|
||||
$headers = BuildAuthHeaders -method 'GET' -uri $uri -region $Region -accessKey $AccessKeyId -secretKey $SecretAccessKey -payloadHash $payloadHash
|
||||
$request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Get, $uri)
|
||||
foreach ($kvp in $headers.GetEnumerator()) {
|
||||
$request.Headers.TryAddWithoutValidation($kvp.Key, $kvp.Value) | Out-Null
|
||||
}
|
||||
|
||||
$response = $client.SendAsync($request, [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead).GetAwaiter().GetResult()
|
||||
$null = $response.EnsureSuccessStatusCode()
|
||||
|
||||
$totalBytes = $response.Content.Headers.ContentLength
|
||||
$stream = $response.Content.ReadAsStreamAsync().Result
|
||||
$tmpPath = $TargetPath
|
||||
$fs = [System.IO.File]::Create($tmpPath)
|
||||
try {
|
||||
$buffer = New-Object byte[] 8192
|
||||
$totalRead = 0
|
||||
while (($read = $stream.Read($buffer, 0, $buffer.Length)) -gt 0) {
|
||||
$fs.Write($buffer, 0, $read)
|
||||
$totalRead += $read
|
||||
if ($totalBytes) { $pct = ($totalRead * 100.0) / $totalBytes; Write-Progress -Activity "Downloading khdb.txt.zip" -Status ("{0:N2}% Complete" -f $pct) -PercentComplete $pct } else { Write-Progress -Activity "Downloading khdb.txt.zip" -Status ("Downloaded {0:N0} bytes" -f $totalRead) -PercentComplete 0 }
|
||||
if ($totalBytes) {
|
||||
$pct = ($totalRead * 100.0) / $totalBytes
|
||||
Write-Progress -Activity $Activity -Status ("{0:N2}% Complete" -f $pct) -PercentComplete $pct
|
||||
} else {
|
||||
Write-Progress -Activity $Activity -Status ("Downloaded {0:N0} bytes" -f $totalRead) -PercentComplete 0
|
||||
}
|
||||
}
|
||||
} finally { $fs.Close(); $stream.Close() }
|
||||
if ($resp) { $resp.Dispose() }
|
||||
} finally {
|
||||
$fs.Close()
|
||||
$stream.Close()
|
||||
}
|
||||
|
||||
if ($response) { $response.Dispose() }
|
||||
Write-Progress -Activity $Activity -Completed -Status 'Completed'
|
||||
return
|
||||
} catch {
|
||||
if ($i -lt ($retries - 1)) { Write-Warning "Download failed (attempt $($i+1)/$retries): $($_.Exception.Message). Retrying in ${delay}s..."; Start-Sleep -Seconds $delay; $delay=[Math]::Min($delay*2,30) }
|
||||
else { throw }
|
||||
} finally { if ($req){ $req.Dispose() } }
|
||||
if ($attempt -lt ($retries - 1)) {
|
||||
Write-Warning "Download of '$Key' failed (attempt $($attempt + 1)/$retries): $($_.Exception.Message). Retrying in ${delay}s..."
|
||||
Start-Sleep -Seconds $delay
|
||||
$delay = [Math]::Min($delay * 2, 30)
|
||||
} else {
|
||||
throw
|
||||
}
|
||||
} finally {
|
||||
if ($request) { $request.Dispose() }
|
||||
}
|
||||
}
|
||||
} finally { $client.Dispose() }
|
||||
} finally {
|
||||
$client.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-DownloadWithRetry([System.Net.Http.HttpClient]$client, [string]$uri, [string]$targetPath) {
|
||||
function Invoke-DownloadWithRetry {
|
||||
param(
|
||||
[System.Net.Http.HttpClient]$Client,
|
||||
[string]$Uri,
|
||||
[string]$TargetPath,
|
||||
[string]$Activity
|
||||
)
|
||||
|
||||
$retries = 5
|
||||
$delay = 2
|
||||
for ($i = 0; $i -lt $retries; $i++) {
|
||||
for ($attempt = 0; $attempt -lt $retries; $attempt++) {
|
||||
try {
|
||||
$resp = $client.GetAsync($uri, [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead).Result
|
||||
if (-not $resp.IsSuccessStatusCode) {
|
||||
$code = [int]$resp.StatusCode
|
||||
$response = $Client.GetAsync($Uri, [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead).Result
|
||||
if (-not $response.IsSuccessStatusCode) {
|
||||
$code = [int]$response.StatusCode
|
||||
if (($code -ge 500 -and $code -lt 600) -or $code -eq 429 -or $code -eq 408) { throw "Transient HTTP error $code" }
|
||||
throw "HTTP error $code"
|
||||
}
|
||||
$totalBytes = $resp.Content.Headers.ContentLength
|
||||
$stream = $resp.Content.ReadAsStreamAsync().Result
|
||||
$fs = [System.IO.File]::Create($targetPath)
|
||||
|
||||
$totalBytes = $response.Content.Headers.ContentLength
|
||||
$stream = $response.Content.ReadAsStreamAsync().Result
|
||||
$fs = [System.IO.File]::Create($TargetPath)
|
||||
try {
|
||||
$buffer = New-Object byte[] 8192
|
||||
$totalRead = 0
|
||||
@@ -230,18 +307,21 @@ function Invoke-DownloadWithRetry([System.Net.Http.HttpClient]$client, [string]$
|
||||
$totalRead += $read
|
||||
if ($totalBytes) {
|
||||
$pct = ($totalRead * 100.0) / $totalBytes
|
||||
Write-Progress -Activity "Downloading khdb.txt.zip" -Status ("{0:N2}% Complete" -f $pct) -PercentComplete $pct
|
||||
Write-Progress -Activity $Activity -Status ("{0:N2}% Complete" -f $pct) -PercentComplete $pct
|
||||
} else {
|
||||
Write-Progress -Activity "Downloading khdb.txt.zip" -Status ("Downloaded {0:N0} bytes" -f $totalRead) -PercentComplete 0
|
||||
Write-Progress -Activity $Activity -Status ("Downloaded {0:N0} bytes" -f $totalRead) -PercentComplete 0
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$fs.Close(); $stream.Close()
|
||||
$fs.Close()
|
||||
$stream.Close()
|
||||
}
|
||||
|
||||
Write-Progress -Activity $Activity -Completed -Status 'Completed'
|
||||
return
|
||||
} catch {
|
||||
if ($i -lt ($retries - 1)) {
|
||||
Write-Warning "Download failed (attempt $($i+1)/$retries): $($_.Exception.Message). Retrying in ${delay}s..."
|
||||
if ($attempt -lt ($retries - 1)) {
|
||||
Write-Warning "Download of '$Uri' failed (attempt $($attempt + 1)/$retries): $($_.Exception.Message). Retrying in ${delay}s..."
|
||||
Start-Sleep -Seconds $delay
|
||||
$delay = [Math]::Min($delay * 2, 30)
|
||||
} else {
|
||||
@@ -251,18 +331,174 @@ function Invoke-DownloadWithRetry([System.Net.Http.HttpClient]$client, [string]$
|
||||
}
|
||||
}
|
||||
|
||||
function Validate-KHDBFile([string]$path) {
|
||||
if (-not (Test-Path $path)) { throw "Validation failed: $path not found" }
|
||||
$lines = Get-Content -Path $path -Encoding UTF8
|
||||
if (-not $lines -or $lines.Count -eq 0) { throw 'Validation failed: file is empty.' }
|
||||
$regex = '^[0-9A-Fa-f]{32}$'
|
||||
$invalid = $lines | Where-Object { $_ -notmatch $regex }
|
||||
if ($invalid.Count -gt 0) {
|
||||
throw ("Validation failed: {0} invalid lines detected." -f $invalid.Count)
|
||||
function Get-FileSha256Lower {
|
||||
param([string]$Path)
|
||||
if (-not (Test-Path -LiteralPath $Path)) { throw "File not found: $Path" }
|
||||
return (Get-FileHash -Path $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Ensure-Directory {
|
||||
param([string]$Path)
|
||||
if ([string]::IsNullOrWhiteSpace($Path)) { return }
|
||||
if (-not (Test-Path -LiteralPath $Path)) {
|
||||
New-Item -Path $Path -ItemType Directory -Force | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function Get-BooleanSetting {
|
||||
param(
|
||||
[string]$Value,
|
||||
[bool]$Default = $false
|
||||
)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Value)) { return $Default }
|
||||
$parsed = $Default
|
||||
if ([System.Boolean]::TryParse($Value, [ref]$parsed)) { return $parsed }
|
||||
return $Default
|
||||
}
|
||||
|
||||
function Get-RelativePath {
|
||||
param(
|
||||
[string]$BasePath,
|
||||
[string]$FullPath
|
||||
)
|
||||
|
||||
$base = (Resolve-Path -LiteralPath $BasePath).ProviderPath
|
||||
$full = (Resolve-Path -LiteralPath $FullPath).ProviderPath
|
||||
if (-not $base.EndsWith([System.IO.Path]::DirectorySeparatorChar)) {
|
||||
$base = $base + [System.IO.Path]::DirectorySeparatorChar
|
||||
}
|
||||
|
||||
$baseUri = New-Object System.Uri($base, [System.UriKind]::Absolute)
|
||||
$fullUri = New-Object System.Uri($full, [System.UriKind]::Absolute)
|
||||
$relativeUri = $baseUri.MakeRelativeUri($fullUri)
|
||||
$relativePath = [System.Uri]::UnescapeDataString($relativeUri.ToString())
|
||||
return $relativePath.Replace('/', [System.IO.Path]::DirectorySeparatorChar)
|
||||
}
|
||||
|
||||
function Combine-StoragePath {
|
||||
param(
|
||||
[string]$Prefix,
|
||||
[string]$Name
|
||||
)
|
||||
|
||||
$cleanName = $Name.Replace('\', '/').TrimStart('/')
|
||||
if ([string]::IsNullOrWhiteSpace($Prefix)) { return $cleanName }
|
||||
$normalizedPrefix = $Prefix.Replace('\', '/').Trim('/')
|
||||
if ([string]::IsNullOrEmpty($normalizedPrefix)) { return $cleanName }
|
||||
return "$normalizedPrefix/$cleanName"
|
||||
}
|
||||
|
||||
function Load-Manifest {
|
||||
param([string]$Path)
|
||||
$raw = Get-Content -LiteralPath $Path -Encoding UTF8 -Raw
|
||||
return $raw | ConvertFrom-Json -Depth 8
|
||||
}
|
||||
|
||||
function Validate-Manifest {
|
||||
param([psobject]$Manifest)
|
||||
|
||||
if (-not $Manifest) { throw 'Manifest is empty or invalid JSON.' }
|
||||
if (-not $Manifest.shards) { throw 'Manifest does not contain a shards collection.' }
|
||||
|
||||
$seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase)
|
||||
foreach ($entry in $Manifest.shards) {
|
||||
if (-not $entry) { throw 'Manifest contains null shard entries.' }
|
||||
$name = [string]$entry.name
|
||||
if ([string]::IsNullOrWhiteSpace($name)) { throw 'Manifest shard entry is missing name.' }
|
||||
$hash = [string]$entry.sha256
|
||||
if ([string]::IsNullOrWhiteSpace($hash) -or $hash.Length -ne 64) { throw "Manifest shard '$name' is missing a valid sha256." }
|
||||
$sizeValue = [string]$entry.size
|
||||
$sizeParsed = 0L
|
||||
if (-not [long]::TryParse($sizeValue, [ref]$sizeParsed) -or $sizeParsed -lt 0) {
|
||||
throw "Manifest shard '$name' has invalid size."
|
||||
}
|
||||
if (-not $seen.Add($name)) { throw "Manifest contains duplicate shard name '$name'." }
|
||||
}
|
||||
|
||||
if ($Manifest.shardSize -and [int]$Manifest.shardSize -ne 2) {
|
||||
throw "Manifest shardSize $($Manifest.shardSize) is not supported. Expected shardSize 2."
|
||||
}
|
||||
}
|
||||
|
||||
function Merge-ShardsToFile {
|
||||
param(
|
||||
[psobject]$Manifest,
|
||||
[string]$ShardsRoot,
|
||||
[string]$TargetPath
|
||||
)
|
||||
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
$writer = New-Object System.IO.StreamWriter($TargetPath, $false, $encoding)
|
||||
try {
|
||||
foreach ($entry in ($Manifest.shards | Sort-Object name)) {
|
||||
$relative = [string]$entry.name
|
||||
$shardPath = Join-Path -Path $ShardsRoot -ChildPath $relative
|
||||
if (-not (Test-Path -LiteralPath $shardPath)) {
|
||||
throw "Missing shard on disk: $relative"
|
||||
}
|
||||
$reader = New-Object System.IO.StreamReader($shardPath, [System.Text.Encoding]::UTF8, $true)
|
||||
try {
|
||||
while (($line = $reader.ReadLine()) -ne $null) {
|
||||
$trimmed = $line.Trim()
|
||||
if ($trimmed.Length -gt 0) {
|
||||
$writer.WriteLine($trimmed)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$reader.Dispose()
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$writer.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Validate-KHDBFile {
|
||||
param([string]$Path)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $Path)) { throw "Validation failed: $Path not found." }
|
||||
|
||||
$regex = '^[0-9A-Fa-f]{32}(:\d+)?$'
|
||||
$lineNumber = 0
|
||||
$previous = $null
|
||||
$duplicates = 0
|
||||
$reader = New-Object System.IO.StreamReader($Path, [System.Text.Encoding]::UTF8, $true)
|
||||
try {
|
||||
while (($line = $reader.ReadLine()) -ne $null) {
|
||||
$lineNumber++
|
||||
$trimmed = $line.Trim()
|
||||
if ($trimmed.Length -eq 0) { continue }
|
||||
if ($trimmed -notmatch $regex) {
|
||||
throw "Validation failed: unexpected format at line $lineNumber ('$trimmed')."
|
||||
}
|
||||
$normalized = $trimmed.ToUpperInvariant()
|
||||
if ($previous -and $normalized -lt $previous) {
|
||||
Write-Warning ("Validation warning: line {0} is out of order." -f $lineNumber)
|
||||
}
|
||||
if ($normalized -eq $previous) { $duplicates++ }
|
||||
$previous = $normalized
|
||||
}
|
||||
} finally {
|
||||
$reader.Dispose()
|
||||
}
|
||||
|
||||
if ($lineNumber -eq 0) { throw 'Validation failed: KHDB file is empty.' }
|
||||
if ($duplicates -gt 0) {
|
||||
Write-Warning ("Validation warning: detected {0} duplicate hash entries (file remains unchanged)." -f $duplicates)
|
||||
}
|
||||
}
|
||||
|
||||
function Remove-EmptyDirectories {
|
||||
param([string]$Root)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $Root)) { return }
|
||||
Get-ChildItem -LiteralPath $Root -Directory -Recurse | Sort-Object FullName -Descending | ForEach-Object {
|
||||
$childItems = Get-ChildItem -LiteralPath $_.FullName -Force
|
||||
if (-not $childItems) {
|
||||
Remove-Item -LiteralPath $_.FullName -Force
|
||||
}
|
||||
}
|
||||
# Deduplicate and normalize line endings
|
||||
$unique = $lines | ForEach-Object { $_.Trim() } | Where-Object { $_ } | Sort-Object -Unique
|
||||
Set-Content -Path $path -Value $unique -Encoding ASCII
|
||||
}
|
||||
|
||||
function Update-KHDB {
|
||||
@@ -270,79 +506,273 @@ function Update-KHDB {
|
||||
try {
|
||||
$settings = Read-ElysiumSettings
|
||||
$installPath = Get-InstallationPath $settings
|
||||
if (-not (Test-Path $installPath)) { New-Item -Path $installPath -ItemType Directory -Force | Out-Null }
|
||||
Ensure-Directory $installPath
|
||||
|
||||
$storageProvider = $settings['StorageProvider']
|
||||
if ([string]::IsNullOrWhiteSpace($storageProvider)) { $storageProvider = 'Azure' }
|
||||
|
||||
$client = $null
|
||||
$s3Bucket = $settings['s3BucketName']
|
||||
$s3EndpointUrl = $settings['s3EndpointUrl']
|
||||
$s3Region = $settings['s3Region']
|
||||
$s3AK = $settings['s3AccessKeyId']
|
||||
$s3SK = $settings['s3SecretAccessKey']
|
||||
$s3Force = $settings['s3ForcePathStyle']
|
||||
$s3UseAwsTools = $settings['s3UseAwsTools']
|
||||
try { $s3Force = [System.Convert]::ToBoolean($s3Force) } catch { $s3Force = $true }
|
||||
try { $s3UseAwsTools = [System.Convert]::ToBoolean($s3UseAwsTools) } catch { $s3UseAwsTools = $false }
|
||||
$manifestBlobPath = $settings['KhdbManifestPath']
|
||||
if ([string]::IsNullOrWhiteSpace($manifestBlobPath)) { $manifestBlobPath = 'khdb/manifest.json' }
|
||||
|
||||
$remoteShardPrefix = $settings['KhdbShardPrefix']
|
||||
if ([string]::IsNullOrWhiteSpace($remoteShardPrefix)) { $remoteShardPrefix = 'khdb/shards' }
|
||||
|
||||
$localShardDirName = $settings['KhdbLocalShardDir']
|
||||
if ([string]::IsNullOrWhiteSpace($localShardDirName)) { $localShardDirName = 'khdb-shards' }
|
||||
|
||||
$localShardRoot = Join-Path -Path $installPath -ChildPath $localShardDirName
|
||||
Ensure-Directory $localShardRoot
|
||||
|
||||
$localManifestPath = Join-Path -Path $installPath -ChildPath 'khdb-manifest.json'
|
||||
$tmpDir = New-Item -ItemType Directory -Path ([System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), "elysium-khdb-" + [System.Guid]::NewGuid())) -Force
|
||||
$zipPath = Join-Path -Path $tmpDir.FullName -ChildPath 'khdb.txt.zip'
|
||||
$extractDir = Join-Path -Path $tmpDir.FullName -ChildPath 'extract'
|
||||
New-Item -ItemType Directory -Path $extractDir -Force | Out-Null
|
||||
$manifestTempPath = Join-Path -Path $tmpDir.FullName -ChildPath 'manifest.json'
|
||||
$downloadTempRoot = Join-Path -Path $tmpDir.FullName -ChildPath 'shards'
|
||||
Ensure-Directory $downloadTempRoot
|
||||
|
||||
Write-Host "Fetching manifest ($manifestBlobPath) from $storageProvider storage..."
|
||||
|
||||
if ($storageProvider -ieq 'S3') {
|
||||
$s3Bucket = $settings['s3BucketName']
|
||||
$s3EndpointUrl = $settings['s3EndpointUrl']
|
||||
$s3Region = $settings['s3Region']
|
||||
$s3AK = $settings['s3AccessKeyId']
|
||||
$s3SK = $settings['s3SecretAccessKey']
|
||||
$s3Force = $settings['s3ForcePathStyle']
|
||||
$s3UseAwsTools = $settings['s3UseAwsTools']
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($s3Bucket)) { throw 's3BucketName is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($s3AK) -or [string]::IsNullOrWhiteSpace($s3SK)) { throw 's3AccessKeyId / s3SecretAccessKey missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($s3EndpointUrl)) { throw 's3EndpointUrl is required for S3-compatible storage.' }
|
||||
$forcePathStyle = Get-BooleanSetting -Value $s3Force -Default $true
|
||||
try { $s3UseAwsTools = [System.Convert]::ToBoolean($s3UseAwsTools) } catch { $s3UseAwsTools = $false }
|
||||
|
||||
$downloadKey = Combine-StoragePath -Prefix $null -Name $manifestBlobPath
|
||||
if ($s3UseAwsTools) {
|
||||
try {
|
||||
$s3Client = New-S3Client -EndpointUrl $s3EndpointUrl -Region $s3Region -AccessKeyId $s3AK -SecretAccessKey $s3SK -ForcePathStyle:$s3Force
|
||||
Write-Host "Downloading KHDB from S3-compatible storage (AWS Tools)..."
|
||||
# Use AWS SDK stream method into file for progress parity
|
||||
$req = New-Object Amazon.S3.Model.GetObjectRequest -Property @{ BucketName = $s3Bucket; Key = 'khdb.txt.zip' }
|
||||
$resp = $s3Client.GetObject($req)
|
||||
try { $resp.WriteResponseStreamToFile($zipPath, $true) } finally { $resp.Dispose() }
|
||||
$client = New-S3Client -EndpointUrl $s3EndpointUrl -Region $s3Region -AccessKeyId $s3AK -SecretAccessKey $s3SK -ForcePathStyle:$forcePathStyle
|
||||
$req = New-Object Amazon.S3.Model.GetObjectRequest -Property @{ BucketName = $s3Bucket; Key = $downloadKey }
|
||||
$resp = $client.GetObject($req)
|
||||
try { $resp.WriteResponseStreamToFile($manifestTempPath, $true) } finally { $resp.Dispose() }
|
||||
} catch {
|
||||
Write-Warning "AWS Tools path failed or not available. Falling back to native HTTP (SigV4). Details: $($_.Exception.Message)"
|
||||
Write-Host "Downloading KHDB from S3-compatible storage..."
|
||||
Invoke-S3HttpDownloadWithRetry -endpointUrl $s3EndpointUrl -bucket $s3Bucket -key 'khdb.txt.zip' -targetPath $zipPath -region $s3Region -ak $s3AK -sk $s3SK -forcePathStyle:$s3Force
|
||||
Write-Warning "AWS Tools download failed for manifest: $($_.Exception.Message). Falling back to SigV4 HTTP."
|
||||
Invoke-S3HttpDownloadWithRetry -EndpointUrl $s3EndpointUrl -Bucket $s3Bucket -Key $downloadKey -TargetPath $manifestTempPath -Region $s3Region -AccessKeyId $s3AK -SecretAccessKey $s3SK -ForcePathStyle:$forcePathStyle -Activity 'Downloading manifest'
|
||||
}
|
||||
} else {
|
||||
Write-Host "Downloading KHDB from S3-compatible storage..."
|
||||
Invoke-S3HttpDownloadWithRetry -endpointUrl $s3EndpointUrl -bucket $s3Bucket -key 'khdb.txt.zip' -targetPath $zipPath -region $s3Region -ak $s3AK -sk $s3SK -forcePathStyle:$s3Force
|
||||
Invoke-S3HttpDownloadWithRetry -EndpointUrl $s3EndpointUrl -Bucket $s3Bucket -Key $downloadKey -TargetPath $manifestTempPath -Region $s3Region -AccessKeyId $s3AK -SecretAccessKey $s3SK -ForcePathStyle:$forcePathStyle -Activity 'Downloading manifest'
|
||||
}
|
||||
} else {
|
||||
$storageAccountName = $settings['storageAccountName']
|
||||
$containerName = $settings['containerName']
|
||||
$sasToken = $settings['sasToken']
|
||||
$uri = Build-BlobUri -account $storageAccountName -container $containerName -sas $sasToken
|
||||
$client = New-HttpClient
|
||||
Write-Host "Downloading KHDB from Azure Blob Storage..."
|
||||
Invoke-DownloadWithRetry -client $client -uri $uri -targetPath $zipPath
|
||||
try {
|
||||
$uri = Build-BlobUri -Account $storageAccountName -Container $containerName -Sas $sasToken -BlobName $manifestBlobPath
|
||||
Invoke-DownloadWithRetry -Client $client -Uri $uri -TargetPath $manifestTempPath -Activity 'Downloading manifest'
|
||||
} finally {
|
||||
if ($client) { $client.Dispose() }
|
||||
}
|
||||
}
|
||||
Write-Host "Download completed. Extracting archive..."
|
||||
|
||||
Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force
|
||||
$manifest = Load-Manifest -Path $manifestTempPath
|
||||
Validate-Manifest -Manifest $manifest
|
||||
if ($manifest.shardPrefix) {
|
||||
$remoteShardPrefix = [string]$manifest.shardPrefix
|
||||
Write-Verbose "Using shard prefix from manifest: $remoteShardPrefix"
|
||||
}
|
||||
|
||||
$extractedKHDB = Get-ChildItem -Path $extractDir -Recurse -Filter 'khdb.txt' | Select-Object -First 1
|
||||
if (-not $extractedKHDB) { throw 'Extracted archive does not contain khdb.txt.' }
|
||||
$remoteShardPrefix = $remoteShardPrefix.Replace('\', '/')
|
||||
|
||||
# Validate content
|
||||
Validate-KHDBFile -path $extractedKHDB.FullName
|
||||
Write-Host ("Manifest downloaded. Found {0} shard(s)." -f $manifest.shards.Count)
|
||||
if ($manifest.version) {
|
||||
Write-Host ("Remote version: {0}" -f $manifest.version)
|
||||
}
|
||||
|
||||
# Compute target path and backup
|
||||
$targetKHDB = Join-Path -Path $installPath -ChildPath 'khdb.txt'
|
||||
if (Test-Path $targetKHDB) {
|
||||
$localManifest = $null
|
||||
if (Test-Path -LiteralPath $localManifestPath) {
|
||||
try {
|
||||
$localManifest = Load-Manifest -Path $localManifestPath
|
||||
} catch {
|
||||
Write-Warning ("Failed to parse existing manifest. Full refresh will occur: {0}" -f $_.Exception.Message)
|
||||
}
|
||||
}
|
||||
|
||||
$localManifestMap = @{}
|
||||
if ($localManifest -and $localManifest.shards) {
|
||||
foreach ($entry in $localManifest.shards) {
|
||||
if ($entry.name) { $localManifestMap[$entry.name] = $entry }
|
||||
}
|
||||
}
|
||||
|
||||
$downloadQueue = New-Object System.Collections.Generic.List[psobject]
|
||||
$remoteNameSet = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase)
|
||||
foreach ($entry in $manifest.shards) {
|
||||
$name = [string]$entry.name
|
||||
[void]$remoteNameSet.Add($name)
|
||||
$expectedHash = ([string]$entry.sha256).ToLowerInvariant()
|
||||
$expectedSize = 0L
|
||||
if (-not [long]::TryParse([string]$entry.size, [ref]$expectedSize)) {
|
||||
throw "Cannot parse size for shard '$name'."
|
||||
}
|
||||
|
||||
$localPath = Join-Path -Path $localShardRoot -ChildPath $name
|
||||
$needsDownload = $true
|
||||
|
||||
if (Test-Path -LiteralPath $localPath) {
|
||||
$localInfo = Get-Item -LiteralPath $localPath
|
||||
if ($localInfo.Length -eq $expectedSize) {
|
||||
$localManifestEntry = $null
|
||||
if ($localManifestMap.ContainsKey($name)) {
|
||||
$localManifestEntry = $localManifestMap[$name]
|
||||
}
|
||||
if ($localManifestEntry -and ([string]$localManifestEntry.sha256).ToLowerInvariant() -eq $expectedHash) {
|
||||
$needsDownload = $false
|
||||
} else {
|
||||
$localHash = Get-FileSha256Lower -Path $localPath
|
||||
if ($localHash -eq $expectedHash) { $needsDownload = $false }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($needsDownload) {
|
||||
$downloadQueue.Add($entry)
|
||||
}
|
||||
}
|
||||
|
||||
if ($downloadQueue.Count -gt 0) {
|
||||
Write-Host ("{0} shard(s) require download or refresh." -f $downloadQueue.Count)
|
||||
} else {
|
||||
Write-Host 'All shards already up to date; verifying manifest and combined file.'
|
||||
}
|
||||
|
||||
$storageClient = $null
|
||||
$storageHttpClient = $null
|
||||
$isS3 = ($storageProvider -ieq 'S3')
|
||||
|
||||
try {
|
||||
if ($isS3) {
|
||||
$s3Bucket = $settings['s3BucketName']
|
||||
$s3EndpointUrl = $settings['s3EndpointUrl']
|
||||
$s3Region = $settings['s3Region']
|
||||
$s3AK = $settings['s3AccessKeyId']
|
||||
$s3SK = $settings['s3SecretAccessKey']
|
||||
$s3Force = $settings['s3ForcePathStyle']
|
||||
$forcePathStyle = Get-BooleanSetting -Value $s3Force -Default $true
|
||||
$useAwsTools = $settings['s3UseAwsTools']
|
||||
try { $useAwsTools = [System.Convert]::ToBoolean($useAwsTools) } catch { $useAwsTools = $false }
|
||||
|
||||
if ($downloadQueue.Count -gt 0) {
|
||||
if ($useAwsTools) {
|
||||
$storageClient = New-S3Client -EndpointUrl $s3EndpointUrl -Region $s3Region -AccessKeyId $s3AK -SecretAccessKey $s3SK -ForcePathStyle:$forcePathStyle
|
||||
} else {
|
||||
$storageHttpClient = @{
|
||||
Endpoint = $s3EndpointUrl
|
||||
Bucket = $s3Bucket
|
||||
Region = $s3Region
|
||||
AccessKey = $s3AK
|
||||
SecretKey = $s3SK
|
||||
ForcePath = $forcePathStyle
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if ($downloadQueue.Count -gt 0) {
|
||||
$storageHttpClient = New-HttpClient
|
||||
}
|
||||
}
|
||||
|
||||
$downloadIndex = 0
|
||||
foreach ($entry in $downloadQueue) {
|
||||
$downloadIndex++
|
||||
$name = [string]$entry.name
|
||||
$expectedHash = ([string]$entry.sha256).ToLowerInvariant()
|
||||
$expectedSize = 0L
|
||||
[void][long]::TryParse([string]$entry.size, [ref]$expectedSize)
|
||||
|
||||
$activity = "Downloading shard $downloadIndex/$($downloadQueue.Count): $name"
|
||||
$remoteKey = Combine-StoragePath -Prefix $remoteShardPrefix -Name $name
|
||||
$stagingPath = Join-Path -Path $downloadTempRoot -ChildPath $name
|
||||
Ensure-Directory (Split-Path -Path $stagingPath -Parent)
|
||||
|
||||
if ($isS3) {
|
||||
if ($storageClient) {
|
||||
try {
|
||||
$request = New-Object Amazon.S3.Model.GetObjectRequest -Property @{ BucketName = $settings['s3BucketName']; Key = $remoteKey }
|
||||
$response = $storageClient.GetObject($request)
|
||||
try { $response.WriteResponseStreamToFile($stagingPath, $true) } finally { $response.Dispose() }
|
||||
} catch {
|
||||
Write-Warning "AWS Tools download failed for shard '$name': $($_.Exception.Message). Falling back to SigV4 HTTP."
|
||||
Invoke-S3HttpDownloadWithRetry -EndpointUrl $storageHttpClient.Endpoint -Bucket $storageHttpClient.Bucket -Key $remoteKey -TargetPath $stagingPath -Region $storageHttpClient.Region -AccessKeyId $storageHttpClient.AccessKey -SecretAccessKey $storageHttpClient.SecretKey -ForcePathStyle:$storageHttpClient.ForcePath -Activity $activity
|
||||
}
|
||||
} else {
|
||||
Invoke-S3HttpDownloadWithRetry -EndpointUrl $storageHttpClient.Endpoint -Bucket $storageHttpClient.Bucket -Key $remoteKey -TargetPath $stagingPath -Region $storageHttpClient.Region -AccessKeyId $storageHttpClient.AccessKey -SecretAccessKey $storageHttpClient.SecretKey -ForcePathStyle:$storageHttpClient.ForcePath -Activity $activity
|
||||
}
|
||||
} else {
|
||||
$storageAccountName = $settings['storageAccountName']
|
||||
$containerName = $settings['containerName']
|
||||
$sasToken = $settings['sasToken']
|
||||
$blobUri = Build-BlobUri -Account $storageAccountName -Container $containerName -Sas $sasToken -BlobName $remoteKey
|
||||
Invoke-DownloadWithRetry -Client $storageHttpClient -Uri $blobUri -TargetPath $stagingPath -Activity $activity
|
||||
}
|
||||
|
||||
$downloadInfo = Get-Item -LiteralPath $stagingPath
|
||||
if ($downloadInfo.Length -ne $expectedSize) {
|
||||
throw "Shard '$name' size mismatch. Expected $expectedSize bytes, got $($downloadInfo.Length)."
|
||||
}
|
||||
|
||||
$actualHash = Get-FileSha256Lower -Path $stagingPath
|
||||
if ($actualHash -ne $expectedHash) {
|
||||
throw "Shard '$name' checksum mismatch. Expected $expectedHash, got $actualHash."
|
||||
}
|
||||
|
||||
$finalPath = Join-Path -Path $localShardRoot -ChildPath $name
|
||||
Ensure-Directory (Split-Path -Path $finalPath -Parent)
|
||||
Move-Item -LiteralPath $stagingPath -Destination $finalPath -Force
|
||||
Write-Host ("Shard '{0}' updated." -f $name)
|
||||
}
|
||||
} finally {
|
||||
if ($storageClient) { $storageClient.Dispose() }
|
||||
if ($storageHttpClient -is [System.Net.Http.HttpClient]) { $storageHttpClient.Dispose() }
|
||||
}
|
||||
|
||||
$existingShards = @()
|
||||
if (Test-Path -LiteralPath $localShardRoot) {
|
||||
$existingShards = Get-ChildItem -LiteralPath $localShardRoot -File -Recurse
|
||||
}
|
||||
|
||||
$removed = 0
|
||||
foreach ($file in $existingShards) {
|
||||
$relative = Get-RelativePath -BasePath $localShardRoot -FullPath $file.FullName
|
||||
if (-not $remoteNameSet.Contains($relative)) {
|
||||
Remove-Item -LiteralPath $file.FullName -Force
|
||||
$removed++
|
||||
}
|
||||
}
|
||||
|
||||
if ($removed -gt 0) {
|
||||
Write-Host ("Removed {0} stale shard(s)." -f $removed)
|
||||
Remove-EmptyDirectories -Root $localShardRoot
|
||||
}
|
||||
|
||||
Copy-Item -LiteralPath $manifestTempPath -Destination $localManifestPath -Force
|
||||
Write-Host ("Manifest saved locally to {0}" -f $localManifestPath)
|
||||
|
||||
$khdbName = if ([string]::IsNullOrWhiteSpace($settings['WeakPasswordsDatabase'])) { 'khdb.txt' } else { $settings['WeakPasswordsDatabase'] }
|
||||
$combinedTarget = Join-Path -Path $installPath -ChildPath $khdbName
|
||||
$combinedTemp = Join-Path -Path $tmpDir.FullName -ChildPath 'khdb-combined.txt'
|
||||
|
||||
Write-Host "Rebuilding combined KHDB file..."
|
||||
Merge-ShardsToFile -Manifest $manifest -ShardsRoot $localShardRoot -TargetPath $combinedTemp
|
||||
Validate-KHDBFile -Path $combinedTemp
|
||||
|
||||
if (Test-Path -LiteralPath $combinedTarget) {
|
||||
$ts = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$backupPath = Join-Path -Path $installPath -ChildPath ("khdb.txt.bak-$ts")
|
||||
Copy-Item -Path $targetKHDB -Destination $backupPath -Force
|
||||
Write-Host "Existing KHDB backed up to $backupPath"
|
||||
$backupPath = Join-Path -Path $installPath -ChildPath ("$khdbName.bak-$ts")
|
||||
Copy-Item -LiteralPath $combinedTarget -Destination $backupPath -Force
|
||||
Write-Host ("Existing KHDB backed up to {0}" -f $backupPath)
|
||||
}
|
||||
|
||||
# Atomic-ish replace: move validated file into place
|
||||
Move-Item -Path $extractedKHDB.FullName -Destination $targetKHDB -Force
|
||||
Write-Host "KHDB updated at $targetKHDB"
|
||||
Move-Item -LiteralPath $combinedTemp -Destination $combinedTarget -Force
|
||||
Write-Host ("KHDB merged file refreshed at {0}" -f $combinedTarget)
|
||||
Write-Host "KHDB update completed successfully."
|
||||
} catch {
|
||||
Write-Error ("KHDB update failed: {0}" -f $_.Exception.Message)
|
||||
@@ -353,6 +783,5 @@ function Update-KHDB {
|
||||
}
|
||||
}
|
||||
|
||||
# Execute the update function
|
||||
Update-KHDB
|
||||
Write-Host "Script execution completed."
|
||||
|
||||
Reference in New Issue
Block a user