cqrenet/elysium
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Release v2.2.1: DRY refactoring and housekeeping

Browse Source 
Consolidated duplicated helpers into Elysium.Common.ps1:
- Settings parsing (Read-KeyValueSettingsFile, Read-ElysiumSettings, Get-SettingsValue)
- Azure Blob URI builder (Build-BlobUri)
- S3 SigV4 signing helpers and AWS module bootstrap
- AD credential validation and replication permission pre-check
- Parallel execution helper (Get-FunctionDefinitionText)

Test-WeakADPasswords.ps1 and Extract-NTHashes.ps1 now import
Elysium.Common.ps1 for the first time. Update-KHDB.ps1 and
Prepare-KHDBStorage.ps1 removed their local duplicates.

Deleted legacy Settings.ps1 (superseded by ElysiumSettings.txt).
Removed stray placeholder comment in Elysium.ps1.

All versions bumped to unified v2.2.1.
This commit is contained in:
Tomas Kracmar
2026-06-09 10:52:19 +02:00
parent 5127c2d096
commit 09c30f97e9
11 changed files with 616 additions and 845 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Update-KHDB.ps1
+98 -261
View File
@@ -7,7 +7,7 @@
##################################################
## Project: Elysium ##
## File: Update-KHDB.ps1 ##
## Version: 2.2.0 ##
## Version: 2.2.1 ##
## Support: support@cqre.net ##
##################################################
@@ -50,21 +50,6 @@ function Stop-UpdateTranscript {
try { Stop-Transcript | Out-Null } catch {}
}
function Read-ElysiumSettings {
$settings = @{}
$settingsPath = Join-Path -Path $scriptRoot -ChildPath 'ElysiumSettings.txt'
if (-not (Test-Path $settingsPath)) { throw "Settings file not found at $settingsPath" }
Get-Content $settingsPath | ForEach-Object {
if ($_ -and -not $_.Trim().StartsWith('#')) {
$kv = $_ -split '=', 2
if ($kv.Count -eq 2) {
$settings[$kv[0].Trim()] = $kv[1].Trim().Trim("'")
}
}
}
return $settings
}
function Get-InstallationPath([hashtable]$settings) {
$p = $settings['InstallationPath']
if ([string]::IsNullOrWhiteSpace($p)) { return $scriptRoot }
@@ -76,218 +61,10 @@ function New-HttpClient {
Add-Type -AssemblyName System.Net.Http -ErrorAction SilentlyContinue
$client = [System.Net.Http.HttpClient]::new()
$client.Timeout = [TimeSpan]::FromSeconds(600)
$client.DefaultRequestHeaders.UserAgent.ParseAdd('Elysium/2.1.1 (+Update-KHDB)')
$client.DefaultRequestHeaders.UserAgent.ParseAdd('Elysium/2.2.1 (+Update-KHDB)')
return $client
}
function Build-BlobUri {
param(
[string]$Account,
[string]$Container,
[string]$Sas,
[string]$BlobName
)
if ([string]::IsNullOrWhiteSpace($Account)) { throw 'storageAccountName is missing or empty.' }
if ([string]::IsNullOrWhiteSpace($Container)) { throw 'containerName is missing or empty.' }
if ([string]::IsNullOrWhiteSpace($Sas)) { throw 'sasToken is missing or empty.' }
if ([string]::IsNullOrWhiteSpace($BlobName)) { throw 'BlobName cannot be empty.' }
$sas = $Sas.Trim()
if (-not $sas.StartsWith('?')) { $sas = '?' + $sas }
$normalizedBlob = $BlobName.Replace('\', '/').TrimStart('/')
$uriBuilder = [System.UriBuilder]::new("https://$Account.blob.core.windows.net/$Container/$normalizedBlob")
$uriBuilder.Query = $sas.TrimStart('?')
return $uriBuilder.Uri.AbsoluteUri
}
function Ensure-AWSS3Module {
try { $null = [Amazon.S3.AmazonS3Client]; return } catch {}
try { Import-Module -Name AWS.Tools.S3 -ErrorAction Stop; return } catch {}
try { Import-Module -Name AWSPowerShell.NetCore -ErrorAction Stop; return } catch {}
throw "AWS Tools for PowerShell not found. Install with: Install-Module AWS.Tools.S3 -Scope CurrentUser"
}
function Get-FunctionDefinitionText {
param([Parameter(Mandatory = $true)][string]$Name)
$cmd = Get-Command -Name $Name -CommandType Function -ErrorAction Stop
return $cmd.ScriptBlock.Ast.Extent.Text
}
function New-S3Client {
param(
[string]$EndpointUrl,
[string]$Region,
[string]$AccessKeyId,
[string]$SecretAccessKey,
[bool]$ForcePathStyle = $true
)
Ensure-AWSS3Module
$creds = New-Object Amazon.Runtime.BasicAWSCredentials($AccessKeyId, $SecretAccessKey)
$cfg = New-Object Amazon.S3.AmazonS3Config
if ($EndpointUrl) { $cfg.ServiceURL = $EndpointUrl }
if ($Region) { try { $cfg.RegionEndpoint = [Amazon.RegionEndpoint]::GetBySystemName($Region) } catch {} }
$cfg.ForcePathStyle = [bool]$ForcePathStyle
return (New-Object Amazon.S3.AmazonS3Client($creds, $cfg))
}
function Get-Bytes([string]$s) { return [System.Text.Encoding]::UTF8.GetBytes($s) }
function Get-HashHex([byte[]]$bytes) {
if ($null -eq $bytes) { $bytes = [byte[]]@() }
$sha = [System.Security.Cryptography.SHA256]::Create()
try {
$ms = New-Object System.IO.MemoryStream -ArgumentList (,$bytes)
try {
$hash = $sha.ComputeHash([System.IO.Stream]$ms)
} finally { $ms.Dispose() }
return ([BitConverter]::ToString($hash)).Replace('-', '').ToLowerInvariant()
} finally { $sha.Dispose() }
}
function HmacSha256([byte[]]$key, [string]$data) {
$h = [System.Security.Cryptography.HMACSHA256]::new($key)
try {
$b = [System.Text.Encoding]::UTF8.GetBytes($data)
$ms = New-Object System.IO.MemoryStream -ArgumentList (,$b)
try {
return $h.ComputeHash([System.IO.Stream]$ms)
} finally { $ms.Dispose() }
} finally { $h.Dispose() }
}
function GetSignatureKey([string]$secret, [string]$dateStamp, [string]$regionName, [string]$serviceName) {
$kDate = HmacSha256 (Get-Bytes ('AWS4' + $secret)) $dateStamp
$kRegion = HmacSha256 $kDate $regionName
$kService = HmacSha256 $kRegion $serviceName
HmacSha256 $kService 'aws4_request'
}
function UriEncode([string]$data, [bool]$encodeSlash) {
$enc = [System.Uri]::EscapeDataString($data)
if (-not $encodeSlash) { $enc = $enc -replace '%2F', '/' }
return $enc
}
function BuildCanonicalPath([System.Uri]$uri) {
$segments = $uri.AbsolutePath.Split('/')
$encoded = @()
foreach ($s in $segments) { $encoded += (UriEncode $s $false) }
$path = ($encoded -join '/')
if (-not $path.StartsWith('/')) { $path = '/' + $path }
return $path
}
function ToHex([byte[]]$b) { ([BitConverter]::ToString($b)).Replace('-', '').ToLowerInvariant() }
function BuildAuthHeaders($method, [System.Uri]$uri, [string]$region, [string]$accessKey, [string]$secretKey, [string]$payloadHash) {
$algorithm = 'AWS4-HMAC-SHA256'
$timestamp = (Get-Date).ToUniversalTime()
$amzDate = $timestamp.ToString('yyyyMMddTHHmmssZ')
$dateStamp = $timestamp.ToString('yyyyMMdd')
$hostHeader = $uri.Host
if (-not $uri.IsDefaultPort) { $hostHeader = "${hostHeader}:$($uri.Port)" }
$canonicalUri = BuildCanonicalPath $uri
$canonicalQueryString = ''
$canonicalHeaders = "host:$hostHeader`n" + "x-amz-content-sha256:$payloadHash`n" + "x-amz-date:$amzDate`n"
$signedHeaders = 'host;x-amz-content-sha256;x-amz-date'
$canonicalRequest = "$method`n$canonicalUri`n$canonicalQueryString`n$canonicalHeaders`n$signedHeaders`n$payloadHash"
$credentialScope = "$dateStamp/$region/s3/aws4_request"
$stringToSign = "$algorithm`n$amzDate`n$credentialScope`n$((Get-HashHex (Get-Bytes $canonicalRequest)))"
$signingKey = GetSignatureKey $secretKey $dateStamp $region 's3'
$signature = ToHex (HmacSha256 $signingKey $stringToSign)
$authHeader = "$algorithm Credential=$accessKey/$credentialScope, SignedHeaders=$signedHeaders, Signature=$signature"
@{
'x-amz-date' = $amzDate
'x-amz-content-sha256' = $payloadHash
'Authorization' = $authHeader
}
}
function BuildS3Uri([string]$endpointUrl, [string]$bucket, [string]$key, [bool]$forcePathStyle) {
$base = [System.Uri]$endpointUrl
$builder = [System.UriBuilder]::new($base)
$normalizedKey = $key.Replace('\', '/').TrimStart('/')
if ($forcePathStyle) {
$path = $builder.Path.TrimEnd('/')
if ([string]::IsNullOrEmpty($path)) { $path = '/' }
$builder.Path = ($path.TrimEnd('/') + '/' + $bucket + '/' + $normalizedKey)
} else {
$builder.Host = "$bucket." + $builder.Host
$path = $builder.Path.TrimEnd('/')
if ([string]::IsNullOrEmpty($path)) { $path = '/' }
$builder.Path = ($path.TrimEnd('/') + '/' + $normalizedKey)
}
return $builder.Uri
}
function Invoke-S3HttpDownloadWithRetry {
param(
[string]$EndpointUrl,
[string]$Bucket,
[string]$Key,
[string]$TargetPath,
[string]$Region,
[string]$AccessKeyId,
[string]$SecretAccessKey,
[bool]$ForcePathStyle,
[string]$Activity
)
Add-Type -AssemblyName System.Net.Http -ErrorAction SilentlyContinue
[System.Net.Http.HttpClient]$client = [System.Net.Http.HttpClient]::new()
$retries = 5
$delay = 2
try {
for ($attempt = 0; $attempt -lt $retries; $attempt++) {
$request = $null
try {
$uri = BuildS3Uri -endpointUrl $EndpointUrl -bucket $Bucket -key $Key -forcePathStyle $ForcePathStyle
$payloadHash = (Get-HashHex (Get-Bytes ''))
$headers = BuildAuthHeaders -method 'GET' -uri $uri -region $Region -accessKey $AccessKeyId -secretKey $SecretAccessKey -payloadHash $payloadHash
$request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Get, $uri)
foreach ($kvp in $headers.GetEnumerator()) {
$request.Headers.TryAddWithoutValidation($kvp.Key, $kvp.Value) | Out-Null
}
$response = $client.SendAsync($request, [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead).GetAwaiter().GetResult()
$null = $response.EnsureSuccessStatusCode()
$totalBytes = $response.Content.Headers.ContentLength
$stream = $response.Content.ReadAsStreamAsync().Result
$tmpPath = $TargetPath
$fs = [System.IO.File]::Create($tmpPath)
try {
$buffer = New-Object byte[] 8192
$totalRead = 0
while (($read = $stream.Read($buffer, 0, $buffer.Length)) -gt 0) {
$fs.Write($buffer, 0, $read)
$totalRead += $read
if ($totalBytes) {
$pct = ($totalRead * 100.0) / $totalBytes
Write-Progress -Activity $Activity -Status ("{0:N2}% Complete" -f $pct) -PercentComplete $pct
} else {
Write-Progress -Activity $Activity -Status ("Downloaded {0:N0} bytes" -f $totalRead) -PercentComplete 0
}
}
} finally {
$fs.Close()
$stream.Close()
}
if ($response) { $response.Dispose() }
Write-Progress -Activity $Activity -Completed -Status 'Completed'
return
} catch {
if ($attempt -lt ($retries - 1)) {
Write-Warning "Download of '$Key' failed (attempt $($attempt + 1)/$retries): $($_.Exception.Message). Retrying in ${delay}s..."
Start-Sleep -Seconds $delay
$delay = [Math]::Min($delay * 2, 30)
} else {
throw
}
} finally {
if ($request) { $request.Dispose() }
}
}
} finally {
$client.Dispose()
}
}
function Invoke-DownloadWithRetry {
param(
[System.Net.Http.HttpClient]$Client,
@@ -342,6 +119,79 @@ function Invoke-DownloadWithRetry {
}
}
function Invoke-S3HttpDownloadWithRetry {
param(
[string]$EndpointUrl,
[string]$Bucket,
[string]$Key,
[string]$TargetPath,
[string]$Region,
[string]$AccessKeyId,
[string]$SecretAccessKey,
[bool]$ForcePathStyle,
[string]$Activity
)
Add-Type -AssemblyName System.Net.Http -ErrorAction SilentlyContinue
[System.Net.Http.HttpClient]$client = [System.Net.Http.HttpClient]::new()
$retries = 5
$delay = 2
try {
for ($attempt = 0; $attempt -lt $retries; $attempt++) {
$request = $null
try {
$uri = BuildS3Uri -endpointUrl $EndpointUrl -bucket $Bucket -key $Key -forcePathStyle $ForcePathStyle
$payloadHash = (Get-HashHex (Get-Bytes ''))
$headers = BuildAuthHeaders -method 'GET' -uri $uri -region $Region -accessKey $AccessKeyId -secretKey $SecretAccessKey -payloadHash $payloadHash
$request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Get, $uri)
foreach ($kvp in $headers.GetEnumerator()) {
$request.Headers.TryAddWithoutValidation($kvp.Key, $kvp.Value) | Out-Null
}
$response = $client.SendAsync($request, [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead).GetAwaiter().GetResult()
$null = $response.EnsureSuccessStatusCode()
$totalBytes = $response.Content.Headers.ContentLength
$stream = $response.Content.ReadAsStreamAsync().Result
$fs = [System.IO.File]::Create($TargetPath)
try {
$buffer = New-Object byte[] 8192
$totalRead = 0
while (($read = $stream.Read($buffer, 0, $buffer.Length)) -gt 0) {
$fs.Write($buffer, 0, $read)
$totalRead += $read
if ($totalBytes) {
$pct = ($totalRead * 100.0) / $totalBytes
Write-Progress -Activity $Activity -Status ("{0:N2}% Complete" -f $pct) -PercentComplete $pct
} else {
Write-Progress -Activity $Activity -Status ("Downloaded {0:N0} bytes" -f $totalRead) -PercentComplete 0
}
}
} finally {
$fs.Close()
$stream.Close()
}
if ($response) { $response.Dispose() }
Write-Progress -Activity $Activity -Completed -Status 'Completed'
return
} catch {
if ($attempt -lt ($retries - 1)) {
Write-Warning "Download of '$Key' failed (attempt $($attempt + 1)/$retries): $($_.Exception.Message). Retrying in ${delay}s..."
Start-Sleep -Seconds $delay
$delay = [Math]::Min($delay * 2, 30)
} else {
throw
}
} finally {
if ($request) { $request.Dispose() }
}
}
} finally {
$client.Dispose()
}
}
function Get-FileSha256Lower {
param([string]$Path)
if (-not (Test-Path -LiteralPath $Path)) { throw "File not found: $Path" }
@@ -387,19 +237,6 @@ function Get-RelativePath {
return $relativePath.Replace('/', [System.IO.Path]::DirectorySeparatorChar)
}
function Combine-StoragePath {
param(
[string]$Prefix,
[string]$Name
)
$cleanName = $Name.Replace('\', '/').TrimStart('/')
if ([string]::IsNullOrWhiteSpace($Prefix)) { return $cleanName }
$normalizedPrefix = $Prefix.Replace('\', '/').Trim('/')
if ([string]::IsNullOrEmpty($normalizedPrefix)) { return $cleanName }
return "$normalizedPrefix/$cleanName"
}
function Load-Manifest {
param([string]$Path)
$raw = Get-Content -LiteralPath $Path -Encoding UTF8 -Raw
@@ -542,7 +379,7 @@ function Update-KHDB {
)
Start-UpdateTranscript -BasePath $scriptRoot
try {
$settings = Read-ElysiumSettings
$settings = Read-ElysiumSettings -ScriptRoot $scriptRoot
$installPath = Get-InstallationPath $settings
Ensure-Directory $installPath
@@ -558,10 +395,10 @@ function Update-KHDB {
$parallelS3DownloadHelperList = @()
if ($parallelDownloadsEnabled) {
$parallelAzureDownloadHelpers = @{
'Build-BlobUri' = Get-FunctionDefinitionText 'Build-BlobUri'
'Build-BlobUri' = Get-FunctionDefinitionText 'Build-BlobUri'
'Invoke-DownloadWithRetry' = Get-FunctionDefinitionText 'Invoke-DownloadWithRetry'
'New-HttpClient' = Get-FunctionDefinitionText 'New-HttpClient'
'Get-FileSha256Lower' = Get-FunctionDefinitionText 'Get-FileSha256Lower'
'New-HttpClient' = Get-FunctionDefinitionText 'New-HttpClient'
'Get-FileSha256Lower' = Get-FunctionDefinitionText 'Get-FileSha256Lower'
}
$parallelAzureDownloadHelperList = $parallelAzureDownloadHelpers.GetEnumerator() | ForEach-Object {
[pscustomobject]@{ Name = $_.Key; Definition = $_.Value }
@@ -727,14 +564,14 @@ function Update-KHDB {
}
}
if ($needsDownload) {
[void]$downloadQueue.Add([pscustomobject]@{
Name = $name
Sha256 = $expectedHash
Size = $expectedSize
})
if ($needsDownload) {
[void]$downloadQueue.Add([pscustomobject]@{
Name = $name
Sha256 = $expectedHash
Size = $expectedSize
})
}
}
}
if ($downloadQueue.Count -gt 0) {
Write-Host ("{0} shard(s) require download or refresh." -f $downloadQueue.Count)
@@ -763,12 +600,12 @@ function Update-KHDB {
$storageClient = New-S3Client -EndpointUrl $s3EndpointUrl -Region $s3Region -AccessKeyId $s3AK -SecretAccessKey $s3SK -ForcePathStyle:$forcePathStyle
}
$storageHttpClient = @{
Endpoint = $s3EndpointUrl
Bucket = $s3Bucket
Region = $s3Region
AccessKey = $s3AK
SecretKey = $s3SK
ForcePath = $forcePathStyle
Endpoint = $s3EndpointUrl
Bucket = $s3Bucket
Region = $s3Region
AccessKey = $s3AK
SecretKey = $s3SK
ForcePath = $forcePathStyle
}
}
} else {
@@ -846,11 +683,11 @@ function Update-KHDB {
$downloadIndex = 0
foreach ($entry in $downloadQueue.ToArray()) {
$downloadIndex++
if ($null -eq $entry) { continue }
$name = [string]$entry.Name
if ([string]::IsNullOrWhiteSpace($name)) {
throw "Shard entry missing name: $(ConvertTo-Json $entry -Compress)"
}
if ($null -eq $entry) { continue }
$name = [string]$entry.Name
if ([string]::IsNullOrWhiteSpace($name)) {
throw "Shard entry missing name: $(ConvertTo-Json $entry -Compress)"
}
$expectedHash = ([string]$entry.Sha256).ToLowerInvariant()
$expectedSize = [long]$entry.Size
@@ -862,7 +699,7 @@ function Update-KHDB {
if ($isS3) {
if ($storageClient) {
try {
$request = New-Object Amazon.S3.Model.GetObjectRequest -Property @{ BucketName = $s3Bucket; Key = $remoteKey }
$request = New-Object Amazon.S3.Model.GetObjectRequest -Property @{ BucketName = $s3Bucket; Key = $remoteKey }
$response = $storageClient.GetObject($request)
try { $response.WriteResponseStreamToFile($stagingPath, $true) } finally { $response.Dispose() }
} catch {