Bug fixes
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -2,4 +2,5 @@
|
||||
khdb.txt
|
||||
khdb.txt.zip
|
||||
ElysiumSettings.txt
|
||||
/Reports
|
||||
/ReportsElysium/khdb.csv
|
||||
Settings.ps1
|
||||
41
Elysium/Elysium.ps1
Normal file
41
Elysium/Elysium.ps1
Normal file
@@ -0,0 +1,41 @@
|
||||
#Global settings
|
||||
. "../Settings.ps1"
|
||||
function Show-Menu {
|
||||
param (
|
||||
[string]$Title = 'Project Elysium'
|
||||
)
|
||||
Clear-Host
|
||||
Write-Host "================ $Title ================"
|
||||
|
||||
Write-Host "1: Update Known Hashes Database"
|
||||
Write-Host "2: Run Weak Password Test"
|
||||
Write-Host "3: Extract and Send Current Hashes"
|
||||
Write-Host "Q: Exit"
|
||||
}
|
||||
|
||||
do {
|
||||
Show-Menu
|
||||
$input = Read-Host "Please make a selection"
|
||||
switch ($input) {
|
||||
'1' {
|
||||
# Call Script 1
|
||||
.\UpdateKHDB.ps1
|
||||
break
|
||||
}
|
||||
'2' {
|
||||
# Call Script 2
|
||||
.\TestADAccounts.ps1
|
||||
break
|
||||
}
|
||||
'3' {
|
||||
# Call Script 3
|
||||
.\ExportHashes.ps1
|
||||
break
|
||||
}
|
||||
'Q' {
|
||||
return
|
||||
}
|
||||
}
|
||||
pause
|
||||
}
|
||||
until ($input -eq 'Q')
|
||||
60
Elysium/ExportHashes.ps1
Normal file
60
Elysium/ExportHashes.ps1
Normal file
@@ -0,0 +1,60 @@
|
||||
#Global settings
|
||||
. "../Settings.ps1"
|
||||
|
||||
# Import Required Modules
|
||||
Import-Module DSInternals
|
||||
Add-Type -AssemblyName System.IO.Compression.FileSystem
|
||||
|
||||
# Define Domains and Associated Usernames
|
||||
$domains = @{
|
||||
"Domain1" = "username1";
|
||||
"Domain2" = "username2";
|
||||
# Add more domains and usernames as needed
|
||||
}
|
||||
|
||||
# Present Choice of Domains to User
|
||||
$selectedDomain = $domains.Keys | Out-GridView -Title "Select a Domain" -PassThru
|
||||
$selectedUsername = $domains[$selectedDomain]
|
||||
|
||||
# Ask User to Enter Password for Chosen Account
|
||||
Write-Host "Enter password for account $selectedUsername in domain $selectedDomain:"
|
||||
$password = Read-Host -AsSecureString
|
||||
|
||||
# Define Domain Controller (Modify as needed)
|
||||
$domainController = "$selectedDomain" + "Controller" # Example: Domain1Controller
|
||||
|
||||
# Credential Object
|
||||
$credential = New-Object System.Management.Automation.PSCredential ($selectedUsername, $password)
|
||||
|
||||
# Get Current Timestamp
|
||||
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
|
||||
|
||||
# Define Export Path and Filename
|
||||
$exportPath = "C:\Path\To\Export" # Configure this path as needed
|
||||
$exportFilename = "extractedHashes_" + $selectedDomain + "_" + $timestamp + ".csv"
|
||||
$exportFullPath = Join-Path $exportPath $exportFilename
|
||||
|
||||
# Extract Non-Disabled Account Hashes
|
||||
Get-ADReplAccount -All -Server $domainController -Credential $credential |
|
||||
Where-Object { -not $_.AccountDisabled } |
|
||||
Select-Object -Property SamAccountName, NTHash |
|
||||
Export-Csv -Path $exportFullPath -NoTypeInformation
|
||||
|
||||
# Ask User for a Secure Password for Encryption
|
||||
Write-Host "Enter a secure password to encrypt the file:"
|
||||
$encryptionPassword = Read-Host -AsSecureString
|
||||
|
||||
# Compress and Encrypt File
|
||||
$compressedFile = $exportFullPath + ".zip"
|
||||
[IO.Compression.ZipFile]::CreateFromDirectory($exportPath, $compressedFile)
|
||||
$encryptedFile = $compressedFile + ".encrypted"
|
||||
|
||||
# Encrypt the Compressed File
|
||||
ConvertFrom-SecureString $encryptionPassword | Out-File "$encryptedFile"
|
||||
|
||||
# Clean Up
|
||||
Remove-Item -Path $exportFullPath # Remove the original CSV file
|
||||
Remove-Item -Path $compressedFile # Remove the compressed ZIP file
|
||||
|
||||
# Output
|
||||
Write-Host "Hashes exported, compressed, and encrypted to: $encryptedFile"
|
||||
3
Elysium/TestADAccounts.ps1
Normal file
3
Elysium/TestADAccounts.ps1
Normal file
@@ -0,0 +1,3 @@
|
||||
#Global settings
|
||||
. "../Settings.ps1"
|
||||
|
||||
52
Elysium/UpdateKHDB.ps1
Normal file
52
Elysium/UpdateKHDB.ps1
Normal file
@@ -0,0 +1,52 @@
|
||||
#Global settings
|
||||
. "../Settings.ps1"
|
||||
|
||||
# Function to extract version number from filename
|
||||
function Extract-VersionNumber($filename) {
|
||||
if ($filename -match "known-hashes-v(\d+\.\d+)\.encrypted\.zip") {
|
||||
return $matches[1]
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
# Get the list of available files (assuming a directory listing is available)
|
||||
$response = Invoke-WebRequest -Uri $baseUrl
|
||||
$files = $response.Links | Where-Object { $_.href -like "known-hashes-v*.encrypted.zip" } | Select-Object -ExpandProperty href
|
||||
|
||||
# Determine the latest version
|
||||
$latestVersion = "0.0"
|
||||
$latestFile = $null
|
||||
foreach ($file in $files) {
|
||||
$version = Extract-VersionNumber $file
|
||||
if ([version]$version -gt [version]$latestVersion) {
|
||||
$latestVersion = $version
|
||||
$latestFile = $file
|
||||
}
|
||||
}
|
||||
|
||||
# Check local file version
|
||||
$localVersion = "0.0"
|
||||
if (Test-Path "$localFilePath.encrypted") {
|
||||
$localVersion = Extract-VersionNumber (Get-Item "$localFilePath.encrypted").Name
|
||||
}
|
||||
|
||||
# Download and extract if the online version is newer
|
||||
if ([version]$latestVersion -gt [version]$localVersion) {
|
||||
$downloadUrl = $baseUrl + $latestFile
|
||||
$localZipPath = "$localFilePath-v$latestVersion.encrypted.zip"
|
||||
Invoke-WebRequest -Uri $downloadUrl -OutFile $localZipPath
|
||||
|
||||
# Ask for the ZIP password
|
||||
Write-Host "Enter the password to unzip the file:"
|
||||
$zipPassword = Read-Host -AsSecureString
|
||||
|
||||
# Unzip the file (requires .NET 4.5 or higher and external tools like 7-Zip)
|
||||
$zipPasswordPlainText = [Runtime.InteropServices.Marshal]::PtrToStringBSTR([Runtime.InteropServices.Marshal]::SecureStringToBSTR($zipPassword))
|
||||
$7zipPath = "C:\Path\To\7Zip\7z.exe" # Update with the actual path to 7-Zip executable
|
||||
$arguments = "x `"$localZipPath`" -p$zipPasswordPlainText -o`"$localFilePath`" -y"
|
||||
Start-Process $7zipPath -ArgumentList $arguments -NoNewWindow -Wait
|
||||
|
||||
Write-Host "File downloaded and extracted successfully. Latest version: v$latestVersion"
|
||||
} else {
|
||||
Write-Host "Local known-hashes file is up-to-date. Current version: v$localVersion"
|
||||
}
|
||||
32
README.md
32
README.md
@@ -17,29 +17,25 @@ Sensitive operations are confined only to the dedicated host. In the third step,
|
||||
---
|
||||
## Operation
|
||||
### Install and update
|
||||
This tool is provided in private git repository. Installation and updating is done with cloning and pulling from this repository.
|
||||
During first run, the tool will ask for passphrase that will be used to encrypt/decrypt sensitive content.
|
||||
After installation, edit ElysiumSettings.txt, check all variables and add domains to test.
|
||||
Clone this private git repository to install or update the tool. During the first run, you will be prompted for a passphrase to encrypt/decrypt sensitive content. After installation, edit `ElysiumSettings.txt`, check all variables, and add domains to test.
|
||||
|
||||
### Update Known-Hashed Database (KHDB)
|
||||
Run script Elysium.ps1 as an administrator and choose option 1 (Update Known-Hashes Database).
|
||||
The script downloads the database from the configured storage (Azure Blob or S3-compatible), decompresses it and updates the current database.
|
||||
Run either `Elysium.ps1` or `Start.ps1` as an administrator and choose option 1 (Update Known-Hashes Database). The script will check for a newer version online and, if found, download and decompress it. If the KHDB content is encrypted, you will be prompted for the decryption password. The database is then updated from the configured storage (Azure Blob or S3-compatible).
|
||||
|
||||
### Test Weak AD passwords
|
||||
Run script Elysium.ps1 as an administrator and choose option 2 (Test Weak AD Passwords).
|
||||
The script will then ask for the domain to be tested and upon choice will ask for domain administrator password. The DA username is already provided in the script for each domain.
|
||||
The tool then connects to Domain Controller and tests all enabled users in the domain against KHDB. PDF report with findings is then generated.
|
||||
### Send current hashes for update KHDB
|
||||
Run script Elysium.ps1 as an administrator and choose option 3 (Extract and Send Hashes).
|
||||
The tool will then ask for domain and password of domain administrator. With correct credentials, the tool will then extract current hashes (no history) of non-disabled users, compresses and encrypts them and uploads them to the configured storage (Azure Blob or S3-compatible) for pickup by the tool provider.
|
||||
Run either `Elysium.ps1` or `Start.ps1` as an administrator and choose option 2 (Test Weak AD Passwords). The script will ask for the domain to be tested and then for the domain administrator password. The DA username is already provided in the script for each domain. The tool connects to the Domain Controller and tests all enabled users in the domain against KHDB. A PDF report with findings is generated.
|
||||
|
||||
### Send current hashes for KHDB update
|
||||
Run either `Elysium.ps1` or `Start.ps1` as an administrator and choose option 3 (Extract and Send Hashes). The tool will ask for the domain and password of the domain administrator. With correct credentials, the tool extracts current hashes (no history) of non-disabled users, compresses and encrypts them, and uploads them to the configured storage (Azure Blob or S3-compatible) for pickup by the tool provider.
|
||||
|
||||
S3-compatible usage notes:
|
||||
- No AWS Tools required. The scripts can sign requests using native SigV4 via .NET and HttpClient.
|
||||
- To force using AWS Tools instead, set `s3UseAwsTools = true` in `ElysiumSettings.txt` and install `AWS.Tools.S3`.
|
||||
### Uninstallation
|
||||
Run script Elysium.ps1 as an administrator and choose option 4 (Uninstall).
|
||||
The script will then delete everything and remove the passphrase variable.
|
||||
---
|
||||
## FAQ
|
||||
|
||||
### Uninstallation
|
||||
Run `Elysium.ps1` as an administrator and choose option 4 (Uninstall) to delete all files and remove the passphrase variable. Alternatively, you can manually remove the cloned repository.
|
||||
|
||||
## FAQ
|
||||
### What happens to the hashes we uploaded?
|
||||
These hashes are subjected to cracking. Any cracked hash is then added to KHDB. Hash cracking happens on dedicated air-gapped machine and all sensitive material is never decrypted outside this machine. Secure exchange of decryption keys is arranged beforehand with every client.
|
||||
### Do we need to upload the hashes?
|
||||
@@ -71,10 +67,10 @@ It should, as it is extremely sensitive operation that should never happen outsi
|
||||
---
|
||||
|
||||
## Weak password report
|
||||
This section explains in detail individual parts of weak password report.
|
||||
This section explains in detail individual parts of the weak password report.
|
||||
|
||||
1. Reversible Encryption:
|
||||
* ****Explanation:**** Accounts have passwords stored in a reversible format that can be decrypted.
|
||||
* **Explanation:** Accounts have passwords stored in a reversible format that can be decrypted.
|
||||
* **Risk Assessment:** High. Decrypted passwords can be misused easily.
|
||||
* **Possible Cause:** Legacy applications requiring plaintext password equivalents.
|
||||
* **Use:** Compatibility with older applications.
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
##################################################
|
||||
## ____ ___ ____ _____ _ _ _____ _____ ##
|
||||
## / ___/ _ \| _ \| ____| | \ | | ____|_ _| ##
|
||||
## | | | | | | |_) | _| | \| | _| | | ##
|
||||
## | |__| |_| | _ <| |___ _| |\ | |___ | | ##
|
||||
## \____\__\_\_| \_\_____(_)_| \_|_____| |_| ##
|
||||
##################################################
|
||||
## Project: Elysium ##
|
||||
## File: Uninstall.ps1 ##
|
||||
## Version: 1.1.0 ##
|
||||
## Support: support@cqre.net ##
|
||||
##################################################
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Uninstall script for the Elysium AD password testing tool.
|
||||
|
||||
.DESCRIPTION
|
||||
This script will remove the Elysium tool and its components (scripts, configurations, and any generated data) from the system, and then delete itself.
|
||||
#>
|
||||
|
||||
function Start-UninstallTranscript {
|
||||
try {
|
||||
$base = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), 'Elysium', 'logs')
|
||||
if (-not (Test-Path $base)) { New-Item -Path $base -ItemType Directory -Force | Out-Null }
|
||||
$ts = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$logPath = Join-Path -Path $base -ChildPath "uninstall-$ts.log"
|
||||
Start-Transcript -Path $logPath -Force | Out-Null
|
||||
} catch {
|
||||
Write-Warning "Could not start transcript: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
|
||||
function Stop-UninstallTranscript { try { Stop-Transcript | Out-Null } catch {} }
|
||||
|
||||
function Uninstall-Elysium {
|
||||
$ElysiumPath = Get-Location
|
||||
|
||||
Write-Host "Uninstalling Elysium tool from $ElysiumPath..."
|
||||
|
||||
# Check if the Elysium directory exists
|
||||
if (Test-Path $ElysiumPath) {
|
||||
# Schedule the script file for deletion
|
||||
$scriptPath = $MyInvocation.MyCommand.Path
|
||||
$deleteScript = { param($path) Remove-Item -Path $path -Force }
|
||||
Start-Sleep -Seconds 3 # Delay to ensure the script finishes
|
||||
Start-Process -FilePath "powershell.exe" -ArgumentList "-Command", $deleteScript, "-ArgumentList", $scriptPath -WindowStyle Hidden
|
||||
|
||||
# Remove the Elysium directory and all its contents
|
||||
Remove-Item -Path $ElysiumPath -Recurse -Force -Exclude $scriptPath
|
||||
Write-Host "Elysium tool and all related files have been removed, excluding this script. This script will be deleted shortly."
|
||||
} else {
|
||||
Write-Host "Elysium directory not found. It might have been removed already, or the path is incorrect."
|
||||
}
|
||||
|
||||
# Additional cleanup actions can be added here if needed
|
||||
}
|
||||
|
||||
Start-UninstallTranscript
|
||||
try {
|
||||
# Execute the uninstall function
|
||||
Uninstall-Elysium
|
||||
|
||||
# Check if the Elysium passphrase environment variable exists
|
||||
$passphraseEnvVar = [System.Environment]::GetEnvironmentVariable("ELYSIUM_PASSPHRASE", [System.EnvironmentVariableTarget]::User)
|
||||
|
||||
if ([string]::IsNullOrEmpty($passphraseEnvVar)) {
|
||||
Write-Host "No passphrase environment variable to remove."
|
||||
} else {
|
||||
# Remove the Elysium passphrase environment variable
|
||||
[System.Environment]::SetEnvironmentVariable("ELYSIUM_PASSPHRASE", $null, [System.EnvironmentVariableTarget]::User)
|
||||
Write-Host "Elysium passphrase environment variable has been removed."
|
||||
}
|
||||
|
||||
# Confirm uninstallation
|
||||
Write-Host "Elysium tool has been successfully uninstalled. Exiting script." -ForegroundColor Green
|
||||
} finally {
|
||||
Stop-UninstallTranscript
|
||||
}
|
||||
335
Update-KHDB.ps1
335
Update-KHDB.ps1
@@ -1,335 +0,0 @@
|
||||
##################################################
|
||||
## ____ ___ ____ _____ _ _ _____ _____ ##
|
||||
## / ___/ _ \| _ \| ____| | \ | | ____|_ _| ##
|
||||
## | | | | | | |_) | _| | \| | _| | | ##
|
||||
## | |__| |_| | _ <| |___ _| |\ | |___ | | ##
|
||||
## \____\__\_\_| \_\_____(_)_| \_|_____| |_| ##
|
||||
##################################################
|
||||
## Project: Elysium ##
|
||||
## File: Update-KHDB.ps1 ##
|
||||
## Version: 1.1.0 ##
|
||||
## Support: support@cqre.net ##
|
||||
##################################################
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Known hashes database update script for the Elysium AD password testing tool.
|
||||
|
||||
.DESCRIPTION
|
||||
This script downloads khdb.txt.zip from the designated storage (Azure Blob or S3-compatible), validates and decompresses it, and atomically updates the current version with backup and logging.
|
||||
#>
|
||||
|
||||
# safer defaults
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
# ensure TLS 1.2
|
||||
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor [System.Net.SecurityProtocolType]::Tls12
|
||||
|
||||
# Resolve paths
|
||||
$scriptRoot = $PSScriptRoot
|
||||
|
||||
function Start-UpdateTranscript {
|
||||
param(
|
||||
[string]$BasePath
|
||||
)
|
||||
try {
|
||||
$logsDir = Join-Path -Path $BasePath -ChildPath 'Reports/logs'
|
||||
if (-not (Test-Path $logsDir)) { New-Item -Path $logsDir -ItemType Directory -Force | Out-Null }
|
||||
$ts = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$logPath = Join-Path -Path $logsDir -ChildPath "update-khdb-$ts.log"
|
||||
Start-Transcript -Path $logPath -Force | Out-Null
|
||||
} catch {
|
||||
Write-Warning "Could not start transcript: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
|
||||
function Stop-UpdateTranscript {
|
||||
try { Stop-Transcript | Out-Null } catch {}
|
||||
}
|
||||
|
||||
function Read-ElysiumSettings {
|
||||
$settings = @{}
|
||||
$settingsPath = Join-Path -Path $scriptRoot -ChildPath 'ElysiumSettings.txt'
|
||||
if (-not (Test-Path $settingsPath)) { throw "Settings file not found at $settingsPath" }
|
||||
Get-Content $settingsPath | ForEach-Object {
|
||||
if ($_ -and -not $_.Trim().StartsWith('#')) {
|
||||
$kv = $_ -split '=', 2
|
||||
if ($kv.Count -eq 2) { $settings[$kv[0].Trim()] = $kv[1].Trim().Trim("'") }
|
||||
}
|
||||
}
|
||||
return $settings
|
||||
}
|
||||
|
||||
function Get-InstallationPath([hashtable]$settings) {
|
||||
$p = $settings['InstallationPath']
|
||||
if ([string]::IsNullOrWhiteSpace($p)) { return $scriptRoot }
|
||||
if ([System.IO.Path]::IsPathRooted($p)) { return $p }
|
||||
return (Join-Path -Path $scriptRoot -ChildPath $p)
|
||||
}
|
||||
|
||||
function New-HttpClient {
|
||||
Add-Type -AssemblyName System.Net.Http
|
||||
$client = [System.Net.Http.HttpClient]::new()
|
||||
$client.Timeout = [TimeSpan]::FromSeconds(600)
|
||||
$client.DefaultRequestHeaders.UserAgent.ParseAdd('Elysium/1.1 (+Update-KHDB)')
|
||||
return $client
|
||||
}
|
||||
|
||||
function Build-BlobUri([string]$account, [string]$container, [string]$sas) {
|
||||
if ([string]::IsNullOrWhiteSpace($account)) { throw 'storageAccountName is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($container)) { throw 'containerName is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($sas)) { throw 'sasToken is missing or empty.' }
|
||||
$sas = $sas.Trim()
|
||||
if (-not $sas.StartsWith('?')) { $sas = '?' + $sas }
|
||||
$ub = [System.UriBuilder]::new("https://$account.blob.core.windows.net/$container/khdb.txt.zip")
|
||||
$ub.Query = $sas.TrimStart('?')
|
||||
return $ub.Uri.AbsoluteUri
|
||||
}
|
||||
|
||||
function Ensure-AWSS3Module {
|
||||
try { $null = [Amazon.S3.AmazonS3Client]; return } catch {}
|
||||
try { Import-Module -Name AWS.Tools.S3 -ErrorAction Stop; return } catch {}
|
||||
try { Import-Module -Name AWSPowerShell.NetCore -ErrorAction Stop; return } catch {}
|
||||
throw "AWS Tools for PowerShell not found. Install with: Install-Module AWS.Tools.S3 -Scope CurrentUser"
|
||||
}
|
||||
|
||||
function New-S3Client {
|
||||
param(
|
||||
[string]$EndpointUrl,
|
||||
[string]$Region,
|
||||
[string]$AccessKeyId,
|
||||
[string]$SecretAccessKey,
|
||||
[bool]$ForcePathStyle = $true
|
||||
)
|
||||
Ensure-AWSS3Module
|
||||
$creds = New-Object Amazon.Runtime.BasicAWSCredentials($AccessKeyId, $SecretAccessKey)
|
||||
$cfg = New-Object Amazon.S3.AmazonS3Config
|
||||
if ($EndpointUrl) { $cfg.ServiceURL = $EndpointUrl }
|
||||
if ($Region) { try { $cfg.RegionEndpoint = [Amazon.RegionEndpoint]::GetBySystemName($Region) } catch {} }
|
||||
$cfg.ForcePathStyle = [bool]$ForcePathStyle
|
||||
return (New-Object Amazon.S3.AmazonS3Client($creds, $cfg))
|
||||
}
|
||||
|
||||
# Native S3 SigV4 (no AWS Tools) helpers
|
||||
function Get-Bytes([string]$s) { return [System.Text.Encoding]::UTF8.GetBytes($s) }
|
||||
function Get-HashHex([byte[]]$bytes) { $sha=[System.Security.Cryptography.SHA256]::Create(); try { ([BitConverter]::ToString($sha.ComputeHash($bytes))).Replace('-', '').ToLowerInvariant() } finally { $sha.Dispose() } }
|
||||
function HmacSha256([byte[]]$key, [string]$data) { $h=[System.Security.Cryptography.HMACSHA256]::new($key); try { $h.ComputeHash((Get-Bytes $data)) } finally { $h.Dispose() } }
|
||||
function GetSignatureKey([string]$secret, [string]$dateStamp, [string]$regionName, [string]$serviceName) {
|
||||
$kDate = HmacSha256 (Get-Bytes ('AWS4' + $secret)) $dateStamp
|
||||
$kRegion = HmacSha256 $kDate $regionName
|
||||
$kService = HmacSha256 $kRegion $serviceName
|
||||
HmacSha256 $kService 'aws4_request'
|
||||
}
|
||||
function UriEncode([string]$data, [bool]$encodeSlash) { $enc=[System.Uri]::EscapeDataString($data); if (-not $encodeSlash) { $enc = $enc -replace '%2F','/' }; $enc }
|
||||
function BuildCanonicalPath([System.Uri]$uri) { $segments=$uri.AbsolutePath.Split('/'); $encoded=@(); foreach($s in $segments){ $encoded += (UriEncode $s $false) }; $p=($encoded -join '/'); if (-not $p.StartsWith('/')){ $p='/' + $p }; $p }
|
||||
function ToHex([byte[]]$b) { ([BitConverter]::ToString($b)).Replace('-', '').ToLowerInvariant() }
|
||||
function BuildAuthHeaders($method, [System.Uri]$uri, [string]$region, [string]$accessKey, [string]$secretKey, [string]$payloadHash) {
|
||||
$algorithm = 'AWS4-HMAC-SHA256'
|
||||
$amzdate = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssZ')
|
||||
$datestamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd')
|
||||
$hostHeader = $uri.Host; if (-not $uri.IsDefaultPort) { $hostHeader = "$hostHeader:$($uri.Port)" }
|
||||
$canonicalUri = BuildCanonicalPath $uri
|
||||
$canonicalQueryString = ''
|
||||
$canonicalHeaders = "host:$hostHeader`n" + "x-amz-content-sha256:$payloadHash`n" + "x-amz-date:$amzdate`n"
|
||||
$signedHeaders = 'host;x-amz-content-sha256;x-amz-date'
|
||||
$canonicalRequest = "$method`n$canonicalUri`n$canonicalQueryString`n$canonicalHeaders`n$signedHeaders`n$payloadHash"
|
||||
$credentialScope = "$datestamp/$region/s3/aws4_request"
|
||||
$stringToSign = "$algorithm`n$amzdate`n$credentialScope`n$((Get-HashHex (Get-Bytes $canonicalRequest)))"
|
||||
$signingKey = GetSignatureKey $secretKey $datestamp $region 's3'
|
||||
$signature = ToHex (HmacSha256 $signingKey $stringToSign)
|
||||
$authHeader = "$algorithm Credential=$accessKey/$credentialScope, SignedHeaders=$signedHeaders, Signature=$signature"
|
||||
@{ 'x-amz-date' = $amzdate; 'x-amz-content-sha256' = $payloadHash; 'Authorization' = $authHeader }
|
||||
}
|
||||
function BuildS3Uri([string]$endpointUrl, [string]$bucket, [string]$key, [bool]$forcePathStyle) {
|
||||
$base=[System.Uri]$endpointUrl; $ub=[System.UriBuilder]::new($base)
|
||||
if ($forcePathStyle) { $p=$ub.Path.TrimEnd('/'); if ([string]::IsNullOrEmpty($p)){ $p='/' }; $ub.Path = ($p.TrimEnd('/') + '/' + $bucket + '/' + $key) }
|
||||
else { $ub.Host = "$bucket." + $ub.Host; $p=$ub.Path.TrimEnd('/'); if ([string]::IsNullOrEmpty($p)){ $p='/' }; $ub.Path = ($p.TrimEnd('/') + '/' + $key) }
|
||||
$ub.Uri
|
||||
}
|
||||
function Invoke-S3HttpDownloadWithRetry([string]$endpointUrl, [string]$bucket, [string]$key, [string]$targetPath, [string]$region, [string]$ak, [string]$sk, [bool]$forcePathStyle) {
|
||||
Add-Type -AssemblyName System.Net.Http -ErrorAction SilentlyContinue
|
||||
$client = [System.Net.Http.HttpClient]::new()
|
||||
$retries=5; $delay=2
|
||||
try {
|
||||
for($i=0;$i -lt $retries;$i++){
|
||||
try {
|
||||
$uri = BuildS3Uri -endpointUrl $endpointUrl -bucket $bucket -key $key -forcePathStyle $forcePathStyle
|
||||
$payloadHash = (Get-HashHex (Get-Bytes ''))
|
||||
$req = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Get, $uri)
|
||||
$hdrs = BuildAuthHeaders -method 'GET' -uri $uri -region $region -accessKey $ak -secretKey $sk -payloadHash $payloadHash
|
||||
$req.Headers.TryAddWithoutValidation('x-amz-date', $hdrs['x-amz-date']) | Out-Null
|
||||
$req.Headers.TryAddWithoutValidation('Authorization', $hdrs['Authorization']) | Out-Null
|
||||
$req.Headers.TryAddWithoutValidation('x-amz-content-sha256', $hdrs['x-amz-content-sha256']) | Out-Null
|
||||
$resp = $client.SendAsync($req).Result
|
||||
if (-not $resp.IsSuccessStatusCode) { throw "HTTP $([int]$resp.StatusCode) $($resp.ReasonPhrase)" }
|
||||
$totalBytes = $resp.Content.Headers.ContentLength
|
||||
$stream = $resp.Content.ReadAsStreamAsync().Result
|
||||
$fs = [System.IO.File]::Create($targetPath)
|
||||
try {
|
||||
$buffer = New-Object byte[] 8192
|
||||
$totalRead = 0
|
||||
while (($read = $stream.Read($buffer, 0, $buffer.Length)) -gt 0) {
|
||||
$fs.Write($buffer, 0, $read)
|
||||
$totalRead += $read
|
||||
if ($totalBytes) { $pct = ($totalRead * 100.0) / $totalBytes; Write-Progress -Activity "Downloading khdb.txt.zip" -Status ("{0:N2}% Complete" -f $pct) -PercentComplete $pct } else { Write-Progress -Activity "Downloading khdb.txt.zip" -Status ("Downloaded {0:N0} bytes" -f $totalRead) -PercentComplete 0 }
|
||||
}
|
||||
} finally { $fs.Close(); $stream.Close() }
|
||||
if ($resp) { $resp.Dispose() }
|
||||
return
|
||||
} catch {
|
||||
if ($i -lt ($retries - 1)) { Write-Warning "Download failed (attempt $($i+1)/$retries): $($_.Exception.Message). Retrying in ${delay}s..."; Start-Sleep -Seconds $delay; $delay=[Math]::Min($delay*2,30) }
|
||||
else { throw }
|
||||
} finally { if ($req){ $req.Dispose() } }
|
||||
}
|
||||
} finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Invoke-DownloadWithRetry([System.Net.Http.HttpClient]$client, [string]$uri, [string]$targetPath) {
|
||||
$retries = 5
|
||||
$delay = 2
|
||||
for ($i = 0; $i -lt $retries; $i++) {
|
||||
try {
|
||||
$resp = $client.GetAsync($uri, [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead).Result
|
||||
if (-not $resp.IsSuccessStatusCode) {
|
||||
$code = [int]$resp.StatusCode
|
||||
if (($code -ge 500 -and $code -lt 600) -or $code -eq 429 -or $code -eq 408) { throw "Transient HTTP error $code" }
|
||||
throw "HTTP error $code"
|
||||
}
|
||||
$totalBytes = $resp.Content.Headers.ContentLength
|
||||
$stream = $resp.Content.ReadAsStreamAsync().Result
|
||||
$fs = [System.IO.File]::Create($targetPath)
|
||||
try {
|
||||
$buffer = New-Object byte[] 8192
|
||||
$totalRead = 0
|
||||
while (($read = $stream.Read($buffer, 0, $buffer.Length)) -gt 0) {
|
||||
$fs.Write($buffer, 0, $read)
|
||||
$totalRead += $read
|
||||
if ($totalBytes) {
|
||||
$pct = ($totalRead * 100.0) / $totalBytes
|
||||
Write-Progress -Activity "Downloading khdb.txt.zip" -Status ("{0:N2}% Complete" -f $pct) -PercentComplete $pct
|
||||
} else {
|
||||
Write-Progress -Activity "Downloading khdb.txt.zip" -Status ("Downloaded {0:N0} bytes" -f $totalRead) -PercentComplete 0
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$fs.Close(); $stream.Close()
|
||||
}
|
||||
return
|
||||
} catch {
|
||||
if ($i -lt ($retries - 1)) {
|
||||
Write-Warning "Download failed (attempt $($i+1)/$retries): $($_.Exception.Message). Retrying in ${delay}s..."
|
||||
Start-Sleep -Seconds $delay
|
||||
$delay = [Math]::Min($delay * 2, 30)
|
||||
} else {
|
||||
throw
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Validate-KHDBFile([string]$path) {
|
||||
if (-not (Test-Path $path)) { throw "Validation failed: $path not found" }
|
||||
$lines = Get-Content -Path $path -Encoding UTF8
|
||||
if (-not $lines -or $lines.Count -eq 0) { throw 'Validation failed: file is empty.' }
|
||||
$regex = '^[0-9A-Fa-f]{32}$'
|
||||
$invalid = $lines | Where-Object { $_ -notmatch $regex }
|
||||
if ($invalid.Count -gt 0) {
|
||||
throw ("Validation failed: {0} invalid lines detected." -f $invalid.Count)
|
||||
}
|
||||
# Deduplicate and normalize line endings
|
||||
$unique = $lines | ForEach-Object { $_.Trim() } | Where-Object { $_ } | Sort-Object -Unique
|
||||
Set-Content -Path $path -Value $unique -Encoding ASCII
|
||||
}
|
||||
|
||||
function Update-KHDB {
|
||||
Start-UpdateTranscript -BasePath $scriptRoot
|
||||
try {
|
||||
$settings = Read-ElysiumSettings
|
||||
$installPath = Get-InstallationPath $settings
|
||||
if (-not (Test-Path $installPath)) { New-Item -Path $installPath -ItemType Directory -Force | Out-Null }
|
||||
|
||||
$storageProvider = $settings['StorageProvider']
|
||||
if ([string]::IsNullOrWhiteSpace($storageProvider)) { $storageProvider = 'Azure' }
|
||||
|
||||
$client = $null
|
||||
$s3Bucket = $settings['s3BucketName']
|
||||
$s3EndpointUrl = $settings['s3EndpointUrl']
|
||||
$s3Region = $settings['s3Region']
|
||||
$s3AK = $settings['s3AccessKeyId']
|
||||
$s3SK = $settings['s3SecretAccessKey']
|
||||
$s3Force = $settings['s3ForcePathStyle']
|
||||
$s3UseAwsTools = $settings['s3UseAwsTools']
|
||||
try { $s3Force = [System.Convert]::ToBoolean($s3Force) } catch { $s3Force = $true }
|
||||
try { $s3UseAwsTools = [System.Convert]::ToBoolean($s3UseAwsTools) } catch { $s3UseAwsTools = $false }
|
||||
|
||||
$tmpDir = New-Item -ItemType Directory -Path ([System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), "elysium-khdb-" + [System.Guid]::NewGuid())) -Force
|
||||
$zipPath = Join-Path -Path $tmpDir.FullName -ChildPath 'khdb.txt.zip'
|
||||
$extractDir = Join-Path -Path $tmpDir.FullName -ChildPath 'extract'
|
||||
New-Item -ItemType Directory -Path $extractDir -Force | Out-Null
|
||||
|
||||
if ($storageProvider -ieq 'S3') {
|
||||
if ([string]::IsNullOrWhiteSpace($s3Bucket)) { throw 's3BucketName is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($s3AK) -or [string]::IsNullOrWhiteSpace($s3SK)) { throw 's3AccessKeyId / s3SecretAccessKey missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($s3EndpointUrl)) { throw 's3EndpointUrl is required for S3-compatible storage.' }
|
||||
if ($s3UseAwsTools) {
|
||||
try {
|
||||
$s3Client = New-S3Client -EndpointUrl $s3EndpointUrl -Region $s3Region -AccessKeyId $s3AK -SecretAccessKey $s3SK -ForcePathStyle:$s3Force
|
||||
Write-Host "Downloading KHDB from S3-compatible storage (AWS Tools)..."
|
||||
# Use AWS SDK stream method into file for progress parity
|
||||
$req = New-Object Amazon.S3.Model.GetObjectRequest -Property @{ BucketName = $s3Bucket; Key = 'khdb.txt.zip' }
|
||||
$resp = $s3Client.GetObject($req)
|
||||
try { $resp.WriteResponseStreamToFile($zipPath, $true) } finally { $resp.Dispose() }
|
||||
} catch {
|
||||
Write-Warning "AWS Tools path failed or not available. Falling back to native HTTP (SigV4). Details: $($_.Exception.Message)"
|
||||
Write-Host "Downloading KHDB from S3-compatible storage..."
|
||||
Invoke-S3HttpDownloadWithRetry -endpointUrl $s3EndpointUrl -bucket $s3Bucket -key 'khdb.txt.zip' -targetPath $zipPath -region $s3Region -ak $s3AK -sk $s3SK -forcePathStyle:$s3Force
|
||||
}
|
||||
} else {
|
||||
Write-Host "Downloading KHDB from S3-compatible storage..."
|
||||
Invoke-S3HttpDownloadWithRetry -endpointUrl $s3EndpointUrl -bucket $s3Bucket -key 'khdb.txt.zip' -targetPath $zipPath -region $s3Region -ak $s3AK -sk $s3SK -forcePathStyle:$s3Force
|
||||
}
|
||||
} else {
|
||||
$storageAccountName = $settings['storageAccountName']
|
||||
$containerName = $settings['containerName']
|
||||
$sasToken = $settings['sasToken']
|
||||
$uri = Build-BlobUri -account $storageAccountName -container $containerName -sas $sasToken
|
||||
$client = New-HttpClient
|
||||
Write-Host "Downloading KHDB from Azure Blob Storage..."
|
||||
Invoke-DownloadWithRetry -client $client -uri $uri -targetPath $zipPath
|
||||
}
|
||||
Write-Host "Download completed. Extracting archive..."
|
||||
|
||||
Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force
|
||||
|
||||
$extractedKHDB = Get-ChildItem -Path $extractDir -Recurse -Filter 'khdb.txt' | Select-Object -First 1
|
||||
if (-not $extractedKHDB) { throw 'Extracted archive does not contain khdb.txt.' }
|
||||
|
||||
# Validate content
|
||||
Validate-KHDBFile -path $extractedKHDB.FullName
|
||||
|
||||
# Compute target path and backup
|
||||
$targetKHDB = Join-Path -Path $installPath -ChildPath 'khdb.txt'
|
||||
if (Test-Path $targetKHDB) {
|
||||
$ts = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$backupPath = Join-Path -Path $installPath -ChildPath ("khdb.txt.bak-$ts")
|
||||
Copy-Item -Path $targetKHDB -Destination $backupPath -Force
|
||||
Write-Host "Existing KHDB backed up to $backupPath"
|
||||
}
|
||||
|
||||
# Atomic-ish replace: move validated file into place
|
||||
Move-Item -Path $extractedKHDB.FullName -Destination $targetKHDB -Force
|
||||
Write-Host "KHDB updated at $targetKHDB"
|
||||
Write-Host "KHDB update completed successfully."
|
||||
} catch {
|
||||
Write-Error ("KHDB update failed: {0}" -f $_.Exception.Message)
|
||||
throw
|
||||
} finally {
|
||||
try { if ($tmpDir -and (Test-Path $tmpDir.FullName)) { Remove-Item -Path $tmpDir.FullName -Recurse -Force } } catch {}
|
||||
Stop-UpdateTranscript
|
||||
}
|
||||
}
|
||||
|
||||
# Execute the update function
|
||||
Update-KHDB
|
||||
Write-Host "Script execution completed."
|
||||
Reference in New Issue
Block a user