Compare commits
1 Commits
main
...
backup/ori
| Author | SHA1 | Date | |
|---|---|---|---|
| e7a01f52a2 |
7
.gitignore
vendored
7
.gitignore
vendored
@@ -2,8 +2,5 @@
|
||||
khdb.txt
|
||||
khdb.txt.zip
|
||||
ElysiumSettings.txt
|
||||
/Reports
|
||||
/khdb-shards
|
||||
khdb-manifest.json
|
||||
/elysium/.vscode
|
||||
/.vscode
|
||||
/ReportsElysium/khdb.csv
|
||||
Settings.ps1
|
||||
58
CHANGELOG.md
58
CHANGELOG.md
@@ -1,63 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-10-30
|
||||
|
||||
### Update-KHDB.ps1 v2.0.0
|
||||
Changed:
|
||||
- Replaced single-archive workflow with manifest-driven, two-hex shard downloads that verify SHA256/size before in-place updates.
|
||||
- Added incremental refresh logic, stale shard cleanup, and automatic rebuild of the merged `khdb.txt` for downstream scripts.
|
||||
- Hardened validation to stream-check merged output while preserving strict TLS, retry, and transcript behaviour.
|
||||
|
||||
### ElysiumSettings.txt.sample v1.3.0
|
||||
Added:
|
||||
- Documented `KhdbManifestPath`, `KhdbShardPrefix`, and `KhdbLocalShardDir` defaults for the shard-aware updater.
|
||||
|
||||
### README.md
|
||||
Changed:
|
||||
- Described the manifest/shard update flow so operators understand the incremental download model and automatic cleanup.
|
||||
|
||||
### Prepare-KHDBStorage.ps1 v1.0.0
|
||||
Added:
|
||||
- Helper script to split `khdb.txt` (or a directory/list of `.gz` HIBP slices) into two-hex shards, build the JSON manifest, and push the package to Azure Blob Storage or S3-compatible endpoints.
|
||||
- Validation step that tallies and quarantines malformed hashes before sharding, writing `invalid-hashes.txt` plus a console summary so bad data never reaches storage.
|
||||
- Optional `-ShowProgress` mode emitting periodic `Write-Progress` updates (interval configurable) so large ingests visibly tick forward.
|
||||
- Automatic reconstruction of HIBP NTLM hashes (file-prefix + suffix) so partially stored hashes still produce full 32-hex values in the shards, plus per-prefix deduplication that keeps the highest observed count.
|
||||
- `-ForcePlainText` switch to skip `.gz` expansions entirely and treat the source as pre-built hash lines (skipped entries are reported separately).
|
||||
- Emits a merged `khdb-clean.txt` alongside the shards for DSInternals or offline review, including SHA256 fingerprints for both manifest and clean output.
|
||||
- Automatic checkpoint/resume when `-ForcePlainText` is used (configurable via `-CheckpointPath`, disable with `-NoCheckpoint`) so large ingests can be paused and resumed without reprocessing prior shards.
|
||||
|
||||
## 2025-10-26
|
||||
|
||||
### Test-WeakADPasswords.ps1 v1.3.3
|
||||
Added:
|
||||
- Opt-in usage beacon that fires a single HTTP request (GET/POST/PUT) after settings load, suitable for pre-signed S3 URLs, and only includes script name, version, and a UTC timestamp (plus optional instance ID).
|
||||
- Instance identifier header/body support and configurable timeout so adopters can differentiate deployments without collecting user data.
|
||||
|
||||
### ElysiumSettings.txt.sample v1.2.0
|
||||
Added:
|
||||
- Documented `UsageBeacon*` keys (URL, method, instance ID, timeout) so telemetry stays disabled by default but easy to enable.
|
||||
|
||||
### README.md
|
||||
Added:
|
||||
- Usage beacon section explaining how to configure the lightweight tracking call and what metadata is transmitted.
|
||||
|
||||
## 2025-10-21
|
||||
|
||||
### Extract-NTHashes.ps1 v1.2.1
|
||||
Fixed:
|
||||
- Corrected SigV4 host header formatting so non-default ports serialize without parser errors.
|
||||
- Hardened hashing helpers to avoid `ComputeHash` overload ambiguity under Windows PowerShell.
|
||||
- Domain selection menu now respects the configured numeric order.
|
||||
|
||||
### Test-WeakADPasswords.ps1 v1.3.2
|
||||
Changed:
|
||||
- Switched to the sorted KHDB path when driving `Test-PasswordQuality`, eliminating full linear scans and avoiding malformed-line crashes on massive datasets.
|
||||
|
||||
### Test-WeakADPasswords.ps1 v1.3.1
|
||||
Fixed:
|
||||
- Domain picker now renders in numeric order from settings for predictable operator workflows.
|
||||
- UPN export now relies on structured weak-password results, so dictionary hit UPN lists are populated reliably.
|
||||
|
||||
## 2025-10-10
|
||||
|
||||
### Test-WeakADPasswords.ps1 v1.3.0
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
function Invoke-RestartWithExecutable {
|
||||
param(
|
||||
[string]$ExecutablePath,
|
||||
[hashtable]$BoundParameters,
|
||||
[object[]]$UnboundArguments
|
||||
)
|
||||
|
||||
if (-not $ExecutablePath) { return }
|
||||
if (-not $PSCommandPath) { return }
|
||||
|
||||
$argList = @('-NoLogo', '-NoProfile', '-File', $PSCommandPath)
|
||||
|
||||
if ($BoundParameters) {
|
||||
foreach ($entry in $BoundParameters.GetEnumerator()) {
|
||||
$key = "-$($entry.Key)"
|
||||
$value = $entry.Value
|
||||
if ($value -is [System.Management.Automation.SwitchParameter]) {
|
||||
if ($value.IsPresent) { $argList += $key }
|
||||
} else {
|
||||
$argList += $key
|
||||
$argList += $value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($UnboundArguments) {
|
||||
$argList += $UnboundArguments
|
||||
}
|
||||
|
||||
& $ExecutablePath @argList
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
|
||||
function Restart-WithPwshIfAvailable {
|
||||
param(
|
||||
[hashtable]$BoundParameters,
|
||||
[object[]]$UnboundArguments
|
||||
)
|
||||
|
||||
if ($PSVersionTable.PSVersion.Major -ge 7 -or $PSVersionTable.PSEdition -eq 'Core') { return }
|
||||
$pwsh = Get-Command -Name 'pwsh' -ErrorAction SilentlyContinue
|
||||
if (-not $pwsh) { return }
|
||||
Write-Host ("PowerShell 7 detected at '{0}'; relaunching script under pwsh..." -f $pwsh.Path)
|
||||
Invoke-RestartWithExecutable -ExecutablePath $pwsh.Path -BoundParameters $BoundParameters -UnboundArguments $UnboundArguments
|
||||
}
|
||||
|
||||
function Restart-WithWindowsPowerShellIfAvailable {
|
||||
param(
|
||||
[hashtable]$BoundParameters,
|
||||
[object[]]$UnboundArguments
|
||||
)
|
||||
|
||||
if ($PSVersionTable.PSEdition -eq 'Desktop') { return }
|
||||
$powershellCmd = Get-Command -Name 'powershell.exe' -ErrorAction SilentlyContinue
|
||||
$powershellPath = $null
|
||||
if ($powershellCmd) {
|
||||
$powershellPath = $powershellCmd.Path
|
||||
} else {
|
||||
$defaultPath = Join-Path -Path $env:SystemRoot -ChildPath 'System32\WindowsPowerShell\v1.0\powershell.exe'
|
||||
if (Test-Path -LiteralPath $defaultPath) {
|
||||
$powershellPath = $defaultPath
|
||||
}
|
||||
}
|
||||
if (-not $powershellPath) {
|
||||
Write-Warning 'Windows PowerShell (powershell.exe) was not found; continuing under current host.'
|
||||
return
|
||||
}
|
||||
Write-Host ("Windows PowerShell detected at '{0}'; relaunching script under powershell.exe..." -f $powershellPath)
|
||||
Invoke-RestartWithExecutable -ExecutablePath $powershellPath -BoundParameters $BoundParameters -UnboundArguments $UnboundArguments
|
||||
}
|
||||
49
Elysium.ps1
49
Elysium.ps1
@@ -7,7 +7,7 @@
|
||||
##################################################
|
||||
## Project: Elysium ##
|
||||
## File: Elysium.ps1 ##
|
||||
## Version: 1.3.0 ##
|
||||
## Version: 1.2.0 ##
|
||||
## Support: support@cqre.net ##
|
||||
##################################################
|
||||
|
||||
@@ -21,13 +21,13 @@ Elysium.ps1 offers a menu to perform various actions:
|
||||
2. Test Weak AD Passwords
|
||||
3. Extract and Send Current Hashes for KHDB Update
|
||||
4. Uninstall the tool
|
||||
5. Update Lithnet Password Protection store
|
||||
6. Exit
|
||||
5. Exit
|
||||
#>
|
||||
|
||||
# Safer defaults
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
# Define the path to the settings file
|
||||
$settingsFilePath = Join-Path -Path $PSScriptRoot -ChildPath "ElysiumSettings.txt"
|
||||
|
||||
@@ -69,20 +69,6 @@ function Start-OrchestratorTranscript {
|
||||
|
||||
function Stop-OrchestratorTranscript { try { Stop-Transcript | Out-Null } catch {} }
|
||||
|
||||
function Invoke-WindowsPowerShellScript {
|
||||
param([string]$ScriptPath)
|
||||
$powershellCmd = Get-Command -Name 'powershell.exe' -ErrorAction SilentlyContinue
|
||||
if (-not $powershellCmd) {
|
||||
throw "Windows PowerShell (powershell.exe) was not found. Install it or run the script from a Desktop edition session."
|
||||
}
|
||||
$args = @('-NoLogo', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', $ScriptPath)
|
||||
& $powershellCmd.Path @args
|
||||
$exitCode = $LASTEXITCODE
|
||||
if ($exitCode -ne 0) {
|
||||
throw ("Windows PowerShell script '{0}' exited with code {1}." -f $ScriptPath, $exitCode)
|
||||
}
|
||||
}
|
||||
|
||||
function Show-Menu {
|
||||
param (
|
||||
[string]$Title = 'Elysium Tool Main Menu'
|
||||
@@ -92,9 +78,8 @@ function Show-Menu {
|
||||
Write-Host "1: Download/Update Known-Hashes Database (KHDB)"
|
||||
Write-Host "2: Test Weak AD Passwords"
|
||||
Write-Host "3: Extract and Send Current Hashes for KHDB Update"
|
||||
Write-Host "4: Update Lithnet Password Protection Store"
|
||||
Write-Host "5: Uninstall"
|
||||
Write-Host "6: Exit"
|
||||
Write-Host "4: Uninstall"
|
||||
Write-Host "5: Exit"
|
||||
}
|
||||
|
||||
Start-OrchestratorTranscript -BasePath $PSScriptRoot
|
||||
@@ -109,41 +94,27 @@ do {
|
||||
}
|
||||
'2' {
|
||||
Write-Host "Testing Weak AD Passwords..."
|
||||
$testScript = Join-Path -Path $PSScriptRoot -ChildPath 'Test-WeakADPasswords.ps1'
|
||||
if ($PSVersionTable.PSEdition -eq 'Desktop') {
|
||||
& $testScript
|
||||
} else {
|
||||
Invoke-WindowsPowerShellScript -ScriptPath $testScript
|
||||
}
|
||||
& (Join-Path -Path $PSScriptRoot -ChildPath 'Test-WeakADPasswords.ps1')
|
||||
}
|
||||
'3' {
|
||||
Write-Host "Extracting and Sending Current Hashes..."
|
||||
$extractScript = Join-Path -Path $PSScriptRoot -ChildPath 'Extract-NTHashes.ps1'
|
||||
if ($PSVersionTable.PSEdition -eq 'Desktop') {
|
||||
& $extractScript
|
||||
} else {
|
||||
Invoke-WindowsPowerShellScript -ScriptPath $extractScript
|
||||
}
|
||||
& (Join-Path -Path $PSScriptRoot -ChildPath 'Extract-NTHashes.ps1')
|
||||
}
|
||||
'4' {
|
||||
Write-Host "Updating Lithnet Password Protection store..."
|
||||
& (Join-Path -Path $PSScriptRoot -ChildPath 'Update-LithnetStore.ps1')
|
||||
}
|
||||
'5' {
|
||||
Write-Host "Uninstalling..."
|
||||
& (Join-Path -Path $PSScriptRoot -ChildPath 'Uninstall.ps1')
|
||||
}
|
||||
'6' {
|
||||
'5' {
|
||||
Write-Host "Exiting..."
|
||||
# end loop; transcript will be stopped after the loop
|
||||
$userSelection = '6'
|
||||
$userSelection = '5'
|
||||
}
|
||||
default {
|
||||
Write-Host "Invalid selection, please try again."
|
||||
}
|
||||
}
|
||||
pause
|
||||
} while ($userSelection -ne '6')
|
||||
} while ($userSelection -ne '5')
|
||||
} finally {
|
||||
Stop-OrchestratorTranscript
|
||||
}
|
||||
|
||||
41
Elysium/Elysium.ps1
Normal file
41
Elysium/Elysium.ps1
Normal file
@@ -0,0 +1,41 @@
|
||||
#Global settings
|
||||
. "../Settings.ps1"
|
||||
function Show-Menu {
|
||||
param (
|
||||
[string]$Title = 'Project Elysium'
|
||||
)
|
||||
Clear-Host
|
||||
Write-Host "================ $Title ================"
|
||||
|
||||
Write-Host "1: Update Known Hashes Database"
|
||||
Write-Host "2: Run Weak Password Test"
|
||||
Write-Host "3: Extract and Send Current Hashes"
|
||||
Write-Host "Q: Exit"
|
||||
}
|
||||
|
||||
do {
|
||||
Show-Menu
|
||||
$input = Read-Host "Please make a selection"
|
||||
switch ($input) {
|
||||
'1' {
|
||||
# Call Script 1
|
||||
.\UpdateKHDB.ps1
|
||||
break
|
||||
}
|
||||
'2' {
|
||||
# Call Script 2
|
||||
.\TestADAccounts.ps1
|
||||
break
|
||||
}
|
||||
'3' {
|
||||
# Call Script 3
|
||||
.\ExportHashes.ps1
|
||||
break
|
||||
}
|
||||
'Q' {
|
||||
return
|
||||
}
|
||||
}
|
||||
pause
|
||||
}
|
||||
until ($input -eq 'Q')
|
||||
60
Elysium/ExportHashes.ps1
Normal file
60
Elysium/ExportHashes.ps1
Normal file
@@ -0,0 +1,60 @@
|
||||
#Global settings
|
||||
. "../Settings.ps1"
|
||||
|
||||
# Import Required Modules
|
||||
Import-Module DSInternals
|
||||
Add-Type -AssemblyName System.IO.Compression.FileSystem
|
||||
|
||||
# Define Domains and Associated Usernames
|
||||
$domains = @{
|
||||
"Domain1" = "username1";
|
||||
"Domain2" = "username2";
|
||||
# Add more domains and usernames as needed
|
||||
}
|
||||
|
||||
# Present Choice of Domains to User
|
||||
$selectedDomain = $domains.Keys | Out-GridView -Title "Select a Domain" -PassThru
|
||||
$selectedUsername = $domains[$selectedDomain]
|
||||
|
||||
# Ask User to Enter Password for Chosen Account
|
||||
Write-Host "Enter password for account $selectedUsername in domain $selectedDomain:"
|
||||
$password = Read-Host -AsSecureString
|
||||
|
||||
# Define Domain Controller (Modify as needed)
|
||||
$domainController = "$selectedDomain" + "Controller" # Example: Domain1Controller
|
||||
|
||||
# Credential Object
|
||||
$credential = New-Object System.Management.Automation.PSCredential ($selectedUsername, $password)
|
||||
|
||||
# Get Current Timestamp
|
||||
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
|
||||
|
||||
# Define Export Path and Filename
|
||||
$exportPath = "C:\Path\To\Export" # Configure this path as needed
|
||||
$exportFilename = "extractedHashes_" + $selectedDomain + "_" + $timestamp + ".csv"
|
||||
$exportFullPath = Join-Path $exportPath $exportFilename
|
||||
|
||||
# Extract Non-Disabled Account Hashes
|
||||
Get-ADReplAccount -All -Server $domainController -Credential $credential |
|
||||
Where-Object { -not $_.AccountDisabled } |
|
||||
Select-Object -Property SamAccountName, NTHash |
|
||||
Export-Csv -Path $exportFullPath -NoTypeInformation
|
||||
|
||||
# Ask User for a Secure Password for Encryption
|
||||
Write-Host "Enter a secure password to encrypt the file:"
|
||||
$encryptionPassword = Read-Host -AsSecureString
|
||||
|
||||
# Compress and Encrypt File
|
||||
$compressedFile = $exportFullPath + ".zip"
|
||||
[IO.Compression.ZipFile]::CreateFromDirectory($exportPath, $compressedFile)
|
||||
$encryptedFile = $compressedFile + ".encrypted"
|
||||
|
||||
# Encrypt the Compressed File
|
||||
ConvertFrom-SecureString $encryptionPassword | Out-File "$encryptedFile"
|
||||
|
||||
# Clean Up
|
||||
Remove-Item -Path $exportFullPath # Remove the original CSV file
|
||||
Remove-Item -Path $compressedFile # Remove the compressed ZIP file
|
||||
|
||||
# Output
|
||||
Write-Host "Hashes exported, compressed, and encrypted to: $encryptedFile"
|
||||
3
Elysium/TestADAccounts.ps1
Normal file
3
Elysium/TestADAccounts.ps1
Normal file
@@ -0,0 +1,3 @@
|
||||
#Global settings
|
||||
. "../Settings.ps1"
|
||||
|
||||
52
Elysium/UpdateKHDB.ps1
Normal file
52
Elysium/UpdateKHDB.ps1
Normal file
@@ -0,0 +1,52 @@
|
||||
#Global settings
|
||||
. "../Settings.ps1"
|
||||
|
||||
# Function to extract version number from filename
|
||||
function Extract-VersionNumber($filename) {
|
||||
if ($filename -match "known-hashes-v(\d+\.\d+)\.encrypted\.zip") {
|
||||
return $matches[1]
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
# Get the list of available files (assuming a directory listing is available)
|
||||
$response = Invoke-WebRequest -Uri $baseUrl
|
||||
$files = $response.Links | Where-Object { $_.href -like "known-hashes-v*.encrypted.zip" } | Select-Object -ExpandProperty href
|
||||
|
||||
# Determine the latest version
|
||||
$latestVersion = "0.0"
|
||||
$latestFile = $null
|
||||
foreach ($file in $files) {
|
||||
$version = Extract-VersionNumber $file
|
||||
if ([version]$version -gt [version]$latestVersion) {
|
||||
$latestVersion = $version
|
||||
$latestFile = $file
|
||||
}
|
||||
}
|
||||
|
||||
# Check local file version
|
||||
$localVersion = "0.0"
|
||||
if (Test-Path "$localFilePath.encrypted") {
|
||||
$localVersion = Extract-VersionNumber (Get-Item "$localFilePath.encrypted").Name
|
||||
}
|
||||
|
||||
# Download and extract if the online version is newer
|
||||
if ([version]$latestVersion -gt [version]$localVersion) {
|
||||
$downloadUrl = $baseUrl + $latestFile
|
||||
$localZipPath = "$localFilePath-v$latestVersion.encrypted.zip"
|
||||
Invoke-WebRequest -Uri $downloadUrl -OutFile $localZipPath
|
||||
|
||||
# Ask for the ZIP password
|
||||
Write-Host "Enter the password to unzip the file:"
|
||||
$zipPassword = Read-Host -AsSecureString
|
||||
|
||||
# Unzip the file (requires .NET 4.5 or higher and external tools like 7-Zip)
|
||||
$zipPasswordPlainText = [Runtime.InteropServices.Marshal]::PtrToStringBSTR([Runtime.InteropServices.Marshal]::SecureStringToBSTR($zipPassword))
|
||||
$7zipPath = "C:\Path\To\7Zip\7z.exe" # Update with the actual path to 7-Zip executable
|
||||
$arguments = "x `"$localZipPath`" -p$zipPasswordPlainText -o`"$localFilePath`" -y"
|
||||
Start-Process $7zipPath -ArgumentList $arguments -NoNewWindow -Wait
|
||||
|
||||
Write-Host "File downloaded and extracted successfully. Latest version: v$latestVersion"
|
||||
} else {
|
||||
Write-Host "Local known-hashes file is up-to-date. Current version: v$localVersion"
|
||||
}
|
||||
@@ -8,7 +8,7 @@
|
||||
##################################################
|
||||
## Project: Elysium ##
|
||||
## File: ElysiumSettings.txt ##
|
||||
## Version: 1.3.0 ##
|
||||
## Version: 1.1.0 ##
|
||||
## Support: support@cqre.net ##
|
||||
##################################################
|
||||
|
||||
@@ -36,14 +36,6 @@ s3SecretAccessKey =
|
||||
s3ForcePathStyle = true
|
||||
s3UseAwsTools = false
|
||||
|
||||
# KHDB Shard Settings
|
||||
#####################
|
||||
# The KHDB update script downloads a manifest plus per-prefix shards (default shard size 2).
|
||||
# These values control the remote object names and local storage directory.
|
||||
KhdbManifestPath=khdb/manifest.json
|
||||
KhdbShardPrefix=khdb/shards
|
||||
KhdbLocalShardDir=khdb-shards
|
||||
|
||||
# Application Settings
|
||||
######################
|
||||
InstallationPath=
|
||||
@@ -51,25 +43,6 @@ ReportPathBase=Reports
|
||||
WeakPasswordsDatabase=khdb.txt
|
||||
# CheckOnlyEnabledUsers=true
|
||||
|
||||
# Lithnet Password Protection Settings
|
||||
######################################
|
||||
LithnetStorePath=
|
||||
LithnetSyncHibp=false
|
||||
LithnetHashSources=khdb.txt
|
||||
LithnetPlaintextSources=
|
||||
LithnetBannedWordSources=
|
||||
|
||||
# Telemetry (optional)
|
||||
######################
|
||||
# These values are empty by default so no telemetry is sent.
|
||||
# Provide a pre-signed URL (for example, an S3 PUT) to receive a single beacon
|
||||
# when the weak-password test starts. Only script name, version, and timestamp
|
||||
# are transmitted; you can set UsageBeaconInstanceId to differentiate deployments.
|
||||
UsageBeaconUrl=
|
||||
UsageBeaconMethod=GET # GET, POST, or PUT
|
||||
UsageBeaconInstanceId=
|
||||
UsageBeaconTimeoutSeconds=5
|
||||
|
||||
# Notes:
|
||||
# - Required PowerShell modules: DSInternals, ActiveDirectory
|
||||
# For Azure uploads: Az.Storage
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
##################################################
|
||||
## Project: Elysium ##
|
||||
## File: Extract-NTLMHashes.ps1 ##
|
||||
## Version: 1.3.0 ##
|
||||
## Version: 1.2.0 ##
|
||||
## Support: support@cqre.net ##
|
||||
##################################################
|
||||
|
||||
@@ -22,7 +22,6 @@ This script will connect to selected domain (defined in ElysiumSettings.txt) usi
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
$scriptRoot = $PSScriptRoot
|
||||
|
||||
function Start-ExtractTranscript {
|
||||
@@ -123,14 +122,7 @@ function New-S3Client {
|
||||
function Get-Bytes([string]$s) { return [System.Text.Encoding]::UTF8.GetBytes($s) }
|
||||
function Get-HashHex([byte[]]$bytes) {
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
if ($null -eq $bytes) { $bytes = [byte[]]@() }
|
||||
$ms = [System.IO.MemoryStream]::new($bytes)
|
||||
try {
|
||||
$hashBytes = $sha.ComputeHash($ms)
|
||||
return ([BitConverter]::ToString($hashBytes)).Replace('-', '').ToLowerInvariant()
|
||||
} finally { $ms.Dispose() }
|
||||
} finally { $sha.Dispose() }
|
||||
try { return ([BitConverter]::ToString($sha.ComputeHash($bytes))).Replace('-', '').ToLowerInvariant() } finally { $sha.Dispose() }
|
||||
}
|
||||
function Get-FileSha256Hex([string]$path) {
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
@@ -139,11 +131,7 @@ function Get-FileSha256Hex([string]$path) {
|
||||
}
|
||||
function HmacSha256([byte[]]$key, [string]$data) {
|
||||
$h = [System.Security.Cryptography.HMACSHA256]::new($key)
|
||||
try {
|
||||
$dataBytes = Get-Bytes $data
|
||||
$ms = [System.IO.MemoryStream]::new($dataBytes)
|
||||
try { return $h.ComputeHash($ms) } finally { $ms.Dispose() }
|
||||
} finally { $h.Dispose() }
|
||||
try { return $h.ComputeHash((Get-Bytes $data)) } finally { $h.Dispose() }
|
||||
}
|
||||
function GetSignatureKey([string]$secret, [string]$dateStamp, [string]$regionName, [string]$serviceName) {
|
||||
$kDate = HmacSha256 (Get-Bytes ('AWS4' + $secret)) $dateStamp
|
||||
@@ -171,7 +159,7 @@ function BuildAuthHeaders($method, [System.Uri]$uri, [string]$region, [string]$a
|
||||
$amzdate = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssZ')
|
||||
$datestamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd')
|
||||
$hostHeader = $uri.Host
|
||||
if (-not $uri.IsDefaultPort) { $hostHeader = "{0}:{1}" -f $hostHeader, $uri.Port }
|
||||
if (-not $uri.IsDefaultPort) { $hostHeader = "$hostHeader:$($uri.Port)" }
|
||||
|
||||
$canonicalUri = BuildCanonicalPath $uri
|
||||
$canonicalQueryString = ''
|
||||
@@ -320,8 +308,8 @@ function Get-FileChecksum {
|
||||
$reportBase = Normalize-ReportPath -p $ElysiumSettings['ReportPathBase']
|
||||
if (-not (Test-Path $reportBase)) { New-Item -Path $reportBase -ItemType Directory -Force | Out-Null }
|
||||
|
||||
# Build domain details from settings (ordered to keep numeric index order)
|
||||
$DomainDetails = [ordered]@{}
|
||||
# Build domain details from settings
|
||||
$DomainDetails = @{}
|
||||
for ($i = 1; $ElysiumSettings.ContainsKey("Domain${i}Name"); $i++) {
|
||||
$DomainDetails["$i"] = @{
|
||||
Name = $ElysiumSettings["Domain${i}Name"]
|
||||
@@ -332,7 +320,7 @@ for ($i = 1; $ElysiumSettings.ContainsKey("Domain${i}Name"); $i++) {
|
||||
|
||||
# User selects a domain
|
||||
Write-Host "Select a domain to extract NTLM hashes:"
|
||||
$DomainDetails.GetEnumerator() | Sort-Object { [int]$_.Key } | ForEach-Object { Write-Host "$($_.Key): $($_.Value.Name)" }
|
||||
$DomainDetails.GetEnumerator() | ForEach-Object { Write-Host "$($_.Key): $($_.Value.Name)" }
|
||||
$selection = Read-Host "Enter the number of the domain"
|
||||
$selectedDomain = $DomainDetails[$selection]
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
70
README.md
70
README.md
@@ -12,72 +12,30 @@ Sensitive operations are confined only to the dedicated host. In the third step,
|
||||
## Prerequisities
|
||||
* **Windows Host:** A Windows machine with PowerShell and DSInternals suite installed.
|
||||
* **Administrative Access:** Local admin privileges on the host for installation and updating.
|
||||
* **Domain Credentials:** For weak-password testing (option 2), an account with the three replication rights (`Replicating Directory Changes`, `Replicating Directory Changes All`, `Replicating Directory Changes In Filtered Set`) on the domain naming context; Domain Admin also works but is not required. Keep this account disabled and enable only when running tests.
|
||||
* **Domain Credentials:** A domain user account with Domain Admin privileges for each tested AD domain. This account should be active only during testing.
|
||||
* **Network Requirements:** A stable connection to the domain controller in each tested AD domain and internet access (specific hostnames/IP addresses will be provided).
|
||||
---
|
||||
## Operation
|
||||
### Install and update
|
||||
This tool is provided in private git repository. Installation and updating is done with cloning and pulling from this repository.
|
||||
During first run, the tool will ask for passphrase that will be used to encrypt/decrypt sensitive content.
|
||||
After installation, edit ElysiumSettings.txt, check all variables and add domains to test.
|
||||
All scripts automatically relaunch under PowerShell 7 (`pwsh`) when it is installed so that features like parallel transfers are available; if pwsh is missing they continue under Windows PowerShell 5.1 with the legacy single-threaded behavior. The two DSInternals-driven workflows (menu options 2 and 3) load the legacy `ActiveDirectory` and `DSInternals` modules, so they automatically fall back to Windows PowerShell even if pwsh is present.
|
||||
Clone this private git repository to install or update the tool. During the first run, you will be prompted for a passphrase to encrypt/decrypt sensitive content. After installation, edit `ElysiumSettings.txt`, check all variables, and add domains to test.
|
||||
|
||||
### Update Known-Hashed Database (KHDB)
|
||||
Run script Elysium.ps1 as an administrator and choose option 1 (Update Known-Hashes Database).
|
||||
The updater now pulls a manifest plus individual hash shards (two-hex prefix layout) from the configured storage (Azure Blob or S3-compatible), verifies checksums, replaces only changed shards, and rebuilds `khdb.txt` for local use. Deleted shards listed in the manifest are removed automatically. When PowerShell 7 is available the downloader automatically fetches up to `-MaxParallelTransfers` shards in parallel (default `5`); on Windows PowerShell 5.1 it reverts to the original sequential behavior. Override the concurrency as needed when running the script directly (for example `.\Update-KHDB.ps1 -MaxParallelTransfers 8`).
|
||||
Run either `Elysium.ps1` or `Start.ps1` as an administrator and choose option 1 (Update Known-Hashes Database). The script will check for a newer version online and, if found, download and decompress it. If the KHDB content is encrypted, you will be prompted for the decryption password. The database is then updated from the configured storage (Azure Blob or S3-compatible).
|
||||
|
||||
To publish an updated shard set, run `Prepare-KHDBStorage.ps1` against your sorted `khdb.txt` (or point it at the directory/list of the Have I Been Pwned `.gz` slices). The helper reconstructs the full 32‑hex NTLM values (prefix + remainder), deduplicates per hash (keeping the largest count), splits by the first two hex characters, writes a manifest (`version`, `sha256`, `size`, entry counts), and can upload the resulting files directly to Azure Blob Storage (via SAS) or S3-compatible endpoints using SigV4. Invalid or malformed entries are omitted automatically, and a short report (aggregate counts + `invalid-hashes.txt`) is produced for review. Example:
|
||||
|
||||
```powershell
|
||||
.\Prepare-KHDBStorage.ps1 -SourcePath .\khdb.txt `
|
||||
-OutputRoot .\publish `
|
||||
-StorageProvider S3 `
|
||||
-S3EndpointUrl https://s3.example.com `
|
||||
-S3BucketName private-khdb `
|
||||
-S3AccessKeyId AKIA... `
|
||||
-S3SecretAccessKey ... `
|
||||
-ManifestRemotePath khdb/manifest.json `
|
||||
-ShardRemotePrefix khdb/shards
|
||||
```
|
||||
|
||||
Use `-SkipUpload` to stage files locally, or `-StorageProvider Azure` with `storageAccountName`/`containerName`/`sasToken` when targeting Azure Blob Storage. Add `-ShowProgress` (optionally tune `-ProgressUpdateInterval`) if you want a running `Write-Progress` indicator while the hashes are being split. Pass `-ForcePlainText` when your `khdb.txt` already contains complete hashes and you want `.gz` references treated as invalid instead of being expanded. When you only need to push an already prepared package, combine `-UploadOnly` with `-OutputRoot` pointing at the existing shard directory and choose the storage provider to perform an upload-only run. Missing storage values are pulled from `ElysiumSettings.txt` automatically (override the path with `-SettingsPath`) so you don’t have to retype S3/Azure credentials for every run. On PowerShell 7, `Prepare-KHDBStorage.ps1` can push shards concurrently by setting `-MaxParallelTransfers` (default `5`); Windows PowerShell 5.1 automatically falls back to serial uploads.
|
||||
|
||||
Every run also emits a cleaned, DSInternals-friendly `khdb-clean.txt` beside the shards so you can inspect or distribute the merged list before publishing.
|
||||
|
||||
When `-ForcePlainText` is specified the script automatically keeps a checkpoint (default: `<output>/khdb.checkpoint.json`) and resumes from it on the next run so massive inputs don’t restart from scratch. Use `-CheckpointPath` to relocate that file or `-NoCheckpoint` to disable the behavior entirely.
|
||||
### Test Weak AD passwords
|
||||
Run script Elysium.ps1 as an administrator and choose option 2 (Test Weak AD Passwords).
|
||||
The script will list domains in the same order as they appear in `ElysiumSettings.txt` and, after you pick one, prompt for the corresponding domain administrator password (the username is taken from the settings file).
|
||||
The tool connects to the selected Domain Controller and compares accounts against KHDB (respecting the optional `CheckOnlyEnabledUsers` flag if configured). A timestamped text report is saved under `Reports`, and accounts with dictionary hits are also exported to a dedicated UPN-only text file to support follow-up automation.
|
||||
The KHDB file is consumed via binary search as a sorted hash list (plain text lines like `HASH:count`); ensure the file you place at `khdb.txt` keeps that ordering and omits stray blank lines.
|
||||
Run either `Elysium.ps1` or `Start.ps1` as an administrator and choose option 2 (Test Weak AD Passwords). The script will ask for the domain to be tested and then for the domain administrator password. The DA username is already provided in the script for each domain. The tool connects to the Domain Controller and tests all enabled users in the domain against KHDB. A PDF report with findings is generated.
|
||||
|
||||
#### Least privileges for password-quality testing
|
||||
The DSInternals cmdlets (`Get-ADReplAccount`/`Test-PasswordQuality`) pull replicated password data, which requires DCSync-style rights. The account that runs option 2 does not have to be a Domain Admin if it has these permissions on the domain naming context:
|
||||
- `Replicating Directory Changes`
|
||||
- `Replicating Directory Changes All`
|
||||
- `Replicating Directory Changes In Filtered Set` (needed on 2008 R2+ to read password hashes)
|
||||
|
||||
To delegate, enable Advanced Features in ADUC, right-click the domain, choose *Delegate Control…*, pick the service account, select *Create a custom task*, apply to *This object and all descendant objects*, and tick the three replication permissions above. Keep this account disabled and only activate it for scheduled tests.
|
||||
|
||||
#### Optional usage beacon
|
||||
If you want to know the script was executed without collecting telemetry, set a pre-signed URL (for example, an S3 `PUT` URL) in `UsageBeaconUrl` inside `ElysiumSettings.txt`. When present, the weak-password script issues a single request as soon as it loads the settings. Only the script name, its version, a UTC timestamp, and the optional `UsageBeaconInstanceId` value are sent, and network failures never block the run. Choose the HTTP verb via `UsageBeaconMethod` (`GET`, `POST`, or `PUT`) and adjust the timeout with `UsageBeaconTimeoutSeconds` if your storage endpoint needs more time.
|
||||
|
||||
### Send current hashes for update KHDB
|
||||
Run script Elysium.ps1 as an administrator and choose option 3 (Extract and Send Hashes).
|
||||
Domains are listed in configuration order, after which the script prompts for the replication-capable account password. With valid credentials, it extracts current NTLM hashes (no history) for active accounts, compresses the results, encrypts them with the configured passphrase, and uploads the payload to the configured storage (Azure Blob or S3-compatible). A checksum-verified round-trip download confirms the upload before local artifacts are removed.
|
||||
|
||||
### Update Lithnet Password Protection store
|
||||
Run script Elysium.ps1 as an administrator and choose option 5 (Update Lithnet Password Protection Store).
|
||||
Configure the target folder via `LithnetStorePath` in `ElysiumSettings.txt` (the location created with `Open-Store`). The script automatically imports the `khdb.txt` file unless you override/add additional NTLM hash lists in `LithnetHashSources` (comma or semicolon separated). You can also populate plaintext password lists (`LithnetPlaintextSources`) and banned-word files (`LithnetBannedWordSources`), or enable `LithnetSyncHibp=true` to seed the store directly from the Have I Been Pwned API (using `Sync-HashesFromHibp`). Behind the scenes the helper loads the `LithnetPasswordProtection` module, opens the store, runs [`Import-CompromisedPasswordHashes`](https://docs.lithnet.io/password-protection/advanced-help/powershell-reference/import-compromisedpasswordhashes)/`Import-CompromisedPasswords`/`Import-BannedWords` for each configured file, and then closes the store.
|
||||
### Send current hashes for KHDB update
|
||||
Run either `Elysium.ps1` or `Start.ps1` as an administrator and choose option 3 (Extract and Send Hashes). The tool will ask for the domain and password of the domain administrator. With correct credentials, the tool extracts current hashes (no history) of non-disabled users, compresses and encrypts them, and uploads them to the configured storage (Azure Blob or S3-compatible) for pickup by the tool provider.
|
||||
|
||||
S3-compatible usage notes:
|
||||
- No AWS Tools required. The scripts sign requests using native SigV4 via .NET and HttpClient, including non-default endpoint ports.
|
||||
- No AWS Tools required. The scripts can sign requests using native SigV4 via .NET and HttpClient.
|
||||
- To force using AWS Tools instead, set `s3UseAwsTools = true` in `ElysiumSettings.txt` and install `AWS.Tools.S3`.
|
||||
### Uninstallation
|
||||
Run script Elysium.ps1 as an administrator and choose option 4 (Uninstall).
|
||||
The script will then delete everything and remove the passphrase variable.
|
||||
---
|
||||
## FAQ
|
||||
|
||||
### Uninstallation
|
||||
Run `Elysium.ps1` as an administrator and choose option 4 (Uninstall) to delete all files and remove the passphrase variable. Alternatively, you can manually remove the cloned repository.
|
||||
|
||||
## FAQ
|
||||
### What happens to the hashes we uploaded?
|
||||
These hashes are subjected to cracking. Any cracked hash is then added to KHDB. Hash cracking happens on dedicated air-gapped machine and all sensitive material is never decrypted outside this machine. Secure exchange of decryption keys is arranged beforehand with every client.
|
||||
### Do we need to upload the hashes?
|
||||
@@ -109,10 +67,10 @@ It should, as it is extremely sensitive operation that should never happen outsi
|
||||
---
|
||||
|
||||
## Weak password report
|
||||
This section explains in detail individual parts of weak password report.
|
||||
This section explains in detail individual parts of the weak password report.
|
||||
|
||||
1. Reversible Encryption:
|
||||
* ****Explanation:**** Accounts have passwords stored in a reversible format that can be decrypted.
|
||||
* **Explanation:** Accounts have passwords stored in a reversible format that can be decrypted.
|
||||
* **Risk Assessment:** High. Decrypted passwords can be misused easily.
|
||||
* **Possible Cause:** Legacy applications requiring plaintext password equivalents.
|
||||
* **Use:** Compatibility with older applications.
|
||||
|
||||
22
Settings.ps1
22
Settings.ps1
@@ -1,22 +0,0 @@
|
||||
# Settings for Elysium Tool
|
||||
|
||||
# General Settings
|
||||
$Global:ToolRepositoryUrl = "https://example.com/git/elysium.git"
|
||||
|
||||
# KHDB Update Settings
|
||||
$Global:KnownHashesBaseUrl = "https://example.com/known-hashes/"
|
||||
$Global:LocalKnownHashesPath = "C:\Elysium\known-hashes"
|
||||
|
||||
# Test Weak AD Passwords Settings
|
||||
$Global:DomainAdminUsernames = @{
|
||||
"Domain1" = "admin1";
|
||||
"Domain2" = "admin2";
|
||||
# Add more domains and usernames as needed
|
||||
}
|
||||
$Global:PdfReportPath = "C:\Elysium\Reports"
|
||||
|
||||
# Extract and Send Hashes Settings
|
||||
$Global:HashesExportPath = "C:\Elysium\Hashes"
|
||||
$Global:ToolProviderUploadUrl = "https://upload.example.com/hashes"
|
||||
|
||||
# Any additional settings...
|
||||
@@ -8,7 +8,7 @@
|
||||
##################################################
|
||||
## Project: Elysium ##
|
||||
## File: Test-WeakADPasswords.ps1 ##
|
||||
## Version: 1.4.0 ##
|
||||
## Version: 1.3.0 ##
|
||||
## Support: support@cqre.net ##
|
||||
##################################################
|
||||
|
||||
@@ -24,8 +24,7 @@ This script will test the passwords of selected domain (defined in ElysiumSettin
|
||||
# Enable verbose output
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
$VerbosePreference = "SilentlyContinue"
|
||||
$VerbosePreference = "Continue"
|
||||
|
||||
$scriptRoot = $PSScriptRoot
|
||||
|
||||
@@ -53,63 +52,6 @@ function Start-TestTranscript {
|
||||
|
||||
function Stop-TestTranscript { try { Stop-Transcript | Out-Null } catch {} }
|
||||
|
||||
function Invoke-UsageBeacon {
|
||||
param(
|
||||
[string]$Url,
|
||||
[string]$Method = 'GET',
|
||||
[int]$TimeoutSeconds = 5,
|
||||
[string]$InstanceId
|
||||
)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Url)) { return }
|
||||
|
||||
$normalizedMethod = 'GET'
|
||||
if (-not [string]::IsNullOrWhiteSpace($Method)) {
|
||||
$normalizedMethod = $Method.ToUpperInvariant()
|
||||
}
|
||||
if ($normalizedMethod -notin @('GET', 'POST', 'PUT')) {
|
||||
$normalizedMethod = 'GET'
|
||||
}
|
||||
|
||||
$requestParams = @{
|
||||
Uri = $Url
|
||||
Method = $normalizedMethod
|
||||
ErrorAction = 'Stop'
|
||||
}
|
||||
|
||||
$invokeWebRequestCmd = $null
|
||||
try { $invokeWebRequestCmd = Get-Command -Name Invoke-WebRequest -ErrorAction Stop } catch { }
|
||||
if ($invokeWebRequestCmd -and $invokeWebRequestCmd.Parameters.ContainsKey('UseBasicParsing')) {
|
||||
$requestParams['UseBasicParsing'] = $true
|
||||
}
|
||||
if ($TimeoutSeconds -gt 0 -and $invokeWebRequestCmd -and $invokeWebRequestCmd.Parameters.ContainsKey('TimeoutSec')) {
|
||||
$requestParams['TimeoutSec'] = $TimeoutSeconds
|
||||
}
|
||||
if (-not [string]::IsNullOrWhiteSpace($InstanceId)) {
|
||||
$requestParams['Headers'] = @{ 'X-Elysium-Instance' = $InstanceId }
|
||||
}
|
||||
|
||||
if ($normalizedMethod -in @('POST', 'PUT')) {
|
||||
$payload = [ordered]@{
|
||||
script = 'Test-WeakADPasswords'
|
||||
version = '1.3.3'
|
||||
ranAtUtc = (Get-Date).ToUniversalTime().ToString('o')
|
||||
}
|
||||
if (-not [string]::IsNullOrWhiteSpace($InstanceId)) {
|
||||
$payload['instanceId'] = $InstanceId
|
||||
}
|
||||
$requestParams['ContentType'] = 'application/json'
|
||||
$requestParams['Body'] = ($payload | ConvertTo-Json -Depth 3 -Compress)
|
||||
}
|
||||
|
||||
try {
|
||||
Invoke-WebRequest @requestParams | Out-Null
|
||||
Write-Verbose ("Usage beacon sent via {0}." -f $normalizedMethod)
|
||||
} catch {
|
||||
Write-Verbose ("Usage beacon failed: {0}" -f $_.Exception.Message)
|
||||
}
|
||||
}
|
||||
|
||||
# Current timestamp for both report generation and header
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
|
||||
@@ -151,37 +93,13 @@ try {
|
||||
exit
|
||||
}
|
||||
|
||||
$usageBeaconUrl = $ElysiumSettings['UsageBeaconUrl']
|
||||
$usageBeaconMethod = $ElysiumSettings['UsageBeaconMethod']
|
||||
$usageBeaconInstanceId = $ElysiumSettings['UsageBeaconInstanceId']
|
||||
$usageBeaconTimeoutSeconds = $null
|
||||
if ($ElysiumSettings.ContainsKey('UsageBeaconTimeoutSeconds')) {
|
||||
$parsedTimeout = 0
|
||||
if ([int]::TryParse($ElysiumSettings['UsageBeaconTimeoutSeconds'], [ref]$parsedTimeout)) {
|
||||
$usageBeaconTimeoutSeconds = $parsedTimeout
|
||||
}
|
||||
}
|
||||
if (-not [string]::IsNullOrWhiteSpace($usageBeaconUrl)) {
|
||||
$beaconParams = @{ Url = $usageBeaconUrl }
|
||||
if (-not [string]::IsNullOrWhiteSpace($usageBeaconMethod)) {
|
||||
$beaconParams['Method'] = $usageBeaconMethod
|
||||
}
|
||||
if (-not [string]::IsNullOrWhiteSpace($usageBeaconInstanceId)) {
|
||||
$beaconParams['InstanceId'] = $usageBeaconInstanceId
|
||||
}
|
||||
if ($null -ne $usageBeaconTimeoutSeconds) {
|
||||
$beaconParams['TimeoutSeconds'] = $usageBeaconTimeoutSeconds
|
||||
}
|
||||
Invoke-UsageBeacon @beaconParams
|
||||
}
|
||||
|
||||
# Define the function to extract domain details from settings
|
||||
function Get-DomainDetailsFromSettings {
|
||||
param (
|
||||
[hashtable]$Settings
|
||||
)
|
||||
|
||||
$domainDetails = [ordered]@{}
|
||||
$domainDetails = @{}
|
||||
$counter = 1
|
||||
while ($true) {
|
||||
$nameKey = "Domain${counter}Name"
|
||||
@@ -412,7 +330,7 @@ function Test-WeakADPasswords {
|
||||
|
||||
# User selects a domain
|
||||
Write-Host "Select a domain to test:"
|
||||
$DomainDetails.GetEnumerator() | Sort-Object { [int]$_.Key } | ForEach-Object { Write-Host "$($_.Key): $($_.Value.Name)" }
|
||||
$DomainDetails.GetEnumerator() | ForEach-Object { Write-Host "$($_.Key): $($_.Value.Name)" }
|
||||
$selection = Read-Host "Enter the number of the domain"
|
||||
|
||||
if (-not ($DomainDetails.ContainsKey($selection))) {
|
||||
@@ -437,7 +355,7 @@ function Test-WeakADPasswords {
|
||||
if ($_.PSObject.Properties.Name -contains 'Enabled') { $_.Enabled } else { $true }
|
||||
}
|
||||
}
|
||||
$testResults = $accounts | Test-PasswordQuality -WeakPasswordHashesSortedFile $FilePath
|
||||
$testResults = $accounts | Test-PasswordQuality -WeakPasswordHashesFile $FilePath
|
||||
Write-Verbose "Password quality test completed."
|
||||
} catch {
|
||||
Write-Error ("An error occurred while testing passwords: {0}" -f $_.Exception.Message)
|
||||
@@ -447,62 +365,40 @@ function Test-WeakADPasswords {
|
||||
# Report generation with dynamic content and UPNs
|
||||
$reportPath = Join-Path -Path $reportPathBase -ChildPath "$($selectedDomain.Name)_WeakPasswordReport_$timestamp.txt"
|
||||
$upnOnlyReportPath = Join-Path -Path $reportPathBase -ChildPath "$($selectedDomain.Name)_DictionaryPasswordUPNs_$timestamp.txt"
|
||||
|
||||
# Build a lookup of SAM account names to UPNs for dictionary hits by leveraging structured results
|
||||
$dictionaryLogonNames = @()
|
||||
foreach ($result in @($testResults)) {
|
||||
if ($null -ne $result -and $null -ne $result.WeakPassword) {
|
||||
$dictionaryLogonNames += $result.WeakPassword
|
||||
}
|
||||
}
|
||||
$dictionaryLogonNames = $dictionaryLogonNames | Sort-Object -Unique
|
||||
|
||||
$dictionarySamToUpn = @{}
|
||||
$upnReportContent = @()
|
||||
|
||||
foreach ($logonName in $dictionaryLogonNames) {
|
||||
$samAccountName = $logonName -replace '^.*\\', ''
|
||||
if (-not [string]::IsNullOrWhiteSpace($samAccountName) -and -not $dictionarySamToUpn.ContainsKey($samAccountName)) {
|
||||
Write-Verbose "Looking up UPN for $samAccountName (dictionary hit)"
|
||||
$upn = Get-UserUPN -SamAccountName $samAccountName -Domain $selectedDomain.DC -Credential $credential
|
||||
$dictionarySamToUpn[$samAccountName] = $upn
|
||||
if ($upn -ne "UPN not found") {
|
||||
$upnReportContent += $upn
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Write-Verbose "Generating report at $reportPath"
|
||||
$reportContent = @($header, ($testResults | Out-String).Trim(), $footer) -join "`r`n"
|
||||
|
||||
|
||||
$lines = $reportContent -split "`r`n"
|
||||
$newReportContent = @()
|
||||
$upnReportContent = @()
|
||||
|
||||
$collectingUPNs = $false
|
||||
|
||||
foreach ($line in $lines) {
|
||||
$newReportContent += $line
|
||||
|
||||
|
||||
# Start collecting UPNs after detecting the relevant section in the report
|
||||
if ($line -match "Passwords of these accounts have been found in the dictionary:") {
|
||||
$collectingUPNs = $true
|
||||
continue
|
||||
}
|
||||
|
||||
if ($collectingUPNs) {
|
||||
if ($line -match '^\s*$') { continue }
|
||||
if ($line -match '^\s*-{2,}') { continue }
|
||||
if ($line -match '^\s*(SamAccountName|LogonName)\b') { continue }
|
||||
if ($line -match '^[^\s].*:\s*$' -and $line -notmatch 'dictionary') {
|
||||
$collectingUPNs = $false
|
||||
continue
|
||||
}
|
||||
# Stop collecting UPNs if a new section starts or end of section is detected
|
||||
if ($collectingUPNs -and $line -match "^\s*$") {
|
||||
$collectingUPNs = $false
|
||||
}
|
||||
|
||||
$firstToken = ($line.Trim() -split '\s+')[0]
|
||||
if (-not [string]::IsNullOrWhiteSpace($firstToken)) {
|
||||
$samAccountName = $firstToken -replace '^.*\\', ''
|
||||
if ($dictionarySamToUpn.ContainsKey($samAccountName)) {
|
||||
$upnValue = $dictionarySamToUpn[$samAccountName]
|
||||
$newReportContent += " UPN: $upnValue"
|
||||
}
|
||||
# Regex to match the SAMAccountName from the report line and collect UPNs if in the target section
|
||||
if ($collectingUPNs -and $line -match "^\s*(\S+)\s*$") {
|
||||
$samAccountName = $matches[1]
|
||||
Write-Verbose "Looking up UPN for $samAccountName"
|
||||
$upn = Get-UserUPN -SamAccountName $samAccountName -Domain $selectedDomain.DC -Credential $credential
|
||||
$newReportContent += " UPN: $upn"
|
||||
|
||||
# Collect UPNs only for accounts found in the dictionary section
|
||||
if ($upn -ne "UPN not found") {
|
||||
$upnReportContent += $upn
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
##################################################
|
||||
## ____ ___ ____ _____ _ _ _____ _____ ##
|
||||
## / ___/ _ \| _ \| ____| | \ | | ____|_ _| ##
|
||||
## | | | | | | |_) | _| | \| | _| | | ##
|
||||
## | |__| |_| | _ <| |___ _| |\ | |___ | | ##
|
||||
## \____\__\_\_| \_\_____(_)_| \_|_____| |_| ##
|
||||
##################################################
|
||||
## Project: Elysium ##
|
||||
## File: Uninstall.ps1 ##
|
||||
## Version: 1.2.0 ##
|
||||
## Support: support@cqre.net ##
|
||||
##################################################
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Uninstall script for the Elysium AD password testing tool.
|
||||
|
||||
.DESCRIPTION
|
||||
This script will remove the Elysium tool and its components (scripts, configurations, and any generated data) from the system, and then delete itself.
|
||||
#>
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
[string]$commonHelper = Join-Path -Path $PSScriptRoot -ChildPath 'Elysium.Common.ps1'
|
||||
if (-not (Test-Path -LiteralPath $commonHelper)) { throw "Common helper not found at $commonHelper" }
|
||||
. $commonHelper
|
||||
Restart-WithPwshIfAvailable -BoundParameters $PSBoundParameters -UnboundArguments $MyInvocation.UnboundArguments
|
||||
|
||||
function Start-UninstallTranscript {
|
||||
try {
|
||||
$base = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), 'Elysium', 'logs')
|
||||
if (-not (Test-Path $base)) { New-Item -Path $base -ItemType Directory -Force | Out-Null }
|
||||
$ts = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$logPath = Join-Path -Path $base -ChildPath "uninstall-$ts.log"
|
||||
Start-Transcript -Path $logPath -Force | Out-Null
|
||||
} catch {
|
||||
Write-Warning "Could not start transcript: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
|
||||
function Stop-UninstallTranscript { try { Stop-Transcript | Out-Null } catch {} }
|
||||
|
||||
function Uninstall-Elysium {
|
||||
$ElysiumPath = Get-Location
|
||||
|
||||
Write-Host "Uninstalling Elysium tool from $ElysiumPath..."
|
||||
|
||||
# Check if the Elysium directory exists
|
||||
if (Test-Path $ElysiumPath) {
|
||||
# Schedule the script file for deletion
|
||||
$scriptPath = $MyInvocation.MyCommand.Path
|
||||
$deleteScript = { param($path) Remove-Item -Path $path -Force }
|
||||
Start-Sleep -Seconds 3 # Delay to ensure the script finishes
|
||||
Start-Process -FilePath "powershell.exe" -ArgumentList "-Command", $deleteScript, "-ArgumentList", $scriptPath -WindowStyle Hidden
|
||||
|
||||
# Remove the Elysium directory and all its contents
|
||||
Remove-Item -Path $ElysiumPath -Recurse -Force -Exclude $scriptPath
|
||||
Write-Host "Elysium tool and all related files have been removed, excluding this script. This script will be deleted shortly."
|
||||
} else {
|
||||
Write-Host "Elysium directory not found. It might have been removed already, or the path is incorrect."
|
||||
}
|
||||
|
||||
# Additional cleanup actions can be added here if needed
|
||||
}
|
||||
|
||||
Start-UninstallTranscript
|
||||
try {
|
||||
# Execute the uninstall function
|
||||
Uninstall-Elysium
|
||||
|
||||
# Check if the Elysium passphrase environment variable exists
|
||||
$passphraseEnvVar = [System.Environment]::GetEnvironmentVariable("ELYSIUM_PASSPHRASE", [System.EnvironmentVariableTarget]::User)
|
||||
|
||||
if ([string]::IsNullOrEmpty($passphraseEnvVar)) {
|
||||
Write-Host "No passphrase environment variable to remove."
|
||||
} else {
|
||||
# Remove the Elysium passphrase environment variable
|
||||
[System.Environment]::SetEnvironmentVariable("ELYSIUM_PASSPHRASE", $null, [System.EnvironmentVariableTarget]::User)
|
||||
Write-Host "Elysium passphrase environment variable has been removed."
|
||||
}
|
||||
|
||||
# Confirm uninstallation
|
||||
Write-Host "Elysium tool has been successfully uninstalled. Exiting script." -ForegroundColor Green
|
||||
} finally {
|
||||
Stop-UninstallTranscript
|
||||
}
|
||||
928
Update-KHDB.ps1
928
Update-KHDB.ps1
@@ -1,928 +0,0 @@
|
||||
##################################################
|
||||
## ____ ___ ____ _____ _ _ _____ _____ ##
|
||||
## / ___/ _ \| _ \| ____| | \ | | ____|_ _| ##
|
||||
## | | | | | | |_) | _| | \| | _| | | ##
|
||||
## | |__| |_| | _ <| |___ _| |\ | |___ | | ##
|
||||
## \____\__\_\_| \_\_____(_)_| \_|_____| |_| ##
|
||||
##################################################
|
||||
## Project: Elysium ##
|
||||
## File: Update-KHDB.ps1 ##
|
||||
## Version: 2.1.0 ##
|
||||
## Support: support@cqre.net ##
|
||||
##################################################
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Known-hashes database updater for the Elysium AD password testing tool.
|
||||
|
||||
.DESCRIPTION
|
||||
Downloads a sharded KHDB manifest, performs incremental shard updates, validates
|
||||
checksums, and atomically refreshes the merged khdb.txt for downstream scripts.
|
||||
Supports Azure Blob Storage (via SAS) and S3-compatible endpoints (SigV4).
|
||||
#>
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
[string]$commonHelper = Join-Path -Path $PSScriptRoot -ChildPath 'Elysium.Common.ps1'
|
||||
if (-not (Test-Path -LiteralPath $commonHelper)) { throw "Common helper not found at $commonHelper" }
|
||||
. $commonHelper
|
||||
Restart-WithPwshIfAvailable -BoundParameters $PSBoundParameters -UnboundArguments $MyInvocation.UnboundArguments
|
||||
|
||||
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor [System.Net.SecurityProtocolType]::Tls12
|
||||
|
||||
$scriptRoot = $PSScriptRoot
|
||||
|
||||
function Start-UpdateTranscript {
|
||||
param([string]$BasePath)
|
||||
try {
|
||||
$logsDir = Join-Path -Path $BasePath -ChildPath 'Reports/logs'
|
||||
if (-not (Test-Path $logsDir)) { New-Item -Path $logsDir -ItemType Directory -Force | Out-Null }
|
||||
$ts = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$logPath = Join-Path -Path $logsDir -ChildPath "update-khdb-$ts.log"
|
||||
Start-Transcript -Path $logPath -Force | Out-Null
|
||||
} catch {
|
||||
Write-Warning "Could not start transcript: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
|
||||
function Stop-UpdateTranscript {
|
||||
try { Stop-Transcript | Out-Null } catch {}
|
||||
}
|
||||
|
||||
function Read-ElysiumSettings {
|
||||
$settings = @{}
|
||||
$settingsPath = Join-Path -Path $scriptRoot -ChildPath 'ElysiumSettings.txt'
|
||||
if (-not (Test-Path $settingsPath)) { throw "Settings file not found at $settingsPath" }
|
||||
Get-Content $settingsPath | ForEach-Object {
|
||||
if ($_ -and -not $_.Trim().StartsWith('#')) {
|
||||
$kv = $_ -split '=', 2
|
||||
if ($kv.Count -eq 2) {
|
||||
$settings[$kv[0].Trim()] = $kv[1].Trim().Trim("'")
|
||||
}
|
||||
}
|
||||
}
|
||||
return $settings
|
||||
}
|
||||
|
||||
function Get-InstallationPath([hashtable]$settings) {
|
||||
$p = $settings['InstallationPath']
|
||||
if ([string]::IsNullOrWhiteSpace($p)) { return $scriptRoot }
|
||||
if ([System.IO.Path]::IsPathRooted($p)) { return $p }
|
||||
return (Join-Path -Path $scriptRoot -ChildPath $p)
|
||||
}
|
||||
|
||||
function New-HttpClient {
|
||||
Add-Type -AssemblyName System.Net.Http -ErrorAction SilentlyContinue
|
||||
$client = [System.Net.Http.HttpClient]::new()
|
||||
$client.Timeout = [TimeSpan]::FromSeconds(600)
|
||||
$client.DefaultRequestHeaders.UserAgent.ParseAdd('Elysium/2.0 (+Update-KHDB)')
|
||||
return $client
|
||||
}
|
||||
|
||||
function Build-BlobUri {
|
||||
param(
|
||||
[string]$Account,
|
||||
[string]$Container,
|
||||
[string]$Sas,
|
||||
[string]$BlobName
|
||||
)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Account)) { throw 'storageAccountName is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($Container)) { throw 'containerName is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($Sas)) { throw 'sasToken is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($BlobName)) { throw 'BlobName cannot be empty.' }
|
||||
|
||||
$sas = $Sas.Trim()
|
||||
if (-not $sas.StartsWith('?')) { $sas = '?' + $sas }
|
||||
$normalizedBlob = $BlobName.Replace('\', '/').TrimStart('/')
|
||||
$uriBuilder = [System.UriBuilder]::new("https://$Account.blob.core.windows.net/$Container/$normalizedBlob")
|
||||
$uriBuilder.Query = $sas.TrimStart('?')
|
||||
return $uriBuilder.Uri.AbsoluteUri
|
||||
}
|
||||
|
||||
function Ensure-AWSS3Module {
|
||||
try { $null = [Amazon.S3.AmazonS3Client]; return } catch {}
|
||||
try { Import-Module -Name AWS.Tools.S3 -ErrorAction Stop; return } catch {}
|
||||
try { Import-Module -Name AWSPowerShell.NetCore -ErrorAction Stop; return } catch {}
|
||||
throw "AWS Tools for PowerShell not found. Install with: Install-Module AWS.Tools.S3 -Scope CurrentUser"
|
||||
}
|
||||
|
||||
function Get-FunctionDefinitionText {
|
||||
param([Parameter(Mandatory = $true)][string]$Name)
|
||||
$cmd = Get-Command -Name $Name -CommandType Function -ErrorAction Stop
|
||||
return $cmd.ScriptBlock.Ast.Extent.Text
|
||||
}
|
||||
|
||||
function New-S3Client {
|
||||
param(
|
||||
[string]$EndpointUrl,
|
||||
[string]$Region,
|
||||
[string]$AccessKeyId,
|
||||
[string]$SecretAccessKey,
|
||||
[bool]$ForcePathStyle = $true
|
||||
)
|
||||
|
||||
Ensure-AWSS3Module
|
||||
$creds = New-Object Amazon.Runtime.BasicAWSCredentials($AccessKeyId, $SecretAccessKey)
|
||||
$cfg = New-Object Amazon.S3.AmazonS3Config
|
||||
if ($EndpointUrl) { $cfg.ServiceURL = $EndpointUrl }
|
||||
if ($Region) { try { $cfg.RegionEndpoint = [Amazon.RegionEndpoint]::GetBySystemName($Region) } catch {} }
|
||||
$cfg.ForcePathStyle = [bool]$ForcePathStyle
|
||||
return (New-Object Amazon.S3.AmazonS3Client($creds, $cfg))
|
||||
}
|
||||
|
||||
function Get-Bytes([string]$s) { return [System.Text.Encoding]::UTF8.GetBytes($s) }
|
||||
function Get-HashHex([byte[]]$bytes) {
|
||||
if ($null -eq $bytes) { $bytes = [byte[]]@() }
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
$ms = New-Object System.IO.MemoryStream -ArgumentList (,$bytes)
|
||||
try {
|
||||
$hash = $sha.ComputeHash([System.IO.Stream]$ms)
|
||||
} finally { $ms.Dispose() }
|
||||
return ([BitConverter]::ToString($hash)).Replace('-', '').ToLowerInvariant()
|
||||
} finally { $sha.Dispose() }
|
||||
}
|
||||
function HmacSha256([byte[]]$key, [string]$data) {
|
||||
$h = [System.Security.Cryptography.HMACSHA256]::new($key)
|
||||
try {
|
||||
$b = [System.Text.Encoding]::UTF8.GetBytes($data)
|
||||
$ms = New-Object System.IO.MemoryStream -ArgumentList (,$b)
|
||||
try {
|
||||
return $h.ComputeHash([System.IO.Stream]$ms)
|
||||
} finally { $ms.Dispose() }
|
||||
} finally { $h.Dispose() }
|
||||
}
|
||||
function GetSignatureKey([string]$secret, [string]$dateStamp, [string]$regionName, [string]$serviceName) {
|
||||
$kDate = HmacSha256 (Get-Bytes ('AWS4' + $secret)) $dateStamp
|
||||
$kRegion = HmacSha256 $kDate $regionName
|
||||
$kService = HmacSha256 $kRegion $serviceName
|
||||
HmacSha256 $kService 'aws4_request'
|
||||
}
|
||||
function UriEncode([string]$data, [bool]$encodeSlash) {
|
||||
$enc = [System.Uri]::EscapeDataString($data)
|
||||
if (-not $encodeSlash) { $enc = $enc -replace '%2F', '/' }
|
||||
return $enc
|
||||
}
|
||||
function BuildCanonicalPath([System.Uri]$uri) {
|
||||
$segments = $uri.AbsolutePath.Split('/')
|
||||
$encoded = @()
|
||||
foreach ($s in $segments) { $encoded += (UriEncode $s $false) }
|
||||
$path = ($encoded -join '/')
|
||||
if (-not $path.StartsWith('/')) { $path = '/' + $path }
|
||||
return $path
|
||||
}
|
||||
function ToHex([byte[]]$b) { ([BitConverter]::ToString($b)).Replace('-', '').ToLowerInvariant() }
|
||||
function BuildAuthHeaders($method, [System.Uri]$uri, [string]$region, [string]$accessKey, [string]$secretKey, [string]$payloadHash) {
|
||||
$algorithm = 'AWS4-HMAC-SHA256'
|
||||
$timestamp = (Get-Date).ToUniversalTime()
|
||||
$amzDate = $timestamp.ToString('yyyyMMddTHHmmssZ')
|
||||
$dateStamp = $timestamp.ToString('yyyyMMdd')
|
||||
$hostHeader = $uri.Host
|
||||
if (-not $uri.IsDefaultPort) { $hostHeader = "${hostHeader}:$($uri.Port)" }
|
||||
$canonicalUri = BuildCanonicalPath $uri
|
||||
$canonicalQueryString = ''
|
||||
$canonicalHeaders = "host:$hostHeader`n" + "x-amz-content-sha256:$payloadHash`n" + "x-amz-date:$amzDate`n"
|
||||
$signedHeaders = 'host;x-amz-content-sha256;x-amz-date'
|
||||
$canonicalRequest = "$method`n$canonicalUri`n$canonicalQueryString`n$canonicalHeaders`n$signedHeaders`n$payloadHash"
|
||||
$credentialScope = "$dateStamp/$region/s3/aws4_request"
|
||||
$stringToSign = "$algorithm`n$amzDate`n$credentialScope`n$((Get-HashHex (Get-Bytes $canonicalRequest)))"
|
||||
$signingKey = GetSignatureKey $secretKey $dateStamp $region 's3'
|
||||
$signature = ToHex (HmacSha256 $signingKey $stringToSign)
|
||||
$authHeader = "$algorithm Credential=$accessKey/$credentialScope, SignedHeaders=$signedHeaders, Signature=$signature"
|
||||
@{
|
||||
'x-amz-date' = $amzDate
|
||||
'x-amz-content-sha256' = $payloadHash
|
||||
'Authorization' = $authHeader
|
||||
}
|
||||
}
|
||||
function BuildS3Uri([string]$endpointUrl, [string]$bucket, [string]$key, [bool]$forcePathStyle) {
|
||||
$base = [System.Uri]$endpointUrl
|
||||
$builder = [System.UriBuilder]::new($base)
|
||||
$normalizedKey = $key.Replace('\', '/').TrimStart('/')
|
||||
if ($forcePathStyle) {
|
||||
$path = $builder.Path.TrimEnd('/')
|
||||
if ([string]::IsNullOrEmpty($path)) { $path = '/' }
|
||||
$builder.Path = ($path.TrimEnd('/') + '/' + $bucket + '/' + $normalizedKey)
|
||||
} else {
|
||||
$builder.Host = "$bucket." + $builder.Host
|
||||
$path = $builder.Path.TrimEnd('/')
|
||||
if ([string]::IsNullOrEmpty($path)) { $path = '/' }
|
||||
$builder.Path = ($path.TrimEnd('/') + '/' + $normalizedKey)
|
||||
}
|
||||
return $builder.Uri
|
||||
}
|
||||
|
||||
function Invoke-S3HttpDownloadWithRetry {
|
||||
param(
|
||||
[string]$EndpointUrl,
|
||||
[string]$Bucket,
|
||||
[string]$Key,
|
||||
[string]$TargetPath,
|
||||
[string]$Region,
|
||||
[string]$AccessKeyId,
|
||||
[string]$SecretAccessKey,
|
||||
[bool]$ForcePathStyle,
|
||||
[string]$Activity
|
||||
)
|
||||
|
||||
Add-Type -AssemblyName System.Net.Http -ErrorAction SilentlyContinue
|
||||
[System.Net.Http.HttpClient]$client = [System.Net.Http.HttpClient]::new()
|
||||
$retries = 5
|
||||
$delay = 2
|
||||
try {
|
||||
for ($attempt = 0; $attempt -lt $retries; $attempt++) {
|
||||
$request = $null
|
||||
try {
|
||||
$uri = BuildS3Uri -endpointUrl $EndpointUrl -bucket $Bucket -key $Key -forcePathStyle $ForcePathStyle
|
||||
$payloadHash = (Get-HashHex (Get-Bytes ''))
|
||||
$headers = BuildAuthHeaders -method 'GET' -uri $uri -region $Region -accessKey $AccessKeyId -secretKey $SecretAccessKey -payloadHash $payloadHash
|
||||
$request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Get, $uri)
|
||||
foreach ($kvp in $headers.GetEnumerator()) {
|
||||
$request.Headers.TryAddWithoutValidation($kvp.Key, $kvp.Value) | Out-Null
|
||||
}
|
||||
|
||||
$response = $client.SendAsync($request, [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead).GetAwaiter().GetResult()
|
||||
$null = $response.EnsureSuccessStatusCode()
|
||||
|
||||
$totalBytes = $response.Content.Headers.ContentLength
|
||||
$stream = $response.Content.ReadAsStreamAsync().Result
|
||||
$tmpPath = $TargetPath
|
||||
$fs = [System.IO.File]::Create($tmpPath)
|
||||
try {
|
||||
$buffer = New-Object byte[] 8192
|
||||
$totalRead = 0
|
||||
while (($read = $stream.Read($buffer, 0, $buffer.Length)) -gt 0) {
|
||||
$fs.Write($buffer, 0, $read)
|
||||
$totalRead += $read
|
||||
if ($totalBytes) {
|
||||
$pct = ($totalRead * 100.0) / $totalBytes
|
||||
Write-Progress -Activity $Activity -Status ("{0:N2}% Complete" -f $pct) -PercentComplete $pct
|
||||
} else {
|
||||
Write-Progress -Activity $Activity -Status ("Downloaded {0:N0} bytes" -f $totalRead) -PercentComplete 0
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$fs.Close()
|
||||
$stream.Close()
|
||||
}
|
||||
|
||||
if ($response) { $response.Dispose() }
|
||||
Write-Progress -Activity $Activity -Completed -Status 'Completed'
|
||||
return
|
||||
} catch {
|
||||
if ($attempt -lt ($retries - 1)) {
|
||||
Write-Warning "Download of '$Key' failed (attempt $($attempt + 1)/$retries): $($_.Exception.Message). Retrying in ${delay}s..."
|
||||
Start-Sleep -Seconds $delay
|
||||
$delay = [Math]::Min($delay * 2, 30)
|
||||
} else {
|
||||
throw
|
||||
}
|
||||
} finally {
|
||||
if ($request) { $request.Dispose() }
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$client.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-DownloadWithRetry {
|
||||
param(
|
||||
[System.Net.Http.HttpClient]$Client,
|
||||
[string]$Uri,
|
||||
[string]$TargetPath,
|
||||
[string]$Activity
|
||||
)
|
||||
|
||||
$retries = 5
|
||||
$delay = 2
|
||||
for ($attempt = 0; $attempt -lt $retries; $attempt++) {
|
||||
try {
|
||||
$response = $Client.GetAsync($Uri, [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead).Result
|
||||
if (-not $response.IsSuccessStatusCode) {
|
||||
$code = [int]$response.StatusCode
|
||||
if (($code -ge 500 -and $code -lt 600) -or $code -eq 429 -or $code -eq 408) { throw "Transient HTTP error $code" }
|
||||
throw "HTTP error $code"
|
||||
}
|
||||
|
||||
$totalBytes = $response.Content.Headers.ContentLength
|
||||
$stream = $response.Content.ReadAsStreamAsync().Result
|
||||
$fs = [System.IO.File]::Create($TargetPath)
|
||||
try {
|
||||
$buffer = New-Object byte[] 8192
|
||||
$totalRead = 0
|
||||
while (($read = $stream.Read($buffer, 0, $buffer.Length)) -gt 0) {
|
||||
$fs.Write($buffer, 0, $read)
|
||||
$totalRead += $read
|
||||
if ($totalBytes) {
|
||||
$pct = ($totalRead * 100.0) / $totalBytes
|
||||
Write-Progress -Activity $Activity -Status ("{0:N2}% Complete" -f $pct) -PercentComplete $pct
|
||||
} else {
|
||||
Write-Progress -Activity $Activity -Status ("Downloaded {0:N0} bytes" -f $totalRead) -PercentComplete 0
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$fs.Close()
|
||||
$stream.Close()
|
||||
}
|
||||
|
||||
Write-Progress -Activity $Activity -Completed -Status 'Completed'
|
||||
return
|
||||
} catch {
|
||||
if ($attempt -lt ($retries - 1)) {
|
||||
Write-Warning "Download of '$Uri' failed (attempt $($attempt + 1)/$retries): $($_.Exception.Message). Retrying in ${delay}s..."
|
||||
Start-Sleep -Seconds $delay
|
||||
$delay = [Math]::Min($delay * 2, 30)
|
||||
} else {
|
||||
throw
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-FileSha256Lower {
|
||||
param([string]$Path)
|
||||
if (-not (Test-Path -LiteralPath $Path)) { throw "File not found: $Path" }
|
||||
return (Get-FileHash -Path $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Ensure-Directory {
|
||||
param([string]$Path)
|
||||
if ([string]::IsNullOrWhiteSpace($Path)) { return }
|
||||
if (-not (Test-Path -LiteralPath $Path)) {
|
||||
New-Item -Path $Path -ItemType Directory -Force | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function Get-BooleanSetting {
|
||||
param(
|
||||
[string]$Value,
|
||||
[bool]$Default = $false
|
||||
)
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Value)) { return $Default }
|
||||
$parsed = $Default
|
||||
if ([System.Boolean]::TryParse($Value, [ref]$parsed)) { return $parsed }
|
||||
return $Default
|
||||
}
|
||||
|
||||
function Get-RelativePath {
|
||||
param(
|
||||
[string]$BasePath,
|
||||
[string]$FullPath
|
||||
)
|
||||
|
||||
$base = (Resolve-Path -LiteralPath $BasePath).ProviderPath
|
||||
$full = (Resolve-Path -LiteralPath $FullPath).ProviderPath
|
||||
if (-not $base.EndsWith([System.IO.Path]::DirectorySeparatorChar)) {
|
||||
$base = $base + [System.IO.Path]::DirectorySeparatorChar
|
||||
}
|
||||
|
||||
$baseUri = New-Object System.Uri($base, [System.UriKind]::Absolute)
|
||||
$fullUri = New-Object System.Uri($full, [System.UriKind]::Absolute)
|
||||
$relativeUri = $baseUri.MakeRelativeUri($fullUri)
|
||||
$relativePath = [System.Uri]::UnescapeDataString($relativeUri.ToString())
|
||||
return $relativePath.Replace('/', [System.IO.Path]::DirectorySeparatorChar)
|
||||
}
|
||||
|
||||
function Combine-StoragePath {
|
||||
param(
|
||||
[string]$Prefix,
|
||||
[string]$Name
|
||||
)
|
||||
|
||||
$cleanName = $Name.Replace('\', '/').TrimStart('/')
|
||||
if ([string]::IsNullOrWhiteSpace($Prefix)) { return $cleanName }
|
||||
$normalizedPrefix = $Prefix.Replace('\', '/').Trim('/')
|
||||
if ([string]::IsNullOrEmpty($normalizedPrefix)) { return $cleanName }
|
||||
return "$normalizedPrefix/$cleanName"
|
||||
}
|
||||
|
||||
function Load-Manifest {
|
||||
param([string]$Path)
|
||||
$raw = Get-Content -LiteralPath $Path -Encoding UTF8 -Raw
|
||||
return $raw | ConvertFrom-Json
|
||||
}
|
||||
|
||||
function Validate-Manifest {
|
||||
param([psobject]$Manifest)
|
||||
|
||||
if (-not $Manifest) { throw 'Manifest is empty or invalid JSON.' }
|
||||
if (-not $Manifest.shards) { throw 'Manifest does not contain a shards collection.' }
|
||||
|
||||
$seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase)
|
||||
foreach ($entry in $Manifest.shards) {
|
||||
if (-not $entry) { throw 'Manifest contains null shard entries.' }
|
||||
$name = [string]$entry.name
|
||||
if ([string]::IsNullOrWhiteSpace($name)) { throw 'Manifest shard entry is missing name.' }
|
||||
$hash = [string]$entry.sha256
|
||||
if ([string]::IsNullOrWhiteSpace($hash) -or $hash.Length -ne 64) { throw "Manifest shard '$name' is missing a valid sha256." }
|
||||
$sizeValue = [string]$entry.size
|
||||
$sizeParsed = 0L
|
||||
if (-not [long]::TryParse($sizeValue, [ref]$sizeParsed) -or $sizeParsed -lt 0) {
|
||||
throw "Manifest shard '$name' has invalid size."
|
||||
}
|
||||
if (-not $seen.Add($name)) { throw "Manifest contains duplicate shard name '$name'." }
|
||||
}
|
||||
|
||||
if ($Manifest.shardSize -and [int]$Manifest.shardSize -ne 2) {
|
||||
throw "Manifest shardSize $($Manifest.shardSize) is not supported. Expected shardSize 2."
|
||||
}
|
||||
}
|
||||
|
||||
function Merge-ShardsToFile {
|
||||
param(
|
||||
[psobject]$Manifest,
|
||||
[string]$ShardsRoot,
|
||||
[string]$TargetPath
|
||||
)
|
||||
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
$writer = New-Object System.IO.StreamWriter($TargetPath, $false, $encoding)
|
||||
try {
|
||||
foreach ($entry in ($Manifest.shards | Sort-Object name)) {
|
||||
$relative = [string]$entry.name
|
||||
$shardPath = Join-Path -Path $ShardsRoot -ChildPath $relative
|
||||
if (-not (Test-Path -LiteralPath $shardPath)) {
|
||||
throw "Missing shard on disk: $relative"
|
||||
}
|
||||
$reader = New-Object System.IO.StreamReader($shardPath, [System.Text.Encoding]::UTF8, $true)
|
||||
try {
|
||||
while (($line = $reader.ReadLine()) -ne $null) {
|
||||
$trimmed = $line.Trim()
|
||||
if ($trimmed.Length -gt 0) {
|
||||
$writer.WriteLine($trimmed)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$reader.Dispose()
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$writer.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Validate-KHDBFile {
|
||||
param([string]$Path)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $Path)) { throw "Validation failed: $Path not found." }
|
||||
|
||||
$regex = '^[0-9A-Fa-f]{32}(:\d+)?$'
|
||||
$lineNumber = 0
|
||||
$previous = $null
|
||||
$duplicates = 0
|
||||
$reader = New-Object System.IO.StreamReader($Path, [System.Text.Encoding]::UTF8, $true)
|
||||
try {
|
||||
while (($line = $reader.ReadLine()) -ne $null) {
|
||||
$lineNumber++
|
||||
$trimmed = $line.Trim()
|
||||
if ($trimmed.Length -eq 0) { continue }
|
||||
if ($trimmed -notmatch $regex) {
|
||||
throw "Validation failed: unexpected format at line $lineNumber ('$trimmed')."
|
||||
}
|
||||
$normalized = $trimmed.ToUpperInvariant()
|
||||
if ($previous -and $normalized -lt $previous) {
|
||||
Write-Warning ("Validation warning: line {0} is out of order." -f $lineNumber)
|
||||
}
|
||||
if ($normalized -eq $previous) { $duplicates++ }
|
||||
$previous = $normalized
|
||||
}
|
||||
} finally {
|
||||
$reader.Dispose()
|
||||
}
|
||||
|
||||
if ($lineNumber -eq 0) { throw 'Validation failed: KHDB file is empty.' }
|
||||
if ($duplicates -gt 0) {
|
||||
Write-Warning ("Validation warning: detected {0} duplicate hash entries (file remains unchanged)." -f $duplicates)
|
||||
}
|
||||
}
|
||||
|
||||
function Remove-EmptyDirectories {
|
||||
param([string]$Root)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $Root)) { return }
|
||||
Get-ChildItem -LiteralPath $Root -Directory -Recurse | Sort-Object FullName -Descending | ForEach-Object {
|
||||
$childItems = Get-ChildItem -LiteralPath $_.FullName -Force
|
||||
if (-not $childItems) {
|
||||
Remove-Item -LiteralPath $_.FullName -Force
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Update-KHDB {
|
||||
param(
|
||||
[ValidateRange(1, 64)]
|
||||
[int]$MaxParallelTransfers = 5
|
||||
)
|
||||
Start-UpdateTranscript -BasePath $scriptRoot
|
||||
try {
|
||||
$settings = Read-ElysiumSettings
|
||||
$installPath = Get-InstallationPath $settings
|
||||
Ensure-Directory $installPath
|
||||
|
||||
$psSupportsParallel = ($PSVersionTable.PSVersion.Major -ge 7)
|
||||
$effectiveParallelTransfers = if ($MaxParallelTransfers -lt 1) { 1 } else { [int]$MaxParallelTransfers }
|
||||
$parallelDownloadsEnabled = $psSupportsParallel -and $effectiveParallelTransfers -gt 1
|
||||
if (-not $psSupportsParallel -and $effectiveParallelTransfers -gt 1) {
|
||||
Write-Verbose "Parallel transfers requested but PowerShell $($PSVersionTable.PSVersion) does not support ForEach-Object -Parallel; using serial downloads."
|
||||
}
|
||||
$parallelAzureDownloadHelpers = $null
|
||||
$parallelAzureDownloadHelperList = @()
|
||||
$parallelS3DownloadHelpers = $null
|
||||
$parallelS3DownloadHelperList = @()
|
||||
if ($parallelDownloadsEnabled) {
|
||||
$parallelAzureDownloadHelpers = @{
|
||||
'Build-BlobUri' = Get-FunctionDefinitionText 'Build-BlobUri'
|
||||
'Invoke-DownloadWithRetry' = Get-FunctionDefinitionText 'Invoke-DownloadWithRetry'
|
||||
'New-HttpClient' = Get-FunctionDefinitionText 'New-HttpClient'
|
||||
'Get-FileSha256Lower' = Get-FunctionDefinitionText 'Get-FileSha256Lower'
|
||||
}
|
||||
$parallelAzureDownloadHelperList = $parallelAzureDownloadHelpers.GetEnumerator() | ForEach-Object {
|
||||
[pscustomobject]@{ Name = $_.Key; Definition = $_.Value }
|
||||
}
|
||||
$parallelS3DownloadHelpers = @{}
|
||||
@(
|
||||
'Get-Bytes',
|
||||
'Get-HashHex',
|
||||
'HmacSha256',
|
||||
'ToHex',
|
||||
'GetSignatureKey',
|
||||
'UriEncode',
|
||||
'BuildCanonicalPath',
|
||||
'BuildAuthHeaders',
|
||||
'BuildS3Uri',
|
||||
'Invoke-S3HttpDownloadWithRetry',
|
||||
'Get-FileSha256Lower'
|
||||
) | ForEach-Object {
|
||||
$parallelS3DownloadHelpers[$_] = Get-FunctionDefinitionText $_
|
||||
}
|
||||
$parallelS3DownloadHelperList = $parallelS3DownloadHelpers.GetEnumerator() | ForEach-Object {
|
||||
[pscustomobject]@{ Name = $_.Key; Definition = $_.Value }
|
||||
}
|
||||
}
|
||||
|
||||
$storageProvider = $settings['StorageProvider']
|
||||
if ([string]::IsNullOrWhiteSpace($storageProvider)) { $storageProvider = 'Azure' }
|
||||
|
||||
$manifestBlobPath = $settings['KhdbManifestPath']
|
||||
if ([string]::IsNullOrWhiteSpace($manifestBlobPath)) { $manifestBlobPath = 'khdb/manifest.json' }
|
||||
|
||||
$remoteShardPrefix = $settings['KhdbShardPrefix']
|
||||
if ([string]::IsNullOrWhiteSpace($remoteShardPrefix)) { $remoteShardPrefix = 'khdb/shards' }
|
||||
|
||||
$localShardDirName = $settings['KhdbLocalShardDir']
|
||||
if ([string]::IsNullOrWhiteSpace($localShardDirName)) { $localShardDirName = 'khdb-shards' }
|
||||
|
||||
$localShardRoot = Join-Path -Path $installPath -ChildPath $localShardDirName
|
||||
Ensure-Directory $localShardRoot
|
||||
|
||||
$localManifestPath = Join-Path -Path $installPath -ChildPath 'khdb-manifest.json'
|
||||
$tmpDir = New-Item -ItemType Directory -Path ([System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), "elysium-khdb-" + [System.Guid]::NewGuid())) -Force
|
||||
$manifestTempPath = Join-Path -Path $tmpDir.FullName -ChildPath 'manifest.json'
|
||||
$downloadTempRoot = Join-Path -Path $tmpDir.FullName -ChildPath 'shards'
|
||||
Ensure-Directory $downloadTempRoot
|
||||
|
||||
Write-Host "Fetching manifest ($manifestBlobPath) from $storageProvider storage..."
|
||||
|
||||
$s3Bucket = $null
|
||||
$s3EndpointUrl = $null
|
||||
$s3Region = $null
|
||||
$s3AK = $null
|
||||
$s3SK = $null
|
||||
$forcePathStyle = $true
|
||||
$s3UseAwsTools = $false
|
||||
$storageAccountName = $null
|
||||
$containerName = $null
|
||||
$sasToken = $null
|
||||
|
||||
if ($storageProvider -ieq 'S3') {
|
||||
$s3Bucket = $settings['s3BucketName']
|
||||
$s3EndpointUrl = $settings['s3EndpointUrl']
|
||||
$s3Region = $settings['s3Region']
|
||||
$s3AK = $settings['s3AccessKeyId']
|
||||
$s3SK = $settings['s3SecretAccessKey']
|
||||
$s3Force = $settings['s3ForcePathStyle']
|
||||
$s3UseAwsTools = $settings['s3UseAwsTools']
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($s3Bucket)) { throw 's3BucketName is missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($s3AK) -or [string]::IsNullOrWhiteSpace($s3SK)) { throw 's3AccessKeyId / s3SecretAccessKey missing or empty.' }
|
||||
if ([string]::IsNullOrWhiteSpace($s3EndpointUrl)) { throw 's3EndpointUrl is required for S3-compatible storage.' }
|
||||
$forcePathStyle = Get-BooleanSetting -Value $s3Force -Default $true
|
||||
try { $s3UseAwsTools = [System.Convert]::ToBoolean($s3UseAwsTools) } catch { $s3UseAwsTools = $false }
|
||||
if ($parallelDownloadsEnabled -and $s3UseAwsTools) {
|
||||
Write-Warning 'Parallel shard downloads require the SigV4 HTTP path; disabling AWS Tools mode for this run.'
|
||||
$s3UseAwsTools = $false
|
||||
}
|
||||
|
||||
$downloadKey = Combine-StoragePath -Prefix $null -Name $manifestBlobPath
|
||||
if ($s3UseAwsTools) {
|
||||
try {
|
||||
$client = New-S3Client -EndpointUrl $s3EndpointUrl -Region $s3Region -AccessKeyId $s3AK -SecretAccessKey $s3SK -ForcePathStyle:$forcePathStyle
|
||||
$req = New-Object Amazon.S3.Model.GetObjectRequest -Property @{ BucketName = $s3Bucket; Key = $downloadKey }
|
||||
$resp = $client.GetObject($req)
|
||||
try { $resp.WriteResponseStreamToFile($manifestTempPath, $true) } finally { $resp.Dispose() }
|
||||
} catch {
|
||||
Write-Warning "AWS Tools download failed for manifest: $($_.Exception.Message). Falling back to SigV4 HTTP."
|
||||
Invoke-S3HttpDownloadWithRetry -EndpointUrl $s3EndpointUrl -Bucket $s3Bucket -Key $downloadKey -TargetPath $manifestTempPath -Region $s3Region -AccessKeyId $s3AK -SecretAccessKey $s3SK -ForcePathStyle:$forcePathStyle -Activity 'Downloading manifest'
|
||||
}
|
||||
} else {
|
||||
Invoke-S3HttpDownloadWithRetry -EndpointUrl $s3EndpointUrl -Bucket $s3Bucket -Key $downloadKey -TargetPath $manifestTempPath -Region $s3Region -AccessKeyId $s3AK -SecretAccessKey $s3SK -ForcePathStyle:$forcePathStyle -Activity 'Downloading manifest'
|
||||
}
|
||||
} else {
|
||||
$storageAccountName = $settings['storageAccountName']
|
||||
$containerName = $settings['containerName']
|
||||
$sasToken = $settings['sasToken']
|
||||
$client = New-HttpClient
|
||||
try {
|
||||
$uri = Build-BlobUri -Account $storageAccountName -Container $containerName -Sas $sasToken -BlobName $manifestBlobPath
|
||||
Invoke-DownloadWithRetry -Client $client -Uri $uri -TargetPath $manifestTempPath -Activity 'Downloading manifest'
|
||||
} finally {
|
||||
if ($client) { $client.Dispose() }
|
||||
}
|
||||
}
|
||||
|
||||
$manifest = Load-Manifest -Path $manifestTempPath
|
||||
Validate-Manifest -Manifest $manifest
|
||||
if ($manifest.shardPrefix) {
|
||||
$remoteShardPrefix = [string]$manifest.shardPrefix
|
||||
Write-Verbose "Using shard prefix from manifest: $remoteShardPrefix"
|
||||
}
|
||||
|
||||
$remoteShardPrefix = $remoteShardPrefix.Replace('\', '/')
|
||||
|
||||
Write-Host ("Manifest downloaded. Found {0} shard(s)." -f $manifest.shards.Count)
|
||||
if ($manifest.version) {
|
||||
Write-Host ("Remote version: {0}" -f $manifest.version)
|
||||
}
|
||||
|
||||
$localManifest = $null
|
||||
if (Test-Path -LiteralPath $localManifestPath) {
|
||||
try {
|
||||
$localManifest = Load-Manifest -Path $localManifestPath
|
||||
} catch {
|
||||
Write-Warning ("Failed to parse existing manifest. Full refresh will occur: {0}" -f $_.Exception.Message)
|
||||
}
|
||||
}
|
||||
|
||||
$localManifestMap = @{}
|
||||
if ($localManifest -and $localManifest.shards) {
|
||||
foreach ($entry in $localManifest.shards) {
|
||||
if ($entry.name) { $localManifestMap[$entry.name] = $entry }
|
||||
}
|
||||
}
|
||||
|
||||
$downloadQueue = [System.Collections.ArrayList]::new()
|
||||
$remoteNameSet = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase)
|
||||
foreach ($entry in $manifest.shards) {
|
||||
$name = [string]$entry.name
|
||||
[void]$remoteNameSet.Add($name)
|
||||
$expectedHash = ([string]$entry.sha256).ToLowerInvariant()
|
||||
$expectedSize = 0L
|
||||
if (-not [long]::TryParse([string]$entry.size, [ref]$expectedSize)) {
|
||||
throw "Cannot parse size for shard '$name'."
|
||||
}
|
||||
|
||||
$localPath = Join-Path -Path $localShardRoot -ChildPath $name
|
||||
$needsDownload = $true
|
||||
|
||||
if (Test-Path -LiteralPath $localPath) {
|
||||
$localInfo = Get-Item -LiteralPath $localPath
|
||||
if ($localInfo.Length -eq $expectedSize) {
|
||||
$localManifestEntry = $null
|
||||
if ($localManifestMap.ContainsKey($name)) {
|
||||
$localManifestEntry = $localManifestMap[$name]
|
||||
}
|
||||
if ($localManifestEntry -and ([string]$localManifestEntry.sha256).ToLowerInvariant() -eq $expectedHash) {
|
||||
$needsDownload = $false
|
||||
} else {
|
||||
$localHash = Get-FileSha256Lower -Path $localPath
|
||||
if ($localHash -eq $expectedHash) { $needsDownload = $false }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($needsDownload) {
|
||||
[void]$downloadQueue.Add([pscustomobject]@{
|
||||
Name = $name
|
||||
Sha256 = $expectedHash
|
||||
Size = $expectedSize
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if ($downloadQueue.Count -gt 0) {
|
||||
Write-Host ("{0} shard(s) require download or refresh." -f $downloadQueue.Count)
|
||||
} else {
|
||||
Write-Host 'All shards already up to date; verifying manifest and combined file.'
|
||||
}
|
||||
|
||||
$storageClient = $null
|
||||
$storageHttpClient = $null
|
||||
$isS3 = ($storageProvider -ieq 'S3')
|
||||
|
||||
try {
|
||||
if ($isS3) {
|
||||
$s3Bucket = $settings['s3BucketName']
|
||||
$s3EndpointUrl = $settings['s3EndpointUrl']
|
||||
$s3Region = $settings['s3Region']
|
||||
$s3AK = $settings['s3AccessKeyId']
|
||||
$s3SK = $settings['s3SecretAccessKey']
|
||||
$s3Force = $settings['s3ForcePathStyle']
|
||||
$forcePathStyle = Get-BooleanSetting -Value $s3Force -Default $true
|
||||
$useAwsTools = $settings['s3UseAwsTools']
|
||||
try { $useAwsTools = [System.Convert]::ToBoolean($useAwsTools) } catch { $useAwsTools = $false }
|
||||
|
||||
if ($downloadQueue.Count -gt 0 -and -not $parallelDownloadsEnabled) {
|
||||
if ($useAwsTools) {
|
||||
$storageClient = New-S3Client -EndpointUrl $s3EndpointUrl -Region $s3Region -AccessKeyId $s3AK -SecretAccessKey $s3SK -ForcePathStyle:$forcePathStyle
|
||||
}
|
||||
$storageHttpClient = @{
|
||||
Endpoint = $s3EndpointUrl
|
||||
Bucket = $s3Bucket
|
||||
Region = $s3Region
|
||||
AccessKey = $s3AK
|
||||
SecretKey = $s3SK
|
||||
ForcePath = $forcePathStyle
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if ($downloadQueue.Count -gt 0 -and -not $parallelDownloadsEnabled) {
|
||||
$storageHttpClient = New-HttpClient
|
||||
}
|
||||
}
|
||||
|
||||
if ($parallelDownloadsEnabled -and $downloadQueue.Count -gt 0) {
|
||||
Write-Host ("Downloading shards with up to {0} concurrent transfer(s)..." -f $effectiveParallelTransfers)
|
||||
$remotePrefixForParallel = if ([string]::IsNullOrWhiteSpace($remoteShardPrefix)) { $null } else { $remoteShardPrefix.Replace('\', '/').Trim('/') }
|
||||
$parallelDownloadHelpers = if ($isS3) { $parallelS3DownloadHelperList } else { $parallelAzureDownloadHelperList }
|
||||
$downloadQueue.ToArray() | ForEach-Object -Parallel {
|
||||
$entry = $PSItem
|
||||
try {
|
||||
if ($null -eq $entry) { return }
|
||||
foreach ($helper in $using:parallelDownloadHelpers) {
|
||||
if (-not (Get-Command $helper.Name -ErrorAction SilentlyContinue)) {
|
||||
Invoke-Expression $helper.Definition
|
||||
}
|
||||
}
|
||||
$name = [string]$entry.Name
|
||||
if ([string]::IsNullOrWhiteSpace($name)) {
|
||||
throw "Parallel shard entry missing name: $(ConvertTo-Json $entry -Compress)"
|
||||
}
|
||||
$expectedHash = ([string]$entry.Sha256).ToLowerInvariant()
|
||||
$expectedSize = [long]$entry.Size
|
||||
|
||||
$remoteKey = $name.Replace('\', '/').TrimStart('/')
|
||||
if (-not [string]::IsNullOrWhiteSpace($using:remotePrefixForParallel)) {
|
||||
$remoteKey = $using:remotePrefixForParallel + '/' + $remoteKey
|
||||
}
|
||||
$stagingPath = Join-Path -Path $using:downloadTempRoot -ChildPath $name
|
||||
$stagingParent = Split-Path -Path $stagingPath -Parent
|
||||
if ($stagingParent -and -not (Test-Path -LiteralPath $stagingParent)) {
|
||||
[System.IO.Directory]::CreateDirectory($stagingParent) | Out-Null
|
||||
}
|
||||
|
||||
$activity = ("Downloading shard: {0}" -f $name)
|
||||
if ($using:isS3) {
|
||||
Invoke-S3HttpDownloadWithRetry -EndpointUrl $using:s3EndpointUrl -Bucket $using:s3Bucket -Key $remoteKey -TargetPath $stagingPath -Region $using:s3Region -AccessKeyId $using:s3AK -SecretAccessKey $using:s3SK -ForcePathStyle:$using:forcePathStyle -Activity $activity
|
||||
} else {
|
||||
$client = $null
|
||||
try {
|
||||
$client = New-HttpClient
|
||||
$blobUri = Build-BlobUri -Account $using:storageAccountName -Container $using:containerName -Sas $using:sasToken -BlobName $remoteKey
|
||||
Invoke-DownloadWithRetry -Client $client -Uri $blobUri -TargetPath $stagingPath -Activity $activity
|
||||
} finally {
|
||||
if ($client) { $client.Dispose() }
|
||||
}
|
||||
}
|
||||
|
||||
$downloadInfo = Get-Item -LiteralPath $stagingPath
|
||||
if ($downloadInfo.Length -ne $expectedSize) {
|
||||
throw "Shard '$name' size mismatch. Expected $expectedSize bytes, got $($downloadInfo.Length)."
|
||||
}
|
||||
|
||||
$actualHash = Get-FileSha256Lower -Path $stagingPath
|
||||
if ($actualHash -ne $expectedHash) {
|
||||
throw "Shard '$name' checksum mismatch. Expected $expectedHash, got $actualHash."
|
||||
}
|
||||
|
||||
$finalPath = Join-Path -Path $using:localShardRoot -ChildPath $name
|
||||
$parentDir = Split-Path -Path $finalPath -Parent
|
||||
if ($parentDir -and -not (Test-Path -LiteralPath $parentDir)) {
|
||||
[System.IO.Directory]::CreateDirectory($parentDir) | Out-Null
|
||||
}
|
||||
Move-Item -LiteralPath $stagingPath -Destination $finalPath -Force
|
||||
Write-Host ("Shard '{0}' updated." -f $name)
|
||||
} catch {
|
||||
throw ("Shard '{0}': {1}" -f $entry.name, $_.Exception.Message)
|
||||
}
|
||||
} -ThrottleLimit $effectiveParallelTransfers
|
||||
} else {
|
||||
$downloadIndex = 0
|
||||
foreach ($entry in $downloadQueue.ToArray()) {
|
||||
$downloadIndex++
|
||||
if ($null -eq $entry) { continue }
|
||||
$name = [string]$entry.Name
|
||||
if ([string]::IsNullOrWhiteSpace($name)) {
|
||||
throw "Shard entry missing name: $(ConvertTo-Json $entry -Compress)"
|
||||
}
|
||||
$expectedHash = ([string]$entry.Sha256).ToLowerInvariant()
|
||||
$expectedSize = [long]$entry.Size
|
||||
|
||||
$activity = "Downloading shard $downloadIndex/$($downloadQueue.Count): $name"
|
||||
$remoteKey = Combine-StoragePath -Prefix $remoteShardPrefix -Name $name
|
||||
$stagingPath = Join-Path -Path $downloadTempRoot -ChildPath $name
|
||||
Ensure-Directory (Split-Path -Path $stagingPath -Parent)
|
||||
|
||||
if ($isS3) {
|
||||
if ($storageClient) {
|
||||
try {
|
||||
$request = New-Object Amazon.S3.Model.GetObjectRequest -Property @{ BucketName = $s3Bucket; Key = $remoteKey }
|
||||
$response = $storageClient.GetObject($request)
|
||||
try { $response.WriteResponseStreamToFile($stagingPath, $true) } finally { $response.Dispose() }
|
||||
} catch {
|
||||
Write-Warning "AWS Tools download failed for shard '$name': $($_.Exception.Message). Falling back to SigV4 HTTP."
|
||||
Invoke-S3HttpDownloadWithRetry -EndpointUrl $storageHttpClient.Endpoint -Bucket $storageHttpClient.Bucket -Key $remoteKey -TargetPath $stagingPath -Region $storageHttpClient.Region -AccessKeyId $storageHttpClient.AccessKey -SecretAccessKey $storageHttpClient.SecretKey -ForcePathStyle:$storageHttpClient.ForcePath -Activity $activity
|
||||
}
|
||||
} else {
|
||||
Invoke-S3HttpDownloadWithRetry -EndpointUrl $storageHttpClient.Endpoint -Bucket $storageHttpClient.Bucket -Key $remoteKey -TargetPath $stagingPath -Region $storageHttpClient.Region -AccessKeyId $storageHttpClient.AccessKey -SecretAccessKey $storageHttpClient.SecretKey -ForcePathStyle:$storageHttpClient.ForcePath -Activity $activity
|
||||
}
|
||||
} else {
|
||||
$blobUri = Build-BlobUri -Account $storageAccountName -Container $containerName -Sas $sasToken -BlobName $remoteKey
|
||||
Invoke-DownloadWithRetry -Client $storageHttpClient -Uri $blobUri -TargetPath $stagingPath -Activity $activity
|
||||
}
|
||||
|
||||
$downloadInfo = Get-Item -LiteralPath $stagingPath
|
||||
if ($downloadInfo.Length -ne $expectedSize) {
|
||||
throw "Shard '$name' size mismatch. Expected $expectedSize bytes, got $($downloadInfo.Length)."
|
||||
}
|
||||
|
||||
$actualHash = Get-FileSha256Lower -Path $stagingPath
|
||||
if ($actualHash -ne $expectedHash) {
|
||||
throw "Shard '$name' checksum mismatch. Expected $expectedHash, got $actualHash."
|
||||
}
|
||||
|
||||
$finalPath = Join-Path -Path $localShardRoot -ChildPath $name
|
||||
Ensure-Directory (Split-Path -Path $finalPath -Parent)
|
||||
Move-Item -LiteralPath $stagingPath -Destination $finalPath -Force
|
||||
Write-Host ("Shard '{0}' updated." -f $name)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if ($storageClient) { $storageClient.Dispose() }
|
||||
if ($storageHttpClient -is [System.Net.Http.HttpClient]) { $storageHttpClient.Dispose() }
|
||||
}
|
||||
|
||||
$existingShards = @()
|
||||
if (Test-Path -LiteralPath $localShardRoot) {
|
||||
$existingShards = Get-ChildItem -LiteralPath $localShardRoot -File -Recurse
|
||||
}
|
||||
|
||||
$removed = 0
|
||||
foreach ($file in $existingShards) {
|
||||
$relative = Get-RelativePath -BasePath $localShardRoot -FullPath $file.FullName
|
||||
if (-not $remoteNameSet.Contains($relative)) {
|
||||
Remove-Item -LiteralPath $file.FullName -Force
|
||||
$removed++
|
||||
}
|
||||
}
|
||||
|
||||
if ($removed -gt 0) {
|
||||
Write-Host ("Removed {0} stale shard(s)." -f $removed)
|
||||
Remove-EmptyDirectories -Root $localShardRoot
|
||||
}
|
||||
|
||||
Copy-Item -LiteralPath $manifestTempPath -Destination $localManifestPath -Force
|
||||
Write-Host ("Manifest saved locally to {0}" -f $localManifestPath)
|
||||
|
||||
$khdbName = if ([string]::IsNullOrWhiteSpace($settings['WeakPasswordsDatabase'])) { 'khdb.txt' } else { $settings['WeakPasswordsDatabase'] }
|
||||
$combinedTarget = Join-Path -Path $installPath -ChildPath $khdbName
|
||||
$combinedTemp = Join-Path -Path $tmpDir.FullName -ChildPath 'khdb-combined.txt'
|
||||
|
||||
Write-Host "Rebuilding combined KHDB file..."
|
||||
Merge-ShardsToFile -Manifest $manifest -ShardsRoot $localShardRoot -TargetPath $combinedTemp
|
||||
Validate-KHDBFile -Path $combinedTemp
|
||||
|
||||
if (Test-Path -LiteralPath $combinedTarget) {
|
||||
$ts = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$backupPath = Join-Path -Path $installPath -ChildPath ("$khdbName.bak-$ts")
|
||||
Copy-Item -LiteralPath $combinedTarget -Destination $backupPath -Force
|
||||
Write-Host ("Existing KHDB backed up to {0}" -f $backupPath)
|
||||
}
|
||||
|
||||
Move-Item -LiteralPath $combinedTemp -Destination $combinedTarget -Force
|
||||
Write-Host ("KHDB merged file refreshed at {0}" -f $combinedTarget)
|
||||
Write-Host "KHDB update completed successfully."
|
||||
} catch {
|
||||
Write-Error ("KHDB update failed: {0}" -f $_.Exception.Message)
|
||||
throw
|
||||
} finally {
|
||||
try { if ($tmpDir -and (Test-Path $tmpDir.FullName)) { Remove-Item -Path $tmpDir.FullName -Recurse -Force } } catch {}
|
||||
Stop-UpdateTranscript
|
||||
}
|
||||
}
|
||||
|
||||
Update-KHDB
|
||||
Write-Host "Script execution completed."
|
||||
@@ -1,167 +0,0 @@
|
||||
##################################################
|
||||
## ____ ___ ____ _____ _ _ _____ _____ ##
|
||||
## / ___/ _ \| _ \| ____| | \ | | ____|_ _| ##
|
||||
## | | | | | | |_) | _| | \| | _| | | ##
|
||||
## | |__| |_| | _ <| |___ _| |\ | |___ | | ##
|
||||
## \____\__\_\_| \_\_____(_)_| \_|_____| |_| ##
|
||||
##################################################
|
||||
## Project: Elysium ##
|
||||
## File: Update-LithnetStore.ps1 ##
|
||||
## Version: 1.1.0 ##
|
||||
## Support: support@cqre.net ##
|
||||
##################################################
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Populates a Lithnet Password Protection store with compromised passwords and banned words.
|
||||
|
||||
.DESCRIPTION
|
||||
Reads configuration from ElysiumSettings.txt (or a provided settings file), opens the Lithnet
|
||||
Password Protection store, optionally synchronizes with Have I Been Pwned, imports local NTLM
|
||||
hash lists, plaintext password lists, and banned-word files.
|
||||
#>
|
||||
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$SettingsPath,
|
||||
[string[]]$HashFiles,
|
||||
[string[]]$PlaintextFiles,
|
||||
[string[]]$BannedWordFiles,
|
||||
[switch]$SkipHibpSync
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
[string]$commonHelper = Join-Path -Path $PSScriptRoot -ChildPath 'Elysium.Common.ps1'
|
||||
if (-not (Test-Path -LiteralPath $commonHelper)) { throw "Common helper not found at $commonHelper" }
|
||||
. $commonHelper
|
||||
Restart-WithPwshIfAvailable -BoundParameters $PSBoundParameters -UnboundArguments $MyInvocation.UnboundArguments
|
||||
|
||||
function Read-KeyValueSettings {
|
||||
param([string]$Path)
|
||||
$result = @{}
|
||||
if (-not (Test-Path -LiteralPath $Path)) { throw "Settings file not found at $Path" }
|
||||
Get-Content -LiteralPath $Path | ForEach-Object {
|
||||
$line = $_
|
||||
if (-not $line) { return }
|
||||
$trimmed = $line.Trim()
|
||||
if (-not $trimmed) { return }
|
||||
if ($trimmed.StartsWith('#')) { return }
|
||||
$kv = $line -split '=', 2
|
||||
if ($kv.Count -ne 2) { return }
|
||||
$key = $kv[0].Trim()
|
||||
$value = $kv[1].Trim()
|
||||
if (-not $key) { return }
|
||||
if ($value.StartsWith("'") -and $value.EndsWith("'") -and $value.Length -ge 2) {
|
||||
$value = $value.Substring(1, $value.Length - 2)
|
||||
}
|
||||
$result[$key] = $value
|
||||
}
|
||||
return $result
|
||||
}
|
||||
|
||||
function Get-BooleanSetting {
|
||||
param(
|
||||
[string]$Value,
|
||||
[bool]$Default = $false
|
||||
)
|
||||
if ([string]::IsNullOrWhiteSpace($Value)) { return $Default }
|
||||
try { return [System.Convert]::ToBoolean($Value) } catch { return $Default }
|
||||
}
|
||||
|
||||
function Get-ListFromSetting {
|
||||
param([string]$Value)
|
||||
if ([string]::IsNullOrWhiteSpace($Value)) { return @() }
|
||||
return ($Value -split '[,;]' | ForEach-Object { $_.Trim() } | Where-Object { $_ })
|
||||
}
|
||||
|
||||
function Resolve-ExistingPath {
|
||||
param([string]$PathValue, [string]$Description)
|
||||
if ([string]::IsNullOrWhiteSpace($PathValue)) { throw "$Description path was not provided." }
|
||||
if (-not (Test-Path -LiteralPath $PathValue)) {
|
||||
throw "$Description not found at '$PathValue'."
|
||||
}
|
||||
return (Resolve-Path -LiteralPath $PathValue).Path
|
||||
}
|
||||
|
||||
if (-not $SettingsPath) {
|
||||
$SettingsPath = Join-Path -Path $PSScriptRoot -ChildPath 'ElysiumSettings.txt'
|
||||
}
|
||||
$settings = Read-KeyValueSettings -Path $SettingsPath
|
||||
|
||||
$storePathSetting = $settings['LithnetStorePath']
|
||||
$storePath = Resolve-ExistingPath -PathValue $storePathSetting -Description 'Lithnet store'
|
||||
|
||||
$settingsHashSources = Get-ListFromSetting -Value $settings['LithnetHashSources']
|
||||
$settingsPlainSources = Get-ListFromSetting -Value $settings['LithnetPlaintextSources']
|
||||
$settingsBannedSources = Get-ListFromSetting -Value $settings['LithnetBannedWordSources']
|
||||
|
||||
$hashSourcePaths = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase)
|
||||
foreach ($path in @($HashFiles) + $settingsHashSources) {
|
||||
if ([string]::IsNullOrWhiteSpace($path)) { continue }
|
||||
$resolved = Resolve-ExistingPath -PathValue $path -Description 'Hash list'
|
||||
[void]$hashSourcePaths.Add($resolved)
|
||||
}
|
||||
if ($hashSourcePaths.Count -eq 0) {
|
||||
$defaultKhdb = Join-Path -Path $PSScriptRoot -ChildPath 'khdb.txt'
|
||||
if (Test-Path -LiteralPath $defaultKhdb) {
|
||||
[void]$hashSourcePaths.Add((Resolve-Path -LiteralPath $defaultKhdb).Path)
|
||||
} else {
|
||||
throw 'No hash files were provided via parameters or LithnetHashSources, and khdb.txt was not found.'
|
||||
}
|
||||
}
|
||||
|
||||
$plaintextSourcePaths = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase)
|
||||
foreach ($path in @($PlaintextFiles) + $settingsPlainSources) {
|
||||
if ([string]::IsNullOrWhiteSpace($path)) { continue }
|
||||
$resolved = Resolve-ExistingPath -PathValue $path -Description 'Plaintext password list'
|
||||
[void]$plaintextSourcePaths.Add($resolved)
|
||||
}
|
||||
|
||||
$bannedWordSourcePaths = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase)
|
||||
foreach ($path in @($BannedWordFiles) + $settingsBannedSources) {
|
||||
if ([string]::IsNullOrWhiteSpace($path)) { continue }
|
||||
$resolved = Resolve-ExistingPath -PathValue $path -Description 'Banned word list'
|
||||
[void]$bannedWordSourcePaths.Add($resolved)
|
||||
}
|
||||
|
||||
$syncHibp = Get-BooleanSetting -Value $settings['LithnetSyncHibp'] -Default:$false
|
||||
if ($SkipHibpSync) { $syncHibp = $false }
|
||||
|
||||
Write-Host "Importing LithnetPasswordProtection module..."
|
||||
try {
|
||||
Import-Module -Name LithnetPasswordProtection -ErrorAction Stop | Out-Null
|
||||
} catch {
|
||||
throw "LithnetPasswordProtection module not found: $($_.Exception.Message). Install it from https://github.com/lithnet/password-protection."
|
||||
}
|
||||
|
||||
Write-Host ("Opening Lithnet store at '{0}'..." -f $storePath)
|
||||
Open-Store -Path $storePath
|
||||
$storeOpened = $true
|
||||
try {
|
||||
if ($syncHibp) {
|
||||
Write-Host 'Synchronizing compromised hashes from Have I Been Pwned (this can take a while)...'
|
||||
Sync-HashesFromHibp
|
||||
}
|
||||
|
||||
foreach ($hashFile in ($hashSourcePaths.ToArray() | Sort-Object)) {
|
||||
Write-Host ("Importing NTLM hash list '{0}'..." -f $hashFile)
|
||||
Import-CompromisedPasswordHashes -Filename $hashFile
|
||||
}
|
||||
|
||||
foreach ($plainFile in ($plaintextSourcePaths.ToArray() | Sort-Object)) {
|
||||
Write-Host ("Importing plaintext password list '{0}'..." -f $plainFile)
|
||||
Import-CompromisedPasswords -Filename $plainFile
|
||||
}
|
||||
|
||||
foreach ($bannedFile in ($bannedWordSourcePaths.ToArray() | Sort-Object)) {
|
||||
Write-Host ("Importing banned word list '{0}'..." -f $bannedFile)
|
||||
Import-BannedWords -Filename $bannedFile
|
||||
}
|
||||
|
||||
Write-Host 'Lithnet store update completed successfully.'
|
||||
} finally {
|
||||
if ($storeOpened) {
|
||||
try { Close-Store } catch {}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user