From 60a7671cebabea2c602517704c64b85309ed78b9 Mon Sep 17 00:00:00 2001 From: Tomas Kracmar Date: Mon, 16 Mar 2026 16:38:19 +0100 Subject: [PATCH] Fix KHDB password match format handling --- CHANGELOG.md | 20 +++++++ Prepare-KHDBStorage.ps1 | 20 +++---- README.md | 2 +- Test-WeakADPasswords.ps1 | 111 +++++++++++++++++++++++++++++++++++++-- Update-KHDB.ps1 | 35 +++++++++--- 5 files changed, 169 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 171890e..64b7cf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## 2026-03-16 + +### Test-WeakADPasswords.ps1 v1.4.5 +Fixed: +- Normalizes legacy `HASH:count` KHDB files into a temporary hash-only list before calling `DSInternals`, so dictionary matches no longer fail silently when clients have older database content. +- Warns when KHDB normalization is required instead of leaving the weak-password match section empty without explanation. + +### Update-KHDB.ps1 v2.1.1 +Fixed: +- Rebuilds the merged local `khdb.txt` as a DSInternals-compatible hash-only file even when upstream shards still contain legacy `HASH:count` lines. +- Tightened KHDB merge validation so malformed shard content is surfaced during update rather than producing a silently unusable weak-password database. + +### Prepare-KHDBStorage.ps1 v1.1.1 +Fixed: +- Accepts legacy `HASH:count` source input but writes deduplicated hash-only shards for downstream DSInternals consumers. + +### README.md +Changed: +- Corrected the KHDB format documentation to require one NT hash per line and documented the automatic legacy-format normalization. + ## 2026-02-17 ### Test-WeakADPasswords.ps1 v1.4.4 diff --git a/Prepare-KHDBStorage.ps1 b/Prepare-KHDBStorage.ps1 index 69d171f..c1f15a9 100644 --- a/Prepare-KHDBStorage.ps1 +++ b/Prepare-KHDBStorage.ps1 @@ -7,7 +7,7 @@ ################################################## ## Project: Elysium ## ## File: Prepare-KHDBStorage.ps1 ## -## Version: 1.1.0 ## +## Version: 1.1.1 ## ## Support: support@cqre.net ## ################################################## @@ -385,7 +385,7 @@ function Split-KhdbIntoShards { [psobject]$ResumeState ) - $hashRegex = '^[0-9A-Fa-f]{32}$' + $hashRegex = '^[0-9A-Fa-f]{32}(:\d+)?$' $encoding = New-Object System.Text.UTF8Encoding($false) $ShardRoot = [System.IO.Path]::GetFullPath($ShardRoot) Ensure-Directory $ShardRoot @@ -402,6 +402,7 @@ function Split-KhdbIntoShards { TotalLines = 0L InvalidLines = 0L SkippedLines = 0L + LegacyLines = 0L InvalidSamples = New-Object System.Collections.Generic.List[string] } @@ -596,13 +597,13 @@ function Split-KhdbIntoShards { if (-not [string]::IsNullOrWhiteSpace($prefix)) { $hashPortion = ($prefix.Trim() + $hashPortion) } - if ($hashPortion.Length -ne 32 -or $hashPortion -notmatch $hashRegex) { + if ($hashPortion.Length -ne 32 -or $hashPortion -notmatch '^[0-9A-Fa-f]{32}$') { $match = [regex]::Match($hashPortion, '[0-9A-Fa-f]{32}') if ($match.Success) { $hashPortion = $match.Value } } - if ($hashPortion.Length -ne 32 -or $hashPortion -notmatch $hashRegex) { + if ($hashPortion.Length -ne 32 -or $hashPortion -notmatch '^[0-9A-Fa-f]{32}$') { $meta.InvalidLines++ if ($meta.InvalidSamples.Count -lt $maxInvalidSamples) { [void]$meta.InvalidSamples.Add($trimmed) @@ -625,17 +626,14 @@ function Split-KhdbIntoShards { $normalizedHash = $hashPortion.ToUpperInvariant() $countValue = 0 if ($parts.Count -gt 1) { + $meta.LegacyLines++ $countText = $parts[1].Trim() if (-not [string]::IsNullOrWhiteSpace($countText)) { $null = [int]::TryParse($countText, [ref]$countValue) if ($countValue -lt 0) { $countValue = 0 } } } - $normalizedLine = if ($parts.Count -gt 1 -and -not [string]::IsNullOrWhiteSpace($parts[1])) { - "{0}:{1}" -f $normalizedHash, $parts[1].Trim() - } else { - $normalizedHash - } + $normalizedLine = $normalizedHash $prefixKey = $normalizedHash.Substring(0, $PrefixLength).ToLowerInvariant() if (-not $shardStates.ContainsKey($prefixKey)) { @@ -828,6 +826,7 @@ function Split-KhdbIntoShards { TotalLines = [long]$meta.TotalLines InvalidLines = [long]$meta.InvalidLines SkippedLines = [long]$meta.SkippedLines + LegacyCount = [long]$meta.LegacyLines InvalidSamples = $meta.InvalidSamples.ToArray() InvalidOutputPath = if ($meta.InvalidLines -gt 0 -and $InvalidOutputPath) { $InvalidOutputPath } else { $null } } @@ -1131,6 +1130,9 @@ if ($UploadOnly) { $invalidCount = [long]$splitResult.InvalidLines $skippedCount = [long]$splitResult.SkippedLines Write-Host ("Input summary: {0} non-empty line(s) -> {1} valid hash(es), {2} invalid entr(y/ies), {3} skipped." -f $totalLines, $totalEntries, $invalidCount, $skippedCount) + if ([long]$splitResult.LegacyCount -gt 0) { + Write-Warning ("Detected {0} legacy HASH:count entries. Output shards and khdb-clean.txt were normalized to hash-only lines for DSInternals compatibility." -f [long]$splitResult.LegacyCount) + } if ($invalidCount -gt 0) { if ($splitResult.InvalidOutputPath) { Write-Warning ("Invalid lines saved to {0}" -f $splitResult.InvalidOutputPath) diff --git a/README.md b/README.md index e6a1b38..521d403 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ When `-ForcePlainText` is specified the script automatically keeps a checkpoint Run script Elysium.ps1 as an administrator and choose option 2 (Test Weak AD Passwords). The script lists domains in the same order as they appear in `ElysiumSettings.txt`. After you pick one, it prompts for credentials and validates them against the selected domain controller before running the password-quality test. The tool connects to the selected Domain Controller and compares accounts against KHDB (respecting the optional `CheckOnlyEnabledUsers` flag if configured). A timestamped text report is saved under `Reports`, and accounts with dictionary hits are also exported to a dedicated UPN-only text file to support follow-up automation. -The KHDB file is consumed via binary search as a sorted hash list (plain text lines like `HASH:count`); ensure the file you place at `khdb.txt` keeps that ordering and omits stray blank lines. +The KHDB file is consumed by DSInternals as a sorted hash list with one NT hash per line (for example `HASH`). Do not include `:count` suffixes in `khdb.txt`; the packaging and update scripts normalize legacy `HASH:count` input to the hash-only format automatically. #### Least privileges for password-quality testing The DSInternals cmdlets (`Get-ADReplAccount`/`Test-PasswordQuality`) pull replicated password data, which requires DCSync-style rights. The account that runs option 2 does not have to be a Domain Admin if it has these permissions on the domain naming context: diff --git a/Test-WeakADPasswords.ps1 b/Test-WeakADPasswords.ps1 index 53dcee1..26adce5 100644 --- a/Test-WeakADPasswords.ps1 +++ b/Test-WeakADPasswords.ps1 @@ -8,7 +8,7 @@ ################################################## ## Project: Elysium ## ## File: Test-WeakADPasswords.ps1 ## -## Version: 1.4.4 ## +## Version: 1.4.5 ## ## Support: support@cqre.net ## ################################################## @@ -92,7 +92,7 @@ function Invoke-UsageBeacon { if ($normalizedMethod -in @('POST', 'PUT')) { $payload = [ordered]@{ script = 'Test-WeakADPasswords' - version = '1.4.4' + version = '1.4.5' ranAtUtc = (Get-Date).ToUniversalTime().ToString('o') } if (-not [string]::IsNullOrWhiteSpace($InstanceId)) { @@ -462,6 +462,105 @@ function Get-UserUPN { # (removed stray top-level loop; UPN enrichment happens during report generation below) +function Resolve-DSInternalsWeakHashFile { + param( + [Parameter(Mandatory)][string]$Path + ) + + if (-not (Test-Path -LiteralPath $Path)) { + throw "Weak password hashes file not found at '$Path'." + } + + $compatibleRegex = '^[0-9A-F]{32}$' + $legacyRegex = '^[0-9A-Fa-f]{32}(:\d+)?$' + $lineNumber = 0 + $previousHash = $null + $duplicateCount = 0 + $legacyEntryCount = 0 + $needsNormalization = $false + $reader = $null + + try { + $reader = New-Object System.IO.StreamReader($Path, [System.Text.Encoding]::UTF8, $true) + while (($line = $reader.ReadLine()) -ne $null) { + $lineNumber++ + $trimmed = $line.Trim() + if ($trimmed.Length -eq 0) { continue } + + if ($trimmed -notmatch $legacyRegex) { + throw ("Weak password hashes file '{0}' contains invalid content at line {1}: '{2}'." -f $Path, $lineNumber, $trimmed) + } + + if ($trimmed -notmatch $compatibleRegex) { + $needsNormalization = $true + if ($trimmed.Contains(':')) { $legacyEntryCount++ } + } + + $normalizedHash = ($trimmed.Split(':', 2)[0]).ToUpperInvariant() + if ($line -cne $trimmed -or $trimmed -cne $normalizedHash) { + $needsNormalization = $true + } + + if ($null -ne $previousHash) { + if ($normalizedHash -lt $previousHash) { + throw "Weak password hashes file '$Path' is not sorted alphabetically at line $lineNumber." + } + if ($normalizedHash -eq $previousHash) { + $duplicateCount++ + $needsNormalization = $true + } + } + + $previousHash = $normalizedHash + } + } finally { + if ($reader) { $reader.Dispose() } + } + + if (-not $needsNormalization) { + return [pscustomobject]@{ + Path = $Path + IsTemporary = $false + } + } + + $tmpPath = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), ('elysium-khdb-' + [System.Guid]::NewGuid().ToString() + '.txt')) + $encoding = New-Object System.Text.UTF8Encoding($false) + $reader = $null + $writer = $null + $lastWrittenHash = $null + + try { + $reader = New-Object System.IO.StreamReader($Path, [System.Text.Encoding]::UTF8, $true) + $writer = New-Object System.IO.StreamWriter($tmpPath, $false, $encoding) + + while (($line = $reader.ReadLine()) -ne $null) { + $trimmed = $line.Trim() + if ($trimmed.Length -eq 0) { continue } + + $normalizedHash = ($trimmed.Split(':', 2)[0]).ToUpperInvariant() + if ($normalizedHash -eq $lastWrittenHash) { continue } + + $writer.WriteLine($normalizedHash) + $lastWrittenHash = $normalizedHash + } + } finally { + if ($reader) { $reader.Dispose() } + if ($writer) { $writer.Dispose() } + } + + $normalizationReasons = @() + if ($legacyEntryCount -gt 0) { $normalizationReasons += "$legacyEntryCount legacy HASH:count entries" } + if ($duplicateCount -gt 0) { $normalizationReasons += "$duplicateCount duplicate hashes" } + if ($normalizationReasons.Count -eq 0) { $normalizationReasons += 'format normalization' } + Write-Warning ("Normalized weak password hashes file for DSInternals compatibility ({0}). Temporary file: {1}" -f ($normalizationReasons -join ', '), $tmpPath) + + return [pscustomobject]@{ + Path = $tmpPath + IsTemporary = $true + } +} + function Get-ValidatedADCredential { param ( [Parameter(Mandatory)][string]$DomainName, @@ -527,7 +626,9 @@ function Test-WeakADPasswords { # Performing the test Write-Verbose "Testing password quality for $($selectedDomain.Name)..." + $resolvedHashFile = $null try { + $resolvedHashFile = Resolve-DSInternalsWeakHashFile -Path $FilePath $accounts = Get-ADReplAccount -All -Server $selectedDomain["DC"] -Credential $credential if ($CheckOnlyEnabledUsers) { Write-Verbose "Filtering to only enabled users per settings." @@ -536,7 +637,7 @@ function Test-WeakADPasswords { if ($_.PSObject.Properties.Name -contains 'Enabled') { $_.Enabled } else { $true } } } - $testResults = $accounts | Test-PasswordQuality -WeakPasswordHashesSortedFile $FilePath + $testResults = $accounts | Test-PasswordQuality -WeakPasswordHashesSortedFile $resolvedHashFile.Path Write-Verbose "Password quality test completed." } catch { $message = $_.Exception.Message @@ -550,6 +651,10 @@ function Test-WeakADPasswords { } Write-Error ("An error occurred while testing passwords: {0}" -f $message) return + } finally { + if ($resolvedHashFile -and $resolvedHashFile.IsTemporary -and (Test-Path -LiteralPath $resolvedHashFile.Path)) { + try { Remove-Item -LiteralPath $resolvedHashFile.Path -Force -ErrorAction Stop } catch { } + } } # Report generation with dynamic content and UPNs diff --git a/Update-KHDB.ps1 b/Update-KHDB.ps1 index 5b61f95..ace9f27 100644 --- a/Update-KHDB.ps1 +++ b/Update-KHDB.ps1 @@ -7,7 +7,7 @@ ################################################## ## Project: Elysium ## ## File: Update-KHDB.ps1 ## -## Version: 2.1.0 ## +## Version: 2.1.1 ## ## Support: support@cqre.net ## ################################################## @@ -76,7 +76,7 @@ function New-HttpClient { Add-Type -AssemblyName System.Net.Http -ErrorAction SilentlyContinue $client = [System.Net.Http.HttpClient]::new() $client.Timeout = [TimeSpan]::FromSeconds(600) - $client.DefaultRequestHeaders.UserAgent.ParseAdd('Elysium/2.0 (+Update-KHDB)') + $client.DefaultRequestHeaders.UserAgent.ParseAdd('Elysium/2.1.1 (+Update-KHDB)') return $client } @@ -432,6 +432,22 @@ function Validate-Manifest { } } +function Convert-KHDBLineToHash { + param( + [Parameter(Mandatory)][string]$Line, + [string]$SourceName, + [int]$LineNumber + ) + + $trimmed = $Line.Trim() + if ($trimmed.Length -eq 0) { return $null } + if ($trimmed -notmatch '^[0-9A-Fa-f]{32}(:\d+)?$') { + throw ("Invalid KHDB content in '{0}' at line {1}: '{2}'." -f $SourceName, $LineNumber, $trimmed) + } + + return ($trimmed.Split(':', 2)[0]).ToUpperInvariant() +} + function Merge-ShardsToFile { param( [psobject]$Manifest, @@ -441,6 +457,7 @@ function Merge-ShardsToFile { $encoding = New-Object System.Text.UTF8Encoding($false) $writer = New-Object System.IO.StreamWriter($TargetPath, $false, $encoding) + $previousHash = $null try { foreach ($entry in ($Manifest.shards | Sort-Object name)) { $relative = [string]$entry.name @@ -449,12 +466,18 @@ function Merge-ShardsToFile { throw "Missing shard on disk: $relative" } $reader = New-Object System.IO.StreamReader($shardPath, [System.Text.Encoding]::UTF8, $true) + $lineNumber = 0 try { while (($line = $reader.ReadLine()) -ne $null) { - $trimmed = $line.Trim() - if ($trimmed.Length -gt 0) { - $writer.WriteLine($trimmed) + $lineNumber++ + $normalizedHash = Convert-KHDBLineToHash -Line $line -SourceName $relative -LineNumber $lineNumber + if ($null -eq $normalizedHash) { continue } + if ($previousHash -and $normalizedHash -lt $previousHash) { + throw "Shard merge would produce an unsorted KHDB file at '$relative' line $lineNumber." } + if ($normalizedHash -eq $previousHash) { continue } + $writer.WriteLine($normalizedHash) + $previousHash = $normalizedHash } } finally { $reader.Dispose() @@ -470,7 +493,7 @@ function Validate-KHDBFile { if (-not (Test-Path -LiteralPath $Path)) { throw "Validation failed: $Path not found." } - $regex = '^[0-9A-Fa-f]{32}(:\d+)?$' + $regex = '^[0-9A-Fa-f]{32}$' $lineNumber = 0 $previous = $null $duplicates = 0