From 69c81856970066c67ee5f9332fcea9a4e92002e4 Mon Sep 17 00:00:00 2001 From: Tomas Kracmar Date: Thu, 5 Nov 2026 19:17:33 +0100 Subject: [PATCH] perf(khdb): compiled check of the weak-hash file before the DSInternals test Resolve-DSInternalsWeakHashFile read the whole KHDB with a per-line PowerShell loop (regex, Trim, Split, ToUpper, comparisons) to decide whether DSInternals could use it as is, and a second such loop to rewrite it if not. On the real 1.3 billion-hash KHDB that is the same hours-long cost the rebuild had before the compiled merge, paid at the start of every Test-WeakADPasswords run. - Elysium.KhdbText.Inspect does the check in one pass with the same rules and messages: invalid content and wrong order throw, legacy HASH:count entries and duplicates are counted. - Elysium.KhdbText.WriteNormalized does the rewrite when needed. - TryParse is shared with the merge's Normalize. - The resolver announces the file and its size before checking. - Blank lines now also mark the file as needing normalization; before, they were handed to DSInternals unchanged. Measured on pwsh 7 (macOS), 5,000,000 hashes: clean file 1.6 s with no copy; lowercase "HASH:3" file 13.8 s including the rewrite. 40/40 Pester tests pass; new test covers blank and padded lines. The Windows PowerShell 5.1 timing on the real KHDB is still to be measured. --- CHANGELOG.md | 1 + src/Elysium/Private/KhdbMerge.ps1 | 97 ++++++++++++++++++++++--- src/Elysium/Private/PasswordQuality.ps1 | 74 ++----------------- tests/Elysium.Tests.ps1 | 14 ++++ 4 files changed, 107 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44c0ce7..e2f96e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Starting with **v2.2.0**, Elysium uses a **unified project version**. All script ## [Unreleased] ### Fixed +- **`Test-WeakADPasswords` spent hours checking the KHDB before testing:** `Resolve-DSInternalsWeakHashFile` validated the whole file with a per-line PowerShell loop, the same pattern that made the KHDB rebuild take hours. It now uses the compiled checker (`Elysium.KhdbText.Inspect` / `WriteNormalized`): one pass over a clean KHDB (about 1.6 s per 5 million hashes on pwsh 7) and no temporary copy. Files with blank or whitespace-padded lines are now normalized too; before, they were passed to DSInternals unchanged. - **KHDB rebuild took hours (or days) on Windows PowerShell 5.1:** the merged `khdb.txt` was built and validated with PowerShell code per hash line - a function call, a regex, a split and an uppercase conversion for every line, then a second regex pass to validate. Full-size KHDBs have hundreds of millions of lines. The merge and the validation now run in a small compiled class (`Elysium.KhdbMerger` / `Elysium.KhdbText`, built with `Add-Type` on 5.1 and 7, no extra install) with the same rules. 5 million hashes merge in about 4 s on pwsh 7; the 5.1 timing is still to be measured. The rebuild also prints progress per shard. - **Unchanged KHDB is no longer rebuilt:** if the manifest is identical to the last successful build, no shard was downloaded or removed, and `khdb.txt` exists, the update keeps the current file. The `khdb-build.sha256` marker is written only after a successful build. - **Security:** the KHDB updater joined shard names from the downloaded manifest straight onto local paths, so a tampered manifest (for example from a compromised KHDB bucket) could name a shard `../../something` and have a file of its choosing written outside the shard directory - including over Elysium's own scripts on the admin host the tool runs on. Manifest validation now rejects shard names that are rooted, contain `..` segments, or contain `:` (Windows drive-relative paths, NTFS alternate data streams), before anything is downloaded. diff --git a/src/Elysium/Private/KhdbMerge.ps1 b/src/Elysium/Private/KhdbMerge.ps1 index 373e525..7fcc469 100644 --- a/src/Elysium/Private/KhdbMerge.ps1 +++ b/src/Elysium/Private/KhdbMerge.ps1 @@ -66,28 +66,101 @@ namespace Elysium return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } + // Splits a trimmed "HASH" or "HASH:digits" line. False when it is anything else. + private static bool TryParse(string trimmed, out string hash, out bool hasCount) + { + int colon = trimmed.IndexOf(':'); + hasCount = colon >= 0; + hash = hasCount ? trimmed.Substring(0, colon) : trimmed; + bool valid = hash.Length == 32; + for (int i = 0; valid && i < hash.Length; i++) valid = IsHex(hash[i]); + if (valid && hasCount) + { + valid = colon + 1 < trimmed.Length; + for (int i = colon + 1; valid && i < trimmed.Length; i++) valid = trimmed[i] >= '0' && trimmed[i] <= '9'; + } + return valid; + } + // Returns the uppercase hash of a "HASH" or "HASH:count" line, or null for a blank line. public static string Normalize(string line, string source, long lineNumber) { string trimmed = line.Trim(); if (trimmed.Length == 0) return null; - - int colon = trimmed.IndexOf(':'); - string hash = colon < 0 ? trimmed : trimmed.Substring(0, colon); - bool valid = hash.Length == 32; - for (int i = 0; valid && i < hash.Length; i++) valid = IsHex(hash[i]); - if (valid && colon >= 0) - { - valid = colon + 1 < trimmed.Length; - for (int i = colon + 1; valid && i < trimmed.Length; i++) valid = trimmed[i] >= '0' && trimmed[i] <= '9'; - } - if (!valid) + string hash; + bool hasCount; + if (!TryParse(trimmed, out hash, out hasCount)) throw new InvalidDataException(string.Format( "Invalid KHDB content in '{0}' at line {1}: '{2}'.", source, lineNumber, trimmed)); - return hash.ToUpperInvariant(); } + // Reads a KHDB-style file once and reports whether DSInternals can use it as is: a sorted list of + // bare, upper-case, unique 32-hex hashes, one per line. Throws on invalid content or wrong order. + public static void Inspect(string path, out long legacyEntries, out long duplicates, out bool needsNormalization) + { + legacyEntries = 0; + duplicates = 0; + needsNormalization = false; + string previous = null; + using (var reader = new StreamReader(path, Encoding.UTF8, true, 1 << 20)) + { + string line; + long lineNumber = 0; + while ((line = reader.ReadLine()) != null) + { + lineNumber++; + string trimmed = line.Trim(); + if (trimmed.Length == 0) { needsNormalization = true; continue; } + + string hash; + bool hasCount; + if (!TryParse(trimmed, out hash, out hasCount)) + throw new InvalidDataException(string.Format( + "Weak password hashes file '{0}' contains invalid content at line {1}: '{2}'.", path, lineNumber, trimmed)); + if (hasCount) { legacyEntries++; needsNormalization = true; } + + string upper = hash.ToUpperInvariant(); + if (trimmed != line || hash != upper) needsNormalization = true; + + if (previous != null) + { + int order = string.CompareOrdinal(upper, previous); + if (order < 0) + throw new InvalidDataException(string.Format( + "Weak password hashes file '{0}' is not sorted alphabetically at line {1}.", path, lineNumber)); + if (order == 0) { duplicates++; needsNormalization = true; } + } + previous = upper; + } + } + } + + // Writes the bare, upper-case, de-duplicated hashes of a (previously inspected) file to targetPath. + public static long WriteNormalized(string sourcePath, string targetPath) + { + long written = 0; + string last = null; + using (var reader = new StreamReader(sourcePath, Encoding.UTF8, true, 1 << 20)) + using (var writer = new StreamWriter(targetPath, false, new UTF8Encoding(false), 1 << 20)) + { + string line; + while ((line = reader.ReadLine()) != null) + { + string trimmed = line.Trim(); + if (trimmed.Length == 0) continue; + int colon = trimmed.IndexOf(':'); + string hash = (colon < 0 ? trimmed : trimmed.Substring(0, colon)).ToUpperInvariant(); + if (hash == last) continue; + writer.Write(hash); + writer.Write(Environment.NewLine); + last = hash; + written++; + } + } + return written; + } + // Checks a finished KHDB file (one bare hash per line) in a single pass. Returns the hash count; // the duplicate and out-of-order counts come back through the out parameters. public static long Validate(string path, out long duplicates, out long outOfOrder) diff --git a/src/Elysium/Private/PasswordQuality.ps1 b/src/Elysium/Private/PasswordQuality.ps1 index 699b7e0..eae4998 100644 --- a/src/Elysium/Private/PasswordQuality.ps1 +++ b/src/Elysium/Private/PasswordQuality.ps1 @@ -224,6 +224,8 @@ function Import-CompatModule { } function Resolve-DSInternalsWeakHashFile { + # DSInternals wants a sorted file of bare, upper-case, unique hashes. The KHDB that Update-ElysiumKhdb + # builds already is one, so the normal case is one compiled pass over the file and no copy. param( [Parameter(Mandatory)][string]$Path ) @@ -232,51 +234,11 @@ function Resolve-DSInternalsWeakHashFile { throw "Weak password hashes file not found at '$Path'." } - $compatibleRegex = '^[0-9A-F]{32}$' - $legacyRegex = '^[0-9A-Fa-f]{32}(:\d+)?$' - $lineNumber = 0 - $previousHash = $null - $duplicateCount = 0 - $legacyEntryCount = 0 + Write-Host ("Checking weak password hashes file '{0}' ({1:N1} GB)..." -f $Path, ((Get-Item -LiteralPath $Path).Length / 1GB)) + $legacyEntryCount = [long]0 + $duplicateCount = [long]0 $needsNormalization = $false - $reader = $null - - try { - $reader = New-Object System.IO.StreamReader($Path, [System.Text.Encoding]::UTF8, $true) - while (($line = $reader.ReadLine()) -ne $null) { - $lineNumber++ - $trimmed = $line.Trim() - if ($trimmed.Length -eq 0) { continue } - - if ($trimmed -notmatch $legacyRegex) { - throw ("Weak password hashes file '{0}' contains invalid content at line {1}: '{2}'." -f $Path, $lineNumber, $trimmed) - } - - if ($trimmed -notmatch $compatibleRegex) { - $needsNormalization = $true - if ($trimmed.Contains(':')) { $legacyEntryCount++ } - } - - $normalizedHash = ($trimmed.Split(':', 2)[0]).ToUpperInvariant() - if ($line -cne $trimmed -or $trimmed -cne $normalizedHash) { - $needsNormalization = $true - } - - if ($null -ne $previousHash) { - if ($normalizedHash -lt $previousHash) { - throw "Weak password hashes file '$Path' is not sorted alphabetically at line $lineNumber." - } - if ($normalizedHash -eq $previousHash) { - $duplicateCount++ - $needsNormalization = $true - } - } - - $previousHash = $normalizedHash - } - } finally { - if ($reader) { $reader.Dispose() } - } + [Elysium.KhdbText]::Inspect($Path, [ref]$legacyEntryCount, [ref]$duplicateCount, [ref]$needsNormalization) if (-not $needsNormalization) { return [pscustomobject]@{ @@ -286,29 +248,7 @@ function Resolve-DSInternalsWeakHashFile { } $tmpPath = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), ('elysium-khdb-' + [System.Guid]::NewGuid().ToString() + '.txt')) - $encoding = New-Object System.Text.UTF8Encoding($false) - $reader = $null - $writer = $null - $lastWrittenHash = $null - - try { - $reader = New-Object System.IO.StreamReader($Path, [System.Text.Encoding]::UTF8, $true) - $writer = New-Object System.IO.StreamWriter($tmpPath, $false, $encoding) - - while (($line = $reader.ReadLine()) -ne $null) { - $trimmed = $line.Trim() - if ($trimmed.Length -eq 0) { continue } - - $normalizedHash = ($trimmed.Split(':', 2)[0]).ToUpperInvariant() - if ($normalizedHash -eq $lastWrittenHash) { continue } - - $writer.WriteLine($normalizedHash) - $lastWrittenHash = $normalizedHash - } - } finally { - if ($reader) { $reader.Dispose() } - if ($writer) { $writer.Dispose() } - } + [void][Elysium.KhdbText]::WriteNormalized($Path, $tmpPath) $normalizationReasons = @() if ($legacyEntryCount -gt 0) { $normalizationReasons += "$legacyEntryCount legacy HASH:count entries" } diff --git a/tests/Elysium.Tests.ps1 b/tests/Elysium.Tests.ps1 index 4f0d757..08726a2 100644 --- a/tests/Elysium.Tests.ps1 +++ b/tests/Elysium.Tests.ps1 @@ -325,6 +325,20 @@ Describe 'Password-quality helpers' { } } + It 'normalizes a file with a blank line or a padded line instead of handing it to DSInternals as is' { + foreach ($case in @{ name = 'blank'; lines = @(('0' * 32), '', ('A' * 32)) }, @{ name = 'padded'; lines = @(('0' * 32), (' ' + 'A' * 32)) }) { + $path = Join-Path $TestDrive "$($case.name).txt" + Set-Content -LiteralPath $path -Value $case.lines + $resolved = Resolve-DSInternalsWeakHashFile -Path $path 3>$null 6>$null + try { + $resolved.IsTemporary | Should -BeTrue + Get-Content -LiteralPath $resolved.Path | Should -Be @(('0' * 32), ('A' * 32)) + } finally { + Remove-Item -LiteralPath $resolved.Path -ErrorAction SilentlyContinue + } + } + } + It 'rejects unsorted and malformed KHDB files' { $unsorted = Join-Path $TestDrive 'unsorted.txt' Set-Content -LiteralPath $unsorted -Value @(('B' * 32), ('A' * 32))