From 06e7607eff529bd3888d566f84b40b46aaf1bfc1 Mon Sep 17 00:00:00 2001 From: Tomas Kracmar Date: Wed, 29 Jul 2026 12:01:28 +0200 Subject: [PATCH] fix(Prepare-KHDBStorage): fast-path dedup, and a scope bug hiding it Two related bugs in Split-KhdbIntoShards: 1. The fast path for plain 32-hex-char lines wrote every line straight to its shard with no deduplication, unlike the slow path which merges adjacent duplicate hashes via a per-shard pending-line buffer. Routed the fast path through the same pending-line buffer so adjacent duplicates are dropped there too. 2. While fixing that, found $total (the valid-entry counter) was a plain scalar incremented inside $processHashLine, which is invoked via the call operator (&). Scriptblocks invoked with & run in their own child scope, so '$total++' inside them silently incremented a local shadow copy and never updated the real counter in the enclosing function - meaning TotalEntries/ validEntries (used in the returned result, the checkpoint state, and the live progress status) were wrong on every run, previously masked because actual shard file writes go through $state.Writer objects (reference types, unaffected by the scoping issue). Moved the counter onto $meta (already a hashtable used the same way for TotalLines/InvalidLines/etc.) so it mutates by reference across the scope boundary. Verified both with an isolated test harness (function extracted and dot-sourced, no AD/Windows dependency): adjacent AAAA/CCCC duplicates in a 6-line sample now collapse to 3 shard entries, and TotalEntries correctly reports 3 instead of the previous 0/wrong value. Co-Authored-By: Claude Sonnet 5 --- Prepare-KHDBStorage.ps1 | 45 +++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/Prepare-KHDBStorage.ps1 b/Prepare-KHDBStorage.ps1 index e6bf187..829b665 100644 --- a/Prepare-KHDBStorage.ps1 +++ b/Prepare-KHDBStorage.ps1 @@ -241,14 +241,19 @@ function Split-KhdbIntoShards { $shardStates = @{} $stats = @{} - $total = 0L $resumeFilePosition = 0L + # ValidEntries lives on $meta (a hashtable, i.e. reference type) rather than as its own scalar + # variable because $processHashLine below is invoked via the call operator (&), which runs in + # its own child scope - a bare scalar '$total++' inside it would silently increment a local + # shadow copy and never update the caller's variable. Mutating a property on a shared + # hashtable/object works fine across that scope boundary. $meta = @{ TotalLines = 0L InvalidLines = 0L SkippedLines = 0L LegacyLines = 0L + ValidEntries = 0L InvalidSamples = New-Object System.Collections.Generic.List[string] } @@ -271,7 +276,7 @@ function Split-KhdbIntoShards { if ($ResumeState.totalLines) { $meta.TotalLines = [long]$ResumeState.totalLines } if ($ResumeState.invalidLines) { $meta.InvalidLines = [long]$ResumeState.invalidLines } if ($ResumeState.skippedLines) { $meta.SkippedLines = [long]$ResumeState.skippedLines } - if ($ResumeState.validEntries) { $total = [long]$ResumeState.validEntries } + if ($ResumeState.validEntries) { $meta.ValidEntries = [long]$ResumeState.validEntries } if ($ResumeState.filePosition) { $resumeFilePosition = [long]$ResumeState.filePosition } } $plainReader = $null @@ -371,7 +376,7 @@ function Split-KhdbIntoShards { totalLines = [long]$meta.TotalLines invalidLines = [long]$meta.InvalidLines skippedLines = [long]$meta.SkippedLines - validEntries = [long]$total + validEntries = [long]$meta.ValidEntries shardStates = @() } foreach ($entry in ($shardStates.GetEnumerator() | Sort-Object Key)) { @@ -403,7 +408,7 @@ function Split-KhdbIntoShards { if ([string]::IsNullOrWhiteSpace($statusContext)) { $statusContext = if ($currentSource) { Split-Path -Leaf $currentSource } else { 'input' } } - $status = "Processed {0:N0} hashes (+{1:N0} invalid, {2:N0} skipped, {3:N0} lines) [{4}]" -f $total, $meta.InvalidLines, $meta.SkippedLines, $meta.TotalLines, $statusContext + $status = "Processed {0:N0} hashes (+{1:N0} invalid, {2:N0} skipped, {3:N0} lines) [{4}]" -f $meta.ValidEntries, $meta.InvalidLines, $meta.SkippedLines, $meta.TotalLines, $statusContext Write-Progress -Activity $ProgressActivity -Status $status -PercentComplete 0 if ($EnableCheckpoint -and $plainReader) { $checkpointPosition = if ($plainBaseStream) { $plainBaseStream.Position } else { $plainReader.BaseStream.Position } @@ -427,12 +432,26 @@ function Split-KhdbIntoShards { return } - # Fast path for valid 32-char hex lines + # Fast path for valid 32-char hex lines. Routes through the same pending-line dedup as the + # slow path below (adjacent identical hashes are dropped) instead of writing straight + # through - the source is expected to be pre-sorted, so only *adjacent* duplicates are + # caught either way, but the fast path used to skip this entirely and write every + # duplicate straight to the shard. if ($rawLine.Length -eq 32 -and $rawLine -match '^[0-9A-Fa-f]{32}$') { - $prefixKey = $rawLine.Substring(0, $PrefixLength).ToLowerInvariant() - $shardStates[$prefixKey].Writer.WriteLine($rawLine.ToUpperInvariant()) + $normalizedHash = $rawLine.ToUpperInvariant() + $prefixKey = $normalizedHash.Substring(0, $PrefixLength).ToLowerInvariant() + $state = $shardStates[$prefixKey] + if ($state.PendingHash -ne $normalizedHash) { + if ($state.PendingLine) { + $state.Writer.WriteLine($state.PendingLine) + $state.Count++ + $meta.ValidEntries++ + } + $state.PendingLine = $normalizedHash + $state.PendingHash = $normalizedHash + $state.PendingCount = 0 + } $meta.TotalLines++ - $total++ return } @@ -517,7 +536,7 @@ function Split-KhdbIntoShards { if ($state.PendingLine) { $state.Writer.WriteLine($state.PendingLine) $state.Count++ - $total++ + $meta.ValidEntries++ } $state.PendingLine = $normalizedLine $state.PendingHash = $normalizedHash @@ -651,7 +670,7 @@ function Split-KhdbIntoShards { if ($state.PendingLine) { $state.Writer.WriteLine($state.PendingLine) $state.Count++ - $total++ + $meta.ValidEntries++ $state.PendingLine = $null } $state.Writer.Dispose() @@ -661,13 +680,13 @@ function Split-KhdbIntoShards { } } - if ($total -eq 0) { throw 'Source did not contain any valid hashes after processing.' } + if ($meta.ValidEntries -eq 0) { throw 'Source did not contain any valid hashes after processing.' } if ($ShowProgress) { - $status = "Processed {0:N0} hashes (+{1:N0} invalid, {2:N0} skipped, {3:N0} lines)" -f $total, $meta.InvalidLines, $meta.SkippedLines, $meta.TotalLines + $status = "Processed {0:N0} hashes (+{1:N0} invalid, {2:N0} skipped, {3:N0} lines)" -f $meta.ValidEntries, $meta.InvalidLines, $meta.SkippedLines, $meta.TotalLines Write-Progress -Activity $ProgressActivity -Status $status -Completed } return [pscustomobject]@{ - TotalEntries = [long]$total + TotalEntries = [long]$meta.ValidEntries ShardStats = $stats TotalLines = [long]$meta.TotalLines InvalidLines = [long]$meta.InvalidLines