Merge pull request #108 from CriticalSolutionsNetwork/release-branch

fix: working and verbose confirmation included

CHANGELOG.md

@@ -4,6 +4,16 @@ The format is based on and uses the types of changes according to [Keep a Change
 
 ## [Unreleased]
 
+### Fixed
+
+- Fixed bug in 1.1.1 that caused the test to fail/pass incorrectly. Added verbose output.
+
+### Docs
+
+- Updated helper csv formatting for one cis control.
+
+## [0.1.8] - 2024-06-09
+
 ### Added
 
 - Added output type to functions.

helpers/Build-Help.ps1

@@ -4,7 +4,7 @@ Import-Module .\output\module\M365FoundationsCISReport\*\*.psd1
 
 
 <#
-    $ver = "v0.1.7"
+    $ver = "v0.1.8"
     git checkout main
     git pull origin main
     git tag -a $ver -m "Release version $ver refactor Update"

source/helper/TestDefinitions.csv

@@ -10,7 +10,7 @@
 9,Test-CommonAttachmentFilter.ps1,2.1.2,Ensure the Common Attachment Types Filter is enabled,E3,L1,9.6,Block Unnecessary File Types,FALSE,TRUE,TRUE,TRUE,EXO
 10,Test-NotifyMalwareInternal.ps1,2.1.3,Ensure notifications for internal users sending malware is Enabled,E3,L1,17.5,Assign Key Roles and Responsibilities,FALSE,TRUE,TRUE,TRUE,EXO
 11,Test-SafeAttachmentsPolicy.ps1,2.1.4,Ensure Safe Attachments policy is enabled,E5,L2,9.7,Deploy and Maintain Email Server Anti-Malware Protections,FALSE,FALSE,TRUE,TRUE,EXO
-12,Test-SafeAttachmentsTeams.ps1,2.1.5,"Ensure Safe Attachments for SharePoint, OneDrive, and Microsoft Teams is Enabled",E5,L2,"9.7, 10.1","Deploy and Maintain Email Server Anti-Malware Protections, Deploy and Maintain Anti-Malware Software",TRUE,TRUE,TRUE,TRUE,EXO
+12,Test-SafeAttachmentsTeams.ps1,2.1.5,"Ensure Safe Attachments for SharePoint, OneDrive, and Microsoft Teams is Enabled",E5,L2,"9.7,10.1","Deploy and Maintain Email Server Anti-Malware Protections, Deploy and Maintain Anti-Malware Software",TRUE,TRUE,TRUE,TRUE,EXO
 13,Test-SpamPolicyAdminNotify.ps1,2.1.6,Ensure Exchange Online Spam Policies are set to notify administrators,E3,L1,17.5,Assign Key Roles and Responsibilities,FALSE,TRUE,TRUE,TRUE,EXO
 14,Test-AntiPhishingPolicy.ps1,2.1.7,Ensure that an anti-phishing policy has been created,E5,L1,9.7,Deploy and Maintain Email Server Anti-Malware Protections,FALSE,FALSE,TRUE,TRUE,EXO
 15,Test-EnableDKIM.ps1,2.1.9,Ensure that DKIM is enabled for all Exchange Online Domains,E3,L1,9.5,Implement DMARC,FALSE,TRUE,TRUE,TRUE,EXO

source/tests/Test-AdministrativeAccountCompliance.ps1

@@ -1,91 +1,111 @@
 function Test-AdministrativeAccountCompliance {
     [CmdletBinding()]
-    [OutputType([CISAuditResult])]
     param (
+        # Aligned
         # Parameters can be added if needed
     )
 
     begin {
+        # The following conditions are checked:
+        # Condition A: The administrative account is cloud-only (not synced).
+        # Condition B: The account is assigned a valid license (e.g., Microsoft Entra ID P1 or P2).
+        # Condition C: The administrative account does not have any other application assignments (only valid licenses).
+
         $validLicenses = @('AAD_PREMIUM', 'AAD_PREMIUM_P2')
         $recnum = "1.1.1"
+        Write-Verbose "Starting Test-AdministrativeAccountCompliance with Rec: $recnum"
     }
 
     process {
         try {
-            # Retrieve all necessary data outside the loops
+            # Retrieve all admin roles
+            Write-Verbose "Retrieving all admin roles"
             $adminRoles = Get-MgRoleManagementDirectoryRoleDefinition | Where-Object { $_.DisplayName -like "*Admin*" }
-            $roleAssignments = Get-MgRoleManagementDirectoryRoleAssignment
-            $principalIds = $roleAssignments.PrincipalId | Select-Object -Unique
-
-            # Fetch user details using filter
-            $userDetailsList = @{}
-            $licensesList = @{}
-
-            $userDetails = Get-MgUser -Filter "id in ('$($principalIds -join "','")')" -Property "DisplayName, UserPrincipalName, Id, OnPremisesSyncEnabled" -ErrorAction SilentlyContinue
-            foreach ($user in $userDetails) {
-                $userDetailsList[$user.Id] = $user
-            }
-
-            # Fetch user licenses for each unique principal ID
-            foreach ($principalId in $principalIds) {
-                $licensesList[$principalId] = Get-MgUserLicenseDetail -UserId $principalId -ErrorAction SilentlyContinue
-            }
-
             $adminRoleUsers = @()
 
+            # Loop through each admin role to get role assignments and user details
             foreach ($role in $adminRoles) {
-                foreach ($assignment in $roleAssignments | Where-Object { $_.RoleDefinitionId -eq $role.Id }) {
+                Write-Verbose "Processing role: $($role.DisplayName)"
-                    $userDetails = $userDetailsList[$assignment.PrincipalId]
+                $roleAssignments = Get-MgRoleManagementDirectoryRoleAssignment -Filter "roleDefinitionId eq '$($role.Id)'"
+
+                foreach ($assignment in $roleAssignments) {
+                    Write-Verbose "Processing role assignment for principal ID: $($assignment.PrincipalId)"
+                    # Get user details for each principal ID
+                    $userDetails = Get-MgUser -UserId $assignment.PrincipalId -Property "DisplayName, UserPrincipalName, Id, OnPremisesSyncEnabled" -ErrorAction SilentlyContinue
                     if ($userDetails) {
-                        $licenses = $licensesList[$assignment.PrincipalId]
+                        Write-Verbose "Retrieved user details for: $($userDetails.UserPrincipalName)"
+                        # Get user license details
+                        $licenses = Get-MgUserLicenseDetail -UserId $assignment.PrincipalId -ErrorAction SilentlyContinue
                         $licenseString = if ($licenses) { ($licenses.SkuPartNumber -join '|') } else { "No Licenses Found" }
 
+                        # Condition A: Check if the account is cloud-only
+                        $cloudOnlyStatus = if ($userDetails.OnPremisesSyncEnabled) { "Fail" } else { "Pass" }
+
+                        # Condition B: Check if the account has valid licenses
+                        $hasValidLicense = $licenses.SkuPartNumber | ForEach-Object { $validLicenses -contains $_ }
+                        $validLicensesStatus = if ($hasValidLicense) { "Pass" } else { "Fail" }
+
+                        # Condition C: Check if the account has no other licenses
+                        $hasInvalidLicense = $licenses.SkuPartNumber | ForEach-Object { $validLicenses -notcontains $_ }
+                        $applicationAssignmentStatus = if ($hasInvalidLicense) { "Fail" } else { "Pass" }
+
+                        Write-Verbose "User: $($userDetails.UserPrincipalName), Cloud-Only: $cloudOnlyStatus, Valid Licenses: $validLicensesStatus, Other Applications Assigned: $applicationAssignmentStatus"
+
+                        # Collect user information
                         $adminRoleUsers += [PSCustomObject]@{
-                            UserName   = $userDetails.UserPrincipalName
+                            UserName                    = $userDetails.UserPrincipalName
-                            RoleName   = $role.DisplayName
+                            RoleName                    = $role.DisplayName
-                            UserId     = $userDetails.Id
+                            UserId                      = $userDetails.Id
-                            HybridUser = $userDetails.OnPremisesSyncEnabled
+                            HybridUser                  = $userDetails.OnPremisesSyncEnabled
-                            Licenses   = $licenseString
+                            Licenses                    = $licenseString
+                            CloudOnlyStatus             = $cloudOnlyStatus
+                            ValidLicensesStatus         = $validLicensesStatus
+                            ApplicationAssignmentStatus = $applicationAssignmentStatus
                         }
                     }
+                    else {
+                        Write-Verbose "No user details found for principal ID: $($assignment.PrincipalId)"
+                    }
                 }
             }
 
+            # Group admin role users by UserName and collect unique roles and licenses
+            Write-Verbose "Grouping admin role users by UserName"
             $uniqueAdminRoleUsers = $adminRoleUsers | Group-Object -Property UserName | ForEach-Object {
                 $first = $_.Group | Select-Object -First 1
                 $roles = ($_.Group.RoleName -join ', ')
                 $licenses = (($_.Group | Select-Object -ExpandProperty Licenses) -join ',').Split(',') | Select-Object -Unique
 
-                $first | Select-Object UserName, UserId, HybridUser, @{Name = 'Roles'; Expression = { $roles } }, @{Name = 'Licenses'; Expression = { $licenses -join '|' } }
+                $first | Select-Object UserName, UserId, HybridUser, @{Name = 'Roles'; Expression = { $roles } }, @{Name = 'Licenses'; Expression = { $licenses -join '|' } }, CloudOnlyStatus, ValidLicensesStatus, ApplicationAssignmentStatus
             }
 
+            # Identify non-compliant users based on conditions A, B, and C
+            Write-Verbose "Identifying non-compliant users based on conditions"
             $nonCompliantUsers = $uniqueAdminRoleUsers | Where-Object {
-                $_.HybridUser -or
+                $_.HybridUser -or # Fails Condition A
-                -not ($_.Licenses -split '\|' | Where-Object { $validLicenses -contains $_ })
+                $_.ValidLicensesStatus -eq "Fail" -or # Fails Condition B
+                $_.ApplicationAssignmentStatus -eq "Fail" # Fails Condition C
             }
 
+            # Generate failure reasons
+            Write-Verbose "Generating failure reasons for non-compliant users"
             $failureReasons = $nonCompliantUsers | ForEach-Object {
-                $accountType = if ($_.HybridUser) { "Hybrid" } else { "Cloud-Only" }
+                "$($_.UserName)|$($_.Roles)|$($_.CloudOnlyStatus)|$($_.ValidLicensesStatus)|$($_.ApplicationAssignmentStatus)"
-                $missingLicenses = $validLicenses | Where-Object { $_ -notin ($_.Licenses -split '\|') }
-                "$($_.UserName)|$($_.Roles)|$accountType|$($missingLicenses -join ',')"
             }
             $failureReasons = $failureReasons -join "`n"
-
+            $failureReason = if ($nonCompliantUsers) {
-            $details = if ($nonCompliantUsers) {
+                "Non-Compliant Accounts: $($nonCompliantUsers.Count)"
-                "Non-compliant accounts: `nUsername | Roles | HybridStatus | Missing Licence`n$failureReasons"
             } else {
                 "Compliant Accounts: $($uniqueAdminRoleUsers.Count)"
             }
 
-            $failureReason = if ($nonCompliantUsers) {
-                "Non-Compliant Accounts: $($nonCompliantUsers.Count)`nDetails:`n" + ($nonCompliantUsers | ForEach-Object { $_.UserName }) -join "`n"
-            } else {
-                "N/A"
-            }
-
             $result = $nonCompliantUsers.Count -eq 0
             $status = if ($result) { 'Pass' } else { 'Fail' }
+            $details = if ($nonCompliantUsers) { "Non-compliant accounts: `nUsername | Roles | Cloud-Only Status | Entra ID License Status | Other Applications Assigned Status`n$failureReasons" } else { "N/A" }
 
+            Write-Verbose "Assessment completed. Result: $status"
+
+            # Create the parameter splat
             $params = @{
                 Rec           = $recnum
                 Result        = $result
@@ -99,6 +119,7 @@ function Test-AdministrativeAccountCompliance {
         catch {
             Write-Error "An error occurred during the test: $_"
 
+            # Handle the error and create a failure result
             $testDefinition = $script:TestDefinitionsObject | Where-Object { $_.Rec -eq $recnum }
             $description = if ($testDefinition) { $testDefinition.RecDescription } else { "Description not found" }
 
@@ -107,7 +128,9 @@ function Test-AdministrativeAccountCompliance {
             $auditResult = Initialize-CISAuditResult -Rec $recnum -Failure
         }
     }
+
     end {
+        # Output the result
         return $auditResult
     }
 }

test-gh.ps1

@@ -0,0 +1,212 @@
+$repoOwner = "CriticalSolutionsNetwork"
+$repoName = "M365FoundationsCISReport"
+$directoryPath = ".\source\tests"
+$projectName = "Test Validation Project"
+
+# Function to create GitHub issues
+function Create-GitHubIssue {
+    param (
+        [string]$title,
+        [string]$body,
+        [string]$project
+    )
+
+    # Create the issue and add it to the specified project
+    $issue = gh issue create --repo "$repoOwner/$repoName" --title "$title" --body "$body" --project "$project"
+    return $issue
+}
+
+# Load test definitions from CSV
+$testDefinitionsPath = ".\source\helper\TestDefinitions.csv"
+$testDefinitions = Import-Csv -Path $testDefinitionsPath
+
+# Iterate over each .ps1 file in the directory
+Get-ChildItem -Path $directoryPath -Filter "*.ps1" | ForEach-Object {
+    $fileName = $_.Name
+    $testDefinition = $testDefinitions | Where-Object { $_.TestFileName -eq $fileName }
+
+    if ($testDefinition) {
+        $rec = $testDefinition.Rec
+        $elevel = $testDefinition.ELevel
+        $profileLevel = $testDefinition.ProfileLevel
+        $ig1 = $testDefinition.IG1
+        $ig2 = $testDefinition.IG2
+        $ig3 = $testDefinition.IG3
+        $connection = $testDefinition.Connection
+
+        $issueTitle = "Rec: $rec - Validate $fileName, ELevel: $elevel, ProfileLevel: $profileLevel, IG1: $ig1, IG2: $ig2, IG3: $ig3, Connection: $connection"
+        $issueBody = @"
+# Validation for $fileName
+
+## Tasks
+- [ ] Validate test for a pass
+  - Description of passing criteria:
+- [ ] Validate test for a fail
+  - Description of failing criteria:
+- [ ] Add notes and observations
+  - Placeholder for additional notes:
+"@
+
+        # Create the issue using GitHub CLI
+        try {
+            Create-GitHubIssue -title "$issueTitle" -body "$issueBody" -project "$projectName"
+            Write-Output "Created issue for $fileName"
+        }
+        catch {
+            Write-Error "Failed to create issue for $fileName`: $_"
+        }
+
+        # Introduce a delay of 2 seconds
+        Start-Sleep -Seconds 2
+    }
+    else {
+        Write-Warning "No matching test definition found for $fileName"
+    }
+}
+######################################
+$repoOwner = "CriticalSolutionsNetwork"
+$repoName = "M365FoundationsCISReport"
+
+# Function to update GitHub issue
+function Update-GitHubTIssue {
+    param (
+        [int]$issueNumber,
+        [string]$title,
+        [string]$body,
+        [string]$owner,
+        [string]$repositoryName
+    )
+
+    # Update the issue using Set-GitHubIssue
+    Set-GitHubIssue -OwnerName $owner -RepositoryName $repositoryName -Issue $issueNumber -Title $title -Body $body -Label @("documentation", "help wanted", "question") -Confirm:$false
+}
+
+# Load test definitions from CSV
+$testDefinitionsPath = ".\source\helper\TestDefinitions.csv"
+$testDefinitions = Import-Csv -Path $testDefinitionsPath
+
+# Fetch existing issues that start with "Rec:"
+$existingIssues = Get-GitHubIssue -OwnerName 'CriticalSolutionsNetwork' -RepositoryName 'M365FoundationsCISReport'
+
+# Create a list to hold matched issues
+$matchedIssues = @()
+$warnings = @()
+
+# Iterate over each existing issue
+$existingIssues | ForEach-Object {
+    $issueNumber = $_.Number
+    $issueTitle = $_.Title
+    $issueBody = $_.Body
+
+    # Extract the rec number from the issue title
+    if ($issueTitle -match "Rec: (\d+\.\d+\.\d+)") {
+        $rec = $matches[1]
+
+        # Find the matching test definition based on rec number
+        $testDefinition = $testDefinitions | Where-Object { $_.Rec -eq $rec }
+
+        if ($testDefinition) {
+            # Create the new issue body
+            $newIssueBody = @"
+# Validation for $($testDefinition.TestFileName)
+
+## Recommendation Details
+- **Recommendation**: $($testDefinition.Rec)
+- **Description**: $($testDefinition.RecDescription)
+- **ELevel**: $($testDefinition.ELevel)
+- **Profile Level**: $($testDefinition.ProfileLevel)
+- **CIS Control**: $($testDefinition.CISControl)
+- **CIS Description**: $($testDefinition.CISDescription)
+- **Implementation Group 1**: $($testDefinition.IG1)
+- **Implementation Group 2**: $($testDefinition.IG2)
+- **Implementation Group 3**: $($testDefinition.IG3)
+- **Automated**: $($testDefinition.Automated)
+- **Connection**: $($testDefinition.Connection)
+
+## [$($testDefinition.TestFileName)](https://github.com/CriticalSolutionsNetwork/M365FoundationsCISReport/blob/main/source/tests/$($testDefinition.TestFileName))
+
+## Tasks
+
+### Validate recommendation details
+- [ ] Confirm that the recommendation details are accurate and complete as per the CIS benchmark.
+
+### Validate test for a pass
+- [ ] Confirm that the automated test results align with the manual audit steps outlined in the CIS benchmark.
+  - Specific conditions to check:
+    - Condition A: (Detail about what constitutes Condition A)
+    - Condition B: (Detail about what constitutes Condition B)
+    - Condition C: (Detail about what constitutes Condition C)
+
+### Validate test for a fail
+- [ ] Confirm that the failure conditions in the automated test are consistent with the manual audit results.
+  - Specific conditions to check:
+    - Condition A: (Detail about what constitutes Condition A)
+    - Condition B: (Detail about what constitutes Condition B)
+    - Condition C: (Detail about what constitutes Condition C)
+
+### Add notes and observations
+- [ ] Compare the automated audit results with the manual audit steps and provide detailed observations.
+  - Automated audit produced info consistent with the manual audit test results? (Yes/No)
+  - Without disclosing any sensitive information, document any discrepancies between the actual output and the expected output.
+  - Document any error messages, removing any sensitive information before submitting.
+  - Identify the specific function, line, or section of the script that failed, if known.
+  - Provide any additional context or observations that might help in troubleshooting.
+
+If needed, the helpers folder in .\source\helpers contains a CSV to assist with locating the test definition.
+"@
+
+            # Add to matched issues list
+            $matchedIssues += [PSCustomObject]@{
+                IssueNumber = $issueNumber
+                Title = $issueTitle
+                NewBody = $newIssueBody
+            }
+        } else {
+            $warnings += "No matching test definition found for Rec: $rec"
+        }
+    } else {
+        $warnings += "No matching rec number found in issue title #$issueNumber"
+    }
+}
+
+# Display matched issues for confirmation
+if ($matchedIssues.Count -gt 0) {
+    Write-Output "Matched Issues:"
+    $matchedIssues | ForEach-Object {
+        Write-Output $_.Title
+    }
+
+    $confirmation = Read-Host "Do you want to proceed with updating these issues? (yes/no)"
+
+    if ($confirmation -eq 'yes') {
+        # Update the issues
+        $matchedIssues | ForEach-Object {
+            try {
+                Update-GitHubTIssue -issueNumber $_.IssueNumber -title $_.Title -body $_.NewBody -owner $repoOwner -repositoryName $repoName
+                Write-Output "Updated issue #$($_.IssueNumber)"
+            } catch {
+                Write-Error "Failed to update issue #$($_.IssueNumber): $_"
+            }
+
+            # Introduce a delay of 2 seconds
+            Start-Sleep -Seconds 2
+        }
+    } else {
+        Write-Output "Update canceled by user."
+    }
+} else {
+    Write-Output "No matched issues found to update."
+}
+
+# Display any warnings that were captured
+if ($warnings.Count -gt 0) {
+    Write-Output "Warnings:"
+    $warnings | ForEach-Object {
+        Write-Output $_
+    }
+}
+
+# Test command to verify GitHub access
+Get-GitHubRepository -OwnerName 'CriticalSolutionsNetwork' -RepositoryName 'M365FoundationsCISReport'
+
+