Detecting and Restarting High-CPU Windows Services with PowerShell

By /
PowerShell
<#
.SYNOPSIS
Monitors Windows service threads for sustained high CPU usage and restarts
the affected service when the configured CPU threshold is exceeded.

.DESCRIPTION
Samples the processor time of each thread belonging to the configured
Windows services over a specified interval. If a thread's average CPU
usage meets or exceeds the configured threshold, the associated service
is restarted.

The service is first given an opportunity to stop normally. If its process
does not exit within the configured timeout, the process is forcefully
terminated before the service is restarted.
#>

# Configuration
$sampleTimeSeconds = 30
$usageThresholdPercentage = 90
$maxServiceWaitTimeSeconds = 30
$servicesToCheck = "FCCServerSvc", "IvAppMonSvc"

# Gets the threads associated with each monitored service and captures
# the total processor time used by each thread.
function Get-ServiceThreads($Services) {
    $Services | ForEach-Object {

        # Get the service and its associated process ID.
        $service = Get-CimInstance -Query "SELECT Name, ProcessId FROM Win32_Service WHERE Name = '$($_)'"
        if ($null -eq $service) { return }

        # Get the process and enumerate its threads.
        # If the service is not currently running, no threads are returned.
        $proc = Get-Process -Id $service.ProcessId -ErrorAction SilentlyContinue
        $proc.Threads | ForEach-Object {
            [PSCustomObject]@{
                ServiceName = $service.Name
                ProcessName = $proc.Name
                ProcessId = $proc.Id
                ThreadId = $_.Id
                TotalProcessorTime = $_.TotalProcessorTime
            }
        }
    }
}

# Take the first CPU time sample for all service threads.
$threads1 = Get-ServiceThreads $servicesToCheck

# Wait before taking the second sample so CPU usage can be calculated
# from the change in processor time.
Start-Sleep -Seconds $sampleTimeSeconds

# Take the second CPU time sample.
$threads2 = Get-ServiceThreads $servicesToCheck


# Compare the two samples and identify threads that consumed at least
# the configured percentage of a logical processor during the sample period.
$spinningThreads = $threads1 | ForEach-Object {
    $thread1 = $_
    $threads2 | ForEach-Object {
        $thread2 = $_
        if ($thread1.ThreadId -eq $thread2.ThreadId) {
            $threadTimeDelta = $thread2.TotalProcessorTime - $thread1.TotalProcessorTime

            # Calculate average CPU utilization for this thread over the sample period.
            $threadUsage = (($threadTimeDelta.TotalMilliseconds / ($sampleTimeSeconds * 1000)) * 100)
            if ($threadUsage -ge $usageThresholdPercentage) {
                $thread1 | Add-Member -NotePropertyName TotalProcessorTimeDelta -NotePropertyValue $threadTimeDelta
                $thread1 | Add-Member -NotePropertyName TotalProcessorUsagePercentage -NotePropertyValue $threadUsage
                $thread1
            }
        }
    }
}

# Restart each service containing one or more threads that exceeded
# the CPU usage threshold. Selecting unique service/process combinations
# prevents a service from being restarted multiple times when several
# of its threads exceed the threshold.
$spinningThreads | select ServiceName, ProcessId | select -Unique | ForEach-Object {

    # Start a background watchdog that gives the service time to stop normally.
    # If the process is still running after the timeout, repeatedly force
    # termination until the process is confirmed to have exited.
    $killJob = Start-Job -ArgumentList $_.ProcessId, $maxServiceWaitTimeSeconds -ScriptBlock {
        $elapsedSeconds = 0
        while ($true) {
            $proc = Get-Process -Id $args[0] -ErrorAction SilentlyContinue
            if ($null -eq $proc) {
                
                # Exit the watchdog once the service process no longer exists.
                break
            }
            Start-Sleep -Seconds 1
            $elapsedSeconds += 1

            # Allow the service to stop normally until the timeout expires,
            # then force termination on each subsequent check.
            if ($elapsedSeconds -ge $args[1]) {
                Stop-Process -Id $args[0] -Force -ErrorAction SilentlyContinue
            }
        }
    }

    # Request that the service stop without blocking this script.
    Stop-Service -Name $_.ServiceName -Force -NoWait -ErrorAction SilentlyContinue

    # Do not continue until the service process has exited or been terminated.
    Wait-Job $killJob | Out-Null

    # Give the system a short settling period before restarting the service.
    Start-Sleep -Seconds 5
    Start-Service -Name $_.ServiceName -ErrorAction SilentlyContinue
}