monitor.go

// Package monitor implements a goroutine based monitoring for
// detecting stuck scanner processes and dumping stack and other
// relevant information for investigation.
package monitor

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/DataDog/gostackparse"
	"github.com/projectdiscovery/gologger"
	permissionutil "github.com/projectdiscovery/utils/permission"
	unitutils "github.com/projectdiscovery/utils/unit"
	"github.com/rs/xid"
)

// Agent is an agent for monitoring hanging programs
type Agent struct {
	lastStack []string
	callbacks []Callback

	goroutineCount   int
	currentIteration int // number of times we've checked hang

	lock sync.Mutex
}

const defaultMonitorIteration = 6

// NewStackMonitor returns a new stack monitor instance
func NewStackMonitor() *Agent {
	return &Agent{}
}

// Callback when crash is detected and stack trace is saved to disk
type Callback func(dumpID string) error

// RegisterCallback adds a callback to perform additional operations before bailing out.
func (s *Agent) RegisterCallback(callback Callback) {
	s.lock.Lock()
	defer s.lock.Unlock()

	s.callbacks = append(s.callbacks, callback)
}

func (s *Agent) Start(interval time.Duration) context.CancelFunc {
	ctx, cancel := context.WithCancel(context.Background())
	ticker := time.NewTicker(interval)

	go func() {
		for {
			select {
			case <-ctx.Done():
				ticker.Stop()
			case <-ticker.C:
				s.monitorWorker(cancel)
			default:
				continue
			}
		}
	}()
	return cancel
}

// monitorWorker is a worker for monitoring running goroutines
func (s *Agent) monitorWorker(cancel context.CancelFunc) {
	current := runtime.NumGoroutine()
	if current != s.goroutineCount {
		s.goroutineCount = current
		s.currentIteration = 0
		return
	}
	s.currentIteration++

	if s.currentIteration == defaultMonitorIteration-1 {
		lastStackTrace := generateStackTraceSlice(getStack(true))
		s.lastStack = lastStackTrace
		return
	}

	// cancel the monitoring goroutine if we discover
	// we've been stuck for some iterations.
	if s.currentIteration == defaultMonitorIteration {
		currentStack := getStack(true)

		// Bail out if the stacks don't match from previous iteration
		newStack := generateStackTraceSlice(currentStack)
		if !compareStringSliceEqual(s.lastStack, newStack) {
			s.currentIteration = 0
			return
		}

		cancel()
		dumpID := xid.New().String()
		stackTraceFile := fmt.Sprintf("nuclei-stacktrace-%s.dump", dumpID)
		gologger.Error().Msgf("Detected hanging goroutine (count=%d/%d) = %s\n", current, s.goroutineCount, stackTraceFile)
		if err := os.WriteFile(stackTraceFile, currentStack, permissionutil.ConfigFilePermission); err != nil {
			gologger.Error().Msgf("Could not write stack trace for goroutines: %s\n", err)
		}

		s.lock.Lock()
		callbacks := s.callbacks
		s.lock.Unlock()
		for _, callback := range callbacks {
			if err := callback(dumpID); err != nil {
				gologger.Error().Msgf("Stack monitor callback error: %s\n", err)
			}
		}

		os.Exit(1) // exit forcefully if we've been stuck
	}
}

// getStack returns full stack trace of the program
var getStack = func(all bool) []byte {
	for i := unitutils.Mega; ; i *= 2 {
		buf := make([]byte, i)
		if n := runtime.Stack(buf, all); n < i {
			return buf[:n-1]
		}
	}
}

// generateStackTraceSlice returns a list of current stack in string slice format
func generateStackTraceSlice(stack []byte) []string {
	goroutines, _ := gostackparse.Parse(bytes.NewReader(stack))

	var builder strings.Builder
	var stackList []string
	for _, goroutine := range goroutines {
		builder.WriteString(goroutine.State)
		builder.WriteString("|")

		for _, frame := range goroutine.Stack {
			builder.WriteString(frame.Func)
			builder.WriteString(";")
		}
		stackList = append(stackList, builder.String())
		builder.Reset()
	}
	return stackList
}

// compareStringSliceEqual compares two string slices for equality without order
func compareStringSliceEqual(first, second []string) bool {
	if len(first) != len(second) {
		return false
	}
	diff := make(map[string]int, len(first))
	for _, x := range first {
		diff[x]++
	}
	for _, y := range second {
		if _, ok := diff[y]; !ok {
			return false
		}
		diff[y] -= 1
		if diff[y] == 0 {
			delete(diff, y)
		}
	}
	return len(diff) == 0
}