src

monitor.go (11335B)

---

 // Copyright 2024 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 package crashmonitor
 
 // This file defines a monitor that reports arbitrary Go runtime
 // crashes to telemetry.
 
 import (
 	"bytes"
 	"fmt"
 	"io"
 	"log"
 	"os"
 	"reflect"
 	"runtime/debug"
 	"strconv"
 	"strings"
 
 	"golang.org/x/telemetry/internal/counter"
 )
 
 // Parent sets up the parent side of the crashmonitor. It requires
 // exclusive use of a writable pipe connected to the child process's stdin.
 func Parent(pipe *os.File) {
 	writeSentinel(pipe)
 	// Ensure that we get pc=0x%x values in the traceback.
 	debug.SetTraceback("system")
 	debug.SetCrashOutput(pipe, debug.CrashOptions{}) // ignore error
 }
 
 // Child runs the part of the crashmonitor that runs in the child process.
 // It expects its stdin to be connected via a pipe to the parent which has
 // run Parent.
 func Child() {
 	// Wait for parent process's dying gasp.
 	// If the parent dies for any reason this read will return.
 	data, err := io.ReadAll(os.Stdin)
 	if err != nil {
 		log.Fatalf("failed to read from input pipe: %v", err)
 	}
 
 	// If the only line is the sentinel, it wasn't a crash.
 	if bytes.Count(data, []byte("\n")) < 2 {
 		childExitHook()
 		os.Exit(0) // parent exited without crash report
 	}
 
 	log.Printf("parent reported crash:\n%s", data)
 
 	// Parse the stack out of the crash report
 	// and record a telemetry count for it.
 	name, err := telemetryCounterName(data)
 	if err != nil {
 		// Keep count of how often this happens
 		// so that we can investigate if necessary.
 		incrementCounter("crash/malformed")
 
 		// Something went wrong.
 		// Save the crash securely in the file system.
 		f, err := os.CreateTemp(os.TempDir(), "*.crash")
 		if err != nil {
 			log.Fatal(err)
 		}
 		if _, err := f.Write(data); err != nil {
 			log.Fatal(err)
 		}
 		if err := f.Close(); err != nil {
 			log.Fatal(err)
 		}
 		log.Printf("failed to report crash to telemetry: %v", err)
 		log.Fatalf("crash report saved at %s", f.Name())
 	}
 
 	incrementCounter(name)
 
 	childExitHook()
 	log.Fatalf("telemetry crash recorded")
 }
 
 // (stubbed by test)
 var (
 	incrementCounter = func(name string) { counter.New(name).Inc() }
 	childExitHook    = func() {}
 )
 
 // The sentinel function returns its address. The difference between
 // this value as observed by calls in two different processes of the
 // same executable tells us the relative offset of their text segments.
 //
 // It would be nice if SetCrashOutput took care of this as it's fiddly
 // and likely to confuse every user at first.
 func sentinel() uint64 {
 	return uint64(reflect.ValueOf(sentinel).Pointer())
 }
 
 func writeSentinel(out io.Writer) {
 	fmt.Fprintf(out, "sentinel %x\n", sentinel())
 }
 
 // telemetryCounterName parses a crash report produced by the Go
 // runtime, extracts the stack of the first runnable goroutine,
 // converts each line into telemetry form ("symbol:relative-line"),
 // and returns this as the name of a counter.
 func telemetryCounterName(crash []byte) (string, error) {
 	pcs, err := parseStackPCs(string(crash))
 	if err != nil {
 		return "", err
 	}
 
 	// Limit the number of frames we request.
 	pcs = pcs[:min(len(pcs), 16)]
 
 	if len(pcs) == 0 {
 		// This can occur if all goroutines are idle, as when
 		// caught in a deadlock, or killed by an async signal
 		// while blocked.
 		//
 		// TODO(adonovan): consider how to report such
 		// situations. Reporting a goroutine in [sleep] or
 		// [select] state could be quite confusing without
 		// further information about the nature of the crash,
 		// as the problem is not local to the code location.
 		//
 		// For now, we keep count of this situation so that we
 		// can access whether it needs a more involved solution.
 		return "crash/no-running-goroutine", nil
 	}
 
 	// This string appears at the start of all
 	// crashmonitor-generated counter names.
 	//
 	// It is tempting to expose this as a parameter of Start, but
 	// it is not without risk. What value should most programs
 	// provide? There's no point giving the name of the executable
 	// as this is already recorded by telemetry. What if the
 	// application runs in multiple modes? Then it might be useful
 	// to record the mode. The problem is that an application with
 	// multiple modes probably doesn't know its mode by line 1 of
 	// main.main: it might require flag or argument parsing, or
 	// even validation of an environment variable, and we really
 	// want to steer users aware from any logic before Start. The
 	// flags and arguments will be wrong in the child process, and
 	// every extra conditional branch creates a risk that the
 	// recursively executed child program will behave not like the
 	// monitor but like the application. If the child process
 	// exits before calling Start, then the parent application
 	// will not have a monitor, and its crash reports will be
 	// discarded (written in to a pipe that is never read).
 	//
 	// So for now, we use this constant string.
 	const prefix = "crash/crash"
 	return counter.EncodeStack(pcs, prefix), nil
 }
 
 // parseStackPCs parses the parent process's program counters for the
 // first running goroutine out of a GOTRACEBACK=system traceback,
 // adjusting them so that they are valid for the child process's text
 // segment.
 //
 // This function returns only program counter values, ensuring that
 // there is no possibility of strings from the crash report (which may
 // contain PII) leaking into the telemetry system.
 func parseStackPCs(crash string) ([]uintptr, error) {
 	// getSymbol parses the symbol name out of a line of the form:
 	// SYMBOL(ARGS)
 	//
 	// Note: SYMBOL may contain parens "pkg.(*T).method". However, type
 	// parameters are always replaced with ..., so they cannot introduce
 	// more parens. e.g., "pkg.(*T[...]).method".
 	//
 	// ARGS can contain parens. We want the first paren that is not
 	// immediately preceded by a ".".
 	//
 	// TODO(prattmic): This is mildly complicated and is only used to find
 	// runtime.sigpanic, so perhaps simplify this by checking explicitly
 	// for sigpanic.
 	getSymbol := func(line string) (string, error) {
 		var prev rune
 		for i, c := range line {
 			if line[i] != '(' {
 				prev = c
 				continue
 			}
 			if prev == '.' {
 				prev = c
 				continue
 			}
 			return line[:i], nil
 		}
 		return "", fmt.Errorf("no symbol for stack frame: %s", line)
 	}
 
 	// getPC parses the PC out of a line of the form:
 	//     \tFILE:LINE +0xRELPC sp=... fp=... pc=...
 	getPC := func(line string) (uint64, error) {
 		_, pcstr, ok := strings.Cut(line, " pc=") // e.g. pc=0x%x
 		if !ok {
 			return 0, fmt.Errorf("no pc= for stack frame: %s", line)
 		}
 		return strconv.ParseUint(pcstr, 0, 64) // 0 => allow 0x prefix
 	}
 
 	type goroutine struct {
 		id     uint64
 		pcs    []uintptr
 		system bool // stack starts with runtime.systemstack_switch
 	}
 
 	var (
 		glist          []*goroutine
 		parentSentinel uint64
 		childSentinel  = sentinel()
 		lines          = strings.Split(crash, "\n")
 
 		// Parsing state for the current goroutine.
 		g          *goroutine
 		symLine    = true // a symbol (not a filename) is expected on this line
 		currSymbol string
 		prevSymbol string
 	)
 
 	for _, line := range lines {
 		// Read sentinel value.
 		if parentSentinel == 0 && strings.HasPrefix(line, "sentinel ") {
 			_, err := fmt.Sscanf(line, "sentinel %x", &parentSentinel)
 			if err != nil {
 				return nil, fmt.Errorf("can't read sentinel line")
 			}
 			continue
 		}
 
 		// Check for a "goroutine GID [STATUS]" header.
 		if strings.HasPrefix(line, "goroutine ") {
 			if parentSentinel == 0 {
 				return nil, fmt.Errorf("no sentinel value in crash report")
 			}
 			isG0 := strings.HasPrefix(line, "goroutine 0 ")
 			isRunning := strings.Contains(line, " [running]:")
 			if isG0 || isRunning {
 				var id uint64
 				if parts := strings.Fields(line); len(parts) > 1 {
 					id, _ = strconv.ParseUint(parts[1], 10, 64)
 				}
 				g = &goroutine{id: id}
 				glist = append(glist, g)
 				symLine = true
 				currSymbol = ""
 				prevSymbol = ""
 			} else {
 				g = nil
 			}
 			continue
 		}
 
 		if g == nil {
 			continue
 		}
 
 		// A blank line or "created by " marks the end of the goroutine stack.
 		if line == "" || strings.HasPrefix(line, "created by ") {
 			g = nil
 			continue
 		}
 
 		// Expect a pair of lines:
 		//   SYMBOL(ARGS)
 		//   \tFILE:LINE +0xRELPC sp=0x%x fp=0x%x pc=0x%x
 		// Note: SYMBOL may contain parens "pkg.(*T).method"
 		// The RELPC is sometimes missing.
 
 		if symLine {
 			var err error
 			currSymbol, err = getSymbol(line)
 			if err != nil {
 				return nil, fmt.Errorf("error extracting symbol: %v", err)
 			}
 
 			symLine = false // Next line is FILE:LINE.
 		} else {
 			// Parse the PC, and correct for the parent and child's
 			// different mappings of the text section.
 			pc, err := getPC(line)
 			if err != nil {
 				// Inlined frame, perhaps; skip it.
 
 				// Done with this frame. Next line is a new frame.
 				//
 				// Don't update prevSymbol; we only want to
 				// track frames with a PC.
 				currSymbol = ""
 				symLine = true
 				continue
 			}
 
 			pc = pc - parentSentinel + childSentinel
 
 			// If the previous frame was sigpanic, then this frame
 			// was a trap (e.g., SIGSEGV).
 			//
 			// Typically all middle frames are calls, and report
 			// the "return PC". That is, the instruction following
 			// the CALL where the callee will eventually return to.
 			//
 			// runtime.CallersFrames is aware of this property and
 			// will decrement each PC by 1 to "back up" to the
 			// location of the CALL, which is the actual line
 			// number the user expects.
 			//
 			// This does not work for traps, as a trap is not a
 			// call, so the reported PC is not the return PC, but
 			// the actual PC of the trap.
 			//
 			// runtime.Callers is aware of this and will
 			// intentionally increment trap PCs in order to correct
 			// for the decrement performed by
 			// runtime.CallersFrames. See runtime.tracebackPCs and
 			// runtume.(*unwinder).symPC.
 			//
 			// We must emulate the same behavior, otherwise we will
 			// report the location of the instruction immediately
 			// prior to the trap, which may be on a different line,
 			// or even a different inlined functions.
 			//
 			// TODO(prattmic): The runtime applies the same trap
 			// behavior for other "injected calls", see injectCall
 			// in runtime.(*unwinder).next. Do we want to handle
 			// those as well? I don't believe we'd ever see
 			// runtime.asyncPreempt or runtime.debugCallV2 in a
 			// typical crash.
 			if prevSymbol == "runtime.sigpanic" {
 				pc++
 			}
 
 			if len(g.pcs) == 0 && currSymbol == "runtime.systemstack_switch" {
 				g.system = true
 			}
 			g.pcs = append(g.pcs, uintptr(pc))
 
 			// Done with this frame. Next line is a new frame.
 			prevSymbol = currSymbol
 			currSymbol = ""
 			symLine = true
 		}
 	}
 
 	if len(glist) == 0 {
 		return nil, nil
 	}
 
 	// The first goroutine in the dump is the one that crashed.
 	firstG := glist[0]
 
 	// If the first goroutine is g0 (the system stack), we want to find the user
 	// goroutine that called it, which will start with systemstack_switch.
 	if firstG.id == 0 {
 		for _, g := range glist[1:] {
 			if g.system && len(g.pcs) > 0 {
 				// Stitch the g0 stack and user stack (skipping systemstack_switch).
 				return append(firstG.pcs, g.pcs[1:]...), nil
 			}
 		}
 	}
 
 	return firstG.pcs, nil
 }