src

reports.go (10671B)

---

 // Copyright 2023 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 package upload
 
 import (
 	"crypto/rand"
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"math"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"golang.org/x/telemetry/internal/config"
 	"golang.org/x/telemetry/internal/counter"
 	"golang.org/x/telemetry/internal/telemetry"
 )
 
 // reports generates reports from inactive count files
 func (u *uploader) reports(todo *work) ([]string, error) {
 	if mode, _ := u.dir.Mode(); mode == "off" {
 		return nil, nil // no reports
 	}
 	thisInstant := u.startTime
 	today := thisInstant.Format(telemetry.DateOnly)
 	lastWeek := latestReport(todo.uploaded)
 	if lastWeek >= today { //should never happen
 		lastWeek = ""
 	}
 	u.logger.Printf("Last week: %s, today: %s", lastWeek, today)
 	countFiles := make(map[string][]string) // expiry date string->filenames
 	earliest := make(map[string]time.Time)  // earliest begin time for any counter
 	for _, f := range todo.countfiles {
 		begin, end, err := u.counterDateSpan(f)
 		if err != nil {
 			// This shouldn't happen: we should have already skipped count files that
 			// don't contain valid start or end times.
 			u.logger.Printf("BUG: failed to parse expiry for collected count file: %v", err)
 			continue
 		}
 
 		if end.Before(thisInstant) {
 			expiry := end.Format(dateFormat)
 			countFiles[expiry] = append(countFiles[expiry], f)
 			if earliest[expiry].IsZero() || earliest[expiry].After(begin) {
 				earliest[expiry] = begin
 			}
 		}
 	}
 	for expiry, files := range countFiles {
 		if notNeeded(expiry, *todo) {
 			u.logger.Printf("Files for %s not needed, deleting %v", expiry, files)
 			// The report already exists.
 			// There's another check in createReport.
 			u.deleteFiles(files)
 			continue
 		}
 		fname, err := u.createReport(earliest[expiry], expiry, files, lastWeek)
 		if err != nil {
 			u.logger.Printf("Failed to create report for %s: %v", expiry, err)
 			continue
 		}
 		if fname != "" {
 			u.logger.Printf("Ready to upload: %s", filepath.Base(fname))
 			todo.readyfiles = append(todo.readyfiles, fname)
 		}
 	}
 	return todo.readyfiles, nil
 }
 
 // latestReport returns the YYYY-MM-DD of the last report uploaded
 // or the empty string if there are no reports.
 func latestReport(uploaded map[string]bool) string {
 	var latest string
 	for name := range uploaded {
 		if strings.HasSuffix(name, ".json") {
 			if name > latest {
 				latest = name
 			}
 		}
 	}
 	if latest == "" {
 		return ""
 	}
 	// strip off the .json
 	return latest[:len(latest)-len(".json")]
 }
 
 // notNeeded returns true if the report for date has already been created
 func notNeeded(date string, todo work) bool {
 	if todo.uploaded != nil && todo.uploaded[date+".json"] {
 		return true
 	}
 	// maybe the report is already in todo.readyfiles
 	for _, f := range todo.readyfiles {
 		if strings.Contains(f, date) {
 			return true
 		}
 	}
 	return false
 }
 
 func (u *uploader) deleteFiles(files []string) {
 	for _, f := range files {
 		if err := os.Remove(f); err != nil {
 			// this could be a race condition.
 			// conversely, on Windows, err may be nil and
 			// the file not deleted if anyone has it open.
 			u.logger.Printf("%v failed to remove %s", err, f)
 		}
 	}
 }
 
 // createReport creates local and upload report files by
 // combining all the count files for the expiryDate, and
 // returns the upload report file's path.
 // It may delete the count files once local and upload report
 // files are successfully created.
 func (u *uploader) createReport(start time.Time, expiryDate string, countFiles []string, lastWeek string) (string, error) {
 	uploadOK := true
 	mode, asof := u.dir.Mode()
 	if mode != "on" {
 		u.logger.Printf("No upload config or mode %q is not 'on'", mode)
 		uploadOK = false // no config, nothing to upload
 	}
 	if u.tooOld(expiryDate, u.startTime) {
 		u.logger.Printf("Expiry date %s is too old", expiryDate)
 		uploadOK = false
 	}
 	// If the mode is recorded with an asof date, don't upload if the report
 	// includes any data on or before the asof date.
 	if !asof.IsZero() && !asof.Before(start) {
 		u.logger.Printf("As-of date %s is not before start %s", asof, start)
 		uploadOK = false
 	}
 	// TODO(rfindley): check that all the x.Meta are consistent for GOOS, GOARCH, etc.
 	report := &telemetry.Report{
 		Config:   u.configVersion,
 		X:        computeRandom(), // json encodes all the bits
 		Week:     expiryDate,
 		LastWeek: lastWeek,
 	}
 	if report.X > u.config.SampleRate && u.config.SampleRate > 0 {
 		u.logger.Printf("X: %f > SampleRate:%f, not uploadable", report.X, u.config.SampleRate)
 		uploadOK = false
 	}
 	var succeeded bool
 	for _, f := range countFiles {
 		fok := false
 		x, err := u.parseCountFile(f)
 		if err != nil {
 			u.logger.Printf("Unparseable count file %s: %v", filepath.Base(f), err)
 			continue
 		}
 		prog := findProgReport(x.Meta, report)
 		for k, v := range x.Count {
 			if counter.IsStackCounter(k) {
 				// stack
 				prog.Stacks[k] += int64(v)
 			} else {
 				// counter
 				prog.Counters[k] += int64(v)
 			}
 			succeeded = true
 			fok = true
 		}
 		if !fok {
 			u.logger.Printf("no counters found in %s", f)
 		}
 	}
 	if !succeeded {
 		return "", fmt.Errorf("none of the %d count files for %s contained counters", len(countFiles), expiryDate)
 	}
 	// 1. generate the local report
 	localContents, err := json.MarshalIndent(report, "", " ")
 	if err != nil {
 		return "", fmt.Errorf("failed to marshal report for %s: %v", expiryDate, err)
 	}
 	// check that the report can be read back
 	// TODO(pjw): remove for production?
 	var report2 telemetry.Report
 	if err := json.Unmarshal(localContents, &report2); err != nil {
 		return "", fmt.Errorf("failed to unmarshal local report for %s: %v", expiryDate, err)
 	}
 
 	var uploadContents []byte
 	if uploadOK {
 		// 2. create the uploadable version
 		cfg := config.NewConfig(u.config)
 		upload := &telemetry.Report{
 			Week:     report.Week,
 			LastWeek: report.LastWeek,
 			X:        report.X,
 			Config:   report.Config,
 		}
 		for _, p := range report.Programs {
 			// does the uploadConfig want this program?
 			// if so, copy over the Stacks and Counters
 			// that the uploadConfig mentions.
 			if !cfg.HasGoVersion(p.GoVersion) || !cfg.HasProgram(p.Program) || !cfg.HasVersion(p.Program, p.Version) {
 				continue
 			}
 			x := &telemetry.ProgramReport{
 				Program:   p.Program,
 				Version:   p.Version,
 				GOOS:      p.GOOS,
 				GOARCH:    p.GOARCH,
 				GoVersion: p.GoVersion,
 				Counters:  make(map[string]int64),
 				Stacks:    make(map[string]int64),
 			}
 			upload.Programs = append(upload.Programs, x)
 			for k, v := range p.Counters {
 				if cfg.HasCounter(p.Program, k) && report.X <= cfg.Rate(p.Program, k) {
 					x.Counters[k] = v
 				}
 			}
 			// and the same for Stacks
 			// this can be made more efficient, when it matters
 			for k, v := range p.Stacks {
 				before, _, _ := strings.Cut(k, "\n")
 				if cfg.HasStack(p.Program, before) && report.X <= cfg.Rate(p.Program, before) {
 					x.Stacks[k] = v
 				}
 			}
 		}
 
 		uploadContents, err = json.MarshalIndent(upload, "", " ")
 		if err != nil {
 			return "", fmt.Errorf("failed to marshal upload report for %s: %v", expiryDate, err)
 		}
 	}
 	localFileName := filepath.Join(u.dir.LocalDir(), "local."+expiryDate+".json")
 	uploadFileName := filepath.Join(u.dir.LocalDir(), expiryDate+".json")
 
 	/* Prepare to write files */
 	// if either file exists, someone has been here ahead of us
 	// (there is still a race, but this check shortens the open window)
 	if _, err := os.Stat(localFileName); err == nil {
 		u.deleteFiles(countFiles)
 		return "", fmt.Errorf("local report %s already exists", localFileName)
 	}
 	if _, err := os.Stat(uploadFileName); err == nil {
 		u.deleteFiles(countFiles)
 		return "", fmt.Errorf("report %s already exists", uploadFileName)
 	}
 	// write the uploadable file
 	var errUpload, errLocal error
 	if uploadOK {
 		_, errUpload = exclusiveWrite(uploadFileName, uploadContents)
 	}
 	// write the local file
 	_, errLocal = exclusiveWrite(localFileName, localContents)
 	/*  Wrote the files */
 
 	// even though these errors won't occur, what should happen
 	// if errUpload == nil and it is ok to upload, and errLocal != nil?
 	if errLocal != nil {
 		return "", fmt.Errorf("failed to write local file %s (%v)", localFileName, errLocal)
 	}
 	if errUpload != nil {
 		return "", fmt.Errorf("failed to write upload file %s (%v)", uploadFileName, errUpload)
 	}
 	u.logger.Printf("Created %s, deleting %d count files", filepath.Base(uploadFileName), len(countFiles))
 	u.deleteFiles(countFiles)
 	if uploadOK {
 		return uploadFileName, nil
 	}
 	return "", nil
 }
 
 // exclusiveWrite attempts to create filename exclusively, and if successful,
 // writes content to the resulting file handle.
 //
 // It returns a boolean indicating whether the exclusive handle was acquired,
 // and an error indicating whether the operation succeeded.
 // If the file already exists, exclusiveWrite returns (false, nil).
 func exclusiveWrite(filename string, content []byte) (_ bool, rerr error) {
 	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
 	if err != nil {
 		if os.IsExist(err) {
 			return false, nil
 		}
 		return false, err
 	}
 	defer func() {
 		if err := f.Close(); err != nil && rerr == nil {
 			rerr = err
 		}
 	}()
 	if _, err := f.Write(content); err != nil {
 		return false, err
 	}
 	return true, nil
 }
 
 // return an existing ProgremReport, or create anew
 func findProgReport(meta map[string]string, report *telemetry.Report) *telemetry.ProgramReport {
 	for _, prog := range report.Programs {
 		if prog.Program == meta["Program"] && prog.Version == meta["Version"] &&
 			prog.GoVersion == meta["GoVersion"] && prog.GOOS == meta["GOOS"] &&
 			prog.GOARCH == meta["GOARCH"] {
 			return prog
 		}
 	}
 	prog := telemetry.ProgramReport{
 		Program:   meta["Program"],
 		Version:   meta["Version"],
 		GoVersion: meta["GoVersion"],
 		GOOS:      meta["GOOS"],
 		GOARCH:    meta["GOARCH"],
 		Counters:  make(map[string]int64),
 		Stacks:    make(map[string]int64),
 	}
 	report.Programs = append(report.Programs, &prog)
 	return &prog
 }
 
 // computeRandom returns a cryptographic random float64 in the range [0, 1],
 // with 52 bits of precision.
 func computeRandom() float64 {
 	for {
 		b := make([]byte, 8)
 		_, err := rand.Read(b)
 		if err != nil {
 			panic(fmt.Sprintf("rand.Read failed: %v", err))
 		}
 		// and turn it into a float64
 		x := math.Float64frombits(binary.LittleEndian.Uint64(b))
 		if math.IsNaN(x) || math.IsInf(x, 0) {
 			continue
 		}
 		x = math.Abs(x)
 		if x < 0x1p-1000 { // avoid underflow patterns
 			continue
 		}
 		frac, _ := math.Frexp(x) // 52 bits of randomness
 		return frac*2 - 1
 	}
 }