findwork.go (3462B)
1 // Copyright 2023 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package upload 6 7 import ( 8 "os" 9 "path/filepath" 10 "strings" 11 ) 12 13 // files to handle 14 type work struct { 15 // absolute file names 16 countfiles []string // count files to process 17 readyfiles []string // old reports to upload 18 // relative names 19 uploaded map[string]bool // reports that have been uploaded 20 } 21 22 // find all the files that look like counter files or reports 23 // that need to be uploaded. (There may be unexpected leftover files 24 // and uploading is supposed to be idempotent.) 25 func (u *uploader) findWork() work { 26 localdir, uploaddir := u.dir.LocalDir(), u.dir.UploadDir() 27 var ans work 28 fis, err := os.ReadDir(localdir) 29 if err != nil { 30 u.logger.Printf("Could not find work: failed to read local dir %s: %v", localdir, err) 31 return ans 32 } 33 34 mode, asof := u.dir.Mode() 35 u.logger.Printf("Finding work: mode %s asof %s", mode, asof) 36 37 // count files end in .v1.count 38 // reports end in .json. If they are not to be uploaded they 39 // start with local. 40 for _, fi := range fis { 41 if strings.HasSuffix(fi.Name(), ".v1.count") { 42 fname := filepath.Join(localdir, fi.Name()) 43 _, expiry, err := u.counterDateSpan(fname) 44 switch { 45 case err != nil: 46 u.logger.Printf("Error reading expiry for count file %s: %v", fi.Name(), err) 47 case expiry.After(u.startTime): 48 u.logger.Printf("Skipping count file %s: still active", fi.Name()) 49 default: 50 u.logger.Printf("Collecting count file %s", fi.Name()) 51 ans.countfiles = append(ans.countfiles, fname) 52 } 53 } else if strings.HasPrefix(fi.Name(), "local.") { 54 // skip 55 } else if strings.HasSuffix(fi.Name(), ".json") && mode == "on" { 56 // Collect reports that are ready for upload. 57 reportDate := u.uploadReportDate(fi.Name()) 58 if !asof.IsZero() && !reportDate.IsZero() { 59 // If both the mode asof date and the report date are present, do the 60 // right thing... 61 // 62 // (see https://github.com/golang/go/issues/63142#issuecomment-1734025130) 63 if asof.Before(reportDate) { 64 // Note: since this report was created after telemetry was enabled, 65 // we can only assume that the process that created it checked that 66 // the counter data contained therein was all from after the asof 67 // date. 68 // 69 // TODO(rfindley): store the begin date in reports, so that we can 70 // verify this assumption. 71 u.logger.Printf("Uploadable: %s", fi.Name()) 72 ans.readyfiles = append(ans.readyfiles, filepath.Join(localdir, fi.Name())) 73 } 74 } else { 75 // ...otherwise fall back on the old behavior of uploading all 76 // unuploaded files. 77 // 78 // TODO(rfindley): invert this logic following more testing. We 79 // should only upload if we know both the asof date and the report 80 // date, and they are acceptable. 81 u.logger.Printf("Uploadable (missing date): %s", fi.Name()) 82 ans.readyfiles = append(ans.readyfiles, filepath.Join(localdir, fi.Name())) 83 } 84 } 85 } 86 87 fis, err = os.ReadDir(uploaddir) 88 if err != nil { 89 os.MkdirAll(uploaddir, 0777) 90 return ans 91 } 92 // There should be only one of these per day; maybe sometime 93 // we'll want to clean the directory. 94 ans.uploaded = make(map[string]bool) 95 for _, fi := range fis { 96 if strings.HasSuffix(fi.Name(), ".json") { 97 u.logger.Printf("Already uploaded: %s", fi.Name()) 98 ans.uploaded[fi.Name()] = true 99 } 100 } 101 return ans 102 }