packages.go (52121B)
1 // Copyright 2018 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package packages 6 7 // See doc.go for package documentation and implementation notes. 8 9 import ( 10 "context" 11 "encoding/json" 12 "errors" 13 "fmt" 14 "go/ast" 15 "go/parser" 16 "go/scanner" 17 "go/token" 18 "go/types" 19 "log" 20 "os" 21 "path/filepath" 22 "runtime" 23 "strings" 24 "sync" 25 "sync/atomic" 26 "time" 27 28 "golang.org/x/sync/errgroup" 29 30 "golang.org/x/tools/go/gcexportdata" 31 "golang.org/x/tools/internal/gocommand" 32 "golang.org/x/tools/internal/packagesinternal" 33 "golang.org/x/tools/internal/typesinternal" 34 ) 35 36 // A LoadMode controls the amount of detail to return when loading. 37 // The bits below can be combined to specify which fields should be 38 // filled in the result packages. 39 // 40 // The zero value is a special case, equivalent to combining 41 // the NeedName, NeedFiles, and NeedCompiledGoFiles bits. 42 // 43 // ID and Errors (if present) will always be filled. 44 // [Load] may return more information than requested. 45 // 46 // The Mode flag is a union of several bits named NeedName, 47 // NeedFiles, and so on, each of which determines whether 48 // a given field of Package (Name, Files, etc) should be 49 // populated. 50 // 51 // For convenience, we provide named constants for the most 52 // common combinations of Need flags: 53 // 54 // [LoadFiles] lists of files in each package 55 // [LoadImports] ... plus imports 56 // [LoadTypes] ... plus type information 57 // [LoadSyntax] ... plus type-annotated syntax 58 // [LoadAllSyntax] ... for all dependencies 59 // 60 // Unfortunately there are a number of open bugs related to 61 // interactions among the LoadMode bits: 62 // - https://go.dev/issue/56633 63 // - https://go.dev/issue/56677 64 // - https://go.dev/issue/58726 65 // - https://go.dev/issue/63517 66 type LoadMode int 67 68 const ( 69 // NeedName adds Name and PkgPath. 70 NeedName LoadMode = 1 << iota 71 72 // NeedFiles adds Dir, GoFiles, OtherFiles, and IgnoredFiles 73 NeedFiles 74 75 // NeedCompiledGoFiles adds CompiledGoFiles. 76 NeedCompiledGoFiles 77 78 // NeedImports adds Imports. If NeedDeps is not set, the Imports field will contain 79 // "placeholder" Packages with only the ID set. 80 NeedImports 81 82 // NeedDeps adds the fields requested by the LoadMode in the packages in Imports. 83 NeedDeps 84 85 // NeedExportFile adds ExportFile. 86 NeedExportFile 87 88 // NeedTypes adds Types, Fset, and IllTyped. 89 NeedTypes 90 91 // NeedSyntax adds Syntax and Fset. 92 NeedSyntax 93 94 // NeedTypesInfo adds TypesInfo and Fset. 95 NeedTypesInfo 96 97 // NeedTypesSizes adds TypesSizes. 98 NeedTypesSizes 99 100 // needInternalDepsErrors adds the internal deps errors field for use by gopls. 101 needInternalDepsErrors 102 103 // NeedForTest adds ForTest. 104 // 105 // Tests must also be set on the context for this field to be populated. 106 NeedForTest 107 108 // typecheckCgo enables full support for type checking cgo. Requires Go 1.15+. 109 // Modifies CompiledGoFiles and Types, and has no effect on its own. 110 typecheckCgo 111 112 // NeedModule adds Module. 113 NeedModule 114 115 // NeedEmbedFiles adds EmbedFiles. 116 NeedEmbedFiles 117 118 // NeedEmbedPatterns adds EmbedPatterns. 119 NeedEmbedPatterns 120 121 // NeedTarget adds Target. 122 NeedTarget 123 124 // Be sure to update loadmode_string.go when adding new items! 125 ) 126 127 const ( 128 // LoadFiles loads the name and file names for the initial packages. 129 LoadFiles = NeedName | NeedFiles | NeedCompiledGoFiles 130 131 // LoadImports loads the name, file names, and import mapping for the initial packages. 132 LoadImports = LoadFiles | NeedImports 133 134 // LoadTypes loads exported type information for the initial packages. 135 LoadTypes = LoadImports | NeedTypes | NeedTypesSizes 136 137 // LoadSyntax loads typed syntax for the initial packages. 138 LoadSyntax = LoadTypes | NeedSyntax | NeedTypesInfo 139 140 // LoadAllSyntax loads typed syntax for the initial packages and all dependencies. 141 LoadAllSyntax = LoadSyntax | NeedDeps 142 143 // Deprecated: NeedExportsFile is a historical misspelling of NeedExportFile. 144 // 145 //go:fix inline 146 NeedExportsFile = NeedExportFile 147 ) 148 149 // A Config specifies details about how packages should be loaded. 150 // The zero value is a valid configuration. 151 // 152 // Calls to [Load] do not modify this struct. 153 type Config struct { 154 // Mode controls the level of information returned for each package. 155 Mode LoadMode 156 157 // Context specifies the context for the load operation. 158 // Cancelling the context may cause [Load] to abort and 159 // return an error. 160 Context context.Context 161 162 // Logf is the logger for the config. 163 // If the user provides a logger, debug logging is enabled. 164 // If the GOPACKAGESDEBUG environment variable is set to true, 165 // but the logger is nil, default to log.Printf. 166 Logf func(format string, args ...any) 167 168 // Dir is the directory in which to run the build system's query tool 169 // that provides information about the packages. 170 // If Dir is empty, the tool is run in the current directory. 171 Dir string 172 173 // Env is the environment to use when invoking the build system's query tool. 174 // If Env is nil, the current environment is used. 175 // As in os/exec's Cmd, only the last value in the slice for 176 // each environment key is used. To specify the setting of only 177 // a few variables, append to the current environment, as in: 178 // 179 // opt.Env = append(os.Environ(), "GOOS=plan9", "GOARCH=386") 180 // 181 Env []string 182 183 // BuildFlags is a list of command-line flags to be passed through to 184 // the build system's query tool. 185 BuildFlags []string 186 187 // Fset provides source position information for syntax trees and types. 188 // If Fset is nil, Load will use a new fileset, but preserve Fset's value. 189 Fset *token.FileSet 190 191 // ParseFile is called to read and parse each file 192 // when preparing a package's type-checked syntax tree. 193 // It must be safe to call ParseFile simultaneously from multiple goroutines. 194 // If ParseFile is nil, the loader will uses parser.ParseFile. 195 // 196 // ParseFile should parse the source from src and use filename only for 197 // recording position information. 198 // 199 // An application may supply a custom implementation of ParseFile 200 // to change the effective file contents or the behavior of the parser, 201 // or to modify the syntax tree. For example, selectively eliminating 202 // unwanted function bodies can significantly accelerate type checking. 203 ParseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) 204 205 // If Tests is set, the loader includes not just the packages 206 // matching a particular pattern but also any related test packages, 207 // including test-only variants of the package and the test executable. 208 // 209 // For example, when using the go command, loading "fmt" with Tests=true 210 // returns four packages, with IDs "fmt" (the standard package), 211 // "fmt [fmt.test]" (the package as compiled for the test), 212 // "fmt_test" (the test functions from source files in package fmt_test), 213 // and "fmt.test" (the test binary). 214 // 215 // In build systems with explicit names for tests, 216 // setting Tests may have no effect. 217 Tests bool 218 219 // Overlay is a mapping from absolute file paths to file contents. 220 // 221 // For each map entry, [Load] uses the alternative file 222 // contents provided by the overlay mapping instead of reading 223 // from the file system. This mechanism can be used to enable 224 // editor-integrated tools to correctly analyze the contents 225 // of modified but unsaved buffers, for example. 226 // 227 // The overlay mapping is passed to the build system's driver 228 // (see "The driver protocol") so that it too can report 229 // consistent package metadata about unsaved files. However, 230 // drivers may vary in their level of support for overlays. 231 Overlay map[string][]byte 232 } 233 234 // Load loads and returns the Go packages named by the given patterns. 235 // 236 // The cfg parameter specifies loading options; nil behaves the same as an empty [Config]. 237 // 238 // The [Config.Mode] field is a set of bits that determine what kinds 239 // of information should be computed and returned. Modes that require 240 // more information tend to be slower. See [LoadMode] for details 241 // and important caveats. Its zero value is equivalent to 242 // [NeedName] | [NeedFiles] | [NeedCompiledGoFiles]. 243 // 244 // Each call to Load returns a new set of [Package] instances. 245 // The Packages and their Imports form a directed acyclic graph. 246 // 247 // If the [NeedTypes] mode flag was set, each call to Load uses a new 248 // [types.Importer], so [types.Object] and [types.Type] values from 249 // different calls to Load must not be mixed as they will have 250 // inconsistent notions of type identity. 251 // 252 // If any of the patterns was invalid as defined by the 253 // underlying build system, Load returns an error. 254 // It may return an empty list of packages without an error, 255 // for instance for an empty expansion of a valid wildcard. 256 // Errors associated with a particular package are recorded in the 257 // corresponding Package's Errors list, and do not cause Load to 258 // return an error. Clients may need to handle such errors before 259 // proceeding with further analysis. The [PrintErrors] function is 260 // provided for convenient display of all errors. 261 func Load(cfg *Config, patterns ...string) ([]*Package, error) { 262 ld := newLoader(cfg) 263 response, external, err := defaultDriver(&ld.Config, patterns...) 264 if err != nil { 265 return nil, err 266 } 267 268 ld.sizes = types.SizesFor(response.Compiler, response.Arch) 269 if ld.sizes == nil && ld.Config.Mode&(NeedTypes|NeedTypesSizes|NeedTypesInfo) != 0 { 270 // Type size information is needed but unavailable. 271 if external { 272 // An external driver may fail to populate the Compiler/GOARCH fields, 273 // especially since they are relatively new (see #63700). 274 // Provide a sensible fallback in this case. 275 ld.sizes = types.SizesFor("gc", runtime.GOARCH) 276 if ld.sizes == nil { // gccgo-only arch 277 ld.sizes = types.SizesFor("gc", "amd64") 278 } 279 } else { 280 // Go list should never fail to deliver accurate size information. 281 // Reject the whole Load since the error is the same for every package. 282 return nil, fmt.Errorf("can't determine type sizes for compiler %q on GOARCH %q", 283 response.Compiler, response.Arch) 284 } 285 } 286 287 ld.externalDriver = external 288 289 return ld.refine(response) 290 } 291 292 // defaultDriver is a driver that implements go/packages' fallback behavior. 293 // It will try to request to an external driver, if one exists. If there's 294 // no external driver, or the driver returns a response with NotHandled set, 295 // defaultDriver will fall back to the go list driver. 296 // The boolean result indicates that an external driver handled the request. 297 func defaultDriver(cfg *Config, patterns ...string) (*DriverResponse, bool, error) { 298 const ( 299 // windowsArgMax specifies the maximum command line length for 300 // the Windows' CreateProcess function. 301 windowsArgMax = 32767 302 // maxEnvSize is a very rough estimation of the maximum environment 303 // size of a user. 304 maxEnvSize = 16384 305 // safeArgMax specifies the maximum safe command line length to use 306 // by the underlying driver excl. the environment. We choose the Windows' 307 // ARG_MAX as the starting point because it's one of the lowest ARG_MAX 308 // constants out of the different supported platforms, 309 // e.g., https://www.in-ulm.de/~mascheck/various/argmax/#results. 310 safeArgMax = windowsArgMax - maxEnvSize 311 ) 312 chunks, err := splitIntoChunks(patterns, safeArgMax) 313 if err != nil { 314 return nil, false, err 315 } 316 317 if driver := findExternalDriver(cfg); driver != nil { 318 response, err := callDriverOnChunks(driver, cfg, chunks) 319 if err != nil { 320 return nil, false, err 321 } else if !response.NotHandled { 322 return response, true, nil 323 } 324 // not handled: fall through 325 } 326 327 // go list fallback 328 329 // Write overlays once, as there are many calls 330 // to 'go list' (one per chunk plus others too). 331 overlayFile, cleanupOverlay, err := gocommand.WriteOverlays(cfg.Overlay) 332 if err != nil { 333 return nil, false, err 334 } 335 defer cleanupOverlay() 336 337 var runner gocommand.Runner // (shared across many 'go list' calls) 338 driver := func(cfg *Config, patterns []string) (*DriverResponse, error) { 339 return goListDriver(cfg, &runner, overlayFile, patterns) 340 } 341 response, err := callDriverOnChunks(driver, cfg, chunks) 342 if err != nil { 343 return nil, false, err 344 } 345 return response, false, err 346 } 347 348 // splitIntoChunks chunks the slice so that the total number of characters 349 // in a chunk is no longer than argMax. 350 func splitIntoChunks(patterns []string, argMax int) ([][]string, error) { 351 if argMax <= 0 { 352 return nil, errors.New("failed to split patterns into chunks, negative safe argMax value") 353 } 354 var chunks [][]string 355 charsInChunk := 0 356 nextChunkStart := 0 357 for i, v := range patterns { 358 vChars := len(v) 359 if vChars > argMax { 360 // a single pattern is longer than the maximum safe ARG_MAX, hardly should happen 361 return nil, errors.New("failed to split patterns into chunks, a pattern is too long") 362 } 363 charsInChunk += vChars + 1 // +1 is for a whitespace between patterns that has to be counted too 364 if charsInChunk > argMax { 365 chunks = append(chunks, patterns[nextChunkStart:i]) 366 nextChunkStart = i 367 charsInChunk = vChars 368 } 369 } 370 // add the last chunk 371 if nextChunkStart < len(patterns) { 372 chunks = append(chunks, patterns[nextChunkStart:]) 373 } 374 return chunks, nil 375 } 376 377 func callDriverOnChunks(driver driver, cfg *Config, chunks [][]string) (*DriverResponse, error) { 378 if len(chunks) == 0 { 379 return driver(cfg, nil) 380 } 381 responses := make([]*DriverResponse, len(chunks)) 382 errNotHandled := errors.New("driver returned NotHandled") 383 var g errgroup.Group 384 for i, chunk := range chunks { 385 g.Go(func() (err error) { 386 responses[i], err = driver(cfg, chunk) 387 if responses[i] != nil && responses[i].NotHandled { 388 err = errNotHandled 389 } 390 return err 391 }) 392 } 393 if err := g.Wait(); err != nil { 394 if errors.Is(err, errNotHandled) { 395 return &DriverResponse{NotHandled: true}, nil 396 } 397 return nil, err 398 } 399 return mergeResponses(responses...), nil 400 } 401 402 func mergeResponses(responses ...*DriverResponse) *DriverResponse { 403 if len(responses) == 0 { 404 return nil 405 } 406 // No dedup needed 407 if len(responses) == 1 { 408 return responses[0] 409 } 410 response := newDeduper() 411 response.dr.NotHandled = false 412 response.dr.Compiler = responses[0].Compiler 413 response.dr.Arch = responses[0].Arch 414 response.dr.GoVersion = responses[0].GoVersion 415 for _, v := range responses { 416 response.addAll(v) 417 } 418 return response.dr 419 } 420 421 // A Package describes a loaded Go package. 422 // 423 // It also defines part of the JSON schema of [DriverResponse]. 424 // See the package documentation for an overview. 425 type Package struct { 426 // ID is a unique identifier for a package, 427 // in a syntax provided by the underlying build system. 428 // 429 // Because the syntax varies based on the build system, 430 // clients should treat IDs as opaque and not attempt to 431 // interpret them. 432 ID string 433 434 // Name is the package name as it appears in the package source code. 435 Name string 436 437 // PkgPath is the package path as used by the go/types package. 438 PkgPath string 439 440 // Dir is the directory associated with the package, if it exists. 441 // 442 // For packages listed by the go command, this is the directory containing 443 // the package files. 444 Dir string 445 446 // Errors contains any errors encountered querying the metadata 447 // of the package, or while parsing or type-checking its files. 448 Errors []Error 449 450 // TypeErrors contains the subset of errors produced during type checking. 451 TypeErrors []types.Error 452 453 // GoFiles lists the absolute file paths of the package's Go source files. 454 // It may include files that should not be compiled, for example because 455 // they contain non-matching build tags, are documentary pseudo-files such as 456 // unsafe/unsafe.go or builtin/builtin.go, or are subject to cgo preprocessing. 457 GoFiles []string 458 459 // CompiledGoFiles lists the absolute file paths of the package's source 460 // files that are suitable for type checking. 461 // This may differ from GoFiles if files are processed before compilation. 462 CompiledGoFiles []string 463 464 // OtherFiles lists the absolute file paths of the package's non-Go source files, 465 // including assembly, C, C++, Fortran, Objective-C, SWIG, and so on. 466 OtherFiles []string 467 468 // EmbedFiles lists the absolute file paths of the package's files 469 // embedded with go:embed. 470 EmbedFiles []string 471 472 // EmbedPatterns lists the absolute file patterns of the package's 473 // files embedded with go:embed. 474 EmbedPatterns []string 475 476 // IgnoredFiles lists source files that are not part of the package 477 // using the current build configuration but that might be part of 478 // the package using other build configurations. 479 IgnoredFiles []string 480 481 // ExportFile is the absolute path to a file containing type 482 // information for the package as provided by the build system. 483 ExportFile string 484 485 // Target is the absolute install path of the .a file, for libraries, 486 // and of the executable file, for binaries. 487 Target string 488 489 // Imports maps import paths appearing in the package's Go source files 490 // to corresponding loaded Packages. 491 Imports map[string]*Package 492 493 // Module is the module information for the package if it exists. 494 // 495 // Note: it may be missing for std and cmd; see Go issue #65816. 496 Module *Module 497 498 // -- The following fields are not part of the driver JSON schema. -- 499 500 // Types provides type information for the package. 501 // The NeedTypes LoadMode bit sets this field for packages matching the 502 // patterns; type information for dependencies may be missing or incomplete, 503 // unless NeedDeps and NeedImports are also set. 504 // 505 // Each call to [Load] returns a consistent set of type 506 // symbols, as defined by the comment at [types.Identical]. 507 // Avoid mixing type information from two or more calls to [Load]. 508 Types *types.Package `json:"-"` 509 510 // Fset provides position information for Types, TypesInfo, and Syntax. 511 // It is set only when Types is set. 512 Fset *token.FileSet `json:"-"` 513 514 // IllTyped indicates whether the package or any dependency contains errors. 515 // It is set only when Types is set. 516 IllTyped bool `json:"-"` 517 518 // Syntax is the package's syntax trees, for the files listed in CompiledGoFiles. 519 // 520 // The NeedSyntax LoadMode bit populates this field for packages matching the patterns. 521 // If NeedDeps and NeedImports are also set, this field will also be populated 522 // for dependencies. 523 // 524 // Syntax is kept in the same order as CompiledGoFiles, with the caveat that nils are 525 // removed. If parsing returned nil, Syntax may be shorter than CompiledGoFiles. 526 Syntax []*ast.File `json:"-"` 527 528 // TypesInfo provides type information about the package's syntax trees. 529 // It is set only when Syntax is set. 530 TypesInfo *types.Info `json:"-"` 531 532 // TypesSizes provides the effective size function for types in TypesInfo. 533 TypesSizes types.Sizes `json:"-"` 534 535 // -- internal -- 536 537 // ForTest is the package under test, if any. 538 ForTest string 539 540 // depsErrors is the DepsErrors field from the go list response, if any. 541 depsErrors []*packagesinternal.PackageError 542 543 // exportDataError is the error encountered reading export data, if any. 544 // Decoding export data should ordinarily be infallible, so this typically 545 // indicates a producer/consumer version skew. 546 exportDataError error 547 } 548 549 // Module provides module information for a package. 550 // 551 // It also defines part of the JSON schema of [DriverResponse]. 552 // See the package documentation for an overview. 553 type Module struct { 554 Path string // module path 555 Version string // module version 556 Replace *Module // replaced by this module 557 Time *time.Time // time version was created 558 Main bool // is this the main module? 559 Indirect bool // is this module only an indirect dependency of main module? 560 Dir string // directory holding files for this module, if any 561 GoMod string // path to go.mod file used when loading this module, if any 562 GoVersion string // go version used in module 563 Error *ModuleError // error loading module 564 } 565 566 // ModuleError holds errors loading a module. 567 type ModuleError struct { 568 Err string // the error itself 569 } 570 571 func init() { 572 packagesinternal.GetDepsErrors = func(p any) []*packagesinternal.PackageError { 573 return p.(*Package).depsErrors 574 } 575 packagesinternal.TypecheckCgo = int(typecheckCgo) 576 packagesinternal.DepsErrors = int(needInternalDepsErrors) 577 } 578 579 // An Error describes a problem with a package's metadata, syntax, or types. 580 type Error struct { 581 Pos string // "file:line:col" or "file:line" or "" or "-" 582 Msg string 583 Kind ErrorKind 584 } 585 586 // ErrorKind describes the source of the error, allowing the user to 587 // differentiate between errors generated by the driver, the parser, or the 588 // type-checker. 589 type ErrorKind int 590 591 const ( 592 UnknownError ErrorKind = iota 593 ListError 594 ParseError 595 TypeError 596 ) 597 598 func (err Error) Error() string { 599 pos := err.Pos 600 if pos == "" { 601 pos = "-" // like token.Position{}.String() 602 } 603 return pos + ": " + err.Msg 604 } 605 606 // flatPackage is the JSON form of Package 607 // It drops all the type and syntax fields, and transforms the Imports 608 // 609 // TODO(adonovan): identify this struct with Package, effectively 610 // publishing the JSON protocol. 611 type flatPackage struct { 612 ID string 613 Name string `json:",omitempty"` 614 PkgPath string `json:",omitempty"` 615 Errors []Error `json:",omitempty"` 616 GoFiles []string `json:",omitempty"` 617 CompiledGoFiles []string `json:",omitempty"` 618 OtherFiles []string `json:",omitempty"` 619 EmbedFiles []string `json:",omitempty"` 620 EmbedPatterns []string `json:",omitempty"` 621 IgnoredFiles []string `json:",omitempty"` 622 ExportFile string `json:",omitempty"` 623 Imports map[string]string `json:",omitempty"` 624 } 625 626 // MarshalJSON returns the Package in its JSON form. 627 // For the most part, the structure fields are written out unmodified, and 628 // the type and syntax fields are skipped. 629 // The imports are written out as just a map of path to package id. 630 // The errors are written using a custom type that tries to preserve the 631 // structure of error types we know about. 632 // 633 // This method exists to enable support for additional build systems. It is 634 // not intended for use by clients of the API and we may change the format. 635 func (p *Package) MarshalJSON() ([]byte, error) { 636 flat := &flatPackage{ 637 ID: p.ID, 638 Name: p.Name, 639 PkgPath: p.PkgPath, 640 Errors: p.Errors, 641 GoFiles: p.GoFiles, 642 CompiledGoFiles: p.CompiledGoFiles, 643 OtherFiles: p.OtherFiles, 644 EmbedFiles: p.EmbedFiles, 645 EmbedPatterns: p.EmbedPatterns, 646 IgnoredFiles: p.IgnoredFiles, 647 ExportFile: p.ExportFile, 648 } 649 if len(p.Imports) > 0 { 650 flat.Imports = make(map[string]string, len(p.Imports)) 651 for path, ipkg := range p.Imports { 652 flat.Imports[path] = ipkg.ID 653 } 654 } 655 return json.Marshal(flat) 656 } 657 658 // UnmarshalJSON reads in a Package from its JSON format. 659 // See MarshalJSON for details about the format accepted. 660 func (p *Package) UnmarshalJSON(b []byte) error { 661 flat := &flatPackage{} 662 if err := json.Unmarshal(b, &flat); err != nil { 663 return err 664 } 665 *p = Package{ 666 ID: flat.ID, 667 Name: flat.Name, 668 PkgPath: flat.PkgPath, 669 Errors: flat.Errors, 670 GoFiles: flat.GoFiles, 671 CompiledGoFiles: flat.CompiledGoFiles, 672 OtherFiles: flat.OtherFiles, 673 EmbedFiles: flat.EmbedFiles, 674 EmbedPatterns: flat.EmbedPatterns, 675 IgnoredFiles: flat.IgnoredFiles, 676 ExportFile: flat.ExportFile, 677 } 678 if len(flat.Imports) > 0 { 679 p.Imports = make(map[string]*Package, len(flat.Imports)) 680 for path, id := range flat.Imports { 681 p.Imports[path] = &Package{ID: id} 682 } 683 } 684 return nil 685 } 686 687 func (p *Package) String() string { return p.ID } 688 689 // loaderPackage augments Package with state used during the loading phase 690 type loaderPackage struct { 691 *Package 692 importErrors map[string]error // maps each bad import to its error 693 preds []*loaderPackage // packages that import this one 694 unfinishedSuccs atomic.Int32 // number of direct imports not yet loaded 695 color uint8 // for cycle detection 696 needsrc bool // load from source (Mode >= LoadTypes) 697 needtypes bool // type information is either requested or depended on 698 initial bool // package was matched by a pattern 699 goVersion int // minor version number of go command on PATH 700 } 701 702 // loader holds the working state of a single call to load. 703 type loader struct { 704 pkgs map[string]*loaderPackage // keyed by Package.ID 705 Config 706 sizes types.Sizes // non-nil if needed by mode 707 parseCache map[string]*parseValue 708 parseCacheMu sync.Mutex 709 exportMu sync.Mutex // enforces mutual exclusion of exportdata operations 710 externalDriver bool // true if an external GOPACKAGESDRIVER handled the request 711 712 // Config.Mode contains the implied mode (see impliedLoadMode). 713 // Implied mode contains all the fields we need the data for. 714 // In requestedMode there are the actually requested fields. 715 // We'll zero them out before returning packages to the user. 716 // This makes it easier for us to get the conditions where 717 // we need certain modes right. 718 requestedMode LoadMode 719 } 720 721 type parseValue struct { 722 f *ast.File 723 err error 724 ready chan struct{} 725 } 726 727 func newLoader(cfg *Config) *loader { 728 ld := &loader{ 729 parseCache: map[string]*parseValue{}, 730 } 731 if cfg != nil { 732 ld.Config = *cfg 733 // If the user has provided a logger, use it. 734 ld.Config.Logf = cfg.Logf 735 } 736 if ld.Config.Logf == nil { 737 // If the GOPACKAGESDEBUG environment variable is set to true, 738 // but the user has not provided a logger, default to log.Printf. 739 if debug { 740 ld.Config.Logf = log.Printf 741 } else { 742 ld.Config.Logf = func(format string, args ...any) {} 743 } 744 } 745 if ld.Config.Mode == 0 { 746 ld.Config.Mode = NeedName | NeedFiles | NeedCompiledGoFiles // Preserve zero behavior of Mode for backwards compatibility. 747 } 748 if ld.Config.Env == nil { 749 ld.Config.Env = os.Environ() 750 } 751 if ld.Context == nil { 752 ld.Context = context.Background() 753 } 754 if ld.Dir == "" { 755 if dir, err := os.Getwd(); err == nil { 756 ld.Dir = dir 757 } 758 } 759 760 // Save the actually requested fields. We'll zero them out before returning packages to the user. 761 ld.requestedMode = ld.Mode 762 ld.Mode = impliedLoadMode(ld.Mode) 763 764 if ld.Mode&(NeedSyntax|NeedTypes|NeedTypesInfo) != 0 { 765 if ld.Fset == nil { 766 ld.Fset = token.NewFileSet() 767 } 768 769 // ParseFile is required even in LoadTypes mode 770 // because we load source if export data is missing. 771 if ld.ParseFile == nil { 772 ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) { 773 // We implicitly promise to keep doing ast.Object resolution. :( 774 const mode = parser.AllErrors | parser.ParseComments 775 return parser.ParseFile(fset, filename, src, mode) 776 } 777 } 778 } 779 780 return ld 781 } 782 783 // refine connects the supplied packages into a graph and then adds type 784 // and syntax information as requested by the LoadMode. 785 func (ld *loader) refine(response *DriverResponse) ([]*Package, error) { 786 roots := response.Roots 787 rootMap := make(map[string]int, len(roots)) 788 for i, root := range roots { 789 rootMap[root] = i 790 } 791 ld.pkgs = make(map[string]*loaderPackage) 792 // first pass, fixup and build the map and roots 793 var initial = make([]*loaderPackage, len(roots)) 794 for _, pkg := range response.Packages { 795 rootIndex := -1 796 if i, found := rootMap[pkg.ID]; found { 797 rootIndex = i 798 } 799 800 // Overlays can invalidate export data. 801 // TODO(matloob): make this check fine-grained based on dependencies on overlaid files 802 exportDataInvalid := len(ld.Overlay) > 0 || pkg.ExportFile == "" && pkg.PkgPath != "unsafe" 803 // This package needs type information if the caller requested types and the package is 804 // either a root, or it's a non-root and the user requested dependencies ... 805 needtypes := (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0)) 806 // This package needs source if the call requested source (or types info, which implies source) 807 // and the package is either a root, or itas a non- root and the user requested dependencies... 808 needsrc := ((ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0)) || 809 // ... or if we need types and the exportData is invalid. We fall back to (incompletely) 810 // typechecking packages from source if they fail to compile. 811 (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && exportDataInvalid)) && pkg.PkgPath != "unsafe" 812 lpkg := &loaderPackage{ 813 Package: pkg, 814 needtypes: needtypes, 815 needsrc: needsrc, 816 goVersion: response.GoVersion, 817 } 818 // Don't trust the driver to respond with duplicate-free 819 // package names (go.dev/issue/63822). 820 if _, ok := ld.pkgs[lpkg.ID]; ok { 821 return nil, fmt.Errorf("%s response contained duplicate packages for ID %q", 822 cond(ld.externalDriver, "go/packages driver", "go list"), lpkg.ID) 823 } 824 ld.pkgs[lpkg.ID] = lpkg 825 if rootIndex >= 0 { 826 initial[rootIndex] = lpkg 827 lpkg.initial = true 828 } 829 } 830 for i, root := range roots { 831 if initial[i] == nil { 832 return nil, fmt.Errorf("root package %v is missing", root) 833 } 834 } 835 836 // Materialize the import graph if it is needed (NeedImports), 837 // or if we'll be using loadPackages (Need{Syntax|Types|TypesInfo}). 838 var leaves []*loaderPackage // packages with no unfinished successors 839 if ld.Mode&(NeedImports|NeedSyntax|NeedTypes|NeedTypesInfo) != 0 { 840 const ( 841 white = 0 // new 842 grey = 1 // in progress 843 black = 2 // complete 844 ) 845 846 // visit traverses the import graph, depth-first, 847 // and materializes the graph as Packages.Imports. 848 // 849 // Valid imports are saved in the Packages.Import map. 850 // Invalid imports (cycles and missing nodes) are saved in the importErrors map. 851 // Thus, even in the presence of both kinds of errors, 852 // the Import graph remains a DAG. 853 // 854 // visit returns whether the package needs src or has a transitive 855 // dependency on a package that does. These are the only packages 856 // for which we load source code. 857 var stack []*loaderPackage 858 var visit func(from, lpkg *loaderPackage) bool 859 visit = func(from, lpkg *loaderPackage) bool { 860 if lpkg.color == grey { 861 panic("internal error: grey node") 862 } 863 if lpkg.color == white { 864 lpkg.color = grey 865 stack = append(stack, lpkg) // push 866 stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports 867 lpkg.Imports = make(map[string]*Package, len(stubs)) 868 for importPath, ipkg := range stubs { 869 var importErr error 870 imp := ld.pkgs[ipkg.ID] 871 if imp == nil { 872 // (includes package "C" when DisableCgo) 873 importErr = fmt.Errorf("missing package: %q", ipkg.ID) 874 } else if imp.color == grey { 875 importErr = fmt.Errorf("import cycle: %s", stack) 876 } 877 if importErr != nil { 878 if lpkg.importErrors == nil { 879 lpkg.importErrors = make(map[string]error) 880 } 881 lpkg.importErrors[importPath] = importErr 882 continue 883 } 884 885 if visit(lpkg, imp) { 886 lpkg.needsrc = true 887 } 888 lpkg.Imports[importPath] = imp.Package 889 } 890 891 // -- postorder -- 892 893 // Complete type information is required for the 894 // immediate dependencies of each source package. 895 if lpkg.needsrc && ld.Mode&NeedTypes != 0 { 896 for _, ipkg := range lpkg.Imports { 897 ld.pkgs[ipkg.ID].needtypes = true 898 } 899 } 900 901 // NeedTypeSizes causes TypeSizes to be set even 902 // on packages for which types aren't needed. 903 if ld.Mode&NeedTypesSizes != 0 { 904 lpkg.TypesSizes = ld.sizes 905 } 906 907 // Add packages with no imports directly to the queue of leaves. 908 if len(lpkg.Imports) == 0 { 909 leaves = append(leaves, lpkg) 910 } 911 912 stack = stack[:len(stack)-1] // pop 913 lpkg.color = black 914 } 915 916 // Add edge from predecessor. 917 if from != nil { 918 from.unfinishedSuccs.Add(+1) // incref 919 lpkg.preds = append(lpkg.preds, from) 920 } 921 922 return lpkg.needsrc 923 } 924 925 // For each initial package, create its import DAG. 926 for _, lpkg := range initial { 927 visit(nil, lpkg) 928 } 929 930 } else { 931 // !NeedImports: drop the stub (ID-only) import packages 932 // that we are not even going to try to resolve. 933 for _, lpkg := range initial { 934 lpkg.Imports = nil 935 } 936 } 937 938 // Load type data and syntax if needed, starting at 939 // the initial packages (roots of the import DAG). 940 if ld.Mode&(NeedSyntax|NeedTypes|NeedTypesInfo) != 0 { 941 942 // We avoid using g.SetLimit to limit concurrency as 943 // it makes g.Go stop accepting work, which prevents 944 // workers from enqeuing, and thus finishing, and thus 945 // allowing the group to make progress: deadlock. 946 // 947 // Instead we use the ioLimit and cpuLimit semaphores. 948 g, _ := errgroup.WithContext(ld.Context) 949 950 // enqueues adds a package to the type-checking queue. 951 // It must have no unfinished successors. 952 var enqueue func(*loaderPackage) 953 enqueue = func(lpkg *loaderPackage) { 954 g.Go(func() error { 955 // Parse and type-check. 956 ld.loadPackage(lpkg) 957 958 // Notify each waiting predecessor, 959 // and enqueue it when it becomes a leaf. 960 for _, pred := range lpkg.preds { 961 if pred.unfinishedSuccs.Add(-1) == 0 { // decref 962 enqueue(pred) 963 } 964 } 965 966 return nil 967 }) 968 } 969 970 // Load leaves first, adding new packages 971 // to the queue as they become leaves. 972 for _, leaf := range leaves { 973 enqueue(leaf) 974 } 975 976 if err := g.Wait(); err != nil { 977 return nil, err // cancelled 978 } 979 } 980 981 // If the context is done, return its error and 982 // throw out [likely] incomplete packages. 983 if err := ld.Context.Err(); err != nil { 984 return nil, err 985 } 986 987 result := make([]*Package, len(initial)) 988 for i, lpkg := range initial { 989 result[i] = lpkg.Package 990 } 991 for i := range ld.pkgs { 992 // Clear all unrequested fields, 993 // to catch programs that use more than they request. 994 if ld.requestedMode&NeedName == 0 { 995 ld.pkgs[i].Name = "" 996 ld.pkgs[i].PkgPath = "" 997 } 998 if ld.requestedMode&NeedFiles == 0 { 999 ld.pkgs[i].GoFiles = nil 1000 ld.pkgs[i].OtherFiles = nil 1001 ld.pkgs[i].IgnoredFiles = nil 1002 } 1003 if ld.requestedMode&NeedEmbedFiles == 0 { 1004 ld.pkgs[i].EmbedFiles = nil 1005 } 1006 if ld.requestedMode&NeedEmbedPatterns == 0 { 1007 ld.pkgs[i].EmbedPatterns = nil 1008 } 1009 if ld.requestedMode&NeedCompiledGoFiles == 0 { 1010 ld.pkgs[i].CompiledGoFiles = nil 1011 } 1012 if ld.requestedMode&NeedImports == 0 { 1013 ld.pkgs[i].Imports = nil 1014 } 1015 if ld.requestedMode&NeedExportFile == 0 { 1016 ld.pkgs[i].ExportFile = "" 1017 } 1018 if ld.requestedMode&NeedTypes == 0 { 1019 ld.pkgs[i].Types = nil 1020 ld.pkgs[i].IllTyped = false 1021 } 1022 if ld.requestedMode&NeedSyntax == 0 { 1023 ld.pkgs[i].Syntax = nil 1024 } 1025 if ld.requestedMode&(NeedSyntax|NeedTypes|NeedTypesInfo) == 0 { 1026 ld.pkgs[i].Fset = nil 1027 } 1028 if ld.requestedMode&NeedTypesInfo == 0 { 1029 ld.pkgs[i].TypesInfo = nil 1030 } 1031 if ld.requestedMode&NeedTypesSizes == 0 { 1032 ld.pkgs[i].TypesSizes = nil 1033 } 1034 if ld.requestedMode&NeedModule == 0 { 1035 ld.pkgs[i].Module = nil 1036 } 1037 } 1038 1039 return result, nil 1040 } 1041 1042 // loadPackage loads/parses/typechecks the specified package. 1043 // It must be called only once per Package, 1044 // after immediate dependencies are loaded. 1045 // Precondition: ld.Mode&(NeedSyntax|NeedTypes|NeedTypesInfo) != 0. 1046 func (ld *loader) loadPackage(lpkg *loaderPackage) { 1047 if lpkg.PkgPath == "unsafe" { 1048 // To avoid surprises, fill in the blanks consistent 1049 // with other packages. (For example, some analyzers 1050 // assert that each needed types.Info map is non-nil 1051 // even when there is no syntax that would cause them 1052 // to consult the map.) 1053 lpkg.Types = types.Unsafe 1054 lpkg.Fset = ld.Fset 1055 lpkg.Syntax = []*ast.File{} 1056 lpkg.TypesInfo = ld.newTypesInfo() 1057 lpkg.TypesSizes = ld.sizes 1058 return 1059 } 1060 1061 // Call NewPackage directly with explicit name. 1062 // This avoids skew between golist and go/types when the files' 1063 // package declarations are inconsistent. 1064 lpkg.Types = types.NewPackage(lpkg.PkgPath, lpkg.Name) 1065 lpkg.Fset = ld.Fset 1066 1067 // Start shutting down if the context is done and do not load 1068 // source or export data files. 1069 // Packages that import this one will have ld.Context.Err() != nil. 1070 // ld.Context.Err() will be returned later by refine. 1071 if ld.Context.Err() != nil { 1072 return 1073 } 1074 1075 // Subtle: we populate all Types fields with an empty Package 1076 // before loading export data so that export data processing 1077 // never has to create a types.Package for an indirect dependency, 1078 // which would then require that such created packages be explicitly 1079 // inserted back into the Import graph as a final step after export data loading. 1080 // (Hence this return is after the Types assignment.) 1081 // The Diamond test exercises this case. 1082 if !lpkg.needtypes && !lpkg.needsrc { 1083 return 1084 } 1085 1086 // TODO(adonovan): this condition looks wrong: 1087 // I think it should be lpkg.needtypes && !lpkg.needsrc, 1088 // so that NeedSyntax without NeedTypes can be satisfied by export data. 1089 if !lpkg.needsrc { 1090 if err := ld.loadFromExportData(lpkg); err != nil { 1091 lpkg.exportDataError = err 1092 lpkg.Errors = append(lpkg.Errors, Error{ 1093 Pos: "-", 1094 Msg: err.Error(), 1095 Kind: UnknownError, // e.g. can't find/open/parse export data 1096 }) 1097 } 1098 return // not a source package, don't get syntax trees 1099 } 1100 1101 appendError := func(err error) { 1102 // Convert various error types into the one true Error. 1103 var errs []Error 1104 switch err := err.(type) { 1105 case Error: 1106 // from driver 1107 errs = append(errs, err) 1108 1109 case *os.PathError: 1110 // from parser 1111 errs = append(errs, Error{ 1112 Pos: err.Path + ":1", 1113 Msg: err.Err.Error(), 1114 Kind: ParseError, 1115 }) 1116 1117 case scanner.ErrorList: 1118 // from parser 1119 for _, err := range err { 1120 errs = append(errs, Error{ 1121 Pos: err.Pos.String(), 1122 Msg: err.Msg, 1123 Kind: ParseError, 1124 }) 1125 } 1126 1127 case types.Error: 1128 // from type checker 1129 lpkg.TypeErrors = append(lpkg.TypeErrors, err) 1130 errs = append(errs, Error{ 1131 Pos: err.Fset.Position(err.Pos).String(), 1132 Msg: err.Msg, 1133 Kind: TypeError, 1134 }) 1135 1136 default: 1137 // unexpected impoverished error from parser? 1138 errs = append(errs, Error{ 1139 Pos: "-", 1140 Msg: err.Error(), 1141 Kind: UnknownError, 1142 }) 1143 1144 // If you see this error message, please file a bug. 1145 log.Printf("internal error: error %q (%T) without position", err, err) 1146 } 1147 1148 lpkg.Errors = append(lpkg.Errors, errs...) 1149 } 1150 1151 // If the go command on the PATH is newer than the runtime, 1152 // then the go/{scanner,ast,parser,types} packages from the 1153 // standard library may be unable to process the files 1154 // selected by go list. 1155 // 1156 // There is currently no way to downgrade the effective 1157 // version of the go command (see issue 52078), so we proceed 1158 // with the newer go command but, in case of parse or type 1159 // errors, we emit an additional diagnostic. 1160 // 1161 // See: 1162 // - golang.org/issue/52078 (flag to set release tags) 1163 // - golang.org/issue/50825 (gopls legacy version support) 1164 // - golang.org/issue/55883 (go/packages confusing error) 1165 // 1166 // Should we assert a hard minimum of (currently) go1.16 here? 1167 var runtimeVersion int 1168 if _, err := fmt.Sscanf(runtime.Version(), "go1.%d", &runtimeVersion); err == nil && runtimeVersion < lpkg.goVersion { 1169 defer func() { 1170 if len(lpkg.Errors) > 0 { 1171 appendError(Error{ 1172 Pos: "-", 1173 Msg: fmt.Sprintf("This application uses version go1.%d of the source-processing packages but runs version go1.%d of 'go list'. It may fail to process source files that rely on newer language features. If so, rebuild the application using a newer version of Go.", runtimeVersion, lpkg.goVersion), 1174 Kind: UnknownError, 1175 }) 1176 } 1177 }() 1178 } 1179 1180 if ld.Config.Mode&NeedTypes != 0 && len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" { 1181 // The config requested loading sources and types, but sources are missing. 1182 // Add an error to the package and fall back to loading from export data. 1183 appendError(Error{"-", fmt.Sprintf("sources missing for package %s", lpkg.ID), ParseError}) 1184 _ = ld.loadFromExportData(lpkg) // ignore any secondary errors 1185 1186 return // can't get syntax trees for this package 1187 } 1188 1189 files, errs := ld.parseFiles(lpkg.CompiledGoFiles) 1190 for _, err := range errs { 1191 appendError(err) 1192 } 1193 1194 lpkg.Syntax = files 1195 if ld.Config.Mode&(NeedTypes|NeedTypesInfo) == 0 { 1196 return 1197 } 1198 1199 // Start shutting down if the context is done and do not type check. 1200 // Packages that import this one will have ld.Context.Err() != nil. 1201 // ld.Context.Err() will be returned later by refine. 1202 if ld.Context.Err() != nil { 1203 return 1204 } 1205 1206 lpkg.TypesInfo = ld.newTypesInfo() 1207 lpkg.TypesSizes = ld.sizes 1208 1209 importer := importerFunc(func(path string) (*types.Package, error) { 1210 if path == "unsafe" { 1211 return types.Unsafe, nil 1212 } 1213 1214 // The imports map is keyed by import path. 1215 ipkg := lpkg.Imports[path] 1216 if ipkg == nil { 1217 if err := lpkg.importErrors[path]; err != nil { 1218 return nil, err 1219 } 1220 // There was skew between the metadata and the 1221 // import declarations, likely due to an edit 1222 // race, or because the ParseFile feature was 1223 // used to supply alternative file contents. 1224 return nil, fmt.Errorf("no metadata for %s", path) 1225 } 1226 1227 if ipkg.Types != nil && ipkg.Types.Complete() { 1228 return ipkg.Types, nil 1229 } 1230 1231 // If types are unavailable, there must be an export data error. 1232 if ipkg.exportDataError != nil { 1233 return nil, ipkg.exportDataError 1234 } 1235 1236 log.Fatalf("internal error: expected complete types for package %q", path) 1237 panic("unreachable") 1238 }) 1239 1240 // type-check 1241 tc := &types.Config{ 1242 Importer: importer, 1243 1244 // Type-check bodies of functions only in initial packages. 1245 // Example: for import graph A->B->C and initial packages {A,C}, 1246 // we can ignore function bodies in B. 1247 IgnoreFuncBodies: ld.Mode&NeedDeps == 0 && !lpkg.initial, 1248 1249 Error: appendError, 1250 Sizes: ld.sizes, // may be nil 1251 } 1252 if lpkg.Module != nil && lpkg.Module.GoVersion != "" { 1253 tc.GoVersion = "go" + lpkg.Module.GoVersion 1254 } else if ld.externalDriver && lpkg.goVersion != 0 { 1255 // Module information is missing when GOPACKAGESDRIVER is used, 1256 // so use the go version from the driver response. 1257 tc.GoVersion = fmt.Sprintf("go1.%d", lpkg.goVersion) 1258 } 1259 if (ld.Mode & typecheckCgo) != 0 { 1260 if !typesinternal.SetUsesCgo(tc) { 1261 appendError(Error{ 1262 Msg: "typecheckCgo requires Go 1.15+", 1263 Kind: ListError, 1264 }) 1265 return 1266 } 1267 } 1268 1269 // Type-checking is CPU intensive. 1270 cpuLimit <- unit{} // acquire a token 1271 defer func() { <-cpuLimit }() // release a token 1272 1273 typErr := types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax) 1274 lpkg.importErrors = nil // no longer needed 1275 1276 // In go/types go1.21 and go1.22, Checker.Files failed fast with a 1277 // a "too new" error, without calling tc.Error and without 1278 // proceeding to type-check the package (#66525). 1279 // We rely on the runtimeVersion error to give the suggested remedy. 1280 if typErr != nil && len(lpkg.Errors) == 0 && len(lpkg.Syntax) > 0 { 1281 if msg := typErr.Error(); strings.HasPrefix(msg, "package requires newer Go version") { 1282 appendError(types.Error{ 1283 Fset: ld.Fset, 1284 Pos: lpkg.Syntax[0].Package, 1285 Msg: msg, 1286 }) 1287 } 1288 } 1289 1290 // If !Cgo, the type-checker uses FakeImportC mode, so 1291 // it doesn't invoke the importer for import "C", 1292 // nor report an error for the import, 1293 // or for any undefined C.f reference. 1294 // We must detect this explicitly and correctly 1295 // mark the package as IllTyped (by reporting an error). 1296 // TODO(adonovan): if these errors are annoying, 1297 // we could just set IllTyped quietly. 1298 if tc.FakeImportC { 1299 outer: 1300 for _, f := range lpkg.Syntax { 1301 for _, imp := range f.Imports { 1302 if imp.Path.Value == `"C"` { 1303 err := types.Error{Fset: ld.Fset, Pos: imp.Pos(), Msg: `import "C" ignored`} 1304 appendError(err) 1305 break outer 1306 } 1307 } 1308 } 1309 } 1310 1311 // If types.Checker.Files had an error that was unreported, 1312 // make sure to report the unknown error so the package is illTyped. 1313 if typErr != nil && len(lpkg.Errors) == 0 { 1314 appendError(typErr) 1315 } 1316 1317 // Record accumulated errors. 1318 illTyped := len(lpkg.Errors) > 0 1319 if !illTyped { 1320 for _, imp := range lpkg.Imports { 1321 if imp.IllTyped { 1322 illTyped = true 1323 break 1324 } 1325 } 1326 } 1327 lpkg.IllTyped = illTyped 1328 } 1329 1330 func (ld *loader) newTypesInfo() *types.Info { 1331 // Populate TypesInfo only if needed, as it 1332 // causes the type checker to work much harder. 1333 if ld.Config.Mode&NeedTypesInfo == 0 { 1334 return nil 1335 } 1336 return &types.Info{ 1337 Types: make(map[ast.Expr]types.TypeAndValue), 1338 Defs: make(map[*ast.Ident]types.Object), 1339 Uses: make(map[*ast.Ident]types.Object), 1340 Implicits: make(map[ast.Node]types.Object), 1341 Instances: make(map[*ast.Ident]types.Instance), 1342 Scopes: make(map[ast.Node]*types.Scope), 1343 Selections: make(map[*ast.SelectorExpr]*types.Selection), 1344 FileVersions: make(map[*ast.File]string), 1345 } 1346 } 1347 1348 // An importFunc is an implementation of the single-method 1349 // types.Importer interface based on a function value. 1350 type importerFunc func(path string) (*types.Package, error) 1351 1352 func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } 1353 1354 // We use a counting semaphore to limit 1355 // the number of parallel I/O calls or CPU threads per process. 1356 var ( 1357 ioLimit = make(chan unit, 20) 1358 cpuLimit = make(chan unit, runtime.GOMAXPROCS(0)) 1359 ) 1360 1361 func (ld *loader) parseFile(filename string) (*ast.File, error) { 1362 ld.parseCacheMu.Lock() 1363 v, ok := ld.parseCache[filename] 1364 if ok { 1365 // cache hit 1366 ld.parseCacheMu.Unlock() 1367 <-v.ready 1368 } else { 1369 // cache miss 1370 v = &parseValue{ready: make(chan struct{})} 1371 ld.parseCache[filename] = v 1372 ld.parseCacheMu.Unlock() 1373 1374 var src []byte 1375 for f, contents := range ld.Config.Overlay { 1376 // TODO(adonovan): Inefficient for large overlays. 1377 // Do an exact name-based map lookup 1378 // (for nonexistent files) followed by a 1379 // FileID-based map lookup (for existing ones). 1380 if sameFile(f, filename) { 1381 src = contents 1382 break 1383 } 1384 } 1385 var err error 1386 if src == nil { 1387 ioLimit <- unit{} // acquire a token 1388 src, err = os.ReadFile(filename) 1389 <-ioLimit // release a token 1390 } 1391 if err != nil { 1392 v.err = err 1393 } else { 1394 // Parsing is CPU intensive. 1395 cpuLimit <- unit{} // acquire a token 1396 v.f, v.err = ld.ParseFile(ld.Fset, filename, src) 1397 <-cpuLimit // release a token 1398 } 1399 1400 close(v.ready) 1401 } 1402 return v.f, v.err 1403 } 1404 1405 // parseFiles reads and parses the Go source files and returns the ASTs 1406 // of the ones that could be at least partially parsed, along with a 1407 // list of I/O and parse errors encountered. 1408 // 1409 // Because files are scanned in parallel, the token.Pos 1410 // positions of the resulting ast.Files are not ordered. 1411 func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) { 1412 var ( 1413 n = len(filenames) 1414 parsed = make([]*ast.File, n) 1415 errors = make([]error, n) 1416 ) 1417 var g errgroup.Group 1418 for i, filename := range filenames { 1419 // This creates goroutines unnecessarily in the 1420 // cache-hit case, but that case is uncommon. 1421 g.Go(func() error { 1422 parsed[i], errors[i] = ld.parseFile(filename) 1423 return nil 1424 }) 1425 } 1426 g.Wait() 1427 1428 // Eliminate nils, preserving order. 1429 var o int 1430 for _, f := range parsed { 1431 if f != nil { 1432 parsed[o] = f 1433 o++ 1434 } 1435 } 1436 parsed = parsed[:o] 1437 1438 o = 0 1439 for _, err := range errors { 1440 if err != nil { 1441 errors[o] = err 1442 o++ 1443 } 1444 } 1445 errors = errors[:o] 1446 1447 return parsed, errors 1448 } 1449 1450 // sameFile returns true if x and y have the same basename and denote 1451 // the same file. 1452 func sameFile(x, y string) bool { 1453 if x == y { 1454 // It could be the case that y doesn't exist. 1455 // For instance, it may be an overlay file that 1456 // hasn't been written to disk. To handle that case 1457 // let x == y through. (We added the exact absolute path 1458 // string to the CompiledGoFiles list, so the unwritten 1459 // overlay case implies x==y.) 1460 return true 1461 } 1462 if strings.EqualFold(filepath.Base(x), filepath.Base(y)) { // (optimisation) 1463 if xi, err := os.Stat(x); err == nil { 1464 if yi, err := os.Stat(y); err == nil { 1465 return os.SameFile(xi, yi) 1466 } 1467 } 1468 } 1469 return false 1470 } 1471 1472 // loadFromExportData ensures that type information is present for the specified 1473 // package, loading it from an export data file on the first request. 1474 // On success it sets lpkg.Types to a new Package. 1475 func (ld *loader) loadFromExportData(lpkg *loaderPackage) error { 1476 if lpkg.PkgPath == "" { 1477 log.Fatalf("internal error: Package %s has no PkgPath", lpkg) 1478 } 1479 1480 // Because gcexportdata.Read has the potential to create or 1481 // modify the types.Package for each node in the transitive 1482 // closure of dependencies of lpkg, all exportdata operations 1483 // must be sequential. (Finer-grained locking would require 1484 // changes to the gcexportdata API.) 1485 // 1486 // The exportMu lock guards the lpkg.Types field and the 1487 // types.Package it points to, for each loaderPackage in the graph. 1488 // 1489 // Not all accesses to Package.Pkg need to be protected by exportMu: 1490 // graph ordering ensures that direct dependencies of source 1491 // packages are fully loaded before the importer reads their Pkg field. 1492 ld.exportMu.Lock() 1493 defer ld.exportMu.Unlock() 1494 1495 if tpkg := lpkg.Types; tpkg != nil && tpkg.Complete() { 1496 return nil // cache hit 1497 } 1498 1499 lpkg.IllTyped = true // fail safe 1500 1501 if lpkg.ExportFile == "" { 1502 // Errors while building export data will have been printed to stderr. 1503 return fmt.Errorf("no export data file") 1504 } 1505 f, err := os.Open(lpkg.ExportFile) 1506 if err != nil { 1507 return err 1508 } 1509 defer f.Close() 1510 1511 // Read gc export data. 1512 // 1513 // We don't currently support gccgo export data because all 1514 // underlying workspaces use the gc toolchain. (Even build 1515 // systems that support gccgo don't use it for workspace 1516 // queries.) 1517 r, err := gcexportdata.NewReader(f) 1518 if err != nil { 1519 return fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) 1520 } 1521 1522 // Build the view. 1523 // 1524 // The gcexportdata machinery has no concept of package ID. 1525 // It identifies packages by their PkgPath, which although not 1526 // globally unique is unique within the scope of one invocation 1527 // of the linker, type-checker, or gcexportdata. 1528 // 1529 // So, we must build a PkgPath-keyed view of the global 1530 // (conceptually ID-keyed) cache of packages and pass it to 1531 // gcexportdata. The view must contain every existing 1532 // package that might possibly be mentioned by the 1533 // current package---its transitive closure. 1534 // 1535 // In loadPackage, we unconditionally create a types.Package for 1536 // each dependency so that export data loading does not 1537 // create new ones. 1538 // 1539 // TODO(adonovan): it would be simpler and more efficient 1540 // if the export data machinery invoked a callback to 1541 // get-or-create a package instead of a map. 1542 // 1543 view := make(map[string]*types.Package) // view seen by gcexportdata 1544 seen := make(map[*loaderPackage]bool) // all visited packages 1545 var visit func(pkgs map[string]*Package) 1546 visit = func(pkgs map[string]*Package) { 1547 for _, p := range pkgs { 1548 lpkg := ld.pkgs[p.ID] 1549 if !seen[lpkg] { 1550 seen[lpkg] = true 1551 view[lpkg.PkgPath] = lpkg.Types 1552 visit(lpkg.Imports) 1553 } 1554 } 1555 } 1556 visit(lpkg.Imports) 1557 1558 viewLen := len(view) + 1 // adding the self package 1559 // Parse the export data. 1560 // (May modify incomplete packages in view but not create new ones.) 1561 tpkg, err := gcexportdata.Read(r, ld.Fset, view, lpkg.PkgPath) 1562 if err != nil { 1563 return fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) 1564 } 1565 if _, ok := view["go.shape"]; ok { 1566 // Account for the pseudopackage "go.shape" that gets 1567 // created by generic code. 1568 viewLen++ 1569 } 1570 if viewLen != len(view) { 1571 log.Panicf("golang.org/x/tools/go/packages: unexpected new packages during load of %s", lpkg.PkgPath) 1572 } 1573 1574 lpkg.Types = tpkg 1575 lpkg.IllTyped = false 1576 return nil 1577 } 1578 1579 // impliedLoadMode returns loadMode with its dependencies. 1580 func impliedLoadMode(loadMode LoadMode) LoadMode { 1581 if loadMode&(NeedDeps|NeedTypes|NeedTypesInfo) != 0 { 1582 // All these things require knowing the import graph. 1583 loadMode |= NeedImports 1584 } 1585 if loadMode&NeedTypes != 0 { 1586 // Types require the GoVersion from Module. 1587 loadMode |= NeedModule 1588 } 1589 1590 return loadMode 1591 } 1592 1593 func usesExportData(cfg *Config) bool { 1594 return cfg.Mode&NeedExportFile != 0 || cfg.Mode&NeedTypes != 0 && cfg.Mode&NeedDeps == 0 1595 } 1596 1597 type unit struct{} 1598 1599 func cond[T any](cond bool, t, f T) T { 1600 if cond { 1601 return t 1602 } else { 1603 return f 1604 } 1605 }