src

Go monorepo.
git clone git://code.dwrz.net/src
Log | Files | Refs

client.go (7912B)

      1 // Copyright 2023 The Go Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style
      3 // license that can be found in the LICENSE file.
      4 
      5 // Package client provides an interface for accessing vulnerability
      6 // databases, via either HTTP or local filesystem access.
      7 //
      8 // The protocol is described at https://go.dev/security/vuln/database.
      9 package client
     10 
     11 import (
     12 	"bytes"
     13 	"context"
     14 	"encoding/json"
     15 	"errors"
     16 	"fmt"
     17 	"net/http"
     18 	"net/url"
     19 	"os"
     20 	"path/filepath"
     21 	"sort"
     22 	"strings"
     23 	"time"
     24 
     25 	"golang.org/x/sync/errgroup"
     26 	"golang.org/x/vuln/internal/derrors"
     27 	"golang.org/x/vuln/internal/osv"
     28 	isem "golang.org/x/vuln/internal/semver"
     29 	"golang.org/x/vuln/internal/web"
     30 )
     31 
     32 // A Client for reading vulnerability databases.
     33 type Client struct {
     34 	source
     35 }
     36 
     37 type Options struct {
     38 	HTTPClient *http.Client
     39 }
     40 
     41 // NewClient returns a client that reads the vulnerability database
     42 // in source (an "http" or "file" prefixed URL).
     43 //
     44 // It supports databases following the API described
     45 // in https://go.dev/security/vuln/database#api.
     46 func NewClient(source string, opts *Options) (_ *Client, err error) {
     47 	source = strings.TrimRight(source, "/")
     48 	uri, err := url.Parse(source)
     49 	if err != nil {
     50 		return nil, err
     51 	}
     52 	switch uri.Scheme {
     53 	case "http", "https":
     54 		return newHTTPClient(uri, opts)
     55 	case "file":
     56 		return newLocalClient(uri)
     57 	default:
     58 		return nil, fmt.Errorf("source %q has unsupported scheme", uri)
     59 	}
     60 }
     61 
     62 var errUnknownSchema = errors.New("unrecognized vulndb format; see https://go.dev/security/vuln/database#api for accepted schema")
     63 
     64 func newHTTPClient(uri *url.URL, opts *Options) (*Client, error) {
     65 	source := uri.String()
     66 
     67 	// v1 returns true if the source likely follows the V1 schema.
     68 	v1 := func() bool {
     69 		return source == "https://vuln.go.dev" ||
     70 			endpointExistsHTTP(source, "index/modules.json.gz")
     71 	}
     72 
     73 	if v1() {
     74 		return &Client{source: newHTTPSource(uri.String(), opts)}, nil
     75 	}
     76 
     77 	return nil, errUnknownSchema
     78 }
     79 
     80 func endpointExistsHTTP(source, endpoint string) bool {
     81 	r, err := http.Head(source + "/" + endpoint)
     82 	return err == nil && r.StatusCode == http.StatusOK
     83 }
     84 
     85 func newLocalClient(uri *url.URL) (*Client, error) {
     86 	dir, err := toDir(uri)
     87 	if err != nil {
     88 		return nil, err
     89 	}
     90 
     91 	// Check if the DB likely follows the v1 schema by
     92 	// looking for the "index/modules.json" endpoint.
     93 	if endpointExistsDir(dir, modulesEndpoint+".json") {
     94 		return &Client{source: newLocalSource(dir)}, nil
     95 	}
     96 
     97 	// If the DB doesn't follow the v1 schema,
     98 	// attempt to intepret it as a flat list of OSV files.
     99 	// This is currently a "hidden" feature, so don't output the
    100 	// specific error if this fails.
    101 	src, err := newHybridSource(dir)
    102 	if err != nil {
    103 		return nil, errUnknownSchema
    104 	}
    105 	return &Client{source: src}, nil
    106 }
    107 
    108 func toDir(uri *url.URL) (string, error) {
    109 	dir, err := web.URLToFilePath(uri)
    110 	if err != nil {
    111 		return "", err
    112 	}
    113 	fi, err := os.Stat(dir)
    114 	if err != nil {
    115 		return "", err
    116 	}
    117 	if !fi.IsDir() {
    118 		return "", fmt.Errorf("%s is not a directory", dir)
    119 	}
    120 	return dir, nil
    121 }
    122 
    123 func endpointExistsDir(dir, endpoint string) bool {
    124 	_, err := os.Stat(filepath.Join(dir, endpoint))
    125 	return err == nil
    126 }
    127 
    128 func NewInMemoryClient(entries []*osv.Entry) (*Client, error) {
    129 	s, err := newInMemorySource(entries)
    130 	if err != nil {
    131 		return nil, err
    132 	}
    133 	return &Client{source: s}, nil
    134 }
    135 
    136 func (c *Client) LastModifiedTime(ctx context.Context) (_ time.Time, err error) {
    137 	derrors.Wrap(&err, "LastModifiedTime()")
    138 
    139 	b, err := c.source.get(ctx, dbEndpoint)
    140 	if err != nil {
    141 		return time.Time{}, err
    142 	}
    143 
    144 	var dbMeta dbMeta
    145 	if err := json.Unmarshal(b, &dbMeta); err != nil {
    146 		return time.Time{}, err
    147 	}
    148 
    149 	return dbMeta.Modified, nil
    150 }
    151 
    152 type ModuleRequest struct {
    153 	// The module path to filter on.
    154 	// This must be set (if empty, ByModule errors).
    155 	Path string
    156 	// (Optional) If set, only return vulnerabilities affected
    157 	// at this version.
    158 	Version string
    159 }
    160 
    161 type ModuleResponse struct {
    162 	Path    string
    163 	Version string
    164 	Entries []*osv.Entry
    165 }
    166 
    167 // ByModules returns a list of responses
    168 // containing the OSV entries corresponding to each request.
    169 //
    170 // The order of the requests is preserved, and each request has
    171 // a response even if there are no entries (in which case the Entries
    172 // field is nil).
    173 func (c *Client) ByModules(ctx context.Context, reqs []*ModuleRequest) (_ []*ModuleResponse, err error) {
    174 	derrors.Wrap(&err, "ByModules(%v)", reqs)
    175 
    176 	metas, err := c.moduleMetas(ctx, reqs)
    177 	if err != nil {
    178 		return nil, err
    179 	}
    180 
    181 	resps := make([]*ModuleResponse, len(reqs))
    182 	g, gctx := errgroup.WithContext(ctx)
    183 	g.SetLimit(10)
    184 	for i, req := range reqs {
    185 		i, req := i, req
    186 		g.Go(func() error {
    187 			entries, err := c.byModule(gctx, req, metas[i])
    188 			if err != nil {
    189 				return err
    190 			}
    191 			resps[i] = &ModuleResponse{
    192 				Path:    req.Path,
    193 				Version: req.Version,
    194 				Entries: entries,
    195 			}
    196 			return nil
    197 		})
    198 	}
    199 	if err := g.Wait(); err != nil {
    200 		return nil, err
    201 	}
    202 
    203 	return resps, nil
    204 }
    205 
    206 func (c *Client) moduleMetas(ctx context.Context, reqs []*ModuleRequest) (_ []*moduleMeta, err error) {
    207 	b, err := c.source.get(ctx, modulesEndpoint)
    208 	if err != nil {
    209 		return nil, err
    210 	}
    211 
    212 	dec, err := newStreamDecoder(b)
    213 	if err != nil {
    214 		return nil, err
    215 	}
    216 
    217 	metas := make([]*moduleMeta, len(reqs))
    218 	for dec.More() {
    219 		var m moduleMeta
    220 		err := dec.Decode(&m)
    221 		if err != nil {
    222 			return nil, err
    223 		}
    224 		for i, req := range reqs {
    225 			if m.Path == req.Path {
    226 				metas[i] = &m
    227 			}
    228 		}
    229 	}
    230 
    231 	return metas, nil
    232 }
    233 
    234 // byModule returns the OSV entries matching the ModuleRequest,
    235 // or (nil, nil) if there are none.
    236 func (c *Client) byModule(ctx context.Context, req *ModuleRequest, m *moduleMeta) (_ []*osv.Entry, err error) {
    237 	// This module isn't in the database.
    238 	if m == nil {
    239 		return nil, nil
    240 	}
    241 
    242 	if req.Path == "" {
    243 		return nil, fmt.Errorf("module path must be set")
    244 	}
    245 
    246 	if req.Version != "" && !isem.Valid(req.Version) {
    247 		return nil, fmt.Errorf("version %s is not valid semver", req.Version)
    248 	}
    249 
    250 	var ids []string
    251 	for _, v := range m.Vulns {
    252 		if v.Fixed == "" || isem.Less(req.Version, v.Fixed) {
    253 			ids = append(ids, v.ID)
    254 		}
    255 	}
    256 
    257 	if len(ids) == 0 {
    258 		return nil, nil
    259 	}
    260 
    261 	entries, err := c.byIDs(ctx, ids)
    262 	if err != nil {
    263 		return nil, err
    264 	}
    265 
    266 	// Filter by version.
    267 	if req.Version != "" {
    268 		affected := func(e *osv.Entry) bool {
    269 			for _, a := range e.Affected {
    270 				if a.Module.Path == req.Path && isem.Affects(a.Ranges, req.Version) {
    271 					return true
    272 				}
    273 			}
    274 			return false
    275 		}
    276 
    277 		var filtered []*osv.Entry
    278 		for _, entry := range entries {
    279 			if affected(entry) {
    280 				filtered = append(filtered, entry)
    281 			}
    282 		}
    283 		if len(filtered) == 0 {
    284 			return nil, nil
    285 		}
    286 	}
    287 
    288 	sort.SliceStable(entries, func(i, j int) bool {
    289 		return entries[i].ID < entries[j].ID
    290 	})
    291 
    292 	return entries, nil
    293 }
    294 
    295 func (c *Client) byIDs(ctx context.Context, ids []string) (_ []*osv.Entry, err error) {
    296 	entries := make([]*osv.Entry, len(ids))
    297 	g, gctx := errgroup.WithContext(ctx)
    298 	g.SetLimit(10)
    299 	for i, id := range ids {
    300 		i, id := i, id
    301 		g.Go(func() error {
    302 			e, err := c.byID(gctx, id)
    303 			if err != nil {
    304 				return err
    305 			}
    306 			entries[i] = e
    307 			return nil
    308 		})
    309 	}
    310 	if err := g.Wait(); err != nil {
    311 		return nil, err
    312 	}
    313 
    314 	return entries, nil
    315 }
    316 
    317 // byID returns the OSV entry with the given ID,
    318 // or an error if it does not exist / cannot be unmarshaled.
    319 func (c *Client) byID(ctx context.Context, id string) (_ *osv.Entry, err error) {
    320 	derrors.Wrap(&err, "byID(%s)", id)
    321 
    322 	b, err := c.source.get(ctx, entryEndpoint(id))
    323 	if err != nil {
    324 		return nil, err
    325 	}
    326 
    327 	var entry osv.Entry
    328 	if err := json.Unmarshal(b, &entry); err != nil {
    329 		return nil, err
    330 	}
    331 
    332 	return &entry, nil
    333 }
    334 
    335 // newStreamDecoder returns a decoder that can be used
    336 // to read an array of JSON objects.
    337 func newStreamDecoder(b []byte) (*json.Decoder, error) {
    338 	dec := json.NewDecoder(bytes.NewBuffer(b))
    339 
    340 	// skip open bracket
    341 	_, err := dec.Token()
    342 	if err != nil {
    343 		return nil, err
    344 	}
    345 
    346 	return dec, nil
    347 }