Source file src/cmd/go/proxy_test.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main_test
     6  
     7  import (
     8  	"archive/zip"
     9  	"bytes"
    10  	"encoding/json"
    11  	"errors"
    12  	"flag"
    13  	"fmt"
    14  	"internal/txtar"
    15  	"io"
    16  	"io/fs"
    17  	"log"
    18  	"net"
    19  	"net/http"
    20  	"os"
    21  	"path/filepath"
    22  	"strconv"
    23  	"strings"
    24  	"sync"
    25  	"testing"
    26  
    27  	"cmd/go/internal/modfetch/codehost"
    28  	"cmd/internal/par"
    29  
    30  	"golang.org/x/mod/module"
    31  	"golang.org/x/mod/semver"
    32  	"golang.org/x/mod/sumdb"
    33  	"golang.org/x/mod/sumdb/dirhash"
    34  )
    35  
    36  var (
    37  	proxyAddr = flag.String("proxy", "", "run proxy on this network address instead of running any tests")
    38  	proxyURL  string
    39  )
    40  
    41  var proxyOnce sync.Once
    42  
    43  // StartProxy starts the Go module proxy running on *proxyAddr (like "localhost:1234")
    44  // and sets proxyURL to the GOPROXY setting to use to access the proxy.
    45  // Subsequent calls are no-ops.
    46  //
    47  // The proxy serves from testdata/mod. See testdata/mod/README.
    48  func StartProxy() {
    49  	proxyOnce.Do(func() {
    50  		readModList()
    51  		addr := *proxyAddr
    52  		if addr == "" {
    53  			addr = "localhost:0"
    54  		}
    55  		l, err := net.Listen("tcp", addr)
    56  		if err != nil {
    57  			log.Fatal(err)
    58  		}
    59  		*proxyAddr = l.Addr().String()
    60  		proxyURL = "http://" + *proxyAddr + "/mod"
    61  		fmt.Fprintf(os.Stderr, "go test proxy running at GOPROXY=%s\n", proxyURL)
    62  		go func() {
    63  			log.Fatalf("go proxy: http.Serve: %v", http.Serve(l, http.HandlerFunc(proxyHandler)))
    64  		}()
    65  
    66  		// Prepopulate main sumdb.
    67  		for _, mod := range modList {
    68  			sumdbOps.Lookup(nil, mod)
    69  		}
    70  	})
    71  }
    72  
    73  var modList []module.Version
    74  
    75  func readModList() {
    76  	files, err := os.ReadDir("testdata/mod")
    77  	if err != nil {
    78  		log.Fatal(err)
    79  	}
    80  	for _, f := range files {
    81  		name := f.Name()
    82  		if !strings.HasSuffix(name, ".txt") {
    83  			continue
    84  		}
    85  		name = strings.TrimSuffix(name, ".txt")
    86  		i := strings.LastIndex(name, "_v")
    87  		if i < 0 {
    88  			continue
    89  		}
    90  		encPath := strings.ReplaceAll(name[:i], "_", "/")
    91  		path, err := module.UnescapePath(encPath)
    92  		if err != nil {
    93  			if testing.Verbose() && encPath != "example.com/invalidpath/v1" {
    94  				fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
    95  			}
    96  			continue
    97  		}
    98  		encVers := name[i+1:]
    99  		vers, err := module.UnescapeVersion(encVers)
   100  		if err != nil {
   101  			fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
   102  			continue
   103  		}
   104  		modList = append(modList, module.Version{Path: path, Version: vers})
   105  	}
   106  }
   107  
   108  var zipCache par.ErrCache[*txtar.Archive, []byte]
   109  
   110  const (
   111  	testSumDBName        = "localhost.localdev/sumdb"
   112  	testSumDBVerifierKey = "localhost.localdev/sumdb+00000c67+AcTrnkbUA+TU4heY3hkjiSES/DSQniBqIeQ/YppAUtK6"
   113  	testSumDBSignerKey   = "PRIVATE+KEY+localhost.localdev/sumdb+00000c67+AXu6+oaVaOYuQOFrf1V59JK1owcFlJcHwwXHDfDGxSPk"
   114  )
   115  
   116  var (
   117  	sumdbOps    = sumdb.NewTestServer(testSumDBSignerKey, proxyGoSum)
   118  	sumdbServer = sumdb.NewServer(sumdbOps)
   119  
   120  	sumdbWrongOps    = sumdb.NewTestServer(testSumDBSignerKey, proxyGoSumWrong)
   121  	sumdbWrongServer = sumdb.NewServer(sumdbWrongOps)
   122  )
   123  
   124  // proxyHandler serves the Go module proxy protocol.
   125  // See the proxy section of https://research.swtch.com/vgo-module.
   126  func proxyHandler(w http.ResponseWriter, r *http.Request) {
   127  	if !strings.HasPrefix(r.URL.Path, "/mod/") {
   128  		http.NotFound(w, r)
   129  		return
   130  	}
   131  	path := r.URL.Path[len("/mod/"):]
   132  
   133  	// /mod/invalid returns faulty responses.
   134  	if strings.HasPrefix(path, "invalid/") {
   135  		w.Write([]byte("invalid"))
   136  		return
   137  	}
   138  
   139  	// Next element may opt into special behavior.
   140  	if j := strings.Index(path, "/"); j >= 0 {
   141  		n, err := strconv.Atoi(path[:j])
   142  		if err == nil && n >= 200 {
   143  			w.WriteHeader(n)
   144  			return
   145  		}
   146  		if strings.HasPrefix(path, "sumdb-") {
   147  			n, err := strconv.Atoi(path[len("sumdb-"):j])
   148  			if err == nil && n >= 200 {
   149  				if strings.HasPrefix(path[j:], "/sumdb/") {
   150  					w.WriteHeader(n)
   151  					return
   152  				}
   153  				path = path[j+1:]
   154  			}
   155  		}
   156  	}
   157  
   158  	// Request for $GOPROXY/sumdb-direct is direct sumdb access.
   159  	// (Client thinks it is talking directly to a sumdb.)
   160  	if strings.HasPrefix(path, "sumdb-direct/") {
   161  		r.URL.Path = path[len("sumdb-direct"):]
   162  		sumdbServer.ServeHTTP(w, r)
   163  		return
   164  	}
   165  
   166  	// Request for $GOPROXY/sumdb-wrong is direct sumdb access
   167  	// but all the hashes are wrong.
   168  	// (Client thinks it is talking directly to a sumdb.)
   169  	if strings.HasPrefix(path, "sumdb-wrong/") {
   170  		r.URL.Path = path[len("sumdb-wrong"):]
   171  		sumdbWrongServer.ServeHTTP(w, r)
   172  		return
   173  	}
   174  
   175  	// Request for $GOPROXY/sumdb-redirect/module@version:/lookup/...
   176  	// performs a lookup for module@version rather than the requested module.
   177  	if strings.HasPrefix(path, "sumdb-redirect/") {
   178  		redirect, rest, ok := strings.Cut(path[len("sumdb-redirect"):], ":")
   179  		if !ok {
   180  			w.WriteHeader(500)
   181  			return
   182  		}
   183  		if strings.HasPrefix(rest, "/lookup/") {
   184  			r.URL.Path = "/lookup" + redirect
   185  		} else {
   186  			r.URL.Path = rest
   187  		}
   188  		sumdbServer.ServeHTTP(w, r)
   189  		return
   190  	}
   191  
   192  	// Request for $GOPROXY/redirect/<count>/... goes to redirects.
   193  	if strings.HasPrefix(path, "redirect/") {
   194  		path = path[len("redirect/"):]
   195  		if j := strings.Index(path, "/"); j >= 0 {
   196  			count, err := strconv.Atoi(path[:j])
   197  			if err != nil {
   198  				return
   199  			}
   200  
   201  			// The last redirect.
   202  			if count <= 1 {
   203  				http.Redirect(w, r, fmt.Sprintf("/mod/%s", path[j+1:]), 302)
   204  				return
   205  			}
   206  			http.Redirect(w, r, fmt.Sprintf("/mod/redirect/%d/%s", count-1, path[j+1:]), 302)
   207  			return
   208  		}
   209  	}
   210  
   211  	// Request for $GOPROXY/sumdb/<name>/supported
   212  	// is checking whether it's OK to access sumdb via the proxy.
   213  	if path == "sumdb/"+testSumDBName+"/supported" {
   214  		w.WriteHeader(200)
   215  		return
   216  	}
   217  
   218  	// Request for $GOPROXY/sumdb/<name>/... goes to sumdb.
   219  	if sumdbPrefix := "sumdb/" + testSumDBName + "/"; strings.HasPrefix(path, sumdbPrefix) {
   220  		r.URL.Path = path[len(sumdbPrefix)-1:]
   221  		sumdbServer.ServeHTTP(w, r)
   222  		return
   223  	}
   224  
   225  	// Module proxy request: /mod/path/@latest
   226  	// Rewrite to /mod/path/@v/<latest>.info where <latest> is the semantically
   227  	// latest version, including pseudo-versions.
   228  	if i := strings.LastIndex(path, "/@latest"); i >= 0 {
   229  		enc := path[:i]
   230  		modPath, err := module.UnescapePath(enc)
   231  		if err != nil {
   232  			if testing.Verbose() {
   233  				fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
   234  			}
   235  			http.NotFound(w, r)
   236  			return
   237  		}
   238  
   239  		// Imitate what "latest" does in direct mode and what proxy.golang.org does.
   240  		// Use the latest released version.
   241  		// If there is no released version, use the latest prereleased version.
   242  		// Otherwise, use the latest pseudoversion.
   243  		var latestRelease, latestPrerelease, latestPseudo string
   244  		for _, m := range modList {
   245  			if m.Path != modPath {
   246  				continue
   247  			}
   248  			if module.IsPseudoVersion(m.Version) && (latestPseudo == "" || semver.Compare(latestPseudo, m.Version) > 0) {
   249  				latestPseudo = m.Version
   250  			} else if semver.Prerelease(m.Version) != "" && (latestPrerelease == "" || semver.Compare(latestPrerelease, m.Version) > 0) {
   251  				latestPrerelease = m.Version
   252  			} else if latestRelease == "" || semver.Compare(latestRelease, m.Version) > 0 {
   253  				latestRelease = m.Version
   254  			}
   255  		}
   256  		var latest string
   257  		if latestRelease != "" {
   258  			latest = latestRelease
   259  		} else if latestPrerelease != "" {
   260  			latest = latestPrerelease
   261  		} else if latestPseudo != "" {
   262  			latest = latestPseudo
   263  		} else {
   264  			http.NotFound(w, r)
   265  			return
   266  		}
   267  
   268  		encVers, err := module.EscapeVersion(latest)
   269  		if err != nil {
   270  			http.Error(w, err.Error(), http.StatusInternalServerError)
   271  			return
   272  		}
   273  		path = fmt.Sprintf("%s/@v/%s.info", enc, encVers)
   274  	}
   275  
   276  	// Module proxy request: /mod/path/@v/version[.suffix]
   277  	i := strings.Index(path, "/@v/")
   278  	if i < 0 {
   279  		http.NotFound(w, r)
   280  		return
   281  	}
   282  	enc, file := path[:i], path[i+len("/@v/"):]
   283  	path, err := module.UnescapePath(enc)
   284  	if err != nil {
   285  		if testing.Verbose() {
   286  			fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
   287  		}
   288  		http.NotFound(w, r)
   289  		return
   290  	}
   291  	if file == "list" {
   292  		// list returns a list of versions, not including pseudo-versions.
   293  		// If the module has no tagged versions, we should serve an empty 200.
   294  		// If the module doesn't exist, we should serve 404 or 410.
   295  		found := false
   296  		for _, m := range modList {
   297  			if m.Path != path {
   298  				continue
   299  			}
   300  			found = true
   301  			if !module.IsPseudoVersion(m.Version) {
   302  				if err := module.Check(m.Path, m.Version); err == nil {
   303  					fmt.Fprintf(w, "%s\n", m.Version)
   304  				}
   305  			}
   306  		}
   307  		if !found {
   308  			http.NotFound(w, r)
   309  		}
   310  		return
   311  	}
   312  
   313  	i = strings.LastIndex(file, ".")
   314  	if i < 0 {
   315  		http.NotFound(w, r)
   316  		return
   317  	}
   318  	encVers, ext := file[:i], file[i+1:]
   319  	vers, err := module.UnescapeVersion(encVers)
   320  	if err != nil {
   321  		fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err)
   322  		http.NotFound(w, r)
   323  		return
   324  	}
   325  
   326  	if codehost.AllHex(vers) {
   327  		var best string
   328  		// Convert commit hash (only) to known version.
   329  		// Use latest version in semver priority, to match similar logic
   330  		// in the repo-based module server (see modfetch.(*codeRepo).convert).
   331  		for _, m := range modList {
   332  			if m.Path == path && semver.Compare(best, m.Version) < 0 {
   333  				var hash string
   334  				if module.IsPseudoVersion(m.Version) {
   335  					hash = m.Version[strings.LastIndex(m.Version, "-")+1:]
   336  				} else {
   337  					hash = findHash(m)
   338  				}
   339  				if strings.HasPrefix(hash, vers) || strings.HasPrefix(vers, hash) {
   340  					best = m.Version
   341  				}
   342  			}
   343  		}
   344  		if best != "" {
   345  			vers = best
   346  		}
   347  	}
   348  
   349  	a, err := readArchive(path, vers)
   350  	if err != nil {
   351  		if testing.Verbose() {
   352  			fmt.Fprintf(os.Stderr, "go proxy: no archive %s %s: %v\n", path, vers, err)
   353  		}
   354  		if errors.Is(err, fs.ErrNotExist) {
   355  			http.NotFound(w, r)
   356  		} else {
   357  			http.Error(w, "cannot load archive", 500)
   358  		}
   359  		return
   360  	}
   361  
   362  	switch ext {
   363  	case "info", "mod":
   364  		want := "." + ext
   365  		for _, f := range a.Files {
   366  			if f.Name == want {
   367  				w.Write(f.Data)
   368  				return
   369  			}
   370  		}
   371  
   372  	case "zip":
   373  		zipBytes, err := zipCache.Do(a, func() ([]byte, error) {
   374  			var buf bytes.Buffer
   375  			z := zip.NewWriter(&buf)
   376  			for _, f := range a.Files {
   377  				if f.Name == ".info" || f.Name == ".mod" || f.Name == ".zip" {
   378  					continue
   379  				}
   380  				var zipName string
   381  				if strings.HasPrefix(f.Name, "/") {
   382  					zipName = f.Name[1:]
   383  				} else {
   384  					zipName = path + "@" + vers + "/" + f.Name
   385  				}
   386  				zf, err := z.Create(zipName)
   387  				if err != nil {
   388  					return nil, err
   389  				}
   390  				if _, err := zf.Write(f.Data); err != nil {
   391  					return nil, err
   392  				}
   393  			}
   394  			if err := z.Close(); err != nil {
   395  				return nil, err
   396  			}
   397  			return buf.Bytes(), nil
   398  		})
   399  
   400  		if err != nil {
   401  			if testing.Verbose() {
   402  				fmt.Fprintf(os.Stderr, "go proxy: %v\n", err)
   403  			}
   404  			http.Error(w, err.Error(), 500)
   405  			return
   406  		}
   407  		w.Write(zipBytes)
   408  		return
   409  
   410  	}
   411  	http.NotFound(w, r)
   412  }
   413  
   414  func findHash(m module.Version) string {
   415  	a, err := readArchive(m.Path, m.Version)
   416  	if err != nil {
   417  		return ""
   418  	}
   419  	var data []byte
   420  	for _, f := range a.Files {
   421  		if f.Name == ".info" {
   422  			data = f.Data
   423  			break
   424  		}
   425  	}
   426  	var info struct{ Short string }
   427  	json.Unmarshal(data, &info)
   428  	return info.Short
   429  }
   430  
   431  var archiveCache par.Cache[string, *txtar.Archive]
   432  
   433  var cmdGoDir, _ = os.Getwd()
   434  
   435  func readArchive(path, vers string) (*txtar.Archive, error) {
   436  	enc, err := module.EscapePath(path)
   437  	if err != nil {
   438  		return nil, err
   439  	}
   440  	encVers, err := module.EscapeVersion(vers)
   441  	if err != nil {
   442  		return nil, err
   443  	}
   444  
   445  	prefix := strings.ReplaceAll(enc, "/", "_")
   446  	name := filepath.Join(cmdGoDir, "testdata/mod", prefix+"_"+encVers+".txt")
   447  	a := archiveCache.Do(name, func() *txtar.Archive {
   448  		a, err := txtar.ParseFile(name)
   449  		if err != nil {
   450  			if testing.Verbose() || !os.IsNotExist(err) {
   451  				fmt.Fprintf(os.Stderr, "go proxy: %v\n", err)
   452  			}
   453  			a = nil
   454  		}
   455  		return a
   456  	})
   457  	if a == nil {
   458  		return nil, fs.ErrNotExist
   459  	}
   460  	return a, nil
   461  }
   462  
   463  // proxyGoSum returns the two go.sum lines for path@vers.
   464  func proxyGoSum(path, vers string) ([]byte, error) {
   465  	a, err := readArchive(path, vers)
   466  	if err != nil {
   467  		return nil, err
   468  	}
   469  	var names []string
   470  	files := make(map[string][]byte)
   471  	var gomod []byte
   472  	for _, f := range a.Files {
   473  		if strings.HasPrefix(f.Name, ".") {
   474  			if f.Name == ".mod" {
   475  				gomod = f.Data
   476  			}
   477  			continue
   478  		}
   479  		name := path + "@" + vers + "/" + f.Name
   480  		names = append(names, name)
   481  		files[name] = f.Data
   482  	}
   483  	h1, err := dirhash.Hash1(names, func(name string) (io.ReadCloser, error) {
   484  		data := files[name]
   485  		return io.NopCloser(bytes.NewReader(data)), nil
   486  	})
   487  	if err != nil {
   488  		return nil, err
   489  	}
   490  	h1mod, err := dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
   491  		return io.NopCloser(bytes.NewReader(gomod)), nil
   492  	})
   493  	if err != nil {
   494  		return nil, err
   495  	}
   496  	data := []byte(fmt.Sprintf("%s %s %s\n%s %s/go.mod %s\n", path, vers, h1, path, vers, h1mod))
   497  	return data, nil
   498  }
   499  
   500  // proxyGoSumWrong returns the wrong lines.
   501  func proxyGoSumWrong(path, vers string) ([]byte, error) {
   502  	data := []byte(fmt.Sprintf("%s %s %s\n%s %s/go.mod %s\n", path, vers, "h1:wrong", path, vers, "h1:wrong"))
   503  	return data, nil
   504  }
   505  

View as plain text