From 76f63ee890170f4884f4d213e8150d39d6758ad3 Mon Sep 17 00:00:00 2001 From: David Finkel Date: Fri, 13 Dec 2024 20:36:30 -0500 Subject: [PATCH] cmd/go: add support for git sha256 hashes Git's supported SHA 256 object hashes since 2.29[1] in 2021, and Gitlab now has experimental support for sha256 repos. Take rsc@'s suggestion of checking the of the length of the hashes from git ls-remote to determine whether a git repo is using sha256 hashes and decide whether to pass --object-format=sha256 to git init. Unfortunately, just passing --object-format=sha256 wasn't quite enough, though. We also need to decide whether the hash-length is 64 hex bytes or 40 hex bytes when resolving refs to decide whether we've been passed a full commit-hash. To that end, we use git config extensions.objectformat to decide whether the (now guaranteed local) repo is using sha256 hashes and hence 64-hex-byte strings. [1]: lost experimental status in 2.42 from Aug 2023 (https://lore.kernel.org/git/xmqqr0nwp8mv.fsf@gitster.g/) For: #68359 Change-Id: I47f480ab8334128c5d17570fe76722367d0d8ed8 Reviewed-on: https://go-review.googlesource.com/c/go/+/636475 Auto-Submit: Sam Thanawalla LUCI-TryBot-Result: Go LUCI Reviewed-by: Sam Thanawalla Reviewed-by: Michael Matloob Reviewed-by: Michael Matloob Reviewed-by: David Finkel --- src/cmd/go/internal/modfetch/codehost/git.go | 57 ++++- .../go/internal/modfetch/codehost/git_test.go | 215 +++++++++++++++++- ...mod_download_git_bareRepository_sha256.txt | 30 +++ .../testdata/vcstest/git/gitrepo-sha256.txt | 69 ++++++ .../vcstest/go/mod/gitrepo-sha256.txt | 6 + 5 files changed, 370 insertions(+), 7 deletions(-) create mode 100644 src/cmd/go/testdata/script/mod_download_git_bareRepository_sha256.txt create mode 100644 src/cmd/go/testdata/vcstest/git/gitrepo-sha256.txt create mode 100644 src/cmd/go/testdata/vcstest/go/mod/gitrepo-sha256.txt diff --git a/src/cmd/go/internal/modfetch/codehost/git.go b/src/cmd/go/internal/modfetch/codehost/git.go index dfb3667889..b445ac2486 100644 --- a/src/cmd/go/internal/modfetch/codehost/git.go +++ b/src/cmd/go/internal/modfetch/codehost/git.go @@ -59,6 +59,7 @@ func newGitRepo(ctx context.Context, remote string, local bool) (Repo, error) { } r.dir = remote r.mu.Path = r.dir + ".lock" + r.sha256Hashes = r.checkConfigSHA256(ctx) return r, nil } // This is a remote path lookup. @@ -81,7 +82,20 @@ func newGitRepo(ctx context.Context, remote string, local bool) (Repo, error) { defer unlock() if _, err := os.Stat(filepath.Join(r.dir, "objects")); err != nil { - if _, err := Run(ctx, r.dir, "git", "init", "--bare"); err != nil { + repoSha256Hash := false + if refs, lrErr := r.loadRefs(ctx); lrErr == nil { + // Check any ref's hash, it doesn't matter which; they won't be mixed + // between sha1 and sha256 for the moment. + for _, refHash := range refs { + repoSha256Hash = len(refHash) == (256 / 4) + break + } + } + objFormatFlag := []string{} + if repoSha256Hash { + objFormatFlag = []string{"--object-format=sha256"} + } + if _, err := Run(ctx, r.dir, "git", "init", "--bare", objFormatFlag); err != nil { os.RemoveAll(r.dir) return nil, err } @@ -109,6 +123,7 @@ func newGitRepo(ctx context.Context, remote string, local bool) (Repo, error) { } } } + r.sha256Hashes = r.checkConfigSHA256(ctx) r.remoteURL = r.remote r.remote = "origin" return r, nil @@ -121,6 +136,9 @@ type gitRepo struct { local bool // local only lookups; no remote fetches dir string + // Repo uses the SHA256 for hashes, so expect the hashes to be 256/4 == 64-bytes in hex. + sha256Hashes bool + mu lockedfile.Mutex // protects fetchLevel and git repo state fetchLevel int @@ -386,6 +404,32 @@ func (r *gitRepo) findRef(ctx context.Context, hash string) (ref string, ok bool return "", false } +func (r *gitRepo) checkConfigSHA256(ctx context.Context) bool { + if hashType, sha256CfgErr := r.runGit(ctx, "git", "config", "extensions.objectformat"); sha256CfgErr == nil { + return "sha256" == strings.TrimSpace(string(hashType)) + } + return false +} + +func (r *gitRepo) hexHashLen() int { + if !r.sha256Hashes { + return 160 / 4 + } + return 256 / 4 +} + +// shortenObjectHash shortens a SHA1 or SHA256 hash (40 or 64 hex digits) to +// the canonical length used in pseudo-versions (12 hex digits). +func (r *gitRepo) shortenObjectHash(rev string) string { + if !r.sha256Hashes { + return ShortenSHA1(rev) + } + if AllHex(rev) && len(rev) == 256/4 { + return rev[:12] + } + return rev +} + // minHashDigits is the minimum number of digits to require // before accepting a hex digit sequence as potentially identifying // a specific commit in a git repo. (Of course, users can always @@ -399,7 +443,7 @@ const minHashDigits = 7 func (r *gitRepo) stat(ctx context.Context, rev string) (info *RevInfo, err error) { // Fast path: maybe rev is a hash we already have locally. didStatLocal := false - if len(rev) >= minHashDigits && len(rev) <= 40 && AllHex(rev) { + if len(rev) >= minHashDigits && len(rev) <= r.hexHashLen() && AllHex(rev) { if info, err := r.statLocal(ctx, rev, rev); err == nil { return info, nil } @@ -415,7 +459,8 @@ func (r *gitRepo) stat(ctx context.Context, rev string) (info *RevInfo, err erro // Maybe rev is the name of a tag or branch on the remote server. // Or maybe it's the prefix of a hash of a named ref. - // Try to resolve to both a ref (git name) and full (40-hex-digit) commit hash. + // Try to resolve to both a ref (git name) and full (40-hex-digit for + // sha1 64 for sha256) commit hash. refs, err := r.loadRefs(ctx) if err != nil { return nil, err @@ -436,7 +481,7 @@ func (r *gitRepo) stat(ctx context.Context, rev string) (info *RevInfo, err erro ref = "HEAD" hash = refs[ref] rev = hash // Replace rev, because meaning of HEAD can change. - } else if len(rev) >= minHashDigits && len(rev) <= 40 && AllHex(rev) { + } else if len(rev) >= minHashDigits && len(rev) <= r.hexHashLen() && AllHex(rev) { // At the least, we have a hash prefix we can look up after the fetch below. // Maybe we can map it to a full hash using the known refs. prefix := rev @@ -455,7 +500,7 @@ func (r *gitRepo) stat(ctx context.Context, rev string) (info *RevInfo, err erro hash = h } } - if hash == "" && len(rev) == 40 { // Didn't find a ref, but rev is a full hash. + if hash == "" && len(rev) == r.hexHashLen() { // Didn't find a ref, but rev is a full hash. hash = rev } } else { @@ -631,7 +676,7 @@ func (r *gitRepo) statLocal(ctx context.Context, version, rev string) (*RevInfo, Hash: hash, }, Name: hash, - Short: ShortenSHA1(hash), + Short: r.shortenObjectHash(hash), Time: time.Unix(t, 0).UTC(), Version: hash, } diff --git a/src/cmd/go/internal/modfetch/codehost/git_test.go b/src/cmd/go/internal/modfetch/codehost/git_test.go index eb06d3d7a9..cf89548f50 100644 --- a/src/cmd/go/internal/modfetch/codehost/git_test.go +++ b/src/cmd/go/internal/modfetch/codehost/git_test.go @@ -33,7 +33,7 @@ func TestMain(m *testing.M) { } } -var gitrepo1, hgrepo1, vgotest1 string +var gitrepo1, gitsha256repo, hgrepo1, vgotest1 string var altRepos = func() []string { return []string{ @@ -69,6 +69,7 @@ func localGitURL(t testing.TB) string { } repo := gitRepo{dir: localGitRepo} _, localGitURLErr = repo.runGit(context.Background(), "git", "config", "daemon.uploadarch", "true") + // TODO(david.finkel): do the same with the git repo using sha256 object hashes }) if localGitURLErr != nil { @@ -103,6 +104,7 @@ func testMain(m *testing.M) (err error) { }() gitrepo1 = srv.HTTP.URL + "/git/gitrepo1" + gitsha256repo = srv.HTTP.URL + "/git/gitrepo-sha256" hgrepo1 = srv.HTTP.URL + "/hg/hgrepo1" vgotest1 = srv.HTTP.URL + "/git/vgotest1" @@ -239,6 +241,26 @@ func TestTags(t *testing.T) { {"v1.2.4-annotated", "ede458df7cd0fdca520df19a33158086a8a68e81"}, }}, {gitrepo1, "2", []Tag{}}, + {gitsha256repo, "xxx", []Tag{}}, + {gitsha256repo, "", []Tag{ + {"v1.2.3", "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c"}, + {"v1.2.4-annotated", "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c"}, + {"v2.0.1", "b7550fd9d2129c724c39ae0536e8b2fae4364d8c82bb8b0880c9b71f67295d09"}, + {"v2.0.2", "1401e4e1fdb4169b51d44a1ff62af63ccc708bf5c12d15051268b51bbb6cbd82"}, + {"v2.3", "b7550fd9d2129c724c39ae0536e8b2fae4364d8c82bb8b0880c9b71f67295d09"}, + }}, + {gitsha256repo, "v", []Tag{ + {"v1.2.3", "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c"}, + {"v1.2.4-annotated", "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c"}, + {"v2.0.1", "b7550fd9d2129c724c39ae0536e8b2fae4364d8c82bb8b0880c9b71f67295d09"}, + {"v2.0.2", "1401e4e1fdb4169b51d44a1ff62af63ccc708bf5c12d15051268b51bbb6cbd82"}, + {"v2.3", "b7550fd9d2129c724c39ae0536e8b2fae4364d8c82bb8b0880c9b71f67295d09"}, + }}, + {gitsha256repo, "v1", []Tag{ + {"v1.2.3", "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c"}, + {"v1.2.4-annotated", "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c"}, + }}, + {gitsha256repo, "2", []Tag{}}, } { t.Run(path.Base(tt.repo)+"/"+tt.prefix, runTest(tt)) if tt.repo == gitrepo1 { @@ -303,6 +325,22 @@ func TestLatest(t *testing.T) { Tags: []string{"v1.2.3", "v1.2.4-annotated"}, }, }, + { + gitsha256repo, + &RevInfo{ + Origin: &Origin{ + VCS: "git", + URL: gitsha256repo, + Ref: "HEAD", + Hash: "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c", + }, + Name: "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c", + Short: "47b8b51b2a2d", + Version: "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c", + Time: time.Date(2018, 4, 17, 19, 43, 22, 0, time.UTC), + Tags: []string{"v1.2.3", "v1.2.4-annotated"}, + }, + }, { hgrepo1, &RevInfo{ @@ -391,6 +429,24 @@ func TestReadFile(t *testing.T) { file: "another.txt", err: fs.ErrNotExist.Error(), }, + { + repo: gitsha256repo, + rev: "latest", + file: "README", + data: "", + }, + { + repo: gitsha256repo, + rev: "v2", + file: "another.txt", + data: "another\n", + }, + { + repo: gitsha256repo, + rev: "v2.3.4", + file: "another.txt", + err: fs.ErrNotExist.Error(), + }, } { t.Run(path.Base(tt.repo)+"/"+tt.rev+"/"+tt.file, runTest(tt)) if tt.repo == gitrepo1 { @@ -481,6 +537,16 @@ func TestReadZip(t *testing.T) { "prefix/v2": 3, }, }, + { + repo: gitsha256repo, + rev: "v2.3.4", + subdir: "", + files: map[string]uint64{ + "prefix/": 0, + "prefix/README": 0, + "prefix/v2": 3, + }, + }, { repo: hgrepo1, rev: "v2.3.4", @@ -504,6 +570,18 @@ func TestReadZip(t *testing.T) { "prefix/foo.txt": 13, }, }, + { + repo: gitsha256repo, + rev: "v2", + subdir: "", + files: map[string]uint64{ + "prefix/": 0, + "prefix/README": 0, + "prefix/v2": 3, + "prefix/another.txt": 8, + "prefix/foo.txt": 13, + }, + }, { repo: hgrepo1, rev: "v2", @@ -530,6 +608,19 @@ func TestReadZip(t *testing.T) { "prefix/README": 0, }, }, + { + repo: gitsha256repo, + rev: "v3", + subdir: "", + files: map[string]uint64{ + "prefix/": 0, + "prefix/v3/": 0, + "prefix/v3/sub/": 0, + "prefix/v3/sub/dir/": 0, + "prefix/v3/sub/dir/file.txt": 16, + "prefix/README": 0, + }, + }, { repo: hgrepo1, rev: "v3", @@ -554,6 +645,18 @@ func TestReadZip(t *testing.T) { "prefix/v3/sub/dir/file.txt": 16, }, }, + { + repo: gitsha256repo, + rev: "v3", + subdir: "v3/sub/dir", + files: map[string]uint64{ + "prefix/": 0, + "prefix/v3/": 0, + "prefix/v3/sub/": 0, + "prefix/v3/sub/dir/": 0, + "prefix/v3/sub/dir/file.txt": 16, + }, + }, { repo: hgrepo1, rev: "v3", @@ -575,6 +678,18 @@ func TestReadZip(t *testing.T) { "prefix/v3/sub/dir/file.txt": 16, }, }, + { + repo: gitsha256repo, + rev: "v3", + subdir: "v3/sub", + files: map[string]uint64{ + "prefix/": 0, + "prefix/v3/": 0, + "prefix/v3/sub/": 0, + "prefix/v3/sub/dir/": 0, + "prefix/v3/sub/dir/file.txt": 16, + }, + }, { repo: hgrepo1, rev: "v3", @@ -590,6 +705,12 @@ func TestReadZip(t *testing.T) { subdir: "", err: "unknown revision", }, + { + repo: gitsha256repo, + rev: "aaaaaaaaab", + subdir: "", + err: "unknown revision", + }, { repo: hgrepo1, rev: "aaaaaaaaab", @@ -757,6 +878,98 @@ func TestStat(t *testing.T) { rev: "aaaaaaaaab", err: "unknown revision", }, + { + repo: gitsha256repo, + rev: "HEAD", + info: &RevInfo{ + Name: "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c", + Short: "47b8b51b2a2d", + Version: "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c", + Time: time.Date(2018, 4, 17, 19, 43, 22, 0, time.UTC), + Tags: []string{"v1.2.3", "v1.2.4-annotated"}, + }, + }, + { + repo: gitsha256repo, + rev: "v2", // branch + info: &RevInfo{ + Name: "1401e4e1fdb4169b51d44a1ff62af63ccc708bf5c12d15051268b51bbb6cbd82", + Short: "1401e4e1fdb4", + Version: "1401e4e1fdb4169b51d44a1ff62af63ccc708bf5c12d15051268b51bbb6cbd82", + Time: time.Date(2018, 4, 17, 20, 00, 32, 0, time.UTC), + Tags: []string{"v2.0.2"}, + }, + }, + { + repo: gitsha256repo, + rev: "v2.3.4", // badly-named branch (semver should be a tag) + info: &RevInfo{ + Name: "b7550fd9d2129c724c39ae0536e8b2fae4364d8c82bb8b0880c9b71f67295d09", + Short: "b7550fd9d212", + Version: "b7550fd9d2129c724c39ae0536e8b2fae4364d8c82bb8b0880c9b71f67295d09", + Time: time.Date(2018, 4, 17, 19, 45, 48, 0, time.UTC), + Tags: []string{"v2.0.1", "v2.3"}, + }, + }, + { + repo: gitsha256repo, + rev: "v2.3", // badly-named tag (we only respect full semver v2.3.0) + info: &RevInfo{ + Name: "b7550fd9d2129c724c39ae0536e8b2fae4364d8c82bb8b0880c9b71f67295d09", + Short: "b7550fd9d212", + Version: "v2.3", + Time: time.Date(2018, 4, 17, 19, 45, 48, 0, time.UTC), + Tags: []string{"v2.0.1", "v2.3"}, + }, + }, + { + repo: gitsha256repo, + rev: "v1.2.3", // tag + info: &RevInfo{ + Name: "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c", + Short: "47b8b51b2a2d", + Version: "v1.2.3", + Time: time.Date(2018, 4, 17, 19, 43, 22, 0, time.UTC), + Tags: []string{"v1.2.3", "v1.2.4-annotated"}, + }, + }, + { + repo: gitsha256repo, + rev: "47b8b51b", // hash prefix in refs + info: &RevInfo{ + Name: "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c", + Short: "47b8b51b2a2d", + Version: "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c", + Time: time.Date(2018, 4, 17, 19, 43, 22, 0, time.UTC), + Tags: []string{"v1.2.3", "v1.2.4-annotated"}, + }, + }, + { + repo: gitsha256repo, + rev: "0be440b6", // hash prefix not in refs + info: &RevInfo{ + Name: "0be440b60b6c81be26c7256781d8e57112ec46c8cd1a9481a8e78a283f10570c", + Short: "0be440b60b6c", + Version: "0be440b60b6c81be26c7256781d8e57112ec46c8cd1a9481a8e78a283f10570c", + Time: time.Date(2018, 4, 17, 20, 0, 19, 0, time.UTC), + }, + }, + { + repo: gitsha256repo, + rev: "v1.2.4-annotated", // annotated tag uses unwrapped commit hash + info: &RevInfo{ + Name: "47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c", + Short: "47b8b51b2a2d", + Version: "v1.2.4-annotated", + Time: time.Date(2018, 4, 17, 19, 43, 22, 0, time.UTC), + Tags: []string{"v1.2.3", "v1.2.4-annotated"}, + }, + }, + { + repo: gitsha256repo, + rev: "aaaaaaaaab", + err: "unknown revision", + }, } { t.Run(path.Base(tt.repo)+"/"+tt.rev, runTest(tt)) if tt.repo == gitrepo1 { diff --git a/src/cmd/go/testdata/script/mod_download_git_bareRepository_sha256.txt b/src/cmd/go/testdata/script/mod_download_git_bareRepository_sha256.txt new file mode 100644 index 0000000000..9e8dc3c015 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_download_git_bareRepository_sha256.txt @@ -0,0 +1,30 @@ +[short] skip +[!git] skip + +# This is a git sha256-mode copy of mod_download_git_bareRepository + +# Redirect git to a test-specific .gitconfig. +# GIT_CONFIG_GLOBAL suffices for git 2.32.0 and newer. +# For older git versions we also set $HOME. +env GIT_CONFIG_GLOBAL=$WORK${/}home${/}gopher${/}.gitconfig +env HOME=$WORK${/}home${/}gopher +exec git config --global --show-origin user.name +stdout 'Go Gopher' + +env GOPRIVATE=vcs-test.golang.org + +go mod download -x + +-- go.mod -- +module test + +go 1.18 + +require vcs-test.golang.org/git/gitrepo-sha256.git v1.2.3 + +-- $WORK/home/gopher/.gitconfig -- +[user] + name = Go Gopher + email = gopher@golang.org +[safe] + bareRepository = explicit diff --git a/src/cmd/go/testdata/vcstest/git/gitrepo-sha256.txt b/src/cmd/go/testdata/vcstest/git/gitrepo-sha256.txt new file mode 100644 index 0000000000..ae68a9ad8f --- /dev/null +++ b/src/cmd/go/testdata/vcstest/git/gitrepo-sha256.txt @@ -0,0 +1,69 @@ +handle git + +# This is a sha256 version of gitrepo1.txt (which uses sha1 hashes) +env GIT_AUTHOR_NAME='David Finkel' +env GIT_AUTHOR_EMAIL='david.finkel@gmail.com' +env GIT_COMMITTER_NAME=$GIT_AUTHOR_NAME +env GIT_COMMITTER_EMAIL=$GIT_AUTHOR_EMAIL + +git init --object-format=sha256 + +at 2018-04-17T15:43:22-04:00 +unquote '' +cp stdout README +git add README +git commit -a -m 'empty README' +git branch -m main +git tag v1.2.3 + +at 2018-04-17T15:45:48-04:00 +git branch v2 +git checkout v2 +echo 'v2' +cp stdout v2 +git add v2 +git commit -a -m 'v2' +git tag v2.3 +git tag v2.0.1 +git branch v2.3.4 + +at 2018-04-17T16:00:19-04:00 +echo 'intermediate' +cp stdout foo.txt +git add foo.txt +git commit -a -m 'intermediate' + +at 2018-04-17T16:00:32-04:00 +echo 'another' +cp stdout another.txt +git add another.txt +git commit -a -m 'another' +git tag v2.0.2 + +at 2018-04-17T16:16:52-04:00 +git checkout main +git branch v3 +git checkout v3 +mkdir v3/sub/dir +echo 'v3/sub/dir/file' +cp stdout v3/sub/dir/file.txt +git add v3 +git commit -a -m 'add v3/sub/dir/file.txt' + +at 2018-04-17T22:23:00-04:00 +git checkout main +git tag -a v1.2.4-annotated -m 'v1.2.4-annotated' + +git show-ref --tags --heads +cmp stdout .git-refs + +-- .git-refs -- +47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c refs/heads/main +1401e4e1fdb4169b51d44a1ff62af63ccc708bf5c12d15051268b51bbb6cbd82 refs/heads/v2 +b7550fd9d2129c724c39ae0536e8b2fae4364d8c82bb8b0880c9b71f67295d09 refs/heads/v2.3.4 +c2a5bbdbeb8b2c82e819a4af94ea59f7d67faeb6df7cb4907c3f0d70836a977b refs/heads/v3 +47b8b51b2a2d9d5caa3d460096c4e01f05700ce3a9390deb54400bd508214c5c refs/tags/v1.2.3 +f88263be2704531e0f664784b66c2f84dea6d0dc4231cf9c6be482af0400c607 refs/tags/v1.2.4-annotated +b7550fd9d2129c724c39ae0536e8b2fae4364d8c82bb8b0880c9b71f67295d09 refs/tags/v2.0.1 +1401e4e1fdb4169b51d44a1ff62af63ccc708bf5c12d15051268b51bbb6cbd82 refs/tags/v2.0.2 +b7550fd9d2129c724c39ae0536e8b2fae4364d8c82bb8b0880c9b71f67295d09 refs/tags/v2.3 diff --git a/src/cmd/go/testdata/vcstest/go/mod/gitrepo-sha256.txt b/src/cmd/go/testdata/vcstest/go/mod/gitrepo-sha256.txt new file mode 100644 index 0000000000..5349b3a2ec --- /dev/null +++ b/src/cmd/go/testdata/vcstest/go/mod/gitrepo-sha256.txt @@ -0,0 +1,6 @@ +handle dir + +-- index.html -- + + + -- 2.51.0