Copied and updated import paths.
Eventually we will probably publish
these packages somewhere in golang.org/x
(as non-internal packages) and then we will
be able to vendor them properly.
For now, copy.
sumweb.globsMatchPath moved to str.GlobsMatchPath.
Change-Id: I4585e6dc5daa423e4ca9669195d41e58e7c8c275
Reviewed-on: https://go-review.googlesource.com/c/go/+/173950
Reviewed-by: Jay Conrod <jayconrod@google.com>
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package note_test
+
+import (
+ "fmt"
+ "io"
+ "os"
+
+ "cmd/go/internal/note"
+)
+
+func ExampleSign() {
+ skey := "PRIVATE+KEY+PeterNeumann+c74f20a3+AYEKFALVFGyNhPJEMzD1QIDr+Y7hfZx09iUvxdXHKDFz"
+ text := "If you think cryptography is the answer to your problem,\n" +
+ "then you don't know what your problem is.\n"
+
+ signer, err := note.NewSigner(skey)
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+
+ msg, err := note.Sign(¬e.Note{Text: text}, signer)
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+ os.Stdout.Write(msg)
+
+ // Output:
+ // If you think cryptography is the answer to your problem,
+ // then you don't know what your problem is.
+ //
+ // — PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=
+}
+
+func ExampleOpen() {
+ vkey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW"
+ msg := []byte("If you think cryptography is the answer to your problem,\n" +
+ "then you don't know what your problem is.\n" +
+ "\n" +
+ "— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=\n")
+
+ verifier, err := note.NewVerifier(vkey)
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+ verifiers := note.VerifierList(verifier)
+
+ n, err := note.Open(msg, verifiers)
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+ fmt.Printf("%s (%08x):\n%s", n.Sigs[0].Name, n.Sigs[0].Hash, n.Text)
+
+ // Output:
+ // PeterNeumann (c74f20a3):
+ // If you think cryptography is the answer to your problem,
+ // then you don't know what your problem is.
+}
+
+var rand = struct {
+ Reader io.Reader
+}{
+ zeroReader{},
+}
+
+type zeroReader struct{}
+
+func (zeroReader) Read(buf []byte) (int, error) {
+ for i := range buf {
+ buf[i] = 0
+ }
+ return len(buf), nil
+}
+
+func ExampleSign_add_signatures() {
+ vkey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW"
+ msg := []byte("If you think cryptography is the answer to your problem,\n" +
+ "then you don't know what your problem is.\n" +
+ "\n" +
+ "— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=\n")
+
+ verifier, err := note.NewVerifier(vkey)
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+ verifiers := note.VerifierList(verifier)
+
+ n, err := note.Open([]byte(msg), verifiers)
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+
+ skey, vkey, err := note.GenerateKey(rand.Reader, "EnochRoot")
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+ _ = vkey // give to verifiers
+
+ me, err := note.NewSigner(skey)
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+
+ msg, err = note.Sign(n, me)
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+ os.Stdout.Write(msg)
+
+ // Output:
+ // If you think cryptography is the answer to your problem,
+ // then you don't know what your problem is.
+ //
+ // — PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=
+ // — EnochRoot rwz+eBzmZa0SO3NbfRGzPCpDckykFXSdeX+MNtCOXm2/5n2tiOHp+vAF1aGrQ5ovTG01oOTGwnWLox33WWd1RvMc+QQ=
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package note defines the notes signed by the Go module database server.
+//
+// This package is part of a DRAFT of what the Go module database server will look like.
+// Do not assume the details here are final!
+//
+// A note is text signed by one or more server keys.
+// The text should be ignored unless the note is signed by
+// a trusted server key and the signature has been verified
+// using the server's public key.
+//
+// A server's public key is identified by a name, typically the "host[/path]"
+// giving the base URL of the server's transparency log.
+// The syntactic restrictions on a name are that it be non-empty,
+// well-formed UTF-8 containing neither Unicode spaces nor plus (U+002B).
+//
+// A Go module database server signs texts using public key cryptography.
+// A given server may have multiple public keys, each
+// identified by the first 32 bits of the SHA-256 hash of
+// the concatenation of the server name, a newline, and
+// the encoded public key.
+//
+// Verifying Notes
+//
+// A Verifier allows verification of signatures by one server public key.
+// It can report the name of the server and the uint32 hash of the key,
+// and it can verify a purported signature by that key.
+//
+// The standard implementation of a Verifier is constructed
+// by NewVerifier starting from a verifier key, which is a
+// plain text string of the form "<name>+<hash>+<keydata>".
+//
+// A Verifiers allows looking up a Verifier by the combination
+// of server name and key hash.
+//
+// The standard implementation of a Verifiers is constructed
+// by VerifierList from a list of known verifiers.
+//
+// A Note represents a text with one or more signatures.
+// An implementation can reject a note with too many signatures
+// (for example, more than 100 signatures).
+//
+// A Signature represents a signature on a note, verified or not.
+//
+// The Open function takes as input a signed message
+// and a set of known verifiers. It decodes and verifies
+// the message signatures and returns a Note structure
+// containing the message text and (verified or unverified) signatures.
+//
+// Signing Notes
+//
+// A Signer allows signing a text with a given key.
+// It can report the name of the server and the hash of the key
+// and can sign a raw text using that key.
+//
+// The standard implementation of a Signer is constructed
+// by NewSigner starting from an encoded signer key, which is a
+// plain text string of the form "PRIVATE+KEY+<name>+<hash>+<keydata>".
+// Anyone with an encoded signer key can sign messages using that key,
+// so it must be kept secret. The encoding begins with the literal text
+// "PRIVATE+KEY" to avoid confusion with the public server key.
+//
+// The Sign function takes as input a Note and a list of Signers
+// and returns an encoded, signed message.
+//
+// Signed Note Format
+//
+// A signed note consists of a text ending in newline (U+000A),
+// followed by a blank line (only a newline),
+// followed by one or more signature lines of this form:
+// em dash (U+2014), space (U+0020),
+// server name, space, base64-encoded signature, newline.
+//
+// Signed notes must be valid UTF-8 and must not contain any
+// ASCII control characters (those below U+0020) other than newline.
+//
+// A signature is a base64 encoding of 4+n bytes.
+//
+// The first four bytes in the signature are the uint32 key hash
+// stored in big-endian order, which is to say they are the first
+// four bytes of the truncated SHA-256 used to derive the key hash
+// in the first place.
+//
+// The remaining n bytes are the result of using the specified key
+// to sign the note text (including the final newline but not the
+// separating blank line).
+//
+// Generating Keys
+//
+// There is only one key type, Ed25519 with algorithm identifier 1.
+// New key types may be introduced in the future as needed,
+// although doing so will require deploying the new algorithms to all clients
+// before starting to depend on them for signatures.
+//
+// The GenerateKey function generates and returns a new signer
+// and corresponding verifier.
+//
+// Example
+//
+// Here is a well-formed signed note:
+//
+// If you think cryptography is the answer to your problem,
+// then you don't know what your problem is.
+//
+// — PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=
+//
+// It can be constructed and displayed using:
+//
+// skey := "PRIVATE+KEY+PeterNeumann+c74f20a3+AYEKFALVFGyNhPJEMzD1QIDr+Y7hfZx09iUvxdXHKDFz"
+// text := "If you think cryptography is the answer to your problem,\n" +
+// "then you don't know what your problem is.\n"
+//
+// signer, err := note.NewSigner(skey)
+// if err != nil {
+// log.Fatal(err)
+// }
+//
+// msg, err := note.Sign(¬e.Note{Text: text}, signer)
+// if err != nil {
+// log.Fatal(err)
+// }
+// os.Stdout.Write(msg)
+//
+// The note's text is two lines, including the final newline,
+// and the text is purportedly signed by a server named
+// "PeterNeumann". (Although server names are canonically
+// base URLs, the only syntactic requirement is that they
+// not contain spaces or newlines).
+//
+// If Open is given access to a Verifiers including the
+// Verifier for this key, then it will succeed at verifiying
+// the encoded message and returning the parsed Note:
+//
+// vkey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW"
+// msg := []byte("If you think cryptography is the answer to your problem,\n" +
+// "then you don't know what your problem is.\n" +
+// "\n" +
+// "— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=\n")
+//
+// verifier, err := note.NewVerifier(vkey)
+// if err != nil {
+// log.Fatal(err)
+// }
+// verifiers := note.VerifierList(verifier)
+//
+// n, err := note.Open([]byte(msg), verifiers)
+// if err != nil {
+// log.Fatal(err)
+// }
+// fmt.Printf("%s (%08x):\n%s", n.Sigs[0].Name, n.Sigs[0].Hash, n.Text)
+//
+// You can add your own signature to this message by re-signing the note:
+//
+// skey, vkey, err := note.GenerateKey(rand.Reader, "EnochRoot")
+// if err != nil {
+// log.Fatal(err)
+// }
+// _ = vkey // give to verifiers
+//
+// me, err := note.NewSigner(skey)
+// if err != nil {
+// log.Fatal(err)
+// }
+//
+// msg, err := note.Sign(n, me)
+// if err != nil {
+// log.Fatal(err)
+// }
+// os.Stdout.Write(msg)
+//
+// This will print a doubly-signed message, like:
+//
+// If you think cryptography is the answer to your problem,
+// then you don't know what your problem is.
+//
+// — PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=
+// — EnochRoot rwz+eBzmZa0SO3NbfRGzPCpDckykFXSdeX+MNtCOXm2/5n2tiOHp+vAF1aGrQ5ovTG01oOTGwnWLox33WWd1RvMc+QQ=
+//
+package note
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+ "strconv"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+
+ "golang.org/x/crypto/ed25519"
+)
+
+// A Verifier verifies messages signed with a specific key.
+type Verifier interface {
+ // Name returns the server name associated with the key.
+ Name() string
+
+ // KeyHash returns the key hash.
+ KeyHash() uint32
+
+ // Verify reports whether sig is a valid signature of msg.
+ Verify(msg, sig []byte) bool
+}
+
+// A Signer signs messages using a specific key.
+type Signer interface {
+ // Name returns the server name associated with the key.
+ Name() string
+
+ // KeyHash returns the key hash.
+ KeyHash() uint32
+
+ // Sign returns a signature for the given message.
+ Sign(msg []byte) ([]byte, error)
+}
+
+// keyHash computes the key hash for the given server name and encoded public key.
+func keyHash(name string, key []byte) uint32 {
+ h := sha256.New()
+ h.Write([]byte(name))
+ h.Write([]byte("\n"))
+ h.Write(key)
+ sum := h.Sum(nil)
+ return binary.BigEndian.Uint32(sum)
+}
+
+var (
+ errVerifierID = errors.New("malformed verifier id")
+ errVerifierAlg = errors.New("unknown verifier algorithm")
+ errVerifierHash = errors.New("invalid verifier hash")
+)
+
+const (
+ algEd25519 = 1
+)
+
+// isValidName reports whether name is valid.
+// It must be non-empty and not have any Unicode spaces or pluses.
+func isValidName(name string) bool {
+ return name != "" && utf8.ValidString(name) && strings.IndexFunc(name, unicode.IsSpace) < 0 && !strings.Contains(name, "+")
+}
+
+// NewVerifier construct a new Verifier from an encoded verifier key.
+func NewVerifier(vkey string) (Verifier, error) {
+ name, vkey := chop(vkey, "+")
+ hash16, key64 := chop(vkey, "+")
+ hash, err1 := strconv.ParseUint(hash16, 16, 32)
+ key, err2 := base64.StdEncoding.DecodeString(key64)
+ if len(hash16) != 8 || err1 != nil || err2 != nil || !isValidName(name) || len(key) == 0 {
+ return nil, errVerifierID
+ }
+ if uint32(hash) != keyHash(name, key) {
+ return nil, errVerifierHash
+ }
+
+ v := &verifier{
+ name: name,
+ hash: uint32(hash),
+ }
+
+ alg, key := key[0], key[1:]
+ switch alg {
+ default:
+ return nil, errVerifierAlg
+
+ case algEd25519:
+ if len(key) != 32 {
+ return nil, errVerifierID
+ }
+ v.verify = func(msg, sig []byte) bool {
+ return ed25519.Verify(key, msg, sig)
+ }
+ }
+
+ return v, nil
+}
+
+// chop chops s at the first instance of sep, if any,
+// and returns the text before and after sep.
+// If sep is not present, chop returns before is s and after is empty.
+func chop(s, sep string) (before, after string) {
+ i := strings.Index(s, sep)
+ if i < 0 {
+ return s, ""
+ }
+ return s[:i], s[i+len(sep):]
+}
+
+// verifier is a trivial Verifier implementation.
+type verifier struct {
+ name string
+ hash uint32
+ verify func([]byte, []byte) bool
+}
+
+func (v *verifier) Name() string { return v.name }
+func (v *verifier) KeyHash() uint32 { return v.hash }
+func (v *verifier) Verify(msg, sig []byte) bool { return v.verify(msg, sig) }
+
+// NewSigner constructs a new Signer from an encoded signer key.
+func NewSigner(skey string) (Signer, error) {
+ priv1, skey := chop(skey, "+")
+ priv2, skey := chop(skey, "+")
+ name, skey := chop(skey, "+")
+ hash16, key64 := chop(skey, "+")
+ hash, err1 := strconv.ParseUint(hash16, 16, 32)
+ key, err2 := base64.StdEncoding.DecodeString(key64)
+ if priv1 != "PRIVATE" || priv2 != "KEY" || len(hash16) != 8 || err1 != nil || err2 != nil || !isValidName(name) || len(key) == 0 {
+ return nil, errSignerID
+ }
+
+ // Note: hash is the hash of the public key and we have the private key.
+ // Must verify hash after deriving public key.
+
+ s := &signer{
+ name: name,
+ hash: uint32(hash),
+ }
+
+ var pubkey []byte
+
+ alg, key := key[0], key[1:]
+ switch alg {
+ default:
+ return nil, errSignerAlg
+
+ case algEd25519:
+ if len(key) != 32 {
+ return nil, errSignerID
+ }
+ key = ed25519.NewKeyFromSeed(key)
+ pubkey = append([]byte{algEd25519}, key[32:]...)
+ s.sign = func(msg []byte) ([]byte, error) {
+ return ed25519.Sign(key, msg), nil
+ }
+ }
+
+ if uint32(hash) != keyHash(name, pubkey) {
+ return nil, errSignerHash
+ }
+
+ return s, nil
+}
+
+var (
+ errSignerID = errors.New("malformed verifier id")
+ errSignerAlg = errors.New("unknown verifier algorithm")
+ errSignerHash = errors.New("invalid verifier hash")
+)
+
+// signer is a trivial Signer implementation.
+type signer struct {
+ name string
+ hash uint32
+ sign func([]byte) ([]byte, error)
+}
+
+func (s *signer) Name() string { return s.name }
+func (s *signer) KeyHash() uint32 { return s.hash }
+func (s *signer) Sign(msg []byte) ([]byte, error) { return s.sign(msg) }
+
+// GenerateKey generates a signer and verifier key pair for a named server.
+// The signer key skey is private and must be kept secret.
+func GenerateKey(rand io.Reader, name string) (skey, vkey string, err error) {
+ pub, priv, err := ed25519.GenerateKey(rand)
+ if err != nil {
+ return "", "", err
+ }
+ pubkey := append([]byte{algEd25519}, pub...)
+ privkey := append([]byte{algEd25519}, priv.Seed()...)
+ h := keyHash(name, pubkey)
+
+ skey = fmt.Sprintf("PRIVATE+KEY+%s+%08x+%s", name, h, base64.StdEncoding.EncodeToString(privkey))
+ vkey = fmt.Sprintf("%s+%08x+%s", name, h, base64.StdEncoding.EncodeToString(pubkey))
+ return skey, vkey, nil
+}
+
+// NewEd25519VerifierKey returns an encoded verifier key using the given name
+// and Ed25519 public key.
+func NewEd25519VerifierKey(name string, key ed25519.PublicKey) (string, error) {
+ if len(key) != ed25519.PublicKeySize {
+ return "", fmt.Errorf("invalid public key size %d, expected %d", len(key), ed25519.PublicKeySize)
+ }
+
+ pubkey := append([]byte{algEd25519}, key...)
+ hash := keyHash(name, pubkey)
+
+ b64Key := base64.StdEncoding.EncodeToString(pubkey)
+ return fmt.Sprintf("%s+%08x+%s", name, hash, b64Key), nil
+}
+
+// A Verifiers is a collection of known verifier keys.
+type Verifiers interface {
+ // Verifier returns the Verifier associated with the key
+ // identified by the name and hash.
+ // If the name, hash pair is unknown, Verifier should return
+ // an UnknownVerifierError.
+ Verifier(name string, hash uint32) (Verifier, error)
+}
+
+// An UnknownVerifierError indicates that the given key is not known.
+// The Open function records signatures without associated verifiers as
+// unverified signatures.
+type UnknownVerifierError struct {
+ Name string
+ KeyHash uint32
+}
+
+func (e *UnknownVerifierError) Error() string {
+ return fmt.Sprintf("unknown key %s+%08x", e.Name, e.KeyHash)
+}
+
+// An ambiguousVerifierError indicates that the given name and hash
+// match multiple keys passed to VerifierList.
+// (If this happens, some malicious actor has taken control of the
+// verifier list, at which point we may as well give up entirely,
+// but we diagnose the problem instead.)
+type ambiguousVerifierError struct {
+ name string
+ hash uint32
+}
+
+func (e *ambiguousVerifierError) Error() string {
+ return fmt.Sprintf("ambiguous key %s+%08x", e.name, e.hash)
+}
+
+// VerifierList returns a Verifiers implementation that uses the given list of verifiers.
+func VerifierList(list ...Verifier) Verifiers {
+ m := make(verifierMap)
+ for _, v := range list {
+ k := nameHash{v.Name(), v.KeyHash()}
+ m[k] = append(m[k], v)
+ }
+ return m
+}
+
+type nameHash struct {
+ name string
+ hash uint32
+}
+
+type verifierMap map[nameHash][]Verifier
+
+func (m verifierMap) Verifier(name string, hash uint32) (Verifier, error) {
+ v, ok := m[nameHash{name, hash}]
+ if !ok {
+ return nil, &UnknownVerifierError{name, hash}
+ }
+ if len(v) > 1 {
+ return nil, &ambiguousVerifierError{name, hash}
+ }
+ return v[0], nil
+}
+
+// A Note is a text and signatures.
+type Note struct {
+ Text string // text of note
+ Sigs []Signature // verified signatures
+ UnverifiedSigs []Signature // unverified signatures
+}
+
+// A Signature is a single signature found in a note.
+type Signature struct {
+ // Name and Hash give the name and key hash
+ // for the key that generated the signature.
+ Name string
+ Hash uint32
+
+ // Base64 records the base64-encoded signature bytes.
+ Base64 string
+}
+
+// An UnverifiedNoteError indicates that the note
+// successfully parsed but had no verifiable signatures.
+type UnverifiedNoteError struct {
+ Note *Note
+}
+
+func (e *UnverifiedNoteError) Error() string {
+ return "note has no verifiable signatures"
+}
+
+// An InvalidSignatureError indicates that the given key was known
+// and the associated Verifier rejected the signature.
+type InvalidSignatureError struct {
+ Name string
+ Hash uint32
+}
+
+func (e *InvalidSignatureError) Error() string {
+ return fmt.Sprintf("invalid signature for key %s+%08x", e.Name, e.Hash)
+}
+
+var (
+ errMalformedNote = errors.New("malformed note")
+ errInvalidSigner = errors.New("invalid signer")
+
+ sigSplit = []byte("\n\n")
+ sigPrefix = []byte("— ")
+)
+
+// Open opens and parses the message msg, checking signatures from the known verifiers.
+//
+// For each signature in the message, Open calls known.Verifier to find a verifier.
+// If known.Verifier returns a verifier and the verifier accepts the signature,
+// Open records the signature in the returned note's Sigs field.
+// If known.Verifier returns a verifier but the verifier rejects the signature,
+// Open returns an InvalidSignatureError.
+// If known.Verifier returns an UnknownVerifierError,
+// Open records the signature in the returned note's UnverifiedSigs field.
+// If known.Verifier returns any other error, Open returns that error.
+//
+// If no known verifier has signed an otherwise valid note,
+// Open returns an UnverifiedNoteError.
+// In this case, the unverified note can be fetched from inside the error.
+func Open(msg []byte, known Verifiers) (*Note, error) {
+ if known == nil {
+ // Treat nil Verifiers as empty list, to produce useful error instead of crash.
+ known = VerifierList()
+ }
+
+ // Must have valid UTF-8 with no non-newline ASCII control characters.
+ for i := 0; i < len(msg); {
+ r, size := utf8.DecodeRune(msg[i:])
+ if r < 0x20 && r != '\n' || r == utf8.RuneError && size == 1 {
+ return nil, errMalformedNote
+ }
+ i += size
+ }
+
+ // Must end with signature block preceded by blank line.
+ split := bytes.LastIndex(msg, sigSplit)
+ if split < 0 {
+ return nil, errMalformedNote
+ }
+ text, sigs := msg[:split+1], msg[split+2:]
+ if len(sigs) == 0 || sigs[len(sigs)-1] != '\n' {
+ return nil, errMalformedNote
+ }
+
+ n := &Note{
+ Text: string(text),
+ }
+
+ var buf bytes.Buffer
+ buf.Write(text)
+
+ // Parse and verify signatures.
+ // Ignore duplicate signatures.
+ seen := make(map[nameHash]bool)
+ seenUnverified := make(map[string]bool)
+ numSig := 0
+ for len(sigs) > 0 {
+ // Pull out next signature line.
+ // We know sigs[len(sigs)-1] == '\n', so IndexByte always finds one.
+ i := bytes.IndexByte(sigs, '\n')
+ line := sigs[:i]
+ sigs = sigs[i+1:]
+
+ if !bytes.HasPrefix(line, sigPrefix) {
+ return nil, errMalformedNote
+ }
+ line = line[len(sigPrefix):]
+ name, b64 := chop(string(line), " ")
+ sig, err := base64.StdEncoding.DecodeString(b64)
+ if err != nil || !isValidName(name) || b64 == "" || len(sig) < 5 {
+ return nil, errMalformedNote
+ }
+ hash := binary.BigEndian.Uint32(sig[0:4])
+ sig = sig[4:]
+
+ if numSig++; numSig > 100 {
+ // Avoid spending forever parsing a note with many signatures.
+ return nil, errMalformedNote
+ }
+
+ v, err := known.Verifier(name, hash)
+ if _, ok := err.(*UnknownVerifierError); ok {
+ // Drop repeated identical unverified signatures.
+ if seenUnverified[string(line)] {
+ continue
+ }
+ seenUnverified[string(line)] = true
+ n.UnverifiedSigs = append(n.UnverifiedSigs, Signature{Name: name, Hash: hash, Base64: b64})
+ continue
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ // Drop repeated signatures by a single verifier.
+ if seen[nameHash{name, hash}] {
+ continue
+ }
+ seen[nameHash{name, hash}] = true
+
+ ok := v.Verify(text, sig)
+ if !ok {
+ return nil, &InvalidSignatureError{name, hash}
+ }
+
+ n.Sigs = append(n.Sigs, Signature{Name: name, Hash: hash, Base64: b64})
+ }
+
+ // Parsed and verified all the signatures.
+ if len(n.Sigs) == 0 {
+ return nil, &UnverifiedNoteError{n}
+ }
+ return n, nil
+}
+
+// Sign signs the note with the given signers and returns the encoded message.
+// The new signatures from signers are listed in the encoded message after
+// the existing signatures already present in n.Sigs.
+// If any signer uses the same key as an existing signature,
+// the existing signature is elided from the output.
+func Sign(n *Note, signers ...Signer) ([]byte, error) {
+ var buf bytes.Buffer
+ if !strings.HasSuffix(n.Text, "\n") {
+ return nil, errMalformedNote
+ }
+ buf.WriteString(n.Text)
+
+ // Prepare signatures.
+ var sigs bytes.Buffer
+ have := make(map[nameHash]bool)
+ for _, s := range signers {
+ name := s.Name()
+ hash := s.KeyHash()
+ have[nameHash{name, hash}] = true
+ if !isValidName(name) {
+ return nil, errInvalidSigner
+ }
+
+ sig, err := s.Sign(buf.Bytes()) // buf holds n.Text
+ if err != nil {
+ return nil, err
+ }
+
+ var hbuf [4]byte
+ binary.BigEndian.PutUint32(hbuf[:], hash)
+ b64 := base64.StdEncoding.EncodeToString(append(hbuf[:], sig...))
+ sigs.WriteString("— ")
+ sigs.WriteString(name)
+ sigs.WriteString(" ")
+ sigs.WriteString(b64)
+ sigs.WriteString("\n")
+ }
+
+ buf.WriteString("\n")
+
+ // Emit existing signatures not replaced by new ones.
+ for _, list := range [][]Signature{n.Sigs, n.UnverifiedSigs} {
+ for _, sig := range list {
+ name, hash := sig.Name, sig.Hash
+ if !isValidName(name) {
+ return nil, errMalformedNote
+ }
+ if have[nameHash{name, hash}] {
+ continue
+ }
+ // Double-check hash against base64.
+ raw, err := base64.StdEncoding.DecodeString(sig.Base64)
+ if err != nil || len(raw) < 4 || binary.BigEndian.Uint32(raw) != hash {
+ return nil, errMalformedNote
+ }
+ buf.WriteString("— ")
+ buf.WriteString(sig.Name)
+ buf.WriteString(" ")
+ buf.WriteString(sig.Base64)
+ buf.WriteString("\n")
+ }
+ }
+ buf.Write(sigs.Bytes())
+
+ return buf.Bytes(), nil
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package note
+
+import (
+ "crypto/rand"
+ "errors"
+ "strings"
+ "testing"
+ "testing/iotest"
+
+ "golang.org/x/crypto/ed25519"
+)
+
+func TestNewVerifier(t *testing.T) {
+ vkey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW"
+ _, err := NewVerifier(vkey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Check various manglings are not accepted.
+ badKey := func(k string) {
+ _, err := NewVerifier(k)
+ if err == nil {
+ t.Errorf("NewVerifier(%q) succeeded, should have failed", k)
+ }
+ }
+
+ b := []byte(vkey)
+ for i := 0; i <= len(b); i++ {
+ for j := i + 1; j <= len(b); j++ {
+ if i != 0 || j != len(b) {
+ badKey(string(b[i:j]))
+ }
+ }
+ }
+ for i := 0; i < len(b); i++ {
+ b[i]++
+ badKey(string(b))
+ b[i]--
+ }
+
+ badKey("PeterNeumann+cc469956+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TWBADKEY==") // wrong length key, with adjusted key hash
+ badKey("PeterNeumann+173116ae+ZRpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW") // unknown algorithm, with adjusted key hash
+}
+
+func TestNewSigner(t *testing.T) {
+ skey := "PRIVATE+KEY+PeterNeumann+c74f20a3+AYEKFALVFGyNhPJEMzD1QIDr+Y7hfZx09iUvxdXHKDFz"
+ _, err := NewSigner(skey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Check various manglings are not accepted.
+ b := []byte(skey)
+ for i := 0; i <= len(b); i++ {
+ for j := i + 1; j <= len(b); j++ {
+ if i == 0 && j == len(b) {
+ continue
+ }
+ _, err := NewSigner(string(b[i:j]))
+ if err == nil {
+ t.Errorf("NewSigner(%q) succeeded, should have failed", b[i:j])
+ }
+ }
+ }
+ for i := 0; i < len(b); i++ {
+ b[i]++
+ _, err := NewSigner(string(b))
+ if err == nil {
+ t.Errorf("NewSigner(%q) succeeded, should have failed", b)
+ }
+ b[i]--
+ }
+}
+
+func testSignerAndVerifier(t *testing.T, Name string, signer Signer, verifier Verifier) {
+ if name := signer.Name(); name != Name {
+ t.Errorf("signer.Name() = %q, want %q", name, Name)
+ }
+ if name := verifier.Name(); name != Name {
+ t.Errorf("verifier.Name() = %q, want %q", name, Name)
+ }
+ shash := signer.KeyHash()
+ vhash := verifier.KeyHash()
+ if shash != vhash {
+ t.Errorf("signer.KeyHash() = %#08x != verifier.KeyHash() = %#08x", shash, vhash)
+ }
+
+ msg := []byte("hi")
+ sig, err := signer.Sign(msg)
+ if err != nil {
+ t.Fatalf("signer.Sign: %v", err)
+ }
+ if !verifier.Verify(msg, sig) {
+ t.Fatalf("verifier.Verify failed on signature returned by signer.Sign")
+ }
+ sig[0]++
+ if verifier.Verify(msg, sig) {
+ t.Fatalf("verifier.Verify succceeded on corrupt signature")
+ }
+ sig[0]--
+ msg[0]++
+ if verifier.Verify(msg, sig) {
+ t.Fatalf("verifier.Verify succceeded on corrupt message")
+ }
+}
+
+func TestGenerateKey(t *testing.T) {
+ // Generate key pair, make sure it is all self-consistent.
+ const Name = "EnochRoot"
+
+ skey, vkey, err := GenerateKey(rand.Reader, Name)
+ if err != nil {
+ t.Fatalf("GenerateKey: %v", err)
+ }
+ signer, err := NewSigner(skey)
+ if err != nil {
+ t.Fatalf("NewSigner: %v", err)
+ }
+ verifier, err := NewVerifier(vkey)
+ if err != nil {
+ t.Fatalf("NewVerifier: %v", err)
+ }
+
+ testSignerAndVerifier(t, Name, signer, verifier)
+
+ // Check that GenerateKey returns error from rand reader.
+ _, _, err = GenerateKey(iotest.TimeoutReader(iotest.OneByteReader(rand.Reader)), Name)
+ if err == nil {
+ t.Fatalf("GenerateKey succeeded with error-returning rand reader")
+ }
+}
+
+func TestFromEd25519(t *testing.T) {
+ const Name = "EnochRoot"
+
+ pub, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatalf("GenerateKey: %v", err)
+ }
+ signer, err := newSignerFromEd25519Seed(Name, priv.Seed())
+ if err != nil {
+ t.Fatalf("newSignerFromEd25519Seed: %v", err)
+ }
+ vkey, err := NewEd25519VerifierKey(Name, pub)
+ if err != nil {
+ t.Fatalf("NewEd25519VerifierKey: %v", err)
+ }
+ verifier, err := NewVerifier(vkey)
+ if err != nil {
+ t.Fatalf("NewVerifier: %v", err)
+ }
+
+ testSignerAndVerifier(t, Name, signer, verifier)
+
+ // Check that wrong key sizes return errors.
+ _, err = NewEd25519VerifierKey(Name, pub[:len(pub)-1])
+ if err == nil {
+ t.Errorf("NewEd25519VerifierKey succeeded with a seed of the wrong size")
+ }
+}
+
+// newSignerFromEd25519Seed constructs a new signer from a verifier name and a
+// golang.org/x/crypto/ed25519 private key seed.
+func newSignerFromEd25519Seed(name string, seed []byte) (Signer, error) {
+ if len(seed) != ed25519.SeedSize {
+ return nil, errors.New("invalid seed size")
+ }
+ priv := ed25519.NewKeyFromSeed(seed)
+ pub := priv[32:]
+
+ pubkey := append([]byte{algEd25519}, pub...)
+ hash := keyHash(name, pubkey)
+
+ s := &signer{
+ name: name,
+ hash: uint32(hash),
+ sign: func(msg []byte) ([]byte, error) {
+ return ed25519.Sign(priv, msg), nil
+ },
+ }
+ return s, nil
+}
+
+func TestSign(t *testing.T) {
+ skey := "PRIVATE+KEY+PeterNeumann+c74f20a3+AYEKFALVFGyNhPJEMzD1QIDr+Y7hfZx09iUvxdXHKDFz"
+ text := "If you think cryptography is the answer to your problem,\n" +
+ "then you don't know what your problem is.\n"
+
+ signer, err := NewSigner(skey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ msg, err := Sign(&Note{Text: text}, signer)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ want := `If you think cryptography is the answer to your problem,
+then you don't know what your problem is.
+
+— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=
+`
+ if string(msg) != want {
+ t.Errorf("Sign: wrong output\nhave:\n%s\nwant:\n%s", msg, want)
+ }
+
+ // Check that existing signature is replaced by new one.
+ msg, err = Sign(&Note{Text: text, Sigs: []Signature{{Name: "PeterNeumann", Hash: 0xc74f20a3, Base64: "BADSIGN="}}}, signer)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(msg) != want {
+ t.Errorf("Sign replacing signature: wrong output\nhave:\n%s\nwant:\n%s", msg, want)
+ }
+
+ // Check various bad inputs.
+ _, err = Sign(&Note{Text: "abc"}, signer)
+ if err == nil || err.Error() != "malformed note" {
+ t.Fatalf("Sign with short text: %v, want malformed note error", err)
+ }
+
+ _, err = Sign(&Note{Text: text, Sigs: []Signature{{Name: "a+b", Base64: "ABCD"}}})
+ if err == nil || err.Error() != "malformed note" {
+ t.Fatalf("Sign with bad name: %v, want malformed note error", err)
+ }
+
+ _, err = Sign(&Note{Text: text, Sigs: []Signature{{Name: "PeterNeumann", Hash: 0xc74f20a3, Base64: "BADHASH="}}})
+ if err == nil || err.Error() != "malformed note" {
+ t.Fatalf("Sign with bad pre-filled signature: %v, want malformed note error", err)
+ }
+
+ _, err = Sign(&Note{Text: text}, &badSigner{signer})
+ if err == nil || err.Error() != "invalid signer" {
+ t.Fatalf("Sign with bad signer: %v, want invalid signer error", err)
+ }
+
+ _, err = Sign(&Note{Text: text}, &errSigner{signer})
+ if err != errSurprise {
+ t.Fatalf("Sign with failing signer: %v, want errSurprise", err)
+ }
+}
+
+func TestVerifierList(t *testing.T) {
+ peterKey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW"
+ peterVerifier, err := NewVerifier(peterKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ enochKey := "EnochRoot+af0cfe78+ATtqJ7zOtqQtYqOo0CpvDXNlMhV3HeJDpjrASKGLWdop"
+ enochVerifier, err := NewVerifier(enochKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ list := VerifierList(peterVerifier, enochVerifier, enochVerifier)
+ v, err := list.Verifier("PeterNeumann", 0xc74f20a3)
+ if v != peterVerifier || err != nil {
+ t.Fatalf("list.Verifier(peter) = %v, %v, want %v, nil", v, err, peterVerifier)
+ }
+ v, err = list.Verifier("PeterNeumann", 0xc74f20a4)
+ if v != nil || err == nil || err.Error() != "unknown key PeterNeumann+c74f20a4" {
+ t.Fatalf("list.Verifier(peter bad hash) = %v, %v, want nil, unknown key error", v, err)
+ }
+
+ v, err = list.Verifier("PeterNeuman", 0xc74f20a3)
+ if v != nil || err == nil || err.Error() != "unknown key PeterNeuman+c74f20a3" {
+ t.Fatalf("list.Verifier(peter bad name) = %v, %v, want nil, unknown key error", v, err)
+ }
+ v, err = list.Verifier("EnochRoot", 0xaf0cfe78)
+ if v != nil || err == nil || err.Error() != "ambiguous key EnochRoot+af0cfe78" {
+ t.Fatalf("list.Verifier(enoch) = %v, %v, want nil, ambiguous key error", v, err)
+ }
+}
+
+type badSigner struct {
+ Signer
+}
+
+func (b *badSigner) Name() string {
+ return "bad name"
+}
+
+var errSurprise = errors.New("surprise!")
+
+type errSigner struct {
+ Signer
+}
+
+func (e *errSigner) Sign([]byte) ([]byte, error) {
+ return nil, errSurprise
+}
+
+func TestOpen(t *testing.T) {
+ peterKey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW"
+ peterVerifier, err := NewVerifier(peterKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ enochKey := "EnochRoot+af0cfe78+ATtqJ7zOtqQtYqOo0CpvDXNlMhV3HeJDpjrASKGLWdop"
+ enochVerifier, err := NewVerifier(enochKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ text := `If you think cryptography is the answer to your problem,
+then you don't know what your problem is.
+`
+ peterSig := "— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=\n"
+ enochSig := "— EnochRoot rwz+eBzmZa0SO3NbfRGzPCpDckykFXSdeX+MNtCOXm2/5n2tiOHp+vAF1aGrQ5ovTG01oOTGwnWLox33WWd1RvMc+QQ=\n"
+
+ peter := Signature{"PeterNeumann", 0xc74f20a3, "x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM="}
+ enoch := Signature{"EnochRoot", 0xaf0cfe78, "rwz+eBzmZa0SO3NbfRGzPCpDckykFXSdeX+MNtCOXm2/5n2tiOHp+vAF1aGrQ5ovTG01oOTGwnWLox33WWd1RvMc+QQ="}
+
+ // Check one signature verified, one not.
+ n, err := Open([]byte(text+"\n"+peterSig+enochSig), VerifierList(peterVerifier))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n.Text != text {
+ t.Errorf("n.Text = %q, want %q", n.Text, text)
+ }
+ if len(n.Sigs) != 1 || n.Sigs[0] != peter {
+ t.Errorf("n.Sigs:\nhave %v\nwant %v", n.Sigs, []Signature{peter})
+ }
+ if len(n.UnverifiedSigs) != 1 || n.UnverifiedSigs[0] != enoch {
+ t.Errorf("n.UnverifiedSigs:\nhave %v\nwant %v", n.Sigs, []Signature{peter})
+ }
+
+ // Check both verified.
+ n, err = Open([]byte(text+"\n"+peterSig+enochSig), VerifierList(peterVerifier, enochVerifier))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(n.Sigs) != 2 || n.Sigs[0] != peter || n.Sigs[1] != enoch {
+ t.Errorf("n.Sigs:\nhave %v\nwant %v", n.Sigs, []Signature{peter, enoch})
+ }
+ if len(n.UnverifiedSigs) != 0 {
+ t.Errorf("n.UnverifiedSigs:\nhave %v\nwant %v", n.Sigs, []Signature{})
+ }
+
+ // Check both unverified.
+ n, err = Open([]byte(text+"\n"+peterSig+enochSig), VerifierList())
+ if n != nil || err == nil {
+ t.Fatalf("Open unverified = %v, %v, want nil, error", n, err)
+ }
+ e, ok := err.(*UnverifiedNoteError)
+ if !ok {
+ t.Fatalf("Open unverified: err is %T, want *UnverifiedNoteError", err)
+ }
+ if err.Error() != "note has no verifiable signatures" {
+ t.Fatalf("Open unverified: err.Error() = %q, want %q", err.Error(), "note has no verifiable signatures")
+ }
+
+ n = e.Note
+ if n == nil {
+ t.Fatalf("Open unverified: missing note in UnverifiedNoteError")
+ }
+ if len(n.Sigs) != 0 {
+ t.Errorf("n.Sigs:\nhave %v\nwant %v", n.Sigs, []Signature{})
+ }
+ if len(n.UnverifiedSigs) != 2 || n.UnverifiedSigs[0] != peter || n.UnverifiedSigs[1] != enoch {
+ t.Errorf("n.UnverifiedSigs:\nhave %v\nwant %v", n.Sigs, []Signature{peter, enoch})
+ }
+
+ // Check duplicated verifier.
+ _, err = Open([]byte(text+"\n"+enochSig), VerifierList(enochVerifier, peterVerifier, enochVerifier))
+ if err == nil || err.Error() != "ambiguous key EnochRoot+af0cfe78" {
+ t.Fatalf("Open with duplicated verifier: err=%v, want ambiguous key", err)
+ }
+
+ // Check unused duplicated verifier.
+ _, err = Open([]byte(text+"\n"+peterSig), VerifierList(enochVerifier, peterVerifier, enochVerifier))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Check too many signatures.
+ n, err = Open([]byte(text+"\n"+strings.Repeat(peterSig, 101)), VerifierList(peterVerifier))
+ if n != nil || err == nil || err.Error() != "malformed note" {
+ t.Fatalf("Open too many verified signatures = %v, %v, want nil, malformed note error", n, err)
+ }
+ n, err = Open([]byte(text+"\n"+strings.Repeat(peterSig, 101)), VerifierList())
+ if n != nil || err == nil || err.Error() != "malformed note" {
+ t.Fatalf("Open too many verified signatures = %v, %v, want nil, malformed note error", n, err)
+ }
+
+ // Invalid signature.
+ n, err = Open([]byte(text+"\n"+peterSig[:60]+"ABCD"+peterSig[60:]), VerifierList(peterVerifier))
+ if n != nil || err == nil || err.Error() != "invalid signature for key PeterNeumann+c74f20a3" {
+ t.Fatalf("Open too many verified signatures = %v, %v, want nil, invalid signature error", n, err)
+ }
+
+ // Duplicated verified and unverified signatures.
+ enochABCD := Signature{"EnochRoot", 0xaf0cfe78, "rwz+eBzmZa0SO3NbfRGzPCpDckykFXSdeX+MNtCOXm2/5n" + "ABCD" + "2tiOHp+vAF1aGrQ5ovTG01oOTGwnWLox33WWd1RvMc+QQ="}
+ n, err = Open([]byte(text+"\n"+peterSig+peterSig+enochSig+enochSig+enochSig[:60]+"ABCD"+enochSig[60:]), VerifierList(peterVerifier))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(n.Sigs) != 1 || n.Sigs[0] != peter {
+ t.Errorf("n.Sigs:\nhave %v\nwant %v", n.Sigs, []Signature{peter})
+ }
+ if len(n.UnverifiedSigs) != 2 || n.UnverifiedSigs[0] != enoch || n.UnverifiedSigs[1] != enochABCD {
+ t.Errorf("n.UnverifiedSigs:\nhave %v\nwant %v", n.UnverifiedSigs, []Signature{enoch, enochABCD})
+ }
+
+ // Invalid encoded message syntax.
+ badMsgs := []string{
+ text,
+ text + "\n",
+ text + "\n" + peterSig[:len(peterSig)-1],
+ "\x01" + text + "\n" + peterSig,
+ "\xff" + text + "\n" + peterSig,
+ text + "\n" + "— Bad Name x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=",
+ text + "\n" + peterSig + "Unexpected line.\n",
+ }
+ for _, msg := range badMsgs {
+ n, err := Open([]byte(msg), VerifierList(peterVerifier))
+ if n != nil || err == nil || err.Error() != "malformed note" {
+ t.Fatalf("Open bad msg = %v, %v, want nil, malformed note error\nmsg:\n%s", n, err, msg)
+ }
+ }
+}
+
+func BenchmarkOpen(b *testing.B) {
+ vkey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW"
+ msg := []byte("If you think cryptography is the answer to your problem,\n" +
+ "then you don't know what your problem is.\n" +
+ "\n" +
+ "— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=\n")
+
+ verifier, err := NewVerifier(vkey)
+ if err != nil {
+ b.Fatal(err)
+ }
+ verifiers := VerifierList(verifier)
+ verifiers0 := VerifierList()
+
+ // Try with 0 signatures and 1 signature so we can tell how much each signature adds.
+
+ b.Run("Sig0", func(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ _, err := Open(msg, verifiers0)
+ e, ok := err.(*UnverifiedNoteError)
+ if !ok {
+ b.Fatal("expected UnverifiedNoteError")
+ }
+ n := e.Note
+ if len(n.Sigs) != 0 || len(n.UnverifiedSigs) != 1 {
+ b.Fatal("wrong signature count")
+ }
+ }
+ })
+
+ b.Run("Sig1", func(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ n, err := Open(msg, verifiers)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if len(n.Sigs) != 1 || len(n.UnverifiedSigs) != 0 {
+ b.Fatal("wrong signature count")
+ }
+ }
+ })
+}
package str
import (
+ "path"
"path/filepath"
"strings"
)
return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix
}
}
+
+// GlobsMatchPath reports whether any path prefix of target
+// matches one of the glob patterns (as defined by path.Match)
+// in the comma-separated globs list.
+// It ignores any empty or malformed patterns in the list.
+func GlobsMatchPath(globs, target string) bool {
+ for globs != "" {
+ // Extract next non-empty glob in comma-separated list.
+ var glob string
+ if i := strings.Index(globs, ","); i >= 0 {
+ glob, globs = globs[:i], globs[i+1:]
+ } else {
+ glob, globs = globs, ""
+ }
+ if glob == "" {
+ continue
+ }
+
+ // A glob with N+1 path elements (N slashes) needs to be matched
+ // against the first N+1 path elements of target,
+ // which end just before the N+1'th slash.
+ n := strings.Count(glob, "/")
+ prefix := target
+ // Walk target, counting slashes, truncating at the N+1'th slash.
+ for i := 0; i < len(target); i++ {
+ if target[i] == '/' {
+ if n == 0 {
+ prefix = target[:i]
+ break
+ }
+ n--
+ }
+ }
+ if n > 0 {
+ // Not enough prefix elements.
+ continue
+ }
+ matched, _ := path.Match(glob, prefix)
+ if matched {
+ return true
+ }
+ }
+ return false
+}
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Parallel cache.
+// This file is copied from cmd/go/internal/par.
+
+package sumweb
+
+import (
+ "sync"
+ "sync/atomic"
+)
+
+// parCache runs an action once per key and caches the result.
+type parCache struct {
+ m sync.Map
+}
+
+type cacheEntry struct {
+ done uint32
+ mu sync.Mutex
+ result interface{}
+}
+
+// Do calls the function f if and only if Do is being called for the first time with this key.
+// No call to Do with a given key returns until the one call to f returns.
+// Do returns the value returned by the one call to f.
+func (c *parCache) Do(key interface{}, f func() interface{}) interface{} {
+ entryIface, ok := c.m.Load(key)
+ if !ok {
+ entryIface, _ = c.m.LoadOrStore(key, new(cacheEntry))
+ }
+ e := entryIface.(*cacheEntry)
+ if atomic.LoadUint32(&e.done) == 0 {
+ e.mu.Lock()
+ if atomic.LoadUint32(&e.done) == 0 {
+ e.result = f()
+ atomic.StoreUint32(&e.done, 1)
+ }
+ e.mu.Unlock()
+ }
+ return e.result
+}
+
+// Get returns the cached result associated with key.
+// It returns nil if there is no such result.
+// If the result for key is being computed, Get does not wait for the computation to finish.
+func (c *parCache) Get(key interface{}) interface{} {
+ entryIface, ok := c.m.Load(key)
+ if !ok {
+ return nil
+ }
+ e := entryIface.(*cacheEntry)
+ if atomic.LoadUint32(&e.done) == 0 {
+ return nil
+ }
+ return e.result
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sumweb
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "strings"
+ "sync"
+ "sync/atomic"
+
+ "cmd/go/internal/note"
+ "cmd/go/internal/str"
+ "cmd/go/internal/tlog"
+)
+
+// A Client provides the external operations
+// (file caching, HTTP fetches, and so on)
+// needed to implement the HTTP client Conn.
+// The methods must be safe for concurrent use by multiple goroutines.
+type Client interface {
+ // ReadRemote reads and returns the content served at the given path
+ // on the remote database server. The path begins with "/lookup" or "/tile/".
+ // It is the implementation's responsibility to turn that path into a full URL
+ // and make the HTTP request. ReadRemote should return an error for
+ // any non-200 HTTP response status.
+ ReadRemote(path string) ([]byte, error)
+
+ // ReadConfig reads and returns the content of the named configuration file.
+ // There are only a fixed set of configuration files.
+ //
+ // "key" returns a file containing the verifier key for the server.
+ //
+ // serverName + "/latest" returns a file containing the latest known
+ // signed tree from the server. It is read and written (using WriteConfig).
+ // To signal that the client wishes to start with an "empty" signed tree,
+ // ReadConfig can return a successful empty result (0 bytes of data).
+ ReadConfig(file string) ([]byte, error)
+
+ // WriteConfig updates the content of the named configuration file,
+ // changing it from the old []byte to the new []byte.
+ // If the old []byte does not match the stored configuration,
+ // WriteConfig must return ErrWriteConflict.
+ // Otherwise, WriteConfig should atomically replace old with new.
+ WriteConfig(file string, old, new []byte) error
+
+ // ReadCache reads and returns the content of the named cache file.
+ // Any returned error will be treated as equivalent to the file not existing.
+ // There can be arbitrarily many cache files, such as:
+ // serverName/lookup/pkg@version
+ // serverName/tile/8/1/x123/456
+ ReadCache(file string) ([]byte, error)
+
+ // WriteCache writes the named cache file.
+ WriteCache(file string, data []byte)
+
+ // Log prints the given log message (such as with log.Print)
+ Log(msg string)
+
+ // SecurityError prints the given security error log message.
+ // The Conn returns ErrSecurity from any operation that invokes SecurityError,
+ // but the return value is mainly for testing. In a real program,
+ // SecurityError should typically print the message and call log.Fatal or os.Exit.
+ SecurityError(msg string)
+}
+
+// ErrWriteConflict signals a write conflict during Client.WriteConfig.
+var ErrWriteConflict = errors.New("write conflict")
+
+// ErrSecurity is returned by Conn operations that invoke Client.SecurityError.
+var ErrSecurity = errors.New("security error: misbehaving server")
+
+// A Conn is a client connection to a go.sum database.
+// All the methods are safe for simultaneous use by multiple goroutines.
+type Conn struct {
+ client Client // client-provided external world
+
+ didLookup uint32
+
+ // one-time initialized data
+ initOnce sync.Once
+ initErr error // init error, if any
+ name string // name of accepted verifier
+ verifiers note.Verifiers // accepted verifiers (just one, but Verifiers for note.Open)
+ tileReader tileReader
+ tileHeight int
+ nosumdb string
+
+ record parCache // cache of record lookup, keyed by path@vers
+ tileCache parCache // cache of c.readTile, keyed by tile
+
+ latestMu sync.Mutex
+ latest tlog.Tree // latest known tree head
+ latestMsg []byte // encoded signed note for latest
+
+ tileSavedMu sync.Mutex
+ tileSaved map[tlog.Tile]bool // which tiles have been saved using c.client.WriteCache already
+}
+
+// NewConn returns a new Conn using the given Client.
+func NewConn(client Client) *Conn {
+ return &Conn{
+ client: client,
+ }
+}
+
+// init initiailzes the conn (if not already initialized)
+// and returns any initialization error.
+func (c *Conn) init() error {
+ c.initOnce.Do(c.initWork)
+ return c.initErr
+}
+
+// initWork does the actual initialization work.
+func (c *Conn) initWork() {
+ defer func() {
+ if c.initErr != nil {
+ c.initErr = fmt.Errorf("initializing sumweb.Conn: %v", c.initErr)
+ }
+ }()
+
+ c.tileReader.c = c
+ if c.tileHeight == 0 {
+ c.tileHeight = 8
+ }
+ c.tileSaved = make(map[tlog.Tile]bool)
+
+ vkey, err := c.client.ReadConfig("key")
+ if err != nil {
+ c.initErr = err
+ return
+ }
+ verifier, err := note.NewVerifier(strings.TrimSpace(string(vkey)))
+ if err != nil {
+ c.initErr = err
+ return
+ }
+ c.verifiers = note.VerifierList(verifier)
+ c.name = verifier.Name()
+
+ data, err := c.client.ReadConfig(c.name + "/latest")
+ if err != nil {
+ c.initErr = err
+ return
+ }
+ if err := c.mergeLatest(data); err != nil {
+ c.initErr = err
+ return
+ }
+}
+
+// SetTileHeight sets the tile height for the Conn.
+// Any call to SetTileHeight must happen before the first call to Lookup.
+// If SetTileHeight is not called, the Conn defaults to tile height 8.
+func (c *Conn) SetTileHeight(height int) {
+ if atomic.LoadUint32(&c.didLookup) != 0 {
+ panic("SetTileHeight used after Lookup")
+ }
+ if c.tileHeight != 0 {
+ panic("multiple calls to SetTileHeight")
+ }
+ c.tileHeight = height
+}
+
+// SetGONOSUMDB sets the list of comma-separated GONOSUMDB patterns for the Conn.
+// For any module path matching one of the patterns,
+// Lookup will return ErrGONOSUMDB.
+// Any call to SetGONOSUMDB must happen before the first call to Lookup.
+func (c *Conn) SetGONOSUMDB(list string) {
+ if atomic.LoadUint32(&c.didLookup) != 0 {
+ panic("SetGONOSUMDB used after Lookup")
+ }
+ if c.nosumdb != "" {
+ panic("multiple calls to SetGONOSUMDB")
+ }
+ c.nosumdb = list
+}
+
+// ErrGONOSUMDB is returned by Lookup for paths that match
+// a pattern listed in the GONOSUMDB list (set by SetGONOSUMDB,
+// usually from the environment variable).
+var ErrGONOSUMDB = errors.New("skipped (listed in GONOSUMDB)")
+
+func (c *Conn) skip(target string) bool {
+ return str.GlobsMatchPath(c.nosumdb, target)
+}
+
+// Lookup returns the go.sum lines for the given module path and version.
+// The version may end in a /go.mod suffix, in which case Lookup returns
+// the go.sum lines for the module's go.mod-only hash.
+func (c *Conn) Lookup(path, vers string) (lines []string, err error) {
+ atomic.StoreUint32(&c.didLookup, 1)
+
+ if c.skip(path) {
+ return nil, ErrGONOSUMDB
+ }
+
+ defer func() {
+ if err != nil {
+ err = fmt.Errorf("%s@%s: %v", path, vers, err)
+ }
+ }()
+
+ if err := c.init(); err != nil {
+ return nil, err
+ }
+
+ // Prepare encoded cache filename / URL.
+ epath, err := encodePath(path)
+ if err != nil {
+ return nil, err
+ }
+ evers, err := encodeVersion(strings.TrimSuffix(vers, "/go.mod"))
+ if err != nil {
+ return nil, err
+ }
+ file := c.name + "/lookup/" + epath + "@" + evers
+ remotePath := "/lookup/" + epath + "@" + evers
+
+ // Fetch the data.
+ // The lookupCache avoids redundant ReadCache/GetURL operations
+ // (especially since go.sum lines tend to come in pairs for a given
+ // path and version) and also avoids having multiple of the same
+ // request in flight at once.
+ type cached struct {
+ data []byte
+ err error
+ }
+ result := c.record.Do(file, func() interface{} {
+ // Try the on-disk cache, or else get from web.
+ writeCache := false
+ data, err := c.client.ReadCache(file)
+ if err != nil {
+ data, err = c.client.ReadRemote(remotePath)
+ if err != nil {
+ return cached{nil, err}
+ }
+ writeCache = true
+ }
+
+ // Validate the record before using it for anything.
+ id, text, treeMsg, err := tlog.ParseRecord(data)
+ if err != nil {
+ return cached{nil, err}
+ }
+ if err := c.mergeLatest(treeMsg); err != nil {
+ return cached{nil, err}
+ }
+ if err := c.checkRecord(id, text); err != nil {
+ return cached{nil, err}
+ }
+
+ // Now that we've validated the record,
+ // save it to the on-disk cache (unless that's where it came from).
+ if writeCache {
+ c.client.WriteCache(file, data)
+ }
+
+ return cached{data, nil}
+ }).(cached)
+ if result.err != nil {
+ return nil, result.err
+ }
+
+ // Extract the lines for the specific version we want
+ // (with or without /go.mod).
+ prefix := path + " " + vers + " "
+ var hashes []string
+ for _, line := range strings.Split(string(result.data), "\n") {
+ if strings.HasPrefix(line, prefix) {
+ hashes = append(hashes, line)
+ }
+ }
+ return hashes, nil
+}
+
+// mergeLatest merges the tree head in msg
+// with the Conn's current latest tree head,
+// ensuring the result is a consistent timeline.
+// If the result is inconsistent, mergeLatest calls c.client.SecurityError
+// with a detailed security error message and then
+// (only if c.client.SecurityError does not exit the program) returns ErrSecurity.
+// If the Conn's current latest tree head moves forward,
+// mergeLatest updates the underlying configuration file as well,
+// taking care to merge any independent updates to that configuration.
+func (c *Conn) mergeLatest(msg []byte) error {
+ // Merge msg into our in-memory copy of the latest tree head.
+ when, err := c.mergeLatestMem(msg)
+ if err != nil {
+ return err
+ }
+ if when != msgFuture {
+ // msg matched our present or was in the past.
+ // No change to our present, so no update of config file.
+ return nil
+ }
+
+ // Flush our extended timeline back out to the configuration file.
+ // If the configuration file has been updated in the interim,
+ // we need to merge any updates made there as well.
+ // Note that writeConfig is an atomic compare-and-swap.
+ for {
+ msg, err := c.client.ReadConfig(c.name + "/latest")
+ if err != nil {
+ return err
+ }
+ when, err := c.mergeLatestMem(msg)
+ if err != nil {
+ return err
+ }
+ if when != msgPast {
+ // msg matched our present or was from the future,
+ // and now our in-memory copy matches.
+ return nil
+ }
+
+ // msg (== config) is in the past, so we need to update it.
+ c.latestMu.Lock()
+ latestMsg := c.latestMsg
+ c.latestMu.Unlock()
+ if err := c.client.WriteConfig(c.name+"/latest", msg, latestMsg); err != ErrWriteConflict {
+ // Success or a non-write-conflict error.
+ return err
+ }
+ }
+}
+
+const (
+ msgPast = 1 + iota
+ msgNow
+ msgFuture
+)
+
+// mergeLatestMem is like mergeLatest but is only concerned with
+// updating the in-memory copy of the latest tree head (c.latest)
+// not the configuration file.
+// The when result explains when msg happened relative to our
+// previous idea of c.latest:
+// msgPast means msg was from before c.latest,
+// msgNow means msg was exactly c.latest, and
+// msgFuture means msg was from after c.latest, which has now been updated.
+func (c *Conn) mergeLatestMem(msg []byte) (when int, err error) {
+ if len(msg) == 0 {
+ // Accept empty msg as the unsigned, empty timeline.
+ c.latestMu.Lock()
+ latest := c.latest
+ c.latestMu.Unlock()
+ if latest.N == 0 {
+ return msgNow, nil
+ }
+ return msgPast, nil
+ }
+
+ note, err := note.Open(msg, c.verifiers)
+ if err != nil {
+ return 0, fmt.Errorf("reading tree note: %v\nnote:\n%s", err, msg)
+ }
+ tree, err := tlog.ParseTree([]byte(note.Text))
+ if err != nil {
+ return 0, fmt.Errorf("reading tree: %v\ntree:\n%s", err, note.Text)
+ }
+
+ // Other lookups may be calling mergeLatest with other heads,
+ // so c.latest is changing underfoot. We don't want to hold the
+ // c.mu lock during tile fetches, so loop trying to update c.latest.
+ c.latestMu.Lock()
+ latest := c.latest
+ latestMsg := c.latestMsg
+ c.latestMu.Unlock()
+
+ for {
+ // If the tree head looks old, check that it is on our timeline.
+ if tree.N <= latest.N {
+ if err := c.checkTrees(tree, msg, latest, latestMsg); err != nil {
+ return 0, err
+ }
+ if tree.N < latest.N {
+ return msgPast, nil
+ }
+ return msgNow, nil
+ }
+
+ // The tree head looks new. Check that we are on its timeline and try to move our timeline forward.
+ if err := c.checkTrees(latest, latestMsg, tree, msg); err != nil {
+ return 0, err
+ }
+
+ // Install our msg if possible.
+ // Otherwise we will go around again.
+ c.latestMu.Lock()
+ installed := false
+ if c.latest == latest {
+ installed = true
+ c.latest = tree
+ c.latestMsg = msg
+ } else {
+ latest = c.latest
+ latestMsg = c.latestMsg
+ }
+ c.latestMu.Unlock()
+
+ if installed {
+ return msgFuture, nil
+ }
+ }
+}
+
+// checkTrees checks that older (from olderNote) is contained in newer (from newerNote).
+// If an error occurs, such as malformed data or a network problem, checkTrees returns that error.
+// If on the other hand checkTrees finds evidence of misbehavior, it prepares a detailed
+// message and calls log.Fatal.
+func (c *Conn) checkTrees(older tlog.Tree, olderNote []byte, newer tlog.Tree, newerNote []byte) error {
+ thr := tlog.TileHashReader(newer, &c.tileReader)
+ h, err := tlog.TreeHash(older.N, thr)
+ if err != nil {
+ if older.N == newer.N {
+ return fmt.Errorf("checking tree#%d: %v", older.N, err)
+ }
+ return fmt.Errorf("checking tree#%d against tree#%d: %v", older.N, newer.N, err)
+ }
+ if h == older.Hash {
+ return nil
+ }
+
+ // Detected a fork in the tree timeline.
+ // Start by reporting the inconsistent signed tree notes.
+ var buf bytes.Buffer
+ fmt.Fprintf(&buf, "SECURITY ERROR\n")
+ fmt.Fprintf(&buf, "go.sum database server misbehavior detected!\n\n")
+ indent := func(b []byte) []byte {
+ return bytes.Replace(b, []byte("\n"), []byte("\n\t"), -1)
+ }
+ fmt.Fprintf(&buf, "old database:\n\t%s\n", indent(olderNote))
+ fmt.Fprintf(&buf, "new database:\n\t%s\n", indent(newerNote))
+
+ // The notes alone are not enough to prove the inconsistency.
+ // We also need to show that the newer note's tree hash for older.N
+ // does not match older.Hash. The consumer of this report could
+ // of course consult the server to try to verify the inconsistency,
+ // but we are holding all the bits we need to prove it right now,
+ // so we might as well print them and make the report not depend
+ // on the continued availability of the misbehaving server.
+ // Preparing this data only reuses the tiled hashes needed for
+ // tlog.TreeHash(older.N, thr) above, so assuming thr is caching tiles,
+ // there are no new access to the server here, and these operations cannot fail.
+ fmt.Fprintf(&buf, "proof of misbehavior:\n\t%v", h)
+ if p, err := tlog.ProveTree(newer.N, older.N, thr); err != nil {
+ fmt.Fprintf(&buf, "\tinternal error: %v\n", err)
+ } else if err := tlog.CheckTree(p, newer.N, newer.Hash, older.N, h); err != nil {
+ fmt.Fprintf(&buf, "\tinternal error: generated inconsistent proof\n")
+ } else {
+ for _, h := range p {
+ fmt.Fprintf(&buf, "\n\t%v", h)
+ }
+ }
+ c.client.SecurityError(buf.String())
+ return ErrSecurity
+}
+
+// checkRecord checks that record #id's hash matches data.
+func (c *Conn) checkRecord(id int64, data []byte) error {
+ c.latestMu.Lock()
+ latest := c.latest
+ c.latestMu.Unlock()
+
+ if id >= latest.N {
+ return fmt.Errorf("cannot validate record %d in tree of size %d", id, latest.N)
+ }
+ hashes, err := tlog.TileHashReader(latest, &c.tileReader).ReadHashes([]int64{tlog.StoredHashIndex(0, id)})
+ if err != nil {
+ return err
+ }
+ if hashes[0] == tlog.RecordHash(data) {
+ return nil
+ }
+ return fmt.Errorf("cannot authenticate record data in server response")
+}
+
+// tileReader is a *Conn wrapper that implements tlog.TileReader.
+// The separate type avoids exposing the ReadTiles and SaveTiles
+// methods on Conn itself.
+type tileReader struct {
+ c *Conn
+}
+
+func (r *tileReader) Height() int {
+ return r.c.tileHeight
+}
+
+// ReadTiles reads and returns the requested tiles,
+// either from the on-disk cache or the server.
+func (r *tileReader) ReadTiles(tiles []tlog.Tile) ([][]byte, error) {
+ // Read all the tiles in parallel.
+ data := make([][]byte, len(tiles))
+ errs := make([]error, len(tiles))
+ var wg sync.WaitGroup
+ for i, tile := range tiles {
+ wg.Add(1)
+ go func(i int, tile tlog.Tile) {
+ defer wg.Done()
+ data[i], errs[i] = r.c.readTile(tile)
+ }(i, tile)
+ }
+ wg.Wait()
+
+ for _, err := range errs {
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return data, nil
+}
+
+// tileCacheKey returns the cache key for the tile.
+func (c *Conn) tileCacheKey(tile tlog.Tile) string {
+ return c.name + "/" + tile.Path()
+}
+
+// tileRemotePath returns the remote path for the tile.
+func (c *Conn) tileRemotePath(tile tlog.Tile) string {
+ return "/" + tile.Path()
+}
+
+// readTile reads a single tile, either from the on-disk cache or the server.
+func (c *Conn) readTile(tile tlog.Tile) ([]byte, error) {
+ type cached struct {
+ data []byte
+ err error
+ }
+
+ result := c.tileCache.Do(tile, func() interface{} {
+ // Try the requested tile in on-disk cache.
+ data, err := c.client.ReadCache(c.tileCacheKey(tile))
+ if err == nil {
+ c.markTileSaved(tile)
+ return cached{data, nil}
+ }
+
+ // Try the full tile in on-disk cache (if requested tile not already full).
+ // We only save authenticated tiles to the on-disk cache,
+ // so the recreated prefix is equally authenticated.
+ full := tile
+ full.W = 1 << tile.H
+ if tile != full {
+ data, err := c.client.ReadCache(c.tileCacheKey(full))
+ if err == nil {
+ c.markTileSaved(tile) // don't save tile later; we already have full
+ return cached{data[:len(data)/full.W*tile.W], nil}
+ }
+ }
+
+ // Try requested tile from server.
+ data, err = c.client.ReadRemote(c.tileRemotePath(tile))
+ if err == nil {
+ return cached{data, nil}
+ }
+
+ // Try full tile on server.
+ // If the partial tile does not exist, it should be because
+ // the tile has been completed and only the complete one
+ // is available.
+ if tile != full {
+ data, err := c.client.ReadRemote(c.tileRemotePath(full))
+ if err == nil {
+ // Note: We could save the full tile in the on-disk cache here,
+ // but we don't know if it is valid yet, and we will only find out
+ // about the partial data, not the full data. So let SaveTiles
+ // save the partial tile, and we'll just refetch the full tile later
+ // once we can validate more (or all) of it.
+ return cached{data[:len(data)/full.W*tile.W], nil}
+ }
+ }
+
+ // Nothing worked.
+ // Return the error from the server fetch for the requested (not full) tile.
+ return cached{nil, err}
+ }).(cached)
+
+ return result.data, result.err
+}
+
+// markTileSaved records that tile is already present in the on-disk cache,
+// so that a future SaveTiles for that tile can be ignored.
+func (c *Conn) markTileSaved(tile tlog.Tile) {
+ c.tileSavedMu.Lock()
+ c.tileSaved[tile] = true
+ c.tileSavedMu.Unlock()
+}
+
+// SaveTiles saves the now validated tiles.
+func (r *tileReader) SaveTiles(tiles []tlog.Tile, data [][]byte) {
+ c := r.c
+
+ // Determine which tiles need saving.
+ // (Tiles that came from the cache need not be saved back.)
+ save := make([]bool, len(tiles))
+ c.tileSavedMu.Lock()
+ for i, tile := range tiles {
+ if !c.tileSaved[tile] {
+ save[i] = true
+ c.tileSaved[tile] = true
+ }
+ }
+ c.tileSavedMu.Unlock()
+
+ for i, tile := range tiles {
+ if save[i] {
+ // If WriteCache fails here (out of disk space? i/o error?),
+ // c.tileSaved[tile] is still true and we will not try to write it again.
+ // Next time we run maybe we'll redownload it again and be
+ // more successful.
+ c.client.WriteCache(c.name+"/"+tile.Path(), data[i])
+ }
+ }
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sumweb
+
+import (
+ "bytes"
+ "fmt"
+ "strings"
+ "sync"
+ "testing"
+
+ "cmd/go/internal/note"
+ "cmd/go/internal/tlog"
+)
+
+const (
+ testName = "localhost.localdev/sumdb"
+ testVerifierKey = "localhost.localdev/sumdb+00000c67+AcTrnkbUA+TU4heY3hkjiSES/DSQniBqIeQ/YppAUtK6"
+ testSignerKey = "PRIVATE+KEY+localhost.localdev/sumdb+00000c67+AXu6+oaVaOYuQOFrf1V59JK1owcFlJcHwwXHDfDGxSPk"
+)
+
+func TestConnLookup(t *testing.T) {
+ tc := newTestClient(t)
+ tc.mustHaveLatest(1)
+
+ // Basic lookup.
+ tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=")
+ tc.mustHaveLatest(3)
+
+ // Everything should now be cached, both for the original package and its /go.mod.
+ tc.getOK = false
+ tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=")
+ tc.mustLookup("rsc.io/sampler", "v1.3.0/go.mod", "rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=")
+ tc.mustHaveLatest(3)
+ tc.getOK = true
+ tc.getTileOK = false // the cache has what we need
+
+ // Lookup with multiple returned lines.
+ tc.mustLookup("rsc.io/quote", "v1.5.2", "rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=\nrsc.io/quote v1.5.2 h2:xyzzy")
+ tc.mustHaveLatest(3)
+
+ // Lookup with need for !-encoding.
+ // rsc.io/Quote is the only record written after rsc.io/samper,
+ // so it is the only one that should need more tiles.
+ tc.getTileOK = true
+ tc.mustLookup("rsc.io/Quote", "v1.5.2", "rsc.io/Quote v1.5.2 h1:uppercase!=")
+ tc.mustHaveLatest(4)
+}
+
+func TestConnBadTiles(t *testing.T) {
+ tc := newTestClient(t)
+
+ flipBits := func() {
+ for url, data := range tc.remote {
+ if strings.Contains(url, "/tile/") {
+ for i := range data {
+ data[i] ^= 0x80
+ }
+ }
+ }
+ }
+
+ // Bad tiles in initial download.
+ tc.mustHaveLatest(1)
+ flipBits()
+ _, err := tc.conn.Lookup("rsc.io/sampler", "v1.3.0")
+ tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumweb.Conn: checking tree#1: downloaded inconsistent tile")
+ flipBits()
+ tc.newConn()
+ tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=")
+
+ // Bad tiles after initial download.
+ flipBits()
+ _, err = tc.conn.Lookup("rsc.io/Quote", "v1.5.2")
+ tc.mustError(err, "rsc.io/Quote@v1.5.2: checking tree#3 against tree#4: downloaded inconsistent tile")
+ flipBits()
+ tc.newConn()
+ tc.mustLookup("rsc.io/Quote", "v1.5.2", "rsc.io/Quote v1.5.2 h1:uppercase!=")
+
+ // Bad starting tree hash looks like bad tiles.
+ tc.newConn()
+ text := tlog.FormatTree(tlog.Tree{N: 1, Hash: tlog.Hash{}})
+ data, err := note.Sign(¬e.Note{Text: string(text)}, tc.signer)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ tc.config[testName+"/latest"] = data
+ _, err = tc.conn.Lookup("rsc.io/sampler", "v1.3.0")
+ tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumweb.Conn: checking tree#1: downloaded inconsistent tile")
+}
+
+func TestConnFork(t *testing.T) {
+ tc := newTestClient(t)
+ tc2 := tc.fork()
+
+ tc.addRecord("rsc.io/pkg1@v1.5.2", `rsc.io/pkg1 v1.5.2 h1:hash!=
+`)
+ tc.addRecord("rsc.io/pkg1@v1.5.4", `rsc.io/pkg1 v1.5.4 h1:hash!=
+`)
+ tc.mustLookup("rsc.io/pkg1", "v1.5.2", "rsc.io/pkg1 v1.5.2 h1:hash!=")
+
+ tc2.addRecord("rsc.io/pkg1@v1.5.3", `rsc.io/pkg1 v1.5.3 h1:hash!=
+`)
+ tc2.addRecord("rsc.io/pkg1@v1.5.4", `rsc.io/pkg1 v1.5.4 h1:hash!=
+`)
+ tc2.mustLookup("rsc.io/pkg1", "v1.5.4", "rsc.io/pkg1 v1.5.4 h1:hash!=")
+
+ key := "/lookup/rsc.io/pkg1@v1.5.2"
+ tc2.remote[key] = tc.remote[key]
+ _, err := tc2.conn.Lookup("rsc.io/pkg1", "v1.5.2")
+ tc2.mustError(err, ErrSecurity.Error())
+
+ /*
+ SECURITY ERROR
+ go.sum database server misbehavior detected!
+
+ old database:
+ go.sum database tree!
+ 5
+ nWzN20+pwMt62p7jbv1/NlN95ePTlHijabv5zO/s36w=
+
+ — localhost.localdev/sumdb AAAMZ5/2FVAdMH58kmnz/0h299pwyskEbzDzoa2/YaPdhvLya4YWDFQQxu2TQb5GpwAH4NdWnTwuhILafisyf3CNbgg=
+
+ new database:
+ go.sum database tree
+ 6
+ wc4SkQt52o5W2nQ8To2ARs+mWuUJjss+sdleoiqxMmM=
+
+ — localhost.localdev/sumdb AAAMZ6oRNswlEZ6ZZhxrCvgl1MBy+nusq4JU+TG6Fe2NihWLqOzb+y2c2kzRLoCr4tvw9o36ucQEnhc20e4nA4Qc/wc=
+
+ proof of misbehavior:
+ T7i+H/8ER4nXOiw4Bj0koZOkGjkxoNvlI34GpvhHhQg=
+ Nsuejv72de9hYNM5bqFv8rv3gm3zJQwv/DT/WNbLDLA=
+ mOmqqZ1aI/lzS94oq/JSbj7pD8Rv9S+xDyi12BtVSHo=
+ /7Aw5jVSMM9sFjQhaMg+iiDYPMk6decH7QLOGrL9Lx0=
+ */
+
+ wants := []string{
+ "SECURITY ERROR",
+ "go.sum database server misbehavior detected!",
+ "old database:\n\tgo.sum database tree\n\t5\n",
+ "— localhost.localdev/sumdb AAAMZ5/2FVAd",
+ "new database:\n\tgo.sum database tree\n\t6\n",
+ "— localhost.localdev/sumdb AAAMZ6oRNswl",
+ "proof of misbehavior:\n\tT7i+H/8ER4nXOiw4Bj0k",
+ }
+ text := tc2.security.String()
+ for _, want := range wants {
+ if !strings.Contains(text, want) {
+ t.Fatalf("cannot find %q in security text:\n%s", want, text)
+ }
+ }
+}
+
+func TestConnGONOSUMDB(t *testing.T) {
+ tc := newTestClient(t)
+ tc.conn.SetGONOSUMDB("p,*/q")
+ tc.conn.Lookup("rsc.io/sampler", "v1.3.0") // initialize before we turn off network
+ tc.getOK = false
+
+ ok := []string{
+ "abc",
+ "a/p",
+ "pq",
+ "q",
+ "n/o/p/q",
+ }
+ skip := []string{
+ "p",
+ "p/x",
+ "x/q",
+ "x/q/z",
+ }
+
+ for _, path := range ok {
+ _, err := tc.conn.Lookup(path, "v1.0.0")
+ if err == ErrGONOSUMDB {
+ t.Errorf("Lookup(%q): ErrGONOSUMDB, wanted failed actual lookup", path)
+ }
+ }
+ for _, path := range skip {
+ _, err := tc.conn.Lookup(path, "v1.0.0")
+ if err != ErrGONOSUMDB {
+ t.Errorf("Lookup(%q): %v, wanted ErrGONOSUMDB", path, err)
+ }
+ }
+}
+
+// A testClient is a self-contained client-side testing environment.
+type testClient struct {
+ t *testing.T // active test
+ conn *Conn // conn being tested
+ tileHeight int // tile height to use (default 2)
+ getOK bool // should tc.GetURL succeed?
+ getTileOK bool // should tc.GetURL of tiles succeed?
+ treeSize int64
+ hashes []tlog.Hash
+ remote map[string][]byte
+ signer note.Signer
+
+ // mu protects config, cache, log, security
+ // during concurrent use of the exported methods
+ // by the conn itself (testClient is the Conn's Client,
+ // and the Client methods can both read and write these fields).
+ // Unexported methods invoked directly by the test
+ // (for example, addRecord) need not hold the mutex:
+ // for proper test execution those methods should only
+ // be called when the Conn is idle and not using its Client.
+ // Not holding the mutex in those methods ensures
+ // that if a mistake is made, go test -race will report it.
+ // (Holding the mutex would eliminate the race report but
+ // not the underlying problem.)
+ // Similarly, the get map is not protected by the mutex,
+ // because the Client methods only read it.
+ mu sync.Mutex // prot
+ config map[string][]byte
+ cache map[string][]byte
+ security bytes.Buffer
+}
+
+// newTestClient returns a new testClient that will call t.Fatal on error
+// and has a few records already available on the remote server.
+func newTestClient(t *testing.T) *testClient {
+ tc := &testClient{
+ t: t,
+ tileHeight: 2,
+ getOK: true,
+ getTileOK: true,
+ config: make(map[string][]byte),
+ cache: make(map[string][]byte),
+ remote: make(map[string][]byte),
+ }
+
+ tc.config["key"] = []byte(testVerifierKey + "\n")
+ var err error
+ tc.signer, err = note.NewSigner(testSignerKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ tc.newConn()
+
+ tc.addRecord("rsc.io/quote@v1.5.2", `rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=
+rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0=
+rsc.io/quote v1.5.2 h2:xyzzy
+`)
+
+ tc.addRecord("golang.org/x/text@v0.0.0-20170915032832-14c0d48ead0c", `golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+`)
+ tc.addRecord("rsc.io/sampler@v1.3.0", `rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+`)
+ tc.config[testName+"/latest"] = tc.signTree(1)
+
+ tc.addRecord("rsc.io/!quote@v1.5.2", `rsc.io/Quote v1.5.2 h1:uppercase!=
+`)
+ return tc
+}
+
+// newConn resets the Conn associated with tc.
+// This clears any in-memory cache from the Conn
+// but not tc's on-disk cache.
+func (tc *testClient) newConn() {
+ tc.conn = NewConn(tc)
+ tc.conn.SetTileHeight(tc.tileHeight)
+}
+
+// mustLookup does a lookup for path@vers and checks that the lines that come back match want.
+func (tc *testClient) mustLookup(path, vers, want string) {
+ tc.t.Helper()
+ lines, err := tc.conn.Lookup(path, vers)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ if strings.Join(lines, "\n") != want {
+ tc.t.Fatalf("Lookup(%q, %q):\n\t%s\nwant:\n\t%s", path, vers, strings.Join(lines, "\n\t"), strings.Replace(want, "\n", "\n\t", -1))
+ }
+}
+
+// mustHaveLatest checks that the on-disk configuration
+// for latest is a tree of size n.
+func (tc *testClient) mustHaveLatest(n int64) {
+ tc.t.Helper()
+
+ latest := tc.config[testName+"/latest"]
+ lines := strings.Split(string(latest), "\n")
+ if len(lines) < 2 || lines[1] != fmt.Sprint(n) {
+ tc.t.Fatalf("/latest should have tree %d, but has:\n%s", n, latest)
+ }
+}
+
+// mustError checks that err's error string contains the text.
+func (tc *testClient) mustError(err error, text string) {
+ tc.t.Helper()
+ if err == nil || !strings.Contains(err.Error(), text) {
+ tc.t.Fatalf("err = %v, want %q", err, text)
+ }
+}
+
+// fork returns a copy of tc.
+// Changes made to the new copy or to tc are not reflected in the other.
+func (tc *testClient) fork() *testClient {
+ tc2 := &testClient{
+ t: tc.t,
+ getOK: tc.getOK,
+ getTileOK: tc.getTileOK,
+ tileHeight: tc.tileHeight,
+ treeSize: tc.treeSize,
+ hashes: append([]tlog.Hash{}, tc.hashes...),
+ signer: tc.signer,
+ config: copyMap(tc.config),
+ cache: copyMap(tc.cache),
+ remote: copyMap(tc.remote),
+ }
+ tc2.newConn()
+ return tc2
+}
+
+func copyMap(m map[string][]byte) map[string][]byte {
+ m2 := make(map[string][]byte)
+ for k, v := range m {
+ m2[k] = v
+ }
+ return m2
+}
+
+// ReadHashes is tc's implementation of tlog.HashReader, for use with
+// tlog.TreeHash and so on.
+func (tc *testClient) ReadHashes(indexes []int64) ([]tlog.Hash, error) {
+ var list []tlog.Hash
+ for _, id := range indexes {
+ list = append(list, tc.hashes[id])
+ }
+ return list, nil
+}
+
+// addRecord adds a log record using the given (!-encoded) key and data.
+func (tc *testClient) addRecord(key, data string) {
+ tc.t.Helper()
+
+ // Create record, add hashes to log tree.
+ id := tc.treeSize
+ tc.treeSize++
+ rec, err := tlog.FormatRecord(id, []byte(data))
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ hashes, err := tlog.StoredHashesForRecordHash(id, tlog.RecordHash([]byte(data)), tc)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ tc.hashes = append(tc.hashes, hashes...)
+
+ // Create lookup result.
+ tc.remote["/lookup/"+key] = append(rec, tc.signTree(tc.treeSize)...)
+
+ // Create new tiles.
+ tiles := tlog.NewTiles(tc.tileHeight, id, tc.treeSize)
+ for _, tile := range tiles {
+ data, err := tlog.ReadTileData(tile, tc)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ tc.remote["/"+tile.Path()] = data
+ // TODO delete old partial tiles
+ }
+}
+
+// signTree returns the signed head for the tree of the given size.
+func (tc *testClient) signTree(size int64) []byte {
+ h, err := tlog.TreeHash(size, tc)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ text := tlog.FormatTree(tlog.Tree{N: size, Hash: h})
+ data, err := note.Sign(¬e.Note{Text: string(text)}, tc.signer)
+ if err != nil {
+ tc.t.Fatal(err)
+ }
+ return data
+}
+
+// ReadRemote is for tc's implementation of Client.
+func (tc *testClient) ReadRemote(path string) ([]byte, error) {
+ // No mutex here because only the Client should be running
+ // and the Client cannot change tc.get.
+ if !tc.getOK {
+ return nil, fmt.Errorf("disallowed remote read %s", path)
+ }
+ if strings.Contains(path, "/tile/") && !tc.getTileOK {
+ return nil, fmt.Errorf("disallowed remote tile read %s", path)
+ }
+
+ data, ok := tc.remote[path]
+ if !ok {
+ return nil, fmt.Errorf("no remote path %s", path)
+ }
+ return data, nil
+}
+
+// ReadConfig is for tc's implementation of Client.
+func (tc *testClient) ReadConfig(file string) ([]byte, error) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ data, ok := tc.config[file]
+ if !ok {
+ return nil, fmt.Errorf("no config %s", file)
+ }
+ return data, nil
+}
+
+// WriteConfig is for tc's implementation of Client.
+func (tc *testClient) WriteConfig(file string, old, new []byte) error {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ data := tc.config[file]
+ if !bytes.Equal(old, data) {
+ return ErrWriteConflict
+ }
+ tc.config[file] = new
+ return nil
+}
+
+// ReadCache is for tc's implementation of Client.
+func (tc *testClient) ReadCache(file string) ([]byte, error) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ data, ok := tc.cache[file]
+ if !ok {
+ return nil, fmt.Errorf("no cache %s", file)
+ }
+ return data, nil
+}
+
+// WriteCache is for tc's implementation of Client.
+func (tc *testClient) WriteCache(file string, data []byte) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ tc.cache[file] = data
+}
+
+// Log is for tc's implementation of Client.
+func (tc *testClient) Log(msg string) {
+ tc.t.Log(msg)
+}
+
+// SecurityError is for tc's implementation of Client.
+func (tc *testClient) SecurityError(msg string) {
+ tc.mu.Lock()
+ defer tc.mu.Unlock()
+
+ fmt.Fprintf(&tc.security, "%s\n", strings.TrimRight(msg, "\n"))
+}
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// FS-safe encoding of module paths and versions.
+// Copied from cmd/go/internal/module and unexported.
+
+package sumweb
+
+import (
+ "fmt"
+ "unicode/utf8"
+)
+
+// Safe encodings
+//
+// Module paths appear as substrings of file system paths
+// (in the download cache) and of web server URLs in the proxy protocol.
+// In general we cannot rely on file systems to be case-sensitive,
+// nor can we rely on web servers, since they read from file systems.
+// That is, we cannot rely on the file system to keep rsc.io/QUOTE
+// and rsc.io/quote separate. Windows and macOS don't.
+// Instead, we must never require two different casings of a file path.
+// Because we want the download cache to match the proxy protocol,
+// and because we want the proxy protocol to be possible to serve
+// from a tree of static files (which might be stored on a case-insensitive
+// file system), the proxy protocol must never require two different casings
+// of a URL path either.
+//
+// One possibility would be to make the safe encoding be the lowercase
+// hexadecimal encoding of the actual path bytes. This would avoid ever
+// needing different casings of a file path, but it would be fairly illegible
+// to most programmers when those paths appeared in the file system
+// (including in file paths in compiler errors and stack traces)
+// in web server logs, and so on. Instead, we want a safe encoding that
+// leaves most paths unaltered.
+//
+// The safe encoding is this:
+// replace every uppercase letter with an exclamation mark
+// followed by the letter's lowercase equivalent.
+//
+// For example,
+// github.com/Azure/azure-sdk-for-go -> github.com/!azure/azure-sdk-for-go.
+// github.com/GoogleCloudPlatform/cloudsql-proxy -> github.com/!google!cloud!platform/cloudsql-proxy
+// github.com/Sirupsen/logrus -> github.com/!sirupsen/logrus.
+//
+// Import paths that avoid upper-case letters are left unchanged.
+// Note that because import paths are ASCII-only and avoid various
+// problematic punctuation (like : < and >), the safe encoding is also ASCII-only
+// and avoids the same problematic punctuation.
+//
+// Import paths have never allowed exclamation marks, so there is no
+// need to define how to encode a literal !.
+//
+// Although paths are disallowed from using Unicode (see pathOK above),
+// the eventual plan is to allow Unicode letters as well, to assume that
+// file systems and URLs are Unicode-safe (storing UTF-8), and apply
+// the !-for-uppercase convention. Note however that not all runes that
+// are different but case-fold equivalent are an upper/lower pair.
+// For example, U+004B ('K'), U+006B ('k'), and U+212A ('K' for Kelvin)
+// are considered to case-fold to each other. When we do add Unicode
+// letters, we must not assume that upper/lower are the only case-equivalent pairs.
+// Perhaps the Kelvin symbol would be disallowed entirely, for example.
+// Or perhaps it would encode as "!!k", or perhaps as "(212A)".
+//
+// Also, it would be nice to allow Unicode marks as well as letters,
+// but marks include combining marks, and then we must deal not
+// only with case folding but also normalization: both U+00E9 ('é')
+// and U+0065 U+0301 ('e' followed by combining acute accent)
+// look the same on the page and are treated by some file systems
+// as the same path. If we do allow Unicode marks in paths, there
+// must be some kind of normalization to allow only one canonical
+// encoding of any character used in an import path.
+
+// encodePath returns the safe encoding of the given module path.
+// It fails if the module path is invalid.
+func encodePath(path string) (encoding string, err error) {
+ return encodeString(path)
+}
+
+// encodeVersion returns the safe encoding of the given module version.
+// Versions are allowed to be in non-semver form but must be valid file names
+// and not contain exclamation marks.
+func encodeVersion(v string) (encoding string, err error) {
+ return encodeString(v)
+}
+
+func encodeString(s string) (encoding string, err error) {
+ haveUpper := false
+ for _, r := range s {
+ if r == '!' || r >= utf8.RuneSelf {
+ // This should be disallowed by CheckPath, but diagnose anyway.
+ // The correctness of the encoding loop below depends on it.
+ return "", fmt.Errorf("internal error: inconsistency in EncodePath")
+ }
+ if 'A' <= r && r <= 'Z' {
+ haveUpper = true
+ }
+ }
+
+ if !haveUpper {
+ return s, nil
+ }
+
+ var buf []byte
+ for _, r := range s {
+ if 'A' <= r && r <= 'Z' {
+ buf = append(buf, '!', byte(r+'a'-'A'))
+ } else {
+ buf = append(buf, byte(r))
+ }
+ }
+ return string(buf), nil
+}
+
+// decodePath returns the module path of the given safe encoding.
+// It fails if the encoding is invalid or encodes an invalid path.
+func decodePath(encoding string) (path string, err error) {
+ path, ok := decodeString(encoding)
+ if !ok {
+ return "", fmt.Errorf("invalid module path encoding %q", encoding)
+ }
+ return path, nil
+}
+
+// decodeVersion returns the version string for the given safe encoding.
+// It fails if the encoding is invalid or encodes an invalid version.
+// Versions are allowed to be in non-semver form but must be valid file names
+// and not contain exclamation marks.
+func decodeVersion(encoding string) (v string, err error) {
+ v, ok := decodeString(encoding)
+ if !ok {
+ return "", fmt.Errorf("invalid version encoding %q", encoding)
+ }
+ return v, nil
+}
+
+func decodeString(encoding string) (string, bool) {
+ var buf []byte
+
+ bang := false
+ for _, r := range encoding {
+ if r >= utf8.RuneSelf {
+ return "", false
+ }
+ if bang {
+ bang = false
+ if r < 'a' || 'z' < r {
+ return "", false
+ }
+ buf = append(buf, byte(r+'A'-'a'))
+ continue
+ }
+ if r == '!' {
+ bang = true
+ continue
+ }
+ if 'A' <= r && r <= 'Z' {
+ return "", false
+ }
+ buf = append(buf, byte(r))
+ }
+ if bang {
+ return "", false
+ }
+ return string(buf), true
+}
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sumweb
+
+import "testing"
+
+var encodeTests = []struct {
+ path string
+ enc string // empty means same as path
+}{
+ {path: "ascii.com/abcdefghijklmnopqrstuvwxyz.-+/~_0123456789"},
+ {path: "github.com/GoogleCloudPlatform/omega", enc: "github.com/!google!cloud!platform/omega"},
+}
+
+func TestEncodePath(t *testing.T) {
+ // Check encodings.
+ for _, tt := range encodeTests {
+ enc, err := encodePath(tt.path)
+ if err != nil {
+ t.Errorf("encodePath(%q): unexpected error: %v", tt.path, err)
+ continue
+ }
+ want := tt.enc
+ if want == "" {
+ want = tt.path
+ }
+ if enc != want {
+ t.Errorf("encodePath(%q) = %q, want %q", tt.path, enc, want)
+ }
+ }
+}
+
+var badDecode = []string{
+ "github.com/GoogleCloudPlatform/omega",
+ "github.com/!google!cloud!platform!/omega",
+ "github.com/!0google!cloud!platform/omega",
+ "github.com/!_google!cloud!platform/omega",
+ "github.com/!!google!cloud!platform/omega",
+}
+
+func TestDecodePath(t *testing.T) {
+ // Check invalid decodings.
+ for _, bad := range badDecode {
+ _, err := decodePath(bad)
+ if err == nil {
+ t.Errorf("DecodePath(%q): succeeded, want error (invalid decoding)", bad)
+ }
+ }
+
+ // Check encodings.
+ for _, tt := range encodeTests {
+ enc := tt.enc
+ if enc == "" {
+ enc = tt.path
+ }
+ path, err := decodePath(enc)
+ if err != nil {
+ t.Errorf("decodePath(%q): unexpected error: %v", enc, err)
+ continue
+ }
+ if path != tt.path {
+ t.Errorf("decodePath(%q) = %q, want %q", enc, path, tt.path)
+ }
+ }
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package sumweb implements the HTTP protocols for serving or accessing a go.sum database.
+package sumweb
+
+import (
+ "context"
+ "net/http"
+ "os"
+ "regexp"
+ "strings"
+
+ "cmd/go/internal/tlog"
+)
+
+// A Server provides the external operations
+// (underlying database access and so on)
+// needed to implement the HTTP server Handler.
+type Server interface {
+ // NewContext returns the context to use for the request r.
+ NewContext(r *http.Request) (context.Context, error)
+
+ // Signed returns the signed hash of the latest tree.
+ Signed(ctx context.Context) ([]byte, error)
+
+ // ReadRecords returns the content for the n records id through id+n-1.
+ ReadRecords(ctx context.Context, id, n int64) ([][]byte, error)
+
+ // Lookup looks up a record by its associated key ("module@version"),
+ // returning the record ID.
+ Lookup(ctx context.Context, key string) (int64, error)
+
+ // ReadTileData reads the content of tile t.
+ // It is only invoked for hash tiles (t.L ≥ 0).
+ ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error)
+}
+
+// A Handler is the go.sum database server handler,
+// which should be invoked to serve the paths listed in Paths.
+// The calling code is responsible for initializing Server.
+type Handler struct {
+ Server Server
+}
+
+// Paths are the URL paths for which Handler should be invoked.
+//
+// Typically a server will do:
+//
+// handler := &sumweb.Handler{Server: srv}
+// for _, path := range sumweb.Paths {
+// http.HandleFunc(path, handler)
+// }
+//
+var Paths = []string{
+ "/lookup/",
+ "/latest",
+ "/tile/",
+}
+
+var modVerRE = regexp.MustCompile(`^[^@]+@v[0-9]+\.[0-9]+\.[0-9]+(-[^@]*)?$`)
+
+func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ ctx, err := h.Server.NewContext(r)
+ if err != nil {
+ http.Error(w, err.Error(), 500)
+ return
+ }
+
+ switch {
+ default:
+ http.NotFound(w, r)
+
+ case strings.HasPrefix(r.URL.Path, "/lookup/"):
+ mod := strings.TrimPrefix(r.URL.Path, "/lookup/")
+ if !modVerRE.MatchString(mod) {
+ http.Error(w, "invalid module@version syntax", http.StatusBadRequest)
+ return
+ }
+ i := strings.Index(mod, "@")
+ encPath, encVers := mod[:i], mod[i+1:]
+ path, err := decodePath(encPath)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ vers, err := decodeVersion(encVers)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ id, err := h.Server.Lookup(ctx, path+"@"+vers)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ records, err := h.Server.ReadRecords(ctx, id, 1)
+ if err != nil {
+ // This should never happen - the lookup says the record exists.
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if len(records) != 1 {
+ http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError)
+ return
+ }
+ msg, err := tlog.FormatRecord(id, records[0])
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ signed, err := h.Server.Signed(ctx)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
+ w.Write(msg)
+ w.Write(signed)
+
+ case r.URL.Path == "/latest":
+ data, err := h.Server.Signed(ctx)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
+ w.Write(data)
+
+ case strings.HasPrefix(r.URL.Path, "/tile/"):
+ t, err := tlog.ParseTilePath(r.URL.Path[1:])
+ if err != nil {
+ http.Error(w, "invalid tile syntax", http.StatusBadRequest)
+ return
+ }
+ if t.L == -1 {
+ // Record data.
+ start := t.N << uint(t.H)
+ records, err := h.Server.ReadRecords(ctx, start, int64(t.W))
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ if len(records) != t.W {
+ http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError)
+ return
+ }
+ var data []byte
+ for i, text := range records {
+ msg, err := tlog.FormatRecord(start+int64(i), text)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ }
+ data = append(data, msg...)
+ }
+ w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
+ w.Write(data)
+ return
+ }
+
+ data, err := h.Server.ReadTileData(ctx, t)
+ if err != nil {
+ reportError(w, r, err)
+ return
+ }
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Write(data)
+ }
+}
+
+// reportError reports err to w.
+// If it's a not-found, the reported error is 404.
+// Otherwise it is an internal server error.
+// The caller must only call reportError in contexts where
+// a not-found err should be reported as 404.
+func reportError(w http.ResponseWriter, r *http.Request, err error) {
+ if os.IsNotExist(err) {
+ http.Error(w, err.Error(), http.StatusNotFound)
+ return
+ }
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sumweb
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strings"
+ "sync"
+
+ "cmd/go/internal/note"
+ "cmd/go/internal/tlog"
+)
+
+// NewTestServer constructs a new TestServer
+// that will sign its tree with the given signer key
+// (see cmd/go/internal/note)
+// and fetch new records as needed by calling gosum.
+func NewTestServer(signer string, gosum func(path, vers string) ([]byte, error)) *TestServer {
+ return &TestServer{signer: signer, gosum: gosum}
+}
+
+// A TestServer is an in-memory implementation of Server for testing.
+type TestServer struct {
+ signer string
+ gosum func(path, vers string) ([]byte, error)
+
+ mu sync.Mutex
+ hashes testHashes
+ records [][]byte
+ lookup map[string]int64
+}
+
+// testHashes implements tlog.HashReader, reading from a slice.
+type testHashes []tlog.Hash
+
+func (h testHashes) ReadHashes(indexes []int64) ([]tlog.Hash, error) {
+ var list []tlog.Hash
+ for _, id := range indexes {
+ list = append(list, h[id])
+ }
+ return list, nil
+}
+
+func (s *TestServer) NewContext(r *http.Request) (context.Context, error) {
+ return nil, nil
+}
+
+func (s *TestServer) Signed(ctx context.Context) ([]byte, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ size := int64(len(s.records))
+ h, err := tlog.TreeHash(size, s.hashes)
+ if err != nil {
+ return nil, err
+ }
+ text := tlog.FormatTree(tlog.Tree{N: size, Hash: h})
+ signer, err := note.NewSigner(s.signer)
+ if err != nil {
+ return nil, err
+ }
+ return note.Sign(¬e.Note{Text: string(text)}, signer)
+}
+
+func (s *TestServer) ReadRecords(ctx context.Context, id, n int64) ([][]byte, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ var list [][]byte
+ for i := int64(0); i < n; i++ {
+ if id+i >= int64(len(s.records)) {
+ return nil, fmt.Errorf("missing records")
+ }
+ list = append(list, s.records[id+i])
+ }
+ return list, nil
+}
+
+func (s *TestServer) Lookup(ctx context.Context, key string) (int64, error) {
+ s.mu.Lock()
+ id, ok := s.lookup[key]
+ s.mu.Unlock()
+ if ok {
+ return id, nil
+ }
+
+ // Look up module and compute go.sum lines.
+ i := strings.Index(key, "@")
+ if i < 0 {
+ return 0, fmt.Errorf("invalid lookup key %q", key)
+ }
+ path, vers := key[:i], key[i+1:]
+ data, err := s.gosum(path, vers)
+ if err != nil {
+ return 0, err
+ }
+
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ // We ran the fetch without the lock.
+ // If another fetch happened and committed, use it instead.
+ id, ok = s.lookup[key]
+ if ok {
+ return id, nil
+ }
+
+ // Add record.
+ id = int64(len(s.records))
+ s.records = append(s.records, data)
+ if s.lookup == nil {
+ s.lookup = make(map[string]int64)
+ }
+ s.lookup[key] = id
+ hashes, err := tlog.StoredHashesForRecordHash(id, tlog.RecordHash([]byte(data)), s.hashes)
+ if err != nil {
+ panic(err)
+ }
+ s.hashes = append(s.hashes, hashes...)
+
+ return id, nil
+}
+
+func (s *TestServer) ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ return tlog.ReadTileData(t, s.hashes)
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tlog
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "os"
+ "testing"
+)
+
+func TestCertificateTransparency(t *testing.T) {
+ // Test that we can verify actual Certificate Transparency proofs.
+ // (The other tests check that we can verify our own proofs;
+ // this is a test that the two are compatible.)
+
+ if testing.Short() {
+ t.Skip("skipping in -short mode")
+ }
+
+ var root ctTree
+ httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-sth", &root)
+
+ var leaf ctEntries
+ httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-entries?start=10000&end=10000", &leaf)
+ hash := RecordHash(leaf.Entries[0].Data)
+
+ var rp ctRecordProof
+ httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-proof-by-hash?tree_size="+fmt.Sprint(root.Size)+"&hash="+url.QueryEscape(hash.String()), &rp)
+
+ err := CheckRecord(rp.Proof, root.Size, root.Hash, 10000, hash)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var tp ctTreeProof
+ httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-sth-consistency?first=3654490&second="+fmt.Sprint(root.Size), &tp)
+
+ oh, _ := ParseHash("AuIZ5V6sDUj1vn3Y1K85oOaQ7y+FJJKtyRTl1edIKBQ=")
+ err = CheckTree(tp.Proof, root.Size, root.Hash, 3654490, oh)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+type ctTree struct {
+ Size int64 `json:"tree_size"`
+ Hash Hash `json:"sha256_root_hash"`
+}
+
+type ctEntries struct {
+ Entries []*ctEntry
+}
+
+type ctEntry struct {
+ Data []byte `json:"leaf_input"`
+}
+
+type ctRecordProof struct {
+ Index int64 `json:"leaf_index"`
+ Proof RecordProof `json:"audit_path"`
+}
+
+type ctTreeProof struct {
+ Proof TreeProof `json:"consistency"`
+}
+
+func httpGET(t *testing.T, url string, targ interface{}) {
+ if testing.Verbose() {
+ println()
+ println(url)
+ }
+ resp, err := http.Get(url)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ data, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if testing.Verbose() {
+ os.Stdout.Write(data)
+ }
+ err = json.Unmarshal(data, targ)
+ if err != nil {
+ println(url)
+ os.Stdout.Write(data)
+ t.Fatal(err)
+ }
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tlog
+
+import (
+ "bytes"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "strconv"
+ "strings"
+ "unicode/utf8"
+)
+
+// A Tree is a tree description, to be signed by a go.sum database server.
+type Tree struct {
+ N int64
+ Hash Hash
+}
+
+// FormatTree formats a tree description for inclusion in a note.
+//
+// The encoded form is three lines, each ending in a newline (U+000A):
+//
+// go.sum database tree
+// N
+// Hash
+//
+// where N is in decimal and Hash is in base64.
+//
+// A future backwards-compatible encoding may add additional lines,
+// which the parser can ignore.
+// A future backwards-incompatible encoding would use a different
+// first line (for example, "go.sum database tree v2").
+func FormatTree(tree Tree) []byte {
+ return []byte(fmt.Sprintf("go.sum database tree\n%d\n%s\n", tree.N, tree.Hash))
+}
+
+var errMalformedTree = errors.New("malformed tree note")
+var treePrefix = []byte("go.sum database tree\n")
+
+// ParseTree parses a tree root description.
+func ParseTree(text []byte) (tree Tree, err error) {
+ // The message looks like:
+ //
+ // go.sum database tree
+ // 2
+ // nND/nri/U0xuHUrYSy0HtMeal2vzD9V4k/BO79C+QeI=
+ //
+ // For forwards compatibility, extra text lines after the encoding are ignored.
+ if !bytes.HasPrefix(text, treePrefix) || bytes.Count(text, []byte("\n")) < 3 || len(text) > 1e6 {
+ return Tree{}, errMalformedTree
+ }
+
+ lines := strings.SplitN(string(text), "\n", 4)
+ n, err := strconv.ParseInt(lines[1], 10, 64)
+ if err != nil || n < 0 || lines[1] != strconv.FormatInt(n, 10) {
+ return Tree{}, errMalformedTree
+ }
+
+ h, err := base64.StdEncoding.DecodeString(lines[2])
+ if err != nil || len(h) != HashSize {
+ return Tree{}, errMalformedTree
+ }
+
+ var hash Hash
+ copy(hash[:], h)
+ return Tree{n, hash}, nil
+}
+
+var errMalformedRecord = errors.New("malformed record data")
+
+// FormatRecord formats a record for serving to a client
+// in a lookup response or data tile.
+//
+// The encoded form is the record ID as a single number,
+// then the text of the record, and then a terminating blank line.
+// Record text must be valid UTF-8 and must not contain any ASCII control
+// characters (those below U+0020) other than newline (U+000A).
+// It must end in a terminating newline and not contain any blank lines.
+func FormatRecord(id int64, text []byte) (msg []byte, err error) {
+ if !isValidRecordText(text) {
+ return nil, errMalformedRecord
+ }
+ msg = []byte(fmt.Sprintf("%d\n", id))
+ msg = append(msg, text...)
+ msg = append(msg, '\n')
+ return msg, nil
+}
+
+// isValidRecordText reports whether text is syntactically valid record text.
+func isValidRecordText(text []byte) bool {
+ var last rune
+ for i := 0; i < len(text); {
+ r, size := utf8.DecodeRune(text[i:])
+ if r < 0x20 && r != '\n' || r == utf8.RuneError && size == 1 || last == '\n' && r == '\n' {
+ return false
+ }
+ i += size
+ last = r
+ }
+ if last != '\n' {
+ return false
+ }
+ return true
+}
+
+// ParseRecord parses a record description at the start of text,
+// stopping immediately after the terminating blank line.
+// It returns the record id, the record text, and the remainder of text.
+func ParseRecord(msg []byte) (id int64, text, rest []byte, err error) {
+ // Leading record id.
+ i := bytes.IndexByte(msg, '\n')
+ if i < 0 {
+ return 0, nil, nil, errMalformedRecord
+ }
+ id, err = strconv.ParseInt(string(msg[:i]), 10, 64)
+ if err != nil {
+ return 0, nil, nil, errMalformedRecord
+ }
+ msg = msg[i+1:]
+
+ // Record text.
+ i = bytes.Index(msg, []byte("\n\n"))
+ if i < 0 {
+ return 0, nil, nil, errMalformedRecord
+ }
+ text, rest = msg[:i+1], msg[i+2:]
+ if !isValidRecordText(text) {
+ return 0, nil, nil, errMalformedRecord
+ }
+ return id, text, rest, nil
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tlog
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestFormatTree(t *testing.T) {
+ n := int64(123456789012)
+ h := RecordHash([]byte("hello world"))
+ golden := "go.sum database tree\n123456789012\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBb0=\n"
+ b := FormatTree(Tree{n, h})
+ if string(b) != golden {
+ t.Errorf("FormatTree(...) = %q, want %q", b, golden)
+ }
+}
+
+func TestParseTree(t *testing.T) {
+ in := "go.sum database tree\n123456789012\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBb0=\n"
+ goldH := RecordHash([]byte("hello world"))
+ goldN := int64(123456789012)
+ tree, err := ParseTree([]byte(in))
+ if tree.N != goldN || tree.Hash != goldH || err != nil {
+ t.Fatalf("ParseTree(...) = Tree{%d, %v}, %v, want Tree{%d, %v}, nil", tree.N, tree.Hash, err, goldN, goldH)
+ }
+
+ // Check invalid trees.
+ var badTrees = []string{
+ "not-" + in,
+ "go.sum database tree\n0xabcdef\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBb0=\n",
+ "go.sum database tree\n123456789012\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBTOOBIG=\n",
+ }
+ for _, bad := range badTrees {
+ _, err := ParseTree([]byte(bad))
+ if err == nil {
+ t.Fatalf("ParseTree(%q) succeeded, want failure", in)
+ }
+ }
+
+ // Check junk on end is ignored.
+ var goodTrees = []string{
+ in + "JOE",
+ in + "JOE\n",
+ in + strings.Repeat("JOE\n", 1000),
+ }
+ for _, good := range goodTrees {
+ _, err := ParseTree([]byte(good))
+ if tree.N != goldN || tree.Hash != goldH || err != nil {
+ t.Fatalf("ParseTree(...+%q) = Tree{%d, %v}, %v, want Tree{%d, %v}, nil", good[len(in):], tree.N, tree.Hash, err, goldN, goldH)
+ }
+ }
+}
+
+func TestFormatRecord(t *testing.T) {
+ id := int64(123456789012)
+ text := "hello, world\n"
+ golden := "123456789012\nhello, world\n\n"
+ msg, err := FormatRecord(id, []byte(text))
+ if err != nil {
+ t.Fatalf("FormatRecord: %v", err)
+ }
+ if string(msg) != golden {
+ t.Fatalf("FormatRecord(...) = %q, want %q", msg, golden)
+ }
+
+ var badTexts = []string{
+ "",
+ "hello\nworld",
+ "hello\n\nworld\n",
+ "hello\x01world\n",
+ }
+ for _, bad := range badTexts {
+ msg, err := FormatRecord(id, []byte(bad))
+ if err == nil {
+ t.Errorf("FormatRecord(id, %q) = %q, want error", bad, msg)
+ }
+ }
+}
+
+func TestParseRecord(t *testing.T) {
+ in := "123456789012\nhello, world\n\njunk on end\x01\xff"
+ goldID := int64(123456789012)
+ goldText := "hello, world\n"
+ goldRest := "junk on end\x01\xff"
+ id, text, rest, err := ParseRecord([]byte(in))
+ if id != goldID || string(text) != goldText || string(rest) != goldRest || err != nil {
+ t.Fatalf("ParseRecord(%q) = %d, %q, %q, %v, want %d, %q, %q, nil", in, id, text, rest, err, goldID, goldText, goldRest)
+ }
+
+ in = "123456789012\nhello, world\n\n"
+ id, text, rest, err = ParseRecord([]byte(in))
+ if id != goldID || string(text) != goldText || len(rest) != 0 || err != nil {
+ t.Fatalf("ParseRecord(%q) = %d, %q, %q, %v, want %d, %q, %q, nil", in, id, text, rest, err, goldID, goldText, "")
+ }
+ if rest == nil {
+ t.Fatalf("ParseRecord(%q): rest = []byte(nil), want []byte{}", in)
+ }
+
+ // Check invalid records.
+ var badRecords = []string{
+ "not-" + in,
+ "123\nhello\x01world\n\n",
+ "123\nhello\xffworld\n\n",
+ "123\nhello world\n",
+ "0x123\nhello world\n\n",
+ }
+ for _, bad := range badRecords {
+ id, text, rest, err := ParseRecord([]byte(bad))
+ if err == nil {
+ t.Fatalf("ParseRecord(%q) = %d, %q, %q, nil, want error", in, id, text, rest)
+ }
+ }
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tlog
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+)
+
+// A Tile is a description of a transparency log tile.
+// A tile of height H at level L offset N lists W consecutive hashes
+// at level H*L of the tree starting at offset N*(2**H).
+// A complete tile lists 2**H hashes; a partial tile lists fewer.
+// Note that a tile represents the entire subtree of height H
+// with those hashes as the leaves. The levels above H*L
+// can be reconstructed by hashing the leaves.
+//
+// Each Tile can be encoded as a “tile coordinate path”
+// of the form tile/H/L/NNN[.p/W].
+// The .p/W suffix is present only for partial tiles, meaning W < 2**H.
+// The NNN element is an encoding of N into 3-digit path elements.
+// All but the last path element begins with an "x".
+// For example,
+// Tile{H: 3, L: 4, N: 1234067, W: 1}'s path
+// is tile/3/4/x001/x234/067.p/1, and
+// Tile{H: 3, L: 4, N: 1234067, W: 8}'s path
+// is tile/3/4/x001/x234/067.
+// See Tile's Path method and the ParseTilePath function.
+//
+// The special level L=-1 holds raw record data instead of hashes.
+// In this case, the level encodes into a tile path as the path element
+// "data" instead of "-1".
+type Tile struct {
+ H int // height of tile (1 ≤ H ≤ 30)
+ L int // level in tiling (-1 ≤ L ≤ 63)
+ N int64 // number within level (0 ≤ N, unbounded)
+ W int // width of tile (1 ≤ W ≤ 2**H; 2**H is complete tile)
+}
+
+// TileForIndex returns the tile of height h ≥ 1
+// and least width storing the given hash storage index.
+func TileForIndex(h int, index int64) Tile {
+ if h < 1 {
+ panic("TileForIndex: invalid height")
+ }
+ t, _, _ := tileForIndex(h, index)
+ return t
+}
+
+// tileForIndex returns the tile of height h ≥ 1
+// storing the given hash index, which can be
+// reconstructed using tileHash(data[start:end]).
+func tileForIndex(h int, index int64) (t Tile, start, end int) {
+ level, n := SplitStoredHashIndex(index)
+ t.H = h
+ t.L = level / h
+ level -= t.L * h // now level within tile
+ t.N = n << uint(level) >> uint(t.H)
+ n -= t.N << uint(t.H) >> uint(level) // now n within tile at level
+ t.W = int((n + 1) << uint(level))
+ return t, int(n<<uint(level)) * HashSize, int((n+1)<<uint(level)) * HashSize
+}
+
+// HashFromTile returns the hash at the given storage index,
+// provided that t == TileForIndex(t.H, index) or a wider version,
+// and data is t's tile data (of length at least t.W*HashSize).
+func HashFromTile(t Tile, data []byte, index int64) (Hash, error) {
+ if t.H < 1 || t.H > 30 || t.L < 0 || t.L >= 64 || t.W < 1 || t.W > 1<<uint(t.H) {
+ return Hash{}, fmt.Errorf("invalid tile %v", t.Path())
+ }
+ if len(data) < t.W*HashSize {
+ return Hash{}, fmt.Errorf("data len %d too short for tile %v", len(data), t.Path())
+ }
+ t1, start, end := tileForIndex(t.H, index)
+ if t.L != t1.L || t.N != t1.N || t.W < t1.W {
+ return Hash{}, fmt.Errorf("index %v is in %v not %v", index, t1.Path(), t.Path())
+ }
+ return tileHash(data[start:end]), nil
+}
+
+// tileHash computes the subtree hash corresponding to the (2^K)-1 hashes in data.
+func tileHash(data []byte) Hash {
+ if len(data) == 0 {
+ panic("bad math in tileHash")
+ }
+ if len(data) == HashSize {
+ var h Hash
+ copy(h[:], data)
+ return h
+ }
+ n := len(data) / 2
+ return NodeHash(tileHash(data[:n]), tileHash(data[n:]))
+}
+
+// NewTiles returns the coordinates of the tiles of height h ≥ 1
+// that must be published when publishing from a tree of
+// size newTreeSize to replace a tree of size oldTreeSize.
+// (No tiles need to be published for a tree of size zero.)
+func NewTiles(h int, oldTreeSize, newTreeSize int64) []Tile {
+ if h < 1 {
+ panic(fmt.Sprintf("NewTiles: invalid height %d", h))
+ }
+ H := uint(h)
+ var tiles []Tile
+ for level := uint(0); newTreeSize>>(H*level) > 0; level++ {
+ oldN := oldTreeSize >> (H * level)
+ newN := newTreeSize >> (H * level)
+ for n := oldN >> H; n < newN>>H; n++ {
+ tiles = append(tiles, Tile{H: h, L: int(level), N: n, W: 1 << H})
+ }
+ n := newN >> H
+ maxW := int(newN - n<<H)
+ minW := 1
+ if oldN > n<<H {
+ minW = int(oldN - n<<H)
+ }
+ for w := minW; w <= maxW; w++ {
+ tiles = append(tiles, Tile{H: h, L: int(level), N: n, W: w})
+ }
+ }
+ return tiles
+}
+
+// ReadTileData reads the hashes for tile t from r
+// and returns the corresponding tile data.
+func ReadTileData(t Tile, r HashReader) ([]byte, error) {
+ size := t.W
+ if size == 0 {
+ size = 1 << uint(t.H)
+ }
+ start := t.N << uint(t.H)
+ indexes := make([]int64, size)
+ for i := 0; i < size; i++ {
+ indexes[i] = StoredHashIndex(t.H*t.L, start+int64(i))
+ }
+
+ hashes, err := r.ReadHashes(indexes)
+ if err != nil {
+ return nil, err
+ }
+ if len(hashes) != len(indexes) {
+ return nil, fmt.Errorf("tlog: ReadHashes(%d indexes) = %d hashes", len(indexes), len(hashes))
+ }
+
+ tile := make([]byte, size*HashSize)
+ for i := 0; i < size; i++ {
+ copy(tile[i*HashSize:], hashes[i][:])
+ }
+ return tile, nil
+}
+
+// To limit the size of any particular directory listing,
+// we encode the (possibly very large) number N
+// by encoding three digits at a time.
+// For example, 123456789 encodes as x123/x456/789.
+// Each directory has at most 1000 each xNNN, NNN, and NNN.p children,
+// so there are at most 3000 entries in any one directory.
+const pathBase = 1000
+
+// Path returns a tile coordinate path describing t.
+func (t Tile) Path() string {
+ n := t.N
+ nStr := fmt.Sprintf("%03d", n%pathBase)
+ for n >= pathBase {
+ n /= pathBase
+ nStr = fmt.Sprintf("x%03d/%s", n%pathBase, nStr)
+ }
+ pStr := ""
+ if t.W != 1<<uint(t.H) {
+ pStr = fmt.Sprintf(".p/%d", t.W)
+ }
+ var L string
+ if t.L == -1 {
+ L = "data"
+ } else {
+ L = fmt.Sprintf("%d", t.L)
+ }
+ return fmt.Sprintf("tile/%d/%s/%s%s", t.H, L, nStr, pStr)
+}
+
+// ParseTilePath parses a tile coordinate path.
+func ParseTilePath(path string) (Tile, error) {
+ f := strings.Split(path, "/")
+ if len(f) < 4 || f[0] != "tile" {
+ return Tile{}, &badPathError{path}
+ }
+ h, err1 := strconv.Atoi(f[1])
+ isData := false
+ if f[2] == "data" {
+ isData = true
+ f[2] = "0"
+ }
+ l, err2 := strconv.Atoi(f[2])
+ if err1 != nil || err2 != nil || h < 1 || l < 0 || h > 30 {
+ return Tile{}, &badPathError{path}
+ }
+ w := 1 << uint(h)
+ if dotP := f[len(f)-2]; strings.HasSuffix(dotP, ".p") {
+ ww, err := strconv.Atoi(f[len(f)-1])
+ if err != nil || ww <= 0 || ww >= w {
+ return Tile{}, &badPathError{path}
+ }
+ w = ww
+ f[len(f)-2] = dotP[:len(dotP)-len(".p")]
+ f = f[:len(f)-1]
+ }
+ f = f[3:]
+ n := int64(0)
+ for _, s := range f {
+ nn, err := strconv.Atoi(strings.TrimPrefix(s, "x"))
+ if err != nil || nn < 0 || nn >= pathBase {
+ return Tile{}, &badPathError{path}
+ }
+ n = n*pathBase + int64(nn)
+ }
+ if isData {
+ l = -1
+ }
+ t := Tile{H: h, L: l, N: n, W: w}
+ if path != t.Path() {
+ return Tile{}, &badPathError{path}
+ }
+ return t, nil
+}
+
+type badPathError struct {
+ path string
+}
+
+func (e *badPathError) Error() string {
+ return fmt.Sprintf("malformed tile path %q", e.path)
+}
+
+// A TileReader reads tiles from a go.sum database log.
+type TileReader interface {
+ // Height returns the height of the available tiles.
+ Height() int
+
+ // ReadTiles returns the data for each requested tile.
+ // If ReadTiles returns err == nil, it must also return
+ // a data record for each tile (len(data) == len(tiles))
+ // and each data record must be the correct length
+ // (len(data[i]) == tiles[i].W*HashSize).
+ ReadTiles(tiles []Tile) (data [][]byte, err error)
+
+ // SaveTiles informs the TileReader that the tile data
+ // returned by ReadTiles has been confirmed as valid
+ // and can be saved in persistent storage (on disk).
+ SaveTiles(tiles []Tile, data [][]byte)
+}
+
+// TileHashReader returns a HashReader that satisfies requests
+// by loading tiles of the given tree.
+//
+// The returned HashReader checks that loaded tiles are
+// valid for the given tree. Therefore, any hashes returned
+// by the HashReader are already proven to be in the tree.
+func TileHashReader(tree Tree, tr TileReader) HashReader {
+ return &tileHashReader{tree: tree, tr: tr}
+}
+
+type tileHashReader struct {
+ tree Tree
+ tr TileReader
+}
+
+// tileParent returns t's k'th tile parent in the tiles for a tree of size n.
+// If there is no such parent, tileParent returns Tile{}.
+func tileParent(t Tile, k int, n int64) Tile {
+ t.L += k
+ t.N >>= uint(k * t.H)
+ t.W = 1 << uint(t.H)
+ if max := n >> uint(t.L*t.H); t.N<<uint(t.H)+int64(t.W) >= max {
+ if t.N<<uint(t.H) >= max {
+ return Tile{}
+ }
+ t.W = int(max - t.N<<uint(t.H))
+ }
+ return t
+}
+
+func (r *tileHashReader) ReadHashes(indexes []int64) ([]Hash, error) {
+ h := r.tr.Height()
+
+ tileOrder := make(map[Tile]int) // tileOrder[tileKey(tiles[i])] = i
+ var tiles []Tile
+
+ // Plan to fetch tiles necessary to recompute tree hash.
+ // If it matches, those tiles are authenticated.
+ stx := subTreeIndex(0, r.tree.N, nil)
+ stxTileOrder := make([]int, len(stx))
+ for i, x := range stx {
+ tile, _, _ := tileForIndex(h, x)
+ tile = tileParent(tile, 0, r.tree.N)
+ if j, ok := tileOrder[tile]; ok {
+ stxTileOrder[i] = j
+ continue
+ }
+ stxTileOrder[i] = len(tiles)
+ tileOrder[tile] = len(tiles)
+ tiles = append(tiles, tile)
+ }
+
+ // Plan to fetch tiles containing the indexes,
+ // along with any parent tiles needed
+ // for authentication. For most calls,
+ // the parents are being fetched anyway.
+ indexTileOrder := make([]int, len(indexes))
+ for i, x := range indexes {
+ if x >= StoredHashIndex(0, r.tree.N) {
+ return nil, fmt.Errorf("indexes not in tree")
+ }
+
+ tile, _, _ := tileForIndex(h, x)
+
+ // Walk up parent tiles until we find one we've requested.
+ // That one will be authenticated.
+ k := 0
+ for ; ; k++ {
+ p := tileParent(tile, k, r.tree.N)
+ if j, ok := tileOrder[p]; ok {
+ if k == 0 {
+ indexTileOrder[i] = j
+ }
+ break
+ }
+ }
+
+ // Walk back down recording child tiles after parents.
+ // This loop ends by revisiting the tile for this index
+ // (tileParent(tile, 0, r.tree.N)) unless k == 0, in which
+ // case the previous loop did it.
+ for k--; k >= 0; k-- {
+ p := tileParent(tile, k, r.tree.N)
+ if p.W != 1<<uint(p.H) {
+ // Only full tiles have parents.
+ // This tile has a parent, so it must be full.
+ return nil, fmt.Errorf("bad math in tileHashReader: %d %d %v", r.tree.N, x, p)
+ }
+ tileOrder[p] = len(tiles)
+ if k == 0 {
+ indexTileOrder[i] = len(tiles)
+ }
+ tiles = append(tiles, p)
+ }
+ }
+
+ // Fetch all the tile data.
+ data, err := r.tr.ReadTiles(tiles)
+ if err != nil {
+ return nil, err
+ }
+ if len(data) != len(tiles) {
+ return nil, fmt.Errorf("TileReader returned bad result slice (len=%d, want %d)", len(data), len(tiles))
+ }
+ for i, tile := range tiles {
+ if len(data[i]) != tile.W*HashSize {
+ return nil, fmt.Errorf("TileReader returned bad result slice (%v len=%d, want %d)", tile.Path(), len(data[i]), tile.W*HashSize)
+ }
+ }
+
+ // Authenticate the initial tiles against the tree hash.
+ // They are arranged so that parents are authenticated before children.
+ // First the tiles needed for the tree hash.
+ th, err := HashFromTile(tiles[stxTileOrder[len(stx)-1]], data[stxTileOrder[len(stx)-1]], stx[len(stx)-1])
+ if err != nil {
+ return nil, err
+ }
+ for i := len(stx) - 2; i >= 0; i-- {
+ h, err := HashFromTile(tiles[stxTileOrder[i]], data[stxTileOrder[i]], stx[i])
+ if err != nil {
+ return nil, err
+ }
+ th = NodeHash(h, th)
+ }
+ if th != r.tree.Hash {
+ // The tiles do not support the tree hash.
+ // We know at least one is wrong, but not which one.
+ return nil, fmt.Errorf("downloaded inconsistent tile")
+ }
+
+ // Authenticate full tiles against their parents.
+ for i := len(stx); i < len(tiles); i++ {
+ tile := tiles[i]
+ p := tileParent(tile, 1, r.tree.N)
+ j, ok := tileOrder[p]
+ if !ok {
+ return nil, fmt.Errorf("bad math in tileHashReader %d %v: lost parent of %v", r.tree.N, indexes, tile)
+ }
+ h, err := HashFromTile(p, data[j], StoredHashIndex(p.L*p.H, tile.N))
+ if err != nil {
+ return nil, fmt.Errorf("bad math in tileHashReader %d %v: lost hash of %v: %v", r.tree.N, indexes, tile, err)
+ }
+ if h != tileHash(data[i]) {
+ return nil, fmt.Errorf("downloaded inconsistent tile")
+ }
+ }
+
+ // Now we have all the tiles needed for the requested hashes,
+ // and we've authenticated the full tile set against the trusted tree hash.
+ r.tr.SaveTiles(tiles, data)
+
+ // Pull out the requested hashes.
+ hashes := make([]Hash, len(indexes))
+ for i, x := range indexes {
+ j := indexTileOrder[i]
+ h, err := HashFromTile(tiles[j], data[j], x)
+ if err != nil {
+ return nil, fmt.Errorf("bad math in tileHashReader %d %v: lost hash %v: %v", r.tree.N, indexes, x, err)
+ }
+ hashes[i] = h
+ }
+
+ return hashes, nil
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tlog implements a tamper-evident log
+// used in the Go module go.sum database server.
+//
+// This package is part of a DRAFT of what the go.sum database server will look like.
+// Do not assume the details here are final!
+//
+// This package follows the design of Certificate Transparency (RFC 6962)
+// and its proofs are compatible with that system.
+// See TestCertificateTransparency.
+//
+package tlog
+
+import (
+ "crypto/sha256"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "math/bits"
+)
+
+// A Hash is a hash identifying a log record or tree root.
+type Hash [HashSize]byte
+
+// HashSize is the size of a Hash in bytes.
+const HashSize = 32
+
+// String returns a base64 representation of the hash for printing.
+func (h Hash) String() string {
+ return base64.StdEncoding.EncodeToString(h[:])
+}
+
+// MarshalJSON marshals the hash as a JSON string containing the base64-encoded hash.
+func (h Hash) MarshalJSON() ([]byte, error) {
+ return []byte(`"` + h.String() + `"`), nil
+}
+
+// UnmarshalJSON unmarshals a hash from JSON string containing the a base64-encoded hash.
+func (h *Hash) UnmarshalJSON(data []byte) error {
+ if len(data) != 1+44+1 || data[0] != '"' || data[len(data)-2] != '=' || data[len(data)-1] != '"' {
+ return errors.New("cannot decode hash")
+ }
+
+ // As of Go 1.12, base64.StdEncoding.Decode insists on
+ // slicing into target[33:] even when it only writes 32 bytes.
+ // Since we already checked that the hash ends in = above,
+ // we can use base64.RawStdEncoding with the = removed;
+ // RawStdEncoding does not exhibit the same bug.
+ // We decode into a temporary to avoid writing anything to *h
+ // unless the entire input is well-formed.
+ var tmp Hash
+ n, err := base64.RawStdEncoding.Decode(tmp[:], data[1:len(data)-2])
+ if err != nil || n != HashSize {
+ return errors.New("cannot decode hash")
+ }
+ *h = tmp
+ return nil
+}
+
+// ParseHash parses the base64-encoded string form of a hash.
+func ParseHash(s string) (Hash, error) {
+ data, err := base64.StdEncoding.DecodeString(s)
+ if err != nil || len(data) != HashSize {
+ return Hash{}, fmt.Errorf("malformed hash")
+ }
+ var h Hash
+ copy(h[:], data)
+ return h, nil
+}
+
+// maxpow2 returns k, the maximum power of 2 smaller than n,
+// as well as l = log₂ k (so k = 1<<l).
+func maxpow2(n int64) (k int64, l int) {
+ l = 0
+ for 1<<uint(l+1) < n {
+ l++
+ }
+ return 1 << uint(l), l
+}
+
+var zeroPrefix = []byte{0x00}
+
+// RecordHash returns the content hash for the given record data.
+func RecordHash(data []byte) Hash {
+ // SHA256(0x00 || data)
+ // https://tools.ietf.org/html/rfc6962#section-2.1
+ h := sha256.New()
+ h.Write(zeroPrefix)
+ h.Write(data)
+ var h1 Hash
+ h.Sum(h1[:0])
+ return h1
+}
+
+// NodeHash returns the hash for an interior tree node with the given left and right hashes.
+func NodeHash(left, right Hash) Hash {
+ // SHA256(0x01 || left || right)
+ // https://tools.ietf.org/html/rfc6962#section-2.1
+ // We use a stack buffer to assemble the hash input
+ // to avoid allocating a hash struct with sha256.New.
+ var buf [1 + HashSize + HashSize]byte
+ buf[0] = 0x01
+ copy(buf[1:], left[:])
+ copy(buf[1+HashSize:], right[:])
+ return sha256.Sum256(buf[:])
+}
+
+// For information about the stored hash index ordering,
+// see section 3.3 of Crosby and Wallach's paper
+// "Efficient Data Structures for Tamper-Evident Logging".
+// https://www.usenix.org/legacy/event/sec09/tech/full_papers/crosby.pdf
+
+// StoredHashIndex maps the tree coordinates (level, n)
+// to a dense linear ordering that can be used for hash storage.
+// Hash storage implementations that store hashes in sequential
+// storage can use this function to compute where to read or write
+// a given hash.
+func StoredHashIndex(level int, n int64) int64 {
+ // Level L's n'th hash is written right after level L+1's 2n+1'th hash.
+ // Work our way down to the level 0 ordering.
+ // We'll add back the orignal level count at the end.
+ for l := level; l > 0; l-- {
+ n = 2*n + 1
+ }
+
+ // Level 0's n'th hash is written at n+n/2+n/4+... (eventually n/2ⁱ hits zero).
+ i := int64(0)
+ for ; n > 0; n >>= 1 {
+ i += n
+ }
+
+ return i + int64(level)
+}
+
+// SplitStoredHashIndex is the inverse of StoredHashIndex.
+// That is, SplitStoredHashIndex(StoredHashIndex(level, n)) == level, n.
+func SplitStoredHashIndex(index int64) (level int, n int64) {
+ // Determine level 0 record before index.
+ // StoredHashIndex(0, n) < 2*n,
+ // so the n we want is in [index/2, index/2+log₂(index)].
+ n = index / 2
+ indexN := StoredHashIndex(0, n)
+ if indexN > index {
+ panic("bad math")
+ }
+ for {
+ // Each new record n adds 1 + trailingZeros(n) hashes.
+ x := indexN + 1 + int64(bits.TrailingZeros64(uint64(n+1)))
+ if x > index {
+ break
+ }
+ n++
+ indexN = x
+ }
+ // The hash we want was commited with record n,
+ // meaning it is one of (0, n), (1, n/2), (2, n/4), ...
+ level = int(index - indexN)
+ return level, n >> uint(level)
+}
+
+// StoredHashCount returns the number of stored hashes
+// that are expected for a tree with n records.
+func StoredHashCount(n int64) int64 {
+ if n == 0 {
+ return 0
+ }
+ // The tree will have the hashes up to the last leaf hash.
+ numHash := StoredHashIndex(0, n-1) + 1
+ // And it will have any hashes for subtrees completed by that leaf.
+ for i := uint64(n - 1); i&1 != 0; i >>= 1 {
+ numHash++
+ }
+ return numHash
+}
+
+// StoredHashes returns the hashes that must be stored when writing
+// record n with the given data. The hashes should be stored starting
+// at StoredHashIndex(0, n). The result will have at most 1 + log₂ n hashes,
+// but it will average just under two per call for a sequence of calls for n=1..k.
+//
+// StoredHashes may read up to log n earlier hashes from r
+// in order to compute hashes for completed subtrees.
+func StoredHashes(n int64, data []byte, r HashReader) ([]Hash, error) {
+ return StoredHashesForRecordHash(n, RecordHash(data), r)
+}
+
+// StoredHashesForRecordHash is like StoredHashes but takes
+// as its second argument RecordHash(data) instead of data itself.
+func StoredHashesForRecordHash(n int64, h Hash, r HashReader) ([]Hash, error) {
+ // Start with the record hash.
+ hashes := []Hash{h}
+
+ // Build list of indexes needed for hashes for completed subtrees.
+ // Each trailing 1 bit in the binary representation of n completes a subtree
+ // and consumes a hash from an adjacent subtree.
+ m := int(bits.TrailingZeros64(uint64(n + 1)))
+ indexes := make([]int64, m)
+ for i := 0; i < m; i++ {
+ // We arrange indexes in sorted order.
+ // Note that n>>i is always odd.
+ indexes[m-1-i] = StoredHashIndex(i, n>>uint(i)-1)
+ }
+
+ // Fetch hashes.
+ old, err := r.ReadHashes(indexes)
+ if err != nil {
+ return nil, err
+ }
+ if len(old) != len(indexes) {
+ return nil, fmt.Errorf("tlog: ReadHashes(%d indexes) = %d hashes", len(indexes), len(old))
+ }
+
+ // Build new hashes.
+ for i := 0; i < m; i++ {
+ h = NodeHash(old[m-1-i], h)
+ hashes = append(hashes, h)
+ }
+ return hashes, nil
+}
+
+// A HashReader can read hashes for nodes in the log's tree structure.
+type HashReader interface {
+ // ReadHashes returns the hashes with the given stored hash indexes
+ // (see StoredHashIndex and SplitStoredHashIndex).
+ // ReadHashes must return a slice of hashes the same length as indexes,
+ // or else it must return a non-nil error.
+ // ReadHashes may run faster if indexes is sorted in increasing order.
+ ReadHashes(indexes []int64) ([]Hash, error)
+}
+
+// A HashReaderFunc is a function implementing HashReader.
+type HashReaderFunc func([]int64) ([]Hash, error)
+
+func (f HashReaderFunc) ReadHashes(indexes []int64) ([]Hash, error) {
+ return f(indexes)
+}
+
+// TreeHash computes the hash for the root of the tree with n records,
+// using the HashReader to obtain previously stored hashes
+// (those returned by StoredHashes during the writes of those n records).
+// TreeHash makes a single call to ReadHash requesting at most 1 + log₂ n hashes.
+// The tree of size zero is defined to have an all-zero Hash.
+func TreeHash(n int64, r HashReader) (Hash, error) {
+ if n == 0 {
+ return Hash{}, nil
+ }
+ indexes := subTreeIndex(0, n, nil)
+ hashes, err := r.ReadHashes(indexes)
+ if err != nil {
+ return Hash{}, err
+ }
+ if len(hashes) != len(indexes) {
+ return Hash{}, fmt.Errorf("tlog: ReadHashes(%d indexes) = %d hashes", len(indexes), len(hashes))
+ }
+ hash, hashes := subTreeHash(0, n, hashes)
+ if len(hashes) != 0 {
+ panic("tlog: bad index math in TreeHash")
+ }
+ return hash, nil
+}
+
+// subTreeIndex returns the storage indexes needed to compute
+// the hash for the subtree containing records [lo, hi),
+// appending them to need and returning the result.
+// See https://tools.ietf.org/html/rfc6962#section-2.1
+func subTreeIndex(lo, hi int64, need []int64) []int64 {
+ // See subTreeHash below for commentary.
+ for lo < hi {
+ k, level := maxpow2(hi - lo + 1)
+ if lo&(k-1) != 0 {
+ panic("tlog: bad math in subTreeIndex")
+ }
+ need = append(need, StoredHashIndex(level, lo>>uint(level)))
+ lo += k
+ }
+ return need
+}
+
+// subTreeHash computes the hash for the subtree containing records [lo, hi),
+// assuming that hashes are the hashes corresponding to the indexes
+// returned by subTreeIndex(lo, hi).
+// It returns any leftover hashes.
+func subTreeHash(lo, hi int64, hashes []Hash) (Hash, []Hash) {
+ // Repeatedly partition the tree into a left side with 2^level nodes,
+ // for as large a level as possible, and a right side with the fringe.
+ // The left hash is stored directly and can be read from storage.
+ // The right side needs further computation.
+ numTree := 0
+ for lo < hi {
+ k, _ := maxpow2(hi - lo + 1)
+ if lo&(k-1) != 0 || lo >= hi {
+ panic("tlog: bad math in subTreeHash")
+ }
+ numTree++
+ lo += k
+ }
+
+ if len(hashes) < numTree {
+ panic("tlog: bad index math in subTreeHash")
+ }
+
+ // Reconstruct hash.
+ h := hashes[numTree-1]
+ for i := numTree - 2; i >= 0; i-- {
+ h = NodeHash(hashes[i], h)
+ }
+ return h, hashes[numTree:]
+}
+
+// A RecordProof is a verifiable proof that a particular log root contains a particular record.
+// RFC 6962 calls this a “Merkle audit path.”
+type RecordProof []Hash
+
+// ProveRecord returns the proof that the tree of size t contains the record with index n.
+func ProveRecord(t, n int64, r HashReader) (RecordProof, error) {
+ if t < 0 || n < 0 || n >= t {
+ return nil, fmt.Errorf("tlog: invalid inputs in ProveRecord")
+ }
+ indexes := leafProofIndex(0, t, n, nil)
+ if len(indexes) == 0 {
+ return RecordProof{}, nil
+ }
+ hashes, err := r.ReadHashes(indexes)
+ if err != nil {
+ return nil, err
+ }
+ if len(hashes) != len(indexes) {
+ return nil, fmt.Errorf("tlog: ReadHashes(%d indexes) = %d hashes", len(indexes), len(hashes))
+ }
+
+ p, hashes := leafProof(0, t, n, hashes)
+ if len(hashes) != 0 {
+ panic("tlog: bad index math in ProveRecord")
+ }
+ return p, nil
+}
+
+// leafProofIndex builds the list of indexes needed to construct the proof
+// that leaf n is contained in the subtree with leaves [lo, hi).
+// It appends those indexes to need and returns the result.
+// See https://tools.ietf.org/html/rfc6962#section-2.1.1
+func leafProofIndex(lo, hi, n int64, need []int64) []int64 {
+ // See leafProof below for commentary.
+ if !(lo <= n && n < hi) {
+ panic("tlog: bad math in leafProofIndex")
+ }
+ if lo+1 == hi {
+ return need
+ }
+ if k, _ := maxpow2(hi - lo); n < lo+k {
+ need = leafProofIndex(lo, lo+k, n, need)
+ need = subTreeIndex(lo+k, hi, need)
+ } else {
+ need = subTreeIndex(lo, lo+k, need)
+ need = leafProofIndex(lo+k, hi, n, need)
+ }
+ return need
+}
+
+// leafProof constructs the proof that leaf n is contained in the subtree with leaves [lo, hi).
+// It returns any leftover hashes as well.
+// See https://tools.ietf.org/html/rfc6962#section-2.1.1
+func leafProof(lo, hi, n int64, hashes []Hash) (RecordProof, []Hash) {
+ // We must have lo <= n < hi or else the code here has a bug.
+ if !(lo <= n && n < hi) {
+ panic("tlog: bad math in leafProof")
+ }
+
+ if lo+1 == hi { // n == lo
+ // Reached the leaf node.
+ // The verifier knows what the leaf hash is, so we don't need to send it.
+ return RecordProof{}, hashes
+ }
+
+ // Walk down the tree toward n.
+ // Record the hash of the path not taken (needed for verifying the proof).
+ var p RecordProof
+ var th Hash
+ if k, _ := maxpow2(hi - lo); n < lo+k {
+ // n is on left side
+ p, hashes = leafProof(lo, lo+k, n, hashes)
+ th, hashes = subTreeHash(lo+k, hi, hashes)
+ } else {
+ // n is on right side
+ th, hashes = subTreeHash(lo, lo+k, hashes)
+ p, hashes = leafProof(lo+k, hi, n, hashes)
+ }
+ return append(p, th), hashes
+}
+
+var errProofFailed = errors.New("invalid transparency proof")
+
+// CheckRecord verifies that p is a valid proof that the tree of size t
+// with hash th has an n'th record with hash h.
+func CheckRecord(p RecordProof, t int64, th Hash, n int64, h Hash) error {
+ if t < 0 || n < 0 || n >= t {
+ return fmt.Errorf("tlog: invalid inputs in CheckRecord")
+ }
+ th2, err := runRecordProof(p, 0, t, n, h)
+ if err != nil {
+ return err
+ }
+ if th2 == th {
+ return nil
+ }
+ return errProofFailed
+}
+
+// runRecordProof runs the proof p that leaf n is contained in the subtree with leaves [lo, hi).
+// Running the proof means constructing and returning the implied hash of that
+// subtree.
+func runRecordProof(p RecordProof, lo, hi, n int64, leafHash Hash) (Hash, error) {
+ // We must have lo <= n < hi or else the code here has a bug.
+ if !(lo <= n && n < hi) {
+ panic("tlog: bad math in runRecordProof")
+ }
+
+ if lo+1 == hi { // m == lo
+ // Reached the leaf node.
+ // The proof must not have any unnecessary hashes.
+ if len(p) != 0 {
+ return Hash{}, errProofFailed
+ }
+ return leafHash, nil
+ }
+
+ if len(p) == 0 {
+ return Hash{}, errProofFailed
+ }
+
+ k, _ := maxpow2(hi - lo)
+ if n < lo+k {
+ th, err := runRecordProof(p[:len(p)-1], lo, lo+k, n, leafHash)
+ if err != nil {
+ return Hash{}, err
+ }
+ return NodeHash(th, p[len(p)-1]), nil
+ } else {
+ th, err := runRecordProof(p[:len(p)-1], lo+k, hi, n, leafHash)
+ if err != nil {
+ return Hash{}, err
+ }
+ return NodeHash(p[len(p)-1], th), nil
+ }
+}
+
+// A TreeProof is a verifiable proof that a particular log tree contains
+// as a prefix all records present in an earlier tree.
+// RFC 6962 calls this a “Merkle consistency proof.”
+type TreeProof []Hash
+
+// ProveTree returns the proof that the tree of size t contains
+// as a prefix all the records from the tree of smaller size n.
+func ProveTree(t, n int64, h HashReader) (TreeProof, error) {
+ if t < 1 || n < 1 || n > t {
+ return nil, fmt.Errorf("tlog: invalid inputs in ProveTree")
+ }
+ indexes := treeProofIndex(0, t, n, nil)
+ if len(indexes) == 0 {
+ return TreeProof{}, nil
+ }
+ hashes, err := h.ReadHashes(indexes)
+ if err != nil {
+ return nil, err
+ }
+ if len(hashes) != len(indexes) {
+ return nil, fmt.Errorf("tlog: ReadHashes(%d indexes) = %d hashes", len(indexes), len(hashes))
+ }
+
+ p, hashes := treeProof(0, t, n, hashes)
+ if len(hashes) != 0 {
+ panic("tlog: bad index math in ProveTree")
+ }
+ return p, nil
+}
+
+// treeProofIndex builds the list of indexes needed to construct
+// the sub-proof related to the subtree containing records [lo, hi).
+// See https://tools.ietf.org/html/rfc6962#section-2.1.2.
+func treeProofIndex(lo, hi, n int64, need []int64) []int64 {
+ // See treeProof below for commentary.
+ if !(lo < n && n <= hi) {
+ panic("tlog: bad math in treeProofIndex")
+ }
+
+ if n == hi {
+ if lo == 0 {
+ return need
+ }
+ return subTreeIndex(lo, hi, need)
+ }
+
+ if k, _ := maxpow2(hi - lo); n <= lo+k {
+ need = treeProofIndex(lo, lo+k, n, need)
+ need = subTreeIndex(lo+k, hi, need)
+ } else {
+ need = subTreeIndex(lo, lo+k, need)
+ need = treeProofIndex(lo+k, hi, n, need)
+ }
+ return need
+}
+
+// treeProof constructs the sub-proof related to the subtree containing records [lo, hi).
+// It returns any leftover hashes as well.
+// See https://tools.ietf.org/html/rfc6962#section-2.1.2.
+func treeProof(lo, hi, n int64, hashes []Hash) (TreeProof, []Hash) {
+ // We must have lo < n <= hi or else the code here has a bug.
+ if !(lo < n && n <= hi) {
+ panic("tlog: bad math in treeProof")
+ }
+
+ // Reached common ground.
+ if n == hi {
+ if lo == 0 {
+ // This subtree corresponds exactly to the old tree.
+ // The verifier knows that hash, so we don't need to send it.
+ return TreeProof{}, hashes
+ }
+ th, hashes := subTreeHash(lo, hi, hashes)
+ return TreeProof{th}, hashes
+ }
+
+ // Interior node for the proof.
+ // Decide whether to walk down the left or right side.
+ var p TreeProof
+ var th Hash
+ if k, _ := maxpow2(hi - lo); n <= lo+k {
+ // m is on left side
+ p, hashes = treeProof(lo, lo+k, n, hashes)
+ th, hashes = subTreeHash(lo+k, hi, hashes)
+ } else {
+ // m is on right side
+ th, hashes = subTreeHash(lo, lo+k, hashes)
+ p, hashes = treeProof(lo+k, hi, n, hashes)
+ }
+ return append(p, th), hashes
+}
+
+// CheckTree verifies that p is a valid proof that the tree of size t with hash th
+// contains as a prefix the tree of size n with hash h.
+func CheckTree(p TreeProof, t int64, th Hash, n int64, h Hash) error {
+ if t < 1 || n < 1 || n > t {
+ return fmt.Errorf("tlog: invalid inputs in CheckTree")
+ }
+ h2, th2, err := runTreeProof(p, 0, t, n, h)
+ if err != nil {
+ return err
+ }
+ if th2 == th && h2 == h {
+ return nil
+ }
+ return errProofFailed
+}
+
+// runTreeProof runs the sub-proof p related to the subtree containing records [lo, hi),
+// where old is the hash of the old tree with n records.
+// Running the proof means constructing and returning the implied hashes of that
+// subtree in both the old and new tree.
+func runTreeProof(p TreeProof, lo, hi, n int64, old Hash) (Hash, Hash, error) {
+ // We must have lo < n <= hi or else the code here has a bug.
+ if !(lo < n && n <= hi) {
+ panic("tlog: bad math in runTreeProof")
+ }
+
+ // Reached common ground.
+ if n == hi {
+ if lo == 0 {
+ if len(p) != 0 {
+ return Hash{}, Hash{}, errProofFailed
+ }
+ return old, old, nil
+ }
+ if len(p) != 1 {
+ return Hash{}, Hash{}, errProofFailed
+ }
+ return p[0], p[0], nil
+ }
+
+ if len(p) == 0 {
+ return Hash{}, Hash{}, errProofFailed
+ }
+
+ // Interior node for the proof.
+ k, _ := maxpow2(hi - lo)
+ if n <= lo+k {
+ oh, th, err := runTreeProof(p[:len(p)-1], lo, lo+k, n, old)
+ if err != nil {
+ return Hash{}, Hash{}, err
+ }
+ return oh, NodeHash(th, p[len(p)-1]), nil
+ } else {
+ oh, th, err := runTreeProof(p[:len(p)-1], lo+k, hi, n, old)
+ if err != nil {
+ return Hash{}, Hash{}, err
+ }
+ return NodeHash(p[len(p)-1], oh), NodeHash(p[len(p)-1], th), nil
+ }
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tlog
+
+import (
+ "bytes"
+ "fmt"
+ "testing"
+)
+
+type testHashStorage []Hash
+
+func (t testHashStorage) ReadHash(level int, n int64) (Hash, error) {
+ return t[StoredHashIndex(level, n)], nil
+}
+
+func (t testHashStorage) ReadHashes(index []int64) ([]Hash, error) {
+ // It's not required by HashReader that indexes be in increasing order,
+ // but check that the functions we are testing only ever ask for
+ // indexes in increasing order.
+ for i := 1; i < len(index); i++ {
+ if index[i-1] >= index[i] {
+ panic("indexes out of order")
+ }
+ }
+
+ out := make([]Hash, len(index))
+ for i, x := range index {
+ out[i] = t[x]
+ }
+ return out, nil
+}
+
+type testTilesStorage struct {
+ unsaved int
+ m map[Tile][]byte
+}
+
+func (t testTilesStorage) Height() int {
+ return 2
+}
+
+func (t *testTilesStorage) SaveTiles(tiles []Tile, data [][]byte) {
+ t.unsaved -= len(tiles)
+}
+
+func (t *testTilesStorage) ReadTiles(tiles []Tile) ([][]byte, error) {
+ out := make([][]byte, len(tiles))
+ for i, tile := range tiles {
+ out[i] = t.m[tile]
+ }
+ t.unsaved += len(tiles)
+ return out, nil
+}
+
+func TestTree(t *testing.T) {
+ var trees []Hash
+ var leafhashes []Hash
+ var storage testHashStorage
+ tiles := make(map[Tile][]byte)
+ const testH = 2
+ for i := int64(0); i < 100; i++ {
+ data := []byte(fmt.Sprintf("leaf %d", i))
+ hashes, err := StoredHashes(i, data, storage)
+ if err != nil {
+ t.Fatal(err)
+ }
+ leafhashes = append(leafhashes, RecordHash(data))
+ oldStorage := len(storage)
+ storage = append(storage, hashes...)
+ if count := StoredHashCount(i + 1); count != int64(len(storage)) {
+ t.Errorf("StoredHashCount(%d) = %d, have %d StoredHashes", i+1, count, len(storage))
+ }
+ th, err := TreeHash(i+1, storage)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for _, tile := range NewTiles(testH, i, i+1) {
+ data, err := ReadTileData(tile, storage)
+ if err != nil {
+ t.Fatal(err)
+ }
+ old := Tile{H: tile.H, L: tile.L, N: tile.N, W: tile.W - 1}
+ oldData := tiles[old]
+ if len(oldData) != len(data)-HashSize || !bytes.Equal(oldData, data[:len(oldData)]) {
+ t.Fatalf("tile %v not extending earlier tile %v", tile.Path(), old.Path())
+ }
+ tiles[tile] = data
+ }
+ for _, tile := range NewTiles(testH, 0, i+1) {
+ data, err := ReadTileData(tile, storage)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(tiles[tile], data) {
+ t.Fatalf("mismatch at %+v", tile)
+ }
+ }
+ for _, tile := range NewTiles(testH, i/2, i+1) {
+ data, err := ReadTileData(tile, storage)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(tiles[tile], data) {
+ t.Fatalf("mismatch at %+v", tile)
+ }
+ }
+
+ // Check that all the new hashes are readable from their tiles.
+ for j := oldStorage; j < len(storage); j++ {
+ tile := TileForIndex(testH, int64(j))
+ data, ok := tiles[tile]
+ if !ok {
+ t.Log(NewTiles(testH, 0, i+1))
+ t.Fatalf("TileForIndex(%d, %d) = %v, not yet stored (i=%d, stored %d)", testH, j, tile.Path(), i, len(storage))
+ continue
+ }
+ h, err := HashFromTile(tile, data, int64(j))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if h != storage[j] {
+ t.Errorf("HashFromTile(%v, %d) = %v, want %v", tile.Path(), int64(j), h, storage[j])
+ }
+ }
+
+ trees = append(trees, th)
+
+ // Check that leaf proofs work, for all trees and leaves so far.
+ for j := int64(0); j <= i; j++ {
+ p, err := ProveRecord(i+1, j, storage)
+ if err != nil {
+ t.Fatalf("ProveRecord(%d, %d): %v", i+1, j, err)
+ }
+ if err := CheckRecord(p, i+1, th, j, leafhashes[j]); err != nil {
+ t.Fatalf("CheckRecord(%d, %d): %v", i+1, j, err)
+ }
+ for k := range p {
+ p[k][0] ^= 1
+ if err := CheckRecord(p, i+1, th, j, leafhashes[j]); err == nil {
+ t.Fatalf("CheckRecord(%d, %d) succeeded with corrupt proof hash #%d!", i+1, j, k)
+ }
+ p[k][0] ^= 1
+ }
+ }
+
+ // Check that leaf proofs work using TileReader.
+ // To prove a leaf that way, all you have to do is read and verify its hash.
+ storage := &testTilesStorage{m: tiles}
+ thr := TileHashReader(Tree{i + 1, th}, storage)
+ for j := int64(0); j <= i; j++ {
+ h, err := thr.ReadHashes([]int64{StoredHashIndex(0, j)})
+ if err != nil {
+ t.Fatalf("TileHashReader(%d).ReadHashes(%d): %v", i+1, j, err)
+ }
+ if h[0] != leafhashes[j] {
+ t.Fatalf("TileHashReader(%d).ReadHashes(%d) returned wrong hash", i+1, j)
+ }
+
+ // Even though reading the hash suffices,
+ // check we can generate the proof too.
+ p, err := ProveRecord(i+1, j, thr)
+ if err != nil {
+ t.Fatalf("ProveRecord(%d, %d, TileHashReader(%d)): %v", i+1, j, i+1, err)
+ }
+ if err := CheckRecord(p, i+1, th, j, leafhashes[j]); err != nil {
+ t.Fatalf("CheckRecord(%d, %d, TileHashReader(%d)): %v", i+1, j, i+1, err)
+ }
+ }
+ if storage.unsaved != 0 {
+ t.Fatalf("TileHashReader(%d) did not save %d tiles", i+1, storage.unsaved)
+ }
+
+ // Check that ReadHashes will give an error if the index is not in the tree.
+ if _, err := thr.ReadHashes([]int64{(i + 1) * 2}); err == nil {
+ t.Fatalf("TileHashReader(%d).ReadHashes(%d) for index not in tree <nil>, want err", i, i+1)
+ }
+ if storage.unsaved != 0 {
+ t.Fatalf("TileHashReader(%d) did not save %d tiles", i+1, storage.unsaved)
+ }
+
+ // Check that tree proofs work, for all trees so far, using TileReader.
+ // To prove a tree that way, all you have to do is compute and verify its hash.
+ for j := int64(0); j <= i; j++ {
+ h, err := TreeHash(j+1, thr)
+ if err != nil {
+ t.Fatalf("TreeHash(%d, TileHashReader(%d)): %v", j, i+1, err)
+ }
+ if h != trees[j] {
+ t.Fatalf("TreeHash(%d, TileHashReader(%d)) = %x, want %x (%v)", j, i+1, h[:], trees[j][:], trees[j])
+ }
+
+ // Even though computing the subtree hash suffices,
+ // check that we can generate the proof too.
+ p, err := ProveTree(i+1, j+1, thr)
+ if err != nil {
+ t.Fatalf("ProveTree(%d, %d): %v", i+1, j+1, err)
+ }
+ if err := CheckTree(p, i+1, th, j+1, trees[j]); err != nil {
+ t.Fatalf("CheckTree(%d, %d): %v [%v]", i+1, j+1, err, p)
+ }
+ for k := range p {
+ p[k][0] ^= 1
+ if err := CheckTree(p, i+1, th, j+1, trees[j]); err == nil {
+ t.Fatalf("CheckTree(%d, %d) succeeded with corrupt proof hash #%d!", i+1, j+1, k)
+ }
+ p[k][0] ^= 1
+ }
+ }
+ if storage.unsaved != 0 {
+ t.Fatalf("TileHashReader(%d) did not save %d tiles", i+1, storage.unsaved)
+ }
+ }
+}
+
+func TestSplitStoredHashIndex(t *testing.T) {
+ for l := 0; l < 10; l++ {
+ for n := int64(0); n < 100; n++ {
+ x := StoredHashIndex(l, n)
+ l1, n1 := SplitStoredHashIndex(x)
+ if l1 != l || n1 != n {
+ t.Fatalf("StoredHashIndex(%d, %d) = %d, but SplitStoredHashIndex(%d) = %d, %d", l, n, x, x, l1, n1)
+ }
+ }
+ }
+}
+
+// TODO(rsc): Test invalid paths too, like "tile/3/5/123/456/078".
+var tilePaths = []struct {
+ path string
+ tile Tile
+}{
+ {"tile/4/0/001", Tile{4, 0, 1, 16}},
+ {"tile/4/0/001.p/5", Tile{4, 0, 1, 5}},
+ {"tile/3/5/x123/x456/078", Tile{3, 5, 123456078, 8}},
+ {"tile/3/5/x123/x456/078.p/2", Tile{3, 5, 123456078, 2}},
+ {"tile/1/0/x003/x057/500", Tile{1, 0, 3057500, 2}},
+ {"tile/3/5/123/456/078", Tile{}},
+ {"tile/3/-1/123/456/078", Tile{}},
+ {"tile/1/data/x003/x057/500", Tile{1, -1, 3057500, 2}},
+}
+
+func TestTilePath(t *testing.T) {
+ for _, tt := range tilePaths {
+ if tt.tile.H > 0 {
+ p := tt.tile.Path()
+ if p != tt.path {
+ t.Errorf("%+v.Path() = %q, want %q", tt.tile, p, tt.path)
+ }
+ }
+ tile, err := ParseTilePath(tt.path)
+ if err != nil {
+ if tt.tile.H == 0 {
+ // Expected error.
+ continue
+ }
+ t.Errorf("ParseTilePath(%q): %v", tt.path, err)
+ } else if tile != tt.tile {
+ if tt.tile.H == 0 {
+ t.Errorf("ParseTilePath(%q): expected error, got %+v", tt.path, tt.tile)
+ continue
+ }
+ t.Errorf("ParseTilePath(%q) = %+v, want %+v", tt.path, tile, tt.tile)
+ }
+ }
+}