From: Sergey Matveev Date: Tue, 28 Jan 2025 08:57:59 +0000 (+0300) Subject: Merkle-tree hashers X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=d3cf6eaa826cf3f55c1be0c73aa6f49a732a4ee0507f34bf84c97f6727f8bf57;p=keks.git Merkle-tree hashers --- diff --git a/go/pki/hash/merkle/cmd/main.go b/go/pki/hash/merkle/cmd/main.go new file mode 100644 index 0000000..1bca4fa --- /dev/null +++ b/go/pki/hash/merkle/cmd/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "crypto/sha512" + "encoding/hex" + "flag" + "fmt" + "hash" + "io" + "log" + "os" + "runtime" + + "go.cypherpunks.su/keks/pki/hash/merkle" + "golang.org/x/crypto/blake2b" +) + +func main() { + workers := flag.Int("p", runtime.NumCPU(), "Parallel workers") + chunkLen := flag.Int("c", 8, "Chunk size, KiB") + algo := flag.String("a", "blake2b", "TODO") + flag.Parse() + var hasher hash.Hash + switch *algo { + case "sha512": + hasher = merkle.NewHasherPrefixed(sha512.New, *chunkLen*1024, *workers) + case "blake2b": + hasher = merkle.NewHasher( + func() hash.Hash { + h, e := blake2b.New512([]byte("LEAF")) + if e != nil { + panic(e) + } + return h + }, + func() hash.Hash { + h, e := blake2b.New512([]byte("NODE")) + if e != nil { + panic(e) + } + return h + }, + func(h hash.Hash) { h.Reset() }, + func(h hash.Hash) { h.Reset() }, + *chunkLen*1024, + *workers, + ) + default: + log.Fatal("unknown -a") + } + if _, err := io.CopyBuffer(hasher, os.Stdin, make([]byte, 128*1024)); err != nil { + log.Fatal(err) + } + fmt.Println(hex.EncodeToString(hasher.Sum(nil))) +} diff --git a/go/pki/hash/merkle/hash.go b/go/pki/hash/merkle/hash.go new file mode 100644 index 0000000..e1308a8 --- /dev/null +++ b/go/pki/hash/merkle/hash.go @@ -0,0 +1,240 @@ +// GoKEKS/PKI -- PKI-related capabilities based on KEKS encoded formats +// Copyright (C) 2024-2025 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program. If not, see . + +package merkle + +import ( + "hash" + "io" + "sync" +) + +const MaxDepth = 64 + +type job struct { + reply chan []byte + chunk []byte +} + +type Hasher struct { + nodeHash hash.Hash + leafNew func() hash.Hash + leafReset func(hash.Hash) + nodeReset func(hash.Hash) + pr *io.PipeReader + pw *io.PipeWriter + hashes [2 * MaxDepth][]byte + frees [2 * MaxDepth]bool + count int + workersLen int + chunkLen int + + freeChunks chan []byte + freeHshes chan []byte + freeReplies chan chan []byte + jobs chan job + replies chan chan []byte + finished chan struct{} + workers sync.WaitGroup +} + +func (h *Hasher) Size() int { + return h.nodeHash.Size() +} + +func (h *Hasher) BlockSize() int { + return h.nodeHash.BlockSize() +} + +func NewHasher( + leafNew, nodeNew func() hash.Hash, + leafReset, nodeReset func(hash.Hash), + chunkLen, workers int, +) *Hasher { + h := Hasher{ + leafNew: leafNew, + nodeHash: nodeNew(), + leafReset: leafReset, + nodeReset: nodeReset, + freeChunks: make(chan []byte, workers), + freeHshes: make(chan []byte, workers), + freeReplies: make(chan chan []byte, workers), + workersLen: workers, + chunkLen: chunkLen, + } + hashSize := h.Size() + for i := 0; i < 2*MaxDepth; i++ { + h.hashes[i] = make([]byte, hashSize) + h.frees[i] = true + } + for range workers { + h.freeChunks <- make([]byte, chunkLen) + h.freeHshes <- make([]byte, hashSize) + h.freeReplies <- make(chan []byte) + } + h.prepare() + return &h +} + +func (h *Hasher) prepare() { + h.jobs = make(chan job, h.workersLen) + h.replies = make(chan chan []byte, h.workersLen) + h.finished = make(chan struct{}) + for range h.workersLen { + go h.worker() + } + h.workers.Add(h.workersLen) + go h.aggregator() +} + +func (h *Hasher) Reset() { + for i := 0; i < 2*MaxDepth; i++ { + h.frees[i] = true + } + h.pw.Close() + <-h.finished + h.prepare() +} + +func (h *Hasher) get(l int) []byte { + if l >= MaxDepth { + panic("too deep") + } + i := l * 2 + if !h.frees[i] { + i++ + } + h.frees[i] = false + return h.hashes[i] +} + +func (h *Hasher) fold() { + var err error + for l := 0; l < MaxDepth; l++ { + if h.frees[l*2+0] || h.frees[l*2+1] { + continue + } + h.nodeReset(h.nodeHash) + if _, err = h.nodeHash.Write(h.hashes[l*2+0]); err != nil { + panic(err) + } + if _, err = h.nodeHash.Write(h.hashes[l*2+1]); err != nil { + panic(err) + } + h.nodeHash.Sum(h.get(l + 1)[:0]) + h.frees[l*2+0] = true + h.frees[l*2+1] = true + h.count-- + } +} + +func (h *Hasher) makePipe() { + h.pr, h.pw = io.Pipe() + go func() { + h.ReadFrom(h.pr) + h.pr.Close() + }() +} + +func (h *Hasher) Write(p []byte) (int, error) { + if h.pr == nil { + h.makePipe() + } + return h.pw.Write(p) +} + +func (h *Hasher) ReadFrom(r io.Reader) (total int64, err error) { + var n int + var eof bool + var chunk []byte + var reply chan []byte + for !eof { + chunk = <-h.freeChunks + n, err = io.ReadFull(r, chunk) + total += int64(n) + if err != nil { + if err != io.ErrUnexpectedEOF { + return + } + err = nil + eof = true + } + if n == 0 { + continue + } + reply = <-h.freeReplies + h.jobs <- job{reply: reply, chunk: chunk[:n]} + h.replies <- reply + } + close(h.jobs) + h.workers.Wait() + close(h.replies) + return +} + +func (h *Hasher) worker() { + hasher := h.leafNew() + var err error + var hsh []byte + for j := range h.jobs { + h.leafReset(hasher) + if _, err = hasher.Write(j.chunk); err != nil { + panic(err) + } + h.freeChunks <- j.chunk + hsh = <-h.freeHshes + hasher.Sum(hsh[:0]) + j.reply <- hsh + } + h.workers.Done() +} + +func (h *Hasher) aggregator() { + var hsh []byte + for reply := range h.replies { + hsh = <-reply + h.freeReplies <- reply + copy(h.get(0), hsh) + h.count++ + h.fold() + h.freeHshes <- hsh + } + close(h.finished) +} + +func (h *Hasher) Sum(b []byte) []byte { + if h.pw != nil { + h.pw.Close() + } + <-h.finished + if h.count == 0 { + return append(b, h.leafNew().Sum(nil)...) + } + for l := 0; l < MaxDepth; l++ { + if h.count == 1 { + for ; l < MaxDepth; l++ { + if !h.frees[l*2+0] { + return append(b, h.hashes[l*2+0]...) + } + } + } + if !h.frees[l*2+0] { + copy(h.get(l+1), h.hashes[l*2+0]) + h.frees[l*2+0] = true + h.fold() + } + } + panic("did not reach the end") +} diff --git a/go/pki/hash/merkle/hash_test.go b/go/pki/hash/merkle/hash_test.go new file mode 100644 index 0000000..e59c715 --- /dev/null +++ b/go/pki/hash/merkle/hash_test.go @@ -0,0 +1,142 @@ +// GoKEKS/PKI -- PKI-related capabilities based on KEKS encoded formats +// Copyright (C) 2024-2025 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program. If not, see . + +package merkle + +import ( + "bytes" + "crypto/sha512" + "testing" +) + +func TestEmpty(t *testing.T) { + h0 := sha512.Sum512(nil) + h1 := NewHasherPrefixed(sha512.New, 8, 2) + if !bytes.Equal(h1.Sum(nil), h0[:]) { + t.FailNow() + } +} + +func Test1ChunkShort(t *testing.T) { + data := []byte("some") + h0 := sha512.Sum512(append([]byte{0}, data...)) + h1 := NewHasherPrefixed(sha512.New, 8, 2) + if _, err := h1.Write(data); err != nil { + t.Fatal(err) + } + if !bytes.Equal(h1.Sum(nil), h0[:]) { + t.FailNow() + } +} + +func Test1Chunk(t *testing.T) { + data := []byte("somethin") + h0 := sha512.Sum512(append([]byte{0}, data...)) + h1 := NewHasherPrefixed(sha512.New, 8, 2) + if _, err := h1.Write(data); err != nil { + t.Fatal(err) + } + if !bytes.Equal(h1.Sum(nil), h0[:]) { + t.FailNow() + } +} + +func Test2ChunkShort(t *testing.T) { + data := []byte("something") + c0 := sha512.Sum512([]byte("\x00somethin")) + c1 := sha512.Sum512([]byte("\x00g")) + c2 := append(c0[:], c1[:]...) + c3 := sha512.Sum512(append([]byte{0x01}, c2...)) + h1 := NewHasherPrefixed(sha512.New, 8, 2) + if _, err := h1.Write(data); err != nil { + t.Fatal(err) + } + if !bytes.Equal(h1.Sum(nil), c3[:]) { + t.FailNow() + } +} + +func Test2Chunk(t *testing.T) { + data := []byte("somethingsomethi") + c0 := sha512.Sum512([]byte("\x00somethin")) + c1 := sha512.Sum512([]byte("\x00gsomethi")) + c2 := sha512.Sum512(append([]byte{0x01}, append(c0[:], c1[:]...)...)) + h1 := NewHasherPrefixed(sha512.New, 8, 2) + if _, err := h1.Write(data); err != nil { + t.Fatal(err) + } + if !bytes.Equal(h1.Sum(nil), c2[:]) { + t.FailNow() + } +} + +func Test3Chunk(t *testing.T) { + data := []byte("123456") + c0 := sha512.Sum512([]byte("\x0012")) + c1 := sha512.Sum512([]byte("\x0034")) + c2 := sha512.Sum512([]byte("\x0056")) + c3 := sha512.Sum512(append([]byte{0x01}, append(c0[:], c1[:]...)...)) + c4 := sha512.Sum512(append([]byte{0x01}, append(c3[:], c2[:]...)...)) + h1 := NewHasherPrefixed(sha512.New, 2, 2) + if _, err := h1.Write(data); err != nil { + t.Fatal(err) + } + if !bytes.Equal(h1.Sum(nil), c4[:]) { + t.FailNow() + } +} + +func Test4Chunk(t *testing.T) { + data := []byte("12345678") + c0 := sha512.Sum512([]byte("\x0012")) + c1 := sha512.Sum512([]byte("\x0034")) + c2 := sha512.Sum512([]byte("\x0056")) + c3 := sha512.Sum512([]byte("\x0078")) + c4 := sha512.Sum512(append([]byte{0x01}, append(c0[:], c1[:]...)...)) + c5 := sha512.Sum512(append([]byte{0x01}, append(c2[:], c3[:]...)...)) + c6 := sha512.Sum512(append([]byte{0x01}, append(c4[:], c5[:]...)...)) + h1 := NewHasherPrefixed(sha512.New, 2, 2) + if _, err := h1.Write(data); err != nil { + t.Fatal(err) + } + if !bytes.Equal(h1.Sum(nil), c6[:]) { + t.FailNow() + } +} + +// RFC 9162, 2.1.5 Example +func TestRFC7Leaves(t *testing.T) { + data := []byte("d0d1d2d3d4d5d6") + a := sha512.Sum512([]byte("\x00d0")) + b := sha512.Sum512([]byte("\x00d1")) + c := sha512.Sum512([]byte("\x00d2")) + d := sha512.Sum512([]byte("\x00d3")) + e := sha512.Sum512([]byte("\x00d4")) + f := sha512.Sum512([]byte("\x00d5")) + g := sha512.Sum512(append([]byte{0x01}, append(a[:], b[:]...)...)) + h := sha512.Sum512(append([]byte{0x01}, append(c[:], d[:]...)...)) + i := sha512.Sum512(append([]byte{0x01}, append(e[:], f[:]...)...)) + j := sha512.Sum512([]byte("\x00d6")) + k := sha512.Sum512(append([]byte{0x01}, append(g[:], h[:]...)...)) + l := sha512.Sum512(append([]byte{0x01}, append(i[:], j[:]...)...)) + root := sha512.Sum512(append([]byte{0x01}, append(k[:], l[:]...)...)) + h1 := NewHasherPrefixed(sha512.New, 2, 2) + if _, err := h1.Write(data); err != nil { + t.Fatal(err) + } + if !bytes.Equal(h1.Sum(nil), root[:]) { + t.FailNow() + } +} diff --git a/go/pki/hash/merkle/prefixed.go b/go/pki/hash/merkle/prefixed.go new file mode 100644 index 0000000..207bdd6 --- /dev/null +++ b/go/pki/hash/merkle/prefixed.go @@ -0,0 +1,39 @@ +// GoKEKS/PKI -- PKI-related capabilities based on KEKS encoded formats +// Copyright (C) 2024-2025 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program. If not, see . + +package merkle + +import "hash" + +func NewHasherPrefixed(h func() hash.Hash, chunkLen, workers int) *Hasher { + return NewHasher( + h, + h, + func(h hash.Hash) { + h.Reset() + if _, err := h.Write([]byte{0x00}); err != nil { + panic(err) + } + }, + func(h hash.Hash) { + h.Reset() + if _, err := h.Write([]byte{0x01}); err != nil { + panic(err) + } + }, + chunkLen, + workers, + ) +} diff --git a/spec/format/hashed.texi b/spec/format/hashed.texi index 8bf30f9..ae127b7 100644 --- a/spec/format/hashed.texi +++ b/spec/format/hashed.texi @@ -34,7 +34,9 @@ algorithms. @url{https://datatracker.ietf.org/doc/html/rfc9162, RFC 9162}, except that no @code{0x00}/@code{0x01} constants are appended to the hashed data, but BLAKE2b is initialised in keyed mode with - either "LEAF" or "NODE" keys. + either "LEAF" or "NODE" keys. Although BLAKE2 has ability to set + tree-hashing parameters on its own, many implementations do not + provide necessary API for that. @code{blake2b-merkle} algorithm identifier is used. @@ -79,7 +81,8 @@ algorithms. Streebog-512 is used in Merkle tree hashing mode, as described in @url{https://datatracker.ietf.org/doc/html/rfc9162, RFC 9162}. - @code{streebog512-merkle} algorithm identifier is used. + @code{streebog256-merkle}, @code{streebog512-merkle} algorithm + identifiers are used. @node pki-hashed-xxh3-128 @subsection pki-hashed with XXH3-128