--- /dev/null
+package main
+
+import (
+ "crypto/sha512"
+ "encoding/hex"
+ "flag"
+ "fmt"
+ "hash"
+ "io"
+ "log"
+ "os"
+ "runtime"
+
+ "go.cypherpunks.su/keks/pki/hash/merkle"
+ "golang.org/x/crypto/blake2b"
+)
+
+func main() {
+ workers := flag.Int("p", runtime.NumCPU(), "Parallel workers")
+ chunkLen := flag.Int("c", 8, "Chunk size, KiB")
+ algo := flag.String("a", "blake2b", "TODO")
+ flag.Parse()
+ var hasher hash.Hash
+ switch *algo {
+ case "sha512":
+ hasher = merkle.NewHasherPrefixed(sha512.New, *chunkLen*1024, *workers)
+ case "blake2b":
+ hasher = merkle.NewHasher(
+ func() hash.Hash {
+ h, e := blake2b.New512([]byte("LEAF"))
+ if e != nil {
+ panic(e)
+ }
+ return h
+ },
+ func() hash.Hash {
+ h, e := blake2b.New512([]byte("NODE"))
+ if e != nil {
+ panic(e)
+ }
+ return h
+ },
+ func(h hash.Hash) { h.Reset() },
+ func(h hash.Hash) { h.Reset() },
+ *chunkLen*1024,
+ *workers,
+ )
+ default:
+ log.Fatal("unknown -a")
+ }
+ if _, err := io.CopyBuffer(hasher, os.Stdin, make([]byte, 128*1024)); err != nil {
+ log.Fatal(err)
+ }
+ fmt.Println(hex.EncodeToString(hasher.Sum(nil)))
+}
--- /dev/null
+// GoKEKS/PKI -- PKI-related capabilities based on KEKS encoded formats
+// Copyright (C) 2024-2025 Sergey Matveev <stargrave@stargrave.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation, version 3 of the License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package merkle
+
+import (
+ "hash"
+ "io"
+ "sync"
+)
+
+const MaxDepth = 64
+
+type job struct {
+ reply chan []byte
+ chunk []byte
+}
+
+type Hasher struct {
+ nodeHash hash.Hash
+ leafNew func() hash.Hash
+ leafReset func(hash.Hash)
+ nodeReset func(hash.Hash)
+ pr *io.PipeReader
+ pw *io.PipeWriter
+ hashes [2 * MaxDepth][]byte
+ frees [2 * MaxDepth]bool
+ count int
+ workersLen int
+ chunkLen int
+
+ freeChunks chan []byte
+ freeHshes chan []byte
+ freeReplies chan chan []byte
+ jobs chan job
+ replies chan chan []byte
+ finished chan struct{}
+ workers sync.WaitGroup
+}
+
+func (h *Hasher) Size() int {
+ return h.nodeHash.Size()
+}
+
+func (h *Hasher) BlockSize() int {
+ return h.nodeHash.BlockSize()
+}
+
+func NewHasher(
+ leafNew, nodeNew func() hash.Hash,
+ leafReset, nodeReset func(hash.Hash),
+ chunkLen, workers int,
+) *Hasher {
+ h := Hasher{
+ leafNew: leafNew,
+ nodeHash: nodeNew(),
+ leafReset: leafReset,
+ nodeReset: nodeReset,
+ freeChunks: make(chan []byte, workers),
+ freeHshes: make(chan []byte, workers),
+ freeReplies: make(chan chan []byte, workers),
+ workersLen: workers,
+ chunkLen: chunkLen,
+ }
+ hashSize := h.Size()
+ for i := 0; i < 2*MaxDepth; i++ {
+ h.hashes[i] = make([]byte, hashSize)
+ h.frees[i] = true
+ }
+ for range workers {
+ h.freeChunks <- make([]byte, chunkLen)
+ h.freeHshes <- make([]byte, hashSize)
+ h.freeReplies <- make(chan []byte)
+ }
+ h.prepare()
+ return &h
+}
+
+func (h *Hasher) prepare() {
+ h.jobs = make(chan job, h.workersLen)
+ h.replies = make(chan chan []byte, h.workersLen)
+ h.finished = make(chan struct{})
+ for range h.workersLen {
+ go h.worker()
+ }
+ h.workers.Add(h.workersLen)
+ go h.aggregator()
+}
+
+func (h *Hasher) Reset() {
+ for i := 0; i < 2*MaxDepth; i++ {
+ h.frees[i] = true
+ }
+ h.pw.Close()
+ <-h.finished
+ h.prepare()
+}
+
+func (h *Hasher) get(l int) []byte {
+ if l >= MaxDepth {
+ panic("too deep")
+ }
+ i := l * 2
+ if !h.frees[i] {
+ i++
+ }
+ h.frees[i] = false
+ return h.hashes[i]
+}
+
+func (h *Hasher) fold() {
+ var err error
+ for l := 0; l < MaxDepth; l++ {
+ if h.frees[l*2+0] || h.frees[l*2+1] {
+ continue
+ }
+ h.nodeReset(h.nodeHash)
+ if _, err = h.nodeHash.Write(h.hashes[l*2+0]); err != nil {
+ panic(err)
+ }
+ if _, err = h.nodeHash.Write(h.hashes[l*2+1]); err != nil {
+ panic(err)
+ }
+ h.nodeHash.Sum(h.get(l + 1)[:0])
+ h.frees[l*2+0] = true
+ h.frees[l*2+1] = true
+ h.count--
+ }
+}
+
+func (h *Hasher) makePipe() {
+ h.pr, h.pw = io.Pipe()
+ go func() {
+ h.ReadFrom(h.pr)
+ h.pr.Close()
+ }()
+}
+
+func (h *Hasher) Write(p []byte) (int, error) {
+ if h.pr == nil {
+ h.makePipe()
+ }
+ return h.pw.Write(p)
+}
+
+func (h *Hasher) ReadFrom(r io.Reader) (total int64, err error) {
+ var n int
+ var eof bool
+ var chunk []byte
+ var reply chan []byte
+ for !eof {
+ chunk = <-h.freeChunks
+ n, err = io.ReadFull(r, chunk)
+ total += int64(n)
+ if err != nil {
+ if err != io.ErrUnexpectedEOF {
+ return
+ }
+ err = nil
+ eof = true
+ }
+ if n == 0 {
+ continue
+ }
+ reply = <-h.freeReplies
+ h.jobs <- job{reply: reply, chunk: chunk[:n]}
+ h.replies <- reply
+ }
+ close(h.jobs)
+ h.workers.Wait()
+ close(h.replies)
+ return
+}
+
+func (h *Hasher) worker() {
+ hasher := h.leafNew()
+ var err error
+ var hsh []byte
+ for j := range h.jobs {
+ h.leafReset(hasher)
+ if _, err = hasher.Write(j.chunk); err != nil {
+ panic(err)
+ }
+ h.freeChunks <- j.chunk
+ hsh = <-h.freeHshes
+ hasher.Sum(hsh[:0])
+ j.reply <- hsh
+ }
+ h.workers.Done()
+}
+
+func (h *Hasher) aggregator() {
+ var hsh []byte
+ for reply := range h.replies {
+ hsh = <-reply
+ h.freeReplies <- reply
+ copy(h.get(0), hsh)
+ h.count++
+ h.fold()
+ h.freeHshes <- hsh
+ }
+ close(h.finished)
+}
+
+func (h *Hasher) Sum(b []byte) []byte {
+ if h.pw != nil {
+ h.pw.Close()
+ }
+ <-h.finished
+ if h.count == 0 {
+ return append(b, h.leafNew().Sum(nil)...)
+ }
+ for l := 0; l < MaxDepth; l++ {
+ if h.count == 1 {
+ for ; l < MaxDepth; l++ {
+ if !h.frees[l*2+0] {
+ return append(b, h.hashes[l*2+0]...)
+ }
+ }
+ }
+ if !h.frees[l*2+0] {
+ copy(h.get(l+1), h.hashes[l*2+0])
+ h.frees[l*2+0] = true
+ h.fold()
+ }
+ }
+ panic("did not reach the end")
+}
--- /dev/null
+// GoKEKS/PKI -- PKI-related capabilities based on KEKS encoded formats
+// Copyright (C) 2024-2025 Sergey Matveev <stargrave@stargrave.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation, version 3 of the License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package merkle
+
+import (
+ "bytes"
+ "crypto/sha512"
+ "testing"
+)
+
+func TestEmpty(t *testing.T) {
+ h0 := sha512.Sum512(nil)
+ h1 := NewHasherPrefixed(sha512.New, 8, 2)
+ if !bytes.Equal(h1.Sum(nil), h0[:]) {
+ t.FailNow()
+ }
+}
+
+func Test1ChunkShort(t *testing.T) {
+ data := []byte("some")
+ h0 := sha512.Sum512(append([]byte{0}, data...))
+ h1 := NewHasherPrefixed(sha512.New, 8, 2)
+ if _, err := h1.Write(data); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(h1.Sum(nil), h0[:]) {
+ t.FailNow()
+ }
+}
+
+func Test1Chunk(t *testing.T) {
+ data := []byte("somethin")
+ h0 := sha512.Sum512(append([]byte{0}, data...))
+ h1 := NewHasherPrefixed(sha512.New, 8, 2)
+ if _, err := h1.Write(data); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(h1.Sum(nil), h0[:]) {
+ t.FailNow()
+ }
+}
+
+func Test2ChunkShort(t *testing.T) {
+ data := []byte("something")
+ c0 := sha512.Sum512([]byte("\x00somethin"))
+ c1 := sha512.Sum512([]byte("\x00g"))
+ c2 := append(c0[:], c1[:]...)
+ c3 := sha512.Sum512(append([]byte{0x01}, c2...))
+ h1 := NewHasherPrefixed(sha512.New, 8, 2)
+ if _, err := h1.Write(data); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(h1.Sum(nil), c3[:]) {
+ t.FailNow()
+ }
+}
+
+func Test2Chunk(t *testing.T) {
+ data := []byte("somethingsomethi")
+ c0 := sha512.Sum512([]byte("\x00somethin"))
+ c1 := sha512.Sum512([]byte("\x00gsomethi"))
+ c2 := sha512.Sum512(append([]byte{0x01}, append(c0[:], c1[:]...)...))
+ h1 := NewHasherPrefixed(sha512.New, 8, 2)
+ if _, err := h1.Write(data); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(h1.Sum(nil), c2[:]) {
+ t.FailNow()
+ }
+}
+
+func Test3Chunk(t *testing.T) {
+ data := []byte("123456")
+ c0 := sha512.Sum512([]byte("\x0012"))
+ c1 := sha512.Sum512([]byte("\x0034"))
+ c2 := sha512.Sum512([]byte("\x0056"))
+ c3 := sha512.Sum512(append([]byte{0x01}, append(c0[:], c1[:]...)...))
+ c4 := sha512.Sum512(append([]byte{0x01}, append(c3[:], c2[:]...)...))
+ h1 := NewHasherPrefixed(sha512.New, 2, 2)
+ if _, err := h1.Write(data); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(h1.Sum(nil), c4[:]) {
+ t.FailNow()
+ }
+}
+
+func Test4Chunk(t *testing.T) {
+ data := []byte("12345678")
+ c0 := sha512.Sum512([]byte("\x0012"))
+ c1 := sha512.Sum512([]byte("\x0034"))
+ c2 := sha512.Sum512([]byte("\x0056"))
+ c3 := sha512.Sum512([]byte("\x0078"))
+ c4 := sha512.Sum512(append([]byte{0x01}, append(c0[:], c1[:]...)...))
+ c5 := sha512.Sum512(append([]byte{0x01}, append(c2[:], c3[:]...)...))
+ c6 := sha512.Sum512(append([]byte{0x01}, append(c4[:], c5[:]...)...))
+ h1 := NewHasherPrefixed(sha512.New, 2, 2)
+ if _, err := h1.Write(data); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(h1.Sum(nil), c6[:]) {
+ t.FailNow()
+ }
+}
+
+// RFC 9162, 2.1.5 Example
+func TestRFC7Leaves(t *testing.T) {
+ data := []byte("d0d1d2d3d4d5d6")
+ a := sha512.Sum512([]byte("\x00d0"))
+ b := sha512.Sum512([]byte("\x00d1"))
+ c := sha512.Sum512([]byte("\x00d2"))
+ d := sha512.Sum512([]byte("\x00d3"))
+ e := sha512.Sum512([]byte("\x00d4"))
+ f := sha512.Sum512([]byte("\x00d5"))
+ g := sha512.Sum512(append([]byte{0x01}, append(a[:], b[:]...)...))
+ h := sha512.Sum512(append([]byte{0x01}, append(c[:], d[:]...)...))
+ i := sha512.Sum512(append([]byte{0x01}, append(e[:], f[:]...)...))
+ j := sha512.Sum512([]byte("\x00d6"))
+ k := sha512.Sum512(append([]byte{0x01}, append(g[:], h[:]...)...))
+ l := sha512.Sum512(append([]byte{0x01}, append(i[:], j[:]...)...))
+ root := sha512.Sum512(append([]byte{0x01}, append(k[:], l[:]...)...))
+ h1 := NewHasherPrefixed(sha512.New, 2, 2)
+ if _, err := h1.Write(data); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(h1.Sum(nil), root[:]) {
+ t.FailNow()
+ }
+}
--- /dev/null
+// GoKEKS/PKI -- PKI-related capabilities based on KEKS encoded formats
+// Copyright (C) 2024-2025 Sergey Matveev <stargrave@stargrave.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation, version 3 of the License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package merkle
+
+import "hash"
+
+func NewHasherPrefixed(h func() hash.Hash, chunkLen, workers int) *Hasher {
+ return NewHasher(
+ h,
+ h,
+ func(h hash.Hash) {
+ h.Reset()
+ if _, err := h.Write([]byte{0x00}); err != nil {
+ panic(err)
+ }
+ },
+ func(h hash.Hash) {
+ h.Reset()
+ if _, err := h.Write([]byte{0x01}); err != nil {
+ panic(err)
+ }
+ },
+ chunkLen,
+ workers,
+ )
+}
@url{https://datatracker.ietf.org/doc/html/rfc9162, RFC 9162},
except that no @code{0x00}/@code{0x01} constants are appended to
the hashed data, but BLAKE2b is initialised in keyed mode with
- either "LEAF" or "NODE" keys.
+ either "LEAF" or "NODE" keys. Although BLAKE2 has ability to set
+ tree-hashing parameters on its own, many implementations do not
+ provide necessary API for that.
@code{blake2b-merkle} algorithm identifier is used.
Streebog-512 is used in Merkle tree hashing mode, as described in
@url{https://datatracker.ietf.org/doc/html/rfc9162, RFC 9162}.
- @code{streebog512-merkle} algorithm identifier is used.
+ @code{streebog256-merkle}, @code{streebog512-merkle} algorithm
+ identifiers are used.
@node pki-hashed-xxh3-128
@subsection pki-hashed with XXH3-128