From: Sergey Matveev Date: Tue, 14 Jan 2025 12:03:44 +0000 (+0300) Subject: textdump-tester X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=fc405f429d89f8e49b4c0000d976d062440972dcd61d7607f3e082abe76db2b4;p=keks.git textdump-tester --- diff --git a/go/cmd/textdump-tester/main.go b/go/cmd/textdump-tester/main.go new file mode 100644 index 0000000..717738d --- /dev/null +++ b/go/cmd/textdump-tester/main.go @@ -0,0 +1,281 @@ +// GoKEKS -- Go KEKS codec implementation +// Copyright (C) 2024-2025 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this program. If not, see . + +// Look for py3/tests/textdump-tester tools for more description. + +package main + +import ( + "bufio" + "bytes" + "encoding/hex" + "io" + "log" + "math/big" + "os" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "go.cypherpunks.su/keks" + "go.cypherpunks.su/tai64n/v4" +) + +var Scanner *bufio.Scanner + +func getFields() []string { + if !Scanner.Scan() { + if err := Scanner.Err(); err != nil { + log.Fatal(err) + } + return []string{""} + } + t := Scanner.Text() + if t == "" { + return []string{""} + } + return strings.Fields(t) +} + +func mustDecodeHex(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + log.Fatal(err) + } + return b +} + +func checker(v any) { + fields := getFields() + switch f := fields[0]; f { + case "NIL": + if v != nil { + log.Fatalf("expected NIL, got %+v\n", v) + } + case "FALSE": + our, ok := v.(bool) + if !ok { + log.Fatalf("expected bool, got %+v\n", v) + } + if our { + log.Fatalf("expected FALSE, got %+v\n", our) + } + case "TRUE": + our, ok := v.(bool) + if !ok { + log.Fatalf("expected bool, got %+v\n", v) + } + if !our { + log.Fatalf("expected TRUE, got %+v\n", our) + } + case "UUID": + our, ok := v.(uuid.UUID) + if !ok { + log.Fatalf("expected UUID, got %+v\n", v) + } + their, err := uuid.Parse(fields[1]) + if err != nil { + log.Fatal(err) + } + if our != their { + log.Fatalln("UUID differs:", our, their) + } + case "UTC": + our, ok := v.(time.Time) + if !ok { + log.Fatalf("expected Time, got %+v\n", v) + } + var their time.Time + var err error + if our.Nanosecond() > 0 { + their, err = time.Parse(time.RFC3339Nano, fields[1]) + } else { + their, err = time.Parse(time.RFC3339, fields[1]) + } + if err != nil { + log.Fatal(err) + } + if !their.Equal(our) { + log.Fatalln("UTC differs:", our, their) + } + case "TAI64NA": + our, ok := v.(*tai64n.TAI64NA) + if !ok { + log.Fatalf("expected TAI64NA, got %+v\n", v) + } + their := tai64n.TAI64NA(mustDecodeHex(fields[1])) + if *our != their { + log.Fatalln("TAI64NA differs:", our, their) + } + case "BIN": + our, ok := v.([]byte) + if !ok { + log.Fatalf("expected []byte, got %+v\n", v) + } + var their []byte + if len(fields) > 1 { + their = mustDecodeHex(fields[1]) + } + if !bytes.Equal(our, their) { + log.Fatalln("BIN differs:", our, their) + } + case "STR": + our, ok := v.(string) + if !ok { + log.Fatalf("expected string, got %+v\n", v) + } + var their string + if len(fields) > 1 { + their = string(mustDecodeHex(fields[1])) + } + if our != their { + log.Fatalln("STR differs:", our, their) + } + case "INT": + their, ok := new(big.Int).SetString(fields[1], 10) + if !ok { + log.Fatal("can not parse INT") + } + if their.Sign() >= 0 { + if their.BitLen() > 64 { + goto BigIntCheck + } + var our uint64 + our, ok = v.(uint64) + if !ok { + log.Fatalf("expected uint64, got %+v\n", v) + } + if our != their.Uint64() { + log.Fatalln("INT differs:", our, their.Uint64()) + } + } else { + if their.BitLen() > 63 { + goto BigIntCheck + } + var our int64 + our, ok = v.(int64) + if !ok { + log.Fatalf("expected int64, got %+v\n", v) + } + if our != their.Int64() { + log.Fatalln("INT differs:", our, their.Int64()) + } + } + break + BigIntCheck: + our, ok := v.(*big.Int) + if !ok { + log.Fatalf("expected big.Int, got %+v\n", v) + } + if our.Cmp(their) != 0 { + log.Fatalln("INT differs:", our, their) + } + case "BLOB": + blob, ok := v.(keks.BlobChunked) + if !ok { + log.Fatalf("expected BlobChunked, got %+v\n", v) + } + chunkLen, err := strconv.Atoi(fields[1]) + if err != nil { + log.Fatal(err) + } + if blob.ChunkLen != int64(chunkLen) { + log.Fatalln("chunkLen differs:", blob.ChunkLen, chunkLen) + } + var their []byte + if len(fields) > 2 { + their = mustDecodeHex(fields[2]) + } + our, err := io.ReadAll(blob.Reader()) + if err != nil { + log.Fatal(err) + } + if !bytes.Equal(our, their) { + log.Fatalln("BLOB differs:", our, their) + } + case "LIST": + our, ok := v.([]any) + if !ok { + log.Fatalf("expected []any, got %+v\n", v) + } + their, err := strconv.Atoi(fields[1]) + if err != nil { + log.Fatal(err) + } + if len(our) != their { + log.Fatalln("LIST len differs:", our, their) + } + for _, item := range our { + checker(item) + } + case "MAP": + our, ok := v.(map[string]any) + if !ok { + log.Fatalf("expected map[string]any, got %+v\n", v) + } + their, err := strconv.Atoi(fields[1]) + if err != nil { + log.Fatal(err) + } + if len(our) != their { + log.Fatalln("MAP len differs:", our, their) + } + for i := 0; i < their; i++ { + fields = getFields() + k := string(mustDecodeHex(fields[1])) + item, ok := our[k] + if !ok { + log.Fatalln("no key found:", k) + } + checker(item) + } + default: + log.Fatalln("unknown:", f) + } +} + +func main() { + log.SetFlags(log.Lshortfile) + Scanner = bufio.NewScanner(os.Stdin) + var fields []string + for { + fields = getFields() + if fields[0] != "KEKS" { + log.Fatalln("KEKS was expected, got", fields[0]) + } + their := mustDecodeHex(fields[1]) + decoder := keks.NewDecoderFromBytes(their, nil) + v, err := decoder.Decode() + if err != nil { + log.Fatal(err) + } + log.Println("parsed:", decoder.Read, "bytes") + { + our, err := keks.EncodeBuf(v, nil) + if err != nil { + log.Fatal(err) + } + if !bytes.Equal(our, their) { + log.Fatal("encoded version differs") + } + } + checker(v) + fields = getFields() + if fields[0] != "EOC" { + log.Fatalln("EOC was expected, got", fields[0]) + } + } +} diff --git a/py3/tests/textdump-tester b/py3/tests/textdump-tester new file mode 100755 index 0000000..84282bc --- /dev/null +++ b/py3/tests/textdump-tester @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# hypothesis library is convenient tool for generation of various complex +# data structures. That is tedious work to do in strongly typed Go. So +# let's generate KEKS structures in Python, feed them to Go's +# implementation and supply them with additional data describing what is +# exactly expected to be decoded. +# +# Simple text-based protocol is made for that task. You run data +# generator and feed its output to go/cmd/textdump-tester's stdin. Text +# protocol is a flow of ASCII lines, containing space-separated fields: +# * KEKS HEX(complex-data) -- provides the data for decoding and verifying +# * NIL -- NIL is expected +# * FALSE -- FALSE is expected +# * TRUE -- TRUE is expected +# * UUID xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -- UUID is expected +# * UTC xxxx-xx-xxTxx:xx:xxZ -- TAI64-encoded UTC is expected +# * UTC xxxx-xx-xxTxx:xx:xx.xxxxxxZ -- TAI64N-encoded UTC is expected +# * TAI64NA HEX(...) -- external TAI64NA-encoded time +# * BIN HEX(...) -- BIN is expected +# * STR HEX(...) -- STR is expected +# * INT DEC(...) -- ±INT is expected +# * BLOB DEC(chunkLen) HEX(content) -- BLOB is expected +# * LIST DEC(len) -- LEN is expected with exact number of elements. +# Their similar text descriptions follow +# * MAP DEC(len) -- MAP is expected with exact number of elements. +# It is followed by pairs of STR(key) and element's value +# * EOC -- nothing more expected, end of contents + +from datetime import datetime +from uuid import UUID + +from keks import Blob +from keks import Raw +from keks import TagTAI64NA + + +def textdump(v): + if v is None: + print("NIL") + elif v is False: + print("FALSE") + elif v is True: + print("TRUE") + elif isinstance(v, UUID): + print("UUID " + str(v)) + elif isinstance(v, float): + raise NotImplementedError("no FLOAT* support") + elif isinstance(v, datetime): + print("UTC %sZ" % v.isoformat()) + elif isinstance(v, bytes): + print("BIN " + v.hex()) + elif isinstance(v, str): + print("STR " + v.encode("utf-8").hex()) + elif isinstance(v, int): + print("INT %d" % v) + elif isinstance(v, Blob): + print("BLOB %d %s" % (v.l, v.v.hex())) + elif isinstance(v, (list, tuple)): + print("LIST %d" % len(v)) + for i in v: + textdump(i) + elif isinstance(v, set): + textdump({i: None for i in v}) + elif isinstance(v, dict): + print("MAP %d" % len(v)) + for k, v in v.items(): + textdump(k) + textdump(v) + elif isinstance(v, Raw): + v = bytes(v) + if v[0] != TagTAI64NA: + raise NotImplementedError("unsupported Raw type") + print("TAI64NA " + v[1:].hex()) + else: + raise NotImplementedError("unsupported type") + + +if __name__ == "__main__": + from hypothesis.strategies import dictionaries + from keks import dumps + from tests.strategies import everything_st + from tests.strategies import mapkey_st + st = dictionaries(mapkey_st, everything_st, max_size=4) + while True: + ex = st.example() + print("KEKS", dumps(ex).hex()) + textdump(ex) + print("EOC")