From 7c9e2c2b6c2e0aa3090dbd5183809e1b2f53359b Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 12 Sep 2008 16:42:53 -0700 Subject: [PATCH] buffered input & output R=r DELTA=812 (803 added, 0 deleted, 9 changed) OCL=15225 CL=15280 --- src/lib/bufio.go | 402 ++++++++++++++++++++++++++++++++++++++++++ src/lib/io.go | 36 ++++ src/lib/make.bash | 2 +- src/lib/os/os_file.go | 16 +- test/bufiolib.go | 377 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 824 insertions(+), 9 deletions(-) create mode 100644 src/lib/bufio.go create mode 100644 src/lib/io.go create mode 100644 test/bufiolib.go diff --git a/src/lib/bufio.go b/src/lib/bufio.go new file mode 100644 index 0000000000..323e39ff66 --- /dev/null +++ b/src/lib/bufio.go @@ -0,0 +1,402 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bufio +import "os" +import "io" + + +// TODO: +// - maybe define an interface +// - BufRead: ReadRune, UnreadRune ? +// could make ReadRune generic if we dropped UnreadRune +// - buffered output + +const ( + DefaultBufSize = 4096 +) + +func NewError(s string) *os.Error { + // BUG return &os.Error{s}; + e := new(os.Error); + e.s = s; + return e +} + +export var ( + EndOfFile = NewError("end of file"); + PhaseError = NewError("phase error"); + BufferFull = NewError("buffer full"); + InternalError = NewError("bufio internal error"); + BadBufSize = NewError("bad bufio size"); + ShortWrite = NewError("short write"); +) + +func CopySlice(dst *[]byte, src *[]byte) { + for i := 0; i < len(dst); i++ { + dst[i] = src[i] + } +} + + +// Buffered input. + +export type BufRead struct { + err *os.Error; + buf *[]byte; + r, w int; + rd io.Read; +} + +export func NewBufReadSize(rd io.Read, size int) (b *BufRead, err *os.Error) { + if size <= 0 { + return nil, BadBufSize + } + b = new(BufRead); + b.buf = new([]byte, size); + b.rd = rd + return b, nil +} + +export func NewBufRead(rd io.Read) (b *BufRead, err *os.Error) { + // 6g BUG return NewBufReadSize(rd, DefaultBufSize) + r, e := NewBufReadSize(rd, DefaultBufSize) + return r, e +} + +// Read a new chunk into the buffer. +func (b *BufRead) Fill() *os.Error { + if b.err != nil { + return b.err + } + + // Slide existing data to beginning. + if b.w > b.r { + CopySlice(b.buf[0:b.w-b.r], b.buf[b.r:b.w]); + b.w -= b.r; + } else { + b.w = 0 + } + b.r = 0; + + // Read new data. + n, e := b.rd.Read(b.buf[b.w:len(b.buf)]) + if e != nil { + b.err = e + return e + } + b.w += n + return nil +} + +// Read into p. +// Returns the number of bytes read into p. +// If nn < len(p), also returns an error explaining +// why the read is short. +func (b *BufRead) Read(p *[]byte) (nn int, err *os.Error) { + nn = 0 + for len(p) > 0 { + n := len(p) + if b.w == b.r { + b.Fill() + if b.err != nil { + return nn, b.err + } + if b.w == b.r { + return nn, EndOfFile + } + } + if n > b.w - b.r { + n = b.w - b.r + } + CopySlice(p[0:n], b.buf[b.r:b.r+n]); + p = p[n:len(p)]; + b.r += n; + nn += n + } + return nn, nil +} + +// Read a single byte. +// If no byte available, returns error. +func (b *BufRead) ReadByte() (c byte, err *os.Error) { + if b.w == b.r { + b.Fill() + if b.err != nil { + return 0, b.err + } + if b.w == b.r { + return 0, EndOfFile + } + } + c = b.buf[b.r]; + b.r++ + return c, nil +} + +// Unread the last byte. Only guaranteed to be able to unread one byte. +func (b *BufRead) UnreadByte() *os.Error { + if b.err != nil { + return b.err + } + if b.r <= 0 { + return PhaseError + } + b.r-- + return nil +} + +// Helper function: look for byte c in array p, +// returning its index or -1. +func FindByte(p *[]byte, c byte) int { + for i := 0; i < len(p); i++ { + if p[i] == c { + return i + } + } + return -1 +} + +// Returns the number of bytes that can be read. +func (b *BufRead) Buffered() int { + return b.w - b.r; +} + +// Read until the first occurrence of delim in the input, +// returning a slice pointing at the bytes in the buffer. +// The bytes stop being valid at the next read call. +// Fails if the line doesn't fit in the buffer. +// For internal (or advanced) use only. +// Use ReadLineString or ReadLineBytes instead. +func (b *BufRead) ReadLineSlice(delim byte) (line *[]byte, err *os.Error) { + if b.err != nil { + return nil, b.err + } + + // Look in buffer. + if i := FindByte(b.buf[b.r:b.w], delim); i >= 0 { + line1 := b.buf[b.r:b.r+i+1]; + b.r += i+1; + return line1, nil + } + + // Read more into buffer, until buffer fills or we find delim. + for { + n := b.Buffered(); + b.Fill(); + if b.err != nil { + return nil, b.err + } + if b.Buffered() == n { // no data added; end of file + return nil, EndOfFile + } + + // Search new part of buffer + if i := FindByte(b.buf[n:b.w], delim); i >= 0 { + line := b.buf[0:n+i+1]; + b.r = n+i+1 + return line, nil + } + + // Buffer is full? + if b.Buffered() >= len(b.buf) { + return nil, BufferFull + } + } + + // BUG 6g bug100 + return nil, nil +} + +// Read until the first occurrence of delim in the input, +// returning a new byte array containing the line. +// If an error happens, returns the data (without a delimiter) +// and the error. (Can't leave the data in the buffer because +// we might have read more than the buffer size.) +func (b *BufRead) ReadLineBytes(delim byte) (line *[]byte, err *os.Error) { + if b.err != nil { + return nil, b.err + } + + // Use ReadLineSlice to look for array, + // accumulating full buffers. + var frag *[]byte; + var full *[]*[]byte; + nfull := 0; + err = nil; + + for { + var e *os.Error; + frag, e = b.ReadLineSlice(delim); + if e == nil { // got final fragment + break + } + if e != BufferFull { // unexpected error + err = e; + break + } + + // Read bytes out of buffer. + buf := new([]byte, b.Buffered()); + var n int; + n, e = b.Read(buf); + if e != nil { + frag = buf[0:n]; + err = e + break + } + if n != len(buf) { + frag = buf[0:n]; + err = InternalError + break + } + + // Grow list if needed. + if full == nil { + full = new([]*[]byte, 16); + } else if nfull >= len(full) { + newfull := new([]*[]byte, len(full)*2); + // BUG slice assignment + for i := 0; i < len(full); i++ { + newfull[i] = full[i]; + } + full = newfull + } + + // Save buffer + full[nfull] = buf; + nfull++ + } + + // Allocate new buffer to hold the full pieces and the fragment. + n := 0 + for i := 0; i < nfull; i++ { + n += len(full[i]) + } + if frag != nil { + n += len(frag); + } + + // Copy full pieces and fragment in. + buf := new([]byte, n); + n = 0 + for i := 0; i < nfull; i++ { + CopySlice(buf[n:n+len(full[i])], full[i]); + n += len(full[i]) + } + if frag != nil { + CopySlice(buf[n:n+len(frag)], frag) + } + return buf, err +} + +// Read until the first occurrence of delim in the input, +// returning a new string containing the line. +// If savedelim, keep delim in the result; otherwise chop it off. +func (b *BufRead) ReadLineString(delim byte, savedelim bool) (line string, err *os.Error) { + bytes, e := b.ReadLineBytes(delim) + if e != nil { + return string(bytes), e + } + if !savedelim { + bytes = bytes[0:len(bytes)-1] + } + return string(bytes), nil +} + + +// buffered output + +export type BufWrite struct { + err *os.Error; + buf *[]byte; + n int; + wr io.Write; +} + +export func NewBufWriteSize(wr io.Write, size int) (b *BufWrite, err *os.Error) { + if size <= 0 { + return nil, BadBufSize + } + b = new(BufWrite); + b.buf = new([]byte, size); + b.wr = wr + return b, nil +} + +export func NewBufWrite(wr io.Write) (b *BufWrite, err *os.Error) { + // 6g BUG return NewBufWriteSize(wr, DefaultBufSize) + r, e := NewBufWriteSize(wr, DefaultBufSize) + return r, e +} + +// Flush the output buffer. +func (b *BufWrite) Flush() *os.Error { + if b.err != nil { + return b.err + } + n := 0 + for n < b.n { + m, e := b.wr.Write(b.buf[n:b.n]); + n += m + if m == 0 && e == nil { + e = ShortWrite + } + if e != nil { + if n < b.n { + CopySlice(b.buf[0:b.n-n], b.buf[n:b.n]) + } + b.n -= n; + b.err = e + return e + } + } + b.n = 0 + return nil +} + +func (b *BufWrite) Available() int { + return len(b.buf) - b.n +} + +func (b *BufWrite) Buffered() int { + return b.n +} + +func (b *BufWrite) Write(p *[]byte) (nn int, err *os.Error) { + if b.err != nil { + return 0, b.err + } + nn = 0 + for len(p) > 0 { + n := b.Available() + if n <= 0 { + if b.Flush(); b.err != nil { + break + } + n = b.Available() + } + if n > len(p) { + n = len(p) + } + CopySlice(b.buf[b.n:b.n+n], p[0:n]); + b.n += n; + nn += n; + p = p[n:len(p)] + } + return nn, b.err +} + +func (b *BufWrite) WriteByte(c byte) *os.Error { + if b.err != nil { + return b.err + } + if b.Available() <= 0 && b.Flush() != nil { + return b.err + } + b.buf[b.n] = c; + b.n++ + return nil +} + diff --git a/src/lib/io.go b/src/lib/io.go new file mode 100644 index 0000000000..d7770ebe7d --- /dev/null +++ b/src/lib/io.go @@ -0,0 +1,36 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package io +import os "os" + +export func StringToBytes(b *[]byte, s string) bool { + if len(s) >= len(b) { + return false + } + for i := 0; i < len(s); i++ { + b[i] = s[i] + } + b[len(s)] = '\000'; // not necessary - memory is zeroed - but be explicit + return true +} + +export type Read interface { + Read(p *[]byte) (n int, err *os.Error); +} + +export type Write interface { + Write(p *[]byte) (n int, err *os.Error); +} + +export func WriteString(w Write, s string) (n int, err *os.Error) { + b := new([]byte, len(s)+1) + if !StringToBytes(b, s) { + return -1, os.EINVAL + } + // BUG return w.Write(b[0:len(s)]) + r, e := w.Write(b[0:len(s)]) + return r, e +} + diff --git a/src/lib/make.bash b/src/lib/make.bash index cd5c0cd8d3..3786f7fa4f 100755 --- a/src/lib/make.bash +++ b/src/lib/make.bash @@ -15,7 +15,7 @@ do done rm -f *.6 -for i in fmt.go flag.go container/vector.go rand.go sort.go strings.go +for i in fmt.go flag.go container/vector.go rand.go sort.go io.go bufio.go strings.go do base=$(basename $i .go) echo 6g -o $GOROOT/pkg/$base.6 $i diff --git a/src/lib/os/os_file.go b/src/lib/os/os_file.go index 0867b60853..c6c641bb55 100644 --- a/src/lib/os/os_file.go +++ b/src/lib/os/os_file.go @@ -41,8 +41,8 @@ export const ( O_TRUNC = syscall.O_TRUNC; ) -export func Open(name string, mode int64, flags int64) (fd *FD, err *Error) { - r, e := syscall.open(name, mode, flags); +export func Open(name string, mode int, flags int) (fd *FD, err *Error) { + r, e := syscall.open(name, int64(mode), int64(flags)); return NewFD(r), ErrnoToError(e) } @@ -55,23 +55,23 @@ func (fd *FD) Close() *Error { return ErrnoToError(e) } -func (fd *FD) Read(b *[]byte) (ret int64, err *Error) { +func (fd *FD) Read(b *[]byte) (ret int, err *Error) { if fd == nil { return -1, EINVAL } r, e := syscall.read(fd.fd, &b[0], int64(len(b))); - return r, ErrnoToError(e) + return int(r), ErrnoToError(e) } -func (fd *FD) Write(b *[]byte) (ret int64, err *Error) { +func (fd *FD) Write(b *[]byte) (ret int, err *Error) { if fd == nil { return -1, EINVAL } r, e := syscall.write(fd.fd, &b[0], int64(len(b))); - return r, ErrnoToError(e) + return int(r), ErrnoToError(e) } -func (fd *FD) WriteString(s string) (ret int64, err *Error) { +func (fd *FD) WriteString(s string) (ret int, err *Error) { if fd == nil { return -1, EINVAL } @@ -80,5 +80,5 @@ func (fd *FD) WriteString(s string) (ret int64, err *Error) { return -1, EINVAL } r, e := syscall.write(fd.fd, &b[0], int64(len(s))); - return r, ErrnoToError(e) + return int(r), ErrnoToError(e) } diff --git a/test/bufiolib.go b/test/bufiolib.go new file mode 100644 index 0000000000..b5d7453c1b --- /dev/null +++ b/test/bufiolib.go @@ -0,0 +1,377 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// $G $F.go && $L $F.$A && ./$A.out + +package main + +import ( + "os"; + "io"; + "bufio"; + "syscall"; + "rand" +) + +func StringToBytes(s string) *[]byte { + b := new([]byte, len(s)); + for i := 0; i < len(s); i++ { + b[i] = s[i] + } + return b +} + +// Should be in language! +func Copy(p *[]byte, q *[]byte) { + for i := 0; i < len(p); i++ { + p[i] = q[i] + } +} + +// Reads from p. +type ByteReader struct { + p *[]byte +} + +func NewByteReader(p *[]byte) io.Read { + b := new(ByteReader); + b.p = p + return b +} + +func (b *ByteReader) Read(p *[]byte) (int, *os.Error) { + n := len(p) + if n > len(b.p) { + n = len(b.p) + } + Copy(p[0:n], b.p[0:n]); + b.p = b.p[n:len(b.p)] + return n, nil +} + + +// Reads from p but only returns half of what you asked for. +type HalfByteReader struct { + p *[]byte +} + +func NewHalfByteReader(p *[]byte) io.Read { + b := new(HalfByteReader); + b.p = p + return b +} + +func (b *HalfByteReader) Read(p *[]byte) (int, *os.Error) { + n := len(p)/2 + if n == 0 && len(p) > 0 { + n = 1 + } + if n > len(b.p) { + n = len(b.p) + } + Copy(p[0:n], b.p[0:n]); + b.p = b.p[n:len(b.p)] + return n, nil +} + +// Reads from a reader and rot13s the result. +type Rot13Reader struct { + r io.Read +} + +func NewRot13Reader(r io.Read) *Rot13Reader { + r13 := new(Rot13Reader); + r13.r = r + return r13 +} + +func (r13 *Rot13Reader) Read(p *[]byte) (int, *os.Error) { + n, e := r13.r.Read(p) + if e != nil { + return n, e + } + for i := 0; i < n; i++ { + if 'a' <= p[i] && p[i] <= 'z' || 'A' <= p[i] && p[i] <= 'Z' { + if 'a' <= p[i] && p[i] <= 'm' || 'A' <= p[i] && p[i] <= 'M' { + p[i] += 13; + } else { + p[i] -= 13; + } + } + } + return n, nil +} + +func MakeByteReader(p *[]byte) io.Read { + return NewByteReader(p) +} +func MakeHalfByteReader(p *[]byte) io.Read { + return NewHalfByteReader(p) +} + +var readmakers = []*(p *[]byte) io.Read { + &NewByteReader, + &NewHalfByteReader +} + + +// Call ReadLineString (which ends up calling everything else) +// to accumulate the text of a file. +func ReadLines(b *bufio.BufRead) string { + s := "" + for { + s1, e := b.ReadLineString('\n', false) + if e == bufio.EndOfFile { + break + } + if e != nil { + panic("GetLines: "+e.String()) + } + s += s1 + } + return s +} + +// Call ReadByte to accumulate the text of a file +func ReadBytes(buf *bufio.BufRead) string { + var b [1000]byte; + nb := 0 + for { + c, e := buf.ReadByte() + if e == bufio.EndOfFile { + break + } + if e != nil { + panic("GetBytes: "+e.String()) + } + b[nb] = c; + nb++ + } + // BUG return string(b[0:nb]) ? + return string(b)[0:nb] +} + +// Call Read to accumulate the text of a file +func Reads(buf *bufio.BufRead, m int) string { + var b [1000]byte; + nb := 0 + for { + // BUG parens around (&b) should not be needed + n, e := buf.Read((&b)[nb:nb+m]); + nb += n + if e == bufio.EndOfFile { + break + } + } + // BUG 6g bug102 - out of bounds error on empty byte array -> string + if nb == 0 { return "" } + return string((&b)[0:nb]) +} + +func Read1(b *bufio.BufRead) string { return Reads(b, 1) } +func Read2(b *bufio.BufRead) string { return Reads(b, 2) } +func Read3(b *bufio.BufRead) string { return Reads(b, 3) } +func Read4(b *bufio.BufRead) string { return Reads(b, 4) } +func Read5(b *bufio.BufRead) string { return Reads(b, 5) } +func Read7(b *bufio.BufRead) string { return Reads(b, 7) } + +var bufreaders = []*(b *bufio.BufRead) string { + &Read1, &Read2, &Read3, &Read4, &Read5, &Read7, + &ReadBytes, &ReadLines +} + +var bufsizes = []int { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 23, 32, 46, 64, 93, 128, 1024, 4096 +} + +func TestBufRead() { + // work around 6g bug101 + readmakers[0] = &NewByteReader; + readmakers[1] = &NewHalfByteReader; + + bufreaders[0] = &Read1; + bufreaders[1] = &Read2; + bufreaders[2] = &Read3; + bufreaders[3] = &Read4; + bufreaders[4] = &Read5; + bufreaders[5] = &Read7; + bufreaders[6] = &ReadBytes; + bufreaders[7] = &ReadLines; + + bufsizes[0] = 1; + bufsizes[1] = 2; + bufsizes[2] = 3; + bufsizes[3] = 4; + bufsizes[4] = 5; + bufsizes[5] = 6; + bufsizes[6] = 7; + bufsizes[7] = 8; + bufsizes[8] = 9; + bufsizes[9] = 10; + bufsizes[10] = 23; + bufsizes[11] = 32; + bufsizes[12] = 46; + bufsizes[13] = 64; + bufsizes[14] = 93; + bufsizes[15] = 128; + bufsizes[16] = 1024; + bufsizes[17] = 4096; + + var texts [31]string + str := ""; + all := "" + for i := 0; i < len(texts)-1; i++ { + texts[i] = str + "\n"; + all += texts[i]; + str += string(i%26+'a') + } + texts[len(texts)-1] = all; + + // BUG 6g should not need nbr temporary (bug099) + nbr := NewByteReader(StringToBytes("hello world")); + b, e := bufio.NewBufRead(nbr); + if ReadBytes(b) != "hello world" { panic("hello world") } + + // BUG 6g should not need nbr nor nbr1 (bug009) + nbr = NewByteReader(StringToBytes("hello world")); + nbr1 := NewRot13Reader(nbr); + b, e = bufio.NewBufRead(nbr1); + if ReadBytes(b) != "uryyb jbeyq" { panic("hello world") } + + for h := 0; h < len(texts); h++ { + text := texts[h]; + textbytes := StringToBytes(text) + for i := 0; i < len(readmakers); i++ { + readmaker := readmakers[i] + for j := 0; j < len(bufreaders); j++ { + bufreader := bufreaders[j] + for k := 0; k < len(bufsizes); k++ { + bufsize := bufsizes[k]; + read := readmaker(textbytes); + buf, e := bufio.NewBufReadSize(read, bufsize); + s := bufreader(buf); + if s != text { + print("Failed: ", h, " ", i, " ", j, " ", k, " ", len(s), " ", len(text), "\n"); + print("<", s, ">\nshould be <", text, ">\n"); + panic("bufio result") + } + } + } + } + } +} + + +type WriteBuffer interface { + Write(p *[]byte) (int, *os.Error); + GetBytes() *[]byte +} + +// Accumulates bytes into a byte array. +type ByteWriter struct { + p *[]byte; + n int +} + +func NewByteWriter() WriteBuffer { + return new(ByteWriter) +} + +func (w *ByteWriter) Write(p *[]byte) (int, *os.Error) { + if w.p == nil { + w.p = new([]byte, len(p)+100) + } else if w.n + len(p) >= len(w.p) { + newp := new([]byte, len(w.p)*2 + len(p)); + Copy(newp[0:w.n], w.p[0:w.n]); + w.p = newp + } + Copy(w.p[w.n:w.n+len(p)], p); + w.n += len(p) + return len(p), nil +} + +func (w *ByteWriter) GetBytes() *[]byte { + return w.p[0:w.n] +} + +// Accumulates bytes written into a byte array +// but Write only takes half of what you give it. +type HalfByteWriter struct { + bw WriteBuffer +} + +func NewHalfByteWriter() WriteBuffer { + w := new(HalfByteWriter); + w.bw = NewByteWriter() + return w +} + +func (w *HalfByteWriter) Write(p *[]byte) (int, *os.Error) { + n := (len(p)+1) / 2; + // BUG return w.bw.Write(p[0:n]) + r, e := w.bw.Write(p[0:n]) + return r, e +} + +func (w *HalfByteWriter) GetBytes() *[]byte { + return w.bw.GetBytes() +} + +func TestBufWrite() { + var data [8192]byte + + var writers [2]*()WriteBuffer; + writers[0] = &NewByteWriter; + writers[1] = &NewHalfByteWriter; + + for i := 0; i < len(data); i++ { + data[i] = byte(rand.rand()) + } + for i := 0; i < len(bufsizes); i++ { + for j := 0; j < len(bufsizes); j++ { + for k := 0; k < len(writers); k++ { + nwrite := bufsizes[i]; + bs := bufsizes[j]; + + // Write nwrite bytes using buffer size bs. + // Check that the right amount makes it out + // and that the data is correct. + + write := writers[k](); + buf, e := bufio.NewBufWriteSize(write, bs); + if e != nil { + panic("NewBufWriteSize error: "+e.String()) + } + n, e1 := buf.Write((&data)[0:nwrite]) + if e1 != nil { + panic("buf.Write error "+e1.String()) + } + if n != nwrite { + panic("buf.Write wrong count") + } + e = buf.Flush() + if e != nil { + panic("buf.Flush error "+e.String()) + } + + written := write.GetBytes(); + if len(written) != nwrite { + panic("wrong amount written") + } + for l := 0; l < len(written); l++ { + if written[i] != data[i] { + panic("wrong bytes written") + } + } + } + } + } +} + + +func main() { + TestBufRead(); + TestBufWrite() +} -- 2.48.1