From 37519d950d24a466ea96d7a65164ceaab17a40e0 Mon Sep 17 00:00:00 2001
From: Rob Pike <r@golang.org>
Date: Thu, 12 Jul 2012 20:53:17 -0700
Subject: [PATCH] encoding/gob: reduce decoder memory Gob decoding reads a
 whole message into memory and then copies it into a bytes.Buffer. For large
 messages this wastes an entire copy of the message. In this CL, we use a
 staging buffer to avoid the large temporary.

Update #2539
RSS drops to 775MB from 1GB.
Active memory drops to 858317048 from 1027878136,
essentially the size of one copy of the input file.

R=dsymonds, nigeltao
CC=golang-dev
https://golang.org/cl/6392057
---
 src/pkg/encoding/gob/decoder.go | 37 ++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/src/pkg/encoding/gob/decoder.go b/src/pkg/encoding/gob/decoder.go
index c5c7d3fdb1..04f706ca54 100644
--- a/src/pkg/encoding/gob/decoder.go
+++ b/src/pkg/encoding/gob/decoder.go
@@ -87,21 +87,38 @@ func (dec *Decoder) recvMessage() bool {
 
 // readMessage reads the next nbytes bytes from the input.
 func (dec *Decoder) readMessage(nbytes int) {
-	// Allocate the buffer.
-	if cap(dec.tmp) < nbytes {
-		dec.tmp = make([]byte, nbytes+100) // room to grow
+	// Allocate the dec.tmp buffer, up to 10KB.
+	const maxBuf = 10 * 1024
+	nTmp := nbytes
+	if nTmp > maxBuf {
+		nTmp = maxBuf
 	}
-	dec.tmp = dec.tmp[:nbytes]
+	if cap(dec.tmp) < nTmp {
+		nAlloc := nTmp + 100 // A little extra for growth.
+		if nAlloc > maxBuf {
+			nAlloc = maxBuf
+		}
+		dec.tmp = make([]byte, nAlloc)
+	}
+	dec.tmp = dec.tmp[:nTmp]
 
 	// Read the data
-	_, dec.err = io.ReadFull(dec.r, dec.tmp)
-	if dec.err != nil {
-		if dec.err == io.EOF {
-			dec.err = io.ErrUnexpectedEOF
+	dec.buf.Grow(nbytes)
+	for nbytes > 0 {
+		if nbytes < nTmp {
+			dec.tmp = dec.tmp[:nbytes]
 		}
-		return
+		var nRead int
+		nRead, dec.err = io.ReadFull(dec.r, dec.tmp)
+		if dec.err != nil {
+			if dec.err == io.EOF {
+				dec.err = io.ErrUnexpectedEOF
+			}
+			return
+		}
+		dec.buf.Write(dec.tmp)
+		nbytes -= nRead
 	}
-	dec.buf.Write(dec.tmp)
 }
 
 // toInt turns an encoded uint64 into an int, according to the marshaling rules.
-- 
2.48.1