bytes: speed up Fields and FieldsFunc

author Martin Möhrmann <moehrmann@google.com>

Sat, 12 Aug 2017 08:20:30 +0000 (10:20 +0200)

committer Martin Möhrmann <moehrmann@google.com>

Mon, 14 Aug 2017 21:08:51 +0000 (21:08 +0000)
author Martin Möhrmann <moehrmann@google.com>
Sat, 12 Aug 2017 08:20:30 +0000 (10:20 +0200)
committer Martin Möhrmann <moehrmann@google.com>
Mon, 14 Aug 2017 21:08:51 +0000 (21:08 +0000)
diff --git a/src/bytes/bytes.go b/src/bytes/bytes.go

index 7c878af688cc71462ab4e8bcbfa7cdaf6256f7b2..457e1494104833dd8cb5ad64dc6b225c97480e26 100644 (file)
--- a/src/bytes/bytes.go
+++ b/src/bytes/bytes.go
@@ -265,9 +265,57 @@ func SplitAfter(s, sep []byte) [][]byte {
         return genSplit(s, sep, len(sep), -1)
  }
  
+var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1}
+
  // Fields splits the slice s around each instance of one or more consecutive white space
-// characters, returning a slice of subslices of s or an empty list if s contains only white space.
+// characters, as defined by unicode.IsSpace, returning a slice of subslices of s or an
+// empty slice if s contains only white space.
  func Fields(s []byte) [][]byte {
+       // First count the fields.
+       // This is an exact count if s is ASCII, otherwise it is an approximation.
+       n := 0
+       wasSpace := 1
+       // setBits is used to track which bits are set in the bytes of s.
+       setBits := uint8(0)
+       for i := 0; i < len(s); i++ {
+               r := s[i]
+               setBits |= r
+               isSpace := int(asciiSpace[r])
+               n += wasSpace & ^isSpace
+               wasSpace = isSpace
+       }
+
+       if setBits < utf8.RuneSelf { // ASCII fast path
+               a := make([][]byte, n)
+               na := 0
+               fieldStart := 0
+               i := 0
+               // Skip spaces in the front of the input.
+               for i < len(s) && asciiSpace[s[i]] != 0 {
+                       i++
+               }
+               fieldStart = i
+               for i < len(s) {
+                       if asciiSpace[s[i]] == 0 {
+                               i++
+                               continue
+                       }
+                       a[na] = s[fieldStart:i]
+                       na++
+                       i++
+                       // Skip spaces in between fields.
+                       for i < len(s) && asciiSpace[s[i]] != 0 {
+                               i++
+                       }
+                       fieldStart = i
+               }
+               if fieldStart < len(s) { // Last field might end at EOF.
+                       a[na] = s[fieldStart:]
+               }
+               return a
+       }
+
+       // Some runes in the input slice are not ASCII.
         return FieldsFunc(s, unicode.IsSpace)
  }
  
@@ -278,39 +326,49 @@ func Fields(s []byte) [][]byte {
  // FieldsFunc makes no guarantees about the order in which it calls f(c).
  // If f does not return consistent results for a given c, FieldsFunc may crash.
  func FieldsFunc(s []byte, f func(rune) bool) [][]byte {
-       n := 0
-       inField := false
-       for i := 0; i < len(s); {
-               r, size := utf8.DecodeRune(s[i:])
-               wasInField := inField
-               inField = !f(r)
-               if inField && !wasInField {
-                       n++
-               }
-               i += size
+       // A span is used to record a slice of s of the form s[start:end].
+       // The start index is inclusive and the end index is exclusive.
+       type span struct {
+               start int
+               end   int
         }
+       spans := make([]span, 0, 32)
  
-       a := make([][]byte, n)
-       na := 0
-       fieldStart := -1
-       for i := 0; i <= len(s) && na < n; {
-               r, size := utf8.DecodeRune(s[i:])
-               if fieldStart < 0 && size > 0 && !f(r) {
-                       fieldStart = i
-                       i += size
-                       continue
-               }
-               if fieldStart >= 0 && (size == 0 || f(r)) {
-                       a[na] = s[fieldStart:i]
-                       na++
-                       fieldStart = -1
+       // Find the field start and end indices.
+       wasField := false
+       fromIndex := 0
+       for i := 0; i < len(s); {
+               size := 1
+               r := rune(s[i])
+               if r >= utf8.RuneSelf {
+                       r, size = utf8.DecodeRune(s[i:])
                 }
-               if size == 0 {
-                       break
+               if f(r) {
+                       if wasField {
+                               spans = append(spans, span{start: fromIndex, end: i})
+                               wasField = false
+                       }
+               } else {
+                       if !wasField {
+                               fromIndex = i
+                               wasField = true
+                       }
                 }
                 i += size
         }
-       return a[0:na]
+
+       // Last field might end at EOF.
+       if wasField {
+               spans = append(spans, span{fromIndex, len(s)})
+       }
+
+       // Create subslices from recorded field indices.
+       a := make([][]byte, len(spans))
+       for i, span := range spans {
+               a[i] = s[span.start:span.end]
+       }
+
+       return a
  }
  
  // Join concatenates the elements of s to create a new byte slice. The separator
diff --git a/src/bytes/bytes_test.go b/src/bytes/bytes_test.go

index ca0cdbb7c9f02bc78a092cf3cb14e89cb93870cd..db28497e39e4d16a713c3daa7982694759c20558 100644 (file)
--- a/src/bytes/bytes_test.go
+++ b/src/bytes/bytes_test.go
@@ -1502,19 +1502,58 @@ var makeFieldsInput = func() []byte {
         return x
  }
  
-var fieldsInput = makeFieldsInput()
+var makeFieldsInputASCII = func() []byte {
+       x := make([]byte, 1<<20)
+       // Input is ~10% space, rest ASCII non-space.
+       for i := range x {
+               if rand.Intn(10) == 0 {
+                       x[i] = ' '
+               } else {
+                       x[i] = 'x'
+               }
+       }
+       return x
+}
+
+var bytesdata = []struct {
+       name string
+       data []byte
+}{
+       {"ASCII", makeFieldsInputASCII()},
+       {"Mixed", makeFieldsInput()},
+}
  
  func BenchmarkFields(b *testing.B) {
-       b.SetBytes(int64(len(fieldsInput)))
-       for i := 0; i < b.N; i++ {
-               Fields(fieldsInput)
+       for _, sd := range bytesdata {
+               b.Run(sd.name, func(b *testing.B) {
+                       for j := 1 << 4; j <= 1<<20; j <<= 4 {
+                               b.Run(fmt.Sprintf("%d", j), func(b *testing.B) {
+                                       b.ReportAllocs()
+                                       b.SetBytes(int64(j))
+                                       data := sd.data[:j]
+                                       for i := 0; i < b.N; i++ {
+                                               Fields(data)
+                                       }
+                               })
+                       }
+               })
         }
  }
  
  func BenchmarkFieldsFunc(b *testing.B) {
-       b.SetBytes(int64(len(fieldsInput)))
-       for i := 0; i < b.N; i++ {
-               FieldsFunc(fieldsInput, unicode.IsSpace)
+       for _, sd := range bytesdata {
+               b.Run(sd.name, func(b *testing.B) {
+                       for j := 1 << 4; j <= 1<<20; j <<= 4 {
+                               b.Run(fmt.Sprintf("%d", j), func(b *testing.B) {
+                                       b.ReportAllocs()
+                                       b.SetBytes(int64(j))
+                                       data := sd.data[:j]
+                                       for i := 0; i < b.N; i++ {
+                                               FieldsFunc(data, unicode.IsSpace)
+                                       }
+                               })
+                       }
+               })
         }
  }
author	Martin Möhrmann <moehrmann@google.com>
	Sat, 12 Aug 2017 08:20:30 +0000 (10:20 +0200)
committer	Martin Möhrmann <moehrmann@google.com>
	Mon, 14 Aug 2017 21:08:51 +0000 (21:08 +0000)
src/bytes/bytes.go		patch \| blob \| history
src/bytes/bytes_test.go		patch \| blob \| history