iterator.go (3704B)
1 package graphemes 2 3 import "unicode/utf8" 4 5 // FromString returns an iterator for the grapheme clusters in the input string. 6 // Iterate while Next() is true, and access the grapheme via Value(). 7 func FromString(s string) *Iterator[string] { 8 return &Iterator[string]{ 9 split: splitFuncString, 10 data: s, 11 } 12 } 13 14 // FromBytes returns an iterator for the grapheme clusters in the input bytes. 15 // Iterate while Next() is true, and access the grapheme via Value(). 16 func FromBytes(b []byte) *Iterator[[]byte] { 17 return &Iterator[[]byte]{ 18 split: splitFuncBytes, 19 data: b, 20 } 21 } 22 23 // Iterator is a generic iterator for grapheme clusters in strings or byte slices, 24 // with an ASCII hot path optimization. 25 type Iterator[T ~string | ~[]byte] struct { 26 split func(T, bool) (int, T, error) 27 data T 28 pos int 29 start int 30 // AnsiEscapeSequences treats 7-bit ANSI escape sequences (ECMA-48) as 31 // single grapheme clusters when true. The default is false. 32 // 33 // 8-bit controls are not enabled by this option. See [AnsiEscapeSequences8Bit]. 34 AnsiEscapeSequences bool 35 // AnsiEscapeSequences8Bit treats 8-bit C1 ANSI escape sequences (ECMA-48) as single 36 // grapheme clusters when true. The default is false. 37 // 38 // 8-bit control bytes are not UTF-8 encoded, i.e. not valid UTF-8. If you 39 // choose this option, you are choosing to interpret non-UTF-8 data, caveat 40 // emptor. 41 AnsiEscapeSequences8Bit bool 42 } 43 44 var ( 45 splitFuncString = splitFunc[string] 46 splitFuncBytes = splitFunc[[]byte] 47 ) 48 49 const ( 50 esc = 0x1B 51 cr = 0x0D 52 bel = 0x07 53 can = 0x18 54 sub = 0x1A 55 st = 0x9C 56 ) 57 58 // Next advances the iterator to the next grapheme cluster. 59 // Returns false when there are no more grapheme clusters. 60 func (iter *Iterator[T]) Next() bool { 61 if iter.pos >= len(iter.data) { 62 return false 63 } 64 iter.start = iter.pos 65 66 b := iter.data[iter.pos] 67 if iter.AnsiEscapeSequences && b == esc { 68 if a := ansiEscapeLength(iter.data[iter.pos:]); a > 0 { 69 iter.pos += a 70 return true 71 } 72 } 73 if iter.AnsiEscapeSequences8Bit && b >= 0x80 && b <= 0x9F { 74 if a := ansiEscapeLength8Bit(iter.data[iter.pos:]); a > 0 { 75 iter.pos += a 76 return true 77 } 78 } 79 80 // ASCII hot path: any ASCII is one grapheme when next byte is ASCII or end. 81 if b < utf8.RuneSelf && b != cr { 82 if iter.pos+1 >= len(iter.data) || iter.data[iter.pos+1] < utf8.RuneSelf { 83 iter.pos++ 84 return true 85 } 86 } 87 88 // Fall back to UAX29 grapheme parsing 89 remaining := iter.data[iter.pos:] 90 advance, _, err := iter.split(remaining, true) 91 if err != nil { 92 panic(err) 93 } 94 if advance <= 0 { 95 panic("splitFunc returned a zero or negative advance") 96 } 97 iter.pos += advance 98 if iter.pos > len(iter.data) { 99 panic("splitFunc advanced beyond end of data") 100 } 101 return true 102 } 103 104 // Value returns the current grapheme cluster. 105 func (iter *Iterator[T]) Value() T { 106 return iter.data[iter.start:iter.pos] 107 } 108 109 // Start returns the byte position of the current grapheme in the original data. 110 func (iter *Iterator[T]) Start() int { 111 return iter.start 112 } 113 114 // End returns the byte position after the current grapheme in the original data. 115 func (iter *Iterator[T]) End() int { 116 return iter.pos 117 } 118 119 // Reset resets the iterator to the beginning of the data. 120 func (iter *Iterator[T]) Reset() { 121 iter.start = 0 122 iter.pos = 0 123 } 124 125 // SetText sets the data for the iterator to operate on, and resets all state. 126 func (iter *Iterator[T]) SetText(data T) { 127 iter.data = data 128 iter.start = 0 129 iter.pos = 0 130 } 131 132 // First returns the first grapheme cluster without advancing the iterator. 133 func (iter *Iterator[T]) First() T { 134 if len(iter.data) == 0 { 135 return iter.data 136 } 137 138 // Use a copy to leverage Next()'s ASCII optimization 139 cp := *iter 140 cp.pos = 0 141 cp.start = 0 142 cp.Next() 143 return cp.Value() 144 }