src

splitfunc.go (5073B)

---

 package graphemes
 
 import (
 	"bufio"
 )
 
 // is determines if lookup intersects propert(ies)
 func (lookup property) is(properties property) bool {
 	return (lookup & properties) != 0
 }
 
 const _Ignore = _Extend
 
 // incbState tracks state for GB9c rule (Indic conjunct clusters)
 // Pattern: Consonant (Extend|Linker)* Linker (Extend|Linker)* × Consonant
 type incbState int
 
 const (
 	incbNone      incbState = iota // initial/reset
 	incbConsonant                  // seen Consonant, awaiting Linker
 	incbLinker                     // seen Consonant and Linker (conjunct ready)
 )
 
 // SplitFunc is a bufio.SplitFunc implementation of Unicode grapheme cluster segmentation, for use with bufio.Scanner.
 //
 // See https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries.
 var SplitFunc bufio.SplitFunc = splitFunc[[]byte]
 
 func splitFunc[T ~string | ~[]byte](data T, atEOF bool) (advance int, token T, err error) {
 	var empty T
 	if len(data) == 0 {
 		return 0, empty, nil
 	}
 
 	// These vars are stateful across loop iterations
 	var pos int
 	var lastExIgnore property = 0     // "last excluding ignored categories"
 	var lastLastExIgnore property = 0 // "last one before that"
 	var regionalIndicatorCount int
 
 	// GB9c state: tracking Indic conjunct clusters
 	var incb incbState
 
 	// Rules are usually of the form Cat1 × Cat2; "current" refers to the first property
 	// to the right of the ×, from which we look back or forward
 
 	current, w := lookup(data[pos:])
 	if w == 0 {
 		if !atEOF {
 			// Rune extends past current data, request more
 			return 0, empty, nil
 		}
 		pos = len(data)
 		return pos, data[:pos], nil
 	}
 
 	// https://unicode.org/reports/tr29/#GB1
 	// Start of text always advances
 	pos += w
 
 	for {
 		eot := pos == len(data) // "end of text"
 
 		if eot {
 			if !atEOF {
 				// Token extends past current data, request more
 				return 0, empty, nil
 			}
 
 			// https://unicode.org/reports/tr29/#GB2
 			break
 		}
 
 		/*
 			We've switched the evaluation order of GB1↓ and GB2↑. It's ok:
 			because we've checked for len(data) at the top of this function,
 			sot and eot are mutually exclusive, order doesn't matter.
 		*/
 
 		// Rules are usually of the form Cat1 × Cat2; "current" refers to the first property
 		// to the right of the ×, from which we look back or forward
 
 		// Remember previous properties to avoid lookups/lookbacks
 		last := current
 		if !last.is(_Ignore) {
 			lastLastExIgnore = lastExIgnore
 			lastExIgnore = last
 		}
 
 		// Update GB9c state based on what we just advanced past
 		if last.is(_InCBConsonant | _InCBLinker | _InCBExtend) {
 			switch {
 			case last.is(_InCBConsonant):
 				if incb != incbLinker {
 					incb = incbConsonant
 				}
 			case last.is(_InCBLinker):
 				if incb >= incbConsonant {
 					incb = incbLinker
 				}
 				// case last.is(_InCBExtend): stay in current state
 			}
 		} else {
 			incb = incbNone
 		}
 
 		current, w = lookup(data[pos:])
 		if w == 0 {
 			if atEOF {
 				// Just return the bytes, we can't do anything with them
 				pos = len(data)
 				break
 			}
 			// Rune extends past current data, request more
 			return 0, empty, nil
 		}
 
 		// Optimization: no rule can possibly apply
 		if current|last == 0 { // i.e. both are zero
 			break
 		}
 
 		// https://unicode.org/reports/tr29/#GB3
 		if current.is(_LF) && last.is(_CR) {
 			pos += w
 			continue
 		}
 
 		// https://unicode.org/reports/tr29/#GB4
 		// https://unicode.org/reports/tr29/#GB5
 		if (current | last).is(_Control | _CR | _LF) {
 			break
 		}
 
 		// https://unicode.org/reports/tr29/#GB6
 		if current.is(_L|_V|_LV|_LVT) && last.is(_L) {
 			pos += w
 			continue
 		}
 
 		// https://unicode.org/reports/tr29/#GB7
 		if current.is(_V|_T) && last.is(_LV|_V) {
 			pos += w
 			continue
 		}
 
 		// https://unicode.org/reports/tr29/#GB8
 		if current.is(_T) && last.is(_LVT|_T) {
 			pos += w
 			continue
 		}
 
 		// https://unicode.org/reports/tr29/#GB9
 		if current.is(_Extend | _ZWJ) {
 			pos += w
 			continue
 		}
 
 		// https://unicode.org/reports/tr29/#GB9a
 		if current.is(_SpacingMark) {
 			pos += w
 			continue
 		}
 
 		// https://unicode.org/reports/tr29/#GB9b
 		if last.is(_Prepend) {
 			pos += w
 			continue
 		}
 
 		// https://unicode.org/reports/tr29/#GB9c
 		// Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker.
 		if incb == incbLinker && current.is(_InCBConsonant) {
 			// After matching the pattern, reset state to start tracking a new pattern
 			// The current Consonant becomes the start of the new pattern
 			incb = incbConsonant
 			pos += w
 			continue
 		}
 
 		// https://unicode.org/reports/tr29/#GB11
 		if current.is(_ExtendedPictographic) && last.is(_ZWJ) && lastLastExIgnore.is(_ExtendedPictographic) {
 			pos += w
 			continue
 		}
 
 		// https://unicode.org/reports/tr29/#GB12
 		// https://unicode.org/reports/tr29/#GB13
 		if (current & last).is(_RegionalIndicator) {
 			regionalIndicatorCount++
 
 			odd := regionalIndicatorCount%2 == 1
 			if odd {
 				pos += w
 				continue
 			}
 		}
 
 		// If we fall through all the above rules, it's a grapheme cluster break
 		break
 	}
 
 	// Return token
 	return pos, data[:pos], nil
 }