src

Go monorepo.
git clone git://code.dwrz.net/src
Log | Files | Refs

tokenize.go (2290B)

      1 package ini
      2 
      3 import (
      4 	"strings"
      5 )
      6 
      7 func tokenize(lines []string) ([]lineToken, error) {
      8 	tokens := make([]lineToken, 0, len(lines))
      9 	for _, line := range lines {
     10 		if len(strings.TrimSpace(line)) == 0 || isLineComment(line) {
     11 			continue
     12 		}
     13 
     14 		if tok := asProfile(line); tok != nil {
     15 			tokens = append(tokens, tok)
     16 		} else if tok := asProperty(line); tok != nil {
     17 			tokens = append(tokens, tok)
     18 		} else if tok := asSubProperty(line); tok != nil {
     19 			tokens = append(tokens, tok)
     20 		} else if tok := asContinuation(line); tok != nil {
     21 			tokens = append(tokens, tok)
     22 		} // unrecognized tokens are effectively ignored
     23 	}
     24 	return tokens, nil
     25 }
     26 
     27 func isLineComment(line string) bool {
     28 	trimmed := strings.TrimLeft(line, " \t")
     29 	return strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, ";")
     30 }
     31 
     32 func asProfile(line string) *lineTokenProfile { // " [ type name ] ; comment"
     33 	trimmed := strings.TrimSpace(trimProfileComment(line)) // "[ type name ]"
     34 	if !isBracketed(trimmed) {
     35 		return nil
     36 	}
     37 	trimmed = trimmed[1 : len(trimmed)-1] // " type name " (or just " name ")
     38 	trimmed = strings.TrimSpace(trimmed)  // "type name" / "name"
     39 	typ, name := splitProfile(trimmed)
     40 	return &lineTokenProfile{
     41 		Type: typ,
     42 		Name: name,
     43 	}
     44 }
     45 
     46 func asProperty(line string) *lineTokenProperty {
     47 	if isLineSpace(rune(line[0])) {
     48 		return nil
     49 	}
     50 
     51 	trimmed := trimPropertyComment(line)
     52 	trimmed = strings.TrimRight(trimmed, " \t")
     53 	k, v, ok := splitProperty(trimmed)
     54 	if !ok {
     55 		return nil
     56 	}
     57 
     58 	return &lineTokenProperty{
     59 		Key:   strings.ToLower(k), // LEGACY: normalize key case
     60 		Value: legacyStrconv(v),   // LEGACY: see func docs
     61 	}
     62 }
     63 
     64 func asSubProperty(line string) *lineTokenSubProperty {
     65 	if !isLineSpace(rune(line[0])) {
     66 		return nil
     67 	}
     68 
     69 	// comments on sub-properties are included in the value
     70 	trimmed := strings.TrimLeft(line, " \t")
     71 	k, v, ok := splitProperty(trimmed)
     72 	if !ok {
     73 		return nil
     74 	}
     75 
     76 	return &lineTokenSubProperty{ // same LEGACY constraints as in normal property
     77 		Key:   strings.ToLower(k),
     78 		Value: legacyStrconv(v),
     79 	}
     80 }
     81 
     82 func asContinuation(line string) *lineTokenContinuation {
     83 	if !isLineSpace(rune(line[0])) {
     84 		return nil
     85 	}
     86 
     87 	// includes comments like sub-properties
     88 	trimmed := strings.TrimLeft(line, " \t")
     89 	return &lineTokenContinuation{
     90 		Value: trimmed,
     91 	}
     92 }