Documentation
¶
Overview ¶
Package utf8 implements functions and constants to support text encoded in UTF-8. It includes functions to translate between runes and UTF-8 byte sequences. See https://en.wikipedia.org/wiki/UTF-8
Index ¶
- Constants
- func AppendRunes(dst []byte, src ...rune) []byte
- func Count(s string) (n int)
- func EncodeRune(dst []byte, r rune) ([]byte, int)
- func EncodeRuneNonASCII(dst []byte, r rune) ([]byte, int)
- func First(s string) (r rune, size int)
- func FullRune(s string) bool
- func IterRune(p string, pos int) (r rune, next int)
- func Last(s string) (r rune, size int)
- func RuneLen(r rune) int
- func RuneLenNonASCII(r rune) int
- func RuneStart(b byte) bool
- func RuneValid(r rune) bool
- func Valid(s string) bool
- type String
Examples ¶
Constants ¶
const ( RuneSelf = 0x80 // characters below RuneSelf are represented as themselves in a single byte. RuneErrorLen = 3 // encoded UTF-8 length of a unicodeconst.RuneError MaxRuneLen = 4 // maximum number of bytes of a UTF-8 encoded Unicode character. )
const ( Rune1Max = 1<<7 - 1 Rune2Max = 1<<11 - 1 Rune3Max = 1<<16 - 1 )
Variables ¶
This section is empty.
Functions ¶
func AppendRunes ¶
AppendRunes appends the UTF-8 encoding of r to the end of p and returns the extended buffer. If the rune is out of range, it appends the encoding of RuneError.
Example ¶
package main
import (
"fmt"
"github.com/primecitizens/pcz/std/text/unicode/utf8"
)
func main() {
buf1 := utf8.AppendRunes(nil, 0x10000)
buf2 := utf8.AppendRunes([]byte("init"), 0x10000)
fmt.Println(string(buf1))
fmt.Println(string(buf2))
}
Output: 𐀀 init𐀀
func Count ¶
Count returns the number of runes in s. Erroneous and short encodings are treated as single runes of width 1 byte.
Example ¶
package main
import (
"fmt"
"github.com/primecitizens/pcz/std/text/unicode/utf8"
)
func main() {
str := "Hello, 世界"
fmt.Println("bytes =", len(str))
fmt.Println("runes =", utf8.Count(str))
}
Output: bytes = 13 runes = 9
func EncodeRune ¶
EncodeRune writes into dst the UTF-8 encoding of the rune.
If the rune is out of range, it writes the encoding of RuneError.
It returns the number of bytes written, and when dst is too small to write the complete byte sequence of the rune, it returns 0.
Example ¶
package main
import (
"fmt"
"github.com/primecitizens/pcz/std/text/unicode/utf8"
)
func main() {
r := '世'
buf := make([]byte, 0, 3)
buf, n := utf8.EncodeRune(buf, r)
fmt.Println(buf)
fmt.Println(n)
}
Output: [228 184 150] 3
Example (OutOfRange) ¶
package main
import (
"fmt"
"github.com/primecitizens/pcz/std/text/unicode/common"
"github.com/primecitizens/pcz/std/text/unicode/utf8"
)
func main() {
runes := []rune{
// Less than 0, out of range.
-1,
// Greater than 0x10FFFF, out of range.
0x110000,
// The Unicode replacement character.
common.RuneError,
}
for i, c := range runes {
buf := make([]byte, 0, 3)
buf, size := utf8.EncodeRune(buf, c)
fmt.Printf("%d: %d %[2]s %d\n", i, buf, size)
}
}
Output: 0: [239 191 189] � 3 1: [239 191 189] � 3 2: [239 191 189] � 3
func First ¶
First unpacks the first UTF-8 encoding in s and returns the rune and its width in bytes. If p is empty it returns (RuneError, 0).
Otherwise, if the encoding is invalid, it returns (RuneError, 1). Both are impossible results for correct, non-empty UTF-8.
An encoding is invalid if it is incorrect UTF-8, encodes a rune that is out of range, or is not the shortest possible UTF-8 encoding for the value. No other validation is performed.
Example ¶
package main
import (
"fmt"
"github.com/primecitizens/pcz/std/text/unicode/utf8"
)
func main() {
str := "Hello, 世界"
for len(str) > 0 {
r, size := utf8.First(str)
fmt.Printf("%c %v\n", r, size)
str = str[size:]
}
}
Output: H 1 e 1 l 1 l 1 o 1 , 1 1 世 3 界 3
func FullRune ¶
FullRune reports whether the bytes in s begin with a full UTF-8 encoding of a rune.
An invalid encoding is considered a full Rune since it will convert as a width-1 error rune.
Example ¶
package main
import (
"fmt"
"github.com/primecitizens/pcz/std/text/unicode/utf8"
)
func main() {
str := "世"
fmt.Println(utf8.FullRune(str))
fmt.Println(utf8.FullRune(str[:2]))
}
Output: true false
func IterRune ¶
IterRune returns the rune at the start of p[pos:] and the index after the rune in p.
If the string appears to be incomplete or decoding problems are encountered (RuneError, pos+1) is returned to ensure progress when IterRune is used to iterate over a string.
func Last ¶
Last unpacks the last UTF-8 encoding in p and returns the rune and its width in bytes. If p is empty it returns (RuneError, 0). Otherwise, if the encoding is invalid, it returns (RuneError, 1). Both are impossible results for correct, non-empty UTF-8.
An encoding is invalid if it is incorrect UTF-8, encodes a rune that is out of range, or is not the shortest possible UTF-8 encoding for the value. No other validation is performed.
Example ¶
package main
import (
"fmt"
"github.com/primecitizens/pcz/std/text/unicode/utf8"
)
func main() {
str := "Hello, 世界"
for len(str) > 0 {
r, size := utf8.Last(str)
fmt.Printf("%c %v\n", r, size)
str = str[:len(str)-size]
}
}
Output: 界 3 世 3 1 , 1 o 1 l 1 l 1 e 1 H 1
func RuneLen ¶
RuneLen returns the number of bytes required to encode the rune.
It returns 0 if the rune is not a valid value to encode in UTF-8.
Example ¶
package main
import (
"fmt"
"github.com/primecitizens/pcz/std/text/unicode/utf8"
)
func main() {
fmt.Println(utf8.RuneLen('a'))
fmt.Println(utf8.RuneLen('界'))
}
Output: 1 3
func RuneLenNonASCII ¶
func RuneStart ¶
RuneStart reports whether the byte could be the first byte of an encoded, possibly invalid rune. Second and subsequent bytes always have the top two bits set to 10.
Example ¶
package main
import (
"fmt"
"github.com/primecitizens/pcz/std/text/unicode/utf8"
)
func main() {
buf := []byte("a界")
fmt.Println(utf8.RuneStart(buf[0]))
fmt.Println(utf8.RuneStart(buf[1]))
fmt.Println(utf8.RuneStart(buf[2]))
}
Output: true true false
func RuneValid ¶
RuneValid reports whether r can be legally encoded as UTF-8. Code points that are out of range or a surrogate half are illegal.
Example ¶
package main
import (
"fmt"
"github.com/primecitizens/pcz/std/text/unicode/utf8"
)
func main() {
valid := 'a'
invalid := rune(0xfffffff)
fmt.Println(utf8.RuneValid(valid))
fmt.Println(utf8.RuneValid(invalid))
}
Output: true false
func Valid ¶
Valid reports whether s consists entirely of valid UTF-8-encoded runes.
Example ¶
package main
import (
"fmt"
"github.com/primecitizens/pcz/std/text/unicode/utf8"
)
func main() {
valid := "Hello, 世界"
invalid := string([]byte{0xff, 0xfe, 0xfd})
fmt.Println(utf8.Valid(valid))
fmt.Println(utf8.Valid(invalid))
}
Output: true false
Source Files