From 754bbf7a25a8dda49b5d08ef0d0443bbf5af0e36 Mon Sep 17 00:00:00 2001 From: Craig Jennings Date: Sun, 7 Apr 2024 13:41:34 -0500 Subject: new repository --- devdocs/go/unicode%2Futf8%2Findex.html | 205 +++++++++++++++++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 devdocs/go/unicode%2Futf8%2Findex.html (limited to 'devdocs/go/unicode%2Futf8%2Findex.html') diff --git a/devdocs/go/unicode%2Futf8%2Findex.html b/devdocs/go/unicode%2Futf8%2Findex.html new file mode 100644 index 00000000..b27d81f9 --- /dev/null +++ b/devdocs/go/unicode%2Futf8%2Findex.html @@ -0,0 +1,205 @@ +

Package utf8

Overview

Package utf8 implements functions and constants to support text encoded in UTF-8. It includes functions to translate between runes and UTF-8 byte sequences. See https://en.wikipedia.org/wiki/UTF-8

Index

Examples

AppendRune
DecodeLastRune
DecodeLastRuneInString
DecodeRune
DecodeRuneInString
EncodeRune
EncodeRune (OutOfRange)
FullRune
FullRuneInString
RuneCount
RuneCountInString
RuneLen
RuneStart
Valid
ValidRune
ValidString

Package files

utf8.go

Constants

Numbers fundamental to the encoding.

+
const (
+    RuneError = '\uFFFD'     // the "error" Rune or "Unicode replacement character"
+    RuneSelf  = 0x80         // characters below RuneSelf are represented as themselves in a single byte.
+    MaxRune   = '\U0010FFFF' // Maximum valid Unicode code point.
+    UTFMax    = 4            // maximum number of bytes of a UTF-8 encoded Unicode character.
+)

func AppendRune 1.18 

func AppendRune(p []byte, r rune) []byte

AppendRune appends the UTF-8 encoding of r to the end of p and returns the extended buffer. If the rune is out of range, it appends the encoding of RuneError.

Example +

Code:

buf1 := utf8.AppendRune(nil, 0x10000)
+buf2 := utf8.AppendRune([]byte("init"), 0x10000)
+fmt.Println(string(buf1))
+fmt.Println(string(buf2))
+

Output:

๐€€
+init๐€€
+

func DecodeLastRune 

func DecodeLastRune(p []byte) (r rune, size int)

DecodeLastRune unpacks the last UTF-8 encoding in p and returns the rune and its width in bytes. If p is empty it returns (RuneError, 0). Otherwise, if the encoding is invalid, it returns (RuneError, 1). Both are impossible results for correct, non-empty UTF-8.

+

An encoding is invalid if it is incorrect UTF-8, encodes a rune that is out of range, or is not the shortest possible UTF-8 encoding for the value. No other validation is performed.

Example +

Code:

b := []byte("Hello, ไธ–็•Œ")
+
+for len(b) > 0 {
+    r, size := utf8.DecodeLastRune(b)
+    fmt.Printf("%c %v\n", r, size)
+
+    b = b[:len(b)-size]
+}
+

Output:

็•Œ 3
+ไธ– 3
+  1
+, 1
+o 1
+l 1
+l 1
+e 1
+H 1
+

func DecodeLastRuneInString 

func DecodeLastRuneInString(s string) (r rune, size int)

DecodeLastRuneInString is like DecodeLastRune but its input is a string. If s is empty it returns (RuneError, 0). Otherwise, if the encoding is invalid, it returns (RuneError, 1). Both are impossible results for correct, non-empty UTF-8.

+

An encoding is invalid if it is incorrect UTF-8, encodes a rune that is out of range, or is not the shortest possible UTF-8 encoding for the value. No other validation is performed.

Example +

Code:

str := "Hello, ไธ–็•Œ"
+
+for len(str) > 0 {
+    r, size := utf8.DecodeLastRuneInString(str)
+    fmt.Printf("%c %v\n", r, size)
+
+    str = str[:len(str)-size]
+}
+

Output:

็•Œ 3
+ไธ– 3
+  1
+, 1
+o 1
+l 1
+l 1
+e 1
+H 1
+

func DecodeRune 

func DecodeRune(p []byte) (r rune, size int)

DecodeRune unpacks the first UTF-8 encoding in p and returns the rune and its width in bytes. If p is empty it returns (RuneError, 0). Otherwise, if the encoding is invalid, it returns (RuneError, 1). Both are impossible results for correct, non-empty UTF-8.

+

An encoding is invalid if it is incorrect UTF-8, encodes a rune that is out of range, or is not the shortest possible UTF-8 encoding for the value. No other validation is performed.

Example +

Code:

b := []byte("Hello, ไธ–็•Œ")
+
+for len(b) > 0 {
+    r, size := utf8.DecodeRune(b)
+    fmt.Printf("%c %v\n", r, size)
+
+    b = b[size:]
+}
+

Output:

H 1
+e 1
+l 1
+l 1
+o 1
+, 1
+  1
+ไธ– 3
+็•Œ 3
+

func DecodeRuneInString 

func DecodeRuneInString(s string) (r rune, size int)

DecodeRuneInString is like DecodeRune but its input is a string. If s is empty it returns (RuneError, 0). Otherwise, if the encoding is invalid, it returns (RuneError, 1). Both are impossible results for correct, non-empty UTF-8.

+

An encoding is invalid if it is incorrect UTF-8, encodes a rune that is out of range, or is not the shortest possible UTF-8 encoding for the value. No other validation is performed.

Example +

Code:

str := "Hello, ไธ–็•Œ"
+
+for len(str) > 0 {
+    r, size := utf8.DecodeRuneInString(str)
+    fmt.Printf("%c %v\n", r, size)
+
+    str = str[size:]
+}
+

Output:

H 1
+e 1
+l 1
+l 1
+o 1
+, 1
+  1
+ไธ– 3
+็•Œ 3
+

func EncodeRune 

func EncodeRune(p []byte, r rune) int

EncodeRune writes into p (which must be large enough) the UTF-8 encoding of the rune. If the rune is out of range, it writes the encoding of RuneError. It returns the number of bytes written.

Example +

Code:

r := 'ไธ–'
+buf := make([]byte, 3)
+
+n := utf8.EncodeRune(buf, r)
+
+fmt.Println(buf)
+fmt.Println(n)
+

Output:

[228 184 150]
+3
+

Example (OutOfRange) +

Code:

runes := []rune{
+    // Less than 0, out of range.
+    -1,
+    // Greater than 0x10FFFF, out of range.
+    0x110000,
+    // The Unicode replacement character.
+    utf8.RuneError,
+}
+for i, c := range runes {
+    buf := make([]byte, 3)
+    size := utf8.EncodeRune(buf, c)
+    fmt.Printf("%d: %d %[2]s %d\n", i, buf, size)
+}
+

Output:

0: [239 191 189] ๏ฟฝ 3
+1: [239 191 189] ๏ฟฝ 3
+2: [239 191 189] ๏ฟฝ 3
+

func FullRune 

func FullRune(p []byte) bool

FullRune reports whether the bytes in p begin with a full UTF-8 encoding of a rune. An invalid encoding is considered a full Rune since it will convert as a width-1 error rune.

Example +

Code:

buf := []byte{228, 184, 150} // ไธ–
+fmt.Println(utf8.FullRune(buf))
+fmt.Println(utf8.FullRune(buf[:2]))
+

Output:

true
+false
+

func FullRuneInString 

func FullRuneInString(s string) bool

FullRuneInString is like FullRune but its input is a string.

Example +

Code:

str := "ไธ–"
+fmt.Println(utf8.FullRuneInString(str))
+fmt.Println(utf8.FullRuneInString(str[:2]))
+

Output:

true
+false
+

func RuneCount 

func RuneCount(p []byte) int

RuneCount returns the number of runes in p. Erroneous and short encodings are treated as single runes of width 1 byte.

Example +

Code:

buf := []byte("Hello, ไธ–็•Œ")
+fmt.Println("bytes =", len(buf))
+fmt.Println("runes =", utf8.RuneCount(buf))
+

Output:

bytes = 13
+runes = 9
+

func RuneCountInString 

func RuneCountInString(s string) (n int)

RuneCountInString is like RuneCount but its input is a string.

Example +

Code:

str := "Hello, ไธ–็•Œ"
+fmt.Println("bytes =", len(str))
+fmt.Println("runes =", utf8.RuneCountInString(str))
+

Output:

bytes = 13
+runes = 9
+

func RuneLen 

func RuneLen(r rune) int

RuneLen returns the number of bytes required to encode the rune. It returns -1 if the rune is not a valid value to encode in UTF-8.

Example +

Code:

fmt.Println(utf8.RuneLen('a'))
+fmt.Println(utf8.RuneLen('็•Œ'))
+

Output:

1
+3
+

func RuneStart 

func RuneStart(b byte) bool

RuneStart reports whether the byte could be the first byte of an encoded, possibly invalid rune. Second and subsequent bytes always have the top two bits set to 10.

Example +

Code:

buf := []byte("a็•Œ")
+fmt.Println(utf8.RuneStart(buf[0]))
+fmt.Println(utf8.RuneStart(buf[1]))
+fmt.Println(utf8.RuneStart(buf[2]))
+

Output:

true
+true
+false
+

func Valid 

func Valid(p []byte) bool

Valid reports whether p consists entirely of valid UTF-8-encoded runes.

Example +

Code:

valid := []byte("Hello, ไธ–็•Œ")
+invalid := []byte{0xff, 0xfe, 0xfd}
+
+fmt.Println(utf8.Valid(valid))
+fmt.Println(utf8.Valid(invalid))
+

Output:

true
+false
+

func ValidRune 1.1 

func ValidRune(r rune) bool

ValidRune reports whether r can be legally encoded as UTF-8. Code points that are out of range or a surrogate half are illegal.

Example +

Code:

valid := 'a'
+invalid := rune(0xfffffff)
+
+fmt.Println(utf8.ValidRune(valid))
+fmt.Println(utf8.ValidRune(invalid))
+

Output:

true
+false
+

func ValidString 

func ValidString(s string) bool

ValidString reports whether s consists entirely of valid UTF-8-encoded runes.

Example +

Code:

valid := "Hello, ไธ–็•Œ"
+invalid := string([]byte{0xff, 0xfe, 0xfd})
+
+fmt.Println(utf8.ValidString(valid))
+fmt.Println(utf8.ValidString(invalid))
+

Output:

true
+false
+
+

+ © Google, Inc.
Licensed under the Creative Commons Attribution License 3.0.
+ http://golang.org/pkg/unicode/utf8/ +

+
-- cgit v1.2.3