From 423cdd4d9475a8e9622844594172ca5dce2dea96 Mon Sep 17 00:00:00 2001 From: silverwind Date: Mon, 6 Apr 2026 13:07:33 +0200 Subject: [PATCH] Improve control char rendering and escape button styling (#37094) Follow-up to #37078. - Use Unicode Control Pictures](U+2400-U+2421) to render C0 control characters - Make it work in diff view too - Replace escape warning emoji with SVG - Align escape warning button with code lines --------- Co-authored-by: wxiaoguang --- modules/charset/ambiguous.go | 14 +- modules/charset/ambiguous/generate.go | 188 -- modules/charset/ambiguous_gen.go | 1590 +++++++++-------- modules/charset/ambiguous_gen_test.go | 8 +- modules/charset/breakwriter.go | 43 - modules/charset/breakwriter_test.go | 68 - modules/charset/charset.go | 20 +- modules/charset/escape.go | 40 +- modules/charset/escape_status.go | 8 +- modules/charset/escape_stream.go | 579 +++--- modules/charset/escape_test.go | 99 +- .../{ambiguous => generate}/ambiguous.json | 0 .../{invisible => generate}/generate.go | 158 +- modules/charset/htmlstream.go | 200 --- modules/charset/invisible_gen.go | 60 +- modules/highlight/highlight.go | 97 +- modules/highlight/highlight_test.go | 11 - modules/indexer/code/search.go | 2 +- modules/templates/util_render.go | 22 + options/locale/locale_en-US.json | 2 +- routers/web/devtest/devtest.go | 26 + routers/web/repo/view.go | 2 +- routers/web/repo/wiki.go | 3 +- services/markup/renderhelper_codepreview.go | 2 +- .../markup/renderhelper_codepreview_test.go | 9 +- templates/base/markup_codepreview.tmpl | 5 +- templates/devtest/unicode-escape.tmpl | 17 + templates/repo/blame.tmpl | 10 +- templates/repo/diff/blob_excerpt.tmpl | 8 +- templates/repo/diff/escape_title.tmpl | 2 - templates/repo/diff/section_code.tmpl | 2 +- templates/repo/diff/section_split.tmpl | 10 +- templates/repo/diff/section_unified.tmpl | 4 +- templates/repo/view_file.tmpl | 4 +- web_src/css/base.css | 2 + web_src/css/modules/charescape.css | 23 +- web_src/css/review.css | 17 +- 37 files changed, 1561 insertions(+), 1794 deletions(-) delete mode 100644 modules/charset/ambiguous/generate.go delete mode 100644 modules/charset/breakwriter.go delete mode 100644 modules/charset/breakwriter_test.go rename modules/charset/{ambiguous => generate}/ambiguous.json (100%) rename modules/charset/{invisible => generate}/generate.go (57%) delete mode 100644 modules/charset/htmlstream.go create mode 100644 templates/devtest/unicode-escape.tmpl delete mode 100644 templates/repo/diff/escape_title.tmpl diff --git a/modules/charset/ambiguous.go b/modules/charset/ambiguous.go index 96e0561e15..c87d3cfa5a 100644 --- a/modules/charset/ambiguous.go +++ b/modules/charset/ambiguous.go @@ -1,4 +1,3 @@ -// This file is generated by modules/charset/ambiguous/generate.go DO NOT EDIT // Copyright 2022 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT @@ -14,11 +13,12 @@ import ( // AmbiguousTablesForLocale provides the table of ambiguous characters for this locale. func AmbiguousTablesForLocale(locale translation.Locale) []*AmbiguousTable { + ambiguousTableMap := globalVars().ambiguousTableMap key := locale.Language() var table *AmbiguousTable var ok bool for len(key) > 0 { - if table, ok = AmbiguousCharacters[key]; ok { + if table, ok = ambiguousTableMap[key]; ok { break } idx := strings.LastIndexAny(key, "-_") @@ -29,18 +29,18 @@ func AmbiguousTablesForLocale(locale translation.Locale) []*AmbiguousTable { } } if table == nil && (locale.Language() == "zh-CN" || locale.Language() == "zh_CN") { - table = AmbiguousCharacters["zh-hans"] + table = ambiguousTableMap["zh-hans"] } if table == nil && strings.HasPrefix(locale.Language(), "zh") { - table = AmbiguousCharacters["zh-hant"] + table = ambiguousTableMap["zh-hant"] } if table == nil { - table = AmbiguousCharacters["_default"] + table = ambiguousTableMap["_default"] } return []*AmbiguousTable{ table, - AmbiguousCharacters["_common"], + ambiguousTableMap["_common"], } } @@ -52,7 +52,7 @@ func isAmbiguous(r rune, confusableTo *rune, tables ...*AmbiguousTable) bool { i := sort.Search(len(table.Confusable), func(i int) bool { return table.Confusable[i] >= r }) - (*confusableTo) = table.With[i] + *confusableTo = table.With[i] return true } return false diff --git a/modules/charset/ambiguous/generate.go b/modules/charset/ambiguous/generate.go deleted file mode 100644 index e3fda5be98..0000000000 --- a/modules/charset/ambiguous/generate.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package main - -import ( - "bytes" - "flag" - "fmt" - "go/format" - "os" - "sort" - "text/template" - "unicode" - - "code.gitea.io/gitea/modules/json" - - "golang.org/x/text/unicode/rangetable" -) - -// ambiguous.json provides a one to one mapping of ambiguous characters to other characters -// See https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json - -type AmbiguousTable struct { - Confusable []rune - With []rune - Locale string - RangeTable *unicode.RangeTable -} - -type RunePair struct { - Confusable rune - With rune -} - -var verbose bool - -func main() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, `%s: Generate AmbiguousCharacter - -Usage: %[1]s [-v] [-o output.go] ambiguous.json -`, os.Args[0]) - flag.PrintDefaults() - } - - output := "" - flag.BoolVar(&verbose, "v", false, "verbose output") - flag.StringVar(&output, "o", "ambiguous_gen.go", "file to output to") - flag.Parse() - input := flag.Arg(0) - if input == "" { - input = "ambiguous.json" - } - - bs, err := os.ReadFile(input) - if err != nil { - fatalf("Unable to read: %s Err: %v", input, err) - } - - var unwrapped string - if err := json.Unmarshal(bs, &unwrapped); err != nil { - fatalf("Unable to unwrap content in: %s Err: %v", input, err) - } - - fromJSON := map[string][]uint32{} - if err := json.Unmarshal([]byte(unwrapped), &fromJSON); err != nil { - fatalf("Unable to unmarshal content in: %s Err: %v", input, err) - } - - tables := make([]*AmbiguousTable, 0, len(fromJSON)) - for locale, chars := range fromJSON { - table := &AmbiguousTable{Locale: locale} - table.Confusable = make([]rune, 0, len(chars)/2) - table.With = make([]rune, 0, len(chars)/2) - pairs := make([]RunePair, len(chars)/2) - for i := 0; i < len(chars); i += 2 { - pairs[i/2].Confusable, pairs[i/2].With = rune(chars[i]), rune(chars[i+1]) - } - sort.Slice(pairs, func(i, j int) bool { - return pairs[i].Confusable < pairs[j].Confusable - }) - for _, pair := range pairs { - table.Confusable = append(table.Confusable, pair.Confusable) - table.With = append(table.With, pair.With) - } - table.RangeTable = rangetable.New(table.Confusable...) - tables = append(tables, table) - } - sort.Slice(tables, func(i, j int) bool { - return tables[i].Locale < tables[j].Locale - }) - data := map[string]any{ - "Tables": tables, - } - - if err := runTemplate(generatorTemplate, output, &data); err != nil { - fatalf("Unable to run template: %v", err) - } -} - -func runTemplate(t *template.Template, filename string, data any) error { - buf := bytes.NewBuffer(nil) - if err := t.Execute(buf, data); err != nil { - return fmt.Errorf("unable to execute template: %w", err) - } - bs, err := format.Source(buf.Bytes()) - if err != nil { - verbosef("Bad source:\n%s", buf.String()) - return fmt.Errorf("unable to format source: %w", err) - } - - old, err := os.ReadFile(filename) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to read old file %s because %w", filename, err) - } else if err == nil { - if bytes.Equal(bs, old) { - // files are the same don't rewrite it. - return nil - } - } - - file, err := os.Create(filename) - if err != nil { - return fmt.Errorf("failed to create file %s because %w", filename, err) - } - defer file.Close() - _, err = file.Write(bs) - if err != nil { - return fmt.Errorf("unable to write generated source: %w", err) - } - return nil -} - -var generatorTemplate = template.Must(template.New("ambiguousTemplate").Parse(`// This file is generated by modules/charset/ambiguous/generate.go DO NOT EDIT -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - - -package charset - -import "unicode" - -// This file is generated from https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json - -// AmbiguousTable matches a confusable rune with its partner for the Locale -type AmbiguousTable struct { - Confusable []rune - With []rune - Locale string - RangeTable *unicode.RangeTable -} - -// AmbiguousCharacters provides a map by locale name to the confusable characters in that locale -var AmbiguousCharacters = map[string]*AmbiguousTable{ - {{range .Tables}}{{printf "%q:" .Locale}} { - Confusable: []rune{ {{range .Confusable}}{{.}},{{end}} }, - With: []rune{ {{range .With}}{{.}},{{end}} }, - Locale: {{printf "%q" .Locale}}, - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {{range .RangeTable.R16 }} {Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}}, - {{end}} }, - R32: []unicode.Range32{ - {{range .RangeTable.R32}} {Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}}, - {{end}} }, - LatinOffset: {{.RangeTable.LatinOffset}}, - }, - }, - {{end}} -} - -`)) - -func logf(format string, args ...any) { - fmt.Fprintf(os.Stderr, format+"\n", args...) -} - -func verbosef(format string, args ...any) { - if verbose { - logf(format, args...) - } -} - -func fatalf(format string, args ...any) { - logf("fatal: "+format+"\n", args...) - os.Exit(1) -} diff --git a/modules/charset/ambiguous_gen.go b/modules/charset/ambiguous_gen.go index c88ffd5aa5..669a46c91a 100644 --- a/modules/charset/ambiguous_gen.go +++ b/modules/charset/ambiguous_gen.go @@ -1,5 +1,5 @@ -// This file is generated by modules/charset/ambiguous/generate.go DO NOT EDIT -// Copyright 2022 The Gitea Authors. All rights reserved. +// This file is generated by modules/charset/generate/generate.go DO NOT EDIT +// Copyright 2026 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package charset @@ -16,821 +16,837 @@ type AmbiguousTable struct { RangeTable *unicode.RangeTable } -// AmbiguousCharacters provides a map by locale name to the confusable characters in that locale -var AmbiguousCharacters = map[string]*AmbiguousTable{ - "_common": { - Confusable: []rune{184, 383, 388, 397, 422, 423, 439, 444, 445, 448, 451, 540, 546, 547, 577, 593, 609, 611, 617, 618, 623, 651, 655, 660, 697, 699, 700, 701, 702, 706, 707, 708, 710, 712, 714, 715, 720, 727, 731, 732, 756, 760, 884, 890, 894, 895, 900, 913, 914, 917, 918, 919, 922, 924, 925, 927, 929, 932, 933, 935, 945, 947, 953, 957, 959, 961, 963, 965, 978, 988, 1000, 1010, 1011, 1017, 1018, 1029, 1030, 1032, 1109, 1110, 1112, 1121, 1140, 1141, 1198, 1199, 1211, 1213, 1216, 1231, 1248, 1281, 1292, 1307, 1308, 1309, 1357, 1359, 1365, 1370, 1373, 1377, 1379, 1382, 1392, 1400, 1404, 1405, 1409, 1412, 1413, 1417, 1472, 1475, 1493, 1496, 1497, 1503, 1505, 1523, 1549, 1575, 1607, 1632, 1633, 1637, 1639, 1643, 1645, 1726, 1729, 1748, 1749, 1776, 1777, 1781, 1783, 1793, 1794, 1795, 1796, 1984, 1994, 2036, 2037, 2042, 2307, 2406, 2429, 2534, 2538, 2541, 2662, 2663, 2666, 2691, 2790, 2819, 2848, 2918, 2920, 3046, 3074, 3174, 3202, 3302, 3330, 3360, 3430, 3437, 3458, 3664, 3792, 4125, 4160, 4327, 4351, 4608, 4816, 5024, 5025, 5026, 5029, 5033, 5034, 5035, 5036, 5038, 5043, 5047, 5051, 5053, 5056, 5058, 5059, 5070, 5071, 5074, 5076, 5077, 5081, 5082, 5086, 5087, 5090, 5094, 5095, 5102, 5107, 5108, 5120, 5167, 5171, 5176, 5194, 5196, 5229, 5231, 5234, 5261, 5290, 5311, 5441, 5500, 5501, 5511, 5551, 5556, 5573, 5598, 5610, 5616, 5623, 5741, 5742, 5760, 5810, 5815, 5825, 5836, 5845, 5846, 5868, 5869, 5941, 6147, 6153, 7428, 7439, 7441, 7452, 7456, 7457, 7458, 7462, 7555, 7564, 7837, 7935, 8125, 8126, 8127, 8128, 8175, 8189, 8190, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200, 8201, 8202, 8208, 8209, 8210, 8218, 8219, 8228, 8232, 8233, 8239, 8242, 8249, 8250, 8257, 8259, 8260, 8270, 8275, 8282, 8287, 8450, 8458, 8459, 8460, 8461, 8462, 8464, 8465, 8466, 8467, 8469, 8473, 8474, 8475, 8476, 8477, 8484, 8488, 8490, 8492, 8493, 8494, 8495, 8496, 8497, 8499, 8500, 8505, 8509, 8517, 8518, 8519, 8520, 8521, 8544, 8548, 8553, 8556, 8557, 8558, 8559, 8560, 8564, 8569, 8572, 8573, 8574, 8722, 8725, 8726, 8727, 8739, 8744, 8746, 8758, 8764, 8868, 8897, 8899, 8959, 9075, 9076, 9082, 9213, 9585, 9587, 10088, 10089, 10094, 10095, 10098, 10099, 10100, 10101, 10133, 10134, 10187, 10189, 10201, 10539, 10540, 10741, 10744, 10745, 10799, 11397, 11406, 11410, 11412, 11416, 11418, 11422, 11423, 11426, 11427, 11428, 11429, 11430, 11432, 11436, 11450, 11462, 11466, 11468, 11472, 11474, 11576, 11577, 11599, 11601, 11604, 11605, 11613, 11840, 12034, 12035, 12295, 12308, 12309, 12339, 12448, 12755, 12756, 20022, 20031, 42192, 42193, 42194, 42195, 42196, 42198, 42199, 42201, 42202, 42204, 42205, 42207, 42208, 42209, 42210, 42211, 42214, 42215, 42218, 42219, 42220, 42222, 42224, 42226, 42227, 42228, 42232, 42233, 42237, 42239, 42510, 42564, 42567, 42719, 42731, 42735, 42801, 42842, 42858, 42862, 42872, 42889, 42892, 42904, 42905, 42911, 42923, 42930, 42931, 42932, 43826, 43829, 43837, 43847, 43848, 43854, 43858, 43866, 43893, 43905, 43907, 43923, 43945, 43946, 43951, 64422, 64423, 64424, 64425, 64426, 64427, 64428, 64429, 64830, 64831, 65072, 65101, 65102, 65103, 65112, 65128, 65165, 65166, 65257, 65258, 65259, 65260, 65282, 65284, 65285, 65286, 65287, 65290, 65291, 65293, 65294, 65295, 65296, 65297, 65298, 65299, 65300, 65301, 65302, 65303, 65304, 65305, 65308, 65309, 65310, 65312, 65313, 65314, 65315, 65316, 65317, 65318, 65319, 65320, 65321, 65322, 65323, 65324, 65325, 65326, 65327, 65328, 65329, 65330, 65331, 65332, 65333, 65334, 65335, 65336, 65337, 65338, 65339, 65340, 65341, 65342, 65343, 65344, 65345, 65346, 65347, 65348, 65349, 65350, 65351, 65352, 65353, 65354, 65355, 65356, 65357, 65358, 65359, 65360, 65361, 65362, 65363, 65364, 65365, 65366, 65367, 65368, 65369, 65370, 65371, 65372, 65373, 65512, 66178, 66182, 66183, 66186, 66192, 66194, 66197, 66198, 66199, 66203, 66208, 66209, 66210, 66213, 66219, 66224, 66225, 66226, 66228, 66255, 66293, 66305, 66306, 66313, 66321, 66325, 66327, 66330, 66335, 66336, 66338, 66564, 66581, 66587, 66592, 66604, 66621, 66632, 66740, 66754, 66766, 66770, 66794, 66806, 66835, 66838, 66840, 66844, 66845, 66853, 66854, 66855, 68176, 70864, 71430, 71434, 71438, 71439, 71840, 71842, 71843, 71844, 71846, 71849, 71852, 71854, 71855, 71858, 71861, 71864, 71867, 71868, 71872, 71873, 71874, 71875, 71876, 71878, 71880, 71882, 71884, 71893, 71894, 71895, 71896, 71900, 71904, 71909, 71910, 71913, 71916, 71919, 71922, 93960, 93962, 93974, 93992, 94005, 94010, 94011, 94015, 94016, 94018, 94019, 94033, 94034, 119060, 119149, 119302, 119309, 119311, 119314, 119315, 119318, 119338, 119350, 119351, 119354, 119355, 119808, 119809, 119810, 119811, 119812, 119813, 119814, 119815, 119816, 119817, 119818, 119819, 119820, 119821, 119822, 119823, 119824, 119825, 119826, 119827, 119828, 119829, 119830, 119831, 119832, 119833, 119834, 119835, 119836, 119837, 119838, 119839, 119840, 119841, 119842, 119843, 119844, 119845, 119847, 119848, 119849, 119850, 119851, 119852, 119853, 119854, 119855, 119856, 119857, 119858, 119859, 119860, 119861, 119862, 119863, 119864, 119865, 119866, 119867, 119868, 119869, 119870, 119871, 119872, 119873, 119874, 119875, 119876, 119877, 119878, 119879, 119880, 119881, 119882, 119883, 119884, 119885, 119886, 119887, 119888, 119889, 119890, 119891, 119892, 119894, 119895, 119896, 119897, 119899, 119900, 119901, 119902, 119903, 119904, 119905, 119906, 119907, 119908, 119909, 119910, 119911, 119912, 119913, 119914, 119915, 119916, 119917, 119918, 119919, 119920, 119921, 119922, 119923, 119924, 119925, 119926, 119927, 119928, 119929, 119930, 119931, 119932, 119933, 119934, 119935, 119936, 119937, 119938, 119939, 119940, 119941, 119942, 119943, 119944, 119945, 119946, 119947, 119948, 119949, 119951, 119952, 119953, 119954, 119955, 119956, 119957, 119958, 119959, 119960, 119961, 119962, 119963, 119964, 119966, 119967, 119970, 119973, 119974, 119977, 119978, 119979, 119980, 119982, 119983, 119984, 119985, 119986, 119987, 119988, 119989, 119990, 119991, 119992, 119993, 119995, 119997, 119998, 119999, 120000, 120001, 120003, 120005, 120006, 120007, 120008, 120009, 120010, 120011, 120012, 120013, 120014, 120015, 120016, 120017, 120018, 120019, 120020, 120021, 120022, 120023, 120024, 120025, 120026, 120027, 120028, 120029, 120030, 120031, 120032, 120033, 120034, 120035, 120036, 120037, 120038, 120039, 120040, 120041, 120042, 120043, 120044, 120045, 120046, 120047, 120048, 120049, 120050, 120051, 120052, 120053, 120055, 120056, 120057, 120058, 120059, 120060, 120061, 120062, 120063, 120064, 120065, 120066, 120067, 120068, 120069, 120071, 120072, 120073, 120074, 120077, 120078, 120079, 120080, 120081, 120082, 120083, 120084, 120086, 120087, 120088, 120089, 120090, 120091, 120092, 120094, 120095, 120096, 120097, 120098, 120099, 120100, 120101, 120102, 120103, 120104, 120105, 120107, 120108, 120109, 120110, 120111, 120112, 120113, 120114, 120115, 120116, 120117, 120118, 120119, 120120, 120121, 120123, 120124, 120125, 120126, 120128, 120129, 120130, 120131, 120132, 120134, 120138, 120139, 120140, 120141, 120142, 120143, 120144, 120146, 120147, 120148, 120149, 120150, 120151, 120152, 120153, 120154, 120155, 120156, 120157, 120159, 120160, 120161, 120162, 120163, 120164, 120165, 120166, 120167, 120168, 120169, 120170, 120171, 120172, 120173, 120174, 120175, 120176, 120177, 120178, 120179, 120180, 120181, 120182, 120183, 120184, 120185, 120186, 120187, 120188, 120189, 120190, 120191, 120192, 120193, 120194, 120195, 120196, 120197, 120198, 120199, 120200, 120201, 120202, 120203, 120204, 120205, 120206, 120207, 120208, 120209, 120211, 120212, 120213, 120214, 120215, 120216, 120217, 120218, 120219, 120220, 120221, 120222, 120223, 120224, 120225, 120226, 120227, 120228, 120229, 120230, 120231, 120232, 120233, 120234, 120235, 120236, 120237, 120238, 120239, 120240, 120241, 120242, 120243, 120244, 120245, 120246, 120247, 120248, 120249, 120250, 120251, 120252, 120253, 120254, 120255, 120256, 120257, 120258, 120259, 120260, 120261, 120263, 120264, 120265, 120266, 120267, 120268, 120269, 120270, 120271, 120272, 120273, 120274, 120275, 120276, 120277, 120278, 120279, 120280, 120281, 120282, 120283, 120284, 120285, 120286, 120287, 120288, 120289, 120290, 120291, 120292, 120293, 120294, 120295, 120296, 120297, 120298, 120299, 120300, 120301, 120302, 120303, 120304, 120305, 120306, 120307, 120308, 120309, 120310, 120311, 120312, 120313, 120315, 120316, 120317, 120318, 120319, 120320, 120321, 120322, 120323, 120324, 120325, 120326, 120327, 120328, 120329, 120330, 120331, 120332, 120333, 120334, 120335, 120336, 120337, 120338, 120339, 120340, 120341, 120342, 120343, 120344, 120345, 120346, 120347, 120348, 120349, 120350, 120351, 120352, 120353, 120354, 120355, 120356, 120357, 120358, 120359, 120360, 120361, 120362, 120363, 120364, 120365, 120367, 120368, 120369, 120370, 120371, 120372, 120373, 120374, 120375, 120376, 120377, 120378, 120379, 120380, 120381, 120382, 120383, 120384, 120385, 120386, 120387, 120388, 120389, 120390, 120391, 120392, 120393, 120394, 120395, 120396, 120397, 120398, 120399, 120400, 120401, 120402, 120403, 120404, 120405, 120406, 120407, 120408, 120409, 120410, 120411, 120412, 120413, 120414, 120415, 120416, 120417, 120419, 120420, 120421, 120422, 120423, 120424, 120425, 120426, 120427, 120428, 120429, 120430, 120431, 120432, 120433, 120434, 120435, 120436, 120437, 120438, 120439, 120440, 120441, 120442, 120443, 120444, 120445, 120446, 120447, 120448, 120449, 120450, 120451, 120452, 120453, 120454, 120455, 120456, 120457, 120458, 120459, 120460, 120461, 120462, 120463, 120464, 120465, 120466, 120467, 120468, 120469, 120471, 120472, 120473, 120474, 120475, 120476, 120477, 120478, 120479, 120480, 120481, 120482, 120483, 120484, 120488, 120489, 120492, 120493, 120494, 120496, 120497, 120499, 120500, 120502, 120504, 120507, 120508, 120510, 120514, 120516, 120522, 120526, 120528, 120530, 120532, 120534, 120544, 120546, 120547, 120550, 120551, 120552, 120554, 120555, 120557, 120558, 120560, 120562, 120565, 120566, 120568, 120572, 120574, 120580, 120584, 120586, 120588, 120590, 120592, 120602, 120604, 120605, 120608, 120609, 120610, 120612, 120613, 120615, 120616, 120618, 120620, 120623, 120624, 120626, 120630, 120632, 120638, 120642, 120644, 120646, 120648, 120650, 120660, 120662, 120663, 120666, 120667, 120668, 120670, 120671, 120673, 120674, 120676, 120678, 120681, 120682, 120684, 120688, 120690, 120696, 120700, 120702, 120704, 120706, 120708, 120718, 120720, 120721, 120724, 120725, 120726, 120728, 120729, 120731, 120732, 120734, 120736, 120739, 120740, 120742, 120746, 120748, 120754, 120758, 120760, 120762, 120764, 120766, 120776, 120778, 120782, 120783, 120784, 120785, 120786, 120787, 120788, 120789, 120790, 120791, 120792, 120793, 120794, 120795, 120796, 120797, 120798, 120799, 120800, 120801, 120802, 120803, 120804, 120805, 120806, 120807, 120808, 120809, 120810, 120811, 120812, 120813, 120814, 120815, 120816, 120817, 120818, 120819, 120820, 120821, 120822, 120823, 120824, 120825, 120826, 120827, 120828, 120829, 120830, 120831, 125127, 125131, 126464, 126500, 126564, 126592, 126596, 128844, 128872, 130032, 130033, 130034, 130035, 130036, 130037, 130038, 130039, 130040, 130041}, - With: []rune{44, 102, 98, 103, 82, 50, 51, 53, 115, 73, 33, 51, 56, 56, 63, 97, 103, 121, 105, 105, 119, 117, 121, 63, 96, 96, 96, 96, 96, 60, 62, 94, 94, 96, 96, 96, 58, 45, 105, 126, 96, 58, 96, 105, 59, 74, 96, 65, 66, 69, 90, 72, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 89, 70, 50, 99, 106, 67, 77, 83, 73, 74, 115, 105, 106, 119, 86, 118, 89, 121, 104, 101, 73, 105, 51, 100, 71, 113, 87, 119, 85, 83, 79, 96, 96, 119, 113, 113, 104, 110, 110, 117, 103, 102, 111, 58, 108, 58, 108, 118, 96, 108, 111, 96, 44, 108, 111, 46, 108, 111, 86, 44, 42, 111, 111, 45, 111, 46, 73, 111, 86, 46, 46, 58, 58, 79, 108, 96, 96, 95, 58, 111, 63, 79, 56, 57, 111, 57, 56, 58, 111, 56, 79, 79, 57, 111, 111, 111, 111, 111, 111, 111, 111, 57, 111, 111, 111, 111, 111, 121, 111, 85, 79, 68, 82, 84, 105, 89, 65, 74, 69, 63, 87, 77, 72, 89, 71, 104, 90, 52, 98, 82, 87, 83, 86, 83, 76, 67, 80, 75, 100, 54, 71, 66, 61, 86, 62, 60, 96, 85, 80, 100, 98, 74, 76, 50, 120, 72, 120, 82, 98, 70, 65, 68, 68, 77, 66, 88, 120, 32, 60, 88, 73, 96, 75, 77, 58, 43, 47, 58, 58, 99, 111, 111, 117, 118, 119, 122, 114, 103, 121, 102, 121, 96, 105, 96, 126, 96, 96, 96, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 45, 44, 96, 46, 32, 32, 32, 96, 60, 62, 47, 45, 47, 42, 126, 58, 32, 67, 103, 72, 72, 72, 104, 73, 73, 76, 108, 78, 80, 81, 82, 82, 82, 90, 90, 75, 66, 67, 101, 101, 69, 70, 77, 111, 105, 121, 68, 100, 101, 105, 106, 73, 86, 88, 76, 67, 68, 77, 105, 118, 120, 73, 99, 100, 45, 47, 92, 42, 73, 118, 85, 58, 126, 84, 118, 85, 69, 105, 112, 97, 73, 47, 88, 40, 41, 60, 62, 40, 41, 123, 125, 43, 45, 47, 92, 84, 120, 120, 92, 47, 92, 120, 114, 72, 73, 75, 77, 78, 79, 111, 80, 112, 67, 99, 84, 89, 88, 45, 47, 57, 51, 76, 54, 86, 69, 73, 33, 79, 81, 88, 61, 92, 47, 79, 40, 41, 47, 61, 47, 92, 92, 47, 66, 80, 100, 68, 84, 71, 75, 74, 67, 90, 70, 77, 78, 76, 83, 82, 86, 72, 87, 88, 89, 65, 69, 73, 79, 85, 46, 44, 58, 61, 46, 50, 105, 86, 63, 50, 115, 50, 51, 57, 38, 58, 96, 70, 102, 117, 51, 74, 88, 66, 101, 102, 111, 114, 114, 117, 117, 121, 105, 114, 119, 122, 118, 115, 99, 111, 111, 111, 111, 111, 111, 111, 111, 40, 41, 58, 95, 95, 95, 45, 92, 108, 108, 111, 111, 111, 111, 34, 36, 37, 38, 96, 42, 43, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 60, 61, 62, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 73, 66, 69, 70, 124, 88, 79, 80, 83, 84, 43, 65, 66, 67, 70, 79, 77, 84, 89, 88, 72, 90, 66, 67, 124, 77, 84, 88, 56, 42, 108, 88, 79, 67, 76, 83, 111, 99, 115, 82, 79, 85, 55, 111, 117, 78, 79, 75, 67, 86, 70, 76, 88, 46, 79, 118, 119, 119, 119, 86, 70, 76, 89, 69, 90, 57, 69, 52, 76, 79, 85, 53, 84, 118, 115, 70, 105, 122, 55, 111, 51, 57, 54, 57, 111, 117, 121, 79, 90, 87, 67, 88, 87, 67, 86, 84, 76, 73, 82, 83, 51, 62, 65, 85, 89, 96, 96, 123, 46, 51, 86, 92, 55, 70, 82, 76, 60, 62, 47, 92, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 67, 68, 71, 74, 75, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 102, 104, 105, 106, 107, 108, 110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 68, 69, 70, 71, 74, 75, 76, 77, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 68, 69, 70, 71, 73, 74, 75, 76, 77, 79, 83, 84, 85, 86, 87, 88, 89, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 105, 65, 66, 69, 90, 72, 73, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 112, 65, 66, 69, 90, 72, 73, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 112, 65, 66, 69, 90, 72, 73, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 112, 65, 66, 69, 90, 72, 73, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 112, 65, 66, 69, 90, 72, 73, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 112, 70, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57, 108, 56, 108, 111, 111, 108, 111, 67, 84, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57}, - Locale: "_common", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 184, Hi: 383, Stride: 199}, - {Lo: 388, Hi: 397, Stride: 9}, - {Lo: 422, Hi: 423, Stride: 1}, - {Lo: 439, Hi: 444, Stride: 5}, - {Lo: 445, Hi: 451, Stride: 3}, - {Lo: 540, Hi: 546, Stride: 6}, - {Lo: 547, Hi: 577, Stride: 30}, - {Lo: 593, Hi: 609, Stride: 16}, - {Lo: 611, Hi: 617, Stride: 6}, - {Lo: 618, Hi: 623, Stride: 5}, - {Lo: 651, Hi: 655, Stride: 4}, - {Lo: 660, Hi: 697, Stride: 37}, - {Lo: 699, Hi: 702, Stride: 1}, - {Lo: 706, Hi: 708, Stride: 1}, - {Lo: 710, Hi: 714, Stride: 2}, - {Lo: 715, Hi: 720, Stride: 5}, - {Lo: 727, Hi: 731, Stride: 4}, - {Lo: 732, Hi: 756, Stride: 24}, - {Lo: 760, Hi: 884, Stride: 124}, - {Lo: 890, Hi: 894, Stride: 4}, - {Lo: 895, Hi: 900, Stride: 5}, - {Lo: 913, Hi: 914, Stride: 1}, - {Lo: 917, Hi: 919, Stride: 1}, - {Lo: 922, Hi: 924, Stride: 2}, - {Lo: 925, Hi: 929, Stride: 2}, - {Lo: 932, Hi: 933, Stride: 1}, - {Lo: 935, Hi: 945, Stride: 10}, - {Lo: 947, Hi: 953, Stride: 6}, - {Lo: 957, Hi: 965, Stride: 2}, - {Lo: 978, Hi: 988, Stride: 10}, - {Lo: 1000, Hi: 1010, Stride: 10}, - {Lo: 1011, Hi: 1017, Stride: 6}, - {Lo: 1018, Hi: 1029, Stride: 11}, - {Lo: 1030, Hi: 1032, Stride: 2}, - {Lo: 1109, Hi: 1110, Stride: 1}, - {Lo: 1112, Hi: 1121, Stride: 9}, - {Lo: 1140, Hi: 1141, Stride: 1}, - {Lo: 1198, Hi: 1199, Stride: 1}, - {Lo: 1211, Hi: 1213, Stride: 2}, - {Lo: 1216, Hi: 1231, Stride: 15}, - {Lo: 1248, Hi: 1281, Stride: 33}, - {Lo: 1292, Hi: 1307, Stride: 15}, - {Lo: 1308, Hi: 1309, Stride: 1}, - {Lo: 1357, Hi: 1359, Stride: 2}, - {Lo: 1365, Hi: 1370, Stride: 5}, - {Lo: 1373, Hi: 1377, Stride: 4}, - {Lo: 1379, Hi: 1382, Stride: 3}, - {Lo: 1392, Hi: 1400, Stride: 8}, - {Lo: 1404, Hi: 1405, Stride: 1}, - {Lo: 1409, Hi: 1412, Stride: 3}, - {Lo: 1413, Hi: 1417, Stride: 4}, - {Lo: 1472, Hi: 1475, Stride: 3}, - {Lo: 1493, Hi: 1496, Stride: 3}, - {Lo: 1497, Hi: 1503, Stride: 6}, - {Lo: 1505, Hi: 1523, Stride: 18}, - {Lo: 1549, Hi: 1575, Stride: 26}, - {Lo: 1607, Hi: 1632, Stride: 25}, - {Lo: 1633, Hi: 1637, Stride: 4}, - {Lo: 1639, Hi: 1643, Stride: 4}, - {Lo: 1645, Hi: 1726, Stride: 81}, - {Lo: 1729, Hi: 1748, Stride: 19}, - {Lo: 1749, Hi: 1776, Stride: 27}, - {Lo: 1777, Hi: 1781, Stride: 4}, - {Lo: 1783, Hi: 1793, Stride: 10}, - {Lo: 1794, Hi: 1796, Stride: 1}, - {Lo: 1984, Hi: 1994, Stride: 10}, - {Lo: 2036, Hi: 2037, Stride: 1}, - {Lo: 2042, Hi: 2307, Stride: 265}, - {Lo: 2406, Hi: 2429, Stride: 23}, - {Lo: 2534, Hi: 2538, Stride: 4}, - {Lo: 2541, Hi: 2662, Stride: 121}, - {Lo: 2663, Hi: 2666, Stride: 3}, - {Lo: 2691, Hi: 2790, Stride: 99}, - {Lo: 2819, Hi: 2848, Stride: 29}, - {Lo: 2918, Hi: 2920, Stride: 2}, - {Lo: 3046, Hi: 3074, Stride: 28}, - {Lo: 3174, Hi: 3202, Stride: 28}, - {Lo: 3302, Hi: 3330, Stride: 28}, - {Lo: 3360, Hi: 3430, Stride: 70}, - {Lo: 3437, Hi: 3458, Stride: 21}, - {Lo: 3664, Hi: 3792, Stride: 128}, - {Lo: 4125, Hi: 4160, Stride: 35}, - {Lo: 4327, Hi: 4351, Stride: 24}, - {Lo: 4608, Hi: 5024, Stride: 208}, - {Lo: 5025, Hi: 5026, Stride: 1}, - {Lo: 5029, Hi: 5033, Stride: 4}, - {Lo: 5034, Hi: 5036, Stride: 1}, - {Lo: 5038, Hi: 5043, Stride: 5}, - {Lo: 5047, Hi: 5051, Stride: 4}, - {Lo: 5053, Hi: 5056, Stride: 3}, - {Lo: 5058, Hi: 5059, Stride: 1}, - {Lo: 5070, Hi: 5071, Stride: 1}, - {Lo: 5074, Hi: 5076, Stride: 2}, - {Lo: 5077, Hi: 5081, Stride: 4}, - {Lo: 5082, Hi: 5086, Stride: 4}, - {Lo: 5087, Hi: 5090, Stride: 3}, - {Lo: 5094, Hi: 5095, Stride: 1}, - {Lo: 5102, Hi: 5107, Stride: 5}, - {Lo: 5108, Hi: 5120, Stride: 12}, - {Lo: 5167, Hi: 5171, Stride: 4}, - {Lo: 5176, Hi: 5194, Stride: 18}, - {Lo: 5196, Hi: 5229, Stride: 33}, - {Lo: 5231, Hi: 5234, Stride: 3}, - {Lo: 5261, Hi: 5290, Stride: 29}, - {Lo: 5311, Hi: 5441, Stride: 130}, - {Lo: 5500, Hi: 5501, Stride: 1}, - {Lo: 5511, Hi: 5551, Stride: 40}, - {Lo: 5556, Hi: 5573, Stride: 17}, - {Lo: 5598, Hi: 5610, Stride: 12}, - {Lo: 5616, Hi: 5623, Stride: 7}, - {Lo: 5741, Hi: 5742, Stride: 1}, - {Lo: 5760, Hi: 5810, Stride: 50}, - {Lo: 5815, Hi: 5825, Stride: 10}, - {Lo: 5836, Hi: 5845, Stride: 9}, - {Lo: 5846, Hi: 5868, Stride: 22}, - {Lo: 5869, Hi: 5941, Stride: 72}, - {Lo: 6147, Hi: 6153, Stride: 6}, - {Lo: 7428, Hi: 7439, Stride: 11}, - {Lo: 7441, Hi: 7452, Stride: 11}, - {Lo: 7456, Hi: 7458, Stride: 1}, - {Lo: 7462, Hi: 7555, Stride: 93}, - {Lo: 7564, Hi: 7837, Stride: 273}, - {Lo: 7935, Hi: 8125, Stride: 190}, - {Lo: 8126, Hi: 8128, Stride: 1}, - {Lo: 8175, Hi: 8189, Stride: 14}, - {Lo: 8190, Hi: 8192, Stride: 2}, - {Lo: 8193, Hi: 8202, Stride: 1}, - {Lo: 8208, Hi: 8210, Stride: 1}, - {Lo: 8218, Hi: 8219, Stride: 1}, - {Lo: 8228, Hi: 8232, Stride: 4}, - {Lo: 8233, Hi: 8239, Stride: 6}, - {Lo: 8242, Hi: 8249, Stride: 7}, - {Lo: 8250, Hi: 8257, Stride: 7}, - {Lo: 8259, Hi: 8260, Stride: 1}, - {Lo: 8270, Hi: 8275, Stride: 5}, - {Lo: 8282, Hi: 8287, Stride: 5}, - {Lo: 8450, Hi: 8458, Stride: 8}, - {Lo: 8459, Hi: 8462, Stride: 1}, - {Lo: 8464, Hi: 8467, Stride: 1}, - {Lo: 8469, Hi: 8473, Stride: 4}, - {Lo: 8474, Hi: 8477, Stride: 1}, - {Lo: 8484, Hi: 8488, Stride: 4}, - {Lo: 8490, Hi: 8492, Stride: 2}, - {Lo: 8493, Hi: 8497, Stride: 1}, - {Lo: 8499, Hi: 8500, Stride: 1}, - {Lo: 8505, Hi: 8509, Stride: 4}, - {Lo: 8517, Hi: 8521, Stride: 1}, - {Lo: 8544, Hi: 8548, Stride: 4}, - {Lo: 8553, Hi: 8556, Stride: 3}, - {Lo: 8557, Hi: 8560, Stride: 1}, - {Lo: 8564, Hi: 8569, Stride: 5}, - {Lo: 8572, Hi: 8574, Stride: 1}, - {Lo: 8722, Hi: 8725, Stride: 3}, - {Lo: 8726, Hi: 8727, Stride: 1}, - {Lo: 8739, Hi: 8744, Stride: 5}, - {Lo: 8746, Hi: 8758, Stride: 12}, - {Lo: 8764, Hi: 8868, Stride: 104}, - {Lo: 8897, Hi: 8899, Stride: 2}, - {Lo: 8959, Hi: 9075, Stride: 116}, - {Lo: 9076, Hi: 9082, Stride: 6}, - {Lo: 9213, Hi: 9585, Stride: 372}, - {Lo: 9587, Hi: 10088, Stride: 501}, - {Lo: 10089, Hi: 10094, Stride: 5}, - {Lo: 10095, Hi: 10098, Stride: 3}, - {Lo: 10099, Hi: 10101, Stride: 1}, - {Lo: 10133, Hi: 10134, Stride: 1}, - {Lo: 10187, Hi: 10189, Stride: 2}, - {Lo: 10201, Hi: 10539, Stride: 338}, - {Lo: 10540, Hi: 10741, Stride: 201}, - {Lo: 10744, Hi: 10745, Stride: 1}, - {Lo: 10799, Hi: 11397, Stride: 598}, - {Lo: 11406, Hi: 11410, Stride: 4}, - {Lo: 11412, Hi: 11416, Stride: 4}, - {Lo: 11418, Hi: 11422, Stride: 4}, - {Lo: 11423, Hi: 11426, Stride: 3}, - {Lo: 11427, Hi: 11430, Stride: 1}, - {Lo: 11432, Hi: 11436, Stride: 4}, - {Lo: 11450, Hi: 11462, Stride: 12}, - {Lo: 11466, Hi: 11468, Stride: 2}, - {Lo: 11472, Hi: 11474, Stride: 2}, - {Lo: 11576, Hi: 11577, Stride: 1}, - {Lo: 11599, Hi: 11601, Stride: 2}, - {Lo: 11604, Hi: 11605, Stride: 1}, - {Lo: 11613, Hi: 11840, Stride: 227}, - {Lo: 12034, Hi: 12035, Stride: 1}, - {Lo: 12295, Hi: 12308, Stride: 13}, - {Lo: 12309, Hi: 12339, Stride: 30}, - {Lo: 12448, Hi: 12755, Stride: 307}, - {Lo: 12756, Hi: 20022, Stride: 7266}, - {Lo: 20031, Hi: 42192, Stride: 22161}, - {Lo: 42193, Hi: 42196, Stride: 1}, - {Lo: 42198, Hi: 42199, Stride: 1}, - {Lo: 42201, Hi: 42202, Stride: 1}, - {Lo: 42204, Hi: 42205, Stride: 1}, - {Lo: 42207, Hi: 42211, Stride: 1}, - {Lo: 42214, Hi: 42215, Stride: 1}, - {Lo: 42218, Hi: 42220, Stride: 1}, - {Lo: 42222, Hi: 42226, Stride: 2}, - {Lo: 42227, Hi: 42228, Stride: 1}, - {Lo: 42232, Hi: 42233, Stride: 1}, - {Lo: 42237, Hi: 42239, Stride: 2}, - {Lo: 42510, Hi: 42564, Stride: 54}, - {Lo: 42567, Hi: 42719, Stride: 152}, - {Lo: 42731, Hi: 42735, Stride: 4}, - {Lo: 42801, Hi: 42842, Stride: 41}, - {Lo: 42858, Hi: 42862, Stride: 4}, - {Lo: 42872, Hi: 42889, Stride: 17}, - {Lo: 42892, Hi: 42904, Stride: 12}, - {Lo: 42905, Hi: 42911, Stride: 6}, - {Lo: 42923, Hi: 42930, Stride: 7}, - {Lo: 42931, Hi: 42932, Stride: 1}, - {Lo: 43826, Hi: 43829, Stride: 3}, - {Lo: 43837, Hi: 43847, Stride: 10}, - {Lo: 43848, Hi: 43854, Stride: 6}, - {Lo: 43858, Hi: 43866, Stride: 8}, - {Lo: 43893, Hi: 43905, Stride: 12}, - {Lo: 43907, Hi: 43923, Stride: 16}, - {Lo: 43945, Hi: 43946, Stride: 1}, - {Lo: 43951, Hi: 64422, Stride: 20471}, - {Lo: 64423, Hi: 64429, Stride: 1}, - {Lo: 64830, Hi: 64831, Stride: 1}, - {Lo: 65072, Hi: 65101, Stride: 29}, - {Lo: 65102, Hi: 65103, Stride: 1}, - {Lo: 65112, Hi: 65128, Stride: 16}, - {Lo: 65165, Hi: 65166, Stride: 1}, - {Lo: 65257, Hi: 65260, Stride: 1}, - {Lo: 65282, Hi: 65284, Stride: 2}, - {Lo: 65285, Hi: 65287, Stride: 1}, - {Lo: 65290, Hi: 65291, Stride: 1}, - {Lo: 65293, Hi: 65305, Stride: 1}, - {Lo: 65308, Hi: 65310, Stride: 1}, - {Lo: 65312, Hi: 65373, Stride: 1}, - {Lo: 65512, Hi: 65512, Stride: 1}, +func newAmbiguousTableMap() map[string]*AmbiguousTable { + return map[string]*AmbiguousTable{ + "_common": { + Confusable: []rune{184, 383, 388, 397, 422, 423, 439, 444, 445, 448, 451, 540, 546, 547, 577, 593, 609, 611, 617, 618, 623, 651, 655, 660, 697, 699, 700, 701, 702, 706, 707, 708, 710, 712, 714, 715, 720, 727, 731, 732, 756, 760, 884, 890, 894, 895, 900, 913, 914, 917, 918, 919, 922, 924, 925, 927, 929, 932, 933, 935, 945, 947, 953, 957, 959, 961, 963, 965, 978, 988, 1000, 1010, 1011, 1017, 1018, 1029, 1030, 1032, 1109, 1110, 1112, 1121, 1140, 1141, 1198, 1199, 1211, 1213, 1216, 1231, 1248, 1281, 1292, 1307, 1308, 1309, 1357, 1359, 1365, 1370, 1373, 1377, 1379, 1382, 1392, 1400, 1404, 1405, 1409, 1412, 1413, 1417, 1472, 1475, 1493, 1496, 1497, 1503, 1505, 1523, 1549, 1575, 1607, 1632, 1633, 1637, 1639, 1643, 1645, 1726, 1729, 1748, 1749, 1776, 1777, 1781, 1783, 1793, 1794, 1795, 1796, 1984, 1994, 2036, 2037, 2042, 2307, 2406, 2429, 2534, 2538, 2541, 2662, 2663, 2666, 2691, 2790, 2819, 2848, 2918, 2920, 3046, 3074, 3174, 3202, 3302, 3330, 3360, 3430, 3437, 3458, 3664, 3792, 4125, 4160, 4327, 4351, 4608, 4816, 5024, 5025, 5026, 5029, 5033, 5034, 5035, 5036, 5038, 5043, 5047, 5051, 5053, 5056, 5058, 5059, 5070, 5071, 5074, 5076, 5077, 5081, 5082, 5086, 5087, 5090, 5094, 5095, 5102, 5107, 5108, 5120, 5167, 5171, 5176, 5194, 5196, 5229, 5231, 5234, 5261, 5290, 5311, 5441, 5500, 5501, 5511, 5551, 5556, 5573, 5598, 5610, 5616, 5623, 5741, 5742, 5760, 5810, 5815, 5825, 5836, 5845, 5846, 5868, 5869, 5941, 6147, 6153, 7428, 7439, 7441, 7452, 7456, 7457, 7458, 7462, 7555, 7564, 7837, 7935, 8125, 8126, 8127, 8128, 8175, 8189, 8190, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200, 8201, 8202, 8208, 8209, 8210, 8218, 8219, 8228, 8232, 8233, 8239, 8242, 8249, 8250, 8257, 8259, 8260, 8270, 8275, 8282, 8287, 8450, 8458, 8459, 8460, 8461, 8462, 8464, 8465, 8466, 8467, 8469, 8473, 8474, 8475, 8476, 8477, 8484, 8488, 8490, 8492, 8493, 8494, 8495, 8496, 8497, 8499, 8500, 8505, 8509, 8517, 8518, 8519, 8520, 8521, 8544, 8548, 8553, 8556, 8557, 8558, 8559, 8560, 8564, 8569, 8572, 8573, 8574, 8722, 8725, 8726, 8727, 8739, 8744, 8746, 8758, 8764, 8868, 8897, 8899, 8959, 9075, 9076, 9082, 9213, 9585, 9587, 10088, 10089, 10094, 10095, 10098, 10099, 10100, 10101, 10133, 10134, 10187, 10189, 10201, 10539, 10540, 10741, 10744, 10745, 10799, 11397, 11406, 11410, 11412, 11416, 11418, 11422, 11423, 11426, 11427, 11428, 11429, 11430, 11432, 11436, 11450, 11462, 11466, 11468, 11472, 11474, 11576, 11577, 11599, 11601, 11604, 11605, 11613, 11840, 12034, 12035, 12295, 12308, 12309, 12339, 12448, 12755, 12756, 20022, 20031, 42192, 42193, 42194, 42195, 42196, 42198, 42199, 42201, 42202, 42204, 42205, 42207, 42208, 42209, 42210, 42211, 42214, 42215, 42218, 42219, 42220, 42222, 42224, 42226, 42227, 42228, 42232, 42233, 42237, 42239, 42510, 42564, 42567, 42719, 42731, 42735, 42801, 42842, 42858, 42862, 42872, 42889, 42892, 42904, 42905, 42911, 42923, 42930, 42931, 42932, 43826, 43829, 43837, 43847, 43848, 43854, 43858, 43866, 43893, 43905, 43907, 43923, 43945, 43946, 43951, 64422, 64423, 64424, 64425, 64426, 64427, 64428, 64429, 64830, 64831, 65072, 65101, 65102, 65103, 65112, 65128, 65165, 65166, 65257, 65258, 65259, 65260, 65282, 65284, 65285, 65286, 65287, 65290, 65291, 65293, 65294, 65295, 65296, 65297, 65298, 65299, 65300, 65301, 65302, 65303, 65304, 65305, 65308, 65309, 65310, 65312, 65313, 65314, 65315, 65316, 65317, 65318, 65319, 65320, 65321, 65322, 65323, 65324, 65325, 65326, 65327, 65328, 65329, 65330, 65331, 65332, 65333, 65334, 65335, 65336, 65337, 65338, 65339, 65340, 65341, 65342, 65343, 65344, 65345, 65346, 65347, 65348, 65349, 65350, 65351, 65352, 65353, 65354, 65355, 65356, 65357, 65358, 65359, 65360, 65361, 65362, 65363, 65364, 65365, 65366, 65367, 65368, 65369, 65370, 65371, 65372, 65373, 65512, 66178, 66182, 66183, 66186, 66192, 66194, 66197, 66198, 66199, 66203, 66208, 66209, 66210, 66213, 66219, 66224, 66225, 66226, 66228, 66255, 66293, 66305, 66306, 66313, 66321, 66325, 66327, 66330, 66335, 66336, 66338, 66564, 66581, 66587, 66592, 66604, 66621, 66632, 66740, 66754, 66766, 66770, 66794, 66806, 66835, 66838, 66840, 66844, 66845, 66853, 66854, 66855, 68176, 70864, 71430, 71434, 71438, 71439, 71840, 71842, 71843, 71844, 71846, 71849, 71852, 71854, 71855, 71858, 71861, 71864, 71867, 71868, 71872, 71873, 71874, 71875, 71876, 71878, 71880, 71882, 71884, 71893, 71894, 71895, 71896, 71900, 71904, 71909, 71910, 71913, 71916, 71919, 71922, 93960, 93962, 93974, 93992, 94005, 94010, 94011, 94015, 94016, 94018, 94019, 94033, 94034, 119060, 119149, 119302, 119309, 119311, 119314, 119315, 119318, 119338, 119350, 119351, 119354, 119355, 119808, 119809, 119810, 119811, 119812, 119813, 119814, 119815, 119816, 119817, 119818, 119819, 119820, 119821, 119822, 119823, 119824, 119825, 119826, 119827, 119828, 119829, 119830, 119831, 119832, 119833, 119834, 119835, 119836, 119837, 119838, 119839, 119840, 119841, 119842, 119843, 119844, 119845, 119847, 119848, 119849, 119850, 119851, 119852, 119853, 119854, 119855, 119856, 119857, 119858, 119859, 119860, 119861, 119862, 119863, 119864, 119865, 119866, 119867, 119868, 119869, 119870, 119871, 119872, 119873, 119874, 119875, 119876, 119877, 119878, 119879, 119880, 119881, 119882, 119883, 119884, 119885, 119886, 119887, 119888, 119889, 119890, 119891, 119892, 119894, 119895, 119896, 119897, 119899, 119900, 119901, 119902, 119903, 119904, 119905, 119906, 119907, 119908, 119909, 119910, 119911, 119912, 119913, 119914, 119915, 119916, 119917, 119918, 119919, 119920, 119921, 119922, 119923, 119924, 119925, 119926, 119927, 119928, 119929, 119930, 119931, 119932, 119933, 119934, 119935, 119936, 119937, 119938, 119939, 119940, 119941, 119942, 119943, 119944, 119945, 119946, 119947, 119948, 119949, 119951, 119952, 119953, 119954, 119955, 119956, 119957, 119958, 119959, 119960, 119961, 119962, 119963, 119964, 119966, 119967, 119970, 119973, 119974, 119977, 119978, 119979, 119980, 119982, 119983, 119984, 119985, 119986, 119987, 119988, 119989, 119990, 119991, 119992, 119993, 119995, 119997, 119998, 119999, 120000, 120001, 120003, 120005, 120006, 120007, 120008, 120009, 120010, 120011, 120012, 120013, 120014, 120015, 120016, 120017, 120018, 120019, 120020, 120021, 120022, 120023, 120024, 120025, 120026, 120027, 120028, 120029, 120030, 120031, 120032, 120033, 120034, 120035, 120036, 120037, 120038, 120039, 120040, 120041, 120042, 120043, 120044, 120045, 120046, 120047, 120048, 120049, 120050, 120051, 120052, 120053, 120055, 120056, 120057, 120058, 120059, 120060, 120061, 120062, 120063, 120064, 120065, 120066, 120067, 120068, 120069, 120071, 120072, 120073, 120074, 120077, 120078, 120079, 120080, 120081, 120082, 120083, 120084, 120086, 120087, 120088, 120089, 120090, 120091, 120092, 120094, 120095, 120096, 120097, 120098, 120099, 120100, 120101, 120102, 120103, 120104, 120105, 120107, 120108, 120109, 120110, 120111, 120112, 120113, 120114, 120115, 120116, 120117, 120118, 120119, 120120, 120121, 120123, 120124, 120125, 120126, 120128, 120129, 120130, 120131, 120132, 120134, 120138, 120139, 120140, 120141, 120142, 120143, 120144, 120146, 120147, 120148, 120149, 120150, 120151, 120152, 120153, 120154, 120155, 120156, 120157, 120159, 120160, 120161, 120162, 120163, 120164, 120165, 120166, 120167, 120168, 120169, 120170, 120171, 120172, 120173, 120174, 120175, 120176, 120177, 120178, 120179, 120180, 120181, 120182, 120183, 120184, 120185, 120186, 120187, 120188, 120189, 120190, 120191, 120192, 120193, 120194, 120195, 120196, 120197, 120198, 120199, 120200, 120201, 120202, 120203, 120204, 120205, 120206, 120207, 120208, 120209, 120211, 120212, 120213, 120214, 120215, 120216, 120217, 120218, 120219, 120220, 120221, 120222, 120223, 120224, 120225, 120226, 120227, 120228, 120229, 120230, 120231, 120232, 120233, 120234, 120235, 120236, 120237, 120238, 120239, 120240, 120241, 120242, 120243, 120244, 120245, 120246, 120247, 120248, 120249, 120250, 120251, 120252, 120253, 120254, 120255, 120256, 120257, 120258, 120259, 120260, 120261, 120263, 120264, 120265, 120266, 120267, 120268, 120269, 120270, 120271, 120272, 120273, 120274, 120275, 120276, 120277, 120278, 120279, 120280, 120281, 120282, 120283, 120284, 120285, 120286, 120287, 120288, 120289, 120290, 120291, 120292, 120293, 120294, 120295, 120296, 120297, 120298, 120299, 120300, 120301, 120302, 120303, 120304, 120305, 120306, 120307, 120308, 120309, 120310, 120311, 120312, 120313, 120315, 120316, 120317, 120318, 120319, 120320, 120321, 120322, 120323, 120324, 120325, 120326, 120327, 120328, 120329, 120330, 120331, 120332, 120333, 120334, 120335, 120336, 120337, 120338, 120339, 120340, 120341, 120342, 120343, 120344, 120345, 120346, 120347, 120348, 120349, 120350, 120351, 120352, 120353, 120354, 120355, 120356, 120357, 120358, 120359, 120360, 120361, 120362, 120363, 120364, 120365, 120367, 120368, 120369, 120370, 120371, 120372, 120373, 120374, 120375, 120376, 120377, 120378, 120379, 120380, 120381, 120382, 120383, 120384, 120385, 120386, 120387, 120388, 120389, 120390, 120391, 120392, 120393, 120394, 120395, 120396, 120397, 120398, 120399, 120400, 120401, 120402, 120403, 120404, 120405, 120406, 120407, 120408, 120409, 120410, 120411, 120412, 120413, 120414, 120415, 120416, 120417, 120419, 120420, 120421, 120422, 120423, 120424, 120425, 120426, 120427, 120428, 120429, 120430, 120431, 120432, 120433, 120434, 120435, 120436, 120437, 120438, 120439, 120440, 120441, 120442, 120443, 120444, 120445, 120446, 120447, 120448, 120449, 120450, 120451, 120452, 120453, 120454, 120455, 120456, 120457, 120458, 120459, 120460, 120461, 120462, 120463, 120464, 120465, 120466, 120467, 120468, 120469, 120471, 120472, 120473, 120474, 120475, 120476, 120477, 120478, 120479, 120480, 120481, 120482, 120483, 120484, 120488, 120489, 120492, 120493, 120494, 120496, 120497, 120499, 120500, 120502, 120504, 120507, 120508, 120510, 120514, 120516, 120522, 120526, 120528, 120530, 120532, 120534, 120544, 120546, 120547, 120550, 120551, 120552, 120554, 120555, 120557, 120558, 120560, 120562, 120565, 120566, 120568, 120572, 120574, 120580, 120584, 120586, 120588, 120590, 120592, 120602, 120604, 120605, 120608, 120609, 120610, 120612, 120613, 120615, 120616, 120618, 120620, 120623, 120624, 120626, 120630, 120632, 120638, 120642, 120644, 120646, 120648, 120650, 120660, 120662, 120663, 120666, 120667, 120668, 120670, 120671, 120673, 120674, 120676, 120678, 120681, 120682, 120684, 120688, 120690, 120696, 120700, 120702, 120704, 120706, 120708, 120718, 120720, 120721, 120724, 120725, 120726, 120728, 120729, 120731, 120732, 120734, 120736, 120739, 120740, 120742, 120746, 120748, 120754, 120758, 120760, 120762, 120764, 120766, 120776, 120778, 120782, 120783, 120784, 120785, 120786, 120787, 120788, 120789, 120790, 120791, 120792, 120793, 120794, 120795, 120796, 120797, 120798, 120799, 120800, 120801, 120802, 120803, 120804, 120805, 120806, 120807, 120808, 120809, 120810, 120811, 120812, 120813, 120814, 120815, 120816, 120817, 120818, 120819, 120820, 120821, 120822, 120823, 120824, 120825, 120826, 120827, 120828, 120829, 120830, 120831, 125127, 125131, 126464, 126500, 126564, 126592, 126596, 128844, 128872, 130032, 130033, 130034, 130035, 130036, 130037, 130038, 130039, 130040, 130041}, + With: []rune{44, 102, 98, 103, 82, 50, 51, 53, 115, 73, 33, 51, 56, 56, 63, 97, 103, 121, 105, 105, 119, 117, 121, 63, 96, 96, 96, 96, 96, 60, 62, 94, 94, 96, 96, 96, 58, 45, 105, 126, 96, 58, 96, 105, 59, 74, 96, 65, 66, 69, 90, 72, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 89, 70, 50, 99, 106, 67, 77, 83, 73, 74, 115, 105, 106, 119, 86, 118, 89, 121, 104, 101, 73, 105, 51, 100, 71, 113, 87, 119, 85, 83, 79, 96, 96, 119, 113, 113, 104, 110, 110, 117, 103, 102, 111, 58, 108, 58, 108, 118, 96, 108, 111, 96, 44, 108, 111, 46, 108, 111, 86, 44, 42, 111, 111, 45, 111, 46, 73, 111, 86, 46, 46, 58, 58, 79, 108, 96, 96, 95, 58, 111, 63, 79, 56, 57, 111, 57, 56, 58, 111, 56, 79, 79, 57, 111, 111, 111, 111, 111, 111, 111, 111, 57, 111, 111, 111, 111, 111, 121, 111, 85, 79, 68, 82, 84, 105, 89, 65, 74, 69, 63, 87, 77, 72, 89, 71, 104, 90, 52, 98, 82, 87, 83, 86, 83, 76, 67, 80, 75, 100, 54, 71, 66, 61, 86, 62, 60, 96, 85, 80, 100, 98, 74, 76, 50, 120, 72, 120, 82, 98, 70, 65, 68, 68, 77, 66, 88, 120, 32, 60, 88, 73, 96, 75, 77, 58, 43, 47, 58, 58, 99, 111, 111, 117, 118, 119, 122, 114, 103, 121, 102, 121, 96, 105, 96, 126, 96, 96, 96, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 45, 44, 96, 46, 32, 32, 32, 96, 60, 62, 47, 45, 47, 42, 126, 58, 32, 67, 103, 72, 72, 72, 104, 73, 73, 76, 108, 78, 80, 81, 82, 82, 82, 90, 90, 75, 66, 67, 101, 101, 69, 70, 77, 111, 105, 121, 68, 100, 101, 105, 106, 73, 86, 88, 76, 67, 68, 77, 105, 118, 120, 73, 99, 100, 45, 47, 92, 42, 73, 118, 85, 58, 126, 84, 118, 85, 69, 105, 112, 97, 73, 47, 88, 40, 41, 60, 62, 40, 41, 123, 125, 43, 45, 47, 92, 84, 120, 120, 92, 47, 92, 120, 114, 72, 73, 75, 77, 78, 79, 111, 80, 112, 67, 99, 84, 89, 88, 45, 47, 57, 51, 76, 54, 86, 69, 73, 33, 79, 81, 88, 61, 92, 47, 79, 40, 41, 47, 61, 47, 92, 92, 47, 66, 80, 100, 68, 84, 71, 75, 74, 67, 90, 70, 77, 78, 76, 83, 82, 86, 72, 87, 88, 89, 65, 69, 73, 79, 85, 46, 44, 58, 61, 46, 50, 105, 86, 63, 50, 115, 50, 51, 57, 38, 58, 96, 70, 102, 117, 51, 74, 88, 66, 101, 102, 111, 114, 114, 117, 117, 121, 105, 114, 119, 122, 118, 115, 99, 111, 111, 111, 111, 111, 111, 111, 111, 40, 41, 58, 95, 95, 95, 45, 92, 108, 108, 111, 111, 111, 111, 34, 36, 37, 38, 96, 42, 43, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 60, 61, 62, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 73, 66, 69, 70, 124, 88, 79, 80, 83, 84, 43, 65, 66, 67, 70, 79, 77, 84, 89, 88, 72, 90, 66, 67, 124, 77, 84, 88, 56, 42, 108, 88, 79, 67, 76, 83, 111, 99, 115, 82, 79, 85, 55, 111, 117, 78, 79, 75, 67, 86, 70, 76, 88, 46, 79, 118, 119, 119, 119, 86, 70, 76, 89, 69, 90, 57, 69, 52, 76, 79, 85, 53, 84, 118, 115, 70, 105, 122, 55, 111, 51, 57, 54, 57, 111, 117, 121, 79, 90, 87, 67, 88, 87, 67, 86, 84, 76, 73, 82, 83, 51, 62, 65, 85, 89, 96, 96, 123, 46, 51, 86, 92, 55, 70, 82, 76, 60, 62, 47, 92, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 67, 68, 71, 74, 75, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 102, 104, 105, 106, 107, 108, 110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 68, 69, 70, 71, 74, 75, 76, 77, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 68, 69, 70, 71, 73, 74, 75, 76, 77, 79, 83, 84, 85, 86, 87, 88, 89, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 73, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 105, 65, 66, 69, 90, 72, 73, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 112, 65, 66, 69, 90, 72, 73, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 112, 65, 66, 69, 90, 72, 73, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 112, 65, 66, 69, 90, 72, 73, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 112, 65, 66, 69, 90, 72, 73, 75, 77, 78, 79, 80, 84, 89, 88, 97, 121, 105, 118, 111, 112, 111, 117, 112, 70, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57, 108, 56, 108, 111, 111, 108, 111, 67, 84, 79, 73, 50, 51, 52, 53, 54, 55, 56, 57}, + Locale: "_common", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 184, Hi: 383, Stride: 199}, + {Lo: 388, Hi: 397, Stride: 9}, + {Lo: 422, Hi: 423, Stride: 1}, + {Lo: 439, Hi: 444, Stride: 5}, + {Lo: 445, Hi: 451, Stride: 3}, + {Lo: 540, Hi: 546, Stride: 6}, + {Lo: 547, Hi: 577, Stride: 30}, + {Lo: 593, Hi: 609, Stride: 16}, + {Lo: 611, Hi: 617, Stride: 6}, + {Lo: 618, Hi: 623, Stride: 5}, + {Lo: 651, Hi: 655, Stride: 4}, + {Lo: 660, Hi: 697, Stride: 37}, + {Lo: 699, Hi: 702, Stride: 1}, + {Lo: 706, Hi: 708, Stride: 1}, + {Lo: 710, Hi: 714, Stride: 2}, + {Lo: 715, Hi: 720, Stride: 5}, + {Lo: 727, Hi: 731, Stride: 4}, + {Lo: 732, Hi: 756, Stride: 24}, + {Lo: 760, Hi: 884, Stride: 124}, + {Lo: 890, Hi: 894, Stride: 4}, + {Lo: 895, Hi: 900, Stride: 5}, + {Lo: 913, Hi: 914, Stride: 1}, + {Lo: 917, Hi: 919, Stride: 1}, + {Lo: 922, Hi: 924, Stride: 2}, + {Lo: 925, Hi: 929, Stride: 2}, + {Lo: 932, Hi: 933, Stride: 1}, + {Lo: 935, Hi: 945, Stride: 10}, + {Lo: 947, Hi: 953, Stride: 6}, + {Lo: 957, Hi: 965, Stride: 2}, + {Lo: 978, Hi: 988, Stride: 10}, + {Lo: 1000, Hi: 1010, Stride: 10}, + {Lo: 1011, Hi: 1017, Stride: 6}, + {Lo: 1018, Hi: 1029, Stride: 11}, + {Lo: 1030, Hi: 1032, Stride: 2}, + {Lo: 1109, Hi: 1110, Stride: 1}, + {Lo: 1112, Hi: 1121, Stride: 9}, + {Lo: 1140, Hi: 1141, Stride: 1}, + {Lo: 1198, Hi: 1199, Stride: 1}, + {Lo: 1211, Hi: 1213, Stride: 2}, + {Lo: 1216, Hi: 1231, Stride: 15}, + {Lo: 1248, Hi: 1281, Stride: 33}, + {Lo: 1292, Hi: 1307, Stride: 15}, + {Lo: 1308, Hi: 1309, Stride: 1}, + {Lo: 1357, Hi: 1359, Stride: 2}, + {Lo: 1365, Hi: 1370, Stride: 5}, + {Lo: 1373, Hi: 1377, Stride: 4}, + {Lo: 1379, Hi: 1382, Stride: 3}, + {Lo: 1392, Hi: 1400, Stride: 8}, + {Lo: 1404, Hi: 1405, Stride: 1}, + {Lo: 1409, Hi: 1412, Stride: 3}, + {Lo: 1413, Hi: 1417, Stride: 4}, + {Lo: 1472, Hi: 1475, Stride: 3}, + {Lo: 1493, Hi: 1496, Stride: 3}, + {Lo: 1497, Hi: 1503, Stride: 6}, + {Lo: 1505, Hi: 1523, Stride: 18}, + {Lo: 1549, Hi: 1575, Stride: 26}, + {Lo: 1607, Hi: 1632, Stride: 25}, + {Lo: 1633, Hi: 1637, Stride: 4}, + {Lo: 1639, Hi: 1643, Stride: 4}, + {Lo: 1645, Hi: 1726, Stride: 81}, + {Lo: 1729, Hi: 1748, Stride: 19}, + {Lo: 1749, Hi: 1776, Stride: 27}, + {Lo: 1777, Hi: 1781, Stride: 4}, + {Lo: 1783, Hi: 1793, Stride: 10}, + {Lo: 1794, Hi: 1796, Stride: 1}, + {Lo: 1984, Hi: 1994, Stride: 10}, + {Lo: 2036, Hi: 2037, Stride: 1}, + {Lo: 2042, Hi: 2307, Stride: 265}, + {Lo: 2406, Hi: 2429, Stride: 23}, + {Lo: 2534, Hi: 2538, Stride: 4}, + {Lo: 2541, Hi: 2662, Stride: 121}, + {Lo: 2663, Hi: 2666, Stride: 3}, + {Lo: 2691, Hi: 2790, Stride: 99}, + {Lo: 2819, Hi: 2848, Stride: 29}, + {Lo: 2918, Hi: 2920, Stride: 2}, + {Lo: 3046, Hi: 3074, Stride: 28}, + {Lo: 3174, Hi: 3202, Stride: 28}, + {Lo: 3302, Hi: 3330, Stride: 28}, + {Lo: 3360, Hi: 3430, Stride: 70}, + {Lo: 3437, Hi: 3458, Stride: 21}, + {Lo: 3664, Hi: 3792, Stride: 128}, + {Lo: 4125, Hi: 4160, Stride: 35}, + {Lo: 4327, Hi: 4351, Stride: 24}, + {Lo: 4608, Hi: 5024, Stride: 208}, + {Lo: 5025, Hi: 5026, Stride: 1}, + {Lo: 5029, Hi: 5033, Stride: 4}, + {Lo: 5034, Hi: 5036, Stride: 1}, + {Lo: 5038, Hi: 5043, Stride: 5}, + {Lo: 5047, Hi: 5051, Stride: 4}, + {Lo: 5053, Hi: 5056, Stride: 3}, + {Lo: 5058, Hi: 5059, Stride: 1}, + {Lo: 5070, Hi: 5071, Stride: 1}, + {Lo: 5074, Hi: 5076, Stride: 2}, + {Lo: 5077, Hi: 5081, Stride: 4}, + {Lo: 5082, Hi: 5086, Stride: 4}, + {Lo: 5087, Hi: 5090, Stride: 3}, + {Lo: 5094, Hi: 5095, Stride: 1}, + {Lo: 5102, Hi: 5107, Stride: 5}, + {Lo: 5108, Hi: 5120, Stride: 12}, + {Lo: 5167, Hi: 5171, Stride: 4}, + {Lo: 5176, Hi: 5194, Stride: 18}, + {Lo: 5196, Hi: 5229, Stride: 33}, + {Lo: 5231, Hi: 5234, Stride: 3}, + {Lo: 5261, Hi: 5290, Stride: 29}, + {Lo: 5311, Hi: 5441, Stride: 130}, + {Lo: 5500, Hi: 5501, Stride: 1}, + {Lo: 5511, Hi: 5551, Stride: 40}, + {Lo: 5556, Hi: 5573, Stride: 17}, + {Lo: 5598, Hi: 5610, Stride: 12}, + {Lo: 5616, Hi: 5623, Stride: 7}, + {Lo: 5741, Hi: 5742, Stride: 1}, + {Lo: 5760, Hi: 5810, Stride: 50}, + {Lo: 5815, Hi: 5825, Stride: 10}, + {Lo: 5836, Hi: 5845, Stride: 9}, + {Lo: 5846, Hi: 5868, Stride: 22}, + {Lo: 5869, Hi: 5941, Stride: 72}, + {Lo: 6147, Hi: 6153, Stride: 6}, + {Lo: 7428, Hi: 7439, Stride: 11}, + {Lo: 7441, Hi: 7452, Stride: 11}, + {Lo: 7456, Hi: 7458, Stride: 1}, + {Lo: 7462, Hi: 7555, Stride: 93}, + {Lo: 7564, Hi: 7837, Stride: 273}, + {Lo: 7935, Hi: 8125, Stride: 190}, + {Lo: 8126, Hi: 8128, Stride: 1}, + {Lo: 8175, Hi: 8189, Stride: 14}, + {Lo: 8190, Hi: 8192, Stride: 2}, + {Lo: 8193, Hi: 8202, Stride: 1}, + {Lo: 8208, Hi: 8210, Stride: 1}, + {Lo: 8218, Hi: 8219, Stride: 1}, + {Lo: 8228, Hi: 8232, Stride: 4}, + {Lo: 8233, Hi: 8239, Stride: 6}, + {Lo: 8242, Hi: 8249, Stride: 7}, + {Lo: 8250, Hi: 8257, Stride: 7}, + {Lo: 8259, Hi: 8260, Stride: 1}, + {Lo: 8270, Hi: 8275, Stride: 5}, + {Lo: 8282, Hi: 8287, Stride: 5}, + {Lo: 8450, Hi: 8458, Stride: 8}, + {Lo: 8459, Hi: 8462, Stride: 1}, + {Lo: 8464, Hi: 8467, Stride: 1}, + {Lo: 8469, Hi: 8473, Stride: 4}, + {Lo: 8474, Hi: 8477, Stride: 1}, + {Lo: 8484, Hi: 8488, Stride: 4}, + {Lo: 8490, Hi: 8492, Stride: 2}, + {Lo: 8493, Hi: 8497, Stride: 1}, + {Lo: 8499, Hi: 8500, Stride: 1}, + {Lo: 8505, Hi: 8509, Stride: 4}, + {Lo: 8517, Hi: 8521, Stride: 1}, + {Lo: 8544, Hi: 8548, Stride: 4}, + {Lo: 8553, Hi: 8556, Stride: 3}, + {Lo: 8557, Hi: 8560, Stride: 1}, + {Lo: 8564, Hi: 8569, Stride: 5}, + {Lo: 8572, Hi: 8574, Stride: 1}, + {Lo: 8722, Hi: 8725, Stride: 3}, + {Lo: 8726, Hi: 8727, Stride: 1}, + {Lo: 8739, Hi: 8744, Stride: 5}, + {Lo: 8746, Hi: 8758, Stride: 12}, + {Lo: 8764, Hi: 8868, Stride: 104}, + {Lo: 8897, Hi: 8899, Stride: 2}, + {Lo: 8959, Hi: 9075, Stride: 116}, + {Lo: 9076, Hi: 9082, Stride: 6}, + {Lo: 9213, Hi: 9585, Stride: 372}, + {Lo: 9587, Hi: 10088, Stride: 501}, + {Lo: 10089, Hi: 10094, Stride: 5}, + {Lo: 10095, Hi: 10098, Stride: 3}, + {Lo: 10099, Hi: 10101, Stride: 1}, + {Lo: 10133, Hi: 10134, Stride: 1}, + {Lo: 10187, Hi: 10189, Stride: 2}, + {Lo: 10201, Hi: 10539, Stride: 338}, + {Lo: 10540, Hi: 10741, Stride: 201}, + {Lo: 10744, Hi: 10745, Stride: 1}, + {Lo: 10799, Hi: 11397, Stride: 598}, + {Lo: 11406, Hi: 11410, Stride: 4}, + {Lo: 11412, Hi: 11416, Stride: 4}, + {Lo: 11418, Hi: 11422, Stride: 4}, + {Lo: 11423, Hi: 11426, Stride: 3}, + {Lo: 11427, Hi: 11430, Stride: 1}, + {Lo: 11432, Hi: 11436, Stride: 4}, + {Lo: 11450, Hi: 11462, Stride: 12}, + {Lo: 11466, Hi: 11468, Stride: 2}, + {Lo: 11472, Hi: 11474, Stride: 2}, + {Lo: 11576, Hi: 11577, Stride: 1}, + {Lo: 11599, Hi: 11601, Stride: 2}, + {Lo: 11604, Hi: 11605, Stride: 1}, + {Lo: 11613, Hi: 11840, Stride: 227}, + {Lo: 12034, Hi: 12035, Stride: 1}, + {Lo: 12295, Hi: 12308, Stride: 13}, + {Lo: 12309, Hi: 12339, Stride: 30}, + {Lo: 12448, Hi: 12755, Stride: 307}, + {Lo: 12756, Hi: 20022, Stride: 7266}, + {Lo: 20031, Hi: 42192, Stride: 22161}, + {Lo: 42193, Hi: 42196, Stride: 1}, + {Lo: 42198, Hi: 42199, Stride: 1}, + {Lo: 42201, Hi: 42202, Stride: 1}, + {Lo: 42204, Hi: 42205, Stride: 1}, + {Lo: 42207, Hi: 42211, Stride: 1}, + {Lo: 42214, Hi: 42215, Stride: 1}, + {Lo: 42218, Hi: 42220, Stride: 1}, + {Lo: 42222, Hi: 42226, Stride: 2}, + {Lo: 42227, Hi: 42228, Stride: 1}, + {Lo: 42232, Hi: 42233, Stride: 1}, + {Lo: 42237, Hi: 42239, Stride: 2}, + {Lo: 42510, Hi: 42564, Stride: 54}, + {Lo: 42567, Hi: 42719, Stride: 152}, + {Lo: 42731, Hi: 42735, Stride: 4}, + {Lo: 42801, Hi: 42842, Stride: 41}, + {Lo: 42858, Hi: 42862, Stride: 4}, + {Lo: 42872, Hi: 42889, Stride: 17}, + {Lo: 42892, Hi: 42904, Stride: 12}, + {Lo: 42905, Hi: 42911, Stride: 6}, + {Lo: 42923, Hi: 42930, Stride: 7}, + {Lo: 42931, Hi: 42932, Stride: 1}, + {Lo: 43826, Hi: 43829, Stride: 3}, + {Lo: 43837, Hi: 43847, Stride: 10}, + {Lo: 43848, Hi: 43854, Stride: 6}, + {Lo: 43858, Hi: 43866, Stride: 8}, + {Lo: 43893, Hi: 43905, Stride: 12}, + {Lo: 43907, Hi: 43923, Stride: 16}, + {Lo: 43945, Hi: 43946, Stride: 1}, + {Lo: 43951, Hi: 64422, Stride: 20471}, + {Lo: 64423, Hi: 64429, Stride: 1}, + {Lo: 64830, Hi: 64831, Stride: 1}, + {Lo: 65072, Hi: 65101, Stride: 29}, + {Lo: 65102, Hi: 65103, Stride: 1}, + {Lo: 65112, Hi: 65128, Stride: 16}, + {Lo: 65165, Hi: 65166, Stride: 1}, + {Lo: 65257, Hi: 65260, Stride: 1}, + {Lo: 65282, Hi: 65284, Stride: 2}, + {Lo: 65285, Hi: 65287, Stride: 1}, + {Lo: 65290, Hi: 65291, Stride: 1}, + {Lo: 65293, Hi: 65305, Stride: 1}, + {Lo: 65308, Hi: 65310, Stride: 1}, + {Lo: 65312, Hi: 65373, Stride: 1}, + {Lo: 65512, Hi: 65512, Stride: 1}, + }, + R32: []unicode.Range32{ + {Lo: 66178, Hi: 66182, Stride: 4}, + {Lo: 66183, Hi: 66186, Stride: 3}, + {Lo: 66192, Hi: 66194, Stride: 2}, + {Lo: 66197, Hi: 66199, Stride: 1}, + {Lo: 66203, Hi: 66208, Stride: 5}, + {Lo: 66209, Hi: 66210, Stride: 1}, + {Lo: 66213, Hi: 66219, Stride: 6}, + {Lo: 66224, Hi: 66226, Stride: 1}, + {Lo: 66228, Hi: 66255, Stride: 27}, + {Lo: 66293, Hi: 66305, Stride: 12}, + {Lo: 66306, Hi: 66313, Stride: 7}, + {Lo: 66321, Hi: 66325, Stride: 4}, + {Lo: 66327, Hi: 66330, Stride: 3}, + {Lo: 66335, Hi: 66336, Stride: 1}, + {Lo: 66338, Hi: 66564, Stride: 226}, + {Lo: 66581, Hi: 66587, Stride: 6}, + {Lo: 66592, Hi: 66604, Stride: 12}, + {Lo: 66621, Hi: 66632, Stride: 11}, + {Lo: 66740, Hi: 66754, Stride: 14}, + {Lo: 66766, Hi: 66770, Stride: 4}, + {Lo: 66794, Hi: 66806, Stride: 12}, + {Lo: 66835, Hi: 66838, Stride: 3}, + {Lo: 66840, Hi: 66844, Stride: 4}, + {Lo: 66845, Hi: 66853, Stride: 8}, + {Lo: 66854, Hi: 66855, Stride: 1}, + {Lo: 68176, Hi: 70864, Stride: 2688}, + {Lo: 71430, Hi: 71438, Stride: 4}, + {Lo: 71439, Hi: 71840, Stride: 401}, + {Lo: 71842, Hi: 71844, Stride: 1}, + {Lo: 71846, Hi: 71852, Stride: 3}, + {Lo: 71854, Hi: 71855, Stride: 1}, + {Lo: 71858, Hi: 71867, Stride: 3}, + {Lo: 71868, Hi: 71872, Stride: 4}, + {Lo: 71873, Hi: 71876, Stride: 1}, + {Lo: 71878, Hi: 71884, Stride: 2}, + {Lo: 71893, Hi: 71896, Stride: 1}, + {Lo: 71900, Hi: 71904, Stride: 4}, + {Lo: 71909, Hi: 71910, Stride: 1}, + {Lo: 71913, Hi: 71922, Stride: 3}, + {Lo: 93960, Hi: 93962, Stride: 2}, + {Lo: 93974, Hi: 93992, Stride: 18}, + {Lo: 94005, Hi: 94010, Stride: 5}, + {Lo: 94011, Hi: 94015, Stride: 4}, + {Lo: 94016, Hi: 94018, Stride: 2}, + {Lo: 94019, Hi: 94033, Stride: 14}, + {Lo: 94034, Hi: 119060, Stride: 25026}, + {Lo: 119149, Hi: 119302, Stride: 153}, + {Lo: 119309, Hi: 119311, Stride: 2}, + {Lo: 119314, Hi: 119315, Stride: 1}, + {Lo: 119318, Hi: 119338, Stride: 20}, + {Lo: 119350, Hi: 119351, Stride: 1}, + {Lo: 119354, Hi: 119355, Stride: 1}, + {Lo: 119808, Hi: 119845, Stride: 1}, + {Lo: 119847, Hi: 119892, Stride: 1}, + {Lo: 119894, Hi: 119897, Stride: 1}, + {Lo: 119899, Hi: 119949, Stride: 1}, + {Lo: 119951, Hi: 119964, Stride: 1}, + {Lo: 119966, Hi: 119967, Stride: 1}, + {Lo: 119970, Hi: 119973, Stride: 3}, + {Lo: 119974, Hi: 119977, Stride: 3}, + {Lo: 119978, Hi: 119980, Stride: 1}, + {Lo: 119982, Hi: 119993, Stride: 1}, + {Lo: 119995, Hi: 119997, Stride: 2}, + {Lo: 119998, Hi: 120001, Stride: 1}, + {Lo: 120003, Hi: 120005, Stride: 2}, + {Lo: 120006, Hi: 120053, Stride: 1}, + {Lo: 120055, Hi: 120069, Stride: 1}, + {Lo: 120071, Hi: 120074, Stride: 1}, + {Lo: 120077, Hi: 120084, Stride: 1}, + {Lo: 120086, Hi: 120092, Stride: 1}, + {Lo: 120094, Hi: 120105, Stride: 1}, + {Lo: 120107, Hi: 120121, Stride: 1}, + {Lo: 120123, Hi: 120126, Stride: 1}, + {Lo: 120128, Hi: 120132, Stride: 1}, + {Lo: 120134, Hi: 120138, Stride: 4}, + {Lo: 120139, Hi: 120144, Stride: 1}, + {Lo: 120146, Hi: 120157, Stride: 1}, + {Lo: 120159, Hi: 120209, Stride: 1}, + {Lo: 120211, Hi: 120261, Stride: 1}, + {Lo: 120263, Hi: 120313, Stride: 1}, + {Lo: 120315, Hi: 120365, Stride: 1}, + {Lo: 120367, Hi: 120417, Stride: 1}, + {Lo: 120419, Hi: 120469, Stride: 1}, + {Lo: 120471, Hi: 120484, Stride: 1}, + {Lo: 120488, Hi: 120489, Stride: 1}, + {Lo: 120492, Hi: 120494, Stride: 1}, + {Lo: 120496, Hi: 120497, Stride: 1}, + {Lo: 120499, Hi: 120500, Stride: 1}, + {Lo: 120502, Hi: 120504, Stride: 2}, + {Lo: 120507, Hi: 120508, Stride: 1}, + {Lo: 120510, Hi: 120514, Stride: 4}, + {Lo: 120516, Hi: 120522, Stride: 6}, + {Lo: 120526, Hi: 120534, Stride: 2}, + {Lo: 120544, Hi: 120546, Stride: 2}, + {Lo: 120547, Hi: 120550, Stride: 3}, + {Lo: 120551, Hi: 120552, Stride: 1}, + {Lo: 120554, Hi: 120555, Stride: 1}, + {Lo: 120557, Hi: 120558, Stride: 1}, + {Lo: 120560, Hi: 120562, Stride: 2}, + {Lo: 120565, Hi: 120566, Stride: 1}, + {Lo: 120568, Hi: 120572, Stride: 4}, + {Lo: 120574, Hi: 120580, Stride: 6}, + {Lo: 120584, Hi: 120592, Stride: 2}, + {Lo: 120602, Hi: 120604, Stride: 2}, + {Lo: 120605, Hi: 120608, Stride: 3}, + {Lo: 120609, Hi: 120610, Stride: 1}, + {Lo: 120612, Hi: 120613, Stride: 1}, + {Lo: 120615, Hi: 120616, Stride: 1}, + {Lo: 120618, Hi: 120620, Stride: 2}, + {Lo: 120623, Hi: 120624, Stride: 1}, + {Lo: 120626, Hi: 120630, Stride: 4}, + {Lo: 120632, Hi: 120638, Stride: 6}, + {Lo: 120642, Hi: 120650, Stride: 2}, + {Lo: 120660, Hi: 120662, Stride: 2}, + {Lo: 120663, Hi: 120666, Stride: 3}, + {Lo: 120667, Hi: 120668, Stride: 1}, + {Lo: 120670, Hi: 120671, Stride: 1}, + {Lo: 120673, Hi: 120674, Stride: 1}, + {Lo: 120676, Hi: 120678, Stride: 2}, + {Lo: 120681, Hi: 120682, Stride: 1}, + {Lo: 120684, Hi: 120688, Stride: 4}, + {Lo: 120690, Hi: 120696, Stride: 6}, + {Lo: 120700, Hi: 120708, Stride: 2}, + {Lo: 120718, Hi: 120720, Stride: 2}, + {Lo: 120721, Hi: 120724, Stride: 3}, + {Lo: 120725, Hi: 120726, Stride: 1}, + {Lo: 120728, Hi: 120729, Stride: 1}, + {Lo: 120731, Hi: 120732, Stride: 1}, + {Lo: 120734, Hi: 120736, Stride: 2}, + {Lo: 120739, Hi: 120740, Stride: 1}, + {Lo: 120742, Hi: 120746, Stride: 4}, + {Lo: 120748, Hi: 120754, Stride: 6}, + {Lo: 120758, Hi: 120766, Stride: 2}, + {Lo: 120776, Hi: 120778, Stride: 2}, + {Lo: 120782, Hi: 120831, Stride: 1}, + {Lo: 125127, Hi: 125131, Stride: 4}, + {Lo: 126464, Hi: 126500, Stride: 36}, + {Lo: 126564, Hi: 126592, Stride: 28}, + {Lo: 126596, Hi: 128844, Stride: 2248}, + {Lo: 128872, Hi: 130032, Stride: 1160}, + {Lo: 130033, Hi: 130041, Stride: 1}, + }, + LatinOffset: 0, }, - R32: []unicode.Range32{ - {Lo: 66178, Hi: 66182, Stride: 4}, - {Lo: 66183, Hi: 66186, Stride: 3}, - {Lo: 66192, Hi: 66194, Stride: 2}, - {Lo: 66197, Hi: 66199, Stride: 1}, - {Lo: 66203, Hi: 66208, Stride: 5}, - {Lo: 66209, Hi: 66210, Stride: 1}, - {Lo: 66213, Hi: 66219, Stride: 6}, - {Lo: 66224, Hi: 66226, Stride: 1}, - {Lo: 66228, Hi: 66255, Stride: 27}, - {Lo: 66293, Hi: 66305, Stride: 12}, - {Lo: 66306, Hi: 66313, Stride: 7}, - {Lo: 66321, Hi: 66325, Stride: 4}, - {Lo: 66327, Hi: 66330, Stride: 3}, - {Lo: 66335, Hi: 66336, Stride: 1}, - {Lo: 66338, Hi: 66564, Stride: 226}, - {Lo: 66581, Hi: 66587, Stride: 6}, - {Lo: 66592, Hi: 66604, Stride: 12}, - {Lo: 66621, Hi: 66632, Stride: 11}, - {Lo: 66740, Hi: 66754, Stride: 14}, - {Lo: 66766, Hi: 66770, Stride: 4}, - {Lo: 66794, Hi: 66806, Stride: 12}, - {Lo: 66835, Hi: 66838, Stride: 3}, - {Lo: 66840, Hi: 66844, Stride: 4}, - {Lo: 66845, Hi: 66853, Stride: 8}, - {Lo: 66854, Hi: 66855, Stride: 1}, - {Lo: 68176, Hi: 70864, Stride: 2688}, - {Lo: 71430, Hi: 71438, Stride: 4}, - {Lo: 71439, Hi: 71840, Stride: 401}, - {Lo: 71842, Hi: 71844, Stride: 1}, - {Lo: 71846, Hi: 71852, Stride: 3}, - {Lo: 71854, Hi: 71855, Stride: 1}, - {Lo: 71858, Hi: 71867, Stride: 3}, - {Lo: 71868, Hi: 71872, Stride: 4}, - {Lo: 71873, Hi: 71876, Stride: 1}, - {Lo: 71878, Hi: 71884, Stride: 2}, - {Lo: 71893, Hi: 71896, Stride: 1}, - {Lo: 71900, Hi: 71904, Stride: 4}, - {Lo: 71909, Hi: 71910, Stride: 1}, - {Lo: 71913, Hi: 71922, Stride: 3}, - {Lo: 93960, Hi: 93962, Stride: 2}, - {Lo: 93974, Hi: 93992, Stride: 18}, - {Lo: 94005, Hi: 94010, Stride: 5}, - {Lo: 94011, Hi: 94015, Stride: 4}, - {Lo: 94016, Hi: 94018, Stride: 2}, - {Lo: 94019, Hi: 94033, Stride: 14}, - {Lo: 94034, Hi: 119060, Stride: 25026}, - {Lo: 119149, Hi: 119302, Stride: 153}, - {Lo: 119309, Hi: 119311, Stride: 2}, - {Lo: 119314, Hi: 119315, Stride: 1}, - {Lo: 119318, Hi: 119338, Stride: 20}, - {Lo: 119350, Hi: 119351, Stride: 1}, - {Lo: 119354, Hi: 119355, Stride: 1}, - {Lo: 119808, Hi: 119845, Stride: 1}, - {Lo: 119847, Hi: 119892, Stride: 1}, - {Lo: 119894, Hi: 119897, Stride: 1}, - {Lo: 119899, Hi: 119949, Stride: 1}, - {Lo: 119951, Hi: 119964, Stride: 1}, - {Lo: 119966, Hi: 119967, Stride: 1}, - {Lo: 119970, Hi: 119973, Stride: 3}, - {Lo: 119974, Hi: 119977, Stride: 3}, - {Lo: 119978, Hi: 119980, Stride: 1}, - {Lo: 119982, Hi: 119993, Stride: 1}, - {Lo: 119995, Hi: 119997, Stride: 2}, - {Lo: 119998, Hi: 120001, Stride: 1}, - {Lo: 120003, Hi: 120005, Stride: 2}, - {Lo: 120006, Hi: 120053, Stride: 1}, - {Lo: 120055, Hi: 120069, Stride: 1}, - {Lo: 120071, Hi: 120074, Stride: 1}, - {Lo: 120077, Hi: 120084, Stride: 1}, - {Lo: 120086, Hi: 120092, Stride: 1}, - {Lo: 120094, Hi: 120105, Stride: 1}, - {Lo: 120107, Hi: 120121, Stride: 1}, - {Lo: 120123, Hi: 120126, Stride: 1}, - {Lo: 120128, Hi: 120132, Stride: 1}, - {Lo: 120134, Hi: 120138, Stride: 4}, - {Lo: 120139, Hi: 120144, Stride: 1}, - {Lo: 120146, Hi: 120157, Stride: 1}, - {Lo: 120159, Hi: 120209, Stride: 1}, - {Lo: 120211, Hi: 120261, Stride: 1}, - {Lo: 120263, Hi: 120313, Stride: 1}, - {Lo: 120315, Hi: 120365, Stride: 1}, - {Lo: 120367, Hi: 120417, Stride: 1}, - {Lo: 120419, Hi: 120469, Stride: 1}, - {Lo: 120471, Hi: 120484, Stride: 1}, - {Lo: 120488, Hi: 120489, Stride: 1}, - {Lo: 120492, Hi: 120494, Stride: 1}, - {Lo: 120496, Hi: 120497, Stride: 1}, - {Lo: 120499, Hi: 120500, Stride: 1}, - {Lo: 120502, Hi: 120504, Stride: 2}, - {Lo: 120507, Hi: 120508, Stride: 1}, - {Lo: 120510, Hi: 120514, Stride: 4}, - {Lo: 120516, Hi: 120522, Stride: 6}, - {Lo: 120526, Hi: 120534, Stride: 2}, - {Lo: 120544, Hi: 120546, Stride: 2}, - {Lo: 120547, Hi: 120550, Stride: 3}, - {Lo: 120551, Hi: 120552, Stride: 1}, - {Lo: 120554, Hi: 120555, Stride: 1}, - {Lo: 120557, Hi: 120558, Stride: 1}, - {Lo: 120560, Hi: 120562, Stride: 2}, - {Lo: 120565, Hi: 120566, Stride: 1}, - {Lo: 120568, Hi: 120572, Stride: 4}, - {Lo: 120574, Hi: 120580, Stride: 6}, - {Lo: 120584, Hi: 120592, Stride: 2}, - {Lo: 120602, Hi: 120604, Stride: 2}, - {Lo: 120605, Hi: 120608, Stride: 3}, - {Lo: 120609, Hi: 120610, Stride: 1}, - {Lo: 120612, Hi: 120613, Stride: 1}, - {Lo: 120615, Hi: 120616, Stride: 1}, - {Lo: 120618, Hi: 120620, Stride: 2}, - {Lo: 120623, Hi: 120624, Stride: 1}, - {Lo: 120626, Hi: 120630, Stride: 4}, - {Lo: 120632, Hi: 120638, Stride: 6}, - {Lo: 120642, Hi: 120650, Stride: 2}, - {Lo: 120660, Hi: 120662, Stride: 2}, - {Lo: 120663, Hi: 120666, Stride: 3}, - {Lo: 120667, Hi: 120668, Stride: 1}, - {Lo: 120670, Hi: 120671, Stride: 1}, - {Lo: 120673, Hi: 120674, Stride: 1}, - {Lo: 120676, Hi: 120678, Stride: 2}, - {Lo: 120681, Hi: 120682, Stride: 1}, - {Lo: 120684, Hi: 120688, Stride: 4}, - {Lo: 120690, Hi: 120696, Stride: 6}, - {Lo: 120700, Hi: 120708, Stride: 2}, - {Lo: 120718, Hi: 120720, Stride: 2}, - {Lo: 120721, Hi: 120724, Stride: 3}, - {Lo: 120725, Hi: 120726, Stride: 1}, - {Lo: 120728, Hi: 120729, Stride: 1}, - {Lo: 120731, Hi: 120732, Stride: 1}, - {Lo: 120734, Hi: 120736, Stride: 2}, - {Lo: 120739, Hi: 120740, Stride: 1}, - {Lo: 120742, Hi: 120746, Stride: 4}, - {Lo: 120748, Hi: 120754, Stride: 6}, - {Lo: 120758, Hi: 120766, Stride: 2}, - {Lo: 120776, Hi: 120778, Stride: 2}, - {Lo: 120782, Hi: 120831, Stride: 1}, - {Lo: 125127, Hi: 125131, Stride: 4}, - {Lo: 126464, Hi: 126500, Stride: 36}, - {Lo: 126564, Hi: 126592, Stride: 28}, - {Lo: 126596, Hi: 128844, Stride: 2248}, - {Lo: 128872, Hi: 130032, Stride: 1160}, - {Lo: 130033, Hi: 130041, Stride: 1}, - }, - LatinOffset: 0, }, - }, - "_default": { - Confusable: []rune{160, 180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{32, 96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "_default", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 160, Hi: 180, Stride: 20}, - {Lo: 215, Hi: 305, Stride: 90}, - {Lo: 921, Hi: 1009, Stride: 88}, - {Lo: 1040, Hi: 1042, Stride: 2}, - {Lo: 1045, Hi: 1047, Stride: 2}, - {Lo: 1050, Hi: 1052, Stride: 2}, - {Lo: 1053, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8211, Hi: 8216, Stride: 5}, - {Lo: 8217, Hi: 8245, Stride: 28}, - {Lo: 12494, Hi: 65281, Stride: 52787}, - {Lo: 65283, Hi: 65288, Stride: 5}, - {Lo: 65289, Hi: 65292, Stride: 3}, - {Lo: 65306, Hi: 65307, Stride: 1}, - {Lo: 65311, Hi: 65374, Stride: 63}, + + "_default": { + Confusable: []rune{160, 180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{32, 96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "_default", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 160, Hi: 180, Stride: 20}, + {Lo: 215, Hi: 305, Stride: 90}, + {Lo: 921, Hi: 1009, Stride: 88}, + {Lo: 1040, Hi: 1042, Stride: 2}, + {Lo: 1045, Hi: 1047, Stride: 2}, + {Lo: 1050, Hi: 1052, Stride: 2}, + {Lo: 1053, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8211, Hi: 8216, Stride: 5}, + {Lo: 8217, Hi: 8245, Stride: 28}, + {Lo: 12494, Hi: 65281, Stride: 52787}, + {Lo: 65283, Hi: 65288, Stride: 5}, + {Lo: 65289, Hi: 65292, Stride: 3}, + {Lo: 65306, Hi: 65307, Stride: 1}, + {Lo: 65311, Hi: 65374, Stride: 63}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, - "cs": { - Confusable: []rune{180, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{96, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "cs", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 180, Hi: 305, Stride: 125}, - {Lo: 921, Hi: 1009, Stride: 88}, - {Lo: 1040, Hi: 1042, Stride: 2}, - {Lo: 1045, Hi: 1047, Stride: 2}, - {Lo: 1050, Hi: 1052, Stride: 2}, - {Lo: 1053, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8216, Hi: 8217, Stride: 1}, - {Lo: 8245, Hi: 12494, Stride: 4249}, - {Lo: 65281, Hi: 65283, Stride: 2}, - {Lo: 65288, Hi: 65289, Stride: 1}, - {Lo: 65292, Hi: 65306, Stride: 14}, - {Lo: 65307, Hi: 65311, Stride: 4}, - {Lo: 65374, Hi: 65374, Stride: 1}, + + "cs": { + Confusable: []rune{180, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{96, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "cs", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 180, Hi: 305, Stride: 125}, + {Lo: 921, Hi: 1009, Stride: 88}, + {Lo: 1040, Hi: 1042, Stride: 2}, + {Lo: 1045, Hi: 1047, Stride: 2}, + {Lo: 1050, Hi: 1052, Stride: 2}, + {Lo: 1053, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8216, Hi: 8217, Stride: 1}, + {Lo: 8245, Hi: 12494, Stride: 4249}, + {Lo: 65281, Hi: 65283, Stride: 2}, + {Lo: 65288, Hi: 65289, Stride: 1}, + {Lo: 65292, Hi: 65306, Stride: 14}, + {Lo: 65307, Hi: 65311, Stride: 4}, + {Lo: 65374, Hi: 65374, Stride: 1}, + }, + R32: []unicode.Range32{}, + LatinOffset: 0, }, - R32: []unicode.Range32{}, - LatinOffset: 0, }, - }, - "de": { - Confusable: []rune{180, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{96, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "de", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 180, Hi: 305, Stride: 125}, - {Lo: 921, Hi: 1009, Stride: 88}, - {Lo: 1040, Hi: 1042, Stride: 2}, - {Lo: 1045, Hi: 1047, Stride: 2}, - {Lo: 1050, Hi: 1052, Stride: 2}, - {Lo: 1053, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8216, Hi: 8217, Stride: 1}, - {Lo: 8245, Hi: 12494, Stride: 4249}, - {Lo: 65281, Hi: 65283, Stride: 2}, - {Lo: 65288, Hi: 65289, Stride: 1}, - {Lo: 65292, Hi: 65306, Stride: 14}, - {Lo: 65307, Hi: 65311, Stride: 4}, - {Lo: 65374, Hi: 65374, Stride: 1}, + + "de": { + Confusable: []rune{180, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{96, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "de", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 180, Hi: 305, Stride: 125}, + {Lo: 921, Hi: 1009, Stride: 88}, + {Lo: 1040, Hi: 1042, Stride: 2}, + {Lo: 1045, Hi: 1047, Stride: 2}, + {Lo: 1050, Hi: 1052, Stride: 2}, + {Lo: 1053, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8216, Hi: 8217, Stride: 1}, + {Lo: 8245, Hi: 12494, Stride: 4249}, + {Lo: 65281, Hi: 65283, Stride: 2}, + {Lo: 65288, Hi: 65289, Stride: 1}, + {Lo: 65292, Hi: 65306, Stride: 14}, + {Lo: 65307, Hi: 65311, Stride: 4}, + {Lo: 65374, Hi: 65374, Stride: 1}, + }, + R32: []unicode.Range32{}, + LatinOffset: 0, }, - R32: []unicode.Range32{}, - LatinOffset: 0, }, - }, - "es": { - Confusable: []rune{180, 215, 305, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{96, 120, 105, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "es", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 180, Hi: 215, Stride: 35}, - {Lo: 305, Hi: 1009, Stride: 704}, - {Lo: 1040, Hi: 1042, Stride: 2}, - {Lo: 1045, Hi: 1047, Stride: 2}, - {Lo: 1050, Hi: 1052, Stride: 2}, - {Lo: 1053, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8211, Hi: 8245, Stride: 34}, - {Lo: 12494, Hi: 65281, Stride: 52787}, - {Lo: 65283, Hi: 65288, Stride: 5}, - {Lo: 65289, Hi: 65292, Stride: 3}, - {Lo: 65306, Hi: 65307, Stride: 1}, - {Lo: 65311, Hi: 65374, Stride: 63}, + + "es": { + Confusable: []rune{180, 215, 305, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{96, 120, 105, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "es", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 180, Hi: 215, Stride: 35}, + {Lo: 305, Hi: 1009, Stride: 704}, + {Lo: 1040, Hi: 1042, Stride: 2}, + {Lo: 1045, Hi: 1047, Stride: 2}, + {Lo: 1050, Hi: 1052, Stride: 2}, + {Lo: 1053, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8211, Hi: 8245, Stride: 34}, + {Lo: 12494, Hi: 65281, Stride: 52787}, + {Lo: 65283, Hi: 65288, Stride: 5}, + {Lo: 65289, Hi: 65292, Stride: 3}, + {Lo: 65306, Hi: 65307, Stride: 1}, + {Lo: 65311, Hi: 65374, Stride: 63}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, - "fr": { - Confusable: []rune{215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8216, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "fr", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 215, Hi: 305, Stride: 90}, - {Lo: 921, Hi: 1009, Stride: 88}, - {Lo: 1040, Hi: 1042, Stride: 2}, - {Lo: 1045, Hi: 1047, Stride: 2}, - {Lo: 1050, Hi: 1052, Stride: 2}, - {Lo: 1053, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8216, Hi: 8245, Stride: 29}, - {Lo: 12494, Hi: 65281, Stride: 52787}, - {Lo: 65283, Hi: 65288, Stride: 5}, - {Lo: 65289, Hi: 65292, Stride: 3}, - {Lo: 65306, Hi: 65307, Stride: 1}, - {Lo: 65311, Hi: 65374, Stride: 63}, + + "fr": { + Confusable: []rune{215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8216, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "fr", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 215, Hi: 305, Stride: 90}, + {Lo: 921, Hi: 1009, Stride: 88}, + {Lo: 1040, Hi: 1042, Stride: 2}, + {Lo: 1045, Hi: 1047, Stride: 2}, + {Lo: 1050, Hi: 1052, Stride: 2}, + {Lo: 1053, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8216, Hi: 8245, Stride: 29}, + {Lo: 12494, Hi: 65281, Stride: 52787}, + {Lo: 65283, Hi: 65288, Stride: 5}, + {Lo: 65289, Hi: 65292, Stride: 3}, + {Lo: 65306, Hi: 65307, Stride: 1}, + {Lo: 65311, Hi: 65374, Stride: 63}, + }, + R32: []unicode.Range32{}, + LatinOffset: 0, }, - R32: []unicode.Range32{}, - LatinOffset: 0, }, - }, - "it": { - Confusable: []rune{160, 180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8216, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{32, 96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "it", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 160, Hi: 180, Stride: 20}, - {Lo: 215, Hi: 305, Stride: 90}, - {Lo: 921, Hi: 1009, Stride: 88}, - {Lo: 1040, Hi: 1042, Stride: 2}, - {Lo: 1045, Hi: 1047, Stride: 2}, - {Lo: 1050, Hi: 1052, Stride: 2}, - {Lo: 1053, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8211, Hi: 8216, Stride: 5}, - {Lo: 8245, Hi: 12494, Stride: 4249}, - {Lo: 65281, Hi: 65283, Stride: 2}, - {Lo: 65288, Hi: 65289, Stride: 1}, - {Lo: 65292, Hi: 65306, Stride: 14}, - {Lo: 65307, Hi: 65311, Stride: 4}, - {Lo: 65374, Hi: 65374, Stride: 1}, + + "it": { + Confusable: []rune{160, 180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8216, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{32, 96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "it", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 160, Hi: 180, Stride: 20}, + {Lo: 215, Hi: 305, Stride: 90}, + {Lo: 921, Hi: 1009, Stride: 88}, + {Lo: 1040, Hi: 1042, Stride: 2}, + {Lo: 1045, Hi: 1047, Stride: 2}, + {Lo: 1050, Hi: 1052, Stride: 2}, + {Lo: 1053, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8211, Hi: 8216, Stride: 5}, + {Lo: 8245, Hi: 12494, Stride: 4249}, + {Lo: 65281, Hi: 65283, Stride: 2}, + {Lo: 65288, Hi: 65289, Stride: 1}, + {Lo: 65292, Hi: 65306, Stride: 14}, + {Lo: 65307, Hi: 65311, Stride: 4}, + {Lo: 65374, Hi: 65374, Stride: 1}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, - "ja": { - Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8216, 8217, 8245, 65281, 65283, 65292, 65306, 65307}, - With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 96, 96, 33, 35, 44, 58, 59}, - Locale: "ja", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 180, Hi: 215, Stride: 35}, - {Lo: 305, Hi: 921, Stride: 616}, - {Lo: 1009, Hi: 1040, Stride: 31}, - {Lo: 1042, Hi: 1045, Stride: 3}, - {Lo: 1047, Hi: 1050, Stride: 3}, - {Lo: 1052, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8211, Hi: 8216, Stride: 5}, - {Lo: 8217, Hi: 8245, Stride: 28}, - {Lo: 65281, Hi: 65283, Stride: 2}, - {Lo: 65292, Hi: 65306, Stride: 14}, - {Lo: 65307, Hi: 65307, Stride: 1}, + + "ja": { + Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8216, 8217, 8245, 65281, 65283, 65292, 65306, 65307}, + With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 96, 96, 33, 35, 44, 58, 59}, + Locale: "ja", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 180, Hi: 215, Stride: 35}, + {Lo: 305, Hi: 921, Stride: 616}, + {Lo: 1009, Hi: 1040, Stride: 31}, + {Lo: 1042, Hi: 1045, Stride: 3}, + {Lo: 1047, Hi: 1050, Stride: 3}, + {Lo: 1052, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8211, Hi: 8216, Stride: 5}, + {Lo: 8217, Hi: 8245, Stride: 28}, + {Lo: 65281, Hi: 65283, Stride: 2}, + {Lo: 65292, Hi: 65306, Stride: 14}, + {Lo: 65307, Hi: 65307, Stride: 1}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, - "ko": { - Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "ko", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 180, Hi: 215, Stride: 35}, - {Lo: 305, Hi: 921, Stride: 616}, - {Lo: 1009, Hi: 1040, Stride: 31}, - {Lo: 1042, Hi: 1045, Stride: 3}, - {Lo: 1047, Hi: 1050, Stride: 3}, - {Lo: 1052, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8211, Hi: 8245, Stride: 34}, - {Lo: 12494, Hi: 65281, Stride: 52787}, - {Lo: 65283, Hi: 65288, Stride: 5}, - {Lo: 65289, Hi: 65292, Stride: 3}, - {Lo: 65306, Hi: 65307, Stride: 1}, - {Lo: 65311, Hi: 65374, Stride: 63}, + + "ko": { + Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "ko", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 180, Hi: 215, Stride: 35}, + {Lo: 305, Hi: 921, Stride: 616}, + {Lo: 1009, Hi: 1040, Stride: 31}, + {Lo: 1042, Hi: 1045, Stride: 3}, + {Lo: 1047, Hi: 1050, Stride: 3}, + {Lo: 1052, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8211, Hi: 8245, Stride: 34}, + {Lo: 12494, Hi: 65281, Stride: 52787}, + {Lo: 65283, Hi: 65288, Stride: 5}, + {Lo: 65289, Hi: 65292, Stride: 3}, + {Lo: 65306, Hi: 65307, Stride: 1}, + {Lo: 65311, Hi: 65374, Stride: 63}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, - "pl": { - Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "pl", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 180, Hi: 215, Stride: 35}, - {Lo: 305, Hi: 921, Stride: 616}, - {Lo: 1009, Hi: 1040, Stride: 31}, - {Lo: 1042, Hi: 1045, Stride: 3}, - {Lo: 1047, Hi: 1050, Stride: 3}, - {Lo: 1052, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8216, Hi: 8217, Stride: 1}, - {Lo: 8245, Hi: 12494, Stride: 4249}, - {Lo: 65281, Hi: 65283, Stride: 2}, - {Lo: 65288, Hi: 65289, Stride: 1}, - {Lo: 65292, Hi: 65306, Stride: 14}, - {Lo: 65307, Hi: 65311, Stride: 4}, - {Lo: 65374, Hi: 65374, Stride: 1}, + + "pl": { + Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "pl", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 180, Hi: 215, Stride: 35}, + {Lo: 305, Hi: 921, Stride: 616}, + {Lo: 1009, Hi: 1040, Stride: 31}, + {Lo: 1042, Hi: 1045, Stride: 3}, + {Lo: 1047, Hi: 1050, Stride: 3}, + {Lo: 1052, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8216, Hi: 8217, Stride: 1}, + {Lo: 8245, Hi: 12494, Stride: 4249}, + {Lo: 65281, Hi: 65283, Stride: 2}, + {Lo: 65288, Hi: 65289, Stride: 1}, + {Lo: 65292, Hi: 65306, Stride: 14}, + {Lo: 65307, Hi: 65311, Stride: 4}, + {Lo: 65374, Hi: 65374, Stride: 1}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, - "pt-BR": { - Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "pt-BR", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 180, Hi: 215, Stride: 35}, - {Lo: 305, Hi: 921, Stride: 616}, - {Lo: 1009, Hi: 1040, Stride: 31}, - {Lo: 1042, Hi: 1045, Stride: 3}, - {Lo: 1047, Hi: 1050, Stride: 3}, - {Lo: 1052, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8216, Hi: 8217, Stride: 1}, - {Lo: 8245, Hi: 12494, Stride: 4249}, - {Lo: 65281, Hi: 65283, Stride: 2}, - {Lo: 65288, Hi: 65289, Stride: 1}, - {Lo: 65292, Hi: 65306, Stride: 14}, - {Lo: 65307, Hi: 65311, Stride: 4}, - {Lo: 65374, Hi: 65374, Stride: 1}, + + "pt-BR": { + Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "pt-BR", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 180, Hi: 215, Stride: 35}, + {Lo: 305, Hi: 921, Stride: 616}, + {Lo: 1009, Hi: 1040, Stride: 31}, + {Lo: 1042, Hi: 1045, Stride: 3}, + {Lo: 1047, Hi: 1050, Stride: 3}, + {Lo: 1052, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8216, Hi: 8217, Stride: 1}, + {Lo: 8245, Hi: 12494, Stride: 4249}, + {Lo: 65281, Hi: 65283, Stride: 2}, + {Lo: 65288, Hi: 65289, Stride: 1}, + {Lo: 65292, Hi: 65306, Stride: 14}, + {Lo: 65307, Hi: 65311, Stride: 4}, + {Lo: 65374, Hi: 65374, Stride: 1}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, - "qps-ploc": { - Confusable: []rune{160, 180, 215, 305, 921, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{32, 96, 120, 105, 73, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "qps-ploc", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 160, Hi: 180, Stride: 20}, - {Lo: 215, Hi: 305, Stride: 90}, - {Lo: 921, Hi: 1040, Stride: 119}, - {Lo: 1042, Hi: 1045, Stride: 3}, - {Lo: 1047, Hi: 1050, Stride: 3}, - {Lo: 1052, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8211, Hi: 8216, Stride: 5}, - {Lo: 8217, Hi: 8245, Stride: 28}, - {Lo: 12494, Hi: 65281, Stride: 52787}, - {Lo: 65283, Hi: 65288, Stride: 5}, - {Lo: 65289, Hi: 65292, Stride: 3}, - {Lo: 65306, Hi: 65307, Stride: 1}, - {Lo: 65311, Hi: 65374, Stride: 63}, + + "qps-ploc": { + Confusable: []rune{160, 180, 215, 305, 921, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{32, 96, 120, 105, 73, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "qps-ploc", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 160, Hi: 180, Stride: 20}, + {Lo: 215, Hi: 305, Stride: 90}, + {Lo: 921, Hi: 1040, Stride: 119}, + {Lo: 1042, Hi: 1045, Stride: 3}, + {Lo: 1047, Hi: 1050, Stride: 3}, + {Lo: 1052, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8211, Hi: 8216, Stride: 5}, + {Lo: 8217, Hi: 8245, Stride: 28}, + {Lo: 12494, Hi: 65281, Stride: 52787}, + {Lo: 65283, Hi: 65288, Stride: 5}, + {Lo: 65289, Hi: 65292, Stride: 3}, + {Lo: 65306, Hi: 65307, Stride: 1}, + {Lo: 65311, Hi: 65374, Stride: 63}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, - "ru": { - Confusable: []rune{180, 215, 305, 921, 1009, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{96, 120, 105, 73, 112, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "ru", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 180, Hi: 215, Stride: 35}, - {Lo: 305, Hi: 921, Stride: 616}, - {Lo: 1009, Hi: 8216, Stride: 7207}, - {Lo: 8217, Hi: 8245, Stride: 28}, - {Lo: 12494, Hi: 65281, Stride: 52787}, - {Lo: 65283, Hi: 65288, Stride: 5}, - {Lo: 65289, Hi: 65292, Stride: 3}, - {Lo: 65306, Hi: 65307, Stride: 1}, - {Lo: 65311, Hi: 65374, Stride: 63}, + + "ru": { + Confusable: []rune{180, 215, 305, 921, 1009, 8216, 8217, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{96, 120, 105, 73, 112, 96, 96, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "ru", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 180, Hi: 215, Stride: 35}, + {Lo: 305, Hi: 921, Stride: 616}, + {Lo: 1009, Hi: 8216, Stride: 7207}, + {Lo: 8217, Hi: 8245, Stride: 28}, + {Lo: 12494, Hi: 65281, Stride: 52787}, + {Lo: 65283, Hi: 65288, Stride: 5}, + {Lo: 65289, Hi: 65292, Stride: 3}, + {Lo: 65306, Hi: 65307, Stride: 1}, + {Lo: 65311, Hi: 65374, Stride: 63}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, - "tr": { - Confusable: []rune{160, 180, 215, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, - With: []rune{32, 96, 120, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, - Locale: "tr", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 160, Hi: 180, Stride: 20}, - {Lo: 215, Hi: 921, Stride: 706}, - {Lo: 1009, Hi: 1040, Stride: 31}, - {Lo: 1042, Hi: 1045, Stride: 3}, - {Lo: 1047, Hi: 1050, Stride: 3}, - {Lo: 1052, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8211, Hi: 8245, Stride: 34}, - {Lo: 12494, Hi: 65281, Stride: 52787}, - {Lo: 65283, Hi: 65288, Stride: 5}, - {Lo: 65289, Hi: 65292, Stride: 3}, - {Lo: 65306, Hi: 65307, Stride: 1}, - {Lo: 65311, Hi: 65374, Stride: 63}, + + "tr": { + Confusable: []rune{160, 180, 215, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 8245, 12494, 65281, 65283, 65288, 65289, 65292, 65306, 65307, 65311, 65374}, + With: []rune{32, 96, 120, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 96, 47, 33, 35, 40, 41, 44, 58, 59, 63, 126}, + Locale: "tr", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 160, Hi: 180, Stride: 20}, + {Lo: 215, Hi: 921, Stride: 706}, + {Lo: 1009, Hi: 1040, Stride: 31}, + {Lo: 1042, Hi: 1045, Stride: 3}, + {Lo: 1047, Hi: 1050, Stride: 3}, + {Lo: 1052, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8211, Hi: 8245, Stride: 34}, + {Lo: 12494, Hi: 65281, Stride: 52787}, + {Lo: 65283, Hi: 65288, Stride: 5}, + {Lo: 65289, Hi: 65292, Stride: 3}, + {Lo: 65306, Hi: 65307, Stride: 1}, + {Lo: 65311, Hi: 65374, Stride: 63}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, - "zh-hans": { - Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8245, 12494, 65281, 65288, 65289, 65306, 65374}, - With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 47, 33, 40, 41, 58, 126}, - Locale: "zh-hans", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 180, Hi: 215, Stride: 35}, - {Lo: 305, Hi: 921, Stride: 616}, - {Lo: 1009, Hi: 1040, Stride: 31}, - {Lo: 1042, Hi: 1045, Stride: 3}, - {Lo: 1047, Hi: 1050, Stride: 3}, - {Lo: 1052, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8245, Hi: 12494, Stride: 4249}, - {Lo: 65281, Hi: 65288, Stride: 7}, - {Lo: 65289, Hi: 65306, Stride: 17}, - {Lo: 65374, Hi: 65374, Stride: 1}, + + "zh-hans": { + Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8245, 12494, 65281, 65288, 65289, 65306, 65374}, + With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 96, 47, 33, 40, 41, 58, 126}, + Locale: "zh-hans", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 180, Hi: 215, Stride: 35}, + {Lo: 305, Hi: 921, Stride: 616}, + {Lo: 1009, Hi: 1040, Stride: 31}, + {Lo: 1042, Hi: 1045, Stride: 3}, + {Lo: 1047, Hi: 1050, Stride: 3}, + {Lo: 1052, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8245, Hi: 12494, Stride: 4249}, + {Lo: 65281, Hi: 65288, Stride: 7}, + {Lo: 65289, Hi: 65306, Stride: 17}, + {Lo: 65374, Hi: 65374, Stride: 1}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, - "zh-hant": { - Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 12494, 65283, 65307, 65374}, - With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 47, 35, 59, 126}, - Locale: "zh-hant", - RangeTable: &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 180, Hi: 215, Stride: 35}, - {Lo: 305, Hi: 921, Stride: 616}, - {Lo: 1009, Hi: 1040, Stride: 31}, - {Lo: 1042, Hi: 1045, Stride: 3}, - {Lo: 1047, Hi: 1050, Stride: 3}, - {Lo: 1052, Hi: 1054, Stride: 1}, - {Lo: 1056, Hi: 1059, Stride: 1}, - {Lo: 1061, Hi: 1068, Stride: 7}, - {Lo: 1072, Hi: 1073, Stride: 1}, - {Lo: 1075, Hi: 1077, Stride: 2}, - {Lo: 1086, Hi: 1088, Stride: 2}, - {Lo: 1089, Hi: 1093, Stride: 2}, - {Lo: 8211, Hi: 12494, Stride: 4283}, - {Lo: 65283, Hi: 65307, Stride: 24}, - {Lo: 65374, Hi: 65374, Stride: 1}, + + "zh-hant": { + Confusable: []rune{180, 215, 305, 921, 1009, 1040, 1042, 1045, 1047, 1050, 1052, 1053, 1054, 1056, 1057, 1058, 1059, 1061, 1068, 1072, 1073, 1075, 1077, 1086, 1088, 1089, 1091, 1093, 8211, 12494, 65283, 65307, 65374}, + With: []rune{96, 120, 105, 73, 112, 65, 66, 69, 51, 75, 77, 72, 79, 80, 67, 84, 89, 88, 98, 97, 54, 114, 101, 111, 112, 99, 121, 120, 45, 47, 35, 59, 126}, + Locale: "zh-hant", + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 180, Hi: 215, Stride: 35}, + {Lo: 305, Hi: 921, Stride: 616}, + {Lo: 1009, Hi: 1040, Stride: 31}, + {Lo: 1042, Hi: 1045, Stride: 3}, + {Lo: 1047, Hi: 1050, Stride: 3}, + {Lo: 1052, Hi: 1054, Stride: 1}, + {Lo: 1056, Hi: 1059, Stride: 1}, + {Lo: 1061, Hi: 1068, Stride: 7}, + {Lo: 1072, Hi: 1073, Stride: 1}, + {Lo: 1075, Hi: 1077, Stride: 2}, + {Lo: 1086, Hi: 1088, Stride: 2}, + {Lo: 1089, Hi: 1093, Stride: 2}, + {Lo: 8211, Hi: 12494, Stride: 4283}, + {Lo: 65283, Hi: 65307, Stride: 24}, + {Lo: 65374, Hi: 65374, Stride: 1}, + }, + R32: []unicode.Range32{}, + LatinOffset: 1, }, - R32: []unicode.Range32{}, - LatinOffset: 1, }, - }, + } } diff --git a/modules/charset/ambiguous_gen_test.go b/modules/charset/ambiguous_gen_test.go index d3be0b1a13..81d2e8065b 100644 --- a/modules/charset/ambiguous_gen_test.go +++ b/modules/charset/ambiguous_gen_test.go @@ -8,11 +8,13 @@ import ( "testing" "unicode" + "code.gitea.io/gitea/modules/translation" + "github.com/stretchr/testify/assert" ) func TestAmbiguousCharacters(t *testing.T) { - for locale, ambiguous := range AmbiguousCharacters { + for locale, ambiguous := range globalVars().ambiguousTableMap { assert.Equal(t, locale, ambiguous.Locale) assert.Len(t, ambiguous.With, len(ambiguous.Confusable)) assert.True(t, sort.SliceIsSorted(ambiguous.Confusable, func(i, j int) bool { @@ -28,4 +30,8 @@ func TestAmbiguousCharacters(t *testing.T) { assert.True(t, found, "%c is not in %d", confusable, i) } } + + var confusableTo rune + ret := isAmbiguous('๐พ', &confusableTo, AmbiguousTablesForLocale(&translation.MockLocale{})...) + assert.True(t, ret) } diff --git a/modules/charset/breakwriter.go b/modules/charset/breakwriter.go deleted file mode 100644 index a87e846466..0000000000 --- a/modules/charset/breakwriter.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package charset - -import ( - "bytes" - "io" -) - -// BreakWriter wraps an io.Writer to always write '\n' as '
' -type BreakWriter struct { - io.Writer -} - -// Write writes the provided byte slice transparently replacing '\n' with '
' -func (b *BreakWriter) Write(bs []byte) (n int, err error) { - pos := 0 - for pos < len(bs) { - idx := bytes.IndexByte(bs[pos:], '\n') - if idx < 0 { - wn, err := b.Writer.Write(bs[pos:]) - return n + wn, err - } - - if idx > 0 { - wn, err := b.Writer.Write(bs[pos : pos+idx]) - n += wn - if err != nil { - return n, err - } - } - - if _, err = b.Writer.Write([]byte("
")); err != nil { - return n, err - } - pos += idx + 1 - - n++ - } - - return n, err -} diff --git a/modules/charset/breakwriter_test.go b/modules/charset/breakwriter_test.go deleted file mode 100644 index 5eeeedc4e2..0000000000 --- a/modules/charset/breakwriter_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package charset - -import ( - "strings" - "testing" -) - -func TestBreakWriter_Write(t *testing.T) { - tests := []struct { - name string - kase string - expect string - wantErr bool - }{ - { - name: "noline", - kase: "abcdefghijklmnopqrstuvwxyz", - expect: "abcdefghijklmnopqrstuvwxyz", - }, - { - name: "endline", - kase: "abcdefghijklmnopqrstuvwxyz\n", - expect: "abcdefghijklmnopqrstuvwxyz
", - }, - { - name: "startline", - kase: "\nabcdefghijklmnopqrstuvwxyz", - expect: "
abcdefghijklmnopqrstuvwxyz", - }, - { - name: "onlyline", - kase: "\n\n\n", - expect: "


", - }, - { - name: "empty", - kase: "", - expect: "", - }, - { - name: "midline", - kase: "\nabc\ndefghijkl\nmnopqrstuvwxy\nz", - expect: "
abc
defghijkl
mnopqrstuvwxy
z", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - buf := &strings.Builder{} - b := &BreakWriter{ - Writer: buf, - } - n, err := b.Write([]byte(tt.kase)) - if (err != nil) != tt.wantErr { - t.Errorf("BreakWriter.Write() error = %v, wantErr %v", err, tt.wantErr) - return - } - if n != len(tt.kase) { - t.Errorf("BreakWriter.Write() = %v, want %v", n, len(tt.kase)) - } - if buf.String() != tt.expect { - t.Errorf("BreakWriter.Write() wrote %q, want %v", buf.String(), tt.expect) - } - }) - } -} diff --git a/modules/charset/charset.go b/modules/charset/charset.go index b156654973..96de1c9fcc 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -6,7 +6,10 @@ package charset import ( "bytes" "io" + "regexp" "strings" + "sync" + "unicode" "unicode/utf8" "code.gitea.io/gitea/modules/setting" @@ -17,8 +20,19 @@ import ( "golang.org/x/text/transform" ) -// UTF8BOM is the utf-8 byte-order marker -var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'} +var globalVars = sync.OnceValue(func() (ret struct { + utf8Bom []byte + + defaultWordRegexp *regexp.Regexp + ambiguousTableMap map[string]*AmbiguousTable + invisibleRangeTable *unicode.RangeTable +}, +) { + ret.utf8Bom = []byte{'\xef', '\xbb', '\xbf'} + ret.ambiguousTableMap = newAmbiguousTableMap() + ret.invisibleRangeTable = newInvisibleRangeTable() + return ret +}) type ConvertOpts struct { KeepBOM bool @@ -105,7 +119,7 @@ func maybeRemoveBOM(content []byte, opts ConvertOpts) []byte { if opts.KeepBOM { return content } - return bytes.TrimPrefix(content, UTF8BOM) + return bytes.TrimPrefix(content, globalVars().utf8Bom) } // DetectEncoding detect the encoding of content diff --git a/modules/charset/escape.go b/modules/charset/escape.go index 167683a298..8f25e7876d 100644 --- a/modules/charset/escape.go +++ b/modules/charset/escape.go @@ -1,10 +1,6 @@ // Copyright 2022 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:generate go run invisible/generate.go -v -o ./invisible_gen.go - -//go:generate go run ambiguous/generate.go -v -o ./ambiguous_gen.go ambiguous/ambiguous.json - package charset import ( @@ -12,36 +8,36 @@ import ( "io" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" ) -// RuneNBSP is the codepoint for NBSP -const RuneNBSP = 0xa0 +type EscapeOptions struct { + Allowed map[rune]bool +} + +func AllowRuneNBSP() map[rune]bool { + return map[rune]bool{0xa0: true} +} + +func EscapeOptionsForView() EscapeOptions { + return EscapeOptions{ + // it's safe to see NBSP in the view, but maybe not in the diff + Allowed: AllowRuneNBSP(), + } +} // EscapeControlHTML escapes the Unicode control sequences in a provided html document -func EscapeControlHTML(html template.HTML, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output template.HTML) { +func EscapeControlHTML(html template.HTML, locale translation.Locale, opts ...EscapeOptions) (escaped *EscapeStatus, output template.HTML) { if !setting.UI.AmbiguousUnicodeDetection { return &EscapeStatus{}, html } sb := &strings.Builder{} - escaped, _ = EscapeControlReader(strings.NewReader(string(html)), sb, locale, allowed...) // err has been handled in EscapeControlReader + escaped, _ = EscapeControlReader(strings.NewReader(string(html)), sb, locale, opts...) // err has been handled in EscapeControlReader return escaped, template.HTML(sb.String()) } // EscapeControlReader escapes the Unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus -func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) { - if !setting.UI.AmbiguousUnicodeDetection { - _, err = io.Copy(writer, reader) - return &EscapeStatus{}, err - } - outputStream := &HTMLStreamerWriter{Writer: writer} - streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer) - - if err = StreamHTML(reader, streamer); err != nil { - streamer.escaped.HasError = true - log.Error("Error whilst escaping: %v", err) - } - return streamer.escaped, err +func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, opts ...EscapeOptions) (*EscapeStatus, error) { + return escapeStream(locale, reader, writer, opts...) } diff --git a/modules/charset/escape_status.go b/modules/charset/escape_status.go index 37b6ad86d4..fb9ebbb228 100644 --- a/modules/charset/escape_status.go +++ b/modules/charset/escape_status.go @@ -3,11 +3,9 @@ package charset -// EscapeStatus represents the findings of the unicode escaper +// EscapeStatus represents the findings of the Unicode escaper type EscapeStatus struct { - Escaped bool - HasError bool - HasBadRunes bool + Escaped bool // it means that some characters were escaped, and they can also be unescaped back HasInvisible bool HasAmbiguous bool } @@ -19,8 +17,6 @@ func (status *EscapeStatus) Or(other *EscapeStatus) *EscapeStatus { st = &EscapeStatus{} } st.Escaped = st.Escaped || other.Escaped - st.HasError = st.HasError || other.HasError - st.HasBadRunes = st.HasBadRunes || other.HasBadRunes st.HasAmbiguous = st.HasAmbiguous || other.HasAmbiguous st.HasInvisible = st.HasInvisible || other.HasInvisible return st diff --git a/modules/charset/escape_stream.go b/modules/charset/escape_stream.go index 22e7f14f39..11d09083fc 100644 --- a/modules/charset/escape_stream.go +++ b/modules/charset/escape_stream.go @@ -4,288 +4,415 @@ package charset import ( + "bytes" "fmt" - "regexp" - "strings" + "html" + "io" "unicode" "unicode/utf8" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" - - "golang.org/x/net/html" ) -// VScode defaultWordRegexp -var defaultWordRegexp = regexp.MustCompile(`(-?\d*\.\d\w*)|([^\` + "`" + `\~\!\@\#\$\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s\x00-\x1f]+)`) - -func NewEscapeStreamer(locale translation.Locale, next HTMLStreamer, allowed ...rune) HTMLStreamer { - allowedM := make(map[rune]bool, len(allowed)) - for _, v := range allowed { - allowedM[v] = true - } - return &escapeStreamer{ - escaped: &EscapeStatus{}, - PassthroughHTMLStreamer: *NewPassthroughStreamer(next), - locale: locale, - ambiguousTables: AmbiguousTablesForLocale(locale), - allowed: allowedM, - } +type htmlChunkReader struct { + in io.Reader + readErr error + readBuf []byte + curInTag bool } type escapeStreamer struct { - PassthroughHTMLStreamer + htmlChunkReader + escaped *EscapeStatus locale translation.Locale ambiguousTables []*AmbiguousTable allowed map[rune]bool + + out io.Writer } -func (e *escapeStreamer) EscapeStatus() *EscapeStatus { - return e.escaped +func escapeStream(locale translation.Locale, in io.Reader, out io.Writer, opts ...EscapeOptions) (*EscapeStatus, error) { + es := &escapeStreamer{ + escaped: &EscapeStatus{}, + locale: locale, + ambiguousTables: AmbiguousTablesForLocale(locale), + htmlChunkReader: htmlChunkReader{ + in: in, + readBuf: make([]byte, 0, 32*1024), + }, + out: out, + } + + if len(opts) > 0 { + es.allowed = opts[0].Allowed + } + + readCount := 0 + lastIsTag := false + for { + parts, partInTag, err := es.readRunes() + readCount++ + if err == io.EOF { + return es.escaped, nil + } else if err != nil { + return nil, err + } + for i, part := range parts { + if partInTag[i] { + lastIsTag = true + if _, err := out.Write(part); err != nil { + return nil, err + } + } else { + // if last part is tag, then this part is content begin + // if the content is the first part of the first read, then it's also content begin + isContentBegin := lastIsTag || (readCount == 1 && i == 0) + lastIsTag = false + if isContentBegin { + if part, err = es.trimAndWriteBom(part); err != nil { + return nil, err + } + } + if err = es.detectAndWriteRunes(part); err != nil { + return nil, err + } + } + } + } } -// Text tells the next streamer there is a text -func (e *escapeStreamer) Text(data string) error { - sb := &strings.Builder{} - var until int - var next int +func (e *escapeStreamer) trimAndWriteBom(part []byte) ([]byte, error) { + remaining, ok := bytes.CutPrefix(part, globalVars().utf8Bom) + if ok { + part = remaining + if _, err := e.out.Write(globalVars().utf8Bom); err != nil { + return part, err + } + } + return part, nil +} + +const longSentenceDetectionLimit = 20 + +func (e *escapeStreamer) possibleLongSentence(results []detectResult, pos int) bool { + countBasic := 0 + countNonASCII := 0 + for i := max(pos-longSentenceDetectionLimit, 0); i < min(pos+longSentenceDetectionLimit, len(results)); i++ { + if results[i].runeType == runeTypeBasic && results[i].runeChar != ' ' { + countBasic++ + } + if results[i].runeType == runeTypeNonASCII || results[i].runeType == runeTypeAmbiguous { + countNonASCII++ + } + } + countChar := countBasic + countNonASCII + // many non-ASCII runes around, it seems to be a sentence, + // don't handle the invisible/ambiguous chars in it, otherwise it will be too noisy + return countChar != 0 && countNonASCII*100/countChar >= 50 +} + +func (e *escapeStreamer) analyzeDetectResults(results []detectResult) { + for i := range results { + res := &results[i] + if res.runeType == runeTypeInvisible || res.runeType == runeTypeAmbiguous { + leftIsNonASCII := i > 0 && (results[i-1].runeType == runeTypeNonASCII || results[i-1].runeType == runeTypeAmbiguous) + rightIsNonASCII := i < len(results)-1 && (results[i+1].runeType == runeTypeNonASCII || results[i+1].runeType == runeTypeAmbiguous) + surroundingNonASCII := leftIsNonASCII || rightIsNonASCII + if !surroundingNonASCII { + if len(results) < longSentenceDetectionLimit { + res.needEscape = setting.UI.AmbiguousUnicodeDetection + } else if !e.possibleLongSentence(results, i) { + res.needEscape = setting.UI.AmbiguousUnicodeDetection + } + } + } + } +} + +func (e *escapeStreamer) detectAndWriteRunes(part []byte) error { + results := e.detectRunes(part) + e.analyzeDetectResults(results) + return e.writeDetectResults(part, results) +} + +func (e *htmlChunkReader) readRunes() (parts [][]byte, partInTag []bool, _ error) { + // we have read everything, eof + if e.readErr != nil && len(e.readBuf) == 0 { + return nil, nil, e.readErr + } + + // not eof, and the there is space in the buffer, try to read more data + if e.readErr == nil && len(e.readBuf) <= cap(e.readBuf)*3/4 { + n, err := e.in.Read(e.readBuf[len(e.readBuf):cap(e.readBuf)]) + e.readErr = err + e.readBuf = e.readBuf[:len(e.readBuf)+n] + } + if len(e.readBuf) == 0 { + return nil, nil, e.readErr + } + + // try to exact tag parts and content parts pos := 0 - if len(data) > len(UTF8BOM) && data[:len(UTF8BOM)] == string(UTF8BOM) { - _, _ = sb.WriteString(data[:len(UTF8BOM)]) - pos = len(UTF8BOM) - } - dataBytes := []byte(data) - for pos < len(data) { - nextIdxs := defaultWordRegexp.FindStringIndex(data[pos:]) - if nextIdxs == nil { - until = len(data) - next = until - } else { - until = min(nextIdxs[0]+pos, len(data)) - next = min(nextIdxs[1]+pos, len(data)) - } - - // from pos until we know that the runes are not \r\t\n or even ' ' - n := next - until - runes := make([]rune, 0, n) - positions := make([]int, 0, n+1) - - for pos < until { - r, sz := utf8.DecodeRune(dataBytes[pos:]) - positions = positions[:0] - positions = append(positions, pos, pos+sz) - types, confusables, _ := e.runeTypes(r) - if err := e.handleRunes(dataBytes, []rune{r}, positions, types, confusables, sb); err != nil { - return err - } - pos += sz - } - - for i := pos; i < next; { - r, sz := utf8.DecodeRune(dataBytes[i:]) - runes = append(runes, r) - positions = append(positions, i) - i += sz - } - positions = append(positions, next) - types, confusables, runeCounts := e.runeTypes(runes...) - if runeCounts.needsEscape() { - if err := e.handleRunes(dataBytes, runes, positions, types, confusables, sb); err != nil { - return err + for pos < len(e.readBuf) { + var curPartEnd int + nextInTag := e.curInTag + if e.curInTag { + // if cur part is in tag, try to find the tag close char '>' + idx := bytes.IndexByte(e.readBuf[pos:], '>') + if idx == -1 { + // if no tag close char, then the whole buffer is in tag + curPartEnd = len(e.readBuf) + } else { + // tag part ends, switch to content part + curPartEnd = pos + idx + 1 + nextInTag = !nextInTag } } else { - _, _ = sb.Write(dataBytes[pos:next]) + // if cur part is in content, try to find the tag open char '<' + idx := bytes.IndexByte(e.readBuf[pos:], '<') + if idx == -1 { + // if no tag open char, then the whole buffer is in content + curPartEnd = len(e.readBuf) + } else { + // content part ends, switch to tag part + curPartEnd = pos + idx + nextInTag = !nextInTag + } + } + + curPartLen := curPartEnd - pos + if curPartLen == 0 { + // if cur part is empty, only need to switch the part type + if e.curInTag == nextInTag { + panic("impossible, curPartLen is 0 but the part in tag status is not switched") + } + e.curInTag = nextInTag + continue + } + + // now, curPartLen can't be 0 + curPart := make([]byte, curPartLen) + copy(curPart, e.readBuf[pos:curPartEnd]) + // now we get the curPart bytes, but we can't directly use it, the last rune in it might have been cut + // try to decode the last rune, if it's invalid, then we cut the last byte and try again until we get a valid rune or no byte left + for i := curPartLen - 1; i >= 0; i-- { + last, lastSize := utf8.DecodeRune(curPart[i:]) + if last == utf8.RuneError && lastSize == 1 { + curPartLen-- + } else { + curPartLen += lastSize - 1 + break + } + } + if curPartLen == 0 { + // actually it's impossible that the part doesn't contain any valid rune, + // the only case is that the cap(readBuf) is too small, or the origin contain indeed doesn't contain any valid rune + // * try to leave the last 4 bytes (possible longest utf-8 encoding) to next round + // * at least consume 1 byte to avoid infinite loop + curPartLen = max(len(curPart)-utf8.UTFMax, 1) + } + + // if curPartLen is not the same as curPart, it means we have cut some bytes, + // need to wait for more data if not eof + trailingCorrupted := curPartLen != len(curPart) + + // finally, we get the real part we need + curPart = curPart[:curPartLen] + parts = append(parts, curPart) + partInTag = append(partInTag, e.curInTag) + + pos += curPartLen + e.curInTag = nextInTag + + if trailingCorrupted && e.readErr == nil { + // if the last part is corrupted, and we haven't reach eof, then we need to wait for more data to get the complete part + break } - pos = next } - if sb.Len() > 0 { - if err := e.PassthroughHTMLStreamer.Text(sb.String()); err != nil { + + copy(e.readBuf, e.readBuf[pos:]) + e.readBuf = e.readBuf[:len(e.readBuf)-pos] + return parts, partInTag, nil +} + +func (e *escapeStreamer) writeDetectResults(data []byte, results []detectResult) error { + lastWriteRawIdx := -1 + for idx := range results { + res := &results[idx] + if !res.needEscape { + if lastWriteRawIdx == -1 { + lastWriteRawIdx = idx + } + continue + } + + if lastWriteRawIdx != -1 { + if _, err := e.out.Write(data[results[lastWriteRawIdx].position:res.position]); err != nil { + return err + } + lastWriteRawIdx = -1 + } + switch res.runeType { + case runeTypeBroken: + if err := e.writeBrokenRune(data[res.position : res.position+res.runeSize]); err != nil { + return err + } + case runeTypeAmbiguous: + if err := e.writeAmbiguousRune(res.runeChar, res.confusable); err != nil { + return err + } + case runeTypeInvisible: + if err := e.writeInvisibleRune(res.runeChar); err != nil { + return err + } + case runeTypeControlChar: + if err := e.writeControlRune(res.runeChar); err != nil { + return err + } + default: + panic("unreachable") + } + } + if lastWriteRawIdx != -1 { + lastResult := results[len(results)-1] + if _, err := e.out.Write(data[results[lastWriteRawIdx].position : lastResult.position+lastResult.runeSize]); err != nil { return err } } return nil } -func (e *escapeStreamer) handleRunes(data []byte, runes []rune, positions []int, types []runeType, confusables []rune, sb *strings.Builder) error { - for i, r := range runes { - switch types[i] { - case brokenRuneType: - if sb.Len() > 0 { - if err := e.PassthroughHTMLStreamer.Text(sb.String()); err != nil { - return err - } - sb.Reset() - } - end := positions[i+1] - start := positions[i] - if err := e.brokenRune(data[start:end]); err != nil { - return err - } - case ambiguousRuneType: - if sb.Len() > 0 { - if err := e.PassthroughHTMLStreamer.Text(sb.String()); err != nil { - return err - } - sb.Reset() - } - if err := e.ambiguousRune(r, confusables[0]); err != nil { - return err - } - confusables = confusables[1:] - case invisibleRuneType: - if sb.Len() > 0 { - if err := e.PassthroughHTMLStreamer.Text(sb.String()); err != nil { - return err - } - sb.Reset() - } - if err := e.invisibleRune(r); err != nil { - return err - } - default: - _, _ = sb.WriteRune(r) - } - } - return nil +func (e *escapeStreamer) writeBrokenRune(_ []byte) (err error) { + // Although we'd like to use the original bytes to display (show the real broken content to users), + // however, when this "escape stream" module is applied to the content, the content has already been processed by other modules. + // So the invalid bytes just can't be kept till this step, in most (all) cases, the only thing we see here is utf8.RuneError + _, err = io.WriteString(e.out, `๏ฟฝ`) + return err } -func (e *escapeStreamer) brokenRune(bs []byte) error { - e.escaped.Escaped = true - e.escaped.HasBadRunes = true - - if err := e.PassthroughHTMLStreamer.StartTag("span", html.Attribute{ - Key: "class", - Val: "broken-code-point", - }); err != nil { +func (e *escapeStreamer) writeEscapedCharHTML(tag1, attr, tag2, content, tag3 string) (err error) { + _, err = io.WriteString(e.out, tag1) + if err != nil { return err } - if err := e.PassthroughHTMLStreamer.Text(fmt.Sprintf("<%X>", bs)); err != nil { + _, err = io.WriteString(e.out, html.EscapeString(attr)) + if err != nil { return err } - - return e.PassthroughHTMLStreamer.EndTag("span") + _, err = io.WriteString(e.out, tag2) + if err != nil { + return err + } + _, err = io.WriteString(e.out, html.EscapeString(content)) + if err != nil { + return err + } + _, err = io.WriteString(e.out, tag3) + return err } -func (e *escapeStreamer) ambiguousRune(r, c rune) error { +func runeToHex(r rune) string { + return fmt.Sprintf("[U+%04X]", r) +} + +func (e *escapeStreamer) writeAmbiguousRune(r, c rune) (err error) { e.escaped.Escaped = true e.escaped.HasAmbiguous = true - - if err := e.PassthroughHTMLStreamer.StartTag("span", html.Attribute{ - Key: "class", - Val: "ambiguous-code-point", - }, html.Attribute{ - Key: "data-tooltip-content", - Val: e.locale.TrString("repo.ambiguous_character", r, c), - }); err != nil { - return err - } - if err := e.PassthroughHTMLStreamer.StartTag("span", html.Attribute{ - Key: "class", - Val: "char", - }); err != nil { - return err - } - if err := e.PassthroughHTMLStreamer.Text(string(r)); err != nil { - return err - } - if err := e.PassthroughHTMLStreamer.EndTag("span"); err != nil { - return err - } - - return e.PassthroughHTMLStreamer.EndTag("span") + return e.writeEscapedCharHTML( + ``, + string(r), + ``, + ) } -func (e *escapeStreamer) invisibleRune(r rune) error { +func (e *escapeStreamer) writeInvisibleRune(r rune) error { e.escaped.Escaped = true e.escaped.HasInvisible = true - - if err := e.PassthroughHTMLStreamer.StartTag("span", html.Attribute{ - Key: "class", - Val: "escaped-code-point", - }, html.Attribute{ - Key: "data-escaped", - Val: fmt.Sprintf("[U+%04X]", r), - }); err != nil { - return err - } - if err := e.PassthroughHTMLStreamer.StartTag("span", html.Attribute{ - Key: "class", - Val: "char", - }); err != nil { - return err - } - if err := e.PassthroughHTMLStreamer.Text(string(r)); err != nil { - return err - } - if err := e.PassthroughHTMLStreamer.EndTag("span"); err != nil { - return err - } - - return e.PassthroughHTMLStreamer.EndTag("span") + return e.writeEscapedCharHTML( + ``, + string(r), + ``, + ) } -type runeCountType struct { - numBasicRunes int - numNonConfusingNonBasicRunes int - numAmbiguousRunes int - numInvisibleRunes int - numBrokenRunes int +func (e *escapeStreamer) writeControlRune(r rune) error { + var display string + if r >= 0 && r <= 0x1f { + display = string(0x2400 + r) + } else if r == 0x7f { + display = string(rune(0x2421)) + } else { + display = runeToHex(r) + } + return e.writeEscapedCharHTML( + ``, + string(r), + ``, + ) } -func (counts runeCountType) needsEscape() bool { - if counts.numBrokenRunes > 0 { - return true - } - if counts.numBasicRunes == 0 && - counts.numNonConfusingNonBasicRunes > 0 { - return false - } - return counts.numAmbiguousRunes > 0 || counts.numInvisibleRunes > 0 +type detectResult struct { + runeChar rune + runeType int + runeSize int + position int + confusable rune + needEscape bool } -type runeType int - const ( - basicASCIIRuneType runeType = iota // <- This is technically deadcode but its self-documenting so it should stay - brokenRuneType - nonBasicASCIIRuneType - ambiguousRuneType - invisibleRuneType + runeTypeBasic int = iota + runeTypeBroken + runeTypeNonASCII + runeTypeAmbiguous + runeTypeInvisible + runeTypeControlChar ) -func (e *escapeStreamer) runeTypes(runes ...rune) (types []runeType, confusables []rune, runeCounts runeCountType) { - types = make([]runeType, len(runes)) - for i, r := range runes { - var confusable rune +func (e *escapeStreamer) detectRunes(data []byte) []detectResult { + runeCount := utf8.RuneCount(data) + results := make([]detectResult, runeCount) + invisibleRangeTable := globalVars().invisibleRangeTable + var i int + var confusable rune + for pos := 0; pos < len(data); i++ { + r, runeSize := utf8.DecodeRune(data[pos:]) + results[i].runeChar = r + results[i].runeSize = runeSize + results[i].position = pos + pos += runeSize + switch { case r == utf8.RuneError: - types[i] = brokenRuneType - runeCounts.numBrokenRunes++ - case r == ' ' || r == '\t' || r == '\n': - runeCounts.numBasicRunes++ - case e.allowed[r]: - if r > 0x7e || r < 0x20 { - types[i] = nonBasicASCIIRuneType - runeCounts.numNonConfusingNonBasicRunes++ - } else { - runeCounts.numBasicRunes++ + results[i].runeType = runeTypeBroken + results[i].needEscape = true + case r == ' ' || r == '\t' || r == '\n' || e.allowed[r]: + results[i].runeType = runeTypeBasic + if r >= 0x80 { + results[i].runeType = runeTypeNonASCII } - case unicode.Is(InvisibleRanges, r): - types[i] = invisibleRuneType - runeCounts.numInvisibleRunes++ - case unicode.IsControl(r): - types[i] = invisibleRuneType - runeCounts.numInvisibleRunes++ + case r < 0x20 || r == 0x7f: + results[i].runeType = runeTypeControlChar + results[i].needEscape = true + case unicode.Is(invisibleRangeTable, r): + results[i].runeType = runeTypeInvisible + // not sure about results[i].needEscape, will be detected separately case isAmbiguous(r, &confusable, e.ambiguousTables...): - confusables = append(confusables, confusable) - types[i] = ambiguousRuneType - runeCounts.numAmbiguousRunes++ - case r > 0x7e || r < 0x20: - types[i] = nonBasicASCIIRuneType - runeCounts.numNonConfusingNonBasicRunes++ - default: - runeCounts.numBasicRunes++ + results[i].runeType = runeTypeAmbiguous + results[i].confusable = confusable + // not sure about results[i].needEscape, will be detected separately + case r >= 0x80: + results[i].runeType = runeTypeNonASCII + default: // details to basic runes } } - return types, confusables, runeCounts + return results } diff --git a/modules/charset/escape_test.go b/modules/charset/escape_test.go index 9d796a0c18..4e1ff0fcf4 100644 --- a/modules/charset/escape_test.go +++ b/modules/charset/escape_test.go @@ -4,7 +4,6 @@ package charset import ( - "regexp" "strings" "testing" @@ -13,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/translation" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type escapeControlTest struct { @@ -57,24 +57,24 @@ var escapeControlTests = []escapeControlTest{ status: EscapeStatus{}, }, { - name: "hebrew", + name: "hebrew", // old test was wrong, such text shouldn't be escaped text: "ืขื“ ืชืงื•ืคืช ื™ื•ื•ืŸ ื”ืขืชื™ืงื” ื”ื™ื” ื”ืขื™ืกื•ืง ื‘ืžืชืžื˜ื™ืงื” ืชื›ืœื™ืชื™ ื‘ืœื‘ื“: ื”ื™ื ืฉื™ืžืฉื” ื›ืื•ืกืฃ ืฉืœ ื ื•ืกื—ืื•ืช ืœื—ื™ืฉื•ื‘ ืงืจืงืข, ืื•ื›ืœื•ืกื™ืŸ ื•ื›ื•'. ืคืจื™ืฆืช ื”ื“ืจืš ืฉืœ ื”ื™ื•ื•ื ื™ื, ืคืจื˜ ืœืชืจื•ืžื•ืชื™ื”ื ื”ื’ื“ื•ืœื•ืช ืœื™ื“ืข ื”ืžืชืžื˜ื™, ื”ื™ื™ืชื” ื‘ืœื™ืžื•ื“ ื”ืžืชืžื˜ื™ืงื” ื›ืฉืœืขืฆืžื”, ืžืชื•ืงืฃ ืขืจื›ื” ื”ืจื•ื—ื ื™. ื™ื—ืกื ืฉืœ ื—ืœืง ืžื”ื™ื•ื•ื ื™ื ื”ืงื“ืžื•ื ื™ื ืœืžืชืžื˜ื™ืงื” ื”ื™ื” ื“ืชื™ - ืœืžืฉืœ, ื”ื›ืช ืฉืืกืฃ ืกื‘ื™ื‘ื• ืคื™ืชื’ื•ืจืก ื”ืืžื™ื ื” ื›ื™ ื”ืžืชืžื˜ื™ืงื” ื”ื™ื ื”ื‘ืกื™ืก ืœื›ืœ ื”ื“ื‘ืจื™ื. ื”ื™ื•ื•ื ื™ื ื ื—ืฉื‘ื™ื ืœื™ื•ืฆืจื™ ืžื•ืฉื’ ื”ื”ื•ื›ื—ื” ื”ืžืชืžื˜ื™ืช, ื•ื›ืŸ ืœืจืืฉื•ื ื™ื ืฉืขืกืงื• ื‘ืžืชืžื˜ื™ืงื” ืœืฉื ืขืฆืžื”, ื›ืœื•ืžืจ ื›ืชื—ื•ื ืžื—ืงืจื™ ืขื™ื•ื ื™ ื•ืžื•ืคืฉื˜ ื•ืœื ืจืง ื›ืขื–ืจ ืฉื™ืžื•ืฉื™. ืขื ื–ืืช, ืœืฆื“ื”", - result: `ืขื“ ืชืงื•ืคืช ื™ื•ื•ืŸ ื”ืขืชื™ืงื” ื”ื™ื” ื”ืขื™ืกื•ืง ื‘ืžืชืžื˜ื™ืงื” ืชื›ืœื™ืชื™ ื‘ืœื‘ื“: ื”ื™ื ืฉื™ืžืฉื” ื›ืื•ืกืฃ ืฉืœ ื ื•ืกื—ืื•ืช ืœื—ื™ืฉื•ื‘ ืงืจืงืข, ืื•ื›ืœื•ืกื™ืŸ ื•ื›ื•'. ืคืจื™ืฆืช ื”ื“ืจืš ืฉืœ ื”ื™ื•ื•ื ื™ื, ืคืจื˜ ืœืชืจื•ืžื•ืชื™ื”ื ื”ื’ื“ื•ืœื•ืช ืœื™ื“ืข ื”ืžืชืžื˜ื™, ื”ื™ื™ืชื” ื‘ืœื™ืžื•ื“ ื”ืžืชืžื˜ื™ืงื” ื›ืฉืœืขืฆืžื”, ืžืชื•ืงืฃ ืขืจื›ื” ื”ืจื•ื—ื ื™. ื™ื—ืกื ืฉืœ ื—ืœืง ืžื”ื™ื•ื•ื ื™ื ื”ืงื“ืžื•ื ื™ื ืœืžืชืžื˜ื™ืงื” ื”ื™ื” ื“ืชื™ - ืœืžืฉืœ, ื”ื›ืช ืฉืืกืฃ ืกื‘ื™ื‘ื• ืคื™ืชื’ื•ืจืก ื”ืืžื™ื ื” ื›ื™ ื”ืžืชืžื˜ื™ืงื” ื”ื™ื ื”ื‘ืกื™ืก ืœื›ืœ ื”ื“ื‘ืจื™ื. ื”ื™ื•ื•ื ื™ื ื ื—ืฉื‘ื™ื ืœื™ื•ืฆืจื™ ืžื•ืฉื’ ื”ื”ื•ื›ื—ื” ื”ืžืชืžื˜ื™ืช, ื•ื›ืŸ ืœืจืืฉื•ื ื™ื ืฉืขืกืงื• ื‘ืžืชืžื˜ื™ืงื” ืœืฉื ืขืฆืžื”, ื›ืœื•ืžืจ ื›ืชื—ื•ื ืžื—ืงืจื™ ืขื™ื•ื ื™ ื•ืžื•ืคืฉื˜ ื•ืœื ืจืง ื›ืขื–ืจ ืฉื™ืžื•ืฉื™. ืขื ื–ืืช, ืœืฆื“ื”`, - status: EscapeStatus{Escaped: true, HasAmbiguous: true}, + result: "ืขื“ ืชืงื•ืคืช ื™ื•ื•ืŸ ื”ืขืชื™ืงื” ื”ื™ื” ื”ืขื™ืกื•ืง ื‘ืžืชืžื˜ื™ืงื” ืชื›ืœื™ืชื™ ื‘ืœื‘ื“: ื”ื™ื ืฉื™ืžืฉื” ื›ืื•ืกืฃ ืฉืœ ื ื•ืกื—ืื•ืช ืœื—ื™ืฉื•ื‘ ืงืจืงืข, ืื•ื›ืœื•ืกื™ืŸ ื•ื›ื•'. ืคืจื™ืฆืช ื”ื“ืจืš ืฉืœ ื”ื™ื•ื•ื ื™ื, ืคืจื˜ ืœืชืจื•ืžื•ืชื™ื”ื ื”ื’ื“ื•ืœื•ืช ืœื™ื“ืข ื”ืžืชืžื˜ื™, ื”ื™ื™ืชื” ื‘ืœื™ืžื•ื“ ื”ืžืชืžื˜ื™ืงื” ื›ืฉืœืขืฆืžื”, ืžืชื•ืงืฃ ืขืจื›ื” ื”ืจื•ื—ื ื™. ื™ื—ืกื ืฉืœ ื—ืœืง ืžื”ื™ื•ื•ื ื™ื ื”ืงื“ืžื•ื ื™ื ืœืžืชืžื˜ื™ืงื” ื”ื™ื” ื“ืชื™ - ืœืžืฉืœ, ื”ื›ืช ืฉืืกืฃ ืกื‘ื™ื‘ื• ืคื™ืชื’ื•ืจืก ื”ืืžื™ื ื” ื›ื™ ื”ืžืชืžื˜ื™ืงื” ื”ื™ื ื”ื‘ืกื™ืก ืœื›ืœ ื”ื“ื‘ืจื™ื. ื”ื™ื•ื•ื ื™ื ื ื—ืฉื‘ื™ื ืœื™ื•ืฆืจื™ ืžื•ืฉื’ ื”ื”ื•ื›ื—ื” ื”ืžืชืžื˜ื™ืช, ื•ื›ืŸ ืœืจืืฉื•ื ื™ื ืฉืขืกืงื• ื‘ืžืชืžื˜ื™ืงื” ืœืฉื ืขืฆืžื”, ื›ืœื•ืžืจ ื›ืชื—ื•ื ืžื—ืงืจื™ ืขื™ื•ื ื™ ื•ืžื•ืคืฉื˜ ื•ืœื ืจืง ื›ืขื–ืจ ืฉื™ืžื•ืฉื™. ืขื ื–ืืช, ืœืฆื“ื”", + status: EscapeStatus{}, }, { - name: "more hebrew", + name: "more hebrew", // old test was wrong, such text shouldn't be escaped text: `ื‘ืชืงื•ืคื” ืžืื•ื—ืจืช ื™ื•ืชืจ, ื”ืฉืชืžืฉื• ื”ื™ื•ื•ื ื™ื ื‘ืฉื™ื˜ืช ืกื™ืžื•ืŸ ืžืชืงื“ืžืช ื™ื•ืชืจ, ืฉื‘ื” ื”ื•ืฆื’ื• ื”ืžืกืคืจื™ื ืœืคื™ 22 ืื•ืชื™ื•ืช ื”ืืœืคื‘ื™ืช ื”ื™ื•ื•ื ื™. ืœืกื™ืžื•ืŸ ื”ืžืกืคืจื™ื ื‘ื™ืŸ 1 ืœ-9 ื ืงื‘ืขื• ืชืฉืข ื”ืื•ืชื™ื•ืช ื”ืจืืฉื•ื ื•ืช, ื‘ืชื•ืกืคืช ื’ืจืฉ ( ' ) ื‘ืฆื“ ื™ืžื™ืŸ ืฉืœ ื”ืื•ืช, ืœืžืขืœื”; ืชืฉืข ื”ืื•ืชื™ื•ืช ื”ื‘ืื•ืช ื™ื™ืฆื’ื• ืืช ื”ืขืฉืจื•ืช ืž-10 ืขื“ 90, ื•ื”ื‘ืื•ืช ืืช ื”ืžืื•ืช. ืœืกื™ืžื•ืŸ ื”ืกืคืจื•ืช ื‘ื™ืŸ 1000 ืœ-900,000, ื”ืฉืชืžืฉื• ื”ื™ื•ื•ื ื™ื ื‘ืื•ืชืŸ ืื•ืชื™ื•ืช, ืืš ื”ื•ืกื™ืคื• ืœืื•ืชื™ื•ืช ืืช ื”ื’ืจืฉ ื“ื•ื•ืงื ืžืฆื“ ืฉืžืืœ ืฉืœ ื”ืื•ืชื™ื•ืช, ืœืžื˜ื”. ืžืžื™ืœื™ื•ืŸ ื•ืžืขืœื”, ื›ื ืจืื” ื”ืฉืชืžืฉื• ื”ื™ื•ื•ื ื™ื ื‘ืฉื ื™ ืชื’ื™ื ื‘ืžืงื•ื ืื—ื“. ื”ืžืชืžื˜ื™ืงืื™ ื”ื‘ื•ืœื˜ ื”ืจืืฉื•ืŸ ื‘ื™ื•ื•ืŸ ื”ืขืชื™ืงื”, ื•ื™ืฉ ื”ืื•ืžืจื™ื ื‘ืชื•ืœื“ื•ืช ื”ืื ื•ืฉื•ืช, ื”ื•ื ืชืืœืก (624 ืœืคื ื”"ืก - 546 ืœืคื ื”"ืก ื‘ืงื™ืจื•ื‘).[1] ืœื ื™ื”ื™ื” ื–ื” ืžืฉื•ืœืœ ื™ืกื•ื“ ืœื”ื ื™ื— ืฉื”ื•ื ื”ืื“ื ื”ืจืืฉื•ืŸ ืฉื”ื•ื›ื™ื— ืžืฉืคื˜ ืžืชืžื˜ื™, ื•ืœื ืจืง ื’ื™ืœื” ืื•ืชื•. ืชืืœืก ื”ื•ื›ื™ื— ืฉื™ืฉืจื™ื ืžืงื‘ื™ืœื™ื ื—ื•ืชื›ื™ื ืžืฆื“ ืื—ื“ ืฉืœ ืฉื•ืงื™ ื–ื•ื•ื™ืช ืงื˜ืขื™ื ื‘ืขืœื™ ื™ื—ืกื™ื ืฉื•ื•ื™ื (ืžืฉืคื˜ ืชืืœืก ื”ืจืืฉื•ืŸ), ืฉื”ื–ื•ื•ื™ืช ื”ืžื•ื ื—ืช ืขืœ ืงื•ื˜ืจ ื‘ืžืขื’ืœ ื”ื™ื ื–ื•ื•ื™ืช ื™ืฉืจื” (ืžืฉืคื˜ ืชืืœืก ื”ืฉื ื™), ืฉื”ืงื•ื˜ืจ ืžื—ืœืง ืืช ื”ืžืขื’ืœ ืœืฉื ื™ ื—ืœืงื™ื ืฉื•ื•ื™ื, ื•ืฉื–ื•ื•ื™ื•ืช ื”ื‘ืกื™ืก ื‘ืžืฉื•ืœืฉ ืฉื•ื•ื”-ืฉื•ืงื™ื™ื ืฉื•ื•ืช ื–ื• ืœื–ื•. ืžื™ื•ื—ืกื•ืช ืœื• ื’ื ืฉื™ื˜ื•ืช ืœืžื“ื™ื“ืช ื’ื•ื‘ื”ืŸ ืฉืœ ื”ืคื™ืจืžื™ื“ื•ืช ื‘ืขื–ืจืช ืžื“ื™ื“ืช ืฆื™ืœืŸ ื•ืœืงื‘ื™ืขืช ืžื™ืงื•ืžื” ืฉืœ ืกืคื™ื ื” ื”ื ืจืื™ืช ืžืŸ ื”ื—ื•ืฃ. ื‘ืฉื ื™ื 582 ืœืคื ื”"ืก ืขื“ 496 ืœืคื ื”"ืก, ื‘ืงื™ืจื•ื‘, ื—ื™ ืžืชืžื˜ื™ืงืื™ ื—ืฉื•ื‘ ื‘ืžื™ื•ื—ื“ - ืคื™ืชื’ื•ืจืก. ื”ืžืงื•ืจื•ืช ื”ืจืืฉื•ื ื™ื™ื ืขืœื™ื• ืžื•ืขื˜ื™ื, ื•ื”ื”ื™ืกื˜ื•ืจื™ื•ื ื™ื ืžืชืงืฉื™ื ืœื”ืคืจื™ื“ ืืช ื”ืขื•ื‘ื“ื•ืช ืžืฉื›ื‘ืช ื”ืžืกืชื•ืจื™ืŸ ื•ื”ืื’ื“ื•ืช ืฉื ืงืฉืจื• ื‘ื•. ื™ื“ื•ืข ืฉืกื‘ื™ื‘ื• ื”ืชืงื‘ืฆื” ื”ืืกื›ื•ืœื” ื”ืคื™ืชื’ื•ืจืื™ืช ืžืขื™ืŸ ื›ืช ืคืกื‘ื“ื•-ืžืชืžื˜ื™ืช ืฉื”ืืžื™ื ื” ืฉ"ื”ื›ืœ ืžืกืคืจ", ืื• ืœื™ืชืจ ื“ื™ื•ืง ื”ื›ืœ ื ื™ืชืŸ ืœื›ื™ืžื•ืช, ื•ื™ื™ื—ืกื” ืœืžืกืคืจื™ื ืžืฉืžืขื•ื™ื•ืช ืžื™ืกื˜ื™ื•ืช. ื›ื›ืœ ื”ื ืจืื” ื”ืคื™ืชื’ื•ืจืื™ื ื™ื“ืขื• ืœื‘ื ื•ืช ืืช ื”ื’ื•ืคื™ื ื”ืืคืœื˜ื•ื ื™ื™ื, ื”ื›ื™ืจื• ืืช ื”ืžืžื•ืฆืข ื”ืืจื™ืชืžื˜ื™, ื”ืžืžื•ืฆืข ื”ื’ืื•ืžื˜ืจื™ ื•ื”ืžืžื•ืฆืข ื”ื”ืจืžื•ื ื™ ื•ื”ื’ื™ืขื• ืœื”ื™ืฉื’ื™ื ื—ืฉื•ื‘ื™ื ื ื•ืกืคื™ื. ื ื™ืชืŸ ืœื•ืžืจ ืฉื”ืคื™ืชื’ื•ืจืื™ื ื’ื™ืœื• ืืช ื”ื™ื•ืชื• ืฉืœ ื”ืฉื•ืจืฉ ื”ืจื™ื‘ื•ืขื™ ืฉืœ 2, ืฉื”ื•ื ื’ื ื”ืืœื›ืกื•ืŸ ื‘ืจื™ื‘ื•ืข ืฉืื•ืจืš ืฆืœืขื•ืชื™ื• 1, ืื™ ืจืฆื™ื•ื ืœื™, ืืš ืชื’ืœื™ืชื ื”ื™ื™ืชื” ืœืžืขืฉื” ืจืง ืฉื”ืงื˜ืขื™ื "ื—ืกืจื™ ืžื™ื“ื” ืžืฉื•ืชืคืช", ื•ืžื•ืฉื’ ื”ืžืกืคืจ ื”ืื™ ืจืฆื™ื•ื ืœื™ ืžืื•ื—ืจ ื™ื•ืชืจ.[2] ืื–ื›ื•ืจ ืจืืฉื•ืŸ ืœืงื™ื•ืžื ืฉืœ ืงื˜ืขื™ื ื—ืกืจื™ ืžื™ื“ื” ืžืฉื•ืชืคืช ืžื•ืคื™ืข ื‘ื“ื™ืืœื•ื’ "ืชืื™ื˜ื™ื˜ื•ืก" ืฉืœ ืืคืœื˜ื•ืŸ, ืืš ืจืขื™ื•ืŸ ื–ื” ื”ื™ื” ืžื•ื›ืจ ืขื•ื“ ืงื•ื“ื ืœื›ืŸ, ื‘ืžืื” ื”ื—ืžื™ืฉื™ืช ืœืคื ื”"ืก ืœื”ื™ืคืืกื•ืก, ื‘ืŸ ื”ืืกื›ื•ืœื” ื”ืคื™ืชื’ื•ืจืื™ืช, ื•ืื•ืœื™ ืœืคื™ืชื’ื•ืจืก ืขืฆืžื•.[3]`, - result: `ื‘ืชืงื•ืคื” ืžืื•ื—ืจืช ื™ื•ืชืจ, ื”ืฉืชืžืฉื• ื”ื™ื•ื•ื ื™ื ื‘ืฉื™ื˜ืช ืกื™ืžื•ืŸ ืžืชืงื“ืžืช ื™ื•ืชืจ, ืฉื‘ื” ื”ื•ืฆื’ื• ื”ืžืกืคืจื™ื ืœืคื™ 22 ืื•ืชื™ื•ืช ื”ืืœืคื‘ื™ืช ื”ื™ื•ื•ื ื™. ืœืกื™ืžื•ืŸ ื”ืžืกืคืจื™ื ื‘ื™ืŸ 1 ืœ-9 ื ืงื‘ืขื• ืชืฉืข ื”ืื•ืชื™ื•ืช ื”ืจืืฉื•ื ื•ืช, ื‘ืชื•ืกืคืช ื’ืจืฉ ( ' ) ื‘ืฆื“ ื™ืžื™ืŸ ืฉืœ ื”ืื•ืช, ืœืžืขืœื”; ืชืฉืข ื”ืื•ืชื™ื•ืช ื”ื‘ืื•ืช ื™ื™ืฆื’ื• ืืช ื”ืขืฉืจื•ืช ืž-10 ืขื“ 90, ื•ื”ื‘ืื•ืช ืืช ื”ืžืื•ืช. ืœืกื™ืžื•ืŸ ื”ืกืคืจื•ืช ื‘ื™ืŸ 1000 ืœ-900,000, ื”ืฉืชืžืฉื• ื”ื™ื•ื•ื ื™ื ื‘ืื•ืชืŸ ืื•ืชื™ื•ืช, ืืš ื”ื•ืกื™ืคื• ืœืื•ืชื™ื•ืช ืืช ื”ื’ืจืฉ ื“ื•ื•ืงื ืžืฆื“ ืฉืžืืœ ืฉืœ ื”ืื•ืชื™ื•ืช, ืœืžื˜ื”. ืžืžื™ืœื™ื•ืŸ ื•ืžืขืœื”, ื›ื ืจืื” ื”ืฉืชืžืฉื• ื”ื™ื•ื•ื ื™ื ื‘ืฉื ื™ ืชื’ื™ื ื‘ืžืงื•ื ืื—ื“. + result: `ื‘ืชืงื•ืคื” ืžืื•ื—ืจืช ื™ื•ืชืจ, ื”ืฉืชืžืฉื• ื”ื™ื•ื•ื ื™ื ื‘ืฉื™ื˜ืช ืกื™ืžื•ืŸ ืžืชืงื“ืžืช ื™ื•ืชืจ, ืฉื‘ื” ื”ื•ืฆื’ื• ื”ืžืกืคืจื™ื ืœืคื™ 22 ืื•ืชื™ื•ืช ื”ืืœืคื‘ื™ืช ื”ื™ื•ื•ื ื™. ืœืกื™ืžื•ืŸ ื”ืžืกืคืจื™ื ื‘ื™ืŸ 1 ืœ-9 ื ืงื‘ืขื• ืชืฉืข ื”ืื•ืชื™ื•ืช ื”ืจืืฉื•ื ื•ืช, ื‘ืชื•ืกืคืช ื’ืจืฉ ( ' ) ื‘ืฆื“ ื™ืžื™ืŸ ืฉืœ ื”ืื•ืช, ืœืžืขืœื”; ืชืฉืข ื”ืื•ืชื™ื•ืช ื”ื‘ืื•ืช ื™ื™ืฆื’ื• ืืช ื”ืขืฉืจื•ืช ืž-10 ืขื“ 90, ื•ื”ื‘ืื•ืช ืืช ื”ืžืื•ืช. ืœืกื™ืžื•ืŸ ื”ืกืคืจื•ืช ื‘ื™ืŸ 1000 ืœ-900,000, ื”ืฉืชืžืฉื• ื”ื™ื•ื•ื ื™ื ื‘ืื•ืชืŸ ืื•ืชื™ื•ืช, ืืš ื”ื•ืกื™ืคื• ืœืื•ืชื™ื•ืช ืืช ื”ื’ืจืฉ ื“ื•ื•ืงื ืžืฆื“ ืฉืžืืœ ืฉืœ ื”ืื•ืชื™ื•ืช, ืœืžื˜ื”. ืžืžื™ืœื™ื•ืŸ ื•ืžืขืœื”, ื›ื ืจืื” ื”ืฉืชืžืฉื• ื”ื™ื•ื•ื ื™ื ื‘ืฉื ื™ ืชื’ื™ื ื‘ืžืงื•ื ืื—ื“. - ื”ืžืชืžื˜ื™ืงืื™ ื”ื‘ื•ืœื˜ ื”ืจืืฉื•ืŸ ื‘ื™ื•ื•ืŸ ื”ืขืชื™ืงื”, ื•ื™ืฉ ื”ืื•ืžืจื™ื ื‘ืชื•ืœื“ื•ืช ื”ืื ื•ืฉื•ืช, ื”ื•ื ืชืืœืก (624 ืœืคื ื”"ืก - 546 ืœืคื ื”"ืก ื‘ืงื™ืจื•ื‘).[1] ืœื ื™ื”ื™ื” ื–ื” ืžืฉื•ืœืœ ื™ืกื•ื“ ืœื”ื ื™ื— ืฉื”ื•ื ื”ืื“ื ื”ืจืืฉื•ืŸ ืฉื”ื•ื›ื™ื— ืžืฉืคื˜ ืžืชืžื˜ื™, ื•ืœื ืจืง ื’ื™ืœื” ืื•ืชื•. ืชืืœืก ื”ื•ื›ื™ื— ืฉื™ืฉืจื™ื ืžืงื‘ื™ืœื™ื ื—ื•ืชื›ื™ื ืžืฆื“ ืื—ื“ ืฉืœ ืฉื•ืงื™ ื–ื•ื•ื™ืช ืงื˜ืขื™ื ื‘ืขืœื™ ื™ื—ืกื™ื ืฉื•ื•ื™ื (ืžืฉืคื˜ ืชืืœืก ื”ืจืืฉื•ืŸ), ืฉื”ื–ื•ื•ื™ืช ื”ืžื•ื ื—ืช ืขืœ ืงื•ื˜ืจ ื‘ืžืขื’ืœ ื”ื™ื ื–ื•ื•ื™ืช ื™ืฉืจื” (ืžืฉืคื˜ ืชืืœืก ื”ืฉื ื™), ืฉื”ืงื•ื˜ืจ ืžื—ืœืง ืืช ื”ืžืขื’ืœ ืœืฉื ื™ ื—ืœืงื™ื ืฉื•ื•ื™ื, ื•ืฉื–ื•ื•ื™ื•ืช ื”ื‘ืกื™ืก ื‘ืžืฉื•ืœืฉ ืฉื•ื•ื”-ืฉื•ืงื™ื™ื ืฉื•ื•ืช ื–ื• ืœื–ื•. ืžื™ื•ื—ืกื•ืช ืœื• ื’ื ืฉื™ื˜ื•ืช ืœืžื“ื™ื“ืช ื’ื•ื‘ื”ืŸ ืฉืœ ื”ืคื™ืจืžื™ื“ื•ืช ื‘ืขื–ืจืช ืžื“ื™ื“ืช ืฆื™ืœืŸ ื•ืœืงื‘ื™ืขืช ืžื™ืงื•ืžื” ืฉืœ ืกืคื™ื ื” ื”ื ืจืื™ืช ืžืŸ ื”ื—ื•ืฃ. + ื”ืžืชืžื˜ื™ืงืื™ ื”ื‘ื•ืœื˜ ื”ืจืืฉื•ืŸ ื‘ื™ื•ื•ืŸ ื”ืขืชื™ืงื”, ื•ื™ืฉ ื”ืื•ืžืจื™ื ื‘ืชื•ืœื“ื•ืช ื”ืื ื•ืฉื•ืช, ื”ื•ื ืชืืœืก (624 ืœืคื ื”"ืก - 546 ืœืคื ื”"ืก ื‘ืงื™ืจื•ื‘).[1] ืœื ื™ื”ื™ื” ื–ื” ืžืฉื•ืœืœ ื™ืกื•ื“ ืœื”ื ื™ื— ืฉื”ื•ื ื”ืื“ื ื”ืจืืฉื•ืŸ ืฉื”ื•ื›ื™ื— ืžืฉืคื˜ ืžืชืžื˜ื™, ื•ืœื ืจืง ื’ื™ืœื” ืื•ืชื•. ืชืืœืก ื”ื•ื›ื™ื— ืฉื™ืฉืจื™ื ืžืงื‘ื™ืœื™ื ื—ื•ืชื›ื™ื ืžืฆื“ ืื—ื“ ืฉืœ ืฉื•ืงื™ ื–ื•ื•ื™ืช ืงื˜ืขื™ื ื‘ืขืœื™ ื™ื—ืกื™ื ืฉื•ื•ื™ื (ืžืฉืคื˜ ืชืืœืก ื”ืจืืฉื•ืŸ), ืฉื”ื–ื•ื•ื™ืช ื”ืžื•ื ื—ืช ืขืœ ืงื•ื˜ืจ ื‘ืžืขื’ืœ ื”ื™ื ื–ื•ื•ื™ืช ื™ืฉืจื” (ืžืฉืคื˜ ืชืืœืก ื”ืฉื ื™), ืฉื”ืงื•ื˜ืจ ืžื—ืœืง ืืช ื”ืžืขื’ืœ ืœืฉื ื™ ื—ืœืงื™ื ืฉื•ื•ื™ื, ื•ืฉื–ื•ื•ื™ื•ืช ื”ื‘ืกื™ืก ื‘ืžืฉื•ืœืฉ ืฉื•ื•ื”-ืฉื•ืงื™ื™ื ืฉื•ื•ืช ื–ื• ืœื–ื•. ืžื™ื•ื—ืกื•ืช ืœื• ื’ื ืฉื™ื˜ื•ืช ืœืžื“ื™ื“ืช ื’ื•ื‘ื”ืŸ ืฉืœ ื”ืคื™ืจืžื™ื“ื•ืช ื‘ืขื–ืจืช ืžื“ื™ื“ืช ืฆื™ืœืŸ ื•ืœืงื‘ื™ืขืช ืžื™ืงื•ืžื” ืฉืœ ืกืคื™ื ื” ื”ื ืจืื™ืช ืžืŸ ื”ื—ื•ืฃ. - ื‘ืฉื ื™ื 582 ืœืคื ื”"ืก ืขื“ 496 ืœืคื ื”"ืก, ื‘ืงื™ืจื•ื‘, ื—ื™ ืžืชืžื˜ื™ืงืื™ ื—ืฉื•ื‘ ื‘ืžื™ื•ื—ื“ - ืคื™ืชื’ื•ืจืก. ื”ืžืงื•ืจื•ืช ื”ืจืืฉื•ื ื™ื™ื ืขืœื™ื• ืžื•ืขื˜ื™ื, ื•ื”ื”ื™ืกื˜ื•ืจื™ื•ื ื™ื ืžืชืงืฉื™ื ืœื”ืคืจื™ื“ ืืช ื”ืขื•ื‘ื“ื•ืช ืžืฉื›ื‘ืช ื”ืžืกืชื•ืจื™ืŸ ื•ื”ืื’ื“ื•ืช ืฉื ืงืฉืจื• ื‘ื•. ื™ื“ื•ืข ืฉืกื‘ื™ื‘ื• ื”ืชืงื‘ืฆื” ื”ืืกื›ื•ืœื” ื”ืคื™ืชื’ื•ืจืื™ืช ืžืขื™ืŸ ื›ืช ืคืกื‘ื“ื•-ืžืชืžื˜ื™ืช ืฉื”ืืžื™ื ื” ืฉ"ื”ื›ืœ ืžืกืคืจ", ืื• ืœื™ืชืจ ื“ื™ื•ืง ื”ื›ืœ ื ื™ืชืŸ ืœื›ื™ืžื•ืช, ื•ื™ื™ื—ืกื” ืœืžืกืคืจื™ื ืžืฉืžืขื•ื™ื•ืช ืžื™ืกื˜ื™ื•ืช. ื›ื›ืœ ื”ื ืจืื” ื”ืคื™ืชื’ื•ืจืื™ื ื™ื“ืขื• ืœื‘ื ื•ืช ืืช ื”ื’ื•ืคื™ื ื”ืืคืœื˜ื•ื ื™ื™ื, ื”ื›ื™ืจื• ืืช ื”ืžืžื•ืฆืข ื”ืืจื™ืชืžื˜ื™, ื”ืžืžื•ืฆืข ื”ื’ืื•ืžื˜ืจื™ ื•ื”ืžืžื•ืฆืข ื”ื”ืจืžื•ื ื™ ื•ื”ื’ื™ืขื• ืœื”ื™ืฉื’ื™ื ื—ืฉื•ื‘ื™ื ื ื•ืกืคื™ื. ื ื™ืชืŸ ืœื•ืžืจ ืฉื”ืคื™ืชื’ื•ืจืื™ื ื’ื™ืœื• ืืช ื”ื™ื•ืชื• ืฉืœ ื”ืฉื•ืจืฉ ื”ืจื™ื‘ื•ืขื™ ืฉืœ 2, ืฉื”ื•ื ื’ื ื”ืืœื›ืกื•ืŸ ื‘ืจื™ื‘ื•ืข ืฉืื•ืจืš ืฆืœืขื•ืชื™ื• 1, ืื™ ืจืฆื™ื•ื ืœื™, ืืš ืชื’ืœื™ืชื ื”ื™ื™ืชื” ืœืžืขืฉื” ืจืง ืฉื”ืงื˜ืขื™ื "ื—ืกืจื™ ืžื™ื“ื” ืžืฉื•ืชืคืช", ื•ืžื•ืฉื’ ื”ืžืกืคืจ ื”ืื™ ืจืฆื™ื•ื ืœื™ ืžืื•ื—ืจ ื™ื•ืชืจ.[2] ืื–ื›ื•ืจ ืจืืฉื•ืŸ ืœืงื™ื•ืžื ืฉืœ ืงื˜ืขื™ื ื—ืกืจื™ ืžื™ื“ื” ืžืฉื•ืชืคืช ืžื•ืคื™ืข ื‘ื“ื™ืืœื•ื’ "ืชืื™ื˜ื™ื˜ื•ืก" ืฉืœ ืืคืœื˜ื•ืŸ, ืืš ืจืขื™ื•ืŸ ื–ื” ื”ื™ื” ืžื•ื›ืจ ืขื•ื“ ืงื•ื“ื ืœื›ืŸ, ื‘ืžืื” ื”ื—ืžื™ืฉื™ืช ืœืคื ื”"ืก ืœื”ื™ืคืืกื•ืก, ื‘ืŸ ื”ืืกื›ื•ืœื” ื”ืคื™ืชื’ื•ืจืื™ืช, ื•ืื•ืœื™ ืœืคื™ืชื’ื•ืจืก ืขืฆืžื•.[3]`, - status: EscapeStatus{Escaped: true, HasAmbiguous: true}, + ื‘ืฉื ื™ื 582 ืœืคื ื”"ืก ืขื“ 496 ืœืคื ื”"ืก, ื‘ืงื™ืจื•ื‘, ื—ื™ ืžืชืžื˜ื™ืงืื™ ื—ืฉื•ื‘ ื‘ืžื™ื•ื—ื“ - ืคื™ืชื’ื•ืจืก. ื”ืžืงื•ืจื•ืช ื”ืจืืฉื•ื ื™ื™ื ืขืœื™ื• ืžื•ืขื˜ื™ื, ื•ื”ื”ื™ืกื˜ื•ืจื™ื•ื ื™ื ืžืชืงืฉื™ื ืœื”ืคืจื™ื“ ืืช ื”ืขื•ื‘ื“ื•ืช ืžืฉื›ื‘ืช ื”ืžืกืชื•ืจื™ืŸ ื•ื”ืื’ื“ื•ืช ืฉื ืงืฉืจื• ื‘ื•. ื™ื“ื•ืข ืฉืกื‘ื™ื‘ื• ื”ืชืงื‘ืฆื” ื”ืืกื›ื•ืœื” ื”ืคื™ืชื’ื•ืจืื™ืช ืžืขื™ืŸ ื›ืช ืคืกื‘ื“ื•-ืžืชืžื˜ื™ืช ืฉื”ืืžื™ื ื” ืฉ"ื”ื›ืœ ืžืกืคืจ", ืื• ืœื™ืชืจ ื“ื™ื•ืง ื”ื›ืœ ื ื™ืชืŸ ืœื›ื™ืžื•ืช, ื•ื™ื™ื—ืกื” ืœืžืกืคืจื™ื ืžืฉืžืขื•ื™ื•ืช ืžื™ืกื˜ื™ื•ืช. ื›ื›ืœ ื”ื ืจืื” ื”ืคื™ืชื’ื•ืจืื™ื ื™ื“ืขื• ืœื‘ื ื•ืช ืืช ื”ื’ื•ืคื™ื ื”ืืคืœื˜ื•ื ื™ื™ื, ื”ื›ื™ืจื• ืืช ื”ืžืžื•ืฆืข ื”ืืจื™ืชืžื˜ื™, ื”ืžืžื•ืฆืข ื”ื’ืื•ืžื˜ืจื™ ื•ื”ืžืžื•ืฆืข ื”ื”ืจืžื•ื ื™ ื•ื”ื’ื™ืขื• ืœื”ื™ืฉื’ื™ื ื—ืฉื•ื‘ื™ื ื ื•ืกืคื™ื. ื ื™ืชืŸ ืœื•ืžืจ ืฉื”ืคื™ืชื’ื•ืจืื™ื ื’ื™ืœื• ืืช ื”ื™ื•ืชื• ืฉืœ ื”ืฉื•ืจืฉ ื”ืจื™ื‘ื•ืขื™ ืฉืœ 2, ืฉื”ื•ื ื’ื ื”ืืœื›ืกื•ืŸ ื‘ืจื™ื‘ื•ืข ืฉืื•ืจืš ืฆืœืขื•ืชื™ื• 1, ืื™ ืจืฆื™ื•ื ืœื™, ืืš ืชื’ืœื™ืชื ื”ื™ื™ืชื” ืœืžืขืฉื” ืจืง ืฉื”ืงื˜ืขื™ื "ื—ืกืจื™ ืžื™ื“ื” ืžืฉื•ืชืคืช", ื•ืžื•ืฉื’ ื”ืžืกืคืจ ื”ืื™ ืจืฆื™ื•ื ืœื™ ืžืื•ื—ืจ ื™ื•ืชืจ.[2] ืื–ื›ื•ืจ ืจืืฉื•ืŸ ืœืงื™ื•ืžื ืฉืœ ืงื˜ืขื™ื ื—ืกืจื™ ืžื™ื“ื” ืžืฉื•ืชืคืช ืžื•ืคื™ืข ื‘ื“ื™ืืœื•ื’ "ืชืื™ื˜ื™ื˜ื•ืก" ืฉืœ ืืคืœื˜ื•ืŸ, ืืš ืจืขื™ื•ืŸ ื–ื” ื”ื™ื” ืžื•ื›ืจ ืขื•ื“ ืงื•ื“ื ืœื›ืŸ, ื‘ืžืื” ื”ื—ืžื™ืฉื™ืช ืœืคื ื”"ืก ืœื”ื™ืคืืกื•ืก, ื‘ืŸ ื”ืืกื›ื•ืœื” ื”ืคื™ืชื’ื•ืจืื™ืช, ื•ืื•ืœื™ ืœืคื™ืชื’ื•ืจืก ืขืฆืžื•.[3]`, + status: EscapeStatus{}, }, { name: "Mixed RTL+LTR", @@ -111,7 +111,7 @@ then resh (ืจ), and finally heh (ื”) (which should appear leftmost).`, { name: "CVE testcase", text: "if access_level != \"user\u202E \u2066// Check if admin\u2069 \u2066\" {", - result: `if access_level != "user` + "\u202e" + ` ` + "\u2066" + `// Check if admin` + "\u2069" + ` ` + "\u2066" + `" {`, + result: `if access_level != "user` + "\u202e" + ` ` + "\u2066" + `// Check if admin` + "\u2069" + ` ` + "\u2066" + `" {`, status: EscapeStatus{Escaped: true, HasInvisible: true}, }, { @@ -123,7 +123,7 @@ then resh (ืจ), and finally heh (ื”) (which should appear leftmost).`, result: `Many computer programs fail to display bidirectional text correctly. For example, the Hebrew name Sarah ` + "\u2067" + `ืฉืจื”` + "\u2066\n" + `sin (ืฉ) (which appears rightmost), then resh (ืจ), and finally heh (ื”) (which should appear leftmost).` + - "\n" + `if access_level != "user` + "\u202e" + ` ` + "\u2066" + `// Check if admin` + "\u2069" + ` ` + "\u2066" + `" {` + "\n", + "\n" + `if access_level != "user` + "\u202e" + ` ` + "\u2066" + `// Check if admin` + "\u2069" + ` ` + "\u2066" + `" {` + "\n", status: EscapeStatus{Escaped: true, HasInvisible: true}, }, { @@ -134,38 +134,22 @@ then resh (ืจ), and finally heh (ื”) (which should appear leftmost).`, result: "\xef\xbb\xbftest", status: EscapeStatus{}, }, + { + name: "ambiguous", + text: "O๐พ", + result: `O๐พ`, + status: EscapeStatus{Escaped: true, HasAmbiguous: true}, + }, } func TestEscapeControlReader(t *testing.T) { - // add some control characters to the tests - tests := make([]escapeControlTest, 0, len(escapeControlTests)*3) - copy(tests, escapeControlTests) - - // if there is a BOM, we should keep the BOM - addPrefix := func(prefix, s string) string { - if strings.HasPrefix(s, "\xef\xbb\xbf") { - return s[:3] + prefix + s[3:] - } - return prefix + s - } - for _, test := range escapeControlTests { - test.name += " (+Control)" - test.text = addPrefix("\u001E", test.text) - test.result = addPrefix(``+"\u001e"+``, test.result) - test.status.Escaped = true - test.status.HasInvisible = true - tests = append(tests, test) - } - - re := regexp.MustCompile(`repo.ambiguous_character:\d+,\d+`) // simplify the output for the tests, remove the translation variants - for _, tt := range tests { + for _, tt := range escapeControlTests { t.Run(tt.name, func(t *testing.T) { output := &strings.Builder{} status, err := EscapeControlReader(strings.NewReader(tt.text), output, &translation.MockLocale{}) assert.NoError(t, err) assert.Equal(t, tt.status, *status) outStr := output.String() - outStr = re.ReplaceAllString(outStr, "repo.ambiguous_character") assert.Equal(t, tt.result, outStr) }) } @@ -179,3 +163,50 @@ func TestSettingAmbiguousUnicodeDetection(t *testing.T) { _, out = EscapeControlHTML("aย test", &translation.MockLocale{}) assert.EqualValues(t, `aย test`, out) } + +func TestHTMLChunkReader(t *testing.T) { + type textPart struct { + text string + isTag bool + } + testReadChunks := func(t *testing.T, chunkSize int, input string, expected []textPart) { + r := &htmlChunkReader{in: strings.NewReader(input), readBuf: make([]byte, 0, chunkSize)} + var results []textPart + for { + parts, partIsTag, err := r.readRunes() + if err != nil { + break + } + for i, part := range parts { + results = append(results, textPart{string(part), partIsTag[i]}) + } + } + assert.Equal(t, expected, results, "chunk size: %d, input: %s", chunkSize, input) + } + + testReadChunks(t, 10, "abcghi", []textPart{ + {text: "abc", isTag: false}, + {text: "", isTag: true}, + {text: "gh", isTag: false}, + // -- chunk + {text: "i", isTag: false}, + }) + + testReadChunks(t, 10, "ghi", []textPart{ + {text: "", isTag: true}, + {text: "", isTag: true}, + // -- chunk + {text: "ghi", isTag: false}, + }) + + rune1, rune2, rune3, rune4 := "A", "รฉ", "ๅ•Š", "๐ŸŒž" + require.Len(t, rune1, 1) + require.Len(t, rune2, 2) + require.Len(t, rune3, 3) + require.Len(t, rune4, 4) + input := "<" + rune1 + rune2 + rune3 + rune4 + ">" + rune1 + rune2 + rune3 + rune4 + testReadChunks(t, 4, input, []textPart{{"", true}, {"Aรฉ", false}, {"ๅ•Š", false}, {"๐ŸŒž", false}}) + testReadChunks(t, 5, input, []textPart{{"", true}, {"Aรฉ", false}, {"ๅ•Š", false}, {"๐ŸŒž", false}}) + testReadChunks(t, 6, input, []textPart{{"", true}, {"A", false}, {"รฉๅ•Š", false}, {"๐ŸŒž", false}}) + testReadChunks(t, 7, input, []textPart{{"", true}, {"A", false}, {"รฉๅ•Š", false}, {"๐ŸŒž", false}}) +} diff --git a/modules/charset/ambiguous/ambiguous.json b/modules/charset/generate/ambiguous.json similarity index 100% rename from modules/charset/ambiguous/ambiguous.json rename to modules/charset/generate/ambiguous.json diff --git a/modules/charset/invisible/generate.go b/modules/charset/generate/generate.go similarity index 57% rename from modules/charset/invisible/generate.go rename to modules/charset/generate/generate.go index bd57dd6c4d..16ea53fda1 100644 --- a/modules/charset/invisible/generate.go +++ b/modules/charset/generate/generate.go @@ -5,37 +5,86 @@ package main import ( "bytes" - "flag" "fmt" "go/format" + "log" "os" + "sort" "text/template" + "unicode" + + "code.gitea.io/gitea/modules/json" "golang.org/x/text/unicode/rangetable" ) +// ambiguous.json provides a one to one mapping of ambiguous characters to other characters +// See https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json + +type AmbiguousTable struct { + Confusable []rune + With []rune + Locale string + RangeTable *unicode.RangeTable +} + +type RunePair struct { + Confusable rune + With rune +} + // InvisibleRunes these are runes that vscode has assigned to be invisible // See https://github.com/hediet/vscode-unicode-data var InvisibleRunes = []rune{ 9, 10, 11, 12, 13, 32, 127, 160, 173, 847, 1564, 4447, 4448, 6068, 6069, 6155, 6156, 6157, 6158, 7355, 7356, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200, 8201, 8202, 8203, 8204, 8205, 8206, 8207, 8234, 8235, 8236, 8237, 8238, 8239, 8287, 8288, 8289, 8290, 8291, 8292, 8293, 8294, 8295, 8296, 8297, 8298, 8299, 8300, 8301, 8302, 8303, 10240, 12288, 12644, 65024, 65025, 65026, 65027, 65028, 65029, 65030, 65031, 65032, 65033, 65034, 65035, 65036, 65037, 65038, 65039, 65279, 65440, 65520, 65521, 65522, 65523, 65524, 65525, 65526, 65527, 65528, 65532, 78844, 119155, 119156, 119157, 119158, 119159, 119160, 119161, 119162, 917504, 917505, 917506, 917507, 917508, 917509, 917510, 917511, 917512, 917513, 917514, 917515, 917516, 917517, 917518, 917519, 917520, 917521, 917522, 917523, 917524, 917525, 917526, 917527, 917528, 917529, 917530, 917531, 917532, 917533, 917534, 917535, 917536, 917537, 917538, 917539, 917540, 917541, 917542, 917543, 917544, 917545, 917546, 917547, 917548, 917549, 917550, 917551, 917552, 917553, 917554, 917555, 917556, 917557, 917558, 917559, 917560, 917561, 917562, 917563, 917564, 917565, 917566, 917567, 917568, 917569, 917570, 917571, 917572, 917573, 917574, 917575, 917576, 917577, 917578, 917579, 917580, 917581, 917582, 917583, 917584, 917585, 917586, 917587, 917588, 917589, 917590, 917591, 917592, 917593, 917594, 917595, 917596, 917597, 917598, 917599, 917600, 917601, 917602, 917603, 917604, 917605, 917606, 917607, 917608, 917609, 917610, 917611, 917612, 917613, 917614, 917615, 917616, 917617, 917618, 917619, 917620, 917621, 917622, 917623, 917624, 917625, 917626, 917627, 917628, 917629, 917630, 917631, 917760, 917761, 917762, 917763, 917764, 917765, 917766, 917767, 917768, 917769, 917770, 917771, 917772, 917773, 917774, 917775, 917776, 917777, 917778, 917779, 917780, 917781, 917782, 917783, 917784, 917785, 917786, 917787, 917788, 917789, 917790, 917791, 917792, 917793, 917794, 917795, 917796, 917797, 917798, 917799, 917800, 917801, 917802, 917803, 917804, 917805, 917806, 917807, 917808, 917809, 917810, 917811, 917812, 917813, 917814, 917815, 917816, 917817, 917818, 917819, 917820, 917821, 917822, 917823, 917824, 917825, 917826, 917827, 917828, 917829, 917830, 917831, 917832, 917833, 917834, 917835, 917836, 917837, 917838, 917839, 917840, 917841, 917842, 917843, 917844, 917845, 917846, 917847, 917848, 917849, 917850, 917851, 917852, 917853, 917854, 917855, 917856, 917857, 917858, 917859, 917860, 917861, 917862, 917863, 917864, 917865, 917866, 917867, 917868, 917869, 917870, 917871, 917872, 917873, 917874, 917875, 917876, 917877, 917878, 917879, 917880, 917881, 917882, 917883, 917884, 917885, 917886, 917887, 917888, 917889, 917890, 917891, 917892, 917893, 917894, 917895, 917896, 917897, 917898, 917899, 917900, 917901, 917902, 917903, 917904, 917905, 917906, 917907, 917908, 917909, 917910, 917911, 917912, 917913, 917914, 917915, 917916, 917917, 917918, 917919, 917920, 917921, 917922, 917923, 917924, 917925, 917926, 917927, 917928, 917929, 917930, 917931, 917932, 917933, 917934, 917935, 917936, 917937, 917938, 917939, 917940, 917941, 917942, 917943, 917944, 917945, 917946, 917947, 917948, 917949, 917950, 917951, 917952, 917953, 917954, 917955, 917956, 917957, 917958, 917959, 917960, 917961, 917962, 917963, 917964, 917965, 917966, 917967, 917968, 917969, 917970, 917971, 917972, 917973, 917974, 917975, 917976, 917977, 917978, 917979, 917980, 917981, 917982, 917983, 917984, 917985, 917986, 917987, 917988, 917989, 917990, 917991, 917992, 917993, 917994, 917995, 917996, 917997, 917998, 917999, } -var verbose bool - -func main() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, `%s: Generate InvisibleRunesRange - -Usage: %[1]s [-v] [-o output.go] -`, os.Args[0]) - flag.PrintDefaults() +func generateAmbiguous() { + bs, err := os.ReadFile("ambiguous.json") + if err != nil { + log.Fatalf("Unable to read, err: %v", err) } - output := "" - flag.BoolVar(&verbose, "v", false, "verbose output") - flag.StringVar(&output, "o", "invisible_gen.go", "file to output to") - flag.Parse() + var unwrapped string + if err := json.Unmarshal(bs, &unwrapped); err != nil { + log.Fatalf("Unable to unwrap content in, err: %v", err) + } + fromJSON := map[string][]uint32{} + if err := json.Unmarshal([]byte(unwrapped), &fromJSON); err != nil { + log.Fatalf("Unable to unmarshal content in, err: %v", err) + } + + tables := make([]*AmbiguousTable, 0, len(fromJSON)) + for locale, chars := range fromJSON { + table := &AmbiguousTable{Locale: locale} + table.Confusable = make([]rune, 0, len(chars)/2) + table.With = make([]rune, 0, len(chars)/2) + pairs := make([]RunePair, len(chars)/2) + for i := 0; i < len(chars); i += 2 { + pairs[i/2].Confusable, pairs[i/2].With = rune(chars[i]), rune(chars[i+1]) + } + sort.Slice(pairs, func(i, j int) bool { + return pairs[i].Confusable < pairs[j].Confusable + }) + for _, pair := range pairs { + table.Confusable = append(table.Confusable, pair.Confusable) + table.With = append(table.With, pair.With) + } + table.RangeTable = rangetable.New(table.Confusable...) + tables = append(tables, table) + } + sort.Slice(tables, func(i, j int) bool { + return tables[i].Locale < tables[j].Locale + }) + data := map[string]any{"Tables": tables} + + if err := runTemplate(templateAmbiguous, "../ambiguous_gen.go", &data); err != nil { + log.Fatalf("Unable to run template: %v", err) + } +} + +func generateInvisible() { // First we filter the runes to remove // filtered := make([]rune, 0, len(InvisibleRunes)) @@ -47,8 +96,8 @@ Usage: %[1]s [-v] [-o output.go] } table := rangetable.New(filtered...) - if err := runTemplate(generatorTemplate, output, table); err != nil { - fatalf("Unable to run template: %v", err) + if err := runTemplate(generatorInvisible, "../invisible_gen.go", table); err != nil { + log.Fatalf("Unable to run template: %v", err) } } @@ -59,7 +108,7 @@ func runTemplate(t *template.Template, filename string, data any) error { } bs, err := format.Source(buf.Bytes()) if err != nil { - verbosef("Bad source:\n%s", buf.String()) + log.Printf("Bad source:\n%s", buf.String()) return fmt.Errorf("unable to format source: %w", err) } @@ -85,37 +134,68 @@ func runTemplate(t *template.Template, filename string, data any) error { return nil } -var generatorTemplate = template.Must(template.New("invisibleTemplate").Parse(`// This file is generated by modules/charset/invisible/generate.go DO NOT EDIT -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT +func main() { + generateAmbiguous() + generateInvisible() +} +var templateAmbiguous = template.Must(template.New("ambiguousTemplate").Parse(`// This file is generated by modules/charset/generate/generate.go DO NOT EDIT +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT package charset import "unicode" -var InvisibleRanges = &unicode.RangeTable{ - R16: []unicode.Range16{ -{{range .R16 }} {Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}}, -{{end}} }, - R32: []unicode.Range32{ -{{range .R32}} {Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}}, -{{end}} }, - LatinOffset: {{.LatinOffset}}, +// This file is generated from https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json + +// AmbiguousTable matches a confusable rune with its partner for the Locale +type AmbiguousTable struct { + Confusable []rune + With []rune + Locale string + RangeTable *unicode.RangeTable +} + +func newAmbiguousTableMap() map[string]*AmbiguousTable { + return map[string]*AmbiguousTable { + {{- range .Tables}} + {{printf "%q" .Locale}}: { + Confusable: []rune{ {{range .Confusable}}{{.}},{{end}} }, + With: []rune{ {{range .With}}{{.}},{{end}} }, + Locale: {{printf "%q" .Locale}}, + RangeTable: &unicode.RangeTable{ + R16: []unicode.Range16{ + {{range .RangeTable.R16 }} {Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}}, + {{end}} }, + R32: []unicode.Range32{ + {{range .RangeTable.R32}} {Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}}, + {{end}} }, + LatinOffset: {{.RangeTable.LatinOffset}}, + }, + }, + {{end}} + } } `)) -func logf(format string, args ...any) { - fmt.Fprintf(os.Stderr, format+"\n", args...) -} +var generatorInvisible = template.Must(template.New("invisibleTemplate").Parse(`// This file is generated by modules/charset/generate/generate.go DO NOT EDIT +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT -func verbosef(format string, args ...any) { - if verbose { - logf(format, args...) +package charset + +import "unicode" + +func newInvisibleRangeTable() *unicode.RangeTable { + return &unicode.RangeTable{ + R16: []unicode.Range16{ +{{range .R16 }} {Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}}, + {{end}}}, + R32: []unicode.Range32{ +{{range .R32}} {Lo:{{.Lo}}, Hi:{{.Hi}}, Stride: {{.Stride}}}, + {{end}}}, + LatinOffset: {{.LatinOffset}}, } } - -func fatalf(format string, args ...any) { - logf("fatal: "+format+"\n", args...) - os.Exit(1) -} +`)) diff --git a/modules/charset/htmlstream.go b/modules/charset/htmlstream.go deleted file mode 100644 index 61f29120a6..0000000000 --- a/modules/charset/htmlstream.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package charset - -import ( - "fmt" - "io" - - "golang.org/x/net/html" -) - -// HTMLStreamer represents a SAX-like interface for HTML -type HTMLStreamer interface { - Error(err error) error - Doctype(data string) error - Comment(data string) error - StartTag(data string, attrs ...html.Attribute) error - SelfClosingTag(data string, attrs ...html.Attribute) error - EndTag(data string) error - Text(data string) error -} - -// PassthroughHTMLStreamer is a passthrough streamer -type PassthroughHTMLStreamer struct { - next HTMLStreamer -} - -func NewPassthroughStreamer(next HTMLStreamer) *PassthroughHTMLStreamer { - return &PassthroughHTMLStreamer{next: next} -} - -var _ (HTMLStreamer) = &PassthroughHTMLStreamer{} - -// Error tells the next streamer in line that there is an error -func (p *PassthroughHTMLStreamer) Error(err error) error { - return p.next.Error(err) -} - -// Doctype tells the next streamer what the doctype is -func (p *PassthroughHTMLStreamer) Doctype(data string) error { - return p.next.Doctype(data) -} - -// Comment tells the next streamer there is a comment -func (p *PassthroughHTMLStreamer) Comment(data string) error { - return p.next.Comment(data) -} - -// StartTag tells the next streamer there is a starting tag -func (p *PassthroughHTMLStreamer) StartTag(data string, attrs ...html.Attribute) error { - return p.next.StartTag(data, attrs...) -} - -// SelfClosingTag tells the next streamer there is a self-closing tag -func (p *PassthroughHTMLStreamer) SelfClosingTag(data string, attrs ...html.Attribute) error { - return p.next.SelfClosingTag(data, attrs...) -} - -// EndTag tells the next streamer there is a end tag -func (p *PassthroughHTMLStreamer) EndTag(data string) error { - return p.next.EndTag(data) -} - -// Text tells the next streamer there is a text -func (p *PassthroughHTMLStreamer) Text(data string) error { - return p.next.Text(data) -} - -// HTMLStreamWriter acts as a writing sink -type HTMLStreamerWriter struct { - io.Writer - err error -} - -// Write implements io.Writer -func (h *HTMLStreamerWriter) Write(data []byte) (int, error) { - if h.err != nil { - return 0, h.err - } - return h.Writer.Write(data) -} - -// Write implements io.StringWriter -func (h *HTMLStreamerWriter) WriteString(data string) (int, error) { - if h.err != nil { - return 0, h.err - } - return h.Writer.Write([]byte(data)) -} - -// Error tells the next streamer in line that there is an error -func (h *HTMLStreamerWriter) Error(err error) error { - if h.err == nil { - h.err = err - } - return h.err -} - -// Doctype tells the next streamer what the doctype is -func (h *HTMLStreamerWriter) Doctype(data string) error { - _, h.err = h.WriteString("") - return h.err -} - -// Comment tells the next streamer there is a comment -func (h *HTMLStreamerWriter) Comment(data string) error { - _, h.err = h.WriteString("") - return h.err -} - -// StartTag tells the next streamer there is a starting tag -func (h *HTMLStreamerWriter) StartTag(data string, attrs ...html.Attribute) error { - return h.startTag(data, attrs, false) -} - -// SelfClosingTag tells the next streamer there is a self-closing tag -func (h *HTMLStreamerWriter) SelfClosingTag(data string, attrs ...html.Attribute) error { - return h.startTag(data, attrs, true) -} - -func (h *HTMLStreamerWriter) startTag(data string, attrs []html.Attribute, selfclosing bool) error { - if _, h.err = h.WriteString("<" + data); h.err != nil { - return h.err - } - for _, attr := range attrs { - if _, h.err = h.WriteString(" " + attr.Key + "=\"" + html.EscapeString(attr.Val) + "\""); h.err != nil { - return h.err - } - } - if selfclosing { - if _, h.err = h.WriteString("/>"); h.err != nil { - return h.err - } - } else { - if _, h.err = h.WriteString(">"); h.err != nil { - return h.err - } - } - return h.err -} - -// EndTag tells the next streamer there is a end tag -func (h *HTMLStreamerWriter) EndTag(data string) error { - _, h.err = h.WriteString("") - return h.err -} - -// Text tells the next streamer there is a text -func (h *HTMLStreamerWriter) Text(data string) error { - _, h.err = h.WriteString(html.EscapeString(data)) - return h.err -} - -// StreamHTML streams an html to a provided streamer -func StreamHTML(source io.Reader, streamer HTMLStreamer) error { - tokenizer := html.NewTokenizer(source) - for { - tt := tokenizer.Next() - switch tt { - case html.ErrorToken: - if tokenizer.Err() != io.EOF { - return tokenizer.Err() - } - return nil - case html.DoctypeToken: - token := tokenizer.Token() - if err := streamer.Doctype(token.Data); err != nil { - return err - } - case html.CommentToken: - token := tokenizer.Token() - if err := streamer.Comment(token.Data); err != nil { - return err - } - case html.StartTagToken: - token := tokenizer.Token() - if err := streamer.StartTag(token.Data, token.Attr...); err != nil { - return err - } - case html.SelfClosingTagToken: - token := tokenizer.Token() - if err := streamer.StartTag(token.Data, token.Attr...); err != nil { - return err - } - case html.EndTagToken: - token := tokenizer.Token() - if err := streamer.EndTag(token.Data); err != nil { - return err - } - case html.TextToken: - token := tokenizer.Token() - if err := streamer.Text(token.Data); err != nil { - return err - } - default: - return fmt.Errorf("unknown type of token: %d", tt) - } - } -} diff --git a/modules/charset/invisible_gen.go b/modules/charset/invisible_gen.go index 812f0e34b3..ddda875a9f 100644 --- a/modules/charset/invisible_gen.go +++ b/modules/charset/invisible_gen.go @@ -1,36 +1,38 @@ -// This file is generated by modules/charset/invisible/generate.go DO NOT EDIT -// Copyright 2022 The Gitea Authors. All rights reserved. +// This file is generated by modules/charset/generate/generate.go DO NOT EDIT +// Copyright 2026 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package charset import "unicode" -var InvisibleRanges = &unicode.RangeTable{ - R16: []unicode.Range16{ - {Lo: 11, Hi: 13, Stride: 1}, - {Lo: 127, Hi: 160, Stride: 33}, - {Lo: 173, Hi: 847, Stride: 674}, - {Lo: 1564, Hi: 4447, Stride: 2883}, - {Lo: 4448, Hi: 6068, Stride: 1620}, - {Lo: 6069, Hi: 6155, Stride: 86}, - {Lo: 6156, Hi: 6158, Stride: 1}, - {Lo: 7355, Hi: 7356, Stride: 1}, - {Lo: 8192, Hi: 8207, Stride: 1}, - {Lo: 8234, Hi: 8239, Stride: 1}, - {Lo: 8287, Hi: 8303, Stride: 1}, - {Lo: 10240, Hi: 12288, Stride: 2048}, - {Lo: 12644, Hi: 65024, Stride: 52380}, - {Lo: 65025, Hi: 65039, Stride: 1}, - {Lo: 65279, Hi: 65440, Stride: 161}, - {Lo: 65520, Hi: 65528, Stride: 1}, - {Lo: 65532, Hi: 65532, Stride: 1}, - }, - R32: []unicode.Range32{ - {Lo: 78844, Hi: 119155, Stride: 40311}, - {Lo: 119156, Hi: 119162, Stride: 1}, - {Lo: 917504, Hi: 917631, Stride: 1}, - {Lo: 917760, Hi: 917999, Stride: 1}, - }, - LatinOffset: 2, +func newInvisibleRangeTable() *unicode.RangeTable { + return &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 11, Hi: 13, Stride: 1}, + {Lo: 127, Hi: 160, Stride: 33}, + {Lo: 173, Hi: 847, Stride: 674}, + {Lo: 1564, Hi: 4447, Stride: 2883}, + {Lo: 4448, Hi: 6068, Stride: 1620}, + {Lo: 6069, Hi: 6155, Stride: 86}, + {Lo: 6156, Hi: 6158, Stride: 1}, + {Lo: 7355, Hi: 7356, Stride: 1}, + {Lo: 8192, Hi: 8207, Stride: 1}, + {Lo: 8234, Hi: 8239, Stride: 1}, + {Lo: 8287, Hi: 8303, Stride: 1}, + {Lo: 10240, Hi: 12288, Stride: 2048}, + {Lo: 12644, Hi: 65024, Stride: 52380}, + {Lo: 65025, Hi: 65039, Stride: 1}, + {Lo: 65279, Hi: 65440, Stride: 161}, + {Lo: 65520, Hi: 65528, Stride: 1}, + {Lo: 65532, Hi: 65532, Stride: 1}, + }, + R32: []unicode.Range32{ + {Lo: 78844, Hi: 119155, Stride: 40311}, + {Lo: 119156, Hi: 119162, Stride: 1}, + {Lo: 917504, Hi: 917631, Stride: 1}, + {Lo: 917760, Hi: 917999, Stride: 1}, + }, + LatinOffset: 2, + } } diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index addc372f85..dca28588e4 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -6,8 +6,8 @@ package highlight import ( "bytes" + gohtml "html" "html/template" - "slices" "sync" "code.gitea.io/gitea/modules/log" @@ -15,7 +15,7 @@ import ( "code.gitea.io/gitea/modules/util" "github.com/alecthomas/chroma/v2" - "github.com/alecthomas/chroma/v2/formatters/html" + chromahtml "github.com/alecthomas/chroma/v2/formatters/html" "github.com/alecthomas/chroma/v2/styles" ) @@ -25,8 +25,6 @@ const sizeLimit = 1024 * 1024 type globalVarsType struct { highlightMapping map[string]string githubStyles *chroma.Style - escapeFull []template.HTML - escCtrlCharsMap []template.HTML } var ( @@ -42,69 +40,10 @@ func globalVars() *globalVarsType { globalVarsPtr = &globalVarsType{} globalVarsPtr.githubStyles = styles.Get("github") globalVarsPtr.highlightMapping = setting.GetHighlightMapping() - globalVarsPtr.escCtrlCharsMap = make([]template.HTML, 256) - // ASCII Table 0x00 - 0x1F - controlCharNames := []string{ - "NUL", "SOH", "STX", "ETX", "EOT", "ENQ", "ACK", "BEL", - "BS", "HT", "LF", "VT", "FF", "CR", "SO", "SI", - "DLE", "DC1", "DC2", "DC3", "DC4", "NAK", "SYN", "ETB", - "CAN", "EM", "SUB", "ESC", "FS", "GS", "RS", "US", - } - // Uncomment this line if you'd debug the layout without creating a special file, then Space (0x20) will also be escaped. - // Don't worry, even if you forget to comment it out and push it to git repo, the CI tests will catch it and fail. - // controlCharNames = append(controlCharNames, "SP") - for i, s := range controlCharNames { - globalVarsPtr.escCtrlCharsMap[i] = template.HTML(`` + string(byte(i)) + ``) - } - globalVarsPtr.escCtrlCharsMap[0x7f] = template.HTML(`` + string(byte(0x7f)) + ``) - globalVarsPtr.escCtrlCharsMap['\t'] = "" - globalVarsPtr.escCtrlCharsMap['\n'] = "" - globalVarsPtr.escCtrlCharsMap['\r'] = "" - - globalVarsPtr.escapeFull = slices.Clone(globalVarsPtr.escCtrlCharsMap) - // exactly the same as Golang's html.EscapeString - globalVarsPtr.escapeFull['&'] = "&" - globalVarsPtr.escapeFull['\''] = "'" - globalVarsPtr.escapeFull['<'] = "<" - globalVarsPtr.escapeFull['>'] = ">" - globalVarsPtr.escapeFull['"'] = """ } return globalVarsPtr } -func escapeByMap(code []byte, escapeMap []template.HTML) template.HTML { - firstEscapePos := -1 - for i, c := range code { - if escapeMap[c] != "" { - firstEscapePos = i - break - } - } - if firstEscapePos == -1 { - return template.HTML(util.UnsafeBytesToString(code)) - } - - buf := make([]byte, firstEscapePos, len(code)*2) - copy(buf[:firstEscapePos], code[:firstEscapePos]) - for i := firstEscapePos; i < len(code); i++ { - c := code[i] - if esc := escapeMap[c]; esc != "" { - buf = append(buf, esc...) - } else { - buf = append(buf, c) - } - } - return template.HTML(util.UnsafeBytesToString(buf)) -} - -func escapeFullString(code string) template.HTML { - return escapeByMap(util.UnsafeStringToBytes(code), globalVars().escapeFull) -} - -func escapeControlChars(code []byte) template.HTML { - return escapeByMap(code, globalVars().escCtrlCharsMap) -} - // UnsafeSplitHighlightedLines splits highlighted code into lines preserving HTML tags // It always includes '\n', '\n' can appear at the end of each line or in the middle of HTML tags // The '\n' is necessary for copying code from web UI to preserve original code lines @@ -137,6 +76,10 @@ func UnsafeSplitHighlightedLines(code template.HTML) (ret [][]byte) { } } +func htmlEscape(code string) template.HTML { + return template.HTML(gohtml.EscapeString(code)) +} + // RenderCodeSlowGuess tries to get a lexer by file name and language first, // if not found, it will try to guess the lexer by code content, which is slow (more than several hundreds of milliseconds). func RenderCodeSlowGuess(fileName, language, code string) (output template.HTML, lexer chroma.Lexer, lexerDisplayName string) { @@ -147,7 +90,7 @@ func RenderCodeSlowGuess(fileName, language, code string) (output template.HTML, } if len(code) > sizeLimit { - return escapeFullString(code), nil, "" + return htmlEscape(code), nil, "" } lexer = detectChromaLexerWithAnalyze(fileName, language, util.UnsafeStringToBytes(code)) // it is also slow @@ -156,15 +99,15 @@ func RenderCodeSlowGuess(fileName, language, code string) (output template.HTML, // RenderCodeByLexer returns a HTML version of code string with chroma syntax highlighting classes func RenderCodeByLexer(lexer chroma.Lexer, code string) template.HTML { - formatter := html.New(html.WithClasses(true), - html.WithLineNumbers(false), - html.PreventSurroundingPre(true), + formatter := chromahtml.New(chromahtml.WithClasses(true), + chromahtml.WithLineNumbers(false), + chromahtml.PreventSurroundingPre(true), ) iterator, err := lexer.Tokenise(nil, code) if err != nil { log.Error("Can't tokenize code: %v", err) - return escapeFullString(code) + return htmlEscape(code) } htmlBuf := &bytes.Buffer{} @@ -172,14 +115,9 @@ func RenderCodeByLexer(lexer chroma.Lexer, code string) template.HTML { err = formatter.Format(htmlBuf, globalVars().githubStyles, iterator) if err != nil { log.Error("Can't format code: %v", err) - return escapeFullString(code) + return htmlEscape(code) } - - // At the moment, we do not escape control chars here (unlike RenderFullFile which escapes control chars). - // The reason is: it is a very rare case that a text file contains control chars. - // This function is usually used by highlight diff and blame, not quite sure whether there will be side effects. - // If there would be new user feedback about this, we can re-consider about various edge cases. - return template.HTML(htmlBuf.String()) + return template.HTML(util.UnsafeBytesToString(htmlBuf.Bytes())) } // RenderFullFile returns a slice of chroma syntax highlighted HTML lines of code and the matched lexer name @@ -191,10 +129,9 @@ func RenderFullFile(fileName, language string, code []byte) ([]template.HTML, st lexerName := formatLexerName(lexer.Config().Name) rendered := RenderCodeByLexer(lexer, util.UnsafeBytesToString(code)) unsafeLines := UnsafeSplitHighlightedLines(rendered) - lines := make([]template.HTML, 0, len(unsafeLines)) - for _, lineBytes := range unsafeLines { - line := escapeControlChars(lineBytes) - lines = append(lines, line) + lines := make([]template.HTML, len(unsafeLines)) + for idx, lineBytes := range unsafeLines { + lines[idx] = template.HTML(util.UnsafeBytesToString(lineBytes)) } return lines, lexerName } @@ -213,7 +150,7 @@ func renderPlainText(code []byte) []template.HTML { content = code[pos : pos+nextPos+1] pos += nextPos + 1 } - lines = append(lines, escapeFullString(util.UnsafeBytesToString(content))) + lines = append(lines, htmlEscape(util.UnsafeBytesToString(content))) } return lines } diff --git a/modules/highlight/highlight_test.go b/modules/highlight/highlight_test.go index cad22ba9bb..211132b255 100644 --- a/modules/highlight/highlight_test.go +++ b/modules/highlight/highlight_test.go @@ -204,14 +204,3 @@ func TestUnsafeSplitHighlightedLines(t *testing.T) { assert.Equal(t, "a\n", string(ret[0])) assert.Equal(t, "b\n", string(ret[1])) } - -func TestEscape(t *testing.T) { - assert.Equal(t, template.HTML("\t\r\n\x00\x1f&'\"<>"), escapeControlChars([]byte("\t\r\n\x00\x1f&'\"<>"))) - assert.Equal(t, template.HTML("\x00\x1f&'"<>\t\r\n"), escapeFullString("\x00\x1f&'\"<>\t\r\n")) - - out, _ := RenderFullFile("a.py", "", []byte("# \x7f<>")) - assert.Equal(t, template.HTML(`# `+string(byte(0x7f))+`<>`), out[0]) - - out = renderPlainText([]byte("# \x7f<>")) - assert.Equal(t, template.HTML(`# `+string(byte(0x7f))+`<>`), out[0]) -} diff --git a/modules/indexer/code/search.go b/modules/indexer/code/search.go index 009d659d76..b5eb5116b0 100644 --- a/modules/indexer/code/search.go +++ b/modules/indexer/code/search.go @@ -74,7 +74,7 @@ func HighlightSearchResultCode(filename, language string, lineNums []int, code s // we should highlight the whole code block first, otherwise it doesn't work well with multiple line highlighting lexer := highlight.DetectChromaLexerByFileName(filename, language) hl := highlight.RenderCodeByLexer(lexer, code) - highlightedLines := strings.Split(string(hl), "\n") + highlightedLines := highlight.UnsafeSplitHighlightedLines(hl) // The lineNums outputted by render might not match the original lineNums, because "highlight" removes the last `\n` lines := make([]*ResultLine, min(len(highlightedLines), len(lineNums))) diff --git a/modules/templates/util_render.go b/modules/templates/util_render.go index 7ff0b204f0..081a13fb9e 100644 --- a/modules/templates/util_render.go +++ b/modules/templates/util_render.go @@ -16,6 +16,7 @@ import ( issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/renderhelper" "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/emoji" "code.gitea.io/gitea/modules/htmlutil" "code.gitea.io/gitea/modules/log" @@ -277,3 +278,24 @@ func (ut *RenderUtils) RenderThemeItem(info *webtheme.ThemeMetaInfo, iconSize in extraIcon := svg.RenderHTML(info.GetExtraIconName(), iconSize) return htmlutil.HTMLFormat(`
%s %s %s
`, info.GetDescription(), icon, info.DisplayName, extraIcon) } + +func (ut *RenderUtils) RenderUnicodeEscapeToggleButton(escapeStatus *charset.EscapeStatus) template.HTML { + if escapeStatus == nil || !escapeStatus.Escaped { + return "" + } + locale := ut.ctx.Value(translation.ContextKey).(translation.Locale) + var title template.HTML + if escapeStatus.HasAmbiguous { + title += locale.Tr("repo.ambiguous_runes_line") + } else if escapeStatus.HasInvisible { + title += locale.Tr("repo.invisible_runes_line") + } + return htmlutil.HTMLFormat(``, title) +} + +func (ut *RenderUtils) RenderUnicodeEscapeToggleTd(combined, escapeStatus *charset.EscapeStatus) template.HTML { + if combined == nil || !combined.Escaped { + return "" + } + return `` + ut.RenderUnicodeEscapeToggleButton(escapeStatus) + `` +} diff --git a/options/locale/locale_en-US.json b/options/locale/locale_en-US.json index acb5a25087..a361781768 100644 --- a/options/locale/locale_en-US.json +++ b/options/locale/locale_en-US.json @@ -1216,7 +1216,7 @@ "repo.ambiguous_runes_description": "This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.", "repo.invisible_runes_line": "This line has invisible unicode characters", "repo.ambiguous_runes_line": "This line has ambiguous unicode characters", - "repo.ambiguous_character": "%[1]c [U+%04[1]X] can be confused with %[2]c [U+%04[2]X]", + "repo.ambiguous_character": "%[1]s can be confused with %[2]s", "repo.escape_control_characters": "Escape", "repo.unescape_control_characters": "Unescape", "repo.file_copy_permalink": "Copy Permalink", diff --git a/routers/web/devtest/devtest.go b/routers/web/devtest/devtest.go index 8283d3ad9d..5cfe08c7fe 100644 --- a/routers/web/devtest/devtest.go +++ b/routers/web/devtest/devtest.go @@ -17,7 +17,9 @@ import ( "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/badge" + "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/indexer/code" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/context" @@ -190,9 +192,33 @@ func prepareMockData(ctx *context.Context) { prepareMockDataBadgeActionsSvg(ctx) case "/devtest/relative-time": prepareMockDataRelativeTime(ctx) + case "/devtest/unicode-escape": + prepareMockDataUnicodeEscape(ctx) } } +func prepareMockDataUnicodeEscape(ctx *context.Context) { + content := "// demo code\n" + content += "if accessLevel != \"user\u202E \u2066// Check if admin (invisible char)\u2069 \u2066\" { }\n" + content += "if O๐พ { } // ambiguous char\n" + content += "if O๐พ && accessLevel != \"user\u202E \u2066// ambiguous char + invisible char\u2069 \u2066\" { }\n" + content += "str := `\xef` // broken char\n" + content += "str := `\x00 \x19 \x7f` // control char\n" + + lineNums := []int{1, 2, 3, 4, 5, 6, 7, 8, 9} + + highlightLines := code.HighlightSearchResultCode("demo.go", "", lineNums, content) + escapeStatus := &charset.EscapeStatus{} + lineEscapeStatus := make([]*charset.EscapeStatus, len(highlightLines)) + for i, hl := range highlightLines { + lineEscapeStatus[i], hl.FormattedContent = charset.EscapeControlHTML(hl.FormattedContent, ctx.Locale) + escapeStatus = escapeStatus.Or(lineEscapeStatus[i]) + } + ctx.Data["HighlightLines"] = highlightLines + ctx.Data["EscapeStatus"] = escapeStatus + ctx.Data["LineEscapeStatus"] = lineEscapeStatus +} + func TmplCommon(ctx *context.Context) { prepareMockData(ctx) if ctx.Req.Method == http.MethodPost { diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 7136b87058..46661f0df0 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -159,7 +159,7 @@ func markupRenderToHTML(ctx *context.Context, renderCtx *markup.RenderContext, r go func() { sb := &strings.Builder{} if markup.RendererNeedPostProcess(renderer) { - escaped, _ = charset.EscapeControlReader(markupRd, sb, ctx.Locale, charset.RuneNBSP) // We allow NBSP here this is rendered + escaped, _ = charset.EscapeControlReader(markupRd, sb, ctx.Locale, charset.EscapeOptionsForView()) } else { escaped = &charset.EscapeStatus{} _, _ = io.Copy(sb, markupRd) diff --git a/routers/web/repo/wiki.go b/routers/web/repo/wiki.go index e5b07633a2..1826ca54e1 100644 --- a/routers/web/repo/wiki.go +++ b/routers/web/repo/wiki.go @@ -258,8 +258,7 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) { defer markupWr.Close() done := make(chan struct{}) go func() { - // We allow NBSP here this is rendered - escaped, _ = charset.EscapeControlReader(markupRd, buf, ctx.Locale, charset.RuneNBSP) + escaped, _ = charset.EscapeControlReader(markupRd, buf, ctx.Locale, charset.EscapeOptionsForView()) output = template.HTML(buf.String()) buf.Reset() close(done) diff --git a/services/markup/renderhelper_codepreview.go b/services/markup/renderhelper_codepreview.go index 87c3bd1d48..fd2db0d10f 100644 --- a/services/markup/renderhelper_codepreview.go +++ b/services/markup/renderhelper_codepreview.go @@ -101,7 +101,7 @@ func renderRepoFileCodePreview(ctx context.Context, opts markup.RenderCodePrevie escapeStatus := &charset.EscapeStatus{} lineEscapeStatus := make([]*charset.EscapeStatus, len(highlightLines)) for i, hl := range highlightLines { - lineEscapeStatus[i], hl.FormattedContent = charset.EscapeControlHTML(hl.FormattedContent, webCtx.Base.Locale, charset.RuneNBSP) + lineEscapeStatus[i], hl.FormattedContent = charset.EscapeControlHTML(hl.FormattedContent, webCtx.Base.Locale, charset.EscapeOptionsForView()) escapeStatus = escapeStatus.Or(lineEscapeStatus[i]) } diff --git a/services/markup/renderhelper_codepreview_test.go b/services/markup/renderhelper_codepreview_test.go index c84845e7ea..94ed5b418a 100644 --- a/services/markup/renderhelper_codepreview_test.go +++ b/services/markup/renderhelper_codepreview_test.go @@ -37,10 +37,12 @@ func TestRenderHelperCodePreview(t *testing.T) { - + - +
# repo1
# repo1 +
+
@@ -64,7 +66,8 @@ func TestRenderHelperCodePreview(t *testing.T) { - +
# repo1
# repo1 +
diff --git a/templates/base/markup_codepreview.tmpl b/templates/base/markup_codepreview.tmpl index e16848581d..4c04b5092c 100644 --- a/templates/base/markup_codepreview.tmpl +++ b/templates/base/markup_codepreview.tmpl @@ -13,10 +13,7 @@ {{- range $idx, $line := .HighlightLines -}} - {{- if $.EscapeStatus.Escaped -}} - {{- $lineEscapeStatus := index $.LineEscapeStatus $idx -}} - {{if $lineEscapeStatus.Escaped}}{{end}} - {{- end}} + {{- ctx.RenderUtils.RenderUnicodeEscapeToggleTd $.EscapeStatus (index $.LineEscapeStatus $idx)}}
{{$line.FormattedContent}}
{{/* only div works, span generates incorrect HTML structure */}} {{- end -}} diff --git a/templates/devtest/unicode-escape.tmpl b/templates/devtest/unicode-escape.tmpl new file mode 100644 index 0000000000..a61813f5c8 --- /dev/null +++ b/templates/devtest/unicode-escape.tmpl @@ -0,0 +1,17 @@ +{{template "devtest/devtest-header"}} +
+
+ + + {{range $idx, $line := .HighlightLines}} + + + {{ctx.RenderUtils.RenderUnicodeEscapeToggleTd $.EscapeStatus (index $.LineEscapeStatus $idx)}} + + + {{end}} + +
{{$line.FormattedContent}}
+
+
+{{template "devtest/devtest-footer"}} diff --git a/templates/repo/blame.tmpl b/templates/repo/blame.tmpl index bc91adb64f..51052d9359 100644 --- a/templates/repo/blame.tmpl +++ b/templates/repo/blame.tmpl @@ -66,13 +66,9 @@ - {{if $.EscapeStatus.Escaped}} - - {{if $row.EscapeStatus.Escaped}} - - {{end}} - - {{end}} + + {{ctx.RenderUtils.RenderUnicodeEscapeToggleTd $.EscapeStatus $row.EscapeStatus}} + {{$row.Code}} diff --git a/templates/repo/diff/blob_excerpt.tmpl b/templates/repo/diff/blob_excerpt.tmpl index 916d589839..a2dcf0d091 100644 --- a/templates/repo/diff/blob_excerpt.tmpl +++ b/templates/repo/diff/blob_excerpt.tmpl @@ -12,7 +12,7 @@ {{else}} {{$inlineDiff := $.section.GetComputedInlineDiffFor $line ctx.Locale}} - {{if and $line.LeftIdx $inlineDiff.EscapeStatus.Escaped}}{{end}} + {{if $line.LeftIdx}}{{ctx.RenderUtils.RenderUnicodeEscapeToggleButton $inlineDiff.EscapeStatus}}{{end}} {{if $line.LeftIdx}}{{end}} {{/* ATTENTION: BLOB-EXCERPT-COMMENT-RIGHT: here it intentionally use "right" side to comment, because the backend code depends on the assumption that the comment only happens on right side*/}} @@ -28,7 +28,7 @@ {{- end -}} - {{if and $line.RightIdx $inlineDiff.EscapeStatus.Escaped}}{{end}} + {{if and $line.RightIdx}}{{ctx.RenderUtils.RenderUnicodeEscapeToggleButton $inlineDiff.EscapeStatus}}{{end}} {{if $line.RightIdx}}{{end}} {{- if and $canCreateComment $line.RightIdx -}} @@ -69,7 +69,7 @@ {{end}} {{$inlineDiff := $.section.GetComputedInlineDiffFor $line ctx.Locale}} - {{if $inlineDiff.EscapeStatus.Escaped}}{{end}} + {{ctx.RenderUtils.RenderUnicodeEscapeToggleButton $inlineDiff.EscapeStatus}} {{- if and $canCreateComment -}} @@ -77,7 +77,7 @@ {{- svg "octicon-plus" -}} {{- end -}} - {{$inlineDiff.Content}} + {{$inlineDiff.Content}} {{if $line.Comments}} diff --git a/templates/repo/diff/escape_title.tmpl b/templates/repo/diff/escape_title.tmpl deleted file mode 100644 index 9787ae1d42..0000000000 --- a/templates/repo/diff/escape_title.tmpl +++ /dev/null @@ -1,2 +0,0 @@ -{{if .diff.EscapeStatus.HasInvisible}}{{ctx.Locale.Tr "repo.invisible_runes_line"}} {{end -}} -{{- if .diff.EscapeStatus.HasAmbiguous}}{{ctx.Locale.Tr "repo.ambiguous_runes_line"}}{{end}} diff --git a/templates/repo/diff/section_code.tmpl b/templates/repo/diff/section_code.tmpl index 3e8303eda6..3b9ed0c88c 100644 --- a/templates/repo/diff/section_code.tmpl +++ b/templates/repo/diff/section_code.tmpl @@ -1 +1 @@ -{{.diff.Content}} +{{.diff.Content}} diff --git a/templates/repo/diff/section_split.tmpl b/templates/repo/diff/section_split.tmpl index ab23b1b934..c13b205518 100644 --- a/templates/repo/diff/section_split.tmpl +++ b/templates/repo/diff/section_split.tmpl @@ -18,14 +18,14 @@ {{if eq .GetType 4}} {{$inlineDiff := $section.GetComputedInlineDiffFor $line ctx.Locale}} {{$line.RenderBlobExcerptButtons $file.NameHash $diffBlobExcerptData}} - {{if $inlineDiff.EscapeStatus.Escaped}}{{end}} + {{ctx.RenderUtils.RenderUnicodeEscapeToggleButton $inlineDiff.EscapeStatus}} {{template "repo/diff/section_code" dict "diff" $inlineDiff}} {{else if and (eq .GetType 3) $hasmatch}}{{/* DEL */}} {{$match := index $section.Lines $line.Match}} {{- $leftDiff := ""}}{{if $line.LeftIdx}}{{$leftDiff = $section.GetComputedInlineDiffFor $line ctx.Locale}}{{end}} {{- $rightDiff := ""}}{{if $match.RightIdx}}{{$rightDiff = $section.GetComputedInlineDiffFor $match ctx.Locale}}{{end}} - {{if $line.LeftIdx}}{{if $leftDiff.EscapeStatus.Escaped}}{{end}}{{end}} + {{if $line.LeftIdx}}{{ctx.RenderUtils.RenderUnicodeEscapeToggleButton $leftDiff.EscapeStatus}}{{end}} {{- if and $.root.SignedUserID $.root.PageIsPullFiles -}} @@ -40,7 +40,7 @@ {{- end -}} - {{if $match.RightIdx}}{{if $rightDiff.EscapeStatus.Escaped}}{{end}}{{end}} + {{if $match.RightIdx}}{{ctx.RenderUtils.RenderUnicodeEscapeToggleButton $rightDiff.EscapeStatus}}{{end}} {{if $match.RightIdx}}{{end}} {{- if and $.root.SignedUserID $.root.PageIsPullFiles -}} @@ -57,7 +57,7 @@ {{else}} {{$inlineDiff := $section.GetComputedInlineDiffFor $line ctx.Locale}} - {{if $line.LeftIdx}}{{if $inlineDiff.EscapeStatus.Escaped}}{{end}}{{end}} + {{if $line.LeftIdx}}{{ctx.RenderUtils.RenderUnicodeEscapeToggleButton $inlineDiff.EscapeStatus}}{{end}} {{if $line.LeftIdx}}{{end}} {{- if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 2)) -}} @@ -72,7 +72,7 @@ {{- end -}} - {{if $line.RightIdx}}{{if $inlineDiff.EscapeStatus.Escaped}}{{end}}{{end}} + {{if $line.RightIdx}}{{ctx.RenderUtils.RenderUnicodeEscapeToggleButton $inlineDiff.EscapeStatus}}{{end}} {{if $line.RightIdx}}{{end}} {{- if and $.root.SignedUserID $.root.PageIsPullFiles (not (eq .GetType 3)) -}} diff --git a/templates/repo/diff/section_unified.tmpl b/templates/repo/diff/section_unified.tmpl index 6776198b75..4dee648cdd 100644 --- a/templates/repo/diff/section_unified.tmpl +++ b/templates/repo/diff/section_unified.tmpl @@ -24,9 +24,7 @@ {{end}} {{$inlineDiff := $section.GetComputedInlineDiffFor $line ctx.Locale -}} - {{- if $inlineDiff.EscapeStatus.Escaped -}} - - {{- end -}} + {{ctx.RenderUtils.RenderUnicodeEscapeToggleButton $inlineDiff.EscapeStatus}} {{if eq .GetType 4}} diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index d91235e0ea..72b5a83b65 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -109,9 +109,7 @@ {{$line := Eval $idx "+" 1}} - {{if $.EscapeStatus.Escaped}} - {{if (index $.LineEscapeStatus $idx).Escaped}}{{end}} - {{end}} + {{ctx.RenderUtils.RenderUnicodeEscapeToggleTd $.EscapeStatus (index $.LineEscapeStatus $idx)}} {{$code}} {{end}} diff --git a/web_src/css/base.css b/web_src/css/base.css index bb16b9fe21..a8d9dea2a2 100644 --- a/web_src/css/base.css +++ b/web_src/css/base.css @@ -17,6 +17,7 @@ /* images */ --checkbox-mask-checked: url('data:image/svg+xml;utf8,'); --checkbox-mask-indeterminate: url('data:image/svg+xml;utf8,'); + --octicon-alert-fill: url('data:image/svg+xml;utf8,'); --octicon-chevron-right: url('data:image/svg+xml;utf8,'); --octicon-x: url('data:image/svg+xml;utf8,'); --select-arrows: url('data:image/svg+xml;utf8,'); @@ -686,6 +687,7 @@ overflow-menu .ui.label { } .lines-num, +.lines-escape, .lines-code { font-size: 12px; font-family: var(--fonts-monospace); diff --git a/web_src/css/modules/charescape.css b/web_src/css/modules/charescape.css index 0c9cbb55b5..35439ebc27 100644 --- a/web_src/css/modules/charescape.css +++ b/web_src/css/modules/charescape.css @@ -1,24 +1,31 @@ /* Show the escaped and hide the real char: - {real-char} + {real-char} Only show the real-char: {real-char} */ -.broken-code-point:not([data-escaped]), -.broken-code-point[data-escaped]::before { +.broken-code-point:not([data-escaped]) { border-radius: 4px; padding: 0 2px; - color: var(--color-body); - background: var(--color-text-light-1); + border: 1px var(--color-yellow) solid; +} + +.broken-code-point[data-escaped] { + position: relative; } .broken-code-point[data-escaped]::before { visibility: visible; content: attr(data-escaped); + border-radius: 2px; + padding: 0 1px; + border: 1px var(--color-yellow) solid; } + .broken-code-point[data-escaped] .char { - /* make it copyable by selecting the text (AI suggestion, no other solution) */ + /* keep the original character selectable/copyable while showing the escaped label via ::before */ position: absolute; + left: 0; opacity: 0; pointer-events: none; } @@ -26,11 +33,11 @@ Only show the real-char: /* Show the escaped and hide the real-char: - {real-char} + {real-char} Hide the escaped and show the real-char: - {real-char} + {real-char} */ .unicode-escaped .escaped-code-point[data-escaped]::before { diff --git a/web_src/css/review.css b/web_src/css/review.css index 9e320346d8..f192b0322b 100644 --- a/web_src/css/review.css +++ b/web_src/css/review.css @@ -15,11 +15,20 @@ transform: scale(1.1); } +.lines-escape .toggle-escape-button { + margin: -1px 2px 0; +} + .lines-escape .toggle-escape-button::before { - visibility: visible; - content: "โš ๏ธ"; - font-family: var(--fonts-emoji); - color: var(--color-red); + content: ""; + display: inline-flex; + width: 14px; + height: 14px; + background-color: var(--color-yellow); /* TODO: maybe it needs a new kind of color, there is no suitable "warning" color in the current palette */ + mask-image: var(--octicon-alert-fill); + -webkit-mask-image: var(--octicon-alert-fill); + mask-size: contain; + -webkit-mask-size: contain; } .repository .diff-file-box .code-diff td.lines-escape {