1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
|
//===- unittests/Support/UnicodeTest.cpp - Unicode.h tests ----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "llvm/Support/Unicode.h"
#include "llvm/Support/ConvertUTF.h"
#include "gtest/gtest.h"
namespace llvm {
namespace sys {
namespace unicode {
namespace {
TEST(Unicode, columnWidthUTF8) {
EXPECT_EQ(0, columnWidthUTF8(""));
EXPECT_EQ(1, columnWidthUTF8(" "));
EXPECT_EQ(1, columnWidthUTF8("a"));
EXPECT_EQ(1, columnWidthUTF8("~"));
EXPECT_EQ(6, columnWidthUTF8("abcdef"));
EXPECT_EQ(-1, columnWidthUTF8("\x01"));
EXPECT_EQ(-1, columnWidthUTF8("\t"));
EXPECT_EQ(-1, columnWidthUTF8("aaaaaaaaaa\x01"));
EXPECT_EQ(-1, columnWidthUTF8("\342\200\213")); // 200B ZERO WIDTH SPACE
// 00AD SOFT HYPHEN is displayed on most terminals as a space or a dash. Some
// text editors display it only when a line is broken at it, some use it as a
// line-break hint, but don't display. We choose terminal-oriented
// interpretation.
EXPECT_EQ(1, columnWidthUTF8("\302\255"));
EXPECT_EQ(0, columnWidthUTF8("\314\200")); // 0300 COMBINING GRAVE ACCENT
EXPECT_EQ(1, columnWidthUTF8("\340\270\201")); // 0E01 THAI CHARACTER KO KAI
EXPECT_EQ(2, columnWidthUTF8("\344\270\200")); // CJK UNIFIED IDEOGRAPH-4E00
EXPECT_EQ(4, columnWidthUTF8("\344\270\200\344\270\200"));
EXPECT_EQ(3, columnWidthUTF8("q\344\270\200"));
EXPECT_EQ(3, columnWidthUTF8("\314\200\340\270\201\344\270\200"));
// Invalid UTF-8 strings, columnWidthUTF8 should error out.
EXPECT_EQ(-2, columnWidthUTF8("\344"));
EXPECT_EQ(-2, columnWidthUTF8("\344\270"));
EXPECT_EQ(-2, columnWidthUTF8("\344\270\033"));
EXPECT_EQ(-2, columnWidthUTF8("\344\270\300"));
EXPECT_EQ(-2, columnWidthUTF8("\377\366\355"));
EXPECT_EQ(-2, columnWidthUTF8("qwer\344"));
EXPECT_EQ(-2, columnWidthUTF8("qwer\344\270"));
EXPECT_EQ(-2, columnWidthUTF8("qwer\344\270\033"));
EXPECT_EQ(-2, columnWidthUTF8("qwer\344\270\300"));
EXPECT_EQ(-2, columnWidthUTF8("qwer\377\366\355"));
// UTF-8 sequences longer than 4 bytes correspond to unallocated Unicode
// characters.
EXPECT_EQ(-2, columnWidthUTF8("\370\200\200\200\200")); // U+200000
EXPECT_EQ(-2, columnWidthUTF8("\374\200\200\200\200\200")); // U+4000000
}
TEST(Unicode, isPrintable) {
EXPECT_FALSE(isPrintable(0)); // <control-0000>-<control-001F>
EXPECT_FALSE(isPrintable(0x01));
EXPECT_FALSE(isPrintable(0x1F));
EXPECT_TRUE(isPrintable(' '));
EXPECT_TRUE(isPrintable('A'));
EXPECT_TRUE(isPrintable('~'));
EXPECT_FALSE(isPrintable(0x7F)); // <control-007F>..<control-009F>
EXPECT_FALSE(isPrintable(0x90));
EXPECT_FALSE(isPrintable(0x9F));
EXPECT_TRUE(isPrintable(0xAC));
EXPECT_TRUE(isPrintable(0xAD)); // SOFT HYPHEN is displayed on most terminals
// as either a space or a dash.
EXPECT_TRUE(isPrintable(0xAE));
EXPECT_TRUE(isPrintable(0x0377)); // GREEK SMALL LETTER PAMPHYLIAN DIGAMMA
EXPECT_FALSE(isPrintable(0x0378)); // <reserved-0378>..<reserved-0379>
EXPECT_FALSE(isPrintable(0x0600)); // ARABIC NUMBER SIGN
EXPECT_FALSE(isPrintable(0x1FFFF)); // <reserved-1F774>..<noncharacter-1FFFF>
EXPECT_TRUE(isPrintable(0x20000)); // CJK UNIFIED IDEOGRAPH-20000
EXPECT_FALSE(isPrintable(0x10FFFF)); // noncharacter
// test the validity of a fast path in columnWidthUTF8
for (unsigned char c = 0; c < 128; ++c) {
const UTF8 buf8[2] = {c, 0};
const UTF8 *Target8 = &buf8[0];
UTF32 buf32[1];
UTF32 *Target32 = &buf32[0];
auto status = ConvertUTF8toUTF32(&Target8, Target8 + 1, &Target32,
Target32 + 1, strictConversion);
EXPECT_TRUE(status == conversionOK);
EXPECT_TRUE((columnWidthUTF8(reinterpret_cast<const char *>(buf8)) == 1) ==
(bool)isPrintable(buf32[0]));
}
}
} // namespace
} // namespace unicode
} // namespace sys
} // namespace llvm
|
