index.html
171 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xml:lang="en" lang="en" xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>Character Model for the World Wide Web 1.0: Fundamentals </title><style type="text/css">
code { font-family: monospace; }
div.constraint,
div.issue,
div.note,
div.example,
div.notice { margin-left: 2em; }
.example-head, .note-head { font-weight: bold }
li p { margin-top: 0.3em;
margin-bottom: 0.3em; }
.rfc2119, .uname { text-transform: lowercase; font-variant: small-caps; }
.new-term { font-weight: bold }
.quote { font-style: italic }
.figure { margin-bottom: 2em; }
.caption {
text-align: center;
margin: 0.5em 2em;
font-style: italic;
}
.editor-note { font-style: italic; color: red; }
.req { background: #ffffcc; }
.reqId, .reqId a {
color: #005A9C;
background: white;
font-weight: bold;
font-style: italic;
text-decoration: none;
}
img { border: 0; }
@media print {
.req { background: #ffcc99 }
}
div.exampleInner pre { margin-left: 1em;
margin-top: 0em; margin-bottom: 0em}
div.exampleOuter {border: 4px double gray;
margin: 0em; padding: 0em}
div.exampleInner { background-color: #d5dee3;
border-top-width: 4px;
border-top-style: double;
border-top-color: #d3d3d3;
border-bottom-width: 4px;
border-bottom-style: double;
border-bottom-color: #d3d3d3;
padding: 4px; margin: 0em }
div.exampleWrapper { margin: 4px }
div.exampleHeader { font-weight: bold;
margin: 4px}
</style><link rel="stylesheet" type="text/css" href="http://www.w3.org/StyleSheets/TR/W3C-REC" /></head><body><div class="head"><p><a href="http://www.w3.org/"><img src="http://www.w3.org/Icons/w3c_home" alt="W3C" height="48" width="72" /></a></p> <h1><a name="title" id="title" />Character Model for the World Wide Web 1.0: Fundamentals </h1> <h2><a name="w3c-doctype" id="w3c-doctype" />W3C Recommendation 15 February 2005</h2><dl><dt>This version:</dt><dd>
<a href="http://www.w3.org/TR/2005/REC-charmod-20050215/">http://www.w3.org/TR/2005/REC-charmod-20050215/</a></dd><dt>Latest version:</dt><dd>
<a href="http://www.w3.org/TR/charmod/">http://www.w3.org/TR/charmod/</a>
</dd><dt>Previous version:</dt><dd><a href="http://www.w3.org/TR/2004/PR-charmod-20041122/">http://www.w3.org/TR/2004/PR-charmod-20041122/</a></dd><dt>Editors:</dt><dd>Martin J. Dürst, W3C <a href="mailto:duerst@w3.org"><duerst@w3.org></a></dd><dd>François Yergeau (Invited Expert)</dd><dd>Richard Ishida, W3C <a href="mailto:ishida@w3.org"><ishida@w3.org></a></dd><dd>Misha Wolf (until Dec 2002), Reuters
Ltd. <a href="mailto:misha.wolf@reuters.com"><misha.wolf@reuters.com></a></dd><dd>Tex Texin (Invited Expert), XenCraft <a href="mailto:tex@XenCraft.com"><tex@XenCraft.com></a></dd></dl><p>Please refer to the <a href="http://www.w3.org/2005/02/charmod-fundamentals-errata.html"><strong>errata</strong></a> for this document, which may include some normative corrections.</p><p>See also <a href="http://www.w3.org/2003/03/Translations/byTechnology?technology=charmod"><strong>translations</strong></a>.</p><p class="copyright"><a href="http://www.w3.org/Consortium/Legal/ipr-notice#Copyright">Copyright</a> © 2005 <a href="http://www.w3.org/"><acronym title="World Wide Web Consortium">W3C</acronym></a><sup>®</sup> (<a href="http://www.csail.mit.edu/"><acronym title="Massachusetts Institute of Technology">MIT</acronym></a>, <a href="http://www.ercim.org/"><acronym title="European Research Consortium for Informatics and Mathematics">ERCIM</acronym></a>, <a href="http://www.keio.ac.jp/">Keio</a>), All Rights Reserved. W3C <a href="http://www.w3.org/Consortium/Legal/ipr-notice#Legal_Disclaimer">liability</a>, <a href="http://www.w3.org/Consortium/Legal/ipr-notice#W3C_Trademarks">trademark</a> and <a href="http://www.w3.org/Consortium/Legal/copyright-documents">document use</a> rules apply.</p></div><hr /><div> <h2><a name="abstract" id="abstract" />Abstract</h2><p>This Architectural Specification provides authors of specifications,
software developers, and content developers with a common reference for
interoperable text manipulation on the World Wide Web, building on the Universal Character Set,
defined jointly by the Unicode Standard and
ISO/IEC 10646. Topics addressed include
use of the terms '<span class="qterm">character</span>', '<span class="qterm">encoding</span>' and '<span class="qterm">string</span>', a reference processing model, choice and identification of character encodings, character escaping, and string indexing.</p><p>For normalization and string identity matching, see the companion document <cite>Character Model for the World Wide Web 1.0: Normalization</cite> <a href="#charnorm">[CharNorm]</a>. For resource identifiers, see the companion document <cite>Character Model for the World Wide Web 1.0: Resource Identifiers</cite> <a href="#charmod3">[CharIRI]</a>.</p></div><div> <h2><a name="status" id="status" />Status of this Document</h2><p><em>This section describes the status of this document at the time
of its publication. Other documents may supersede this document. A list of current W3C publications and the latest revision of this technical report can be found in the <a href="http://www.w3.org/TR/">W3C technical reports index</a> at http://www.w3.org/TR/.</em></p><p>This document contains the <cite>Character Model for the World Wide Web 1.0: Fundamentals</cite> specification, and is a <a href="http://www.w3.org/2003/06/Process-20030618/tr.html#RecsW3C"> W3C Recommendation</a>. It has been reviewed by W3C Members and other interested parties and has been endorsed by the Director. It is a stable document and may be used as reference material or cited as a normative reference from another document. W3C's role in making the Recommendation is to draw attention to the specification and to promote its widespread deployment. This enhances the functionality and interoperability of the Web.</p><p>This document was developed as part of the
<a href="http://www.w3.org/International/Activity">W3C
Internationalization Activity</a> by the <a href="http://www.w3.org/International/core/">W3C Internationalization Core Working Group</a>,
with the help of the Internationalization Interest Group. </p><p>If you have comments on this document, send them to <a href="mailto:www-i18n-comments@w3.org">www-i18n-comments@w3.org</a> (<a href="http://lists.w3.org/Archives/Public/www-i18n-comments/">public archive</a>). Last Call dispositions are available in a <a href="http://www.w3.org/2004/02/charmod1-lastcall/">public version</a> and a <a href="http://www.w3.org/International/Group/2004/charmod1-lc/">Members-only version</a>. There is also an <a href="/2004/11/charmod-implementation/">implementation report</a>. Changes to this document since the Proposed Recommendation version are detailed in <a href="#sec-Changes"><b>E Changes since the Proposed Recommendation</b></a>.</p><p>This document was produced under the <a href="http://www.w3.org/TR/2002/NOTE-patent-practice-20020124">24 January 2002 CPP</a> as amended by the <a href="http://www.w3.org/2004/02/05-pp-transition">W3C Patent Policy Transition Procedure</a>. The Working Group maintains a <a href="http://www.w3.org/2004/01/pp-impl/32113/status">public list of patent disclosures</a> relevant to this document; that page also includes instructions for disclosing a patent. An individual who has actual knowledge of a patent which the individual believes contains Essential Claim(s) with respect to this specification should disclose the information in accordance with <a href="http://www.w3.org/Consortium/Patent-Policy-20040205/#sec-Disclosure">section 6 of the W3C Patent Policy</a>.</p></div><div class="toc"> <h2><a name="contents" id="contents" />Table of Contents</h2><p class="toc">1 <a href="#sec-Intro">Introduction</a><br /> 1.1 <a href="#sec-GoalsScope">Goals and Scope</a><br /> 1.2 <a href="#sec-Background">Background</a><br /> 1.3 <a href="#sec-Notation">Terminology and Notation</a><br /> 2 <a href="#sec-Conformance">Conformance</a><br /> 3 <a href="#sec-Perceptions">Perceptions of Characters</a><br /> 3.1 <a href="#sec-PerceptionsIntro">Introduction</a><br /> 3.2 <a href="#sec-WritingSystem">Units of aural rendering</a><br /> 3.3 <a href="#sec-VisualRenderingUnits">Units of visual
rendering</a><br /> 3.3.1 <a href="#sec-LogicalOrder">Visual Rendering and Logical Order</a><br /> 3.4 <a href="#sec-InputUnits">Units of input</a><br /> 3.5 <a href="#sec-CollationUnits">Units of collation</a><br /> 3.6 <a href="#sec-Storage">Units of storage</a><br /> 3.7 <a href="#sec-PerceptionsOutro">Summary</a><br /> 4 <a href="#sec-Characters">Digital Encoding of Characters</a><br /> 4.1 <a href="#sec-Digital">Character Encoding</a><br /> 4.2 <a href="#sec-Transcoding">Transcoding</a><br /> 4.3 <a href="#sec-RefProcModel">Reference Processing Model</a><br /> 4.4 <a href="#sec-Encodings">Choice and Identification of Character
Encodings</a><br /> 4.4.1 <a href="#sec-UniqueEncoding">Mandating a unique character
encoding</a><br /> 4.4.2 <a href="#sec-EncodingIdent">Character encoding
identification</a><br /> 4.5 <a href="#sec-PrivateUse">Private use code points</a><br /> 4.6 <a href="#sec-Escaping">Character Escaping</a><br /> 5 <a href="#sec-Compatibility">Compatibility and Formatting
Characters</a><br /> 6 <a href="#sec-Indexing">Strings</a><br /> 6.1 <a href="#sec-Strings">String concepts</a><br /> 6.2 <a href="#sec-stringIndexing">String indexing</a><br /> 7 <a href="#sec-RefUnicode">Referencing the Unicode Standard and
ISO/IEC 10646</a><br /> </p> <h3><a name="appendices" id="appendices" />Appendices</h3><p class="toc">A <a href="#sec-References">References</a><br /> A.1 <a href="#sec-NormativeReferences">Normative
References</a><br /> A.2 <a href="#sec-OtherReferences">Other References</a><br /> B <a href="#sec-CharExamples">Examples of Characters, Keystrokes
and Glyphs</a> (Non-Normative)<br /> C <a href="#sec-ExampleText">Example text</a> (Non-Normative)<br /> D <a href="#sec-Checklist">List of conformance criteria</a> (Non-Normative)<br /> E <a href="#sec-Changes">Changes since the Proposed Recommendation</a> (Non-Normative)<br /> F <a href="#sec-Acknowledgements">Acknowledgements</a> (Non-Normative)<br /> </p></div><hr /><div class="body"><div class="div1"> <h2><a name="sec-Intro" id="sec-Intro" />1 Introduction</h2><div class="div2"> <h3><a name="sec-GoalsScope" id="sec-GoalsScope" />1.1 Goals and Scope</h3><p>The goal of the Character Model for the World Wide
Web is to facilitate use of the Web by all people,
regardless of their language, script, writing system, and cultural conventions,
in accordance with the <a href=" http://www.w3.org/Consortium/mission.html"><cite>W3C
goal of universal access</cite></a>. One basic prerequisite to achieve this goal
is to be able to transmit and process the characters used around the world in a
well-defined and well-understood way.</p><p>The main target audience of this specification
is W3C specification developers. This specification
and parts of it can be referenced from other W3C specifications. It defines conformance criteria for W3C specifications
as well as other specifications.</p><p>Other audiences of this specification
include software developers, content
developers, and authors of specifications outside the W3C. Software developers
and content developers implement and use W3C specifications. This
specification
defines some conformance criteria for implementations (software) and content
that implement and use W3C specifications. It also helps software developers and
content developers to understand the character-related provisions in W3C
specifications.</p><p>The character model described in this specification
provides authors of
specifications, software developers, and content developers with a common
reference for consistent, interoperable text manipulation on the World Wide Web.
Working together, these three groups can build a more international Web.</p><p>Topics addressed in this part of the Character Model for the World Wide Web
include use of the terms '<span class="qterm">character</span>', '<span class="qterm">encoding</span>' and
'<span class="qterm">string</span>', a reference processing model, choice and identification
of character encodings, character escaping, and string indexing.</p><p>Other parts of the Character Model address
normalization and string identity matching (<a href="#charnorm">[CharNorm]</a>) and Internationalized Resource Identifiers (IRI) conventions
(<a href="#charmod3">[CharIRI]</a>).</p><p>Topics as yet not addressed or barely touched include fuzzy
matching, and language tagging. Some of these topics may be addressed in a
future version of this specification.</p><p>At the core of the model is the Universal Character Set (UCS), defined
jointly by the Unicode Standard <a href="#unicode">[Unicode]</a> and ISO/IEC 10646
<a href="#iso10646">[ISO/IEC 10646]</a>. In this document, <span class="new-term"> Unicode</span> is used as a
synonym for the Universal Character Set. The model will allow Web documents
authored in the world's scripts (and on different platforms) to be exchanged,
read, and searched by Web users around the world.</p></div><div class="div2"> <h3><a name="sec-Background" id="sec-Background" />1.2 Background</h3><p>This section provides some historical background on the topics
addressed in this specification.</p><p>Starting with <cite>Internationalization of the Hypertext Markup Language
</cite> <a href="#rfc2070">[RFC 2070]</a>, the Web community has recognized the need
for a character model for the World Wide Web. The first step towards building
this model was the adoption of Unicode as the document character set for HTML.</p><p>The choice of Unicode was motivated by the fact that Unicode:
</p><ul><li><p>is the only universal character repertoire available,</p></li><li><p>provides a way of referencing characters independent of the
encoding of the text,</p></li><li><p>is being updated/completed carefully,</p></li><li><p>is widely accepted and implemented by industry.</p></li></ul><p>
</p><p>W3C adopted Unicode as the document character set for HTML in <a href="#html40">[HTML 4.0]</a>. The same approach was later used for specifications such as XML 1.0
<a href="#xml10">[XML 1.0]</a> and CSS2 <a href="#css2">[CSS2]</a>. W3C specifications and
applications now use Unicode as the common reference character set.</p><p>When data transfer on the Web remained mostly unidirectional (from server to
browser), and where the main purpose was to render documents, the use of Unicode
without specifying additional details was sufficient. However, the Web has
grown:
</p><ul><li><p>Data transfers among servers, proxies, and clients, in all
directions, have increased.</p></li><li><p>Characters outside the US-ASCII
<a href="#iso646">[ISO/IEC 646]</a><a href="#MIME-charset">[MIME-charset]</a> repertoire
are being used in more and more places.</p></li><li><p>Data transfers between different protocol/format elements (such as
element/attribute names, URI components, and textual content) have
increased.</p></li><li><p>More and more APIs are defined, not just protocols and
formats.</p></li></ul><p>
</p><p>In short, the Web may be seen as a single, very large application (see
<a href="#Nicol">[Nicol]</a>), rather than as a collection of small independent
applications.</p><p>While these developments strengthen the requirement that Unicode be the basis
of a character model for the Web, they also create the need for additional
specifications on the application of Unicode to the Web. Some aspects of Unicode
that require additional specification for the Web include:
</p><ul><li><p>Choice of Unicode encoding forms (UTF-8, UTF-16, UTF-32).</p></li><li><p>Counting characters, measuring string length in the presence
of variable-length character encodings and combining characters.</p></li><li><p>Duplicate encodings of characters (e.g. precomposed vs decomposed).</p></li><li><p>Use of control codes for various purposes (e.g. bidirectionality
control, symmetric swapping, etc.).</p></li></ul><p>
</p><p id="def-legacyEnc">It should be noted that such aspects also exist in various
encodings, and in many cases have been inherited by Unicode
in one way or another from these encodings.</p><p>The remainder of this specification presents
additional
requirements to ensure an interoperable character model for the Web, taking into
account earlier work (from W3C, ISO and IETF).</p><p>The first few chapters of the Unicode Standard <a href="#unicode">[Unicode]</a>
provide very useful background reading. The policies adopted by the IETF for on
the use of character sets on the Internet are documented in <a href="#rfc2277">[RFC 2277]</a>.</p></div><div class="div2"> <h3><a name="sec-Notation" id="sec-Notation" />1.3 Terminology and Notation</h3><p>Unicode code points are denoted as U+hhhh, where "hhhh" is a
sequence of at least four, and at most six hexadecimal digits.</p><p>Text has been used for examples to allow them to be cut and pasted by the
reader. Characters used will not appear as intended unless you have the
appropriate font, but care has been taken to annotate the examples so that they
remain understandable even if you do not. In some cases it is important to see
the result of an example, so images have been used; by clicking on the image it
is possible to link to the text for these examples in <a href="#sec-ExampleText"><b>C Example text</b></a>.</p></div></div><div class="div1"> <h2><a name="sec-Conformance" id="sec-Conformance" />2 Conformance</h2><p>This section explains the conditions that specifications, software, and Web content have to fulfill to be able to claim conformance to this specification.</p><p>The key words "<span class="rfc2119">MUST</span>", "<span class="rfc2119">MUST
NOT</span>", "<span class="rfc2119">REQUIRED</span>", "<span class="rfc2119">SHALL</span>",
"<span class="rfc2119">SHALL NOT</span>", <span class="rfc2119">SHOULD</span>", "<span class="rfc2119">SHOULD
NOT</span>", "<span class="rfc2119">RECOMMENDED</span>", "<span class="rfc2119">MAY</span>" and
"<span class="rfc2119">OPTIONAL</span>" in this document are to be interpreted as
described in RFC 2119 <a href="#rfc2119">[RFC 2119]</a>.</p><div class="note"><p><span class="note-head">NOTE: </span>RFC 2119 makes it clear that requirements that use
<span class="rfc2119">SHOULD</span> are not optional and must be complied with unless
there are specific reasons not to: "<span class="quote">This word, or the adjective
"RECOMMENDED", mean that there may exist valid reasons in particular
circumstances to ignore a particular item, but the full implications must be
understood and carefully weighed before choosing a different
course.</span>"
</p></div><p>This specification defines conformance criteria
for specifications, for software, and for Web content. To aid the reader, all
conformance criteria are
preceded by '<span class="qterm">[X]</span>' where '<span class="qchar">X</span>' is one of
'<span class="qchar">S</span>' for specifications, '<span class="qchar">I</span>' for software
implementations, and '<span class="qchar">C</span>' for Web content. These markers indicate
the relevance of the conformance criteria and allow the
reader to quickly locate relevant conformance criteria by searching through this document.</p><p>A specification conforms to this document if it:</p><ol type="1"><li><p> does not violate any conformance criteria preceded by [S],</p></li><li><p>documents the reason for any deviation from criteria where the imperative is <span class="rfc2119">SHOULD</span>, <span class="rfc2119">SHOULD NOT</span>, or <span class="rfc2119">RECOMMENDED</span>,</p></li><li><p>where applicable, requires implementations conforming to the specification to conform to this document,</p></li><li><p> where applicable, requires content conforming to the specification to conform to this document.</p></li></ol><p>An implementation (software) conforms to this document if it does not
violate any conformance criteria preceded by [I].</p><p>Content conforms to this document if it does not violate any conformance criteria preceded by [C].</p><div class="note"><p><span class="note-head">NOTE: </span>Requirements placed on specifications might indirectly cause requirements to be placed on implementations or content that claim to conform to those specifications. Likewise, requirements placed on content may affect implementations designed to produce such content, and so on.</p></div><p>Where this specification places requirements on processing, it is to be understood as a way to
specify the desired external behavior. Implementations can
use other means of achieving the same results, as
long as observable behavior is not affected.</p></div><div class="div1"> <h2><a name="sec-Perceptions" id="sec-Perceptions" />3 Perceptions of Characters</h2><div class="div2"> <h3><a name="sec-PerceptionsIntro" id="sec-PerceptionsIntro" />3.1 Introduction</h3><p>The glossary entry in the Unicode Standard <a href="#unicode40">[Unicode 4.0]</a> gives:</p><p>
"<span class="quote">Character. (1) The smallest component of written language
that has semantic value; refers to the abstract meaning and/or shape
...</span>"</p><p>The word '<span class="qterm">character</span>' is used in many contexts, with
different meanings. Human cultures have radically differing writing systems,
leading to radically differing concepts of a character. Such wide variation in
end user experience can, and often does, result in misunderstanding. This
variation is sometimes mistakenly seen as the consequence of imperfect
technology. Instead, it derives from the great flexibility and creativity of
the human mind and the long tradition of writing as an important part of the
human cultural heritage. The alphabetic approach used by scripts such as Latin,
Cyrillic and Greek is only one of several possibilities.</p><div class="example"><p><span class="example-head">EXAMPLE: </span>A character in Japanese hiragana and katakana scripts corresponds
to a syllable (usually a combination of consonant plus vowel).</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>Korean Hangul combines symbols for
individual sounds of the language into square blocks, each of which represents a syllable. Depending on the
user and the application, either the individual symbols or the syllabic
clusters can be considered to be characters.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>In Indic scripts
each consonant letter carries an inherent vowel that is
eliminated or replaced using semi-regular or irregular ways to combine
consonants and vowels into clusters. Depending on the user and the application,
either individual consonants or vowels, or the consonant or consonant-vowel
clusters can be perceived as characters.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>In Arabic and Hebrew vowel sounds are typically not written at all.
When they are written they are indicated by the use of combining marks placed
above and below the consonantal letters.</p></div><p>The developers of specifications, and the developers of
software based on those specifications, are likely to be more familiar with
usages of the term '<span class="qterm">character</span>' they have experienced
and less familiar with the wide variety of usages in an international context.
Furthermore, within a computing context, characters are often confused with
related concepts, resulting in incomplete or inappropriate specifications and
software.</p><p>This section examines some of these contexts, meanings and
confusions.</p></div><div class="div2"> <h3><a name="sec-WritingSystem" id="sec-WritingSystem" />3.2 Units of aural rendering</h3><p>In some scripts, characters have a close relationship to phonemes
(a <span class="new-term">phoneme</span> is a minimally distinct sound in the context of a
particular spoken language), while in others they are closely related to
meanings. Even when characters (loosely) correspond to phonemes, this
relationship may not be simple, and there is rarely a one-to-one correspondence
between character and phoneme.</p><div class="example"><p><span class="example-head">EXAMPLE: </span>In the English sentence,
"<span class="quote">They were too close to the door to close it.</span>" the same character
'<span class="qchar">s</span>' is used to represent both /s/ and /z/ phonemes.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>In the English language the phoneme /k/ of "<span class="quote">cool</span>" is like the phoneme /k/ of "<span class="quote">keel</span>".</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>In many scripts a single character may represent a sequence of
phonemes, such as the syllabic characters of Japanese hiragana.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>In many writing systems a sequence of characters may represent a
single phoneme, for example '<span class="qchar">th</span>' and '<span class="qchar">ng</span>' in
"<span class="quote">thing</span>".</p></div><p>
<a id="C001" name="C001" href="#C001"><span class="reqId">C001</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
<span class="requirement-type">[C]</span>
Specifications,
software and content <span class="rfc2119">MUST NOT</span> require or depend on a one-to-one
correspondence between characters and the sounds of a
language.
</span>
</p></div><div class="div2"> <h3><a name="sec-VisualRenderingUnits" id="sec-VisualRenderingUnits" />3.3 Units of visual
rendering</h3><p id="def-glyph">Visual rendering introduces the notion of a
<em>glyph</em>. <span class="new-term">Glyphs</span> are defined by ISO/IEC 9541-1
<a href="#iso9541">[ISO/IEC 9541-1]</a> as "<span class="quote">a recognizable abstract graphic symbol which
is independent of a specific design</span>". There is <em>not</em> a
one-to-one correspondence between characters and glyphs:
</p><ul><li><p>A single character can be represented by multiple glyphs
(each glyph is then part of the representation of that character). These glyphs
may be physically separated from one another. </p></li><li><p>A single glyph may represent a sequence of characters (this
is the case with ligatures, among others).</p></li><li><p>A character may be rendered with very different glyphs
depending on the context.</p></li><li><p>A single glyph may represent different characters (e.g.
capital Latin A, capital Greek A and capital Cyrillic A).</p></li></ul><p>
</p><p>A set of glyphs makes up a <span class="new-term">font</span>. Glyphs can be
construed as the basic units of organization of the visual rendering of text,
just as characters are the basic unit of organization of encoded text.</p><p>
<a id="C002" name="C002" href="#C002"><span class="reqId">C002</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
<span class="requirement-type">[C]</span>
Specifications,
software and content <span class="rfc2119">MUST NOT</span> require or depend on a one-to-one mapping between
characters and units of displayed text.
</span>
</p><p>See the appendix <a href="#sec-CharExamples"><b>B Examples of Characters, Keystrokes
and Glyphs</b></a> for examples of
the complexities of character to glyph mapping.</p><div class="div3"> <h4><a name="sec-LogicalOrder" id="sec-LogicalOrder" />3.3.1 Visual Rendering and Logical Order</h4><p>Some scripts, in particular Arabic and Hebrew, are written from
right to left. Text including characters from these scripts can run in both
directions and is therefore called bidirectional text. The Unicode Standard
<a href="#unicode">[Unicode]</a> requires that characters be stored and interchanged in
<span class="new-term">logical order</span>, i.e. roughly corresponding to the order in which text
is typed in via the keyboard or spoken (for a more detailed definition see
<a href="#unicode40">[Unicode 4.0]</a>, Section 2.2). Logical ordering is
important to ensure interoperability of data, and also benefits accessibility,
searching, and collation.
</p><p><a id="C003" name="C003" href="#C003"><span class="reqId">C003</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
<span class="requirement-type">[C]</span>
Protocols,
data formats and APIs <span class="rfc2119">MUST</span> store, interchange or process
text data in logical order.
</span>
</p><p>In the presence of bidirectional text, two possible selection
modes can be considered. The first is <span class="new-term">logical selection mode</span>,
which selects all the characters <em>logically</em> located between the
end-points of the user's mouse gesture. Here the user selects from between the
first and second letters of the second word to the middle of the number.
Logical selection looks like this:</p><div class="figure" align="center"><table border="1" cellspacing="0" cellpadding="5" summary="Two images contrasting a single logical selection in memory and the resulting two selections on screen, in a bidi context"><tbody><tr><th>Visual display</th><td align="center">
<a href="#arabic-select"><img align="middle" src="images/logSelScreen.gif" alt="The same
 example, showing how the text would look on-screen when highlighted, showing two
 separate highlighted character ranges." height="32" width="144" /></a>
</td></tr><tr><th>Logical order</th><td align="center">
<a href="#arabic-select"><img align="middle" src="images/logSelMemory.gif" alt="An example
 showing the logical order of characters in a string containing two Arabic words
 followed by a year number. In logical selection mode, the range of characters
 selected by starting the selection in the middle of the second word and ending
 in the middle of the year number is depicted using highlighting. The
 highlighting covers a single block of contiguous characters." height="27" width="323" /></a>
</td></tr></tbody></table><div class="caption">Logical selection resulting in discontiguous visual ranges</div></div><p>It is a consequence of the bidirectionality of the text that a
single, continuous logical selection in memory results in a <em>discontinuous
selection appearing on the screen</em>. This discontinuity makes some users prefer a
<span class="new-term">visual selection mode</span>, which selects all the characters
<em>visually</em> located between the end-points of the user's mouse
gesture. With the same mouse gesture as before, we now obtain:</p><div class="figure" align="center"><table border="1" cellpadding="5" cellspacing="0" summary="Two images contrasting a single visual selection on screen and the resulting two selections in memory, in a bidi context"><tbody><tr><th>Visual display</th><td align="center">
<a href="#arabic-select"><img align="middle" src="images/visSelScreen.gif" alt="
 The
 same example, showing how the text would look on-screen when highlighted, showing
 a single highlighted block of contiguous
 characters." height="33" width="141" /></a>
</td></tr><tr><th>Logical order</th><td align="center">
<a href="#arabic-select"><img align="middle" src="images/visSelMemory.gif" alt="
 An
 example showing the logical order of characters in a string containing two
 Arabic words followed by a year number. In visual selection mode, the range of
 characters selected by starting the selection in the middle of the second word
 and ending in the middle of the year number is depicted using highlighting. The
 highlighting covers two separate blocks of
 characters." height="27" width="343" /></a>
</td></tr></tbody></table><div class="caption">Visual selection resulting in discontiguous logical ranges</div></div><p>In visual selection mode, as seen in the example above, a single visual selection range may result in
<em>two or more</em> logical ranges, which may have to be accommodated by protocols,
APIs and implementations. Other, related aspects of a user interface for bidirectional text include caret movement, behavior of backspace/delete keys, and so on.</p><p>Currently, most implementations provide logical selection, while only very few provide visual selection. </p><p><a id="C075" name="C075" href="#C075"><span class="reqId">C075</span></a> <span class="req"><span class="requirement-type">[I]</span> Independent of whether some implementation uses logical selection or visual selection, characters selected <span class="rfc2119">MUST</span> be kept in logical order in storage.</span></p><p>
<a id="C004" name="C004" href="#C004"><span class="reqId">C004</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications of protocols
and APIs that involve selection of ranges <span class="rfc2119">SHOULD</span> provide for
discontiguous logical selections, at least to the extent necessary to support
implementation of visual selection on screen on top of those protocols and
APIs.
</span>
</p></div></div><div class="div2"> <h3><a name="sec-InputUnits" id="sec-InputUnits" />3.4 Units of input</h3><p>In keyboard input, it is <em>not</em> always the case that
keystrokes and input characters correspond one-to-one. A limited number of keys
can fit on a keyboard. Some keyboards will generate multiple characters from a
single keypress. In other cases ('<span class="qterm">dead keys</span>') a key will generate
no characters, but affect the results of subsequent keypresses. Many writing
systems have far too many characters to fit on a keyboard and must rely on more
complex <span class="new-term">input methods</span>, which transform keystroke sequences into
character sequences. Other languages may make it necessary to input some
characters with special modifier keys. See <a href="#sec-CharExamples"><b>B Examples of Characters, Keystrokes
and Glyphs</b></a>
for examples of non-trivial input.</p><p>
<a id="C005" name="C005" href="#C005"><span class="reqId">C005</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
Specifications
and software <span class="rfc2119">MUST NOT</span> require nor depend on a single keystroke resulting
in a single character, nor that a single character be input with a single
keystroke (even with modifiers), nor that keyboards are the same all over the
world.
</span>
</p></div><div class="div2"> <h3><a name="sec-CollationUnits" id="sec-CollationUnits" />3.5 Units of collation</h3><p>String comparison as used in sorting and searching is based on
units which do not in general have a one-to-one relationship to encoded
characters. Such string comparison can aggregate a character sequence into a
single <span class="new-term">collation unit</span> with its own position in the sorting order,
can separate a single character into multiple collation units, and can
distinguish various aspects of a character (case, presence of diacritics, etc.)
to be sorted separately (multi-level sorting).</p><p>In addition, a certain amount of pre-processing may also be
required, and in some languages (such as Japanese and Arabic) sort order may be
governed by higher order factors such as phonetics or word roots. Collation
methods may also vary by application.</p><div class="example"><p><span class="example-head">EXAMPLE: </span>In traditional Spanish sorting, the character sequences 'ch' and 'll' are treated as atomic collation units.
Although Spanish sorting, and to some extent Spanish everyday use, treat
'<span class="qchar">ch</span>' as a single unit, current digital encodings treat it as two
characters, and keyboards do the same (the user types '<span class="qchar">c</span>', then
'<span class="qchar">h</span>').</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>In some languages, the letter
'<span class="qchar">æ</span>' is sorted as two consecutive collation units: '<span class="qchar">a</span>'
and '<span class="qchar">e</span>'.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>The sorting of text written in a
bicameral script (i.e. a script which has distinct upper and lower case
letters) is usually required to ignore case differences in a first pass; case
is then used to break ties in a later pass.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>Treatment of
accented letters in sorting is dependent on the script or language in question.
The letter '<span class="qchar">ö</span>' is treated as a modified '<span class="qchar">o</span>' in
French, but as a letter completely independent from '<span class="qchar">o</span>' (and
sorting after '<span class="qchar">z</span>') in Swedish. In German certain applications
treat the letter '<span class="qchar">ö</span>' as if it were the sequence
'<span class="qchar">oe</span>'.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>In Thai the sequence '<span class="qchar">ไก</span>' (U+0E44 U+0E01) must
be sorted as if it were written '<span class="qchar">กไ</span>' (U+0E01 U+0E44). Reordering is typically done
during an initial pre-processing stage.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>German dictionaries typically sort '<span class="qchar">ä</span>', '<span class="qchar">ö</span>' and '<span class="qchar">ü</span>' together with '<span class="qchar">a</span>', '<span class="qchar">o</span>' and '<span class="qchar">u</span>' respectively. On the other hand, German telephone books typically sort '<span class="qchar">ä</span>', '<span class="qchar">ö</span>' and '<span class="qchar">ü</span>' as if they were spelled '<span class="qchar">ae</span>', '<span class="qchar">oe</span>' and '<span class="qchar">ue</span>'. Here the application is affecting the collation algorithm used.</p></div><p>
<a id="C006" name="C006" href="#C006"><span class="reqId">C006</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
Software
that sorts or searches text for users
<span class="rfc2119">SHOULD</span> do so on
the basis of appropriate collation units and ordering rules for the relevant
language and/or application.
</span>
</p><p>
<a id="C007" name="C007" href="#C007"><span class="reqId">C007</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
Where searching or sorting is done dynamically,
particularly in a multilingual environment, the 'relevant language'
<span class="rfc2119">SHOULD</span> be determined to be that of the current user, and may
thus differ from user to user.
</span></p><p><a id="C066" name="C066" href="#C066"><span class="reqId">C066</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
Software that allows
users to sort or search text <span class="rfc2119">SHOULD</span> allow the user to select
alternative rules for collation units and ordering.
</span>
</p><p>
<a id="C008" name="C008" href="#C008"><span class="reqId">C008</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
Specifications and implementations of sorting and searching algorithms <span class="rfc2119">SHOULD</span> accommodate text that contains any character in Unicode.
</span>
</p><p>Note that this requires, as a minimum, that a collation algorithm does not break down if the text contains Unicode characters that are not covered by its rules. It does not necessarily require full implementation of complex algorithms for all scripts. One useful way of satisfying the requirement is to apply a default collation algorithm that covers all Unicode characters.</p><p>ISO/IEC 14651 <a href="#iso14651">[ISO/IEC 14651]</a> and Unicode Technical Report #10, the Unicode Collation
Algorithm <a href="#UTR10">[UTR #10]</a>, describe a model for collation that accommodates most
languages and provide a default collation order. They are appropriate
references for collation and provide implementation guidelines.
The default collation order can be used in conjunction with rules tailored for a particular locale
to ensure a predictable ordering and comparison of strings, whatever characters
they include.</p></div><div class="div2"> <h3><a name="sec-Storage" id="sec-Storage" />3.6 Units of storage</h3><p>Computer storage and communication rely on units of physical
storage and information interchange, such as bits and bytes (8-bit units, also called octets). A frequent error in specifications and implementations is
the equating of characters with units of physical storage. The mapping between
characters and such units of storage is actually quite complex, and is
discussed in the next section, <a href="#sec-Digital"><b>4.1 Character Encoding</b></a>.</p><p>
<a id="C009" name="C009" href="#C009"><span class="reqId">C009</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
Specifications,
software and content <span class="rfc2119">MUST NOT</span> require or depend on a one-to-one relationship
between characters and units of physical storage.
</span>
</p></div><div class="div2"> <h3><a name="sec-PerceptionsOutro" id="sec-PerceptionsOutro" />3.7 Summary</h3><p id="def-text">The term character is used differently in a variety
of contexts and often leads to confusion when used outside of these contexts.
In the context of the digital representations of text, a <span class="new-term">character</span> can be
defined as a small logical unit of text. <span class="new-term">Text</span> is then
defined as sequences of characters. While such an informal definition is
sufficient to create or capture a common understanding in many cases, it is
also sufficiently open to create misunderstandings as soon as details start to
matter. In order to write effective specifications, protocol implementations,
and software for end users, it is very important to understand that these
misunderstandings can occur.</p><p>This section, <a href="#sec-Perceptions"><b>3 Perceptions of Characters</b></a>, has discussed terms for units that do not necessarily overlap with the term '<span class="qterm">character</span>', such as phoneme, glyph, and collation unit. The next section, <a href="#sec-Digital"><b>4.1 Character Encoding</b></a>, lists terms that should be used rather than '<span class="qterm">character</span>' to precisely define <span class="new-term">units of encoding</span> (code point, code unit, and byte).</p><p>
<a id="C010" name="C010" href="#C010"><span class="reqId">C010</span></a> <span class="req">
<span class="requirement-type">[S]</span>
When specifications use the
term '<span class="qterm">character</span>' the specifications <span class="rfc2119">MUST</span>
define which meaning they intend.
</span></p><p><a id="C067" name="C067" href="#C067"><span class="reqId">C067</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications <span class="rfc2119">SHOULD</span> use specific terms, when available, instead of the general term 'character'.
</span>
</p></div></div><div class="div1"> <h2><a name="sec-Characters" id="sec-Characters" />4 Digital Encoding of Characters</h2><div class="div2"> <h3><a name="sec-Digital" id="sec-Digital" />4.1 Character Encoding</h3><p>On the WWW, as in
any computing environment, characters must be encoded to be of any
use. To
achieve text encoding, a large variety of character encodings have been devised. Character encodings can loosely be explained as mappings between the character sequences that
users manipulate and the sequences of bits that computers manipulate.</p><p>Given the complexity of text encoding and the large variety of
mechanisms for character encoding invented throughout the computer age, a more
formal description of the encoding process is useful. The process of defining a
text encoding can be described as follows (see Unicode Technical Report #17:
Character Encoding Model <a href="#UTR17">[UTR #17]</a> for a more detailed
description):
</p><ol type="1"><li><a name="def-repertoire" id="def-repertoire"></a><p>A set of characters to be encoded is identified. The
characters are pragmatically chosen to express text and to efficiently allow
various text processes in one or more target languages. They may not correspond
precisely to what users perceive as letters and other characters. The set of
characters is called a <span class="new-term">repertoire</span>.</p></li><li><a name="def-CCS" id="def-CCS"></a><p>Each character in the repertoire is then associated with a
(mathematical, abstract) non-negative integer, the <span class="new-term">code point</span>
(also known as a <span class="new-term">character number</span> or <span class="new-term">code position</span>).
The result, a mapping from the repertoire to the set of non-negative integers,
is called a <span class="new-term">coded character set (CCS)</span>.</p></li><li><a name="def-CEF" id="def-CEF"></a><p>To enable use in computers, a suitable base datatype is
identified (such as a byte, a 16-bit unit of storage or other) and a
<span class="new-term">character encoding form (CEF)</span> is used, which encodes the abstract
integers of a coded character set (<acronym title="Coded Character Set">CCS</acronym>) into sequences
of the <span class="new-term">code units</span> of the base datatype. The character encoding form can be
extremely simple (for instance, one which encodes the integers of the
<acronym title="Coded Character Set">CCS</acronym> into the natural
representation of integers of the chosen datatype of the computing platform) or
arbitrarily complex (a variable number of code units, where the value of each
unit is a non-trivial function of the encoded integer). </p></li><li><a name="def-CES" id="def-CES"></a><p>To enable transmission or storage using byte-oriented devices,
a <span class="new-term">serialization scheme</span> or <span class="new-term">character encoding scheme
(CES)</span> is next used. A character encoding scheme is a mapping of the code units
of a character encoding form (<acronym title="Character Encoding Form">CEF</acronym>) into well-defined
sequences of bytes, taking into account the necessary specification of
byte-order for multi-byte base datatypes and including in some cases switching
schemes between the code units of multiple
character encoding schemes (an example is ISO
2022). A character encoding scheme, together
with the coded character sets it is used
with, is called a <span class="new-term">character encoding</span>, and is identified by a unique identifier, such as an
<acronym title="Internet Assigned Numbers Authority">IANA</acronym> charset
identifier. Given a sequence of bytes representing text and a character encoding identified by a <code class="keyword">charset</code>
identifier, one can in principle unambiguously recover the sequence of
characters of the text.</p></li></ol><p>
</p><div class="note"><p><span class="note-head">NOTE: </span>See <a href="#sec-EncodingIdent"><b>4.4.2 Character encoding
identification</b></a> for a discussion of the
term '<span class="qterm">charset</span>' and further details on character encodings.</p></div><div class="note"><p><span class="note-head">NOTE: </span>The term '<span class="qterm">character encoding</span>' is somewhat ambiguous,
as it is sometimes used to describe the actual process of encoding characters
and sometimes to denote a particular way to perform that process (as in
"<span class="quote">this file is in the X character encoding</span>"). Context normally
allows the distinction of those uses, once one is aware of the ambiguity.</p></div><div class="note"><p><span class="note-head">NOTE: </span>Given a sequence of characters, a given '<span class="qterm">character encoding</span>' may not always produce the same sequence of bytes. In particular for encodings based on ISO 2022, there may be choices available during the encoding process.</p></div><p>In very simple cases, the whole encoding process can be collapsed to
a single step, a trivial one-to-one mapping from characters to bytes; this is
the case, for instance, for US-ASCII <a href="#iso646">[ISO/IEC 646]</a> and ISO-8859-1.</p><p id="Unicode_Encoding_Form">Text is said to be in a <span class="new-term">Unicode
encoding form</span> if it is encoded in UTF-8, UTF-16 or UTF-32.</p></div><div class="div2"> <h3><a name="sec-Transcoding" id="sec-Transcoding" />4.2 Transcoding</h3><p id="def-transcoding">
<span class="new-term">Transcoding</span> is the process of
converting text from one <a title="" href="#def-CES">character
encoding</a> to another. Transcoders work only at
the level of character
encoding and do not parse the text; consequently, they do not deal with
<a title="" href="#sec-Escaping">character escapes</a> such as numeric
character references (see <a href="#sec-Escaping"><b>4.6 Character Escaping</b></a>) and do not adjust
embedded character encoding information (for instance in an XML declaration or
in an HTML <code>meta</code> element).</p><div class="note"><p><span class="note-head">NOTE: </span>Transcoding may involve one-to-one, many-to-one, one-to-many or
many-to-many mappings. In addition, the storage order of characters varies
between encodings: some, such as the Unicode encoding forms, prescribe
logical ordering, while others use visual ordering; among encodings that have
separate diacritics, some prescribe that they be placed before the base
character, some after. Because of these differences in sequencing characters,
transcoding may involve reordering: thus XYZ may map to yxz.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>This first example shows the transcoding of the Russian word '<span class="qterm">Русский</span>' meaning '<span class="qterm">Russian</span>' (language),
from the UTF-16 encoding of Unicode to the ISO 8859-5 encoding:</p><div class="figure" align="center"><table border="1" cellpadding="5" cellspacing="0" summary="table displaying the mapping from ISO 8859-5 to UTF-16"><tbody><tr align="center"><th colspan="2">UTF-16</th><th colspan="2">ISO 8859-5</th></tr><tr align="center"><th>Code unit</th><th>Char. name (abbreviated)</th><th>Code unit</th><th>Char. name (abbreviated)</th></tr><tr align="center"><td>0420</td><td>CAPITAL ER</td><td>C0</td><td>CAPITAL ER</td></tr><tr align="center"><td>0443</td><td>SMALL U</td><td>E3</td><td>SMALL U</td></tr><tr align="center"><td>0441</td><td>SMALL ES</td><td>E1</td><td>SMALL ES</td></tr><tr align="center"><td>0441</td><td>SMALL ES</td><td>E1</td><td>SMALL ES</td></tr><tr align="center"><td>043A</td><td>SMALL KA</td><td>DA</td><td>SMALL KA</td></tr><tr align="center"><td>0438</td><td>SMALL I</td><td>D8</td><td>SMALL I</td></tr><tr align="center"><td>0439</td><td>SMALL SHORT I</td><td>D9</td><td>SMALL SHORT I</td></tr></tbody></table></div></div><div class="example"><p><span class="example-head">EXAMPLE: </span>This second example shows a much more complex case, where the Arabic word '<span class="qterm">السلام</span>', meaning '<span class="qterm">peace</span>', is transcoded from the
visually-ordered, contextualized encoding IBM CP864 to the UTF-16 encoding of Unicode:</p><div class="figure" align="center"><table border="1" cellpadding="5" cellspacing="0" summary="table displaying the mapping from UTF-16 to IBM CP864"><tbody><tr align="center"><th colspan="2">IBM CP864</th><th colspan="2">UTF-16</th></tr><tr align="center"><th>Code unit</th><th>Char. name (abbreviated)</th><th>Code unit</th><th>Char. name (abbreviated)</th></tr><tr><td>EF</td><td>FINAL MEEM</td><td>0627</td><td>ALEF</td></tr><tr><td>9E</td><td>MEDIAN LAM-ALEF</td><td>0644</td><td>LAM</td></tr><tr><td>D3</td><td>MEDIAN SEEN</td><td>0633</td><td>SEEN</td></tr><tr><td>E4</td><td>MEDIAN LAM</td><td>0644</td><td>LAM</td></tr><tr><td>C7</td><td>INITIAL ALEF</td><td>0627</td><td>ALEF</td></tr><tr><td></td><td></td><td>0645</td><td>MEEM</td></tr></tbody></table></div><p>Notice that the order of the characters has been reversed, that the single LAM-ALEF in CP864 has been converted to a LAM ALEF sequence in UTF-16, and that the contextual variants (initial, median or final) in the source encoding have been converted to generic characters in the target encoding.</p></div></div><div class="div2"> <h3><a name="sec-RefProcModel" id="sec-RefProcModel" />4.3 Reference Processing Model</h3><p id="def-char-data">Many Internet protocols and data formats, most
notably the very important Web formats HTML, CSS and XML, are based on text. In
those formats, everything is text but the relevant specifications impose a
structure on the text, giving meaning to certain constructs so as to obtain
functionality in addition to that provided by <span class="new-term">plain text</span> (text that is not in the context of markup or a programming language). HTML and XML are <span class="new-term">markup
languages</span>, defining
documents entirely composed of text but with
conventions allowing the separation of this text into <span class="new-term">markup</span> and
<span class="new-term">character data</span>. Citing from the XML 1.0 specification
<a href="#xml10">[XML 1.0]</a>,
<a href="http://www.w3.org/TR/2004/REC-xml-20040204/#syntax">section
2.4</a>:</p><p>
"<span class="quote">Text consists of intermingled character data and markup.
[...] All text that is not markup constitutes the character data of the
document.</span>"
</p><p>For the purposes of this section, the important aspect is that
everything is <a title="" href="#def-text">text</a>, that is, a sequence of characters.</p><p>A <span class="new-term">textual data object</span> is a whole text protocol message or
a whole text document, or a part of it that is treated separately for purposes
of external storage and retrieval. Examples include external parsed entities
in XML and textual MIME entity bodies
<a href="#MIME-entity">[MIME-entity]</a>.</p><p>
<a id="C013" name="C013" href="#C013"><span class="reqId">C013</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[C]</span>
Textual data objects defined by
protocol or format specifications <span class="rfc2119">MUST</span> be in a
<em>single</em> character encoding.
</span></p><p>Note that this does not
imply that character set switching schemes such as ISO 2022 cannot be
used, since such schemes perform character set switching within a single
character encoding.</p><p id="def-ref-proc-model">Since its early days, the Web has seen the
development of a <span class="new-term">Reference Processing Model</span>, first described for
HTML in RFC 2070 <a href="#rfc2070">[RFC 2070]</a>. This model was later embraced by XML
and CSS. It is applicable to any data format or protocol that is text-based as
described above. The essence of the Reference Processing Model is the use of
Unicode as a common reference. Use of the Reference Processing Model by a
specification does not, however, require that implementations actually use
Unicode. The requirement is only that the implementations behave as if the
processing took place as described by the Model. Also, while this document uses the term Reference <em>Processing</em> Model and describes its properties in terms of processing, the model also applies to specifications that do not explicitly define a processing model.</p><p><a id="C014" name="C014" href="#C014"><span class="reqId">C014</span></a><span class="req"><span class="requirement-type">[S]</span> </span><span class="req">All specifications that
involve processing of text <span class="rfc2119">MUST</span> specify the processing of
text according to the <a title="" href="#sec-RefProcModel">Reference Processing
Model</a>, namely:</span></p><div class="req"><ol type="1"><li><p>Specifications <span class="rfc2119">MUST</span> define text in terms of
Unicode characters, not bytes or <a title="" href="#def-glyph">glyphs</a>.</p></li><li><p>For their textual data objects specifications <span class="rfc2119">MAY</span> allow use of any
character encoding which can be transcoded to a Unicode encoding form.</p></li><li><p>Specifications <span class="rfc2119">MAY</span> choose to disallow or
deprecate some character encodings and to make others mandatory. Independent of the
actual character encoding, the specified behavior <span class="rfc2119">MUST</span> be the same
<em>as if</em> the processing happened as follows:
</p><ul><li><p>The character encoding of any textual data object received by the
application implementing the specification <span class="rfc2119">MUST</span> be
determined and the data object <span class="rfc2119">MUST</span> be interpreted as a
sequence of Unicode characters - this <span class="rfc2119">MUST</span> be equivalent to
<a title="" href="#def-transcoding">transcoding</a> the data object to some
<a title="" href="#Unicode_Encoding_Form">Unicode encoding form</a>, adjusting
any character encoding label if necessary, and receiving it in that Unicode
encoding form.</p></li><li><p>All processing <span class="rfc2119">MUST</span> take place on
this sequence of Unicode characters.</p></li><li><p>If text is output by the application, the sequence of
Unicode characters <span class="rfc2119">MUST</span> be encoded using a character encoding chosen
among those allowed by the specification.</p></li></ul><p>
</p></li><li><p>If a specification is such that multiple textual data objects are
involved (such as an XML document referring to external parsed entities), it
<span class="rfc2119">MAY</span> choose to allow these data objects to be in different
character encodings. In all cases, the <a title="" href="#sec-RefProcModel">Reference Processing Model</a>
<span class="rfc2119">MUST</span> be applied to all textual data objects.</p></li></ol></div><div class="note"><p><span class="note-head">NOTE: </span>All specifications which define applications of the XML 1.0 specification
<a href="#xml10">[XML 1.0]</a> automatically inherit this Reference Processing Model.
XML is entirely defined in terms of Unicode characters and requires the UTF-8
and UTF-16 character encodings while allowing any other character encoding for parsed entities.</p></div><div class="note"><p><span class="note-head">NOTE: </span>When specifications choose to allow character encodings other than Unicode
encoding forms, implementers should be aware that the correspondence between the
characters of such encodings and
Unicode characters may in practice depend on the software used for
<a title="" href="#def-transcoding">transcoding</a>. See the Japanese XML
Profile <a href="#XML_Japanese_profile">[XML Japanese Profile]</a> for examples of such
inconsistencies.</p></div><p>
<a id="C070" name="C070" href="#C070"><span class="reqId">C070</span></a> <span class="req"><span class="requirement-type">[S]</span>
Specifications <span class="rfc2119">SHOULD NOT</span>
<em>arbitrarily</em> exclude code points from the full range
of Unicode <a title="" href="#def-CCS">code points</a> from U+0000
to U+10FFFF inclusive.</span></p><p>
<a id="C077" name="C077" href="#C077"><span class="reqId">C077</span></a> <span class="req"><span class="requirement-type">[S]</span>
Specifications <span class="rfc2119">MUST NOT</span> allow code
points above U+10FFFF.
</span></p><p>Unicode contains some code points for internal use (such as noncharacters) or
special functions (such as surrogate code points).</p><p><a id="C079" name="C079" href="#C079"><span class="reqId">C079</span></a> <span class="req"><span class="requirement-type">[S]</span> Specifications <span class="rfc2119">SHOULD NOT</span> allow the use of codepoints reserved by Unicode for internal use.</span></p><p><a id="C078" name="C078" href="#C078"><span class="reqId">C078</span></a> <span class="req"><span class="requirement-type">[S]</span>
Specifications <span class="rfc2119">MUST NOT</span> allow the
use of surrogate code points.</span></p><p>Excluding code points without good reason conflicts with the W3C goal of
universal accessibility. Excluding code points would prevent some scripts from
being used which may be important to a user community or communities. For
example, without strong reasons to do so, decisions to exclude code points above
the Basic Multilingual Plane or to limit code points to the US-ASCII or Latin-1
repertoire are inappropriate. Also, please note that the Unicode Standard requires software to not corrupt any
code points.</p><p>Other examples of legitimate and non-arbitrary reasons to exclude characters can
be seen in <cite>Unicode in XML and other Markup Languages</cite> <a href="#UXML">[UXML]</a>, where the
use of certain characters is discouraged for reasons such as:</p><ul><li><p>They are deprecated in the Unicode Standard.</p></li><li><p>They cannot be supported without additional data.</p></li><li><p>They are better handled by markup.</p></li><li><p>They conflict with equivalent markup.</p></li></ul><p></p></div><div class="div2"> <h3><a name="sec-Encodings" id="sec-Encodings" />4.4 Choice and Identification of Character
Encodings</h3><p>Because encoded text <em>cannot</em> be interpreted and
processed without knowing the encoding, it is vitally important that the
character encoding (see <a href="#sec-Digital"><b>4.1 Character Encoding</b></a>) is known at all times and
places where text is exchanged, stored or processed. In what follows we use
'<span class="qterm">character encoding</span>' to mean either <a title="" href="#def-CEF">character encoding form (CEF)</a> or <a title="" href="#def-CES">character encoding scheme (CES)</a> depending
on the context. When text is transmitted or stored as a byte stream, for
instance in a protocol or file system, specification of a <a title="" href="#def-CES">CES</a> is required to ensure proper
interpretation. In contexts such as an API, where the environment (typically
the processor architecture) specifies the byte order of multibyte quantities,
specification of a <a title="" href="#def-CEF">CEF</a> suffices.</p><p><a id="C015" name="C015" href="#C015"><span class="reqId">C015</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications <span class="rfc2119">MUST</span>
either specify a unique character encoding, or provide character encoding identification
mechanisms such that the encoding of text can be reliably
identified.
</span></p><p><a id="C016" name="C016" href="#C016"><span class="reqId">C016</span></a> <span class="req">
<span class="requirement-type">[S]</span>
When
designing a new protocol, format or API, specifications
<span class="rfc2119">SHOULD</span> require a unique character
encoding.
</span></p><p><a id="C017" name="C017" href="#C017"><span class="reqId">C017</span></a> <span class="req">
<span class="requirement-type">[S]</span>
When basing
a protocol, format, or API on a protocol, format, or API that already
has rules for character encoding, specifications
<span class="rfc2119">SHOULD</span> use rather than change these rules.
</span>
</p><div class="example"><p><span class="example-head">EXAMPLE: </span>An XML-based format should use the existing XML rules for choosing and determining
the character encoding of external entities, rather than invent new ones.</p></div><div class="div3"> <h4><a name="sec-UniqueEncoding" id="sec-UniqueEncoding" />4.4.1 Mandating a unique character
encoding</h4><p>Mandating a unique character encoding is simple, efficient, and
robust. There is no need for specifying, producing, transmitting, and
interpreting encoding tags. At the receiver, the character encoding will always be
understood. There is also no ambiguity as to which character encoding to use if data is
transferred non-electronically and later has to be converted back to a digital
representation. Even when there is a need for compatibility with existing data,
systems, protocols and applications, multiple character encodings can often be dealt with
at the boundaries or outside a protocol, format, or API. The
<acronym title="Document Object Model">DOM</acronym>
<a href="#dom1">[DOM Level 1]</a> is an
example of where this was done. The advantages of choosing a unique character encoding
are greater
when text sizes are small or the specification is close to the actual
processing.</p><p>
<a id="C018" name="C018" href="#C018"><span class="reqId">C018</span></a> <span class="req">
<span class="requirement-type">[S]</span>
When a unique character encoding is
required, the character encoding <span class="rfc2119">MUST</span> be UTF-8, UTF-16 or
UTF-32.
</span></p><p>US-ASCII is upwards-compatible with UTF-8 (an US-ASCII string is also a UTF-8 string, see <a href="#rfc3629">[RFC 3629]</a>), and UTF-8 is therefore appropriate if compatibility with US-ASCII is desired. In
other situations, such as for APIs, UTF-16 or UTF-32 may be more appropriate.
Possible reasons for choosing one of these include efficiency of internal
processing and interoperability with other processes.</p><div class="note"><p><span class="note-head">NOTE: </span>The IETF Charset Policy <a href="#rfc2277">[RFC 2277]</a> specifies that
on the Internet "<span class="quote">Protocols MUST be able to use the UTF-8
charset</span>".</p></div><div class="note"><p><span class="note-head">NOTE: </span>The XML 1.0 specification <a href="#xml10">[XML 1.0]</a> requires all
conforming XML processors to accept both UTF-16 and UTF-8.</p></div></div><div class="div3"> <h4><a name="sec-EncodingIdent" id="sec-EncodingIdent" />4.4.2 Character encoding
identification</h4><p>The MIME Internet specification provides a
good example of a mechanism for character encoding identification <a href="#MIME-charset">[MIME-charset]</a><a href="#RFC2978">[RFC 2978]</a>. The MIME
<code class="keyword">charset</code> parameter definition is intended to supply sufficient
information to uniquely decode the sequence of bytes of the received data into
a sequence of characters. The values are drawn from the IANA charset registry
<a href="#iana">[IANA]</a>.</p><div class="note"><p><span class="note-head">NOTE: </span>Unfortunately, some charset identifiers do not represent a
single, unique character encoding. Instead, these identifiers denote a number of
small variations. Even though small, the differences
may be crucial and may vary over time. For these identifiers, recovery of the
character sequence from a byte sequence is ambiguous. For example, the
character encoded as 0x5C in Shift_JIS is ambiguous. This code point sometimes represents a <span class="uname">YEN SIGN</span> and sometimes
represents a <span class="uname">REVERSE SOLIDUS</span>. See the
<a href="#XML_Japanese_profile">[XML Japanese Profile]</a> for more detail on this example and for
additional examples of such ambiguous charset identifiers. </p></div><div class="note"><a name="def-charset" id="def-charset"></a><p><span class="note-head">NOTE: </span>The term <span class="new-term">charset</span> derives from '<span class="qterm">character
set</span>', an expression with a long and tortured history (see
<a href="#connolly">[Connolly]</a> for a discussion).</p></div><p>
<a id="C020" name="C020" href="#C020"><span class="reqId">C020</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications
<span class="rfc2119">SHOULD</span> avoid using the terms '<span class="qterm">character set</span>'
and '<span class="qterm">charset</span>' to refer to a character encoding, except when the
latter is used to refer to the MIME <code class="keyword">charset</code> parameter or its
IANA-registered values. The term '<span class="qterm">character encoding</span>',
or in specific cases the terms '<span class="qterm">character encoding form</span>' or '<span class="qterm">character encoding
scheme</span>', are <span class="rfc2119">RECOMMENDED</span>.
</span>
</p><div class="note"><p><span class="note-head">NOTE: </span>In XML, the XML declaration or the text declaration contains the <code>encoding</code>
pseudo-attribute which identifies the character
encoding using the IANA charset.</p></div><p>The IANA charset registry is the official list of names and
aliases for character encoding schemes on the Internet.</p><p>
<a id="C021" name="C021" href="#C021"><span class="reqId">C021</span></a> <span class="req">
<span class="requirement-type">[S]</span>
If the unique encoding
approach is not taken, specifications <span class="rfc2119">SHOULD</span> require the use
of the IANA charset registry names, and in particular the names identified in
the registry as '<span class="qterm">MIME preferred names</span>', to designate character
encodings in protocols, data formats and APIs.
</span></p><p><a id="C022" name="C022" href="#C022"><span class="reqId">C022</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
<span class="requirement-type">[C]</span>
Character
encodings
that are not in the IANA registry <span class="rfc2119">SHOULD NOT</span> be
used, except by private agreement.
</span></p><p><a id="C023" name="C023" href="#C023"><span class="reqId">C023</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
<span class="requirement-type">[C]</span>
If
an unregistered character encoding is used, the convention of using
'<span class="qterm">x-</span>' at the beginning of the name <span class="rfc2119">MUST</span> be
followed.
</span></p><p>
<a id="C049" name="C049" href="#C049"><span class="reqId">C049</span></a> <span class="req">
<span class="requirement-type">[I]</span>
<span class="requirement-type">[C]</span>
The character encoding of content
<span class="rfc2119">SHOULD</span> be chosen so that it maximizes the opportunity to directly
represent characters (ie. minimizes the need to represent characters by
<a title="" href="#def-char-data">markup</a> means such as <a title="" href="#def-char-escape">character
escapes</a>) while avoiding obscure encodings that are unlikely to be understood by recipients.
</span>
</p><div class="note"><p><span class="note-head">NOTE: </span>Due to Unicode's large repertoire and wide base of
support, a character encoding based on Unicode
is a good choice to encode a
document.</p></div><p>
<a id="C034" name="C034" href="#C034"><span class="reqId">C034</span></a> <span class="req">
<span class="requirement-type">[C]</span>
If facilities are
offered for identifying character encoding, content MUST make use of
them; where the facilities
offered for character encoding identification include defaults (e.g. in XML 1.0
<a href="#xml10">[XML 1.0]</a>), relying on such defaults is sufficient to satisfy this
identification requirement.
</span>
</p><p><a id="C024" name="C024" href="#C024"><span class="reqId">C024</span></a> <span class="req">
<span class="requirement-type">[I]</span>
<span class="requirement-type">[C]</span>
Content and software
that label text data <span class="rfc2119">MUST</span> use one of the names required by
the appropriate specification (e.g. the XML specification when editing XML
text) and <span class="rfc2119">SHOULD</span> use the MIME preferred name of a character encoding
to label data in that character encoding.
</span></p><p><a id="C025" name="C025" href="#C025"><span class="reqId">C025</span></a> <span class="req">
<span class="requirement-type">[I]</span>
<span class="requirement-type">[C]</span>
An IANA-registered
<code class="keyword">charset</code> name <span class="rfc2119">MUST NOT</span> be used to label text data in
a character encoding other than the one identified in the IANA registration of that
name.
</span>
</p><p>
<a id="C026" name="C026" href="#C026"><span class="reqId">C026</span></a> <span class="req">
<span class="requirement-type">[S]</span>
If the unique encoding
approach is not chosen, specifications <span class="rfc2119">MUST</span> designate at
least one of the UTF-8 and UTF-16 encoding forms of Unicode as admissible
character encodings and <span class="rfc2119">SHOULD</span> choose at least one of UTF-8 or UTF-16
as required encoding forms (encoding forms that <span class="rfc2119">MUST</span> be
supported by implementations of the specification).
</span></p><p><a id="C027" name="C027" href="#C027"><span class="reqId">C027</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications that require a default encoding <span class="rfc2119">MUST</span> define either UTF-8 or UTF-16 as the default, or both if they define suitable means of distinguishing them.
</span></p><p><a id="C028" name="C028" href="#C028"><span class="reqId">C028</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications <span class="rfc2119">MUST NOT</span>
propose the use of heuristics to determine the encoding of
data.
</span>
</p><p>Examples of heuristics include the use of statistical analysis of byte
(pattern) frequencies or character (pattern) frequencies. Heuristics are bad
because they will not work consistently across different implementations.
Well-defined instructions of how to unambiguously determine a character encoding,
such as those given in XML 1.0 <a href="#xml10">[XML 1.0]</a>,
<a href="http://www.w3.org/TR/2004/REC-xml-20040204/#sec-guessing">Appendix F</a>,
are not considered heuristics.</p><p>
<a id="C029" name="C029" href="#C029"><span class="reqId">C029</span></a> <span class="req">
<span class="requirement-type">[I]</span>
<em>Receiving</em>
software <span class="rfc2119">MUST</span> determine the encoding of data from available
information according to appropriate specifications.
</span></p><p><a id="C030" name="C030" href="#C030"><span class="reqId">C030</span></a> <span class="req">
<span class="requirement-type">[I]</span>
When an IANA-registered <code class="keyword">charset</code>
name is recognized, receiving software <span class="rfc2119">MUST</span> interpret the
received data according to the encoding associated with the name in the IANA
registry.
</span></p><p><a id="C031" name="C031" href="#C031"><span class="reqId">C031</span></a> <span class="req">
<span class="requirement-type">[I]</span>
When no charset
is provided receiving software <span class="rfc2119">MUST</span> adhere to the default
character encoding(s) specified in the specification.
</span>
</p><p>Receiving software
may recognize as many character encodings and as many charset names and aliases for them as
appropriate.</p><p>A field-upgradeable mechanism may be appropriate
for this purpose. Certain character encodings are more or less associated with certain
languages (e.g. Shift_JIS with Japanese). Trying to support a given language or
set of customers may mean that certain character encodings have to be supported. However, one cannot assume universal support for a favoured but non-required encoding. The
character encodings that need to be supported may change over time. This document does
not give any advice on which character encoding may be appropriate or necessary for the
support of any given language.</p><p>Because of the layered Web architecture (e.g. formats used over
protocols), there may be multiple and at times conflicting information about
character encoding.</p><p><a id="C035" name="C035" href="#C035"><span class="reqId">C035</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications
<span class="rfc2119">MUST</span> define conflict-resolution mechanisms (e.g. priorities)
for cases where there is multiple or conflicting information about character
encoding.
</span></p><p>
<a id="C033" name="C033" href="#C033"><span class="reqId">C033</span></a> <span class="req">
<span class="requirement-type">[I]</span>
Software
<span class="rfc2119">MUST</span> completely implement the mechanisms for character
encoding identification and conflict resolution.
</span>
</p></div></div><div class="div2"> <h3><a name="sec-PrivateUse" id="sec-PrivateUse" />4.5 Private use code points</h3><p>Certain ranges of Unicode <a title="" href="#def-CCS">code points</a> are designated for private use:
the Private Use Area (PUA) (U+E000-F8FF) and planes 15 and 16 (U+F0000-FFFFD and
U+100000-10FFFD). These code points are guaranteed to never be allocated to
standard characters, and are available for use by private agreement. However, private agreements do not
scale on the Web. Code points from different private agreements may collide. Also, a private agreement, and therefore the meaning of the code points, can
quickly become lost.</p><p><a id="C073" name="C073" href="#C073"><span class="reqId">C073</span></a> <span class="req">
<span class="requirement-type">[C]</span>
Publicly interchanged content <span class="rfc2119">SHOULD NOT</span> use
codepoints in the private use area.
</span></p><div class="note"><p><span class="note-head">NOTE: </span>A typical exception would be the use of the PUA to design
and test the encoding of not yet encoded (e.g. historic or rare)
scripts.</p></div><p><a id="C076" name="C076" href="#C076"><span class="reqId">C076</span></a> <span class="req"><span class="requirement-type">[C]</span> Content <span class="rfc2119">MUST NOT</span> use a code point for any purpose other than that defined by its coded character set.</span></p><p>This prohibits, for example, the construction of fonts that misuse the codepoints in the ISO Latin 1 character set to represent different scripts, characters, or symbols than those actually encoded in iso-8859-1.</p><p><a id="C038" name="C038" href="#C038"><span class="reqId">C038</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications <span class="rfc2119">MUST NOT</span> require the use of private use area
characters with particular assignments.
</span></p><p><a id="C039" name="C039" href="#C039"><span class="reqId">C039</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications <span class="rfc2119">MUST
NOT</span> require the use of mechanisms for defining agreements of private
use code points.
</span></p><p><a id="C040" name="C040" href="#C040"><span class="reqId">C040</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
Specifications and
implementations <span class="rfc2119">SHOULD NOT</span> disallow the use of private use code points by private
agreement.
</span></p><p>As an example, XML does not disallow the use of
private use code points.</p><p>
<a id="C041" name="C041" href="#C041"><span class="reqId">C041</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications
<span class="rfc2119">MAY</span> define <a title="" href="#def-char-data">markup</a> to
allow the transmission of symbols not in Unicode or to identify specific
variants of Unicode characters.
</span>
</p><div class="example"><p><span class="example-head">EXAMPLE: </span>MathML (see <a href="#mathml2">[MathML2]</a>
<a href="http://www.w3.org/TR/2003/REC-MathML2-20031021/chapter3.html#presm.mglyph">section
3.2.9</a>) defines an element <code>mglyph</code> for mathematical symbols
not in Unicode.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>SVG (see <a href="#svg">[SVG]</a>
<a href="http://www.w3.org/TR/2003/REC-SVG11-20030114/text.html#AlternateGlyphs">section
10.14</a>) defines an element <code>altglyph</code> which allows the
identification of specific display variants of Unicode characters.</p></div><p><a id="C068" name="C068" href="#C068"><span class="reqId">C068</span></a> <span class="req"><span class="requirement-type">[S]</span> Specifications <span class="rfc2119">SHOULD</span> allow the inclusion of or reference to pictures and graphics where appropriate, to eliminate the need to (mis)use character-oriented mechanisms for pictures or graphics.</span></p></div><div class="div2"> <h3><a name="sec-Escaping" id="sec-Escaping" />4.6 Character Escaping</h3><p id="def-syntax-significant">Markup languages or programming languages often
designate certain characters as <span class="new-term">syntax-significant</span>, giving them
specific functions within the language (e.g. '<span class="qchar"><</span>' and
'<span class="qchar">&</span>' serve as markup delimiters in HTML and XML). As a
consequence, these syntax-significant characters cannot be used to represent
themselves in text in the same way as all other characters do, creating the
need for a mechanism to "<span class="quote">escape</span>" their syntax-significance. There is also a need, often satisfied by the same or similar
mechanisms, to express characters not directly representable in the
character encoding chosen for a particular document or program (an
instance of the markup or programming language).</p><p id="def-char-escape">Formally, a <span class="new-term">character
escape</span> is a syntactic device defined in a markup or programming language
that allows one or more of:
</p><ol type="1"><li><p>expressing syntax-significant characters while disregarding
their significance in the syntax of the language, or</p></li><li><p>expressing characters not representable in the character
encoding chosen for an instance of the language, or</p></li><li><p>expressing characters in general, without use of the
corresponding encoded characters.</p></li></ol><p>
</p><p>Escaping a character means expressing it using such a syntactic device,
appropriate to the format or protocol in which the character appears;
<span class="new-term">expanding a character escape</span> (or <span class="new-term">unescaping</span>) means
replacing it with the character that it represents.</p><div class="example"><p><span class="example-head">EXAMPLE: </span>HTML and XML define '<span class="qterm">Numeric Character References</span>' which allow
both the escaping of syntax-significance and the expression of arbitrary Unicode characters. Expressed
as &#x3C; or &#60; the character '<span class="qchar"><</span>' will not be parsed as
a markup delimiter.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>The programming language Java uses '<span class="qchar">"</span>' to delimit strings.
To express '<span class="qchar">"</span>' within a string, one may escape it as '<span class="qchar">\"</span>'.</p></div><div class="example"><p><span class="example-head">EXAMPLE: </span>XML defines '<span class="qterm">CDATA sections</span>' which allow escaping the
syntax-significance of all characters between the CDATA section delimiters. CDATA sections
prevent the
expression of characters using numeric character references.</p></div><p>The following guidelines apply to the way specifications define character
escapes.</p><ul><li><p id="C0000">
<a id="C042" name="C042" href="#C042"><span class="reqId">C042</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications
<span class="rfc2119">SHOULD NOT</span> invent a new escaping mechanism if an appropriate
one already exists.
</span>
</p></li><li><p>
<a id="C043" name="C043" href="#C043"><span class="reqId">C043</span></a> <span class="req">
<span class="requirement-type">[S]</span>
The number of different
ways to escape a character <span class="rfc2119">SHOULD</span> be minimized (ideally to
one).
</span></p><p>A well-known counter-example is that for historical
reasons, both HTML and XML have redundant decimal (&#ddddd;) and
hexadecimal (&#xhhhh;) character escapes.</p></li><li><p>
<a id="C044" name="C044" href="#C044"><span class="reqId">C044</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Escape syntax
<span class="rfc2119">SHOULD</span> require either explicit end delimiters or a
fixed number of characters in each character escape. Escape syntaxes where the
end is determined by any character outside the set of characters admissible in
the character escape itself <span class="rfc2119">SHOULD</span> be
avoided.
</span></p><p>These character escapes are not clear visually, and
can cause an editor to insert spurious line-breaks when word-wrapping on
spaces. Forms like SPREAD's &UABCD; <a href="#spread">[SPREAD]</a> or XML's
&#xhhhh;, where the character escape is explicitly terminated by a
semicolon, are much better. </p></li><li><p>
<a id="C045" name="C045" href="#C045"><span class="reqId">C045</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Whenever specifications
define character escapes that allow the representation of characters using a
number, the number <span class="rfc2119">MUST</span> represent the Unicode code point
of the character and <span class="rfc2119">SHOULD</span> be in hexadecimal
notation.
</span>
</p></li><li><p>
<a id="C046" name="C046" href="#C046"><span class="reqId">C046</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Escaped characters
<span class="rfc2119">SHOULD</span> be acceptable wherever their unescaped forms are; this does not preclude
that <a title="" href="#def-syntax-significant">syntax-significant</a>
characters, when escaped, lose their
significance in the syntax. In particular, if a character is
acceptable
in identifiers and comments, then its escaped form should also be
acceptable.
</span>
</p></li></ul><p>The following guidelines apply to content developers, as well as to
software that generates content:</p><ul><li><p>
<a id="C047" name="C047" href="#C047"><span class="reqId">C047</span></a> <span class="req">
<span class="requirement-type">[I]</span>
<span class="requirement-type">[C]</span>
Escapes
<span class="rfc2119">SHOULD</span> only be used when the characters to be expressed are
not directly representable in the format or the character encoding of the document, or when the visual representation of the character is unclear.
</span></p><div class="note"><p><span class="note-head">NOTE: </span>An example of when the visual representation of the character is unclear is the use of &nbsp; to distinguish a non-breaking space from a normal space.</p></div></li><li><p>
<a id="C048" name="C048" href="#C048"><span class="reqId">C048</span></a> <span class="req">
<span class="requirement-type">[I]</span>
<span class="requirement-type">[C]</span>
Content
<span class="rfc2119">SHOULD</span> use the hexadecimal form of character escapes rather than the decimal form when
there are both.
</span>
</p><div class="note"><p><span class="note-head">NOTE: </span>The hexadecimal form is preferred because character encoding standards (in particular Unicode) usually list character numbers as hexadecimal, making lookup easier.</p></div></li></ul></div></div><div class="div1"> <h2><a name="sec-Compatibility" id="sec-Compatibility" />5 Compatibility and Formatting
Characters</h2><p>This specification does not address the suitability of particular
characters for use in <a title="" href="#def-char-data">markup languages</a>,
in particular formatting characters and compatibility equivalents. For detailed
recommendations about the use of compatibility and formatting characters, see
<cite>Unicode in XML and other Markup Languages</cite>
<a href="#UXML">[UXML]</a>.</p><p>
<a id="C050" name="C050" href="#C050"><span class="reqId">C050</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications
<span class="rfc2119">SHOULD</span> exclude compatibility characters in the syntactic
elements (markup, delimiters, identifiers) of the formats they
define.
</span>
</p></div><div class="div1"> <h2><a name="sec-Indexing" id="sec-Indexing" />6 Strings</h2><div class="div2"> <h3><a name="sec-Strings" id="sec-Strings" />6.1 String concepts</h3><p>Various specifications use the notion of a '<span class="qterm">string</span>',
sometimes without defining precisely what is meant and sometimes defining it
differently from other specifications. The reason for this variability is that
there are in fact multiple reasonable definitions for a string, depending on
one's intended use of the notion; the term '<span class="qterm">string</span>' is used for
all these different notions because these are actually just different views of
the same reality: a piece of text stored inside a computer.</p><p id="def-byte-string">
<span class="new-term">Byte string</span>: A string viewed as a sequence of bytes representing characters in
a particular character encoding. This corresponds to a <a title="" href="#def-CES">character encoding scheme (CES)</a>.
Text processing of a byte string is dependent on the particular
encoding used. When the encoding changes the processing must also be changed to
reflect the stucture of the new encoding. Such a change could require significant redesign of the functions or API
used to process the byte strings as text. Therefore, this definition is only useful in specifications when the textual nature of a string is unimportant and the string is considered only as a piece of opaque data with a length in bytes (such as when copying a buffer).</p><p><a id="C011" name="C011" href="#C011"><span class="reqId">C011</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications <span class="rfc2119">SHOULD NOT</span> define a string as a '<span class="qterm">byte
string</span>'.
</span>
</p><div class="example"><p><span class="example-head">EXAMPLE: </span>This is a counter-example, illustrating one reason why considering strings as byte strings may be problematic. Consider text containing the character U+233B4 (a Chinese character meaning 'stump of tree') encoded as UTF-16 in big-endian byte order (UTF-16BE). The text will contain the bytes D8 4C DF B4. If one searches this text, considered as a byte string, for the character U+4CDF (another Chinese character meaning 'phoenix'), an erroneous match will be found on the bytes 4C DF that are the UTF-16BE representation of U+4CDF.</p></div><p id="def-physical-string">
<span class="new-term">Code unit string</span>: A string
viewed as a sequence of <a title="" href="#def-CEF">code units</a>
representing characters in a particular character encoding. This corresponds to a
<a title="" href="#def-CEF">character encoding form (CEF)</a>. A definition of a code unit string needs to include the size of the code units (e.g. 16 bits) and the character encoding used (e.g. UTF-16). Code unit strings are useful in APIs that
expose a physical representation of string data based
on reliable knowledge of the encoding forms that are likely
candidates for
implementation. Example: For the DOM
<a href="#dom1">[DOM Level 1]</a>, UTF-16 was chosen based on widespread implementation
practice. In general, 'code unit
string' is only
useful if the implementation candidates are likely to be
either UTF-16 or UTF-32.</p><p id="def-character-string">
<span class="new-term">Character string</span>: A string
viewed as a sequence of characters, each represented by a
<a title="" href="#def-CCS">code point</a> in Unicode <a href="#unicode">[Unicode]</a>.
This is usually what programmers consider to be a string, although it may not
match exactly what most users perceive as characters. This is the highest layer
of abstraction that ensures interoperability with very low implementation
effort. The 'character string'
definition of a
string is generally the most useful. Good examples using this
definition include the Production [2] of XML 1.0 <a href="#xml10">[XML 1.0]</a>, the SGML
declaration of HTML 4.0 <a href="#html401">[HTML 4.01]</a>, and the character model of RFC
2070 <a href="#rfc2070">[RFC 2070]</a>.</p><p><a id="C012" name="C012" href="#C012"><span class="reqId">C012</span></a> <span class="req">
<span class="requirement-type">[S]</span>
The '<span class="qterm">character
string</span>' definition <span class="rfc2119">SHOULD</span> be used by most specifications.
</span>
</p><div class="example"><p><span class="example-head">EXAMPLE: </span>Consider the string comprising the characters U+233B4 (a Chinese character meaning 'stump
of tree'), U+2260 <span class="uname">NOT EQUAL TO</span>, U+0071
<span class="uname">LATIN SMALL LETTER Q</span> and U+030C <span class="uname">COMBINING CARON</span>,
encoded in UTF-16 in big-endian byte order. The rows of the following table show the
string viewed as a <a title="" href="#def-character-string">character string</a>, <a title="" href="#def-physical-string">code unit string</a> and <a title="" href="#def-byte-string">byte string</a>, respectively:</p><div class="figure" align="center"><table border="1" cellpadding="5" cellspacing="0" summary="table displaying a string viewed as characters, code units and bytes"><tbody><tr align="center"><th align="right">Glyphs</th><td colspan="4">
<a href="#stumpOfTree"><img align="middle" src="images/chineseSurrogate.gif" alt="Ideographic supplementary character: Archaic Chinese character meaning "the stump of a tree" (still in current use in Cantonese)" height="25" width="24" /></a>
</td><td colspan="2">
<a href="#stumpOfTree"><img align="middle" src="images/not_equal.gif" alt="NOT EQUAL TO" height="26" width="25" /></a>
</td><td colspan="2">
<a href="#stumpOfTree"><img align="middle" src="images/Q.gif" alt="LATIN SMALL LETTER Q" height="21" width="14" /></a>
</td><td colspan="2">
<a href="#stumpOfTree"><img align="middle" src="images/caron.gif" alt="COMBINING CARON" height="21" width="14" /></a>
</td></tr><tr align="center"><th align="right">Character string</th><td colspan="4">U+233B4 </td><td colspan="2">U+2260 </td><td colspan="2">U+0071 </td><td colspan="2">U+030C </td></tr><tr align="center"><th align="right">Code unit string</th><td colspan="2">D84C</td><td colspan="2">DFB4</td><td colspan="2">2260</td><td colspan="2">0071</td><td colspan="2">030C</td></tr><tr align="center"><th align="right">Byte string</th><td>D8</td><td>4C</td><td>DF</td><td>B4</td><td>22</td><td>60</td><td>00</td><td>71</td><td>03</td><td>0C</td></tr></tbody></table></div></div><div class="note"><a name="def-grapheme-string" id="def-grapheme-string"></a><p><span class="note-head">NOTE: </span>It is also possible to view a string as a sequence of
<span class="new-term">grapheme clusters</span>. Grapheme clusters divide the text into units that
correspond more closely than <a title="" href="#def-character-string">character strings</a> to the user's perception of where character boundaries occur in a
visually rendered text. A discussion of grapheme clusters is given at the end of Section 2.10 of the Unicode Standard, Version 4
<a href="#unicode40">[Unicode 4.0]</a>; a formal definition is given in Unicode Standard Annex #29 <a href="#UTR29">[UTR #29]</a>. The Unicode Standard defines <em>default</em> grapheme clustering. Some languages require tailoring to this default. For example, a Slovak user might wish to treat the default pair of grapheme clusters "ch" as a single grapheme cluster. Note that the interaction between the language of string content and the end-user's preferences may be complex.</p></div></div><div class="div2"> <h3><a name="sec-stringIndexing" id="sec-stringIndexing" />6.2 String indexing</h3><p>There are many situations where a software process needs to access a
substring or to point within a string and does so by the use of
<span class="new-term">indices</span>, i.e. numeric "<span class="quote">positions</span>" within a string.
Where such indices are exchanged between components of the Web, there is a need
for an agreed-upon definition of string indexing in order to ensure consistent
behavior. The requirements for string indexing are discussed in
<cite>Requirements for String Identity Matching</cite>
<a href="#CharReq">[CharReq]</a>,
<a href="http://www.w3.org/TR/WD-charreq#4">section 4</a>. The two
main questions that arise are: "<span class="quote">What is the unit of counting?</span>" and
"<span class="quote">Do we start counting at 0 or 1?</span>".</p><p>The example in the previous section, <a href="#sec-Strings"><b>6.1 String concepts</b></a>, shows a
string viewed as a <a title="" href="#def-character-string">character string</a>, <a title="" href="#def-physical-string">code unit string</a> and <a title="" href="#def-byte-string">byte string</a>, respectively, each of which involves different units for indexing.</p><p>Depending on the particular requirements of a process, the unit of
counting may correspond to definitions of a string provided in section
<a href="#sec-Strings"><b>6.1 String concepts</b></a>. In particular:
</p><ul><li><p>
<a id="C051" name="C051" href="#C051"><span class="reqId">C051</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
The
<a title="" href="#def-character-string">character string</a> is
<span class="rfc2119">RECOMMENDED</span> as a basis for string indexing.
</span></p><p>(Example: the XML Path Language <a href="#xpath">[XPath]</a>).</p></li><li><p>
<a id="C052" name="C052" href="#C052"><span class="reqId">C052</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
A
<a title="" href="#def-physical-string">code unit string</a>
<span class="rfc2119">MAY</span> be used as a basis for string indexing if this results
in a significant improvement in the efficiency of internal operations when
compared to the use of <a title="" href="#def-character-string">character
string</a>.
</span></p><p>(Example: the use of UTF-16 in
<a href="#dom1">[DOM Level 1]</a>).</p></li><li><p>
<a id="C071" name="C071" href="#C071"><span class="reqId">C071</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
<a title="" href="#def-grapheme-string">Grapheme clusters</a> <span class="rfc2119">MAY</span> be used as a basis for string indexing in applications where user interaction is the primary concern.
</span></p><p>See Unicode Standard Annex #29, Text Boundaries <a href="#UTR29">[UTR #29]</a>.</p><p><a id="C074" name="C074" href="#C074"><span class="reqId">C074</span></a> <span class="req"><span class="requirement-type">[S]</span> Specifications that define indexing in terms of grapheme clusters <span class="rfc2119">MUST</span> either: a) define grapheme clusters in terms of default grapheme clusters as defined in Unicode Standard Annex #29, Text Boundaries <a href="#UTR29">[UTR #29]</a>, or b) define specifically how tailoring is applied to the indexing operation.</span></p></li><li><p>
<a id="C072" name="C072" href="#C072"><span class="reqId">C072</span></a> <span class="req">
<span class="requirement-type">[S]</span>
<span class="requirement-type">[I]</span>
The use of
<a title="" href="#def-byte-string">byte strings</a> for indexing is
<span class="rfc2119">NOT RECOMMENDED</span>.
</span></p></li></ul><p>
</p><p>It is noteworthy that there exist other, non-numeric ways of
identifying substrings which have favorable properties. For instance,
substrings based on string matching are quite robust against small edits;
substrings based on document structure (in structured formats such as XML) are
even more robust against edits and even against translation of a document from
one human language to another.</p><p><a id="C053" name="C053" href="#C053"><span class="reqId">C053</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications that need a way to identify
substrings or point within a string <span class="rfc2119">SHOULD</span> provide ways
other than string indexing to perform this operation.
</span></p><p><a id="C054" name="C054" href="#C054"><span class="reqId">C054</span></a> <span class="req">
<span class="requirement-type">[I]</span>
<span class="requirement-type">[C]</span>
Users of
specifications (software developers, content developers)
<span class="rfc2119">SHOULD</span> whenever possible prefer ways other than string
indexing to identify substrings or point within a string.
</span>
</p><p>Experience shows that more general, flexible and robust specifications
result when individual characters are understood and processed as substrings,
identified by a position before and a position after the substring.
Understanding indices as boundary positions <em>between</em> the counting
units also makes it easier to relate the indices resulting from the different
string definitions.</p><p><a id="C055" name="C055" href="#C055"><span class="reqId">C055</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications
<span class="rfc2119">SHOULD</span> understand and process single characters as
substrings, and treat indices as boundary positions <em>between</em>
counting units, regardless of the choice of counting
units.
</span>
</p><p>
<a id="C056" name="C056" href="#C056"><span class="reqId">C056</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Specifications of APIs
<span class="rfc2119">SHOULD NOT</span> specify single characters or single '<span class="qterm">units of encoding</span>'
as argument or return types.
</span>
</p><div class="example"><p><span class="example-head">EXAMPLE: </span>
The function <code>uppercase("ß")</code> cannot return the proper result (the two-character string
'<span class="qchar">SS</span>') if the return type of the <code>uppercase</code>
function is defined to be a single character. Note, also, that there is not necessarily a one-to-one mapping between characters and units of sound, input, etc. as described in <a href="#sec-Perceptions"><b>3 Perceptions of Characters</b></a>.</p></div><p>The issue of index origin, i.e. whether we count from 0 or 1, actually
arises only after a decision has been made on whether it is the units
themselves that are counted or the positions between the units.</p><p><a id="C057" name="C057" href="#C057"><span class="reqId">C057</span></a> <span class="req">
<span class="requirement-type">[S]</span>
When the positions between the units are
counted for string indexing, starting with an index of 0 for the position at
the start of the string is the <span class="rfc2119">RECOMMENDED</span> solution, with
the last index then being equal to the number of counting units in the
string.
</span>
</p></div></div><div class="div1"> <h2><a name="sec-RefUnicode" id="sec-RefUnicode" />7 Referencing the Unicode Standard and
ISO/IEC 10646</h2><p>Specifications often need to make references to
the Unicode Standard or
International Standard ISO/IEC 10646. Such references must be made with care,
especially when normative. The questions to be considered are:
</p><ul><li><p>Which standard should be referenced?</p></li><li><p>How to reference a particular version?</p></li><li><p>When to use versioned vs. unversioned references?</p></li></ul><p>
</p><p>ISO/IEC 10646 is developed and published jointly by
<a href="http://www.iso.org/iso/en/ISOOnline.openerpage">ISO</a> (the International
Organization for Standardization)
and
<a href="http://www.iec.ch/">IEC</a> (the International
Electrotechnical Commission). The Unicode Standard is developed and published
by the
<a href="http://www.unicode.org/">Unicode Consortium</a>, an
organization of major computer corporations, software producers, database
vendors, national governments, research institutions, international agencies,
various user groups, and interested individuals. The Unicode Standard is
comparable in standing to W3C Recommendations.</p><p>ISO/IEC 10646 and the Unicode Standard define exactly the same
<a title="" href="#def-CCS">coded character set (CCS)</a> (same <a title="" href="#def-repertoire">repertoire</a>, same <a title="" href="#def-CCS">code
points</a>) and encoding forms. They are actively maintained in synchrony
by liaisons and overlapping membership between the respective technical
committees. In addition to the jointly defined CCS and encoding forms, the Unicode Standard adds normative and informative lists of character properties,
normative character equivalence and normalization specifications, a normative
algorithm for bidirectional text and a large amount of useful implementation
information. In short, the Unicode Standard adds
semantics to the characters that ISO/IEC 10646 merely enumerates. Conformance
to the Unicode Standard implies conformance to ISO/IEC
10646, see <a href="#unicode40">[Unicode 4.0]</a> Appendix C.</p><p>
<a id="C062" name="C062" href="#C062"><span class="reqId">C062</span></a> <span class="req">
<span class="requirement-type">[S]</span>
Since specifications in general
need both a definition for their characters and the semantics associated with
these characters, specifications <span class="rfc2119">SHOULD</span> include a reference
to the Unicode Standard, whether or not they include a
reference to ISO/IEC 10646.
</span></p><p>By providing a reference to the Unicode Standard implementers can benefit from the wealth of information
provided in the standard and on the Unicode Consortium Web site.</p><p>The fact that both ISO/IEC 10646 and the Unicode Standard are evolving (in
synchrony) raises the issue of versioning: should a specification refer to a
specific version of the standard, or should it make a generic reference, so
that the normative reference is to the version current at the time of
<em>reading</em> the specification? In general the answer is
<em>both</em>.</p><p><a id="C063" name="C063" href="#C063"><span class="reqId">C063</span></a> <span class="req">
<span class="requirement-type">[S]</span>
A generic reference to
the Unicode Standard <span class="rfc2119">MUST</span> be made if
it is desired that characters allocated after a specification is published are
usable with that specification. A specific reference to
the Unicode Standard <span class="rfc2119">MAY</span> be included
to ensure that functionality depending on a particular version is available and
will not change over time.
</span></p><p>An example would be the set of characters acceptable
as Name characters in XML 1.0 <a href="#xml10">[XML 1.0]</a>, which is an enumerated
list that parsers must implement to validate names.</p><div class="note"><p><span class="note-head">NOTE: </span>See<a href="http://www.unicode.org/unicode/standard/versions/#Citations">
http://www.unicode.org/unicode/standard/versions/#Citations</a> for guidance
on referring to specific versions of the Unicode
Standard.</p></div><p>A generic reference can be formulated in two ways:
</p><ol type="1"><li><p>By explicitly including a <em>generic</em> entry in the
bibliography section of a specification and simply referring to that entry in
the body of the specification. Such a generic entry contains text such as
"<span class="quote">... as it may from time to time be revised or amended</span>".</p></li><li><p>By including a <em>specific</em> entry in the bibliography
and adding text such as "<span class="quote">... as it may from time to time be revised or
amended</span>" at the point of reference in the body of the specification.</p></li></ol><p>
</p><p>It is an editorial matter, best left to each specification, which of
these two formulations is used. Examples of the first formulation can be found
in the bibliography of this specification (see the entries for
<a href="#iso10646">[ISO/IEC 10646]</a> and <a href="#unicode">[Unicode]</a>). Examples of the latter,
as well as a discussion of the versioning issue with respect to MIME
<code class="keyword">charset</code> parameters for UCS encodings, can be found in
<a href="#rfc3629">[RFC 3629]</a> and <a href="#rfc2781">[RFC 2781]</a>.</p><p>
<a id="C064" name="C064" href="#C064"><span class="reqId">C064</span></a> <span class="req">
<span class="requirement-type">[S]</span>
All <em>generic</em>
references to the Unicode Standard <a href="#unicode">[Unicode]</a>
<span class="rfc2119">MUST</span> refer to the latest version of the Unicode Standard available at the date of publication of the containing specification.
</span></p><p><a id="C065" name="C065" href="#C065"><span class="reqId">C065</span></a> <span class="req">
<span class="requirement-type">[S]</span>
All <em>generic</em> references to ISO/IEC 10646
<a href="#iso10646">[ISO/IEC 10646]</a>
<span class="rfc2119">MUST</span> refer to the latest version of ISO/IEC 10646 available at the date of publication of the containing specification.
</span>
</p></div></div><div class="back"><div class="div1"> <h2><a name="sec-References" id="sec-References" />A References</h2><div class="div2"> <h3><a name="sec-NormativeReferences" id="sec-NormativeReferences" />A.1 Normative
References</h3><dl><dt class="label"><a name="iana" id="iana" />IANA</dt><dd>Internet Assigned Numbers Authority,
<a href="http://www.iana.org/assignments/character-sets"><cite>Official Names
for Character Sets</cite></a>. (See
<a href="http://www.iana.org/assignments/character-sets">http://www.iana.org/assignments/character-sets</a>.)
</dd><dt class="label"><a name="iso10646" id="iso10646" />ISO/IEC 10646</dt><dd>ISO/IEC 10646:2003,
<a href="http://www.iso.ch/iso/en/CatalogueDetailPage.CatalogueDetail?CSNUMBER=39921"><cite>Information
technology -- Universal Multiple-Octet Coded Character Set (UCS)</cite></a>, as, from time to time, amended, replaced by a
new edition or expanded by the addition of new parts. (See
<a href="http://www.iso.org/iso/en/ISOOnline.openerpage">http://www.iso.org/iso/en/ISOOnline.openerpage</a> for the
latest version.)</dd><dt class="label"><a name="MIME-entity" id="MIME-entity" />MIME-entity</dt><dd>
N.
Freed, N. Borenstein, <a href="http://www.ietf.org/rfc/rfc2045.txt"><cite>Multipurpose Internet Mail
Extensions (MIME). Part One: Format of Internet Message Bodies</cite></a>, RFC 2045, November 1996,
<a href="http://www.ietf.org/rfc/rfc2045.txt">http://www.ietf.org/rfc/rfc2045.txt</a>.</dd><dt class="label"><a name="MIME-charset" id="MIME-charset" />MIME-charset</dt><dd><a href="http://www.ietf.org/rfc/rfc2045.txt"><cite>Multipurpose Internet Mail
Extensions (MIME). Part Two: Media Types</cite></a>, N. Freed, N. Borenstein, RFC 2046,
November 1996, <a href="http://www.ietf.org/rfc/rfc2046.txt">http://www.ietf.org/rfc/rfc2046.txt</a>.</dd><dt class="label"><a name="rfc2119" id="rfc2119" />RFC 2119</dt><dd>S. Bradner,
<a href="http://www.ietf.org/rfc/rfc2119.txt"><cite>Key words for use in RFCs
to Indicate Requirement Levels</cite></a>, IETF RFC 2119. (See
<a href="http://www.ietf.org/rfc/rfc2119.txt">http://www.ietf.org/rfc/rfc2119.txt</a>.)
</dd><dt class="label"><a name="unicode" id="unicode" />Unicode</dt><dd>The Unicode Consortium,
<a href="http://www.unicode.org/unicode/standard/versions/"><cite>The Unicode Standard, Version 4</cite></a>, ISBN 0-321-18578-1, as
updated from time to time by the publication of new versions. (See
<a href="http://www.unicode.org/unicode/standard/versions/">http://www.unicode.org/unicode/standard/versions</a>
for the latest version and additional information on versions of the standard
and of the Unicode Character Database).</dd><dt class="label"><a name="unicode32" id="unicode32" />Unicode 3.2</dt><dd>The Unicode Consortium,
<a href="http://www.unicode.org/unicode/standard/versions/enumeratedversions.html#Unicode_3_2_0"><cite>The Unicode Standard, Version 3.2.0</cite></a> is defined by
<a href="http://www.unicode.org/unicode/standard/versions/enumeratedversions.html#Unicode_3_0_0"><cite>The Unicode Standard, Version 3.0</cite></a> (Reading, MA,
Addison-Wesley, 2000. ISBN 0-201-61633-5), as amended by the <a href="http://www.unicode.org/reports/tr27/"><cite>Unicode
Standard Annex #27: Unicode 3.1</cite></a> (see
<a href="http://www.unicode.org/reports/tr27/">http://www.unicode.org/reports/tr27</a>)
and by the <a href="http://www.unicode.org/reports/tr28/"><cite>Unicode Standard Annex #28: Unicode 3.2</cite></a> (see
<a href="http://www.unicode.org/reports/tr28/">http://www.unicode.org/reports/tr28</a>).</dd><dt class="label"><a name="unicode40" id="unicode40" />Unicode 4.0</dt><dd>The Unicode Consortium. <a href="http://www.unicode.org/versions/Unicode4.0.0/"><cite>The Unicode Standard, Version 4.0</cite></a>, Reading, MA, Addison-Wesley, 2003. ISBN 0-321-18578-1. (See <a href="http://www.unicode.org/versions/Unicode4.0.0/">http://www.unicode.org/versions/Unicode4.0.0/</a>)</dd></dl></div><div class="div2"> <h3><a name="sec-OtherReferences" id="sec-OtherReferences" />A.2 Other References</h3><dl><dt class="label"><a name="charnorm" id="charnorm" />CharNorm</dt><dd>Martin J. Dürst,
François Yergeau, Richard Ishida, Misha Wolf, Tex Texin, Addison Phillips
<a href="http://www.w3.org/TR/charmod-norm/"><cite>Character Model for the World Wide Web 1.0: Normalization</cite></a>,
W3C Working Draft. (See
<a href="http://www.w3.org/TR/charmod-norm/">http://www.w3.org/TR/charmod-norm</a>.)
</dd><dt class="label"><a name="charmod3" id="charmod3" />CharIRI</dt><dd>Martin J. Dürst,
François Yergeau, Richard Ishida, Misha Wolf, Tex Texin,
<a href="http://www.w3.org/TR/charmod-resid/"><cite>Character Model for the World Wide Web 1.0: Resource Identifiers</cite></a>,
W3C Candidate Recommendation. (See
<a href="http://www.w3.org/TR/charmod-resid/">http://www.w3.org/TR/charmod-resid</a>.)
</dd><dt class="label"><a name="CharReq" id="CharReq" />CharReq</dt><dd>Martin J. Dürst,
<a href="http://www.w3.org/TR/WD-charreq"><cite>Requirements for String
Identity Matching and String Indexing</cite></a>, W3C Working Draft. (See
<a href="http://www.w3.org/TR/WD-charreq">http://www.w3.org/TR/WD-charreq</a>.)
</dd><dt class="label"><a name="connolly" id="connolly" />Connolly</dt><dd>D. Connolly,
<a href="http://www.w3.org/MarkUp/html-spec/charset-harmful"><cite>Character
Set Considered Harmful</cite></a>, W3C Note. (See
<a href="http://www.w3.org/MarkUp/html-spec/charset-harmful">http://www.w3.org/MarkUp/html-spec/charset-harmful</a>.)</dd><dt class="label"><a name="css2" id="css2" />CSS2</dt><dd>Bert Bos, Håkon Wium Lie, Chris Lilley,
Ian Jacobs, Eds., <a href="http://www.w3.org/TR/REC-CSS2/"><cite>Cascading
Style Sheets, level 2</cite></a> (CSS2 Specification), W3C Recommendation. (See
<a href="http://www.w3.org/TR/REC-CSS2/">http://www.w3.org/TR/REC-CSS2</a>.)
</dd><dt class="label"><a name="dom1" id="dom1" />DOM Level 1</dt><dd>Vidur Apparao et al.,
<a href="http://www.w3.org/TR/REC-DOM-Level-1/"><cite>Document Object Model
(DOM) Level 1 Specification</cite></a>, W3C Recommendation. (See
<a href="http://www.w3.org/TR/REC-DOM-Level-1/">http://www.w3.org/TR/REC-DOM-Level-1</a>.)
</dd><dt class="label"><a name="html40" id="html40" />HTML 4.0</dt><dd>Dave Raggett, Arnaud Le Hors, Ian
Jacobs, Eds., <a href="http://www.w3.org/TR/REC-html40-971218/"><cite>HTML 4.0
Specification</cite></a>, W3C Recommendation, 18-Dec-1997 (See
<a href="http://www.w3.org/TR/REC-html40-971218/">http://www.w3.org/TR/REC-html40-971218</a>.)</dd><dt class="label"><a name="html401" id="html401" />HTML 4.01</dt><dd>Dave Raggett, Arnaud Le Hors, Ian
Jacobs, Eds., <a href="http://www.w3.org/TR/html401/"><cite>HTML 4.01
Specification</cite></a>, W3C Recommendation. (See
<a href="http://www.w3.org/TR/html401/">http://www.w3.org/TR/html401</a>.)
</dd><dt class="label"><a name="iso646" id="iso646" />ISO/IEC 646</dt><dd>ISO/IEC 646:1991, <cite>Information technology -- ISO 7-bit coded character set for information interchange</cite>. This standard defines an International Reference Version (IRV) which corresponds exactly to what is widely known as ASCII or US-ASCII. ISO/IEC 646 was based on the earlier standard ECMA-6. ECMA has maintained its standard up to date with respect to ISO/IEC 646 and makes an electronic copy available at <a href="http://www.ecma-international.org/publications/standards/Ecma-006.htm">http://www.ecma-international.org/publications/standards/Ecma-006.htm</a>
</dd><dt class="label"><a name="iso9541" id="iso9541" />ISO/IEC 9541-1</dt><dd>ISO/IEC 9541-1:1991,
<a href="http://www.iso.ch/iso/en/CatalogueDetailPage.CatalogueDetail?CSNUMBER=17277"><cite>Information
technology -- Font information interchange -- Part 1: Architecture</cite></a>. (See
<a href="http://www.iso.ch/iso/en/CatalogueDetailPage.CatalogueDetail?CSNUMBER=17277">http://www.iso.ch/iso/en/CatalogueDetailPage.CatalogueDetail?CSNUMBER=17277</a>
for the latest version.) </dd><dt class="label"><a name="iso14651" id="iso14651" />ISO/IEC 14651</dt><dd>ISO/IEC 14651:2000,
<a href="http://www.iso.org/iso/en/ISOOnline.openerpage"><cite>Information technology --
International string ordering and comparison -- Method for comparing character
strings and description of the common template tailorable ordering</cite></a> as,
from time to time, amended, replaced by a new edition or expanded by the
addition of new parts. (See
<a href="http://www.iso.org/iso/en/ISOOnline.openerpage">http://www.iso.org/iso/en/ISOOnline.openerpage</a> for the
latest version.)</dd><dt class="label"><a name="mathml2" id="mathml2" />MathML2</dt><dd>David Carlisle, Patrick Ion, Robert
Miner, Nico Poppelier, Eds., <a href="http://www.w3.org/TR/MathML2/"><cite>Mathematical Markup Language (MathML)
Version 2.0</cite></a>, W3C Recommendation. (See
<a href="http://www.w3.org/TR/MathML2/">http://www.w3.org/TR/MathML2</a>.)
</dd><dt class="label"><a name="Nicol" id="Nicol" />Nicol</dt><dd>Gavin Nicol,
<a href="http://www.mind-to-mind.com/library/papers/multilingual/multilingual-www.html"><cite>The
Multilingual World Wide Web</cite></a>, Chapter 2: The WWW As A Multilingual
Application. (See
<a href="http://www.mind-to-mind.com/library/papers/multilingual/multilingual-www.html">http://www.mind-to-mind.com/library/papers/multilingual/multilingual-www.html</a>.)
</dd><dt class="label"><a name="rfc2070" id="rfc2070" />RFC 2070</dt><dd>F. Yergeau, G. Nicol, G. Adams, M.
Dürst, <a href="http://www.ietf.org/rfc/rfc2070.txt"><cite>Internationalization of the
Hypertext Markup Language</cite></a>, IETF RFC 2070, January 1997. (See
<a href="http://www.ietf.org/rfc/rfc2070.txt">http://www.ietf.org/rfc/rfc2070.txt</a>.)</dd><dt class="label"><a name="rfc2277" id="rfc2277" />RFC 2277</dt><dd>H. Alvestrand,
<a href="http://www.ietf.org/rfc/rfc2277.txt"><cite>IETF Policy on Character
Sets and Languages</cite></a>, IETF RFC 2277, BCP 18, January 1998. (See
<a href="http://www.ietf.org/rfc/rfc2277.txt">http://www.ietf.org/rfc/rfc2277.txt</a>.)
</dd><dt class="label"><a name="RFC2978" id="RFC2978" />RFC 2978</dt><dd>N. Freed, J. Postel,
<a href="http://www.ietf.org/rfc/rfc3629.txt"><cite>IANA Charset Registration Procedures</cite></a>, IETF RFC 2978, BCP 19, October 2000. (See
<a href="http://www.ietf.org/rfc/rfc2978.txt">http://www.ietf.org/rfc/rfc2978.txt</a>.)
</dd><dt class="label"><a name="rfc3629" id="rfc3629" />RFC 3629</dt><dd>F. Yergeau,
<a href="http://www.ietf.org/rfc/rfc3629.txt"><cite>UTF-8, a transformation
format of ISO 10646</cite></a>, IETF RFC 3629, STD 63, November 2003. (See
<a href="http://www.ietf.org/rfc/rfc3629.txt">http://www.ietf.org/rfc/rfc3629.txt</a>.)
</dd><dt class="label"><a name="rfc2781" id="rfc2781" />RFC 2781</dt><dd>P. Hoffman, F. Yergeau,
<a href="http://www.ietf.org/rfc/rfc2781.txt"><cite>UTF-16, an encoding of ISO
10646</cite></a>, IETF RFC 2781, February 2000. (See
<a href="http://www.ietf.org/rfc/rfc2781.txt">http://www.ietf.org/rfc/rfc2781.txt</a>.)</dd><dt class="label"><a name="spread" id="spread" />SPREAD</dt><dd>
<a href="http://xml.ascc.net/resource/entities/index.html"><cite>SPREAD -
Standardization Project for East Asian Documents Universal Public Entity
Set</cite></a>. (See
<a href="http://xml.ascc.net/resource/entities/index.html">http://www.ascc.net/xml/resource/entities/index.html</a>)
</dd><dt class="label"><a name="svg" id="svg" />SVG</dt><dd>Jon Ferraiolo, 藤沢 淳 (FUJISAWA Jun), Dean Jackson, Ed.,
<a href="http://www.w3.org/TR/SVG/"><cite>Scalable Vector Graphics (SVG) 1.1
Specification</cite></a>, W3C Recommendation. (See
<a href="http://www.w3.org/TR/SVG/">http://www.w3.org/TR/SVG</a>.) </dd><dt class="label"><a name="UTR10" id="UTR10" />UTR #10</dt><dd>Mark Davis, Ken Whistler,
<a href="http://www.unicode.org/unicode/reports/tr10/"><cite>Unicode Collation
Algorithm</cite></a>, Unicode Technical Report #10. (See
<a href="http://www.unicode.org/unicode/reports/tr10/">http://www.unicode.org/unicode/reports/tr10</a>.)
</dd><dt class="label"><a name="UTR17" id="UTR17" />UTR #17</dt><dd>Ken Whistler, Mark Davis,
<a href="http://www.unicode.org/unicode/reports/tr17/"><cite>Character
Encoding Model</cite></a>, Unicode Technical Report #17. (See
<a href="http://www.unicode.org/unicode/reports/tr17/">http://www.unicode.org/unicode/reports/tr17</a>.)
</dd><dt class="label"><a name="UTR29" id="UTR29" />UTR #29</dt><dd>Mark Davis,
<a href="http://www.unicode.org/unicode/reports/tr29/"><cite>Text Boundaries</cite></a>, Unicode Standard Annex #29. (See
<a href="http://www.unicode.org/unicode/reports/tr29/">http://www.unicode.org/unicode/reports/tr29</a>
for the latest version).</dd><dt class="label"><a name="UXML" id="UXML" />UXML</dt><dd>Martin Dürst and Asmus Freytag,
<a href="http://www.w3.org/TR/unicode-xml/"><cite>Unicode in XML and other
Markup Languages</cite></a>, Unicode Technical Report #20 and W3C Note. (See
<a href="http://www.w3.org/TR/unicode-xml/">http://www.w3.org/TR/unicode-xml</a>.)</dd><dt class="label"><a name="xml10" id="xml10" />XML 1.0</dt><dd>Tim Bray, Jean Paoli, C. M.
Sperberg-McQueen, Eve Maler, François Yergeau, Eds.,
<a href="http://www.w3.org/TR/REC-xml/"><cite>Extensible Markup Language (XML)
1.0</cite></a>, W3C Recommendation. (See
<a href="http://www.w3.org/TR/REC-xml/">http://www.w3.org/TR/REC-xml</a>.)
</dd><dt class="label"><a name="XML_Japanese_profile" id="XML_Japanese_profile" />XML Japanese Profile</dt><dd>MURATA
Makoto Ed., <a href="http://www.w3.org/TR/japanese-xml/"><cite>XML Japanese
Profile</cite></a>, W3C Note. (See
<a href="http://www.w3.org/TR/japanese-xml/">http://www.w3.org/TR/japanese-xml</a>.)
</dd><dt class="label"><a name="xpath" id="xpath" />XPath</dt><dd>James Clark, Steve DeRose, Eds,
<a href="http://www.w3.org/TR/xpath"><cite>XML Path Language (XPath) Version
1.0</cite></a>, W3C Recommendation. (See
<a href="http://www.w3.org/TR/xpath">http://www.w3.org/TR/xpath</a>.)</dd></dl></div></div><div class="div1"> <h2><a name="sec-CharExamples" id="sec-CharExamples" />B Examples of Characters, Keystrokes
and Glyphs (Non-Normative)</h2><p id="exampleA6">A few examples will help make sense all this complexity
of text in computers (which is mostly a reflection of the complexity of human
writing systems). Let us start with a very simple example: a user, equipped
with a US-English keyboard, types "<span class="quote">Foo</span>", which the computer
encodes as 16-bit values (the UTF-16 encoding of Unicode) and displays on the
screen.</p><div class="figure" align="center"><table border="1" cellpadding="5" cellspacing="0" summary="Table showing keystrokes, input characters, encoded characters and display for user typing Foo on a U.S. keyboard"><tbody><tr><th align="right">Keystrokes</th><td align="center">Shift-f</td><td align="center">o</td><td align="center">o</td></tr><tr><th align="right">Input characters</th><td align="center">F</td><td align="center">o</td><td align="center">o</td></tr><tr><th align="right">Encoded characters (byte values
in hex)</th><td align="center">0046</td><td align="center">006F</td><td align="center">006F</td></tr><tr><th align="right">Display</th><td colspan="3" align="center">Foo</td></tr></tbody></table><div class="caption">Example: Basic Latin</div></div><p>The only complexity here is the use of a modifier (Shift) to input the
capital '<span class="qchar">F</span>'.</p><p>A slightly more complex example is a user typing '<span class="qchar">çé</span>' on
a traditional French-Canadian keyboard, which the computer again encodes in
UTF-16 and displays. We assume that this particular computer uses a fully
composed form of UTF-16.</p><div class="figure" align="center"><table border="1" cellpadding="5" cellspacing="0" summary="Table showing keystrokes, input characters, encoded characters and display for user typing çé on a French-Canadian keyboard"><tbody><tr><th align="right">Keystrokes</th><td align="center"> ¸
</td><td align="center">c</td><td align="center">é</td></tr><tr><th align="right">Input characters</th><td colspan="2" align="center">ç</td><td align="center">é</td></tr><tr><th align="right">Encoded characters (byte values
in hex)</th><td colspan="2" align="center">00E7</td><td align="center">00E9</td></tr><tr><th align="right">Display</th><td colspan="3" align="center">çé</td></tr></tbody></table><div class="caption">Example: Latin with diacritics</div></div><p>A few interesting things are happening here: when the user types the
cedilla ('<span class="qchar">¸</span>'), nothing happens except for a change of state of the
keyboard driver; the cedilla is a <span class="new-term">dead key</span>. When the driver gets
the c keystroke, it provides a complete '<span class="qchar">ç</span>' character to the
system, which represents it as a single 16-bit <a title="" href="#def-CEF">code
unit</a> and displays a '<span class="qchar">ç</span>'
<a title="" href="#def-glyph">glyph</a>. The user then presses the dedicated
'<span class="qchar">é</span>' key, which results in, again, a character represented by two
bytes. Most systems will display this as one glyph, but it is also possible to
combine two glyphs (the base letter and the accent) to obtain the same
rendering.</p><p>On to a Japanese example: our user employs a <span class="new-term">romaji input
method</span> to type '<span class="qterm">日本語</span>' (U+65E5, U+672C, U+8A9E), which the computer encodes in UTF-16 and
displays.</p><div class="figure" align="center"><table border="1" cellpadding="5" cellspacing="0" summary="Table showing keystrokes, input characters, encoded characters and display for user typing nihongo in a Japanese Romaji input method"><tbody><tr><th align="right">Keystrokes</th><td align="center" colspan="4"> n i h o n g o <space> <return></td></tr><tr><th align="right">Input characters</th><td colspan="2" align="center">
日
</td><td align="center">本</td><td align="center">語</td></tr><tr><th align="right">Encoded characters (byte values in hex)</th><td colspan="2" align="center">65E5</td><td align="center">672C</td><td align="center">8A9E</td></tr><tr><th align="right">Display</th><td colspan="4" align="center">
<a href="#nihongo"><img align="middle" src="images/nihongo.gif" alt="Three Kanji characters, U+65E5, U+672C, U+8A9E, pronounced
 'nihongo'." height="16" width="47" /></a>
</td></tr></tbody></table><div class="caption">Example: Japanese</div></div><p>The interesting aspect here is input: the user types Latin characters,
which are converted on the fly to kana (not shown here), and then to kanji when
the user requests conversion by pressing <space>; the kanji characters
are finally sent to the application when the user presses <return>. The
user has to type a total of nine keystrokes before the three characters are
produced, which are then encoded and displayed rather trivially.</p><p>A Persian example, using Arabic script, will show different phenomena:</p><div class="figure" align="center"><table border="1" cellpadding="5" cellspacing="0" summary="Table showing keystrokes, input characters, encoded characters and display for user typing on an Arabic keyboard"><tbody><tr><th align="right">Keystrokes</th><td align="center" colspan="2">
<img align="middle" src="images/fa-lam.gif" alt="ARABIC LETTER LAM" height="23" width="16" />
</td><td align="center">
<img align="middle" src="images/fa-alif.gif" alt="ARABIC LETTER
 ALEF" height="23" width="7" />
</td><td align="center" colspan="2">
<img align="middle" src="images/fa-lamalif.gif" alt="Arabic ligature
 'lam-alef'." height="23" width="16" />
</td><td align="center">
<img align="middle" src="images/fa-yeh.gif" alt="ARABIC LETTER FARSI YEH" height="23" width="20" />
</td><td align="center">
<img align="middle" src="images/fa-yeh.gif" alt="ARABIC LETTER FARSI YEH" height="23" width="20" />
</td></tr><tr><th align="right">Input characters</th><td colspan="2" align="center">ل</td><td align="center">ا</td><td align="center">ل</td><td align="center">ا</td><td align="center">ی</td><td align="center">ی</td></tr><tr><th align="right">Encoded characters (byte values in hex)</th><td colspan="2" align="center">0644</td><td align="center">0627</td><td align="center">0644</td><td align="center">0627</td><td align="center">06CC</td><td align="center">06CC</td></tr><tr><th align="right">Display</th><td colspan="7" align="center">
<a href="#arabicLigature"><img align="middle" src="images/fa-laalaayee.gif" alt="The displayed output appears, from right to left, as: two lam-alef ligatures, and initial farsi yeh glyph attached to a final farsi yeh glyph." height="23" width="53" /></a>
</td></tr></tbody></table><div class="caption">Example: Persian</div></div><p>Here the first two keystrokes each produce an input character and an
encoded character, but the pair is displayed as a single glyph
('<img align="middle" src="images/fa-lamalif.gif" alt="Arabic ligature 'lam-alef'." height="23" width="16" />', a lam-alef ligature). The next
keystroke is a lam-alef, which some Arabic script keyboards have; it produces the same
two characters which are displayed similarly, but this second lam-alef is
placed to the <em>left</em> of the first one when displayed. The last two
keystrokes produce two identical characters which are rendered by two different
glyphs (a medial form followed to its left by a final form). We thus have 5
keystrokes producing 6 characters and 4 glyphs laid out right-to-left.</p><p id="sec-CharExamplesA5">A final example in Tamil, typed with an ISCII
keyboard, will illustrate some additional phenomena:</p><div class="figure" align="center"><table border="1" cellpadding="5" cellspacing="0" summary="Table showing keystrokes, input characters, encoded characters and display for user typing on a Tamil ISCII keyboard"><tbody><tr><th align="right">Keystrokes</th><td align="center" colspan="2">
<img align="middle" src="images/ta-tm.gif" alt="TAMIL LETTER TTA" height="14" width="22" />
</td><td align="center">
<img align="middle" src="images/a-tm.gif" alt="TAMIL VOWEL SIGN AA" height="15" width="26" />
</td><td align="center">
<img align="middle" src="images/nga-tm.gif" alt="TAMIL LETTER NGA" height="14" width="21" />
</td><td align="center">
<img align="middle" src="images/virama-tm.gif" alt="TAMIL SIGN VIRAMA" height="16" width="17" />
</td><td align="center">
<img align="middle" src="images/ka-tm.gif" alt="TAMIL LETTER KA" height="14" width="17" />
</td><td align="center">
<img align="middle" src="images/o-tm.gif" alt="TAMIL VOWEL SIGN OO" height="17" width="40" />
</td></tr><tr><th align="right">Input characters</th><td colspan="2" align="center">ட</td><td align="center"> ா</td><td align="center">ங</td><td align="center"> ்</td><td align="center">க</td><td align="center"> ோ</td></tr><tr><th align="right">Encoded characters (byte values in hex)</th><td colspan="2" align="center">0B9F</td><td align="center">0BBE</td><td align="center">0B99</td><td align="center">0BCD</td><td align="center">0B95</td><td align="center">0BCB</td></tr><tr><th align="right">Display</th><td colspan="7" align="center">
<a href="#tamil"><img align="middle" src="images/tango.gif" alt="'Tango' in Tamil letters." height="17" width="82" /></a>
</td></tr></tbody></table><div class="caption">Example: Tamil</div></div><p>Here input is straightforward, but note that contrary to the preceding
accented Latin example, the virama diacritic '<span class="qchar"> ்</span>' (U+0BCD) is entered
<em>after</em> the '<span class="qchar">ங</span>' (U+0B99) to which it applies. Rendering
is interesting for the last two characters. The last one '<span class="qchar"> ோ</span>' (U+0BCB) clearly consists of two glyphs which <em>surround</em>
the glyph of the next to last character '<span class="qchar">க</span>' (U+0B95).</p></div><div class="div1"> <h2><a name="sec-ExampleText" id="sec-ExampleText" />C Example text (Non-Normative)</h2><p>The following are textual versions of strings or characters used in image-based examples in this document. They are provided here for the benefit of those who want to cut and paste the text for their own testing.</p><ol type="1"><li><p id="arabic-select">Section: <a href="#sec-VisualRenderingUnits"><b>3.3 Units of visual
rendering</b></a></p><p>Example: <img align="middle" src="images/logSelMemory.gif" alt="An example
 showing the logical order of characters in a string containing two Arabic words
 followed by a year number. In logical selection mode, the range of characters
 selected by starting the selection in the middle of the second word and ending
 in the middle of the year number is depicted using highlighting. The
 highlighting covers a single block of contiguous characters." height="27" width="323" /></p><p>Text: <span>عدد مارس ١٩٩٨</span></p></li><li><p id="stumpOfTree">Section: <a href="#sec-Strings"><b>6.1 String concepts</b></a></p><p>Example: <img align="middle" src="images/chineseSurrogate.gif" alt="Ideographic supplementary character: Archaic Chinese character meaning "the stump of a tree" (still in current use in Cantonese)" height="25" width="24" /><img align="middle" src="images/not_equal.gif" alt="NOT EQUAL TO" height="26" width="25" /><img align="middle" src="images/Q.gif" alt="LATIN SMALL LETTER Q" height="21" width="14" /><img align="middle" src="images/caron.gif" alt="COMBINING CARON" height="21" width="14" /></p><p>Text: <span>𣎴≠q̌</span></p></li><li><p id="nihongo">Section: <a href="#sec-CharExamples"><b>B Examples of Characters, Keystrokes
and Glyphs</b></a></p><p>Example: <img align="middle" src="images/nihongo.gif" alt="Three Kanji characters, U+65E5, U+672C, U+8A9E, pronounced
 'nihongo'." height="16" width="47" /></p><p>Text: <span>日本語</span></p></li><li><p id="arabicLigature">Section: <a href="#sec-CharExamples"><b>B Examples of Characters, Keystrokes
and Glyphs</b></a></p><p>Example: <img align="middle" src="images/arabe.gif" alt="The displayed output appears, from right to left, as: two lam-alef ligatures, and initial ghayn glyph attached to a final ghayn glyph." height="26" width="42" /></p><p>Text: <span>لالاغغ</span></p></li><li><p id="tamil">Section: <a href="#sec-CharExamples"><b>B Examples of Characters, Keystrokes
and Glyphs</b></a></p><p>Example: <img align="middle" src="images/tango.gif" alt="'Tango' in Tamil letters." height="17" width="82" /></p><p>Text: <span>டாங்கோ</span></p></li></ol></div><div class="div1"> <h2><a name="sec-Checklist" id="sec-Checklist" />D List of conformance criteria (Non-Normative)</h2><p>This is a list of the conformance criteria in this specification, in document order. This list can be used to check specifications, implementations, and content for conformance to this specification.</p><p>When doing so, the following points should be kept in mind:</p><ul><li><p>To ensure that you understand the meaning, read the whole document first. Use this list as a quick reference only after having first read the conformance criteria in context in the main body of the text.</p></li><li><p>If the meaning of a conformance criterion in this list is still unclear after referring back to the surrounding text in the main body of the document, consider sending a comment to www-i18n-comments@w3.org (<a href="http://lists.w3.org/Archives/Public/www-i18n-comments/">publicly archived</a>).</p></li><li><p>Not all conformance criteria apply to all specifications, implementations,
or content. Before checking for actual conformance, applicability should
be checked. As an example, C010 only applies to specifications.
As another example, C002 applies to specifications, implementations,
and content, but only if it deals with mapping between characters and
units of displayed text.</p></li></ul><table id="req-checklist"><tbody><tr><td class="reqId" style="vertical-align: top;"><a href="#C001">C001</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> <span class="requirement-type">[C]</span> </td><td>Specifications,
software and content <span class="rfc2119">MUST NOT</span> require or depend on a one-to-one
correspondence between characters and the sounds of a
language.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C002">C002</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> <span class="requirement-type">[C]</span> </td><td>Specifications,
software and content <span class="rfc2119">MUST NOT</span> require or depend on a one-to-one mapping between
characters and units of displayed text.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C003">C003</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> <span class="requirement-type">[C]</span> </td><td>Protocols,
data formats and APIs <span class="rfc2119">MUST</span> store, interchange or process
text data in logical order.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C075">C075</a></td><td style="vertical-align: top;"><span class="requirement-type">[I]</span> </td><td>Independent of whether some implementation uses logical selection or visual selection, characters selected <span class="rfc2119">MUST</span> be kept in logical order in storage.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C004">C004</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications of protocols
and APIs that involve selection of ranges <span class="rfc2119">SHOULD</span> provide for
discontiguous logical selections, at least to the extent necessary to support
implementation of visual selection on screen on top of those protocols and
APIs.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C005">C005</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> </td><td>Specifications
and software <span class="rfc2119">MUST NOT</span> require nor depend on a single keystroke resulting
in a single character, nor that a single character be input with a single
keystroke (even with modifiers), nor that keyboards are the same all over the
world.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C006">C006</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> </td><td>Software
that sorts or searches text for users
<span class="rfc2119">SHOULD</span> do so on
the basis of appropriate collation units and ordering rules for the relevant
language and/or application.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C007">C007</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> </td><td>Where searching or sorting is done dynamically,
particularly in a multilingual environment, the 'relevant language'
<span class="rfc2119">SHOULD</span> be determined to be that of the current user, and may
thus differ from user to user.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C066">C066</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> </td><td>Software that allows
users to sort or search text <span class="rfc2119">SHOULD</span> allow the user to select
alternative rules for collation units and ordering.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C008">C008</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> </td><td>Specifications and implementations of sorting and searching algorithms <span class="rfc2119">SHOULD</span> accommodate text that contains any character in Unicode.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C009">C009</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> </td><td>Specifications,
software and content <span class="rfc2119">MUST NOT</span> require or depend on a one-to-one relationship
between characters and units of physical storage.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C010">C010</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>When specifications use the
term '<span class="qterm">character</span>' the specifications <span class="rfc2119">MUST</span>
define which meaning they intend.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C067">C067</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications <span class="rfc2119">SHOULD</span> use specific terms, when available, instead of the general term 'character'.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C013">C013</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[C]</span> </td><td>Textual data objects defined by
protocol or format specifications <span class="rfc2119">MUST</span> be in a
<em>single</em> character encoding.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C014">C014</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>All specifications that
involve processing of text <span class="rfc2119">MUST</span> specify the processing of
text according to the <a title="" href="#sec-RefProcModel">Reference Processing
Model</a>, namely:<ol type="1"><li><p>Specifications <span class="rfc2119">MUST</span> define text in terms of
Unicode characters, not bytes or <a title="" href="#def-glyph">glyphs</a>.</p></li><li><p>For their textual data objects specifications <span class="rfc2119">MAY</span> allow use of any
character encoding which can be transcoded to a Unicode encoding form.</p></li><li><p>Specifications <span class="rfc2119">MAY</span> choose to disallow or
deprecate some character encodings and to make others mandatory. Independent of the
actual character encoding, the specified behavior <span class="rfc2119">MUST</span> be the same
<em>as if</em> the processing happened as follows:
</p><ul><li><p>The character encoding of any textual data object received by the
application implementing the specification <span class="rfc2119">MUST</span> be
determined and the data object <span class="rfc2119">MUST</span> be interpreted as a
sequence of Unicode characters - this <span class="rfc2119">MUST</span> be equivalent to
<a title="" href="#def-transcoding">transcoding</a> the data object to some
<a title="" href="#Unicode_Encoding_Form">Unicode encoding form</a>, adjusting
any character encoding label if necessary, and receiving it in that Unicode
encoding form.</p></li><li><p>All processing <span class="rfc2119">MUST</span> take place on
this sequence of Unicode characters.</p></li><li><p>If text is output by the application, the sequence of
Unicode characters <span class="rfc2119">MUST</span> be encoded using a character encoding chosen
among those allowed by the specification.</p></li></ul><p>
</p></li><li><p>If a specification is such that multiple textual data objects are
involved (such as an XML document referring to external parsed entities), it
<span class="rfc2119">MAY</span> choose to allow these data objects to be in different
character encodings. In all cases, the <a title="" href="#sec-RefProcModel">Reference Processing Model</a>
<span class="rfc2119">MUST</span> be applied to all textual data objects.</p></li></ol></td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C070">C070</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications <span class="rfc2119">SHOULD NOT</span>
<em>arbitrarily</em> exclude code points from the full range
of Unicode <a title="" href="#def-CCS">code points</a> from U+0000
to U+10FFFF inclusive.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C077">C077</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications <span class="rfc2119">MUST NOT</span> allow code
points above U+10FFFF.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C079">C079</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications <span class="rfc2119">SHOULD NOT</span> allow the use of codepoints reserved by Unicode for internal use.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C078">C078</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications <span class="rfc2119">MUST NOT</span> allow the
use of surrogate code points.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C015">C015</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications <span class="rfc2119">MUST</span>
either specify a unique character encoding, or provide character encoding identification
mechanisms such that the encoding of text can be reliably
identified.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C016">C016</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>When
designing a new protocol, format or API, specifications
<span class="rfc2119">SHOULD</span> require a unique character
encoding.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C017">C017</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>When basing
a protocol, format, or API on a protocol, format, or API that already
has rules for character encoding, specifications
<span class="rfc2119">SHOULD</span> use rather than change these rules.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C018">C018</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>When a unique character encoding is
required, the character encoding <span class="rfc2119">MUST</span> be UTF-8, UTF-16 or
UTF-32.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C020">C020</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications
<span class="rfc2119">SHOULD</span> avoid using the terms '<span class="qterm">character set</span>'
and '<span class="qterm">charset</span>' to refer to a character encoding, except when the
latter is used to refer to the MIME <code class="keyword">charset</code> parameter or its
IANA-registered values. The term '<span class="qterm">character encoding</span>',
or in specific cases the terms '<span class="qterm">character encoding form</span>' or '<span class="qterm">character encoding
scheme</span>', are <span class="rfc2119">RECOMMENDED</span>.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C021">C021</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>If the unique encoding
approach is not taken, specifications <span class="rfc2119">SHOULD</span> require the use
of the IANA charset registry names, and in particular the names identified in
the registry as '<span class="qterm">MIME preferred names</span>', to designate character
encodings in protocols, data formats and APIs.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C022">C022</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> <span class="requirement-type">[C]</span> </td><td>Character
encodings
that are not in the IANA registry <span class="rfc2119">SHOULD NOT</span> be
used, except by private agreement.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C023">C023</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> <span class="requirement-type">[C]</span> </td><td>If
an unregistered character encoding is used, the convention of using
'<span class="qterm">x-</span>' at the beginning of the name <span class="rfc2119">MUST</span> be
followed.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C049">C049</a></td><td style="vertical-align: top;"><span class="requirement-type">[I]</span> <span class="requirement-type">[C]</span> </td><td>The character encoding of content
<span class="rfc2119">SHOULD</span> be chosen so that it maximizes the opportunity to directly
represent characters (ie. minimizes the need to represent characters by
<a title="" href="#def-char-data">markup</a> means such as <a title="" href="#def-char-escape">character
escapes</a>) while avoiding obscure encodings that are unlikely to be understood by recipients.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C034">C034</a></td><td style="vertical-align: top;"><span class="requirement-type">[C]</span> </td><td>If facilities are
offered for identifying character encoding, content MUST make use of
them; where the facilities
offered for character encoding identification include defaults (e.g. in XML 1.0
<a href="#xml10">[XML 1.0]</a>), relying on such defaults is sufficient to satisfy this
identification requirement.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C024">C024</a></td><td style="vertical-align: top;"><span class="requirement-type">[I]</span> <span class="requirement-type">[C]</span> </td><td>Content and software
that label text data <span class="rfc2119">MUST</span> use one of the names required by
the appropriate specification (e.g. the XML specification when editing XML
text) and <span class="rfc2119">SHOULD</span> use the MIME preferred name of a character encoding
to label data in that character encoding.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C025">C025</a></td><td style="vertical-align: top;"><span class="requirement-type">[I]</span> <span class="requirement-type">[C]</span> </td><td>An IANA-registered
<code class="keyword">charset</code> name <span class="rfc2119">MUST NOT</span> be used to label text data in
a character encoding other than the one identified in the IANA registration of that
name.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C026">C026</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>If the unique encoding
approach is not chosen, specifications <span class="rfc2119">MUST</span> designate at
least one of the UTF-8 and UTF-16 encoding forms of Unicode as admissible
character encodings and <span class="rfc2119">SHOULD</span> choose at least one of UTF-8 or UTF-16
as required encoding forms (encoding forms that <span class="rfc2119">MUST</span> be
supported by implementations of the specification).</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C027">C027</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications that require a default encoding <span class="rfc2119">MUST</span> define either UTF-8 or UTF-16 as the default, or both if they define suitable means of distinguishing them.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C028">C028</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications <span class="rfc2119">MUST NOT</span>
propose the use of heuristics to determine the encoding of
data.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C029">C029</a></td><td style="vertical-align: top;"><span class="requirement-type">[I]</span> </td><td>
<em>Receiving</em>
software <span class="rfc2119">MUST</span> determine the encoding of data from available
information according to appropriate specifications.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C030">C030</a></td><td style="vertical-align: top;"><span class="requirement-type">[I]</span> </td><td>When an IANA-registered <code class="keyword">charset</code>
name is recognized, receiving software <span class="rfc2119">MUST</span> interpret the
received data according to the encoding associated with the name in the IANA
registry.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C031">C031</a></td><td style="vertical-align: top;"><span class="requirement-type">[I]</span> </td><td>When no charset
is provided receiving software <span class="rfc2119">MUST</span> adhere to the default
character encoding(s) specified in the specification.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C035">C035</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications
<span class="rfc2119">MUST</span> define conflict-resolution mechanisms (e.g. priorities)
for cases where there is multiple or conflicting information about character
encoding.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C033">C033</a></td><td style="vertical-align: top;"><span class="requirement-type">[I]</span> </td><td>Software
<span class="rfc2119">MUST</span> completely implement the mechanisms for character
encoding identification and conflict resolution.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C073">C073</a></td><td style="vertical-align: top;"><span class="requirement-type">[C]</span> </td><td>Publicly interchanged content <span class="rfc2119">SHOULD NOT</span> use
codepoints in the private use area.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C076">C076</a></td><td style="vertical-align: top;"><span class="requirement-type">[C]</span> </td><td>Content <span class="rfc2119">MUST NOT</span> use a code point for any purpose other than that defined by its coded character set.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C038">C038</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications <span class="rfc2119">MUST NOT</span> require the use of private use area
characters with particular assignments.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C039">C039</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications <span class="rfc2119">MUST
NOT</span> require the use of mechanisms for defining agreements of private
use code points.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C040">C040</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> </td><td>Specifications and
implementations <span class="rfc2119">SHOULD NOT</span> disallow the use of private use code points by private
agreement.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C041">C041</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications
<span class="rfc2119">MAY</span> define <a title="" href="#def-char-data">markup</a> to
allow the transmission of symbols not in Unicode or to identify specific
variants of Unicode characters.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C068">C068</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications <span class="rfc2119">SHOULD</span> allow the inclusion of or reference to pictures and graphics where appropriate, to eliminate the need to (mis)use character-oriented mechanisms for pictures or graphics.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C042">C042</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications
<span class="rfc2119">SHOULD NOT</span> invent a new escaping mechanism if an appropriate
one already exists. </td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C043">C043</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>The number of different
ways to escape a character <span class="rfc2119">SHOULD</span> be minimized (ideally to
one).</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C044">C044</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Escape syntax
<span class="rfc2119">SHOULD</span> require either explicit end delimiters or a
fixed number of characters in each character escape. Escape syntaxes where the
end is determined by any character outside the set of characters admissible in
the character escape itself <span class="rfc2119">SHOULD</span> be
avoided.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C045">C045</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Whenever specifications
define character escapes that allow the representation of characters using a
number, the number <span class="rfc2119">MUST</span> represent the Unicode code point
of the character and <span class="rfc2119">SHOULD</span> be in hexadecimal
notation.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C046">C046</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Escaped characters
<span class="rfc2119">SHOULD</span> be acceptable wherever their unescaped forms are; this does not preclude
that <a title="" href="#def-syntax-significant">syntax-significant</a>
characters, when escaped, lose their
significance in the syntax. In particular, if a character is
acceptable
in identifiers and comments, then its escaped form should also be
acceptable.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C047">C047</a></td><td style="vertical-align: top;"><span class="requirement-type">[I]</span> <span class="requirement-type">[C]</span> </td><td>Escapes
<span class="rfc2119">SHOULD</span> only be used when the characters to be expressed are
not directly representable in the format or the character encoding of the document, or when the visual representation of the character is unclear.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C048">C048</a></td><td style="vertical-align: top;"><span class="requirement-type">[I]</span> <span class="requirement-type">[C]</span> </td><td>Content
<span class="rfc2119">SHOULD</span> use the hexadecimal form of character escapes rather than the decimal form when
there are both.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C050">C050</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications
<span class="rfc2119">SHOULD</span> exclude compatibility characters in the syntactic
elements (markup, delimiters, identifiers) of the formats they
define.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C011">C011</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications <span class="rfc2119">SHOULD NOT</span> define a string as a '<span class="qterm">byte
string</span>'.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C012">C012</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>The '<span class="qterm">character
string</span>' definition <span class="rfc2119">SHOULD</span> be used by most specifications.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C051">C051</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> </td><td>The
<a title="" href="#def-character-string">character string</a> is
<span class="rfc2119">RECOMMENDED</span> as a basis for string indexing.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C052">C052</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> </td><td>A
<a title="" href="#def-physical-string">code unit string</a>
<span class="rfc2119">MAY</span> be used as a basis for string indexing if this results
in a significant improvement in the efficiency of internal operations when
compared to the use of <a title="" href="#def-character-string">character
string</a>.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C071">C071</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> </td><td><a title="" href="#def-grapheme-string">Grapheme clusters</a> <span class="rfc2119">MAY</span> be used as a basis for string indexing in applications where user interaction is the primary concern.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C074">C074</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications that define indexing in terms of grapheme clusters <span class="rfc2119">MUST</span> either: a) define grapheme clusters in terms of default grapheme clusters as defined in Unicode Standard Annex #29, Text Boundaries <a href="#UTR29">[UTR #29]</a>, or b) define specifically how tailoring is applied to the indexing operation.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C072">C072</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> <span class="requirement-type">[I]</span> </td><td>The use of
<a title="" href="#def-byte-string">byte strings</a> for indexing is
<span class="rfc2119">NOT RECOMMENDED</span>.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C053">C053</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications that need a way to identify
substrings or point within a string <span class="rfc2119">SHOULD</span> provide ways
other than string indexing to perform this operation.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C054">C054</a></td><td style="vertical-align: top;"><span class="requirement-type">[I]</span> <span class="requirement-type">[C]</span> </td><td>Users of
specifications (software developers, content developers)
<span class="rfc2119">SHOULD</span> whenever possible prefer ways other than string
indexing to identify substrings or point within a string.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C055">C055</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications
<span class="rfc2119">SHOULD</span> understand and process single characters as
substrings, and treat indices as boundary positions <em>between</em>
counting units, regardless of the choice of counting
units.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C056">C056</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Specifications of APIs
<span class="rfc2119">SHOULD NOT</span> specify single characters or single '<span class="qterm">units of encoding</span>'
as argument or return types.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C057">C057</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>When the positions between the units are
counted for string indexing, starting with an index of 0 for the position at
the start of the string is the <span class="rfc2119">RECOMMENDED</span> solution, with
the last index then being equal to the number of counting units in the
string.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C062">C062</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>Since specifications in general
need both a definition for their characters and the semantics associated with
these characters, specifications <span class="rfc2119">SHOULD</span> include a reference
to the Unicode Standard, whether or not they include a
reference to ISO/IEC 10646.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C063">C063</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>A generic reference to
the Unicode Standard <span class="rfc2119">MUST</span> be made if
it is desired that characters allocated after a specification is published are
usable with that specification. A specific reference to
the Unicode Standard <span class="rfc2119">MAY</span> be included
to ensure that functionality depending on a particular version is available and
will not change over time.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C064">C064</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>All <em>generic</em>
references to the Unicode Standard <a href="#unicode">[Unicode]</a>
<span class="rfc2119">MUST</span> refer to the latest version of the Unicode Standard available at the date of publication of the containing specification.</td></tr><tr><td class="reqId" style="vertical-align: top;"><a href="#C065">C065</a></td><td style="vertical-align: top;"><span class="requirement-type">[S]</span> </td><td>All <em>generic</em> references to ISO/IEC 10646
<a href="#iso10646">[ISO/IEC 10646]</a>
<span class="rfc2119">MUST</span> refer to the latest version of ISO/IEC 10646 available at the date of publication of the containing specification.</td></tr></tbody></table></div><div class="div1"> <h2><a name="sec-Changes" id="sec-Changes" />E Changes since the Proposed Recommendation (Non-Normative)</h2><ul><li><p>A small number of links and references were updated in the references section.</p></li><li><p>Minor editorial change to paragraph after C076 to clarify: "This prohibits, for example, the construction of fonts that misuse the repertoire encoded by iso-8859-1 to represent different scripts, characters, or symbols than what is actually encoded in iso-8859-1." changed to "This prohibits, for example, the construction of fonts that misuse the
codepoints in the ISO Latin 1 character set to represent different
scripts, characters, or symbols than those
actually encoded in iso-8859-1.".</p></li></ul></div><div class="div1"> <h2><a name="sec-Acknowledgements" id="sec-Acknowledgements" />F Acknowledgements (Non-Normative)</h2><p>Tim
Berners-Lee and James Clark provided important details in the section on URIs.
Asmus Freytag , Addison Phillips, and in early stages Ian Jacobs,
provided significant help in the authoring and editing process.
The W3C I18N WG and IG, as well as many others, provided many helpful comments and
suggestions.</p></div></div></body></html>