GCC Code Coverage Report
Directory: ./ Exec Total Coverage
File: node_i18n.cc Lines: 370 422 87.7 %
Date: 2022-11-10 04:21:28 Branches: 136 210 64.8 %

Line Branch Exec Source
1
// Copyright Joyent, Inc. and other Node contributors.
2
//
3
// Permission is hereby granted, free of charge, to any person obtaining a
4
// copy of this software and associated documentation files (the
5
// "Software"), to deal in the Software without restriction, including
6
// without limitation the rights to use, copy, modify, merge, publish,
7
// distribute, sublicense, and/or sell copies of the Software, and to permit
8
// persons to whom the Software is furnished to do so, subject to the
9
// following conditions:
10
//
11
// The above copyright notice and this permission notice shall be included
12
// in all copies or substantial portions of the Software.
13
//
14
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20
// USE OR OTHER DEALINGS IN THE SOFTWARE.
21
22
/*
23
 * notes: by srl295
24
 *  - When in NODE_HAVE_SMALL_ICU mode, ICU is linked against "stub" (null) data
25
 *     ( stubdata/libicudata.a ) containing nothing, no data, and it's also
26
 *    linked against a "small" data file which the SMALL_ICUDATA_ENTRY_POINT
27
 *    macro names. That's the "english+root" data.
28
 *
29
 *    If icu_data_path is non-null, the user has provided a path and we assume
30
 *    it goes somewhere useful. We set that path in ICU, and exit.
31
 *    If icu_data_path is null, they haven't set a path and we want the
32
 *    "english+root" data.  We call
33
 *       udata_setCommonData(SMALL_ICUDATA_ENTRY_POINT,...)
34
 *    to load up the english+root data.
35
 *
36
 *  - when NOT in NODE_HAVE_SMALL_ICU mode, ICU is linked directly with its full
37
 *    data. All of the variables and command line options for changing data at
38
 *    runtime are disabled, as they wouldn't fully override the internal data.
39
 *    See:  http://bugs.icu-project.org/trac/ticket/10924
40
 */
41
42
43
#include "node_i18n.h"
44
#include "node_external_reference.h"
45
46
#if defined(NODE_HAVE_I18N_SUPPORT)
47
48
#include "base_object-inl.h"
49
#include "node.h"
50
#include "node_buffer.h"
51
#include "node_errors.h"
52
#include "node_internals.h"
53
#include "string_bytes.h"
54
#include "util-inl.h"
55
#include "v8.h"
56
57
#include <unicode/utypes.h>
58
#include <unicode/putil.h>
59
#include <unicode/uchar.h>
60
#include <unicode/uclean.h>
61
#include <unicode/udata.h>
62
#include <unicode/uidna.h>
63
#include <unicode/ucnv.h>
64
#include <unicode/utf8.h>
65
#include <unicode/utf16.h>
66
#include <unicode/timezone.h>
67
#include <unicode/ulocdata.h>
68
#include <unicode/uvernum.h>
69
#include <unicode/uversion.h>
70
#include <unicode/ustring.h>
71
72
#ifdef NODE_HAVE_SMALL_ICU
73
/* if this is defined, we have a 'secondary' entry point.
74
   compare following to utypes.h defs for U_ICUDATA_ENTRY_POINT */
75
#define SMALL_ICUDATA_ENTRY_POINT \
76
  SMALL_DEF2(U_ICU_VERSION_MAJOR_NUM, U_LIB_SUFFIX_C_NAME)
77
#define SMALL_DEF2(major, suff) SMALL_DEF(major, suff)
78
#ifndef U_LIB_SUFFIX_C_NAME
79
#define SMALL_DEF(major, suff) icusmdt##major##_dat
80
#else
81
#define SMALL_DEF(major, suff) icusmdt##suff##major##_dat
82
#endif
83
84
extern "C" const char U_DATA_API SMALL_ICUDATA_ENTRY_POINT[];
85
#endif
86
87
namespace node {
88
89
using v8::Context;
90
using v8::FunctionCallbackInfo;
91
using v8::FunctionTemplate;
92
using v8::Int32;
93
using v8::Isolate;
94
using v8::Local;
95
using v8::MaybeLocal;
96
using v8::NewStringType;
97
using v8::Object;
98
using v8::ObjectTemplate;
99
using v8::String;
100
using v8::Value;
101
102
namespace i18n {
103
namespace {
104
105
template <typename T>
106
22
MaybeLocal<Object> ToBufferEndian(Environment* env, MaybeStackBuffer<T>* buf) {
107
22
  MaybeLocal<Object> ret = Buffer::New(env, buf);
108
22
  if (ret.IsEmpty())
109
    return ret;
110
111
  static_assert(sizeof(T) == 1 || sizeof(T) == 2,
112
                "Currently only one- or two-byte buffers are supported");
113
12
  if (sizeof(T) > 1 && IsBigEndian()) {
114
    SPREAD_BUFFER_ARG(ret.ToLocalChecked(), retbuf);
115
    SwapBytes16(retbuf_data, retbuf_length);
116
  }
117
118
22
  return ret;
119
}
120
121
// One-Shot Converters
122
123
2
void CopySourceBuffer(MaybeStackBuffer<UChar>* dest,
124
                      const char* data,
125
                      const size_t length,
126
                      const size_t length_in_chars) {
127
2
  dest->AllocateSufficientStorage(length_in_chars);
128
2
  char* dst = reinterpret_cast<char*>(**dest);
129
2
  memcpy(dst, data, length);
130
2
  if (IsBigEndian()) {
131
    SwapBytes16(dst, length);
132
  }
133
2
}
134
135
typedef MaybeLocal<Object> (*TranscodeFunc)(Environment* env,
136
                                            const char* fromEncoding,
137
                                            const char* toEncoding,
138
                                            const char* source,
139
                                            const size_t source_length,
140
                                            UErrorCode* status);
141
142
3
MaybeLocal<Object> Transcode(Environment* env,
143
                             const char* fromEncoding,
144
                             const char* toEncoding,
145
                             const char* source,
146
                             const size_t source_length,
147
                             UErrorCode* status) {
148
3
  *status = U_ZERO_ERROR;
149
  MaybeLocal<Object> ret;
150
6
  MaybeStackBuffer<char> result;
151
6
  Converter to(toEncoding);
152
6
  Converter from(fromEncoding);
153
154
3
  size_t sublen = ucnv_getMinCharSize(to.conv());
155
3
  std::string sub(sublen, '?');
156
3
  to.set_subst_chars(sub.c_str());
157
158
3
  const uint32_t limit = source_length * to.max_char_size();
159
3
  result.AllocateSufficientStorage(limit);
160
3
  char* target = *result;
161
3
  ucnv_convertEx(to.conv(), from.conv(), &target, target + limit,
162
                 &source, source + source_length, nullptr, nullptr,
163
                 nullptr, nullptr, true, true, status);
164
3
  if (U_SUCCESS(*status)) {
165
3
    result.SetLength(target - &result[0]);
166
3
    ret = ToBufferEndian(env, &result);
167
  }
168
3
  return ret;
169
}
170
171
4
MaybeLocal<Object> TranscodeToUcs2(Environment* env,
172
                                   const char* fromEncoding,
173
                                   const char* toEncoding,
174
                                   const char* source,
175
                                   const size_t source_length,
176
                                   UErrorCode* status) {
177
4
  *status = U_ZERO_ERROR;
178
  MaybeLocal<Object> ret;
179
8
  MaybeStackBuffer<UChar> destbuf(source_length);
180
4
  Converter from(fromEncoding);
181
4
  const size_t length_in_chars = source_length * sizeof(UChar);
182
4
  ucnv_toUChars(from.conv(), *destbuf, length_in_chars,
183
                source, source_length, status);
184
4
  if (U_SUCCESS(*status))
185
4
    ret = ToBufferEndian(env, &destbuf);
186
4
  return ret;
187
}
188
189
MaybeLocal<Object> TranscodeFromUcs2(Environment* env,
190
                                     const char* fromEncoding,
191
                                     const char* toEncoding,
192
                                     const char* source,
193
                                     const size_t source_length,
194
                                     UErrorCode* status) {
195
  *status = U_ZERO_ERROR;
196
  MaybeStackBuffer<UChar> sourcebuf;
197
  MaybeLocal<Object> ret;
198
  Converter to(toEncoding);
199
200
  size_t sublen = ucnv_getMinCharSize(to.conv());
201
  std::string sub(sublen, '?');
202
  to.set_subst_chars(sub.c_str());
203
204
  const size_t length_in_chars = source_length / sizeof(UChar);
205
  CopySourceBuffer(&sourcebuf, source, source_length, length_in_chars);
206
  MaybeStackBuffer<char> destbuf(length_in_chars);
207
  const uint32_t len = ucnv_fromUChars(to.conv(), *destbuf, length_in_chars,
208
                                       *sourcebuf, length_in_chars, status);
209
  if (U_SUCCESS(*status)) {
210
    destbuf.SetLength(len);
211
    ret = ToBufferEndian(env, &destbuf);
212
  }
213
  return ret;
214
}
215
216
2
MaybeLocal<Object> TranscodeUcs2FromUtf8(Environment* env,
217
                                         const char* fromEncoding,
218
                                         const char* toEncoding,
219
                                         const char* source,
220
                                         const size_t source_length,
221
                                         UErrorCode* status) {
222
2
  *status = U_ZERO_ERROR;
223
2
  MaybeStackBuffer<UChar> destbuf;
224
  int32_t result_length;
225
2
  u_strFromUTF8(*destbuf, destbuf.capacity(), &result_length,
226
                source, source_length, status);
227
  MaybeLocal<Object> ret;
228
2
  if (U_SUCCESS(*status)) {
229
1
    destbuf.SetLength(result_length);
230
1
    ret = ToBufferEndian(env, &destbuf);
231
1
  } else if (*status == U_BUFFER_OVERFLOW_ERROR) {
232
1
    *status = U_ZERO_ERROR;
233
1
    destbuf.AllocateSufficientStorage(result_length);
234
1
    u_strFromUTF8(*destbuf, result_length, &result_length,
235
                  source, source_length, status);
236
1
    if (U_SUCCESS(*status)) {
237
1
      destbuf.SetLength(result_length);
238
1
      ret = ToBufferEndian(env, &destbuf);
239
    }
240
  }
241
2
  return ret;
242
}
243
244
2
MaybeLocal<Object> TranscodeUtf8FromUcs2(Environment* env,
245
                                         const char* fromEncoding,
246
                                         const char* toEncoding,
247
                                         const char* source,
248
                                         const size_t source_length,
249
                                         UErrorCode* status) {
250
2
  *status = U_ZERO_ERROR;
251
  MaybeLocal<Object> ret;
252
2
  const size_t length_in_chars = source_length / sizeof(UChar);
253
  int32_t result_length;
254
4
  MaybeStackBuffer<UChar> sourcebuf;
255
2
  MaybeStackBuffer<char> destbuf;
256
2
  CopySourceBuffer(&sourcebuf, source, source_length, length_in_chars);
257
2
  u_strToUTF8(*destbuf, destbuf.capacity(), &result_length,
258
2
              *sourcebuf, length_in_chars, status);
259
2
  if (U_SUCCESS(*status)) {
260
1
    destbuf.SetLength(result_length);
261
1
    ret = ToBufferEndian(env, &destbuf);
262
1
  } else if (*status == U_BUFFER_OVERFLOW_ERROR) {
263
1
    *status = U_ZERO_ERROR;
264
1
    destbuf.AllocateSufficientStorage(result_length);
265
1
    u_strToUTF8(*destbuf, result_length, &result_length, *sourcebuf,
266
                length_in_chars, status);
267
1
    if (U_SUCCESS(*status)) {
268
1
      destbuf.SetLength(result_length);
269
1
      ret = ToBufferEndian(env, &destbuf);
270
    }
271
  }
272
2
  return ret;
273
}
274
275
22
const char* EncodingName(const enum encoding encoding) {
276

22
  switch (encoding) {
277
2
    case ASCII: return "us-ascii";
278
4
    case LATIN1: return "iso8859-1";
279
10
    case UCS2: return "utf16le";
280
6
    case UTF8: return "utf-8";
281
    default: return nullptr;
282
  }
283
}
284
285
24
bool SupportedEncoding(const enum encoding encoding) {
286
24
  switch (encoding) {
287
22
    case ASCII:
288
    case LATIN1:
289
    case UCS2:
290
22
    case UTF8: return true;
291
2
    default: return false;
292
  }
293
}
294
295
13
void Transcode(const FunctionCallbackInfo<Value>&args) {
296
13
  Environment* env = Environment::GetCurrent(args);
297
13
  Isolate* isolate = env->isolate();
298
13
  UErrorCode status = U_ZERO_ERROR;
299
  MaybeLocal<Object> result;
300
301
13
  ArrayBufferViewContents<char> input(args[0]);
302
13
  const enum encoding fromEncoding = ParseEncoding(isolate, args[1], BUFFER);
303
13
  const enum encoding toEncoding = ParseEncoding(isolate, args[2], BUFFER);
304
305

13
  if (SupportedEncoding(fromEncoding) && SupportedEncoding(toEncoding)) {
306
11
    TranscodeFunc tfn = &Transcode;
307

11
    switch (fromEncoding) {
308
4
      case ASCII:
309
      case LATIN1:
310
4
        if (toEncoding == UCS2)
311
4
          tfn = &TranscodeToUcs2;
312
4
        break;
313
4
      case UTF8:
314
4
        if (toEncoding == UCS2)
315
2
          tfn = &TranscodeUcs2FromUtf8;
316
4
        break;
317
3
      case UCS2:
318
3
        switch (toEncoding) {
319
1
          case UCS2:
320
1
            tfn = &Transcode;
321
1
            break;
322
2
          case UTF8:
323
2
            tfn = &TranscodeUtf8FromUcs2;
324
2
            break;
325
          default:
326
            tfn = &TranscodeFromUcs2;
327
        }
328
3
        break;
329
      default:
330
        // This should not happen because of the SupportedEncoding checks
331
        ABORT();
332
    }
333
334
    result = tfn(env, EncodingName(fromEncoding), EncodingName(toEncoding),
335
11
                 input.data(), input.length(), &status);
336
  } else {
337
2
    status = U_ILLEGAL_ARGUMENT_ERROR;
338
  }
339
340
13
  if (result.IsEmpty())
341
4
    return args.GetReturnValue().Set(status);
342
343
22
  return args.GetReturnValue().Set(result.ToLocalChecked());
344
}
345
346
2
void ICUErrorName(const FunctionCallbackInfo<Value>& args) {
347
2
  Environment* env = Environment::GetCurrent(args);
348
2
  CHECK(args[0]->IsInt32());
349
4
  UErrorCode status = static_cast<UErrorCode>(args[0].As<Int32>()->Value());
350
6
  args.GetReturnValue().Set(
351
2
      String::NewFromUtf8(env->isolate(),
352
2
                          u_errorName(status)).ToLocalChecked());
353
2
}
354
355
}  // anonymous namespace
356
357
10
Converter::Converter(const char* name, const char* sub) {
358
10
  UErrorCode status = U_ZERO_ERROR;
359
10
  UConverter* conv = ucnv_open(name, &status);
360
10
  CHECK(U_SUCCESS(status));
361
10
  conv_.reset(conv);
362
10
  set_subst_chars(sub);
363
10
}
364
365
916
Converter::Converter(UConverter* converter, const char* sub)
366
916
    : conv_(converter) {
367
916
  set_subst_chars(sub);
368
916
}
369
370
1845
void Converter::set_subst_chars(const char* sub) {
371
1845
  CHECK(conv_);
372
1845
  UErrorCode status = U_ZERO_ERROR;
373
1845
  if (sub != nullptr) {
374
919
    ucnv_setSubstChars(conv_.get(), sub, strlen(sub), &status);
375
919
    CHECK(U_SUCCESS(status));
376
  }
377
1845
}
378
379
1436
void Converter::reset() {
380
1436
  ucnv_reset(conv_.get());
381
1436
}
382
383
2334
size_t Converter::min_char_size() const {
384
2334
  CHECK(conv_);
385
2334
  return ucnv_getMinCharSize(conv_.get());
386
}
387
388
3
size_t Converter::max_char_size() const {
389
3
  CHECK(conv_);
390
3
  return ucnv_getMaxCharSize(conv_.get());
391
}
392
393
2
void ConverterObject::Has(const FunctionCallbackInfo<Value>& args) {
394
2
  Environment* env = Environment::GetCurrent(args);
395
396
2
  CHECK_GE(args.Length(), 1);
397
4
  Utf8Value label(env->isolate(), args[0]);
398
399
2
  UErrorCode status = U_ZERO_ERROR;
400
2
  ConverterPointer conv(ucnv_open(*label, &status));
401
4
  args.GetReturnValue().Set(!!U_SUCCESS(status));
402
2
}
403
404
916
void ConverterObject::Create(const FunctionCallbackInfo<Value>& args) {
405
916
  Environment* env = Environment::GetCurrent(args);
406
407
916
  Local<ObjectTemplate> t = env->i18n_converter_template();
408
  Local<Object> obj;
409
1832
  if (!t->NewInstance(env->context()).ToLocal(&obj)) return;
410
411
916
  CHECK_GE(args.Length(), 2);
412
916
  Utf8Value label(env->isolate(), args[0]);
413
916
  int flags = args[1]->Uint32Value(env->context()).ToChecked();
414
916
  bool fatal =
415
916
      (flags & CONVERTER_FLAGS_FATAL) == CONVERTER_FLAGS_FATAL;
416
417
916
  UErrorCode status = U_ZERO_ERROR;
418
916
  UConverter* conv = ucnv_open(*label, &status);
419
916
  if (U_FAILURE(status))
420
    return;
421
422
916
  if (fatal) {
423
337
    status = U_ZERO_ERROR;
424
337
    ucnv_setToUCallBack(conv, UCNV_TO_U_CALLBACK_STOP,
425
                        nullptr, nullptr, nullptr, &status);
426
  }
427
428
916
  auto converter = new ConverterObject(env, obj, conv, flags);
429
916
  size_t sublen = ucnv_getMinCharSize(conv);
430
916
  std::string sub(sublen, '?');
431
916
  converter->set_subst_chars(sub.c_str());
432
433
1832
  args.GetReturnValue().Set(obj);
434
}
435
436
2334
void ConverterObject::Decode(const FunctionCallbackInfo<Value>& args) {
437
2334
  Environment* env = Environment::GetCurrent(args);
438
439
2334
  CHECK_GE(args.Length(), 3);  // Converter, Buffer, Flags
440
441
  ConverterObject* converter;
442
6905
  ASSIGN_OR_RETURN_UNWRAP(&converter, args[0].As<Object>());
443
2334
  ArrayBufferViewContents<char> input(args[1]);
444
2334
  int flags = args[2]->Uint32Value(env->context()).ToChecked();
445
446
2334
  UErrorCode status = U_ZERO_ERROR;
447
2334
  MaybeStackBuffer<UChar> result;
448
449
2334
  UBool flush = (flags & CONVERTER_FLAGS_FLUSH) == CONVERTER_FLAGS_FLUSH;
450
451
  // When flushing the final chunk, the limit is the maximum
452
  // of either the input buffer length or the number of pending
453
  // characters times the min char size, multiplied by 2 as unicode may
454
  // take up to 2 UChars to encode a character
455
2334
  size_t limit = 2 * converter->min_char_size() *
456
3770
      (!flush ?
457
898
          input.length() :
458
          std::max(
459
5206
              input.length(),
460
1436
              static_cast<size_t>(
461
2872
                  ucnv_toUCountPending(converter->conv(), &status))));
462
2334
  status = U_ZERO_ERROR;
463
464
2334
  if (limit > 0)
465
2113
    result.AllocateSufficientStorage(limit);
466
467
2334
  auto cleanup = OnScopeLeave([&]() {
468
2334
    if (flush) {
469
      // Reset the converter state.
470
1436
      converter->set_bom_seen(false);
471
1436
      converter->reset();
472
    }
473
2334
  });
474
475
2334
  const char* source = input.data();
476
2334
  size_t source_length = input.length();
477
478
2334
  UChar* target = *result;
479
2334
  ucnv_toUnicode(converter->conv(),
480
                 &target,
481
2334
                 target + limit,
482
                 &source,
483
                 source + source_length,
484
                 nullptr,
485
                 flush,
486
                 &status);
487
488
2334
  if (U_SUCCESS(status)) {
489
2237
    bool omit_initial_bom = false;
490
2237
    if (limit > 0) {
491
2016
      result.SetLength(target - &result[0]);
492
3828
      if (result.length() > 0 &&
493
1812
          converter->unicode() &&
494

5534
          !converter->ignore_bom() &&
495
1706
          !converter->bom_seen()) {
496
        // If the very first result in the stream is a BOM, and we are not
497
        // explicitly told to ignore it, then we mark it for discarding.
498
1219
        if (result[0] == 0xFEFF)
499
42
          omit_initial_bom = true;
500
1219
        converter->set_bom_seen(true);
501
      }
502
    }
503
504
    Local<Value> error;
505
2237
    UChar* output = result.out();
506
2237
    size_t beginning = 0;
507
2237
    size_t length = result.length() * sizeof(UChar);
508
509
2237
    if (omit_initial_bom) {
510
      // Perform `ret = ret.slice(2)`.
511
42
      beginning += 2;
512
42
      length -= 2;
513
    }
514
515
2237
    char* value = reinterpret_cast<char*>(output) + beginning;
516
517
2237
    if (IsBigEndian()) {
518
      SwapBytes16(value, length);
519
    }
520
521
    MaybeLocal<Value> encoded =
522
2237
        StringBytes::Encode(env->isolate(), value, length, UCS2, &error);
523
524
    Local<Value> ret;
525
2237
    if (encoded.ToLocal(&ret)) {
526
4474
      args.GetReturnValue().Set(ret);
527
    } else {
528
      args.GetReturnValue().Set(error);
529
    }
530
531
2237
    return;
532
  }
533
534
194
  args.GetReturnValue().Set(status);
535
}
536
537
916
ConverterObject::ConverterObject(
538
    Environment* env,
539
    Local<Object> wrap,
540
    UConverter* converter,
541
    int flags,
542
916
    const char* sub)
543
    : BaseObject(env, wrap),
544
      Converter(converter, sub),
545
916
      flags_(flags) {
546
916
  MakeWeak();
547
548
916
  switch (ucnv_getType(converter)) {
549
906
    case UCNV_UTF8:
550
    case UCNV_UTF16_BigEndian:
551
    case UCNV_UTF16_LittleEndian:
552
906
      flags_ |= CONVERTER_FLAGS_UNICODE;
553
906
      break;
554
916
    default: {
555
      // Fall through
556
    }
557
  }
558
916
}
559
560
561
5636
bool InitializeICUDirectory(const std::string& path) {
562
5636
  UErrorCode status = U_ZERO_ERROR;
563
5636
  if (path.empty()) {
564
#ifdef NODE_HAVE_SMALL_ICU
565
    // install the 'small' data.
566
    udata_setCommonData(&SMALL_ICUDATA_ENTRY_POINT, &status);
567
#else  // !NODE_HAVE_SMALL_ICU
568
    // no small data, so nothing to do.
569
#endif  // !NODE_HAVE_SMALL_ICU
570
  } else {
571
    u_setDataDirectory(path.c_str());
572
    u_init(&status);
573
  }
574
5636
  return status == U_ZERO_ERROR;
575
}
576
577
void SetDefaultTimeZone(const char* tzid) {
578
  size_t tzidlen = strlen(tzid) + 1;
579
  UErrorCode status = U_ZERO_ERROR;
580
  MaybeStackBuffer<UChar, 256> id(tzidlen);
581
  u_charsToUChars(tzid, id.out(), tzidlen);
582
  // This is threadsafe:
583
  ucal_setDefaultTimeZone(id.out(), &status);
584
  CHECK(U_SUCCESS(status));
585
}
586
587
384
int32_t ToUnicode(MaybeStackBuffer<char>* buf,
588
                  const char* input,
589
                  size_t length) {
590
384
  UErrorCode status = U_ZERO_ERROR;
591
384
  uint32_t options = UIDNA_NONTRANSITIONAL_TO_UNICODE;
592
384
  UIDNA* uidna = uidna_openUTS46(options, &status);
593
384
  if (U_FAILURE(status))
594
    return -1;
595
384
  UIDNAInfo info = UIDNA_INFO_INITIALIZER;
596
597
384
  int32_t len = uidna_nameToUnicodeUTF8(uidna,
598
                                        input, length,
599
384
                                        **buf, buf->capacity(),
600
                                        &info,
601
                                        &status);
602
603
  // Do not check info.errors like we do with ToASCII since ToUnicode always
604
  // returns a string, despite any possible errors that may have occurred.
605
606
384
  if (status == U_BUFFER_OVERFLOW_ERROR) {
607
    status = U_ZERO_ERROR;
608
    buf->AllocateSufficientStorage(len);
609
    len = uidna_nameToUnicodeUTF8(uidna,
610
                                  input, length,
611
                                  **buf, buf->capacity(),
612
                                  &info,
613
                                  &status);
614
  }
615
616
  // info.errors is ignored as UTS #46 ToUnicode always produces a Unicode
617
  // string, regardless of whether an error occurred.
618
619
384
  if (U_FAILURE(status)) {
620
    len = -1;
621
    buf->SetLength(0);
622
  } else {
623
384
    buf->SetLength(len);
624
  }
625
626
384
  uidna_close(uidna);
627
384
  return len;
628
}
629
630
14714
int32_t ToASCII(MaybeStackBuffer<char>* buf,
631
                const char* input,
632
                size_t length,
633
                enum idna_mode mode) {
634
14714
  UErrorCode status = U_ZERO_ERROR;
635
14714
  uint32_t options =                  // CheckHyphens = false; handled later
636
    UIDNA_CHECK_BIDI |                // CheckBidi = true
637
    UIDNA_CHECK_CONTEXTJ |            // CheckJoiners = true
638
    UIDNA_NONTRANSITIONAL_TO_ASCII;   // Nontransitional_Processing
639
14714
  if (mode == IDNA_STRICT) {
640
    options |= UIDNA_USE_STD3_RULES;  // UseSTD3ASCIIRules = beStrict
641
                                      // VerifyDnsLength = beStrict;
642
                                      //   handled later
643
  }
644
645
14714
  UIDNA* uidna = uidna_openUTS46(options, &status);
646
14714
  if (U_FAILURE(status))
647
    return -1;
648
14714
  UIDNAInfo info = UIDNA_INFO_INITIALIZER;
649
650
14714
  int32_t len = uidna_nameToASCII_UTF8(uidna,
651
                                       input, length,
652
14714
                                       **buf, buf->capacity(),
653
                                       &info,
654
                                       &status);
655
656
14714
  if (status == U_BUFFER_OVERFLOW_ERROR) {
657
2
    status = U_ZERO_ERROR;
658
2
    buf->AllocateSufficientStorage(len);
659
2
    len = uidna_nameToASCII_UTF8(uidna,
660
                                 input, length,
661
2
                                 **buf, buf->capacity(),
662
                                 &info,
663
                                 &status);
664
  }
665
666
  // In UTS #46 which specifies ToASCII, certain error conditions are
667
  // configurable through options, and the WHATWG URL Standard promptly elects
668
  // to disable some of them to accommodate for real-world use cases.
669
  // Unfortunately, ICU4C's IDNA module does not support disabling some of
670
  // these options through `options` above, and thus continues throwing
671
  // unnecessary errors. To counter this situation, we just filter out the
672
  // errors that may have happened afterwards, before deciding whether to
673
  // return an error from this function.
674
675
  // CheckHyphens = false
676
  // (Specified in the current UTS #46 draft rev. 18.)
677
  // Refs:
678
  // - https://github.com/whatwg/url/issues/53
679
  // - https://github.com/whatwg/url/pull/309
680
  // - http://www.unicode.org/review/pri317/
681
  // - http://www.unicode.org/reports/tr46/tr46-18.html
682
  // - https://www.icann.org/news/announcement-2000-01-07-en
683
14714
  info.errors &= ~UIDNA_ERROR_HYPHEN_3_4;
684
14714
  info.errors &= ~UIDNA_ERROR_LEADING_HYPHEN;
685
14714
  info.errors &= ~UIDNA_ERROR_TRAILING_HYPHEN;
686
687
14714
  if (mode != IDNA_STRICT) {
688
    // VerifyDnsLength = beStrict
689
14714
    info.errors &= ~UIDNA_ERROR_EMPTY_LABEL;
690
14714
    info.errors &= ~UIDNA_ERROR_LABEL_TOO_LONG;
691
14714
    info.errors &= ~UIDNA_ERROR_DOMAIN_NAME_TOO_LONG;
692
  }
693
694


14714
  if (U_FAILURE(status) || (mode != IDNA_LENIENT && info.errors != 0)) {
695
142
    len = -1;
696
142
    buf->SetLength(0);
697
  } else {
698
14572
    buf->SetLength(len);
699
  }
700
701
14714
  uidna_close(uidna);
702
14714
  return len;
703
}
704
705
189
static void ToUnicode(const FunctionCallbackInfo<Value>& args) {
706
189
  Environment* env = Environment::GetCurrent(args);
707
189
  CHECK_GE(args.Length(), 1);
708
378
  CHECK(args[0]->IsString());
709
189
  Utf8Value val(env->isolate(), args[0]);
710
711
189
  MaybeStackBuffer<char> buf;
712
189
  int32_t len = ToUnicode(&buf, *val, val.length());
713
714
189
  if (len < 0) {
715
    return THROW_ERR_INVALID_ARG_VALUE(env, "Cannot convert name to Unicode");
716
  }
717
718
567
  args.GetReturnValue().Set(
719
189
      String::NewFromUtf8(env->isolate(),
720
189
                          *buf,
721
                          NewStringType::kNormal,
722
189
                          len).ToLocalChecked());
723
}
724
725
10420
static void ToASCII(const FunctionCallbackInfo<Value>& args) {
726
10420
  Environment* env = Environment::GetCurrent(args);
727
10420
  CHECK_GE(args.Length(), 1);
728
20840
  CHECK(args[0]->IsString());
729
10420
  Utf8Value val(env->isolate(), args[0]);
730
  // optional arg
731
10420
  bool lenient = args[1]->BooleanValue(env->isolate());
732
10420
  enum idna_mode mode = lenient ? IDNA_LENIENT : IDNA_DEFAULT;
733
734
10420
  MaybeStackBuffer<char> buf;
735
10420
  int32_t len = ToASCII(&buf, *val, val.length(), mode);
736
737
10420
  if (len < 0) {
738
9
    return THROW_ERR_INVALID_ARG_VALUE(env, "Cannot convert name to ASCII");
739
  }
740
741
31233
  args.GetReturnValue().Set(
742
10411
      String::NewFromUtf8(env->isolate(),
743
10411
                          *buf,
744
                          NewStringType::kNormal,
745
10411
                          len).ToLocalChecked());
746
}
747
748
// This is similar to wcwidth except that it takes the current unicode
749
// character properties database into consideration, allowing it to
750
// correctly calculate the column widths of things like emoji's and
751
// newer wide characters. wcwidth, on the other hand, uses a fixed
752
// algorithm that does not take things like emoji into proper
753
// consideration.
754
//
755
// TODO(TimothyGu): Investigate Cc (C0/C1 control codes). Both VTE (used by
756
// GNOME Terminal) and Konsole don't consider them to be zero-width (see refs
757
// below), and when printed in VTE it is Narrow. However GNOME Terminal doesn't
758
// allow it to be input. Linux's PTY terminal prints control characters as
759
// Narrow rhombi.
760
//
761
// TODO(TimothyGu): Investigate Hangul jamo characters. Medial vowels and final
762
// consonants are 0-width when combined with initial consonants; otherwise they
763
// are technically Wide. But many terminals (including Konsole and
764
// VTE/GLib-based) implement all medials and finals as 0-width.
765
//
766
// Refs: https://eev.ee/blog/2015/09/12/dark-corners-of-unicode/#combining-characters-and-character-width
767
// Refs: https://github.com/GNOME/glib/blob/79e4d4c6be/glib/guniprop.c#L388-L420
768
// Refs: https://github.com/KDE/konsole/blob/8c6a5d13c0/src/konsole_wcwidth.cpp#L101-L223
769
1216752
static int GetColumnWidth(UChar32 codepoint,
770
                          bool ambiguous_as_full_width = false) {
771
  // UCHAR_EAST_ASIAN_WIDTH is the Unicode property that identifies a
772
  // codepoint as being full width, wide, ambiguous, neutral, narrow,
773
  // or halfwidth.
774
1216752
  const int eaw = u_getIntPropertyValue(codepoint, UCHAR_EAST_ASIAN_WIDTH);
775

1216752
  switch (eaw) {
776
66268
    case U_EA_FULLWIDTH:
777
    case U_EA_WIDE:
778
66268
      return 2;
779
1008774
    case U_EA_AMBIGUOUS:
780
      // See: http://www.unicode.org/reports/tr11/#Ambiguous for details
781
1008774
      if (ambiguous_as_full_width) {
782
        return 2;
783
      }
784
      // If ambiguous_as_full_width is false:
785
      // Fall through
786
    case U_EA_NEUTRAL:
787
1136974
      if (u_hasBinaryProperty(codepoint, UCHAR_EMOJI_PRESENTATION)) {
788
        return 2;
789
      }
790
      // Fall through
791
    case U_EA_HALFWIDTH:
792
    case U_EA_NARROW:
793
    default:
794
1150484
      const auto zero_width_mask = U_GC_CC_MASK |  // C0/C1 control code
795
                                  U_GC_CF_MASK |  // Format control character
796
                                  U_GC_ME_MASK |  // Enclosing mark
797
                                  U_GC_MN_MASK;   // Nonspacing mark
798

2300932
      if (codepoint != 0x00AD &&  // SOFT HYPHEN is Cf but not zero-width
799
1295703
          ((U_MASK(u_charType(codepoint)) & zero_width_mask) ||
800
1150448
          u_hasBinaryProperty(codepoint, UCHAR_EMOJI_MODIFIER))) {
801
1005193
        return 0;
802
      }
803
145291
      return 1;
804
  }
805
}
806
807
// Returns the column width for the given String.
808
1195362
static void GetStringWidth(const FunctionCallbackInfo<Value>& args) {
809
1195362
  Environment* env = Environment::GetCurrent(args);
810
2390724
  CHECK(args[0]->IsString());
811
812
1195362
  bool ambiguous_as_full_width = args[1]->IsTrue();
813

1195362
  bool expand_emoji_sequence = !args[2]->IsBoolean() || args[2]->IsTrue();
814
815
1195362
  TwoByteValue value(env->isolate(), args[0]);
816
  // reinterpret_cast is required by windows to compile
817
1195362
  UChar* str = reinterpret_cast<UChar*>(*value);
818
  static_assert(sizeof(*str) == sizeof(**value),
819
                "sizeof(*str) == sizeof(**value)");
820
1195362
  UChar32 c = 0;
821
  UChar32 p;
822
1195362
  size_t n = 0;
823
1195362
  uint32_t width = 0;
824
825
2412114
  while (n < value.length()) {
826
1216752
    p = c;
827


1216752
    U16_NEXT(str, n, value.length(), c);
828
    // Don't count individual emoji codepoints that occur within an
829
    // emoji sequence. This is not necessarily foolproof. Some
830
    // environments display emoji sequences in the appropriate
831
    // condensed form (as a single emoji glyph), other environments
832
    // may not understand an emoji sequence and will display each
833
    // individual emoji separately. When this happens, the width
834
    // calculated will be off, and there's no reliable way of knowing
835
    // in advance if a particular sequence is going to be supported.
836
    // The expand_emoji_sequence option allows the caller to skip this
837
    // check and count each code within an emoji sequence separately.
838
    // https://www.unicode.org/reports/tr51/tr51-16.html#Emoji_ZWJ_Sequences
839
2433504
    if (!expand_emoji_sequence &&
840


1216752
        n > 0 && p == 0x200d &&  // 0x200d == ZWJ (zero width joiner)
841
        (u_hasBinaryProperty(c, UCHAR_EMOJI_PRESENTATION) ||
842
         u_hasBinaryProperty(c, UCHAR_EMOJI_MODIFIER))) {
843
      continue;
844
    }
845
1216752
    width += GetColumnWidth(c, ambiguous_as_full_width);
846
  }
847
2390724
  args.GetReturnValue().Set(width);
848
1195362
}
849
850
787
void Initialize(Local<Object> target,
851
                Local<Value> unused,
852
                Local<Context> context,
853
                void* priv) {
854
787
  Environment* env = Environment::GetCurrent(context);
855
787
  SetMethod(context, target, "toUnicode", ToUnicode);
856
787
  SetMethod(context, target, "toASCII", ToASCII);
857
787
  SetMethod(context, target, "getStringWidth", GetStringWidth);
858
859
  // One-shot converters
860
787
  SetMethod(context, target, "icuErrName", ICUErrorName);
861
787
  SetMethod(context, target, "transcode", Transcode);
862
863
  // ConverterObject
864
  {
865
787
    Local<FunctionTemplate> t = NewFunctionTemplate(env->isolate(), nullptr);
866
787
    t->Inherit(BaseObject::GetConstructorTemplate(env));
867
1574
    t->InstanceTemplate()->SetInternalFieldCount(
868
        ConverterObject::kInternalFieldCount);
869
    Local<String> converter_string =
870
787
        FIXED_ONE_BYTE_STRING(env->isolate(), "Converter");
871
787
    t->SetClassName(converter_string);
872
787
    env->set_i18n_converter_template(t->InstanceTemplate());
873
  }
874
875
787
  SetMethod(context, target, "getConverter", ConverterObject::Create);
876
787
  SetMethod(context, target, "decode", ConverterObject::Decode);
877
787
  SetMethod(context, target, "hasConverter", ConverterObject::Has);
878
787
}
879
880
5601
void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
881
5601
  registry->Register(ToUnicode);
882
5601
  registry->Register(ToASCII);
883
5601
  registry->Register(GetStringWidth);
884
5601
  registry->Register(ICUErrorName);
885
5601
  registry->Register(Transcode);
886
5601
  registry->Register(ConverterObject::Create);
887
5601
  registry->Register(ConverterObject::Decode);
888
5601
  registry->Register(ConverterObject::Has);
889
5601
}
890
891
}  // namespace i18n
892
}  // namespace node
893
894
5672
NODE_MODULE_CONTEXT_AWARE_INTERNAL(icu, node::i18n::Initialize)
895
5601
NODE_MODULE_EXTERNAL_REFERENCE(icu, node::i18n::RegisterExternalReferences)
896
897
#endif  // NODE_HAVE_I18N_SUPPORT