GCC Code Coverage Report
Directory: ./ Exec Total Coverage
File: node_i18n.cc Lines: 366 416 88.0 %
Date: 2021-09-20 04:12:42 Branches: 141 216 65.3 %

Line Branch Exec Source
1
// Copyright Joyent, Inc. and other Node contributors.
2
//
3
// Permission is hereby granted, free of charge, to any person obtaining a
4
// copy of this software and associated documentation files (the
5
// "Software"), to deal in the Software without restriction, including
6
// without limitation the rights to use, copy, modify, merge, publish,
7
// distribute, sublicense, and/or sell copies of the Software, and to permit
8
// persons to whom the Software is furnished to do so, subject to the
9
// following conditions:
10
//
11
// The above copyright notice and this permission notice shall be included
12
// in all copies or substantial portions of the Software.
13
//
14
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20
// USE OR OTHER DEALINGS IN THE SOFTWARE.
21
22
/*
23
 * notes: by srl295
24
 *  - When in NODE_HAVE_SMALL_ICU mode, ICU is linked against "stub" (null) data
25
 *     ( stubdata/libicudata.a ) containing nothing, no data, and it's also
26
 *    linked against a "small" data file which the SMALL_ICUDATA_ENTRY_POINT
27
 *    macro names. That's the "english+root" data.
28
 *
29
 *    If icu_data_path is non-null, the user has provided a path and we assume
30
 *    it goes somewhere useful. We set that path in ICU, and exit.
31
 *    If icu_data_path is null, they haven't set a path and we want the
32
 *    "english+root" data.  We call
33
 *       udata_setCommonData(SMALL_ICUDATA_ENTRY_POINT,...)
34
 *    to load up the english+root data.
35
 *
36
 *  - when NOT in NODE_HAVE_SMALL_ICU mode, ICU is linked directly with its full
37
 *    data. All of the variables and command line options for changing data at
38
 *    runtime are disabled, as they wouldn't fully override the internal data.
39
 *    See:  http://bugs.icu-project.org/trac/ticket/10924
40
 */
41
42
43
#include "node_i18n.h"
44
#include "node_external_reference.h"
45
46
#if defined(NODE_HAVE_I18N_SUPPORT)
47
48
#include "base_object-inl.h"
49
#include "node.h"
50
#include "node_buffer.h"
51
#include "node_errors.h"
52
#include "node_internals.h"
53
#include "util-inl.h"
54
#include "v8.h"
55
56
#include <unicode/utypes.h>
57
#include <unicode/putil.h>
58
#include <unicode/uchar.h>
59
#include <unicode/uclean.h>
60
#include <unicode/udata.h>
61
#include <unicode/uidna.h>
62
#include <unicode/ucnv.h>
63
#include <unicode/utf8.h>
64
#include <unicode/utf16.h>
65
#include <unicode/timezone.h>
66
#include <unicode/ulocdata.h>
67
#include <unicode/uvernum.h>
68
#include <unicode/uversion.h>
69
#include <unicode/ustring.h>
70
71
#ifdef NODE_HAVE_SMALL_ICU
72
/* if this is defined, we have a 'secondary' entry point.
73
   compare following to utypes.h defs for U_ICUDATA_ENTRY_POINT */
74
#define SMALL_ICUDATA_ENTRY_POINT \
75
  SMALL_DEF2(U_ICU_VERSION_MAJOR_NUM, U_LIB_SUFFIX_C_NAME)
76
#define SMALL_DEF2(major, suff) SMALL_DEF(major, suff)
77
#ifndef U_LIB_SUFFIX_C_NAME
78
#define SMALL_DEF(major, suff) icusmdt##major##_dat
79
#else
80
#define SMALL_DEF(major, suff) icusmdt##suff##major##_dat
81
#endif
82
83
extern "C" const char U_DATA_API SMALL_ICUDATA_ENTRY_POINT[];
84
#endif
85
86
namespace node {
87
88
using v8::Context;
89
using v8::FunctionCallbackInfo;
90
using v8::FunctionTemplate;
91
using v8::Int32;
92
using v8::Isolate;
93
using v8::Local;
94
using v8::MaybeLocal;
95
using v8::NewStringType;
96
using v8::Object;
97
using v8::ObjectTemplate;
98
using v8::String;
99
using v8::Uint8Array;
100
using v8::Value;
101
102
namespace i18n {
103
namespace {
104
105
template <typename T>
106
3192
MaybeLocal<Object> ToBufferEndian(Environment* env, MaybeStackBuffer<T>* buf) {
107
3192
  MaybeLocal<Object> ret = Buffer::New(env, buf);
108
3192
  if (ret.IsEmpty())
109
    return ret;
110
111
  static_assert(sizeof(T) == 1 || sizeof(T) == 2,
112
                "Currently only one- or two-byte buffers are supported");
113
3182
  if (sizeof(T) > 1 && IsBigEndian()) {
114
    SPREAD_BUFFER_ARG(ret.ToLocalChecked(), retbuf);
115
    SwapBytes16(retbuf_data, retbuf_length);
116
  }
117
118
3192
  return ret;
119
}
120
121
// One-Shot Converters
122
123
2
void CopySourceBuffer(MaybeStackBuffer<UChar>* dest,
124
                      const char* data,
125
                      const size_t length,
126
                      const size_t length_in_chars) {
127
2
  dest->AllocateSufficientStorage(length_in_chars);
128
2
  char* dst = reinterpret_cast<char*>(**dest);
129
2
  memcpy(dst, data, length);
130
2
  if (IsBigEndian()) {
131
    SwapBytes16(dst, length);
132
  }
133
2
}
134
135
typedef MaybeLocal<Object> (*TranscodeFunc)(Environment* env,
136
                                            const char* fromEncoding,
137
                                            const char* toEncoding,
138
                                            const char* source,
139
                                            const size_t source_length,
140
                                            UErrorCode* status);
141
142
3
MaybeLocal<Object> Transcode(Environment* env,
143
                             const char* fromEncoding,
144
                             const char* toEncoding,
145
                             const char* source,
146
                             const size_t source_length,
147
                             UErrorCode* status) {
148
3
  *status = U_ZERO_ERROR;
149
  MaybeLocal<Object> ret;
150
6
  MaybeStackBuffer<char> result;
151
6
  Converter to(toEncoding);
152
6
  Converter from(fromEncoding);
153
154
3
  size_t sublen = ucnv_getMinCharSize(to.conv());
155
3
  std::string sub(sublen, '?');
156
3
  to.set_subst_chars(sub.c_str());
157
158
3
  const uint32_t limit = source_length * to.max_char_size();
159
3
  result.AllocateSufficientStorage(limit);
160
3
  char* target = *result;
161
3
  ucnv_convertEx(to.conv(), from.conv(), &target, target + limit,
162
                 &source, source + source_length, nullptr, nullptr,
163
                 nullptr, nullptr, true, true, status);
164
3
  if (U_SUCCESS(*status)) {
165
3
    result.SetLength(target - &result[0]);
166
3
    ret = ToBufferEndian(env, &result);
167
  }
168
3
  return ret;
169
}
170
171
4
MaybeLocal<Object> TranscodeToUcs2(Environment* env,
172
                                   const char* fromEncoding,
173
                                   const char* toEncoding,
174
                                   const char* source,
175
                                   const size_t source_length,
176
                                   UErrorCode* status) {
177
4
  *status = U_ZERO_ERROR;
178
  MaybeLocal<Object> ret;
179
8
  MaybeStackBuffer<UChar> destbuf(source_length);
180
4
  Converter from(fromEncoding);
181
4
  const size_t length_in_chars = source_length * sizeof(UChar);
182
4
  ucnv_toUChars(from.conv(), *destbuf, length_in_chars,
183
                source, source_length, status);
184
4
  if (U_SUCCESS(*status))
185
4
    ret = ToBufferEndian(env, &destbuf);
186
4
  return ret;
187
}
188
189
MaybeLocal<Object> TranscodeFromUcs2(Environment* env,
190
                                     const char* fromEncoding,
191
                                     const char* toEncoding,
192
                                     const char* source,
193
                                     const size_t source_length,
194
                                     UErrorCode* status) {
195
  *status = U_ZERO_ERROR;
196
  MaybeStackBuffer<UChar> sourcebuf;
197
  MaybeLocal<Object> ret;
198
  Converter to(toEncoding);
199
200
  size_t sublen = ucnv_getMinCharSize(to.conv());
201
  std::string sub(sublen, '?');
202
  to.set_subst_chars(sub.c_str());
203
204
  const size_t length_in_chars = source_length / sizeof(UChar);
205
  CopySourceBuffer(&sourcebuf, source, source_length, length_in_chars);
206
  MaybeStackBuffer<char> destbuf(length_in_chars);
207
  const uint32_t len = ucnv_fromUChars(to.conv(), *destbuf, length_in_chars,
208
                                       *sourcebuf, length_in_chars, status);
209
  if (U_SUCCESS(*status)) {
210
    destbuf.SetLength(len);
211
    ret = ToBufferEndian(env, &destbuf);
212
  }
213
  return ret;
214
}
215
216
2
MaybeLocal<Object> TranscodeUcs2FromUtf8(Environment* env,
217
                                         const char* fromEncoding,
218
                                         const char* toEncoding,
219
                                         const char* source,
220
                                         const size_t source_length,
221
                                         UErrorCode* status) {
222
2
  *status = U_ZERO_ERROR;
223
2
  MaybeStackBuffer<UChar> destbuf;
224
  int32_t result_length;
225
2
  u_strFromUTF8(*destbuf, destbuf.capacity(), &result_length,
226
                source, source_length, status);
227
  MaybeLocal<Object> ret;
228
2
  if (U_SUCCESS(*status)) {
229
1
    destbuf.SetLength(result_length);
230
1
    ret = ToBufferEndian(env, &destbuf);
231
1
  } else if (*status == U_BUFFER_OVERFLOW_ERROR) {
232
1
    *status = U_ZERO_ERROR;
233
1
    destbuf.AllocateSufficientStorage(result_length);
234
1
    u_strFromUTF8(*destbuf, result_length, &result_length,
235
                  source, source_length, status);
236
1
    if (U_SUCCESS(*status)) {
237
1
      destbuf.SetLength(result_length);
238
1
      ret = ToBufferEndian(env, &destbuf);
239
    }
240
  }
241
2
  return ret;
242
}
243
244
2
MaybeLocal<Object> TranscodeUtf8FromUcs2(Environment* env,
245
                                         const char* fromEncoding,
246
                                         const char* toEncoding,
247
                                         const char* source,
248
                                         const size_t source_length,
249
                                         UErrorCode* status) {
250
2
  *status = U_ZERO_ERROR;
251
  MaybeLocal<Object> ret;
252
2
  const size_t length_in_chars = source_length / sizeof(UChar);
253
  int32_t result_length;
254
4
  MaybeStackBuffer<UChar> sourcebuf;
255
2
  MaybeStackBuffer<char> destbuf;
256
2
  CopySourceBuffer(&sourcebuf, source, source_length, length_in_chars);
257
2
  u_strToUTF8(*destbuf, destbuf.capacity(), &result_length,
258
2
              *sourcebuf, length_in_chars, status);
259
2
  if (U_SUCCESS(*status)) {
260
1
    destbuf.SetLength(result_length);
261
1
    ret = ToBufferEndian(env, &destbuf);
262
1
  } else if (*status == U_BUFFER_OVERFLOW_ERROR) {
263
1
    *status = U_ZERO_ERROR;
264
1
    destbuf.AllocateSufficientStorage(result_length);
265
1
    u_strToUTF8(*destbuf, result_length, &result_length, *sourcebuf,
266
                length_in_chars, status);
267
1
    if (U_SUCCESS(*status)) {
268
1
      destbuf.SetLength(result_length);
269
1
      ret = ToBufferEndian(env, &destbuf);
270
    }
271
  }
272
2
  return ret;
273
}
274
275
22
const char* EncodingName(const enum encoding encoding) {
276

22
  switch (encoding) {
277
2
    case ASCII: return "us-ascii";
278
4
    case LATIN1: return "iso8859-1";
279
10
    case UCS2: return "utf16le";
280
6
    case UTF8: return "utf-8";
281
    default: return nullptr;
282
  }
283
}
284
285
24
bool SupportedEncoding(const enum encoding encoding) {
286
24
  switch (encoding) {
287
22
    case ASCII:
288
    case LATIN1:
289
    case UCS2:
290
22
    case UTF8: return true;
291
2
    default: return false;
292
  }
293
}
294
295
13
void Transcode(const FunctionCallbackInfo<Value>&args) {
296
13
  Environment* env = Environment::GetCurrent(args);
297
13
  Isolate* isolate = env->isolate();
298
13
  UErrorCode status = U_ZERO_ERROR;
299
  MaybeLocal<Object> result;
300
301
13
  ArrayBufferViewContents<char> input(args[0]);
302
13
  const enum encoding fromEncoding = ParseEncoding(isolate, args[1], BUFFER);
303
13
  const enum encoding toEncoding = ParseEncoding(isolate, args[2], BUFFER);
304
305

13
  if (SupportedEncoding(fromEncoding) && SupportedEncoding(toEncoding)) {
306
11
    TranscodeFunc tfn = &Transcode;
307

11
    switch (fromEncoding) {
308
4
      case ASCII:
309
      case LATIN1:
310
4
        if (toEncoding == UCS2)
311
4
          tfn = &TranscodeToUcs2;
312
4
        break;
313
4
      case UTF8:
314
4
        if (toEncoding == UCS2)
315
2
          tfn = &TranscodeUcs2FromUtf8;
316
4
        break;
317
3
      case UCS2:
318
3
        switch (toEncoding) {
319
1
          case UCS2:
320
1
            tfn = &Transcode;
321
1
            break;
322
2
          case UTF8:
323
2
            tfn = &TranscodeUtf8FromUcs2;
324
2
            break;
325
          default:
326
            tfn = &TranscodeFromUcs2;
327
        }
328
3
        break;
329
      default:
330
        // This should not happen because of the SupportedEncoding checks
331
        ABORT();
332
    }
333
334
    result = tfn(env, EncodingName(fromEncoding), EncodingName(toEncoding),
335
11
                 input.data(), input.length(), &status);
336
  } else {
337
2
    status = U_ILLEGAL_ARGUMENT_ERROR;
338
  }
339
340
13
  if (result.IsEmpty())
341
4
    return args.GetReturnValue().Set(status);
342
343
22
  return args.GetReturnValue().Set(result.ToLocalChecked());
344
}
345
346
2
void ICUErrorName(const FunctionCallbackInfo<Value>& args) {
347
2
  Environment* env = Environment::GetCurrent(args);
348
2
  CHECK(args[0]->IsInt32());
349
4
  UErrorCode status = static_cast<UErrorCode>(args[0].As<Int32>()->Value());
350
6
  args.GetReturnValue().Set(
351
2
      String::NewFromUtf8(env->isolate(),
352
2
                          u_errorName(status)).ToLocalChecked());
353
2
}
354
355
}  // anonymous namespace
356
357
10
Converter::Converter(const char* name, const char* sub) {
358
10
  UErrorCode status = U_ZERO_ERROR;
359
10
  UConverter* conv = ucnv_open(name, &status);
360
10
  CHECK(U_SUCCESS(status));
361
10
  conv_.reset(conv);
362
10
  set_subst_chars(sub);
363
10
}
364
365
4291
Converter::Converter(UConverter* converter, const char* sub)
366
4291
    : conv_(converter) {
367
4291
  set_subst_chars(sub);
368
4291
}
369
370
4304
void Converter::set_subst_chars(const char* sub) {
371
4304
  CHECK(conv_);
372
4304
  UErrorCode status = U_ZERO_ERROR;
373
4304
  if (sub != nullptr) {
374
3
    ucnv_setSubstChars(conv_.get(), sub, strlen(sub), &status);
375
3
    CHECK(U_SUCCESS(status));
376
  }
377
4304
}
378
379
1001
void Converter::reset() {
380
1001
  ucnv_reset(conv_.get());
381
1001
}
382
383
1677
size_t Converter::min_char_size() const {
384
1677
  CHECK(conv_);
385
1677
  return ucnv_getMinCharSize(conv_.get());
386
}
387
388
3
size_t Converter::max_char_size() const {
389
3
  CHECK(conv_);
390
3
  return ucnv_getMaxCharSize(conv_.get());
391
}
392
393
2
void ConverterObject::Has(const FunctionCallbackInfo<Value>& args) {
394
2
  Environment* env = Environment::GetCurrent(args);
395
396
2
  CHECK_GE(args.Length(), 1);
397
4
  Utf8Value label(env->isolate(), args[0]);
398
399
2
  UErrorCode status = U_ZERO_ERROR;
400
2
  ConverterPointer conv(ucnv_open(*label, &status));
401
4
  args.GetReturnValue().Set(!!U_SUCCESS(status));
402
2
}
403
404
4291
void ConverterObject::Create(const FunctionCallbackInfo<Value>& args) {
405
4291
  Environment* env = Environment::GetCurrent(args);
406
407
4291
  Local<ObjectTemplate> t = env->i18n_converter_template();
408
  Local<Object> obj;
409
8582
  if (!t->NewInstance(env->context()).ToLocal(&obj)) return;
410
411
4291
  CHECK_GE(args.Length(), 2);
412
4291
  Utf8Value label(env->isolate(), args[0]);
413
4291
  int flags = args[1]->Uint32Value(env->context()).ToChecked();
414
4291
  bool fatal =
415
4291
      (flags & CONVERTER_FLAGS_FATAL) == CONVERTER_FLAGS_FATAL;
416
417
4291
  UErrorCode status = U_ZERO_ERROR;
418
4291
  UConverter* conv = ucnv_open(*label, &status);
419
4291
  if (U_FAILURE(status))
420
    return;
421
422
4291
  if (fatal) {
423
327
    status = U_ZERO_ERROR;
424
327
    ucnv_setToUCallBack(conv, UCNV_TO_U_CALLBACK_STOP,
425
                        nullptr, nullptr, nullptr, &status);
426
  }
427
428
4291
  new ConverterObject(env, obj, conv, flags);
429
8582
  args.GetReturnValue().Set(obj);
430
}
431
432
1677
void ConverterObject::Decode(const FunctionCallbackInfo<Value>& args) {
433
1677
  Environment* env = Environment::GetCurrent(args);
434
435
1677
  CHECK_GE(args.Length(), 3);  // Converter, Buffer, Flags
436
437
  ConverterObject* converter;
438
4939
  ASSIGN_OR_RETURN_UNWRAP(&converter, args[0].As<Object>());
439
1677
  ArrayBufferViewContents<char> input(args[1]);
440
1677
  int flags = args[2]->Uint32Value(env->context()).ToChecked();
441
442
1677
  UErrorCode status = U_ZERO_ERROR;
443
1677
  MaybeStackBuffer<UChar> result;
444
  MaybeLocal<Object> ret;
445
446
1677
  UBool flush = (flags & CONVERTER_FLAGS_FLUSH) == CONVERTER_FLAGS_FLUSH;
447
448
  // When flushing the final chunk, the limit is the maximum
449
  // of either the input buffer length or the number of pending
450
  // characters times the min char size.
451
1677
  size_t limit = converter->min_char_size() *
452
2678
      (!flush ?
453
676
          input.length() :
454
          std::max(
455
3679
              input.length(),
456
1001
              static_cast<size_t>(
457
2002
                  ucnv_toUCountPending(converter->conv(), &status))));
458
1677
  status = U_ZERO_ERROR;
459
460
1677
  if (limit > 0)
461
1610
    result.AllocateSufficientStorage(limit);
462
463
1677
  auto cleanup = OnScopeLeave([&]() {
464
1677
    if (flush) {
465
      // Reset the converter state.
466
1001
      converter->set_bom_seen(false);
467
1001
      converter->reset();
468
    }
469
1677
  });
470
471
1677
  const char* source = input.data();
472
1677
  size_t source_length = input.length();
473
474
1677
  UChar* target = *result;
475
1677
  ucnv_toUnicode(converter->conv(),
476
                 &target,
477
1677
                 target + (limit * sizeof(UChar)),
478
                 &source,
479
                 source + source_length,
480
                 nullptr,
481
                 flush,
482
                 &status);
483
484
1677
  if (U_SUCCESS(status)) {
485
1585
    bool omit_initial_bom = false;
486
1585
    if (limit > 0) {
487
1518
      result.SetLength(target - &result[0]);
488
2856
      if (result.length() > 0 &&
489
1338
          converter->unicode() &&
490

4143
          !converter->ignore_bom() &&
491
1287
          !converter->bom_seen()) {
492
        // If the very first result in the stream is a BOM, and we are not
493
        // explicitly told to ignore it, then we mark it for discarding.
494
839
        if (result[0] == 0xFEFF)
495
30
          omit_initial_bom = true;
496
839
        converter->set_bom_seen(true);
497
      }
498
    }
499
1585
    ret = ToBufferEndian(env, &result);
500

1615
    if (omit_initial_bom && !ret.IsEmpty()) {
501
      // Perform `ret = ret.slice(2)`.
502
30
      CHECK(ret.ToLocalChecked()->IsUint8Array());
503
60
      Local<Uint8Array> orig_ret = ret.ToLocalChecked().As<Uint8Array>();
504
30
      ret = Buffer::New(env,
505
30
                        orig_ret->Buffer(),
506
30
                        orig_ret->ByteOffset() + 2,
507
60
                        orig_ret->ByteLength() - 2)
508
                            .FromMaybe(Local<Uint8Array>());
509
    }
510
1585
    if (!ret.IsEmpty())
511
3170
      args.GetReturnValue().Set(ret.ToLocalChecked());
512
1585
    return;
513
  }
514
515
184
  args.GetReturnValue().Set(status);
516
}
517
518
4291
ConverterObject::ConverterObject(
519
    Environment* env,
520
    Local<Object> wrap,
521
    UConverter* converter,
522
    int flags,
523
4291
    const char* sub)
524
    : BaseObject(env, wrap),
525
      Converter(converter, sub),
526
4291
      flags_(flags) {
527
4291
  MakeWeak();
528
529
4291
  switch (ucnv_getType(converter)) {
530
738
    case UCNV_UTF8:
531
    case UCNV_UTF16_BigEndian:
532
    case UCNV_UTF16_LittleEndian:
533
738
      flags_ |= CONVERTER_FLAGS_UNICODE;
534
738
      break;
535
4291
    default: {
536
      // Fall through
537
    }
538
  }
539
4291
}
540
541
542
4895
bool InitializeICUDirectory(const std::string& path) {
543
4895
  UErrorCode status = U_ZERO_ERROR;
544
4895
  if (path.empty()) {
545
#ifdef NODE_HAVE_SMALL_ICU
546
    // install the 'small' data.
547
    udata_setCommonData(&SMALL_ICUDATA_ENTRY_POINT, &status);
548
#else  // !NODE_HAVE_SMALL_ICU
549
    // no small data, so nothing to do.
550
#endif  // !NODE_HAVE_SMALL_ICU
551
  } else {
552
    u_setDataDirectory(path.c_str());
553
    u_init(&status);
554
  }
555
4895
  return status == U_ZERO_ERROR;
556
}
557
558
void SetDefaultTimeZone(const char* tzid) {
559
  size_t tzidlen = strlen(tzid) + 1;
560
  UErrorCode status = U_ZERO_ERROR;
561
  MaybeStackBuffer<UChar, 256> id(tzidlen);
562
  u_charsToUChars(tzid, id.out(), tzidlen);
563
  // This is threadsafe:
564
  ucal_setDefaultTimeZone(id.out(), &status);
565
  CHECK(U_SUCCESS(status));
566
}
567
568
384
int32_t ToUnicode(MaybeStackBuffer<char>* buf,
569
                  const char* input,
570
                  size_t length) {
571
384
  UErrorCode status = U_ZERO_ERROR;
572
384
  uint32_t options = UIDNA_NONTRANSITIONAL_TO_UNICODE;
573
384
  UIDNA* uidna = uidna_openUTS46(options, &status);
574
384
  if (U_FAILURE(status))
575
    return -1;
576
384
  UIDNAInfo info = UIDNA_INFO_INITIALIZER;
577
578
384
  int32_t len = uidna_nameToUnicodeUTF8(uidna,
579
                                        input, length,
580
384
                                        **buf, buf->capacity(),
581
                                        &info,
582
                                        &status);
583
584
  // Do not check info.errors like we do with ToASCII since ToUnicode always
585
  // returns a string, despite any possible errors that may have occurred.
586
587
384
  if (status == U_BUFFER_OVERFLOW_ERROR) {
588
    status = U_ZERO_ERROR;
589
    buf->AllocateSufficientStorage(len);
590
    len = uidna_nameToUnicodeUTF8(uidna,
591
                                  input, length,
592
                                  **buf, buf->capacity(),
593
                                  &info,
594
                                  &status);
595
  }
596
597
  // info.errors is ignored as UTS #46 ToUnicode always produces a Unicode
598
  // string, regardless of whether an error occurred.
599
600
384
  if (U_FAILURE(status)) {
601
    len = -1;
602
    buf->SetLength(0);
603
  } else {
604
384
    buf->SetLength(len);
605
  }
606
607
384
  uidna_close(uidna);
608
384
  return len;
609
}
610
611
13991
int32_t ToASCII(MaybeStackBuffer<char>* buf,
612
                const char* input,
613
                size_t length,
614
                enum idna_mode mode) {
615
13991
  UErrorCode status = U_ZERO_ERROR;
616
13991
  uint32_t options =                  // CheckHyphens = false; handled later
617
    UIDNA_CHECK_BIDI |                // CheckBidi = true
618
    UIDNA_CHECK_CONTEXTJ |            // CheckJoiners = true
619
    UIDNA_NONTRANSITIONAL_TO_ASCII;   // Nontransitional_Processing
620
13991
  if (mode == IDNA_STRICT) {
621
    options |= UIDNA_USE_STD3_RULES;  // UseSTD3ASCIIRules = beStrict
622
                                      // VerifyDnsLength = beStrict;
623
                                      //   handled later
624
  }
625
626
13991
  UIDNA* uidna = uidna_openUTS46(options, &status);
627
13991
  if (U_FAILURE(status))
628
    return -1;
629
13991
  UIDNAInfo info = UIDNA_INFO_INITIALIZER;
630
631
13991
  int32_t len = uidna_nameToASCII_UTF8(uidna,
632
                                       input, length,
633
13991
                                       **buf, buf->capacity(),
634
                                       &info,
635
                                       &status);
636
637
13991
  if (status == U_BUFFER_OVERFLOW_ERROR) {
638
2
    status = U_ZERO_ERROR;
639
2
    buf->AllocateSufficientStorage(len);
640
2
    len = uidna_nameToASCII_UTF8(uidna,
641
                                 input, length,
642
2
                                 **buf, buf->capacity(),
643
                                 &info,
644
                                 &status);
645
  }
646
647
  // In UTS #46 which specifies ToASCII, certain error conditions are
648
  // configurable through options, and the WHATWG URL Standard promptly elects
649
  // to disable some of them to accommodate for real-world use cases.
650
  // Unfortunately, ICU4C's IDNA module does not support disabling some of
651
  // these options through `options` above, and thus continues throwing
652
  // unnecessary errors. To counter this situation, we just filter out the
653
  // errors that may have happened afterwards, before deciding whether to
654
  // return an error from this function.
655
656
  // CheckHyphens = false
657
  // (Specified in the current UTS #46 draft rev. 18.)
658
  // Refs:
659
  // - https://github.com/whatwg/url/issues/53
660
  // - https://github.com/whatwg/url/pull/309
661
  // - http://www.unicode.org/review/pri317/
662
  // - http://www.unicode.org/reports/tr46/tr46-18.html
663
  // - https://www.icann.org/news/announcement-2000-01-07-en
664
13991
  info.errors &= ~UIDNA_ERROR_HYPHEN_3_4;
665
13991
  info.errors &= ~UIDNA_ERROR_LEADING_HYPHEN;
666
13991
  info.errors &= ~UIDNA_ERROR_TRAILING_HYPHEN;
667
668
13991
  if (mode != IDNA_STRICT) {
669
    // VerifyDnsLength = beStrict
670
13991
    info.errors &= ~UIDNA_ERROR_EMPTY_LABEL;
671
13991
    info.errors &= ~UIDNA_ERROR_LABEL_TOO_LONG;
672
13991
    info.errors &= ~UIDNA_ERROR_DOMAIN_NAME_TOO_LONG;
673
  }
674
675


13991
  if (U_FAILURE(status) || (mode != IDNA_LENIENT && info.errors != 0)) {
676
133
    len = -1;
677
133
    buf->SetLength(0);
678
  } else {
679
13858
    buf->SetLength(len);
680
  }
681
682
13991
  uidna_close(uidna);
683
13991
  return len;
684
}
685
686
189
static void ToUnicode(const FunctionCallbackInfo<Value>& args) {
687
189
  Environment* env = Environment::GetCurrent(args);
688
189
  CHECK_GE(args.Length(), 1);
689
378
  CHECK(args[0]->IsString());
690
189
  Utf8Value val(env->isolate(), args[0]);
691
692
189
  MaybeStackBuffer<char> buf;
693
189
  int32_t len = ToUnicode(&buf, *val, val.length());
694
695
189
  if (len < 0) {
696
    return THROW_ERR_INVALID_ARG_VALUE(env, "Cannot convert name to Unicode");
697
  }
698
699
567
  args.GetReturnValue().Set(
700
189
      String::NewFromUtf8(env->isolate(),
701
189
                          *buf,
702
                          NewStringType::kNormal,
703
189
                          len).ToLocalChecked());
704
}
705
706
10084
static void ToASCII(const FunctionCallbackInfo<Value>& args) {
707
10084
  Environment* env = Environment::GetCurrent(args);
708
10084
  CHECK_GE(args.Length(), 1);
709
20168
  CHECK(args[0]->IsString());
710
10084
  Utf8Value val(env->isolate(), args[0]);
711
  // optional arg
712
10084
  bool lenient = args[1]->BooleanValue(env->isolate());
713
10084
  enum idna_mode mode = lenient ? IDNA_LENIENT : IDNA_DEFAULT;
714
715
10084
  MaybeStackBuffer<char> buf;
716
10084
  int32_t len = ToASCII(&buf, *val, val.length(), mode);
717
718
10084
  if (len < 0) {
719
9
    return THROW_ERR_INVALID_ARG_VALUE(env, "Cannot convert name to ASCII");
720
  }
721
722
30225
  args.GetReturnValue().Set(
723
10075
      String::NewFromUtf8(env->isolate(),
724
10075
                          *buf,
725
                          NewStringType::kNormal,
726
10075
                          len).ToLocalChecked());
727
}
728
729
// This is similar to wcwidth except that it takes the current unicode
730
// character properties database into consideration, allowing it to
731
// correctly calculate the column widths of things like emoji's and
732
// newer wide characters. wcwidth, on the other hand, uses a fixed
733
// algorithm that does not take things like emoji into proper
734
// consideration.
735
//
736
// TODO(TimothyGu): Investigate Cc (C0/C1 control codes). Both VTE (used by
737
// GNOME Terminal) and Konsole don't consider them to be zero-width (see refs
738
// below), and when printed in VTE it is Narrow. However GNOME Terminal doesn't
739
// allow it to be input. Linux's PTY terminal prints control characters as
740
// Narrow rhombi.
741
//
742
// TODO(TimothyGu): Investigate Hangul jamo characters. Medial vowels and final
743
// consonants are 0-width when combined with initial consonants; otherwise they
744
// are technically Wide. But many terminals (including Konsole and
745
// VTE/GLib-based) implement all medials and finals as 0-width.
746
//
747
// Refs: https://eev.ee/blog/2015/09/12/dark-corners-of-unicode/#combining-characters-and-character-width
748
// Refs: https://github.com/GNOME/glib/blob/79e4d4c6be/glib/guniprop.c#L388-L420
749
// Refs: https://github.com/KDE/konsole/blob/8c6a5d13c0/src/konsole_wcwidth.cpp#L101-L223
750
629454
static int GetColumnWidth(UChar32 codepoint,
751
                          bool ambiguous_as_full_width = false) {
752
  // UCHAR_EAST_ASIAN_WIDTH is the Unicode property that identifies a
753
  // codepoint as being full width, wide, ambiguous, neutral, narrow,
754
  // or halfwidth.
755
629454
  const int eaw = u_getIntPropertyValue(codepoint, UCHAR_EAST_ASIAN_WIDTH);
756

629454
  switch (eaw) {
757
37151
    case U_EA_FULLWIDTH:
758
    case U_EA_WIDE:
759
37151
      return 2;
760
509576
    case U_EA_AMBIGUOUS:
761
      // See: http://www.unicode.org/reports/tr11/#Ambiguous for details
762
509576
      if (ambiguous_as_full_width) {
763
        return 2;
764
      }
765
      // If ambiguous_as_full_width is false:
766
      // Fall through
767
    case U_EA_NEUTRAL:
768
576702
      if (u_hasBinaryProperty(codepoint, UCHAR_EMOJI_PRESENTATION)) {
769
        return 2;
770
      }
771
      // Fall through
772
    case U_EA_HALFWIDTH:
773
    case U_EA_NARROW:
774
    default:
775
592303
      const auto zero_width_mask = U_GC_CC_MASK |  // C0/C1 control code
776
                                  U_GC_CF_MASK |  // Format control character
777
                                  U_GC_ME_MASK |  // Enclosing mark
778
                                  U_GC_MN_MASK;   // Nonspacing mark
779

1184574
      if (codepoint != 0x00AD &&  // SOFT HYPHEN is Cf but not zero-width
780
679780
          ((U_MASK(u_charType(codepoint)) & zero_width_mask) ||
781
592271
          u_hasBinaryProperty(codepoint, UCHAR_EMOJI_MODIFIER))) {
782
504762
        return 0;
783
      }
784
87541
      return 1;
785
  }
786
}
787
788
// Returns the column width for the given String.
789
601943
static void GetStringWidth(const FunctionCallbackInfo<Value>& args) {
790
601943
  Environment* env = Environment::GetCurrent(args);
791
1203886
  CHECK(args[0]->IsString());
792
793
601943
  bool ambiguous_as_full_width = args[1]->IsTrue();
794

601943
  bool expand_emoji_sequence = !args[2]->IsBoolean() || args[2]->IsTrue();
795
796
601943
  TwoByteValue value(env->isolate(), args[0]);
797
  // reinterpret_cast is required by windows to compile
798
601943
  UChar* str = reinterpret_cast<UChar*>(*value);
799
  static_assert(sizeof(*str) == sizeof(**value),
800
                "sizeof(*str) == sizeof(**value)");
801
601943
  UChar32 c = 0;
802
  UChar32 p;
803
601943
  size_t n = 0;
804
601943
  uint32_t width = 0;
805
806
1231397
  while (n < value.length()) {
807
629454
    p = c;
808


629454
    U16_NEXT(str, n, value.length(), c);
809
    // Don't count individual emoji codepoints that occur within an
810
    // emoji sequence. This is not necessarily foolproof. Some
811
    // environments display emoji sequences in the appropriate
812
    // condensed form (as a single emoji glyph), other environments
813
    // may not understand an emoji sequence and will display each
814
    // individual emoji separately. When this happens, the width
815
    // calculated will be off, and there's no reliable way of knowing
816
    // in advance if a particular sequence is going to be supported.
817
    // The expand_emoji_sequence option allows the caller to skip this
818
    // check and count each code within an emoji sequence separately.
819
    // https://www.unicode.org/reports/tr51/tr51-16.html#Emoji_ZWJ_Sequences
820
1258908
    if (!expand_emoji_sequence &&
821


629454
        n > 0 && p == 0x200d &&  // 0x200d == ZWJ (zero width joiner)
822
        (u_hasBinaryProperty(c, UCHAR_EMOJI_PRESENTATION) ||
823
         u_hasBinaryProperty(c, UCHAR_EMOJI_MODIFIER))) {
824
      continue;
825
    }
826
629454
    width += GetColumnWidth(c, ambiguous_as_full_width);
827
  }
828
1203886
  args.GetReturnValue().Set(width);
829
601943
}
830
831
613
void Initialize(Local<Object> target,
832
                Local<Value> unused,
833
                Local<Context> context,
834
                void* priv) {
835
613
  Environment* env = Environment::GetCurrent(context);
836
613
  env->SetMethod(target, "toUnicode", ToUnicode);
837
613
  env->SetMethod(target, "toASCII", ToASCII);
838
613
  env->SetMethod(target, "getStringWidth", GetStringWidth);
839
840
  // One-shot converters
841
613
  env->SetMethod(target, "icuErrName", ICUErrorName);
842
613
  env->SetMethod(target, "transcode", Transcode);
843
844
  // ConverterObject
845
  {
846
613
    Local<FunctionTemplate> t = FunctionTemplate::New(env->isolate());
847
613
    t->Inherit(BaseObject::GetConstructorTemplate(env));
848
1226
    t->InstanceTemplate()->SetInternalFieldCount(
849
        ConverterObject::kInternalFieldCount);
850
    Local<String> converter_string =
851
613
        FIXED_ONE_BYTE_STRING(env->isolate(), "Converter");
852
613
    t->SetClassName(converter_string);
853
613
    env->set_i18n_converter_template(t->InstanceTemplate());
854
  }
855
856
613
  env->SetMethod(target, "getConverter", ConverterObject::Create);
857
613
  env->SetMethod(target, "decode", ConverterObject::Decode);
858
613
  env->SetMethod(target, "hasConverter", ConverterObject::Has);
859
613
}
860
861
4853
void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
862
4853
  registry->Register(ToUnicode);
863
4853
  registry->Register(ToASCII);
864
4853
  registry->Register(GetStringWidth);
865
4853
  registry->Register(ICUErrorName);
866
4853
  registry->Register(Transcode);
867
4853
  registry->Register(ConverterObject::Create);
868
4853
  registry->Register(ConverterObject::Decode);
869
4853
  registry->Register(ConverterObject::Has);
870
4853
}
871
872
}  // namespace i18n
873
}  // namespace node
874
875
4926
NODE_MODULE_CONTEXT_AWARE_INTERNAL(icu, node::i18n::Initialize)
876
4853
NODE_MODULE_EXTERNAL_REFERENCE(icu, node::i18n::RegisterExternalReferences)
877
878
#endif  // NODE_HAVE_I18N_SUPPORT