[gjs/ewlsh/text-encoding] Updates
- From: Evan Welsh <ewlsh src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gjs/ewlsh/text-encoding] Updates
- Date: Thu, 12 Aug 2021 01:23:18 +0000 (UTC)
commit cc6cfd141bf7b6d4d9cdb1d215bddbf58cfbed0d
Author: Evan Welsh <contact evanwelsh com>
Date: Wed Aug 11 18:07:52 2021 -0700
Updates
.eslintignore | 2 +-
gjs/text-encoding.cpp | 41 ++-
installed-tests/js/.eslintrc.yml | 2 +
installed-tests/js/matchers.js | 29 ++
installed-tests/js/meson.build | 6 +-
installed-tests/js/testEncoding.js | 114 +++++---
js.gresource.xml | 6 +-
jsconfig.json | 5 +
modules/core/_encodings.js | 312 ---------------------
modules/esm/_bootstrap/default.js | 3 +
.../{core/_text.js => esm/_encoding/encoding.js} | 104 ++++---
modules/esm/_encoding/encodingMap.js | 305 ++++++++++++++++++++
modules/esm/_encoding/util.js | 37 +++
modules/script/_bootstrap/default.js | 13 -
14 files changed, 566 insertions(+), 413 deletions(-)
---
diff --git a/.eslintignore b/.eslintignore
index 9e12e9f0..60e57336 100644
--- a/.eslintignore
+++ b/.eslintignore
@@ -5,5 +5,5 @@ installed-tests/js/jasmine.js
installed-tests/js/modules/badOverrides/WarnLib.js
# Until ESLint merges class fields.
# https://github.com/eslint/eslint/issues/14343
-modules/core/_text.js
+modules/esm/_encoding/encoding.js
modules/script/jsUnit.js
diff --git a/gjs/text-encoding.cpp b/gjs/text-encoding.cpp
index 228c8c40..2dd384e8 100644
--- a/gjs/text-encoding.cpp
+++ b/gjs/text-encoding.cpp
@@ -73,13 +73,34 @@ static JSString* gjs_lossy_decode_from_uint8array_slow(
if (error)
return gjs_throw_type_error_from_gerror(cx, error);
+ // This function converts *to* UTF-16, using a std::u16string
+ // as its buffer.
+ //
+ // UTF-16 represents each character with 2 bytes or
+ // 4 bytes, the best case scenario when converting to
+ // UTF-16 is that every input byte encodes to two bytes,
+ // this is typical for ASCII and non-supplementary characters.
+ // Because we are converting from an unknown encoding
+ // technically a single byte could be supplementary in
+ // Unicode (4 bytes) or even represen multiple Unicode characters.
+ //
+ // std::u16string does not care about these implementation
+ // details, its only concern is that is consists of byte pairs.
+ // Given this, a single UTF-16 character could be represented
+ // by one or two std::u16string characters.
+
// Allocate bytes_len * 2 + 12 as our initial buffer.
// bytes_len * 2 is the "best case" for LATIN1 strings
- // and strings are in the basic multilingual plane.
+ // and strings which are in the basic multilingual plane.
// Add 12 as a slight cushion and set the minimum allocation
// at 256 to prefer running a single iteration for
// small strings with supplemental plane characters.
- int buffer_size = std::min(bytes_len * 2 + 12, 256lu);
+ //
+ // When converting Chinese characters, for example,
+ // some dialectal characters are in the supplemental plane
+ // Adding a padding of 12 prevents a few dialectal characters
+ // from requiring a reallocation.
+ size_t buffer_size = std::max(bytes_len * 2 + 12, 256lu);
// Cast data to correct input types
const char* input = reinterpret_cast<const char*>(bytes);
@@ -108,7 +129,7 @@ static JSString* gjs_lossy_decode_from_uint8array_slow(
// accumulator
if (bytes_written > 0) {
char16_t* utf16_buffer = reinterpret_cast<char16_t*>(buffer.data());
- // UTF-16 uses exactly 2 bytes for every character.
+ // std::u16string uses exactly 2 bytes for every character.
output_str.append(utf16_buffer, bytes_written / 2);
} else if (error) {
// A PARTIAL_INPUT error can only occur if the user does not provide
@@ -138,10 +159,16 @@ static JSString* gjs_lossy_decode_from_uint8array_slow(
// If the buffer was full increase the buffer
// size and re-try the conversion.
//
- // In most cases this will allocate bytes_len * 4
- // (the worst case scenario) in the next loop
- // where invalid bytes are not found.
- buffer_size *= 2;
+ // This logic allocates bytes_len * 3 first,
+ // then bytes_len * 4 (the worst case scenario
+ // is nearly impossible) and then continues appending
+ // arbitrary padding because we'll trust Gio and give
+ // it additional space.
+ if (buffer_size > bytes_len * 4) {
+ buffer_size += 256;
+ } else {
+ buffer_size += bytes_len;
+ }
// Clear the error.
g_clear_error(&error);
diff --git a/installed-tests/js/.eslintrc.yml b/installed-tests/js/.eslintrc.yml
index c1a4c9bd..bcdadbea 100644
--- a/installed-tests/js/.eslintrc.yml
+++ b/installed-tests/js/.eslintrc.yml
@@ -31,8 +31,10 @@ globals:
setTimeout: writable
overrides:
- files:
+ - matchers.js
- testCairoModule.js
- testESModules.js
+ - testEncoding.js
- modules/importmeta.js
- modules/exports.js
- modules/say.js
diff --git a/installed-tests/js/matchers.js b/installed-tests/js/matchers.js
new file mode 100644
index 00000000..e4d29835
--- /dev/null
+++ b/installed-tests/js/matchers.js
@@ -0,0 +1,29 @@
+/**
+ * A jasmine asymmetric matcher which expects an array-like object
+ * to contain the given element array in the same order with the
+ * same length. Useful for testing typed arrays.
+ *
+ * @template T
+ * @param {T[]} elements an array of elements to compare with
+ * @returns
+ */
+export function arrayLikeWithExactContents(elements) {
+ return {
+ /**
+ * @param {ArrayLike<T>} compareTo an array-like object to compare to
+ * @returns {boolean}
+ */
+ asymmetricMatch(compareTo) {
+ return (
+ compareTo.length === elements.length &&
+ elements.every((e, i) => e === compareTo[i])
+ );
+ },
+ /**
+ * @returns {string}
+ */
+ jasmineToString() {
+ return `${JSON.stringify(elements)}`;
+ },
+ };
+}
diff --git a/installed-tests/js/meson.build b/installed-tests/js/meson.build
index cca525c1..f5fa8e08 100644
--- a/installed-tests/js/meson.build
+++ b/installed-tests/js/meson.build
@@ -94,7 +94,6 @@ subdir('libgjstesttools')
jasmine_tests = [
'self',
'ByteArray',
- 'Encoding',
'Exceptions',
'Format',
'Fundamental',
@@ -217,7 +216,10 @@ endif
# tests using ES modules are also separate because they need an extra
# minijasmine flag
-modules_tests = ['ESModules']
+modules_tests = [
+ 'ESModules',
+ 'Encoding',
+]
if build_cairo
modules_tests += 'CairoModule'
endif
diff --git a/installed-tests/js/testEncoding.js b/installed-tests/js/testEncoding.js
index dcd05ee6..bd02ee75 100644
--- a/installed-tests/js/testEncoding.js
+++ b/installed-tests/js/testEncoding.js
@@ -4,7 +4,9 @@
// Some test inputs are derived from
https://github.com/denoland/deno/blob/923214c53725651792f6d55c5401bf6b475622ea/op_crates/web/08_text_encoding.js
// Data originally from https://encoding.spec.whatwg.org/encodings.json
-const {Gio} = imports.gi;
+import Gio from 'gi://Gio';
+
+import {arrayLikeWithExactContents} from './matchers.js';
/**
* Loads a JSON file from a URI and parses it.
@@ -23,36 +25,6 @@ function loadJSONFromResource(src) {
return json;
}
-/**
- * A jasmine asymmetric matcher which expects an array-like object
- * to contain the given element array in the same order with the
- * same length. Useful for testing typed arrays.
- *
- * @template T
- * @param {T[]} elements an array of elements to compare with
- * @returns
- */
-function withElements(elements) {
- return {
- /**
- * @param {ArrayLike<T>} compareTo an array-like object to compare to
- * @returns {boolean}
- */
- asymmetricMatch(compareTo) {
- return (
- compareTo.length === elements.length &&
- elements.every((e, i) => e === compareTo[i])
- );
- },
- /**
- * @returns {string}
- */
- jasmineToString() {
- return `${JSON.stringify(elements)}`;
- },
- };
-}
-
/**
* Encoded form of '𝓽𝓮𝔁𝓽'
*
@@ -83,7 +55,7 @@ describe('Text Encoding', function () {
const encoded = encoder.encode(input);
expect(encoded).toEqual(
- withElements([...encodedMultibyteCharArray()])
+ arrayLikeWithExactContents([...encodedMultibyteCharArray()])
);
});
});
@@ -98,7 +70,7 @@ describe('Text Encoding', function () {
expect(result.written).toBe(4);
expect(bytes).toEqual(
- withElements([0x74, 0x65, 0x78, 0x74, 0x00])
+ arrayLikeWithExactContents([0x74, 0x65, 0x78, 0x74, 0x00])
);
});
@@ -111,7 +83,10 @@ describe('Text Encoding', function () {
expect(result.written).toBe(16);
expect(bytes).toEqual(
- withElements([...encodedMultibyteCharArray(), 0x00])
+ arrayLikeWithExactContents([
+ ...encodedMultibyteCharArray(),
+ 0x00,
+ ])
);
});
@@ -124,7 +99,7 @@ describe('Text Encoding', function () {
expect(result.written).toBe(4);
expect(bytes).toEqual(
- withElements([
+ arrayLikeWithExactContents([
...encodedMultibyteCharArray().slice(0, 4),
0x00,
])
@@ -302,6 +277,75 @@ describe('Text Encoding', function () {
expect(decoded).toEqual(longResult);
});
+
+ it('can decode Big-5 HKSCS with supplemental characters', function () {
+ // The characters below roughly mean 'hard' or 'solid' and
+ // 'rooster' respectively. They were chosen for their Unicode
+ // and HKSCS positioning, not meaning.
+
+ // Big5-HKSCS bytes for the supplemental character 𠕇
+ const supplementalBytes = [250, 64];
+ // Big5-HKSCS bytes for the non-supplemental characters 公雞
+ const nonSupplementalBytes = [164, 189, 194, 251];
+
+ const decoder = new TextDecoder('big5-hkscs');
+
+ // We currently allocate 12 additional bytes of padding
+ // and a minimum of 256...
+
+ // This should produce 400 non-supplemental bytes (50 * 2 * 4)
+ // and 16 supplemental bytes (4 * 4)
+ const repeatedNonSupplementalBytes = new Array(50).fill(nonSupplementalBytes).flat();
+ const bytes = [
+ ...repeatedNonSupplementalBytes,
+ ...supplementalBytes,
+ ...repeatedNonSupplementalBytes,
+ ...supplementalBytes,
+ ...repeatedNonSupplementalBytes,
+ ...supplementalBytes,
+ ...repeatedNonSupplementalBytes,
+ ...supplementalBytes,
+ ];
+
+ const expectedNonSupplemental = new Array(50).fill('公雞');
+ const expected = [
+ ...expectedNonSupplemental,
+ '𠕇',
+ ...expectedNonSupplemental,
+ '𠕇',
+ ...expectedNonSupplemental,
+ '𠕇',
+ ...expectedNonSupplemental,
+ '𠕇',
+ ].join('');
+
+ // Calculate the number of bytes the UTF-16 characters should
+ // occupy.
+ const expectedU16Bytes = [...expected].reduce((prev, next) => {
+ const utf16code = next.codePointAt(0);
+
+ // Test whether this unit is supplemental
+ const additionalBytes = utf16code > 0xFFFF ? 2 : 0;
+
+ return prev + 2 + additionalBytes;
+ }, 0);
+
+
+ // We set a minimum buffer allocation of 256 bytes,
+ // this ensures that this test exceeds that.
+ expect(expectedU16Bytes / 2).toBeGreaterThan(256);
+
+ // The length of the input bytes should always be less
+ // than the expected output because UTF-16 uses 4 bytes
+ // to represent some characters HKSCS needs only 2 for.
+ expect(bytes.length).toBeLessThan(expectedU16Bytes);
+ // 4 supplemental characters, each with two additional bytes.
+ expect(bytes.length + 4 * 2).toBe(expectedU16Bytes);
+
+ const decoded = decoder.decode(new Uint8Array(bytes));
+
+ expect(decoded).toBe(expected);
+ });
});
describe('Single Byte Encoding Converter', function () {
diff --git a/js.gresource.xml b/js.gresource.xml
index a809930b..947049c2 100644
--- a/js.gresource.xml
+++ b/js.gresource.xml
@@ -9,6 +9,10 @@
<!-- ESM-based modules -->
<file>modules/esm/_bootstrap/default.js</file>
+
+ <file>modules/esm/_encoding/encoding.js</file>
+ <file>modules/esm/_encoding/encodingMap.js</file>
+ <file>modules/esm/_encoding/util.js</file>
<file>modules/esm/cairo.js</file>
<file>modules/esm/gettext.js</file>
@@ -44,10 +48,8 @@
<file>modules/core/_cairo.js</file>
<file>modules/core/_common.js</file>
- <file>modules/core/_encodings.js</file>
<file>modules/core/_format.js</file>
<file>modules/core/_gettext.js</file>
<file>modules/core/_signals.js</file>
- <file>modules/core/_text.js</file>
</gresource>
</gresources>
diff --git a/jsconfig.json b/jsconfig.json
new file mode 100644
index 00000000..43feb2ed
--- /dev/null
+++ b/jsconfig.json
@@ -0,0 +1,5 @@
+{
+ "compilerOptions": {
+ "lib": ["es2020"],
+ }
+}
\ No newline at end of file
diff --git a/modules/esm/_bootstrap/default.js b/modules/esm/_bootstrap/default.js
index fefeb51b..eb315af7 100644
--- a/modules/esm/_bootstrap/default.js
+++ b/modules/esm/_bootstrap/default.js
@@ -2,3 +2,6 @@
// SPDX-FileCopyrightText: 2021 Evan Welsh <contact evanwelsh com>
// Bootstrap file which supports ESM imports.
+
+// Bootstrap the Encoding API
+import '_encoding/encoding';
diff --git a/modules/core/_text.js b/modules/esm/_encoding/encoding.js
similarity index 53%
rename from modules/core/_text.js
rename to modules/esm/_encoding/encoding.js
index 39413b73..e84b752b 100644
--- a/modules/core/_text.js
+++ b/modules/esm/_encoding/encoding.js
@@ -1,22 +1,22 @@
// SPDX-License-Identifier: MIT OR LGPL-2.0-or-later
-// SPDX-FileCopyrightText: Evan Welsh
+// SPDX-FileCopyrightText: 2021 Evan Welsh <contact evanwelsh com>
-const Encoding = imports._encodingNative;
+const Encoding = import.meta.importSync('_encodingNative');
-const { getEncodingFromLabel } = imports._encodings;
+import {getEncodingFromLabel} from './encodingMap.js';
-var TextDecoder = class TextDecoder {
- /**
+class TextDecoder {
+ /**
* @type {string}
*/
encoding;
- /**
+ /**
* @type {boolean}
*/
ignoreBOM;
- /**
+ /**
* @type {boolean}
*/
fatal;
@@ -26,22 +26,24 @@ var TextDecoder = class TextDecoder {
}
/**
- * @param {string} encoding
- * @param {object} [options]
- * @param {boolean=} options.fatal
- * @param {boolean=} options.ignoreBOM
+ * @param {string} encoding The encoding to decode into
+ * @param {object} [options] Decoding options
+ * @param {boolean=} options.fatal Whether to throw or substitute when invalid characters are encountered
+ * @param {boolean=} options.ignoreBOM Whether to ignore the byte order for UTF-8 arrays
*/
constructor(encoding = 'utf-8', options = {}) {
- const { fatal = false, ignoreBOM = false } = options;
+ const {fatal = false, ignoreBOM = false} = options;
const encodingDefinition = getEncodingFromLabel(`${encoding}`);
- if (!encodingDefinition) {
+ if (!encodingDefinition)
throw new RangeError(`Invalid encoding label: '${encoding}'`);
- }
+
if (encodingDefinition.label === 'replacement') {
- throw new RangeError(`Unsupported replacement encoding: '${encoding}'`);
+ throw new RangeError(
+ `Unsupported replacement encoding: '${encoding}'`
+ );
}
Object.defineProperty(this, '_internalEncoding', {
@@ -73,19 +75,19 @@ var TextDecoder = class TextDecoder {
});
}
-
-
/**
- * @param {unknown} bytes
- * @param {object} [options]
- * @param {boolean=} options.stream
- * @returns
+ * @param {unknown} bytes a typed array of bytes to decode
+ * @param {object} [options] Decoding options
+ * @param {boolean=} options.stream Unsupported option. Whether to stream the decoded bytes.
+ * @returns
*/
decode(bytes, options = {}) {
- const { stream = false } = options;
+ const {stream = false} = options;
if (stream) {
- throw new Error(`TextDecoder does not implement the 'stream' option.`);
+ throw new Error(
+ 'TextDecoder does not implement the \'stream\' option.'
+ );
}
/** @type {Uint8Array} */
@@ -96,32 +98,37 @@ var TextDecoder = class TextDecoder {
} else if (bytes instanceof Uint8Array) {
input = bytes;
} else if (bytes instanceof Object.getPrototypeOf(Uint8Array)) {
- let { buffer, byteLength, byteOffset } = /** @type {Uint32Array} */ (bytes);
+ let {buffer, byteLength, byteOffset} =
+ /** @type {Uint32Array} */ bytes;
input = new Uint8Array(buffer, byteOffset, byteLength);
} else if (
- typeof bytes === "object" &&
+ typeof bytes === 'object' &&
bytes !== null &&
- "buffer" in bytes &&
+ 'buffer' in bytes &&
bytes.buffer instanceof ArrayBuffer
) {
- let { buffer, byteLength, byteOffset } = bytes;
- input = new Uint8Array(
- buffer,
- byteOffset,
- byteLength
- );
+ let {buffer, byteLength, byteOffset} = bytes;
+ input = new Uint8Array(buffer, byteOffset, byteLength);
} else if (bytes === undefined) {
input = new Uint8Array(0);
} else {
- throw new Error(`Provided input cannot be converted to ArrayBufferView or ArrayBuffer`);
+ throw new Error(
+ 'Provided input cannot be converted to ArrayBufferView or ArrayBuffer'
+ );
}
- if (this.ignoreBOM && input.length > 2 && input[0] === 0xEF && input[1] === 0xBB && input[2] ===
0xBF) {
- if (this.encoding !== 'utf-8') {
- throw new Error(`Cannot ignore BOM for non-UTF8 encoding.`);
- }
+ if (
+ this.ignoreBOM &&
+ input.length > 2 &&
+ input[0] === 0xef &&
+ input[1] === 0xbb &&
+ input[2] === 0xbf
+ ) {
+ if (this.encoding !== 'utf-8')
+ throw new Error('Cannot ignore BOM for non-UTF8 encoding.');
+
- let { buffer, byteLength, byteOffset } = input;
+ let {buffer, byteLength, byteOffset} = input;
input = new Uint8Array(buffer, byteOffset + 3, byteLength - 3);
}
@@ -129,7 +136,7 @@ var TextDecoder = class TextDecoder {
}
}
-var TextEncoder = class TextEncoder {
+class TextEncoder {
get [Symbol.toStringTag]() {
return 'TextEncoder';
}
@@ -140,11 +147,26 @@ var TextEncoder = class TextEncoder {
encode(input = '') {
// The TextEncoder specification only allows for UTF-8 encoding.
- return Encoding.encode(`${input}`, 'UTF-8');
+ return Encoding.encode(`${input}`, 'utf-8');
}
encodeInto(input = '', output = new Uint8Array()) {
// The TextEncoder specification only allows for UTF-8 encoding.
return Encoding.encodeInto(`${input}`, output);
}
-}
\ No newline at end of file
+}
+
+Object.defineProperties(globalThis, {
+ TextEncoder: {
+ configurable: false,
+ enumerable: true,
+ writable: false,
+ value: TextEncoder,
+ },
+ TextDecoder: {
+ configurable: false,
+ enumerable: true,
+ writable: false,
+ value: TextDecoder,
+ },
+});
diff --git a/modules/esm/_encoding/encodingMap.js b/modules/esm/_encoding/encodingMap.js
new file mode 100644
index 00000000..b0f17702
--- /dev/null
+++ b/modules/esm/_encoding/encodingMap.js
@@ -0,0 +1,305 @@
+// SPDX-License-Identifier: MIT OR LGPL-2.0-or-later
+// SPDX-FileCopyrightText: 2021 Evan Welsh <contact evanwelsh com>
+
+import {trimAsciiWhitespace} from './util.js';
+
+// Data derived from https://encoding.spec.whatwg.org/encodings.json
+const encodingMap = {
+ 'utf-8': [
+ 'unicode-1-1-utf-8',
+ 'unicode11utf8',
+ 'unicode20utf8',
+ 'utf-8',
+ 'utf8',
+ 'x-unicode20utf8',
+ ],
+ ibm866: ['866', 'cp866', 'csibm866', 'ibm866'],
+ 'iso-8859-2': [
+ 'csisolatin2',
+ 'iso-8859-2',
+ 'iso-ir-101',
+ 'iso8859-2',
+ 'iso88592',
+ 'iso_8859-2',
+ 'iso_8859-2:1987',
+ 'l2',
+ 'latin2',
+ ],
+ 'iso-8859-3': [
+ 'csisolatin3',
+ 'iso-8859-3',
+ 'iso-ir-109',
+ 'iso8859-3',
+ 'iso88593',
+ 'iso_8859-3',
+ 'iso_8859-3:1988',
+ 'l3',
+ 'latin3',
+ ],
+ 'iso-8859-4': [
+ 'csisolatin4',
+ 'iso-8859-4',
+ 'iso-ir-110',
+ 'iso8859-4',
+ 'iso88594',
+ 'iso_8859-4',
+ 'iso_8859-4:1988',
+ 'l4',
+ 'latin4',
+ ],
+ 'iso-8859-5': [
+ 'csisolatincyrillic',
+ 'cyrillic',
+ 'iso-8859-5',
+ 'iso-ir-144',
+ 'iso8859-5',
+ 'iso88595',
+ 'iso_8859-5',
+ 'iso_8859-5:1988',
+ ],
+ 'iso-8859-6': [
+ 'arabic',
+ 'asmo-708',
+ 'csiso88596e',
+ 'csiso88596i',
+ 'csisolatinarabic',
+ 'ecma-114',
+ 'iso-8859-6',
+ 'iso-8859-6-e',
+ 'iso-8859-6-i',
+ 'iso-ir-127',
+ 'iso8859-6',
+ 'iso88596',
+ 'iso_8859-6',
+ 'iso_8859-6:1987',
+ ],
+ 'iso-8859-7': [
+ 'csisolatingreek',
+ 'ecma-118',
+ 'elot_928',
+ 'greek',
+ 'greek8',
+ 'iso-8859-7',
+ 'iso-ir-126',
+ 'iso8859-7',
+ 'iso88597',
+ 'iso_8859-7',
+ 'iso_8859-7:1987',
+ 'sun_eu_greek',
+ ],
+ 'iso-8859-8': [
+ 'csiso88598e',
+ 'csisolatinhebrew',
+ 'hebrew',
+ 'iso-8859-8',
+ 'iso-8859-8-e',
+ 'iso-ir-138',
+ 'iso8859-8',
+ 'iso88598',
+ 'iso_8859-8',
+ 'iso_8859-8:1988',
+ 'visual',
+ ],
+ 'iso-8859-8-i': ['csiso88598i', 'iso-8859-8-i', 'logical'],
+ 'iso-8859-10': [
+ 'csisolatin6',
+ 'iso-8859-10',
+ 'iso-ir-157',
+ 'iso8859-10',
+ 'iso885910',
+ 'l6',
+ 'latin6',
+ ],
+ 'iso-8859-13': ['iso-8859-13', 'iso8859-13', 'iso885913'],
+ 'iso-8859-14': ['iso-8859-14', 'iso8859-14', 'iso885914'],
+ 'iso-8859-15': [
+ 'csisolatin9',
+ 'iso-8859-15',
+ 'iso8859-15',
+ 'iso885915',
+ 'iso_8859-15',
+ 'l9',
+ ],
+ 'iso-8859-16': ['iso-8859-16'],
+ 'koi8-r': ['cskoi8r', 'koi', 'koi8', 'koi8-r', 'koi8_r'],
+ 'koi8-u': ['koi8-ru', 'koi8-u'],
+ macintosh: ['csmacintosh', 'mac', 'macintosh', 'x-mac-roman'],
+ 'windows-874': [
+ 'dos-874',
+ 'iso-8859-11',
+ 'iso8859-11',
+ 'iso885911',
+ 'tis-620',
+ 'windows-874',
+ ],
+ 'windows-1250': ['cp1250', 'windows-1250', 'x-cp1250'],
+ 'windows-1251': ['cp1251', 'windows-1251', 'x-cp1251'],
+ 'windows-1252': [
+ 'ansi_x3.4-1968',
+ 'ascii',
+ 'cp1252',
+ 'cp819',
+ 'csisolatin1',
+ 'ibm819',
+ 'iso-8859-1',
+ 'iso-ir-100',
+ 'iso8859-1',
+ 'iso88591',
+ 'iso_8859-1',
+ 'iso_8859-1:1987',
+ 'l1',
+ 'latin1',
+ 'us-ascii',
+ 'windows-1252',
+ 'x-cp1252',
+ ],
+ 'windows-1253': ['cp1253', 'windows-1253', 'x-cp1253'],
+ 'windows-1254': [
+ 'cp1254',
+ 'csisolatin5',
+ 'iso-8859-9',
+ 'iso-ir-148',
+ 'iso8859-9',
+ 'iso88599',
+ 'iso_8859-9',
+ 'iso_8859-9:1989',
+ 'l5',
+ 'latin5',
+ 'windows-1254',
+ 'x-cp1254',
+ ],
+ 'windows-1255': ['cp1255', 'windows-1255', 'x-cp1255'],
+ 'windows-1256': ['cp1256', 'windows-1256', 'x-cp1256'],
+ 'windows-1257': ['cp1257', 'windows-1257', 'x-cp1257'],
+ 'windows-1258': ['cp1258', 'windows-1258', 'x-cp1258'],
+ 'x-mac-cyrillic': ['x-mac-cyrillic', 'x-mac-ukrainian'],
+ gbk: [
+ 'chinese',
+ 'csgb2312',
+ 'csiso58gb231280',
+ 'gb2312',
+ 'gb_2312',
+ 'gb_2312-80',
+ 'gbk',
+ 'iso-ir-58',
+ 'x-gbk',
+ ],
+ gb18030: ['gb18030'],
+ big5: [
+ 'big5',
+ // Unlike the standard WHATWG encoder
+ // the Hong Kong Supplementary Character Set
+ // is not bundled in big5 by iconv
+ // "big5-hkscs",
+ 'cn-big5',
+ 'csbig5',
+ 'x-x-big5',
+ ],
+ 'euc-jp': ['cseucpkdfmtjapanese', 'euc-jp', 'x-euc-jp'],
+ 'iso-2022-jp': ['csiso2022jp', 'iso-2022-jp'],
+ shift_jis: [
+ 'csshiftjis',
+ 'ms932',
+ 'ms_kanji',
+ 'shift-jis',
+ 'shift_jis',
+ 'sjis',
+ 'windows-31j',
+ 'x-sjis',
+ ],
+ 'euc-kr': [
+ 'cseuckr',
+ 'csksc56011987',
+ 'euc-kr',
+ 'iso-ir-149',
+ 'korean',
+ 'ks_c_5601-1987',
+ 'ks_c_5601-1989',
+ 'ksc5601',
+ 'ksc_5601',
+ 'windows-949',
+ ],
+ 'utf-16be': ['unicodefffe', 'utf-16be'],
+ 'utf-16le': [
+ 'csunicode',
+ 'iso-10646-ucs-2',
+ 'ucs-2',
+ 'unicode',
+ 'unicodefeff',
+ 'utf-16',
+ 'utf-16le',
+ ],
+};
+
+/**
+ * Construct a map from each potential label to the canonical label
+ * for an encoding.
+ */
+const encodings = new Map(
+ Object.entries(encodingMap).flatMap(([encoding, labels]) => {
+ return labels.map(label => [label, encoding]);
+ })
+);
+
+// Maps WHATWG specified labels to the appropriate iconv
+// encoding label if iconv does not support the WHATWG label.
+//
+// Mapping here preserves the WHATWG as the label on the
+// TextDecoder so this change is transparent to API users.
+const internalEncodings = new Map([
+ // iso-8859-8-i is functionally equivalent to iso-8859-8
+ // as we are not encoding or decoding control characters.
+ ['iso-8859-8-i', 'iso-8859-8'],
+ // iconv follows a different naming convention for this
+ // encoding
+ ['x-mac-cyrillic', 'MacCyrillic'],
+ // Support HKSCS as a standalone encoding, iconv doesn't
+ // bundle it with Big5 like WHATWG does...
+ ['big5-hkscs', 'big5-hkscs'],
+]);
+
+/**
+ * @typedef Encoding
+ * @property {string} internalLabel
+ * @property {string} label
+ */
+
+/**
+ * @param {string} label the encoding label
+ * @returns {Encoding | null}
+ */
+export function getEncodingFromLabel(label) {
+ const formattedLabel = trimAsciiWhitespace(label.toLowerCase());
+
+ let canonicalLabel = encodings.get(formattedLabel);
+
+ // Lookup an internal mapping using the canonical name, if found, or
+ // the formatted label otherwise.
+ //
+ // x-mac-ukrainian > x-mac-cyrillic > MacCyrillic
+ // (canonical label) (internal label)
+ //
+ // big5-hkscs > undefined > big5-hkscs
+ // (canonical label) (internal label)
+ //
+ let internalLabel = internalEncodings.get(
+ canonicalLabel ?? formattedLabel
+ );
+
+ // If both the canonical label and the internal encoding
+ // are not found, this encoding is unsupported.
+ if (!canonicalLabel && !internalLabel)
+ return null;
+
+ if (internalLabel) {
+ return {
+ label: canonicalLabel ?? formattedLabel,
+ internalLabel,
+ };
+ }
+
+ return {
+ label: canonicalLabel,
+ internalLabel: canonicalLabel,
+ };
+}
diff --git a/modules/esm/_encoding/util.js b/modules/esm/_encoding/util.js
new file mode 100644
index 00000000..9ee450fe
--- /dev/null
+++ b/modules/esm/_encoding/util.js
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: MIT
+// SPDX-FileCopyrightText: Node.js contributors. All rights reserved.
+
+// Modified from
https://github.com/nodejs/node/blob/78680c1cbc8b0c435963bc512e826b2a6227c315/lib/internal/encoding.js
+
+/**
+ * Trims ASCII whitespace from a string.
+ * `String.prototype.trim` removes non-ASCII whitespace.
+ *
+ * @param {string} label the label to trim
+ * @returns {string}
+ */
+export const trimAsciiWhitespace = label => {
+ let s = 0;
+ let e = label.length;
+ while (
+ s < e &&
+ (label[s] === '\u0009' ||
+ label[s] === '\u000a' ||
+ label[s] === '\u000c' ||
+ label[s] === '\u000d' ||
+ label[s] === '\u0020')
+ )
+ s++;
+
+ while (
+ e > s &&
+ (label[e - 1] === '\u0009' ||
+ label[e - 1] === '\u000a' ||
+ label[e - 1] === '\u000c' ||
+ label[e - 1] === '\u000d' ||
+ label[e - 1] === '\u0020')
+ )
+ e--;
+
+ return label.slice(s, e);
+};
diff --git a/modules/script/_bootstrap/default.js b/modules/script/_bootstrap/default.js
index fe354a02..952d7fe3 100644
--- a/modules/script/_bootstrap/default.js
+++ b/modules/script/_bootstrap/default.js
@@ -6,7 +6,6 @@
'use strict';
const {print, printerr, log, logError} = imports._print;
- const {TextEncoder, TextDecoder} = imports._text;
Object.defineProperties(exports, {
ARGV: {
@@ -17,18 +16,6 @@
return imports.system.programArgs;
},
},
- TextEncoder: {
- configurable: false,
- enumerable: true,
- writable: false,
- value: TextEncoder,
- },
- TextDecoder: {
- configurable: false,
- enumerable: true,
- writable: false,
- value: TextDecoder,
- },
print: {
configurable: false,
enumerable: true,
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
