Skip to content

Commit 663f40a

Browse files
petebacondarwinthePunderWoman
authored andcommitted
refactor(compiler): support encoded entity tokens when lexing markup (#43132)
The lexer now splits encoded entity tokens out from text and attribute value tokens. Previously encoded entities would be decoded and the decoded value would be included as part of the text token of the surrounding text. Now the entities have their own tokens. There are two scenarios: text and attribute values. Previously the contents of `<div>Hello &amp; goodbye</div>` would be a single TEXT token. Now it will be three tokens: ``` TEXT: "Hello " ENCODED_ENTITY: "&", "&amp;" TEXT: " goodbye" ``` Previously the attribute value in `<div title="Hello &amp; goodbye">` would be a single text token. Now it will be three tokens: ``` ATTR_VALUE_TEXT: "Hello " ENCODED_ENTITY: "&", "&amp;" ATTR_VALUE_TEXT: " goodbye" ``` - ENCODED_ENTITY tokens have two parts: "decoded" and "encoded". - ENCODED_ENTITY tokens are always preceded and followed by either TEXT tokens or ATTR_VALUE_TEXT tokens, depending upon the context, even if they represent an empty string. The HTML parser has been modified to recombine these tokens to allow this refactoring to have limited effect in this commit. Further refactorings to use these new tokens will follow in subsequent commits. PR Close #43132
1 parent 8f565a2 commit 663f40a

9 files changed

Lines changed: 281 additions & 64 deletions

File tree

‎packages/compiler-cli/test/compliance/test_cases/source_mapping/inline_templates/GOLDEN_PARTIAL.js‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1652,6 +1652,78 @@ export declare class TestCmp {
16521652
static ɵcmp: i0.ɵɵComponentDeclaration<TestCmp, "test-cmp", never, {}, {}, never, never>;
16531653
}
16541654

1655+
/****************************************************************************************************
1656+
* PARTIAL FILE: i18n_message_placeholder_entities.js
1657+
****************************************************************************************************/
1658+
import { Component } from '@angular/core';
1659+
import * as i0 from "@angular/core";
1660+
export class TestCmp {
1661+
constructor() {
1662+
this.one = 1;
1663+
this.two = 2;
1664+
}
1665+
}
1666+
TestCmp.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "0.0.0-PLACEHOLDER", ngImport: i0, type: TestCmp, deps: [], target: i0.ɵɵFactoryTarget.Component });
1667+
TestCmp.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "12.0.0", version: "0.0.0-PLACEHOLDER", type: TestCmp, selector: "test-cmp", ngImport: i0, template: '<div i18n>Interpolation: {{ one }}&nbsp;Interpolation: {{ two }}</div>', isInline: true });
1668+
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "0.0.0-PLACEHOLDER", ngImport: i0, type: TestCmp, decorators: [{
1669+
type: Component,
1670+
args: [{
1671+
selector: 'test-cmp',
1672+
template: '<div i18n>Interpolation: {{ one }}&nbsp;Interpolation: {{ two }}</div>',
1673+
}]
1674+
}] });
1675+
//# sourceMappingURL=i18n_message_placeholder_entities.js.map
1676+
/****************************************************************************************************
1677+
* PARTIAL FILE: i18n_message_placeholder_entities.js.map
1678+
****************************************************************************************************/
1679+
{"version":3,"file":"i18n_message_placeholder_entities.js","sourceRoot":"","sources":["../i18n_message_placeholder_entities.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,SAAS,EAAC,MAAM,eAAe,CAAC;;AAMxC,MAAM,OAAO,OAAO;IAJpB;QAKE,QAAG,GAAG,CAAC,CAAC;QACR,QAAG,GAAG,CAAC,CAAC;KACT;;+GAHY,OAAO;mGAAP,OAAO,gDAFR,wEAAwE;sGAEvE,OAAO;kBAJnB,SAAS;mBAAC;oBACT,QAAQ,EAAE,UAAU;oBACpB,QAAQ,EAAE,wEAAwE;iBACnF"}
1680+
/****************************************************************************************************
1681+
* PARTIAL FILE: i18n_message_placeholder_entities.d.ts
1682+
****************************************************************************************************/
1683+
import * as i0 from "@angular/core";
1684+
export declare class TestCmp {
1685+
one: number;
1686+
two: number;
1687+
static ɵfac: i0.ɵɵFactoryDeclaration<TestCmp, never>;
1688+
static ɵcmp: i0.ɵɵComponentDeclaration<TestCmp, "test-cmp", never, {}, {}, never, never>;
1689+
}
1690+
1691+
/****************************************************************************************************
1692+
* PARTIAL FILE: i18n_message_placeholder_entities.js
1693+
****************************************************************************************************/
1694+
import { Component } from '@angular/core';
1695+
import * as i0 from "@angular/core";
1696+
export class TestCmp {
1697+
constructor() {
1698+
this.one = 1;
1699+
this.two = 2;
1700+
}
1701+
}
1702+
TestCmp.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "0.0.0-PLACEHOLDER", ngImport: i0, type: TestCmp, deps: [], target: i0.ɵɵFactoryTarget.Component });
1703+
TestCmp.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "12.0.0", version: "0.0.0-PLACEHOLDER", type: TestCmp, selector: "test-cmp", ngImport: i0, template: '<div i18n>Interpolation: {{ one }}&nbsp;Interpolation: {{ two }}</div>', isInline: true });
1704+
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "0.0.0-PLACEHOLDER", ngImport: i0, type: TestCmp, decorators: [{
1705+
type: Component,
1706+
args: [{
1707+
selector: 'test-cmp',
1708+
template: '<div i18n>Interpolation: {{ one }}&nbsp;Interpolation: {{ two }}</div>',
1709+
}]
1710+
}] });
1711+
//# sourceMappingURL=i18n_message_placeholder_entities.js.map
1712+
/****************************************************************************************************
1713+
* PARTIAL FILE: i18n_message_placeholder_entities.js.map
1714+
****************************************************************************************************/
1715+
{"version":3,"file":"i18n_message_placeholder_entities.js","sourceRoot":"","sources":["../i18n_message_placeholder_entities.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,SAAS,EAAC,MAAM,eAAe,CAAC;;AAMxC,MAAM,OAAO,OAAO;IAJpB;QAKE,QAAG,GAAG,CAAC,CAAC;QACR,QAAG,GAAG,CAAC,CAAC;KACT;;+GAHY,OAAO;mGAAP,OAAO,gDAFR,wEAAwE;sGAEvE,OAAO;kBAJnB,SAAS;mBAAC;oBACT,QAAQ,EAAE,UAAU;oBACpB,QAAQ,EAAE,wEAAwE;iBACnF"}
1716+
/****************************************************************************************************
1717+
* PARTIAL FILE: i18n_message_placeholder_entities.d.ts
1718+
****************************************************************************************************/
1719+
import * as i0 from "@angular/core";
1720+
export declare class TestCmp {
1721+
one: number;
1722+
two: number;
1723+
static ɵfac: i0.ɵɵFactoryDeclaration<TestCmp, never>;
1724+
static ɵcmp: i0.ɵɵComponentDeclaration<TestCmp, "test-cmp", never, {}, {}, never, never>;
1725+
}
1726+
16551727
/****************************************************************************************************
16561728
* PARTIAL FILE: i18n_message_interpolation_whitespace.js
16571729
****************************************************************************************************/

‎packages/compiler-cli/test/compliance/test_cases/source_mapping/inline_templates/TEST_CASES.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -749,6 +749,40 @@
749749
"sourceMap": true
750750
}
751751
},
752+
{
753+
"description": "should handle encoded entities in i18n message source-mappings (full compile)",
754+
"inputFiles": [
755+
"i18n_message_placeholder_entities.ts"
756+
],
757+
"compilationModeFilter": [
758+
"full compile"
759+
],
760+
"compilerOptions": {
761+
"sourceMap": true
762+
}
763+
},
764+
{
765+
"description": "should handle encoded entities in i18n message source-mappings (partial compile)",
766+
"inputFiles": [
767+
"i18n_message_placeholder_entities.ts"
768+
],
769+
"expectations": [
770+
{
771+
"files": [
772+
{
773+
"generated": "i18n_message_placeholder_entities.js",
774+
"expected": "i18n_message_placeholder_entities_partial.js"
775+
}
776+
]
777+
}
778+
],
779+
"compilationModeFilter": [
780+
"linked compile"
781+
],
782+
"compilerOptions": {
783+
"sourceMap": true
784+
}
785+
},
752786
{
753787
"description": "should correctly handle collapsed whitespace in interpolation placeholder i18n message source-mappings (full compile)",
754788
"inputFiles": [
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
`Interpolation: ${ // SOURCE: "/i18n_message_placeholder_entities.ts" "Interpolation: "
2+
…
3+
"\uFFFD0\uFFFD" // SOURCE: "/i18n_message_placeholder_entities.ts" "{{ one }}"
4+
…
5+
}:INTERPOLATION: Interpolation: ${ // SOURCE: "/i18n_message_placeholder_entities.ts" "&nbsp;Interpolation: "
6+
…
7+
"\uFFFD1\uFFFD" // SOURCE: "/i18n_message_placeholder_entities.ts" "{{ two }}"
8+
…
9+
}:INTERPOLATION_1:` // SOURCE: "/i18n_message_placeholder_entities.ts" "</div>"
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import {Component} from '@angular/core';
2+
3+
@Component({
4+
selector: 'test-cmp',
5+
template: '<div i18n>Interpolation: {{ one }}&nbsp;Interpolation: {{ two }}</div>',
6+
})
7+
export class TestCmp {
8+
one = 1;
9+
two = 2;
10+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
$localize`Interpolation: ${ // SOURCE: "/i18n_message_placeholder_entities.ts" "Interpolation: "
2+
…
3+
"\uFFFD0\uFFFD" // SOURCE: "/i18n_message_placeholder_entities.ts" "{{ one }}"
4+
…
5+
}:INTERPOLATION: Interpolation: ${ // SOURCE: "/i18n_message_placeholder_entities.ts" "&nbsp;Interpolation: "
6+
…
7+
"\uFFFD1\uFFFD" // SOURCE: "/i18n_message_placeholder_entities.ts" "{{ two }}"
8+
…
9+
}:INTERPOLATION_1:` // SOURCE: "/i18n_message_placeholder_entities.ts" "</div>'"

‎packages/compiler/src/ml_parser/lexer.ts‎

Lines changed: 44 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ export enum TokenType {
2323
ESCAPABLE_RAW_TEXT,
2424
RAW_TEXT,
2525
INTERPOLATION,
26+
ENCODED_ENTITY,
2627
COMMENT_START,
2728
COMMENT_END,
2829
CDATA_START,
@@ -398,19 +399,16 @@ class _Tokenizer {
398399
}
399400
}
400401

401-
private _readChar(decodeEntities: boolean): string {
402-
if (decodeEntities && this._cursor.peek() === chars.$AMPERSAND) {
403-
return this._decodeEntity();
404-
} else {
405-
// Don't rely upon reading directly from `_input` as the actual char value
406-
// may have been generated from an escape sequence.
407-
const char = String.fromCodePoint(this._cursor.peek());
408-
this._cursor.advance();
409-
return char;
410-
}
402+
private _readChar(): string {
403+
// Don't rely upon reading directly from `_input` as the actual char value
404+
// may have been generated from an escape sequence.
405+
const char = String.fromCodePoint(this._cursor.peek());
406+
this._cursor.advance();
407+
return char;
411408
}
412409

413-
private _decodeEntity(): string {
410+
private _consumeEntity(textTokenType: TokenType): void {
411+
this._beginToken(TokenType.ENCODED_ENTITY);
414412
const start = this._cursor.clone();
415413
this._cursor.advance();
416414
if (this._attemptCharCode(chars.$HASH)) {
@@ -430,7 +428,7 @@ class _Tokenizer {
430428
this._cursor.advance();
431429
try {
432430
const charCode = parseInt(strNum, isHex ? 16 : 10);
433-
return String.fromCharCode(charCode);
431+
this._endToken([String.fromCharCode(charCode), this._cursor.getChars(start)]);
434432
} catch {
435433
throw this._createError(
436434
_unknownEntityErrorMsg(this._cursor.getChars(start)), this._cursor.getSpan());
@@ -439,21 +437,25 @@ class _Tokenizer {
439437
const nameStart = this._cursor.clone();
440438
this._attemptCharCodeUntilFn(isNamedEntityEnd);
441439
if (this._cursor.peek() != chars.$SEMICOLON) {
440+
// No semicolon was found so abort the encoded entity token that was in progress, and treat
441+
// this as a text token
442+
this._beginToken(textTokenType, start);
442443
this._cursor = nameStart;
443-
return '&';
444-
}
445-
const name = this._cursor.getChars(nameStart);
446-
this._cursor.advance();
447-
const char = NAMED_ENTITIES[name];
448-
if (!char) {
449-
throw this._createError(_unknownEntityErrorMsg(name), this._cursor.getSpan(start));
444+
this._endToken(['&']);
445+
} else {
446+
const name = this._cursor.getChars(nameStart);
447+
this._cursor.advance();
448+
const char = NAMED_ENTITIES[name];
449+
if (!char) {
450+
throw this._createError(_unknownEntityErrorMsg(name), this._cursor.getSpan(start));
451+
}
452+
this._endToken([char, `&${name};`]);
450453
}
451-
return char;
452454
}
453455
}
454456

455-
private _consumeRawText(decodeEntities: boolean, endMarkerPredicate: () => boolean): Token {
456-
this._beginToken(decodeEntities ? TokenType.ESCAPABLE_RAW_TEXT : TokenType.RAW_TEXT);
457+
private _consumeRawText(consumeEntities: boolean, endMarkerPredicate: () => boolean): void {
458+
this._beginToken(consumeEntities ? TokenType.ESCAPABLE_RAW_TEXT : TokenType.RAW_TEXT);
457459
const parts: string[] = [];
458460
while (true) {
459461
const tagCloseStart = this._cursor.clone();
@@ -462,9 +464,16 @@ class _Tokenizer {
462464
if (foundEndMarker) {
463465
break;
464466
}
465-
parts.push(this._readChar(decodeEntities));
467+
if (consumeEntities && this._cursor.peek() === chars.$AMPERSAND) {
468+
this._endToken([this._processCarriageReturns(parts.join(''))]);
469+
parts.length = 0;
470+
this._consumeEntity(TokenType.ESCAPABLE_RAW_TEXT);
471+
this._beginToken(TokenType.ESCAPABLE_RAW_TEXT);
472+
} else {
473+
parts.push(this._readChar());
474+
}
466475
}
467-
return this._endToken([this._processCarriageReturns(parts.join(''))]);
476+
this._endToken([this._processCarriageReturns(parts.join(''))]);
468477
}
469478

470479
private _consumeComment(start: CharacterCursor) {
@@ -566,8 +575,8 @@ class _Tokenizer {
566575
}
567576
}
568577

569-
private _consumeRawTextWithTagClose(prefix: string, tagName: string, decodeEntities: boolean) {
570-
this._consumeRawText(decodeEntities, () => {
578+
private _consumeRawTextWithTagClose(prefix: string, tagName: string, consumeEntities: boolean) {
579+
this._consumeRawText(consumeEntities, () => {
571580
if (!this._attemptCharCode(chars.$LT)) return false;
572581
if (!this._attemptCharCode(chars.$SLASH)) return false;
573582
this._attemptCharCodeUntilFn(isNotWhitespace);
@@ -725,11 +734,16 @@ class _Tokenizer {
725734
const current = this._cursor.clone();
726735
if (this._interpolationConfig && this._attemptStr(this._interpolationConfig.start)) {
727736
this._endToken([this._processCarriageReturns(parts.join(''))], current);
737+
parts.length = 0;
728738
this._consumeInterpolation(interpolationTokenType, current, endInterpolation);
739+
this._beginToken(textTokenType);
740+
} else if (this._cursor.peek() === chars.$AMPERSAND) {
741+
this._endToken([this._processCarriageReturns(parts.join(''))]);
729742
parts.length = 0;
743+
this._consumeEntity(textTokenType);
730744
this._beginToken(textTokenType);
731745
} else {
732-
parts.push(this._readChar(true));
746+
parts.push(this._readChar());
733747
}
734748
}
735749

@@ -918,7 +932,9 @@ function mergeTextTokens(srcTokens: Token[]): Token[] {
918932
let lastDstToken: Token|undefined = undefined;
919933
for (let i = 0; i < srcTokens.length; i++) {
920934
const token = srcTokens[i];
921-
if (lastDstToken && lastDstToken.type === TokenType.TEXT && token.type === TokenType.TEXT) {
935+
if ((lastDstToken && lastDstToken.type === TokenType.TEXT && token.type === TokenType.TEXT) ||
936+
(lastDstToken && lastDstToken.type === TokenType.ATTR_VALUE_TEXT &&
937+
token.type === TokenType.ATTR_VALUE_TEXT)) {
922938
lastDstToken.parts[0]! += token.parts[0];
923939
lastDstToken.sourceSpan.end = token.sourceSpan.end;
924940
} else {

‎packages/compiler/src/ml_parser/parser.ts‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -226,20 +226,21 @@ class _TreeBuilder {
226226
}
227227
}
228228

229-
// For now recombine text and interpolation tokens
230-
if (this._peek.type === lex.TokenType.INTERPOLATION) {
231-
while (this._peek.type === lex.TokenType.INTERPOLATION ||
232-
this._peek.type === lex.TokenType.TEXT) {
233-
token = this._advance();
234-
if (token.type === lex.TokenType.INTERPOLATION) {
235-
// For backward compatibility we decode HTML entities that appear in interpolation
236-
// expressions. This is arguably a bug, but it could be a considerable breaking change to
237-
// fix it. It should be addressed in a larger project to refactor the entire parser/lexer
238-
// chain after View Engine has been removed.
239-
text += token.parts.join('').replace(/&([^;]+);/g, decodeEntity);
240-
} else {
241-
text += token.parts.join('');
242-
}
229+
// For now recombine text, interpolation and entity tokens
230+
while (this._peek.type === lex.TokenType.INTERPOLATION ||
231+
this._peek.type === lex.TokenType.TEXT ||
232+
this._peek.type === lex.TokenType.ENCODED_ENTITY) {
233+
token = this._advance();
234+
if (token.type === lex.TokenType.INTERPOLATION) {
235+
// For backward compatibility we decode HTML entities that appear in interpolation
236+
// expressions. This is arguably a bug, but it could be a considerable breaking change to
237+
// fix it. It should be addressed in a larger project to refactor the entire parser/lexer
238+
// chain after View Engine has been removed.
239+
text += token.parts.join('').replace(/&([^;]+);/g, decodeEntity);
240+
} else if (token.type === lex.TokenType.ENCODED_ENTITY) {
241+
text += token.parts[0];
242+
} else {
243+
text += token.parts.join('');
243244
}
244245
}
245246

@@ -369,23 +370,26 @@ class _TreeBuilder {
369370
this._advance();
370371
}
371372

372-
// Consume the value
373+
// Consume the attribute value
373374
let value = '';
374375
let valueStartSpan: ParseSourceSpan|undefined = undefined;
375376
let valueEnd: ParseLocation|undefined = undefined;
376377
if (this._peek.type === lex.TokenType.ATTR_VALUE_TEXT) {
377378
valueStartSpan = this._peek.sourceSpan;
378379
valueEnd = this._peek.sourceSpan.end;
379-
// For now we are recombining text and interpolation tokens
380+
// For now recombine text, interpolation and entity tokens
380381
while (this._peek.type === lex.TokenType.ATTR_VALUE_TEXT ||
381-
this._peek.type === lex.TokenType.ATTR_VALUE_INTERPOLATION) {
382+
this._peek.type === lex.TokenType.ATTR_VALUE_INTERPOLATION ||
383+
this._peek.type === lex.TokenType.ENCODED_ENTITY) {
382384
let valueToken = this._advance();
383385
if (valueToken.type === lex.TokenType.ATTR_VALUE_INTERPOLATION) {
384386
// For backward compatibility we decode HTML entities that appear in interpolation
385387
// expressions. This is arguably a bug, but it could be a considerable breaking change to
386388
// fix it. It should be addressed in a larger project to refactor the entire parser/lexer
387389
// chain after View Engine has been removed.
388390
value += valueToken.parts.join('').replace(/&([^;]+);/g, decodeEntity);
391+
} else if (valueToken.type === lex.TokenType.ENCODED_ENTITY) {
392+
value += valueToken.parts[0];
389393
} else {
390394
value += valueToken.parts.join('');
391395
}

‎packages/compiler/test/ml_parser/html_parser_spec.ts‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -249,9 +249,9 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
249249
'<div>', '</div>'
250250
],
251251
[html.Element, 'span', 1, '<span>x {{ expr }<!---->} y</span>', '<span>', '</span>'],
252-
[html.Text, 'x {{ expr }', 2, 'x {{ expr }'],
252+
[html.Text, 'x {{ expr }', 2, ['x '], ['{{', ' expr }'], [''], 'x {{ expr }'],
253253
[html.Comment, '', 2, '<!--'],
254-
[html.Text, '} y', 2, '} y'],
254+
[html.Text, '} y', 2, ['} y'], '} y'],
255255
[html.Element, 'div', 1, '<div></div>', '<div>', '</div>'],
256256
]);
257257
expect(errors).toEqual([]);
@@ -269,7 +269,8 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
269269

270270
it('should parse attributes containing unquoted interpolation', () => {
271271
expect(humanizeDom(parser.parse('<div foo={{message}}></div>', 'TestComp'))).toEqual([
272-
[html.Element, 'div', 0], [html.Attribute, 'foo', '{{message}}']
272+
[html.Element, 'div', 0],
273+
[html.Attribute, 'foo', '{{message}}', [''], ['{{', 'message', '}}'], ['']]
273274
]);
274275
});
275276

@@ -285,7 +286,10 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
285286
[
286287
html.Attribute, '[attr]', `[
287288
{text: 'some text',url:'//www.google.com'},
288-
{text:'other text',url:'//www.google.com'}]`
289+
{text:'other text',url:'//www.google.com'}]`,
290+
[`[
291+
{text: 'some text',url:'//www.google.com'},
292+
{text:'other text',url:'//www.google.com'}]`]
289293
],
290294
]);
291295
});
@@ -346,7 +350,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
346350
const {errors, rootNodes} = parser.parse('<div p="{{ abc"><span></span>', 'TestComp');
347351
expect(humanizeNodes(rootNodes, true)).toEqual([
348352
[html.Element, 'div', 0, '<div p="{{ abc">', '<div p="{{ abc">', null],
349-
[html.Attribute, 'p', '{{ abc', 'p="{{ abc"'],
353+
[html.Attribute, 'p', '{{ abc', [''], ['{{', ' abc'], [''], 'p="{{ abc"'],
350354
[html.Element, 'span', 1, '<span></span>', '<span>', '</span>'],
351355
]);
352356
expect(humanizeErrors(errors)).toEqual([]);

0 commit comments

Comments
 (0)