wip: parse interpolation

packages/compiler-core/src/parser/Tokenizer.ts

@@ -65,9 +65,15 @@ export const enum CharCodes {
   RightSquare = 93 // "]"
 }
 
+const defaultDelimitersOpen = [123, 123] // "{{"
+const defaultDelimitersClose = [125, 125] // "}}"
+
 /** All the states the tokenizer can be in. */
 const enum State {
   Text = 1,
+  Interpolation,
+
+  // Tags
   BeforeTagName, // After <
   InTagName,
   InSelfClosingTag,
@@ -134,6 +140,8 @@ export interface Callbacks {
   ontext(start: number, endIndex: number): void
   ontextentity(codepoint: number, endIndex: number): void
 
+  oninterpolation(start: number, endIndex: number): void
+
   onopentagname(start: number, endIndex: number): void
   onopentagend(endIndex: number): void
   onselfclosingtag(endIndex: number): void
@@ -190,14 +198,9 @@ export default class Tokenizer {
   /** Reocrd newline positions for fast line / column calculation */
   private newlines: number[] = []
 
-  private readonly decodeEntities: boolean
   private readonly entityDecoder: EntityDecoder
 
-  constructor(
-    { decodeEntities = true }: { decodeEntities?: boolean },
-    private readonly cbs: Callbacks
-  ) {
-    this.decodeEntities = decodeEntities
+  constructor(private readonly cbs: Callbacks) {
     this.entityDecoder = new EntityDecoder(htmlDecodeTree, (cp, consumed) =>
       this.emitCodePoint(cp, consumed)
     )
@@ -211,6 +214,8 @@ export default class Tokenizer {
     this.baseState = State.Text
     this.currentSequence = undefined!
     this.newlines.length = 0
+    this.delimiterOpen = defaultDelimitersOpen
+    this.delimiterClose = defaultDelimitersClose
   }
 
   /**
@@ -238,17 +243,45 @@ export default class Tokenizer {
   }
 
   private stateText(c: number): void {
-    if (
-      c === CharCodes.Lt ||
-      (!this.decodeEntities && this.fastForwardTo(CharCodes.Lt))
-    ) {
+    if (c === CharCodes.Lt) {
       if (this.index > this.sectionStart) {
         this.cbs.ontext(this.sectionStart, this.index)
       }
       this.state = State.BeforeTagName
       this.sectionStart = this.index
-    } else if (this.decodeEntities && c === CharCodes.Amp) {
+    } else if (c === CharCodes.Amp) {
       this.startEntity()
+    } else if (this.matchDelimiter(c, this.delimiterOpen)) {
+      if (this.index > this.sectionStart) {
+        this.cbs.ontext(this.sectionStart, this.index)
+      }
+      this.state = State.Interpolation
+      this.sectionStart = this.index
+      this.index += this.delimiterOpen.length
+    }
+  }
+
+  public delimiterOpen: number[] = defaultDelimitersOpen
+  public delimiterClose: number[] = defaultDelimitersClose
+  private matchDelimiter(c: number, delimiter: number[]): boolean {
+    if (c === delimiter[0]) {
+      const l = delimiter.length
+      for (let i = 1; i < l; i++) {
+        if (this.buffer.charCodeAt(this.index + i) !== delimiter[i]) {
+          return false
+        }
+      }
+      return true
+    }
+    return false
+  }
+
+  private stateInterpolation(c: number): void {
+    if (this.matchDelimiter(c, this.delimiterClose)) {
+      this.index += this.delimiterClose.length
+      this.cbs.oninterpolation(this.sectionStart, this.index)
+      this.state = State.Text
+      this.sectionStart = this.index
     }
   }
 
@@ -302,7 +335,7 @@ export default class Tokenizer {
     } else if (this.sequenceIndex === 0) {
       if (this.currentSequence === Sequences.TitleEnd) {
         // We have to parse entities in <title> tags.
-        if (this.decodeEntities && c === CharCodes.Amp) {
+        if (c === CharCodes.Amp) {
           this.startEntity()
         }
       } else if (this.fastForwardTo(CharCodes.Lt)) {
@@ -592,7 +625,7 @@ export default class Tokenizer {
     }
   }
   private handleInAttributeValue(c: number, quote: number) {
-    if (c === quote || (!this.decodeEntities && this.fastForwardTo(quote))) {
+    if (c === quote) {
       this.cbs.onattribdata(this.sectionStart, this.index)
       this.sectionStart = -1
       this.cbs.onattribend(
@@ -600,7 +633,7 @@ export default class Tokenizer {
         this.index + 1
       )
       this.state = State.BeforeAttributeName
-    } else if (this.decodeEntities && c === CharCodes.Amp) {
+    } else if (c === CharCodes.Amp) {
       this.startEntity()
     }
   }
@@ -617,7 +650,7 @@ export default class Tokenizer {
       this.cbs.onattribend(QuoteType.Unquoted, this.index)
       this.state = State.BeforeAttributeName
       this.stateBeforeAttributeName(c)
-    } else if (this.decodeEntities && c === CharCodes.Amp) {
+    } else if (c === CharCodes.Amp) {
       this.startEntity()
     }
   }
@@ -715,6 +748,10 @@ export default class Tokenizer {
           this.stateText(c)
           break
         }
+        case State.Interpolation: {
+          this.stateInterpolation(c)
+          break
+        }
         case State.SpecialStartSequence: {
           this.stateSpecialStartSequence(c)
           break

packages/compiler-core/src/parser/index.ts

@@ -47,6 +47,7 @@ export const defaultParserOptions: MergedParserOptions = {
   isVoidTag: NO,
   isPreTag: NO,
   isCustomElement: NO,
+  // TODO handle entities
   decodeEntities: (rawText: string): string =>
     rawText.replace(decodeRE, (_, p1) => decodeMap[p1]),
   onError: defaultOnError,
@@ -69,162 +70,182 @@ let inPre = 0
 // let inVPre = 0
 const stack: ElementNode[] = []
 
-const tokenizer = new Tokenizer(
-  // TODO handle entities
-  { decodeEntities: true },
-  {
-    ontext(start, end) {
-      onText(getSlice(start, end), start, end)
-    },
-
-    ontextentity(cp, end) {
-      onText(fromCodePoint(cp), end - 1, end)
-    },
-
-    onopentagname(start, end) {
-      emitOpenTag(getSlice(start, end), start)
-    },
-
-    onopentagend(end) {
-      endOpenTag(end)
-    },
-
-    onclosetag(start, end) {
-      const name = getSlice(start, end)
-      if (!currentOptions.isVoidTag(name)) {
-        const pos = stack.findIndex(e => e.tag === name)
-        if (pos !== -1) {
-          for (let index = 0; index <= pos; index++) {
-            onCloseTag(stack.shift()!, end)
-          }
-        }
-      }
-    },
-
-    onselfclosingtag(end) {
-      closeCurrentTag(end)
-    },
+const tokenizer = new Tokenizer({
+  ontext(start, end) {
+    onText(getSlice(start, end), start, end)
+  },
 
-    onattribname(start, end) {
-      // plain attribute
-      currentProp = {
-        type: NodeTypes.ATTRIBUTE,
-        name: getSlice(start, end),
-        value: undefined,
-        loc: getLoc(start)
-      }
-    },
-
-    ondirname(start, end) {
-      const raw = getSlice(start, end)
-      const name =
-        raw === '.' || raw === ':'
-          ? 'bind'
-          : raw === '@'
-          ? 'on'
-          : raw === '#'
-          ? 'slot'
-          : raw.slice(2)
-      currentProp = {
-        type: NodeTypes.DIRECTIVE,
-        name,
-        exp: undefined,
-        arg: undefined,
-        modifiers: [],
-        loc: getLoc(start)
-      }
-    },
+  ontextentity(cp, end) {
+    onText(fromCodePoint(cp), end - 1, end)
+  },
 
-    ondirarg(start, end) {
-      const arg = getSlice(start, end)
-      const isStatic = arg[0] !== `[`
-      ;(currentProp as DirectiveNode).arg = {
+  oninterpolation(start, end) {
+    let innerStart = start + tokenizer.delimiterOpen.length
+    let innerEnd = end - tokenizer.delimiterClose.length
+    while (isWhitespace(currentInput.charCodeAt(innerStart))) {
+      innerStart++
+    }
+    while (isWhitespace(currentInput.charCodeAt(innerEnd - 1))) {
+      innerEnd--
+    }
+    addNode({
+      type: NodeTypes.INTERPOLATION,
+      content: {
         type: NodeTypes.SIMPLE_EXPRESSION,
-        content: arg,
-        isStatic,
-        constType: isStatic
-          ? ConstantTypes.CAN_STRINGIFY
-          : ConstantTypes.NOT_CONSTANT,
-        loc: getLoc(start, end)
-      }
-    },
-    ondirmodifier(start, end) {
-      ;(currentProp as DirectiveNode).modifiers.push(getSlice(start, end))
-    },
-
-    onattribdata(start, end) {
-      currentAttrValue += getSlice(start, end)
-      if (currentAttrStartIndex < 0) currentAttrStartIndex = start
-      currentAttrEndIndex = end
-    },
-
-    onattribentity(codepoint) {
-      currentAttrValue += fromCodePoint(codepoint)
-    },
-
-    onattribnameend(end) {
-      // check duplicate attrs
-      const start = currentProp!.loc.start.offset
-      const name = getSlice(start, end)
-      if (currentAttrs.has(name)) {
-        currentProp = null
-        // TODO emit error DUPLICATE_ATTRIBUTE
-        throw new Error(`duplicate attr ${name}`)
-      } else {
-        currentAttrs.add(name)
+        isStatic: false,
+        // Set `isConstant` to false by default and will decide in transformExpression
+        constType: ConstantTypes.NOT_CONSTANT,
+        content: getSlice(innerStart, innerEnd),
+        loc: getLoc(innerStart, innerEnd)
+      },
+      loc: getLoc(start, end)
+    })
+  },
+
+  onopentagname(start, end) {
+    emitOpenTag(getSlice(start, end), start)
+  },
+
+  onopentagend(end) {
+    endOpenTag(end)
+  },
+
+  onclosetag(start, end) {
+    const name = getSlice(start, end)
+    if (!currentOptions.isVoidTag(name)) {
+      const pos = stack.findIndex(e => e.tag === name)
+      if (pos !== -1) {
+        for (let index = 0; index <= pos; index++) {
+          onCloseTag(stack.shift()!, end)
+        }
       }
-    },
-
-    onattribend(quote, end) {
-      if (currentElement && currentProp) {
-        if (currentAttrValue) {
-          if (currentProp.type === NodeTypes.ATTRIBUTE) {
-            // assign value
-            currentProp!.value = {
-              type: NodeTypes.TEXT,
-              content: currentAttrValue,
-              loc:
-                quote === QuoteType.Unquoted
-                  ? getLoc(currentAttrStartIndex, currentAttrEndIndex)
-                  : getLoc(currentAttrStartIndex - 1, currentAttrEndIndex + 1)
-            }
-          } else {
-            // directive
-            currentProp.exp = {
-              type: NodeTypes.SIMPLE_EXPRESSION,
-              content: currentAttrValue,
-              isStatic: false,
-              // Treat as non-constant by default. This can be potentially set
-              // to other values by `transformExpression` to make it eligible
-              // for hoisting.
-              constType: ConstantTypes.NOT_CONSTANT,
-              loc: getLoc(currentAttrStartIndex, currentAttrEndIndex)
-            }
+    }
+  },
+
+  onselfclosingtag(end) {
+    closeCurrentTag(end)
+  },
+
+  onattribname(start, end) {
+    // plain attribute
+    currentProp = {
+      type: NodeTypes.ATTRIBUTE,
+      name: getSlice(start, end),
+      value: undefined,
+      loc: getLoc(start)
+    }
+  },
+
+  ondirname(start, end) {
+    const raw = getSlice(start, end)
+    const name =
+      raw === '.' || raw === ':'
+        ? 'bind'
+        : raw === '@'
+        ? 'on'
+        : raw === '#'
+        ? 'slot'
+        : raw.slice(2)
+    currentProp = {
+      type: NodeTypes.DIRECTIVE,
+      name,
+      exp: undefined,
+      arg: undefined,
+      modifiers: [],
+      loc: getLoc(start)
+    }
+  },
+
+  ondirarg(start, end) {
+    const arg = getSlice(start, end)
+    const isStatic = arg[0] !== `[`
+    ;(currentProp as DirectiveNode).arg = {
+      type: NodeTypes.SIMPLE_EXPRESSION,
+      content: arg,
+      isStatic,
+      constType: isStatic
+        ? ConstantTypes.CAN_STRINGIFY
+        : ConstantTypes.NOT_CONSTANT,
+      loc: getLoc(start, end)
+    }
+  },
+
+  ondirmodifier(start, end) {
+    ;(currentProp as DirectiveNode).modifiers.push(getSlice(start, end))
+  },
+
+  onattribdata(start, end) {
+    currentAttrValue += getSlice(start, end)
+    if (currentAttrStartIndex < 0) currentAttrStartIndex = start
+    currentAttrEndIndex = end
+  },
+
+  onattribentity(codepoint) {
+    currentAttrValue += fromCodePoint(codepoint)
+  },
+
+  onattribnameend(end) {
+    // check duplicate attrs
+    const start = currentProp!.loc.start.offset
+    const name = getSlice(start, end)
+    if (currentAttrs.has(name)) {
+      currentProp = null
+      // TODO emit error DUPLICATE_ATTRIBUTE
+      throw new Error(`duplicate attr ${name}`)
+    } else {
+      currentAttrs.add(name)
+    }
+  },
+
+  onattribend(quote, end) {
+    if (currentElement && currentProp) {
+      if (currentAttrValue) {
+        if (currentProp.type === NodeTypes.ATTRIBUTE) {
+          // assign value
+          currentProp!.value = {
+            type: NodeTypes.TEXT,
+            content: currentAttrValue,
+            loc:
+              quote === QuoteType.Unquoted
+                ? getLoc(currentAttrStartIndex, currentAttrEndIndex)
+                : getLoc(currentAttrStartIndex - 1, currentAttrEndIndex + 1)
+          }
+        } else {
+          // directive
+          currentProp.exp = {
+            type: NodeTypes.SIMPLE_EXPRESSION,
+            content: currentAttrValue,
+            isStatic: false,
+            // Treat as non-constant by default. This can be potentially set
+            // to other values by `transformExpression` to make it eligible
+            // for hoisting.
+            constType: ConstantTypes.NOT_CONSTANT,
+            loc: getLoc(currentAttrStartIndex, currentAttrEndIndex)
           }
         }
-        currentProp.loc.end = tokenizer.getPos(end)
-        currentElement.props.push(currentProp!)
-      }
-      currentAttrValue = ''
-      currentAttrStartIndex = currentAttrEndIndex = -1
-    },
-
-    oncomment(start, end, offset) {
-      // TODO oncomment
-    },
-
-    onend() {
-      const end = currentInput.length - 1
-      for (let index = 0; index < stack.length; index++) {
-        onCloseTag(stack[index], end)
       }
-    },
-
-    oncdata(start, end, offset) {
-      // TODO throw error
+      currentProp.loc.end = tokenizer.getPos(end)
+      currentElement.props.push(currentProp!)
     }
+    currentAttrValue = ''
+    currentAttrStartIndex = currentAttrEndIndex = -1
+  },
+
+  oncomment(start, end, offset) {
+    // TODO oncomment
+  },
+
+  onend() {
+    const end = currentInput.length - 1
+    for (let index = 0; index < stack.length; index++) {
+      onCloseTag(stack[index], end)
+    }
+  },
+
+  oncdata(start, end, offset) {
+    // TODO throw error
   }
-)
+})
 
 function getSlice(start: number, end: number) {
   return currentInput.slice(start, end)
@@ -283,7 +304,7 @@ function onText(content: string, start: number, end: number) {
       loc: {
         start: tokenizer.getPos(start),
         end: tokenizer.getPos(end),
-        source: content
+        source: ''
       }
     })
   }
@@ -413,8 +434,17 @@ function reset() {
   stack.length = 0
 }
 
+function toCharCodes(str: string): number[] {
+  return str.split('').map(c => c.charCodeAt(0))
+}
+
 export function baseParse(input: string, options?: ParserOptions): RootNode {
   reset()
+  const delimiters = options?.delimiters
+  if (delimiters) {
+    tokenizer.delimiterOpen = toCharCodes(delimiters[0])
+    tokenizer.delimiterClose = toCharCodes(delimiters[1])
+  }
   currentInput = input
   currentOptions = extend({}, defaultParserOptions, options)
   const root = (currentRoot = createRoot([]))