Parse all normal strings as {js||js} strings.

The compiler processes these strings with js semantics.

Previously {js||js} where interpreted as template literal strings.
The internal encoding has been changed to use an attribute (@res.template) to detect template literal strings

Author: Iwan

Makefile

@@ -75,7 +75,7 @@ test: reanalyze build-native lib/test.exe
 	./lib/test.exe
 	./test.sh
 
-roundtrip-test: reanalyze bootstrap lib/test.exe
+roundtrip-test: reanalyze lib/test.exe
 	./lib/test.exe
 	ROUNDTRIP_TEST=1 ./test.sh
 

src/res_ast_conversion.ml

@@ -323,6 +323,8 @@ let hasUncurriedAttribute attrs = List.exists (fun attr -> match attr with
   | _ -> false
 ) attrs
 
+let templateLiteralAttr = (Location.mknoloc "res.template", Parsetree.PStr [])
+
 let normalize =
   let open Ast_mapper in
   { default_mapper with
@@ -368,7 +370,7 @@ let normalize =
         in
         let s = Parsetree.Pconst_string ((escapeTemplateLiteral txt), newTag) in
         {p with
-          ppat_attributes = mapper.attributes mapper p.ppat_attributes;
+          ppat_attributes = templateLiteralAttr::(mapper.attributes mapper p.ppat_attributes);
           ppat_desc = Ppat_constant s
         }
       | _ ->
@@ -396,7 +398,7 @@ let normalize =
         in
         let s = Parsetree.Pconst_string ((escapeTemplateLiteral txt), newTag) in
         {expr with
-          pexp_attributes = mapper.attributes mapper expr.pexp_attributes;
+          pexp_attributes= templateLiteralAttr::(mapper.attributes mapper expr.pexp_attributes);
           pexp_desc = Pexp_constant s
         }
       | Pexp_apply (

src/res_core.ml

@@ -136,6 +136,7 @@ let ternaryAttr = (Location.mknoloc "ns.ternary", Parsetree.PStr [])
 let ifLetAttr = (Location.mknoloc "ns.iflet", Parsetree.PStr [])
 let suppressFragileMatchWarningAttr = (Location.mknoloc "warning", Parsetree.PStr [Ast_helper.Str.eval (Ast_helper.Exp.constant (Pconst_string ("-4", None)))])
 let makeBracesAttr loc = (Location.mkloc "ns.braces" loc, Parsetree.PStr [])
+let templateLiteralAttr = (Location.mknoloc "res.template", Parsetree.PStr [])
 
 type stringLiteralState =
   | Start
@@ -911,12 +912,10 @@ let parseConstant p =
     let floatTxt = if isNegative then "-" ^ f else f in
     Parsetree.Pconst_float (floatTxt, suffix)
   | String s ->
-    let txt = if p.mode = ParseForTypeChecker then
-      parseStringLiteral s
+    if p.mode = ParseForTypeChecker then
+      Pconst_string (s, Some "js")
     else
-      s
-    in
-    Pconst_string(txt, None)
+      Pconst_string (s, None)
   | Codepoint {c; original} ->
     if p.mode = ParseForTypeChecker then
       Pconst_char c
@@ -1128,7 +1127,7 @@ let rec parsePattern ?(alias=true) ?(or_=true) p =
     end
   | Backtick ->
     let constant = parseTemplateConstant ~prefix:(Some "js") p in
-    Ast_helper.Pat.constant ~loc:(mkLoc startPos p.prevEndPos) constant
+    Ast_helper.Pat.constant ~attrs:[templateLiteralAttr] ~loc:(mkLoc startPos p.prevEndPos) constant
   | Lparen ->
     Parser.next p;
     begin match p.token with
@@ -2228,25 +2227,19 @@ and parseTemplateExpr ?(prefix="js") p =
     | TemplateTail txt ->
       Parser.next p;
       let loc = mkLoc startPos p.prevEndPos in
-      if String.length txt > 0 then
-        let txt = if p.mode = ParseForTypeChecker then parseTemplateStringLiteral txt else txt in
-        let str = Ast_helper.Exp.constant ~loc (Pconst_string(txt, Some prefix)) in
-        Ast_helper.Exp.apply ~loc hiddenOperator
-          [Nolabel, acc; Nolabel, str]
-      else
-        acc
+      let txt = if p.mode = ParseForTypeChecker then parseTemplateStringLiteral txt else txt in
+      let str = Ast_helper.Exp.constant ~attrs:[templateLiteralAttr] ~loc (Pconst_string(txt, Some prefix)) in
+      Ast_helper.Exp.apply ~loc hiddenOperator
+        [Nolabel, acc; Nolabel, str]
     | TemplatePart txt ->
       Parser.next p;
       let loc = mkLoc startPos p.prevEndPos in
       let expr = parseExprBlock p in
       let fullLoc = mkLoc startPos p.prevEndPos in
       let txt = if p.mode = ParseForTypeChecker then parseTemplateStringLiteral txt else txt in
-      let str = Ast_helper.Exp.constant ~loc (Pconst_string(txt, Some prefix)) in
+      let str = Ast_helper.Exp.constant ~attrs:[templateLiteralAttr] ~loc (Pconst_string(txt, Some prefix)) in
       let next =
-        let a = if String.length txt > 0 then
-            Ast_helper.Exp.apply ~loc:fullLoc hiddenOperator [Nolabel, acc; Nolabel, str]
-          else acc
-        in
+        let a = Ast_helper.Exp.apply ~loc:fullLoc hiddenOperator [Nolabel, acc; Nolabel, str] in
         Ast_helper.Exp.apply ~loc:fullLoc hiddenOperator
           [Nolabel, a; Nolabel, expr]
       in
@@ -2261,19 +2254,16 @@ and parseTemplateExpr ?(prefix="js") p =
   | TemplateTail txt ->
     Parser.next p;
     let txt = if p.mode = ParseForTypeChecker then parseTemplateStringLiteral txt else txt in
-    Ast_helper.Exp.constant ~loc:(mkLoc startPos p.prevEndPos) (Pconst_string(txt, Some prefix))
+    Ast_helper.Exp.constant ~attrs:[templateLiteralAttr] ~loc:(mkLoc startPos p.prevEndPos) (Pconst_string(txt, Some prefix))
   | TemplatePart txt ->
     Parser.next p;
     let constantLoc = mkLoc startPos p.prevEndPos in
     let expr = parseExprBlock p in
     let fullLoc = mkLoc startPos p.prevEndPos in
     let txt = if p.mode = ParseForTypeChecker then parseTemplateStringLiteral txt else txt in
-    let str = Ast_helper.Exp.constant ~loc:constantLoc (Pconst_string(txt, Some prefix)) in
+    let str = Ast_helper.Exp.constant ~attrs:[templateLiteralAttr] ~loc:constantLoc (Pconst_string(txt, Some prefix)) in
     let next =
-      if String.length txt > 0 then
-        Ast_helper.Exp.apply ~loc:fullLoc hiddenOperator [Nolabel, str; Nolabel, expr]
-      else
-        expr
+      Ast_helper.Exp.apply ~loc:fullLoc hiddenOperator [Nolabel, str; Nolabel, expr]
     in
     parseParts next
  | token ->

src/res_parsetree_viewer.ml

@@ -153,7 +153,7 @@ let processBracesAttr expr =
 let filterParsingAttrs attrs =
   List.filter (fun attr ->
     match attr with
-    | ({Location.txt = ("ns.ternary" | "ns.braces" | "bs" | "ns.iflet" | "ns.namedArgLoc")}, _) -> false
+    | ({Location.txt = ("ns.ternary" | "ns.braces" | "res.template" | "bs" | "ns.iflet" | "ns.namedArgLoc")}, _) -> false
     | _ -> true
   ) attrs
 
@@ -292,7 +292,7 @@ let isIfLetExpr expr = match expr with
 
 let hasAttributes attrs =
   List.exists (fun attr -> match attr with
-    | ({Location.txt = "bs" | "ns.ternary" | "ns.braces" | "ns.iflet"}, _) -> false
+    | ({Location.txt = "bs" | "res.template" | "ns.ternary" | "ns.braces" | "ns.iflet"}, _) -> false
     (* Remove the fragile pattern warning for iflet expressions *)
     | ({Location.txt="warning"}, PStr [{
       pstr_desc = Pstr_eval ({
@@ -453,13 +453,13 @@ let shouldInlineRhsBinaryExpr rhs = match rhs.pexp_desc with
 
 let filterPrinteableAttributes attrs =
   List.filter (fun attr -> match attr with
-    | ({Location.txt="bs" | "ns.ternary" | "ns.iflet" | "JSX"}, _) -> false
+    | ({Location.txt="bs" | "res.template" | "ns.ternary" | "ns.iflet" | "JSX"}, _) -> false
     | _ -> true
   ) attrs
 
 let partitionPrinteableAttributes attrs =
   List.partition (fun attr -> match attr with
-    | ({Location.txt="bs" | "ns.ternary" | "ns.iflet" | "JSX"}, _) -> false
+    | ({Location.txt="bs" | "res.template"| "ns.ternary" | "ns.iflet" | "JSX"}, _) -> false
     | _ -> true
   ) attrs
 
@@ -511,17 +511,25 @@ let rec collectPatternsFromListConstruct acc pattern =
     collectPatternsFromListConstruct (pat::acc) rest
   | _ -> List.rev acc, pattern
 
-(* Simple heuristic to detect template literal sugar:
- *  `${user.name} lastName` parses internally as user.name ++ ` lastName`.
- *  The thing is: the ++ operator (parsed as `^`)  will always have a ghost loc.
- *  A ghost loc is only produced by our parser.
- *  Hence, if we have that ghost operator, we know for sure it's a template literal. *)
+
+let hasTemplateLiteralAttr attrs = List.exists (fun attr -> match attr with
+| ({Location.txt = "res.template"}, _) -> true
+| _ -> false) attrs
+
 let isTemplateLiteral expr =
   match expr.pexp_desc with
   | Pexp_apply (
-      {pexp_desc = Pexp_ident {txt = Longident.Lident "^"; loc}},
-      [Nolabel, _; Nolabel, _]
-    ) when loc.loc_ghost -> true
+      {pexp_desc = Pexp_ident {txt = Longident.Lident "^"}},
+      [Nolabel, expr1; Nolabel, expr2]
+    ) ->
+    let firstIsTemplate = hasTemplateLiteralAttr expr1.pexp_attributes in
+    let sndIsTemplate = hasTemplateLiteralAttr expr2.pexp_attributes in
+    begin match (firstIsTemplate, sndIsTemplate) with
+    | (true, true) | (false, false) -> false
+    | _ -> true
+    end
+  | Pexp_constant (Pconst_string (_, Some "")) -> true
+  | Pexp_constant _ when hasTemplateLiteralAttr expr.pexp_attributes -> true
   | _ -> false
 
 (* Blue | Red | Green -> [Blue; Red; Green] *)

src/res_parsetree_viewer.mli

@@ -109,6 +109,7 @@ val collectPatternsFromListConstruct:
 val isBlockExpr : Parsetree.expression -> bool
 
 val isTemplateLiteral: Parsetree.expression -> bool
+val hasTemplateLiteralAttr: Parsetree.attributes -> bool
 
 val collectOrPatternChain:
  Parsetree.pattern -> Parsetree.pattern list

src/res_printer.ml

@@ -495,7 +495,7 @@ let printStringContents txt =
   let lines = String.split_on_char '\n' txt in
   Doc.join ~sep:Doc.literalLine (List.map Doc.text lines)
 
-let printConstant c = match c with
+let printConstant ?(templateLiteral=false) c = match c with
   | Parsetree.Pconst_integer (s, suffix) ->
     begin match suffix with
     | Some c -> Doc.text (s ^ (Char.escaped c))
@@ -511,11 +511,14 @@ let printConstant c = match c with
     if prefix = "INTERNAL_RES_CHAR_CONTENTS" then
       Doc.concat [Doc.text "'"; Doc.text txt; Doc.text "'"]
     else
+      let (lquote, rquote) =
+        if templateLiteral then ("`", "`") else ("\"", "\"")
+      in
       Doc.concat [
         if prefix = "js" then Doc.nil else Doc.text prefix;
-        Doc.text "`";
+        Doc.text lquote;
         printStringContents txt;
-        Doc.text "`";
+        Doc.text rquote;
       ]
   | Pconst_float (s, _) -> Doc.text s
   | Pconst_char c ->
@@ -2097,7 +2100,9 @@ and printPattern (p : Parsetree.pattern) cmtTbl =
   let patternWithoutAttributes = match p.ppat_desc with
   | Ppat_any -> Doc.text "_"
   | Ppat_var var -> printIdentLike var.txt
-  | Ppat_constant c -> printConstant c
+  | Ppat_constant c ->
+    let templateLiteral = ParsetreeViewer.hasTemplateLiteralAttr p.ppat_attributes in
+    printConstant ~templateLiteral c
   | Ppat_tuple patterns ->
     Doc.group(
       Doc.concat([
@@ -2540,7 +2545,8 @@ and printIfChain pexp_attributes ifs elseExpr cmtTbl =
 
 and printExpression (e : Parsetree.expression) cmtTbl =
   let printedExpression = match e.pexp_desc with
-  | Parsetree.Pexp_constant c -> printConstant c
+  | Parsetree.Pexp_constant c ->
+    printConstant ~templateLiteral:(ParsetreeViewer.isTemplateLiteral e) c
   | Pexp_construct _ when ParsetreeViewer.hasJsxAttribute e.pexp_attributes ->
     printJsxFragment e cmtTbl
   | Pexp_construct ({txt = Longident.Lident "()"}, _) -> Doc.text "()"

tests/conversion/reason/expected/bracedJsx.re.txt

@@ -114,7 +114,7 @@ let make = () => {
       ref={containerRef->ReactDOMRe.Ref.domRef}>
       {state.history
       ->Array.mapWithIndex((index, item) =>
-        <div key=j`$index` className=Styles.line>
+        <div key={j`$index`} className=Styles.line>
           {ReasonReact.string(
             switch item {
             | User(value) => userPrefix ++ value

tests/parsing/errors/expressions/expected/ambiguousArrow.res.txt

@@ -24,6 +24,6 @@
   1) (pattern): int => "test"
   2) (pattern: int) => "test"
 
-let a b = ("hi" : int)
-let x = ((let a = 1 in let b = 2 in fun pattern -> ("test" : int))
+let a b = ({js|hi|js} : int)
+let x = ((let a = 1 in let b = 2 in fun pattern -> ({js|test|js} : int))
   [@ns.braces ])

tests/parsing/errors/expressions/expected/emptyeof.res.txt

@@ -7,4 +7,5 @@
 
   This string is missing a double quote at the end
 
-let x = "\n"
+let x = {js|
+|js}

tests/parsing/errors/expressions/expected/ifLet.res.txt

@@ -35,10 +35,12 @@ switch result {
 | _ => ()
 }
 
-;;((match result with | Some x -> Js.log "The sky is blue" | _ -> ())
+;;((match result with | Some x -> Js.log {js|The sky is blue|js} | _ -> ())
   [@ns.iflet ][@warning "-4"])
 ;;((match result with
-    | Error x -> Js.log "The sky is red"
+    | Error x -> Js.log {js|The sky is red|js}
     | _ ->
-        (((match result with | Ok y -> Js.log "The sky is blue" | _ -> ()))
+        (((match result with
+           | Ok y -> Js.log {js|The sky is blue|js}
+           | _ -> ()))
         [@ns.iflet ][@warning "-4"]))[@ns.iflet ][@warning "-4"])

tests/parsing/errors/expressions/expected/taggedTemplateLiterals.res.txt

@@ -7,4 +7,4 @@
 
   Tagged template literals are currently restricted to names like: json`null`.
 
-;;{js|null|js}
+;;(({js|null|js})[@res.template ])

tests/parsing/errors/expressions/expected/unexpectedConstraint.res.txt

@@ -24,4 +24,4 @@
   ("hi": string)
 
 let x = ((let a = 1 in let b = 2 in (a + b : int))[@ns.braces ])
-let x = ("hi" : string)
+let x = ({js|hi|js} : string)

tests/parsing/errors/other/expected/breadcrumbs170.res.txt

@@ -18,7 +18,7 @@ module M = struct ;;match l with | None -> [] | Some l -> l#prop end
 ;;all
 ;;syntax
 ;;is
-;;"valid"
+;;{js|valid|js}
 ;;it
 ;;compiles
 ;;will

tests/parsing/errors/pattern/expected/missing.res.txt

@@ -44,5 +44,5 @@
   I was expecting a pattern to match on before the `=>`
 
 let 2 = [%rescript.exprhole ]
-let 4 = for [%rescript.patternhole ] = 0 to 10 do Js.log "for" done
+let 4 = for [%rescript.patternhole ] = 0 to 10 do Js.log {js|for|js} done
 ;;match x with | () -> [%rescript.exprhole ]

tests/parsing/errors/pattern/expected/templateLiteral.res.txt

@@ -34,5 +34,8 @@
 
   String interpolation is not supported in pattern matching.
 
-let zeroCoord = "0.0"
-;;match l with | "" -> () | "" -> () | _ -> ()
+let zeroCoord = {js|0.0|js}
+;;match l with
+  | (("")[@res.template ]) -> ()
+  | (("")[@res.template ]) -> ()
+  | _ -> ()

tests/parsing/errors/scanner/expected/escapeSequence.res.txt

@@ -19,5 +19,5 @@
 
   unknown escape sequence
 
-let x = "\\0"
-let x = "\\oAAA"
+let x = {js|\0|js}
+let x = {js|\oAAA|js}

tests/parsing/errors/scanner/expected/exoticIdent.res.txt

@@ -43,4 +43,5 @@
 
 let a = b
 ;;c
-;;" = 1\n"
+;;{js| = 1
+|js}

tests/parsing/errors/scanner/expected/unclosedString.res.txt

@@ -7,4 +7,5 @@
 
   This string is missing a double quote at the end
 
-let z = "eof\n"
+let z = {js|eof
+|js}

tests/parsing/errors/structure/expected/gh16A.res.txt

@@ -11,4 +11,4 @@
   I'm not sure what to parse here when looking at ")".
 
 module C = struct module T = (Fun)(struct ;;foo (a + c) (b + d) end) end
-;;Js.log "test"
+;;Js.log {js|test|js}