Skip to content
This repository was archived by the owner on Jun 15, 2023. It is now read-only.

Unicode support #433

Merged
merged 4 commits into from
Aug 24, 2021
Merged

Unicode support #433

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f2c9bf2
## Unicode support
May 5, 2021
a6a6d71
Rename Character token to Codepoint token.
Jun 4, 2021
051abc1
Add comment about codepoint literal encoding for printer.
Jun 4, 2021
f5166a0
Parse all normal strings as {js||js} strings.
Aug 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 15 additions & 11 deletions .depend
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@ src/res_comment.cmx : src/res_comment.cmi
src/res_comment.cmi :
src/res_comments_table.cmx : src/res_parsetree_viewer.cmx src/res_doc.cmx \
src/res_comment.cmx
src/res_core.cmx : src/res_token.cmx src/res_scanner.cmx src/res_printer.cmx \
src/res_parser.cmx src/res_js_ffi.cmx src/res_grammar.cmx src/res_doc.cmx \
src/res_diagnostics.cmx src/res_comments_table.cmx src/res_core.cmi
src/res_core.cmx : src/res_utf8.cmx src/res_token.cmx src/res_scanner.cmx \
src/res_printer.cmx src/res_parser.cmx src/res_js_ffi.cmx \
src/res_grammar.cmx src/res_doc.cmx src/res_diagnostics.cmx \
src/res_comments_table.cmx src/res_core.cmi
src/res_core.cmi : src/res_parser.cmi
src/res_diagnostics.cmx : src/res_token.cmx src/res_grammar.cmx \
src/res_diagnostics_printing_utils.cmx src/res_diagnostics.cmi
Expand Down Expand Up @@ -60,16 +61,19 @@ src/res_parser.cmi : src/res_token.cmx src/res_scanner.cmi \
src/res_comment.cmi
src/res_parsetree_viewer.cmx : src/res_parsetree_viewer.cmi
src/res_parsetree_viewer.cmi :
src/res_printer.cmx : src/res_token.cmx src/res_parsetree_viewer.cmx \
src/res_parens.cmx src/res_doc.cmx src/res_comments_table.cmx \
src/res_comment.cmx src/res_printer.cmi
src/res_printer.cmx : src/res_utf8.cmx src/res_token.cmx \
src/res_parsetree_viewer.cmx src/res_parens.cmx src/res_doc.cmx \
src/res_comments_table.cmx src/res_comment.cmx src/res_printer.cmi
src/res_printer.cmi : src/res_doc.cmi src/res_comments_table.cmx \
src/res_comment.cmi
src/res_reporting.cmx : src/res_token.cmx src/res_grammar.cmx
src/res_scanner.cmx : src/res_token.cmx src/res_diagnostics.cmx \
src/res_comment.cmx src/res_scanner.cmi
src/res_scanner.cmx : src/res_utf8.cmx src/res_token.cmx \
src/res_diagnostics.cmx src/res_comment.cmx src/res_scanner.cmi
src/res_scanner.cmi : src/res_token.cmx src/res_diagnostics.cmi
src/res_token.cmx : src/res_comment.cmx
tests/res_test.cmx : src/res_token.cmx src/res_parser.cmx \
src/res_outcome_printer.cmx src/res_multi_printer.cmx src/res_io.cmx \
src/res_driver.cmx src/res_core.cmx
src/res_utf8.cmx : src/res_utf8.cmi
src/res_utf8.cmi :
tests/res_test.cmx : tests/res_utf8_test.cmx src/res_token.cmx \
src/res_parser.cmx src/res_outcome_printer.cmx src/res_multi_printer.cmx \
src/res_io.cmx src/res_driver.cmx src/res_core.cmx
tests/res_utf8_test.cmx : src/res_utf8.cmx
5 changes: 3 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ API_FILES = \
src/res_parsetree_viewer.cmx\
src/res_parens.cmx\
src/res_comments_table.cmx\
src/res_utf8.cmx\
src/res_printer.cmx\
src/res_scanner.cmx\
src/res_js_ffi.cmx\
Expand All @@ -40,7 +41,7 @@ API_FILES = \

CLI_FILES = $(API_FILES) src/res_cli.cmx

TEST_FILES = $(API_FILES) tests/res_test.cmx
TEST_FILES = $(API_FILES) tests/res_utf8_test.cmx tests/res_test.cmx

.DEFAULT_GOAL := build-native

Expand Down Expand Up @@ -74,7 +75,7 @@ test: reanalyze build-native lib/test.exe
./lib/test.exe
./test.sh

roundtrip-test: reanalyze bootstrap lib/test.exe
roundtrip-test: reanalyze lib/test.exe
./lib/test.exe
ROUNDTRIP_TEST=1 ./test.sh

Expand Down
39 changes: 0 additions & 39 deletions benchmarks/Benchmark.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ module Printer = Res_printer

module IO: sig
val readFile: string -> string
val readStdin: unit -> string
end = struct
(* random chunk size: 2^15, TODO: why do we guess randomly? *)
let chunkSize = 32768
Expand All @@ -26,21 +25,6 @@ end = struct
)
in
loop ()

let readStdin () =
let buffer = Buffer.create chunkSize in
let chunk = (Bytes.create [@doesNotRaise]) chunkSize in
let rec loop () =
let len = try input stdin chunk 0 chunkSize with Invalid_argument _ -> 0 in
if len == 0 then (
close_in_noerr stdin;
Buffer.contents buffer
) else (
Buffer.add_subbytes buffer chunk 0 len;
loop ()
)
in
loop ()
end

module Time: sig
Expand Down Expand Up @@ -188,29 +172,6 @@ end = struct
done
end

module Profile: sig
val record : name:string -> (unit -> 'a) -> 'a
val print: unit -> unit
end = struct
let state = Hashtbl.create 2

let record ~name f =
let startTime = Time.now() in
let result = f() in
let endTime = Time.now() in

Hashtbl.add state name (Time.diff startTime endTime);
result

let print () =
let report = Hashtbl.fold (fun k v acc ->
let line = Printf.sprintf "%s: %fms\n" k (Time.print v) in
acc ^ line
) state "\n\n"
in
print_endline report
end

module Benchmarks: sig
val run: unit -> unit
end = struct
Expand Down
6 changes: 4 additions & 2 deletions src/res_ast_conversion.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,8 @@ let hasUncurriedAttribute attrs = List.exists (fun attr -> match attr with
| _ -> false
) attrs

let templateLiteralAttr = (Location.mknoloc "res.template", Parsetree.PStr [])

let normalize =
let open Ast_mapper in
{ default_mapper with
Expand Down Expand Up @@ -368,7 +370,7 @@ let normalize =
in
let s = Parsetree.Pconst_string ((escapeTemplateLiteral txt), newTag) in
{p with
ppat_attributes = mapper.attributes mapper p.ppat_attributes;
ppat_attributes = templateLiteralAttr::(mapper.attributes mapper p.ppat_attributes);
ppat_desc = Ppat_constant s
}
| _ ->
Expand Down Expand Up @@ -396,7 +398,7 @@ let normalize =
in
let s = Parsetree.Pconst_string ((escapeTemplateLiteral txt), newTag) in
{expr with
pexp_attributes = mapper.attributes mapper expr.pexp_attributes;
pexp_attributes= templateLiteralAttr::(mapper.attributes mapper expr.pexp_attributes);
pexp_desc = Pexp_constant s
}
| Pexp_apply (
Expand Down
7 changes: 5 additions & 2 deletions src/res_ast_debugger.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,13 @@ module SexpAst = struct
string txt;
optChar tag;
]
| Pconst_char c ->
| Pconst_char _ ->
Sexp.list [
Sexp.atom "Pconst_char";
]
| Pconst_string(_, Some "INTERNAL_RES_CHAR_CONTENTS") ->
Sexp.list [
Sexp.atom "Pconst_char";
Sexp.atom (Char.escaped c);
]
| Pconst_string (txt, tag) ->
Sexp.list [
Expand Down
8 changes: 6 additions & 2 deletions src/res_cli.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ module ResClflags: sig
val file: string ref
val interface: bool ref
val ppx: string ref
val typechecker: bool ref

val parse: unit -> unit
end = struct
Expand All @@ -176,6 +177,7 @@ end = struct
let interface = ref false
let ppx = ref ""
let file = ref ""
let typechecker = ref false

let usage = "\n**This command line is for the repo developer's testing purpose only. DO NOT use it in production**!\n\n" ^
"Usage:\n rescript <options> <file>\n\n" ^
Expand All @@ -192,6 +194,7 @@ end = struct
("-width", Arg.Int (fun w -> width := w), "Specify the line length for the printer (formatter)");
("-interface", Arg.Unit (fun () -> interface := true), "Parse as interface");
("-ppx", Arg.String (fun txt -> ppx := txt), "Apply a specific built-in ppx before parsing, none or jsx. Default: none");
("-typechecker", Arg.Unit (fun () -> typechecker := true), "Parses the ast as it would be passed to the typechecker and not the printer")
]

let parse () = Arg.parse spec (fun f -> file := f) usage
Expand All @@ -200,7 +203,7 @@ end
module CliArgProcessor = struct
type backend = Parser: ('diagnostics) Res_driver.parsingEngine -> backend [@@unboxed]

let processFile ~isInterface ~width ~recover ~origin ~target ~ppx filename =
let processFile ~isInterface ~width ~recover ~origin ~target ~ppx ~typechecker filename =
let len = String.length filename in
let processInterface =
isInterface || len > 0 && (String.get [@doesNotRaise]) filename (len - 1) = 'i'
Expand Down Expand Up @@ -233,7 +236,7 @@ module CliArgProcessor = struct
in

let forPrinter = match target with
| "res" | "sexp" -> true
| "res" | "sexp" when not typechecker -> true
| _ -> false
in

Expand Down Expand Up @@ -292,5 +295,6 @@ let [@raises Invalid_argument, Failure, exit] () =
~target:!ResClflags.print
~origin:!ResClflags.origin
~ppx:!ResClflags.ppx
~typechecker:!ResClflags.typechecker
!ResClflags.file
end
Loading