Grammar: macros

Author: ehuss

src/expressions.md

@@ -26,6 +26,7 @@
 >       | [_BreakExpression_]\
 >       | [_RangeExpression_]\
 >       | [_ReturnExpression_]\
+>       | [_MacroInvocation_]\
 >    )
 >
 > _ExpressionWithBlock_ :\
@@ -341,6 +342,7 @@ They are never allowed before:
 [_LazyBooleanExpression_]:        expressions/operator-expr.html#lazy-boolean-operators
 [_LiteralExpression_]:            expressions/literal-expr.html
 [_LoopExpression_]:               expressions/loop-expr.html
+[_MacroInvocation_]:              macros.html#macro-invocation
 [_MatchExpression_]:              expressions/match-expr.html
 [_MethodCallExpression_]:         expressions/method-call-expr.html
 [_OperatorExpression_]:           expressions/operator-expr.html

src/items.md

@@ -17,8 +17,8 @@
 >       | [_Trait_]\
 >       | [_Implementation_]\
 >       | [_ExternBlock_]\
->       | _Macro_\
->       | _MacroDefinition_\
+>       | [_MacroInvocationSemi_] [^novis]\
+>       | [_MacroRulesDefinition_] [^novis]\
 >    )
 
 An _item_ is a component of a crate. Items are organized within a crate by a
@@ -55,12 +55,16 @@ qualified by the name of the enclosing item, or is private to the enclosing
 item (in the case of functions). The grammar specifies the exact locations in
 which sub-item declarations may appear.
 
+[^novis]: Macros may not start with a visibility modifier.
+
 [_ConstantItem_]: items/constant-items.html
 [_Enumeration_]: items/enumerations.html
 [_ExternBlock_]: items/external-blocks.html
 [_ExternCrate_]: items/extern-crates.html
 [_Function_]: items/functions.html
 [_Implementation_]: items/implementations.html
+[_MacroInvocationSemi_]: macros.html#macro-invocation
+[_MacroRulesDefinition_]: macros-by-example.html
 [_Module_]: items/modules.html
 [_OuterAttribute_]: attributes.html
 [_StaticItem_]: items/static-items.html

src/items/implementations.md

@@ -13,7 +13,7 @@
 > _InherentImplItem_ :\
 > &nbsp;&nbsp; [_OuterAttribute_]<sup>\*</sup>\
 > &nbsp;&nbsp; [_Visibility_]<sup>?</sup>\
-> &nbsp;&nbsp; ( [_ConstantItem_] | [_Function_] | [_Method_] )
+> &nbsp;&nbsp; ( [_ConstantItem_] | [_Function_] | [_Method_] | [_MacroInvocationSemi_]&nbsp;[^novis] )
 >
 > _TraitImpl_ :\
 > &nbsp;&nbsp; `unsafe`<sup>?</sup> `impl` [_Generics_] `!`<sup>?</sup>
@@ -27,7 +27,7 @@
 > _TraitImplItem_ :\
 > &nbsp;&nbsp; [_OuterAttribute_]<sup>\*</sup>\
 > &nbsp;&nbsp; [_Visibility_]<sup>?</sup>\
-> &nbsp;&nbsp; ( [_TypeAlias_] | [_ConstantItem_] | [_Function_] | [_Method_] )
+> &nbsp;&nbsp; ( [_TypeAlias_] | [_ConstantItem_] | [_Function_] | [_Method_] | [_MacroInvocationSemi_]&nbsp;[^novis] )
 
 An _implementation_ is an item that associates items with an _implementing type_.
 Implementations are defined with the keyword `impl` and contain functions
@@ -175,11 +175,14 @@ attributes must come before any associated items. That attributes that have
 meaning here are [`cfg`], [`deprecated`], [`doc`], and [the lint check
 attributes].
 
+[^novis]: Macro invocations may not start with a visibility modifier.
+
 [IDENTIFIER]: identifiers.html
 [_ConstantItem_]: items/constant-items.html
 [_Function_]: items/functions.html
 [_Generics_]: items/generics.html
 [_InnerAttribute_]: attributes.html
+[_MacroInvocationSemi_]: macros.html#macro-invocation
 [_Method_]: items/associated-items.html#methods
 [_OuterAttribute_]: attributes.html
 [_TypeAlias_]: items/type-aliases.html

src/items/traits.md

@@ -9,7 +9,13 @@
 >    `}`
 >
 > _TraitItem_ :\
-> &nbsp;&nbsp; [_OuterAttribute_]<sup>\*</sup> (_TraitFunc_ | _TraitMethod_ | _TraitConst_ | _TraitType_)
+> &nbsp;&nbsp; [_OuterAttribute_]<sup>\*</sup> (\
+> &nbsp;&nbsp; &nbsp;&nbsp; &nbsp;&nbsp; _TraitFunc_\
+> &nbsp;&nbsp; &nbsp;&nbsp; | _TraitMethod_\
+> &nbsp;&nbsp; &nbsp;&nbsp; | _TraitConst_\
+> &nbsp;&nbsp; &nbsp;&nbsp; | _TraitType_\
+> &nbsp;&nbsp; &nbsp;&nbsp; | [_MacroInvocationSemi_]\
+> &nbsp;&nbsp; )
 >
 > _TraitFunc_ :\
 > &nbsp;&nbsp; &nbsp;&nbsp; _TraitFunctionDecl_ ( `;` | [_BlockExpression_] )
@@ -205,6 +211,7 @@ trait T {
 [_FunctionQualifiers_]: items/functions.html
 [_FunctionReturnType_]: items/functions.html
 [_Generics_]: items/generics.html
+[_MacroInvocationSemi_]: macros.html#macro-invocation
 [_OuterAttribute_]: attributes.html
 [_Pattern_]: patterns.html
 [_SelfParam_]: items/associated-items.html#methods

src/macros-by-example.md

@@ -1,12 +1,51 @@
 # Macros By Example
 
+> **<sup>Syntax</sup>**\
+> _MacroRulesDefinition_ :\
+> &nbsp;&nbsp; `macro_rules` `!` [IDENTIFIER] _MacroRulesDef_
+>
+> _MacroRulesDef_ :\
+> &nbsp;&nbsp; &nbsp;&nbsp; `(` _MacroRules_ `)` `;`\
+> &nbsp;&nbsp; | `[` _MacroRules_ `]` `;`\
+> &nbsp;&nbsp; | `{` _MacroRules_ `}`
+>
+> _MacroRules_ :\
+> &nbsp;&nbsp; _MacroRule_ ( `;` _MacroRule_ )<sup>\*</sup> `;`<sup>?</sup>
+>
+> _MacroRule_ :\
+> &nbsp;&nbsp; _MacroMatcher_ `=>` _MacroTranscriber_
+>
+> _MacroMatcher_ :\
+> &nbsp;&nbsp; &nbsp;&nbsp; `(` _MacroMatch_<sup>\*</sup> `)`\
+> &nbsp;&nbsp; | `[` _MacroMatch_<sup>\*</sup> `]`\
+> &nbsp;&nbsp; | `{` _MacroMatch_<sup>\*</sup> `}`
+>
+> _MacroMatch_ :\
+> &nbsp;&nbsp; &nbsp;&nbsp; [_Token_]<sub>_except $ and delimiters_</sub>\
+> &nbsp;&nbsp; | _MacroMatcher_\
+> &nbsp;&nbsp; | `$` [IDENTIFIER] `:` _MacroFragSpec_\
+> &nbsp;&nbsp; | `$` `(` _MacroMatch_<sup>+</sup> `)` _MacroRepSep_<sup>?</sup> _MacroKleeneOp_
+>
+> _MacroFragSpec_ :\
+> &nbsp;&nbsp; &nbsp;&nbsp; `block` | `expr` | `ident` | `item` | `lifetime` | `literal`\
+> &nbsp;&nbsp; | `meta` | `pat` | `path` | `stmt` | `tt` | `ty` | `vis`
+>
+> _MacroRepSep_ :\
+> &nbsp;&nbsp; [_Token_]<sub>_except delimiters and kleene operators_</sub>
+>
+> _MacroKleeneOp_<sub>2015</sub> :\
+> &nbsp;&nbsp; `*` | `+`
+>
+> _MacroKleeneOp_<sub>2018+</sub> :\
+> &nbsp;&nbsp; `*` | `+` | `?`
+>
+> _MacroTranscriber_ :\
+> &nbsp;&nbsp; [_DelimTokenTree_]
+
 `macro_rules` allows users to define syntax extension in a declarative way.  We
 call such extensions "macros by example" or simply "macros".
 
-Currently, macros can expand to expressions, statements, items, or patterns.
-
-(A `sep_token` is any token other than `*` and `+`. A `non_special_token` is
-any token other than a delimiter or `$`.)
+Macros can expand to expressions, statements, items, types, or patterns.
 
 The macro expander looks up macro invocations by name, and tries each macro
 rule in turn. It transcribes the first successful match. Matching and
@@ -27,11 +66,11 @@ syntax named by _designator_. Valid designators are:
 * `expr`: an [expression]
 * `ty`: a [type]
 * `ident`: an [identifier] or [keyword]
-* `path`: a [path]
-* `tt`: a token tree (a single [token] by matching `()`, `[]`, or `{}`)
+* `path`: a [_TypePath_] style path
+* `tt`: a [token tree]&nbsp;(a single [token] or tokens in matching delimiters `()`, `[]`, or `{}`)
 * `meta`: the contents of an [attribute]
-* `lifetime`: a lifetime. Examples: `'static`, `'a`.
-* `vis`: a (visibility qualifier)[visibility-and-privacy]
+* `lifetime`: a [lifetime]. Examples: `'static`, `'a`.
+* `vis`: a [visibility qualifier]
 
 [item]: items.html
 [block]: expressions/block-expr.html
@@ -41,10 +80,12 @@ syntax named by _designator_. Valid designators are:
 [type]: types.html
 [identifier]: identifiers.html
 [keyword]: keywords.html
-[path]: paths.html
+[_TypePath_]: paths.html#paths-in-types
+[token tree]: macros.html#macro-invocation
 [token]: tokens.html
 [attribute]: attributes.html
-[visibility-and-privacy]: visibility-and-privacy.html
+[lifetime]: tokens.html#lifetimes-and-loop-labels
+[visibility qualifier]: visibility-and-privacy.html
 
 In the transcriber, the
 designator is already known, and so only the name of a matched nonterminal comes
@@ -94,9 +135,36 @@ Rust syntax is restricted in two ways:
    a macro definition like `$i:expr [ , ]` is not legal, because `[` could be part
    of an expression. A macro definition like `$i:expr,` or `$i:expr;` would be legal,
    however, because `,` and `;` are legal separators. See [RFC 550] for more information.
+   Specifically:
+
+   * `expr` and `stmt` may only be followed by one of `=>`, `,`, or `;`.
+   * `pat` may only be followed by one of `=>`, `,`, `=`, `|`, `if`, or `in`.
+   * `path` and `ty` may only be followed by one of `=>`, `,`, `=`, `|`, `;`,
+     `:`, `>`, `[`, `{`, `as`, `where`, or a macro variable of `block`
+     fragment type.
+   * `vis` may only be followed by one of `,`, `priv`, a raw identifier, any
+     token that can begin a type, or a macro variable of `ident`, `ty`, or
+     `path` fragment type.
+   * All other fragment types have no restrictions.
+
 2. The parser must have eliminated all ambiguity by the time it reaches a `$`
    _name_ `:` _designator_. This requirement most often affects name-designator
    pairs when they occur at the beginning of, or immediately after, a `$(...)*`;
-   requiring a distinctive token in front can solve the problem.
+   requiring a distinctive token in front can solve the problem. For example:
+
+   ```rust
+   // The matcher `$($i:ident)* $e:expr` would be ambiguous because the parser
+   // would be forced to choose between an identifier or an expression. Use some
+   // token to distinguish them.
+   macro_rules! example {
+       ($(I $i:ident)* E $e:expr) => { ($($i)-*) * $e };
+   }
+   let foo = 2;
+   let bar = 3;
+   // The following expands to `(foo - bar) * 5`
+   example!(I foo I bar E 5);
+   ```
 
 [RFC 550]: https://github.com/rust-lang/rfcs/blob/master/text/0550-macro-future-proofing.md
+[_DelimTokenTree_]: macros.html
+[_Token_]: tokens.html

src/macros.md

@@ -9,6 +9,94 @@ There are two ways to define new macros:
 * [Macros by Example] define new syntax in a higher-level, declarative way.
 * [Procedural Macros] can be used to implement custom derive.
 
+## Macro Invocation
+
+> **<sup>Syntax</sup>**\
+> _MacroInvocation_ :\
+> &nbsp;&nbsp; [_SimplePath_] `!` _DelimTokenTree_
+>
+> _DelimTokenTree_ :\
+> &nbsp;&nbsp; &nbsp;&nbsp;  `(` _TokenTree_<sup>\*</sup> `)`\
+> &nbsp;&nbsp; | `[` _TokenTree_<sup>\*</sup> `]`\
+> &nbsp;&nbsp; | `{` _TokenTree_<sup>\*</sup> `}`
+>
+> _TokenTree_ :\
+> &nbsp;&nbsp; [_Token_]<sub>_except delimiters_</sub> | _DelimTokenTree_
+>
+> _MacroInvocationSemi_ :\
+> &nbsp;&nbsp; &nbsp;&nbsp; [_SimplePath_] `!` `(` _TokenTree_<sup>\*</sup> `)` `;`\
+> &nbsp;&nbsp; | [_SimplePath_] `!` `[` _TokenTree_<sup>\*</sup> `]` `;`\
+> &nbsp;&nbsp; | [_SimplePath_] `!` `{` _TokenTree_<sup>\*</sup> `}`
+
+A macro invocation executes a macro at compile time and replaces the
+invocation with the result of the macro. Macros may be invoked in the
+following situations:
+
+* [Expressions] and [statements]
+* [Patterns]
+* [Types]
+* [Items] including [associated items]
+* [`macro_rules`] transcribers
+
+When used as an item or a statement, the _MacroInvocationSemi_ form is used
+where a semicolon is required at the end when not using curly braces.
+[Visibility qualifiers] are never allowed before a macro invocation or
+[`macro_rules`] definition.
+
+```rust
+// Used as an expression.
+let x = vec![1,2,3];
+
+// Used as a statement.
+println!("Hello!");
+
+// Used in a pattern.
+macro_rules! pat {
+    ($i:ident) => (Some($i))
+}
+
+if let pat!(x) = Some(1) {
+    assert_eq!(x, 1);
+}
+
+// Used in a type.
+macro_rules! Tuple {
+    { $A:ty, $B:ty } => { ($A, $B) };
+}
+
+type N2 = Tuple!(i32, i32);
+
+// Used as an item.
+# use std::cell::RefCell;
+thread_local!(static FOO: RefCell<u32> = RefCell::new(1));
+
+// Used as an associated item.
+macro_rules! const_maker {
+    ($t:ty, $v:tt) => { const CONST: $t = $v; };
+}
+trait T {
+    const_maker!{i32, 7}
+}
+
+// Macro calls within macros.
+macro_rules! example {
+    () => { println!("Macro call in a macro!") };
+}
+example!();
+```
+
 [Macros by Example]: macros-by-example.html
 [Procedural Macros]: procedural-macros.html
+[_SimplePath_]: paths.html#simple-paths
+[_Token_]: tokens.html
+[associated items]: items/associated-items.html
 [compiler plugins]: ../unstable-book/language-features/plugin.html
+[delimiters]: tokens.html#delimiters
+[expressions]: expressions.html
+[items]: items.html
+[`macro_rules`]: macros-by-example.html
+[patterns]: patterns.html
+[statements]: statements.html
+[tokens]: tokens.html
+[types]: types.html
+[visibility qualifiers]: visibility-and-privacy.html

src/patterns.md

@@ -12,7 +12,8 @@
 >    | [_TuplePattern_]\
 >    | [_GroupedPattern_]\
 >    | [_SlicePattern_]\
->    | [_PathPattern_]
+>    | [_PathPattern_]\
+>    | [_MacroInvocation_]
 
 Patterns are used to match values against structures and to,
 optionally, bind variables to values inside these structures. They are also
@@ -653,6 +654,7 @@ refer to refutable constants or enum variants for enums with multiple variants.
 [_GroupedPattern_]: #grouped-patterns
 [_IdentifierPattern_]: #identifier-patterns
 [_LiteralPattern_]: #literal-patterns
+[_MacroInvocation_]: macros.html#macro-invocation
 [_PathInExpression_]: paths.html#paths-in-expressions
 [_PathPattern_]: #path-patterns
 [_Pattern_]: #patterns

src/statements.md

@@ -5,7 +5,8 @@
 >       `;`\
 >    | [_Item_]\
 >    | [_LetStatement_]\
->    | [_ExpressionStatement_]
+>    | [_ExpressionStatement_]\
+>    | [_MacroInvocationSemi_]
 
 
 A *statement* is a component of a [block], which is in turn a component of an
@@ -130,6 +131,7 @@ statement are [`cfg`], and [the lint check attributes].
 [_Expression_]: expressions.html
 [_Item_]: items.html
 [_LetStatement_]: #let-statements
+[_MacroInvocationSemi_]: macros.html#macro-invocation
 [_OuterAttribute_]: attributes.html
 [_Pattern_]: patterns.html
 [_Type_]: types.html

src/types.md

@@ -19,7 +19,8 @@
 >    | [_SliceType_]\
 >    | [_InferredType_]\
 >    | [_QualifiedPathInType_]\
->    | [_BareFunctionType_]
+>    | [_BareFunctionType_]\
+>    | [_MacroInvocation_]
 
 Every variable, item and value in a Rust program has a type. The _type_ of a
 *value* defines the interpretation of the memory holding it.
@@ -868,6 +869,7 @@ impl Printable for String {
 [_ImplTraitType_]: #impl-trait
 [_InferredType_]: #inferred-type
 [_Lifetime_]: trait-bounds.html
+[_MacroInvocation_]: macros.html#macro-invocation
 [_NeverType_]: #never-type
 [_ParenthesizedType_]: #parenthesized-types
 [_QualifiedPathInType_]: paths.html#qualified-paths