Skip to content

Implement raw/endraw #148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 5, 2019
Merged

Implement raw/endraw #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d7af342
Strip Right function
Oct 3, 2019
85158f1
Support for raw/endraw
Oct 3, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions include/jinja2cpp/error_info.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ enum class ErrorCode
ExpectedToken, //!< Specific token(s) expected. ExtraParams[0] contains the actual token, rest of ExtraParams contain set of expected tokens
ExpectedExpression, //!< Expression expected
ExpectedEndOfStatement, //!< End of statement expected. ExtraParams[0] contains the expected end of statement tag
ExpectedRawEnd, //!< {% endraw %} expected
UnexpectedToken, //!< Unexpected token. ExtraParams[0] contains the invalid token
UnexpectedStatement, //!< Unexpected statement. ExtraParams[0] contains the invalid statement tag
UnexpectedCommentBegin, //!< Unexpected comment block begin (`{#`)
Expand All @@ -40,6 +41,8 @@ enum class ErrorCode
UnexpectedExprEnd, //!< Unexpected expression block end (`}}`)
UnexpectedStmtBegin, //!< Unexpected statement block begin (`{%`)
UnexpectedStmtEnd, //!< Unexpected statment block end (`%}`)
UnexpectedRawBegin, //!< Unexpected raw block begin {% raw %}
UnexpectedRawEnd, //!< Unexpected raw block end {% endraw %}
};

/*!
Expand Down
9 changes: 9 additions & 0 deletions src/error_info.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,9 @@ void RenderErrorInfo(std::basic_string<CharT>& result, const ErrorInfoTpl<CharT>
format_to(out, UNIVERSAL_STR("Expected end of statement, got: '{}'").GetValue<CharT>(), extraParams[0]);
break;
}
case ErrorCode::ExpectedRawEnd:
format_to(out, UNIVERSAL_STR("Expected end of raw block").GetValue<CharT>());
break;
case ErrorCode::UnexpectedToken:
{
auto& extraParams = errInfo.GetExtraParams();
Expand All @@ -194,6 +197,12 @@ void RenderErrorInfo(std::basic_string<CharT>& result, const ErrorInfoTpl<CharT>
case ErrorCode::UnexpectedCommentEnd:
format_to(out, UNIVERSAL_STR("Unexpected comment end").GetValue<CharT>());
break;
case ErrorCode::UnexpectedRawBegin:
format_to(out, UNIVERSAL_STR("Unexpected raw block begin").GetValue<CharT>());
break;
case ErrorCode::UnexpectedRawEnd:
format_to(out, UNIVERSAL_STR("Unexpected raw block end").GetValue<CharT>());
break;
case ErrorCode::UnexpectedExprBegin:
format_to(out, UNIVERSAL_STR("Unexpected expression block begin").GetValue<CharT>());
break;
Expand Down
2 changes: 2 additions & 0 deletions src/lexer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ struct Token
// Template control
CommentBegin,
CommentEnd,
RawBegin,
RawEnd,
StmtBegin,
StmtEnd,
ExprBegin,
Expand Down
126 changes: 101 additions & 25 deletions src/template_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ struct ParserTraits<char> : public ParserTraitsBase<>
{
static std::regex GetRoughTokenizer()
{
return std::regex(R"((\{\{)|(\}\})|(\{%)|(%\})|(\{#)|(#\})|(\n))");
return std::regex(R"((\{\{)|(\}\})|(\{%[\+\-]?\s+raw\s+[\+\-]?%\})|(\{%[\+\-]?\s+endraw\s+[\+\-]?%\})|(\{%)|(%\})|(\{#)|(#\})|(\n))");
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure we should do right trimming for raw and left trimming for endraw. Need to check original jinja implementation.
Also I think we can use UNIVERSAL_STR here and get rid of copy/paste of the regexp string.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me now and I change that.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've just checked. We should do trimming. So the current implementation is correct.

}
static std::regex GetKeywords()
{
Expand Down Expand Up @@ -112,7 +112,7 @@ struct ParserTraits<wchar_t> : public ParserTraitsBase<>
{
static std::wregex GetRoughTokenizer()
{
return std::wregex(LR"((\{\{)|(\}\})|(\{%)|(%\})|(\{#)|(#\})|(\n))");
return std::wregex(LR"((\{\{)|(\}\})|(\{%[\+\-]?\s+raw\s+[\+\-]?%\})|(\{%[\+\-]?\s+endraw\s+[\+\-]?%\})|(\{%)|(%\})|(\{#)|(#\})|(\n))");
}
static std::wregex GetKeywords()
{
Expand Down Expand Up @@ -289,6 +289,8 @@ class TemplateParser : public LexerHelper
RM_Unknown = 0,
RM_ExprBegin = 1,
RM_ExprEnd,
RM_RawBegin,
RM_RawEnd,
RM_StmtBegin,
RM_StmtEnd,
RM_CommentBegin,
Expand All @@ -308,7 +310,8 @@ class TemplateParser : public LexerHelper
Expression,
Statement,
Comment,
LineStatement
LineStatement,
RawBlock
};

struct TextBlockInfo
Expand Down Expand Up @@ -352,6 +355,14 @@ class TemplateParser : public LexerHelper
}
} while (matchBegin != matchEnd);
FinishCurrentLine(m_template->size());

if ( m_currentBlockInfo.type == TextBlockType::RawBlock)
{
nonstd::expected<void, ParseError> result = MakeParseError(ErrorCode::ExpectedRawEnd, MakeToken(Token::RawEnd, {m_template->size(), m_template->size() }));
foundErrors.push_back(result.error());
return nonstd::make_unexpected(std::move(foundErrors));
}

FinishCurrentBlock(m_template->size());

if (!foundErrors.empty())
Expand Down Expand Up @@ -395,6 +406,8 @@ class TemplateParser : public LexerHelper
}
break;
case RM_CommentBegin:
if (m_currentBlockInfo.type == TextBlockType::RawBlock)
break;
if (m_currentBlockInfo.type != TextBlockType::RawText)
{
FinishCurrentLine(match.position() + 2);
Expand All @@ -407,6 +420,8 @@ class TemplateParser : public LexerHelper
break;

case RM_CommentEnd:
if (m_currentBlockInfo.type == TextBlockType::RawBlock)
break;
if (m_currentBlockInfo.type != TextBlockType::Comment)
{
FinishCurrentLine(match.position() + 2);
Expand Down Expand Up @@ -444,6 +459,26 @@ class TemplateParser : public LexerHelper

m_currentBlockInfo.range.startOffset = FinishCurrentBlock(matchStart);
break;
case RM_RawBegin:
if (m_currentBlockInfo.type == TextBlockType::RawBlock)
break;
else if (m_currentBlockInfo.type != TextBlockType::RawText && m_currentBlockInfo.type != TextBlockType::Comment)
{
FinishCurrentLine(match.position() + match.length());
return MakeParseError(ErrorCode::UnexpectedRawBegin, MakeToken(Token::RawBegin, {matchStart, matchStart + match.length()}));
}
StartControlBlock(TextBlockType::RawBlock, matchStart);
break;
case RM_RawEnd:
if (m_currentBlockInfo.type == TextBlockType::Comment)
break;
else if (m_currentBlockInfo.type != TextBlockType::RawBlock)
{
FinishCurrentLine(match.position() + match.length());
return MakeParseError(ErrorCode::UnexpectedRawEnd, MakeToken(Token::RawEnd, {matchStart, matchStart + match.length()}));
}
m_currentBlockInfo.range.startOffset = FinishCurrentBlock(matchStart);
break;
}

return nonstd::expected<void, ParseError>();
Expand All @@ -453,7 +488,7 @@ class TemplateParser : public LexerHelper
{
size_t startOffset = matchStart + 2;
size_t endOffset = matchStart;
if (m_currentBlockInfo.type != TextBlockType::RawText)
if (m_currentBlockInfo.type != TextBlockType::RawText || m_currentBlockInfo.type == TextBlockType::RawBlock )
return;
else
endOffset = StripBlockLeft(m_currentBlockInfo, startOffset, endOffset);
Expand All @@ -465,8 +500,53 @@ class TemplateParser : public LexerHelper
(*m_template)[startOffset] == '-')
++ startOffset;
}
m_currentBlockInfo.range.startOffset = startOffset;

m_currentBlockInfo.type = blockType;

if (blockType==TextBlockType::RawBlock)
startOffset = StripBlockRight(m_currentBlockInfo, matchStart);

m_currentBlockInfo.range.startOffset = startOffset;
}

size_t StripBlockRight(TextBlockInfo& currentBlockInfo, size_t position)
{
bool doTrim = m_settings.trimBlocks && (m_currentBlockInfo.type == TextBlockType::Statement || m_currentBlockInfo.type == TextBlockType::RawBlock);

if (m_currentBlockInfo.type == TextBlockType::RawBlock)
{
position+=2;
for(; position < m_template->size(); ++ position)
{
if ('%' == (*m_template)[position])
break;
}
}

size_t newPos = position + 2;

if ((m_currentBlockInfo.type != TextBlockType::RawText) && position != 0)
{
auto ctrlChar = (*m_template)[position - 1];
doTrim = ctrlChar == '-' ? true : (ctrlChar == '+' ? false : doTrim);
}

if (doTrim)
{
auto locale = std::locale();
for (;newPos < m_template->size(); ++ newPos)
{
auto ch = (*m_template)[newPos];
if (ch == '\n')
{
++ newPos;
break;
}
if (!std::isspace(ch, locale))
break;
}
}
return newPos;
}

size_t StripBlockLeft(TextBlockInfo& currentBlockInfo, size_t ctrlCharPos, size_t endOffset)
Expand All @@ -483,7 +563,7 @@ class TemplateParser : public LexerHelper

doStrip |= doTotalStrip;
}
if (!doStrip || currentBlockInfo.type != TextBlockType::RawText)
if (!doStrip || (currentBlockInfo.type != TextBlockType::RawText && currentBlockInfo.type != TextBlockType::RawBlock))
return endOffset;

auto locale = std::locale();
Expand Down Expand Up @@ -512,6 +592,7 @@ class TemplateParser : public LexerHelper

switch (block.type)
{
case TextBlockType::RawBlock:
case TextBlockType::RawText:
{
if (block.range.size() == 0)
Expand Down Expand Up @@ -648,32 +729,25 @@ class TemplateParser : public LexerHelper

size_t FinishCurrentBlock(size_t position)
{
bool doTrim = m_settings.trimBlocks && m_currentBlockInfo.type == TextBlockType::Statement;
size_t newPos = position + 2;
size_t newPos;

if ((m_currentBlockInfo.type != TextBlockType::RawText) && position != 0)
if (m_currentBlockInfo.type == TextBlockType::RawBlock)
{
auto ctrlChar = (*m_template)[position - 1];
doTrim = ctrlChar == '-' ? true : (ctrlChar == '+' ? false : doTrim);
if (ctrlChar == '+' || ctrlChar == '-')
-- position;
size_t currentPosition = position;
position = StripBlockLeft(m_currentBlockInfo, currentPosition+2, currentPosition);
newPos = StripBlockRight(m_currentBlockInfo, currentPosition);
}

if (doTrim)
else
{
auto locale = std::locale();
for (;newPos < m_template->size(); ++ newPos)
newPos = StripBlockRight(m_currentBlockInfo, position);
if ((m_currentBlockInfo.type != TextBlockType::RawText) && position != 0)
{
auto ch = (*m_template)[newPos];
if (ch == '\n')
{
++ newPos;
break;
}
if (!std::isspace(ch, locale))
break;
auto ctrlChar = (*m_template)[position - 1];
if (ctrlChar == '+' || ctrlChar == '-')
-- position;
}
}

m_currentBlockInfo.range.endOffset = position;
m_textBlocks.push_back(m_currentBlockInfo);
m_currentBlockInfo.type = TextBlockType::RawText;
Expand Down Expand Up @@ -928,6 +1002,8 @@ std::unordered_map<int, MultiStringLiteral> ParserTraitsBase<T>::s_tokens = {
{Token::From, UNIVERSAL_STR("form")},
{Token::As, UNIVERSAL_STR("as")},
{Token::Do, UNIVERSAL_STR("do")},
{Token::RawBegin, UNIVERSAL_STR("{% raw %}")},
{Token::RawEnd, UNIVERSAL_STR("{% endraw %}")},
{Token::CommentBegin, UNIVERSAL_STR("{#")},
{Token::CommentEnd, UNIVERSAL_STR("#}")},
{Token::StmtBegin, UNIVERSAL_STR("{%")},
Expand Down
8 changes: 7 additions & 1 deletion test/errors_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,7 +485,13 @@ INSTANTIATE_TEST_CASE_P(StatementsTest_2, ErrorsGenericTest, ::testing::Values(
InputOutputPair{"{% if a == 42 %}{% endwith %}",
"noname.j2tpl:1:20: error: Unexpected statement: 'endwith'\n{% if a == 42 %}{% endwith %}\n ---^-------"},
InputOutputPair{"{{}}",
"noname.j2tpl:1:3: error: Unexpected token: '<<End of block>>'\n{{}}\n--^-------"}
"noname.j2tpl:1:3: error: Unexpected token: '<<End of block>>'\n{{}}\n--^-------"},
InputOutputPair{"{% raw %}{% raw %}{{ x }{% endraw %}{% endraw %}",
"noname.j2tpl:1:37: error: Unexpected raw block end\n{% raw %}{% raw %}{{ x }{% endraw %}{% endraw %}\n ---^-------"},
InputOutputPair{"{% raw %}",
"noname.j2tpl:1:10: error: Expected end of raw block\n{% raw %}\n ---^-------"},
InputOutputPair{"{{ 2 + 3 + {% raw %} }}",
"noname.j2tpl:1:12: error: Unexpected raw block begin\n{{ 2 + 3 + {% raw %}\n ---^-------"}
));

INSTANTIATE_TEST_CASE_P(ExtensionStatementsTest, ErrorsGenericExtensionsTest, ::testing::Values(
Expand Down
Loading