Skip to content

Commit a6d544c

Browse files
morenolflexferrum
morenol
authored and
flexferrum
committed
Implement raw/endraw (#148)
* Strip Right function * Support for raw/endraw
1 parent bcda100 commit a6d544c

File tree

6 files changed

+282
-27
lines changed

6 files changed

+282
-27
lines changed

‎include/jinja2cpp/error_info.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ enum class ErrorCode
3232
ExpectedToken, //!< Specific token(s) expected. ExtraParams[0] contains the actual token, rest of ExtraParams contain set of expected tokens
3333
ExpectedExpression, //!< Expression expected
3434
ExpectedEndOfStatement, //!< End of statement expected. ExtraParams[0] contains the expected end of statement tag
35+
ExpectedRawEnd, //!< {% endraw %} expected
3536
UnexpectedToken, //!< Unexpected token. ExtraParams[0] contains the invalid token
3637
UnexpectedStatement, //!< Unexpected statement. ExtraParams[0] contains the invalid statement tag
3738
UnexpectedCommentBegin, //!< Unexpected comment block begin (`{#`)
@@ -40,6 +41,8 @@ enum class ErrorCode
4041
UnexpectedExprEnd, //!< Unexpected expression block end (`}}`)
4142
UnexpectedStmtBegin, //!< Unexpected statement block begin (`{%`)
4243
UnexpectedStmtEnd, //!< Unexpected statment block end (`%}`)
44+
UnexpectedRawBegin, //!< Unexpected raw block begin {% raw %}
45+
UnexpectedRawEnd, //!< Unexpected raw block end {% endraw %}
4346
};
4447

4548
/*!

‎src/error_info.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,9 @@ void RenderErrorInfo(std::basic_string<CharT>& result, const ErrorInfoTpl<CharT>
176176
format_to(out, UNIVERSAL_STR("Expected end of statement, got: '{}'").GetValue<CharT>(), extraParams[0]);
177177
break;
178178
}
179+
case ErrorCode::ExpectedRawEnd:
180+
format_to(out, UNIVERSAL_STR("Expected end of raw block").GetValue<CharT>());
181+
break;
179182
case ErrorCode::UnexpectedToken:
180183
{
181184
auto& extraParams = errInfo.GetExtraParams();
@@ -194,6 +197,12 @@ void RenderErrorInfo(std::basic_string<CharT>& result, const ErrorInfoTpl<CharT>
194197
case ErrorCode::UnexpectedCommentEnd:
195198
format_to(out, UNIVERSAL_STR("Unexpected comment end").GetValue<CharT>());
196199
break;
200+
case ErrorCode::UnexpectedRawBegin:
201+
format_to(out, UNIVERSAL_STR("Unexpected raw block begin").GetValue<CharT>());
202+
break;
203+
case ErrorCode::UnexpectedRawEnd:
204+
format_to(out, UNIVERSAL_STR("Unexpected raw block end").GetValue<CharT>());
205+
break;
197206
case ErrorCode::UnexpectedExprBegin:
198207
format_to(out, UNIVERSAL_STR("Unexpected expression block begin").GetValue<CharT>());
199208
break;

‎src/lexer.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,8 @@ struct Token
9898
// Template control
9999
CommentBegin,
100100
CommentEnd,
101+
RawBegin,
102+
RawEnd,
101103
StmtBegin,
102104
StmtEnd,
103105
ExprBegin,

‎src/template_parser.h‎

Lines changed: 101 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ struct ParserTraits<char> : public ParserTraitsBase<>
5656
{
5757
static std::regex GetRoughTokenizer()
5858
{
59-
return std::regex(R"((\{\{)|(\}\})|(\{%)|(%\})|(\{#)|(#\})|(\n))");
59+
return std::regex(R"((\{\{)|(\}\})|(\{%[\+\-]?\s+raw\s+[\+\-]?%\})|(\{%[\+\-]?\s+endraw\s+[\+\-]?%\})|(\{%)|(%\})|(\{#)|(#\})|(\n))");
6060
}
6161
static std::regex GetKeywords()
6262
{
@@ -112,7 +112,7 @@ struct ParserTraits<wchar_t> : public ParserTraitsBase<>
112112
{
113113
static std::wregex GetRoughTokenizer()
114114
{
115-
return std::wregex(LR"((\{\{)|(\}\})|(\{%)|(%\})|(\{#)|(#\})|(\n))");
115+
return std::wregex(LR"((\{\{)|(\}\})|(\{%[\+\-]?\s+raw\s+[\+\-]?%\})|(\{%[\+\-]?\s+endraw\s+[\+\-]?%\})|(\{%)|(%\})|(\{#)|(#\})|(\n))");
116116
}
117117
static std::wregex GetKeywords()
118118
{
@@ -289,6 +289,8 @@ class TemplateParser : public LexerHelper
289289
RM_Unknown = 0,
290290
RM_ExprBegin = 1,
291291
RM_ExprEnd,
292+
RM_RawBegin,
293+
RM_RawEnd,
292294
RM_StmtBegin,
293295
RM_StmtEnd,
294296
RM_CommentBegin,
@@ -308,7 +310,8 @@ class TemplateParser : public LexerHelper
308310
Expression,
309311
Statement,
310312
Comment,
311-
LineStatement
313+
LineStatement,
314+
RawBlock
312315
};
313316

314317
struct TextBlockInfo
@@ -352,6 +355,14 @@ class TemplateParser : public LexerHelper
352355
}
353356
} while (matchBegin != matchEnd);
354357
FinishCurrentLine(m_template->size());
358+
359+
if ( m_currentBlockInfo.type == TextBlockType::RawBlock)
360+
{
361+
nonstd::expected<void, ParseError> result = MakeParseError(ErrorCode::ExpectedRawEnd, MakeToken(Token::RawEnd, {m_template->size(), m_template->size() }));
362+
foundErrors.push_back(result.error());
363+
return nonstd::make_unexpected(std::move(foundErrors));
364+
}
365+
355366
FinishCurrentBlock(m_template->size());
356367

357368
if (!foundErrors.empty())
@@ -395,6 +406,8 @@ class TemplateParser : public LexerHelper
395406
}
396407
break;
397408
case RM_CommentBegin:
409+
if (m_currentBlockInfo.type == TextBlockType::RawBlock)
410+
break;
398411
if (m_currentBlockInfo.type != TextBlockType::RawText)
399412
{
400413
FinishCurrentLine(match.position() + 2);
@@ -407,6 +420,8 @@ class TemplateParser : public LexerHelper
407420
break;
408421

409422
case RM_CommentEnd:
423+
if (m_currentBlockInfo.type == TextBlockType::RawBlock)
424+
break;
410425
if (m_currentBlockInfo.type != TextBlockType::Comment)
411426
{
412427
FinishCurrentLine(match.position() + 2);
@@ -444,6 +459,26 @@ class TemplateParser : public LexerHelper
444459

445460
m_currentBlockInfo.range.startOffset = FinishCurrentBlock(matchStart);
446461
break;
462+
case RM_RawBegin:
463+
if (m_currentBlockInfo.type == TextBlockType::RawBlock)
464+
break;
465+
else if (m_currentBlockInfo.type != TextBlockType::RawText && m_currentBlockInfo.type != TextBlockType::Comment)
466+
{
467+
FinishCurrentLine(match.position() + match.length());
468+
return MakeParseError(ErrorCode::UnexpectedRawBegin, MakeToken(Token::RawBegin, {matchStart, matchStart + match.length()}));
469+
}
470+
StartControlBlock(TextBlockType::RawBlock, matchStart);
471+
break;
472+
case RM_RawEnd:
473+
if (m_currentBlockInfo.type == TextBlockType::Comment)
474+
break;
475+
else if (m_currentBlockInfo.type != TextBlockType::RawBlock)
476+
{
477+
FinishCurrentLine(match.position() + match.length());
478+
return MakeParseError(ErrorCode::UnexpectedRawEnd, MakeToken(Token::RawEnd, {matchStart, matchStart + match.length()}));
479+
}
480+
m_currentBlockInfo.range.startOffset = FinishCurrentBlock(matchStart);
481+
break;
447482
}
448483

449484
return nonstd::expected<void, ParseError>();
@@ -453,7 +488,7 @@ class TemplateParser : public LexerHelper
453488
{
454489
size_t startOffset = matchStart + 2;
455490
size_t endOffset = matchStart;
456-
if (m_currentBlockInfo.type != TextBlockType::RawText)
491+
if (m_currentBlockInfo.type != TextBlockType::RawText || m_currentBlockInfo.type == TextBlockType::RawBlock )
457492
return;
458493
else
459494
endOffset = StripBlockLeft(m_currentBlockInfo, startOffset, endOffset);
@@ -465,8 +500,53 @@ class TemplateParser : public LexerHelper
465500
(*m_template)[startOffset] == '-')
466501
++ startOffset;
467502
}
468-
m_currentBlockInfo.range.startOffset = startOffset;
503+
469504
m_currentBlockInfo.type = blockType;
505+
506+
if (blockType==TextBlockType::RawBlock)
507+
startOffset = StripBlockRight(m_currentBlockInfo, matchStart);
508+
509+
m_currentBlockInfo.range.startOffset = startOffset;
510+
}
511+
512+
size_t StripBlockRight(TextBlockInfo& currentBlockInfo, size_t position)
513+
{
514+
bool doTrim = m_settings.trimBlocks && (m_currentBlockInfo.type == TextBlockType::Statement || m_currentBlockInfo.type == TextBlockType::RawBlock);
515+
516+
if (m_currentBlockInfo.type == TextBlockType::RawBlock)
517+
{
518+
position+=2;
519+
for(; position < m_template->size(); ++ position)
520+
{
521+
if ('%' == (*m_template)[position])
522+
break;
523+
}
524+
}
525+
526+
size_t newPos = position + 2;
527+
528+
if ((m_currentBlockInfo.type != TextBlockType::RawText) && position != 0)
529+
{
530+
auto ctrlChar = (*m_template)[position - 1];
531+
doTrim = ctrlChar == '-' ? true : (ctrlChar == '+' ? false : doTrim);
532+
}
533+
534+
if (doTrim)
535+
{
536+
auto locale = std::locale();
537+
for (;newPos < m_template->size(); ++ newPos)
538+
{
539+
auto ch = (*m_template)[newPos];
540+
if (ch == '\n')
541+
{
542+
++ newPos;
543+
break;
544+
}
545+
if (!std::isspace(ch, locale))
546+
break;
547+
}
548+
}
549+
return newPos;
470550
}
471551

472552
size_t StripBlockLeft(TextBlockInfo& currentBlockInfo, size_t ctrlCharPos, size_t endOffset)
@@ -483,7 +563,7 @@ class TemplateParser : public LexerHelper
483563

484564
doStrip |= doTotalStrip;
485565
}
486-
if (!doStrip || currentBlockInfo.type != TextBlockType::RawText)
566+
if (!doStrip || (currentBlockInfo.type != TextBlockType::RawText && currentBlockInfo.type != TextBlockType::RawBlock))
487567
return endOffset;
488568

489569
auto locale = std::locale();
@@ -512,6 +592,7 @@ class TemplateParser : public LexerHelper
512592

513593
switch (block.type)
514594
{
595+
case TextBlockType::RawBlock:
515596
case TextBlockType::RawText:
516597
{
517598
if (block.range.size() == 0)
@@ -648,32 +729,25 @@ class TemplateParser : public LexerHelper
648729

649730
size_t FinishCurrentBlock(size_t position)
650731
{
651-
bool doTrim = m_settings.trimBlocks && m_currentBlockInfo.type == TextBlockType::Statement;
652-
size_t newPos = position + 2;
732+
size_t newPos;
653733

654-
if ((m_currentBlockInfo.type != TextBlockType::RawText) && position != 0)
734+
if (m_currentBlockInfo.type == TextBlockType::RawBlock)
655735
{
656-
auto ctrlChar = (*m_template)[position - 1];
657-
doTrim = ctrlChar == '-' ? true : (ctrlChar == '+' ? false : doTrim);
658-
if (ctrlChar == '+' || ctrlChar == '-')
659-
-- position;
736+
size_t currentPosition = position;
737+
position = StripBlockLeft(m_currentBlockInfo, currentPosition+2, currentPosition);
738+
newPos = StripBlockRight(m_currentBlockInfo, currentPosition);
660739
}
661-
662-
if (doTrim)
740+
else
663741
{
664-
auto locale = std::locale();
665-
for (;newPos < m_template->size(); ++ newPos)
742+
newPos = StripBlockRight(m_currentBlockInfo, position);
743+
if ((m_currentBlockInfo.type != TextBlockType::RawText) && position != 0)
666744
{
667-
auto ch = (*m_template)[newPos];
668-
if (ch == '\n')
669-
{
670-
++ newPos;
671-
break;
672-
}
673-
if (!std::isspace(ch, locale))
674-
break;
745+
auto ctrlChar = (*m_template)[position - 1];
746+
if (ctrlChar == '+' || ctrlChar == '-')
747+
-- position;
675748
}
676749
}
750+
677751
m_currentBlockInfo.range.endOffset = position;
678752
m_textBlocks.push_back(m_currentBlockInfo);
679753
m_currentBlockInfo.type = TextBlockType::RawText;
@@ -928,6 +1002,8 @@ std::unordered_map<int, MultiStringLiteral> ParserTraitsBase<T>::s_tokens = {
9281002
{Token::From, UNIVERSAL_STR("form")},
9291003
{Token::As, UNIVERSAL_STR("as")},
9301004
{Token::Do, UNIVERSAL_STR("do")},
1005+
{Token::RawBegin, UNIVERSAL_STR("{% raw %}")},
1006+
{Token::RawEnd, UNIVERSAL_STR("{% endraw %}")},
9311007
{Token::CommentBegin, UNIVERSAL_STR("{#")},
9321008
{Token::CommentEnd, UNIVERSAL_STR("#}")},
9331009
{Token::StmtBegin, UNIVERSAL_STR("{%")},

‎test/errors_test.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -485,7 +485,13 @@ INSTANTIATE_TEST_CASE_P(StatementsTest_2, ErrorsGenericTest, ::testing::Values(
485485
InputOutputPair{"{% if a == 42 %}{% endwith %}",
486486
"noname.j2tpl:1:20: error: Unexpected statement: 'endwith'\n{% if a == 42 %}{% endwith %}\n ---^-------"},
487487
InputOutputPair{"{{}}",
488-
"noname.j2tpl:1:3: error: Unexpected token: '<<End of block>>'\n{{}}\n--^-------"}
488+
"noname.j2tpl:1:3: error: Unexpected token: '<<End of block>>'\n{{}}\n--^-------"},
489+
InputOutputPair{"{% raw %}{% raw %}{{ x }{% endraw %}{% endraw %}",
490+
"noname.j2tpl:1:37: error: Unexpected raw block end\n{% raw %}{% raw %}{{ x }{% endraw %}{% endraw %}\n ---^-------"},
491+
InputOutputPair{"{% raw %}",
492+
"noname.j2tpl:1:10: error: Expected end of raw block\n{% raw %}\n ---^-------"},
493+
InputOutputPair{"{{ 2 + 3 + {% raw %} }}",
494+
"noname.j2tpl:1:12: error: Unexpected raw block begin\n{{ 2 + 3 + {% raw %}\n ---^-------"}
489495
));
490496

491497
INSTANTIATE_TEST_CASE_P(ExtensionStatementsTest, ErrorsGenericExtensionsTest, ::testing::Values(

0 commit comments

Comments
 (0)