Boost Spirit 将表达式标记化为向量

Question

我正在尝试解析一个也可以包含标识符的表达式并将每个元素放入 std::vector <std::string>，我想出了以下语法：

#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <vector>

namespace qi = boost::spirit::qi;

struct Tokeniser
    : boost::spirit::qi::grammar <std::string::const_iterator, std::vector <std::string> (), boost::spirit::ascii::space_type>
{
    Tokeniser() : Tokeniser::base_type(expression)
    {
        namespace qi = boost::spirit::qi;
        expression = 
            term >>
            *( (qi::string("+")[qi::_val.push_back(qi::_1)] >> term) |
               (qi::string("-")[qi::_val.push_back(qi::_1)] >> term) );

        term = 
            factor >>
            *( (qi::string("*")[qi::_val.push_back(qi::_1)] >> factor) |
               (qi::string("/")[qi::_val.push_back(qi::_1)] >> factor) );

        factor =
            (identifier | myDouble_)[qi::_val.push_back(qi::_1)] |
            qi::string("(")[qi::_val.push_back(qi::_1)] >> expression >> qi::string(")")[qi::_val.push_back(qi::_1)];

        identifier = qi::raw [ qi::lexeme[ (qi::alpha | '_') >> *(qi::alnum | '_') ] ]; 

        myDouble_ = qi::raw [ qi::double_ ];
    }

    boost::spirit::qi::rule<std::string::const_iterator, std::vector <std::string> (), boost::spirit::ascii::space_type> expression;
    boost::spirit::qi::rule<std::string::const_iterator, boost::spirit::ascii::space_type>                               factor;
    boost::spirit::qi::rule<std::string::const_iterator, boost::spirit::ascii::space_type>                               term;        

    boost::spirit::qi::rule<std::string::const_iterator, std::string(), boost::spirit::ascii::space_type> identifier;
    boost::spirit::qi::rule<std::string::const_iterator, std::string(), boost::spirit::ascii::space_type> myDouble_;
};

但是，我收到以下错误 'const struct boost::phoenix::actor<boost::spirit::attribute<0> >' has no member named 'push_back'。

有没有直接的方法来执行我想做的事情？

Answer 1

是的，占位符类型（显然）没有 push_back 成员。

C++ 是强类型的。任何延迟的动作都是一个 "illusion"：演员在表达式模板中通过组合可以 "evaluated" 之后的特殊用途类型来表示。

表达式模板简介

Live On Coliru

以防万一您想了解它实际的工作原理，从头开始一个简单的示例。注释描述了代码各个部分的作用：

// we have lazy placeholder types:
template <int N> struct placeholder {};
placeholder<1> _1;
placeholder<2> _2;
placeholder<3> _3;

// note that every type here is stateless, and acts just like a more
// complicated placeholder.
// We can have expressions, like binary addition:
template <typename L, typename R> struct addition { };
template <typename L, typename R> struct multiplication { };

// here is the "factory" for our expression template:
template <typename L, typename R> addition<L,R> operator+(L const&, R const&) { return {}; }
template <typename L, typename R> multiplication<L,R> operator*(L const&, R const&) { return {}; }

///////////////////////////////////////////////
// To evaluate/interpret the expressions, we have to define "evaluation" for each type of placeholder:
template <typename Ctx, int N> 
    auto eval(Ctx& ctx, placeholder<N>) { return ctx.arg(N); }
template <typename Ctx, typename L, typename R>
    auto eval(Ctx& ctx, addition<L, R>) { return eval(ctx, L{}) + eval(ctx, R{}); }
template <typename Ctx, typename L, typename R>
    auto eval(Ctx& ctx, multiplication<L, R>) { return eval(ctx, L{}) * eval(ctx, R{}); }

///////////////////////////////////////////////
// A simple real-life context would contain the arguments:
#include <vector>
struct Context {
    std::vector<double> _args;

    // define the operation to get an argument from this context:
    double arg(int i) const { return _args.at(i-1); }
};

#include <iostream>
int main() {
    auto foo = _1 + _2 + _3;

    Context ctx { { 3, 10, -4 } };
    std::cout << "foo: " << eval(ctx, foo) << "\n";
    std::cout << "_1 + _2 * _3: " << eval(ctx, _1 + _2 * _3) << "\n";
}

输出正是您所期望的：

foo: 9
_1 + _2 * _3: -37

修复操作：

您必须 "describe" push_back 操作，而不是尝试在占位符上查找此类操作。凤凰有你的支持：

#include <boost/phoenix/stl.hpp>

现在我将简化操作以使用 phoenix::push_back:

auto push = px::push_back(qi::_val, qi::_1);

expression = 
    term >>
    *( (qi::string("+")[push] >> term) |
       (qi::string("-")[push] >> term) );

term = 
    factor >>
    *( (qi::string("*")[push] >> factor) |
       (qi::string("/")[push] >> factor) );

factor =
    (identifier | myDouble_)[push] |
    qi::string("(")[push] >> expression >> qi::string(")")[push];

// etc.

但是，这还有一个问题，即 _val 被解析为规则的属性类型。但是您的一些规则 没有声明 属性类型，因此它默认为 qi::unused_type。显然，为该属性生成 "push_back" 评估代码不适用于 unused_type。

让我们修复这些声明：

qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> expression;
qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> factor;
qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> term;

其他问题！

当我们这样做时，令牌基本上是空的。给出了什么？

在存在语义动作的情况下，自动属性传播被禁止。因此，您必须努力获取附加到最终标记向量的子表达式的内容。

同样，使用 Phoenix 的 STL 支持：

auto push      = px::push_back(qi::_val, qi::_1);
auto propagate = px::insert(qi::_val, px::end(qi::_val), px::begin(qi::_1), px::end(qi::_1));

expression = 
    term[propagate] >>
    *( (qi::string("+")[push] >> term[propagate]) |
       (qi::string("-")[push] >> term[propagate]) );

term = 
    factor[propagate] >>
    *( (qi::string("*")[push] >> factor[propagate]) |
       (qi::string("/")[push] >> factor[propagate]) );

factor =
    (identifier | myDouble_)[push] |
    qi::string("(")[push] >> expression[propagate] >> qi::string(")")[push];

现在，使用 Live On Coliru

进行测试

#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/phoenix/stl.hpp>
#include <vector>

namespace qi = boost::spirit::qi;
namespace px = boost::phoenix;

template <typename It = std::string::const_iterator>
struct Tokeniser
    : qi::grammar <It, std::vector <std::string> (), boost::spirit::ascii::space_type>
{
    Tokeniser() : Tokeniser::base_type(expression)
    {
        auto push      = px::push_back(qi::_val, qi::_1);
        auto propagate = px::insert(qi::_val, px::end(qi::_val), px::begin(qi::_1), px::end(qi::_1));

        expression = 
            term[propagate] >>
            *( (qi::string("+")[push] >> term[propagate]) |
               (qi::string("-")[push] >> term[propagate]) );

        term = 
            factor[propagate] >>
            *( (qi::string("*")[push] >> factor[propagate]) |
               (qi::string("/")[push] >> factor[propagate]) );

        factor =
            (identifier | myDouble_)[push] |
            qi::string("(")[push] >> expression[propagate] >> qi::string(")")[push];

        identifier = qi::raw [ qi::lexeme[ (qi::alpha | '_') >> *(qi::alnum | '_') ] ]; 

        myDouble_ = qi::raw [ qi::double_ ];

        BOOST_SPIRIT_DEBUG_NODES((expression)(term)(factor)(identifier)(myDouble_))
    }

    qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> expression;
    qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> factor;
    qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> term;
    qi::rule<It, std::string(), boost::spirit::ascii::space_type> identifier;
    qi::rule<It, std::string(), boost::spirit::ascii::space_type> myDouble_;
};

int main() {
    Tokeniser<> tok;

    std::string const input = "x + 89/(y*y)";

    auto f = input.begin(), l = input.end();
    std::vector<std::string> tokens;
    if (phrase_parse(f, l, tok, boost::spirit::ascii::space, tokens)) {
        std::cout << "Parsed " << tokens.size() << " tokens:\n";
        for (auto& token : tokens)
            std::cout << " - '" << token << "'\n";
    } else {
        std::cout << "Parse failed\n";
    }

    if (f != l)
        std::cout << "Remaining unparsed input: '" << std::string(f,l) << "'\n";
}

版画

Parsed 9 tokens:
 - 'x'
 - '+'
 - '89'
 - '/'
 - '('
 - 'y'
 - '*'
 - 'y'
 - ')'

大清理

一般来说，避免语义操作（参见我的回答 Boost Spirit: "Semantic actions are evil"? - 特别是关于副作用的项目符号）。大多数时候，您可以摆脱自动属性传播。我想说这是 Boost Spirit 的主要卖点。

进一步简化 skipping/lexemes (Boost spirit skipper issues) 确实显着减少了代码和编译时间：

Live On Coliru

#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <vector>

namespace qi = boost::spirit::qi;

template <typename It = std::string::const_iterator>
struct Tokeniser : qi::grammar <It, std::vector <std::string>()> {
    Tokeniser() : Tokeniser::base_type(start)
    {
        start = qi::skip(boost::spirit::ascii::space) [expression];
        expression =
            term >>
            *( (qi::string("+") >> term) |
               (qi::string("-") >> term) );

        term =
            factor >>
            *( (qi::string("*") >> factor) |
               (qi::string("/") >> factor) );

        factor =
            (identifier | myDouble_) |
            qi::string("(") >> expression >> qi::string(")");

        identifier = qi::raw [ (qi::alpha | '_') >> *(qi::alnum | '_') ]; 
        myDouble_  = qi::raw [ qi::double_ ];

        BOOST_SPIRIT_DEBUG_NODES((expression)(term)(factor)(identifier)(myDouble_))
    }

    qi::rule<It, std::vector<std::string>()> start;
    qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> expression, factor, term;
    qi::rule<It, std::string()> identifier, myDouble_;
};

int main() {
    Tokeniser<> tok;

    std::string const input = "x + 89/(y*y)";

    auto f = input.begin(), l = input.end();
    std::vector<std::string> tokens;
    if (parse(f, l, tok, tokens)) {
        std::cout << "Parsed " << tokens.size() << " tokens:\n";
        for (auto& token : tokens)
            std::cout << " - '" << token << "'\n";
    } else {
        std::cout << "Parse failed\n";
    }

    if (f != l)
        std::cout << "Remaining unparsed input: '" << std::string(f,l) << "'\n";
}

仍然打印

Parsed 9 tokens:
 - 'x'
 - '+'
 - '89'
 - '/'
 - '('
 - 'y'
 - '*'
 - 'y'
 - ')'

遗留问题

您是否考虑过回溯行为？我认为您需要在规则中明智地放置一些 qi::hold[] 指令，例如参见Understanding Boost.spirit's string parser

大问题：

当你想要的只是代币时，你为什么要"parsing"？
你会用代币做什么？解析器构建了很多信息，只是为了再次丢弃它

Boost Spirit 将表达式标记化为向量

Boost Spirit tokenise expression into vector

boost

boost-spirit

boost-phoenix

表达式模板简介

修复操作：

其他问题！

大清理

遗留问题

大问题：