Token.cpp

#include <sstream>
#include "Token.h"
#include <initializer_list>
#include <exception>


class Character {
    int ch;
public:
    int get() const { return ch; }
    bool isEOF() const { return ch == std::istream::traits_type::eof(); }
    bool isDigit() const { return ch >= '0' && ch <= '9'; }
    bool isAlpha() const { return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z'; }
    bool is(std::initializer_list<int> targetList) const { for (int i : targetList) { if (ch == i) return true; } return false; }

    Character(int ch) :ch(ch) {}
};

//该函数将在自动机解析保留字或标识符结束时调用
Token *parseReservedOrIdentifier(const std::string & str) {
    using Reserved = TokenReserved::Reserved;
    switch (str.length()) { //这里首先根据字符串长短分类，减少搜索次数
    case 2:
        if (str == "if") return new TokenReserved(Reserved::If);
        if (str == "do") return new TokenReserved(Reserved::Do);
        break;
    case 3:
        if (str == "var") return new TokenReserved(Reserved::Var);
        if (str == "end") return new TokenReserved(Reserved::End);
        if (str == "odd") return new TokenReserved(Reserved::Odd);
        break;
    case 4:
        if (str == "then") return new TokenReserved(Reserved::Then);
        if (str == "call") return new TokenReserved(Reserved::Call);
        if (str == "read") return new TokenReserved(Reserved::Read);
        break;
    case 5:
        if (str == "const") return new TokenReserved(Reserved::Const);
        if (str == "begin") return new TokenReserved(Reserved::Begin);
        if (str == "while") return new TokenReserved(Reserved::While);
        if (str == "write") return new TokenReserved(Reserved::Write);
        break;
    case 9:
        if (str == "procedure") return new TokenReserved(Reserved::Procedure);
        break;
    }
    return new TokenIdentifier(str);
}

Token *parseOperator(const std::string &str) {
    using Operator = TokenOperator::Operator;
    switch (str.length()) {
    case 1:
        if (str == "+") return new TokenOperator(Operator::Plus);
        if (str == "-") return new TokenOperator(Operator::Minus);
        if (str == "*") return new TokenOperator(Operator::Multiplication);
        if (str == "/") return new TokenOperator(Operator::Division);
        if (str == "<") return new TokenOperator(Operator::Less);
        if (str == ">") return new TokenOperator(Operator::Greater);
        if (str == "#") return new TokenOperator(Operator::Pound);
        if (str == "=") return new TokenOperator(Operator::Equal);
        break;
    case 2:
        if (str == "<=") return new TokenOperator(Operator::LessEqual);
        if (str == ">=") return new TokenOperator(Operator::GreaterEqual);
        if (str == ":=") return new TokenOperator(Operator::Walrus);
        break;
    }
    return new TokenError(str, "Unknown Operator");
}

Token *parseDelimiter(const std::string &str) {
    using Delimiter = TokenDelimiter::Delimiter;
    if (str == "(") return new TokenDelimiter(Delimiter::LeftBracket);
    if (str == ")") return new TokenDelimiter(Delimiter::RightBracket);
    if (str == ",") return new TokenDelimiter(Delimiter::Comma);
    if (str == ";") return new TokenDelimiter(Delimiter::Semicolons);
    if (str == ".") return new TokenDelimiter(Delimiter::Period);
    return new TokenError(str, "Unknown Delimiter");
}

Token * Token::fromStream(std::istream &input){
    std::ostringstream buffer;
    int state = 0;
    while (true) {
        const Character nextChar = input.peek();
        switch (state) {

        case 0: { //起始状态
            if (nextChar.isAlpha())  //是字母，转移到识别标识符和保留字的分支
                state = 1;
            else if (nextChar.isDigit()) //是数字，转移到识别数字的分支
                state = 2;
            else if (nextChar.is({ '+','-','*','/','<','>','#','=',':' }))
                state = 3;
            else if (nextChar.is({ ' ', '\t' }))
                state = 5;
            else if (nextChar.is({ '\r' }))
                state = 6;
            else if (nextChar.is({ '\n' }))
                state = 7;
            else if (nextChar.is({ '(', ')', ',', ';', '.' }))
                state = 8;
            else if (nextChar.isEOF())
                return new TokenEnd();
            else //无法接受的字符，将其吞掉后再返回错误
                state = 9;
        } break;

        case 1: { //接收标识符和保留字状态
            if (nextChar.isAlpha() || nextChar.isDigit())//如果下一个字符仍然可和当前输入组成标识符或保留字，优先继续输入
                break; 
            return parseReservedOrIdentifier(buffer.str()); //至此，我们已经不能继续追加下一个字符，考虑将当前输入解析成保留字或者标识符
        } break;

        case 2: { // 接收无符号数字状态
            if (nextChar.isDigit())
                break;
            return new TokenUnsigned(buffer.str());
        }break;
        
        case 3: {
            if (nextChar.is({ '=' })) { // 如果下一个字符是双字符运算符的第二个字符，就转到状态4继续接收
                state = 4;
                break;
            }
            return parseOperator(buffer.str());
        } break;

        case 4: return parseOperator(buffer.str()); //运算符最多两个字符，到这里直接接收就行

        case 5: {
            if (nextChar.is({ ' ', '\t' }))
                break;
            return new TokenWhitespace(buffer.str());
        } break;

        case 6: {
            if (nextChar.is({ '\n' })) {
                state = 7;
                break;
            }
            return new TokenError(buffer.str(), "Incomplete line breaks");
        } break;

        case 7: return new TokenLineWrap(buffer.str());
        case 8: return parseDelimiter(buffer.str());
        case 9: return new TokenError(buffer.str(), "Unexcepted Character");

        } // switch
        // 如果能运行到这里，说明我们的自动机接受输入流的下一个字符
        buffer.put(input.get()); //从输入流接收一个字符
    }
}

bool Token::operator==(Token &other) const
{
    return other.type == this->type && other.raw == this->raw;
}

const std::string TokenReserved::getStringByReserved(Reserved reserved)
{
    switch (reserved) {
        case Reserved::Const:     return "const";
        case Reserved::Var:       return "var";
        case Reserved::Procedure: return "procedure";
        case Reserved::Begin:     return "begin";
        case Reserved::End:       return "end";
        case Reserved::Odd:       return "odd";
        case Reserved::If:        return "if";
        case Reserved::Then:      return "then";
        case Reserved::Call:      return "call";
        case Reserved::While:     return "while";
        case Reserved::Do:        return "do";
        case Reserved::Read:      return "read";
        case Reserved::Write:     return "write";

        default: throw std::exception("Unknown Reserved");
    }
}

std::ostream &TokenReserved::printToStream(std::ostream &out) const
{
    return out << R"({"type":"RESERVED","reserved":")" << getStringByReserved(reserved) << R"("})";
}

bool TokenReserved::match(const Token &other) const
{
    auto o = reinterpret_cast<const TokenReserved *>(&other);
    return o != nullptr && this->reserved == o->reserved;
}

std::ostream &TokenReserved::simplePrintToStream(std::ostream &out) const
{
    return out << "RESERVED:'" << getStringByReserved(reserved) << "'";
}


const std::string TokenOperator::getStringByOperator(Operator op)
{
    switch (op) {
        case Operator::Plus:           return "+";
        case Operator::Minus:          return "-";
        case Operator::Multiplication: return "*";
        case Operator::Division:       return "/";
        case Operator::Less:           return "<";
        case Operator::LessEqual:      return "<=";
        case Operator::Greater:        return ">";
        case Operator::GreaterEqual:   return ">=";
        case Operator::Pound:          return "#";
        case Operator::Equal:          return "=";
        case Operator::Walrus:         return ":=";

        default: throw std::exception("Unknown Operator");
    }
}

std::ostream &TokenOperator::printToStream(std::ostream &out) const
{
    return out << R"({"type":"OPERATOR","operator":")" << getStringByOperator(op) << R"("})";
}

bool TokenOperator::match(const Token &other) const
{
    auto o = dynamic_cast<const TokenOperator *>(&other);
    return o != nullptr && this->op == o->op;
}

std::ostream &TokenOperator::simplePrintToStream(std::ostream &out) const
{
    return out << "OPERATOR:'" << getStringByOperator(op)<<"'";
}


const std::string TokenDelimiter::getStringByDelimiter(Delimiter delimiter)
{
    switch (delimiter) {
        case Delimiter::LeftBracket:  return "(";
        case Delimiter::RightBracket: return ")";
        case Delimiter::Comma:        return ",";
        case Delimiter::Semicolons:   return ";";
        case Delimiter::Period:       return ".";

        default: throw std::exception("Unknown Delimiter");
    }
}

std::ostream &TokenDelimiter::printToStream(std::ostream &out) const
{
    return out << R"({"type":"DELIMITER","delimiter":")" << getStringByDelimiter(delimiter) << R"("})";
}

bool TokenDelimiter::match(const Token &other) const
{
    auto o = dynamic_cast<const TokenDelimiter *>(&other);
    return o != nullptr && this->delimiter == o->delimiter;
}

std::ostream &TokenDelimiter::simplePrintToStream(std::ostream &out) const
{
    return out << "DELIMITER:'" << getStringByDelimiter(delimiter) << "'";
}


std::ostream &operator<<(std::ostream &out, const Token &token)
{
    return token.simplePrintToStream(out);
}

std::ostream &TokenIdentifier::printToStream(std::ostream &out) const
{
    return out << R"({"type":"IDENTIFIER","identifier":")" << getRaw() << R"("})";
}

bool TokenIdentifier::match(const Token &other) const
{
    auto o = dynamic_cast<const TokenIdentifier *>(&other);
    return o != nullptr;
}

std::ostream &TokenIdentifier::simplePrintToStream(std::ostream &out) const
{
    return out << "ID:'" << getRaw() << "'";
}


std::ostream &TokenUnsigned::printToStream(std::ostream &out) const
{
    return out << R"({"type":"UNSIGNED","value":)" << value << R"(})";
}

bool TokenUnsigned::match(const Token &other) const
{
    auto o = dynamic_cast<const TokenUnsigned *>(&other);
    return o != nullptr;
}

std::ostream &TokenUnsigned::simplePrintToStream(std::ostream &out) const
{
    return out << "UNSIGNED:" << getRaw();
}

std::ostream &TokenWhitespace::printToStream(std::ostream &out) const
{
    return out << R"({"type":"WHITESPACE"})";
}

bool TokenWhitespace::match(const Token &other) const
{
    auto o = dynamic_cast<const TokenWhitespace *>(&other);
    return o != nullptr;
}

std::ostream &TokenWhitespace::simplePrintToStream(std::ostream &out) const
{
    return out << "WHITESPACE";
}

std::ostream &TokenLineWrap::printToStream(std::ostream &out) const
{
    return out << R"({"type":"LINEWRAP"})";
}

bool TokenLineWrap::match(const Token &other) const
{
    auto o = dynamic_cast<const TokenLineWrap *>(&other);
    return o != nullptr;
}

std::ostream &TokenLineWrap::simplePrintToStream(std::ostream &out) const
{
    return out << "LINEWRAP";
}

std::ostream &TokenError::printToStream(std::ostream &out) const
{
    return out << R"({"type":"ERROR","error":")" << error << R"(","raw":")" << getRaw() << R"("})";
}

bool TokenError::match(const Token &other) const
{
    auto o = dynamic_cast<const TokenError *>(&other);
    return o != nullptr;
}

std::ostream &TokenError::simplePrintToStream(std::ostream &out) const
{
    return out << "ERROR:'" << error << "'";
}

std::ostream &TokenEnd::printToStream(std::ostream &out) const
{
    return out << R"({"type":"END"})";
}

bool TokenEnd::match(const Token &other) const
{
    auto o = dynamic_cast<const TokenEnd *>(&other);
    return o != nullptr;
}

std::ostream &TokenEnd::simplePrintToStream(std::ostream &out) const
{
    return out << "END";
}