Tokenizer.cpp File Reference

#include "Tokenizer.hpp"
#include "../utils/FixedString.hpp"
#include <algorithm>
#include <array>
#include <charconv>
#include <cmath>
#include <cstdlib>
#include <format>
#include <limits>
#include <locale>
#include <optional>
#include <ranges>
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>
#include <ctre.hpp>

Include dependency graph for Tokenizer.cpp:

Go to the source code of this file.

Functions
std::vector< Token >	tokenize (const std::string &expr, int num_inputs, ExprMode mode, int num_intermediate_inputs)
TokenBehavior	get_token_behavior (const Token &token)

Function Documentation

get_token_behavior()

TokenBehavior get_token_behavior

(

const Token &

token

)

Definition at line 1054 of file Tokenizer.cpp.

                                                     {
    constexpr auto TOKEN_DEFS = get_token_definitions();
 
    const auto* it = std::ranges::find_if(
        TOKEN_DEFS, [&](const auto& def) { return def.type == token.type; });
 
    return std::visit(
        [&token](auto&& arg) -> TokenBehavior {
            using T = std::decay_t<decltype(arg)>;
            if constexpr (std::is_same_v<T, TokenBehavior>) {
                return arg;
            } else if constexpr (std::is_same_v<T, DynamicBehaviorFn>) {
                return arg(token);
            }
        },
        it->behavior);
}

References Token::type.

Referenced by infix2postfix::compute_postfix_stack_effect(), analysis::BlockAnalysisPass::run(), analysis::ConstPropPass::run(), and analysis::StaticArrayOptPass::run().

tokenize()

std::vector< Token > tokenize	(	const std::string &	expr,
		int	num_inputs,
		ExprMode	mode,
		int	num_intermediate_inputs )

Definition at line 969 of file Tokenizer.cpp.

                                                                        {
    std::vector<Token> tokens;
    int idx = 0;
 
    auto is_space = [](char c) { return std::isspace(c); };
    auto to_string_view = [](auto r) {
        return std::string_view(r.begin(), r.end());
    };
 
    constexpr auto TOKEN_DEFS = get_token_definitions();
 
    for (const auto str_token_view :
         expr | std::views::chunk_by([=](char a, char b) {
             return is_space(a) == is_space(b);
         }) | std::views::filter([=](auto r) { return !is_space(r.front()); }) |
             std::views::transform(to_string_view)) {
        std::optional<Token> parsed_token;
 
        for (const auto& definition : TOKEN_DEFS) {
            // Check mode restrictions
            if (!supports_mode(definition.availability, mode)) {
                continue;
            }
 
            if ((parsed_token = definition.parser(str_token_view))) {
                break;
            }
        }
 
        if (!parsed_token) {
            throw std::runtime_error(std::format("Invalid token: {} (idx {})",
                                                 std::string(str_token_view),
                                                 idx));
        }
 
        // Post-parse validation for clip indices
        if (parsed_token->type == TokenType::ClipRel ||
            parsed_token->type == TokenType::ClipAbs ||
            parsed_token->type == TokenType::ClipCur) {
            if (std::get<TokenPayloadClipAccess>(parsed_token->payload)
                        .clip_idx < 0 ||
                std::get<TokenPayloadClipAccess>(parsed_token->payload)
                        .clip_idx >= num_inputs) {
                throw std::runtime_error(
                    std::format("Invalid clip index in token: {} (idx {})",
                                std::string(str_token_view), idx));
            }
        } else if (parsed_token->type == TokenType::PropAccess) {
            if (std::get<TokenPayloadPropAccess>(parsed_token->payload)
                        .clip_idx < 0 ||
                std::get<TokenPayloadPropAccess>(parsed_token->payload)
                        .clip_idx >= num_inputs) {
                throw std::runtime_error(
                    std::format("Invalid clip index in token: {} (idx {})",
                                std::string(str_token_view), idx));
            }
        } else if (parsed_token->type == TokenType::ClipAbsPlane) {
            if (std::get<TokenPayloadClipAccessPlane>(parsed_token->payload)
                        .clip_idx < 0 ||
                std::get<TokenPayloadClipAccessPlane>(parsed_token->payload)
                        .clip_idx >= num_inputs) {
                throw std::runtime_error(
                    std::format("Invalid clip index in token: {} (idx {})",
                                std::string(str_token_view), idx));
            }
        } else if (parsed_token->type == TokenType::BufferRel ||
                   parsed_token->type == TokenType::BufferAbs ||
                   parsed_token->type == TokenType::BufferCur) {
            if (std::get<TokenPayloadBufferAccess>(parsed_token->payload)
                        .buffer_idx < 0 ||
                std::get<TokenPayloadBufferAccess>(parsed_token->payload)
                        .buffer_idx >= num_intermediate_inputs) {
                throw std::runtime_error(
                    std::format("Invalid buffer index in token: {} (idx {})",
                                std::string(str_token_view), idx));
            }
        }
 
        tokens.push_back(*parsed_token);
        idx++;
    }
    return tokens;
}

References BufferAbs, BufferCur, BufferRel, ClipAbs, ClipAbsPlane, ClipCur, ClipRel, and PropAccess.

Referenced by infix2postfix::compute_postfix_stack_effect().