Tokenizer.hpp

Go to the documentation of this file.

 
#ifndef LLVMEXPR_FRONTEND_TOKENIZER_HPP
#define LLVMEXPR_FRONTEND_TOKENIZER_HPP
 
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
#include <variant>
#include <vector>
 
enum class TokenType : std::uint8_t {
    // Literals & Constants
    Number,
    ConstantX,               // X
    ConstantY,               // Y
    ConstantWidth,           // width
    ConstantHeight,          // height
    ConstantPlaneWidth,      // width^plane_no
    ConstantPlaneHeight,     // height^plane_no
    ConstantClipWidth,       // srcN:width
    ConstantClipHeight,      // srcN:height
    ConstantClipPlaneWidth,  // srcN:width^plane_no
    ConstantClipPlaneHeight, // srcN:height^plane_no
    ConstantN,               // N
    ConstantPi,              // pi
 
    // Variable Ops
    VarStore, // my_var!
    VarLoad,  // my_var@
 
    // Array Ops
    ArrayAllocStatic, // arr{}^N
    ArrayAllocDyn,    // arr{}^
    ArrayStore,       // arr{}!
    ArrayLoad,        // arr{}@
 
    // Data Access
    ClipRel,       // src[x,y]
    ClipAbs,       // src[]
    ClipCur,       // src
    PropAccess,    // src.prop
    PropExists,    // src.prop?
    ClipAbsPlane,  // src^plane[]
    StoreAbsPlane, // @[]^plane
    PropStore,     // prop$
    BufferRel,     // bufN[x,y]
    BufferAbs,     // bufN[]
    BufferCur,     // bufN
 
    // Binary Operators
    Add,
    Sub,
    Mul,
    Div,
    Mod,
    Gt,
    Lt,
    Ge,
    Le,
    Eq,
    And,
    Or,
    Xor,
    Bitand,
    Bitor,
    Bitxor,
    Pow,
    Atan2,
    Copysign,
    Min,
    Max,
 
    // Unary Operators
    Not,
    Bitnot,
    Sqrt,
    Exp,
    Log,
    Abs,
    Floor,
    Ceil,
    Trunc,
    Round,
    Sin,
    Cos,
    Tan,
    Asin,
    Acos,
    Atan,
    Exp2,
    Log10,
    Log2,
    Sinh,
    Cosh,
    Tanh,
    Sgn,
    Neg,
 
    // Ternary and other multi-arg
    Ternary, // ?
    Clip,
    Clamp, // same op, 3 args
    Fma,   // 3 args
 
    // Stack manipulation
    Dup,
    Drop,
    Swap,
    SortN,
    ArgminN,
    ArgmaxN,
    ArgsortN,
 
    // Control Flow
    LabelDef, // #my_label
    Jump,     // my_label#
 
    // Custom output control
    ExitNoWrite, // ^exit^
    StoreAbs,    // @[]
};
 
struct TokenPayloadNumber {
    double value;
};
 
struct TokenPayloadVar {
    std::string name;
};
 
struct TokenPayloadLabel {
    std::string name;
};
 
struct TokenPayloadStackOp {
    int n;
};
 
struct TokenPayloadClipAccess {
    int clip_idx;
    int rel_x = 0;
    int rel_y = 0;
    bool use_mirror = false;
    bool has_mode = false;
};
 
struct TokenPayloadPropAccess {
    int clip_idx;
    std::string prop_name;
};
 
struct TokenPayloadClipAccessPlane {
    int clip_idx;
    int plane_idx;
};
 
struct TokenPayloadStoreAbsPlane {
    int plane_idx;
};
 
enum class PropWriteType : std::uint8_t {
    Float,     // $f or $
    Int,       // $i
    AutoInt,   // $ai
    AutoFloat, // $af
    Delete,    // $d
};
 
struct TokenPayloadPropStore {
    std::string prop_name;
    PropWriteType type;
};
 
struct TokenPayloadPlaneDim {
    int plane_idx;
};
 
struct TokenPayloadClipDim {
    int clip_idx;
};
 
struct TokenPayloadClipPlaneDim {
    int clip_idx;
    int plane_idx;
};
 
struct TokenPayloadArrayOp {
    std::string name;
    int static_size = 0; // ARRAY_ALLOC_STATIC
};
 
struct TokenPayloadBufferAccess {
    int buffer_idx;
    int rel_x = 0;
    int rel_y = 0;
    bool use_mirror = false;
    bool has_mode = false;
};
 
struct Token {
    using PayloadVariant =
        std::variant<std::monostate, TokenPayloadNumber, TokenPayloadVar,
                     TokenPayloadLabel, TokenPayloadStackOp,
                     TokenPayloadClipAccess, TokenPayloadPropAccess,
                     TokenPayloadClipAccessPlane, TokenPayloadStoreAbsPlane,
                     TokenPayloadPropStore, TokenPayloadPlaneDim,
                     TokenPayloadClipDim, TokenPayloadClipPlaneDim,
                     TokenPayloadArrayOp, TokenPayloadBufferAccess>;
 
    TokenType type;
    std::string text;
    PayloadVariant payload;
};
 
struct TokenBehavior {
    int arity;
    int stack_effect;
};
 
using DynamicBehaviorFn = TokenBehavior (*)(const Token&);
using BehaviorResolver = std::variant<TokenBehavior, DynamicBehaviorFn>;
 
enum class ExprMode : std::uint8_t {
    Expr,
    SingleExpr,
    VkExpr,
};
 
// Utility functions
constexpr int parse_std_clip_idx(char c) {
    if (c >= 'x' && c <= 'z') {
        return c - 'x';
    }
    return c - 'a' + 3;
}
 
using TokenParser = std::optional<Token> (*)(std::string_view);
 
struct TokenDefinition {
    TokenType type;
    std::string_view name;
    BehaviorResolver behavior;
    TokenParser parser;
 
    enum class Availability : std::uint8_t {
        Expr = 1U << 0,
        SingleExpr = 1U << 1,
        VkExpr = 1U << 2,
    };
 
    Availability availability = static_cast<Availability>(
        static_cast<std::uint8_t>(Availability::Expr) |
        static_cast<std::uint8_t>(Availability::SingleExpr) |
        static_cast<std::uint8_t>(Availability::VkExpr));
};
 
std::vector<Token> tokenize(const std::string& expr, int num_inputs,
                            ExprMode mode = ExprMode::Expr,
                            int num_intermediate_inputs = 0);
TokenBehavior get_token_behavior(const Token& token);
 
#endif // LLVMEXPR_FRONTEND_TOKENIZER_HPP