VapourSynth-llvmexpr
Loading...
Searching...
No Matches
Tokenizer.hpp File Reference
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
#include <variant>
#include <vector>
Include dependency graph for Tokenizer.hpp:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes

struct  TokenPayloadNumber
struct  TokenPayloadVar
struct  TokenPayloadLabel
struct  TokenPayloadStackOp
struct  TokenPayloadClipAccess
struct  TokenPayloadPropAccess
struct  TokenPayloadClipAccessPlane
struct  TokenPayloadStoreAbsPlane
struct  TokenPayloadPropStore
struct  TokenPayloadPlaneDim
struct  TokenPayloadClipDim
struct  TokenPayloadClipPlaneDim
struct  TokenPayloadArrayOp
struct  TokenPayloadBufferAccess
struct  Token
struct  TokenBehavior
struct  TokenDefinition

Typedefs

using DynamicBehaviorFn = TokenBehavior (*)(const Token&)
using BehaviorResolver = std::variant<TokenBehavior, DynamicBehaviorFn>
using TokenParser = std::optional<Token> (*)(std::string_view)

Enumerations

enum class  TokenType : std::uint8_t {
  Number , ConstantX , ConstantY , ConstantWidth ,
  ConstantHeight , ConstantPlaneWidth , ConstantPlaneHeight , ConstantClipWidth ,
  ConstantClipHeight , ConstantClipPlaneWidth , ConstantClipPlaneHeight , ConstantN ,
  ConstantPi , VarStore , VarLoad , ArrayAllocStatic ,
  ArrayAllocDyn , ArrayStore , ArrayLoad , ClipRel ,
  ClipAbs , ClipCur , PropAccess , PropExists ,
  ClipAbsPlane , StoreAbsPlane , PropStore , BufferRel ,
  BufferAbs , BufferCur , Add , Sub ,
  Mul , Div , Mod , Gt ,
  Lt , Ge , Le , Eq ,
  And , Or , Xor , Bitand ,
  Bitor , Bitxor , Pow , Atan2 ,
  Copysign , Min , Max , Not ,
  Bitnot , Sqrt , Exp , Log ,
  Abs , Floor , Ceil , Trunc ,
  Round , Sin , Cos , Tan ,
  Asin , Acos , Atan , Exp2 ,
  Log10 , Log2 , Sinh , Cosh ,
  Tanh , Sgn , Neg , Ternary ,
  Clip , Clamp , Fma , Dup ,
  Drop , Swap , SortN , ArgminN ,
  ArgmaxN , ArgsortN , LabelDef , Jump ,
  ExitNoWrite , StoreAbs
}
enum class  PropWriteType : std::uint8_t {
  Float , Int , AutoInt , AutoFloat ,
  Delete
}
enum class  ExprMode : std::uint8_t { Expr , SingleExpr , VkExpr }

Functions

constexpr int parse_std_clip_idx (char c)
std::vector< Token > tokenize (const std::string &expr, int num_inputs, ExprMode mode=ExprMode::Expr, int num_intermediate_inputs=0)
TokenBehavior get_token_behavior (const Token &token)

Typedef Documentation

◆ BehaviorResolver

using BehaviorResolver = std::variant<TokenBehavior, DynamicBehaviorFn>

Definition at line 240 of file Tokenizer.hpp.

◆ DynamicBehaviorFn

using DynamicBehaviorFn = TokenBehavior (*)(const Token&)

Definition at line 239 of file Tokenizer.hpp.

◆ TokenParser

using TokenParser = std::optional<Token> (*)(std::string_view)

Definition at line 256 of file Tokenizer.hpp.

Enumeration Type Documentation

◆ ExprMode

enum class ExprMode : std::uint8_t
strong
Enumerator
Expr 
SingleExpr 
VkExpr 

Definition at line 242 of file Tokenizer.hpp.

242 : std::uint8_t {
243 Expr,
244 SingleExpr,
245 VkExpr,
246};

◆ PropWriteType

enum class PropWriteType : std::uint8_t
strong
Enumerator
Float 
Int 
AutoInt 
AutoFloat 
Delete 

Definition at line 180 of file Tokenizer.hpp.

180 : std::uint8_t {
181 Float, // $f or $
182 Int, // $i
183 AutoInt, // $ai
184 AutoFloat, // $af
185 Delete, // $d
186};

◆ TokenType

enum class TokenType : std::uint8_t
strong

Copyright (C) 2025 yuygfgg

This file is part of Vapoursynth-llvmexpr.

Vapoursynth-llvmexpr is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.

Vapoursynth-llvmexpr is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License along with Vapoursynth-llvmexpr. If not, see https://www.gnu.org/licenses/.

Enumerator
Number 
ConstantX 
ConstantY 
ConstantWidth 
ConstantHeight 
ConstantPlaneWidth 
ConstantPlaneHeight 
ConstantClipWidth 
ConstantClipHeight 
ConstantClipPlaneWidth 
ConstantClipPlaneHeight 
ConstantN 
ConstantPi 
VarStore 
VarLoad 
ArrayAllocStatic 
ArrayAllocDyn 
ArrayStore 
ArrayLoad 
ClipRel 
ClipAbs 
ClipCur 
PropAccess 
PropExists 
ClipAbsPlane 
StoreAbsPlane 
PropStore 
BufferRel 
BufferAbs 
BufferCur 
Add 
Sub 
Mul 
Div 
Mod 
Gt 
Lt 
Ge 
Le 
Eq 
And 
Or 
Xor 
Bitand 
Bitor 
Bitxor 
Pow 
Atan2 
Copysign 
Min 
Max 
Not 
Bitnot 
Sqrt 
Exp 
Log 
Abs 
Floor 
Ceil 
Trunc 
Round 
Sin 
Cos 
Tan 
Asin 
Acos 
Atan 
Exp2 
Log10 
Log2 
Sinh 
Cosh 
Tanh 
Sgn 
Neg 
Ternary 
Clip 
Clamp 
Fma 
Dup 
Drop 
Swap 
SortN 
ArgminN 
ArgmaxN 
ArgsortN 
LabelDef 
Jump 
ExitNoWrite 
StoreAbs 

Definition at line 30 of file Tokenizer.hpp.

30 : std::uint8_t {
31 // Literals & Constants
32 Number,
33 ConstantX, // X
34 ConstantY, // Y
35 ConstantWidth, // width
36 ConstantHeight, // height
37 ConstantPlaneWidth, // width^plane_no
38 ConstantPlaneHeight, // height^plane_no
39 ConstantClipWidth, // srcN:width
40 ConstantClipHeight, // srcN:height
41 ConstantClipPlaneWidth, // srcN:width^plane_no
42 ConstantClipPlaneHeight, // srcN:height^plane_no
43 ConstantN, // N
44 ConstantPi, // pi
45
46 // Variable Ops
47 VarStore, // my_var!
48 VarLoad, // my_var@
49
50 // Array Ops
51 ArrayAllocStatic, // arr{}^N
52 ArrayAllocDyn, // arr{}^
53 ArrayStore, // arr{}!
54 ArrayLoad, // arr{}@
55
56 // Data Access
57 ClipRel, // src[x,y]
58 ClipAbs, // src[]
59 ClipCur, // src
60 PropAccess, // src.prop
61 PropExists, // src.prop?
62 ClipAbsPlane, // src^plane[]
63 StoreAbsPlane, // @[]^plane
64 PropStore, // prop$
65 BufferRel, // bufN[x,y]
66 BufferAbs, // bufN[]
67 BufferCur, // bufN
68
69 // Binary Operators
70 Add,
71 Sub,
72 Mul,
73 Div,
74 Mod,
75 Gt,
76 Lt,
77 Ge,
78 Le,
79 Eq,
80 And,
81 Or,
82 Xor,
83 Bitand,
84 Bitor,
85 Bitxor,
86 Pow,
87 Atan2,
88 Copysign,
89 Min,
90 Max,
91
92 // Unary Operators
93 Not,
94 Bitnot,
95 Sqrt,
96 Exp,
97 Log,
98 Abs,
99 Floor,
100 Ceil,
101 Trunc,
102 Round,
103 Sin,
104 Cos,
105 Tan,
106 Asin,
107 Acos,
108 Atan,
109 Exp2,
110 Log10,
111 Log2,
112 Sinh,
113 Cosh,
114 Tanh,
115 Sgn,
116 Neg,
117
118 // Ternary and other multi-arg
119 Ternary, // ?
120 Clip,
121 Clamp, // same op, 3 args
122 Fma, // 3 args
123
124 // Stack manipulation
125 Dup,
126 Drop,
127 Swap,
128 SortN,
129 ArgminN,
130 ArgmaxN,
131 ArgsortN,
132
133 // Control Flow
134 LabelDef, // #my_label
135 Jump, // my_label#
136
137 // Custom output control
138 ExitNoWrite, // ^exit^
139 StoreAbs, // @[]
140};
TokenType::ConstantClipPlaneHeight
@ ConstantClipPlaneHeight
Definition Tokenizer.hpp:42
TokenType::ConstantPlaneWidth
@ ConstantPlaneWidth
Definition Tokenizer.hpp:37
TokenType::ArrayAllocStatic
@ ArrayAllocStatic
Definition Tokenizer.hpp:51
TokenType::ConstantPlaneHeight
@ ConstantPlaneHeight
Definition Tokenizer.hpp:38
TokenType::ConstantClipHeight
@ ConstantClipHeight
Definition Tokenizer.hpp:40
TokenType::ConstantClipPlaneWidth
@ ConstantClipPlaneWidth
Definition Tokenizer.hpp:41
TokenType::ConstantClipWidth
@ ConstantClipWidth
Definition Tokenizer.hpp:39

Function Documentation

◆ get_token_behavior()

TokenBehavior get_token_behavior ( const Token & token)

Definition at line 1054 of file Tokenizer.cpp.

1054 {
1055 constexpr auto TOKEN_DEFS = get_token_definitions();
1056
1057 const auto* it = std::ranges::find_if(
1058 TOKEN_DEFS, [&](const auto& def) { return def.type == token.type; });
1059
1060 return std::visit(
1061 [&token](auto&& arg) -> TokenBehavior {
1062 using T = std::decay_t<decltype(arg)>;
1063 if constexpr (std::is_same_v<T, TokenBehavior>) {
1064 return arg;
1065 } else if constexpr (std::is_same_v<T, DynamicBehaviorFn>) {
1066 return arg(token);
1067 }
1068 },
1069 it->behavior);
1070}
Token::type
TokenType type
Definition Tokenizer.hpp:229

References Token::type.

Referenced by infix2postfix::compute_postfix_stack_effect(), analysis::BlockAnalysisPass::run(), analysis::ConstPropPass::run(), and analysis::StaticArrayOptPass::run().

◆ parse_std_clip_idx()

int parse_std_clip_idx ( char c)
constexpr

Definition at line 249 of file Tokenizer.hpp.

249 {
250 if (c >= 'x' && c <= 'z') {
251 return c - 'x';
252 }
253 return c - 'a' + 3;
254}

Referenced by infix2postfix::get_clip_index().

◆ tokenize()

std::vector< Token > tokenize ( const std::string & expr,
int num_inputs,
ExprMode mode = ExprMode::Expr,
int num_intermediate_inputs = 0 )

Definition at line 969 of file Tokenizer.cpp.

970 {
971 std::vector<Token> tokens;
972 int idx = 0;
973
974 auto is_space = [](char c) { return std::isspace(c); };
975 auto to_string_view = [](auto r) {
976 return std::string_view(r.begin(), r.end());
977 };
978
979 constexpr auto TOKEN_DEFS = get_token_definitions();
980
981 for (const auto str_token_view :
982 expr | std::views::chunk_by([=](char a, char b) {
983 return is_space(a) == is_space(b);
984 }) | std::views::filter([=](auto r) { return !is_space(r.front()); }) |
985 std::views::transform(to_string_view)) {
986 std::optional<Token> parsed_token;
987
988 for (const auto& definition : TOKEN_DEFS) {
989 // Check mode restrictions
990 if (!supports_mode(definition.availability, mode)) {
991 continue;
992 }
993
994 if ((parsed_token = definition.parser(str_token_view))) {
995 break;
996 }
997 }
998
999 if (!parsed_token) {
1000 throw std::runtime_error(std::format("Invalid token: {} (idx {})",
1001 std::string(str_token_view),
1002 idx));
1003 }
1004
1005 // Post-parse validation for clip indices
1006 if (parsed_token->type == TokenType::ClipRel ||
1007 parsed_token->type == TokenType::ClipAbs ||
1008 parsed_token->type == TokenType::ClipCur) {
1009 if (std::get<TokenPayloadClipAccess>(parsed_token->payload)
1010 .clip_idx < 0 ||
1011 std::get<TokenPayloadClipAccess>(parsed_token->payload)
1012 .clip_idx >= num_inputs) {
1013 throw std::runtime_error(
1014 std::format("Invalid clip index in token: {} (idx {})",
1015 std::string(str_token_view), idx));
1016 }
1017 } else if (parsed_token->type == TokenType::PropAccess) {
1018 if (std::get<TokenPayloadPropAccess>(parsed_token->payload)
1019 .clip_idx < 0 ||
1020 std::get<TokenPayloadPropAccess>(parsed_token->payload)
1021 .clip_idx >= num_inputs) {
1022 throw std::runtime_error(
1023 std::format("Invalid clip index in token: {} (idx {})",
1024 std::string(str_token_view), idx));
1025 }
1026 } else if (parsed_token->type == TokenType::ClipAbsPlane) {
1027 if (std::get<TokenPayloadClipAccessPlane>(parsed_token->payload)
1028 .clip_idx < 0 ||
1029 std::get<TokenPayloadClipAccessPlane>(parsed_token->payload)
1030 .clip_idx >= num_inputs) {
1031 throw std::runtime_error(
1032 std::format("Invalid clip index in token: {} (idx {})",
1033 std::string(str_token_view), idx));
1034 }
1035 } else if (parsed_token->type == TokenType::BufferRel ||
1036 parsed_token->type == TokenType::BufferAbs ||
1037 parsed_token->type == TokenType::BufferCur) {
1038 if (std::get<TokenPayloadBufferAccess>(parsed_token->payload)
1039 .buffer_idx < 0 ||
1040 std::get<TokenPayloadBufferAccess>(parsed_token->payload)
1041 .buffer_idx >= num_intermediate_inputs) {
1042 throw std::runtime_error(
1043 std::format("Invalid buffer index in token: {} (idx {})",
1044 std::string(str_token_view), idx));
1045 }
1046 }
1047
1048 tokens.push_back(*parsed_token);
1049 idx++;
1050 }
1051 return tokens;
1052}

References BufferAbs, BufferCur, BufferRel, ClipAbs, ClipAbsPlane, ClipCur, ClipRel, and PropAccess.

Referenced by infix2postfix::compute_postfix_stack_effect().