sqlglot.tokens
1from __future__ import annotations 2 3import typing as t 4 5from sqlglot import tokenizer_core 6from sqlglot.trie import new_trie 7 8from sqlglot.tokenizer_core import Token, TokenizerCore, TokenType 9 10 11# The sqlglotc distribution ships no importable `sqlglotc` module; it overlays 12# compiled .so files onto sqlglot's modules, so detect it via the compiled core. 13SQLGLOTC_INSTALLED = not getattr(tokenizer_core, "__file__", "py").endswith(".py") 14 15try: 16 import sqlglotrs # type: ignore # noqa: F401 17 18 if not SQLGLOTC_INSTALLED: 19 import warnings 20 21 warnings.warn( 22 "sqlglot[rs] is deprecated and no longer compatible with sqlglot. " 23 "Please use sqlglotc instead for faster parsing: pip install sqlglot[c]", 24 ) 25except ImportError: 26 pass 27 28if t.TYPE_CHECKING: 29 from sqlglot.dialects.dialect import DialectType 30 31 32def _convert_quotes(arr: list[str | tuple[str, str]]) -> dict[str, str]: 33 return dict((item, item) if isinstance(item, str) else (item[0], item[1]) for item in arr) 34 35 36def _quotes_to_format( 37 token_type: TokenType, arr: list[str | tuple[str, str]] 38) -> dict[str, tuple[str, TokenType]]: 39 return {k: (v, token_type) for k, v in _convert_quotes(arr).items()} 40 41 42class _TokenizerBase: 43 QUOTES: t.ClassVar[list[tuple[str, str] | str]] 44 IDENTIFIERS: t.ClassVar[list[str | tuple[str, str]]] 45 BIT_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 46 BYTE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 47 HEX_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 48 RAW_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 49 HEREDOC_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 50 UNICODE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] 51 STRING_ESCAPES: t.ClassVar[list[str]] 52 BYTE_STRING_ESCAPES: t.ClassVar[list[str]] 53 ESCAPE_FOLLOW_CHARS: t.ClassVar[list[str]] 54 IDENTIFIER_ESCAPES: t.ClassVar[list[str]] 55 HINT_START: t.ClassVar[str] 56 KEYWORDS: t.ClassVar[dict[str, TokenType]] 57 SINGLE_TOKENS: t.ClassVar[dict[str, TokenType]] 58 NUMERIC_LITERALS: t.ClassVar[dict[str, str]] 59 VAR_SINGLE_TOKENS: t.ClassVar[set[str]] 60 COMMANDS: t.ClassVar[set[TokenType]] 61 COMMAND_PREFIX_TOKENS: t.ClassVar[set[TokenType]] 62 HEREDOC_TAG_IS_IDENTIFIER: t.ClassVar[bool] 63 STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS: t.ClassVar[bool] 64 NUMERIC_ESCAPES: t.ClassVar[dict[str, tuple[int, int, int, int]]] 65 DROP_UNKNOWN_ESCAPES: t.ClassVar[bool] 66 NESTED_COMMENTS: t.ClassVar[bool] 67 TOKENS_PRECEDING_HINT: t.ClassVar[set[TokenType]] 68 HEREDOC_STRING_ALTERNATIVE: t.ClassVar[TokenType] 69 COMMENTS: t.ClassVar[list[str | tuple[str, str]]] 70 _QUOTES: t.ClassVar[dict[str, str]] 71 _IDENTIFIERS: t.ClassVar[dict[str, str]] 72 _FORMAT_STRINGS: t.ClassVar[dict[str, tuple[str, TokenType]]] 73 _STRING_ESCAPES: t.ClassVar[set[str]] 74 _BYTE_STRING_ESCAPES: t.ClassVar[set[str]] 75 _ESCAPE_FOLLOW_CHARS: t.ClassVar[set[str]] 76 _IDENTIFIER_ESCAPES: t.ClassVar[set[str]] 77 _COMMENTS: t.ClassVar[dict[str, str | None]] 78 _KEYWORD_TRIE: t.ClassVar[dict[str | int, object]] 79 80 @classmethod 81 def __init_subclass__(cls, **kwargs: t.Any) -> None: 82 super().__init_subclass__(**kwargs) 83 cls._QUOTES = _convert_quotes(cls.QUOTES) 84 cls._IDENTIFIERS = _convert_quotes(cls.IDENTIFIERS) 85 cls._FORMAT_STRINGS = { 86 **{ 87 p + s: (e, TokenType.NATIONAL_STRING) 88 for s, e in cls._QUOTES.items() 89 for p in ("n", "N") 90 }, 91 **_quotes_to_format(TokenType.BIT_STRING, cls.BIT_STRINGS), 92 **_quotes_to_format(TokenType.BYTE_STRING, cls.BYTE_STRINGS), 93 **_quotes_to_format(TokenType.HEX_STRING, cls.HEX_STRINGS), 94 **_quotes_to_format(TokenType.RAW_STRING, cls.RAW_STRINGS), 95 **_quotes_to_format(TokenType.HEREDOC_STRING, cls.HEREDOC_STRINGS), 96 **_quotes_to_format(TokenType.UNICODE_STRING, cls.UNICODE_STRINGS), 97 } 98 if "BYTE_STRING_ESCAPES" not in cls.__dict__: 99 cls.BYTE_STRING_ESCAPES = cls.STRING_ESCAPES.copy() 100 cls._STRING_ESCAPES = set(cls.STRING_ESCAPES) 101 cls._BYTE_STRING_ESCAPES = set(cls.BYTE_STRING_ESCAPES) 102 cls._ESCAPE_FOLLOW_CHARS = set(cls.ESCAPE_FOLLOW_CHARS) 103 cls._IDENTIFIER_ESCAPES = set(cls.IDENTIFIER_ESCAPES) 104 cls._COMMENTS = { 105 **{c: None for c in cls.COMMENTS if isinstance(c, str)}, 106 **{c[0]: c[1] for c in cls.COMMENTS if not isinstance(c, str)}, 107 "{#": "#}", # Ensure Jinja comments are tokenized correctly in all dialects 108 } 109 if cls.HINT_START in cls.KEYWORDS: 110 cls._COMMENTS[cls.HINT_START] = "*/" 111 cls._KEYWORD_TRIE = new_trie( 112 key.upper() 113 for key in ( 114 *cls.KEYWORDS, 115 *cls._COMMENTS, 116 *cls._QUOTES, 117 *cls._FORMAT_STRINGS, 118 ) 119 if " " in key or any(single in key for single in cls.SINGLE_TOKENS) 120 ) 121 122 123class Tokenizer(_TokenizerBase): 124 SINGLE_TOKENS = { 125 "(": TokenType.L_PAREN, 126 ")": TokenType.R_PAREN, 127 "[": TokenType.L_BRACKET, 128 "]": TokenType.R_BRACKET, 129 "{": TokenType.L_BRACE, 130 "}": TokenType.R_BRACE, 131 "&": TokenType.AMP, 132 "^": TokenType.CARET, 133 ":": TokenType.COLON, 134 ",": TokenType.COMMA, 135 ".": TokenType.DOT, 136 "-": TokenType.DASH, 137 "=": TokenType.EQ, 138 ">": TokenType.GT, 139 "<": TokenType.LT, 140 "%": TokenType.MOD, 141 "!": TokenType.NOT, 142 "|": TokenType.PIPE, 143 "+": TokenType.PLUS, 144 ";": TokenType.SEMICOLON, 145 "/": TokenType.SLASH, 146 "\\": TokenType.BACKSLASH, 147 "*": TokenType.STAR, 148 "~": TokenType.TILDE, 149 "?": TokenType.PLACEHOLDER, 150 "@": TokenType.PARAMETER, 151 "#": TokenType.HASH, 152 # Used for breaking a var like x'y' but nothing else the token type doesn't matter 153 "'": TokenType.UNKNOWN, 154 "`": TokenType.UNKNOWN, 155 '"': TokenType.UNKNOWN, 156 } 157 158 BIT_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 159 BYTE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 160 HEX_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 161 RAW_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 162 HEREDOC_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 163 UNICODE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 164 IDENTIFIERS: t.ClassVar[list[str | tuple[str, str]]] = ['"'] 165 QUOTES: t.ClassVar[list[tuple[str, str] | str]] = ["'"] 166 STRING_ESCAPES: t.ClassVar[list[str]] = ["'"] 167 BYTE_STRING_ESCAPES: t.ClassVar[list[str]] = [] 168 VAR_SINGLE_TOKENS: t.ClassVar[set[str]] = set() 169 ESCAPE_FOLLOW_CHARS: t.ClassVar[list[str]] = [] 170 171 # The strings in this list can always be used as escapes, regardless of the surrounding 172 # identifier delimiters. By default, the closing delimiter is assumed to also act as an 173 # identifier escape, e.g. if we use double-quotes, then they also act as escapes: "x""" 174 IDENTIFIER_ESCAPES: t.ClassVar[list[str]] = [] 175 176 # Whether the heredoc tags follow the same lexical rules as unquoted identifiers 177 HEREDOC_TAG_IS_IDENTIFIER = False 178 179 # Token that we'll generate as a fallback if the heredoc prefix doesn't correspond to a heredoc 180 HEREDOC_STRING_ALTERNATIVE = TokenType.VAR 181 182 # Whether string escape characters function as such when placed within raw strings 183 STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS = True 184 185 # Maps the char after a backslash to (base, min digits, max digits, max value) for escape 186 # sequences that encode a code point, e.g. {"x": (16, 2, 2, 0xFF)} decodes '\x41' as 'A'. 187 # Digits are read until the next one would exceed the max value; if fewer than the min digits 188 # are read, the sequence isn't decoded. Octal escapes, e.g. '\101', are keyed by "0". 189 NUMERIC_ESCAPES: t.ClassVar[dict[str, tuple[int, int, int, int]]] = {} 190 191 # Whether the backslash is dropped from escape sequences that aren't otherwise decoded, 192 # e.g. '\z' is 'z' 193 DROP_UNKNOWN_ESCAPES = False 194 195 NESTED_COMMENTS = True 196 197 HINT_START = "/*+" 198 199 TOKENS_PRECEDING_HINT = {TokenType.SELECT, TokenType.INSERT, TokenType.UPDATE, TokenType.DELETE} 200 201 # Autofilled 202 _COMMENTS: t.ClassVar[dict[str, str | None]] = {} 203 _FORMAT_STRINGS: t.ClassVar[dict[str, tuple[str, TokenType]]] = {} 204 _IDENTIFIERS: t.ClassVar[dict[str, str]] = {} 205 _IDENTIFIER_ESCAPES: t.ClassVar[set[str]] = set() 206 _QUOTES: t.ClassVar[dict[str, str]] = {} 207 _STRING_ESCAPES: t.ClassVar[set[str]] = set() 208 _BYTE_STRING_ESCAPES: t.ClassVar[set[str]] = set() 209 _KEYWORD_TRIE: t.ClassVar[dict[str | int, object]] = {} 210 _ESCAPE_FOLLOW_CHARS: t.ClassVar[set[str]] = set() 211 212 KEYWORDS: t.ClassVar[dict[str, TokenType]] = { 213 **{f"{{%{postfix}": TokenType.BLOCK_START for postfix in ("", "+", "-")}, 214 **{f"{prefix}%}}": TokenType.BLOCK_END for prefix in ("", "+", "-")}, 215 **{f"{{{{{postfix}": TokenType.BLOCK_START for postfix in ("+", "-")}, 216 **{f"{prefix}}}}}": TokenType.BLOCK_END for prefix in ("+", "-")}, 217 HINT_START: TokenType.HINT, 218 "&<": TokenType.AMP_LT, 219 "&>": TokenType.AMP_GT, 220 "==": TokenType.EQ, 221 "::": TokenType.DCOLON, 222 "?::": TokenType.QDCOLON, 223 "||": TokenType.DPIPE, 224 "|>": TokenType.PIPE_GT, 225 ">=": TokenType.GTE, 226 "<=": TokenType.LTE, 227 "<>": TokenType.NEQ, 228 "!=": TokenType.NEQ, 229 ":=": TokenType.COLON_EQ, 230 "<=>": TokenType.NULLSAFE_EQ, 231 "->": TokenType.ARROW, 232 "->>": TokenType.DARROW, 233 "=>": TokenType.FARROW, 234 "#>": TokenType.HASH_ARROW, 235 "#>>": TokenType.DHASH_ARROW, 236 "<->": TokenType.LR_ARROW, 237 "<<->>": TokenType.LLRR_ARROW, 238 "&&": TokenType.DAMP, 239 "??": TokenType.DQMARK, 240 "~~~": TokenType.GLOB, 241 "~~": TokenType.LIKE, 242 "~~*": TokenType.ILIKE, 243 "~*": TokenType.IRLIKE, 244 "-|-": TokenType.ADJACENT, 245 "ALL": TokenType.ALL, 246 "AND": TokenType.AND, 247 "ANTI": TokenType.ANTI, 248 "ANY": TokenType.ANY, 249 "ASC": TokenType.ASC, 250 "AS": TokenType.ALIAS, 251 "ASOF": TokenType.ASOF, 252 "AUTOINCREMENT": TokenType.AUTO_INCREMENT, 253 "AUTO_INCREMENT": TokenType.AUTO_INCREMENT, 254 "BEGIN": TokenType.BEGIN, 255 "BETWEEN": TokenType.BETWEEN, 256 "CACHE": TokenType.CACHE, 257 "UNCACHE": TokenType.UNCACHE, 258 "CASE": TokenType.CASE, 259 "CLUSTER BY": TokenType.CLUSTER_BY, 260 "COLLATE": TokenType.COLLATE, 261 "COLUMN": TokenType.COLUMN, 262 "COMMIT": TokenType.COMMIT, 263 "CONNECT BY": TokenType.CONNECT_BY, 264 "CONSTRAINT": TokenType.CONSTRAINT, 265 "COPY": TokenType.COPY, 266 "CREATE": TokenType.CREATE, 267 "CROSS": TokenType.CROSS, 268 "CUBE": TokenType.CUBE, 269 "CURRENT_DATE": TokenType.CURRENT_DATE, 270 "CURRENT_SCHEMA": TokenType.CURRENT_SCHEMA, 271 "CURRENT_TIME": TokenType.CURRENT_TIME, 272 "CURRENT_TIMESTAMP": TokenType.CURRENT_TIMESTAMP, 273 "CURRENT_USER": TokenType.CURRENT_USER, 274 "CURRENT_CATALOG": TokenType.CURRENT_CATALOG, 275 "DATABASE": TokenType.DATABASE, 276 "DEFAULT": TokenType.DEFAULT, 277 "DELETE": TokenType.DELETE, 278 "DESC": TokenType.DESC, 279 "DESCRIBE": TokenType.DESCRIBE, 280 "DISTINCT": TokenType.DISTINCT, 281 "DISTRIBUTE BY": TokenType.DISTRIBUTE_BY, 282 "DIV": TokenType.DIV, 283 "DROP": TokenType.DROP, 284 "ELSE": TokenType.ELSE, 285 "END": TokenType.END, 286 "ENUM": TokenType.ENUM, 287 "ESCAPE": TokenType.ESCAPE, 288 "EXCEPT": TokenType.EXCEPT, 289 "EXECUTE": TokenType.EXECUTE, 290 "EXISTS": TokenType.EXISTS, 291 "FALSE": TokenType.FALSE, 292 "FETCH": TokenType.FETCH, 293 "FILTER": TokenType.FILTER, 294 "FILE": TokenType.FILE, 295 "FIRST": TokenType.FIRST, 296 "FULL": TokenType.FULL, 297 "FUNCTION": TokenType.FUNCTION, 298 "FOR": TokenType.FOR, 299 "FOREIGN KEY": TokenType.FOREIGN_KEY, 300 "FORMAT": TokenType.FORMAT, 301 "FROM": TokenType.FROM, 302 "GEOGRAPHY": TokenType.GEOGRAPHY, 303 "GEOMETRY": TokenType.GEOMETRY, 304 "GLOB": TokenType.GLOB, 305 "GROUP BY": TokenType.GROUP_BY, 306 "GROUPING SETS": TokenType.GROUPING_SETS, 307 "HAVING": TokenType.HAVING, 308 "ILIKE": TokenType.ILIKE, 309 "IN": TokenType.IN, 310 "INDEX": TokenType.INDEX, 311 "INET": TokenType.INET, 312 "INNER": TokenType.INNER, 313 "INSERT": TokenType.INSERT, 314 "INTERVAL": TokenType.INTERVAL, 315 "INTERSECT": TokenType.INTERSECT, 316 "INTO": TokenType.INTO, 317 "IS": TokenType.IS, 318 "ISNULL": TokenType.ISNULL, 319 "JOIN": TokenType.JOIN, 320 "KEEP": TokenType.KEEP, 321 "KILL": TokenType.KILL, 322 "LATERAL": TokenType.LATERAL, 323 "LEFT": TokenType.LEFT, 324 "LIKE": TokenType.LIKE, 325 "LIMIT": TokenType.LIMIT, 326 "LOAD": TokenType.LOAD, 327 "LOCALTIME": TokenType.LOCALTIME, 328 "LOCALTIMESTAMP": TokenType.LOCALTIMESTAMP, 329 "LOCK": TokenType.LOCK, 330 "MERGE": TokenType.MERGE, 331 "NAMESPACE": TokenType.NAMESPACE, 332 "NATURAL": TokenType.NATURAL, 333 "NEXT": TokenType.NEXT, 334 "NOT": TokenType.NOT, 335 "NOTNULL": TokenType.NOTNULL, 336 "NULL": TokenType.NULL, 337 "OBJECT": TokenType.OBJECT, 338 "OFFSET": TokenType.OFFSET, 339 "ON": TokenType.ON, 340 "OR": TokenType.OR, 341 "XOR": TokenType.XOR, 342 "ORDER BY": TokenType.ORDER_BY, 343 "ORDINALITY": TokenType.ORDINALITY, 344 "OUT": TokenType.OUT, 345 "OUTER": TokenType.OUTER, 346 "OVER": TokenType.OVER, 347 "OVERLAPS": TokenType.OVERLAPS, 348 "OVERWRITE": TokenType.OVERWRITE, 349 "PARTITION": TokenType.PARTITION, 350 "PARTITION BY": TokenType.PARTITION_BY, 351 "PARTITIONED BY": TokenType.PARTITION_BY, 352 "PARTITIONED_BY": TokenType.PARTITION_BY, 353 "PERCENT": TokenType.PERCENT, 354 "PIVOT": TokenType.PIVOT, 355 "PRAGMA": TokenType.PRAGMA, 356 "PRIMARY KEY": TokenType.PRIMARY_KEY, 357 "PROCEDURE": TokenType.PROCEDURE, 358 "OPERATOR": TokenType.OPERATOR, 359 "QUALIFY": TokenType.QUALIFY, 360 "RANGE": TokenType.RANGE, 361 "RECURSIVE": TokenType.RECURSIVE, 362 "REGEXP": TokenType.RLIKE, 363 "RENAME": TokenType.RENAME, 364 "REPLACE": TokenType.REPLACE, 365 "RETURNING": TokenType.RETURNING, 366 "REFERENCES": TokenType.REFERENCES, 367 "RIGHT": TokenType.RIGHT, 368 "RLIKE": TokenType.RLIKE, 369 "ROLLBACK": TokenType.ROLLBACK, 370 "ROLLUP": TokenType.ROLLUP, 371 "ROW": TokenType.ROW, 372 "ROWS": TokenType.ROWS, 373 "SCHEMA": TokenType.SCHEMA, 374 "SELECT": TokenType.SELECT, 375 "SEMI": TokenType.SEMI, 376 "SESSION": TokenType.SESSION, 377 "SESSION_USER": TokenType.SESSION_USER, 378 "SET": TokenType.SET, 379 "SETTINGS": TokenType.SETTINGS, 380 "SHOW": TokenType.SHOW, 381 "SIMILAR TO": TokenType.SIMILAR_TO, 382 "SOME": TokenType.SOME, 383 "SORT BY": TokenType.SORT_BY, 384 "SQL SECURITY": TokenType.SQL_SECURITY, 385 "STRAIGHT_JOIN": TokenType.STRAIGHT_JOIN, 386 "TABLE": TokenType.TABLE, 387 "TABLESAMPLE": TokenType.TABLE_SAMPLE, 388 "TEMP": TokenType.TEMPORARY, 389 "TEMPORARY": TokenType.TEMPORARY, 390 "THEN": TokenType.THEN, 391 "TRUE": TokenType.TRUE, 392 "TRUNCATE": TokenType.TRUNCATE, 393 "TRIGGER": TokenType.TRIGGER, 394 "UNION": TokenType.UNION, 395 "UNKNOWN": TokenType.UNKNOWN, 396 "UNNEST": TokenType.UNNEST, 397 "UNPIVOT": TokenType.UNPIVOT, 398 "UPDATE": TokenType.UPDATE, 399 "USE": TokenType.USE, 400 "USING": TokenType.USING, 401 "UUID": TokenType.UUID, 402 "VALUES": TokenType.VALUES, 403 "VIEW": TokenType.VIEW, 404 "VOLATILE": TokenType.VOLATILE, 405 "WHEN": TokenType.WHEN, 406 "WHERE": TokenType.WHERE, 407 "WINDOW": TokenType.WINDOW, 408 "WITH": TokenType.WITH, 409 "APPLY": TokenType.APPLY, 410 "ARRAY": TokenType.ARRAY, 411 "BIT": TokenType.BIT, 412 "BOOL": TokenType.BOOLEAN, 413 "BOOLEAN": TokenType.BOOLEAN, 414 "BYTE": TokenType.TINYINT, 415 "MEDIUMINT": TokenType.MEDIUMINT, 416 "INT1": TokenType.TINYINT, 417 "TINYINT": TokenType.TINYINT, 418 "INT16": TokenType.SMALLINT, 419 "SHORT": TokenType.SMALLINT, 420 "SMALLINT": TokenType.SMALLINT, 421 "HUGEINT": TokenType.INT128, 422 "UHUGEINT": TokenType.UINT128, 423 "INT2": TokenType.SMALLINT, 424 "INTEGER": TokenType.INT, 425 "INT": TokenType.INT, 426 "INT4": TokenType.INT, 427 "INT32": TokenType.INT, 428 "INT64": TokenType.BIGINT, 429 "INT128": TokenType.INT128, 430 "INT256": TokenType.INT256, 431 "LONG": TokenType.BIGINT, 432 "BIGINT": TokenType.BIGINT, 433 "INT8": TokenType.TINYINT, 434 "UINT": TokenType.UINT, 435 "UINT128": TokenType.UINT128, 436 "UINT256": TokenType.UINT256, 437 "DEC": TokenType.DECIMAL, 438 "DECIMAL": TokenType.DECIMAL, 439 "DECIMAL32": TokenType.DECIMAL32, 440 "DECIMAL64": TokenType.DECIMAL64, 441 "DECIMAL128": TokenType.DECIMAL128, 442 "DECIMAL256": TokenType.DECIMAL256, 443 "DECFLOAT": TokenType.DECFLOAT, 444 "BIGDECIMAL": TokenType.BIGDECIMAL, 445 "BIGNUMERIC": TokenType.BIGDECIMAL, 446 "BIGNUM": TokenType.BIGNUM, 447 "LIST": TokenType.LIST, 448 "MAP": TokenType.MAP, 449 "NULLABLE": TokenType.NULLABLE, 450 "NUMBER": TokenType.DECIMAL, 451 "NUMERIC": TokenType.DECIMAL, 452 "FIXED": TokenType.DECIMAL, 453 "REAL": TokenType.FLOAT, 454 "FLOAT": TokenType.FLOAT, 455 "FLOAT4": TokenType.FLOAT, 456 "FLOAT8": TokenType.DOUBLE, 457 "DOUBLE": TokenType.DOUBLE, 458 "DOUBLE PRECISION": TokenType.DOUBLE, 459 "JSON": TokenType.JSON, 460 "JSONB": TokenType.JSONB, 461 "CHAR": TokenType.CHAR, 462 "CHARACTER": TokenType.CHAR, 463 "CHAR VARYING": TokenType.VARCHAR, 464 "CHARACTER VARYING": TokenType.VARCHAR, 465 "NCHAR": TokenType.NCHAR, 466 "VARCHAR": TokenType.VARCHAR, 467 "VARCHAR2": TokenType.VARCHAR, 468 "NVARCHAR": TokenType.NVARCHAR, 469 "NVARCHAR2": TokenType.NVARCHAR, 470 "BPCHAR": TokenType.BPCHAR, 471 "STR": TokenType.TEXT, 472 "STRING": TokenType.TEXT, 473 "TEXT": TokenType.TEXT, 474 "LONGTEXT": TokenType.LONGTEXT, 475 "MEDIUMTEXT": TokenType.MEDIUMTEXT, 476 "TINYTEXT": TokenType.TINYTEXT, 477 "CLOB": TokenType.TEXT, 478 "LONGVARCHAR": TokenType.TEXT, 479 "BINARY": TokenType.BINARY, 480 "BLOB": TokenType.VARBINARY, 481 "LONGBLOB": TokenType.LONGBLOB, 482 "MEDIUMBLOB": TokenType.MEDIUMBLOB, 483 "TINYBLOB": TokenType.TINYBLOB, 484 "BYTEA": TokenType.VARBINARY, 485 "VARBINARY": TokenType.VARBINARY, 486 "TIME": TokenType.TIME, 487 "TIMETZ": TokenType.TIMETZ, 488 "TIME_NS": TokenType.TIME_NS, 489 "TIMESTAMP": TokenType.TIMESTAMP, 490 "TIMESTAMPTZ": TokenType.TIMESTAMPTZ, 491 "TIMESTAMPLTZ": TokenType.TIMESTAMPLTZ, 492 "TIMESTAMP_LTZ": TokenType.TIMESTAMPLTZ, 493 "TIMESTAMPNTZ": TokenType.TIMESTAMPNTZ, 494 "TIMESTAMP_NTZ": TokenType.TIMESTAMPNTZ, 495 "DATE": TokenType.DATE, 496 "DATETIME": TokenType.DATETIME, 497 "INT4RANGE": TokenType.INT4RANGE, 498 "INT4MULTIRANGE": TokenType.INT4MULTIRANGE, 499 "INT8RANGE": TokenType.INT8RANGE, 500 "INT8MULTIRANGE": TokenType.INT8MULTIRANGE, 501 "NUMRANGE": TokenType.NUMRANGE, 502 "NUMMULTIRANGE": TokenType.NUMMULTIRANGE, 503 "TSRANGE": TokenType.TSRANGE, 504 "TSMULTIRANGE": TokenType.TSMULTIRANGE, 505 "TSTZRANGE": TokenType.TSTZRANGE, 506 "TSTZMULTIRANGE": TokenType.TSTZMULTIRANGE, 507 "DATERANGE": TokenType.DATERANGE, 508 "DATEMULTIRANGE": TokenType.DATEMULTIRANGE, 509 "UNIQUE": TokenType.UNIQUE, 510 "VECTOR": TokenType.VECTOR, 511 "STRUCT": TokenType.STRUCT, 512 "SEQUENCE": TokenType.SEQUENCE, 513 "VARIANT": TokenType.VARIANT, 514 "ALTER": TokenType.ALTER, 515 "ANALYZE": TokenType.ANALYZE, 516 "CALL": TokenType.COMMAND, 517 "COMMENT": TokenType.COMMENT, 518 "EXPLAIN": TokenType.COMMAND, 519 "GRANT": TokenType.GRANT, 520 "REVOKE": TokenType.REVOKE, 521 "OPTIMIZE": TokenType.COMMAND, 522 "PREPARE": TokenType.COMMAND, 523 "VACUUM": TokenType.COMMAND, 524 "USER-DEFINED": TokenType.USERDEFINED, 525 } 526 527 COMMANDS = { 528 TokenType.COMMAND, 529 TokenType.EXECUTE, 530 TokenType.FETCH, 531 TokenType.SHOW, 532 TokenType.RENAME, 533 } 534 535 COMMAND_PREFIX_TOKENS = {TokenType.SEMICOLON, TokenType.BEGIN} 536 537 # Handle numeric literals like in hive (3L = BIGINT) 538 NUMERIC_LITERALS: t.ClassVar[dict[str, str]] = {} 539 540 # In tokenizers like JSONPath, dots are always key separators, never decimal points 541 NUMBERS_CAN_HAVE_DECIMALS: t.ClassVar[bool] = True 542 543 COMMENTS = ["--", ("/*", "*/")] 544 545 __slots__ = ( 546 "dialect", 547 "_core", 548 ) 549 550 def __init__(self, dialect: DialectType = None) -> None: 551 from sqlglot.dialects.dialect import Dialect 552 553 self.dialect = Dialect.get_or_raise(dialect) 554 self._core = self._init_core() 555 556 def _init_core(self) -> TokenizerCore: 557 return TokenizerCore( 558 single_tokens=self.SINGLE_TOKENS, 559 keywords=self.KEYWORDS, 560 quotes=self._QUOTES, 561 format_strings=self._FORMAT_STRINGS, 562 identifiers=self._IDENTIFIERS, 563 comments=self._COMMENTS, 564 string_escapes=self._STRING_ESCAPES, 565 byte_string_escapes=self._BYTE_STRING_ESCAPES, 566 identifier_escapes=self._IDENTIFIER_ESCAPES, 567 escape_follow_chars=self._ESCAPE_FOLLOW_CHARS, 568 commands=self.COMMANDS, 569 command_prefix_tokens=self.COMMAND_PREFIX_TOKENS, 570 nested_comments=self.NESTED_COMMENTS, 571 hint_start=self.HINT_START, 572 tokens_preceding_hint=self.TOKENS_PRECEDING_HINT, 573 has_bit_strings=bool(self.BIT_STRINGS), 574 has_hex_strings=bool(self.HEX_STRINGS), 575 numeric_literals=self.NUMERIC_LITERALS, 576 var_single_tokens=self.VAR_SINGLE_TOKENS, 577 string_escapes_allowed_in_raw_strings=self.STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS, 578 heredoc_tag_is_identifier=self.HEREDOC_TAG_IS_IDENTIFIER, 579 heredoc_string_alternative=self.HEREDOC_STRING_ALTERNATIVE, 580 keyword_trie=self._KEYWORD_TRIE, 581 numbers_can_be_underscore_separated=self.dialect.NUMBERS_CAN_BE_UNDERSCORE_SEPARATED, 582 numbers_can_have_decimals=self.NUMBERS_CAN_HAVE_DECIMALS, 583 identifiers_can_start_with_digit=self.dialect.IDENTIFIERS_CAN_START_WITH_DIGIT, 584 unescaped_sequences=self.dialect.UNESCAPED_SEQUENCES, 585 numeric_escapes=self.NUMERIC_ESCAPES, 586 drop_unknown_escapes=self.DROP_UNKNOWN_ESCAPES, 587 ) 588 589 def tokenize(self, sql: str) -> list[Token]: 590 """Returns a list of tokens corresponding to the SQL string `sql`.""" 591 return self._core.tokenize(sql) # type: ignore 592 593 @property 594 def sql(self) -> str: 595 """The SQL string being tokenized.""" 596 return self._core.sql 597 598 @property 599 def size(self) -> int: 600 """Length of the SQL string.""" 601 return self._core.size 602 603 @property 604 def tokens(self) -> list[Token]: 605 """The list of tokens produced by tokenization.""" 606 return self._core.tokens
SQLGLOTC_INSTALLED =
False
124class Tokenizer(_TokenizerBase): 125 SINGLE_TOKENS = { 126 "(": TokenType.L_PAREN, 127 ")": TokenType.R_PAREN, 128 "[": TokenType.L_BRACKET, 129 "]": TokenType.R_BRACKET, 130 "{": TokenType.L_BRACE, 131 "}": TokenType.R_BRACE, 132 "&": TokenType.AMP, 133 "^": TokenType.CARET, 134 ":": TokenType.COLON, 135 ",": TokenType.COMMA, 136 ".": TokenType.DOT, 137 "-": TokenType.DASH, 138 "=": TokenType.EQ, 139 ">": TokenType.GT, 140 "<": TokenType.LT, 141 "%": TokenType.MOD, 142 "!": TokenType.NOT, 143 "|": TokenType.PIPE, 144 "+": TokenType.PLUS, 145 ";": TokenType.SEMICOLON, 146 "/": TokenType.SLASH, 147 "\\": TokenType.BACKSLASH, 148 "*": TokenType.STAR, 149 "~": TokenType.TILDE, 150 "?": TokenType.PLACEHOLDER, 151 "@": TokenType.PARAMETER, 152 "#": TokenType.HASH, 153 # Used for breaking a var like x'y' but nothing else the token type doesn't matter 154 "'": TokenType.UNKNOWN, 155 "`": TokenType.UNKNOWN, 156 '"': TokenType.UNKNOWN, 157 } 158 159 BIT_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 160 BYTE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 161 HEX_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 162 RAW_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 163 HEREDOC_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 164 UNICODE_STRINGS: t.ClassVar[list[str | tuple[str, str]]] = [] 165 IDENTIFIERS: t.ClassVar[list[str | tuple[str, str]]] = ['"'] 166 QUOTES: t.ClassVar[list[tuple[str, str] | str]] = ["'"] 167 STRING_ESCAPES: t.ClassVar[list[str]] = ["'"] 168 BYTE_STRING_ESCAPES: t.ClassVar[list[str]] = [] 169 VAR_SINGLE_TOKENS: t.ClassVar[set[str]] = set() 170 ESCAPE_FOLLOW_CHARS: t.ClassVar[list[str]] = [] 171 172 # The strings in this list can always be used as escapes, regardless of the surrounding 173 # identifier delimiters. By default, the closing delimiter is assumed to also act as an 174 # identifier escape, e.g. if we use double-quotes, then they also act as escapes: "x""" 175 IDENTIFIER_ESCAPES: t.ClassVar[list[str]] = [] 176 177 # Whether the heredoc tags follow the same lexical rules as unquoted identifiers 178 HEREDOC_TAG_IS_IDENTIFIER = False 179 180 # Token that we'll generate as a fallback if the heredoc prefix doesn't correspond to a heredoc 181 HEREDOC_STRING_ALTERNATIVE = TokenType.VAR 182 183 # Whether string escape characters function as such when placed within raw strings 184 STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS = True 185 186 # Maps the char after a backslash to (base, min digits, max digits, max value) for escape 187 # sequences that encode a code point, e.g. {"x": (16, 2, 2, 0xFF)} decodes '\x41' as 'A'. 188 # Digits are read until the next one would exceed the max value; if fewer than the min digits 189 # are read, the sequence isn't decoded. Octal escapes, e.g. '\101', are keyed by "0". 190 NUMERIC_ESCAPES: t.ClassVar[dict[str, tuple[int, int, int, int]]] = {} 191 192 # Whether the backslash is dropped from escape sequences that aren't otherwise decoded, 193 # e.g. '\z' is 'z' 194 DROP_UNKNOWN_ESCAPES = False 195 196 NESTED_COMMENTS = True 197 198 HINT_START = "/*+" 199 200 TOKENS_PRECEDING_HINT = {TokenType.SELECT, TokenType.INSERT, TokenType.UPDATE, TokenType.DELETE} 201 202 # Autofilled 203 _COMMENTS: t.ClassVar[dict[str, str | None]] = {} 204 _FORMAT_STRINGS: t.ClassVar[dict[str, tuple[str, TokenType]]] = {} 205 _IDENTIFIERS: t.ClassVar[dict[str, str]] = {} 206 _IDENTIFIER_ESCAPES: t.ClassVar[set[str]] = set() 207 _QUOTES: t.ClassVar[dict[str, str]] = {} 208 _STRING_ESCAPES: t.ClassVar[set[str]] = set() 209 _BYTE_STRING_ESCAPES: t.ClassVar[set[str]] = set() 210 _KEYWORD_TRIE: t.ClassVar[dict[str | int, object]] = {} 211 _ESCAPE_FOLLOW_CHARS: t.ClassVar[set[str]] = set() 212 213 KEYWORDS: t.ClassVar[dict[str, TokenType]] = { 214 **{f"{{%{postfix}": TokenType.BLOCK_START for postfix in ("", "+", "-")}, 215 **{f"{prefix}%}}": TokenType.BLOCK_END for prefix in ("", "+", "-")}, 216 **{f"{{{{{postfix}": TokenType.BLOCK_START for postfix in ("+", "-")}, 217 **{f"{prefix}}}}}": TokenType.BLOCK_END for prefix in ("+", "-")}, 218 HINT_START: TokenType.HINT, 219 "&<": TokenType.AMP_LT, 220 "&>": TokenType.AMP_GT, 221 "==": TokenType.EQ, 222 "::": TokenType.DCOLON, 223 "?::": TokenType.QDCOLON, 224 "||": TokenType.DPIPE, 225 "|>": TokenType.PIPE_GT, 226 ">=": TokenType.GTE, 227 "<=": TokenType.LTE, 228 "<>": TokenType.NEQ, 229 "!=": TokenType.NEQ, 230 ":=": TokenType.COLON_EQ, 231 "<=>": TokenType.NULLSAFE_EQ, 232 "->": TokenType.ARROW, 233 "->>": TokenType.DARROW, 234 "=>": TokenType.FARROW, 235 "#>": TokenType.HASH_ARROW, 236 "#>>": TokenType.DHASH_ARROW, 237 "<->": TokenType.LR_ARROW, 238 "<<->>": TokenType.LLRR_ARROW, 239 "&&": TokenType.DAMP, 240 "??": TokenType.DQMARK, 241 "~~~": TokenType.GLOB, 242 "~~": TokenType.LIKE, 243 "~~*": TokenType.ILIKE, 244 "~*": TokenType.IRLIKE, 245 "-|-": TokenType.ADJACENT, 246 "ALL": TokenType.ALL, 247 "AND": TokenType.AND, 248 "ANTI": TokenType.ANTI, 249 "ANY": TokenType.ANY, 250 "ASC": TokenType.ASC, 251 "AS": TokenType.ALIAS, 252 "ASOF": TokenType.ASOF, 253 "AUTOINCREMENT": TokenType.AUTO_INCREMENT, 254 "AUTO_INCREMENT": TokenType.AUTO_INCREMENT, 255 "BEGIN": TokenType.BEGIN, 256 "BETWEEN": TokenType.BETWEEN, 257 "CACHE": TokenType.CACHE, 258 "UNCACHE": TokenType.UNCACHE, 259 "CASE": TokenType.CASE, 260 "CLUSTER BY": TokenType.CLUSTER_BY, 261 "COLLATE": TokenType.COLLATE, 262 "COLUMN": TokenType.COLUMN, 263 "COMMIT": TokenType.COMMIT, 264 "CONNECT BY": TokenType.CONNECT_BY, 265 "CONSTRAINT": TokenType.CONSTRAINT, 266 "COPY": TokenType.COPY, 267 "CREATE": TokenType.CREATE, 268 "CROSS": TokenType.CROSS, 269 "CUBE": TokenType.CUBE, 270 "CURRENT_DATE": TokenType.CURRENT_DATE, 271 "CURRENT_SCHEMA": TokenType.CURRENT_SCHEMA, 272 "CURRENT_TIME": TokenType.CURRENT_TIME, 273 "CURRENT_TIMESTAMP": TokenType.CURRENT_TIMESTAMP, 274 "CURRENT_USER": TokenType.CURRENT_USER, 275 "CURRENT_CATALOG": TokenType.CURRENT_CATALOG, 276 "DATABASE": TokenType.DATABASE, 277 "DEFAULT": TokenType.DEFAULT, 278 "DELETE": TokenType.DELETE, 279 "DESC": TokenType.DESC, 280 "DESCRIBE": TokenType.DESCRIBE, 281 "DISTINCT": TokenType.DISTINCT, 282 "DISTRIBUTE BY": TokenType.DISTRIBUTE_BY, 283 "DIV": TokenType.DIV, 284 "DROP": TokenType.DROP, 285 "ELSE": TokenType.ELSE, 286 "END": TokenType.END, 287 "ENUM": TokenType.ENUM, 288 "ESCAPE": TokenType.ESCAPE, 289 "EXCEPT": TokenType.EXCEPT, 290 "EXECUTE": TokenType.EXECUTE, 291 "EXISTS": TokenType.EXISTS, 292 "FALSE": TokenType.FALSE, 293 "FETCH": TokenType.FETCH, 294 "FILTER": TokenType.FILTER, 295 "FILE": TokenType.FILE, 296 "FIRST": TokenType.FIRST, 297 "FULL": TokenType.FULL, 298 "FUNCTION": TokenType.FUNCTION, 299 "FOR": TokenType.FOR, 300 "FOREIGN KEY": TokenType.FOREIGN_KEY, 301 "FORMAT": TokenType.FORMAT, 302 "FROM": TokenType.FROM, 303 "GEOGRAPHY": TokenType.GEOGRAPHY, 304 "GEOMETRY": TokenType.GEOMETRY, 305 "GLOB": TokenType.GLOB, 306 "GROUP BY": TokenType.GROUP_BY, 307 "GROUPING SETS": TokenType.GROUPING_SETS, 308 "HAVING": TokenType.HAVING, 309 "ILIKE": TokenType.ILIKE, 310 "IN": TokenType.IN, 311 "INDEX": TokenType.INDEX, 312 "INET": TokenType.INET, 313 "INNER": TokenType.INNER, 314 "INSERT": TokenType.INSERT, 315 "INTERVAL": TokenType.INTERVAL, 316 "INTERSECT": TokenType.INTERSECT, 317 "INTO": TokenType.INTO, 318 "IS": TokenType.IS, 319 "ISNULL": TokenType.ISNULL, 320 "JOIN": TokenType.JOIN, 321 "KEEP": TokenType.KEEP, 322 "KILL": TokenType.KILL, 323 "LATERAL": TokenType.LATERAL, 324 "LEFT": TokenType.LEFT, 325 "LIKE": TokenType.LIKE, 326 "LIMIT": TokenType.LIMIT, 327 "LOAD": TokenType.LOAD, 328 "LOCALTIME": TokenType.LOCALTIME, 329 "LOCALTIMESTAMP": TokenType.LOCALTIMESTAMP, 330 "LOCK": TokenType.LOCK, 331 "MERGE": TokenType.MERGE, 332 "NAMESPACE": TokenType.NAMESPACE, 333 "NATURAL": TokenType.NATURAL, 334 "NEXT": TokenType.NEXT, 335 "NOT": TokenType.NOT, 336 "NOTNULL": TokenType.NOTNULL, 337 "NULL": TokenType.NULL, 338 "OBJECT": TokenType.OBJECT, 339 "OFFSET": TokenType.OFFSET, 340 "ON": TokenType.ON, 341 "OR": TokenType.OR, 342 "XOR": TokenType.XOR, 343 "ORDER BY": TokenType.ORDER_BY, 344 "ORDINALITY": TokenType.ORDINALITY, 345 "OUT": TokenType.OUT, 346 "OUTER": TokenType.OUTER, 347 "OVER": TokenType.OVER, 348 "OVERLAPS": TokenType.OVERLAPS, 349 "OVERWRITE": TokenType.OVERWRITE, 350 "PARTITION": TokenType.PARTITION, 351 "PARTITION BY": TokenType.PARTITION_BY, 352 "PARTITIONED BY": TokenType.PARTITION_BY, 353 "PARTITIONED_BY": TokenType.PARTITION_BY, 354 "PERCENT": TokenType.PERCENT, 355 "PIVOT": TokenType.PIVOT, 356 "PRAGMA": TokenType.PRAGMA, 357 "PRIMARY KEY": TokenType.PRIMARY_KEY, 358 "PROCEDURE": TokenType.PROCEDURE, 359 "OPERATOR": TokenType.OPERATOR, 360 "QUALIFY": TokenType.QUALIFY, 361 "RANGE": TokenType.RANGE, 362 "RECURSIVE": TokenType.RECURSIVE, 363 "REGEXP": TokenType.RLIKE, 364 "RENAME": TokenType.RENAME, 365 "REPLACE": TokenType.REPLACE, 366 "RETURNING": TokenType.RETURNING, 367 "REFERENCES": TokenType.REFERENCES, 368 "RIGHT": TokenType.RIGHT, 369 "RLIKE": TokenType.RLIKE, 370 "ROLLBACK": TokenType.ROLLBACK, 371 "ROLLUP": TokenType.ROLLUP, 372 "ROW": TokenType.ROW, 373 "ROWS": TokenType.ROWS, 374 "SCHEMA": TokenType.SCHEMA, 375 "SELECT": TokenType.SELECT, 376 "SEMI": TokenType.SEMI, 377 "SESSION": TokenType.SESSION, 378 "SESSION_USER": TokenType.SESSION_USER, 379 "SET": TokenType.SET, 380 "SETTINGS": TokenType.SETTINGS, 381 "SHOW": TokenType.SHOW, 382 "SIMILAR TO": TokenType.SIMILAR_TO, 383 "SOME": TokenType.SOME, 384 "SORT BY": TokenType.SORT_BY, 385 "SQL SECURITY": TokenType.SQL_SECURITY, 386 "STRAIGHT_JOIN": TokenType.STRAIGHT_JOIN, 387 "TABLE": TokenType.TABLE, 388 "TABLESAMPLE": TokenType.TABLE_SAMPLE, 389 "TEMP": TokenType.TEMPORARY, 390 "TEMPORARY": TokenType.TEMPORARY, 391 "THEN": TokenType.THEN, 392 "TRUE": TokenType.TRUE, 393 "TRUNCATE": TokenType.TRUNCATE, 394 "TRIGGER": TokenType.TRIGGER, 395 "UNION": TokenType.UNION, 396 "UNKNOWN": TokenType.UNKNOWN, 397 "UNNEST": TokenType.UNNEST, 398 "UNPIVOT": TokenType.UNPIVOT, 399 "UPDATE": TokenType.UPDATE, 400 "USE": TokenType.USE, 401 "USING": TokenType.USING, 402 "UUID": TokenType.UUID, 403 "VALUES": TokenType.VALUES, 404 "VIEW": TokenType.VIEW, 405 "VOLATILE": TokenType.VOLATILE, 406 "WHEN": TokenType.WHEN, 407 "WHERE": TokenType.WHERE, 408 "WINDOW": TokenType.WINDOW, 409 "WITH": TokenType.WITH, 410 "APPLY": TokenType.APPLY, 411 "ARRAY": TokenType.ARRAY, 412 "BIT": TokenType.BIT, 413 "BOOL": TokenType.BOOLEAN, 414 "BOOLEAN": TokenType.BOOLEAN, 415 "BYTE": TokenType.TINYINT, 416 "MEDIUMINT": TokenType.MEDIUMINT, 417 "INT1": TokenType.TINYINT, 418 "TINYINT": TokenType.TINYINT, 419 "INT16": TokenType.SMALLINT, 420 "SHORT": TokenType.SMALLINT, 421 "SMALLINT": TokenType.SMALLINT, 422 "HUGEINT": TokenType.INT128, 423 "UHUGEINT": TokenType.UINT128, 424 "INT2": TokenType.SMALLINT, 425 "INTEGER": TokenType.INT, 426 "INT": TokenType.INT, 427 "INT4": TokenType.INT, 428 "INT32": TokenType.INT, 429 "INT64": TokenType.BIGINT, 430 "INT128": TokenType.INT128, 431 "INT256": TokenType.INT256, 432 "LONG": TokenType.BIGINT, 433 "BIGINT": TokenType.BIGINT, 434 "INT8": TokenType.TINYINT, 435 "UINT": TokenType.UINT, 436 "UINT128": TokenType.UINT128, 437 "UINT256": TokenType.UINT256, 438 "DEC": TokenType.DECIMAL, 439 "DECIMAL": TokenType.DECIMAL, 440 "DECIMAL32": TokenType.DECIMAL32, 441 "DECIMAL64": TokenType.DECIMAL64, 442 "DECIMAL128": TokenType.DECIMAL128, 443 "DECIMAL256": TokenType.DECIMAL256, 444 "DECFLOAT": TokenType.DECFLOAT, 445 "BIGDECIMAL": TokenType.BIGDECIMAL, 446 "BIGNUMERIC": TokenType.BIGDECIMAL, 447 "BIGNUM": TokenType.BIGNUM, 448 "LIST": TokenType.LIST, 449 "MAP": TokenType.MAP, 450 "NULLABLE": TokenType.NULLABLE, 451 "NUMBER": TokenType.DECIMAL, 452 "NUMERIC": TokenType.DECIMAL, 453 "FIXED": TokenType.DECIMAL, 454 "REAL": TokenType.FLOAT, 455 "FLOAT": TokenType.FLOAT, 456 "FLOAT4": TokenType.FLOAT, 457 "FLOAT8": TokenType.DOUBLE, 458 "DOUBLE": TokenType.DOUBLE, 459 "DOUBLE PRECISION": TokenType.DOUBLE, 460 "JSON": TokenType.JSON, 461 "JSONB": TokenType.JSONB, 462 "CHAR": TokenType.CHAR, 463 "CHARACTER": TokenType.CHAR, 464 "CHAR VARYING": TokenType.VARCHAR, 465 "CHARACTER VARYING": TokenType.VARCHAR, 466 "NCHAR": TokenType.NCHAR, 467 "VARCHAR": TokenType.VARCHAR, 468 "VARCHAR2": TokenType.VARCHAR, 469 "NVARCHAR": TokenType.NVARCHAR, 470 "NVARCHAR2": TokenType.NVARCHAR, 471 "BPCHAR": TokenType.BPCHAR, 472 "STR": TokenType.TEXT, 473 "STRING": TokenType.TEXT, 474 "TEXT": TokenType.TEXT, 475 "LONGTEXT": TokenType.LONGTEXT, 476 "MEDIUMTEXT": TokenType.MEDIUMTEXT, 477 "TINYTEXT": TokenType.TINYTEXT, 478 "CLOB": TokenType.TEXT, 479 "LONGVARCHAR": TokenType.TEXT, 480 "BINARY": TokenType.BINARY, 481 "BLOB": TokenType.VARBINARY, 482 "LONGBLOB": TokenType.LONGBLOB, 483 "MEDIUMBLOB": TokenType.MEDIUMBLOB, 484 "TINYBLOB": TokenType.TINYBLOB, 485 "BYTEA": TokenType.VARBINARY, 486 "VARBINARY": TokenType.VARBINARY, 487 "TIME": TokenType.TIME, 488 "TIMETZ": TokenType.TIMETZ, 489 "TIME_NS": TokenType.TIME_NS, 490 "TIMESTAMP": TokenType.TIMESTAMP, 491 "TIMESTAMPTZ": TokenType.TIMESTAMPTZ, 492 "TIMESTAMPLTZ": TokenType.TIMESTAMPLTZ, 493 "TIMESTAMP_LTZ": TokenType.TIMESTAMPLTZ, 494 "TIMESTAMPNTZ": TokenType.TIMESTAMPNTZ, 495 "TIMESTAMP_NTZ": TokenType.TIMESTAMPNTZ, 496 "DATE": TokenType.DATE, 497 "DATETIME": TokenType.DATETIME, 498 "INT4RANGE": TokenType.INT4RANGE, 499 "INT4MULTIRANGE": TokenType.INT4MULTIRANGE, 500 "INT8RANGE": TokenType.INT8RANGE, 501 "INT8MULTIRANGE": TokenType.INT8MULTIRANGE, 502 "NUMRANGE": TokenType.NUMRANGE, 503 "NUMMULTIRANGE": TokenType.NUMMULTIRANGE, 504 "TSRANGE": TokenType.TSRANGE, 505 "TSMULTIRANGE": TokenType.TSMULTIRANGE, 506 "TSTZRANGE": TokenType.TSTZRANGE, 507 "TSTZMULTIRANGE": TokenType.TSTZMULTIRANGE, 508 "DATERANGE": TokenType.DATERANGE, 509 "DATEMULTIRANGE": TokenType.DATEMULTIRANGE, 510 "UNIQUE": TokenType.UNIQUE, 511 "VECTOR": TokenType.VECTOR, 512 "STRUCT": TokenType.STRUCT, 513 "SEQUENCE": TokenType.SEQUENCE, 514 "VARIANT": TokenType.VARIANT, 515 "ALTER": TokenType.ALTER, 516 "ANALYZE": TokenType.ANALYZE, 517 "CALL": TokenType.COMMAND, 518 "COMMENT": TokenType.COMMENT, 519 "EXPLAIN": TokenType.COMMAND, 520 "GRANT": TokenType.GRANT, 521 "REVOKE": TokenType.REVOKE, 522 "OPTIMIZE": TokenType.COMMAND, 523 "PREPARE": TokenType.COMMAND, 524 "VACUUM": TokenType.COMMAND, 525 "USER-DEFINED": TokenType.USERDEFINED, 526 } 527 528 COMMANDS = { 529 TokenType.COMMAND, 530 TokenType.EXECUTE, 531 TokenType.FETCH, 532 TokenType.SHOW, 533 TokenType.RENAME, 534 } 535 536 COMMAND_PREFIX_TOKENS = {TokenType.SEMICOLON, TokenType.BEGIN} 537 538 # Handle numeric literals like in hive (3L = BIGINT) 539 NUMERIC_LITERALS: t.ClassVar[dict[str, str]] = {} 540 541 # In tokenizers like JSONPath, dots are always key separators, never decimal points 542 NUMBERS_CAN_HAVE_DECIMALS: t.ClassVar[bool] = True 543 544 COMMENTS = ["--", ("/*", "*/")] 545 546 __slots__ = ( 547 "dialect", 548 "_core", 549 ) 550 551 def __init__(self, dialect: DialectType = None) -> None: 552 from sqlglot.dialects.dialect import Dialect 553 554 self.dialect = Dialect.get_or_raise(dialect) 555 self._core = self._init_core() 556 557 def _init_core(self) -> TokenizerCore: 558 return TokenizerCore( 559 single_tokens=self.SINGLE_TOKENS, 560 keywords=self.KEYWORDS, 561 quotes=self._QUOTES, 562 format_strings=self._FORMAT_STRINGS, 563 identifiers=self._IDENTIFIERS, 564 comments=self._COMMENTS, 565 string_escapes=self._STRING_ESCAPES, 566 byte_string_escapes=self._BYTE_STRING_ESCAPES, 567 identifier_escapes=self._IDENTIFIER_ESCAPES, 568 escape_follow_chars=self._ESCAPE_FOLLOW_CHARS, 569 commands=self.COMMANDS, 570 command_prefix_tokens=self.COMMAND_PREFIX_TOKENS, 571 nested_comments=self.NESTED_COMMENTS, 572 hint_start=self.HINT_START, 573 tokens_preceding_hint=self.TOKENS_PRECEDING_HINT, 574 has_bit_strings=bool(self.BIT_STRINGS), 575 has_hex_strings=bool(self.HEX_STRINGS), 576 numeric_literals=self.NUMERIC_LITERALS, 577 var_single_tokens=self.VAR_SINGLE_TOKENS, 578 string_escapes_allowed_in_raw_strings=self.STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS, 579 heredoc_tag_is_identifier=self.HEREDOC_TAG_IS_IDENTIFIER, 580 heredoc_string_alternative=self.HEREDOC_STRING_ALTERNATIVE, 581 keyword_trie=self._KEYWORD_TRIE, 582 numbers_can_be_underscore_separated=self.dialect.NUMBERS_CAN_BE_UNDERSCORE_SEPARATED, 583 numbers_can_have_decimals=self.NUMBERS_CAN_HAVE_DECIMALS, 584 identifiers_can_start_with_digit=self.dialect.IDENTIFIERS_CAN_START_WITH_DIGIT, 585 unescaped_sequences=self.dialect.UNESCAPED_SEQUENCES, 586 numeric_escapes=self.NUMERIC_ESCAPES, 587 drop_unknown_escapes=self.DROP_UNKNOWN_ESCAPES, 588 ) 589 590 def tokenize(self, sql: str) -> list[Token]: 591 """Returns a list of tokens corresponding to the SQL string `sql`.""" 592 return self._core.tokenize(sql) # type: ignore 593 594 @property 595 def sql(self) -> str: 596 """The SQL string being tokenized.""" 597 return self._core.sql 598 599 @property 600 def size(self) -> int: 601 """Length of the SQL string.""" 602 return self._core.size 603 604 @property 605 def tokens(self) -> list[Token]: 606 """The list of tokens produced by tokenization.""" 607 return self._core.tokens
Tokenizer( dialect: Union[str, sqlglot.dialects.Dialect, type[sqlglot.dialects.Dialect], NoneType] = None)
SINGLE_TOKENS =
{'(': <TokenType.L_PAREN: 1>, ')': <TokenType.R_PAREN: 2>, '[': <TokenType.L_BRACKET: 3>, ']': <TokenType.R_BRACKET: 4>, '{': <TokenType.L_BRACE: 5>, '}': <TokenType.R_BRACE: 6>, '&': <TokenType.AMP: 36>, '^': <TokenType.CARET: 42>, ':': <TokenType.COLON: 11>, ',': <TokenType.COMMA: 7>, '.': <TokenType.DOT: 8>, '-': <TokenType.DASH: 9>, '=': <TokenType.EQ: 28>, '>': <TokenType.GT: 25>, '<': <TokenType.LT: 23>, '%': <TokenType.MOD: 329>, '!': <TokenType.NOT: 27>, '|': <TokenType.PIPE: 39>, '+': <TokenType.PLUS: 10>, ';': <TokenType.SEMICOLON: 19>, '/': <TokenType.SLASH: 22>, '\\': <TokenType.BACKSLASH: 21>, '*': <TokenType.STAR: 20>, '~': <TokenType.TILDE: 44>, '?': <TokenType.PLACEHOLDER: 356>, '@': <TokenType.PARAMETER: 58>, '#': <TokenType.HASH: 48>, "'": <TokenType.UNKNOWN: 214>, '`': <TokenType.UNKNOWN: 214>, '"': <TokenType.UNKNOWN: 214>}
TOKENS_PRECEDING_HINT =
{<TokenType.DELETE: 257>, <TokenType.SELECT: 387>, <TokenType.INSERT: 300>, <TokenType.UPDATE: 419>}
KEYWORDS: ClassVar[dict[str, sqlglot.tokenizer_core.TokenType]] =
{'{%': <TokenType.BLOCK_START: 73>, '{%+': <TokenType.BLOCK_START: 73>, '{%-': <TokenType.BLOCK_START: 73>, '%}': <TokenType.BLOCK_END: 74>, '+%}': <TokenType.BLOCK_END: 74>, '-%}': <TokenType.BLOCK_END: 74>, '{{+': <TokenType.BLOCK_START: 73>, '{{-': <TokenType.BLOCK_START: 73>, '+}}': <TokenType.BLOCK_END: 74>, '-}}': <TokenType.BLOCK_END: 74>, '/*+': <TokenType.HINT: 293>, '&<': <TokenType.AMP_LT: 63>, '&>': <TokenType.AMP_GT: 64>, '==': <TokenType.EQ: 28>, '::': <TokenType.DCOLON: 14>, '?::': <TokenType.QDCOLON: 370>, '||': <TokenType.DPIPE: 37>, '|>': <TokenType.PIPE_GT: 38>, '>=': <TokenType.GTE: 26>, '<=': <TokenType.LTE: 24>, '<>': <TokenType.NEQ: 29>, '!=': <TokenType.NEQ: 29>, ':=': <TokenType.COLON_EQ: 31>, '<=>': <TokenType.NULLSAFE_EQ: 30>, '->': <TokenType.ARROW: 45>, '->>': <TokenType.DARROW: 46>, '=>': <TokenType.FARROW: 47>, '#>': <TokenType.HASH_ARROW: 49>, '#>>': <TokenType.DHASH_ARROW: 50>, '<->': <TokenType.LR_ARROW: 51>, '<<->>': <TokenType.LLRR_ARROW: 52>, '&&': <TokenType.DAMP: 62>, '??': <TokenType.DQMARK: 18>, '~~~': <TokenType.GLOB: 287>, '~~': <TokenType.LIKE: 318>, '~~*': <TokenType.ILIKE: 295>, '~*': <TokenType.IRLIKE: 307>, '-|-': <TokenType.ADJACENT: 65>, 'ALL': <TokenType.ALL: 220>, 'AND': <TokenType.AND: 34>, 'ANTI': <TokenType.ANTI: 221>, 'ANY': <TokenType.ANY: 222>, 'ASC': <TokenType.ASC: 225>, 'AS': <TokenType.ALIAS: 218>, 'ASOF': <TokenType.ASOF: 226>, 'AUTOINCREMENT': <TokenType.AUTO_INCREMENT: 228>, 'AUTO_INCREMENT': <TokenType.AUTO_INCREMENT: 228>, 'BEGIN': <TokenType.BEGIN: 229>, 'BETWEEN': <TokenType.BETWEEN: 230>, 'CACHE': <TokenType.CACHE: 232>, 'UNCACHE': <TokenType.UNCACHE: 414>, 'CASE': <TokenType.CASE: 233>, 'CLUSTER BY': <TokenType.CLUSTER_BY: 235>, 'COLLATE': <TokenType.COLLATE: 236>, 'COLUMN': <TokenType.COLUMN: 81>, 'COMMIT': <TokenType.COMMIT: 239>, 'CONNECT BY': <TokenType.CONNECT_BY: 240>, 'CONSTRAINT': <TokenType.CONSTRAINT: 241>, 'COPY': <TokenType.COPY: 242>, 'CREATE': <TokenType.CREATE: 243>, 'CROSS': <TokenType.CROSS: 244>, 'CUBE': <TokenType.CUBE: 245>, 'CURRENT_DATE': <TokenType.CURRENT_DATE: 246>, 'CURRENT_SCHEMA': <TokenType.CURRENT_SCHEMA: 248>, 'CURRENT_TIME': <TokenType.CURRENT_TIME: 249>, 'CURRENT_TIMESTAMP': <TokenType.CURRENT_TIMESTAMP: 250>, 'CURRENT_USER': <TokenType.CURRENT_USER: 251>, 'CURRENT_CATALOG': <TokenType.CURRENT_CATALOG: 254>, 'DATABASE': <TokenType.DATABASE: 80>, 'DEFAULT': <TokenType.DEFAULT: 256>, 'DELETE': <TokenType.DELETE: 257>, 'DESC': <TokenType.DESC: 258>, 'DESCRIBE': <TokenType.DESCRIBE: 259>, 'DISTINCT': <TokenType.DISTINCT: 262>, 'DISTRIBUTE BY': <TokenType.DISTRIBUTE_BY: 263>, 'DIV': <TokenType.DIV: 264>, 'DROP': <TokenType.DROP: 265>, 'ELSE': <TokenType.ELSE: 266>, 'END': <TokenType.END: 267>, 'ENUM': <TokenType.ENUM: 205>, 'ESCAPE': <TokenType.ESCAPE: 268>, 'EXCEPT': <TokenType.EXCEPT: 269>, 'EXECUTE': <TokenType.EXECUTE: 270>, 'EXISTS': <TokenType.EXISTS: 271>, 'FALSE': <TokenType.FALSE: 272>, 'FETCH': <TokenType.FETCH: 273>, 'FILTER': <TokenType.FILTER: 276>, 'FILE': <TokenType.FILE: 274>, 'FIRST': <TokenType.FIRST: 278>, 'FULL': <TokenType.FULL: 284>, 'FUNCTION': <TokenType.FUNCTION: 285>, 'FOR': <TokenType.FOR: 279>, 'FOREIGN KEY': <TokenType.FOREIGN_KEY: 281>, 'FORMAT': <TokenType.FORMAT: 282>, 'FROM': <TokenType.FROM: 283>, 'GEOGRAPHY': <TokenType.GEOGRAPHY: 172>, 'GEOMETRY': <TokenType.GEOMETRY: 175>, 'GLOB': <TokenType.GLOB: 287>, 'GROUP BY': <TokenType.GROUP_BY: 290>, 'GROUPING SETS': <TokenType.GROUPING_SETS: 291>, 'HAVING': <TokenType.HAVING: 292>, 'ILIKE': <TokenType.ILIKE: 295>, 'IN': <TokenType.IN: 296>, 'INDEX': <TokenType.INDEX: 297>, 'INET': <TokenType.INET: 200>, 'INNER': <TokenType.INNER: 299>, 'INSERT': <TokenType.INSERT: 300>, 'INTERVAL': <TokenType.INTERVAL: 304>, 'INTERSECT': <TokenType.INTERSECT: 303>, 'INTO': <TokenType.INTO: 305>, 'IS': <TokenType.IS: 308>, 'ISNULL': <TokenType.ISNULL: 309>, 'JOIN': <TokenType.JOIN: 310>, 'KEEP': <TokenType.KEEP: 312>, 'KILL': <TokenType.KILL: 314>, 'LATERAL': <TokenType.LATERAL: 316>, 'LEFT': <TokenType.LEFT: 317>, 'LIKE': <TokenType.LIKE: 318>, 'LIMIT': <TokenType.LIMIT: 319>, 'LOAD': <TokenType.LOAD: 321>, 'LOCALTIME': <TokenType.LOCALTIME: 179>, 'LOCALTIMESTAMP': <TokenType.LOCALTIMESTAMP: 180>, 'LOCK': <TokenType.LOCK: 322>, 'MERGE': <TokenType.MERGE: 328>, 'NAMESPACE': <TokenType.NAMESPACE: 440>, 'NATURAL': <TokenType.NATURAL: 331>, 'NEXT': <TokenType.NEXT: 332>, 'NOT': <TokenType.NOT: 27>, 'NOTNULL': <TokenType.NOTNULL: 334>, 'NULL': <TokenType.NULL: 335>, 'OBJECT': <TokenType.OBJECT: 199>, 'OFFSET': <TokenType.OFFSET: 337>, 'ON': <TokenType.ON: 338>, 'OR': <TokenType.OR: 35>, 'XOR': <TokenType.XOR: 66>, 'ORDER BY': <TokenType.ORDER_BY: 341>, 'ORDINALITY': <TokenType.ORDINALITY: 344>, 'OUT': <TokenType.OUT: 345>, 'OUTER': <TokenType.OUTER: 347>, 'OVER': <TokenType.OVER: 348>, 'OVERLAPS': <TokenType.OVERLAPS: 349>, 'OVERWRITE': <TokenType.OVERWRITE: 350>, 'PARTITION': <TokenType.PARTITION: 352>, 'PARTITION BY': <TokenType.PARTITION_BY: 353>, 'PARTITIONED BY': <TokenType.PARTITION_BY: 353>, 'PARTITIONED_BY': <TokenType.PARTITION_BY: 353>, 'PERCENT': <TokenType.PERCENT: 354>, 'PIVOT': <TokenType.PIVOT: 355>, 'PRAGMA': <TokenType.PRAGMA: 360>, 'PRIMARY KEY': <TokenType.PRIMARY_KEY: 362>, 'PROCEDURE': <TokenType.PROCEDURE: 363>, 'OPERATOR': <TokenType.OPERATOR: 340>, 'QUALIFY': <TokenType.QUALIFY: 368>, 'RANGE': <TokenType.RANGE: 371>, 'RECURSIVE': <TokenType.RECURSIVE: 372>, 'REGEXP': <TokenType.RLIKE: 380>, 'RENAME': <TokenType.RENAME: 374>, 'REPLACE': <TokenType.REPLACE: 375>, 'RETURNING': <TokenType.RETURNING: 376>, 'REFERENCES': <TokenType.REFERENCES: 378>, 'RIGHT': <TokenType.RIGHT: 379>, 'RLIKE': <TokenType.RLIKE: 380>, 'ROLLBACK': <TokenType.ROLLBACK: 382>, 'ROLLUP': <TokenType.ROLLUP: 383>, 'ROW': <TokenType.ROW: 384>, 'ROWS': <TokenType.ROWS: 385>, 'SCHEMA': <TokenType.SCHEMA: 83>, 'SELECT': <TokenType.SELECT: 387>, 'SEMI': <TokenType.SEMI: 388>, 'SESSION': <TokenType.SESSION: 59>, 'SESSION_USER': <TokenType.SESSION_USER: 61>, 'SET': <TokenType.SET: 392>, 'SETTINGS': <TokenType.SETTINGS: 393>, 'SHOW': <TokenType.SHOW: 394>, 'SIMILAR TO': <TokenType.SIMILAR_TO: 395>, 'SOME': <TokenType.SOME: 396>, 'SORT BY': <TokenType.SORT_BY: 397>, 'SQL SECURITY': <TokenType.SQL_SECURITY: 399>, 'STRAIGHT_JOIN': <TokenType.STRAIGHT_JOIN: 402>, 'TABLE': <TokenType.TABLE: 84>, 'TABLESAMPLE': <TokenType.TABLE_SAMPLE: 405>, 'TEMP': <TokenType.TEMPORARY: 407>, 'TEMPORARY': <TokenType.TEMPORARY: 407>, 'THEN': <TokenType.THEN: 409>, 'TRUE': <TokenType.TRUE: 410>, 'TRUNCATE': <TokenType.TRUNCATE: 411>, 'TRIGGER': <TokenType.TRIGGER: 412>, 'UNION': <TokenType.UNION: 416>, 'UNKNOWN': <TokenType.UNKNOWN: 214>, 'UNNEST': <TokenType.UNNEST: 417>, 'UNPIVOT': <TokenType.UNPIVOT: 418>, 'UPDATE': <TokenType.UPDATE: 419>, 'USE': <TokenType.USE: 420>, 'USING': <TokenType.USING: 421>, 'UUID': <TokenType.UUID: 171>, 'VALUES': <TokenType.VALUES: 422>, 'VIEW': <TokenType.VIEW: 424>, 'VOLATILE': <TokenType.VOLATILE: 426>, 'WHEN': <TokenType.WHEN: 428>, 'WHERE': <TokenType.WHERE: 429>, 'WINDOW': <TokenType.WINDOW: 430>, 'WITH': <TokenType.WITH: 431>, 'APPLY': <TokenType.APPLY: 223>, 'ARRAY': <TokenType.ARRAY: 224>, 'BIT': <TokenType.BIT: 97>, 'BOOL': <TokenType.BOOLEAN: 98>, 'BOOLEAN': <TokenType.BOOLEAN: 98>, 'BYTE': <TokenType.TINYINT: 99>, 'MEDIUMINT': <TokenType.MEDIUMINT: 103>, 'INT1': <TokenType.TINYINT: 99>, 'TINYINT': <TokenType.TINYINT: 99>, 'INT16': <TokenType.SMALLINT: 101>, 'SHORT': <TokenType.SMALLINT: 101>, 'SMALLINT': <TokenType.SMALLINT: 101>, 'HUGEINT': <TokenType.INT128: 110>, 'UHUGEINT': <TokenType.UINT128: 111>, 'INT2': <TokenType.SMALLINT: 101>, 'INTEGER': <TokenType.INT: 105>, 'INT': <TokenType.INT: 105>, 'INT4': <TokenType.INT: 105>, 'INT32': <TokenType.INT: 105>, 'INT64': <TokenType.BIGINT: 107>, 'INT128': <TokenType.INT128: 110>, 'INT256': <TokenType.INT256: 112>, 'LONG': <TokenType.BIGINT: 107>, 'BIGINT': <TokenType.BIGINT: 107>, 'INT8': <TokenType.TINYINT: 99>, 'UINT': <TokenType.UINT: 106>, 'UINT128': <TokenType.UINT128: 111>, 'UINT256': <TokenType.UINT256: 113>, 'DEC': <TokenType.DECIMAL: 117>, 'DECIMAL': <TokenType.DECIMAL: 117>, 'DECIMAL32': <TokenType.DECIMAL32: 118>, 'DECIMAL64': <TokenType.DECIMAL64: 119>, 'DECIMAL128': <TokenType.DECIMAL128: 120>, 'DECIMAL256': <TokenType.DECIMAL256: 121>, 'DECFLOAT': <TokenType.DECFLOAT: 122>, 'BIGDECIMAL': <TokenType.BIGDECIMAL: 124>, 'BIGNUMERIC': <TokenType.BIGDECIMAL: 124>, 'BIGNUM': <TokenType.BIGNUM: 109>, 'LIST': <TokenType.LIST: 320>, 'MAP': <TokenType.MAP: 323>, 'NULLABLE': <TokenType.NULLABLE: 174>, 'NUMBER': <TokenType.DECIMAL: 117>, 'NUMERIC': <TokenType.DECIMAL: 117>, 'FIXED': <TokenType.DECIMAL: 117>, 'REAL': <TokenType.FLOAT: 114>, 'FLOAT': <TokenType.FLOAT: 114>, 'FLOAT4': <TokenType.FLOAT: 114>, 'FLOAT8': <TokenType.DOUBLE: 115>, 'DOUBLE': <TokenType.DOUBLE: 115>, 'DOUBLE PRECISION': <TokenType.DOUBLE: 115>, 'JSON': <TokenType.JSON: 141>, 'JSONB': <TokenType.JSONB: 142>, 'CHAR': <TokenType.CHAR: 125>, 'CHARACTER': <TokenType.CHAR: 125>, 'CHAR VARYING': <TokenType.VARCHAR: 127>, 'CHARACTER VARYING': <TokenType.VARCHAR: 127>, 'NCHAR': <TokenType.NCHAR: 126>, 'VARCHAR': <TokenType.VARCHAR: 127>, 'VARCHAR2': <TokenType.VARCHAR: 127>, 'NVARCHAR': <TokenType.NVARCHAR: 128>, 'NVARCHAR2': <TokenType.NVARCHAR: 128>, 'BPCHAR': <TokenType.BPCHAR: 129>, 'STR': <TokenType.TEXT: 130>, 'STRING': <TokenType.TEXT: 130>, 'TEXT': <TokenType.TEXT: 130>, 'LONGTEXT': <TokenType.LONGTEXT: 132>, 'MEDIUMTEXT': <TokenType.MEDIUMTEXT: 131>, 'TINYTEXT': <TokenType.TINYTEXT: 137>, 'CLOB': <TokenType.TEXT: 130>, 'LONGVARCHAR': <TokenType.TEXT: 130>, 'BINARY': <TokenType.BINARY: 139>, 'BLOB': <TokenType.VARBINARY: 140>, 'LONGBLOB': <TokenType.LONGBLOB: 135>, 'MEDIUMBLOB': <TokenType.MEDIUMBLOB: 134>, 'TINYBLOB': <TokenType.TINYBLOB: 136>, 'BYTEA': <TokenType.VARBINARY: 140>, 'VARBINARY': <TokenType.VARBINARY: 140>, 'TIME': <TokenType.TIME: 143>, 'TIMETZ': <TokenType.TIMETZ: 144>, 'TIME_NS': <TokenType.TIME_NS: 145>, 'TIMESTAMP': <TokenType.TIMESTAMP: 146>, 'TIMESTAMPTZ': <TokenType.TIMESTAMPTZ: 147>, 'TIMESTAMPLTZ': <TokenType.TIMESTAMPLTZ: 148>, 'TIMESTAMP_LTZ': <TokenType.TIMESTAMPLTZ: 148>, 'TIMESTAMPNTZ': <TokenType.TIMESTAMPNTZ: 149>, 'TIMESTAMP_NTZ': <TokenType.TIMESTAMPNTZ: 149>, 'DATE': <TokenType.DATE: 157>, 'DATETIME': <TokenType.DATETIME: 153>, 'INT4RANGE': <TokenType.INT4RANGE: 159>, 'INT4MULTIRANGE': <TokenType.INT4MULTIRANGE: 160>, 'INT8RANGE': <TokenType.INT8RANGE: 161>, 'INT8MULTIRANGE': <TokenType.INT8MULTIRANGE: 162>, 'NUMRANGE': <TokenType.NUMRANGE: 163>, 'NUMMULTIRANGE': <TokenType.NUMMULTIRANGE: 164>, 'TSRANGE': <TokenType.TSRANGE: 165>, 'TSMULTIRANGE': <TokenType.TSMULTIRANGE: 166>, 'TSTZRANGE': <TokenType.TSTZRANGE: 167>, 'TSTZMULTIRANGE': <TokenType.TSTZMULTIRANGE: 168>, 'DATERANGE': <TokenType.DATERANGE: 169>, 'DATEMULTIRANGE': <TokenType.DATEMULTIRANGE: 170>, 'UNIQUE': <TokenType.UNIQUE: 432>, 'VECTOR': <TokenType.VECTOR: 215>, 'STRUCT': <TokenType.STRUCT: 403>, 'SEQUENCE': <TokenType.SEQUENCE: 390>, 'VARIANT': <TokenType.VARIANT: 198>, 'ALTER': <TokenType.ALTER: 219>, 'ANALYZE': <TokenType.ANALYZE: 439>, 'CALL': <TokenType.COMMAND: 237>, 'COMMENT': <TokenType.COMMENT: 238>, 'EXPLAIN': <TokenType.COMMAND: 237>, 'GRANT': <TokenType.GRANT: 289>, 'REVOKE': <TokenType.REVOKE: 377>, 'OPTIMIZE': <TokenType.COMMAND: 237>, 'PREPARE': <TokenType.COMMAND: 237>, 'VACUUM': <TokenType.COMMAND: 237>, 'USER-DEFINED': <TokenType.USERDEFINED: 193>}
COMMANDS =
{<TokenType.SHOW: 394>, <TokenType.COMMAND: 237>, <TokenType.EXECUTE: 270>, <TokenType.FETCH: 273>, <TokenType.RENAME: 374>}
590 def tokenize(self, sql: str) -> list[Token]: 591 """Returns a list of tokens corresponding to the SQL string `sql`.""" 592 return self._core.tokenize(sql) # type: ignore
Returns a list of tokens corresponding to the SQL string sql.
sql: str
594 @property 595 def sql(self) -> str: 596 """The SQL string being tokenized.""" 597 return self._core.sql
The SQL string being tokenized.
size: int
599 @property 600 def size(self) -> int: 601 """Length of the SQL string.""" 602 return self._core.size
Length of the SQL string.
tokens: list[sqlglot.tokenizer_core.Token]
604 @property 605 def tokens(self) -> list[Token]: 606 """The list of tokens produced by tokenization.""" 607 return self._core.tokens
The list of tokens produced by tokenization.