Verzeichnisstruktur phpBB-3.3.15
- Veröffentlicht
- 28.08.2024
So funktioniert es
|
|
Auf das letzte Element klicken. Dies geht jeweils ein Schritt zurück |
Auf das Icon klicken, dies öffnet das Verzeichnis. Nochmal klicken schließt das Verzeichnis. |
|
|
(Beispiel Datei-Icons)
|
Auf das Icon klicken um den Quellcode anzuzeigen |
Lexer.php
001 <?php
002
003 /*
004 * This file is part of Twig.
005 *
006 * (c) Fabien Potencier
007 * (c) Armin Ronacher
008 *
009 * For the full copyright and license information, please view the LICENSE
010 * file that was distributed with this source code.
011 */
012
013 namespace Twig;
014
015 use Twig\Error\SyntaxError;
016
017 /**
018 * Lexes a template string.
019 *
020 * @author Fabien Potencier <fabien@symfony.com>
021 */
022 class Lexer
023 {
024 private $isInitialized = false;
025
026 private $tokens;
027 private $code;
028 private $cursor;
029 private $lineno;
030 private $end;
031 private $state;
032 private $states;
033 private $brackets;
034 private $env;
035 private $source;
036 private $options;
037 private $regexes;
038 private $position;
039 private $positions;
040 private $currentVarBlockLine;
041
042 public const STATE_DATA = 0;
043 public const STATE_BLOCK = 1;
044 public const STATE_VAR = 2;
045 public const STATE_STRING = 3;
046 public const STATE_INTERPOLATION = 4;
047
048 public const REGEX_NAME = '/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/A';
049 public const REGEX_NUMBER = '/[0-9]+(?:\.[0-9]+)?([Ee][\+\-][0-9]+)?/A';
050 public const REGEX_STRING = '/"([^#"\\\\]*(?:\\\\.[^#"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As';
051 public const REGEX_DQ_STRING_DELIM = '/"/A';
052 public const REGEX_DQ_STRING_PART = '/[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*/As';
053 public const PUNCTUATION = '()[]{}?:.,|';
054
055 public function __construct(Environment $env, array $options = [])
056 {
057 $this->env = $env;
058
059 $this->options = array_merge([
060 'tag_comment' => ['{#', '#}'],
061 'tag_block' => ['{%', '%}'],
062 'tag_variable' => ['{{', '}}'],
063 'whitespace_trim' => '-',
064 'whitespace_line_trim' => '~',
065 'whitespace_line_chars' => ' \t\0\x0B',
066 'interpolation' => ['#{', '}'],
067 ], $options);
068 }
069
070 private function initialize()
071 {
072 if ($this->isInitialized) {
073 return;
074 }
075
076 $this->isInitialized = true;
077
078 // when PHP 7.3 is the min version, we will be able to remove the '#' part in preg_quote as it's part of the default
079 $this->regexes = [
080 // }}
081 'lex_var' => '{
082 \s*
083 (?:'.
084 preg_quote($this->options['whitespace_trim'].$this->options['tag_variable'][1], '#').'\s*'. // -}}\s*
085 '|'.
086 preg_quote($this->options['whitespace_line_trim'].$this->options['tag_variable'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~}}[ \t\0\x0B]*
087 '|'.
088 preg_quote($this->options['tag_variable'][1], '#'). // }}
089 ')
090 }Ax',
091
092 // %}
093 'lex_block' => '{
094 \s*
095 (?:'.
096 preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*\n?'. // -%}\s*\n?
097 '|'.
098 preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
099 '|'.
100 preg_quote($this->options['tag_block'][1], '#').'\n?'. // %}\n?
101 ')
102 }Ax',
103
104 // {% endverbatim %}
105 'lex_raw_data' => '{'.
106 preg_quote($this->options['tag_block'][0], '#'). // {%
107 '('.
108 $this->options['whitespace_trim']. // -
109 '|'.
110 $this->options['whitespace_line_trim']. // ~
111 ')?\s*endverbatim\s*'.
112 '(?:'.
113 preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*'. // -%}
114 '|'.
115 preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
116 '|'.
117 preg_quote($this->options['tag_block'][1], '#'). // %}
118 ')
119 }sx',
120
121 'operator' => $this->getOperatorRegex(),
122
123 // #}
124 'lex_comment' => '{
125 (?:'.
126 preg_quote($this->options['whitespace_trim'].$this->options['tag_comment'][1], '#').'\s*\n?'. // -#}\s*\n?
127 '|'.
128 preg_quote($this->options['whitespace_line_trim'].$this->options['tag_comment'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~#}[ \t\0\x0B]*
129 '|'.
130 preg_quote($this->options['tag_comment'][1], '#').'\n?'. // #}\n?
131 ')
132 }sx',
133
134 // verbatim %}
135 'lex_block_raw' => '{
136 \s*verbatim\s*
137 (?:'.
138 preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*'. // -%}\s*
139 '|'.
140 preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
141 '|'.
142 preg_quote($this->options['tag_block'][1], '#'). // %}
143 ')
144 }Asx',
145
146 'lex_block_line' => '{\s*line\s+(\d+)\s*'.preg_quote($this->options['tag_block'][1], '#').'}As',
147
148 // {{ or {% or {#
149 'lex_tokens_start' => '{
150 ('.
151 preg_quote($this->options['tag_variable'][0], '#'). // {{
152 '|'.
153 preg_quote($this->options['tag_block'][0], '#'). // {%
154 '|'.
155 preg_quote($this->options['tag_comment'][0], '#'). // {#
156 ')('.
157 preg_quote($this->options['whitespace_trim'], '#'). // -
158 '|'.
159 preg_quote($this->options['whitespace_line_trim'], '#'). // ~
160 ')?
161 }sx',
162 'interpolation_start' => '{'.preg_quote($this->options['interpolation'][0], '#').'\s*}A',
163 'interpolation_end' => '{\s*'.preg_quote($this->options['interpolation'][1], '#').'}A',
164 ];
165 }
166
167 public function tokenize(Source $source)
168 {
169 $this->initialize();
170
171 $this->source = $source;
172 $this->code = str_replace(["\r\n", "\r"], "\n", $source->getCode());
173 $this->cursor = 0;
174 $this->lineno = 1;
175 $this->end = \strlen($this->code);
176 $this->tokens = [];
177 $this->state = self::STATE_DATA;
178 $this->states = [];
179 $this->brackets = [];
180 $this->position = -1;
181
182 // find all token starts in one go
183 preg_match_all($this->regexes['lex_tokens_start'], $this->code, $matches, \PREG_OFFSET_CAPTURE);
184 $this->positions = $matches;
185
186 while ($this->cursor < $this->end) {
187 // dispatch to the lexing functions depending
188 // on the current state
189 switch ($this->state) {
190 case self::STATE_DATA:
191 $this->lexData();
192 break;
193
194 case self::STATE_BLOCK:
195 $this->lexBlock();
196 break;
197
198 case self::STATE_VAR:
199 $this->lexVar();
200 break;
201
202 case self::STATE_STRING:
203 $this->lexString();
204 break;
205
206 case self::STATE_INTERPOLATION:
207 $this->lexInterpolation();
208 break;
209 }
210 }
211
212 $this->pushToken(/* Token::EOF_TYPE */ -1);
213
214 if (!empty($this->brackets)) {
215 list($expect, $lineno) = array_pop($this->brackets);
216 throw new SyntaxError(sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
217 }
218
219 return new TokenStream($this->tokens, $this->source);
220 }
221
222 private function lexData()
223 {
224 // if no matches are left we return the rest of the template as simple text token
225 if ($this->position == \count($this->positions[0]) - 1) {
226 $this->pushToken(/* Token::TEXT_TYPE */ 0, substr($this->code, $this->cursor));
227 $this->cursor = $this->end;
228
229 return;
230 }
231
232 // Find the first token after the current cursor
233 $position = $this->positions[0][++$this->position];
234 while ($position[1] < $this->cursor) {
235 if ($this->position == \count($this->positions[0]) - 1) {
236 return;
237 }
238 $position = $this->positions[0][++$this->position];
239 }
240
241 // push the template text first
242 $text = $textContent = substr($this->code, $this->cursor, $position[1] - $this->cursor);
243
244 // trim?
245 if (isset($this->positions[2][$this->position][0])) {
246 if ($this->options['whitespace_trim'] === $this->positions[2][$this->position][0]) {
247 // whitespace_trim detected ({%-, {{- or {#-)
248 $text = rtrim($text);
249 } elseif ($this->options['whitespace_line_trim'] === $this->positions[2][$this->position][0]) {
250 // whitespace_line_trim detected ({%~, {{~ or {#~)
251 // don't trim \r and \n
252 $text = rtrim($text, " \t\0\x0B");
253 }
254 }
255 $this->pushToken(/* Token::TEXT_TYPE */ 0, $text);
256 $this->moveCursor($textContent.$position[0]);
257
258 switch ($this->positions[1][$this->position][0]) {
259 case $this->options['tag_comment'][0]:
260 $this->lexComment();
261 break;
262
263 case $this->options['tag_block'][0]:
264 // raw data?
265 if (preg_match($this->regexes['lex_block_raw'], $this->code, $match, 0, $this->cursor)) {
266 $this->moveCursor($match[0]);
267 $this->lexRawData();
268 // {% line \d+ %}
269 } elseif (preg_match($this->regexes['lex_block_line'], $this->code, $match, 0, $this->cursor)) {
270 $this->moveCursor($match[0]);
271 $this->lineno = (int) $match[1];
272 } else {
273 $this->pushToken(/* Token::BLOCK_START_TYPE */ 1);
274 $this->pushState(self::STATE_BLOCK);
275 $this->currentVarBlockLine = $this->lineno;
276 }
277 break;
278
279 case $this->options['tag_variable'][0]:
280 $this->pushToken(/* Token::VAR_START_TYPE */ 2);
281 $this->pushState(self::STATE_VAR);
282 $this->currentVarBlockLine = $this->lineno;
283 break;
284 }
285 }
286
287 private function lexBlock()
288 {
289 if (empty($this->brackets) && preg_match($this->regexes['lex_block'], $this->code, $match, 0, $this->cursor)) {
290 $this->pushToken(/* Token::BLOCK_END_TYPE */ 3);
291 $this->moveCursor($match[0]);
292 $this->popState();
293 } else {
294 $this->lexExpression();
295 }
296 }
297
298 private function lexVar()
299 {
300 if (empty($this->brackets) && preg_match($this->regexes['lex_var'], $this->code, $match, 0, $this->cursor)) {
301 $this->pushToken(/* Token::VAR_END_TYPE */ 4);
302 $this->moveCursor($match[0]);
303 $this->popState();
304 } else {
305 $this->lexExpression();
306 }
307 }
308
309 private function lexExpression()
310 {
311 // whitespace
312 if (preg_match('/\s+/A', $this->code, $match, 0, $this->cursor)) {
313 $this->moveCursor($match[0]);
314
315 if ($this->cursor >= $this->end) {
316 throw new SyntaxError(sprintf('Unclosed "%s".', self::STATE_BLOCK === $this->state ? 'block' : 'variable'), $this->currentVarBlockLine, $this->source);
317 }
318 }
319
320 // arrow function
321 if ('=' === $this->code[$this->cursor] && '>' === $this->code[$this->cursor + 1]) {
322 $this->pushToken(Token::ARROW_TYPE, '=>');
323 $this->moveCursor('=>');
324 }
325 // operators
326 elseif (preg_match($this->regexes['operator'], $this->code, $match, 0, $this->cursor)) {
327 $this->pushToken(/* Token::OPERATOR_TYPE */ 8, preg_replace('/\s+/', ' ', $match[0]));
328 $this->moveCursor($match[0]);
329 }
330 // names
331 elseif (preg_match(self::REGEX_NAME, $this->code, $match, 0, $this->cursor)) {
332 $this->pushToken(/* Token::NAME_TYPE */ 5, $match[0]);
333 $this->moveCursor($match[0]);
334 }
335 // numbers
336 elseif (preg_match(self::REGEX_NUMBER, $this->code, $match, 0, $this->cursor)) {
337 $number = (float) $match[0]; // floats
338 if (ctype_digit($match[0]) && $number <= \PHP_INT_MAX) {
339 $number = (int) $match[0]; // integers lower than the maximum
340 }
341 $this->pushToken(/* Token::NUMBER_TYPE */ 6, $number);
342 $this->moveCursor($match[0]);
343 }
344 // punctuation
345 elseif (false !== strpos(self::PUNCTUATION, $this->code[$this->cursor])) {
346 // opening bracket
347 if (false !== strpos('([{', $this->code[$this->cursor])) {
348 $this->brackets[] = [$this->code[$this->cursor], $this->lineno];
349 }
350 // closing bracket
351 elseif (false !== strpos(')]}', $this->code[$this->cursor])) {
352 if (empty($this->brackets)) {
353 throw new SyntaxError(sprintf('Unexpected "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
354 }
355
356 list($expect, $lineno) = array_pop($this->brackets);
357 if ($this->code[$this->cursor] != strtr($expect, '([{', ')]}')) {
358 throw new SyntaxError(sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
359 }
360 }
361
362 $this->pushToken(/* Token::PUNCTUATION_TYPE */ 9, $this->code[$this->cursor]);
363 ++$this->cursor;
364 }
365 // strings
366 elseif (preg_match(self::REGEX_STRING, $this->code, $match, 0, $this->cursor)) {
367 $this->pushToken(/* Token::STRING_TYPE */ 7, stripcslashes(substr($match[0], 1, -1)));
368 $this->moveCursor($match[0]);
369 }
370 // opening double quoted string
371 elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
372 $this->brackets[] = ['"', $this->lineno];
373 $this->pushState(self::STATE_STRING);
374 $this->moveCursor($match[0]);
375 }
376 // unlexable
377 else {
378 throw new SyntaxError(sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
379 }
380 }
381
382 private function lexRawData()
383 {
384 if (!preg_match($this->regexes['lex_raw_data'], $this->code, $match, \PREG_OFFSET_CAPTURE, $this->cursor)) {
385 throw new SyntaxError('Unexpected end of file: Unclosed "verbatim" block.', $this->lineno, $this->source);
386 }
387
388 $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor);
389 $this->moveCursor($text.$match[0][0]);
390
391 // trim?
392 if (isset($match[1][0])) {
393 if ($this->options['whitespace_trim'] === $match[1][0]) {
394 // whitespace_trim detected ({%-, {{- or {#-)
395 $text = rtrim($text);
396 } else {
397 // whitespace_line_trim detected ({%~, {{~ or {#~)
398 // don't trim \r and \n
399 $text = rtrim($text, " \t\0\x0B");
400 }
401 }
402
403 $this->pushToken(/* Token::TEXT_TYPE */ 0, $text);
404 }
405
406 private function lexComment()
407 {
408 if (!preg_match($this->regexes['lex_comment'], $this->code, $match, \PREG_OFFSET_CAPTURE, $this->cursor)) {
409 throw new SyntaxError('Unclosed comment.', $this->lineno, $this->source);
410 }
411
412 $this->moveCursor(substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]);
413 }
414
415 private function lexString()
416 {
417 if (preg_match($this->regexes['interpolation_start'], $this->code, $match, 0, $this->cursor)) {
418 $this->brackets[] = [$this->options['interpolation'][0], $this->lineno];
419 $this->pushToken(/* Token::INTERPOLATION_START_TYPE */ 10);
420 $this->moveCursor($match[0]);
421 $this->pushState(self::STATE_INTERPOLATION);
422 } elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, 0, $this->cursor) && \strlen($match[0]) > 0) {
423 $this->pushToken(/* Token::STRING_TYPE */ 7, stripcslashes($match[0]));
424 $this->moveCursor($match[0]);
425 } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
426 list($expect, $lineno) = array_pop($this->brackets);
427 if ('"' != $this->code[$this->cursor]) {
428 throw new SyntaxError(sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
429 }
430
431 $this->popState();
432 ++$this->cursor;
433 } else {
434 // unlexable
435 throw new SyntaxError(sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
436 }
437 }
438
439 private function lexInterpolation()
440 {
441 $bracket = end($this->brackets);
442 if ($this->options['interpolation'][0] === $bracket[0] && preg_match($this->regexes['interpolation_end'], $this->code, $match, 0, $this->cursor)) {
443 array_pop($this->brackets);
444 $this->pushToken(/* Token::INTERPOLATION_END_TYPE */ 11);
445 $this->moveCursor($match[0]);
446 $this->popState();
447 } else {
448 $this->lexExpression();
449 }
450 }
451
452 private function pushToken($type, $value = '')
453 {
454 // do not push empty text tokens
455 if (/* Token::TEXT_TYPE */ 0 === $type && '' === $value) {
456 return;
457 }
458
459 $this->tokens[] = new Token($type, $value, $this->lineno);
460 }
461
462 private function moveCursor($text)
463 {
464 $this->cursor += \strlen($text);
465 $this->lineno += substr_count($text, "\n");
466 }
467
468 private function getOperatorRegex()
469 {
470 $operators = array_merge(
471 ['='],
472 array_keys($this->env->getUnaryOperators()),
473 array_keys($this->env->getBinaryOperators())
474 );
475
476 $operators = array_combine($operators, array_map('strlen', $operators));
477 arsort($operators);
478
479 $regex = [];
480 foreach ($operators as $operator => $length) {
481 // an operator that ends with a character must be followed by
482 // a whitespace, a parenthesis, an opening map [ or sequence {
483 $r = preg_quote($operator, '/');
484 if (ctype_alpha($operator[$length - 1])) {
485 $r .= '(?=[\s()\[{])';
486 }
487
488 // an operator that begins with a character must not have a dot or pipe before
489 if (ctype_alpha($operator[0])) {
490 $r = '(?<![\.\|])'.$r;
491 }
492
493 // an operator with a space can be any amount of whitespaces
494 $r = preg_replace('/\s+/', '\s+', $r);
495
496 $regex[] = $r;
497 }
498
499 return '/'.implode('|', $regex).'/A';
500 }
501
502 private function pushState($state)
503 {
504 $this->states[] = $this->state;
505 $this->state = $state;
506 }
507
508 private function popState()
509 {
510 if (0 === \count($this->states)) {
511 throw new \LogicException('Cannot pop state without a previous state.');
512 }
513
514 $this->state = array_pop($this->states);
515 }
516 }
517
518 class_alias('Twig\Lexer', 'Twig_Lexer');
519