TokenStream.php 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256
  1. <?php declare(strict_types=1);
  2. namespace PhpParser\Internal;
  3. /**
  4. * Provides operations on token streams, for use by pretty printer.
  5. *
  6. * @internal
  7. */
  8. class TokenStream
  9. {
  10. /** @var array Tokens (in token_get_all format) */
  11. private $tokens;
  12. /** @var int[] Map from position to indentation */
  13. private $indentMap;
  14. /**
  15. * Create token stream instance.
  16. *
  17. * @param array $tokens Tokens in token_get_all() format
  18. */
  19. public function __construct(array $tokens) {
  20. $this->tokens = $tokens;
  21. $this->indentMap = $this->calcIndentMap();
  22. }
  23. /**
  24. * Whether the given position is immediately surrounded by parenthesis.
  25. *
  26. * @param int $startPos Start position
  27. * @param int $endPos End position
  28. *
  29. * @return bool
  30. */
  31. public function haveParens(int $startPos, int $endPos) : bool {
  32. return $this->haveTokenImmediativelyBefore($startPos, '(')
  33. && $this->haveTokenImmediatelyAfter($endPos, ')');
  34. }
  35. /**
  36. * Whether the given position is immediately surrounded by braces.
  37. *
  38. * @param int $startPos Start position
  39. * @param int $endPos End position
  40. *
  41. * @return bool
  42. */
  43. public function haveBraces(int $startPos, int $endPos) : bool {
  44. return $this->haveTokenImmediativelyBefore($startPos, '{')
  45. && $this->haveTokenImmediatelyAfter($endPos, '}');
  46. }
  47. /**
  48. * Check whether the position is directly preceded by a certain token type.
  49. *
  50. * During this check whitespace and comments are skipped.
  51. *
  52. * @param int $pos Position before which the token should occur
  53. * @param int|string $expectedTokenType Token to check for
  54. *
  55. * @return bool Whether the expected token was found
  56. */
  57. public function haveTokenImmediativelyBefore(int $pos, $expectedTokenType) : bool {
  58. $tokens = $this->tokens;
  59. $pos--;
  60. for (; $pos >= 0; $pos--) {
  61. $tokenType = $tokens[$pos][0];
  62. if ($tokenType === $expectedTokenType) {
  63. return true;
  64. }
  65. if ($tokenType !== \T_WHITESPACE
  66. && $tokenType !== \T_COMMENT && $tokenType !== \T_DOC_COMMENT) {
  67. break;
  68. }
  69. }
  70. return false;
  71. }
  72. /**
  73. * Check whether the position is directly followed by a certain token type.
  74. *
  75. * During this check whitespace and comments are skipped.
  76. *
  77. * @param int $pos Position after which the token should occur
  78. * @param int|string $expectedTokenType Token to check for
  79. *
  80. * @return bool Whether the expected token was found
  81. */
  82. public function haveTokenImmediatelyAfter(int $pos, $expectedTokenType) : bool {
  83. $tokens = $this->tokens;
  84. $pos++;
  85. for (; $pos < \count($tokens); $pos++) {
  86. $tokenType = $tokens[$pos][0];
  87. if ($tokenType === $expectedTokenType) {
  88. return true;
  89. }
  90. if ($tokenType !== \T_WHITESPACE
  91. && $tokenType !== \T_COMMENT && $tokenType !== \T_DOC_COMMENT) {
  92. break;
  93. }
  94. }
  95. return false;
  96. }
  97. public function skipLeft(int $pos, $skipTokenType) {
  98. $tokens = $this->tokens;
  99. $pos = $this->skipLeftWhitespace($pos);
  100. if ($skipTokenType === \T_WHITESPACE) {
  101. return $pos;
  102. }
  103. if ($tokens[$pos][0] !== $skipTokenType) {
  104. // Shouldn't happen. The skip token MUST be there
  105. throw new \Exception('Encountered unexpected token');
  106. }
  107. $pos--;
  108. return $this->skipLeftWhitespace($pos);
  109. }
  110. public function skipRight(int $pos, $skipTokenType) {
  111. $tokens = $this->tokens;
  112. $pos = $this->skipRightWhitespace($pos);
  113. if ($skipTokenType === \T_WHITESPACE) {
  114. return $pos;
  115. }
  116. if ($tokens[$pos][0] !== $skipTokenType) {
  117. // Shouldn't happen. The skip token MUST be there
  118. throw new \Exception('Encountered unexpected token');
  119. }
  120. $pos++;
  121. return $this->skipRightWhitespace($pos);
  122. }
  123. /**
  124. * Return first non-whitespace token position smaller or equal to passed position.
  125. *
  126. * @param int $pos Token position
  127. * @return int Non-whitespace token position
  128. */
  129. public function skipLeftWhitespace(int $pos) {
  130. $tokens = $this->tokens;
  131. for (; $pos >= 0; $pos--) {
  132. $type = $tokens[$pos][0];
  133. if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
  134. break;
  135. }
  136. }
  137. return $pos;
  138. }
  139. /**
  140. * Return first non-whitespace position greater or equal to passed position.
  141. *
  142. * @param int $pos Token position
  143. * @return int Non-whitespace token position
  144. */
  145. public function skipRightWhitespace(int $pos) {
  146. $tokens = $this->tokens;
  147. for ($count = \count($tokens); $pos < $count; $pos++) {
  148. $type = $tokens[$pos][0];
  149. if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
  150. break;
  151. }
  152. }
  153. return $pos;
  154. }
  155. public function findRight($pos, $findTokenType) {
  156. $tokens = $this->tokens;
  157. for ($count = \count($tokens); $pos < $count; $pos++) {
  158. $type = $tokens[$pos][0];
  159. if ($type === $findTokenType) {
  160. return $pos;
  161. }
  162. }
  163. return -1;
  164. }
  165. /**
  166. * Get indentation before token position.
  167. *
  168. * @param int $pos Token position
  169. *
  170. * @return int Indentation depth (in spaces)
  171. */
  172. public function getIndentationBefore(int $pos) : int {
  173. return $this->indentMap[$pos];
  174. }
  175. /**
  176. * Get the code corresponding to a token offset range, optionally adjusted for indentation.
  177. *
  178. * @param int $from Token start position (inclusive)
  179. * @param int $to Token end position (exclusive)
  180. * @param int $indent By how much the code should be indented (can be negative as well)
  181. *
  182. * @return string Code corresponding to token range, adjusted for indentation
  183. */
  184. public function getTokenCode(int $from, int $to, int $indent) : string {
  185. $tokens = $this->tokens;
  186. $result = '';
  187. for ($pos = $from; $pos < $to; $pos++) {
  188. $token = $tokens[$pos];
  189. if (\is_array($token)) {
  190. $type = $token[0];
  191. $content = $token[1];
  192. if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
  193. $result .= $content;
  194. } else {
  195. // TODO Handle non-space indentation
  196. if ($indent < 0) {
  197. $result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $content);
  198. } elseif ($indent > 0) {
  199. $result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $content);
  200. } else {
  201. $result .= $content;
  202. }
  203. }
  204. } else {
  205. $result .= $token;
  206. }
  207. }
  208. return $result;
  209. }
  210. /**
  211. * Precalculate the indentation at every token position.
  212. *
  213. * @return int[] Token position to indentation map
  214. */
  215. private function calcIndentMap() {
  216. $indentMap = [];
  217. $indent = 0;
  218. foreach ($this->tokens as $token) {
  219. $indentMap[] = $indent;
  220. if ($token[0] === \T_WHITESPACE) {
  221. $content = $token[1];
  222. $newlinePos = \strrpos($content, "\n");
  223. if (false !== $newlinePos) {
  224. $indent = \strlen($content) - $newlinePos - 1;
  225. }
  226. }
  227. }
  228. // Add a sentinel for one past end of the file
  229. $indentMap[] = $indent;
  230. return $indentMap;
  231. }
  232. }