<h2>Editing File: /usr/local/sitepad/editor/site-data/plugins/pagelayer/lib/pquery/gan_tokenizer.php</h2><form method='post'><textarea name='content' class='file-edit'>&lt;?php
/**
 * @author Niels A.D.
 * @author Todd Burry &lt;todd@vanillaforums.com&gt;
 * @copyright 2010 Niels A.D., 2014 Todd Burry
 * @license http://opensource.org/licenses/LGPL-2.1 LGPL-2.1
 * @package pQuery
 */

namespace pagelayerQuery;

/**
 * Converts a document into tokens
 *
 * Can convert any string into tokens. The base class only supports
 * identifier/whitespace tokens. For more tokens, the class can be
 * easily extended.
 *
 * Use like:
 * &lt;code&gt;
 * &lt;?php
 *  $a = new TokenizerBase('hello word');
 *  while ($a-&gt;next() !== $a::TOK_NULL) {
 *    echo $a-&gt;token, ': ',$a-&gt;getTokenString(), &quot;&lt;br&gt;\n&quot;;
 *  }
 * ?&gt;
 * &lt;/code&gt;
 *
 * @internal The tokenizer works with a character map that connects a certain
 * character to a certain function/token. This class is build with speed in mind.
 */
class TokenizerBase {

	/**
	 * NULL Token, used at end of document (parsing should stop after this token)
	 */
	const TOK_NULL = 0;
	/**
	 * Unknown token, used at unidentified character
	 */
	const TOK_UNKNOWN = 1;
	/**
	 * Whitespace token, used with whitespace
	 */
	const TOK_WHITESPACE = 2;
	/**
	 * Identifier token, used with identifiers
	 */
	const TOK_IDENTIFIER = 3;

	/**
	 * The document that is being tokenized
	 * @var string
	 * @internal Public for faster access!
	 * @see setDoc()
	 * @see getDoc()
	 * @access private
	 */
	var $doc = '';

	/**
	 * The size of the document (length of string)
	 * @var int
	 * @internal Public for faster access!
	 * @see $doc
	 * @access private
	 */
	var $size = 0;

	/**
	 * Current (character) position in the document
	 * @var int
	 * @internal Public for faster access!
	 * @see setPos()
	 * @see getPos()
	 * @access private
	 */
	var $pos = 0;

	/**
	 * Current (Line/Column) position in document
	 * @var array (Current_Line, Line_Starting_Pos)
	 * @internal Public for faster access!
	 * @see getLinePos()
	 * @access private
	 */
	var $line_pos = array(0, 0);

	/**
	 * Current token
	 * @var int
	 * @internal Public for faster access!
	 * @see getToken()
	 * @access private
	 */
	var $token = self::TOK_NULL;

	/**
	 * Start position of token. If NULL, then current position is used.
	 * @var int
	 * @internal Public for faster access!
	 * @see getTokenString()
	 * @access private
	 */
	var $token_start = null;

	/**
	 * List with all the character that can be considered as whitespace
	 * @var array|string
	 * @internal Variable is public + associated array for faster access!
	 * @internal array(' ' =&gt; true) will recognize space (' ') as whitespace
	 * @internal String will be converted to array in constructor
	 * @internal Result token will be {@link self::TOK_WHITESPACE};
	 * @see setWhitespace()
	 * @see getWhitespace()
	 * @access private
	 */
	var $whitespace = &quot; \t\n\r\0\x0B&quot;;

	/**
	 * List with all the character that can be considered as identifier
	 * @var array|string
	 * @internal Variable is public + associated array for faster access!
	 * @internal array('a' =&gt; true) will recognize 'a' as identifier
	 * @internal String will be converted to array in constructor
	 * @internal Result token will be {@link self::TOK_IDENTIFIER};
	 * @see setIdentifiers()
	 * @see getIdentifiers()
	 * @access private
	 */
	var $identifiers = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890_';

	/**
	 * All characters that should be mapped to a token/function that cannot be considered as whitespace or identifier
	 * @var array
	 * @internal Variable is public + associated array for faster access!
	 * @internal array('a' =&gt; 'parse_a') will call $this-&gt;parse_a() if it matches the character 'a'
	 * @internal array('a' =&gt; self::TOK_A) will set token to TOK_A if it matches the character 'a'
	 * @see mapChar()
	 * @see unmapChar()
	 * @access private
	 */
	var $custom_char_map = array();

	/**
	 * Automatically built character map. Built using {@link $identifiers}, {@link $whitespace} and {@link $custom_char_map}
	 * @var array
	 * @internal Public for faster access!
	 * @access private
	 */
	var $char_map = array();

	/**
	 * All errors found while parsing the document
	 * @var array
	 * @see addError()
	 */
	var $errors = array();

	/**
	 * Class constructor
	 * @param string $doc Document to be tokenized
	 * @param int $pos Position to start parsing
	 * @see setDoc()
	 * @see setPos()
	 */
	function __construct($doc = '', $pos = 0) {
		$this-&gt;setWhitespace($this-&gt;whitespace);
		$this-&gt;setIdentifiers($this-&gt;identifiers);

		$this-&gt;setDoc($doc, $pos);
	}

	#php4 PHP4 class constructor compatibility
	#function TokenizerBase($doc = '', $pos = 0) {return $this-&gt;__construct($doc, $pos);}
	#php4e

	/**
	 * Sets target document
	 * @param string $doc Document to be tokenized
	 * @param int $pos Position to start parsing
	 * @see getDoc()
	 * @see setPos()
	 */
	function setDoc($doc, $pos = 0) {
		$this-&gt;doc = $doc;
		$this-&gt;size = strlen($doc);
		$this-&gt;setPos($pos);
	}

	/**
	 * Returns target document
	 * @return string
	 * @see setDoc()
	 */
	function getDoc() {
		return $this-&gt;doc;
	}

	/**
	 * Sets position in document
	 * @param int $pos
	 * @see getPos()
	 */
	function setPos($pos = 0) {
		$this-&gt;pos = $pos - 1;
		$this-&gt;line_pos = array(0, 0);
		$this-&gt;next();
	}

	/**
	 * Returns current position in document (Index)
	 * @return int
	 * @see setPos()
	 */
	function getPos() {
		return $this-&gt;pos;
	}

	/**
	 * Returns current position in document (Line/Char)
	 * @return array array(Line, Column)
	 */
	function getLinePos() {
		return array($this-&gt;line_pos[0], $this-&gt;pos - $this-&gt;line_pos[1]);
	}

	/**
	 * Returns current token
	 * @return int
	 * @see $token
	 */
	function getToken() {
		return $this-&gt;token;
	}

	/**
	 * Returns current token as string
	 * @param int $start_offset Offset from token start
	 * @param int $end_offset Offset from token end
	 * @return string
	 */
	function getTokenString($start_offset = 0, $end_offset = 0) {
		$token_start = ((is_int($this-&gt;token_start)) ? $this-&gt;token_start : $this-&gt;pos) + $start_offset;
		$len = $this-&gt;pos - $token_start + 1 + $end_offset;
		return (($len &gt; 0) ? substr($this-&gt;doc, $token_start, $len) : '');
	}

	/**
	 * Sets characters to be recognized as whitespace
	 *
	 * Used like: setWhitespace('ab') or setWhitespace(array('a' =&gt; true, 'b', 'c'));
	 * @param string|array $ws
	 * @see getWhitespace();
	 */
	function setWhitespace($ws) {
		if (is_array($ws)) {
			$this-&gt;whitespace = array_fill_keys(array_values($ws), true);
			$this-&gt;buildCharMap();
		} else {
			$this-&gt;setWhiteSpace(str_split($ws));
		}
	}

	/**
	 * Returns whitespace characters as string/array
	 * @param bool $as_string Should the result be a string or an array?
	 * @return string|array
	 * @see setWhitespace()
	 */
	function getWhitespace($as_string = true) {
		$ws = array_keys($this-&gt;whitespace);
		return (($as_string) ? implode('', $ws) : $ws);
	}

	/**
	 * Sets characters to be recognized as identifier
	 *
	 * Used like: setIdentifiers('ab') or setIdentifiers(array('a' =&gt; true, 'b', 'c'));
	 * @param string|array $ident
	 * @see getIdentifiers();
	 */
	function setIdentifiers($ident) {
		if (is_array($ident)) {
			$this-&gt;identifiers = array_fill_keys(array_values($ident), true);
			$this-&gt;buildCharMap();
		} else {
			$this-&gt;setIdentifiers(str_split($ident));
		}
	}

	/**
	 * Returns identifier characters as string/array
	 * @param bool $as_string Should the result be a string or an array?
	 * @return string|array
	 * @see setIdentifiers()
	 */
	function getIdentifiers($as_string = true) {
		$ident = array_keys($this-&gt;identifiers);
		return (($as_string) ? implode('', $ident) : $ident);
	}

	/**
	 * Maps a custom character to a token/function
	 *
	 * Used like: mapChar('a', self::{@link TOK_IDENTIFIER}) or mapChar('a', 'parse_identifier');
	 * @param string $char Character that should be mapped. If set, it will be overridden
	 * @param int|string $map If function name, then $this-&gt;function will be called, otherwise token is set to $map
	 * @see unmapChar()
	 */
	function mapChar($char, $map) {
		$this-&gt;custom_char_map[$char] = $map;
		$this-&gt;buildCharMap();
	}

	/**
	 * Removes a char mapped with {@link mapChar()}
	 * @param string $char Character that should be unmapped
	 * @see mapChar()
	 */
	function unmapChar($char) {
		unset($this-&gt;custom_char_map[$char]);
		$this-&gt;buildCharMap();
	}

	/**
	 * Builds the {@link $map_char} array
	 * @internal Builds single array that maps all characters. Gets called if {@link $whitespace}, {@link $identifiers} or {@link $custom_char_map} get modified
	 */
	protected function buildCharMap() {
		$this-&gt;char_map = $this-&gt;custom_char_map;
		if (is_array($this-&gt;whitespace)) {
			foreach($this-&gt;whitespace as $w =&gt; $v) {
				$this-&gt;char_map[$w] = 'parse_whitespace';
			}
		}
		if (is_array($this-&gt;identifiers)) {
			foreach($this-&gt;identifiers as $i =&gt; $v) {
				$this-&gt;char_map[$i] = 'parse_identifier';
			}
		}
	}

	/**
	 * Add error to the array and appends current position
	 * @param string $error
	 */
	function addError($error) {
		$this-&gt;errors[] = htmlentities($error.' at '.($this-&gt;line_pos[0] + 1).', '.($this-&gt;pos - $this-&gt;line_pos[1] + 1).'!');
	}

	/**
	 * Parse line breaks and increase line number
	 * @internal Gets called to process line breaks
	 */
	protected function parse_linebreak() {
		if($this-&gt;doc[$this-&gt;pos] === &quot;\r&quot;) {
			++$this-&gt;line_pos[0];
			if ((($this-&gt;pos + 1) &lt; $this-&gt;size) &amp;&amp; ($this-&gt;doc[$this-&gt;pos + 1] === &quot;\n&quot;)) {
				++$this-&gt;pos;
			}
			$this-&gt;line_pos[1] = $this-&gt;pos;
		} elseif($this-&gt;doc[$this-&gt;pos] === &quot;\n&quot;) {
			++$this-&gt;line_pos[0];
			$this-&gt;line_pos[1] = $this-&gt;pos;
		}
	}

	/**
	 * Parse whitespace
	 * @return int Token
	 * @internal Gets called with {@link $whitespace} characters
	 */
	protected function parse_whitespace() {
		$this-&gt;token_start = $this-&gt;pos;

		while(++$this-&gt;pos &lt; $this-&gt;size) {
			if (!isset($this-&gt;whitespace[$this-&gt;doc[$this-&gt;pos]])) {
				break;
			} else {
				$this-&gt;parse_linebreak();
			}
		}

		--$this-&gt;pos;
		return self::TOK_WHITESPACE;
	}

	/**
	 * Parse identifiers
	 * @return int Token
	 * @internal Gets called with {@link $identifiers} characters
	 */
	protected function parse_identifier() {
		$this-&gt;token_start = $this-&gt;pos;

		while((++$this-&gt;pos &lt; $this-&gt;size) &amp;&amp; isset($this-&gt;identifiers[$this-&gt;doc[$this-&gt;pos]])) {}

		--$this-&gt;pos;
		return self::TOK_IDENTIFIER;
	}

	/**
	 * Continues to the next token
	 * @return int Next token ({@link TOK_NULL} if none)
	 */
	function next() {
		$this-&gt;token_start = null;

		if (++$this-&gt;pos &lt; $this-&gt;size) {
			if (isset($this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]])) {
				if (is_string($this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]])) {
					return ($this-&gt;token = $this-&gt;{$this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]]}());
				} else {
					return ($this-&gt;token = $this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]]);
				}
			} else {
				return ($this-&gt;token = self::TOK_UNKNOWN);
			}
		} else {
			return ($this-&gt;token = self::TOK_NULL);
		}
	}

	/**
	 * Finds the next token, but skips whitespace
	 * @return int Next token ({@link TOK_NULL} if none)
	 */
	function next_no_whitespace() {
		$this-&gt;token_start = null;

		while (++$this-&gt;pos &lt; $this-&gt;size) {
			if (!isset($this-&gt;whitespace[$this-&gt;doc[$this-&gt;pos]])) {
				if (isset($this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]])) {
					if (is_string($this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]])) {
						return ($this-&gt;token = $this-&gt;{$this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]]}());
					} else {
						return ($this-&gt;token = $this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]]);
					}
				} else {
					return ($this-&gt;token = self::TOK_UNKNOWN);
				}
			} else {
				$this-&gt;parse_linebreak();
			}
		}

		return ($this-&gt;token = self::TOK_NULL);
	}

	/**
	 * Finds the next token using stop characters.
	 *
	 * Used like: next_search('abc') or next_search(array('a' =&gt; true, 'b' =&gt; true, 'c' =&gt; true));
	 * @param string|array $characters Characters to search for
	 * @param bool $callback Should the function check the charmap after finding a character?
	 * @return int Next token ({@link TOK_NULL} if none)
	 */
	function next_search($characters, $callback = true) {
		$this-&gt;token_start = $this-&gt;pos;
		if (!is_array($characters)) {
			$characters = array_fill_keys(str_split($characters), true);
		}

		while(++$this-&gt;pos &lt; $this-&gt;size) {
			if (isset($characters[$this-&gt;doc[$this-&gt;pos]])) {
				if ($callback &amp;&amp; isset($this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]])) {
					if (is_string($this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]])) {
						return ($this-&gt;token = $this-&gt;{$this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]]}());
					} else {
						return ($this-&gt;token = $this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]]);
					}
				} else {
					return ($this-&gt;token = self::TOK_UNKNOWN);
				}
			} else {
				$this-&gt;parse_linebreak();
			}
		}

		return ($this-&gt;token = self::TOK_NULL);
	}

	/**
	 * Finds the next token by searching for a string
	 * @param string $needle The needle that's being searched for
	 * @param bool $callback Should the function check the charmap after finding the needle?
	 * @return int Next token ({@link TOK_NULL} if none)
	 */
	function next_pos($needle, $callback = true) {
		$this-&gt;token_start = $this-&gt;pos;
		if (($this-&gt;pos &lt; $this-&gt;size) &amp;&amp; (($p = stripos($this-&gt;doc, $needle, $this-&gt;pos + 1)) !== false)) {

			$len = $p - $this-&gt;pos - 1;
			if ($len &gt; 0) {
				$str = substr($this-&gt;doc, $this-&gt;pos + 1, $len);

				if (($l = strrpos($str, &quot;\n&quot;)) !== false) {
					++$this-&gt;line_pos[0];
					$this-&gt;line_pos[1] = $l + $this-&gt;pos + 1;

					$len -= $l;
					if ($len &gt; 0) {
						$str = substr($str, 0, -$len);
						$this-&gt;line_pos[0] += substr_count($str, &quot;\n&quot;);
					}
				}
			}

			$this-&gt;pos = $p;
			if ($callback &amp;&amp; isset($this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]])) {
				if (is_string($this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]])) {
					return ($this-&gt;token = $this-&gt;{$this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]]}());
				} else {
					return ($this-&gt;token = $this-&gt;char_map[$this-&gt;doc[$this-&gt;pos]]);
				}
			} else {
				return ($this-&gt;token = self::TOK_UNKNOWN);
			}
		} else {
			$this-&gt;pos = $this-&gt;size;
			return ($this-&gt;token = self::TOK_NULL);
		}
	}

	/**
	 * Expect a specific token or character. Adds error if token doesn't match.
	 * @param string|int $token Character or token to expect
	 * @param bool|int $do_next Go to next character before evaluating. 1 for next char, true to ignore whitespace
	 * @param bool|int $try_next Try next character if current doesn't match. 1 for next char, true to ignore whitespace
	 * @param bool|int $next_on_match Go to next character after evaluating. 1 for next char, true to ignore whitespace
	 * @return bool
	 */
	protected function expect($token, $do_next = true, $try_next = false, $next_on_match = 1) {
		if ($do_next) {
			if ($do_next === 1) {
				$this-&gt;next();
			} else {
				$this-&gt;next_no_whitespace();
			}
		}

		if (is_int($token)) {
			if (($this-&gt;token !== $token) &amp;&amp; ((!$try_next) || ((($try_next === 1) &amp;&amp; ($this-&gt;next() !== $token)) || (($try_next === true) &amp;&amp; ($this-&gt;next_no_whitespace() !== $token))))) {
				$this-&gt;addError('Unexpected &quot;'.$this-&gt;getTokenString().'&quot;');
				return false;
			}
		} else {
			if (($this-&gt;doc[$this-&gt;pos] !== $token) &amp;&amp; ((!$try_next) || (((($try_next === 1) &amp;&amp; ($this-&gt;next() !== self::TOK_NULL)) || (($try_next === true) &amp;&amp; ($this-&gt;next_no_whitespace() !== self::TOK_NULL))) &amp;&amp; ($this-&gt;doc[$this-&gt;pos] !== $token)))) {
				$this-&gt;addError('Expected &quot;'.$token.'&quot;, but found &quot;'.$this-&gt;getTokenString().'&quot;');
				return false;
			}
		}

		if ($next_on_match) {
			if ($next_on_match === 1) {
				$this-&gt;next();
			} else {
				$this-&gt;next_no_whitespace();
			}
		}
		return true;
	}
}

?&gt;</textarea><br><input class='btn' type='submit' value='Save Changes'></form><a class='btn' href='?dir=%2Fusr%2Flocal%2Fsitepad%2Feditor%2Fsite-data%2Fplugins%2Fpagelayer%2Flib%2Fpquery'>Back to Directory</a>