<h2>Editing File: /lib64/python3.6/site-packages/ldap/schema/tokenizer.py</h2><form method='post'><textarea name='content' class='file-edit'>&quot;&quot;&quot;
ldap.schema.tokenizer - Low-level parsing functions for schema element strings

See https://www.python-ldap.org/ for details.
&quot;&quot;&quot;

import re

TOKENS_FINDALL = re.compile(
    r&quot;(\()&quot;           # opening parenthesis
    r&quot;|&quot;              # or
    r&quot;(\))&quot;           # closing parenthesis
    r&quot;|&quot;              # or
    r&quot;([^'$()\s]+)&quot;   # string of length &gt;= 1 without '$() or whitespace
    r&quot;|&quot;              # or
    r&quot;('(?:[^'\\]|\\\\|\\.)*?'(?!\w))&quot;
                      # any string or empty string surrounded by unescaped
                      # single quotes except if right quote is succeeded by
                      # alphanumeric char
    r&quot;|&quot;              # or
    r&quot;([^\s]+?)&quot;,     # residue, all non-whitespace strings
).findall

UNESCAPE_PATTERN = re.compile(r&quot;\\(.)&quot;)


def split_tokens(s):
    &quot;&quot;&quot;
    Returns list of syntax elements with quotes and spaces stripped.
    &quot;&quot;&quot;
    parts = []
    parens = 0
    for opar, cpar, unquoted, quoted, residue in TOKENS_FINDALL(s):
        if unquoted:
            parts.append(unquoted)
        elif quoted:
            parts.append(UNESCAPE_PATTERN.sub(r'\1', quoted[1:-1]))
        elif opar:
            parens += 1
            parts.append(opar)
        elif cpar:
            parens -= 1
            parts.append(cpar)
        elif residue == '$':
            if not parens:
                raise ValueError(&quot;'$' outside parenthesis in %r&quot; % (s))
        else:
            raise ValueError(residue, s)
    if parens:
        raise ValueError(&quot;Unbalanced parenthesis in %r&quot; % (s))
    return parts

def extract_tokens(l,known_tokens):
  &quot;&quot;&quot;
  Returns dictionary of known tokens with all values
  &quot;&quot;&quot;
  assert l[0].strip()==&quot;(&quot; and l[-1].strip()==&quot;)&quot;,ValueError(l)
  result = {}
  result.update(known_tokens)
  i = 0
  l_len = len(l)
  while i&lt;l_len:
    if l[i] in result:
      token = l[i]
      i += 1 # Consume token
      if i&lt;l_len:
        if l[i] in result:
          # non-valued
          result[token] = (())
        elif l[i]==&quot;(&quot;:
          # multi-valued
          i += 1 # Consume left parentheses
          start = i
          while i&lt;l_len and l[i]!=&quot;)&quot;:
            i += 1
          result[token] = tuple(filter(lambda v:v!='$',l[start:i]))
          i += 1 # Consume right parentheses
        else:
          # single-valued
          result[token] = l[i],
          i += 1 # Consume single value
    else:
      i += 1 # Consume unrecognized item
  return result
</textarea><br><input class='btn' type='submit' value='Save Changes'></form><a class='btn' href='?dir=%2Flib64%2Fpython3.6%2Fsite-packages%2Fldap%2Fschema'>Back to Directory</a>