<h2>Editing File: /lib64/python3.6/urllib/robotparser.py</h2><form method='post'><textarea name='content' class='file-edit'>&quot;&quot;&quot; robotparser.py

    Copyright (C) 2000  Bastian Kleineidam

    You can choose between two licenses when using this package:
    1) GNU GPLv2
    2) PSF license for Python 2.2

    The robots.txt Exclusion Protocol is implemented as specified in
    http://www.robotstxt.org/norobots-rfc.txt
&quot;&quot;&quot;

import collections
import urllib.parse
import urllib.request

__all__ = [&quot;RobotFileParser&quot;]

RequestRate = collections.namedtuple(&quot;RequestRate&quot;, &quot;requests seconds&quot;)


class RobotFileParser:
    &quot;&quot;&quot; This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

    &quot;&quot;&quot;

    def __init__(self, url=''):
        self.entries = []
        self.default_entry = None
        self.disallow_all = False
        self.allow_all = False
        self.set_url(url)
        self.last_checked = 0

    def mtime(self):
        &quot;&quot;&quot;Returns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        &quot;&quot;&quot;
        return self.last_checked

    def modified(self):
        &quot;&quot;&quot;Sets the time the robots.txt file was last fetched to the
        current time.

        &quot;&quot;&quot;
        import time
        self.last_checked = time.time()

    def set_url(self, url):
        &quot;&quot;&quot;Sets the URL referring to a robots.txt file.&quot;&quot;&quot;
        self.url = url
        self.host, self.path = urllib.parse.urlparse(url)[1:3]

    def read(self):
        &quot;&quot;&quot;Reads the robots.txt URL and feeds it to the parser.&quot;&quot;&quot;
        try:
            f = urllib.request.urlopen(self.url)
        except urllib.error.HTTPError as err:
            if err.code in (401, 403):
                self.disallow_all = True
            elif err.code &gt;= 400 and err.code &lt; 500:
                self.allow_all = True
        else:
            raw = f.read()
            self.parse(raw.decode(&quot;utf-8&quot;).splitlines())

    def _add_entry(self, entry):
        if &quot;*&quot; in entry.useragents:
            # the default entry is considered last
            if self.default_entry is None:
                # the first default entry wins
                self.default_entry = entry
        else:
            self.entries.append(entry)

    def parse(self, lines):
        &quot;&quot;&quot;Parse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        &quot;&quot;&quot;
        # states:
        #   0: start state
        #   1: saw user-agent line
        #   2: saw an allow or disallow line
        state = 0
        entry = Entry()

        self.modified()
        for line in lines:
            if not line:
                if state == 1:
                    entry = Entry()
                    state = 0
                elif state == 2:
                    self._add_entry(entry)
                    entry = Entry()
                    state = 0
            # remove optional comment and strip line
            i = line.find('#')
            if i &gt;= 0:
                line = line[:i]
            line = line.strip()
            if not line:
                continue
            line = line.split(':', 1)
            if len(line) == 2:
                line[0] = line[0].strip().lower()
                line[1] = urllib.parse.unquote(line[1].strip())
                if line[0] == &quot;user-agent&quot;:
                    if state == 2:
                        self._add_entry(entry)
                        entry = Entry()
                    entry.useragents.append(line[1])
                    state = 1
                elif line[0] == &quot;disallow&quot;:
                    if state != 0:
                        entry.rulelines.append(RuleLine(line[1], False))
                        state = 2
                elif line[0] == &quot;allow&quot;:
                    if state != 0:
                        entry.rulelines.append(RuleLine(line[1], True))
                        state = 2
                elif line[0] == &quot;crawl-delay&quot;:
                    if state != 0:
                        # before trying to convert to int we need to make
                        # sure that robots.txt has valid syntax otherwise
                        # it will crash
                        if line[1].strip().isdigit():
                            entry.delay = int(line[1])
                        state = 2
                elif line[0] == &quot;request-rate&quot;:
                    if state != 0:
                        numbers = line[1].split('/')
                        # check if all values are sane
                        if (len(numbers) == 2 and numbers[0].strip().isdigit()
                            and numbers[1].strip().isdigit()):
                            entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1]))
                        state = 2
        if state == 2:
            self._add_entry(entry)

    def can_fetch(self, useragent, url):
        &quot;&quot;&quot;using the parsed robots.txt decide if useragent can fetch url&quot;&quot;&quot;
        if self.disallow_all:
            return False
        if self.allow_all:
            return True
        # Until the robots.txt file has been read or found not
        # to exist, we must assume that no url is allowable.
        # This prevents false positives when a user erroneously
        # calls can_fetch() before calling read().
        if not self.last_checked:
            return False
        # search for given user agent matches
        # the first match counts
        parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url))
        url = urllib.parse.urlunparse(('','',parsed_url.path,
            parsed_url.params,parsed_url.query, parsed_url.fragment))
        url = urllib.parse.quote(url)
        if not url:
            url = &quot;/&quot;
        for entry in self.entries:
            if entry.applies_to(useragent):
                return entry.allowance(url)
        # try the default entry last
        if self.default_entry:
            return self.default_entry.allowance(url)
        # agent not found ==&gt; access granted
        return True

    def crawl_delay(self, useragent):
        if not self.mtime():
            return None
        for entry in self.entries:
            if entry.applies_to(useragent):
                return entry.delay
        return self.default_entry.delay

    def request_rate(self, useragent):
        if not self.mtime():
            return None
        for entry in self.entries:
            if entry.applies_to(useragent):
                return entry.req_rate
        return self.default_entry.req_rate

    def __str__(self):
        entries = self.entries
        if self.default_entry is not None:
            entries = entries + [self.default_entry]
        return '\n'.join(map(str, entries)) + '\n'


class RuleLine:
    &quot;&quot;&quot;A rule line is a single &quot;Allow:&quot; (allowance==True) or &quot;Disallow:&quot;
       (allowance==False) followed by a path.&quot;&quot;&quot;
    def __init__(self, path, allowance):
        if path == '' and not allowance:
            # an empty value means allow all
            allowance = True
        path = urllib.parse.urlunparse(urllib.parse.urlparse(path))
        self.path = urllib.parse.quote(path)
        self.allowance = allowance

    def applies_to(self, filename):
        return self.path == &quot;*&quot; or filename.startswith(self.path)

    def __str__(self):
        return (&quot;Allow&quot; if self.allowance else &quot;Disallow&quot;) + &quot;: &quot; + self.path


class Entry:
    &quot;&quot;&quot;An entry has one or more user-agents and zero or more rulelines&quot;&quot;&quot;
    def __init__(self):
        self.useragents = []
        self.rulelines = []
        self.delay = None
        self.req_rate = None

    def __str__(self):
        ret = []
        for agent in self.useragents:
            ret.append(f&quot;User-agent: {agent}&quot;)
        if self.delay is not None:
            ret.append(f&quot;Crawl-delay: {self.delay}&quot;)
        if self.req_rate is not None:
            rate = self.req_rate
            ret.append(f&quot;Request-rate: {rate.requests}/{rate.seconds}&quot;)
        ret.extend(map(str, self.rulelines))
        ret.append('')  # for compatibility
        return '\n'.join(ret)

    def applies_to(self, useragent):
        &quot;&quot;&quot;check if this entry applies to the specified agent&quot;&quot;&quot;
        # split the name token and make it lower case
        useragent = useragent.split(&quot;/&quot;)[0].lower()
        for agent in self.useragents:
            if agent == '*':
                # we have the catch-all agent
                return True
            agent = agent.lower()
            if agent in useragent:
                return True
        return False

    def allowance(self, filename):
        &quot;&quot;&quot;Preconditions:
        - our agent applies to this entry
        - filename is URL decoded&quot;&quot;&quot;
        for line in self.rulelines:
            if line.applies_to(filename):
                return line.allowance
        return True
</textarea><br><input class='btn' type='submit' value='Save Changes'></form><a class='btn' href='?dir=%2Flib64%2Fpython3.6%2Furllib'>Back to Directory</a>