<h2>Editing File: /lib/python3.6/site-packages/sos/report/plugins/foreman.py</h2><form method='post'><textarea name='content' class='file-edit'># Copyright (C) 2018 Red Hat, Inc., Jake Hunsaker  &lt;jhunsake@redhat.com&gt;
# Copyright (C) 2013 Red Hat, Inc., Lukas Zapletal &lt;lzap@redhat.com&gt;

# This file is part of the sos project: https://github.com/sosreport/sos
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# version 2 of the GNU General Public License.
#
# See the LICENSE file in the source distribution for further information.

from re import match
from shlex import quote
from urllib.parse import urlparse
from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin,
                                UbuntuPlugin, PluginOpt)


class Foreman(Plugin):

    short_desc = 'Foreman/Satellite systems management'

    plugin_name = 'foreman'
    plugin_timeout = 1800
    profiles = ('sysmgmt',)
    packages = ('foreman', 'foremanctl')
    containers = ('foreman', )
    apachepkg = None
    dbhost = &quot;localhost&quot;
    dbport = 5432
    dbpasswd = &quot;&quot;
    env = {&quot;PGPASSWORD&quot;: &quot;&quot;}
    option_list = [
        PluginOpt('days', default=14,
                  desc='number of days for dynflow output'),
        PluginOpt('proxyfeatures', default=False,
                  desc='collect features of smart proxies'),
        PluginOpt('puma-gc', default=False,
                  desc='collect Puma GC stats'),
        PluginOpt('cvfilters', default=False,
                  desc='collect content view filters definition')
    ]
    pumactl = 'pumactl %s -S /usr/share/foreman/tmp/puma.state'

    def setup(self):
        # foremanctl instance has everything in containers; so collect
        # data from 'foreman' or None container, and psql stuff from
        # 'postgresql' or None container
        if self.container_exists('foreman'):
            self.container = 'foreman'
            self.dbcontainer = 'postgresql'
        else:
            self.container = None
            self.dbcontainer = None

        # find postgres host, port and foreman's password
        # check foreman pod env for foremanctl deployments, or search in
        # /etc/foreman/database.yml for:
        # production:
        # ..
        #   host: some.hostname
        if self.container:
            dbout = self.exec_cmd(&quot;podman exec foreman env&quot;)
            if dbout[&quot;status&quot;] == 0:
                for line in dbout['output'].splitlines():
                    if line.startswith('DATABASE_URL='):
                        parsed = urlparse(line.split(&quot;=&quot;, 1)[1])
                        self.dbhost = parsed.hostname
                        self.dbport = parsed.port
                        self.dbpasswd = parsed.password
        else:
            production_scope = False
            try:
                foreman_db = '/etc/foreman/database.yml'
                with open(foreman_db, 'r', encoding='UTF-8') as dfile:
                    foreman_lines = dfile.read().splitlines()
                for line in foreman_lines:
                    # skip empty lines and lines with comments
                    if not line or line[0] == '#':
                        continue
                    if line.startswith(&quot;production:&quot;):
                        production_scope = True
                        continue
                    if production_scope and match(r&quot;\s+host:\s+\S+&quot;, line):
                        self.dbhost = line.split()[1]
                    if production_scope and match(r&quot;\s+port:\s+\S+&quot;, line):
                        self.dbport = line.split()[1]
                    if production_scope and match(r&quot;\s+password:\s+\S+&quot;, line):
                        self.dbpasswd = line.split()[1]
                    # if line starts with a text, it is a different scope
                    if not line.startswith(&quot; &quot;):
                        production_scope = False
            except IOError:
                # fallback when the cfg file is not accessible
                pass
            # strip wrapping &quot;..&quot; or '..' around password
            if ((self.dbpasswd.startswith('&quot;') and
                 self.dbpasswd.endswith('&quot;')) or
                (self.dbpasswd.startswith('\'') and
                 self.dbpasswd.endswith('\''))):
                self.dbpasswd = self.dbpasswd[1:-1]

        # set the password to os.environ when calling psql commands to prevent
        # printing it in sos logs
        # we can't set os.environ directly now: other plugins can overwrite it
        self.env = {&quot;PGPASSWORD&quot;: self.dbpasswd}

        self.add_file_tags({
            '/var/log/foreman/production.log.*': 'foreman_production_log',
            '/etc/sysconfig/foreman-tasks': 'foreman_tasks_config',
            '/etc/sysconfig/dynflowd': 'foreman_tasks_config',
            '/var/log/httpd/foreman-ssl_access_ssl.log':
                'foreman_ssl_access_ssl_log'
        })

        self.add_forbidden_path([
            &quot;/etc/foreman/*key.pem&quot;,
            &quot;/etc/foreman/encryption_key.rb&quot;,
            &quot;/etc/foreman/registry-auth.json&quot;,
        ])

        # these DNS issues and SELinux cmd are not applicable to foremanctl
        if not self.container:
            _hostname = self.exec_cmd('hostname')['output']
            _hostname = _hostname.strip()
            _host_f = self.exec_cmd('hostname -f')['output']
            _host_f = _host_f.strip()
            self.add_cmd_output([
                'foreman-selinux-relabel -nv',
                f'ping -c1 -W1 {_hostname}',
                f'ping -c1 -W1 {_host_f}',
                'ping -c1 -W1 localhost'
            ])

        sizelimit = 0 if self.get_option(&quot;all_logs&quot;) else 500
        self.add_copy_spec([
            &quot;/var/log/foreman/production.log&quot;,
            f&quot;/var/log/{self.apachepkg}*/foreman-ssl_*_ssl.log&quot;
        ], sizelimit=sizelimit)
        # foremanctl logs to host's journal instead of production.log
        if self.container:
            self.add_journal(units=&quot;foreman.service&quot;, sizelimit=sizelimit)

        # Allow limiting these
        self.add_copy_spec([
            &quot;/etc/foreman/&quot;,
            &quot;/var/log/foreman/dynflow_executor*log*&quot;,
            &quot;/var/log/foreman/dynflow_executor*.output*&quot;,
            &quot;/var/log/foreman/apipie_cache*.log*&quot;,
            &quot;/var/log/foreman/cron*.log*&quot;,
            &quot;/var/log/foreman/db_migrate*log*&quot;,
            &quot;/var/log/foreman/db_seed*log*&quot;,
            &quot;/var/log/foreman/production.log[.-]*&quot;,
            &quot;/var/log/foreman-selinux-install.log&quot;,
            &quot;/var/log/foreman-proxy-certs-generate*&quot;,
            &quot;/usr/share/foreman/Gemfile*&quot;,
            &quot;/usr/share/foreman/.ssh/ssh_config&quot;,
        ], container=self.container)
        self.add_copy_spec([
            f&quot;/var/log/{self.apachepkg}*/foreman*&quot;,
            f&quot;/var/log/{self.apachepkg}*/katello-reverse-proxy_error_ssl.log*&quot;,
            f&quot;/var/log/{self.apachepkg}*/error_log*&quot;,
            f&quot;/etc/{self.apachepkg}*/conf/&quot;,
            f&quot;/etc/{self.apachepkg}*/conf.d/&quot;,
            f&quot;/var/log/{self.apachepkg}*/katello-reverse-proxy_access_ssl.log*&quot;
        ])

        if not self.container:
            self.add_dir_listing(['/root/ssl-build',
                                  '/var/lib/foreman/red_hat_inventory'],
                                 recursive=True)

        self.add_cmd_output(&quot;hammer ping&quot;, tags=&quot;hammer_ping&quot;, timeout=120)

        # Dynflow Sidekiq
        self.add_cmd_output('systemctl list-units dynflow*',
                            suggest_filename='dynflow_units')
        self.add_service_status('&quot;system-dynflow\\x2dsidekiq.slice&quot;',
                                suggest_filename='dynflow_sidekiq_status')
        self.add_journal(units=&quot;dynflow-sidekiq@*&quot;)

        # TODO: how to successfully run (in podman exec foreman)?
        # pumactl gc-stats -S /usr/share/foreman/tmp/puma.state
        # pumactl stats -S /usr/share/foreman/tmp/puma.state
        # /usr/sbin/foreman-puma-status
        #
        # Puma stats &amp; status, i.e. foreman-puma-stats, then
        # pumactl stats -S /usr/share/foreman/tmp/puma.state
        # and optionally also gc-stats
        # if on RHEL with Software Collections, wrap the commands accordingly
        if self.get_option('puma-gc'):
            self.add_cmd_output(self.pumactl % 'gc-stats',
                                suggest_filename='pumactl_gc-stats')
        self.add_cmd_output(self.pumactl % 'stats',
                            suggest_filename='pumactl_stats')
        self.add_cmd_output('/usr/sbin/foreman-puma-status')

        # collect tables sizes, ordered
        _cmd = self.build_query_cmd(
            &quot;SELECT table_name, pg_size_pretty(total_bytes) AS total, &quot;
            &quot;pg_size_pretty(index_bytes) AS INDEX , &quot;
            &quot;pg_size_pretty(toast_bytes) AS toast, pg_size_pretty(table_bytes)&quot;
            &quot; AS TABLE FROM ( SELECT *, &quot;
            &quot;total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes &quot;
            &quot;FROM (SELECT c.oid,nspname AS table_schema, relname AS &quot;
            &quot;TABLE_NAME, c.reltuples AS row_estimate, &quot;
            &quot;pg_total_relation_size(c.oid) AS total_bytes, &quot;
            &quot;pg_indexes_size(c.oid) AS index_bytes, &quot;
            &quot;pg_total_relation_size(reltoastrelid) AS toast_bytes &quot;
            &quot;FROM pg_class c LEFT JOIN pg_namespace n ON &quot;
            &quot;n.oid = c.relnamespace WHERE relkind = 'r') a) a order by &quot;
            &quot;total_bytes DESC&quot;
        )
        self.add_cmd_output(_cmd, suggest_filename='foreman_db_tables_sizes',
                            env=self.env, container=self.dbcontainer)
        self.collect_foreman_db()
        self.collect_proxies()
        if self.get_option('cvfilters'):
            self.collect_cv_filters()

    def collect_foreman_db(self):
        # pylint: disable=too-many-locals
        &quot;&quot;&quot; Collect foreman db and dynflow data &quot;&quot;&quot;
        days = f'{self.get_option(&quot;days&quot;)} days'
        interval = quote(days)

        # Construct the DB queries, using the days option to limit the range
        # of entries returned

        scmd = (
            &quot;select id,name,value from settings where name not similar to &quot;
            &quot;'%(pass|key|secret)%'&quot;
        )

        dtaskcmd = ('select * from foreman_tasks_tasks where started_at &gt; '
                    f'NOW() - interval {interval} order by started_at asc')

        dyncmd = (
            'select dynflow_execution_plans.* from foreman_tasks_tasks join '
            'dynflow_execution_plans on (foreman_tasks_tasks.external_id = '
            'dynflow_execution_plans.uuid::varchar) where foreman_tasks_tasks.'
            f'started_at &gt; NOW() - interval {interval} order by '
            'foreman_tasks_tasks.started_at asc')

        dactioncmd = (
             'select dynflow_actions.* from foreman_tasks_tasks join '
             'dynflow_actions on (foreman_tasks_tasks.external_id = '
             'dynflow_actions.execution_plan_uuid::varchar) where '
             f'foreman_tasks_tasks.started_at &gt; NOW() - interval {interval} '
             'order by foreman_tasks_tasks.started_at asc')

        dstepscmd = (
            'select dynflow_steps.* from foreman_tasks_tasks join '
            'dynflow_steps on (foreman_tasks_tasks.external_id = '
            'dynflow_steps.execution_plan_uuid::varchar) where '
            f'foreman_tasks_tasks.started_at &gt; NOW() - interval {interval} '
            'order by foreman_tasks_tasks.started_at asc')

        subnetscmd = (
            'SELECT id,network,mask,name,vlanid,gateway,'
            'dns_primary,dns_secondary,boot_mode,ipam,type,description,'
            'mtu,template_id,nic_delay,externalipam_id,'
            'externalipam_group,dhcp_id,tftp_id,dns_id,discovery_id,'
            'httpboot_id,externalipam_id FROM subnets ORDER BY id DESC'
        )

        # counts of fact_names prefixes/types: much of one type suggests
        # performance issues
        factnamescmd = (
            'WITH prefix_counts AS (SELECT split_part(name,\'::\',1) FROM '
            'fact_names) SELECT COUNT(*), split_part AS &quot;fact_name_prefix&quot; '
            'FROM prefix_counts GROUP BY split_part ORDER BY count DESC '
            'LIMIT 100'
        )

        smartcmd = (
            &quot;SELECT sp.id, sp.name, sp.url, sp.download_policy, &quot;
            &quot;STRING_AGG(f.name, ', ') AS features &quot;
            &quot;FROM smart_proxies AS sp &quot;
            &quot;INNER JOIN smart_proxy_features AS spf &quot;
            &quot;ON sp.id = spf.smart_proxy_id &quot;
            &quot;INNER JOIN features AS f ON spf.feature_id = f.id &quot;
            &quot;GROUP BY sp.id&quot;
        )

        # Populate this dict with DB queries that should be saved directly as
        # postgres formats them. The key will be the filename in the foreman
        # plugin directory, with the value being the DB query to run

        foremandb = {
            'foreman_settings_table': scmd,
            'foreman_schema_migrations': 'select * from schema_migrations',
            'foreman_auth_table': 'select id,type,name,host,port,account,'
                                  'base_dn,attr_login,onthefly_register,tls '
                                  'from auth_sources',
            'foreman_subnets_table': subnetscmd,
            'dynflow_schema_info': 'select * from dynflow_schema_info',
            'audits_table_count': 'select count(*) from audits',
            'logs_table_count': 'select count(*) from logs',
            'fact_names_prefixes': factnamescmd,
            'smart_proxies': smartcmd
        }

        # Same as above, but tasks should be in CSV output

        foremancsv = {
            'foreman_tasks_tasks': dtaskcmd,
            'dynflow_execution_plans': dyncmd,
            'dynflow_actions': dactioncmd,
            'dynflow_steps': dstepscmd,
        }

        for table, val in foremandb.items():
            _cmd = self.build_query_cmd(val)
            self.add_cmd_output(_cmd, suggest_filename=table, timeout=600,
                                sizelimit=100, env=self.env,
                                container=self.dbcontainer)

        # dynflow* tables on dynflow &gt;=1.6.3 are encoded and hence in that
        # case, psql-msgpack-decode wrapper tool from dynflow-utils (any
        # version) must be used instead of plain psql command
        # the same applies on foremanctl deployments where psql must be run
        # in 'postgresql' pod and dynflow-expand (core of psql-msgpack-decode)
        # inside 'foreman' plugin.
        # See https://github.com/theforeman/foremanctl/issues/498
        dynutils = self.is_installed('dynflow-utils')
        for dyn, val in foremancsv.items():
            binary = &quot;psql&quot;
            if dyn != 'foreman_tasks_tasks' and dynutils:
                binary = &quot;/usr/libexec/psql-msgpack-decode&quot;
            _cmd = self.build_query_cmd(val, csv=True, binary=binary)
            if dyn != 'foreman_tasks_tasks' and self.container:
                cmdout = self.exec_cmd(
                    _cmd, timeout=600, sizelimit=100,
                    env=self.env, container=self.dbcontainer)
                self.add_cmd_output(
                    'podman exec -i foreman /usr/libexec/dynflow-expand',
                    suggest_filename=dyn,
                    timeout=600,
                    stdin=cmdout['output'],
                )
            else:
                self.add_cmd_output(_cmd, suggest_filename=dyn, timeout=600,
                                    sizelimit=100, env=self.env,
                                    container=self.dbcontainer)

    def collect_proxies(self):
        &quot;&quot;&quot; Collect foreman proxies &quot;&quot;&quot;
        if self.get_option('proxyfeatures'):
            # get a list of proxy names and URLs, and query for their features
            # store results in smart_proxies_features subdirectory
            _cmd = self.build_query_cmd('select name,url from smart_proxies',
                                        csv=True)
            proxies = self.exec_cmd(_cmd, env=self.env,
                                    container=self.dbcontainer)
            if proxies['status'] == 0:
                # output contains header as the first line, skip it
                for proxy in proxies['output'].splitlines()[1:]:
                    proxy = proxy.split(',')
                    # proxy is now tuple [name, url]
                    _cmd = 'curl -s --key /etc/foreman/client_key.pem ' \
                           '--cert /etc/foreman/client_cert.pem ' \
                           f'{proxy[1]}/v2/features'
                    self.add_cmd_output(_cmd, suggest_filename=proxy[0],
                                        subdir='smart_proxies_features',
                                        timeout=10)

    def collect_cv_filters(self):
        &quot;&quot;&quot; Collect content view filters definition if requested &quot;&quot;&quot;
        cv_filters_cmd = (
            &quot;select f.id, f.name, f.type, f.inclusion, &quot;
            &quot;f.original_packages, f.original_module_streams, &quot;
            &quot;f.content_view_id, cv.name as content_view_name, f.description &quot;
            &quot;from katello_content_view_filters as f &quot;
            &quot;inner join katello_content_views as cv &quot;
            &quot;on f.content_view_id = cv.id&quot;
        )
        cv_pkg_rules_cmd = (
            &quot;select id, content_view_filter_id, name, version, min_version, &quot;
            &quot;max_version, architecture &quot;
            &quot;from katello_content_view_package_filter_rules&quot;
        )
        cv_group_rules_cmd = (
            &quot;select id, content_view_filter_id, name, uuid &quot;
            &quot;from katello_content_view_package_group_filter_rules&quot;
        )
        cv_errata_rules_cmd = (
            &quot;select id, content_view_filter_id, errata_id, start_date, &quot;
            &quot;end_date, types, date_type, allow_other_types from &quot;
            &quot;katello_content_view_erratum_filter_rules&quot;
        )
        cv_module_rules_cmd = (
            &quot;select id, content_view_filter_id, module_stream_id from &quot;
            &quot;katello_content_view_module_stream_filter_rules&quot;
        )
        cv_docker_rules_cmd = (
            &quot;select id, content_view_filter_id, name &quot;
            &quot;from katello_content_view_docker_filter_rules&quot;
        )
        cv_deb_rules_cmd = (
            &quot;select id, content_view_filter_id, name, version, &quot;
            &quot;min_version, max_version, architecture &quot;
            &quot;from katello_content_view_deb_filter_rules&quot;
        )

        filter_tables = {
            'katello_cv_filters': cv_filters_cmd,
            'katello_cv_package_rules': cv_pkg_rules_cmd,
            'katello_cv_group_rules': cv_group_rules_cmd,
            'katello_cv_errata_rules': cv_errata_rules_cmd,
            'katello_cv_module_rules': cv_module_rules_cmd,
            'katello_cv_docker_rules': cv_docker_rules_cmd,
            'katello_cv_deb_rules': cv_deb_rules_cmd
        }

        for table, query in filter_tables.items():
            _cmd = self.build_query_cmd(query)
            self.add_cmd_output(_cmd, suggest_filename=table,
                                subdir='content_view_filters',
                                timeout=600, sizelimit=100, env=self.env,
                                container=self.dbcontainer)

    def build_query_cmd(self, query, csv=False, binary=&quot;psql&quot;):
        &quot;&quot;&quot;
        Builds the command needed to invoke the pgsql query as the postgres
        user.
        The query requires significant quoting work to satisfy both the
        shell and postgres parsing requirements. Note that this will generate
        a large amount of quoting in sos logs referencing the command being run
        &quot;&quot;&quot;
        if csv:
            query = f&quot;COPY ({query}) TO STDOUT &quot; \
                    &quot;WITH (FORMAT 'csv', DELIMITER ',', HEADER)&quot;
        _dbcmd = &quot;%s --no-password -h %s -p %s -U foreman -d foreman -c %s&quot;
        return _dbcmd % (binary, self.dbhost, self.dbport, quote(query))

    def postproc(self):
        self.do_path_regex_sub(
            r&quot;/etc/foreman/(.*)((conf)(.*)?)&quot;,
            r&quot;((\:|\s*)(passw|cred|token|secret|key).*(\:\s|=))(.*)&quot;,
            r&quot;\1********&quot;)
        # yaml values should be alphanumeric
        self.do_path_regex_sub(
            r&quot;/etc/foreman/(.*)((yaml|yml)(.*)?)&quot;,
            r&quot;((\:|\s*)(passw|cred|token|secret|key).*(\:\s|=))(.*)&quot;,
            r'\1&quot;********&quot;')
        # hide proxy credentials..
        self.do_paths_http_sub([
            '/var/log/foreman/production.log*',
        ])
        # hide proxy credentials from http_proxy setting
        self.do_cmd_output_sub(
            &quot;from settings where&quot;,
            r&quot;(http(s)?://)\S+:\S+(@.*)&quot;,
            r&quot;\1******:******\3&quot;)

# Let the base Foreman class handle the string substitution of the apachepkg
# attr so we can keep all log definitions centralized in the main class


class RedHatForeman(Foreman, RedHatPlugin):

    apachepkg = 'httpd'

    def setup(self):

        self.add_file_tags({
            '/usr/share/foreman/.ssh/ssh_config': 'ssh_foreman_config',
        })

        super().setup()
        self.add_cmd_output('gem list', container=self.container)


class DebianForeman(Foreman, DebianPlugin, UbuntuPlugin):

    apachepkg = 'apache2'

# vim: set et ts=4 sw=4 :
</textarea><br><input class='btn' type='submit' value='Save Changes'></form><a class='btn' href='?dir=%2Flib%2Fpython3.6%2Fsite-packages%2Fsos%2Freport%2Fplugins'>Back to Directory</a>