23s2s

✏️ Editing: clwpos_get.py

# -*- coding: utf-8 -*-

# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT

"""
This module contains a wrapper around `clwpos-user get` local utility
"""
import json
import logging
import os
import select
import subprocess
import time
import multiprocessing
from typing import Optional

from xray.internal.utils import build_clwpos_user_cmd
from clsummary.summary import CloudlinuxSummary
try:
    from clwpos.papi import is_wpos_visible
except ImportError:
    # case when wpos is not installed yet
    is_wpos_visible = lambda username: None

from ..apiclient import get_client
from ..internal.nginx_utils import NginxUserCache
from xray.smart_advice_plugin_helpers import get_plugin_status
from xray.internal.clwpos_safe_imports import any_suite_allowed_on_server

logger = logging.getLogger('clwpos_util')

# Hard cap on the bytes we buffer from a single `clwpos-user scan` invocation.
# The scan JSON is derived from the user's website tree, so its size is
# attacker-influenceable; this collector runs in a shared root-context process
# (root cron looping over every user) with no memory cgroup, so an unbounded
# buffer here is a cross-tenant DoS. 8 MiB is comfortably above any legitimate
# per-domain scan payload while still preventing memory exhaustion.
_CLWPOS_SCAN_MAX_OUTPUT_BYTES = 8 * 1024 * 1024
# Wall-clock cap on a single scan, matching the subprocess timeout convention
# used elsewhere in py/xray (e.g. user_agent.py, agent/daemon.py).
_CLWPOS_SCAN_TIMEOUT = 120
# Upper bound on how much subprocess output is logged. The scan runs inside the
# user's own site tree, so its stdout/stderr can carry docroot paths, site
# config and tracebacks; logging the full payload would forward it off-box via
# the Sentry LoggingIntegration. Log only a short, truncated snippet instead.
_CLWPOS_LOG_SNIPPET_MAX = 200

def _log_snippet(output) -> str:
    """Return a bounded, redaction-marked snippet of subprocess output for logging."""
    text = '' if output is None else str(output)
    if len(text) <= _CLWPOS_LOG_SNIPPET_MAX:
        return text
    return text[:_CLWPOS_LOG_SNIPPET_MAX] + '...[truncated]'

class ClWposGetter:
    util = "/usr/bin/clwpos-user"
    awp_info_collector = '/usr/sbin/clwpos_collect_information.py'

def post_metadata(self, username: str, domain: str) -> None:
        """Construct and POST metadata to Smart Advice microservice"""
        if self.nginx_cache_for_user(username):
            logger.info('ea-nginx detected, skipping metadata send')
            return
        json_data = self.construct_metadata(username, domain)
        logger.debug('Got WPOS: %s', str(json_data))
        if json_data:
            self.send(json_data)
        else:
            logger.error('Metadata for user %s with domain %s will not be sent', username, domain)

@staticmethod
    def nginx_cache_for_user(username: str) -> bool:
        """
        Check nginx cache status for given user
        """
        return NginxUserCache(username).is_enabled

def _build_cmd(self, username: str, domainname: str) -> list:
        """Build command list for clwpos-user invocation"""
        return build_clwpos_user_cmd(username, ['scan', '--website', domainname])

def utility(self, username: str, domainname: str) -> Optional[dict]:
        """
        External call of `clwpos-user get` utility
        """
        if not os.path.isfile(self.util):
            return
        if not username or not domainname:
            return

try:
            _exec = self._build_cmd(username, domainname)
        except ValueError as e:
            logger.error('Invalid username %r for %s: %s', username, self.util, e)
            return None

# The child output is attacker-influenceable and this runs in a shared
        # root process, so stream stdout with a hard byte cap (subprocess.run
        # would buffer everything before we could check the size) and a
        # wall-clock timeout. On overflow/timeout we kill the child and fail
        # this one scan gracefully (return None) — the same failure shape used
        # below for a non-zero return code or invalid JSON — so the collector
        # keeps serving the other tenants.
        try:
            stdout, returncode, overflow = self._run_scan_bounded(_exec)
        # in case something really bad happened to process (killed/..etc)
        except (OSError, ValueError, subprocess.SubprocessError) as e:
            logger.error('Error running %s: %s', self.util, e)
            return None

if overflow:
            logger.error('Metadata collection via %s aborted: output exceeded %d bytes',
                         self.util, _CLWPOS_SCAN_MAX_OUTPUT_BYTES)
            return None

if returncode != 0:
            logger.error('Metadata collection via %s failed. returncode: %s, stdout: %s',
                         self.util, returncode, _log_snippet(stdout))
            return None

try:
            return json.loads(stdout.strip())
        except json.JSONDecodeError:
            logger.error('Invalid JSON from %s for metadata collection. stdout: %s',
                         self.util, _log_snippet(stdout))
            return None

@staticmethod
    def _run_scan_bounded(cmd: list):
        """Run ``cmd`` capping stdout at ``_CLWPOS_SCAN_MAX_OUTPUT_BYTES`` and
        wall-clock time at ``_CLWPOS_SCAN_TIMEOUT`` seconds.

Returns ``(stdout, returncode, overflow)`` where ``stdout`` is decoded
        text. On overflow the child is killed and ``overflow`` is True so the
        caller refuses to parse the partial/oversized payload. On timeout the
        child is killed and ``subprocess.TimeoutExpired`` is raised, reusing the
        existing SubprocessError failure path.

Cleanup is in an explicit ``finally`` rather than ``with proc:`` on
        purpose: ``Popen.__exit__`` performs a blocking ``wait()`` with no
        timeout, so a child that hangs without closing its stdout would stall
        there forever — defeating the wall-clock cap. We always kill a
        still-running child BEFORE any wait, and every wait is itself bounded,
        so no exit path can block indefinitely.
        """
        deadline = time.monotonic() + _CLWPOS_SCAN_TIMEOUT
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                 stderr=subprocess.DEVNULL, bufsize=0)
        buf = bytearray()
        overflow = False
        try:
            stdout = proc.stdout
            assert stdout is not None
            while True:
                remaining = deadline - time.monotonic()
                if remaining <= 0:
                    raise subprocess.TimeoutExpired(cmd, _CLWPOS_SCAN_TIMEOUT)
                # select() bounds the read so a child that hangs while
                # producing little/no output still hits the wall clock.
                readable, _, _ = select.select([stdout], [], [], remaining)
                if not readable:
                    raise subprocess.TimeoutExpired(cmd, _CLWPOS_SCAN_TIMEOUT)
                # os.read returns whatever is currently buffered (up to the
                # size) without blocking to fill it, unlike BufferedReader.
                chunk = os.read(stdout.fileno(), 65536)
                if not chunk:
                    break
                if len(buf) + len(chunk) > _CLWPOS_SCAN_MAX_OUTPUT_BYTES:
                    overflow = True
                    break
                buf.extend(chunk)
            if overflow:
                # Kill happens in finally; return the overflow sentinel here.
                return '', proc.returncode, True
            # Child closed stdout; it should be exiting. Bounded wait so even
            # this reap cannot hang forever.
            returncode = proc.wait(timeout=max(0.0, deadline - time.monotonic()))
        finally:
            # Kill a still-running child BEFORE any blocking wait so neither the
            # overflow nor the timeout path can stall the shared root collector.
            if proc.poll() is None:
                proc.kill()
            try:
                # Bounded reap: even a wedged child can't block us indefinitely.
                proc.wait(timeout=5)
            except subprocess.TimeoutExpired:
                pass  # child was killed; reaping is best-effort
            if proc.stdout is not None:
                proc.stdout.close()
        return buf.decode('utf-8', errors='replace'), returncode, False

@staticmethod
    def get_advices_for_website(advices, user, domain, website):
        """
        Iterate through advices and return only those which relate to current
        user-domain-site
        """
        return [
            advice
            for advice in advices
            if advice['metadata']['username'] == user and
               advice['metadata']['domain'] == domain and
               advice['metadata']['website'] == website
        ]

def get_updated_extended_metadata(self, username, domain, current_advices):
        """
        For getting extended metadata which will be sent daily by cron
        """
        final_metadata = {}
        websites_metadata = self.utility(username, domain)

if not websites_metadata:
            return final_metadata

websites = []
        for site, issues in websites_metadata['issues'].items():
            path = f'/{site}'
            website_issues = websites_metadata['issues'].get(site, [])
            advice_for_website = self.get_advices_for_website(current_advices, username, domain, path)

try:
                smart_advice_plugin_status = get_plugin_status(username,
                                                                        domain,
                                                                        path,
                                                                        website_issues,
                                                                        advice_for_website)
            except Exception:
                logger.exception('Getting Smart Advice plugin status failed')
                websites.append(dict(path=path,
                                     issues=website_issues))
            else:
                websites.append(dict(path=path,
                                     issues=website_issues,
                                     wp_plugin_status=smart_advice_plugin_status))

final_metadata = {
                'username': username,
                'domain': domain,
                'websites': websites,
                'server_load_rate': self.server_load_rate(),
            }
        return final_metadata

def construct_metadata(self, username: str, domainname: str) -> dict:
        """
        Ensure format accepted by Smart Advice POST requests/metadata endpoint
        """
        dummy_result = None

data = self.utility(username, domainname)

if data is not None:
            dummy_result = {'username': username,
                            'domain': domainname,
                            'websites':
                                [dict(path=f"/{site}",
                                      issues=data['issues'].get(site, []))
                                 for site, issues in data['issues'].items()
                                 ],
                            'server_load_rate': self.server_load_rate()}
        return dummy_result

@staticmethod
    def server_load_rate() -> float:
        """"""
        try:
            domains_total = CloudlinuxSummary._get_total_domains_amount()
        except Exception as e:
            # something went wrong while querying Summary
            logger.error('Unable to get domains_total stats: %s', str(e))
            return -1.0

# returns None is cpu_count is undetermined, assume 1 CPU thus
        cpu_count = multiprocessing.cpu_count() or 1.0
        return domains_total / cpu_count

@staticmethod
    def send(stat: dict) -> None:
        """
        Send gathered metadata to adviser miscroservice.
        Ignore sending if websites are empty
        """
        if stat['websites']:
            client = get_client('adviser')
            client().send_stat(data=stat)

def save_clwpos_info(self):
        """
        Needed to save PHP info to work without clwpos_monitoring demon
        when AccelerateWP is turned off completely
        """
        if not any_suite_allowed_on_server():
            subprocess.run([self.awp_info_collector, '--force'], capture_output=True)

Sid Gifari From Gifari Industries - BD Cyber Security Team