# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT """ This module contains X-Ray User Manager service implementation """ import collections import json import logging import queue import re import socket import struct import subprocess from threading import Lock, current_thread from clcommon.clwpos_lib import get_locale_from_envars from xray import gettext as _ from xray.agent.executor import BoundedThreadExecutor from xray.console_utils.run_user.runners import Runner, get_runner from xray.internal.constants import user_agent_log, user_agent_sock from xray.internal.exceptions import XRayError from xray.internal.user_plugin_utils import check_for_root, error_response, extract_creds, pack_response, unpack_request from xray.internal.utils import configure_logging, create_socket, read_sys_id logger = logging.getLogger('user_agent') _LOCALE_RE = re.compile(r'^[a-zA-Z]{1,5}(_[a-zA-Z]{2})?(\.[a-zA-Z0-9-]+)?$') _MAX_WORKERS = 50 _MAX_CONNECTIONS_PER_UID = 25 _uid_connections = collections.defaultdict(int) _uid_connections_lock = Lock() _MAX_REQUEST_SIZE = 1048576 _READ_CHUNK_SIZE = 4096 # How a bare JSON object may begin: optional whitespace, the opening brace, and # then only what JSON allows after it - whitespace, a key's quote or `}`. _BARE_JSON_HEAD_RE = re.compile(rb'[ \t\r\n]*\{[ \t\r\n]*(?:["}]|\Z)') def general_exec_error() -> tuple[bytes, bytes]: """ General format of message in case of errors during manager execution """ _err = _('X-Ray User Plugin failed to execute your request. Please, contact your server administrator') return error_response(_err).encode(), b'' def log_truncate(orig_msg: bytes) -> str: """ Cut data field from the original message, because it could be huge """ return re.sub('(?<="data": {).+(?=}, "result")', '...', orig_msg.decode()) def redact_cmd(cmd: list) -> list: """ Build a log-safe view of an argv list: keep the target binary and the command name, but mask the value of every ``--opt=value`` token so caller-supplied data (analytics_data, user_hash, emails, url, ...) is not written to the log verbatim. """ redacted = [] for token in cmd: if token.startswith('--') and '=' in token: opt = token.split('=', 1)[0] redacted.append(f'{opt}=') else: redacted.append(token) return redacted def duplicate_warning_cast(orig_msg: bytes) -> bytes: """ Extend warning 'Task is duplicated by URL' for end user """ def gen(m): """ Add more text for duplicate warning, leave others unchanged """ additional = b""". \ In case if you do not see running task for the same URL in your list of tasks below, \ contact your server administrator and ask him to check whether the requested URL \ is in the tasks list of X-Ray Admin Plugin or is scheduled for continuous tracing.""" warn = m.group(0) if warn == b'Task is duplicated by URL': return warn + additional return warn return re.sub(b'(?<="warning": ").+(?="})', gen, orig_msg) def execute_manager(command: dict, user: str, runner: Runner) -> tuple[bytes, bytes]: """ Trigger runner.target utility with requested parameters """ if runner.name == 'manager': try: command['system_id'] = read_sys_id() except XRayError: return general_exec_error() def runner_cast_opt(opt): """runner may define a special cast for each option""" return runner.option_cast[opt](opt) if opt in runner.option_cast else opt def hide_true_val(v): """Hide True value from cmd representation for bool-flag options""" return '' if v is True else f'={v}' api_version_key = 'api_version' locale_option = 'lang' # options that is not needed to be passed to utility skip_options = [api_version_key, locale_option] api_version = command.get(api_version_key) # could be passed from WordPress Plugin; guard against None to avoid # TypeError in subprocess.run() when LANG=None in the env dict. target_locale = command.get(locale_option, get_locale_from_envars()) if not isinstance(target_locale, str) or not _LOCALE_RE.match(target_locale): target_locale = get_locale_from_envars() or 'C.UTF-8' if api_version: casted_api_version = runner_cast_opt(api_version_key) cmd = [f'/usr/sbin/{runner.target}', f'--{casted_api_version}', api_version, command.pop('command')] else: cmd = [f'/usr/sbin/{runner.target}', command.pop('command')] # bool options are added only if they are True cmd.extend([f'--{runner_cast_opt(k)}{hide_true_val(v)}' for k, v in command.items() if v and k not in skip_options]) with_env = {'XRAYEXEC_UID': user, 'LANG': target_locale} logger.info('Going to execute: %s, with environment: %s', redact_cmd(cmd), str(with_env)) try: p = subprocess.run(cmd, capture_output=True, env=with_env, timeout=120, check=False) except (OSError, ValueError, TypeError, subprocess.SubprocessError): return general_exec_error() byte_out, byte_err = p.stdout.strip(), p.stderr.strip() # Log only the size of proxied output; the body carries caller-correlated # data and must not be written to the log verbatim. if byte_out: logger.info('[%s] Proxied command stdout: %d bytes', current_thread().name, len(byte_out)) if byte_err: logger.info('[%s] Proxied command stderr: %d bytes', current_thread().name, len(byte_err)) return duplicate_warning_cast(byte_out), byte_err def _recv_exactly(connection: socket.socket, size: int) -> bytes: """ Read exactly `size` bytes, or fewer if the peer stops sending before that The bytes are gathered in a bytearray: appending to an immutable `bytes` copies everything read so far on every recv(), which makes a request delivered in tiny pieces cost quadratic CPU. """ data = bytearray() while len(data) < size: chunk = connection.recv(size - len(data)) if not chunk: break data += chunk return bytes(data) def _scan_json_object(chunk: bytes, depth: int, in_string: bool, escaped: bool) -> tuple[int, bool, bool, int]: """ Advance a brace-depth scan over `chunk`, resuming from the given state Returns the state after the chunk plus the offset just past the top-level object's closing brace, or -1 if it has not closed yet. Braces are scanned as raw bytes on purpose: `{`, `}`, `"` and `\\` cannot appear inside a UTF-8 multi-byte sequence, so no decoding is needed to find them. """ for index, byte in enumerate(chunk): if escaped: escaped = False elif in_string: if byte == 0x5C: # backslash escaped = True elif byte == 0x22: # double quote in_string = False elif byte == 0x22: in_string = True elif byte == 0x7B: # { depth += 1 elif byte == 0x7D: # } depth -= 1 if depth <= 0: return depth, in_string, escaped, index + 1 return depth, in_string, escaped, -1 def _recv_unframed(connection: socket.socket, head: bytes) -> bytes: """ Read a request that carries no length prefix, `head` being its first bytes Such a client sends the payload alone and then waits for the reply without closing its side, so there is no EOF to read up to: the request ends where its JSON object ends, and the brace depth is what finds that. Each byte is looked at once and copied once - only the newly received chunk is scanned, and the request is gathered in a bytearray - so a caller cannot make the reader do more work by pacing the bytes; validating the result stays `unpack_request`'s job. """ data = bytearray(head) chunk = head depth, in_string, escaped = 0, False, False while True: depth, in_string, escaped, end = _scan_json_object(chunk, depth, in_string, escaped) if end >= 0: return bytes(data[: len(data) - len(chunk) + end]) if len(data) >= _MAX_REQUEST_SIZE: logger.warning('Unframed request exceeds %i bytes, rejecting', _MAX_REQUEST_SIZE) return b'' # Never read past the cap: an object that closes beyond it must not be # accepted just because its closing brace shared a chunk with byte 1 MiB. chunk = connection.recv(min(_READ_CHUNK_SIZE, _MAX_REQUEST_SIZE - len(data))) if not chunk: return bytes(data) data += chunk def read_request(connection: socket.socket) -> bytes: """ Read one request, accepting both framings clients send Current clients prefix the payload with its 4-byte big-endian length (`pack_request`, `ApiSocket::send`). AccelerateWP plugin builds up to 3.23-1.1-46 send bare JSON: they predate the prefix and cannot fetch their own fix through the socket they are unable to talk to, so the agent keeps answering them (CLOS-7023). Telling the two apart is exact, not heuristic: a length within `_MAX_REQUEST_SIZE` leaves 0x00 in the first byte, while JSON starts with a printable ASCII byte, so the first four bytes of a bare payload always decode to a length far above the cap (`{"ru` -> 2065855093). """ head = _recv_exactly(connection, 4) if len(head) < 4: return b'' msglen = struct.unpack('>I', head)[0] if 0 < msglen <= _MAX_REQUEST_SIZE: body = _recv_exactly(connection, msglen) return body if len(body) == msglen else b'' if not _BARE_JSON_HEAD_RE.match(head): # Neither a usable length nor the beginning of a JSON object. The # check covers what follows the brace too, so an over-cap length that # merely starts with 0x7B is dropped here instead of waiting out the # timeout for a closing brace that never comes. return b'' logger.info('Request carries no length prefix, reading it as bare JSON') return _recv_unframed(connection, head) def handle(connection: socket.socket) -> None: """ Handle incoming connection :param connection: socket object usable to send and receive data on the connection """ root_error = json.dumps({'result': _('Commands from root are not accepted')}, ensure_ascii=False) with connection: connection.settimeout(5.0) _pid, _uid, _gid = extract_creds(connection) if check_for_root(_uid): connection.sendall(pack_response(root_error.encode())) return with _uid_connections_lock: if _uid_connections.get(_uid, 0) >= _MAX_CONNECTIONS_PER_UID: logger.warning('Too many concurrent connections from uid=%s, rejecting', _uid) return _uid_connections[_uid] += 1 try: data = read_request(connection) if not data: return try: args = unpack_request(data) except (json.JSONDecodeError, UnicodeDecodeError) as e: connection.sendall(pack_response(json.dumps({'result': str(e)}, ensure_ascii=False).encode())) return try: runner = get_runner(args.pop('runner')) except XRayError as e: connection.sendall(pack_response(str(e).encode())) return try: runner.validator(args) except SystemExit as e: connection.sendall(pack_response(str(e).encode())) return _out, _err = execute_manager(args, str(_uid), runner) connection.sendall(pack_response(_out or _err)) except (TimeoutError, OSError): return finally: with _uid_connections_lock: _uid_connections[_uid] -= 1 if _uid_connections[_uid] <= 0: del _uid_connections[_uid] def run() -> None: """ Run listening service """ configure_logging(user_agent_log) with create_socket(user_agent_sock) as s, BoundedThreadExecutor(max_workers=_MAX_WORKERS, maxqueuesize=0) as pool: while True: conn, _ = s.accept() try: pool.submit(handle, conn) except queue.Full: logger.error('Connection rejected: thread pool is full') conn.close()