# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2026 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT """Per-user domain config persistence (~/.lve/domains.json).""" import contextlib from functools import cache import errno import fcntl import json import logging import os import stat import tempfile import time from dataclasses import dataclass, field from typing import List, Optional from secureio import disable_quota from clcommon.clpwd import ClPwd, drop_privileges from clcommon.cpapi import userdomains, docroot as cpapi_docroot from clcommon.cpapi.cpapiexceptions import CPAPIException from . import domain_lve_sentry_extra from . import id_registry from .exceptions import ErrorCode, LvdError log = logging.getLogger(__name__) LVD_CONFIG_DIR = '.lve' LVD_CONFIG_FILE = 'domains.json' LVD_CONFIG_VERSION = 1 # CLOS-4594/F-33: the per-user .domains.lock lives inside (and is owned by) the tenant's # home directory, so an unprivileged user can hold it. A root-run operation # (`lvectl apply all`, the panel-cron sync) must never block on it forever. # Acquire the lock non-blockingly with a generous bounded budget; on timeout # raise LvdError so the caller skips that user instead of hanging the batch. # The budget is generous relative to a legitimate lvdctl op (a fast local read # / atomic write), so a real holder is essentially never false-skipped. _LOCK_TIMEOUT = 10.0 _LOCK_RETRY_INTERVAL = 0.1 MAX_CONFIG_BYTES = 1024 * 1024 # 1 MiB; legit domains.json is a few KB. Bounds a tenant OOM DoS via an oversized file. MAX_DOMAINS = 5000 # real users have tens-hundreds of domains; cap entry materialization. LIMIT_FIELDS = ('cpu', 'pmem', 'io', 'nproc', 'iops', 'ep', 'vmem') # Limit values reach the kernel through signed 32-bit C struct members (T_INT: # ls_cpu, ls_memory_phy, ... in pylve.c). A finite int outside this range # passes int() but raises an uncaught OverflowError on struct assignment in # lveapi.py during 'lvectl apply all', aborting the root batch. Bound to the # T_INT range so any value the C layer can accept passes and only values that # would OverflowError are rejected as malformed. _INT32_MIN = -2 ** 31 # -2147483648 _INT32_MAX = 2 ** 31 - 1 # 2147483647 # The memory fields are the exception, and bounding them like the rest was # CLOS-7494: they are stored here in BYTES and divided into pages before they # reach any of those struct members, so what has to fit in a T_INT is their PAGE # count, not their byte count. A byte bound made every memory limit from 2 GiB up # unreadable -- an ordinary figure, and what a 75% Resource Share of a 4 GiB # account comes to -- although the kernel accepts it happily. # The conversion sites are commands._call_limits_helper and # lveapi.apply_domain_lve_limits; both divide by this, and the bound below is # only correct for as long as they do. PAGE_SIZE = 4096 _BYTES_FIELDS = ('pmem', 'vmem') def _limit_bounds(field_name): """Return the inclusive (min, max) a stored *field_name* value may take.""" if field_name in _BYTES_FIELDS: return _INT32_MIN * PAGE_SIZE, _INT32_MAX * PAGE_SIZE return _INT32_MIN, _INT32_MAX def _validate_limit(field_name, value): """Coerce *value* to int and bound it to what *field_name* can carry. The single definition of "a usable limit value", shared by the reader and the writer. They disagreed once, and a value that could be written but not read back made the whole account's configuration unreadable (CLOS-7494). """ try: iv = int(value) except (TypeError, ValueError, OverflowError) as e: # OverflowError: int(float('inf')) -- json.load accepts the bare token # Infinity/-Infinity, so a tenant can smuggle a non-finite float into # a limit. # Name the type, never the value: the reader logs this message at # WARNING, and the value is whatever the tenant wrote into the file # (CLOS-7520). `from None` keeps int()'s own message, which quotes # the value, out of any traceback that is logged later. raise LvdError( f"limit '{field_name}' must be an integer, " f"got {type(value).__name__}", code=ErrorCode.INVALID_VALUE) from None low, high = _limit_bounds(field_name) if not low <= iv <= high: raise LvdError( f"limit '{field_name}' out of range [{low}, {high}], got {iv}", code=ErrorCode.INVALID_VALUE) return iv def _reject_sub_page(field_name, value): """Refuse a memory limit smaller than one page, other than 0. The kernel takes memory limits in pages, so a value below PAGE_SIZE reaches it as 0 -- unlimited -- while the caller is told it was set (CLOS-7520). Checked on write only: a stored record with such a value still reads back, so the account's other records stay reachable. """ if field_name in _BYTES_FIELDS and 0 < value < PAGE_SIZE: raise LvdError( f"limit '{field_name}' must be 0 (unlimited) or at least " f"{PAGE_SIZE} bytes, got {value}", code=ErrorCode.INVALID_VALUE) @dataclass class DomainLimits: cpu: Optional[int] = None pmem: Optional[int] = None io: Optional[int] = None nproc: Optional[int] = None iops: Optional[int] = None ep: Optional[int] = None vmem: Optional[int] = None def to_dict(self): """Return only fields that are set (non-None).""" return {k: v for k, v in self.__dict__.items() if v is not None} @classmethod def from_dict(cls, data): if not data: return cls() if not isinstance(data, dict): raise LvdError(f"limits must be an object, got {type(data).__name__}") # A value beyond what the field can carry (e.g. 9999999999, or 1e308 # coerced to a 309-digit int) passes int() but raises an uncaught # OverflowError when assigned into the C T_INT struct member # downstream, aborting the root batch. Reject here -> LvdError -> # _read returns the safe default. values = {k: _validate_limit(k, v) for k, v in data.items() if k in LIMIT_FIELDS} return cls(**values) def update(self, **kwargs): """Apply *kwargs*, bounded as the reader bounds them. Memory values below one page are also refused here (see _reject_sub_page); the reader keeps accepting them. Every value is checked before any is assigned, so a rejected call leaves the entry as it was and the caller's save never happens. Writing a value the reader would refuse is what turned one bad `limits set` into an account whose every domain record was lost (CLOS-7494). """ validated = {k: _validate_limit(k, v) for k, v in kwargs.items() if k in LIMIT_FIELDS and v is not None} for k, v in validated.items(): _reject_sub_page(k, v) for k, v in validated.items(): setattr(self, k, v) def __bool__(self): return any(v is not None for v in self.__dict__.values()) @dataclass class DomainEntry: name: str = '' limits: DomainLimits = field(default_factory=DomainLimits) def to_dict(self): return { 'name': self.name, 'limits': self.limits.to_dict(), } @classmethod def from_dict(cls, data): if not isinstance(data, dict): raise LvdError(f"domain entry must be an object, got {type(data).__name__}") # 'name' is tenant-controlled and flows into hashable contexts # downstream: docroot_by_domain.get(d.name) (lveapi.py) and # find_domain/remove_domain string comparisons. A non-str (list/dict # -> unhashable; int/float/bool -> mismatched key type) would raise an # uncaught TypeError outside this module and abort the root # 'lvectl apply all' batch for every other tenant. Reject it here so # the only outcome of a malformed file is the safe default (via _read). name = data.get('name', '') if not isinstance(name, str): raise LvdError(f"domain 'name' must be a string, got {type(name).__name__}") return cls( name=name, limits=DomainLimits.from_dict(data.get('limits')), ) @dataclass class LvdConfig: version: int = LVD_CONFIG_VERSION domains: List[DomainEntry] = field(default_factory=list) _lve_id: Optional[int] = field(default=None, repr=False, compare=False) # Set when an existing file was present but could not be used. Not part of # the value: two configs with the same domains stay equal. _degraded: bool = field(default=False, repr=False, compare=False) @property def unreadable(self): """True when this config stands in for a file that could not be read. It is empty like a fresh config, but the two mean opposite things: a fresh config says the account has no records, this one says its records could not be reached. Anything that would present it as the account's configuration, or write it back, has to tell them apart (CLOS-7494). """ return self._degraded def find_domain(self, name): """Find domain entry by name. Returns None if not found.""" for d in self.domains: if d.name == name: return d return None def remove_domain(self, name): """Remove domain entry by name.""" self.domains = [d for d in self.domains if d.name != name] def add_domain_by_docroot(self, docroot): """Resolve domain name from *docroot* and add it to the config. """ try: domain_name = _resolve_domain_for_docroot(self._lve_id, docroot) if domain_name and self.find_domain(domain_name) is None: self.domains.append(DomainEntry(name=domain_name)) self.save() except LvdError as exc: # ERROR: the domain is never recorded, so it silently carries no # per-domain limits and nothing later notices the gap. # Fingerprinted on the call site plus the exception type (a bounded # set), never on the uid or docroot -- raven groups on the # formatted message, which would otherwise be one issue per account. log.error("add_domain_by_docroot: uid=%s docroot=%s", self._lve_id, docroot, exc_info=True, extra=domain_lve_sentry_extra('add-domain-by-docroot', type(exc).__name__)) def remove_domain_by_docroot(self, docroot): """Resolve domain name from *docroot* and remove it from the config. """ try: domain_name = _resolve_domain_for_docroot(self._lve_id, docroot) if domain_name and self.find_domain(domain_name) is not None: self.remove_domain(domain_name) self.save() except LvdError as exc: # ERROR: the entry outlives the domain, so its stored limits stay # in domains.json and keep being applied to a docroot that moved on. log.error("remove_domain_by_docroot: uid=%s docroot=%s", self._lve_id, docroot, exc_info=True, extra=domain_lve_sentry_extra('remove-domain-by-docroot', type(exc).__name__)) def to_dict(self): return { 'version': self.version, 'domains': [d.to_dict() for d in self.domains], } @classmethod def from_dict(cls, data, lve_id=None): # A shape we never write (every to_dict carries both keys), so it is a # file we cannot interpret rather than an account with no records -- # and the two must not look alike: returning an empty config here let a # top-level array of domain entries read back as "no limits" and be # overwritten by the next save (CLOS-7494). Raising hands it to _read, # which degrades it to a config that refuses to be saved over. if not isinstance(data, dict): raise LvdError( f"config must be an object, got {type(data).__name__}") if 'domains' not in data: raise LvdError("config has no 'domains' key") domains = data['domains'] if not isinstance(domains, list): raise LvdError(f"'domains' must be a list, got {type(domains).__name__}") if len(domains) > MAX_DOMAINS: log.warning("domains.json for lve_id %s has %d domains; capping at %d", lve_id, len(domains), MAX_DOMAINS) domains = domains[:MAX_DOMAINS] return cls( version=data.get('version', LVD_CONFIG_VERSION), domains=[DomainEntry.from_dict(d) for d in domains], _lve_id=lve_id, ) # --- Persistence --- @classmethod def load(cls, lve_id, lock_timeout=None): """Load domains.json for a user (acquires flock for the read). *lock_timeout* overrides _LOCK_TIMEOUT for this call only. See _flock. """ with cls._flock(lve_id, timeout=lock_timeout): return cls._read(lve_id) def save(self, lock_timeout=None): """Write domains.json for this user (acquires flock for the write). *lock_timeout* overrides _LOCK_TIMEOUT for this call only. See _flock. """ if self._lve_id is None: raise LvdError("cannot save: config has no associated lve_id") if self._degraded: # The single guard behind every read-modify-write path in this # module and in lvectllib: _read fails safe to an EMPTY config so a # root batch can carry on for the other tenants, and writing that # back is how one unreadable record used to delete all the others # (CLOS-7494). The file is the account's own, so the remedy is too. raise LvdError( f"refusing to overwrite {config_path(self._lve_id)}: its " f"current contents could not be read, so this write would " f"discard every record in it. Repair or remove the file first", code=ErrorCode.CONFIG_UNREADABLE) with self._flock(self._lve_id, timeout=lock_timeout): self._write() # --- Internal --- @staticmethod @contextlib.contextmanager def _flock(lve_id, timeout=None): """Exclusive flock on a per-user sidecar file. *timeout* defaults to _LOCK_TIMEOUT, which is sized for operations where skipping a tenant has a cost (`lvectl apply all` leaving its limits stale). A caller that only *reads* to count, and would rather skip this account than spend the budget, passes a shorter one: at the 10s default a handful of tenants holding their own lock can consume a whole batch's time budget and starve every account behind them (CLOS-7026). """ budget = _LOCK_TIMEOUT if timeout is None else timeout with user_context(lve_id): config_dir = ensure_config_dir(lve_id) lock_path = os.path.join(config_dir, '.domains.lock') fd = os.open(lock_path, os.O_CREAT | os.O_RDWR, 0o600) try: deadline = time.monotonic() + budget while True: try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) break except OSError as exc: if exc.errno not in (errno.EAGAIN, errno.EWOULDBLOCK): raise if time.monotonic() >= deadline: raise LvdError( f"timed out after {budget}s acquiring " f"domain lock {lock_path}; another process holds it", code=ErrorCode.BACKEND_UNAVAILABLE, ) from exc time.sleep(_LOCK_RETRY_INTERVAL) yield finally: fcntl.flock(fd, fcntl.LOCK_UN) os.close(fd) @classmethod def _unreadable(cls, lve_id): """The safe default for a file that exists but could not be used. Empty so a root batch carries on for every other tenant, but flagged so it is never mistaken for "this account has no records" -- see LvdConfig.unreadable and save(). """ return cls(_lve_id=lve_id, _degraded=True) @classmethod def _read(cls, lve_id): path = config_path(lve_id) if not os.path.exists(path): # No file is not a failed read: a first save must be able to # create one. return cls(_lve_id=lve_id) try: # domains.json is user-writable; open defensively so a symlink or # special file (e.g. a FIFO) planted by the user cannot redirect the # read or block the privileged apply/sync. O_NOFOLLOW rejects symlinks # (ELOOP); O_NONBLOCK ensures open() does not block on a FIFO so the # fstat regular-file check can reject it (O_NONBLOCK is a no-op for # reads on a regular file). fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) try: st = os.fstat(fd) if not stat.S_ISREG(st.st_mode): log.warning("ignoring non-regular config %s", path) return cls._unreadable(lve_id) with os.fdopen(fd, 'rb') as f: fd = -1 # fdopen now owns the descriptor # st.st_size is a TOCTOU snapshot: the user can grow the file # in place after fstat, and a read-to-EOF (json.load) would # follow it. Bound the bytes actually consumed so the parse # cannot be driven to OOM regardless of post-fstat growth. raw = f.read(MAX_CONFIG_BYTES + 1) if len(raw) > MAX_CONFIG_BYTES: log.warning("ignoring oversized config %s (> %d bytes)", path, MAX_CONFIG_BYTES) return cls._unreadable(lve_id) data = json.loads(raw) finally: if fd >= 0: os.close(fd) except (ValueError, OSError, RecursionError) as e: # ValueError subsumes json.JSONDecodeError and UnicodeDecodeError, so # malformed JSON or non-UTF-8 bytes in the user-controlled file fail # safe to an empty config instead of aborting the privileged apply. log.warning("failed to read config %s: %s", path, e) return cls._unreadable(lve_id) try: return cls.from_dict(data, lve_id=lve_id) except (LvdError, TypeError, ValueError, AttributeError, KeyError) as e: # LvdError messages from from_dict name fields and types, not the # tenant's values; any other exception's text is built by Python # and may quote the file, so above DEBUG it is reduced to its type # (CLOS-7520). reason = str(e) if isinstance(e, LvdError) else type(e).__name__ log.warning("malformed config %s, ignoring: %s", path, reason) log.debug("malformed config %s: %r", path, e) return cls._unreadable(lve_id) def _write(self): config_dir = ensure_config_dir(self._lve_id) path = config_path(self._lve_id) content = json.dumps(self.to_dict(), indent=2) + '\n' try: _write_via_tmp(config_dir, path, content) except IOError as e: raise LvdError(f"failed to write config {path}: {e}") from e @cache def _clpwd(): """One ClPwd per process, not one per lookup (CLOS-6868). ClPwd caches the passwd database per INSTANCE, so a fresh object means a full pwd.getpwall(). get_homedir/get_username are called three times per isolated account during `lvectl apply all` — 600 getpwall() calls over ~3,900 entries at 200 isolated accounts, ~11,600 at 3,868. That was the largest single term in the apply-all profile. Safe to hold for the life of a short-lived CLI. A long-running caller that creates or removes accounts must call invalidate_pwd_cache(). """ return ClPwd(min_uid=0) def invalidate_pwd_cache(): """Drop the cached passwd database after accounts are added or removed. Named wrapper rather than exposing _clpwd.cache_clear() directly: callers need to know *when* to drop it, and the name says so. """ _clpwd.cache_clear() def get_homedir(uid): """Get home directory for a uid.""" try: return _clpwd().get_pw_by_uid(uid)[0].pw_dir except ClPwd.NoSuchUserException as exc: raise LvdError(f"user with uid {uid} not found") from exc def get_uid_by_username(username): """Resolve username to uid.""" try: return _clpwd().get_uid(username) except ClPwd.NoSuchUserException as exc: raise LvdError(f"user '{username}' not found") from exc def get_username(lve_id): """Resolve username from uid.""" try: return _clpwd().get_names(lve_id)[0] except (ClPwd.NoSuchUserException, IndexError) as exc: raise LvdError(f"no user found for uid {lve_id}") from exc def resolve_lve_id(lve_id=None, username=None): """ Resolve lve_id from either --lve-id or --username. If neither, use effective UID (end-user mode). """ if lve_id is not None: return int(lve_id) if username is not None: return get_uid_by_username(username) uid = os.geteuid() if uid == 0: raise LvdError("root must specify --lve-id or --username") return uid def config_path(lve_id): """Return path to the user's domains.json config.""" homedir = get_homedir(lve_id) return os.path.join(homedir, LVD_CONFIG_DIR, LVD_CONFIG_FILE) def ensure_config_dir(lve_id): """Create ~/.lve/ directory with proper permissions if it doesn't exist. Caller must ensure privileges are already dropped to the target user.""" homedir = get_homedir(lve_id) config_dir = os.path.join(homedir, LVD_CONFIG_DIR) if not os.path.isdir(config_dir): os.makedirs(config_dir, mode=0o700, exist_ok=True) return config_dir def _write_via_tmp(directory, filename, content): """Write content to a file atomically via a temporary file.""" temp_path = None try: with tempfile.NamedTemporaryFile('w', dir=directory, delete=False) as tmp: temp_path = tmp.name tmp.write(content) tmp.flush() os.fsync(tmp.fileno()) os.replace(temp_path, filename) finally: if temp_path and os.path.exists(temp_path): try: os.remove(temp_path) except OSError: pass @contextlib.contextmanager def user_context(lve_id): """Drop privileges to the target user if running as root. CLOS-4370: when invoked as root we pair drop_privileges with secureio.disable_quota — creating ~/.lve/ and the lockfile / writing domains.json must succeed even when the user is over disk quota, otherwise OSError(EDQUOT) inside this block aborts batch operations like `lvectl apply all` and leaves the rest of the users with stale limits. disable_quota raises CAP_SYS_RESOURCE in the effective set """ if os.geteuid() == 0: username = get_username(lve_id) with drop_privileges(username), disable_quota(): yield else: yield @cache def _isolated_lve_ids() -> tuple: """Cached scan of the LVD id-registry directory (CLOS-6868). A tuple, not a list, so the cached value cannot be mutated by a caller. `lvectl apply all` calls this roughly twice per account — 7,739 directory scans on a 3,868-account host — for a directory that cannot change while the command runs. """ ids_dir = id_registry.LVD_IDS_DIR if not os.path.isdir(ids_dir): return () result = [] for name in os.listdir(ids_dir): try: result.append(int(name)) except ValueError: continue return tuple(result) def find_all_lve_ids_with_config(): """Return list of LVE IDs (UIDs) that have domain isolation configured. Scans the id_registry directory for per-user registry files. Users only appear here after ``lvdctl set`` or ``lvdctl apply`` has been called at least once for one of their domains. Returns a fresh list each call, as before, so existing callers may mutate the result without disturbing the cache behind it. """ return list(_isolated_lve_ids()) def invalidate_isolated_ids_cache(): """Drop the cached isolated-uid list after a domain is enabled or disabled.""" _isolated_lve_ids.cache_clear() def load_config(lve_id): """Load the domain isolation config for a given user.""" return LvdConfig.load(lve_id) def resolve_docroot(domain_name): """Resolve domain name to document root path via panel API.""" try: return cpapi_docroot(domain_name)[0] except Exception as exc: raise LvdError(f"failed to resolve docroot for domain '{domain_name}'", code=ErrorCode.BACKEND_UNAVAILABLE) from exc def _resolve_domain_for_docroot(uid, docroot): """Return domain name whose document root matches *docroot*, or None.""" try: username = get_username(uid) for domain_name, dr in (userdomains(username) or []): if dr == docroot: return domain_name except (LvdError, CPAPIException, OSError) as exc: # ERROR: the panel could not be asked, so both callers above fall # through to a silent no-op -- the add/remove they were asked to make # never happens and returning None is indistinguishable from "no such # domain". A panel API that fails here fails for every docroot. log.error("Cannot resolve domain for docroot: uid=%s docroot=%s", uid, docroot, exc_info=True, extra=domain_lve_sentry_extra('resolve-domain-for-docroot', type(exc).__name__)) return None