azurelinuxagent/ga/memorycontroller.py

# Copyright 2018 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.6+ and Openssl 1.0+

import errno
import os
import re

from azurelinuxagent.common import logger
from azurelinuxagent.common.exception import CGroupsException
from azurelinuxagent.common.future import ustr
from azurelinuxagent.ga.cgroupcontroller import _CgroupController, CounterNotFound, MetricValue, MetricsCategory, \
    MetricsCounter, _REPORT_EVERY_HOUR


class _MemoryController(_CgroupController):
    def __init__(self, name, cgroup_path):
        super(_MemoryController, self).__init__(name, cgroup_path)
        self._counter_not_found_error_count = 0

    def _get_memory_stat_counter(self, counter_name):
        """
        Gets the value for the provided counter in memory.stat
        """
        try:
            with open(os.path.join(self.path, 'memory.stat')) as memory_stat:
                #
                # Sample file v1:
                #   # cat memory.stat
                #   cache 0
                #   rss 0
                #   rss_huge 0
                #   shmem 0
                #   mapped_file 0
                #   dirty 0
                #   writeback 0
                #   swap 0
                #   ...
                #
                # Sample file v2
                #   # cat memory.stat
                #   anon 0
                #   file 147140608
                #   kernel 1421312
                #   kernel_stack 0
                #   pagetables 0
                #   sec_pagetables 0
                #   percpu 130752
                #   sock 0
                #   ...
                #
                for line in memory_stat:
                    re_memory_counter = r'{0}\s+(\d+)'.format(counter_name)
                    match = re.match(re_memory_counter, line)
                    if match is not None:
                        return int(match.groups()[0])
        except (IOError, OSError) as e:
            if e.errno == errno.ENOENT:
                raise
            raise CGroupsException("Failed to read memory.stat: {0}".format(ustr(e)))
        except Exception as e:
            raise CGroupsException("Failed to read memory.stat: {0}".format(ustr(e)))

        raise CounterNotFound("Cannot find counter: {0}".format(counter_name))

    def get_memory_usage(self):
        """
        Collects anon and cache usage for the cgroup and returns as a tuple
        Returns anon and cache memory usage for the cgroup as a tuple ->  (anon, cache)

        :return: Anon and cache memory usage in bytes
        :rtype: tuple[int, int]
        """
        raise NotImplementedError()

    def try_swap_memory_usage(self):
        """
        Collects swap usage for the cgroup

        :return: Memory usage in bytes
        :rtype: int
        """
        raise NotImplementedError()

    def get_max_memory_usage(self):
        """
        Collect max memory usage for the cgroup.

        :return: Memory usage in bytes
        :rtype: int
        """
        raise NotImplementedError()

    def get_tracked_metrics(self, **_):
        # The log collector monitor tracks anon and cache memory separately.
        anon_mem_usage, cache_mem_usage = self.get_memory_usage()
        total_mem_usage = anon_mem_usage + cache_mem_usage
        return [
            MetricValue(MetricsCategory.MEMORY_CATEGORY, MetricsCounter.TOTAL_MEM_USAGE, self.name, total_mem_usage),
            MetricValue(MetricsCategory.MEMORY_CATEGORY, MetricsCounter.ANON_MEM_USAGE, self.name, anon_mem_usage),
            MetricValue(MetricsCategory.MEMORY_CATEGORY, MetricsCounter.CACHE_MEM_USAGE, self.name, cache_mem_usage),
            MetricValue(MetricsCategory.MEMORY_CATEGORY, MetricsCounter.MAX_MEM_USAGE, self.name,
                        self.get_max_memory_usage(), _REPORT_EVERY_HOUR),
            MetricValue(MetricsCategory.MEMORY_CATEGORY, MetricsCounter.SWAP_MEM_USAGE, self.name,
                        self.try_swap_memory_usage(), _REPORT_EVERY_HOUR)
        ]

    def get_unit_properties(self):
        return["MemoryAccounting"]

    def get_controller_type(self):
        return "memory"


class MemoryControllerV1(_MemoryController):
    def get_memory_usage(self):
        # In v1, anon memory is reported in the 'rss' counter
        return self._get_memory_stat_counter("rss"), self._get_memory_stat_counter("cache")

    def try_swap_memory_usage(self):
        # In v1, swap memory should be collected from memory.stat, because memory.memsw.usage_in_bytes reports total Memory+SWAP.
        try:
            return self._get_memory_stat_counter("swap")
        except CounterNotFound as e:
            if self._counter_not_found_error_count < 1:
                logger.periodic_info(logger.EVERY_HALF_HOUR,
                                     '{0} from "memory.stat" file in the cgroup: {1}---[Note: This log for informational purpose only and can be ignored]'.format(ustr(e), self.path))
                self._counter_not_found_error_count += 1
            return 0

    def get_max_memory_usage(self):
        # In v1, max memory usage is reported in memory.max_usage_in_bytes
        usage = 0
        try:
            usage = int(self._get_parameters('memory.max_usage_in_bytes', first_line_only=True))
        except Exception as e:
            if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT:  # pylint: disable=E1101
                raise
            raise CGroupsException("Exception while attempting to read {0}".format("memory.max_usage_in_bytes"), e)

        return usage


class MemoryControllerV2(_MemoryController):
    def get_memory_usage(self):
        # In v2, cache memory is reported in the 'file' counter
        return self._get_memory_stat_counter("anon"), self._get_memory_stat_counter("file")

    def get_memory_throttled_events(self):
        """
        Returns the number of times processes of the cgroup are throttled and routed to perform memory recliam because
        the high memory boundary was exceeded.

        :return: Number of memory throttling events for the cgroup
        :rtype: int
        """
        try:
            with open(os.path.join(self.path, 'memory.events')) as memory_events:
                #
                # Sample file:
                #   # cat memory.events
                #   low 0
                #   high 0
                #   max 0
                #   oom 0
                #   oom_kill 0
                #   oom_group_kill 0
                #
                for line in memory_events:
                    match = re.match(r'high\s+(\d+)', line)
                    if match is not None:
                        return int(match.groups()[0])
        except (IOError, OSError) as e:
            if e.errno == errno.ENOENT:
                raise
            raise CGroupsException("Failed to read memory.events: {0}".format(ustr(e)))
        except Exception as e:
            raise CGroupsException("Failed to read memory.events: {0}".format(ustr(e)))

        raise CounterNotFound("Cannot find memory.events counter: high")

    def try_swap_memory_usage(self):
        # In v2, swap memory is reported in memory.swap.current
        usage = 0
        try:
            usage = int(self._get_parameters('memory.swap.current', first_line_only=True))
        except Exception as e:
            if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT:  # pylint: disable=E1101
                raise
            raise CGroupsException("Exception while attempting to read {0}".format("memory.swap.current"), e)

        return usage

    def get_max_memory_usage(self):
        # In v2, max memory usage is reported in memory.peak
        usage = 0
        try:
            usage = int(self._get_parameters('memory.peak', first_line_only=True))
        except Exception as e:
            if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT:  # pylint: disable=E1101
                raise
            raise CGroupsException("Exception while attempting to read {0}".format("memory.peak"), e)

        return usage

    def get_tracked_metrics(self, **_):
        metrics = super(MemoryControllerV2, self).get_tracked_metrics()
        throttled_value = MetricValue(MetricsCategory.MEMORY_CATEGORY, MetricsCounter.MEM_THROTTLED, self.name,
                                      self.get_memory_throttled_events())
        metrics.append(throttled_value)
        return metrics