From 4a4eb13e7a0c85563b5283d4c3af2093893948a9 Mon Sep 17 00:00:00 2001
From: Ryota Ozaki <ozaki.ryota@gmail.com>
Date: Thu, 24 Jun 2010 01:00:56 +0900
Subject: [PATCH] cgroup: Enable memory.use_hierarchy of cgroup for domain

Through conversation with Kumar L Srikanth-B22348, I found
that the function of getting memory usage (e.g., virsh dominfo)
doesn't work for lxc with ns subsystem of cgroup enabled.

This is because of features of ns and memory subsystems.
Ns creates child cgroup on every process fork and as a result
processes in a container are not assigned in a cgroup for
domain (e.g., libvirt/lxc/test1/). For example, libvirt_lxc
and init (or somewhat specified in XML) are assigned into
libvirt/lxc/test1/8839/ and libvirt/lxc/test1/8839/8849/,
respectively. On the other hand, memory subsystem accounts
memory usage within a group of processes by default, i.e.,
it does not take any child (and descendant) groups into
account. With the two features, virsh dominfo which just
checks memory usage of a cgroup for domain always returns
zero because the cgroup has no process.

Setting memory.use_hierarchy of a group allows to account
(and limit) memory usage of every descendant groups of the group.
By setting it of a cgroup for domain, we can get proper memory
usage of lxc with ns subsystem enabled. (To be exact, the
setting is required only when memory and ns subsystems are
enabled at the same time, e.g., mount -t cgroup none /cgroup.)
---
 src/util/cgroup.c | 63 ++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 59 insertions(+), 4 deletions(-)

diff --git a/src/util/cgroup.c b/src/util/cgroup.c
index 3c23251a45..177433c03f 100644
--- a/src/util/cgroup.c
+++ b/src/util/cgroup.c
@@ -445,7 +445,38 @@ static int virCgroupCpuSetInherit(virCgroupPtr parent, virCgroupPtr group)
     return rc;
 }
 
-static int virCgroupMakeGroup(virCgroupPtr parent, virCgroupPtr group, int create)
+static int virCgroupSetMemoryUseHierarchy(virCgroupPtr group)
+{
+    int rc = 0;
+    unsigned long long value;
+    const char *filename = "memory.use_hierarchy";
+
+    rc = virCgroupGetValueU64(group,
+                              VIR_CGROUP_CONTROLLER_MEMORY,
+                              filename, &value);
+    if (rc != 0) {
+        VIR_ERROR(_("Failed to read %s/%s (%d)"), group->path, filename, rc);
+        return rc;
+    }
+
+    /* Setting twice causes error, so if already enabled, skip setting */
+    if (value == 1)
+        return 0;
+
+    VIR_DEBUG("Setting up %s/%s", group->path, filename);
+    rc = virCgroupSetValueU64(group,
+                              VIR_CGROUP_CONTROLLER_MEMORY,
+                              filename, 1);
+
+    if (rc != 0) {
+        VIR_ERROR(_("Failed to set %s/%s (%d)"), group->path, filename, rc);
+    }
+
+    return rc;
+}
+
+static int virCgroupMakeGroup(virCgroupPtr parent, virCgroupPtr group,
+                              int create, bool memory_hierarchy)
 {
     int i;
     int rc = 0;
@@ -479,6 +510,20 @@ static int virCgroupMakeGroup(virCgroupPtr parent, virCgroupPtr group, int creat
                     break;
                 }
             }
+            /*
+             * Note that virCgroupSetMemoryUseHierarchy should always be
+             * called prior to creating subcgroups and attaching tasks.
+             */
+            if (memory_hierarchy &&
+                group->controllers[VIR_CGROUP_CONTROLLER_MEMORY].mountPoint != NULL &&
+                (i == VIR_CGROUP_CONTROLLER_MEMORY ||
+                 STREQ(group->controllers[i].mountPoint, group->controllers[VIR_CGROUP_CONTROLLER_MEMORY].mountPoint))) {
+                rc = virCgroupSetMemoryUseHierarchy(group);
+                if (rc != 0) {
+                    VIR_FREE(path);
+                    break;
+                }
+            }
         }
 
         VIR_FREE(path);
@@ -555,7 +600,7 @@ static int virCgroupAppRoot(int privileged,
     if (rc != 0)
         goto cleanup;
 
-    rc = virCgroupMakeGroup(rootgrp, *group, create);
+    rc = virCgroupMakeGroup(rootgrp, *group, create, false);
 
 cleanup:
     virCgroupFree(&rootgrp);
@@ -705,7 +750,7 @@ int virCgroupForDriver(const char *name,
     VIR_FREE(path);
 
     if (rc == 0) {
-        rc = virCgroupMakeGroup(rootgrp, *group, create);
+        rc = virCgroupMakeGroup(rootgrp, *group, create, false);
         if (rc != 0)
             virCgroupFree(group);
     }
@@ -755,7 +800,17 @@ int virCgroupForDomain(virCgroupPtr driver,
     VIR_FREE(path);
 
     if (rc == 0) {
-        rc = virCgroupMakeGroup(driver, *group, create);
+        /*
+         * Create a cgroup with memory.use_hierarchy enabled to
+         * surely account memory usage of lxc with ns subsystem
+         * enabled. (To be exact, memory and ns subsystems are
+         * enabled at the same time.)
+         *
+         * The reason why doing it here, not a upper group, say
+         * a group for driver, is to avoid overhead to track
+         * cumulative usage that we don't need.
+         */
+        rc = virCgroupMakeGroup(driver, *group, create, true);
         if (rc != 0)
             virCgroupFree(group);
     }