conf: Introduce RDT monitor host capability

This patch is introducing cache monitor(CMT) to cache and
memory bandwidth monitor(MBM) for monitoring CPU memory
bandwidth.

The host capability of the two monitors is also introduced
in this patch.

For CMT, the host capability is shown like:
  <host>
  ...
    <cache>
      <bank id='0' level='3' type='both' size='15' unit='MiB' cpus='0-5'>
        <control granularity='768' min='1536' unit='KiB' type='both' maxAllocs='4'/>
      </bank>
      <monitor level='3' 'reuseThreshold'='270336' maxMonitors='176'>
        <feature name='llc_occupancy'/>
      </monitor>
    </cache>
    ...
  </host>

For MBM, the capability is shown like this:
  <host>
    ...
    <memory_bandwidth>
      <node id='1' cpus='6-11'>
        <control granularity='10' min ='10' maxAllocs='4'/>
      </node>
      <monitor maxMonitors='176'>
        <feature name='mbm_total_bytes'/>
        <feature name='mbm_local_bytes'/>
      </monitor>
    </memory_bandwidth>
    ...
  </host>

Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com>
Reviewed-by: John Ferlan <jferlan@redhat.com>
This commit is contained in:
Wang Huaqiang 2018-09-20 18:10:50 +08:00 committed by John Ferlan
parent 8f6887998b
commit 6af8417415
35 changed files with 425 additions and 1 deletions

View File

@ -316,6 +316,9 @@
</zeroOrMore>
</element>
</oneOrMore>
<optional>
<ref name='cpuMonitor'/>
</optional>
</element>
</define>
@ -347,7 +350,7 @@
<optional>
<attribute name='min'>
<ref name='unsignedInt'/>
</attribute>
</attribute>
</optional>
<attribute name='maxAllocs'>
<ref name='unsignedInt'/>
@ -356,9 +359,41 @@
</zeroOrMore>
</element>
</oneOrMore>
<optional>
<ref name='cpuMonitor'/>
</optional>
</element>
</define>
<define name='cpuMonitor'>
<element name='monitor'>
<optional>
<attribute name='level'>
<ref name='unsignedInt'/>
</attribute>
<attribute name='reuseThreshold'>
<ref name='unsignedInt'/>
</attribute>
</optional>
<attribute name='maxMonitors'>
<ref name='unsignedInt'/>
</attribute>
<oneOrMore>
<element name='feature'>
<attribute name='name'>
<ref name='monitorFeature'/>
</attribute>
</element>
</oneOrMore>
</element>
</define>
<define name='monitorFeature'>
<data type='string'>
<param name='pattern'>(llc_|mbm_)[a-zA-Z0-9\-_]+</param>
</data>
</define>
<define name='guestcaps'>
<element name='guest'>
<ref name='ostype'/>

View File

@ -243,10 +243,12 @@ virCapsDispose(void *object)
for (i = 0; i < caps->host.cache.nbanks; i++)
virCapsHostCacheBankFree(caps->host.cache.banks[i]);
virResctrlInfoMonFree(caps->host.cache.monitor);
VIR_FREE(caps->host.cache.banks);
for (i = 0; i < caps->host.memBW.nnodes; i++)
virCapsHostMemBWNodeFree(caps->host.memBW.nodes[i]);
virResctrlInfoMonFree(caps->host.memBW.monitor);
VIR_FREE(caps->host.memBW.nodes);
VIR_FREE(caps->host.netprefix);
@ -862,6 +864,50 @@ virCapabilitiesFormatNUMATopology(virBufferPtr buf,
return 0;
}
static int
virCapabilitiesFormatResctrlMonitor(virBufferPtr buf,
virResctrlInfoMonPtr monitor)
{
size_t i = 0;
virBuffer childrenBuf = VIR_BUFFER_INITIALIZER;
/* monitor not supported, no capability */
if (!monitor)
return 0;
/* no feature found in monitor means no capability, return */
if (monitor->nfeatures == 0)
return 0;
virBufferAddLit(buf, "<monitor ");
/* CMT might not enabled, if enabled show related attributes. */
if (monitor->type == VIR_RESCTRL_MONITOR_TYPE_CACHE)
virBufferAsprintf(buf,
"level='%u' reuseThreshold='%u' ",
monitor->cache_level,
monitor->cache_reuse_threshold);
virBufferAsprintf(buf,
"maxMonitors='%u'>\n",
monitor->max_monitor);
virBufferSetChildIndent(&childrenBuf, buf);
for (i = 0; i < monitor->nfeatures; i++) {
virBufferAsprintf(&childrenBuf,
"<feature name='%s'/>\n",
monitor->features[i]);
}
if (virBufferCheckError(&childrenBuf) < 0)
return -1;
virBufferAddBuffer(buf, &childrenBuf);
virBufferAddLit(buf, "</monitor>\n");
return 0;
}
static int
virCapabilitiesFormatCaches(virBufferPtr buf,
virCapsHostCachePtr cache)
@ -949,6 +995,9 @@ virCapabilitiesFormatCaches(virBufferPtr buf,
}
}
if (virCapabilitiesFormatResctrlMonitor(buf, cache->monitor) < 0)
return -1;
virBufferAdjustIndent(buf, -2);
virBufferAddLit(buf, "</cache>\n");
@ -1000,6 +1049,9 @@ virCapabilitiesFormatMemoryBandwidth(virBufferPtr buf,
}
}
if (virCapabilitiesFormatResctrlMonitor(buf, memBW->monitor) < 0)
return -1;
virBufferAdjustIndent(buf, -2);
virBufferAddLit(buf, "</memory_bandwidth>\n");
@ -1659,6 +1711,8 @@ virCapabilitiesInitResctrlMemory(virCapsPtr caps)
virCapsHostMemBWNodePtr node = NULL;
size_t i = 0;
int ret = -1;
const virResctrlMonitorType montype = VIR_RESCTRL_MONITOR_TYPE_MEMBW;
const char *prefix = virResctrlMonitorPrefixTypeToString(montype);
for (i = 0; i < caps->host.cache.nbanks; i++) {
virCapsHostCacheBankPtr bank = caps->host.cache.banks[i];
@ -1680,6 +1734,10 @@ virCapabilitiesInitResctrlMemory(virCapsPtr caps)
node = NULL;
}
if (virResctrlInfoGetMonitorPrefix(caps->host.resctrl, prefix,
&caps->host.memBW.monitor) < 0)
goto cleanup;
ret = 0;
cleanup:
virCapsHostMemBWNodeFree(node);
@ -1699,6 +1757,8 @@ virCapabilitiesInitCaches(virCapsPtr caps)
char *type = NULL;
struct dirent *ent = NULL;
virCapsHostCacheBankPtr bank = NULL;
const virResctrlMonitorType montype = VIR_RESCTRL_MONITOR_TYPE_CACHE;
const char *prefix = virResctrlMonitorPrefixTypeToString(montype);
/* Minimum level to expose in capabilities. Can be lowered or removed (with
* the appropriate code below), but should not be increased, because we'd
@ -1819,6 +1879,10 @@ virCapabilitiesInitCaches(virCapsPtr caps)
if (virCapabilitiesInitResctrlMemory(caps) < 0)
goto cleanup;
if (virResctrlInfoGetMonitorPrefix(caps->host.resctrl, prefix,
&caps->host.cache.monitor) < 0)
goto cleanup;
ret = 0;
cleanup:
VIR_FREE(type);

View File

@ -156,6 +156,8 @@ typedef virCapsHostCache *virCapsHostCachePtr;
struct _virCapsHostCache {
size_t nbanks;
virCapsHostCacheBankPtr *banks;
virResctrlInfoMonPtr monitor;
};
typedef struct _virCapsHostMemBWNode virCapsHostMemBWNode;
@ -171,6 +173,8 @@ typedef virCapsHostMemBW *virCapsHostMemBWPtr;
struct _virCapsHostMemBW {
size_t nnodes;
virCapsHostMemBWNodePtr *nodes;
virResctrlInfoMonPtr monitor;
};
typedef struct _virCapsHost virCapsHost;

View File

@ -2668,6 +2668,8 @@ virResctrlAllocSetCacheSize;
virResctrlAllocSetID;
virResctrlAllocSetMemoryBandwidth;
virResctrlInfoGetCache;
virResctrlInfoGetMonitorPrefix;
virResctrlInfoMonFree;
virResctrlInfoNew;

View File

@ -70,6 +70,12 @@ VIR_ENUM_IMPL(virResctrl, VIR_CACHE_TYPE_LAST,
"CODE",
"DATA")
/* Monitor feature name prefix mapping for monitor naming */
VIR_ENUM_IMPL(virResctrlMonitorPrefix, VIR_RESCTRL_MONITOR_TYPE_LAST,
"__unsupported__",
"llc_",
"mbm_")
/* All private typedefs so that they exist for all later definitions. This way
* structs can be included in one or another without reorganizing the code every
@ -207,6 +213,17 @@ virResctrlInfoDispose(void *obj)
}
void
virResctrlInfoMonFree(virResctrlInfoMonPtr mon)
{
if (!mon)
return;
virStringListFree(mon->features);
VIR_FREE(mon);
}
/* virResctrlAlloc */
/*
@ -853,6 +870,99 @@ virResctrlInfoGetCache(virResctrlInfoPtr resctrl,
}
/* virResctrlInfoGetMonitorPrefix
*
* @resctrl: Pointer to virResctrlInfo
* @prefix: Monitor prefix name for monitor looking for.
* @monitor: Returns the capability information for target monitor if the
* monitor with @prefex is supported by host.
*
* Return monitor capability information for @prefix through @monitor.
* If monitor with @prefix is not supported in system, @monitor will be
* cleared to NULL.
*
* Returns 0 if @monitor is created or monitor type with @prefix is not
* supported by host, -1 on failure with error message set.
*/
int
virResctrlInfoGetMonitorPrefix(virResctrlInfoPtr resctrl,
const char *prefix,
virResctrlInfoMonPtr *monitor)
{
size_t i = 0;
virResctrlInfoMongrpPtr mongrp_info = NULL;
virResctrlInfoMonPtr mon = NULL;
int ret = -1;
if (!prefix) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Empty prefix name for resctrl monitor"));
return -1;
}
if (virResctrlInfoIsEmpty(resctrl))
return 0;
mongrp_info = resctrl->monitor_info;
if (!mongrp_info) {
VIR_INFO("Monitor is not supported in host");
return 0;
}
for (i = 0; i < VIR_RESCTRL_MONITOR_TYPE_LAST; i++) {
if (STREQ(prefix, virResctrlMonitorPrefixTypeToString(i))) {
if (VIR_ALLOC(mon) < 0)
goto cleanup;
mon->type = i;
break;
}
}
if (!mon) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("Bad prefix name '%s' for resctrl monitor"),
prefix);
return -1;
}
mon->max_monitor = mongrp_info->max_monitor;
if (mon->type == VIR_RESCTRL_MONITOR_TYPE_CACHE) {
mon->cache_reuse_threshold = mongrp_info->cache_reuse_threshold;
mon->cache_level = mongrp_info->cache_level;
}
for (i = 0; i < mongrp_info->nfeatures; i++) {
if (STRPREFIX(mongrp_info->features[i], prefix)) {
if (virStringListAdd(&mon->features,
mongrp_info->features[i]) < 0)
goto cleanup;
mon->nfeatures++;
}
}
ret = 0;
/* In case *monitor is pointed to some monitor, clean it. */
virResctrlInfoMonFree(*monitor);
if (mon->nfeatures == 0) {
/* No feature found for current monitor, means host does not support
* monitor type with @prefix name.
* Telling caller this monitor is supported by hardware specification,
* but not supported by this host. */
VIR_INFO("No resctrl monitor features using prefix '%s' found", prefix);
goto cleanup;
}
VIR_STEAL_PTR(*monitor, mon);
cleanup:
virResctrlInfoMonFree(mon);
return ret;
}
/* virResctrlAlloc-related definitions */
virResctrlAllocPtr
virResctrlAllocNew(void)

View File

@ -36,6 +36,16 @@ typedef enum {
VIR_ENUM_DECL(virCache);
VIR_ENUM_DECL(virCacheKernel);
typedef enum {
VIR_RESCTRL_MONITOR_TYPE_UNSUPPORT,
VIR_RESCTRL_MONITOR_TYPE_CACHE,
VIR_RESCTRL_MONITOR_TYPE_MEMBW,
VIR_RESCTRL_MONITOR_TYPE_LAST
} virResctrlMonitorType;
VIR_ENUM_DECL(virResctrlMonitorPrefix);
typedef struct _virResctrlInfoPerCache virResctrlInfoPerCache;
typedef virResctrlInfoPerCache *virResctrlInfoPerCachePtr;
@ -61,6 +71,29 @@ struct _virResctrlInfoMemBWPerNode {
unsigned int max_allocation;
};
typedef struct _virResctrlInfoMon virResctrlInfoMon;
typedef virResctrlInfoMon *virResctrlInfoMonPtr;
struct _virResctrlInfoMon {
/* Maximum number of simultaneous monitors */
unsigned int max_monitor;
/* null-terminal string list for monitor features */
char **features;
/* Number of monitor features */
size_t nfeatures;
/* Monitor type */
virResctrlMonitorType type;
/* This adjustable value affects the final reuse of resources used by
* monitor. After the action of removing a monitor, the kernel may not
* release all hardware resources that monitor used immediately if the
* cache occupancy value associated with 'removed' monitor is above this
* threshold. Once the cache occupancy is below this threshold, the
* underlying hardware resource will be reclaimed and be put into the
* resource pool for next reusing.*/
unsigned int cache_reuse_threshold;
/* The cache 'level' that has the monitor capability */
unsigned int cache_level;
};
typedef struct _virResctrlInfo virResctrlInfo;
typedef virResctrlInfo *virResctrlInfoPtr;
@ -145,4 +178,11 @@ virResctrlAllocAddPID(virResctrlAllocPtr alloc,
int
virResctrlAllocRemove(virResctrlAllocPtr alloc);
void
virResctrlInfoMonFree(virResctrlInfoMonPtr mon);
int
virResctrlInfoGetMonitorPrefix(virResctrlInfoPtr resctrl,
const char *prefix,
virResctrlInfoMonPtr *monitor);
#endif /* __VIR_RESCTRL_H__ */

View File

@ -0,0 +1 @@
llc_occupancy

View File

@ -0,0 +1 @@
000

View File

@ -0,0 +1 @@
L3:0=e0000;1=e0000

View File

@ -0,0 +1 @@
L3:0=1ff00;1=1ff0f

View File

@ -0,0 +1 @@
../linux-resctrl/system

View File

@ -0,0 +1,10 @@
llc_occupancy
mbm_total_bytes
mbm_local_bytes
llc_new_feature
llc_unknown_feature
mbm_new_feature
mbm_unknown_feature
ukn_feature
fak_feature
fake_unknown_feature

View File

@ -0,0 +1 @@
L3:0=e0000;1=e0000

View File

@ -0,0 +1 @@
L3:0=1ff00;1=1ff0f

View File

@ -0,0 +1 @@
../linux-resctrl/system

View File

@ -0,0 +1,3 @@
llc_occupancy
mbm_total_bytes
mbm_local_bytes

View File

@ -0,0 +1 @@
176

View File

@ -0,0 +1,53 @@
<capabilities>
<host>
<cpu>
<arch>x86_64</arch>
</cpu>
<power_management/>
<iommu support='no'/>
<migration_features>
<live/>
</migration_features>
<topology>
<cells num='2'>
<cell id='0'>
<memory unit='KiB'>1048576</memory>
<pages unit='KiB' size='4'>2048</pages>
<pages unit='KiB' size='2048'>4096</pages>
<pages unit='KiB' size='1048576'>6144</pages>
<cpus num='6'>
<cpu id='0' socket_id='0' core_id='0' siblings='0'/>
<cpu id='1' socket_id='0' core_id='1' siblings='1'/>
<cpu id='2' socket_id='0' core_id='2' siblings='2'/>
<cpu id='3' socket_id='0' core_id='3' siblings='3'/>
<cpu id='4' socket_id='0' core_id='4' siblings='4'/>
<cpu id='5' socket_id='0' core_id='5' siblings='5'/>
</cpus>
</cell>
<cell id='1'>
<memory unit='KiB'>2097152</memory>
<pages unit='KiB' size='4'>4096</pages>
<pages unit='KiB' size='2048'>6144</pages>
<pages unit='KiB' size='1048576'>8192</pages>
<cpus num='6'>
<cpu id='6' socket_id='1' core_id='0' siblings='6'/>
<cpu id='7' socket_id='1' core_id='1' siblings='7'/>
<cpu id='8' socket_id='1' core_id='2' siblings='8'/>
<cpu id='9' socket_id='1' core_id='3' siblings='9'/>
<cpu id='10' socket_id='1' core_id='4' siblings='10'/>
<cpu id='11' socket_id='1' core_id='5' siblings='11'/>
</cpus>
</cell>
</cells>
</topology>
<cache>
<bank id='0' level='3' type='both' size='15' unit='MiB' cpus='0-5'/>
<bank id='1' level='3' type='both' size='15' unit='MiB' cpus='6-11'/>
<monitor level='3' reuseThreshold='270336' maxMonitors='176'>
<feature name='llc_occupancy'/>
</monitor>
</cache>
</host>
</capabilities>

View File

@ -0,0 +1,73 @@
<capabilities>
<host>
<cpu>
<arch>x86_64</arch>
</cpu>
<power_management/>
<iommu support='no'/>
<migration_features>
<live/>
</migration_features>
<topology>
<cells num='2'>
<cell id='0'>
<memory unit='KiB'>1048576</memory>
<pages unit='KiB' size='4'>2048</pages>
<pages unit='KiB' size='2048'>4096</pages>
<pages unit='KiB' size='1048576'>6144</pages>
<cpus num='6'>
<cpu id='0' socket_id='0' core_id='0' siblings='0'/>
<cpu id='1' socket_id='0' core_id='1' siblings='1'/>
<cpu id='2' socket_id='0' core_id='2' siblings='2'/>
<cpu id='3' socket_id='0' core_id='3' siblings='3'/>
<cpu id='4' socket_id='0' core_id='4' siblings='4'/>
<cpu id='5' socket_id='0' core_id='5' siblings='5'/>
</cpus>
</cell>
<cell id='1'>
<memory unit='KiB'>2097152</memory>
<pages unit='KiB' size='4'>4096</pages>
<pages unit='KiB' size='2048'>6144</pages>
<pages unit='KiB' size='1048576'>8192</pages>
<cpus num='6'>
<cpu id='6' socket_id='1' core_id='0' siblings='6'/>
<cpu id='7' socket_id='1' core_id='1' siblings='7'/>
<cpu id='8' socket_id='1' core_id='2' siblings='8'/>
<cpu id='9' socket_id='1' core_id='3' siblings='9'/>
<cpu id='10' socket_id='1' core_id='4' siblings='10'/>
<cpu id='11' socket_id='1' core_id='5' siblings='11'/>
</cpus>
</cell>
</cells>
</topology>
<cache>
<bank id='0' level='3' type='both' size='15' unit='MiB' cpus='0-5'>
<control granularity='768' min='1536' unit='KiB' type='both' maxAllocs='4'/>
</bank>
<bank id='1' level='3' type='both' size='15' unit='MiB' cpus='6-11'>
<control granularity='768' min='1536' unit='KiB' type='both' maxAllocs='4'/>
</bank>
<monitor level='3' reuseThreshold='270336' maxMonitors='176'>
<feature name='llc_occupancy'/>
<feature name='llc_new_feature'/>
<feature name='llc_unknown_feature'/>
</monitor>
</cache>
<memory_bandwidth>
<node id='0' cpus='0-5'>
<control granularity='10' min ='10' maxAllocs='4'/>
</node>
<node id='1' cpus='6-11'>
<control granularity='10' min ='10' maxAllocs='4'/>
</node>
<monitor maxMonitors='176'>
<feature name='mbm_total_bytes'/>
<feature name='mbm_local_bytes'/>
<feature name='mbm_new_feature'/>
<feature name='mbm_unknown_feature'/>
</monitor>
</memory_bandwidth>
</host>
</capabilities>

View File

@ -48,6 +48,9 @@
<bank id='1' level='3' type='both' size='15' unit='MiB' cpus='6-11'>
<control granularity='768' min='1536' unit='KiB' type='both' maxAllocs='4'/>
</bank>
<monitor level='3' reuseThreshold='270336' maxMonitors='176'>
<feature name='llc_occupancy'/>
</monitor>
</cache>
<memory_bandwidth>
<node id='0' cpus='0-5'>
@ -56,6 +59,10 @@
<node id='1' cpus='6-11'>
<control granularity='10' min ='10' maxAllocs='4'/>
</node>
<monitor maxMonitors='176'>
<feature name='mbm_total_bytes'/>
<feature name='mbm_local_bytes'/>
</monitor>
</memory_bandwidth>
</host>

View File

@ -110,9 +110,11 @@ mymain(void)
DO_TEST_FULL("caches", VIR_ARCH_X86_64, true, true);
DO_TEST_FULL("resctrl", VIR_ARCH_X86_64, true, true);
DO_TEST_FULL("resctrl-cmt", VIR_ARCH_X86_64, true, true);
DO_TEST_FULL("resctrl-cdp", VIR_ARCH_X86_64, true, true);
DO_TEST_FULL("resctrl-skx", VIR_ARCH_X86_64, true, true);
DO_TEST_FULL("resctrl-skx-twocaches", VIR_ARCH_X86_64, true, true);
DO_TEST_FULL("resctrl-fake-feature", VIR_ARCH_X86_64, true, true);
return ret;
}