mirror of
https://gitlab.com/libvirt/libvirt.git
synced 2025-03-07 17:28:15 +00:00
conf: Store cpu pinning data in def->vcpus
Now with the new struct the data can be stored in a much saner place.
This commit is contained in:
parent
856f254eef
commit
d2a6fc79e3
@ -1289,6 +1289,9 @@ virDomainVcpuInfoClear(virDomainVcpuInfoPtr info)
|
|||||||
{
|
{
|
||||||
if (!info)
|
if (!info)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
virBitmapFree(info->cpumask);
|
||||||
|
info->cpumask = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -1422,7 +1425,14 @@ virDomainDefGetVcpu(virDomainDefPtr def,
|
|||||||
static bool
|
static bool
|
||||||
virDomainDefHasVcpuPin(const virDomainDef *def)
|
virDomainDefHasVcpuPin(const virDomainDef *def)
|
||||||
{
|
{
|
||||||
return !!def->cputune.vcpupin;
|
size_t i;
|
||||||
|
|
||||||
|
for (i = 0; i < def->maxvcpus; i++) {
|
||||||
|
if (def->vcpus[i].cpumask)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -2593,8 +2603,6 @@ void virDomainDefFree(virDomainDefPtr def)
|
|||||||
|
|
||||||
virDomainIOThreadIDDefArrayFree(def->iothreadids, def->niothreadids);
|
virDomainIOThreadIDDefArrayFree(def->iothreadids, def->niothreadids);
|
||||||
|
|
||||||
virDomainPinDefArrayFree(def->cputune.vcpupin, def->cputune.nvcpupin);
|
|
||||||
|
|
||||||
virBitmapFree(def->cputune.emulatorpin);
|
virBitmapFree(def->cputune.emulatorpin);
|
||||||
|
|
||||||
for (i = 0; i < def->cputune.nvcpusched; i++)
|
for (i = 0; i < def->cputune.nvcpusched; i++)
|
||||||
@ -14110,83 +14118,68 @@ virDomainIOThreadIDDefParseXML(xmlNodePtr node,
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Check if pin with same id already exists. */
|
|
||||||
static bool
|
|
||||||
virDomainPinIsDuplicate(virDomainPinDefPtr *def,
|
|
||||||
int npin,
|
|
||||||
int id)
|
|
||||||
{
|
|
||||||
size_t i;
|
|
||||||
|
|
||||||
if (!def || !npin)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
for (i = 0; i < npin; i++) {
|
|
||||||
if (def[i]->id == id)
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Parse the XML definition for a vcpupin
|
/* Parse the XML definition for a vcpupin
|
||||||
*
|
*
|
||||||
* vcpupin has the form of
|
* vcpupin has the form of
|
||||||
* <vcpupin vcpu='0' cpuset='0'/>
|
* <vcpupin vcpu='0' cpuset='0'/>
|
||||||
*/
|
*/
|
||||||
static virDomainPinDefPtr
|
static int
|
||||||
virDomainVcpuPinDefParseXML(xmlNodePtr node,
|
virDomainVcpuPinDefParseXML(virDomainDefPtr def,
|
||||||
xmlXPathContextPtr ctxt)
|
xmlNodePtr node)
|
||||||
{
|
{
|
||||||
virDomainPinDefPtr def;
|
virDomainVcpuInfoPtr vcpu;
|
||||||
xmlNodePtr oldnode = ctxt->node;
|
|
||||||
unsigned int vcpuid;
|
unsigned int vcpuid;
|
||||||
char *tmp = NULL;
|
char *tmp = NULL;
|
||||||
|
int ret = -1;
|
||||||
|
|
||||||
if (VIR_ALLOC(def) < 0)
|
if (!(tmp = virXMLPropString(node, "vcpu"))) {
|
||||||
return NULL;
|
virReportError(VIR_ERR_XML_ERROR, "%s", _("missing vcpu id in vcpupin"));
|
||||||
|
goto cleanup;
|
||||||
ctxt->node = node;
|
|
||||||
|
|
||||||
if (!(tmp = virXPathString("string(./@vcpu)", ctxt))) {
|
|
||||||
virReportError(VIR_ERR_XML_ERROR, "%s",
|
|
||||||
_("missing vcpu id in vcpupin"));
|
|
||||||
goto error;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (virStrToLong_uip(tmp, NULL, 10, &vcpuid) < 0) {
|
if (virStrToLong_uip(tmp, NULL, 10, &vcpuid) < 0) {
|
||||||
virReportError(VIR_ERR_XML_ERROR,
|
virReportError(VIR_ERR_XML_ERROR,
|
||||||
_("invalid setting for vcpu '%s'"), tmp);
|
_("invalid setting for vcpu '%s'"), tmp);
|
||||||
goto error;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
VIR_FREE(tmp);
|
VIR_FREE(tmp);
|
||||||
|
|
||||||
def->id = vcpuid;
|
if (!(vcpu = virDomainDefGetVcpu(def, vcpuid)) ||
|
||||||
|
!vcpu->online) {
|
||||||
|
/* To avoid the regression when daemon loading domain confs, we can't
|
||||||
|
* simply error out if <vcpupin> nodes greater than current vcpus.
|
||||||
|
* Ignore them instead. */
|
||||||
|
VIR_WARN("Ignoring vcpupin for missing vcpus");
|
||||||
|
ret = 0;
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
|
||||||
if (!(tmp = virXMLPropString(node, "cpuset"))) {
|
if (!(tmp = virXMLPropString(node, "cpuset"))) {
|
||||||
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
|
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
|
||||||
_("missing cpuset for vcpupin"));
|
_("missing cpuset for vcpupin"));
|
||||||
|
goto cleanup;
|
||||||
goto error;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (virBitmapParse(tmp, 0, &def->cpumask, VIR_DOMAIN_CPUMASK_LEN) < 0)
|
if (vcpu->cpumask) {
|
||||||
goto error;
|
virReportError(VIR_ERR_INTERNAL_ERROR,
|
||||||
|
_("duplicate vcpupin for vcpu '%d'"), vcpuid);
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
|
||||||
if (virBitmapIsAllClear(def->cpumask)) {
|
if (virBitmapParse(tmp, 0, &vcpu->cpumask, VIR_DOMAIN_CPUMASK_LEN) < 0)
|
||||||
|
goto cleanup;
|
||||||
|
|
||||||
|
if (virBitmapIsAllClear(vcpu->cpumask)) {
|
||||||
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
|
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
|
||||||
_("Invalid value of 'cpuset': %s"), tmp);
|
_("Invalid value of 'cpuset': %s"), tmp);
|
||||||
goto error;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ret = 0;
|
||||||
|
|
||||||
cleanup:
|
cleanup:
|
||||||
VIR_FREE(tmp);
|
VIR_FREE(tmp);
|
||||||
ctxt->node = oldnode;
|
return ret;
|
||||||
return def;
|
|
||||||
|
|
||||||
error:
|
|
||||||
VIR_FREE(def);
|
|
||||||
goto cleanup;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -15142,34 +15135,9 @@ virDomainDefParseXML(xmlDocPtr xml,
|
|||||||
if ((n = virXPathNodeSet("./cputune/vcpupin", ctxt, &nodes)) < 0)
|
if ((n = virXPathNodeSet("./cputune/vcpupin", ctxt, &nodes)) < 0)
|
||||||
goto error;
|
goto error;
|
||||||
|
|
||||||
if (n && VIR_ALLOC_N(def->cputune.vcpupin, n) < 0)
|
|
||||||
goto error;
|
|
||||||
|
|
||||||
for (i = 0; i < n; i++) {
|
for (i = 0; i < n; i++) {
|
||||||
virDomainPinDefPtr vcpupin;
|
if (virDomainVcpuPinDefParseXML(def, nodes[i]))
|
||||||
if (!(vcpupin = virDomainVcpuPinDefParseXML(nodes[i], ctxt)))
|
|
||||||
goto error;
|
goto error;
|
||||||
|
|
||||||
if (virDomainPinIsDuplicate(def->cputune.vcpupin,
|
|
||||||
def->cputune.nvcpupin,
|
|
||||||
vcpupin->id)) {
|
|
||||||
virReportError(VIR_ERR_INTERNAL_ERROR,
|
|
||||||
"%s", _("duplicate vcpupin for same vcpu"));
|
|
||||||
virDomainPinDefFree(vcpupin);
|
|
||||||
goto error;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (vcpupin->id >= virDomainDefGetVcpus(def)) {
|
|
||||||
/* To avoid the regression when daemon loading
|
|
||||||
* domain confs, we can't simply error out if
|
|
||||||
* <vcpupin> nodes greater than current vcpus,
|
|
||||||
* ignoring them instead.
|
|
||||||
*/
|
|
||||||
VIR_WARN("Ignore vcpupin for missing vcpus");
|
|
||||||
virDomainPinDefFree(vcpupin);
|
|
||||||
} else {
|
|
||||||
def->cputune.vcpupin[def->cputune.nvcpupin++] = vcpupin;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
VIR_FREE(nodes);
|
VIR_FREE(nodes);
|
||||||
|
|
||||||
@ -21845,15 +21813,19 @@ virDomainDefFormatInternal(virDomainDefPtr def,
|
|||||||
"</emulator_quota>\n",
|
"</emulator_quota>\n",
|
||||||
def->cputune.emulator_quota);
|
def->cputune.emulator_quota);
|
||||||
|
|
||||||
for (i = 0; i < def->cputune.nvcpupin; i++) {
|
for (i = 0; i < def->maxvcpus; i++) {
|
||||||
char *cpumask;
|
char *cpumask;
|
||||||
virBufferAsprintf(&childrenBuf, "<vcpupin vcpu='%u' ",
|
virDomainVcpuInfoPtr vcpu = def->vcpus + i;
|
||||||
def->cputune.vcpupin[i]->id);
|
|
||||||
|
|
||||||
if (!(cpumask = virBitmapFormat(def->cputune.vcpupin[i]->cpumask)))
|
if (!vcpu->cpumask)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (!(cpumask = virBitmapFormat(vcpu->cpumask)))
|
||||||
goto error;
|
goto error;
|
||||||
|
|
||||||
virBufferAsprintf(&childrenBuf, "cpuset='%s'/>\n", cpumask);
|
virBufferAsprintf(&childrenBuf,
|
||||||
|
"<vcpupin vcpu='%zu' cpuset='%s'/>\n", i, cpumask);
|
||||||
|
|
||||||
VIR_FREE(cpumask);
|
VIR_FREE(cpumask);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2137,8 +2137,6 @@ struct _virDomainCputune {
|
|||||||
long long quota;
|
long long quota;
|
||||||
unsigned long long emulator_period;
|
unsigned long long emulator_period;
|
||||||
long long emulator_quota;
|
long long emulator_quota;
|
||||||
size_t nvcpupin;
|
|
||||||
virDomainPinDefPtr *vcpupin;
|
|
||||||
virBitmapPtr emulatorpin;
|
virBitmapPtr emulatorpin;
|
||||||
|
|
||||||
size_t nvcpusched;
|
size_t nvcpusched;
|
||||||
@ -2153,6 +2151,7 @@ typedef virDomainVcpuInfo *virDomainVcpuInfoPtr;
|
|||||||
|
|
||||||
struct _virDomainVcpuInfo {
|
struct _virDomainVcpuInfo {
|
||||||
bool online;
|
bool online;
|
||||||
|
virBitmapPtr cpumask;
|
||||||
};
|
};
|
||||||
|
|
||||||
typedef struct _virDomainBlkiotune virDomainBlkiotune;
|
typedef struct _virDomainBlkiotune virDomainBlkiotune;
|
||||||
|
@ -821,7 +821,7 @@ int
|
|||||||
libxlDomainSetVcpuAffinities(libxlDriverPrivatePtr driver, virDomainObjPtr vm)
|
libxlDomainSetVcpuAffinities(libxlDriverPrivatePtr driver, virDomainObjPtr vm)
|
||||||
{
|
{
|
||||||
libxlDriverConfigPtr cfg = libxlDriverConfigGet(driver);
|
libxlDriverConfigPtr cfg = libxlDriverConfigGet(driver);
|
||||||
virDomainPinDefPtr pin;
|
virDomainVcpuInfoPtr vcpu;
|
||||||
libxl_bitmap map;
|
libxl_bitmap map;
|
||||||
virBitmapPtr cpumask = NULL;
|
virBitmapPtr cpumask = NULL;
|
||||||
size_t i;
|
size_t i;
|
||||||
@ -830,13 +830,12 @@ libxlDomainSetVcpuAffinities(libxlDriverPrivatePtr driver, virDomainObjPtr vm)
|
|||||||
libxl_bitmap_init(&map);
|
libxl_bitmap_init(&map);
|
||||||
|
|
||||||
for (i = 0; i < virDomainDefGetVcpus(vm->def); ++i) {
|
for (i = 0; i < virDomainDefGetVcpus(vm->def); ++i) {
|
||||||
pin = virDomainPinFind(vm->def->cputune.vcpupin,
|
vcpu = virDomainDefGetVcpu(vm->def, i);
|
||||||
vm->def->cputune.nvcpupin,
|
|
||||||
i);
|
|
||||||
|
|
||||||
if (pin && pin->cpumask)
|
if (!vcpu->online)
|
||||||
cpumask = pin->cpumask;
|
continue;
|
||||||
else
|
|
||||||
|
if (!(cpumask = vcpu->cpumask))
|
||||||
cpumask = vm->def->cpumask;
|
cpumask = vm->def->cpumask;
|
||||||
|
|
||||||
if (!cpumask)
|
if (!cpumask)
|
||||||
@ -845,9 +844,9 @@ libxlDomainSetVcpuAffinities(libxlDriverPrivatePtr driver, virDomainObjPtr vm)
|
|||||||
if (virBitmapToData(cpumask, &map.map, (int *)&map.size) < 0)
|
if (virBitmapToData(cpumask, &map.map, (int *)&map.size) < 0)
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
|
|
||||||
if (libxl_set_vcpuaffinity(cfg->ctx, vm->def->id, pin->id, &map) != 0) {
|
if (libxl_set_vcpuaffinity(cfg->ctx, vm->def->id, i, &map) != 0) {
|
||||||
virReportError(VIR_ERR_INTERNAL_ERROR,
|
virReportError(VIR_ERR_INTERNAL_ERROR,
|
||||||
_("Failed to pin vcpu '%d' with libxenlight"), pin->id);
|
_("Failed to pin vcpu '%zu' with libxenlight"), i);
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2329,6 +2329,7 @@ libxlDomainPinVcpuFlags(virDomainPtr dom, unsigned int vcpu,
|
|||||||
libxlDriverConfigPtr cfg = libxlDriverConfigGet(driver);
|
libxlDriverConfigPtr cfg = libxlDriverConfigGet(driver);
|
||||||
virDomainDefPtr targetDef = NULL;
|
virDomainDefPtr targetDef = NULL;
|
||||||
virBitmapPtr pcpumap = NULL;
|
virBitmapPtr pcpumap = NULL;
|
||||||
|
virDomainVcpuInfoPtr vcpuinfo;
|
||||||
virDomainObjPtr vm;
|
virDomainObjPtr vm;
|
||||||
int ret = -1;
|
int ret = -1;
|
||||||
|
|
||||||
@ -2364,6 +2365,13 @@ libxlDomainPinVcpuFlags(virDomainPtr dom, unsigned int vcpu,
|
|||||||
if (!pcpumap)
|
if (!pcpumap)
|
||||||
goto endjob;
|
goto endjob;
|
||||||
|
|
||||||
|
if (!(vcpuinfo = virDomainDefGetVcpu(targetDef, vcpu)) ||
|
||||||
|
!vcpuinfo->online) {
|
||||||
|
virReportError(VIR_ERR_INVALID_ARG,
|
||||||
|
_("vcpu '%u' is not active"), vcpu);
|
||||||
|
goto endjob;
|
||||||
|
}
|
||||||
|
|
||||||
if (flags & VIR_DOMAIN_AFFECT_LIVE) {
|
if (flags & VIR_DOMAIN_AFFECT_LIVE) {
|
||||||
libxl_bitmap map = { .size = maplen, .map = cpumap };
|
libxl_bitmap map = { .size = maplen, .map = cpumap };
|
||||||
if (libxl_set_vcpuaffinity(cfg->ctx, vm->def->id, vcpu, &map) != 0) {
|
if (libxl_set_vcpuaffinity(cfg->ctx, vm->def->id, vcpu, &map) != 0) {
|
||||||
@ -2374,20 +2382,9 @@ libxlDomainPinVcpuFlags(virDomainPtr dom, unsigned int vcpu,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!targetDef->cputune.vcpupin) {
|
virBitmapFree(vcpuinfo->cpumask);
|
||||||
if (VIR_ALLOC(targetDef->cputune.vcpupin) < 0)
|
vcpuinfo->cpumask = pcpumap;
|
||||||
goto endjob;
|
pcpumap = NULL;
|
||||||
targetDef->cputune.nvcpupin = 0;
|
|
||||||
}
|
|
||||||
if (virDomainPinAdd(&targetDef->cputune.vcpupin,
|
|
||||||
&targetDef->cputune.nvcpupin,
|
|
||||||
cpumap,
|
|
||||||
maplen,
|
|
||||||
vcpu) < 0) {
|
|
||||||
virReportError(VIR_ERR_INTERNAL_ERROR,
|
|
||||||
"%s", _("failed to update or add vcpupin xml"));
|
|
||||||
goto endjob;
|
|
||||||
}
|
|
||||||
|
|
||||||
ret = 0;
|
ret = 0;
|
||||||
|
|
||||||
@ -2463,15 +2460,14 @@ libxlDomainGetVcpuPinInfo(virDomainPtr dom, int ncpumaps,
|
|||||||
memset(cpumaps, 0x00, maplen * ncpumaps);
|
memset(cpumaps, 0x00, maplen * ncpumaps);
|
||||||
|
|
||||||
for (vcpu = 0; vcpu < ncpumaps; vcpu++) {
|
for (vcpu = 0; vcpu < ncpumaps; vcpu++) {
|
||||||
virDomainPinDefPtr pininfo;
|
virDomainVcpuInfoPtr vcpuinfo = virDomainDefGetVcpu(targetDef, vcpu);
|
||||||
virBitmapPtr bitmap = NULL;
|
virBitmapPtr bitmap = NULL;
|
||||||
|
|
||||||
pininfo = virDomainPinFind(targetDef->cputune.vcpupin,
|
if (!vcpuinfo->online)
|
||||||
targetDef->cputune.nvcpupin,
|
continue;
|
||||||
vcpu);
|
|
||||||
|
|
||||||
if (pininfo && pininfo->cpumask)
|
if (vcpuinfo->cpumask)
|
||||||
bitmap = pininfo->cpumask;
|
bitmap = vcpuinfo->cpumask;
|
||||||
else if (targetDef->cpumask)
|
else if (targetDef->cpumask)
|
||||||
bitmap = targetDef->cpumask;
|
bitmap = targetDef->cpumask;
|
||||||
else
|
else
|
||||||
|
@ -1024,7 +1024,7 @@ qemuSetupCgroupForVcpu(virDomainObjPtr vm)
|
|||||||
virCgroupPtr cgroup_vcpu = NULL;
|
virCgroupPtr cgroup_vcpu = NULL;
|
||||||
qemuDomainObjPrivatePtr priv = vm->privateData;
|
qemuDomainObjPrivatePtr priv = vm->privateData;
|
||||||
virDomainDefPtr def = vm->def;
|
virDomainDefPtr def = vm->def;
|
||||||
size_t i, j;
|
size_t i;
|
||||||
unsigned long long period = vm->def->cputune.period;
|
unsigned long long period = vm->def->cputune.period;
|
||||||
long long quota = vm->def->cputune.quota;
|
long long quota = vm->def->cputune.quota;
|
||||||
char *mem_mask = NULL;
|
char *mem_mask = NULL;
|
||||||
@ -1082,20 +1082,13 @@ qemuSetupCgroupForVcpu(virDomainObjPtr vm)
|
|||||||
virCgroupSetCpusetMems(cgroup_vcpu, mem_mask) < 0)
|
virCgroupSetCpusetMems(cgroup_vcpu, mem_mask) < 0)
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
|
|
||||||
/* try to use the default cpu maps */
|
if (vcpu->cpumask)
|
||||||
if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)
|
cpumap = vcpu->cpumask;
|
||||||
|
else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)
|
||||||
cpumap = priv->autoCpuset;
|
cpumap = priv->autoCpuset;
|
||||||
else
|
else
|
||||||
cpumap = vm->def->cpumask;
|
cpumap = vm->def->cpumask;
|
||||||
|
|
||||||
/* lookup a more specific pinning info */
|
|
||||||
for (j = 0; j < def->cputune.nvcpupin; j++) {
|
|
||||||
if (def->cputune.vcpupin[j]->id == i) {
|
|
||||||
cpumap = def->cputune.vcpupin[j]->cpumask;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (cpumap && qemuSetupCgroupCpusetCpus(cgroup_vcpu, cpumap) < 0)
|
if (cpumap && qemuSetupCgroupCpusetCpus(cgroup_vcpu, cpumap) < 0)
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
@ -4869,10 +4869,8 @@ qemuDomainHotplugDelVcpu(virQEMUDriverPtr driver,
|
|||||||
VIR_CGROUP_THREAD_VCPU, vcpu) < 0)
|
VIR_CGROUP_THREAD_VCPU, vcpu) < 0)
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
|
|
||||||
/* Free vcpupin setting */
|
virBitmapFree(vcpuinfo->cpumask);
|
||||||
virDomainPinDel(&vm->def->cputune.vcpupin,
|
vcpuinfo->cpumask = NULL;
|
||||||
&vm->def->cputune.nvcpupin,
|
|
||||||
vcpu);
|
|
||||||
|
|
||||||
ret = 0;
|
ret = 0;
|
||||||
|
|
||||||
@ -5033,10 +5031,15 @@ qemuDomainSetVcpusFlags(virDomainPtr dom, unsigned int nvcpus,
|
|||||||
if (persistentDef) {
|
if (persistentDef) {
|
||||||
/* remove vcpupin entries for vcpus that were unplugged */
|
/* remove vcpupin entries for vcpus that were unplugged */
|
||||||
if (nvcpus < virDomainDefGetVcpus(persistentDef)) {
|
if (nvcpus < virDomainDefGetVcpus(persistentDef)) {
|
||||||
for (i = virDomainDefGetVcpus(persistentDef) - 1; i >= nvcpus; i--)
|
for (i = virDomainDefGetVcpus(persistentDef) - 1; i >= nvcpus; i--) {
|
||||||
virDomainPinDel(&persistentDef->cputune.vcpupin,
|
virDomainVcpuInfoPtr vcpu = virDomainDefGetVcpu(persistentDef,
|
||||||
&persistentDef->cputune.nvcpupin,
|
|
||||||
i);
|
i);
|
||||||
|
|
||||||
|
if (vcpu) {
|
||||||
|
virBitmapFree(vcpu->cpumask);
|
||||||
|
vcpu->cpumask = NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (flags & VIR_DOMAIN_VCPU_MAXIMUM) {
|
if (flags & VIR_DOMAIN_VCPU_MAXIMUM) {
|
||||||
@ -5096,9 +5099,11 @@ qemuDomainPinVcpuFlags(virDomainPtr dom,
|
|||||||
virCgroupPtr cgroup_vcpu = NULL;
|
virCgroupPtr cgroup_vcpu = NULL;
|
||||||
int ret = -1;
|
int ret = -1;
|
||||||
qemuDomainObjPrivatePtr priv;
|
qemuDomainObjPrivatePtr priv;
|
||||||
size_t newVcpuPinNum = 0;
|
|
||||||
virDomainPinDefPtr *newVcpuPin = NULL;
|
|
||||||
virBitmapPtr pcpumap = NULL;
|
virBitmapPtr pcpumap = NULL;
|
||||||
|
virBitmapPtr pcpumaplive = NULL;
|
||||||
|
virBitmapPtr pcpumappersist = NULL;
|
||||||
|
virDomainVcpuInfoPtr vcpuinfolive = NULL;
|
||||||
|
virDomainVcpuInfoPtr vcpuinfopersist = NULL;
|
||||||
virQEMUDriverConfigPtr cfg = NULL;
|
virQEMUDriverConfigPtr cfg = NULL;
|
||||||
virObjectEventPtr event = NULL;
|
virObjectEventPtr event = NULL;
|
||||||
char paramField[VIR_TYPED_PARAM_FIELD_LENGTH] = "";
|
char paramField[VIR_TYPED_PARAM_FIELD_LENGTH] = "";
|
||||||
@ -5126,20 +5131,38 @@ qemuDomainPinVcpuFlags(virDomainPtr dom,
|
|||||||
|
|
||||||
priv = vm->privateData;
|
priv = vm->privateData;
|
||||||
|
|
||||||
if (def && vcpu >= virDomainDefGetVcpus(def)) {
|
if (def) {
|
||||||
|
if (!(vcpuinfolive = virDomainDefGetVcpu(def, vcpu))) {
|
||||||
virReportError(VIR_ERR_INVALID_ARG,
|
virReportError(VIR_ERR_INVALID_ARG,
|
||||||
_("vcpu %d is out of range of live cpu count %d"),
|
_("vcpu %d is out of range of live cpu count %d"),
|
||||||
vcpu, virDomainDefGetVcpus(def));
|
vcpu, virDomainDefGetVcpus(def));
|
||||||
goto endjob;
|
goto endjob;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (persistentDef && vcpu >= virDomainDefGetVcpus(persistentDef)) {
|
if (!vcpuinfolive->online) {
|
||||||
|
virReportError(VIR_ERR_OPERATION_INVALID,
|
||||||
|
_("setting cpu pinning for inactive vcpu '%d' is not "
|
||||||
|
"supported"), vcpu);
|
||||||
|
goto endjob;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (persistentDef) {
|
||||||
|
if (!(vcpuinfopersist = virDomainDefGetVcpu(persistentDef, vcpu))) {
|
||||||
virReportError(VIR_ERR_INVALID_ARG,
|
virReportError(VIR_ERR_INVALID_ARG,
|
||||||
_("vcpu %d is out of range of persistent cpu count %d"),
|
_("vcpu %d is out of range of persistent cpu count %d"),
|
||||||
vcpu, virDomainDefGetVcpus(persistentDef));
|
vcpu, virDomainDefGetVcpus(persistentDef));
|
||||||
goto endjob;
|
goto endjob;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!vcpuinfopersist->online) {
|
||||||
|
virReportError(VIR_ERR_OPERATION_INVALID,
|
||||||
|
_("setting cpu pinning for inactive vcpu '%d' is not "
|
||||||
|
"supported"), vcpu);
|
||||||
|
goto endjob;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (!(pcpumap = virBitmapNewData(cpumap, maplen)))
|
if (!(pcpumap = virBitmapNewData(cpumap, maplen)))
|
||||||
goto endjob;
|
goto endjob;
|
||||||
|
|
||||||
@ -5149,6 +5172,10 @@ qemuDomainPinVcpuFlags(virDomainPtr dom,
|
|||||||
goto endjob;
|
goto endjob;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ((def && !(pcpumaplive = virBitmapNewCopy(pcpumap))) ||
|
||||||
|
(persistentDef && !(pcpumappersist = virBitmapNewCopy(pcpumap))))
|
||||||
|
goto endjob;
|
||||||
|
|
||||||
if (def) {
|
if (def) {
|
||||||
if (!qemuDomainHasVcpuPids(vm)) {
|
if (!qemuDomainHasVcpuPids(vm)) {
|
||||||
virReportError(VIR_ERR_OPERATION_INVALID,
|
virReportError(VIR_ERR_OPERATION_INVALID,
|
||||||
@ -5156,26 +5183,6 @@ qemuDomainPinVcpuFlags(virDomainPtr dom,
|
|||||||
goto endjob;
|
goto endjob;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (def->cputune.vcpupin) {
|
|
||||||
newVcpuPin = virDomainPinDefCopy(def->cputune.vcpupin,
|
|
||||||
def->cputune.nvcpupin);
|
|
||||||
if (!newVcpuPin)
|
|
||||||
goto endjob;
|
|
||||||
|
|
||||||
newVcpuPinNum = def->cputune.nvcpupin;
|
|
||||||
} else {
|
|
||||||
if (VIR_ALLOC(newVcpuPin) < 0)
|
|
||||||
goto endjob;
|
|
||||||
newVcpuPinNum = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (virDomainPinAdd(&newVcpuPin, &newVcpuPinNum,
|
|
||||||
cpumap, maplen, vcpu) < 0) {
|
|
||||||
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
|
|
||||||
_("failed to update vcpupin"));
|
|
||||||
goto endjob;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Configure the corresponding cpuset cgroup before set affinity. */
|
/* Configure the corresponding cpuset cgroup before set affinity. */
|
||||||
if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
|
if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
|
||||||
if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_VCPU, vcpu,
|
if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_VCPU, vcpu,
|
||||||
@ -5197,13 +5204,9 @@ qemuDomainPinVcpuFlags(virDomainPtr dom,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (def->cputune.vcpupin)
|
virBitmapFree(vcpuinfolive->cpumask);
|
||||||
virDomainPinDefArrayFree(def->cputune.vcpupin,
|
vcpuinfolive->cpumask = pcpumaplive;
|
||||||
def->cputune.nvcpupin);
|
pcpumaplive = NULL;
|
||||||
|
|
||||||
def->cputune.vcpupin = newVcpuPin;
|
|
||||||
def->cputune.nvcpupin = newVcpuPinNum;
|
|
||||||
newVcpuPin = NULL;
|
|
||||||
|
|
||||||
if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm, driver->caps) < 0)
|
if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm, driver->caps) < 0)
|
||||||
goto endjob;
|
goto endjob;
|
||||||
@ -5222,21 +5225,9 @@ qemuDomainPinVcpuFlags(virDomainPtr dom,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (persistentDef) {
|
if (persistentDef) {
|
||||||
if (!persistentDef->cputune.vcpupin) {
|
virBitmapFree(vcpuinfopersist->cpumask);
|
||||||
if (VIR_ALLOC(persistentDef->cputune.vcpupin) < 0)
|
vcpuinfopersist->cpumask = pcpumappersist;
|
||||||
goto endjob;
|
pcpumappersist = NULL;
|
||||||
persistentDef->cputune.nvcpupin = 0;
|
|
||||||
}
|
|
||||||
if (virDomainPinAdd(&persistentDef->cputune.vcpupin,
|
|
||||||
&persistentDef->cputune.nvcpupin,
|
|
||||||
cpumap,
|
|
||||||
maplen,
|
|
||||||
vcpu) < 0) {
|
|
||||||
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
|
|
||||||
_("failed to update or add vcpupin xml of "
|
|
||||||
"a persistent domain"));
|
|
||||||
goto endjob;
|
|
||||||
}
|
|
||||||
|
|
||||||
ret = virDomainSaveConfig(cfg->configDir, driver->caps, persistentDef);
|
ret = virDomainSaveConfig(cfg->configDir, driver->caps, persistentDef);
|
||||||
goto endjob;
|
goto endjob;
|
||||||
@ -5248,14 +5239,14 @@ qemuDomainPinVcpuFlags(virDomainPtr dom,
|
|||||||
qemuDomainObjEndJob(driver, vm);
|
qemuDomainObjEndJob(driver, vm);
|
||||||
|
|
||||||
cleanup:
|
cleanup:
|
||||||
if (newVcpuPin)
|
|
||||||
virDomainPinDefArrayFree(newVcpuPin, newVcpuPinNum);
|
|
||||||
if (cgroup_vcpu)
|
if (cgroup_vcpu)
|
||||||
virCgroupFree(&cgroup_vcpu);
|
virCgroupFree(&cgroup_vcpu);
|
||||||
virDomainObjEndAPI(&vm);
|
virDomainObjEndAPI(&vm);
|
||||||
qemuDomainEventQueue(driver, event);
|
qemuDomainEventQueue(driver, event);
|
||||||
VIR_FREE(str);
|
VIR_FREE(str);
|
||||||
virBitmapFree(pcpumap);
|
virBitmapFree(pcpumap);
|
||||||
|
virBitmapFree(pcpumaplive);
|
||||||
|
virBitmapFree(pcpumappersist);
|
||||||
virObjectUnref(cfg);
|
virObjectUnref(cfg);
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
@ -5280,7 +5271,8 @@ qemuDomainGetVcpuPinInfo(virDomainPtr dom,
|
|||||||
virDomainObjPtr vm = NULL;
|
virDomainObjPtr vm = NULL;
|
||||||
virDomainDefPtr def;
|
virDomainDefPtr def;
|
||||||
int ret = -1;
|
int ret = -1;
|
||||||
int hostcpus, vcpu;
|
int hostcpus;
|
||||||
|
size_t i;
|
||||||
virBitmapPtr allcpumap = NULL;
|
virBitmapPtr allcpumap = NULL;
|
||||||
qemuDomainObjPrivatePtr priv = NULL;
|
qemuDomainObjPrivatePtr priv = NULL;
|
||||||
|
|
||||||
@ -5312,16 +5304,15 @@ qemuDomainGetVcpuPinInfo(virDomainPtr dom,
|
|||||||
if (ncpumaps < 1)
|
if (ncpumaps < 1)
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
|
|
||||||
for (vcpu = 0; vcpu < ncpumaps; vcpu++) {
|
for (i = 0; i < ncpumaps; i++) {
|
||||||
virDomainPinDefPtr pininfo;
|
virDomainVcpuInfoPtr vcpu = virDomainDefGetVcpu(def, i);
|
||||||
virBitmapPtr bitmap = NULL;
|
virBitmapPtr bitmap = NULL;
|
||||||
|
|
||||||
pininfo = virDomainPinFind(def->cputune.vcpupin,
|
if (!vcpu->online)
|
||||||
def->cputune.nvcpupin,
|
continue;
|
||||||
vcpu);
|
|
||||||
|
|
||||||
if (pininfo && pininfo->cpumask)
|
if (vcpu->cpumask)
|
||||||
bitmap = pininfo->cpumask;
|
bitmap = vcpu->cpumask;
|
||||||
else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO &&
|
else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO &&
|
||||||
priv->autoCpuset)
|
priv->autoCpuset)
|
||||||
bitmap = priv->autoCpuset;
|
bitmap = priv->autoCpuset;
|
||||||
@ -5330,7 +5321,7 @@ qemuDomainGetVcpuPinInfo(virDomainPtr dom,
|
|||||||
else
|
else
|
||||||
bitmap = allcpumap;
|
bitmap = allcpumap;
|
||||||
|
|
||||||
virBitmapToDataBuf(bitmap, VIR_GET_CPUMAP(cpumaps, maplen, vcpu), maplen);
|
virBitmapToDataBuf(bitmap, VIR_GET_CPUMAP(cpumaps, maplen, i), maplen);
|
||||||
}
|
}
|
||||||
|
|
||||||
ret = ncpumaps;
|
ret = ncpumaps;
|
||||||
|
@ -2193,23 +2193,23 @@ static int
|
|||||||
qemuProcessSetVcpuAffinities(virDomainObjPtr vm)
|
qemuProcessSetVcpuAffinities(virDomainObjPtr vm)
|
||||||
{
|
{
|
||||||
virDomainDefPtr def = vm->def;
|
virDomainDefPtr def = vm->def;
|
||||||
virDomainPinDefPtr pininfo;
|
virDomainVcpuInfoPtr vcpu;
|
||||||
int n;
|
size_t i;
|
||||||
int ret = -1;
|
int ret = -1;
|
||||||
VIR_DEBUG("Setting affinity on CPUs nvcpupin=%zu nvcpus=%d hasVcpupids=%d",
|
VIR_DEBUG("Setting affinity on CPUs");
|
||||||
def->cputune.nvcpupin, virDomainDefGetVcpus(def),
|
|
||||||
qemuDomainHasVcpuPids(vm));
|
|
||||||
if (!def->cputune.nvcpupin)
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
if (!qemuDomainHasVcpuPids(vm)) {
|
if (!qemuDomainHasVcpuPids(vm)) {
|
||||||
/* If any CPU has custom affinity that differs from the
|
/* If any CPU has custom affinity that differs from the
|
||||||
* VM default affinity, we must reject it
|
* VM default affinity, we must reject it
|
||||||
*/
|
*/
|
||||||
for (n = 0; n < def->cputune.nvcpupin; n++) {
|
for (i = 0; i < virDomainDefGetVcpusMax(def); i++) {
|
||||||
if (def->cputune.vcpupin[n]->cpumask &&
|
vcpu = virDomainDefGetVcpu(def, i);
|
||||||
!virBitmapEqual(def->cpumask,
|
|
||||||
def->cputune.vcpupin[n]->cpumask)) {
|
if (!vcpu->online)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (vcpu->cpumask &&
|
||||||
|
!virBitmapEqual(def->cpumask, vcpu->cpumask)) {
|
||||||
virReportError(VIR_ERR_OPERATION_INVALID,
|
virReportError(VIR_ERR_OPERATION_INVALID,
|
||||||
"%s", _("cpu affinity is not supported"));
|
"%s", _("cpu affinity is not supported"));
|
||||||
return -1;
|
return -1;
|
||||||
@ -2218,19 +2218,19 @@ qemuProcessSetVcpuAffinities(virDomainObjPtr vm)
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (n = 0; n < virDomainDefGetVcpus(def); n++) {
|
for (i = 0; i < virDomainDefGetVcpusMax(def); i++) {
|
||||||
virBitmapPtr bitmap;
|
virBitmapPtr bitmap;
|
||||||
/* set affinity only for existing vcpus */
|
|
||||||
if (!(pininfo = virDomainPinFind(def->cputune.vcpupin,
|
vcpu = virDomainDefGetVcpu(def, i);
|
||||||
def->cputune.nvcpupin,
|
|
||||||
n)))
|
if (!vcpu->online)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
if (!(bitmap = pininfo->cpumask) &&
|
if (!(bitmap = vcpu->cpumask) &&
|
||||||
!(bitmap = def->cpumask))
|
!(bitmap = def->cpumask))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
if (virProcessSetAffinity(qemuDomainGetVcpuPid(vm, n), bitmap) < 0)
|
if (virProcessSetAffinity(qemuDomainGetVcpuPid(vm, i), bitmap) < 0)
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2455,15 +2455,14 @@ static int testDomainGetVcpus(virDomainPtr domain,
|
|||||||
memset(cpumaps, 0, maxinfo * maplen);
|
memset(cpumaps, 0, maxinfo * maplen);
|
||||||
|
|
||||||
for (i = 0; i < maxinfo; i++) {
|
for (i = 0; i < maxinfo; i++) {
|
||||||
virDomainPinDefPtr pininfo;
|
virDomainVcpuInfoPtr vcpu = virDomainDefGetVcpu(def, i);
|
||||||
virBitmapPtr bitmap = NULL;
|
virBitmapPtr bitmap = NULL;
|
||||||
|
|
||||||
pininfo = virDomainPinFind(def->cputune.vcpupin,
|
if (!vcpu->online)
|
||||||
def->cputune.nvcpupin,
|
continue;
|
||||||
i);
|
|
||||||
|
|
||||||
if (pininfo && pininfo->cpumask)
|
if (vcpu->cpumask)
|
||||||
bitmap = pininfo->cpumask;
|
bitmap = vcpu->cpumask;
|
||||||
else if (def->cpumask)
|
else if (def->cpumask)
|
||||||
bitmap = def->cpumask;
|
bitmap = def->cpumask;
|
||||||
else
|
else
|
||||||
@ -2492,6 +2491,7 @@ static int testDomainPinVcpu(virDomainPtr domain,
|
|||||||
unsigned char *cpumap,
|
unsigned char *cpumap,
|
||||||
int maplen)
|
int maplen)
|
||||||
{
|
{
|
||||||
|
virDomainVcpuInfoPtr vcpuinfo;
|
||||||
virDomainObjPtr privdom;
|
virDomainObjPtr privdom;
|
||||||
virDomainDefPtr def;
|
virDomainDefPtr def;
|
||||||
int ret = -1;
|
int ret = -1;
|
||||||
@ -2507,29 +2507,21 @@ static int testDomainPinVcpu(virDomainPtr domain,
|
|||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (vcpu > virDomainDefGetVcpus(privdom->def)) {
|
if (!(vcpuinfo = virDomainDefGetVcpu(def, vcpu)) ||
|
||||||
|
!vcpuinfo->online) {
|
||||||
virReportError(VIR_ERR_INVALID_ARG,
|
virReportError(VIR_ERR_INVALID_ARG,
|
||||||
_("requested vcpu '%d' is not present in the domain"),
|
_("requested vcpu '%d' is not present in the domain"),
|
||||||
vcpu);
|
vcpu);
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!def->cputune.vcpupin) {
|
virBitmapFree(vcpuinfo->cpumask);
|
||||||
if (VIR_ALLOC(def->cputune.vcpupin) < 0)
|
|
||||||
|
if (!(vcpuinfo->cpumask = virBitmapNewData(cpumap, maplen)))
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
def->cputune.nvcpupin = 0;
|
|
||||||
}
|
|
||||||
if (virDomainPinAdd(&def->cputune.vcpupin,
|
|
||||||
&def->cputune.nvcpupin,
|
|
||||||
cpumap,
|
|
||||||
maplen,
|
|
||||||
vcpu) < 0) {
|
|
||||||
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
|
|
||||||
_("failed to update or add vcpupin"));
|
|
||||||
goto cleanup;
|
|
||||||
}
|
|
||||||
|
|
||||||
ret = 0;
|
ret = 0;
|
||||||
|
|
||||||
cleanup:
|
cleanup:
|
||||||
virDomainObjEndAPI(&privdom);
|
virDomainObjEndAPI(&privdom);
|
||||||
return ret;
|
return ret;
|
||||||
@ -2566,15 +2558,14 @@ testDomainGetVcpuPinInfo(virDomainPtr dom,
|
|||||||
ncpumaps = virDomainDefGetVcpus(def);
|
ncpumaps = virDomainDefGetVcpus(def);
|
||||||
|
|
||||||
for (vcpu = 0; vcpu < ncpumaps; vcpu++) {
|
for (vcpu = 0; vcpu < ncpumaps; vcpu++) {
|
||||||
virDomainPinDefPtr pininfo;
|
virDomainVcpuInfoPtr vcpuinfo = virDomainDefGetVcpu(def, vcpu);
|
||||||
virBitmapPtr bitmap = NULL;
|
virBitmapPtr bitmap = NULL;
|
||||||
|
|
||||||
pininfo = virDomainPinFind(def->cputune.vcpupin,
|
if (!vcpuinfo->online)
|
||||||
def->cputune.nvcpupin,
|
continue;
|
||||||
vcpu);
|
|
||||||
|
|
||||||
if (pininfo && pininfo->cpumask)
|
if (vcpuinfo->cpumask)
|
||||||
bitmap = pininfo->cpumask;
|
bitmap = vcpuinfo->cpumask;
|
||||||
else if (def->cpumask)
|
else if (def->cpumask)
|
||||||
bitmap = def->cpumask;
|
bitmap = def->cpumask;
|
||||||
else
|
else
|
||||||
|
@ -1957,16 +1957,16 @@ prlsdkCheckUnsupportedParams(PRL_HANDLE sdkdom, virDomainDefPtr def)
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (def->cputune.vcpupin) {
|
for (i = 0; i < virDomainDefGetVcpusMax(def); i++) {
|
||||||
for (i = 0; i < def->cputune.nvcpupin; i++) {
|
virDomainVcpuInfoPtr vcpu = virDomainDefGetVcpu(def, i);
|
||||||
if (def->cputune.vcpupin[i]->cpumask &&
|
|
||||||
!virBitmapEqual(def->cpumask, def->cputune.vcpupin[i]->cpumask)) {
|
if (vcpu->cpumask &&
|
||||||
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
|
!virBitmapEqual(def->cpumask, vcpu->cpumask)) {
|
||||||
"%s", _("vcpupin cpumask differs from default cpumask"));
|
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
|
||||||
|
_("vcpupin cpumask differs from default cpumask"));
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
Loading…
x
Reference in New Issue
Block a user