2017-04-04 16:22:31 +00:00
|
|
|
/*
|
|
|
|
* qemu_extdevice.c: QEMU external devices support
|
|
|
|
*
|
|
|
|
* Copyright (C) 2014, 2018 IBM Corporation
|
|
|
|
*
|
|
|
|
* This library is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Lesser General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
|
|
* License along with this library. If not, see
|
|
|
|
* <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <config.h>
|
|
|
|
|
|
|
|
#include "qemu_extdevice.h"
|
2019-09-23 10:44:40 +00:00
|
|
|
#include "qemu_vhost_user_gpu.h"
|
2020-04-08 17:23:50 +00:00
|
|
|
#include "qemu_dbus.h"
|
2017-04-04 16:22:31 +00:00
|
|
|
#include "qemu_domain.h"
|
|
|
|
#include "qemu_tpm.h"
|
2022-12-15 19:19:16 +00:00
|
|
|
#include "qemu_passt.h"
|
2019-08-08 14:55:10 +00:00
|
|
|
#include "qemu_slirp.h"
|
2019-11-01 11:34:52 +00:00
|
|
|
#include "qemu_virtiofs.h"
|
2017-04-04 16:22:31 +00:00
|
|
|
|
|
|
|
#include "virlog.h"
|
|
|
|
#include "virtime.h"
|
|
|
|
|
|
|
|
#define VIR_FROM_THIS VIR_FROM_QEMU
|
|
|
|
|
2019-01-20 16:32:42 +00:00
|
|
|
VIR_LOG_INIT("qemu.qemu_extdevice");
|
2017-04-04 16:22:31 +00:00
|
|
|
|
|
|
|
int
|
2021-03-11 07:16:13 +00:00
|
|
|
qemuExtDeviceLogCommand(virQEMUDriver *driver,
|
|
|
|
virDomainObj *vm,
|
|
|
|
virCommand *cmd,
|
2017-04-04 16:22:31 +00:00
|
|
|
const char *info)
|
|
|
|
{
|
2019-10-15 13:16:31 +00:00
|
|
|
g_autofree char *timestamp = virTimeStringNow();
|
|
|
|
g_autofree char *cmds = virCommandToString(cmd, false);
|
2017-04-04 16:22:31 +00:00
|
|
|
|
2019-08-08 14:54:55 +00:00
|
|
|
if (!timestamp || !cmds)
|
|
|
|
return -1;
|
2017-04-04 16:22:31 +00:00
|
|
|
|
2019-08-08 14:54:55 +00:00
|
|
|
return qemuDomainLogAppendMessage(driver, vm,
|
|
|
|
_("%s: Starting external device: %s\n%s\n"),
|
|
|
|
timestamp, info, cmds);
|
2017-04-04 16:22:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-08-08 14:54:55 +00:00
|
|
|
|
2017-04-04 16:22:31 +00:00
|
|
|
/*
|
|
|
|
* qemuExtDevicesInitPaths:
|
|
|
|
*
|
|
|
|
* @driver: QEMU driver
|
|
|
|
* @def: domain definition
|
|
|
|
*
|
|
|
|
* Initialize paths of external devices so that it is known where state is
|
|
|
|
* stored and we can remove directories and files in case of domain XML
|
|
|
|
* changes.
|
|
|
|
*/
|
2022-12-05 12:08:25 +00:00
|
|
|
int
|
2021-03-11 07:16:13 +00:00
|
|
|
qemuExtDevicesInitPaths(virQEMUDriver *driver,
|
|
|
|
virDomainDef *def)
|
2017-04-04 16:22:31 +00:00
|
|
|
{
|
2022-04-29 14:38:33 +00:00
|
|
|
size_t i;
|
|
|
|
|
|
|
|
for (i = 0; i < def->ntpms; i++) {
|
|
|
|
virDomainTPMDef *tpm = def->tpms[i];
|
|
|
|
|
|
|
|
if (tpm->type == VIR_DOMAIN_TPM_TYPE_EMULATOR &&
|
|
|
|
qemuExtTPMInitPaths(driver, def, tpm) < 0)
|
|
|
|
return -1;
|
|
|
|
}
|
2017-04-04 16:22:31 +00:00
|
|
|
|
2020-06-10 18:11:45 +00:00
|
|
|
return 0;
|
2017-04-04 16:22:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-09-23 10:44:40 +00:00
|
|
|
/*
|
|
|
|
* qemuExtDevicesPrepareDomain:
|
|
|
|
*
|
|
|
|
* @driver: QEMU driver
|
|
|
|
* @vm: domain
|
|
|
|
*
|
|
|
|
* Code that modifies live XML of a domain which is about to start.
|
|
|
|
*/
|
|
|
|
int
|
2021-03-11 07:16:13 +00:00
|
|
|
qemuExtDevicesPrepareDomain(virQEMUDriver *driver,
|
|
|
|
virDomainObj *vm)
|
2019-09-23 10:44:40 +00:00
|
|
|
{
|
|
|
|
int ret = 0;
|
|
|
|
size_t i;
|
|
|
|
|
2022-12-05 11:27:24 +00:00
|
|
|
if (qemuExtDevicesInitPaths(driver, vm->def) < 0)
|
|
|
|
return -1;
|
|
|
|
|
2019-09-23 10:44:40 +00:00
|
|
|
for (i = 0; i < vm->def->nvideos; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainVideoDef *video = vm->def->videos[i];
|
2019-09-23 10:44:40 +00:00
|
|
|
|
|
|
|
if (video->backend == VIR_DOMAIN_VIDEO_BACKEND_TYPE_VHOSTUSER) {
|
|
|
|
if ((ret = qemuExtVhostUserGPUPrepareDomain(driver, video)) < 0)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-28 14:38:52 +00:00
|
|
|
for (i = 0; i < vm->def->nfss; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainFSDef *fs = vm->def->fss[i];
|
2020-01-28 14:38:52 +00:00
|
|
|
|
|
|
|
if (fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS) {
|
|
|
|
if (qemuVirtioFSPrepareDomain(driver, fs) < 0)
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-23 10:44:40 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-04-04 16:22:31 +00:00
|
|
|
/*
|
|
|
|
* qemuExtDevicesPrepareHost:
|
|
|
|
*
|
|
|
|
* @driver: QEMU driver
|
|
|
|
* @def: domain definition
|
|
|
|
*
|
|
|
|
* Prepare host storage paths for external devices.
|
|
|
|
*/
|
|
|
|
int
|
2021-03-11 07:16:13 +00:00
|
|
|
qemuExtDevicesPrepareHost(virQEMUDriver *driver,
|
|
|
|
virDomainObj *vm)
|
2017-04-04 16:22:31 +00:00
|
|
|
{
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainDef *def = vm->def;
|
2022-04-29 14:38:33 +00:00
|
|
|
size_t i;
|
2017-04-04 16:22:31 +00:00
|
|
|
|
2022-04-29 14:38:33 +00:00
|
|
|
for (i = 0; i < def->ntpms; i++) {
|
|
|
|
virDomainTPMDef *tpm = def->tpms[i];
|
|
|
|
|
|
|
|
if (tpm->type == VIR_DOMAIN_TPM_TYPE_EMULATOR &&
|
|
|
|
qemuExtTPMPrepareHost(driver, def, tpm) < 0)
|
|
|
|
return -1;
|
|
|
|
}
|
2017-04-04 16:22:31 +00:00
|
|
|
|
2019-08-08 14:55:10 +00:00
|
|
|
return 0;
|
2017-04-04 16:22:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
void
|
2021-03-11 07:16:13 +00:00
|
|
|
qemuExtDevicesCleanupHost(virQEMUDriver *driver,
|
2022-10-04 13:38:13 +00:00
|
|
|
virDomainDef *def,
|
2022-10-24 10:28:48 +00:00
|
|
|
virDomainUndefineFlagsValues flags,
|
|
|
|
bool outgoingMigration)
|
2017-04-04 16:22:31 +00:00
|
|
|
{
|
2022-04-29 14:38:33 +00:00
|
|
|
size_t i;
|
|
|
|
|
2017-04-04 16:22:31 +00:00
|
|
|
if (qemuExtDevicesInitPaths(driver, def) < 0)
|
|
|
|
return;
|
|
|
|
|
2022-04-29 14:38:33 +00:00
|
|
|
for (i = 0; i < def->ntpms; i++) {
|
2023-02-10 08:47:05 +00:00
|
|
|
virDomainTPMDef *tpm = def->tpms[i];
|
|
|
|
|
|
|
|
if (tpm->type == VIR_DOMAIN_TPM_TYPE_EMULATOR)
|
|
|
|
qemuExtTPMCleanupHost(tpm, flags, outgoingMigration);
|
2022-04-29 14:38:33 +00:00
|
|
|
}
|
2017-04-04 16:22:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
int
|
2021-03-11 07:16:13 +00:00
|
|
|
qemuExtDevicesStart(virQEMUDriver *driver,
|
|
|
|
virDomainObj *vm,
|
2019-07-26 20:41:10 +00:00
|
|
|
bool incomingMigration)
|
2017-04-04 16:22:31 +00:00
|
|
|
{
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainDef *def = vm->def;
|
2019-08-08 14:55:10 +00:00
|
|
|
size_t i;
|
2017-04-04 16:22:31 +00:00
|
|
|
|
2020-01-24 18:41:45 +00:00
|
|
|
for (i = 0; i < def->nvideos; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainVideoDef *video = def->videos[i];
|
2019-09-23 10:44:41 +00:00
|
|
|
|
|
|
|
if (video->backend == VIR_DOMAIN_VIDEO_BACKEND_TYPE_VHOSTUSER) {
|
2020-01-24 18:43:18 +00:00
|
|
|
if (qemuExtVhostUserGPUStart(driver, vm, video) < 0)
|
|
|
|
return -1;
|
2019-09-23 10:44:41 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-29 14:38:33 +00:00
|
|
|
for (i = 0; i < def->ntpms; i++) {
|
|
|
|
virDomainTPMDef *tpm = def->tpms[i];
|
|
|
|
|
|
|
|
if (tpm->type == VIR_DOMAIN_TPM_TYPE_EMULATOR &&
|
|
|
|
qemuExtTPMStart(driver, vm, tpm, incomingMigration) < 0)
|
|
|
|
return -1;
|
|
|
|
}
|
2017-04-04 16:22:31 +00:00
|
|
|
|
2019-08-08 14:55:10 +00:00
|
|
|
for (i = 0; i < def->nnets; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainNetDef *net = def->nets[i];
|
2019-08-08 14:55:10 +00:00
|
|
|
|
2022-12-15 19:19:16 +00:00
|
|
|
if (net->type != VIR_DOMAIN_NET_TYPE_USER)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (net->backend.type == VIR_DOMAIN_NET_BACKEND_PASST) {
|
|
|
|
if (qemuPasstStart(vm, net) < 0)
|
|
|
|
return -1;
|
|
|
|
} else {
|
|
|
|
if (qemuSlirpStart(vm, net, incomingMigration) < 0)
|
|
|
|
return -1;
|
|
|
|
}
|
2019-08-08 14:55:10 +00:00
|
|
|
}
|
|
|
|
|
2019-11-01 11:34:52 +00:00
|
|
|
for (i = 0; i < def->nfss; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainFSDef *fs = def->fss[i];
|
2019-11-01 11:34:52 +00:00
|
|
|
|
2021-08-18 12:05:46 +00:00
|
|
|
if (fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS && !fs->sock) {
|
2021-10-06 09:33:08 +00:00
|
|
|
if (qemuVirtioFSStart(driver, vm, fs) < 0)
|
2019-11-01 11:34:52 +00:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-13 18:38:12 +00:00
|
|
|
for (i = 0; i < def->ngraphics; i++) {
|
|
|
|
virDomainGraphicsDef *graphics = def->graphics[i];
|
|
|
|
|
|
|
|
if (graphics->type != VIR_DOMAIN_GRAPHICS_TYPE_DBUS)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (graphics->data.dbus.p2p || graphics->data.dbus.fromConfig)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (qemuDBusStart(driver, vm) < 0)
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2020-01-24 18:43:18 +00:00
|
|
|
return 0;
|
2017-04-04 16:22:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
void
|
2021-03-11 07:16:13 +00:00
|
|
|
qemuExtDevicesStop(virQEMUDriver *driver,
|
2022-10-24 10:28:48 +00:00
|
|
|
virDomainObj *vm,
|
|
|
|
bool outgoingMigration)
|
2017-04-04 16:22:31 +00:00
|
|
|
{
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainDef *def = vm->def;
|
2019-08-08 14:55:10 +00:00
|
|
|
size_t i;
|
|
|
|
|
2019-09-23 10:44:41 +00:00
|
|
|
if (qemuExtDevicesInitPaths(driver, def) < 0)
|
2017-04-04 16:22:31 +00:00
|
|
|
return;
|
|
|
|
|
2019-09-23 10:44:41 +00:00
|
|
|
for (i = 0; i < def->nvideos; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainVideoDef *video = def->videos[i];
|
2019-09-23 10:44:41 +00:00
|
|
|
|
|
|
|
if (video->backend == VIR_DOMAIN_VIDEO_BACKEND_TYPE_VHOSTUSER)
|
|
|
|
qemuExtVhostUserGPUStop(driver, vm, video);
|
|
|
|
}
|
|
|
|
|
2022-04-29 14:38:33 +00:00
|
|
|
for (i = 0; i < def->ntpms; i++) {
|
|
|
|
if (def->tpms[i]->type == VIR_DOMAIN_TPM_TYPE_EMULATOR)
|
2022-10-24 10:28:48 +00:00
|
|
|
qemuExtTPMStop(driver, vm, outgoingMigration);
|
2022-04-29 14:38:33 +00:00
|
|
|
}
|
2019-08-08 14:55:10 +00:00
|
|
|
|
|
|
|
for (i = 0; i < def->nnets; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainNetDef *net = def->nets[i];
|
2020-05-26 00:41:00 +00:00
|
|
|
virDomainNetType actualType = virDomainNetGetActualType(net);
|
2021-03-11 07:16:13 +00:00
|
|
|
qemuSlirp *slirp = QEMU_DOMAIN_NETWORK_PRIVATE(net)->slirp;
|
2019-08-08 14:55:10 +00:00
|
|
|
|
|
|
|
if (slirp)
|
2020-02-25 09:55:05 +00:00
|
|
|
qemuSlirpStop(slirp, vm, driver, net);
|
2022-12-15 19:19:16 +00:00
|
|
|
|
|
|
|
if (net->type == VIR_DOMAIN_NET_TYPE_USER &&
|
|
|
|
net->backend.type == VIR_DOMAIN_NET_BACKEND_PASST) {
|
|
|
|
qemuPasstStop(vm, net);
|
|
|
|
}
|
|
|
|
|
2020-05-26 00:41:00 +00:00
|
|
|
if (actualType == VIR_DOMAIN_NET_TYPE_ETHERNET && net->downscript)
|
|
|
|
virNetDevRunEthernetScript(net->ifname, net->downscript);
|
2019-08-08 14:55:10 +00:00
|
|
|
}
|
2019-11-01 11:34:52 +00:00
|
|
|
|
|
|
|
for (i = 0; i < def->nfss; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainFSDef *fs = def->fss[i];
|
2019-11-01 11:34:52 +00:00
|
|
|
|
2021-02-13 12:22:56 +00:00
|
|
|
if (!fs->sock &&
|
|
|
|
fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS)
|
2019-11-01 11:34:52 +00:00
|
|
|
qemuVirtioFSStop(driver, vm, fs);
|
|
|
|
}
|
2017-04-04 16:22:31 +00:00
|
|
|
}
|
2018-04-05 19:06:55 +00:00
|
|
|
|
|
|
|
|
|
|
|
bool
|
2021-03-11 07:16:13 +00:00
|
|
|
qemuExtDevicesHasDevice(virDomainDef *def)
|
2018-04-05 19:06:55 +00:00
|
|
|
{
|
2019-09-23 10:44:41 +00:00
|
|
|
size_t i;
|
|
|
|
|
|
|
|
for (i = 0; i < def->nvideos; i++) {
|
|
|
|
if (def->videos[i]->backend == VIR_DOMAIN_VIDEO_BACKEND_TYPE_VHOSTUSER)
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2023-02-13 15:01:32 +00:00
|
|
|
for (i = 0; i < def->nnets; i++) {
|
|
|
|
virDomainNetDef *net = def->nets[i];
|
|
|
|
|
|
|
|
if (QEMU_DOMAIN_NETWORK_PRIVATE(net)->slirp)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
if (net->type == VIR_DOMAIN_NET_TYPE_USER &&
|
|
|
|
net->backend.type == VIR_DOMAIN_NET_BACKEND_PASST)
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2020-06-10 18:11:47 +00:00
|
|
|
for (i = 0; i < def->ntpms; i++) {
|
|
|
|
if (def->tpms[i]->type == VIR_DOMAIN_TPM_TYPE_EMULATOR)
|
|
|
|
return true;
|
|
|
|
}
|
2018-04-05 19:06:55 +00:00
|
|
|
|
2020-02-18 15:12:29 +00:00
|
|
|
for (i = 0; i < def->nfss; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainFSDef *fs = def->fss[i];
|
2020-02-18 15:12:29 +00:00
|
|
|
|
|
|
|
if (fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS)
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2018-04-05 19:06:55 +00:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
int
|
2021-03-11 07:16:13 +00:00
|
|
|
qemuExtDevicesSetupCgroup(virQEMUDriver *driver,
|
|
|
|
virDomainObj *vm,
|
|
|
|
virCgroup *cgroup)
|
2018-04-05 19:06:55 +00:00
|
|
|
{
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainDef *def = vm->def;
|
2019-09-23 10:44:41 +00:00
|
|
|
size_t i;
|
2018-04-05 19:06:55 +00:00
|
|
|
|
2023-02-15 14:52:21 +00:00
|
|
|
/* Don't forget to adjust qemuExtDevicesHasDevice() accordingly.
|
|
|
|
* Otherwise, this function might not be called at all. */
|
|
|
|
|
2020-10-28 02:20:37 +00:00
|
|
|
if (qemuDBusSetupCgroup(driver, vm, cgroup) < 0)
|
2020-04-08 17:23:50 +00:00
|
|
|
return -1;
|
|
|
|
|
2019-09-23 10:44:41 +00:00
|
|
|
for (i = 0; i < def->nvideos; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainVideoDef *video = def->videos[i];
|
2018-04-05 19:06:55 +00:00
|
|
|
|
2019-09-23 10:44:41 +00:00
|
|
|
if (video->backend == VIR_DOMAIN_VIDEO_BACKEND_TYPE_VHOSTUSER &&
|
|
|
|
qemuExtVhostUserGPUSetupCgroup(driver, def, video, cgroup) < 0)
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2020-04-08 17:23:47 +00:00
|
|
|
for (i = 0; i < def->nnets; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainNetDef *net = def->nets[i];
|
|
|
|
qemuSlirp *slirp = QEMU_DOMAIN_NETWORK_PRIVATE(net)->slirp;
|
2020-04-08 17:23:47 +00:00
|
|
|
|
|
|
|
if (slirp && qemuSlirpSetupCgroup(slirp, cgroup) < 0)
|
|
|
|
return -1;
|
2022-12-15 19:19:16 +00:00
|
|
|
|
|
|
|
if (net->type == VIR_DOMAIN_NET_TYPE_USER &&
|
|
|
|
net->backend.type == VIR_DOMAIN_NET_BACKEND_PASST &&
|
|
|
|
qemuPasstSetupCgroup(vm, net, cgroup) < 0) {
|
|
|
|
return -1;
|
|
|
|
}
|
2020-04-08 17:23:47 +00:00
|
|
|
}
|
|
|
|
|
2022-04-29 14:38:33 +00:00
|
|
|
for (i = 0; i < def->ntpms; i++) {
|
|
|
|
if (def->tpms[i]->type == VIR_DOMAIN_TPM_TYPE_EMULATOR &&
|
|
|
|
qemuExtTPMSetupCgroup(driver, def, cgroup) < 0)
|
|
|
|
return -1;
|
|
|
|
}
|
2019-09-23 10:44:41 +00:00
|
|
|
|
2020-02-18 15:12:29 +00:00
|
|
|
for (i = 0; i < def->nfss; i++) {
|
2021-03-11 07:16:13 +00:00
|
|
|
virDomainFSDef *fs = def->fss[i];
|
2020-02-18 15:12:29 +00:00
|
|
|
|
2021-02-13 12:22:56 +00:00
|
|
|
if (!fs->sock &&
|
|
|
|
fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS &&
|
2020-02-18 15:12:29 +00:00
|
|
|
qemuVirtioFSSetupCgroup(vm, fs, cgroup) < 0)
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2019-09-23 10:44:41 +00:00
|
|
|
return 0;
|
2018-04-05 19:06:55 +00:00
|
|
|
}
|