passt: Relicense to GPL 2.0, or any later version
In practical terms, passt doesn't benefit from the additional
protection offered by the AGPL over the GPL, because it's not
suitable to be executed over a computer network.
Further, restricting the distribution under the version 3 of the GPL
wouldn't provide any practical advantage either, as long as the passt
codebase is concerned, and might cause unnecessary compatibility
dilemmas.
Change licensing terms to the GNU General Public License Version 2,
or any later version, with written permission from all current and
past contributors, namely: myself, David Gibson, Laine Stump, Andrea
Bolognani, Paul Holzinger, Richard W.M. Jones, Chris Kuhn, Florian
Weimer, Giuseppe Scrivano, Stefan Hajnoczi, and Vasiliy Ulyanov.
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-04-05 18:11:44 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
/* PASTA - Pack A Subtle Tap Abstraction
|
|
|
|
* for network namespace/tap device mode
|
|
|
|
*
|
|
|
|
* tcp_splice.c - direct namespace forwarding for local connections
|
|
|
|
*
|
|
|
|
* Copyright (c) 2020-2022 Red Hat GmbH
|
|
|
|
* Author: Stefano Brivio <sbrivio@redhat.com>
|
|
|
|
*/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* DOC: Theory of Operation
|
|
|
|
*
|
|
|
|
*
|
2022-11-17 05:58:42 +00:00
|
|
|
* For local traffic directed to TCP ports configured for direct
|
|
|
|
* mapping between namespaces, packets are directly translated between
|
|
|
|
* L4 sockets using a pair of splice() syscalls. These connections are
|
2022-11-17 05:58:46 +00:00
|
|
|
* tracked by struct tcp_splice_conn entries in the @tc array, using
|
2022-11-17 05:58:42 +00:00
|
|
|
* these events:
|
2022-03-15 00:07:02 +00:00
|
|
|
*
|
|
|
|
* - SPLICE_CONNECT: connection accepted, connecting to target
|
|
|
|
* - SPLICE_ESTABLISHED: connection to target established
|
2023-11-07 02:42:46 +00:00
|
|
|
* - OUT_WAIT_0: pipe to accepted socket full, wait for EPOLLOUT
|
|
|
|
* - OUT_WAIT_1: pipe to target socket full, wait for EPOLLOUT
|
|
|
|
* - FIN_RCVD_0: FIN (EPOLLRDHUP) seen from accepted socket
|
|
|
|
* - FIN_RCVD_1: FIN (EPOLLRDHUP) seen from target socket
|
|
|
|
* - FIN_SENT_0: FIN (write shutdown) sent to accepted socket
|
|
|
|
* - FIN_SENT_1: FIN (write shutdown) sent to target socket
|
2022-03-15 00:07:02 +00:00
|
|
|
*
|
2024-04-11 15:34:04 +00:00
|
|
|
* #syscalls:pasta pipe2|pipe fcntl arm:fcntl64 ppc64:fcntl64
|
2022-03-15 00:07:02 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <sched.h>
|
2023-03-21 03:54:59 +00:00
|
|
|
#include <unistd.h>
|
2023-03-08 03:00:22 +00:00
|
|
|
#include <signal.h>
|
2022-03-15 00:07:02 +00:00
|
|
|
#include <errno.h>
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <limits.h>
|
|
|
|
#include <stdint.h>
|
tcp, tcp_splice: Fix port remapping for inbound, spliced connections
In pasta mode, when we receive a new inbound connection, we need to
select a socket that was created in the namespace to proceed and
connect() it to its final destination.
The existing condition might pick a wrong socket, though, if the
destination port is remapped, because we'll check the bitmap of
inbound ports using the remapped port (stored in the epoll reference)
as index, and not the original port.
Instead of using the port bitmap for this purpose, store this
information in the epoll reference itself, by adding a new 'outbound'
bit, that's set if the listening socket was created the namespace,
and unset otherwise.
Then, use this bit to pick a socket on the right side.
Suggested-by: David Gibson <david@gibson.dropbear.id.au>
Fixes: 33482d5bf293 ("passt: Add PASTA mode, major rework")
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: David Gibson <david@gibson.dropbear.id.au>
2022-10-10 17:00:43 +00:00
|
|
|
#include <stdbool.h>
|
2022-03-15 00:07:02 +00:00
|
|
|
#include <string.h>
|
|
|
|
#include <time.h>
|
|
|
|
#include <net/ethernet.h>
|
|
|
|
#include <netinet/in.h>
|
|
|
|
#include <netinet/tcp.h>
|
|
|
|
#include <sys/epoll.h>
|
|
|
|
#include <sys/types.h>
|
|
|
|
#include <sys/socket.h>
|
|
|
|
|
|
|
|
#include "util.h"
|
2024-03-06 05:58:33 +00:00
|
|
|
#include "ip.h"
|
2022-03-15 00:07:02 +00:00
|
|
|
#include "passt.h"
|
2022-09-24 07:53:15 +00:00
|
|
|
#include "log.h"
|
2022-11-17 05:58:39 +00:00
|
|
|
#include "tcp_splice.h"
|
2023-09-28 01:21:02 +00:00
|
|
|
#include "siphash.h"
|
2022-11-17 05:58:55 +00:00
|
|
|
#include "inany.h"
|
2023-11-30 02:02:08 +00:00
|
|
|
#include "flow.h"
|
2022-03-15 00:07:02 +00:00
|
|
|
|
2023-11-30 02:02:09 +00:00
|
|
|
#include "flow_table.h"
|
2022-11-17 05:58:43 +00:00
|
|
|
|
2022-04-07 09:41:50 +00:00
|
|
|
#define MAX_PIPE_SIZE (8UL * 1024 * 1024)
|
tcp_splice: Don't pool pipes in pairs
To reduce latencies, the tcp splice code maintains a pool of pre-opened
pipes to use for new connections. This is structured as an array of pairs
of pipes, with each pipe, of course, being a pair of fds. Thus when we
use the pool, a single pool "slot" provides both the a->b and b->a pipes.
There's no strong reason to store the pool in pairs, though - we can
with not much difficulty instead take the a->b and b->a pipes for a new
connection independently from separate slots in the pool, or even take one
from the the pool and create the other as we need it, if there's only one
pipe left in the pool.
This marginally increases the length of code, but simplifies the structure
of the pipe pool. We should be able to re-shrink the code with later
changes, too.
In the process we also fix some minor bugs:
- If we both failed to find a pipe in the pool and to create a new one, we
didn't log an error and would silently drop the connection. That could
make debugging such a situation difficult. Add in an error message for
that case
- When refilling the pool, if we were only able to open a single pipe in
the pair, we attempted to rollback, but instead of closing the opened
pipe, we instead closed the pipe we failed to open (probably leading to
some ignored EBADFD errors).
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-07 02:42:45 +00:00
|
|
|
#define TCP_SPLICE_PIPE_POOL_SIZE 32
|
2022-11-17 05:58:46 +00:00
|
|
|
#define TCP_SPLICE_CONN_PRESSURE 30 /* % of conn_count */
|
2022-03-18 23:33:46 +00:00
|
|
|
#define TCP_SPLICE_FILE_PRESSURE 30 /* % of c->nofile */
|
2022-03-15 00:07:02 +00:00
|
|
|
|
2023-02-13 23:48:21 +00:00
|
|
|
/* Pools for pre-opened sockets (in namespace) */
|
|
|
|
#define TCP_SOCK_POOL_TSH 16 /* Refill in ns if > x used */
|
|
|
|
|
|
|
|
static int ns_sock_pool4 [TCP_SOCK_POOL_SIZE];
|
|
|
|
static int ns_sock_pool6 [TCP_SOCK_POOL_SIZE];
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
/* Pool of pre-opened pipes */
|
tcp_splice: Don't pool pipes in pairs
To reduce latencies, the tcp splice code maintains a pool of pre-opened
pipes to use for new connections. This is structured as an array of pairs
of pipes, with each pipe, of course, being a pair of fds. Thus when we
use the pool, a single pool "slot" provides both the a->b and b->a pipes.
There's no strong reason to store the pool in pairs, though - we can
with not much difficulty instead take the a->b and b->a pipes for a new
connection independently from separate slots in the pool, or even take one
from the the pool and create the other as we need it, if there's only one
pipe left in the pool.
This marginally increases the length of code, but simplifies the structure
of the pipe pool. We should be able to re-shrink the code with later
changes, too.
In the process we also fix some minor bugs:
- If we both failed to find a pipe in the pool and to create a new one, we
didn't log an error and would silently drop the connection. That could
make debugging such a situation difficult. Add in an error message for
that case
- When refilling the pool, if we were only able to open a single pipe in
the pair, we attempted to rollback, but instead of closing the opened
pipe, we instead closed the pipe we failed to open (probably leading to
some ignored EBADFD errors).
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-07 02:42:45 +00:00
|
|
|
static int splice_pipe_pool [TCP_SPLICE_PIPE_POOL_SIZE][2];
|
2022-03-15 00:07:02 +00:00
|
|
|
|
2024-06-06 10:09:45 +00:00
|
|
|
#define CONN_V6(x) ((x)->flags & SPLICE_V6)
|
2022-03-15 00:07:02 +00:00
|
|
|
#define CONN_V4(x) (!CONN_V6(x))
|
2024-06-06 10:09:45 +00:00
|
|
|
#define CONN_HAS(conn, set) (((conn)->events & (set)) == (set))
|
2023-11-30 02:02:10 +00:00
|
|
|
#define CONN(idx) (&FLOW(idx)->tcp_splice)
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
/* Display strings for connection events */
|
|
|
|
static const char *tcp_splice_event_str[] __attribute((__unused__)) = {
|
2023-11-07 02:42:46 +00:00
|
|
|
"SPLICE_CONNECT", "SPLICE_ESTABLISHED", "OUT_WAIT_0", "OUT_WAIT_1",
|
|
|
|
"FIN_RCVD_0", "FIN_RCVD_1", "FIN_SENT_0", "FIN_SENT_1",
|
2022-03-15 00:07:02 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/* Display strings for connection flags */
|
|
|
|
static const char *tcp_splice_flag_str[] __attribute((__unused__)) = {
|
2023-11-07 02:42:46 +00:00
|
|
|
"SPLICE_V6", "RCVLOWAT_SET_0", "RCVLOWAT_SET_1", "RCVLOWAT_ACT_0",
|
|
|
|
"RCVLOWAT_ACT_1", "CLOSING",
|
2022-03-15 00:07:02 +00:00
|
|
|
};
|
|
|
|
|
2023-02-13 23:48:23 +00:00
|
|
|
/* Forward declaration */
|
|
|
|
static int tcp_sock_refill_ns(void *arg);
|
2024-02-28 11:25:13 +00:00
|
|
|
static int tcp_conn_sock_ns(const struct ctx *c, sa_family_t af);
|
2023-02-13 23:48:23 +00:00
|
|
|
|
2022-03-15 00:07:02 +00:00
|
|
|
/**
|
|
|
|
* tcp_splice_conn_epoll_events() - epoll events masks for given state
|
|
|
|
* @events: Connection event flags
|
2023-11-07 02:42:46 +00:00
|
|
|
* @ev: Events to fill in, 0 is accepted socket, 1 is connecting socket
|
2022-03-15 00:07:02 +00:00
|
|
|
*/
|
|
|
|
static void tcp_splice_conn_epoll_events(uint16_t events,
|
2023-11-07 02:42:46 +00:00
|
|
|
struct epoll_event ev[])
|
2022-03-15 00:07:02 +00:00
|
|
|
{
|
2023-11-07 02:42:46 +00:00
|
|
|
ev[0].events = ev[1].events = 0;
|
2022-03-15 00:07:02 +00:00
|
|
|
|
2022-11-17 05:58:43 +00:00
|
|
|
if (events & SPLICE_ESTABLISHED) {
|
2023-11-07 02:42:46 +00:00
|
|
|
if (!(events & FIN_SENT_1))
|
|
|
|
ev[0].events = EPOLLIN | EPOLLRDHUP;
|
|
|
|
if (!(events & FIN_SENT_0))
|
|
|
|
ev[1].events = EPOLLIN | EPOLLRDHUP;
|
2022-11-17 05:58:43 +00:00
|
|
|
} else if (events & SPLICE_CONNECT) {
|
2023-11-07 02:42:46 +00:00
|
|
|
ev[1].events = EPOLLOUT;
|
treewide: Packet abstraction with mandatory boundary checks
Implement a packet abstraction providing boundary and size checks
based on packet descriptors: packets stored in a buffer can be queued
into a pool (without storage of its own), and data can be retrieved
referring to an index in the pool, specifying offset and length.
Checks ensure data is not read outside the boundaries of buffer and
descriptors, and that packets added to a pool are within the buffer
range with valid offset and indices.
This implies a wider rework: usage of the "queueing" part of the
abstraction mostly affects tap_handler_{passt,pasta}() functions and
their callees, while the "fetching" part affects all the guest or tap
facing implementations: TCP, UDP, ICMP, ARP, NDP, DHCP and DHCPv6
handlers.
Suggested-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2022-03-25 12:02:47 +00:00
|
|
|
}
|
2022-03-15 00:07:02 +00:00
|
|
|
|
2023-11-07 02:42:46 +00:00
|
|
|
ev[0].events |= (events & OUT_WAIT_0) ? EPOLLOUT : 0;
|
|
|
|
ev[1].events |= (events & OUT_WAIT_1) ? EPOLLOUT : 0;
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
2023-11-07 02:42:43 +00:00
|
|
|
/**
|
|
|
|
* tcp_splice_epoll_ctl() - Add/modify/delete epoll state from connection events
|
|
|
|
* @c: Execution context
|
|
|
|
* @conn: Connection pointer
|
|
|
|
*
|
|
|
|
* Return: 0 on success, negative error code on failure (not on deletion)
|
|
|
|
*/
|
2022-03-26 06:23:21 +00:00
|
|
|
static int tcp_splice_epoll_ctl(const struct ctx *c,
|
2023-11-07 02:42:43 +00:00
|
|
|
struct tcp_splice_conn *conn)
|
|
|
|
{
|
|
|
|
int m = conn->in_epoll ? EPOLL_CTL_MOD : EPOLL_CTL_ADD;
|
2024-01-15 06:39:43 +00:00
|
|
|
const union epoll_ref ref[SIDES] = {
|
2024-01-16 00:50:38 +00:00
|
|
|
{ .type = EPOLL_TYPE_TCP_SPLICE, .fd = conn->s[0],
|
flow,tcp: Use epoll_ref type including flow and side
Currently TCP uses the 'flow' epoll_ref field for both connected
sockets and timers, which consists of just the index of the relevant
flow (connection).
This is just fine for timers, for while it obviously works, it's
subtly incomplete for sockets on spliced connections. In that case we
want to know which side of the connection the event is occurring on as
well as which connection. At present, we deduce that information by
looking at the actual fd, and comparing it to the fds of the sockets
on each side.
When we use the flow table for more things, we expect more cases where
something will need to know a specific side of a specific flow for an
event, but nothing more.
Therefore add a new 'flowside' epoll_ref field, with exactly that
information. We use it for TCP connected sockets. This allows us to
directly know the side for spliced connections. For "tap"
connections, it's pretty meaningless, since the side is always the
socket side. It still makes logical sense though, and it may become
important for future flow table work.
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-30 02:02:18 +00:00
|
|
|
.flowside = FLOW_SIDX(conn, 0) },
|
2024-01-16 00:50:38 +00:00
|
|
|
{ .type = EPOLL_TYPE_TCP_SPLICE, .fd = conn->s[1],
|
flow,tcp: Use epoll_ref type including flow and side
Currently TCP uses the 'flow' epoll_ref field for both connected
sockets and timers, which consists of just the index of the relevant
flow (connection).
This is just fine for timers, for while it obviously works, it's
subtly incomplete for sockets on spliced connections. In that case we
want to know which side of the connection the event is occurring on as
well as which connection. At present, we deduce that information by
looking at the actual fd, and comparing it to the fds of the sockets
on each side.
When we use the flow table for more things, we expect more cases where
something will need to know a specific side of a specific flow for an
event, but nothing more.
Therefore add a new 'flowside' epoll_ref field, with exactly that
information. We use it for TCP connected sockets. This allows us to
directly know the side for spliced connections. For "tap"
connections, it's pretty meaningless, since the side is always the
socket side. It still makes logical sense though, and it may become
important for future flow table work.
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-30 02:02:18 +00:00
|
|
|
.flowside = FLOW_SIDX(conn, 1) }
|
2023-11-07 02:42:46 +00:00
|
|
|
};
|
|
|
|
struct epoll_event ev[SIDES] = { { .data.u64 = ref[0].u64 },
|
|
|
|
{ .data.u64 = ref[1].u64 } };
|
2023-11-07 02:42:43 +00:00
|
|
|
|
2023-11-07 02:42:46 +00:00
|
|
|
tcp_splice_conn_epoll_events(conn->events, ev);
|
2023-11-07 02:42:43 +00:00
|
|
|
|
2023-11-07 02:42:46 +00:00
|
|
|
if (epoll_ctl(c->epollfd, m, conn->s[0], &ev[0]) ||
|
|
|
|
epoll_ctl(c->epollfd, m, conn->s[1], &ev[1])) {
|
2023-11-07 02:42:43 +00:00
|
|
|
int ret = -errno;
|
2023-11-30 02:02:13 +00:00
|
|
|
flow_err(conn, "ERROR on epoll_ctl(): %s", strerror(errno));
|
2023-11-07 02:42:43 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
conn->in_epoll = true;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
/**
|
treewide: Packet abstraction with mandatory boundary checks
Implement a packet abstraction providing boundary and size checks
based on packet descriptors: packets stored in a buffer can be queued
into a pool (without storage of its own), and data can be retrieved
referring to an index in the pool, specifying offset and length.
Checks ensure data is not read outside the boundaries of buffer and
descriptors, and that packets added to a pool are within the buffer
range with valid offset and indices.
This implies a wider rework: usage of the "queueing" part of the
abstraction mostly affects tap_handler_{passt,pasta}() functions and
their callees, while the "fetching" part affects all the guest or tap
facing implementations: TCP, UDP, ICMP, ARP, NDP, DHCP and DHCPv6
handlers.
Suggested-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2022-03-25 12:02:47 +00:00
|
|
|
* conn_flag_do() - Set/unset given flag, log, update epoll on CLOSING flag
|
2022-03-15 00:07:02 +00:00
|
|
|
* @c: Execution context
|
|
|
|
* @conn: Connection pointer
|
|
|
|
* @flag: Flag to set, or ~flag to unset
|
|
|
|
*/
|
2022-03-26 06:23:21 +00:00
|
|
|
static void conn_flag_do(const struct ctx *c, struct tcp_splice_conn *conn,
|
2022-03-15 00:07:02 +00:00
|
|
|
unsigned long flag)
|
|
|
|
{
|
|
|
|
if (flag & (flag - 1)) {
|
2023-02-27 01:45:42 +00:00
|
|
|
int flag_index = fls(~flag);
|
|
|
|
|
2022-03-15 00:07:02 +00:00
|
|
|
if (!(conn->flags & ~flag))
|
|
|
|
return;
|
|
|
|
|
|
|
|
conn->flags &= flag;
|
2023-11-30 02:02:13 +00:00
|
|
|
if (flag_index >= 0)
|
|
|
|
flow_dbg(conn, "%s dropped",
|
|
|
|
tcp_splice_flag_str[flag_index]);
|
2022-03-15 00:07:02 +00:00
|
|
|
} else {
|
2023-02-27 01:45:42 +00:00
|
|
|
int flag_index = fls(flag);
|
|
|
|
|
2022-03-15 00:07:02 +00:00
|
|
|
if (conn->flags & flag)
|
|
|
|
return;
|
|
|
|
|
|
|
|
conn->flags |= flag;
|
2023-11-30 02:02:13 +00:00
|
|
|
if (flag_index >= 0)
|
|
|
|
flow_dbg(conn, "%s", tcp_splice_flag_str[flag_index]);
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
2023-11-07 02:42:42 +00:00
|
|
|
if (flag == CLOSING) {
|
2023-11-07 02:42:46 +00:00
|
|
|
epoll_ctl(c->epollfd, EPOLL_CTL_DEL, conn->s[0], NULL);
|
|
|
|
epoll_ctl(c->epollfd, EPOLL_CTL_DEL, conn->s[1], NULL);
|
2023-11-07 02:42:42 +00:00
|
|
|
}
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#define conn_flag(c, conn, flag) \
|
|
|
|
do { \
|
2023-11-30 02:02:13 +00:00
|
|
|
flow_trace(conn, "flag at %s:%i", __func__, __LINE__); \
|
2022-03-15 00:07:02 +00:00
|
|
|
conn_flag_do(c, conn, flag); \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
/**
|
|
|
|
* conn_event_do() - Set and log connection events, update epoll state
|
|
|
|
* @c: Execution context
|
|
|
|
* @conn: Connection pointer
|
|
|
|
* @event: Connection event
|
|
|
|
*/
|
2022-03-26 06:23:21 +00:00
|
|
|
static void conn_event_do(const struct ctx *c, struct tcp_splice_conn *conn,
|
2022-03-15 00:07:02 +00:00
|
|
|
unsigned long event)
|
|
|
|
{
|
|
|
|
if (event & (event - 1)) {
|
2023-02-27 01:45:42 +00:00
|
|
|
int flag_index = fls(~event);
|
|
|
|
|
2022-03-15 00:07:02 +00:00
|
|
|
if (!(conn->events & ~event))
|
|
|
|
return;
|
|
|
|
|
|
|
|
conn->events &= event;
|
2023-11-30 02:02:13 +00:00
|
|
|
if (flag_index >= 0)
|
|
|
|
flow_dbg(conn, "~%s", tcp_splice_event_str[flag_index]);
|
2022-03-15 00:07:02 +00:00
|
|
|
} else {
|
2023-02-27 01:45:42 +00:00
|
|
|
int flag_index = fls(event);
|
|
|
|
|
2022-03-15 00:07:02 +00:00
|
|
|
if (conn->events & event)
|
|
|
|
return;
|
|
|
|
|
|
|
|
conn->events |= event;
|
2023-11-30 02:02:13 +00:00
|
|
|
if (flag_index >= 0)
|
|
|
|
flow_dbg(conn, "%s", tcp_splice_event_str[flag_index]);
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (tcp_splice_epoll_ctl(c, conn))
|
treewide: Packet abstraction with mandatory boundary checks
Implement a packet abstraction providing boundary and size checks
based on packet descriptors: packets stored in a buffer can be queued
into a pool (without storage of its own), and data can be retrieved
referring to an index in the pool, specifying offset and length.
Checks ensure data is not read outside the boundaries of buffer and
descriptors, and that packets added to a pool are within the buffer
range with valid offset and indices.
This implies a wider rework: usage of the "queueing" part of the
abstraction mostly affects tap_handler_{passt,pasta}() functions and
their callees, while the "fetching" part affects all the guest or tap
facing implementations: TCP, UDP, ICMP, ARP, NDP, DHCP and DHCPv6
handlers.
Suggested-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2022-03-25 12:02:47 +00:00
|
|
|
conn_flag(c, conn, CLOSING);
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#define conn_event(c, conn, event) \
|
|
|
|
do { \
|
2023-11-30 02:02:13 +00:00
|
|
|
flow_trace(conn, "event at %s:%i",__func__, __LINE__); \
|
2022-03-15 00:07:02 +00:00
|
|
|
conn_event_do(c, conn, event); \
|
|
|
|
} while (0)
|
|
|
|
|
2022-11-17 05:58:45 +00:00
|
|
|
|
2022-03-15 00:07:02 +00:00
|
|
|
/**
|
2024-01-16 00:50:34 +00:00
|
|
|
* tcp_splice_flow_defer() - Deferred per-flow handling (clean up closed)
|
2024-05-21 05:57:03 +00:00
|
|
|
* @conn: Connection entry to handle
|
2024-01-16 00:50:42 +00:00
|
|
|
*
|
|
|
|
* Return: true if the flow is ready to free, false otherwise
|
2022-03-15 00:07:02 +00:00
|
|
|
*/
|
2024-05-21 05:57:03 +00:00
|
|
|
bool tcp_splice_flow_defer(struct tcp_splice_conn *conn)
|
2022-03-15 00:07:02 +00:00
|
|
|
{
|
2023-11-30 02:02:17 +00:00
|
|
|
unsigned side;
|
2022-11-19 08:29:54 +00:00
|
|
|
|
2024-05-21 05:57:03 +00:00
|
|
|
if (!(conn->flags & CLOSING))
|
2024-01-16 00:50:42 +00:00
|
|
|
return false;
|
2024-01-16 00:50:34 +00:00
|
|
|
|
2023-11-07 02:42:49 +00:00
|
|
|
for (side = 0; side < SIDES; side++) {
|
2024-02-28 11:25:08 +00:00
|
|
|
/* Flushing might need to block: don't recycle them. */
|
|
|
|
if (conn->pipe[side][0] >= 0) {
|
|
|
|
close(conn->pipe[side][0]);
|
|
|
|
close(conn->pipe[side][1]);
|
|
|
|
conn->pipe[side][0] = conn->pipe[side][1] = -1;
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
2023-11-07 02:42:49 +00:00
|
|
|
|
2024-02-28 11:25:08 +00:00
|
|
|
if (conn->s[side] >= 0) {
|
2023-11-07 02:42:49 +00:00
|
|
|
close(conn->s[side]);
|
|
|
|
conn->s[side] = -1;
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
2023-11-07 02:42:49 +00:00
|
|
|
conn->read[side] = conn->written[side] = 0;
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
2022-11-17 05:58:43 +00:00
|
|
|
conn->events = SPLICE_CLOSED;
|
treewide: Packet abstraction with mandatory boundary checks
Implement a packet abstraction providing boundary and size checks
based on packet descriptors: packets stored in a buffer can be queued
into a pool (without storage of its own), and data can be retrieved
referring to an index in the pool, specifying offset and length.
Checks ensure data is not read outside the boundaries of buffer and
descriptors, and that packets added to a pool are within the buffer
range with valid offset and indices.
This implies a wider rework: usage of the "queueing" part of the
abstraction mostly affects tap_handler_{passt,pasta}() functions and
their callees, while the "fetching" part affects all the guest or tap
facing implementations: TCP, UDP, ICMP, ARP, NDP, DHCP and DHCPv6
handlers.
Suggested-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2022-03-25 12:02:47 +00:00
|
|
|
conn->flags = 0;
|
2023-11-30 02:02:13 +00:00
|
|
|
flow_dbg(conn, "CLOSED");
|
treewide: Packet abstraction with mandatory boundary checks
Implement a packet abstraction providing boundary and size checks
based on packet descriptors: packets stored in a buffer can be queued
into a pool (without storage of its own), and data can be retrieved
referring to an index in the pool, specifying offset and length.
Checks ensure data is not read outside the boundaries of buffer and
descriptors, and that packets added to a pool are within the buffer
range with valid offset and indices.
This implies a wider rework: usage of the "queueing" part of the
abstraction mostly affects tap_handler_{passt,pasta}() functions and
their callees, while the "fetching" part affects all the guest or tap
facing implementations: TCP, UDP, ICMP, ARP, NDP, DHCP and DHCPv6
handlers.
Suggested-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2022-03-25 12:02:47 +00:00
|
|
|
|
2024-01-16 00:50:42 +00:00
|
|
|
return true;
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* tcp_splice_connect_finish() - Completion of connect() or call on success
|
|
|
|
* @c: Execution context
|
|
|
|
* @conn: Connection pointer
|
|
|
|
*
|
|
|
|
* Return: 0 on success, -EIO on failure
|
|
|
|
*/
|
2022-03-26 06:23:21 +00:00
|
|
|
static int tcp_splice_connect_finish(const struct ctx *c,
|
2022-03-15 00:07:02 +00:00
|
|
|
struct tcp_splice_conn *conn)
|
|
|
|
{
|
2023-11-30 02:02:17 +00:00
|
|
|
unsigned side;
|
2023-11-07 02:42:48 +00:00
|
|
|
int i = 0;
|
|
|
|
|
|
|
|
for (side = 0; side < SIDES; side++) {
|
|
|
|
for (; i < TCP_SPLICE_PIPE_POOL_SIZE; i++) {
|
|
|
|
if (splice_pipe_pool[i][0] >= 0) {
|
|
|
|
SWAP(conn->pipe[side][0],
|
|
|
|
splice_pipe_pool[i][0]);
|
|
|
|
SWAP(conn->pipe[side][1],
|
|
|
|
splice_pipe_pool[i][1]);
|
|
|
|
break;
|
|
|
|
}
|
tcp_splice: Don't pool pipes in pairs
To reduce latencies, the tcp splice code maintains a pool of pre-opened
pipes to use for new connections. This is structured as an array of pairs
of pipes, with each pipe, of course, being a pair of fds. Thus when we
use the pool, a single pool "slot" provides both the a->b and b->a pipes.
There's no strong reason to store the pool in pairs, though - we can
with not much difficulty instead take the a->b and b->a pipes for a new
connection independently from separate slots in the pool, or even take one
from the the pool and create the other as we need it, if there's only one
pipe left in the pool.
This marginally increases the length of code, but simplifies the structure
of the pipe pool. We should be able to re-shrink the code with later
changes, too.
In the process we also fix some minor bugs:
- If we both failed to find a pipe in the pool and to create a new one, we
didn't log an error and would silently drop the connection. That could
make debugging such a situation difficult. Add in an error message for
that case
- When refilling the pool, if we were only able to open a single pipe in
the pair, we attempted to rollback, but instead of closing the opened
pipe, we instead closed the pipe we failed to open (probably leading to
some ignored EBADFD errors).
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-07 02:42:45 +00:00
|
|
|
}
|
2022-04-05 05:10:30 +00:00
|
|
|
|
2023-11-07 02:42:48 +00:00
|
|
|
if (conn->pipe[side][0] < 0) {
|
|
|
|
if (pipe2(conn->pipe[side], O_NONBLOCK | O_CLOEXEC)) {
|
2023-11-30 02:02:13 +00:00
|
|
|
flow_err(conn, "cannot create %d->%d pipe: %s",
|
|
|
|
side, !side, strerror(errno));
|
2023-11-07 02:42:48 +00:00
|
|
|
conn_flag(c, conn, CLOSING);
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (fcntl(conn->pipe[side][0], F_SETPIPE_SZ,
|
|
|
|
c->tcp.pipe_size)) {
|
2023-11-30 02:02:13 +00:00
|
|
|
flow_trace(conn,
|
|
|
|
"cannot set %d->%d pipe size to %zu",
|
|
|
|
side, !side, c->tcp.pipe_size);
|
2023-11-07 02:42:48 +00:00
|
|
|
}
|
2022-04-05 05:10:30 +00:00
|
|
|
}
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
2022-11-17 05:58:43 +00:00
|
|
|
if (!(conn->events & SPLICE_ESTABLISHED))
|
|
|
|
conn_event(c, conn, SPLICE_ESTABLISHED);
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* tcp_splice_connect() - Create and connect socket for new spliced connection
|
|
|
|
* @c: Execution context
|
|
|
|
* @conn: Connection pointer
|
2024-02-28 11:25:13 +00:00
|
|
|
* @af: Address family
|
|
|
|
* @pif: pif on which to create socket
|
2022-03-15 00:07:02 +00:00
|
|
|
* @port: Destination port, host order
|
|
|
|
*
|
|
|
|
* Return: 0 for connect() succeeded or in progress, negative value on error
|
|
|
|
*/
|
2022-03-26 06:23:21 +00:00
|
|
|
static int tcp_splice_connect(const struct ctx *c, struct tcp_splice_conn *conn,
|
2024-02-28 11:25:13 +00:00
|
|
|
sa_family_t af, uint8_t pif, in_port_t port)
|
2022-03-15 00:07:02 +00:00
|
|
|
{
|
|
|
|
struct sockaddr_in6 addr6 = {
|
|
|
|
.sin6_family = AF_INET6,
|
|
|
|
.sin6_port = htons(port),
|
|
|
|
.sin6_addr = IN6ADDR_LOOPBACK_INIT,
|
|
|
|
};
|
|
|
|
struct sockaddr_in addr4 = {
|
|
|
|
.sin_family = AF_INET,
|
|
|
|
.sin_port = htons(port),
|
2023-12-07 14:31:34 +00:00
|
|
|
.sin_addr = IN4ADDR_LOOPBACK_INIT,
|
2022-03-15 00:07:02 +00:00
|
|
|
};
|
|
|
|
const struct sockaddr *sa;
|
|
|
|
socklen_t sl;
|
|
|
|
|
2024-02-28 11:25:13 +00:00
|
|
|
if (pif == PIF_HOST)
|
|
|
|
conn->s[1] = tcp_conn_sock(c, af);
|
|
|
|
else if (pif == PIF_SPLICE)
|
|
|
|
conn->s[1] = tcp_conn_sock_ns(c, af);
|
|
|
|
else
|
|
|
|
ASSERT(0);
|
|
|
|
|
|
|
|
if (conn->s[1] < 0)
|
|
|
|
return -1;
|
2022-03-15 00:07:02 +00:00
|
|
|
|
2023-11-07 02:42:46 +00:00
|
|
|
if (setsockopt(conn->s[1], SOL_TCP, TCP_QUICKACK,
|
2022-04-05 05:10:30 +00:00
|
|
|
&((int){ 1 }), sizeof(int))) {
|
2023-11-30 02:02:13 +00:00
|
|
|
flow_trace(conn, "failed to set TCP_QUICKACK on socket %i",
|
|
|
|
conn->s[1]);
|
2022-04-05 05:10:30 +00:00
|
|
|
}
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
if (CONN_V6(conn)) {
|
|
|
|
sa = (struct sockaddr *)&addr6;
|
|
|
|
sl = sizeof(addr6);
|
|
|
|
} else {
|
|
|
|
sa = (struct sockaddr *)&addr4;
|
|
|
|
sl = sizeof(addr4);
|
|
|
|
}
|
|
|
|
|
2023-11-07 02:42:46 +00:00
|
|
|
if (connect(conn->s[1], sa, sl)) {
|
2024-02-28 11:25:14 +00:00
|
|
|
if (errno != EINPROGRESS) {
|
|
|
|
flow_trace(conn, "Couldn't connect socket for splice: %s",
|
|
|
|
strerror(errno));
|
2024-02-28 11:25:08 +00:00
|
|
|
return -errno;
|
2024-02-28 11:25:14 +00:00
|
|
|
}
|
2022-03-15 00:07:02 +00:00
|
|
|
|
2022-11-17 05:58:43 +00:00
|
|
|
conn_event(c, conn, SPLICE_CONNECT);
|
2022-03-15 00:07:02 +00:00
|
|
|
} else {
|
2022-11-17 05:58:43 +00:00
|
|
|
conn_event(c, conn, SPLICE_ESTABLISHED);
|
2022-03-15 00:07:02 +00:00
|
|
|
return tcp_splice_connect_finish(c, conn);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2024-02-19 07:56:50 +00:00
|
|
|
/**
|
|
|
|
* tcp_conn_sock_ns() - Obtain a connectable socket in the namespace
|
|
|
|
* @c: Execution context
|
|
|
|
* @af: Address family (AF_INET or AF_INET6)
|
|
|
|
*
|
|
|
|
* Return: Socket fd in the namespace on success, -errno on failure
|
|
|
|
*/
|
|
|
|
static int tcp_conn_sock_ns(const struct ctx *c, sa_family_t af)
|
|
|
|
{
|
|
|
|
int *p = af == AF_INET6 ? ns_sock_pool6 : ns_sock_pool4;
|
|
|
|
int s;
|
|
|
|
|
|
|
|
if ((s = tcp_conn_pool_sock(p)) >= 0)
|
|
|
|
return s;
|
|
|
|
|
|
|
|
/* If the pool is empty we have to incur the latency of entering the ns.
|
|
|
|
* Therefore, we might as well refill the whole pool while we're at it.
|
|
|
|
* This differs from tcp_conn_sock().
|
|
|
|
*/
|
|
|
|
NS_CALL(tcp_sock_refill_ns, c);
|
|
|
|
|
|
|
|
if ((s = tcp_conn_pool_sock(p)) >= 0)
|
|
|
|
return s;
|
|
|
|
|
|
|
|
err("TCP: No available ns sockets for new connection");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2022-11-17 05:58:51 +00:00
|
|
|
/**
|
2022-11-17 05:58:52 +00:00
|
|
|
* tcp_splice_conn_from_sock() - Attempt to init state for a spliced connection
|
2022-11-17 05:58:51 +00:00
|
|
|
* @c: Execution context
|
2024-02-28 11:25:16 +00:00
|
|
|
* @pif0: pif id of side 0
|
|
|
|
* @dstport: Side 0 destination port of connection
|
2024-02-28 11:25:10 +00:00
|
|
|
* @flow: flow to initialise
|
2024-02-28 11:25:11 +00:00
|
|
|
* @s0: Accepted (side 0) socket
|
2022-11-17 05:58:52 +00:00
|
|
|
* @sa: Peer address of connection
|
2022-11-17 05:58:51 +00:00
|
|
|
*
|
2022-11-17 05:58:52 +00:00
|
|
|
* Return: true if able to create a spliced connection, false otherwise
|
2022-11-17 05:58:51 +00:00
|
|
|
* #syscalls:pasta setsockopt
|
|
|
|
*/
|
2023-09-29 05:50:19 +00:00
|
|
|
bool tcp_splice_conn_from_sock(const struct ctx *c,
|
2024-02-28 11:25:16 +00:00
|
|
|
uint8_t pif0, in_port_t dstport,
|
|
|
|
union flow *flow, int s0,
|
|
|
|
const union sockaddr_inany *sa)
|
2022-11-17 05:58:51 +00:00
|
|
|
{
|
2024-02-28 11:25:10 +00:00
|
|
|
struct tcp_splice_conn *conn;
|
2024-02-28 11:25:11 +00:00
|
|
|
union inany_addr src;
|
2024-02-28 11:25:16 +00:00
|
|
|
in_port_t srcport;
|
2024-02-28 11:25:12 +00:00
|
|
|
sa_family_t af;
|
2024-05-21 05:57:07 +00:00
|
|
|
uint8_t tgtpif;
|
2022-11-17 05:59:04 +00:00
|
|
|
|
2024-02-28 11:25:15 +00:00
|
|
|
if (c->mode != MODE_PASTA)
|
2024-02-28 11:25:01 +00:00
|
|
|
return false;
|
2022-11-17 05:58:52 +00:00
|
|
|
|
2024-02-28 11:25:15 +00:00
|
|
|
inany_from_sockaddr(&src, &srcport, sa);
|
2024-02-28 11:25:12 +00:00
|
|
|
af = inany_v4(&src) ? AF_INET : AF_INET6;
|
|
|
|
|
2024-02-28 11:25:16 +00:00
|
|
|
switch (pif0) {
|
2024-02-28 11:25:15 +00:00
|
|
|
case PIF_SPLICE:
|
|
|
|
if (!inany_is_loopback(&src)) {
|
|
|
|
char str[INANY_ADDRSTRLEN];
|
|
|
|
|
|
|
|
/* We can't use flow_err() etc. because we haven't set
|
|
|
|
* the flow type yet
|
|
|
|
*/
|
|
|
|
warn("Bad source address %s for splice, closing",
|
|
|
|
inany_ntop(&src, str, sizeof(str)));
|
|
|
|
|
|
|
|
/* We *don't* want to fall back to tap */
|
|
|
|
flow_alloc_cancel(flow);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2024-05-21 05:57:07 +00:00
|
|
|
tgtpif = PIF_HOST;
|
2024-02-28 11:25:15 +00:00
|
|
|
dstport += c->tcp.fwd_out.delta[dstport];
|
|
|
|
break;
|
|
|
|
|
|
|
|
case PIF_HOST:
|
|
|
|
if (!inany_is_loopback(&src))
|
|
|
|
return false;
|
|
|
|
|
2024-05-21 05:57:07 +00:00
|
|
|
tgtpif = PIF_SPLICE;
|
2024-02-28 11:25:15 +00:00
|
|
|
dstport += c->tcp.fwd_in.delta[dstport];
|
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2024-05-21 05:57:07 +00:00
|
|
|
flow_target(flow, tgtpif);
|
2024-05-21 05:57:06 +00:00
|
|
|
conn = FLOW_SET_TYPE(flow, FLOW_TCP_SPLICE, tcp_splice);
|
2024-02-28 11:25:10 +00:00
|
|
|
|
2024-02-28 11:25:12 +00:00
|
|
|
conn->flags = af == AF_INET ? 0 : SPLICE_V6;
|
2024-02-28 11:25:11 +00:00
|
|
|
conn->s[0] = s0;
|
2024-02-28 11:25:08 +00:00
|
|
|
conn->s[1] = -1;
|
|
|
|
conn->pipe[0][0] = conn->pipe[0][1] = -1;
|
|
|
|
conn->pipe[1][0] = conn->pipe[1][1] = -1;
|
2022-11-17 05:58:51 +00:00
|
|
|
|
2024-02-28 11:25:11 +00:00
|
|
|
if (setsockopt(s0, SOL_TCP, TCP_QUICKACK, &((int){ 1 }), sizeof(int)))
|
|
|
|
flow_trace(conn, "failed to set TCP_QUICKACK on %i", s0);
|
2024-02-28 11:25:09 +00:00
|
|
|
|
2024-05-21 05:57:07 +00:00
|
|
|
if (tcp_splice_connect(c, conn, af, tgtpif, dstport))
|
2022-11-17 05:58:51 +00:00
|
|
|
conn_flag(c, conn, CLOSING);
|
2022-11-17 05:58:52 +00:00
|
|
|
|
2024-05-21 05:57:05 +00:00
|
|
|
FLOW_ACTIVATE(conn);
|
|
|
|
|
2022-11-17 05:58:52 +00:00
|
|
|
return true;
|
2022-11-17 05:58:51 +00:00
|
|
|
}
|
|
|
|
|
2022-03-15 00:07:02 +00:00
|
|
|
/**
|
2022-11-17 05:58:53 +00:00
|
|
|
* tcp_splice_sock_handler() - Handler for socket mapped to spliced connection
|
2022-03-15 00:07:02 +00:00
|
|
|
* @c: Execution context
|
2024-01-16 00:50:38 +00:00
|
|
|
* @ref: epoll reference
|
2022-03-15 00:07:02 +00:00
|
|
|
* @events: epoll events bitmap
|
|
|
|
*
|
|
|
|
* #syscalls:pasta splice
|
|
|
|
*/
|
2024-01-16 00:50:38 +00:00
|
|
|
void tcp_splice_sock_handler(struct ctx *c, union epoll_ref ref,
|
|
|
|
uint32_t events)
|
2022-03-15 00:07:02 +00:00
|
|
|
{
|
2024-01-16 00:50:38 +00:00
|
|
|
struct tcp_splice_conn *conn = CONN(ref.flowside.flow);
|
|
|
|
unsigned side = ref.flowside.side, fromside;
|
2022-03-15 00:07:02 +00:00
|
|
|
uint8_t lowat_set_flag, lowat_act_flag;
|
2023-11-30 02:02:17 +00:00
|
|
|
int eof, never_read;
|
2024-01-16 00:50:38 +00:00
|
|
|
|
|
|
|
ASSERT(conn->f.type == FLOW_TCP_SPLICE);
|
2022-03-15 00:07:02 +00:00
|
|
|
|
2022-11-17 05:58:43 +00:00
|
|
|
if (conn->events == SPLICE_CLOSED)
|
treewide: Packet abstraction with mandatory boundary checks
Implement a packet abstraction providing boundary and size checks
based on packet descriptors: packets stored in a buffer can be queued
into a pool (without storage of its own), and data can be retrieved
referring to an index in the pool, specifying offset and length.
Checks ensure data is not read outside the boundaries of buffer and
descriptors, and that packets added to a pool are within the buffer
range with valid offset and indices.
This implies a wider rework: usage of the "queueing" part of the
abstraction mostly affects tap_handler_{passt,pasta}() functions and
their callees, while the "fetching" part affects all the guest or tap
facing implementations: TCP, UDP, ICMP, ARP, NDP, DHCP and DHCPv6
handlers.
Suggested-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2022-03-25 12:02:47 +00:00
|
|
|
return;
|
|
|
|
|
2024-02-28 11:25:14 +00:00
|
|
|
if (events & EPOLLERR) {
|
|
|
|
int err, rc;
|
|
|
|
socklen_t sl = sizeof(err);
|
|
|
|
|
|
|
|
rc = getsockopt(ref.fd, SOL_SOCKET, SO_ERROR, &err, &sl);
|
|
|
|
if (rc)
|
|
|
|
flow_err(conn, "Error retrieving SO_ERROR: %s",
|
|
|
|
strerror(errno));
|
|
|
|
else
|
|
|
|
flow_trace(conn, "Error event on socket: %s",
|
|
|
|
strerror(err));
|
|
|
|
|
2022-03-15 00:07:02 +00:00
|
|
|
goto close;
|
2024-02-28 11:25:14 +00:00
|
|
|
}
|
2022-03-15 00:07:02 +00:00
|
|
|
|
2022-11-17 05:58:43 +00:00
|
|
|
if (conn->events == SPLICE_CONNECT) {
|
2022-03-15 00:07:02 +00:00
|
|
|
if (!(events & EPOLLOUT))
|
|
|
|
goto close;
|
|
|
|
if (tcp_splice_connect_finish(c, conn))
|
|
|
|
goto close;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (events & EPOLLOUT) {
|
flow,tcp: Use epoll_ref type including flow and side
Currently TCP uses the 'flow' epoll_ref field for both connected
sockets and timers, which consists of just the index of the relevant
flow (connection).
This is just fine for timers, for while it obviously works, it's
subtly incomplete for sockets on spliced connections. In that case we
want to know which side of the connection the event is occurring on as
well as which connection. At present, we deduce that information by
looking at the actual fd, and comparing it to the fds of the sockets
on each side.
When we use the flow table for more things, we expect more cases where
something will need to know a specific side of a specific flow for an
event, but nothing more.
Therefore add a new 'flowside' epoll_ref field, with exactly that
information. We use it for TCP connected sockets. This allows us to
directly know the side for spliced connections. For "tap"
connections, it's pretty meaningless, since the side is always the
socket side. It still makes logical sense though, and it may become
important for future flow table work.
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-30 02:02:18 +00:00
|
|
|
fromside = !side;
|
|
|
|
conn_event(c, conn, side == 0 ? ~OUT_WAIT_0 : ~OUT_WAIT_1);
|
2022-03-15 00:07:02 +00:00
|
|
|
} else {
|
flow,tcp: Use epoll_ref type including flow and side
Currently TCP uses the 'flow' epoll_ref field for both connected
sockets and timers, which consists of just the index of the relevant
flow (connection).
This is just fine for timers, for while it obviously works, it's
subtly incomplete for sockets on spliced connections. In that case we
want to know which side of the connection the event is occurring on as
well as which connection. At present, we deduce that information by
looking at the actual fd, and comparing it to the fds of the sockets
on each side.
When we use the flow table for more things, we expect more cases where
something will need to know a specific side of a specific flow for an
event, but nothing more.
Therefore add a new 'flowside' epoll_ref field, with exactly that
information. We use it for TCP connected sockets. This allows us to
directly know the side for spliced connections. For "tap"
connections, it's pretty meaningless, since the side is always the
socket side. It still makes logical sense though, and it may become
important for future flow table work.
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-30 02:02:18 +00:00
|
|
|
fromside = side;
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
flow,tcp: Use epoll_ref type including flow and side
Currently TCP uses the 'flow' epoll_ref field for both connected
sockets and timers, which consists of just the index of the relevant
flow (connection).
This is just fine for timers, for while it obviously works, it's
subtly incomplete for sockets on spliced connections. In that case we
want to know which side of the connection the event is occurring on as
well as which connection. At present, we deduce that information by
looking at the actual fd, and comparing it to the fds of the sockets
on each side.
When we use the flow table for more things, we expect more cases where
something will need to know a specific side of a specific flow for an
event, but nothing more.
Therefore add a new 'flowside' epoll_ref field, with exactly that
information. We use it for TCP connected sockets. This allows us to
directly know the side for spliced connections. For "tap"
connections, it's pretty meaningless, since the side is always the
socket side. It still makes logical sense though, and it may become
important for future flow table work.
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-30 02:02:18 +00:00
|
|
|
if (events & EPOLLRDHUP)
|
|
|
|
/* For side 0 this is fake, but implied */
|
|
|
|
conn_event(c, conn, side == 0 ? FIN_RCVD_0 : FIN_RCVD_1);
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
swap:
|
|
|
|
eof = 0;
|
|
|
|
never_read = 1;
|
|
|
|
|
2023-11-07 02:42:50 +00:00
|
|
|
lowat_set_flag = fromside == 0 ? RCVLOWAT_SET_0 : RCVLOWAT_SET_1;
|
|
|
|
lowat_act_flag = fromside == 0 ? RCVLOWAT_ACT_0 : RCVLOWAT_ACT_1;
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
while (1) {
|
|
|
|
ssize_t readlen, to_write = 0, written;
|
2022-04-05 10:43:09 +00:00
|
|
|
int more = 0;
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
retry:
|
2023-11-07 02:42:50 +00:00
|
|
|
readlen = splice(conn->s[fromside], NULL,
|
|
|
|
conn->pipe[fromside][1], NULL, c->tcp.pipe_size,
|
2022-03-15 00:07:02 +00:00
|
|
|
SPLICE_F_MOVE | SPLICE_F_NONBLOCK);
|
2023-11-30 02:02:13 +00:00
|
|
|
flow_trace(conn, "%zi from read-side call", readlen);
|
2022-03-15 00:07:02 +00:00
|
|
|
if (readlen < 0) {
|
|
|
|
if (errno == EINTR)
|
|
|
|
goto retry;
|
|
|
|
|
|
|
|
if (errno != EAGAIN)
|
|
|
|
goto close;
|
|
|
|
|
|
|
|
to_write = c->tcp.pipe_size;
|
|
|
|
} else if (!readlen) {
|
|
|
|
eof = 1;
|
|
|
|
to_write = c->tcp.pipe_size;
|
|
|
|
} else {
|
|
|
|
never_read = 0;
|
|
|
|
to_write += readlen;
|
|
|
|
if (readlen >= (long)c->tcp.pipe_size * 90 / 100)
|
|
|
|
more = SPLICE_F_MORE;
|
|
|
|
|
|
|
|
if (conn->flags & lowat_set_flag)
|
|
|
|
conn_flag(c, conn, lowat_act_flag);
|
|
|
|
}
|
|
|
|
|
|
|
|
eintr:
|
2023-11-07 02:42:50 +00:00
|
|
|
written = splice(conn->pipe[fromside][0], NULL,
|
|
|
|
conn->s[!fromside], NULL, to_write,
|
2022-03-15 00:07:02 +00:00
|
|
|
SPLICE_F_MOVE | more | SPLICE_F_NONBLOCK);
|
2023-11-30 02:02:13 +00:00
|
|
|
flow_trace(conn, "%zi from write-side call (passed %zi)",
|
|
|
|
written, to_write);
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
/* Most common case: skip updating counters. */
|
|
|
|
if (readlen > 0 && readlen == written) {
|
|
|
|
if (readlen >= (long)c->tcp.pipe_size * 10 / 100)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (conn->flags & lowat_set_flag &&
|
|
|
|
readlen > (long)c->tcp.pipe_size / 10) {
|
|
|
|
int lowat = c->tcp.pipe_size / 4;
|
|
|
|
|
2024-06-26 22:55:07 +00:00
|
|
|
if (setsockopt(conn->s[fromside], SOL_SOCKET,
|
|
|
|
SO_RCVLOWAT,
|
|
|
|
&lowat, sizeof(lowat))) {
|
|
|
|
flow_trace(conn,
|
|
|
|
"Setting SO_RCVLOWAT %i: %s",
|
|
|
|
lowat, strerror(errno));
|
|
|
|
} else {
|
|
|
|
conn_flag(c, conn, lowat_set_flag);
|
|
|
|
conn_flag(c, conn, lowat_act_flag);
|
|
|
|
}
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2023-11-07 02:42:50 +00:00
|
|
|
conn->read[fromside] += readlen > 0 ? readlen : 0;
|
|
|
|
conn->written[fromside] += written > 0 ? written : 0;
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
if (written < 0) {
|
|
|
|
if (errno == EINTR)
|
|
|
|
goto eintr;
|
|
|
|
|
|
|
|
if (errno != EAGAIN)
|
|
|
|
goto close;
|
|
|
|
|
tcp_splice: Set OUT_WAIT_ flag whenever pipe isn't emptied
In tcp_splice_sock_handler(), if we get EAGAIN on the second splice(),
from pipe to receiving socket, that doesn't necessarily mean that the
pipe is empty: the receiver buffer might be full instead.
Hence, we can't use the 'never_read' flag to decide that there's
nothing to wait for: even if we didn't read anything from the sending
side in a given iteration, we might still have data to send in the
pipe. Use read/written counters, instead.
This fixes an issue where large bulk transfers would occasionally
hang. From a corresponding strace:
0.000061 epoll_wait(4, [{events=EPOLLOUT, data={u32=29442, u64=12884931330}}], 8, 1000) = 1
0.005003 epoll_ctl(4, EPOLL_CTL_MOD, 211, {events=EPOLLIN|EPOLLRDHUP, data={u32=54018, u64=8589988610}}) = 0
0.000089 epoll_ctl(4, EPOLL_CTL_MOD, 115, {events=EPOLLIN|EPOLLRDHUP, data={u32=29442, u64=12884931330}}) = 0
0.000081 splice(211, NULL, 151, NULL, 1048576, SPLICE_F_MOVE|SPLICE_F_NONBLOCK) = -1 EAGAIN (Resource temporarily unavailable)
0.000073 splice(150, NULL, 115, NULL, 1048576, SPLICE_F_MOVE|SPLICE_F_NONBLOCK) = 1048576
0.000087 splice(211, NULL, 151, NULL, 1048576, SPLICE_F_MOVE|SPLICE_F_NONBLOCK) = -1 EAGAIN (Resource temporarily unavailable)
0.000045 splice(150, NULL, 115, NULL, 1048576, SPLICE_F_MOVE|SPLICE_F_NONBLOCK) = 520415
0.000060 splice(211, NULL, 151, NULL, 1048576, SPLICE_F_MOVE|SPLICE_F_NONBLOCK) = -1 EAGAIN (Resource temporarily unavailable)
0.000044 splice(150, NULL, 115, NULL, 1048576, SPLICE_F_MOVE|SPLICE_F_NONBLOCK) = -1 EAGAIN (Resource temporarily unavailable)
0.000044 epoll_wait(4, [], 8, 1000) = 0
we're reading from socket 211 into to the pipe end numbered 151,
which connects to pipe end 150, and from there we're writing into
socket 115.
We initially drop EPOLLOUT from the set of monitored flags for socket
115, because it already signaled it's ready for output. Then we read
nothing from socket 211 (the sender had nothing to send), and we keep
emptying the pipe into socket 115 (first 1048576 bytes, then 520415
bytes).
This call of tcp_splice_sock_handler() ends with EAGAIN on the writing
side, and we just exit this function without setting the OUT_WAIT_1
flag (and, in turn, EPOLLOUT for socket 115). However, it turns out,
the pipe wasn't actually emptied, and while socket 211 had nothing
more to send, we should have waited on socket 115 to be ready for
output again.
As a further step, we could consider not clearing EPOLLOUT at all,
unless the read/written counters match, but I'm first trying to fix
this ugly issue with a minimal patch.
Link: https://github.com/containers/podman/issues/22575
Link: https://github.com/containers/podman/issues/22593
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: David Gibson <david@gibson.dropbear.id.au>
2024-05-08 07:25:24 +00:00
|
|
|
if (conn->read[fromside] == conn->written[fromside])
|
2022-03-15 00:07:02 +00:00
|
|
|
break;
|
|
|
|
|
2023-11-07 02:42:50 +00:00
|
|
|
conn_event(c, conn,
|
|
|
|
fromside == 0 ? OUT_WAIT_1 : OUT_WAIT_0);
|
2022-03-15 00:07:02 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (never_read && written == (long)(c->tcp.pipe_size))
|
|
|
|
goto retry;
|
|
|
|
|
|
|
|
if (!never_read && written < to_write) {
|
|
|
|
to_write -= written;
|
|
|
|
goto retry;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (eof)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2023-11-07 02:42:46 +00:00
|
|
|
if ((conn->events & FIN_RCVD_0) && !(conn->events & FIN_SENT_1)) {
|
2023-11-07 02:42:50 +00:00
|
|
|
if (conn->read[fromside] == conn->written[fromside] && eof) {
|
2023-11-07 02:42:46 +00:00
|
|
|
shutdown(conn->s[1], SHUT_WR);
|
|
|
|
conn_event(c, conn, FIN_SENT_1);
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-11-07 02:42:46 +00:00
|
|
|
if ((conn->events & FIN_RCVD_1) && !(conn->events & FIN_SENT_0)) {
|
2023-11-07 02:42:50 +00:00
|
|
|
if (conn->read[fromside] == conn->written[fromside] && eof) {
|
2023-11-07 02:42:46 +00:00
|
|
|
shutdown(conn->s[0], SHUT_WR);
|
|
|
|
conn_event(c, conn, FIN_SENT_0);
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-11-07 02:42:46 +00:00
|
|
|
if (CONN_HAS(conn, FIN_SENT_0 | FIN_SENT_1))
|
2022-03-15 00:07:02 +00:00
|
|
|
goto close;
|
|
|
|
|
|
|
|
if ((events & (EPOLLIN | EPOLLOUT)) == (EPOLLIN | EPOLLOUT)) {
|
|
|
|
events = EPOLLIN;
|
|
|
|
|
2023-11-07 02:42:50 +00:00
|
|
|
fromside = !fromside;
|
2022-03-15 00:07:02 +00:00
|
|
|
goto swap;
|
|
|
|
}
|
|
|
|
|
treewide: Packet abstraction with mandatory boundary checks
Implement a packet abstraction providing boundary and size checks
based on packet descriptors: packets stored in a buffer can be queued
into a pool (without storage of its own), and data can be retrieved
referring to an index in the pool, specifying offset and length.
Checks ensure data is not read outside the boundaries of buffer and
descriptors, and that packets added to a pool are within the buffer
range with valid offset and indices.
This implies a wider rework: usage of the "queueing" part of the
abstraction mostly affects tap_handler_{passt,pasta}() functions and
their callees, while the "fetching" part affects all the guest or tap
facing implementations: TCP, UDP, ICMP, ARP, NDP, DHCP and DHCPv6
handlers.
Suggested-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2022-03-25 12:02:47 +00:00
|
|
|
if (events & EPOLLHUP)
|
|
|
|
goto close;
|
|
|
|
|
2022-03-15 00:07:02 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
close:
|
treewide: Packet abstraction with mandatory boundary checks
Implement a packet abstraction providing boundary and size checks
based on packet descriptors: packets stored in a buffer can be queued
into a pool (without storage of its own), and data can be retrieved
referring to an index in the pool, specifying offset and length.
Checks ensure data is not read outside the boundaries of buffer and
descriptors, and that packets added to a pool are within the buffer
range with valid offset and indices.
This implies a wider rework: usage of the "queueing" part of the
abstraction mostly affects tap_handler_{passt,pasta}() functions and
their callees, while the "fetching" part affects all the guest or tap
facing implementations: TCP, UDP, ICMP, ARP, NDP, DHCP and DHCPv6
handlers.
Suggested-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2022-03-25 12:02:47 +00:00
|
|
|
conn_flag(c, conn, CLOSING);
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* tcp_set_pipe_size() - Set usable pipe size, probe starting from MAX_PIPE_SIZE
|
|
|
|
* @c: Execution context
|
|
|
|
*/
|
|
|
|
static void tcp_set_pipe_size(struct ctx *c)
|
|
|
|
{
|
tcp_splice: Don't pool pipes in pairs
To reduce latencies, the tcp splice code maintains a pool of pre-opened
pipes to use for new connections. This is structured as an array of pairs
of pipes, with each pipe, of course, being a pair of fds. Thus when we
use the pool, a single pool "slot" provides both the a->b and b->a pipes.
There's no strong reason to store the pool in pairs, though - we can
with not much difficulty instead take the a->b and b->a pipes for a new
connection independently from separate slots in the pool, or even take one
from the the pool and create the other as we need it, if there's only one
pipe left in the pool.
This marginally increases the length of code, but simplifies the structure
of the pipe pool. We should be able to re-shrink the code with later
changes, too.
In the process we also fix some minor bugs:
- If we both failed to find a pipe in the pool and to create a new one, we
didn't log an error and would silently drop the connection. That could
make debugging such a situation difficult. Add in an error message for
that case
- When refilling the pool, if we were only able to open a single pipe in
the pair, we attempted to rollback, but instead of closing the opened
pipe, we instead closed the pipe we failed to open (probably leading to
some ignored EBADFD errors).
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-07 02:42:45 +00:00
|
|
|
int probe_pipe[TCP_SPLICE_PIPE_POOL_SIZE][2], i, j;
|
2022-03-15 00:07:02 +00:00
|
|
|
|
|
|
|
c->tcp.pipe_size = MAX_PIPE_SIZE;
|
|
|
|
|
|
|
|
smaller:
|
tcp_splice: Don't pool pipes in pairs
To reduce latencies, the tcp splice code maintains a pool of pre-opened
pipes to use for new connections. This is structured as an array of pairs
of pipes, with each pipe, of course, being a pair of fds. Thus when we
use the pool, a single pool "slot" provides both the a->b and b->a pipes.
There's no strong reason to store the pool in pairs, though - we can
with not much difficulty instead take the a->b and b->a pipes for a new
connection independently from separate slots in the pool, or even take one
from the the pool and create the other as we need it, if there's only one
pipe left in the pool.
This marginally increases the length of code, but simplifies the structure
of the pipe pool. We should be able to re-shrink the code with later
changes, too.
In the process we also fix some minor bugs:
- If we both failed to find a pipe in the pool and to create a new one, we
didn't log an error and would silently drop the connection. That could
make debugging such a situation difficult. Add in an error message for
that case
- When refilling the pool, if we were only able to open a single pipe in
the pair, we attempted to rollback, but instead of closing the opened
pipe, we instead closed the pipe we failed to open (probably leading to
some ignored EBADFD errors).
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-07 02:42:45 +00:00
|
|
|
for (i = 0; i < TCP_SPLICE_PIPE_POOL_SIZE; i++) {
|
2022-03-27 11:10:26 +00:00
|
|
|
if (pipe2(probe_pipe[i], O_CLOEXEC)) {
|
2022-03-15 00:07:02 +00:00
|
|
|
i++;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (fcntl(probe_pipe[i][0], F_SETPIPE_SZ, c->tcp.pipe_size) < 0)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (j = i - 1; j >= 0; j--) {
|
|
|
|
close(probe_pipe[j][0]);
|
|
|
|
close(probe_pipe[j][1]);
|
|
|
|
}
|
|
|
|
|
tcp_splice: Don't pool pipes in pairs
To reduce latencies, the tcp splice code maintains a pool of pre-opened
pipes to use for new connections. This is structured as an array of pairs
of pipes, with each pipe, of course, being a pair of fds. Thus when we
use the pool, a single pool "slot" provides both the a->b and b->a pipes.
There's no strong reason to store the pool in pairs, though - we can
with not much difficulty instead take the a->b and b->a pipes for a new
connection independently from separate slots in the pool, or even take one
from the the pool and create the other as we need it, if there's only one
pipe left in the pool.
This marginally increases the length of code, but simplifies the structure
of the pipe pool. We should be able to re-shrink the code with later
changes, too.
In the process we also fix some minor bugs:
- If we both failed to find a pipe in the pool and to create a new one, we
didn't log an error and would silently drop the connection. That could
make debugging such a situation difficult. Add in an error message for
that case
- When refilling the pool, if we were only able to open a single pipe in
the pair, we attempted to rollback, but instead of closing the opened
pipe, we instead closed the pipe we failed to open (probably leading to
some ignored EBADFD errors).
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-07 02:42:45 +00:00
|
|
|
if (i == TCP_SPLICE_PIPE_POOL_SIZE)
|
2022-03-15 00:07:02 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
if (!(c->tcp.pipe_size /= 2)) {
|
|
|
|
c->tcp.pipe_size = MAX_PIPE_SIZE;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
goto smaller;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* tcp_splice_pipe_refill() - Refill pool of pre-opened pipes
|
|
|
|
* @c: Execution context
|
|
|
|
*/
|
2023-02-13 23:48:21 +00:00
|
|
|
static void tcp_splice_pipe_refill(const struct ctx *c)
|
2022-03-15 00:07:02 +00:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < TCP_SPLICE_PIPE_POOL_SIZE; i++) {
|
tcp_splice: Don't pool pipes in pairs
To reduce latencies, the tcp splice code maintains a pool of pre-opened
pipes to use for new connections. This is structured as an array of pairs
of pipes, with each pipe, of course, being a pair of fds. Thus when we
use the pool, a single pool "slot" provides both the a->b and b->a pipes.
There's no strong reason to store the pool in pairs, though - we can
with not much difficulty instead take the a->b and b->a pipes for a new
connection independently from separate slots in the pool, or even take one
from the the pool and create the other as we need it, if there's only one
pipe left in the pool.
This marginally increases the length of code, but simplifies the structure
of the pipe pool. We should be able to re-shrink the code with later
changes, too.
In the process we also fix some minor bugs:
- If we both failed to find a pipe in the pool and to create a new one, we
didn't log an error and would silently drop the connection. That could
make debugging such a situation difficult. Add in an error message for
that case
- When refilling the pool, if we were only able to open a single pipe in
the pair, we attempted to rollback, but instead of closing the opened
pipe, we instead closed the pipe we failed to open (probably leading to
some ignored EBADFD errors).
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-07 02:42:45 +00:00
|
|
|
if (splice_pipe_pool[i][0] >= 0)
|
2022-03-15 00:07:02 +00:00
|
|
|
break;
|
tcp_splice: Don't pool pipes in pairs
To reduce latencies, the tcp splice code maintains a pool of pre-opened
pipes to use for new connections. This is structured as an array of pairs
of pipes, with each pipe, of course, being a pair of fds. Thus when we
use the pool, a single pool "slot" provides both the a->b and b->a pipes.
There's no strong reason to store the pool in pairs, though - we can
with not much difficulty instead take the a->b and b->a pipes for a new
connection independently from separate slots in the pool, or even take one
from the the pool and create the other as we need it, if there's only one
pipe left in the pool.
This marginally increases the length of code, but simplifies the structure
of the pipe pool. We should be able to re-shrink the code with later
changes, too.
In the process we also fix some minor bugs:
- If we both failed to find a pipe in the pool and to create a new one, we
didn't log an error and would silently drop the connection. That could
make debugging such a situation difficult. Add in an error message for
that case
- When refilling the pool, if we were only able to open a single pipe in
the pair, we attempted to rollback, but instead of closing the opened
pipe, we instead closed the pipe we failed to open (probably leading to
some ignored EBADFD errors).
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-07 02:42:45 +00:00
|
|
|
if (pipe2(splice_pipe_pool[i], O_NONBLOCK | O_CLOEXEC))
|
2022-03-15 00:07:02 +00:00
|
|
|
continue;
|
|
|
|
|
tcp_splice: Don't pool pipes in pairs
To reduce latencies, the tcp splice code maintains a pool of pre-opened
pipes to use for new connections. This is structured as an array of pairs
of pipes, with each pipe, of course, being a pair of fds. Thus when we
use the pool, a single pool "slot" provides both the a->b and b->a pipes.
There's no strong reason to store the pool in pairs, though - we can
with not much difficulty instead take the a->b and b->a pipes for a new
connection independently from separate slots in the pool, or even take one
from the the pool and create the other as we need it, if there's only one
pipe left in the pool.
This marginally increases the length of code, but simplifies the structure
of the pipe pool. We should be able to re-shrink the code with later
changes, too.
In the process we also fix some minor bugs:
- If we both failed to find a pipe in the pool and to create a new one, we
didn't log an error and would silently drop the connection. That could
make debugging such a situation difficult. Add in an error message for
that case
- When refilling the pool, if we were only able to open a single pipe in
the pair, we attempted to rollback, but instead of closing the opened
pipe, we instead closed the pipe we failed to open (probably leading to
some ignored EBADFD errors).
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
2023-11-07 02:42:45 +00:00
|
|
|
if (fcntl(splice_pipe_pool[i][0], F_SETPIPE_SZ,
|
2022-04-05 05:10:30 +00:00
|
|
|
c->tcp.pipe_size)) {
|
2023-11-29 12:17:10 +00:00
|
|
|
trace("TCP (spliced): cannot set pool pipe size to %zu",
|
2022-04-05 05:10:30 +00:00
|
|
|
c->tcp.pipe_size);
|
|
|
|
}
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-02-13 23:48:21 +00:00
|
|
|
/**
|
|
|
|
* tcp_sock_refill_ns() - Refill pools of pre-opened sockets in namespace
|
|
|
|
* @arg: Execution context cast to void *
|
|
|
|
*
|
|
|
|
* Return: 0
|
|
|
|
*/
|
2024-06-08 06:30:40 +00:00
|
|
|
/* cppcheck-suppress [constParameterCallback, unmatchedSuppression] */
|
2023-02-13 23:48:21 +00:00
|
|
|
static int tcp_sock_refill_ns(void *arg)
|
|
|
|
{
|
|
|
|
const struct ctx *c = (const struct ctx *)arg;
|
|
|
|
|
|
|
|
ns_enter(c);
|
|
|
|
|
2024-02-19 07:56:49 +00:00
|
|
|
if (c->ifi4) {
|
|
|
|
int rc = tcp_sock_refill_pool(c, ns_sock_pool4, AF_INET);
|
|
|
|
if (rc < 0)
|
|
|
|
warn("TCP: Error refilling IPv4 ns socket pool: %s",
|
|
|
|
strerror(-rc));
|
|
|
|
}
|
|
|
|
if (c->ifi6) {
|
|
|
|
int rc = tcp_sock_refill_pool(c, ns_sock_pool6, AF_INET6);
|
|
|
|
if (rc < 0)
|
|
|
|
warn("TCP: Error refilling IPv6 ns socket pool: %s",
|
|
|
|
strerror(-rc));
|
|
|
|
}
|
2023-02-13 23:48:21 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* tcp_splice_refill() - Refill pools of resources needed for splicing
|
|
|
|
* @c: Execution context
|
|
|
|
*/
|
|
|
|
void tcp_splice_refill(const struct ctx *c)
|
|
|
|
{
|
|
|
|
if ((c->ifi4 && ns_sock_pool4[TCP_SOCK_POOL_TSH] < 0) ||
|
|
|
|
(c->ifi6 && ns_sock_pool6[TCP_SOCK_POOL_TSH] < 0))
|
|
|
|
NS_CALL(tcp_sock_refill_ns, c);
|
|
|
|
|
|
|
|
tcp_splice_pipe_refill(c);
|
|
|
|
}
|
|
|
|
|
2022-03-15 00:07:02 +00:00
|
|
|
/**
|
|
|
|
* tcp_splice_init() - Initialise pipe pool and size
|
|
|
|
* @c: Execution context
|
|
|
|
*/
|
|
|
|
void tcp_splice_init(struct ctx *c)
|
|
|
|
{
|
|
|
|
memset(splice_pipe_pool, 0xff, sizeof(splice_pipe_pool));
|
|
|
|
tcp_set_pipe_size(c);
|
2023-02-13 23:48:21 +00:00
|
|
|
|
|
|
|
memset(&ns_sock_pool4, 0xff, sizeof(ns_sock_pool4));
|
|
|
|
memset(&ns_sock_pool6, 0xff, sizeof(ns_sock_pool6));
|
|
|
|
NS_CALL(tcp_sock_refill_ns, c);
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* tcp_splice_timer() - Timer for spliced connections
|
|
|
|
* @c: Execution context
|
2024-05-21 05:57:03 +00:00
|
|
|
* @conn: Connection to handle
|
2022-03-15 00:07:02 +00:00
|
|
|
*/
|
2024-05-21 05:57:03 +00:00
|
|
|
void tcp_splice_timer(const struct ctx *c, struct tcp_splice_conn *conn)
|
2022-03-15 00:07:02 +00:00
|
|
|
{
|
2023-11-07 02:42:47 +00:00
|
|
|
int side;
|
2022-11-19 08:29:54 +00:00
|
|
|
|
2024-01-16 00:50:33 +00:00
|
|
|
ASSERT(!(conn->flags & CLOSING));
|
2022-03-15 00:07:02 +00:00
|
|
|
|
2023-11-07 02:42:47 +00:00
|
|
|
for (side = 0; side < SIDES; side++) {
|
|
|
|
uint8_t set = side == 0 ? RCVLOWAT_SET_0 : RCVLOWAT_SET_1;
|
|
|
|
uint8_t act = side == 0 ? RCVLOWAT_ACT_0 : RCVLOWAT_ACT_1;
|
2022-03-15 00:07:02 +00:00
|
|
|
|
2023-11-07 02:42:47 +00:00
|
|
|
if ((conn->flags & set) && !(conn->flags & act)) {
|
|
|
|
if (setsockopt(conn->s[side], SOL_SOCKET, SO_RCVLOWAT,
|
|
|
|
&((int){ 1 }), sizeof(int))) {
|
2023-11-30 02:02:13 +00:00
|
|
|
flow_trace(conn, "can't set SO_RCVLOWAT on %d",
|
|
|
|
conn->s[side]);
|
2023-11-07 02:42:47 +00:00
|
|
|
}
|
|
|
|
conn_flag(c, conn, ~set);
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-11-07 02:42:46 +00:00
|
|
|
conn_flag(c, conn, ~RCVLOWAT_ACT_0);
|
|
|
|
conn_flag(c, conn, ~RCVLOWAT_ACT_1);
|
2022-03-15 00:07:02 +00:00
|
|
|
}
|