// SPDX-License-Identifier: GPL-2.0-or-later /* PASST - Plug A Simple Socket Transport * for qemu/UNIX domain socket mode * * PASTA - Pack A Subtle Tap Abstraction * for network namespace/tap device mode * * passt.c - Daemon implementation * * Copyright (c) 2020-2021 Red Hat GmbH * Author: Stefano Brivio * * Grab Ethernet frames from AF_UNIX socket (in "passt" mode) or tap device (in * "pasta" mode), build SOCK_DGRAM/SOCK_STREAM sockets for each 5-tuple from * TCP, UDP packets, perform connection tracking and forward them. Forward * packets received on sockets back to the UNIX domain socket (typically, a * socket virtio_net file descriptor from qemu) or to the tap device (typically, * created in a separate network namespace). */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "util.h" #include "passt.h" #include "dhcp.h" #include "dhcpv6.h" #include "isolation.h" #include "pcap.h" #include "tap.h" #include "conf.h" #include "pasta.h" #include "arch.h" #include "log.h" #include "tcp_splice.h" #include "ndp.h" #include "vu_common.h" #include "migrate.h" #include "repair.h" #include "netlink.h" #include "epoll_ctl.h" #include "flow_table.h" #include "fuzz.h" #define NUM_EPOLL_EVENTS 8 #define TIMER_INTERVAL_ MIN(TCP_TIMER_INTERVAL, FWD_PORT_SCAN_INTERVAL) #define TIMER_INTERVAL MIN(TIMER_INTERVAL_, FLOW_TIMER_INTERVAL) #ifdef FUZZING /* AFL++ persistent mode / shared memory fuzzing compatibility macros. */ #ifndef __AFL_FUZZ_TESTCASE_LEN ssize_t fuzz_len; unsigned char fuzz_buf[1024 * 1024]; # define __AFL_FUZZ_TESTCASE_LEN fuzz_len # define __AFL_FUZZ_TESTCASE_BUF fuzz_buf # define __AFL_FUZZ_INIT() void sync(void) # define __AFL_LOOP(x) \ ((fuzz_len = read(0, fuzz_buf, sizeof(fuzz_buf))) > 0 ? 1 : 0) # define __AFL_INIT() sync() #endif #ifdef __AFL_HAVE_MANUAL_CONTROL __AFL_FUZZ_INIT(); #endif static struct fuzz_turn *fuzz_turn_ptr; /** * fuzz_turn_connect() - Map the turn flag shared memory * * Return: pointer to mapped turn flag, or NULL on failure */ static struct fuzz_turn *fuzz_turn_connect(void) { struct fuzz_turn *t; int fd; fd = open(FUZZ_TURN_PATH, O_RDWR); if (fd < 0) return NULL; t = mmap(NULL, sizeof(*t), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); close(fd); return (t == MAP_FAILED) ? NULL : t; } #endif char pkt_buf[PKT_BUF_BYTES] __attribute__ ((aligned(PAGE_SIZE))); struct ctx passt_ctx = { .pidfile_fd = -1, .fd_tap = -1, .fd_tap_listen = -1, .fd_control_listen = -1, .fd_repair_listen = -1, .pasta_netns_fd = -1, .device_state_fd = -1, }; char *epoll_type_str[] = { [EPOLL_TYPE_TCP] = "connected TCP socket", [EPOLL_TYPE_TCP_SPLICE] = "connected spliced TCP socket", [EPOLL_TYPE_TCP_LISTEN] = "listening TCP socket", [EPOLL_TYPE_TCP_TIMER] = "TCP timer", [EPOLL_TYPE_UDP_LISTEN] = "listening UDP socket", [EPOLL_TYPE_UDP] = "UDP flow socket", [EPOLL_TYPE_PING] = "ICMP/ICMPv6 ping socket", [EPOLL_TYPE_NSQUIT_INOTIFY] = "namespace inotify watch", [EPOLL_TYPE_NSQUIT_TIMER] = "namespace timer watch", [EPOLL_TYPE_TAP_PASTA] = "/dev/net/tun device", [EPOLL_TYPE_TAP_PASST] = "connected qemu socket", [EPOLL_TYPE_TAP_LISTEN] = "listening qemu socket", [EPOLL_TYPE_VHOST_CMD] = "vhost-user command socket", [EPOLL_TYPE_VHOST_KICK] = "vhost-user kick socket", [EPOLL_TYPE_REPAIR_LISTEN] = "TCP_REPAIR helper listening socket", [EPOLL_TYPE_REPAIR] = "TCP_REPAIR helper socket", [EPOLL_TYPE_NL_NEIGH] = "netlink neighbour notifier socket", [EPOLL_TYPE_CONF_LISTEN] = "configuration listening socket", [EPOLL_TYPE_CONF] = "configuration socket", }; static_assert(ARRAY_SIZE(epoll_type_str) == EPOLL_NUM_TYPES, "epoll_type_str[] doesn't match enum epoll_type"); /** * struct passt_stats - Statistics * @events: Event counters for epoll type events */ struct passt_stats { unsigned long events[EPOLL_NUM_TYPES]; }; /** * post_handler() - Run periodic and deferred tasks for L4 protocol handlers * @c: Execution context * @now: Current timestamp */ static void post_handler(struct ctx *c, const struct timespec *now) { if (!c->no_tcp) tcp_defer_handler(c, now); flow_defer_handler(c, now); fwd_scan_ports_timer(c, now); if (!c->no_ndp) ndp_timer(c, now); } /** * random_init() - Initialise things based on random data * @c: Execution context */ static void random_init(struct ctx *c) { unsigned int seed; /* Create secret value for SipHash calculations */ raw_random(&c->hash_secret, sizeof(c->hash_secret)); /* Seed pseudo-RNG for things that need non-cryptographic random */ raw_random(&seed, sizeof(seed)); srandom(seed); } /** * timer_init() - Set initial timestamp for timer runs to current time * @c: Execution context * @now: Current timestamp */ static void timer_init(struct ctx *c, const struct timespec *now) { c->tcp.timer_run = *now; } /** * proto_update_l2_buf() - Update scatter-gather L2 buffers in protocol handlers * @eth_d: Ethernet destination address, NULL if unchanged */ void proto_update_l2_buf(const unsigned char *eth_d) { tcp_update_l2_buf(eth_d); udp_update_l2_buf(eth_d); } /** * exit_handler() - Signal handler for SIGQUIT and SIGTERM * @unused: Unused, handler deals with SIGQUIT and SIGTERM only * * TODO: After unsharing the PID namespace and forking, SIG_DFL for SIGTERM and * SIGQUIT unexpectedly doesn't cause the process to terminate, figure out why. * * #syscalls exit_group */ static void exit_handler(int signal) { (void)signal; passt_exit(EXIT_SUCCESS); } /** * print_stats() - Print event statistics table to stderr * @c: Execution context * @stats: Event counters * @now: Current timestamp */ static void print_stats(const struct ctx *c, const struct passt_stats *stats, const struct timespec *now) { static struct timespec before; static int lines_printed; long long elapsed_ns; int i; if (!c->stats) return; elapsed_ns = (now->tv_sec - before.tv_sec) * 1000000000LL + (now->tv_nsec - before.tv_nsec); if (elapsed_ns < c->stats * 1000000000LL) return; before = *now; if (!(lines_printed % 20)) { /* Table header */ for (i = 1; i < EPOLL_NUM_TYPES; i++) { int j; for (j = 0; j < i * (6 + 1); j++) { if (j && !(j % (6 + 1))) FPRINTF(stderr, "|"); else FPRINTF(stderr, " "); } FPRINTF(stderr, "%s\n", epoll_type_str[i]); } } FPRINTF(stderr, " "); for (i = 1; i < EPOLL_NUM_TYPES; i++) FPRINTF(stderr, " %6lu", stats->events[i]); FPRINTF(stderr, "\n"); lines_printed++; } /** * passt_worker() - Process epoll events and handle protocol operations * @opaque: Pointer to execution context (struct ctx) * @nfds: Number of file descriptors ready (epoll_wait return value) * @events: epoll_event array of ready file descriptors */ static void passt_worker(void *opaque, int nfds, struct epoll_event *events) { static struct passt_stats stats = { 0 }; struct ctx *c = opaque; struct timespec now; int i; if (clock_gettime(CLOCK_MONOTONIC, &now)) err_perror("Failed to get CLOCK_MONOTONIC time"); for (i = 0; i < nfds; i++) { union epoll_ref ref = *((union epoll_ref *)&events[i].data.u64); uint32_t eventmask = events[i].events; trace("%s: epoll event on %s %i (events: 0x%08x)", c->mode == MODE_PASTA ? "pasta" : "passt", EPOLL_TYPE_STR(ref.type), ref.fd, eventmask); switch (ref.type) { case EPOLL_TYPE_TAP_PASTA: tap_handler_pasta(c, eventmask, &now); break; case EPOLL_TYPE_TAP_PASST: tap_handler_passt(c, eventmask, &now); break; case EPOLL_TYPE_TAP_LISTEN: tap_listen_handler(c, eventmask); break; case EPOLL_TYPE_NSQUIT_INOTIFY: pasta_netns_quit_inotify_handler(c, ref.fd); break; case EPOLL_TYPE_NSQUIT_TIMER: pasta_netns_quit_timer_handler(c, ref); break; case EPOLL_TYPE_TCP: tcp_sock_handler(c, ref, eventmask, &now); break; case EPOLL_TYPE_TCP_SPLICE: tcp_splice_sock_handler(c, ref, eventmask, &now); break; case EPOLL_TYPE_TCP_LISTEN: tcp_listen_handler(c, ref, &now); break; case EPOLL_TYPE_TCP_TIMER: tcp_timer_handler(c, ref, &now); break; case EPOLL_TYPE_UDP_LISTEN: udp_listen_sock_handler(c, ref, eventmask, &now); break; case EPOLL_TYPE_UDP: udp_sock_handler(c, ref, eventmask, &now); break; case EPOLL_TYPE_PING: icmp_sock_handler(c, ref, &now); break; case EPOLL_TYPE_VHOST_CMD: #ifdef FUZZING if (!c->vdev) break; #endif vu_control_handler(c->vdev, c->fd_tap, eventmask); break; case EPOLL_TYPE_VHOST_KICK: #ifdef FUZZING if (!c->vdev) break; #endif vu_kick_cb(c->vdev, ref, &now); break; case EPOLL_TYPE_REPAIR_LISTEN: repair_listen_handler(c, eventmask); break; case EPOLL_TYPE_REPAIR: repair_handler(c, eventmask); break; case EPOLL_TYPE_NL_NEIGH: nl_neigh_notify_handler(c); break; case EPOLL_TYPE_CONF_LISTEN: conf_listen_handler(c, eventmask); break; case EPOLL_TYPE_CONF: conf_handler(c, eventmask); break; default: /* Can't happen */ assert(0); } stats.events[ref.type]++; print_stats(c, &stats, &now); } post_handler(c, &now); migrate_handler(c, &now); } /** * main() - Entry point and main loop * @argc: Argument count * @argv: Options, plus optional target PID for pasta mode * * Return: non-zero on failure * * #syscalls read write writev * #syscalls socket getsockopt setsockopt s390x:socketcall i686:socketcall close * #syscalls bind connect recvfrom sendto shutdown * #syscalls arm:recv ppc64le:recv arm:send ppc64le:send * #syscalls accept4 accept listen epoll_ctl epoll_wait|epoll_pwait epoll_pwait * #syscalls clock_gettime|clock_gettime64 * #syscalls arm:clock_gettime64 i686:clock_gettime64 */ int main(int argc, char **argv) { struct epoll_event events[NUM_EPOLL_EVENTS]; int nfds, devnull_fd = -1, fd; struct ctx *c = &passt_ctx; struct rlimit limit; struct timespec now; struct sigaction sa; if (clock_gettime(CLOCK_MONOTONIC, &log_start)) die_perror("Failed to get CLOCK_MONOTONIC time"); arch_avx2_exec(argv); isolate_initial(); c->fd_tap = isolate_fds(argc, argv); if ((devnull_fd = open("/dev/null", O_RDWR | O_CLOEXEC)) < 0) die_perror("Failed to open /dev/null"); /* Ensure fds 0-2 are populated */ for (fd = 0; fd <= STDERR_FILENO; fd++) { if (fcntl(fd, F_GETFD) < 0 && dup2(devnull_fd, fd) < 0) die_perror("Failed to populate fd %d", fd); } sigemptyset(&sa.sa_mask); sa.sa_flags = 0; sa.sa_handler = exit_handler; sigaction(SIGTERM, &sa, NULL); sigaction(SIGQUIT, &sa, NULL); c->mode = conf_mode(argc, argv); if (c->mode == MODE_PASTA) { sa.sa_handler = pasta_child_handler; if (sigaction(SIGCHLD, &sa, NULL)) die_perror("Couldn't install signal handlers"); } if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) die_perror("Couldn't set disposition for SIGPIPE"); madvise(pkt_buf, sizeof(pkt_buf), MADV_HUGEPAGE); c->epollfd = epoll_create1(EPOLL_CLOEXEC); if (c->epollfd == -1) die_perror("Failed to create epoll file descriptor"); flow_epollid_register(EPOLLFD_ID_DEFAULT, c->epollfd); if (getrlimit(RLIMIT_NOFILE, &limit)) die_perror("Failed to get maximum value of open files limit"); c->nofile = limit.rlim_cur = limit.rlim_max; if (setrlimit(RLIMIT_NOFILE, &limit)) die_perror("Failed to set current limit for open files"); sock_probe_features(c); conf(c, argc, argv); trace_init(c->trace); pasta_netns_quit_init(c); tap_backend_init(c); random_init(c); if (clock_gettime(CLOCK_MONOTONIC, &now)) die_perror("Failed to get CLOCK_MONOTONIC time"); flow_init(); fwd_scan_ports_init(c); if ((!c->no_udp && udp_init(c)) || (!c->no_tcp && tcp_init(c))) passt_exit(EXIT_FAILURE); if (fwd_listen_init(c)) passt_exit(EXIT_FAILURE); proto_update_l2_buf(c->guest_mac); if (c->ifi4 && !c->no_dhcp) dhcp_init(); if (c->ifi6 && !c->no_dhcpv6) dhcpv6_init(c); pcap_init(c); fwd_neigh_table_init(c); nl_neigh_notify_init(c); if (isolate_prefork(c)) die("Failed to sandbox process, exiting"); if (!c->foreground) { __daemon(c->pidfile_fd, devnull_fd); log_stderr = false; } else { pidfile_write(c->pidfile_fd, getpid()); } if (c->pidfile_fd >= 0) { close(c->pidfile_fd); c->pidfile_fd = -1; } if (devnull_fd > STDERR_FILENO) close(devnull_fd); if (pasta_child_pid) { kill(pasta_child_pid, SIGUSR1); log_stderr = false; } isolate_postfork(c); timer_init(c, &now); #ifdef FUZZING fuzz_turn_ptr = fuzz_turn_connect(); #define FUZZ_LOOP_ITERATIONS 10000 #define FUZZ_DRAIN_BUF_SIZE 1600 #ifdef __AFL_HAVE_MANUAL_CONTROL __AFL_INIT(); #endif { unsigned char *buf = __AFL_FUZZ_TESTCASE_BUF; while (__AFL_LOOP(FUZZ_LOOP_ITERATIONS)) { int len = __AFL_FUZZ_TESTCASE_LEN; int injected = 0; int pkt_len, round; struct epoll_event ev; union epoll_ref ref; int min_pkt = sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct tcphdr); if (len < (int)sizeof(ev)) continue; /* Reset clock, flow table and epoll for each * AFL++ iteration. */ fuzz_clock_reset(); clock_gettime(CLOCK_MONOTONIC, &now); timer_init(c, &now); flow_init(); /* Recreate epoll instance */ close(c->epollfd); c->epollfd = epoll_create1(EPOLL_CLOEXEC); flow_epollid_register(EPOLLFD_ID_DEFAULT, c->epollfd); if (c->fd_tap >= 0) { union epoll_ref tref = { .type = EPOLL_TYPE_TAP_PASST, .fd = c->fd_tap }; epoll_add(c->epollfd, EPOLLIN | EPOLLRDHUP, tref); /* Drain stale socket data */ char drain[FUZZ_DRAIN_BUF_SIZE]; while (recv(c->fd_tap, drain, sizeof(drain), MSG_DONTWAIT) > 0); } /* Read epoll event from AFL++ buffer */ memcpy(&ev, buf, sizeof(ev)); ref = *((union epoll_ref *)&ev.data.u64); /* Set recv payload in AFL++ shared memory */ fuzz_recv_data = buf + FUZZ_RECV_OFF; fuzz_recv_data_len = (len > FUZZ_RECV_OFF + FUZZ_RECV_MAX) ? FUZZ_RECV_MAX : ((len > FUZZ_RECV_OFF) ? len - FUZZ_RECV_OFF : 0); /* Inject fuzz packet for TAP events */ if (ref.type == EPOLL_TYPE_TAP_PASST || ref.type == EPOLL_TYPE_TAP_PASTA) { struct iov_tail data; struct ethhdr *eh; struct iphdr *iph; struct tcphdr *th; tap_flush_pools(); memset(pkt_buf, 0, min_pkt); pkt_len = len - (int)sizeof(ev); if (pkt_len > 0) memcpy(pkt_buf, buf + sizeof(ev), pkt_len); if (pkt_len < min_pkt) pkt_len = min_pkt; /* construct ethernet header */ eh = (struct ethhdr *)pkt_buf; memcpy(eh->h_dest, c->our_tap_mac, ETH_ALEN); memcpy(eh->h_source, c->guest_mac, ETH_ALEN); eh->h_proto = htons(ETH_P_IP); /* construct IPv4 header */ iph = (struct iphdr *)(pkt_buf + sizeof(*eh)); iph->version = 4; iph->ihl = 5; iph->protocol = IPPROTO_TCP; iph->saddr = c->ip4.addr.s_addr; iph->daddr = c->ip4.guest_gw.s_addr; iph->tot_len = htons(pkt_len - sizeof(*eh)); /* Fix TCP Header */ th = (struct tcphdr *)(pkt_buf + sizeof(*eh) + sizeof(*iph)); th->dest = htons(9999); if (th->doff < 5) th->doff = 5; data = IOV_TAIL_FROM_BUF(pkt_buf, pkt_len, 0); tap_add_packet(c, &data, &now); tap_handler(c, &now); injected = 1; } /* Turn exchange -- only if data was sent */ if (injected && fuzz_turn_ptr) { __atomic_store_n(&fuzz_turn_ptr->turn, 1, __ATOMIC_RELEASE); while (__atomic_load_n(&fuzz_turn_ptr->turn, __ATOMIC_ACQUIRE) != 0); } passt_worker(c, 1, &ev); /* Process host-side TCP events */ for (round = 0; round < 4; round++) { nfds = epoll_wait(c->epollfd, events, NUM_EPOLL_EVENTS, 0); if (nfds <= 0) break; passt_worker(c, nfds, events); } post_handler(c, &now); } } return 0; #else loop: /* NOLINTBEGIN(bugprone-branch-clone): intervals can be the same */ /* cppcheck-suppress [duplicateValueTernary, unmatchedSuppression] */ nfds = epoll_wait(c->epollfd, events, NUM_EPOLL_EVENTS, TIMER_INTERVAL); /* NOLINTEND(bugprone-branch-clone) */ if (nfds == -1 && errno != EINTR) die_perror("epoll_wait() failed in main loop"); passt_worker(c, nfds, events); goto loop; #endif /* FUZZING */ }