Re: [PATCH] tcp_splice: avoid delay on certain transfers

["Bernhard M. Wiedemann" <bwiedemann@suse.de>]

[-- Attachment #1.1.1: Type: text/plain, Size: 3265 bytes --]



On 05/10/2026 21.37, Stefano Brivio wrote:
> Bernard, thanks for the investigation and for the patch. Just one doubt:
[...]
> I'm a bit worried we might cause unnecessary setsockopt() calls in some
> corner cases if we go this way, even though it's a rather minor concern
> (we just called splice(), and that setsockopt() is not _that_ expensive
> in comparison).
The 'corked' variable should ensure that we only add the setsockopt when 
it would otherwise add the unneccesary 200ms delay.

I attached the reproducers and benchmarking tools.
The finding from running it was that SPLICE_F_MORE gives plenty extra 
performance even for 8K pipes and setsockopt hardly costs anything.

To test:
# maybe with adjustment to /proc/sys/fs/pipe-user-pages-soft
pasta --config-net -t 127.0.0.1/25432:15432 -- python3 rr.py serve 15432
python3 rr.py 25432


   ┌─────────┬─────────┬───────────────┬─────────────────┬────────────────┐
│  Build  │  Pipes  │ Worst latency │ Upload (Gbit/s) │ Download/Gbit  │
├─────────┼─────────┼───────────────┼─────────────────┼────────────────┤
│ base    │ 8 KiB   │ 208 ms        │ 89–91           │ 87–89          │
├─────────┼─────────┼───────────────┼─────────────────┼────────────────┤
│ patched │ 8 KiB   │ 0.1 ms        │ 88–89           │ 86–89          │
├─────────┼─────────┼───────────────┼─────────────────┼────────────────┤
│ nomore  │ 8 KiB   │ 0.1 ms        │ 42–43           │ 40             │
├─────────┼─────────┼───────────────┼─────────────────┼────────────────┤
│ base    │ default │ 0.3 ms        │ 203–204         │ 189–198        │
├─────────┼─────────┼───────────────┼─────────────────┼────────────────┤
│ patched │ default │ 0.1 ms        │ 203–205         │ 185–190        │
├─────────┼─────────┼───────────────┼─────────────────┼────────────────┤
│ nomore  │ default │ 0.1 ms        │ 193–203         │ 177–190        │
└─────────┴─────────┴───────────────┴─────────────────┴────────────────┘


Ciao
Bernhard M.

[-- Attachment #1.1.2: bulk.py --]
[-- Type: text/x-python, Size: 1891 bytes --]

"""Bulk throughput through a TCP port. server: bulk.py serve PORT; client: bulk.py PORT MiB RUNS"""
import socket, statistics, sys, time
CH = 1 << 20
def sink(s):
    buf = bytearray(CH); n = 0
    while (r := s.recv_into(buf)): n += r
    return n
def source(s, total):
    mv = memoryview(bytes(CH)); sent = 0
    while sent < total: s.sendall(mv); sent += CH
    s.shutdown(socket.SHUT_WR)
if sys.argv[1] == "serve":
    l = socket.socket(); l.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    l.bind(("0.0.0.0", int(sys.argv[2]))); l.listen()
    while True:
        s, _ = l.accept(); f = s.makefile("rb"); cmd = f.readline().split()
        if not cmd:              # readiness probe
            f.close(); s.close(); continue
        if cmd[0] == b"up":      # client sends, we count and reply with byte count
            buf = bytearray(CH); n = 0
            while (r := f.readinto(buf)): n += r
            s.sendall(str(n).encode()); f.close(); s.close()
        else:                    # down: we send N bytes
            source(s, int(cmd[1])); sink(s); f.close(); s.close()
else:
    port, mib, runs = int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]); total = mib * CH
    for d in (sys.argv[4:] or ["up", "down"]):
        rates = []
        for _ in range(runs):
            s = socket.create_connection(("127.0.0.1", port)); t0 = time.perf_counter()
            s.sendall(f"{d} {total}\n".encode())
            if d == "up":
                source(s, total)
                got = b""
                while (c := s.recv(64)): got += c
                assert int(got) == total, got
            else:
                assert sink(s) == total
            rates.append(total / (time.perf_counter() - t0) / 1e9 * 8)
            s.close()
        print(f"{d:4}: median {statistics.median(rates):6.2f} Gbit/s  (" + " ".join(f"{r:.2f}" for r in rates) + ")", flush=True)

[-- Attachment #1.1.3: rr.py --]
[-- Type: text/x-python, Size: 1410 bytes --]

"""Request/response latency through a TCP port. server: rr.py serve PORT; client: rr.py PORT"""
import socket, struct, sys, time
def recvn(s, n):
    b = bytearray()
    while len(b) < n:
        c = s.recv(n - len(b))
        if not c: raise EOFError
        b += c
    return bytes(b)
if sys.argv[1] == "serve":
    l = socket.socket(); l.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    l.bind(("0.0.0.0", int(sys.argv[2]))); l.listen()
    while True:
        s, _ = l.accept(); s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        try:
            while True:
                req, resp = struct.unpack("!II", recvn(s, 8)); recvn(s, req); s.sendall(b"r" * resp)
        except EOFError:
            s.close()
else:
    s = socket.create_connection(("127.0.0.1", int(sys.argv[1]))); s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
    for label, req, resp in [("request", n, 16) for n in (4000, 7372, 7380, 8000, 9000, 16000, 20000)] + [("response", 16, n) for n in (4000, 7380, 8000, 16000, 64000)]:
        ts = []
        for _ in range(3):
            t0 = time.perf_counter(); s.sendall(struct.pack("!II", req, resp) + b"q" * req); recvn(s, resp); ts.append((time.perf_counter() - t0) * 1000)
        print(f"{label:8} {max(req, resp) if label == 'request' and False else (req if label=='request' else resp):6d} B: " + " ".join(f"{t:6.1f}" for t in ts) + " ms", flush=True)

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 236 bytes --]