我正在测量两台机器之间的乒乓延迟，两者都是20.04.1-Ubuntu。
我的乒乓延迟：发送方向接收方发送 udp pkt，接收方立即发回 udp pkt。当发送者收到 pkt 时，本轮将结束。
起初，我使用的是5.4.0-100-generic，测得平均延迟是13.632 us。
后来，我将内核更新为5.15.0-41-generic，并测量平均延迟为20.141 us。
系统其余部分的设置是相同的，我多次运行了代码。我想知道是linux内核的问题还是我犯了一些错误？
这是我在旁边的代码sender（你可以关注里面的逻辑main），至于接收者，我在符号中添加了它：
#include <arpa/inet.h>
#include <errno.h>
#include <inttypes.h>
#include <linux/errqueue.h>
#include <linux/net_tstamp.h>
#include <linux/sockios.h>
#include <net/if.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#include <time.h>
#define UDP_MAX_LENGTH 1500
#define WARM_UP_PACKETS 10000
#define TOTAL_PACKETS 1000000
long arr2[TOTAL_PACKETS + 5];
typedef struct {
int fd;
int port;
int err_no;
struct sockaddr_in local;
struct sockaddr_in remote;
struct timeval time_kernel;
struct timeval time_user;
int64_t prev_serialnum;
} socket_info;
typedef struct {
int64_t serialnum;
int64_t user_time_serialnum;
int64_t user_time;
int64_t kernel_time_serialnum;
int64_t kernel_time;
size_t message_bytes;
} message_header;
static int setup_udp_sender(socket_info *inf, int port, char *address) {
inf->port = port;
inf->fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
if (inf->fd < 0) {
inf->err_no = errno;
fprintf(stderr, "setup_udp_client: socket failed: %s\n",
strerror(inf->err_no));
return inf->fd;
}
int timestampOn =
SOF_TIMESTAMPING_RX_SOFTWARE | SOF_TIMESTAMPING_TX_SOFTWARE |
SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_RX_HARDWARE |
SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE |
// SOF_TIMESTAMPING_OPT_TSONLY |
0;
int r = setsockopt(inf->fd, SOL_SOCKET, SO_TIMESTAMPING, ×tampOn,
sizeof timestampOn);
if (r < 0) {
inf->err_no = errno;
fprintf(stderr, "setup_udp_server: setsockopt failed: %s\n",
strerror(inf->err_no));
return r;
}
inf->remote = (struct sockaddr_in){.sin_family = AF_INET,
.sin_port = htons((uint16_t)port)};
r = inet_aton(address, &inf->remote.sin_addr);
if (r == 0) {
fprintf(stderr, "setup_udp_client: inet_aton failed\n");
inf->err_no = 0;
return -1;
}
inf->local = (struct sockaddr_in){.sin_family = AF_INET,
.sin_port = htons(0),
.sin_addr.s_addr = htonl(INADDR_ANY)};
inf->prev_serialnum = -1;
return 0;
}
static int setup_udp_receiver(socket_info *inf, int port) {
inf->port = port;
inf->fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
if (inf->fd < 0) {
inf->err_no = errno;
fprintf(stderr, "setup_udp_server: socket failed: %s\n",
strerror(inf->err_no));
return inf->fd;
}
int timestampOn =
SOF_TIMESTAMPING_RX_SOFTWARE | SOF_TIMESTAMPING_TX_SOFTWARE |
SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_RX_HARDWARE |
SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE |
// SOF_TIMESTAMPING_OPT_TSONLY |
0;
int r = setsockopt(inf->fd, SOL_SOCKET, SO_TIMESTAMPING, ×tampOn,
sizeof timestampOn);
if (r < 0) {
inf->err_no = errno;
fprintf(stderr, "setup_udp_server: setsockopt failed: %s\n",
strerror(inf->err_no));
return r;
}
int on = 1;
r = setsockopt(inf->fd, SOL_SOCKET, SO_REUSEPORT, &on, sizeof on);
if (r < 0) {
inf->err_no = errno;
fprintf(stderr, "setup_udp_server: setsockopt2 failed: %s\n",
strerror(inf->err_no));
return r;
}
inf->local = (struct sockaddr_in){.sin_family = AF_INET,
.sin_port = htons((uint16_t)port),
.sin_addr.s_addr = htonl(INADDR_ANY)};
r = bind(inf->fd, (struct sockaddr *)&inf->local, sizeof inf->local);
if (r < 0) {
inf->err_no = errno;
fprintf(stderr, "setup_udp_server: bind failed: %s\n",
strerror(inf->err_no));
return r;
}
inf->prev_serialnum = -1;
return 0;
}
static void handle_scm_timestamping(struct scm_timestamping *ts) {
struct timespec cur;
if( clock_gettime(CLOCK_REALTIME, &cur) == -1 ) {
perror( "clock gettime" );
exit( EXIT_FAILURE );
}
// printf("pkt: timestamp: %ld.%ld ns\n", ts->ts[0].tv_sec, ts->ts[0].tv_nsec);
// printf("now: timestamp: %ld.%ld ns\n\n", cur.tv_sec, cur.tv_nsec);
// printf("Latency: %.3f us\n\n\n", (cur.tv_nsec - ) * 0.001);
// printf("timestamp: %ld ns\n", ts->ts[0].tv_nsec);
static int count = 0;
++count;
if (count > WARM_UP_PACKETS) {
// arr[++arr[0]] = end.tv_sec * 1000000000LL + end.tv_nsec - start.tv_sec * 1000000000LL - start.tv_nsec;
arr2[++arr2[0]] = ts->ts[0].tv_sec * 1000000000LL + ts->ts[0].tv_nsec;
}
// for (size_t i = 0; i < sizeof ts->ts / sizeof *ts->ts; i++) {
// printf("timestamp: %lld.%.9lds\n", (long long)ts->ts[i].tv_sec,
// ts->ts[i].tv_nsec);
// }
// puts("\n");
}
static void handle_time(struct msghdr *msg) {
for (struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
// printf("level=%d, type=%d, len=%zu\n", cmsg->cmsg_level, cmsg->cmsg_type, cmsg->cmsg_len);
if (cmsg->cmsg_level == SOL_IP && cmsg->cmsg_type == IP_RECVERR) {
struct sock_extended_err *ext =
(struct sock_extended_err *)CMSG_DATA(cmsg);
printf("errno=%d, origin=%d\n", ext->ee_errno, ext->ee_origin);
continue;
}
if (cmsg->cmsg_level != SOL_SOCKET) continue;
switch (cmsg->cmsg_type) {
case SO_TIMESTAMPNS: {
struct scm_timestamping *ts = (struct scm_timestamping *)CMSG_DATA(cmsg);
handle_scm_timestamping(ts);
} break;
case SO_TIMESTAMPING: {
struct scm_timestamping *ts = (struct scm_timestamping *)CMSG_DATA(cmsg);
handle_scm_timestamping(ts);
} break;
default:
/* Ignore other cmsg options */
break;
}
}
// printf("End messages\n");
}
static ssize_t udp_receive(socket_info *inf, char *buf, size_t len) {
char ctrl[2048];
struct iovec iov = (struct iovec){.iov_base = buf, .iov_len = len};
struct msghdr msg = (struct msghdr){.msg_control = ctrl,
.msg_controllen = sizeof ctrl,
.msg_name = &inf->remote,
.msg_namelen = sizeof inf->remote,
.msg_iov = &iov,
.msg_iovlen = 1};
ssize_t recv_len = recvmsg(inf->fd, &msg, 0);
gettimeofday(&inf->time_user, NULL);
if (recv_len < 0) {
inf->err_no = errno;
fprintf(stderr, "udp_receive: recvfrom failed: %s\n",
strerror(inf->err_no));
}
// printf("length: %d\n", );
// handle_time(&msg);
return recv_len;
}
static const size_t payload_max = UDP_MAX_LENGTH - sizeof(message_header);
char type_str[50] = "specpaxos.vr.proto.PrepareMessage";
static ssize_t generate_random_message(socket_info *inf, char *buf,
size_t len) {
size_t payload_len = 10;
buf[0] = 0x18, buf[1] = 0x03, buf[2] = 0x05, buf[3] = 0x20;
buf[4] = 0x21;
int x = 5;
for(; x < 12; ++x) buf[x] = 0x00;
for(int i = 0; i < 33; ++i) buf[x + i] = type_str[i];
x += 33;
buf[x++] = 0x18;
for(int i = 0; i < 7; ++i, ++x) buf[x] = 0x00;
for(int i = 0; i < 24; ++i, ++x) buf[x] = 0x00;
buf[x++] = 0x02;
for(int i = 0; i < 7; ++i, ++x) buf[x] = 0x00;
buf[x++] = 0x00;
buf[x++] = 0x00;
return (ssize_t)x;
}
static socket_info registerReceiver(void) {
socket_info inf;
int ret = setup_udp_receiver(&inf, 12345);
if (ret < 0) {
exit(-1);
}
return inf;
}
static ssize_t udp_send(socket_info *inf, char *buf, size_t len) {
struct iovec iov = (struct iovec){.iov_base = buf, .iov_len = len};
struct msghdr msg = (struct msghdr){.msg_name = &inf->remote,
.msg_namelen = sizeof inf->remote,
.msg_iov = &iov,
.msg_iovlen = 1};
gettimeofday(&inf->time_user, NULL);
ssize_t send_len = sendmsg(inf->fd, &msg, 0);
if (send_len < 0) {
inf->err_no = errno;
fprintf(stderr, "udp_send: sendmsg failed: %s\n", strerror(inf->err_no));
}
return send_len;
}
ssize_t send_packet(socket_info inf) {
char packet_buffer[4096];
struct timespec cur;
ssize_t len = generate_random_message(&inf, packet_buffer, sizeof packet_buffer);
return udp_send(&inf, packet_buffer, (size_t)len);
}
static socket_info registerSender(char *host) {
socket_info inf;
int ret = setup_udp_sender(&inf, 12345, host);
if (ret < 0) {
exit(-1);
}
return inf;
}
long arr[TOTAL_PACKETS + 5];
int cmpfunc (const void * a, const void * b) {
return ( *(int*)a - *(int*)b );
}
void queryStatistic() {
int n = arr[0];
qsort(arr + 1, n, sizeof(arr[0]), cmpfunc);
double sum = 0.0;
for(int i = 1; i <= n; ++i) sum += arr[i];
printf("Average latency: %.3f\n", sum / n * 0.001);
printf("90th tail latency: %.3f\n", arr[(long)(n * 0.9)] * 0.001);
printf("99th tail latency: %.3f\n", arr[(long)(n * 0.99)] * 0.001);
printf("99.9th tail latency: %.3f\n", arr[(long)(n * 0.999)] * 0.001);
}
int main(void) {
socket_info inf_tx, inf_rx;
inf_tx = registerSender("10.10.1.3");
inf_rx = registerReceiver();
int count = 0;
char packet_buffer[4096];
struct timespec start, end;
uint64_t next_send_tick = 0;
/* for receiver:
while (1) {
udp_receive(&inf_rx, packet_buffer, sizeof packet_buffer);
send_packet(inf_tx);
}
*/
while(1) {
// printf("asd123www\n");
struct timespec time = {0, 0};
clock_gettime(CLOCK_MONOTONIC, &time);
uint64_t tick = time.tv_sec * 1000000000LL + time.tv_nsec;
if(tick >= next_send_tick) {
clock_gettime(CLOCK_REALTIME, &start);
send_packet(inf_tx);
udp_receive(&inf_rx, packet_buffer, sizeof packet_buffer);
clock_gettime(CLOCK_REALTIME, &end);
++count;
if (count > WARM_UP_PACKETS) {
arr[++arr[0]] = end.tv_sec * 1000000000LL + end.tv_nsec - start.tv_sec * 1000000000LL - start.tv_nsec;
// arr[++arr[0]] = end.tv_sec * 1000000000LL + end.tv_nsec;
}
// printf("%ld\n", arr[arr[0]]);
// printf("%.3f\n", (end.tv_sec * 1000000000LL + end.tv_nsec - start.tv_sec * 1000000000LL - start.tv_nsec) * 0.001);
next_send_tick = tick + 1500;
if (count == TOTAL_PACKETS) break;
}
}
// for(int i = 1; i <= arr[0]; ++i) arr[i] -= arr2[i];
queryStatistic();
}