gx
4862247fe2
Заменяет cudaMalloc + cudaIpcGetMemHandle на cuMemCreate (VMM) +
cuMemExportToShareableHandle(POSIX_FILE_DESCRIPTOR). FDs передаются consumer'у
через sendmsg(SCM_RIGHTS) в handshake. Frigate (s6-overlay не даёт share PID)
и любой другой consumer работают БЕЗ pid namespace share — только volume mount
unix socket'a /run/cuframes и IPC share для /dev/shm header.
Sync: cudaEventRecord+IPC events → cuStreamSynchronize в do_publish.
Producer ждёт ~1 ms что stream flush'нулся, потом atomic_store(seq).
Consumer читает seq через memory_order_acquire и копирует DtoD без
event wait — HW coherence гарантирована на одном GPU.
ABI break (согласован с user'ом):
- magic 0xCC7C1DCC → 0xCC7C1DCE (старые consumers fail cleanly)
- protocol V3 → V4
- libcuframes.so.0 SOVERSION остаётся, но .so.0.3.0 → .so.0.4.0
- EXTERNAL ownership убран (VMM требует cuMemCreate-allocated memory,
нельзя export'нуть произвольный cudaMalloc-pointer как POSIX FD)
- cuframes-rtsp-source переведён на LIBRARY mode + один D2D memcpy
в acquire'нутый slot (overhead малый — публишер всё равно делал такой
D2D из FFmpeg hwframe pool в EXTERNAL pool раньше)
Размер: granularity 2 MB на 5090 → NV12 1920×1080 (~3.1 MB) округляется до
4 MB, +1 MB на slot × 16 × 4 камеры = +64 MB VRAM. Терпимо.
Packet ring (cuframes_packets://) НЕ затронут — отдельный SHM с своим
magic, работает как раньше.
PoC + smoke в spike/:
- vmm_fd_pingpong/ — minimal cuMemCreate+FD round-trip
- smoke_v04/ — full publisher+subscriber, 100/100 frames без pid share
Base image: Dockerfile.runtime → CUDA 12.4 (был 13.0). Matching prod
pipeline + Frigate base, иначе libcudart conflict при load.
Compose stack (localhost-infra repo) — параллельный commit:
- убран pid: container:cuframes-pub-parking из subscribers
- image теги: gx/cuframes:0.4, gx/cuda-grid-pipeline:phase8,
gx/frigate:cuframes-v0.4
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
98 lines
3.2 KiB
C
98 lines
3.2 KiB
C
#include "common.h"
|
|
#include <errno.h>
|
|
#include <string.h>
|
|
#include <sys/socket.h>
|
|
#include <sys/un.h>
|
|
#include <unistd.h>
|
|
|
|
static int recv_fd(int sock, int *out_fd, uint64_t *out_size, uint8_t *out_magic) {
|
|
struct msghdr msg = {0};
|
|
char ctrl[CMSG_SPACE(sizeof(int))];
|
|
struct iovec iov[2];
|
|
iov[0].iov_base = out_size; iov[0].iov_len = sizeof(*out_size);
|
|
iov[1].iov_base = out_magic; iov[1].iov_len = sizeof(*out_magic);
|
|
msg.msg_iov = iov; msg.msg_iovlen = 2;
|
|
msg.msg_control = ctrl; msg.msg_controllen = sizeof(ctrl);
|
|
ssize_t n = recvmsg(sock, &msg, 0);
|
|
if (n < 0) { perror("recvmsg"); return -1; }
|
|
struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
|
|
if (!cmsg || cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) {
|
|
fprintf(stderr, "no SCM_RIGHTS in msg\n");
|
|
return -1;
|
|
}
|
|
memcpy(out_fd, CMSG_DATA(cmsg), sizeof(int));
|
|
return 0;
|
|
}
|
|
|
|
int main(void) {
|
|
CHECK(cuInit(0));
|
|
CUdevice dev;
|
|
CHECK(cuDeviceGet(&dev, 0));
|
|
CUcontext ctx;
|
|
CHECK(cuCtxCreate(&ctx, 0, dev));
|
|
|
|
/* Connect to producer */
|
|
int sock = socket(AF_UNIX, SOCK_STREAM, 0);
|
|
if (sock < 0) { perror("socket"); return 1; }
|
|
struct sockaddr_un sa = {.sun_family = AF_UNIX};
|
|
strncpy(sa.sun_path, SOCK_PATH, sizeof(sa.sun_path) - 1);
|
|
|
|
for (int retry = 0; retry < 50; retry++) {
|
|
if (connect(sock, (struct sockaddr *)&sa, sizeof(sa)) == 0) break;
|
|
if (retry == 49) { perror("connect (final)"); return 1; }
|
|
usleep(100000);
|
|
}
|
|
fprintf(stderr, "consumer: connected to producer\n");
|
|
|
|
int fd = -1;
|
|
uint64_t size = 0;
|
|
uint8_t magic = 0;
|
|
if (recv_fd(sock, &fd, &size, &magic) < 0) return 1;
|
|
fprintf(stderr, "consumer: recv fd=%d size=%llu magic=0x%02x\n",
|
|
fd, (unsigned long long)size, magic);
|
|
|
|
CUmemGenericAllocationHandle mem;
|
|
CHECK(cuMemImportFromShareableHandle(&mem, (void *)(uintptr_t)fd,
|
|
CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR));
|
|
fprintf(stderr, "consumer: imported handle OK\n");
|
|
|
|
CUdeviceptr ptr;
|
|
CHECK(cuMemAddressReserve(&ptr, size, 0, 0, 0));
|
|
CHECK(cuMemMap(ptr, size, 0, mem, 0));
|
|
|
|
CUmemAccessDesc access = {0};
|
|
access.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
|
|
access.location.id = dev;
|
|
access.flags = CU_MEM_ACCESS_FLAGS_PROT_READ;
|
|
CHECK(cuMemSetAccess(ptr, size, &access, 1));
|
|
fprintf(stderr, "consumer: mapped + access OK\n");
|
|
|
|
/* Copy out 1MB чтобы убедиться что pattern там */
|
|
size_t check = size < (1 << 20) ? size : (1 << 20);
|
|
uint8_t *host = malloc(check);
|
|
CHECK(cuMemcpyDtoH(host, ptr, check));
|
|
CHECK(cuCtxSynchronize());
|
|
|
|
size_t mismatch = 0;
|
|
for (size_t i = 0; i < check; i++) {
|
|
if (host[i] != magic) mismatch++;
|
|
}
|
|
free(host);
|
|
|
|
char ack = (mismatch == 0) ? 'O' : 'X';
|
|
fprintf(stderr, "consumer: verify mismatch=%zu/%zu → ACK=%c\n",
|
|
mismatch, check, ack);
|
|
|
|
write(sock, &ack, 1);
|
|
close(sock);
|
|
close(fd);
|
|
|
|
CHECK(cuMemUnmap(ptr, size));
|
|
CHECK(cuMemAddressFree(ptr, size));
|
|
CHECK(cuMemRelease(mem));
|
|
CHECK(cuCtxDestroy(ctx));
|
|
|
|
fprintf(stderr, "consumer: done (%s)\n", ack == 'O' ? "OK" : "FAIL");
|
|
return ack == 'O' ? 0 : 1;
|
|
}
|