int ff_init(int argc, char * const argv[]);
    // Initialize F-Stack
    // Parameters: Command-line arguments in DPDK style
    // Returns: 0 on success, -1 on failure
    // Must be called first by the primary process

void ff_run(loop_func_t loop, void *arg);
    // Start the main polling loop
    // Parameters: User callback function, user argument
    // Does not return (unless ff_stop_run() is called)
    // Called once per lcore

void ff_stop_run(void);
    // Stop the polling loop
    // Graceful shutdown, waits for all packet processing to complete

int ff_socket(int domain, int type, int protocol);
    // Create a socket
    // domain: AF_INET (IPv4) or AF_INET6 (IPv6)
    // type: SOCK_STREAM (TCP) or SOCK_DGRAM (UDP)
    // Returns: File descriptor (>= 0)

int ff_bind(int sockfd, const struct linux_sockaddr *addr, socklen_t addrlen);
    // Bind a local address
    // addr: Pointer to address structure
    // Note: Uses struct linux_sockaddr, not FreeBSD format

int ff_listen(int sockfd, int backlog);
    // Listen for connections (TCP only)
    // backlog: Connection queue size

int ff_accept(int sockfd, struct linux_sockaddr *addr, socklen_t *addrlen);
    // Accept a connection (TCP only)
    // Returns: fd of the new connection

int ff_connect(int sockfd, const struct linux_sockaddr *addr, socklen_t addrlen);
    // Establish a connection (TCP only)
    // Returns: 0 on success, -1 on failure (non-blocking, may return EINPROGRESS)

int ff_close(int sockfd);
    // Close a socket
    // Waits for all pending data to be sent

ssize_t ff_read(int fd, void *buf, size_t nbytes);
    // Read data
    // Returns: > 0 bytes read, 0 connection closed, -1 error
    // Non-blocking: Returns -1 + errno=EAGAIN when no data available

ssize_t ff_write(int fd, const void *buf, size_t nbytes);
    // Write data
    // Returns: > 0 bytes sent, -1 buffer full or error
    // Note: Returns -1 when buffer is full, NOT partial send
    // Should monitor EVFILT_WRITE events to handle

ssize_t ff_readv(int fd, const struct iovec *iov, int iovcnt);
    // Scatter-gather read

ssize_t ff_writev(int fd, const struct iovec *iov, int iovcnt);
    // Scatter-gather write

ssize_t ff_send(int fd, const void *buf, size_t len, int flags);
ssize_t ff_sendto(int fd, const void *buf, size_t len, int flags,
                  const struct linux_sockaddr *to, socklen_t tolen);
ssize_t ff_sendmsg(int fd, const struct msghdr *msg, int flags);
    // Various send methods (flags support MSG_MORE, etc.)

ssize_t ff_recv(int fd, void *buf, size_t len, int flags);
ssize_t ff_recvfrom(int fd, void *buf, size_t len, int flags,
                    struct linux_sockaddr *from, socklen_t *fromlen);
ssize_t ff_recvmsg(int fd, struct msghdr *msg, int flags);
    // Various receive methods

int ff_kqueue(void);
    // Create an event object (similar to epoll_create)
    // Returns: Event object fd

int ff_kevent(int kq, const struct kevent *changelist, int nchanges,
              struct kevent *eventlist, int nevents,
              const struct timespec *timeout);
    // Register events and wait for events
    // changelist: Array of events to register
    // eventlist: Array of returned ready events
    // timeout: NULL for blocking, 0 for non-blocking, > 0 for timeout wait
    // Returns: Number of returned events

int ff_kevent_do_each(int kq, const struct kevent *changelist, int nchanges,
                      void *eventlist, int nevents,
                      const struct timespec *timeout,
                      void (*do_each)(void **, struct kevent *));
    // Convenience function: internally calls ff_kevent and invokes do_each for each event

int ff_select(int nfds, fd_set *readfds, fd_set *writefds,
              fd_set *exceptfds, struct timeval *timeout);
    // select() implementation (for compatibility only)

int ff_poll(struct pollfd *fds, nfds_t nfds, int timeout);
    // poll() implementation (for compatibility only)

int ff_epoll_create(int size);
    // Create an epoll object (actually calls ff_kqueue)

int ff_epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
    // Register/modify/delete epoll events
    // op: EPOLL_CTL_ADD / EPOLL_CTL_MOD / EPOLL_CTL_DEL

int ff_epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout);
    // Wait for events

// Supported epoll event flags:
#define EPOLLIN   0x001      // Readable
#define EPOLLOUT  0x004      // Writable
#define EPOLLET   0x80000000 // Edge-triggered (EV_CLEAR)
#define EPOLLONESHOT 0x40000000 // One-shot trigger (EV_ONESHOT)

int ff_setsockopt(int s, int level, int optname, const void *optval, socklen_t optlen);
    // Set socket options
    // level: SOL_SOCKET / IPPROTO_IP / IPPROTO_TCP / ...

int ff_getsockopt(int s, int level, int optname, void *optval, socklen_t *optlen);
    // Get socket options

int ff_ioctl(int fd, unsigned long request, ...);
    // I/O control
    // Common: FIONBIO (set non-blocking), FIONREAD (query readable bytes)

int ff_fcntl(int fd, int cmd, ...);
    // File control
    // Common: F_SETFL O_NONBLOCK, F_GETFL

struct hostent * ff_gethostbyname(const char *name);
    // DNS resolution (basic implementation)

struct hostent * ff_gethostbyname2(const char *name, int af);
    // DNS resolution (specify AF_INET or AF_INET6)

int ff_route_ctl(enum FF_ROUTE_CTL req, enum FF_ROUTE_FLAG flag,
                 struct linux_sockaddr *dst, struct linux_sockaddr *gw,
                 struct linux_sockaddr *netmask);
    // Routing operations
    // req: FF_ROUTE_ADD / FF_ROUTE_DEL / FF_ROUTE_CHANGE
    // flag: FF_RTF_HOST / FF_RTF_GATEWAY

struct ff_mbuf * ff_mbuf_gethdr(void);
struct ff_mbuf * ff_mbuf_get(const void *data, uint16_t len);
void ff_mbuf_free(struct ff_mbuf *mbuf);
ssize_t ff_mbuf_copydata(struct ff_mbuf *mbuf, uint16_t off, uint16_t len, void *buf);

int ff_zc_mbuf_get(struct ff_zc_mbuf *m, int len);
    // Get a zero-copy mbuf (0 on success, -1 on failure)
ssize_t ff_zc_mbuf_write(int fd, struct ff_zc_mbuf *zm);
ssize_t ff_zc_mbuf_read(int fd, struct ff_zc_mbuf *zm);
    // Zero-copy I/O  [Note] ff_zc_mbuf_read is not yet implemented

#define FF_LOGTYPE_EAL         0
#define FF_LOGTYPE_MALLOC      1
#define FF_LOGTYPE_RING        2
#define FF_LOGTYPE_MEMPOOL     3
#define FF_LOGTYPE_USER1       20
#define FF_LOGTYPE_USER8       27

int ff_log_open_set(void);
int ff_log_set_level(uint32_t logtype, uint32_t level);
int ff_log(uint32_t level, uint32_t logtype, const char *format, ...);

// Log levels (values start from 1, 0 means disabled)
#define FF_LOG_EMERG     1U
#define FF_LOG_ALERT     2U
#define FF_LOG_CRIT      3U
#define FF_LOG_ERR       4U
#define FF_LOG_WARNING   5U
#define FF_LOG_NOTICE    6U
#define FF_LOG_INFO      7U
#define FF_LOG_DEBUG     8U

int ff_pthread_create(pthread_t *thread, const pthread_attr_t *attr,
                      void *(*start_routine)(void *), void *arg);
int ff_pthread_join(pthread_t thread, void **retval);

int ff_gettimeofday(struct timeval *tv, struct timezone *tz);
int ff_clock_gettime(clockid_t clock_id, struct timespec *tp);
    // High-precision time (supports CLOCK_REALTIME / CLOCK_MONOTONIC)

struct ff_config {
    char *filename;

    struct {  // DPDK configuration
        char *proc_type;
        char *lcore_mask;         // CPU core mask (hexadecimal)
        char *proc_mask;
        char *base_virtaddr;
        char *file_prefix;
        char *allow;              // PCI whitelist
        int nb_channel;           // Number of memory channels
        int memory;               // Reserved memory (MB)
        int no_huge;
        int nb_procs;
        int proc_id;
        int promiscuous;          // Promiscuous mode
        int nb_vdev;
        int nb_bond;
        int numa_on;              // NUMA support
        int tso;
        int tx_csum_offoad_skip;
        int vlan_strip;
        int nb_vlan_filter;
        uint16_t vlan_filter_id[DPDK_MAX_VLAN_FILTER];
        int symmetric_rss;
        unsigned idle_sleep;
        unsigned pkt_tx_delay;
        uint16_t *proc_lcore;
        int nb_ports;
        uint16_t max_portid;
        uint16_t *portid_list;
        uint16_t log_level;
        struct ff_port_cfg *port_cfgs;
        struct ff_vlan_cfg *vlan_cfgs;
        struct ff_vdev_cfg *vdev_cfgs;
        struct ff_bond_cfg *bond_cfgs;
        struct ff_rss_check_cfg *rss_check_cfgs;
    } dpdk;

    struct {  // KNI configuration
        int enable;
        int console_packets_ratelimit;
        int general_packets_ratelimit;
        int kernel_packets_ratelimit;
        char *kni_action;
        char *method;
        char *tcp_port;
        char *udp_port;
    } kni;

    struct {
        int level;
        const char *dir;
        void *f; /* FILE * */
    } log;

    struct {  // FreeBSD boot parameters
        struct ff_freebsd_cfg *boot;
        struct ff_freebsd_cfg *sysctl;
        long physmem;
        int hz;                   // Clock frequency (1000 = 1kHz)
        int fd_reserve;           // Reserved fd count
        int mem_size;
    } freebsd;

    struct {
        uint16_t enable;
        uint16_t snap_len;
        uint32_t save_len;
        char*    save_path;
    } pcap;
};

struct kevent {
    uintptr_t ident;       // Event ID (socket fd)
    int16_t filter;        // Event filter (values are negative)
    uint16_t flags;        // Event flags
    uint32_t fflags;       // Filter flags
    __int64_t data;        // Data (ready count/error, fixed 64-bit)
    void *udata;           // User data pointer
    __uint64_t ext[4];     // FreeBSD 13/15 extended fields (KBI unchanged across 13.0 → 15.0; M2 verify-only)
};

// Supported filters (values are negative!)
#define EVFILT_READ     (-1)  // Socket readable
#define EVFILT_WRITE    (-2)  // Socket writable
#define EVFILT_TIMER    (-7)  // Timer
// ... 10 more types, see freebsd/sys/event.h

// Event flags
#define EV_ADD      0x0001   // Add event
#define EV_DELETE   0x0002   // Delete event
#define EV_ONESHOT  0x0010   // One-shot trigger
#define EV_CLEAR    0x0020   // Auto-clear (edge-triggered)
#define EV_EOF      0x8000   // Connection closed

#define ff_EPERM     1     // Operation not permitted
#define ff_ENOENT    2     // No such file or directory
#define ff_ECONNREFUSED  61  // Connection refused
#define ff_ETIMEDOUT 60   // Connection timed out
#define ff_ECONNRESET 54  // Connection reset
#define ff_EAGAIN    35    // Try again
#define ff_EINPROGRESS 36 // Operation in progress
// ... etc.

#define LINUX_SO_REUSEADDR     2     → SO_REUSEADDR
#define LINUX_SO_TYPE          3     → SO_TYPE
#define LINUX_SO_ERROR         4     → SO_ERROR
#define LINUX_SO_DONTROUTE     5     → SO_DONTROUTE
#define LINUX_SO_BROADCAST     6     → SO_BROADCAST
#define LINUX_SO_SNDBUF        7     → SO_SNDBUF
#define LINUX_SO_RCVBUF        8     → SO_RCVBUF
#define LINUX_SO_KEEPALIVE    10     → SO_KEEPALIVE
#define LINUX_SO_OOBINLINE    11     → SO_OOBINLINE
#define LINUX_SO_REUSEPORT   15     → SO_REUSEPORT

#define LINUX_IP_TOS           1     → IP_TOS
#define LINUX_IP_TTL           2     → IP_TTL
#define LINUX_IP_HDRINCL       3     → IP_HDRINCL
#define LINUX_IP_OPTIONS       4     → IP_OPTIONS
#define LINUX_IP_ROUTER_ALERT  5     → IP_ROUTER_ALERT
#define LINUX_IP_RECVOPTS      6     → IP_RECVOPTS
#define LINUX_IP_RETOPTS       7     → IP_RETOPTS
#define LINUX_IP_MULTICAST_IF  32    → IP_MULTICAST_IF
#define LINUX_IP_MULTICAST_TTL 33    → IP_MULTICAST_TTL

#define LINUX_TCP_NODELAY      1     → TCP_NODELAY
#define LINUX_TCP_MAXSEG       2     → TCP_MAXSEG
#define LINUX_TCP_CORK         3     → TCP_CORK
#define LINUX_TCP_KEEPIDLE     4     → TCP_KEEPIDLE
#define LINUX_TCP_KEEPINTVL    5     → TCP_KEEPINTVL
#define LINUX_TCP_KEEPCNT      6     → TCP_KEEPCNT

#define LINUX_FIONBIO         0x5421    // Set non-blocking
#define LINUX_FIONREAD        0x541B    // Query readable bytes
#define LINUX_SIOCGIFFLAGS    0x8913    // Get NIC flags
#define LINUX_SIOCGIFADDR     0x8915    // Get NIC IP
#define LINUX_SIOCGIFNETMASK  0x891B    // Get NIC netmask

// ff_config.c
ff_load_config(argc, argv)
  ├─ 1. Parse command-line arguments
  │     (e.g., --config-file /path/config.ini)
  ├─ 2. Open configuration file
  ├─ 3. Parse INI format line by line
  │     [section] markers
  │     key = value pairs
  ├─ 4. Validate configuration
  │     (address ranges, core count, etc.)
  ├─ 5. Convert to struct ff_config
  └─ 6. Generate DPDK parameter array
        (passed to rte_eal_init)

#define SOCK_FSTACK 0x01000000   // force the F-Stack user-space stack
#define SOCK_KERNEL 0x02000000   // force the host Linux kernel stack

int fk = ff_socket(AF_INET, SOCK_STREAM | SOCK_KERNEL, 0);  // kernel-stack socket
int ff = ff_socket(AF_INET, SOCK_STREAM | SOCK_FSTACK, 0);  // F-Stack socket
int du = ff_socket(AF_INET, SOCK_STREAM, 0);                // dual-created (default)

// Command-line setting at startup
proc_type=primary proc_id=0

ff_init(argc, argv)
  ├─ Initialize DPDK EAL (rte_eal_init)
  │   ├─ Allocate hugepage memory
  │   ├─ Create shared memory mapping
  │   └─ Initialize core 0
  ├─ Create mempool (shared by all processes)
  ├─ Initialize NIC (rte_eth_dev_configure)
  ├─ Start NIC (rte_eth_dev_start)
  └─ Enter polling loop (ff_run)
       ├─ Receive/process packets
       └─ Handle IPC messages from workers

// Command-line setting at startup
proc_type=secondary proc_id=1  // Different secondary processes use different IDs

ff_init(argc, argv)
  ├─ Connect to DPDK shared memory (created by primary process)
  │   ├─ Map hugepage memory
  │   ├─ Connect to mempool
  │   └─ Access NIC configuration
  ├─ Initialize local FreeBSD stack instance
  └─ Enter polling loop (ff_run)
       ├─ Receive/process packets belonging to this process (via RSS)
       └─ Optional: Send IPC messages to primary process

struct ff_msg {
    uint32_t msg_type;           // Message type
    uint32_t msg_id;             // Message ID (for matching request/response)
    uint32_t msg_len;            // Message length

    union {
        struct {
            uint32_t sysctl_id;
            uint32_t value;
        } sysctl;

        struct {
            uint32_t route_cmd;  // ROUTE_CMD_ADD/DEL/CHANGE
            struct sockaddr addr;
            struct sockaddr mask;
        } route;

        struct {
            uint64_t tx_packets;
            uint64_t rx_packets;
            uint64_t tx_bytes;
            uint64_t rx_bytes;
        } traffic;

        char data[256];          // Generic data
    } msg_body;
};

// Message types
#define FF_MSG_SYSCTL       1
#define FF_MSG_IOCTL        2
#define FF_MSG_ROUTE        3
#define FF_MSG_TOP          4
#define FF_MSG_TRAFFIC      5
#define FF_MSG_NGCTL        6
#define FF_MSG_IPFW_CTL     7
#define FF_MSG_KNICTL       8

// Secondary process querying parameters from primary process

// Step 1: Connect to primary process
ff_ipc_init();  // Initialize IPC (connect to primary process's Ring)

// Step 2: Allocate message
struct ff_msg *msg = ff_ipc_msg_alloc();

// Step 3: Fill in request
msg->msg_type = FF_MSG_TRAFFIC;
msg->msg_id = 1;

// Step 4: Send to primary process
ff_ipc_send(msg);

// Step 5: Wait for response
struct ff_msg *retmsg = NULL;
ff_ipc_recv(&retmsg, FF_MSG_TRAFFIC);

// Step 6: Read response
printf("TX packets: %lu\n", retmsg->msg_body.traffic.tx_packets);

// Step 7: Free messages
ff_ipc_msg_free(msg);
ff_ipc_msg_free(retmsg);

enum FF_RELOAD_CMD {
    FF_RELOAD_CMD_UNKNOWN = 0,
    FF_RELOAD_CMD_READY,          /* worker: generation bound and running */
    FF_RELOAD_CMD_HANDOVER_REQ,   /* master: park rx, hand hardware over */
    FF_RELOAD_CMD_HANDOVER_ACK,
    FF_RELOAD_CMD_DRAIN_PROGRESS, /* worker: conns / snd_pending / syncache */
    FF_RELOAD_CMD_DRAIN_DONE,
    FF_RELOAD_CMD_REJECT,         /* reload refused, old generation resumes */
    FF_RELOAD_CMD_QUERY,          /* observability: current reload view */
};

struct ff_reload_args {
    uint32_t cmd;         /* enum FF_RELOAD_CMD */
    uint32_t gen;         /* sender or target generation */
    uint32_t status;      /* command result / progress */
    uint32_t active_gen;  /* serving generation */
};

// Create a thread
pthread_t thread;
int ret = ff_pthread_create(&thread, NULL, thread_func, arg);

// Each thread must:
// 1. Call ff_init() - Create an independent FreeBSD stack instance
// 2. Obtain an independent lcore number
// 3. Use ff_run() to enter the polling loop

// Thread safety
// ✓ Socket operations within the same thread are safe
// ✗ Sockets cannot be shared across threads
// ✗ Cannot create new threads after ff_run()

struct thread_arg {
    int thread_id;
    const char *bind_addr;
    int bind_port;
};

void *thread_func(void *arg) {
    struct thread_arg *ta = (struct thread_arg *)arg;
    ff_init(0, NULL);  // Obtain independent lcore

    int sockfd = ff_socket(AF_INET, SOCK_STREAM, 0);
    struct sockaddr_in addr = {
        .sin_family = AF_INET,
        .sin_port = htons(ta->bind_port),
        .sin_addr.s_addr = inet_addr(ta->bind_addr)
    };

    ff_bind(sockfd, (struct linux_sockaddr *)&addr, sizeof(addr));
    ff_listen(sockfd, 128);
    ff_run(my_loop, (void *)(intptr_t)sockfd);
    return NULL;
}

int main() {
    pthread_t threads[4];
    struct thread_arg args[4] = {
        {0, "10.0.0.1", 80}, {1, "10.0.0.2", 80},
        {2, "10.0.0.3", 80}, {3, "10.0.0.4", 80}
    };
    for (int i = 0; i < 4; i++)
        ff_pthread_create(&threads[i], NULL, thread_func, &args[i]);
    for (int i = 0; i < 4; i++)
        ff_pthread_join(threads[i], NULL);
    return 0;
}

#include <ff_api.h>

int main_loop(void *arg) {
    // Event-ready handler function
    // Called once per polling cycle; must return within 100μs

    int kq = (int)(intptr_t)arg;
    struct kevent events[64];
    int nevents = ff_kevent(kq, NULL, 0, events, 64, NULL);

    for (int i = 0; i < nevents; i++) {
        if (events[i].flags & EV_EOF) {
            ff_close((int)events[i].ident);
        } else if (events[i].filter == EVFILT_READ) {
            char buf[4096];
            ssize_t n = ff_read((int)events[i].ident, buf, sizeof(buf));
            // Process the read data
        } else if (events[i].filter == EVFILT_WRITE) {
            // Write buffer has space, can continue sending
        }
    }
    return 0;
}

int main(int argc, char *argv[]) {
    ff_init(argc, argv);
    int sockfd = ff_socket(AF_INET, SOCK_STREAM, 0);
    struct sockaddr_in addr = {...};
    ff_bind(sockfd, (struct linux_sockaddr *)&addr, sizeof(addr));
    ff_listen(sockfd, 128);

    int kq = ff_kqueue();
    struct kevent kev;
    EV_SET(&kev, sockfd, EVFILT_READ, EV_ADD, 0, 0, NULL);
    ff_kevent(kq, &kev, 1, NULL, 0, NULL);
    ff_run(main_loop, (void *)(intptr_t)kq);
    return 0;
}

#include <ff_api.h>

int main_loop(void *arg) {
    int epfd = (int)(intptr_t)arg;
    struct epoll_event events[64];
    int nevents = ff_epoll_wait(epfd, events, 64, 0);  // Non-blocking

    for (int i = 0; i < nevents; i++) {
        int fd = events[i].data.fd;
        if (events[i].events & EPOLLIN)
            ff_read(fd, buf, sizeof(buf));
        if (events[i].events & EPOLLOUT)
            ff_write(fd, data, len);
    }
    return 0;
}

int main(int argc, char *argv[]) {
    ff_init(argc, argv);
    int sockfd = ff_socket(AF_INET, SOCK_STREAM, 0);
    // ... bind/listen ...
    int epfd = ff_epoll_create(0);
    struct epoll_event ev = { .events = EPOLLIN | EPOLLOUT, .data.fd = sockfd };
    ff_epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev);
    ff_run(main_loop, (void *)(intptr_t)epfd);
    return 0;
}

int flags = 1;
ff_ioctl(sockfd, FIONBIO, &flags);  // Must set!

// ✗ Wrong: Blocks too long
int main_loop(void *arg) { sleep(1); return 0; }

// ✓ Correct: Keep within 100μs
int main_loop(void *arg) {
    ff_kevent(...);  // Non-blocking query
    return 0;
}

// ✗ Wrong: Assume ff_write always succeeds
ssize_t n = ff_write(fd, data, len);

// ✓ Correct: Monitor EVFILT_WRITE events
ssize_t n = ff_write(fd, data, len);
if (n < 0) {
    struct kevent kev;
    EV_SET(&kev, fd, EVFILT_WRITE, EV_ADD, 0, 0, NULL);
    ff_kevent(kq, &kev, 1, NULL, 0, NULL);
}

// ✓ Correct: Check EV_EOF first
if (events[i].flags & EV_EOF) {
    ff_close(fd);
} else if (events[i].filter == EVFILT_READ) {
    ff_read(fd, buf, size);
}

// ✗ Wrong: Share socket across threads
int sockfd = ff_socket(...);  // Created in main thread
ff_pthread_create(&tid, NULL, worker_thread, (void *)(intptr_t)sockfd);

// ✓ Correct: Each thread creates its own socket
void *worker_thread(void *arg) {
    int sockfd = ff_socket(...);  // Created within thread
}