Server.hpp

Go to the documentation of this file.

 
#pragma once
 
#include "ecs/components/ClientId.hpp"
#include "ecs/registry/Registry.hpp"
#include "ecs/systems/PlayerStatusSystem.hpp"
#include "network/ChatProtocol.hpp"
#include "network/MatchConfig.hpp"
#include "network/MatchStatus.hpp"
#include "network/MessageStream.hpp"
#include "network/NetworkConfig.hpp"
#include "network/OutboundQueue.hpp"
#include "network/RosterEvent.hpp"
#include "network/ShotEvent.hpp"
#include "network/VoiceProtocol.hpp"
#include "network/lobby/LobbyStatus.hpp"
#include "network/transport/UdpEndpoint.hpp"
#include "network/transport/UdpSessionTransport.hpp"
#include "systems/EventQueue.hpp"
 
#include <SDL3/SDL_stdinc.h>
 
#include <SDL3_net/SDL_net.h>
#include <atomic>
#include <deque>
#include <entt/entity/entity.hpp>
#include <memory>
#include <mutex>
#include <shared_mutex>
#include <span>
#include <string_view>
#include <thread>
#include <vector>

class Server
{
public:
    bool init(const char* addr,
              Uint16 port,
              const TransportConfig& transport = {},
              const GlobalDiscoveryConfig& discovery = {});

    void shutdown();

    void poll() {}

    bool isEmpty();

    Event dequeueEvent();

    void drainEvents(std::vector<Event>& out);

    bool notifyPlayerClientId(ClientId clientId, entt::entity playerEntity);

    void broadcastRegistry(const Registry& registry);

    void broadcastParticleEvents(const std::vector<NetParticleEvent>& events);

    void broadcastLobbyUpdate(const LobbyUpdateEvent& event);

    int getClientCount();

    uint32_t globalDirectoryServerId() const noexcept;

    uint16_t getClientRttMs(ClientId clientId);

    struct ClientNetState
    {
        ClientId id;
        uint16_t rttMs;
        uint8_t interpDelaySnapshots;
        uint16_t interpDelayMs;
    };
    void snapshotClientNetStates(std::vector<ClientNetState>& out);

    void resetAppliedInputTicks();

    void broadcastMatchStatus(MatchStatePacket packet);

    void broadcastKillEvents(const std::vector<NetKillEvent>& events);

    void broadcastTextChat(ClientId sender, std::string_view message);

    bool sendVoiceFrameToClient(ClientId recipient,
                                ClientId speaker,
                                std::uint16_t sequence,
                                std::uint8_t frameMs,
                                std::span<const std::uint8_t> opus);

    bool sendVoiceFrameToClients(std::span<const ClientId> recipients,
                                 ClientId speaker,
                                 std::uint16_t sequence,
                                 std::uint8_t frameMs,
                                 std::span<const std::uint8_t> opus);

    bool sendToClient(const ClientId& clientId, const void* data, int len);

    bool sendLobbyStateToClient(ClientId clientId, const std::vector<LobbyPlayer>& players);

    bool sendMatchConfigToClient(ClientId clientId, const MatchConfig& config);

    void broadcastRosterEventExcept(ClientId excluded, const PlayerRosterEvent& event);

    void setMaxPlayers(int maxPlayers);

    void setAdvertiseServer(bool enabled);

    void flushAllOutbound();

    using ClientConnectedCallback = std::function<void(ClientId)>;
    using ClientDisconnectedCallback = std::function<void(ClientId)>;
    void onClientConnected(ClientConnectedCallback fn) { clientConnectedFn_ = std::move(fn); }
    void onClientDisconnected(ClientDisconnectedCallback fn) { clientDisconnectedFn_ = std::move(fn); }

    uint16_t listeningPort() const;
 
    void broadcastMatchConfig(const MatchConfig& config);
 
private:
    ClientConnectedCallback clientConnectedFn_;       
    ClientDisconnectedCallback clientDisconnectedFn_; 

    struct Connection
    {
        MessageStream msgStream{};         
        ClientId clientId{};               
        bool pendingInitialization = true; 

        uint32_t lastAppliedInputTick = 0;

        OutboundQueue outbound;

        std::uint64_t connectionId = 0;

        net::UdpEndpointAddr udpAddr;

        uint16_t udpSnapshotSequence = 0;

        uint16_t reliableNextSequence = 0;

        struct PendingReliableEvent
        {
            uint16_t sequence;                                  
            uint8_t remainingSends;                             
            std::shared_ptr<const std::vector<uint8_t>> framed; 
        };
        std::deque<PendingReliableEvent> reliableQueue;

        uint16_t lastReportedRttMs = 0;

        uint8_t lastReportedInterpDelaySnapshots = 0;

        uint16_t lastReportedInterpDelayMs = 0;
 
        Uint64 chatWindowStartMs = 0;
        std::uint8_t chatMessagesInWindow = 0;
        Uint64 voiceWindowStartMs = 0;
        std::uint8_t voiceFramesInWindow = 0;
    };

    void handleMessage(Connection& client, const void* data, Uint32 len);

    ClientId acceptClients();

    void disconnectClient(Connection& conn);

    void readClients();

    ClientId getNextClientId();

    bool enqueueTo(const ClientId& clientId, uint8_t replaceKey, const void* data, int len);

    void enqueueBroadcast(uint8_t replaceKey, const void* data, int len);

    void enqueueReliableEvent(const void* data, int len);

    void enqueueReliableEventExcept(std::span<const ClientId> excluded, const void* data, int len);

    void networkLoop();

    void
    handleUdpUnreliable(std::uint64_t connId, const net::UdpEndpointAddr& from, const uint8_t* payload, uint32_t len);
    void
    handleSessionPayload(std::uint64_t connId, net::ChannelId channel, const std::uint8_t* payload, std::uint32_t len);
    void handleVoiceFrame(Connection& conn, const std::uint8_t* payload, std::uint32_t len);
    void handleDirectoryEvent(const std::vector<std::uint8_t>& payload, const net::UdpEndpointAddr& from);
    void sendDirectoryHeartbeat(Uint64 nowMs);
 
    NET_Server* server = nullptr;                     
 
    std::unordered_map<ClientId, Connection> clients; 
    EventQueue eventQueue;                            
 
    ClientId nextClientId;                            
 
    // ── Phase 3d: UDP sidecar ─────────────────────────────────────────────
    //
    // Bound during init() to the same port as the TCP listener (different
    // protocol = different socket; OS demuxes). The network thread polls
    // both. Server-assigned 32-bit connection IDs let us match an
    // incoming UDP datagram to a TCP-established Connection — the IDs
    // are minted on TCP accept and shipped to the client in the
    // ASSIGN_CLIENT_ID packet so it can stamp them on outbound UDP.
    net::UdpEndpoint udpEndpoint_;
    net::UdpSessionTransport session_;
    TransportConfig transportConfig_;
    GlobalDiscoveryConfig discoveryConfig_;
    net::UdpEndpointAddr directoryAddr_;
    std::atomic<std::uint32_t> directoryServerId_{0};
    Uint64 lastDirectoryHeartbeatMs_ = 0;
    std::uint32_t nextChatServerSeq_ = 0;
    std::atomic<int> maxPlayers_{128};
    std::atomic<bool> advertiseServer_{true}; 
    bool usingUdpSession_ = false;
    Uint16 listenPort_ = 0;
    std::unordered_map<std::uint64_t, ClientId> connIdToClient_; 
 
    // ── Stage 3b: dedicated network thread ────────────────────────────────
    //
    // The mutex protects every member above (clients map, eventQueue,
    // nextClientId, NET_Server*) — both the game thread (enqueueTo /
    // enqueueBroadcast / dequeueEvent / notifyPlayerClientId) and the
    // network thread (acceptClients / readClients / flushAllOutbound) hold
    // it during state mutation. Lock holds are kept short (one phase at a
    // time) so the game thread isn't starved.
    // PR-6 (server-perf): shared_mutex so read-mostly paths
    // (enqueueBroadcast iterating clients, flushAllOutbound's phase 1
    // snapshot, readClients's per-client poll, UDP receive) can take
    // shared locks and run concurrently. Writers (acceptClients,
    // disconnect-application, reliable-queue mutation) take unique.
    std::shared_mutex stateMutex_;
    std::thread networkThread_;
    std::atomic<bool> shouldStop_{false};
 
    // ── PR-2 (server-perf): deferred snapshot fanout ──────────────────────
    //
    // `broadcastRegistry` no longer does the per-client send loop on the
    // game thread. Instead it serializes the snapshot once, wraps the
    // bytes in `shared_ptr<const ...>`, and stores both the unframed
    // payload (for the UDP path's per-fragment send loop) and the
    // length-prefixed framed bytes (for the TCP fallback's enqueue) in
    // these slots. The network thread picks them up at the start of its
    // next cycle and runs the fanout there.
    //
    // Why two buffers: the UDP path expects the raw payload (the
    // PacketHeader is added per-fragment); the TCP path expects the
    // 4-byte length prefix already prepended (it goes straight into
    // OutboundQueue → NET_WriteToStreamSocket). Pre-PR-2 the framing was
    // done per client; now it's once per snapshot and shared.
    //
    // Both pointers protected by stateMutex_; the network thread
    // exchanges to nullptr atomically under the lock so a fresh snapshot
    // arriving mid-cycle replaces the old one without races.
    std::shared_ptr<const std::vector<uint8_t>> pendingSnapshotPayload_;
    std::shared_ptr<const std::vector<uint8_t>> pendingSnapshotFramed_;
 
    // ── PR-10 (server-perf): snapshot delta encoding state ────────────────
    //
    // `prevSnapshotRaw_` is the *unprefixed* `registry_serialization::
    // serialize()` output from the previous broadcastRegistry call —
    // i.e. NOT the wire-prefixed form, just the entt-snapshot bytes.
    // We diff the next call's serialize() against this; if the patch
    // is at least 25% smaller than a full payload AND the size matches
    // (entity count unchanged), we ship a DELTA packet referencing
    // `prevSnapshotTick_`. Clients drop the DELTA if their last-applied
    // tick != `prevSnapshotTick_` — they wait for the next FULL.
    //
    // `snapshotCounter_` increments each broadcastRegistry call. Every
    // `k_keyframeInterval`-th call is forced to FULL regardless of
    // delta size, so clients that fell behind one delta still resync
    // within ≤ 500 ms (at 32 Hz × 16 = 500 ms).
    //
    // PR-14 (loss resilience): `keyframeRaw_` / `keyframeTick_` now
    // hold the most recent FULL keyframe, NOT the immediately previous
    // snapshot.  Every delta within a keyframe window is encoded
    // against the same fixed baseline, so a single dropped delta no
    // longer cascades — the next-arriving delta still decodes against
    // the keyframe the client holds.  The keyframe is replaced only
    // when a new FULL is sent (forced by `k_keyframeInterval` or
    // size-change fallback).  `snapshotCounter_` tracks the absolute
    // snapshot number for tick stamping and keyframe scheduling.
    //
    // All three fields are touched only on the game thread inside
    // broadcastRegistry — no synchronisation needed.
    std::vector<uint8_t> keyframeRaw_;
    std::uint32_t keyframeTick_ = 0;
    std::uint32_t snapshotCounter_ = 0;
 
    // ── PR-4 (server-perf): atomic-published read snapshots ───────────────
    //
    // Pre-PR-4 the game-thread query path (snapshotClientRtts,
    // getClientCount, broadcastMatchStatus's getClientCount) acquired
    // `stateMutex_` once per call and competed with the network thread's
    // long-running readClients() pass. At 200+ bots that lock-wait was
    // the dominant tick-time spike (50 ms p99 on lagcompTargets, 25 ms
    // on match) — see §9 of docs/server-perf-design.md.
    //
    // PR-4 publishes both a per-client RTT snapshot and a client-count
    // gauge atomically from the network thread. The game thread reads
    // them lock-free. Trade-offs:
    //   - Snapshot is at most one network-thread cycle (~1 ms) stale.
    //     For lag-comp's RTT/2 rewind window that's negligible.
    //   - PR-30 (cross-platform): we use the C++11 free-function API
    //     `std::atomic_load_explicit(&shared_ptr, …)` /
    //     `std::atomic_store_explicit(&shared_ptr, …)` rather than the
    //     C++20 `std::atomic<std::shared_ptr<T>>` partial specialization.
    //     Reason: libstdc++ 12+ ships the C++20 specialization, but
    //     Apple's libc++ (through Xcode 16 / libc++ 19) does NOT.
    //     The free-function API is deprecated in C++20 but remains
    //     available through C++23 across all stdlibs we ship to, and
    //     lowers to the same lock-free atomic ops on x86/ARM.  Local
    //     pragma suppression in `Server.cpp` silences the deprecation
    //     warnings at the call sites.
    struct ClientRttSnapshot
    {
        std::vector<ClientNetState> entries;
    };
    std::shared_ptr<const ClientRttSnapshot> rttSnapshotAtomic_;
    std::atomic<std::uint32_t> clientCountAtomic_{0};
};