From 47d2e192aa6a0294eaed69a95a7755c890759106 Mon Sep 17 00:00:00 2001 From: Lee Salzman Date: Thu, 23 Jul 2020 04:42:59 -0400 Subject: [PATCH] use unified outgoing command queue for reliable and unreliable commands --- include/enet/enet.h | 3 +- peer.c | 11 +- protocol.c | 281 ++++++++++++++++++-------------------------- 3 files changed, 121 insertions(+), 174 deletions(-) diff --git a/include/enet/enet.h b/include/enet/enet.h index 54d91b5..a736173 100644 --- a/include/enet/enet.h +++ b/include/enet/enet.h @@ -313,8 +313,7 @@ typedef struct _ENetPeer ENetList acknowledgements; ENetList sentReliableCommands; ENetList sentUnreliableCommands; - ENetList outgoingReliableCommands; - ENetList outgoingUnreliableCommands; + ENetList outgoingCommands; ENetList dispatchedCommands; enet_uint16 flags; enet_uint8 roundTripTimeRemainder; diff --git a/peer.c b/peer.c index 1278b85..ca93289 100644 --- a/peer.c +++ b/peer.c @@ -318,8 +318,7 @@ enet_peer_reset_queues (ENetPeer * peer) enet_peer_reset_outgoing_commands (& peer -> sentReliableCommands); enet_peer_reset_outgoing_commands (& peer -> sentUnreliableCommands); - enet_peer_reset_outgoing_commands (& peer -> outgoingReliableCommands); - enet_peer_reset_outgoing_commands (& peer -> outgoingUnreliableCommands); + enet_peer_reset_outgoing_commands (& peer -> outgoingCommands); enet_peer_reset_incoming_commands (& peer -> dispatchedCommands); if (peer -> channels != NULL && peer -> channelCount > 0) @@ -573,8 +572,7 @@ void enet_peer_disconnect_later (ENetPeer * peer, enet_uint32 data) { if ((peer -> state == ENET_PEER_STATE_CONNECTED || peer -> state == ENET_PEER_STATE_DISCONNECT_LATER) && - ! (enet_list_empty (& peer -> outgoingReliableCommands) && - enet_list_empty (& peer -> outgoingUnreliableCommands) && + ! (enet_list_empty (& peer -> outgoingCommands) && enet_list_empty (& peer -> sentReliableCommands))) { peer -> state = ENET_PEER_STATE_DISCONNECT_LATER; @@ -676,10 +674,7 @@ enet_peer_setup_outgoing_command (ENetPeer * peer, ENetOutgoingCommand * outgoin break; } - if (outgoingCommand -> command.header.command & ENET_PROTOCOL_COMMAND_FLAG_ACKNOWLEDGE) - enet_list_insert (enet_list_end (& peer -> outgoingReliableCommands), outgoingCommand); - else - enet_list_insert (enet_list_end (& peer -> outgoingUnreliableCommands), outgoingCommand); + enet_list_insert (enet_list_end (& peer -> outgoingCommands), outgoingCommand); } ENetOutgoingCommand * diff --git a/protocol.c b/protocol.c index 0a60253..350337d 100644 --- a/protocol.c +++ b/protocol.c @@ -188,8 +188,7 @@ enet_protocol_remove_sent_unreliable_commands (ENetPeer * peer) } while (! enet_list_empty (& peer -> sentUnreliableCommands)); if (peer -> state == ENET_PEER_STATE_DISCONNECT_LATER && - enet_list_empty (& peer -> outgoingReliableCommands) && - enet_list_empty (& peer -> outgoingUnreliableCommands) && + enet_list_empty (& peer -> outgoingCommands) && enet_list_empty (& peer -> sentReliableCommands)) enet_peer_disconnect (peer, peer -> eventData); } @@ -215,12 +214,15 @@ enet_protocol_remove_sent_reliable_command (ENetPeer * peer, enet_uint16 reliabl if (currentCommand == enet_list_end (& peer -> sentReliableCommands)) { - for (currentCommand = enet_list_begin (& peer -> outgoingReliableCommands); - currentCommand != enet_list_end (& peer -> outgoingReliableCommands); + for (currentCommand = enet_list_begin (& peer -> outgoingCommands); + currentCommand != enet_list_end (& peer -> outgoingCommands); currentCommand = enet_list_next (currentCommand)) { outgoingCommand = (ENetOutgoingCommand *) currentCommand; + if (! (outgoingCommand -> command.header.command & ENET_PROTOCOL_COMMAND_FLAG_ACKNOWLEDGE)) + continue; + if (outgoingCommand -> sendAttempts < 1) return ENET_PROTOCOL_COMMAND_NONE; if (outgoingCommand -> reliableSequenceNumber == reliableSequenceNumber && @@ -228,7 +230,7 @@ enet_protocol_remove_sent_reliable_command (ENetPeer * peer, enet_uint16 reliabl break; } - if (currentCommand == enet_list_end (& peer -> outgoingReliableCommands)) + if (currentCommand == enet_list_end (& peer -> outgoingCommands)) return ENET_PROTOCOL_COMMAND_NONE; wasSent = 0; @@ -916,8 +918,7 @@ enet_protocol_handle_acknowledge (ENetHost * host, ENetEvent * event, ENetPeer * break; case ENET_PEER_STATE_DISCONNECT_LATER: - if (enet_list_empty (& peer -> outgoingReliableCommands) && - enet_list_empty (& peer -> outgoingUnreliableCommands) && + if (enet_list_empty (& peer -> outgoingCommands) && enet_list_empty (& peer -> sentReliableCommands)) enet_peer_disconnect (peer, peer -> eventData); break; @@ -1326,108 +1327,6 @@ enet_protocol_send_acknowledgements (ENetHost * host, ENetPeer * peer) host -> bufferCount = buffer - host -> buffers; } -static void -enet_protocol_send_unreliable_outgoing_commands (ENetHost * host, ENetPeer * peer) -{ - ENetProtocol * command = & host -> commands [host -> commandCount]; - ENetBuffer * buffer = & host -> buffers [host -> bufferCount]; - ENetOutgoingCommand * outgoingCommand; - ENetListIterator currentCommand; - - currentCommand = enet_list_begin (& peer -> outgoingUnreliableCommands); - - while (currentCommand != enet_list_end (& peer -> outgoingUnreliableCommands)) - { - size_t commandSize; - - outgoingCommand = (ENetOutgoingCommand *) currentCommand; - commandSize = commandSizes [outgoingCommand -> command.header.command & ENET_PROTOCOL_COMMAND_MASK]; - - if (command >= & host -> commands [sizeof (host -> commands) / sizeof (ENetProtocol)] || - buffer + 1 >= & host -> buffers [sizeof (host -> buffers) / sizeof (ENetBuffer)] || - peer -> mtu - host -> packetSize < commandSize || - (outgoingCommand -> packet != NULL && - peer -> mtu - host -> packetSize < commandSize + outgoingCommand -> fragmentLength)) - { - host -> continueSending = 1; - - break; - } - - currentCommand = enet_list_next (currentCommand); - - if (outgoingCommand -> packet != NULL && outgoingCommand -> fragmentOffset == 0) - { - peer -> packetThrottleCounter += ENET_PEER_PACKET_THROTTLE_COUNTER; - peer -> packetThrottleCounter %= ENET_PEER_PACKET_THROTTLE_SCALE; - - if (peer -> packetThrottleCounter > peer -> packetThrottle) - { - enet_uint16 reliableSequenceNumber = outgoingCommand -> reliableSequenceNumber, - unreliableSequenceNumber = outgoingCommand -> unreliableSequenceNumber; - for (;;) - { - -- outgoingCommand -> packet -> referenceCount; - - if (outgoingCommand -> packet -> referenceCount == 0) - enet_packet_destroy (outgoingCommand -> packet); - - enet_list_remove (& outgoingCommand -> outgoingCommandList); - enet_free (outgoingCommand); - - if (currentCommand == enet_list_end (& peer -> outgoingUnreliableCommands)) - break; - - outgoingCommand = (ENetOutgoingCommand *) currentCommand; - if (outgoingCommand -> reliableSequenceNumber != reliableSequenceNumber || - outgoingCommand -> unreliableSequenceNumber != unreliableSequenceNumber) - break; - - currentCommand = enet_list_next (currentCommand); - } - - continue; - } - } - - buffer -> data = command; - buffer -> dataLength = commandSize; - - host -> packetSize += buffer -> dataLength; - - * command = outgoingCommand -> command; - - enet_list_remove (& outgoingCommand -> outgoingCommandList); - - if (outgoingCommand -> packet != NULL) - { - ++ buffer; - - buffer -> data = outgoingCommand -> packet -> data + outgoingCommand -> fragmentOffset; - buffer -> dataLength = outgoingCommand -> fragmentLength; - - host -> packetSize += buffer -> dataLength; - - enet_list_insert (enet_list_end (& peer -> sentUnreliableCommands), outgoingCommand); - } - else - enet_free (outgoingCommand); - - ++ command; - ++ buffer; - } - - host -> commandCount = command - host -> commands; - host -> bufferCount = buffer - host -> buffers; - - if (peer -> state == ENET_PEER_STATE_DISCONNECT_LATER && - enet_list_empty (& peer -> outgoingReliableCommands) && - enet_list_empty (& peer -> outgoingUnreliableCommands) && - enet_list_empty (& peer -> sentReliableCommands) && - enet_list_empty (& peer -> sentUnreliableCommands)) - enet_peer_disconnect (peer, peer -> eventData); -} - static int enet_protocol_check_timeouts (ENetHost * host, ENetPeer * peer, ENetEvent * event) { @@ -1435,7 +1334,7 @@ enet_protocol_check_timeouts (ENetHost * host, ENetPeer * peer, ENetEvent * even ENetListIterator currentCommand, insertPosition; currentCommand = enet_list_begin (& peer -> sentReliableCommands); - insertPosition = enet_list_begin (& peer -> outgoingReliableCommands); + insertPosition = enet_list_begin (& peer -> outgoingCommands); while (currentCommand != enet_list_end (& peer -> sentReliableCommands)) { @@ -1482,7 +1381,7 @@ enet_protocol_check_timeouts (ENetHost * host, ENetPeer * peer, ENetEvent * even } static int -enet_protocol_send_reliable_outgoing_commands (ENetHost * host, ENetPeer * peer) +enet_protocol_check_outgoing_commands (ENetHost * host, ENetPeer * peer) { ENetProtocol * command = & host -> commands [host -> commandCount]; ENetBuffer * buffer = & host -> buffers [host -> bufferCount]; @@ -1493,50 +1392,53 @@ enet_protocol_send_reliable_outgoing_commands (ENetHost * host, ENetPeer * peer) size_t commandSize; int windowExceeded = 0, windowWrap = 0, canPing = 1; - currentCommand = enet_list_begin (& peer -> outgoingReliableCommands); + currentCommand = enet_list_begin (& peer -> outgoingCommands); - while (currentCommand != enet_list_end (& peer -> outgoingReliableCommands)) + while (currentCommand != enet_list_end (& peer -> outgoingCommands)) { outgoingCommand = (ENetOutgoingCommand *) currentCommand; - channel = outgoingCommand -> command.header.channelID < peer -> channelCount ? & peer -> channels [outgoingCommand -> command.header.channelID] : NULL; - reliableWindow = outgoingCommand -> reliableSequenceNumber / ENET_PEER_RELIABLE_WINDOW_SIZE; - if (channel != NULL) + if (outgoingCommand -> command.header.command & ENET_PROTOCOL_COMMAND_FLAG_ACKNOWLEDGE) { - if (! windowWrap && - outgoingCommand -> sendAttempts < 1 && - ! (outgoingCommand -> reliableSequenceNumber % ENET_PEER_RELIABLE_WINDOW_SIZE) && - (channel -> reliableWindows [(reliableWindow + ENET_PEER_RELIABLE_WINDOWS - 1) % ENET_PEER_RELIABLE_WINDOWS] >= ENET_PEER_RELIABLE_WINDOW_SIZE || - channel -> usedReliableWindows & ((((1 << ENET_PEER_FREE_RELIABLE_WINDOWS) - 1) << reliableWindow) | - (((1 << ENET_PEER_FREE_RELIABLE_WINDOWS) - 1) >> (ENET_PEER_RELIABLE_WINDOWS - reliableWindow))))) - windowWrap = 1; - if (windowWrap) + channel = outgoingCommand -> command.header.channelID < peer -> channelCount ? & peer -> channels [outgoingCommand -> command.header.channelID] : NULL; + reliableWindow = outgoingCommand -> reliableSequenceNumber / ENET_PEER_RELIABLE_WINDOW_SIZE; + if (channel != NULL) { - currentCommand = enet_list_next (currentCommand); + if (! windowWrap && + outgoingCommand -> sendAttempts < 1 && + ! (outgoingCommand -> reliableSequenceNumber % ENET_PEER_RELIABLE_WINDOW_SIZE) && + (channel -> reliableWindows [(reliableWindow + ENET_PEER_RELIABLE_WINDOWS - 1) % ENET_PEER_RELIABLE_WINDOWS] >= ENET_PEER_RELIABLE_WINDOW_SIZE || + channel -> usedReliableWindows & ((((1 << ENET_PEER_FREE_RELIABLE_WINDOWS) - 1) << reliableWindow) | + (((1 << ENET_PEER_FREE_RELIABLE_WINDOWS) - 1) >> (ENET_PEER_RELIABLE_WINDOWS - reliableWindow))))) + windowWrap = 1; + if (windowWrap) + { + currentCommand = enet_list_next (currentCommand); - continue; + continue; + } } - } - if (outgoingCommand -> packet != NULL) - { - if (! windowExceeded) + if (outgoingCommand -> packet != NULL) { - enet_uint32 windowSize = (peer -> packetThrottle * peer -> windowSize) / ENET_PEER_PACKET_THROTTLE_SCALE; + if (! windowExceeded) + { + enet_uint32 windowSize = (peer -> packetThrottle * peer -> windowSize) / ENET_PEER_PACKET_THROTTLE_SCALE; - if (peer -> reliableDataInTransit + outgoingCommand -> fragmentLength > ENET_MAX (windowSize, peer -> mtu)) - windowExceeded = 1; - } - if (windowExceeded) - { - currentCommand = enet_list_next (currentCommand); + if (peer -> reliableDataInTransit + outgoingCommand -> fragmentLength > ENET_MAX (windowSize, peer -> mtu)) + windowExceeded = 1; + } + if (windowExceeded) + { + currentCommand = enet_list_next (currentCommand); - continue; + continue; + } } + + canPing = 0; } - canPing = 0; - commandSize = commandSizes [outgoingCommand -> command.header.command & ENET_PROTOCOL_COMMAND_MASK]; if (command >= & host -> commands [sizeof (host -> commands) / sizeof (ENetProtocol)] || buffer + 1 >= & host -> buffers [sizeof (host -> buffers) / sizeof (ENetBuffer)] || @@ -1551,33 +1453,80 @@ enet_protocol_send_reliable_outgoing_commands (ENetHost * host, ENetPeer * peer) currentCommand = enet_list_next (currentCommand); - if (channel != NULL && outgoingCommand -> sendAttempts < 1) + if (outgoingCommand -> command.header.command & ENET_PROTOCOL_COMMAND_FLAG_ACKNOWLEDGE) { - channel -> usedReliableWindows |= 1 << reliableWindow; - ++ channel -> reliableWindows [reliableWindow]; - } + if (channel != NULL && outgoingCommand -> sendAttempts < 1) + { + channel -> usedReliableWindows |= 1 << reliableWindow; + ++ channel -> reliableWindows [reliableWindow]; + } - ++ outgoingCommand -> sendAttempts; + ++ outgoingCommand -> sendAttempts; - if (outgoingCommand -> roundTripTimeout == 0) - { - outgoingCommand -> roundTripTimeout = peer -> roundTripTime + 4 * peer -> roundTripTimeVariance; - outgoingCommand -> roundTripTimeoutLimit = peer -> timeoutLimit * outgoingCommand -> roundTripTimeout; + if (outgoingCommand -> roundTripTimeout == 0) + { + outgoingCommand -> roundTripTimeout = peer -> roundTripTime + 4 * peer -> roundTripTimeVariance; + outgoingCommand -> roundTripTimeoutLimit = peer -> timeoutLimit * outgoingCommand -> roundTripTimeout; + } + + if (enet_list_empty (& peer -> sentReliableCommands)) + peer -> nextTimeout = host -> serviceTime + outgoingCommand -> roundTripTimeout; + + enet_list_insert (enet_list_end (& peer -> sentReliableCommands), + enet_list_remove (& outgoingCommand -> outgoingCommandList)); + + outgoingCommand -> sentTime = host -> serviceTime; + + host -> headerFlags |= ENET_PROTOCOL_HEADER_FLAG_SENT_TIME; + + peer -> reliableDataInTransit += outgoingCommand -> fragmentLength; } + else + { + if (outgoingCommand -> packet != NULL && outgoingCommand -> fragmentOffset == 0) + { + peer -> packetThrottleCounter += ENET_PEER_PACKET_THROTTLE_COUNTER; + peer -> packetThrottleCounter %= ENET_PEER_PACKET_THROTTLE_SCALE; - if (enet_list_empty (& peer -> sentReliableCommands)) - peer -> nextTimeout = host -> serviceTime + outgoingCommand -> roundTripTimeout; + if (peer -> packetThrottleCounter > peer -> packetThrottle) + { + enet_uint16 reliableSequenceNumber = outgoingCommand -> reliableSequenceNumber, + unreliableSequenceNumber = outgoingCommand -> unreliableSequenceNumber; + for (;;) + { + -- outgoingCommand -> packet -> referenceCount; - enet_list_insert (enet_list_end (& peer -> sentReliableCommands), - enet_list_remove (& outgoingCommand -> outgoingCommandList)); + if (outgoingCommand -> packet -> referenceCount == 0) + enet_packet_destroy (outgoingCommand -> packet); - outgoingCommand -> sentTime = host -> serviceTime; + enet_list_remove (& outgoingCommand -> outgoingCommandList); + enet_free (outgoingCommand); + + if (currentCommand == enet_list_end (& peer -> outgoingCommands)) + break; + + outgoingCommand = (ENetOutgoingCommand *) currentCommand; + if (outgoingCommand -> reliableSequenceNumber != reliableSequenceNumber || + outgoingCommand -> unreliableSequenceNumber != unreliableSequenceNumber) + break; + + currentCommand = enet_list_next (currentCommand); + } + + continue; + } + } + + enet_list_remove (& outgoingCommand -> outgoingCommandList); + + if (outgoingCommand -> packet != NULL) + enet_list_insert (enet_list_end (& peer -> sentUnreliableCommands), outgoingCommand); + } buffer -> data = command; buffer -> dataLength = commandSize; host -> packetSize += buffer -> dataLength; - host -> headerFlags |= ENET_PROTOCOL_HEADER_FLAG_SENT_TIME; * command = outgoingCommand -> command; @@ -1589,9 +1538,10 @@ enet_protocol_send_reliable_outgoing_commands (ENetHost * host, ENetPeer * peer) buffer -> dataLength = outgoingCommand -> fragmentLength; host -> packetSize += outgoingCommand -> fragmentLength; - - peer -> reliableDataInTransit += outgoingCommand -> fragmentLength; } + else + if (! (outgoingCommand -> command.header.command & ENET_PROTOCOL_COMMAND_FLAG_ACKNOWLEDGE)) + enet_free (outgoingCommand); ++ peer -> packetsSent; @@ -1602,6 +1552,12 @@ enet_protocol_send_reliable_outgoing_commands (ENetHost * host, ENetPeer * peer) host -> commandCount = command - host -> commands; host -> bufferCount = buffer - host -> buffers; + if (peer -> state == ENET_PEER_STATE_DISCONNECT_LATER && + enet_list_empty (& peer -> outgoingCommands) && + enet_list_empty (& peer -> sentReliableCommands) && + enet_list_empty (& peer -> sentUnreliableCommands)) + enet_peer_disconnect (peer, peer -> eventData); + return canPing; } @@ -1645,18 +1601,15 @@ enet_protocol_send_outgoing_commands (ENetHost * host, ENetEvent * event, int ch continue; } - if ((enet_list_empty (& currentPeer -> outgoingReliableCommands) || - enet_protocol_send_reliable_outgoing_commands (host, currentPeer)) && + if ((enet_list_empty (& currentPeer -> outgoingCommands) || + enet_protocol_check_outgoing_commands (host, currentPeer)) && enet_list_empty (& currentPeer -> sentReliableCommands) && ENET_TIME_DIFFERENCE (host -> serviceTime, currentPeer -> lastReceiveTime) >= currentPeer -> pingInterval && currentPeer -> mtu - host -> packetSize >= sizeof (ENetProtocolPing)) { enet_peer_ping (currentPeer); - enet_protocol_send_reliable_outgoing_commands (host, currentPeer); + enet_protocol_check_outgoing_commands (host, currentPeer); } - - if (! enet_list_empty (& currentPeer -> outgoingUnreliableCommands)) - enet_protocol_send_unreliable_outgoing_commands (host, currentPeer); if (host -> commandCount == 0) continue; @@ -1670,7 +1623,7 @@ enet_protocol_send_outgoing_commands (ENetHost * host, ENetEvent * event, int ch enet_uint32 packetLoss = currentPeer -> packetsLost * ENET_PEER_PACKET_LOSS_SCALE / currentPeer -> packetsSent; #ifdef ENET_DEBUG - printf ("peer %u: %f%%+-%f%% packet loss, %u+-%u ms round trip time, %f%% throttle, %u/%u outgoing, %u/%u incoming\n", currentPeer -> incomingPeerID, currentPeer -> packetLoss / (float) ENET_PEER_PACKET_LOSS_SCALE, currentPeer -> packetLossVariance / (float) ENET_PEER_PACKET_LOSS_SCALE, currentPeer -> roundTripTime, currentPeer -> roundTripTimeVariance, currentPeer -> packetThrottle / (float) ENET_PEER_PACKET_THROTTLE_SCALE, enet_list_size (& currentPeer -> outgoingReliableCommands), enet_list_size (& currentPeer -> outgoingUnreliableCommands), currentPeer -> channels != NULL ? enet_list_size (& currentPeer -> channels -> incomingReliableCommands) : 0, currentPeer -> channels != NULL ? enet_list_size (& currentPeer -> channels -> incomingUnreliableCommands) : 0); + printf ("peer %u: %f%%+-%f%% packet loss, %u+-%u ms round trip time, %f%% throttle, %u outgoing, %u/%u incoming\n", currentPeer -> incomingPeerID, currentPeer -> packetLoss / (float) ENET_PEER_PACKET_LOSS_SCALE, currentPeer -> packetLossVariance / (float) ENET_PEER_PACKET_LOSS_SCALE, currentPeer -> roundTripTime, currentPeer -> roundTripTimeVariance, currentPeer -> packetThrottle / (float) ENET_PEER_PACKET_THROTTLE_SCALE, enet_list_size (& currentPeer -> outgoingCommands), currentPeer -> channels != NULL ? enet_list_size (& currentPeer -> channels -> incomingReliableCommands) : 0, currentPeer -> channels != NULL ? enet_list_size (& currentPeer -> channels -> incomingUnreliableCommands) : 0); #endif currentPeer -> packetLossVariance = (currentPeer -> packetLossVariance * 3 + ENET_DIFFERENCE (packetLoss, currentPeer -> packetLoss)) / 4;