diff -uprN linux-2.6.11.10.orig/include/linux/tcp.h linux-2.6.11.10/include/linux/tcp.h --- linux-2.6.11.10.orig/include/linux/tcp.h 2005-05-17 02:51:48.000000000 +0900 +++ linux-2.6.11.10/include/linux/tcp.h 2005-06-13 14:31:41.000000000 +0900 @@ -371,6 +371,24 @@ struct tcp_sock { __u32 total_retrans; /* Total retransmits for entire connection */ +#ifdef CONFIG_RETRANS_HINTS + /* retrans queue hinting */ + struct sk_buff* mark_head_lost_skb_hint; + int mark_head_lost_cnt_hint; + + struct sk_buff* update_scoreboard_skb_hint; + + struct sk_buff* xmit_retransmit_queue_lost_skb_hint; + int xmit_retransmit_queue_lost_cnt_hint; + struct sk_buff* xmit_retransmit_queue_forward_skb_hint; + int xmit_retransmit_queue_forward_cnt_hint; + + /* SACK fastpath */ + struct tcp_sack_block recv_sack_cache[4]; + int sackfastpath_facket_cnt_hint; + struct sk_buff* sackfastpath_skb_hint; +#endif + /* The syn_wait_lock is necessary only to avoid proc interface having * to grab the main lock sock while browsing the listening hash * (otherwise it's deadlock prone). diff -uprN linux-2.6.11.10.orig/include/net/sock.h linux-2.6.11.10/include/net/sock.h --- linux-2.6.11.10.orig/include/net/sock.h 2005-05-17 02:51:51.000000000 +0900 +++ linux-2.6.11.10/include/net/sock.h 2005-06-13 14:31:41.000000000 +0900 @@ -1200,6 +1200,18 @@ static inline struct page *sk_stream_all (skb != (struct sk_buff *)&(sk)->sk_write_queue); \ skb = skb->next) +#ifdef CONFIG_RETRANS_HINTS +#define sk_stream_for_retrans_queue_from(skb, skb_init, sk) \ + for (skb = (struct sk_buff *)(skb_init); \ + (skb != (sk)->sk_send_head) && \ + (skb != (struct sk_buff *)&(sk)->sk_write_queue); \ + skb = skb->next) + +#define hint_advance(skb, snd_una) \ + if ((skb) && !after(TCP_SKB_CB(skb)->end_seq, snd_una)) \ + skb = NULL; +#endif + /* * Default write policy as shown to user space via poll/select/SIGIO */ diff -uprN linux-2.6.11.10.orig/include/net/tcp.h linux-2.6.11.10/include/net/tcp.h --- linux-2.6.11.10.orig/include/net/tcp.h 2005-05-17 02:51:51.000000000 +0900 +++ linux-2.6.11.10/include/net/tcp.h 2005-06-13 15:22:35.000000000 +0900 @@ -930,7 +930,6 @@ extern void tcp_unhash(struct sock *sk extern int tcp_v4_hash_connecting(struct sock *sk); - /* From syncookies.c */ extern struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb, struct ip_options *opt); @@ -944,6 +943,9 @@ extern int tcp_retransmit_skb(struct soc extern void tcp_xmit_retransmit_queue(struct sock *); extern void tcp_simple_retransmit(struct sock *); extern int tcp_trim_head(struct sock *, struct sk_buff *, u32); +#ifdef CONFIG_RETRANS_HINTS +extern void tcp_clear_all_retrans_hints(struct tcp_sock *); +#endif extern void tcp_send_probe0(struct sock *); extern void tcp_send_partial(struct sock *); diff -uprN linux-2.6.11.10.orig/net/ipv4/Kconfig linux-2.6.11.10/net/ipv4/Kconfig --- linux-2.6.11.10.orig/net/ipv4/Kconfig 2005-05-17 02:51:57.000000000 +0900 +++ linux-2.6.11.10/net/ipv4/Kconfig 2005-06-13 14:41:19.000000000 +0900 @@ -365,5 +365,10 @@ config IP_TCPDIAG config IP_TCPDIAG_IPV6 def_bool (IP_TCPDIAG=y && IPV6=y) || (IP_TCPDIAG=m && IPV6) -source "net/ipv4/ipvs/Kconfig" +config RETRANS_HINTS + bool "Retransmit queue hinting and SACK fastpath" + depends on INET && EXPERIMENTAL + help + If unsure, say N. +source "net/ipv4/ipvs/Kconfig" diff -uprN linux-2.6.11.10.orig/net/ipv4/tcp_input.c linux-2.6.11.10/net/ipv4/tcp_input.c --- linux-2.6.11.10.orig/net/ipv4/tcp_input.c 2005-05-17 02:52:00.000000000 +0900 +++ linux-2.6.11.10/net/ipv4/tcp_input.c 2005-06-13 15:49:25.000000000 +0900 @@ -971,6 +971,9 @@ tcp_sacktag_write_queue(struct sock *sk, int prior_fackets; u32 lost_retrans = 0; int flag = 0; +#ifdef CONFIG_RETRANS_HINTS + int dup_sack = 0; +#endif int i; /* So, SACKs for already sent large segments will be lost. @@ -985,13 +988,33 @@ tcp_sacktag_write_queue(struct sock *sk, tp->fackets_out = 0; prior_fackets = tp->fackets_out; - for (i=0; istart_seq); __u32 end_seq = ntohl(sp->end_seq); int fack_count = 0; int dup_sack = 0; - +#else + /* SACK fastpath: + * if the only SACK change is the increase of the end_seq of + * the first block then only apply that SACK block + * and use retrans queue hinting otherwise slowpath */ + flag = 1; + for (i = 0; i < num_sacks; i++) { + __u32 start_seq = ntohl(sp[i].start_seq); + __u32 end_seq = ntohl(sp[i].end_seq); + + if (i == 0 && + (tp->recv_sack_cache[i].start_seq != start_seq)) { + flag = 0; + } else if ((tp->recv_sack_cache[i].start_seq != start_seq) && + (tp->recv_sack_cache[i].end_seq != end_seq)){ + flag = 0; + } + tp->recv_sack_cache[i].start_seq = start_seq; + tp->recv_sack_cache[i].end_seq = end_seq; +#endif /* Check for D-SACK. */ if (i == 0) { u32 ack = TCP_SKB_CB(ack_skb)->ack_seq; @@ -1023,14 +1046,65 @@ tcp_sacktag_write_queue(struct sock *sk, return 0; } +#ifdef CONFIG_RETRANS_HINTS + } + + if (flag) { + num_sacks=1; + } else { + int j; + tp->sackfastpath_skb_hint = NULL; + + /* order SACK blocks to allow in order walk of the retrans queue */ + for (i = num_sacks - 1; i > 0; i--) { + for (j = 0; j < i; j++) { + if (after(ntohl(sp[j].start_seq), ntohl(sp[j+1].start_seq))) { + sp[j].start_seq = htonl(tp->recv_sack_cache[j+1].start_seq); + sp[j].end_seq = htonl(tp->recv_sack_cache[j+1].end_seq); + sp[j+1].start_seq = htonl(tp->recv_sack_cache[j].start_seq); + sp[j+1].end_seq = htonl(tp->recv_sack_cache[j].end_seq); + } + + } + } + + } + + /* clear flag as used for different purpose in following code */ + flag = 0; + + for (i = 0; i < num_sacks; i++, sp++) { + struct sk_buff *skb; + __u32 start_seq = ntohl(sp->start_seq); + __u32 end_seq = ntohl(sp->end_seq); + int fack_count; + + /* Use SACK fastpath hint if valid */ + if (tp->sackfastpath_skb_hint != NULL) { + skb = tp->sackfastpath_skb_hint; + fack_count = tp->sackfastpath_facket_cnt_hint; + } else { + skb = sk->sk_write_queue.next; + fack_count = 0; + } + +#endif /* Event "B" in the comment above. */ if (after(end_seq, tp->high_seq)) flag |= FLAG_DATA_LOST; +#ifndef CONFIG_RETRANS_HINTS sk_stream_for_retrans_queue(skb, sk) { +#else + sk_stream_for_retrans_queue_from(skb, skb, sk) { +#endif u8 sacked = TCP_SKB_CB(skb)->sacked; int in_sack; +#ifdef CONFIG_RETRANS_HINTS + tp->sackfastpath_skb_hint = skb; + tp->sackfastpath_facket_cnt_hint = fack_count; +#endif /* The retransmission queue is always in order, so * we can short-circuit the walk early. */ @@ -1083,6 +1157,10 @@ tcp_sacktag_write_queue(struct sock *sk, TCP_SKB_CB(skb)->sacked &= ~(TCPCB_LOST|TCPCB_SACKED_RETRANS); tp->lost_out -= tcp_skb_pcount(skb); tp->retrans_out -= tcp_skb_pcount(skb); +#ifdef CONFIG_RETRANS_HINTS + /* clear lost hint */ + tp->xmit_retransmit_queue_lost_skb_hint = NULL; +#endif } } else { /* New sack for not retransmitted frame, @@ -1095,6 +1173,10 @@ tcp_sacktag_write_queue(struct sock *sk, if (sacked & TCPCB_LOST) { TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST; tp->lost_out -= tcp_skb_pcount(skb); +#ifdef CONFIG_RETRANS_HINTS + /* clear lost hint */ + tp->xmit_retransmit_queue_lost_skb_hint = NULL; +#endif } } @@ -1150,6 +1232,10 @@ tcp_sacktag_write_queue(struct sock *sk, TCP_SKB_CB(skb)->sacked |= TCPCB_LOST; flag |= FLAG_DATA_SACKED; NET_INC_STATS_BH(LINUX_MIB_TCPLOSTRETRANSMIT); +#ifdef CONFIG_RETRANS_HINTS + /* clear lost hint */ + tp->xmit_retransmit_queue_lost_skb_hint = NULL; +#endif } } } @@ -1257,6 +1343,10 @@ static void tcp_enter_frto_loss(struct s void tcp_clear_retrans(struct tcp_sock *tp) { +#ifdef CONFIG_RETRANS_HINTS + tcp_clear_all_retrans_hints(tp); +#endif + tp->left_out = 0; tp->retrans_out = 0; @@ -1536,17 +1626,46 @@ static void tcp_mark_head_lost(struct so int packets, u32 high_seq) { struct sk_buff *skb; +#ifndef CONFIG_RETRANS_HINTS int cnt = packets; +#else + int cnt = 0; +#endif BUG_TRAP(cnt <= tp->packets_out); +#ifndef CONFIG_RETRANS_HINTS sk_stream_for_retrans_queue(skb, sk) { cnt -= tcp_skb_pcount(skb); if (cnt < 0 || after(TCP_SKB_CB(skb)->end_seq, high_seq)) +#else + if (tp->mark_head_lost_skb_hint != NULL) { + skb = tp->mark_head_lost_skb_hint; + cnt = tp->mark_head_lost_cnt_hint; + } else { + skb = sk->sk_write_queue.next; + } + + sk_stream_for_retrans_queue_from(skb, skb, sk) { + /* TODO: do this better */ + /* this is not the most efficient way to do this... */ + tp->mark_head_lost_skb_hint = skb; + tp->mark_head_lost_cnt_hint = cnt; + if (++cnt > packets || after(TCP_SKB_CB(skb)->end_seq, high_seq)) +#endif break; if (!(TCP_SKB_CB(skb)->sacked&TCPCB_TAGBITS)) { TCP_SKB_CB(skb)->sacked |= TCPCB_LOST; tp->lost_out += tcp_skb_pcount(skb); +#ifdef CONFIG_RETRANS_HINTS + /* clear xmit_retransmit_queue hints + * if this is beyond hint */ + if (tp->xmit_retransmit_queue_lost_skb_hint != NULL && + before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->xmit_retransmit_queue_lost_skb_hint)->seq) ){ + + tp->xmit_retransmit_queue_lost_skb_hint = NULL; + } +#endif } } tcp_sync_left_out(tp); @@ -1573,6 +1692,7 @@ static void tcp_update_scoreboard(struct if (tcp_head_timedout(sk, tp)) { struct sk_buff *skb; +#ifndef CONFIG_RETRANS_HINTS sk_stream_for_retrans_queue(skb, sk) { if (tcp_skb_timedout(tp, skb) && !(TCP_SKB_CB(skb)->sacked&TCPCB_TAGBITS)) { @@ -1580,6 +1700,31 @@ static void tcp_update_scoreboard(struct tp->lost_out += tcp_skb_pcount(skb); } } +#else + if (tp->update_scoreboard_skb_hint != NULL) { + skb = tp->update_scoreboard_skb_hint; + } else { + skb = sk->sk_write_queue.next; + } + + sk_stream_for_retrans_queue_from(skb, skb, sk) { + if (tcp_skb_timedout(tp, skb)) { + if (!(TCP_SKB_CB(skb)->sacked&TCPCB_TAGBITS)) { + TCP_SKB_CB(skb)->sacked |= TCPCB_LOST; + tp->lost_out += tcp_skb_pcount(skb); + /* clear xmit_retrans hint */ + if (tp->xmit_retransmit_queue_lost_skb_hint != NULL && + before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->xmit_retransmit_queue_lost_skb_hint)->seq) ){ + + tp->xmit_retransmit_queue_lost_skb_hint = NULL; + } + } + } else { + break; + } + } + tp->update_scoreboard_skb_hint = skb; +#endif tcp_sync_left_out(tp); } } @@ -1667,6 +1812,12 @@ static void tcp_undo_cwr(struct tcp_sock } tcp_moderate_cwnd(tp); tp->snd_cwnd_stamp = tcp_time_stamp; + +#ifdef CONFIG_RETRANS_HINTS + /* There is something screwy going on with the retrans hints after + an undo */ + tcp_clear_all_retrans_hints(tp); +#endif } static inline int tcp_may_undo(struct tcp_sock *tp) @@ -1750,6 +1901,9 @@ static int tcp_try_undo_loss(struct sock sk_stream_for_retrans_queue(skb, sk) { TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST; } +#ifdef CONFIG_RETRANS_HINTS + tcp_clear_all_retrans_hints(tp); +#endif DBGUNDO(sk, tp, "partial loss"); tp->lost_out = 0; tp->left_out = tp->sacked_out; @@ -2389,8 +2543,12 @@ static int tcp_tso_acked(struct sock *sk *seq_rtt = now - scb->when; if (sacked & TCPCB_SACKED_ACKED) tp->sacked_out -= packets_acked; - if (sacked & TCPCB_LOST) + if (sacked & TCPCB_LOST) { tp->lost_out -= packets_acked; +#ifdef CONFIG_RETRANS_HINTS + tp->xmit_retransmit_queue_lost_cnt_hint--; +#endif + } if (sacked & TCPCB_URG) { if (tp->urg_mode && !before(seq, tp->snd_up)) @@ -2422,6 +2580,17 @@ static int tcp_clean_rtx_queue(struct so int acked = 0; __s32 seq_rtt = -1; +#ifdef CONFIG_RETRANS_HINTS + int clean_cnt = tp->packets_out; + + /* clear retrans queue hinting if necessary */ + hint_advance(tp->mark_head_lost_skb_hint, tp->snd_una); + hint_advance(tp->update_scoreboard_skb_hint, tp->snd_una); + hint_advance(tp->xmit_retransmit_queue_lost_skb_hint, tp->snd_una); + hint_advance(tp->xmit_retransmit_queue_forward_skb_hint, tp->snd_una); + hint_advance(tp->sackfastpath_skb_hint, tp->snd_una); +#endif + while ((skb = skb_peek(&sk->sk_write_queue)) && skb != sk->sk_send_head) { struct tcp_skb_cb *scb = TCP_SKB_CB(skb); @@ -2477,6 +2646,14 @@ static int tcp_clean_rtx_queue(struct so sk_stream_free_skb(sk, skb); } +#ifdef CONFIG_RETRANS_HINTS + /* keep retrans queue hinting valid */ + clean_cnt = clean_cnt - tp->packets_out; + tp->mark_head_lost_cnt_hint -= clean_cnt; + tp->xmit_retransmit_queue_forward_cnt_hint -= clean_cnt; + tp->sackfastpath_facket_cnt_hint -= clean_cnt; +#endif + if (acked&FLAG_ACKED) { tcp_ack_update_rtt(tp, acked, seq_rtt); tcp_ack_packets_out(sk, tp); diff -uprN linux-2.6.11.10.orig/net/ipv4/tcp_output.c linux-2.6.11.10/net/ipv4/tcp_output.c --- linux-2.6.11.10.orig/net/ipv4/tcp_output.c 2005-05-17 02:52:00.000000000 +0900 +++ linux-2.6.11.10/net/ipv4/tcp_output.c 2005-06-13 15:50:04.000000000 +0900 @@ -473,6 +473,10 @@ static int tcp_fragment(struct sock *sk, if (nsize < 0) nsize = 0; +#ifdef CONFIG_RETRANS_HINTS + tcp_clear_all_retrans_hints(tp); +#endif + if (skb_cloned(skb) && skb_is_nonlinear(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) @@ -933,6 +937,11 @@ static void tcp_retrans_try_collapse(str BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1); +#ifdef CONFIG_RETRANS_HINTS + /* changing transmit queue under us so clear hints */ + tcp_clear_all_retrans_hints(tp); +#endif + /* Ok. We will be able to collapse the packet. */ __skb_unlink(next_skb, next_skb->list); @@ -1001,6 +1010,9 @@ void tcp_simple_retransmit(struct sock * } } } +#ifdef CONFIG_RETRANS_HINTS + tcp_clear_all_retrans_hints(tp); +#endif if (!lost) return; @@ -1143,6 +1155,17 @@ int tcp_retransmit_skb(struct sock *sk, return err; } +#ifdef CONFIG_RETRANS_HINTS +void tcp_clear_all_retrans_hints(struct tcp_sock *tp) +{ + tp->mark_head_lost_skb_hint = NULL; + tp->update_scoreboard_skb_hint = NULL; + tp->xmit_retransmit_queue_lost_skb_hint = NULL; + tp->xmit_retransmit_queue_forward_skb_hint = NULL; + tp->sackfastpath_skb_hint = NULL; +} +#endif + /* This gets called after a retransmit timeout, and the initially * retransmitted data is acknowledged. It tries to continue * resending the rest of the retransmit queue, until either