[Openswan dev] RFC KLIPS patch for increased throughput on multicore systems
Jagdish Motwani
jagdish.motwani at elitecore.com
Tue Apr 17 04:41:39 EDT 2012
Ref mail: KLIPS tdb_lock as rwlock instead of spinlock?
Please find attached klips patch for increased throughput of single
tunnel on multicore systems.
This patch is just a RFC and may have side effects. Please report them
by testing or code review.
In-order to get increased throughput,
1)Apply the patch
2)Use Receive Packet Steering to send packets of different
flows on different cpu's (/sys/class/net/eth0/queues/rx-0/rps_cpus)
Regrads,
Jagdish Motwani
Software Engineer
Elitecore Technologies Pvt. Ltd.
-------------- next part --------------
diff --git a/linux/include/openswan/ipsec_proto.h b/linux/include/openswan/ipsec_proto.h
index f40336c..db21b92 100644
--- a/linux/include/openswan/ipsec_proto.h
+++ b/linux/include/openswan/ipsec_proto.h
@@ -41,7 +41,7 @@ extern struct prng ipsec_prng;
/* ipsec_sa.c */
extern struct ipsec_sa *ipsec_sadb_hash[SADB_HASHMOD];
-extern spinlock_t tdb_lock;
+extern rwlock_t tdb_lock;
extern int ipsec_sadb_init(void);
extern int ipsec_sadb_cleanup(__u8);
diff --git a/linux/net/ipsec/ipsec_proc.c b/linux/net/ipsec/ipsec_proc.c
index ec25a1b..9c3a331 100644
--- a/linux/net/ipsec/ipsec_proc.c
+++ b/linux/net/ipsec/ipsec_proc.c
@@ -464,7 +464,7 @@ ipsec_spi_get_info(char *buffer,
(int)offset,
length);
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
for (i = 0; i < SADB_HASHMOD; i++) {
for (sa_p = ipsec_sadb_hash[i];
@@ -492,7 +492,7 @@ ipsec_spi_get_info(char *buffer,
}
done_spi_i:
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
*start = buffer + (offset - begin); /* Start of wanted data */
return len - (offset - begin);
@@ -523,7 +523,7 @@ ipsec_spigrp_get_info(char *buffer,
(int)offset,
length);
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
for (i = 0; i < SADB_HASHMOD; i++) {
for (sa_p = ipsec_sadb_hash[i];
@@ -563,7 +563,7 @@ ipsec_spigrp_get_info(char *buffer,
}
done_spigrp_i:
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
*start = buffer + (offset - begin); /* Start of wanted data */
return len - (offset - begin);
@@ -592,7 +592,7 @@ ipsec_saraw_get_info(char *buffer,
(int)offset,
length);
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
for (sa_p = ipsec_sa_raw; sa_p; sa_p = sa_p->ips_raw) {
@@ -616,7 +616,7 @@ ipsec_saraw_get_info(char *buffer,
}
done_spi_i:
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
*start = buffer + (offset - begin); /* Start of wanted data */
return len - (offset - begin);
diff --git a/linux/net/ipsec/ipsec_rcv.c b/linux/net/ipsec/ipsec_rcv.c
index 8800a5d..9b6d468 100644
--- a/linux/net/ipsec/ipsec_rcv.c
+++ b/linux/net/ipsec/ipsec_rcv.c
@@ -1980,7 +1980,7 @@ ipsec_rsm(struct ipsec_rcv_state *irs)
/*
* make sure nothing is removed from underneath us
*/
- spin_lock_bh(&tdb_lock);
+ read_lock_bh(&tdb_lock);
/*
* if we have a valid said, then we must check it here to ensure it
@@ -2021,7 +2021,7 @@ ipsec_rsm(struct ipsec_rcv_state *irs)
* things are on hold until we return here in the next/new state
* we check our SA is valid when we return
*/
- spin_unlock_bh(&tdb_lock);
+ read_unlock_bh(&tdb_lock);
return;
} else {
/* bad result, force state change to done */
@@ -2036,7 +2036,7 @@ ipsec_rsm(struct ipsec_rcv_state *irs)
/*
* all done with anything needing locks
*/
- spin_unlock_bh(&tdb_lock);
+ read_unlock_bh(&tdb_lock);
if (irs->lastipsp) {
ipsec_sa_put(irs->lastipsp, IPSEC_REFRX);
diff --git a/linux/net/ipsec/ipsec_sa.c b/linux/net/ipsec/ipsec_sa.c
index 7e77f73..ccd2b96 100644
--- a/linux/net/ipsec/ipsec_sa.c
+++ b/linux/net/ipsec/ipsec_sa.c
@@ -83,8 +83,7 @@
#define SENDERR(_x) do { error = -(_x); goto errlab; } while (0)
struct ipsec_sa *ipsec_sadb_hash[SADB_HASHMOD];
-DEFINE_SPINLOCK(tdb_lock);
-
+DEFINE_RWLOCK(tdb_lock);
#ifdef IPSEC_SA_RECOUNT_DEBUG
struct ipsec_sa *ipsec_sa_raw = NULL;
#endif
@@ -714,12 +713,12 @@ ipsec_sa_add(struct ipsec_sa *ips)
hashval = IPS_HASH(&ips->ips_said);
ipsec_sa_get(ips, IPSEC_REFSAADD);
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
ips->ips_hnext = ipsec_sadb_hash[hashval];
ipsec_sadb_hash[hashval] = ips;
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
return error;
}
@@ -882,7 +881,7 @@ ipsec_sadb_cleanup(__u8 proto)
"cleaning up proto=%d.\n",
proto);
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
for (i = 0; i < SADB_HASHMOD; i++) {
ips = ipsec_sadb_hash[i];
@@ -898,7 +897,7 @@ ipsec_sadb_cleanup(__u8 proto)
/* errlab: */
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
#if IPSEC_SA_REF_CODE
diff --git a/linux/net/ipsec/ipsec_tunnel.c b/linux/net/ipsec/ipsec_tunnel.c
index a375b79..59cd74d 100644
--- a/linux/net/ipsec/ipsec_tunnel.c
+++ b/linux/net/ipsec/ipsec_tunnel.c
@@ -2145,7 +2145,8 @@ ipsec_tunnel_init(struct net_device *dev)
dev->mtu = 0;
dev->addr_len = 0;
dev->type = ARPHRD_VOID; /* ARPHRD_TUNNEL; */ /* ARPHRD_ETHER; */
- dev->tx_queue_len = 10; /* Small queue */
+ dev->tx_queue_len = 0; /* No qdisc */
+ dev->features |= NETIF_F_LLTX; /* No tx lock */
#ifdef IFF_XMIT_DST_RELEASE
dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
#endif
diff --git a/linux/net/ipsec/ipsec_xmit.c b/linux/net/ipsec/ipsec_xmit.c
index b9e6a36..46f5126 100644
--- a/linux/net/ipsec/ipsec_xmit.c
+++ b/linux/net/ipsec/ipsec_xmit.c
@@ -2820,7 +2820,7 @@ ipsec_xsm(struct ipsec_xmit_state *ixs)
/*
* make sure nothing is removed from underneath us
*/
- spin_lock_bh(&tdb_lock);
+ read_lock_bh(&tdb_lock);
/*
* if we have a valid said, then we must check it here to ensure it
@@ -2865,7 +2865,7 @@ ipsec_xsm(struct ipsec_xmit_state *ixs)
* things are on hold until we return here in the next/new state
* we check our SA is valid when we return
*/
- spin_unlock_bh(&tdb_lock);
+ read_unlock_bh(&tdb_lock);
return;
} else {
/* bad result, force state change to done */
@@ -2880,7 +2880,7 @@ ipsec_xsm(struct ipsec_xmit_state *ixs)
/*
* all done with anything needing locks
*/
- spin_unlock_bh(&tdb_lock);
+ read_unlock_bh(&tdb_lock);
/*
* let the caller continue with their processing
diff --git a/linux/net/ipsec/pfkey_v2_parser.c b/linux/net/ipsec/pfkey_v2_parser.c
index 6f787e0..5a4364d 100644
--- a/linux/net/ipsec/pfkey_v2_parser.c
+++ b/linux/net/ipsec/pfkey_v2_parser.c
@@ -249,12 +249,12 @@ pfkey_getspi_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_e
if(maxspi == minspi) {
extr->ips->ips_said.spi = maxspi;
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
ipsq = ipsec_sa_getbyid(&(extr->ips->ips_said), IPSEC_REFSA);
if(ipsq != NULL) {
sa_len = KLIPS_SATOT(debug_pfkey, &extr->ips->ips_said, 0, sa, sizeof(sa));
ipsec_sa_put(ipsq, IPSEC_REFSA);
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_PRINT(debug_pfkey,
"klips_debug:pfkey_getspi_parse: "
"EMT_GETSPI found an old ipsec_sa for SA: %s, delete it first.\n",
@@ -263,7 +263,7 @@ pfkey_getspi_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_e
} else {
found_avail = 1;
}
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
} else {
int i = 0;
__u32 rand_val;
@@ -278,14 +278,14 @@ pfkey_getspi_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_e
(rand_val %
(spi_diff + 1)));
i++;
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
ipsq = ipsec_sa_getbyid(&(extr->ips->ips_said), IPSEC_REFSA);
if(ipsq == NULL) {
found_avail = 1;
} else {
ipsec_sa_put(ipsq, IPSEC_REFSA);
}
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
}
}
@@ -443,11 +443,11 @@ pfkey_update_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_e
sa_len = KLIPS_SATOT(debug_pfkey, &extr->ips->ips_said, 0, sa, sizeof(sa));
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
ipsq = ipsec_sa_getbyid(&(extr->ips->ips_said), IPSEC_REFSA);
if (ipsq == NULL) {
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_PRINT(debug_pfkey,
"klips_debug:pfkey_update_parse: "
"reserved ipsec_sa for SA: %s not found. Call K_SADB_GETSPI first or call K_SADB_ADD instead.\n",
@@ -503,7 +503,7 @@ pfkey_update_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_e
extr->ips->ips_rcvif = NULL;
if ((error = pfkey_ipsec_sa_init(extr->ips))) {
ipsec_sa_put(ipsq, IPSEC_REFSA);
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_PRINT(debug_pfkey,
"klips_debug:pfkey_update_parse: "
"not successful for SA: %s, deleting.\n",
@@ -517,7 +517,7 @@ pfkey_update_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_e
ipsec_sa_put(ipsq, IPSEC_REFSA);
}
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
if(!(pfkey_safe_build(error = pfkey_msg_hdr_build(&extensions_reply[0],
K_SADB_UPDATE,
@@ -719,18 +719,18 @@ pfkey_add_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_extr
sa_len = KLIPS_SATOT(debug_pfkey, &extr->ips->ips_said, 0, sa, sizeof(sa));
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
ipsq = ipsec_sa_getbyid(&(extr->ips->ips_said), IPSEC_REFSA);
if(ipsq != NULL) {
ipsec_sa_put(ipsq, IPSEC_REFSA);
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_PRINT(debug_pfkey,
"klips_debug:pfkey_add_parse: "
"found an old ipsec_sa for SA%s, delete it first.\n",
sa_len ? sa : " (error)");
SENDERR(EEXIST);
}
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
#ifdef CONFIG_KLIPS_IPV6
if (ip_address_family(&extr->ips->ips_said.dst) == AF_INET6 &&
@@ -770,16 +770,16 @@ pfkey_add_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_extr
}
/* attach it to the SAref table */
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
if((error = ipsec_sa_intern(extr->ips)) != 0) {
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_ERROR(debug_pfkey,
"pfkey_add_parse: "
"failed to intern SA as SAref#%lu\n"
, (unsigned long)extr->ips->ips_ref);
SENDERR(-error);
}
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
extr->ips->ips_life.ipl_addtime.ipl_count = jiffies / HZ;
if(!extr->ips->ips_life.ipl_allocations.ipl_count) {
@@ -953,11 +953,11 @@ pfkey_delete_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_e
sa_len = KLIPS_SATOT(debug_pfkey, &extr->ips->ips_said, 0, sa, sizeof(sa));
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
ipsp = ipsec_sa_getbyid(&(extr->ips->ips_said), IPSEC_REFSA);
if (ipsp == NULL) {
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_PRINT(debug_pfkey,
"klips_debug:pfkey_delete_parse: "
"ipsec_sa not found for SA:%s, could not delete.\n",
@@ -997,7 +997,7 @@ pfkey_delete_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_e
/* this should cause ipsec_sa_wipe to get called on the SA/group */
ipsec_sa_put(ipsq, IPSEC_REFALLOC);
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
memset(&sab, 0, sizeof(sab));
sab.sa_base.sadb_sa_exttype = K_SADB_EXT_SA;
@@ -1092,11 +1092,11 @@ pfkey_get_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_extr
sa_len = KLIPS_SATOT(debug_pfkey, &extr->ips->ips_said, 0, sa, sizeof(sa));
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
ipsp = ipsec_sa_getbyid(&(extr->ips->ips_said), IPSEC_REFSA);
if (ipsp == NULL) {
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_PRINT(debug_pfkey, "klips_debug:pfkey_get_parse: "
"ipsec_sa not found for SA=%s, could not get.\n",
sa_len ? sa : " (error)");
@@ -1230,14 +1230,14 @@ pfkey_get_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_extr
#endif
)) {
ipsec_sa_put(ipsp, IPSEC_REFSA);
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_PRINT(debug_pfkey, "klips_debug:pfkey_get_parse: "
"failed to build the get reply message extensions\n");
SENDERR(-error);
}
ipsec_sa_put(ipsp, IPSEC_REFSA);
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
if((error = pfkey_msg_build(&pfkey_reply, extensions_reply, EXT_BITS_OUT))) {
KLIPS_PRINT(debug_pfkey, "klips_debug:pfkey_get_parse: "
@@ -1697,11 +1697,11 @@ pfkey_x_grpsa_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_
sa_len2 = KLIPS_SATOT(debug_pfkey, &extr->ips2->ips_said, 0, sa2, sizeof(sa2));
}
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
ips1p = ipsec_sa_getbyid(&(extr->ips->ips_said), IPSEC_REFSA);
if(ips1p == NULL) {
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_ERROR(debug_pfkey,
"klips_debug:pfkey_x_grpsa_parse: "
"reserved ipsec_sa for SA1: %s not found. Call K_SADB_ADD/UPDATE first.\n",
@@ -1716,7 +1716,7 @@ pfkey_x_grpsa_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_
ips2p = ipsec_sa_getbyid(&(extr->ips2->ips_said), IPSEC_REFSA);
if(ips2p == NULL) {
ipsec_sa_put(ips1p, IPSEC_REFSA);
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_PRINT(debug_pfkey,
"klips_debug:pfkey_x_grpsa_parse: "
"reserved ipsec_sa for SA2: %s not found. Call K_SADB_ADD/UPDATE first.\n",
@@ -1738,7 +1738,7 @@ pfkey_x_grpsa_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_
if(ips1p->ips_next) {
ipsec_sa_put(ips1p, IPSEC_REFSA);
ipsec_sa_put(ips2p, IPSEC_REFSA);
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_ERROR(debug_pfkey,
"klips_debug:pfkey_x_grpsa_parse: "
"ipsec_sa for SA: %s is already linked.\n",
@@ -1752,7 +1752,7 @@ pfkey_x_grpsa_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_
if(ipsp == ips1p) {
ipsec_sa_put(ips1p, IPSEC_REFSA);
ipsec_sa_put(ips2p, IPSEC_REFSA);
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_ERROR(debug_pfkey,
"klips_debug:pfkey_x_grpsa_parse: "
"ipsec_sa for SA: %s is already linked to %s.\n",
@@ -1799,7 +1799,7 @@ pfkey_x_grpsa_parse(struct sock *sk, struct sadb_ext **extensions, struct pfkey_
sa_len1 ? sa1 : " (error)");
}
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
/* MCR: not only is this ugly to read, and impossible
* to debug through, but it's also really inefficient.
@@ -1964,11 +1964,11 @@ pfkey_x_addflow_parse(struct sock *sk, struct sadb_ext **extensions, struct pfke
char sa[SATOT_BUF];
size_t sa_len;
- spin_lock_bh(&tdb_lock);
+ write_lock_bh(&tdb_lock);
ipsq = ipsec_sa_getbyid(&(extr->ips->ips_said), IPSEC_REFSA);
if(ipsq == NULL) {
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_PRINT(debug_pfkey,
"klips_debug:pfkey_x_addflow_parse: "
"ipsec_sa not found, cannot set incoming policy.\n");
@@ -1982,7 +1982,7 @@ pfkey_x_addflow_parse(struct sock *sk, struct sadb_ext **extensions, struct pfke
if(ipsp == NULL) {
ipsec_sa_put(ipsq, IPSEC_REFSA);
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_PRINT(debug_pfkey,
"klips_debug:pfkey_x_addflow_parse: "
"SA chain does not have an IPIP SA, cannot set incoming policy.\n");
@@ -2000,7 +2000,7 @@ pfkey_x_addflow_parse(struct sock *sk, struct sadb_ext **extensions, struct pfke
ipsec_sa_put(ipsq, IPSEC_REFSA);
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
KLIPS_PRINT(debug_pfkey,
"klips_debug:pfkey_x_addflow_parse: "
@@ -2470,7 +2470,7 @@ pfkey_expire(struct ipsec_sa *ipsp, int hard)
extensions))) {
KLIPS_PRINT(debug_pfkey, "klips_debug:pfkey_expire: "
"failed to build the expire message extensions\n");
- spin_unlock_bh(&tdb_lock);
+ write_unlock_bh(&tdb_lock);
goto errlab;
}
diff --git a/linux/net/ipsec/prng.c b/linux/net/ipsec/prng.c
index 05a16d0..8c8d7e6 100644
--- a/linux/net/ipsec/prng.c
+++ b/linux/net/ipsec/prng.c
@@ -22,23 +22,21 @@
#include <linux/interrupt.h>
/*
- * A horrible locking hack, we ride on tdb_lock for now since it
- * is basically what we want. Since all calls into prng_bytes pass in
- * a pointer to ipsec_prng, there is contention on the data in ipsec_prng
- * as it is not always locked. TO make sure we never messup the PRNG, just
- * locked it if we don't already have the tdb_lock
+ * TO make sure we never messup the PRNG, just
+ * locked with a new lock prng_lock -- please verify this
*/
+DEFINE_SPINLOCK(prng_lock);
#define LOCK_PRNG() \
int ul = 0; \
- if (spin_trylock_bh(&tdb_lock)) { \
+ if (spin_trylock_bh(&prng_lock)) { \
ul = 1; \
} else
#define UNLOCK_PRNG() \
if (ul) { \
ul = 0; \
- spin_unlock_bh(&tdb_lock); \
+ spin_unlock_bh(&prng_lock); \
} else
#else
More information about the Dev
mailing list