--- linux-2.6.11/include/linux/if.h 2005-03-01 23:38:08.000000000 -0800 +++ linux-2.6.11.p4s/include/linux/if.h 2005-07-11 16:06:34.000000000 -0700 @@ -52,6 +52,20 @@ /* Private (from user) interface flags (netdevice->priv_flags). */ #define IFF_802_1Q_VLAN 0x1 /* 802.1Q VLAN device. */ #define IFF_EBRIDGE 0x2 /* Ethernet bridging device. */ +#define IFF_PKTGEN_RCV 0x4 /* Registered to receive & consume Pktgen skbs */ +#define IFF_ACCEPT_LOCAL_ADDRS 0x8 /** Accept pkts even if they come from a local + * address. This lets use send pkts to ourselves + * over external interfaces (when used in conjunction + * with SO_BINDTODEVICE + */ +#define IFF_ACCEPT_ALL_FRAMES 0x10 /** Accept all frames, even ones with bad CRCs. + * Should only be used in debugging/testing situations + * Do NOT enable this unless you understand the + * consequences! */ +#define IFF_SAVE_FCS 0x20 /** Save the Frame Check Sum (FCS) on receive, if + * possible. */ +#define IFF_MAC_VLAN 0x40 /* MAC VLAN device. */ + #define IF_GET_IFACE 0x0001 /* for querying only */ #define IF_GET_PROTO 0x0002 --- linux-2.6.11/include/linux/netdevice.h 2005-03-01 23:38:26.000000000 -0800 +++ linux-2.6.11.p4s/include/linux/netdevice.h 2005-07-11 16:06:34.000000000 -0700 @@ -327,7 +327,9 @@ unsigned short flags; /* interface flags (a la BSD) */ unsigned short gflags; - unsigned short priv_flags; /* Like 'flags' but invisible to userspace. */ + unsigned short priv_flags; /* Like 'flags' but invisible to userspace, + * see: if.h for flag definitions. + */ unsigned short unused_alignment_fixer; /* Because we need priv_flags, * and we want to be 32-bit aligned. */ @@ -416,6 +418,12 @@ #define NETIF_F_VLAN_CHALLENGED 1024 /* Device cannot handle VLAN packets */ #define NETIF_F_TSO 2048 /* Can offload TCP/IP segmentation */ #define NETIF_F_LLTX 4096 /* LockLess TX */ +#define NETIF_F_RX_ALL (1<<12) /* Can be configured to receive all packets, even + * ones with busted CRC. May disable VLAN filtering + * in the NIC, users should NOT enable this feature + * unless they understand the consequences. */ +#define NETIF_F_SAVE_CRC (1<<13) /* Can save FCS in skb, last 4 bytes for ethernet */ + /* Called after device is detached from network. */ void (*uninit)(struct net_device *dev); @@ -477,9 +485,18 @@ void (*poll_controller)(struct net_device *dev); #endif + /* Callback for when the queue is woken, used by pktgen currently */ + int (*notify_queue_woken)(struct net_device *dev); + void* nqw_data; /* To be used by the method above as needed */ + /* bridge stuff */ struct net_bridge_port *br_port; +#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE) + struct macvlan_port *macvlan_priv; +#endif + + #ifdef CONFIG_NET_DIVERT /* this will get initialized at each interface type init routine */ struct divert_blk *divert; @@ -614,8 +631,13 @@ if (netpoll_trap()) return; #endif - if (test_and_clear_bit(__LINK_STATE_XOFF, &dev->state)) + if (test_and_clear_bit(__LINK_STATE_XOFF, &dev->state)) { __netif_schedule(dev); + + if (dev->notify_queue_woken) { + dev->notify_queue_woken(dev); + } + } } static inline void netif_stop_queue(struct net_device *dev) --- linux-2.6.11/net/core/dev.c 2005-03-01 23:38:09.000000000 -0800 +++ linux-2.6.11.p4s/net/core/dev.c 2005-07-11 16:06:34.000000000 -0700 @@ -1,4 +1,4 @@ -/* +/* -*-linux-c-*- * NET3 Protocol independent device support routines. * * This program is free software; you can redistribute it and/or @@ -88,6 +88,7 @@ #include #include #include +#include #include #include #include @@ -115,6 +116,24 @@ #endif /* CONFIG_NET_RADIO */ #include +#if defined(CONFIG_NET_PKTGEN) || defined(CONFIG_NET_PKTGEN_MODULE) +#include "pktgen.h" + +#warning "Compiling dev.c for pktgen."; + +int (*handle_pktgen_hook)(struct sk_buff *skb) = NULL; +EXPORT_SYMBOL(handle_pktgen_hook); + +static __inline__ int handle_pktgen_rcv(struct sk_buff* skb) { + if (handle_pktgen_hook) { + return handle_pktgen_hook(skb); + } + return -1; +} + +#endif + + /* This define, if set, will randomly drop a packet when congestion * is more than moderate. It helps fairness in the multi-interface * case when one of them is a hog, but it kills performance for the @@ -1214,6 +1233,19 @@ * A negative errno code is returned on a failure. A success does not * guarantee the frame will be transmitted as it may be dropped due * to congestion or traffic shaping. + * + * ----------------------------------------------------------------------------------- + * I notice this method can also return errors from the queue disciplines, + * including NET_XMIT_DROP, which is a positive value. So, errors can also + * be positive. + * + * Regardless of the return value, the skb is consumed, so it is currently + * difficult to retry a send to this method. (You can bump the ref count + * before sending to hold a reference for retry if you are careful.) + * + * When calling this method, interrupts MUST be enabled. This is because + * the BH enable code must have IRQs enabled so that it will not deadlock. + * --BLG */ int dev_queue_xmit(struct sk_buff *skb) @@ -1339,7 +1371,7 @@ =======================================================================*/ int netdev_max_backlog = 300; -int weight_p = 64; /* old backlog weight */ +int weight_p = 64; /* old backlog weight */ /* These numbers are selected based on intuition and some * experimentatiom, if you have more scientific way of doing this * please go ahead and fix things. @@ -1623,6 +1655,24 @@ } #endif + +#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE) +/* Returns >= 0 if we consume the packet. Otherwise, let + * it fall through the rest of the packet processing. + */ +int (*macvlan_handle_frame_hook)(struct sk_buff *skb) = NULL; +EXPORT_SYMBOL(macvlan_handle_frame_hook); +#endif + +/* Returns >= 0 if we consume the packet. Otherwise, let + * it fall through the rest of the packet processing. + */ +static __inline__ int handle_macvlan(struct sk_buff *skb) +{ + return macvlan_handle_frame_hook(skb); +} + + int netif_receive_skb(struct sk_buff *skb) { struct packet_type *ptype, *pt_prev; @@ -1689,6 +1739,32 @@ if (handle_bridge(&skb, &pt_prev, &ret)) goto out; +#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE) + if (skb->dev->macvlan_priv != NULL && + macvlan_handle_frame_hook != NULL) { + if (handle_macvlan(skb) >= 0) { + /* consumed by mac-vlan...it would have been + * re-sent to this method with a different + * device... + */ + goto out; + } + else { + /* Let it fall through and be processed normally */ + } + } +#endif + +#if defined(CONFIG_NET_PKTGEN) || defined(CONFIG_NET_PKTGEN_MODULE) + if ((skb->dev->priv_flags & IFF_PKTGEN_RCV) && + (handle_pktgen_rcv(skb) >= 0)) { + /* Pktgen may consume the packet, no need to send + * to further protocols. + */ + goto out; + } +#endif + type = skb->protocol; list_for_each_entry_rcu(ptype, &ptype_base[ntohs(type)&15], list) { if (ptype->type == type && @@ -2420,6 +2496,24 @@ ifr->ifr_newname[IFNAMSIZ-1] = '\0'; return dev_change_name(dev, ifr->ifr_newname); + case SIOCSACCEPTLOCALADDRS: + if (ifr->ifr_flags) { + dev->priv_flags |= IFF_ACCEPT_LOCAL_ADDRS; + } + else { + dev->priv_flags &= ~IFF_ACCEPT_LOCAL_ADDRS; + } + return 0; + + case SIOCGACCEPTLOCALADDRS: + if (dev->priv_flags & IFF_ACCEPT_LOCAL_ADDRS) { + ifr->ifr_flags = 1; + } + else { + ifr->ifr_flags = 0; + } + return 0; + /* * Unknown or private ioctl */ @@ -2518,6 +2612,7 @@ case SIOCGIFMAP: case SIOCGIFINDEX: case SIOCGIFTXQLEN: + case SIOCGACCEPTLOCALADDRS: dev_load(ifr.ifr_name); read_lock(&dev_base_lock); ret = dev_ifsioc(&ifr, cmd); @@ -2594,6 +2689,7 @@ case SIOCBONDCHANGEACTIVE: case SIOCBRADDIF: case SIOCBRDELIF: + case SIOCSACCEPTLOCALADDRS: if (!capable(CAP_NET_ADMIN)) return -EPERM; dev_load(ifr.ifr_name); @@ -3338,6 +3434,10 @@ EXPORT_SYMBOL(net_enable_timestamp); EXPORT_SYMBOL(net_disable_timestamp); +#if defined(CONFIG_NET_PKTGEN) || defined(CONFIG_NET_PKTGEN_MODULE) +EXPORT_SYMBOL(handle_pktgen_rcv); +#endif + #if defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE) EXPORT_SYMBOL(br_handle_frame_hook); #endif --- linux-2.6.11/net/core/pktgen.c 2005-03-01 23:38:38.000000000 -0800 +++ linux-2.6.11.p4s/net/core/pktgen.c 2005-07-11 16:06:34.000000000 -0700 @@ -1,22 +1,12 @@ -/* - * Authors: - * Copyright 2001, 2002 by Robert Olsson - * Uppsala University and - * Swedish University of Agricultural Sciences - * - * Alexey Kuznetsov - * Ben Greear - * Jens Låås - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. +/* -*-linux-c-*- * + * Copyright 2001, 2002 by Robert Olsson + * Uppsala University, Sweden + * 2002 Ben Greear * * A tool for loading the network with preconfigurated packets. * The tool is implemented as a linux module. Parameters are output - * device, delay (to hard_xmit), number of packets, and whether + * device, IPG (interpacket gap), number of packets, and whether * to use multiple SKBs or just the same one. * pktgen uses the installed interface's output routine. * @@ -43,6 +33,16 @@ * latencies (with micro-second) precision. * * Add IOCTL interface to easily get counters & configuration. * --Ben Greear + + * Fix refcount off by one if first packet fails, potential null deref, + * memleak 030710- KJP + * + * * Added the IPMAC option to allow the MAC addresses to mirror IP addresses. + * -- (dhetheri) Dave Hetherington 03/09/29 + * * Allow the user to change the protocol field via 'pgset "prot 0"' command + * -- (dhetheri) Dave Hetherington 03/10/7 + * Integrated to 2.5.x 021029 --Lucio Maciel (luciomaciel@zipmail.com.br) + * * * Renamed multiskb to clone_skb and cleaned up sending core for two distinct * skb modes. A clone_skb=0 mode for Ben "ranges" work and a clone_skb != 0 @@ -54,305 +54,126 @@ * Also moved to /proc/net/pktgen/ * --ro * - * Sept 10: Fixed threading/locking. Lots of bone-headed and more clever - * mistakes. Also merged in DaveM's patch in the -pre6 patch. - * --Ben Greear - * - * Integrated to 2.5.x 021029 --Lucio Maciel (luciomaciel@zipmail.com.br) - * - * - * 021124 Finished major redesign and rewrite for new functionality. - * See Documentation/networking/pktgen.txt for how to use this. - * - * The new operation: - * For each CPU one thread/process is created at start. This process checks - * for running devices in the if_list and sends packets until count is 0 it - * also the thread checks the thread->control which is used for inter-process - * communication. controlling process "posts" operations to the threads this - * way. The if_lock should be possible to remove when add/rem_device is merged - * into this too. - * - * By design there should only be *one* "controlling" process. In practice - * multiple write accesses gives unpredictable result. Understood by "write" - * to /proc gives result code thats should be read be the "writer". - * For pratical use this should be no problem. - * - * Note when adding devices to a specific CPU there good idea to also assign - * /proc/irq/XX/smp_affinity so TX-interrupts gets bound to the same CPU. - * --ro - * - * Fix refcount off by one if first packet fails, potential null deref, + * Fix refcount off by one if first packet fails, potential null deref, * memleak 030710- KJP * - * First "ranges" functionality for ipv6 030726 --ro - * - * Included flow support. 030802 ANK. - * - * Fixed unaligned access on IA-64 Grant Grundler - * - * Remove if fix from added Harald Welte 040419 - * ia64 compilation fix from Aron Griffis 040604 - * - * New xmit() return, do_div and misc clean up by Stephen Hemminger - * 040923 - * - * Rany Dunlap fixed u64 printk compiler waring * - * Remove FCS from BW calculation. Lennert Buytenhek - * New time handling. Lennert Buytenhek 041213 - * - * Corrections from Nikolai Malykh (nmalykh@bilim.com) - * Removed unused flags F_SET_SRCMAC & F_SET_SRCIP 041230 + * Sept 10: Fixed threading/locking. Lots of bone-headed and more clever + * mistakes. Also merged in DaveM's patch in the -pre6 patch. * - * interruptible_sleep_on_timeout() replaced Nishanth Aravamudan - * 050103 + * See Documentation/networking/pktgen.txt for how to use this. */ -#include -#include + #include -#include #include -#include -#include -#include -#include #include -#include +#include #include #include #include #include +#include #include +#include #include -#include #include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include #include #include #include #include -#include /* do_div */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include /* for lock kernel */ +#include /* do_div */ -#define VERSION "pktgen v2.58: Packet Generator for packet performance testing.\n" -/* #define PG_DEBUG(a) a */ -#define PG_DEBUG(a) +#include +#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE) +#include "../macvlan/macvlan.h" +#endif + +#include "pktgen.h" -/* The buckets are exponential in 'width' */ -#define LAT_BUCKETS_MAX 32 -#define IP_NAME_SZ 32 - -/* Device flag bits */ -#define F_IPSRC_RND (1<<0) /* IP-Src Random */ -#define F_IPDST_RND (1<<1) /* IP-Dst Random */ -#define F_UDPSRC_RND (1<<2) /* UDP-Src Random */ -#define F_UDPDST_RND (1<<3) /* UDP-Dst Random */ -#define F_MACSRC_RND (1<<4) /* MAC-Src Random */ -#define F_MACDST_RND (1<<5) /* MAC-Dst Random */ -#define F_TXSIZE_RND (1<<6) /* Transmit size is random */ -#define F_IPV6 (1<<7) /* Interface in IPV6 Mode */ - -/* Thread control flag bits */ -#define T_TERMINATE (1<<0) -#define T_STOP (1<<1) /* Stop run */ -#define T_RUN (1<<2) /* Start run */ -#define T_REMDEV (1<<3) /* Remove all devs */ - -/* Locks */ -#define thread_lock() spin_lock(&_thread_lock) -#define thread_unlock() spin_unlock(&_thread_lock) - -/* If lock -- can be removed after some work */ -#define if_lock(t) spin_lock(&(t->if_lock)); -#define if_unlock(t) spin_unlock(&(t->if_lock)); + +#define VERSION "pktgen version 1.9.2 (nospin)" +static char version[] __initdata = + "pktgen.c: v1.9.2 (nospin): Packet Generator for packet performance testing.\n"; /* Used to help with determining the pkts on receive */ -#define PKTGEN_MAGIC 0xbe9be955 -#define PG_PROC_DIR "pktgen" -#define MAX_CFLOWS 65536 +#define PKTGEN_MAGIC 0xbe9be955 -struct flow_state -{ - __u32 cur_daddr; - int count; -}; +/* #define PG_DEBUG(a) a */ +#define PG_DEBUG(a) /* a */ -struct pktgen_dev { +/* cycles per micro-second */ +static u32 pg_cycles_per_ns; +static u32 pg_cycles_per_us; +static u32 pg_cycles_per_ms; - /* - * Try to keep frequent/infrequent used vars. separated. - */ +/* Module parameters, defaults. */ +static int pg_count_d = 0; /* run forever by default */ +static int pg_ipg_d = 0; +static int pg_multiskb_d = 0; +static int pg_thread_count = 1; /* Initial threads to create */ +static int debug = 0; - char ifname[32]; - struct proc_dir_entry *proc_ent; - char result[512]; - /* proc file names */ - char fname[80]; - struct pktgen_thread* pg_thread; /* the owner */ - struct pktgen_dev *next; /* Used for chaining in the thread's run-queue */ - int running; /* if this changes to false, the test will stop */ - - /* If min != max, then we will either do a linear iteration, or - * we will do a random selection from within the range. - */ - __u32 flags; +/* List of all running threads */ +static struct pktgen_thread_info* pktgen_threads = NULL; +spinlock_t _pg_threadlist_lock = SPIN_LOCK_UNLOCKED; + +/* Holds interfaces for all threads */ +#define PG_INFO_HASH_MAX 32 +static struct pktgen_interface_info* pg_info_hash[PG_INFO_HASH_MAX]; +spinlock_t _pg_hash_lock = SPIN_LOCK_UNLOCKED; - int min_pkt_size; /* = ETH_ZLEN; */ - int max_pkt_size; /* = ETH_ZLEN; */ - int nfrags; - __u32 delay_us; /* Default delay */ - __u32 delay_ns; - __u64 count; /* Default No packets to send */ - __u64 sofar; /* How many pkts we've sent so far */ - __u64 tx_bytes; /* How many bytes we've transmitted */ - __u64 errors; /* Errors when trying to transmit, pkts will be re-sent */ - - /* runtime counters relating to clone_skb */ - __u64 next_tx_us; /* timestamp of when to tx next */ - __u32 next_tx_ns; - - __u64 allocated_skbs; - __u32 clone_count; - int last_ok; /* Was last skb sent? - * Or a failed transmit of some sort? This will keep - * sequence numbers in order, for example. - */ - __u64 started_at; /* micro-seconds */ - __u64 stopped_at; /* micro-seconds */ - __u64 idle_acc; /* micro-seconds */ - __u32 seq_num; - - int clone_skb; /* Use multiple SKBs during packet gen. If this number - * is greater than 1, then that many coppies of the same - * packet will be sent before a new packet is allocated. - * For instance, if you want to send 1024 identical packets - * before creating a new packet, set clone_skb to 1024. - */ - - char dst_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ - char dst_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ - char src_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ - char src_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ - - struct in6_addr in6_saddr; - struct in6_addr in6_daddr; - struct in6_addr cur_in6_daddr; - struct in6_addr cur_in6_saddr; - /* For ranges */ - struct in6_addr min_in6_daddr; - struct in6_addr max_in6_daddr; - struct in6_addr min_in6_saddr; - struct in6_addr max_in6_saddr; +#define PG_PROC_DIR "pktgen" +static struct proc_dir_entry *pg_proc_dir = NULL; - /* If we're doing ranges, random or incremental, then this - * defines the min/max for those ranges. - */ - __u32 saddr_min; /* inclusive, source IP address */ - __u32 saddr_max; /* exclusive, source IP address */ - __u32 daddr_min; /* inclusive, dest IP address */ - __u32 daddr_max; /* exclusive, dest IP address */ - - __u16 udp_src_min; /* inclusive, source UDP port */ - __u16 udp_src_max; /* exclusive, source UDP port */ - __u16 udp_dst_min; /* inclusive, dest UDP port */ - __u16 udp_dst_max; /* exclusive, dest UDP port */ - - __u32 src_mac_count; /* How many MACs to iterate through */ - __u32 dst_mac_count; /* How many MACs to iterate through */ - - unsigned char dst_mac[6]; - unsigned char src_mac[6]; - - __u32 cur_dst_mac_offset; - __u32 cur_src_mac_offset; - __u32 cur_saddr; - __u32 cur_daddr; - __u16 cur_udp_dst; - __u16 cur_udp_src; - __u32 cur_pkt_size; - - __u8 hh[14]; - /* = { - 0x00, 0x80, 0xC8, 0x79, 0xB3, 0xCB, - - We fill in SRC address later - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x00 - }; - */ - __u16 pad; /* pad out the hh struct to an even 16 bytes */ +char module_fname[128]; +struct proc_dir_entry *module_proc_ent = NULL; - struct sk_buff* skb; /* skb we are to transmit next, mainly used for when we - * are transmitting the same one multiple times - */ - struct net_device* odev; /* The out-going device. Note that the device should - * have it's pg_info pointer pointing back to this - * device. This will be set when the user specifies - * the out-going device name (not when the inject is - * started as it used to do.) - */ - struct flow_state *flows; - unsigned cflows; /* Concurrent flows (config) */ - unsigned lflow; /* Flow length (config) */ - unsigned nflows; /* accumulated flows (stats) */ -}; -struct pktgen_hdr { - __u32 pgh_magic; - __u32 seq_num; - __u32 tv_sec; - __u32 tv_usec; -}; +static void init_pktgen_kthread(struct pktgen_thread_info *kthread, char *name); +static int pg_rem_interface_info(struct pktgen_thread_info* pg_thread, + struct pktgen_interface_info* i); +static int pg_add_interface_info(struct pktgen_thread_info* pg_thread, + const char* ifname); +static void exit_pktgen_kthread(struct pktgen_thread_info *kthread); +static void stop_pktgen_kthread(struct pktgen_thread_info *kthread); +static struct pktgen_thread_info* pg_find_thread(const char* name); +static int pg_add_thread_info(const char* name); +static struct pktgen_interface_info* pg_find_interface(struct pktgen_thread_info* pg_thread, + const char* ifname); +static int pktgen_device_event(struct notifier_block *, unsigned long, void *); -struct pktgen_thread { - spinlock_t if_lock; - struct pktgen_dev *if_list; /* All device here */ - struct pktgen_thread* next; - char name[32]; - char fname[128]; /* name of proc file */ - struct proc_dir_entry *proc_ent; - char result[512]; - u32 max_before_softirq; /* We'll call do_softirq to prevent starvation. */ - - /* Field for thread to receive "posted" events terminate, stop ifs etc.*/ - - u32 control; - int pid; - int cpu; - wait_queue_head_t queue; +struct notifier_block pktgen_notifier_block = { + notifier_call: pktgen_device_event, }; -#define REMOVE 1 -#define FIND 0 - /* This code works around the fact that do_div cannot handle two 64-bit numbers, and regular 64-bit division doesn't work on x86 kernels. --Ben */ #define PG_DIV 0 +#define PG_REM 1 /* This was emailed to LMKL by: Chris Caputo * Function copied/adapted/optimized from: @@ -362,9 +183,10 @@ * Copyright 1994, University of Cambridge Computer Laboratory * All Rights Reserved. * + * TODO: When running on a 64-bit CPU platform, this should no longer be + * TODO: necessary. */ -inline static s64 divremdi3(s64 x, s64 y, int type) -{ +inline static s64 divremdi3(s64 x, s64 y, int type) { u64 a = (x < 0) ? -x : x; u64 b = (y < 0) ? -y : y; u64 res = 0, d = 1; @@ -392,13 +214,76 @@ else { return ((x & (1ll<<63)) == 0) ? a : -(s64)a; } -} +}/* divremdi3 */ /* End of hacks to deal with 64-bit math on x86 */ + + +inline static void pg_lock_thread_list(const char* msg) { + if (debug > 1) { + printk("before pg_lock_thread_list, msg: %s\n", msg); + } + spin_lock(&_pg_threadlist_lock); + if (debug > 1) { + printk("after pg_lock_thread_list, msg: %s\n", msg); + } +} + +inline static void pg_unlock_thread_list(const char* msg) { + if (debug > 1) { + printk("before pg_unlock_thread_list, msg: %s\n", msg); + } + spin_unlock(&_pg_threadlist_lock); + if (debug > 1) { + printk("after pg_unlock_thread_list, msg: %s\n", msg); + } +} + +inline static void pg_lock_hash(const char* msg) { + if (debug > 1) { + printk("before pg_lock_hash, msg: %s\n", msg); + } + spin_lock(&_pg_hash_lock); + if (debug > 1) { + printk("before pg_lock_hash, msg: %s\n", msg); + } +} + +inline static void pg_unlock_hash(const char* msg) { + if (debug > 1) { + printk("before pg_unlock_hash, msg: %s\n", msg); + } + spin_unlock(&_pg_hash_lock); + if (debug > 1) { + printk("after pg_unlock_hash, msg: %s\n", msg); + } +} + +inline static void pg_lock(struct pktgen_thread_info* pg_thread, const char* msg) { + if (debug > 1) { + printk("before pg_lock thread, msg: %s\n", msg); + } + spin_lock(&(pg_thread->pg_threadlock)); + if (debug > 1) { + printk("after pg_lock thread, msg: %s\n", msg); + } +} + +inline static void pg_unlock(struct pktgen_thread_info* pg_thread, const char* msg) { + if (debug > 1) { + printk("before pg_unlock thread, thread: %p msg: %s\n", + pg_thread, msg); + } + spin_unlock(&(pg_thread->pg_threadlock)); + if (debug > 1) { + printk("after pg_unlock thread, thread: %p msg: %s\n", + pg_thread, msg); + } +} + /** Convert to miliseconds */ -static inline __u64 tv_to_ms(const struct timeval* tv) -{ +static inline __u64 tv_to_ms(const struct timeval* tv) { __u64 ms = tv->tv_usec / 1000; ms += (__u64)tv->tv_sec * (__u64)1000; return ms; @@ -406,1138 +291,430 @@ /** Convert to micro-seconds */ -static inline __u64 tv_to_us(const struct timeval* tv) -{ +static inline __u64 tv_to_us(const struct timeval* tv) { __u64 us = tv->tv_usec; us += (__u64)tv->tv_sec * (__u64)1000000; return us; } + static inline __u64 pg_div(__u64 n, __u32 base) { __u64 tmp = n; do_div(tmp, base); - /* printk("pktgen: pg_div, n: %llu base: %d rv: %llu\n", + /* printk("pg_div, n: %llu base: %d rv: %llu\n", n, base, tmp); */ return tmp; } -static inline __u64 pg_div64(__u64 n, __u64 base) -{ - __u64 tmp = n; -/* - * How do we know if the architectrure we are running on - * supports division with 64 bit base? - * - */ -#if defined(__sparc_v9__) || defined(__powerpc64__) || defined(__alpha__) || defined(__x86_64__) || defined(__ia64__) - - do_div(tmp, base); -#else - tmp = divremdi3(n, base, PG_DIV); -#endif - return tmp; -} - -static inline u32 pktgen_random(void) -{ -#if 0 - __u32 n; - get_random_bytes(&n, 4); - return n; -#else - return net_random(); -#endif +/* Fast, not horribly accurate, since the machine started. */ +static inline __u64 getRelativeCurMs(void) { + return pg_div(get_cycles(), pg_cycles_per_ms); } -static inline __u64 getCurMs(void) -{ +/* Since the epoc. More precise over long periods of time than + * getRelativeCurMs + */ +static inline __u64 getCurMs(void) { struct timeval tv; do_gettimeofday(&tv); return tv_to_ms(&tv); } -static inline __u64 getCurUs(void) -{ +/* Since the epoc. More precise over long periods of time than + * getRelativeCurMs + */ +static inline __u64 getCurUs(void) { struct timeval tv; do_gettimeofday(&tv); return tv_to_us(&tv); } -static inline __u64 tv_diff(const struct timeval* a, const struct timeval* b) -{ - return tv_to_us(a) - tv_to_us(b); +/* Since the machine booted. */ +static inline __u64 getRelativeCurUs(void) { + return pg_div(get_cycles(), pg_cycles_per_us); } +/* Since the machine booted. */ +static inline __u64 getRelativeCurNs(void) { + return pg_div(get_cycles(), pg_cycles_per_ns); +} -/* old include end */ +static inline __u64 tv_diff(const struct timeval* a, const struct timeval* b) { + return tv_to_us(a) - tv_to_us(b); +} -static char version[] __initdata = VERSION; -static ssize_t proc_pgctrl_read(struct file* file, char __user * buf, size_t count, loff_t *ppos); -static ssize_t proc_pgctrl_write(struct file* file, const char __user * buf, size_t count, loff_t *ppos); -static int proc_if_read(char *buf , char **start, off_t offset, int len, int *eof, void *data); - -static int proc_thread_read(char *buf , char **start, off_t offset, int len, int *eof, void *data); -static int proc_if_write(struct file *file, const char __user *user_buffer, unsigned long count, void *data); -static int proc_thread_write(struct file *file, const char __user *user_buffer, unsigned long count, void *data); -static int create_proc_dir(void); -static int remove_proc_dir(void); - -static int pktgen_remove_device(struct pktgen_thread* t, struct pktgen_dev *i); -static int pktgen_add_device(struct pktgen_thread* t, const char* ifname); -static struct pktgen_thread* pktgen_find_thread(const char* name); -static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread* t, const char* ifname); -static int pktgen_device_event(struct notifier_block *, unsigned long, void *); -static void pktgen_run_all_threads(void); -static void pktgen_stop_all_threads_ifs(void); -static int pktgen_stop_device(struct pktgen_dev *pkt_dev); -static void pktgen_stop(struct pktgen_thread* t); -static void pktgen_clear_counters(struct pktgen_dev *pkt_dev); -static struct pktgen_dev *pktgen_NN_threads(const char* dev_name, int remove); -static unsigned int scan_ip6(const char *s,char ip[16]); -static unsigned int fmt_ip6(char *s,const char ip[16]); -/* Module parameters, defaults. */ -static int pg_count_d = 1000; /* 1000 pkts by default */ -static int pg_delay_d = 0; -static int pg_clone_skb_d = 0; -static int debug = 0; +int pktgen_proc_ioctl(struct inode* inode, struct file* file, unsigned int cmd, + unsigned long arg) { + int err = 0; + struct pktgen_ioctl_info args; + struct pktgen_thread_info* targ = NULL; -static spinlock_t _thread_lock = SPIN_LOCK_UNLOCKED; -static struct pktgen_thread *pktgen_threads = NULL; + /* + if (!capable(CAP_NET_ADMIN)){ + return -EPERM; + } + */ + + if (copy_from_user(&args, (void*)arg, sizeof(args))) { + return -EFAULT; + } -static char module_fname[128]; -static struct proc_dir_entry *module_proc_ent = NULL; + /* Null terminate the names */ + args.thread_name[31] = 0; + args.interface_name[31] = 0; -static struct notifier_block pktgen_notifier_block = { - .notifier_call = pktgen_device_event, -}; + /* printk("pktgen: thread_name: %s interface_name: %s\n", + * args.thread_name, args.interface_name); + */ + + switch (cmd) { + case GET_PKTGEN_INTERFACE_INFO: { + targ = pg_find_thread(args.thread_name); + if (targ) { + struct pktgen_interface_info* info; + info = pg_find_interface(targ, args.interface_name); + if (info) { + memcpy(&(args.info), info, sizeof(args.info)); + if (copy_to_user((void*)(arg), &args, sizeof(args))) { + printk("ERROR: pktgen: copy_to_user failed.\n"); + err = -EFAULT; + } + else { + err = 0; + } + } + else { + /* printk("ERROR: pktgen: Could not find interface -:%s:-\n", + args.interface_name);*/ + err = -ENODEV; + } + } + else { + printk("ERROR: pktgen: Could not find thread -:%s:-.\n", + args.thread_name); + err = -ENODEV; + } + break; + } + default: + /* pass on to underlying device instead?? */ + printk("%s: Unknown pktgen IOCTL: %x \n", __FUNCTION__, + cmd); + return -EINVAL; + } + + return err; +}/* pktgen_proc_ioctl */ static struct file_operations pktgen_fops = { - .read = proc_pgctrl_read, - .write = proc_pgctrl_write, - /* .ioctl = pktgen_ioctl, later maybe */ + ioctl: pktgen_proc_ioctl, }; -/* - * /proc handling functions - * - */ - -static struct proc_dir_entry *pg_proc_dir = NULL; -static int proc_pgctrl_read_eof=0; +/* Runs from interrupt */ +int pg_notify_queue_woken(struct net_device* dev) { + struct pktgen_thread_info* pg_thread = dev->nqw_data; + /* printk("pg_nqw, pg_thread: %p\n", pg_thread); */ + if (pg_thread && pg_thread->sleeping) { + if (getRelativeCurNs() > (pg_thread->next_tx_ns - 1000)) { + /* See if we should wake up the thread, wake + * slightly early (1000 ns) + */ + pg_thread->sleeping = 0; + wake_up_interruptible(&(pg_thread->queue)); + } + } + return 0; +} -static ssize_t proc_pgctrl_read(struct file* file, char __user * buf, - size_t count, loff_t *ppos) -{ - char data[200]; - int len = 0; - if(proc_pgctrl_read_eof) { - proc_pgctrl_read_eof=0; - len = 0; - goto out; +static void set_nqw_hook(struct pktgen_thread_info* pg_thread, + struct pktgen_interface_info* info, + struct net_device* dev) { + /* The notify-queue-woken magic only works for physical + * devices at this time. So, apply hook to underlying + * device. + */ + struct pktgen_thread_info* pg_thread_nd; + /* printk("In set_nqw_hook, dev: %s\n", dev->name); */ + +#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE) + if (dev->priv_flags & IFF_MAC_VLAN) { + struct macvlan_vlan *vlan = dev->priv; + set_nqw_hook(pg_thread, info, vlan->lowerdev); + return; } +#endif - sprintf(data, "%s", VERSION); + if (dev->priv_flags & IFF_802_1Q_VLAN) { + set_nqw_hook(pg_thread, info, VLAN_DEV_INFO(dev)->real_dev); + return; + } - len = strlen(data); + pg_thread_nd = (struct pktgen_thread_info*)(dev->nqw_data); - if(len > count) { - len =-EFAULT; - goto out; - } + /*printk("pg_thread_nd: %p info: %p\n", pg_thread_nd, info);*/ + + if (pg_thread_nd) { + /* Just bump a reference count, it doesn't really matter which + * info is there since we always look at the parent thread anyway + */ + atomic_inc(&(pg_thread_nd->nqw_ref_count)); - if (copy_to_user(buf, data, len)) { - len =-EFAULT; - goto out; - } + /*printk("Incremented nqw_ref_count: %d device: %s\n", + (int)(atomic_read(&(pg_thread_nd->nqw_ref_count))), dev->name); */ + } + else { + /* Must be a real device, or at least some un-handled fake device. Can't + * hurt to add it here at any rate. + */ + atomic_inc(&(pg_thread->nqw_ref_count)); + dev->nqw_data = pg_thread; + dev->notify_queue_woken = pg_notify_queue_woken; + /*printk("Added nqw callback to device: %s, data: %p data_nd: %p\n", + dev->name, dev->nqw_data, pg_thread_nd);*/ + } +}//set_nqw_hook - *ppos += len; - proc_pgctrl_read_eof=1; /* EOF next call */ - out: - return len; -} +static void clear_nqw_hook(struct pktgen_interface_info* info, + struct net_device* dev) { + /* The notify-queue-woken magic only works for physical + * devices at this time. So, apply hook to underlying + * device. + */ + struct pktgen_thread_info* pg_thread; + /* printk("In clear_nqw_hook, dev: %s\n", dev->name); */ + +#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE) + if (dev->priv_flags & IFF_MAC_VLAN) { + struct macvlan_vlan *vlan = dev->priv; + clear_nqw_hook(info, vlan->lowerdev); + return; + } +#endif -static ssize_t proc_pgctrl_write(struct file* file,const char __user * buf, - size_t count, loff_t *ppos) -{ - char *data = NULL; - int err = 0; + if (dev->priv_flags & IFF_802_1Q_VLAN) { + clear_nqw_hook(info, VLAN_DEV_INFO(dev)->real_dev); + return; + } - if (!capable(CAP_NET_ADMIN)){ - err = -EPERM; - goto out; - } + if (dev->nqw_data) { + pg_thread = (struct pktgen_thread_info*)(dev->nqw_data); + atomic_dec(&(pg_thread->nqw_ref_count)); - data = (void*)vmalloc ((unsigned int)count); + /* printk("Decremented nqw_ref_count: %d device: %s\n", + (int)(atomic_read(&(pg_thread->nqw_ref_count))), dev->name); */ + + BUG_ON(atomic_read(&(pg_thread->nqw_ref_count)) < 0); + + if (atomic_read(&(pg_thread->nqw_ref_count)) == 0) { + /*printk("Removing nqw reference from device: %s\n", dev->name);*/ + dev->notify_queue_woken = NULL; + dev->nqw_data = NULL; + + /* Clear from all other devices too. There is an issue/bug where + * we may at times have more actual references to the pg_thread than + * our reference counter shows. In practice, I do not believe this + * will be a problem, but it will be slightly inefficient. A true + * fix would involve some sort of hash/list/map of physical interfaces + * to reference counts, stored in the pg_thread class. + * --Ben + */ - if(!data) { - err = -ENOMEM; - goto out; + { + int i; + struct pktgen_interface_info* p; + for (i = 0; iodev && p->odev->nqw_data == pg_thread) { + /*printk("Removing nqw reference from dev: %s\n", + p->odev->name);*/ + p->odev->notify_queue_woken = NULL; + p->odev->nqw_data = NULL; + } + p = p->next; + } + } + } + } + else { + /* Else, if we are directly attached to this device, remove the + * nqw callback then too, because we only allow one info struct + * per net-device, so this MUST be the only one. + */ + if (dev == info->odev) { + /*printk("Removing nqw reference from real device: %s\n", + dev->name);*/ + dev->notify_queue_woken = NULL; + dev->nqw_data = NULL; + } + else { + /*printk("pktgen NOTE: dev: %s != info->odev: %s\n", + dev ? dev->name : "NULL", + info->odev ? info->odev->name : "NULL");*/ + } + } } - if (copy_from_user(data, buf, count)) { - err =-EFAULT; - goto out_free; - } - data[count-1] = 0; /* Make string */ - - if (!strcmp(data, "stop")) - pktgen_stop_all_threads_ifs(); + else { + printk("pktgen: Warning: nqw_data is null in clear_nqw_hook, dev: %s\n", + dev->name); + } +}//clear_nqw_hook - else if (!strcmp(data, "start")) - pktgen_run_all_threads(); - else - printk("pktgen: Unknown command: %s\n", data); +static void remove_pg_info_from_hash(struct pktgen_interface_info* info) { + pg_lock_hash(__FUNCTION__); + { + int device_idx = info->odev ? info->odev->ifindex : 0; + int b = device_idx % PG_INFO_HASH_MAX; + struct pktgen_interface_info* p = pg_info_hash[b]; + struct pktgen_interface_info* prev = pg_info_hash[b]; + int found_one = 0; + + PG_DEBUG(printk("remove_pg_info_from_hash, p: %p info: %p device_idx: %i\n", + p, info, device_idx)); + + if (p != NULL) { + + if (p == info) { + pg_info_hash[b] = p->next_hash; + p->next_hash = NULL; + found_one = 1; + } + else { + while (prev->next_hash) { + p = prev->next_hash; + if (p == info) { + prev->next_hash = p->next_hash; + p->next_hash = NULL; + found_one = 1; + break; + } + prev = p; + } + } + } - err = count; + if (found_one) { + if (info->odev) { + info->odev->priv_flags &= ~(IFF_PKTGEN_RCV); + clear_nqw_hook(info, info->odev); + } + } + } + pg_unlock_hash(__FUNCTION__); +}/* remove_pg_info_from_hash */ - out_free: - vfree (data); - out: - return err; -} -static int proc_if_read(char *buf , char **start, off_t offset, - int len, int *eof, void *data) -{ - char *p; - int i; - struct pktgen_dev *pkt_dev = (struct pktgen_dev*)(data); - __u64 sa; - __u64 stopped; - __u64 now = getCurUs(); +static void add_pg_info_to_hash(struct pktgen_thread_info* pg_thread, + struct pktgen_interface_info* info) { + /* First remove it, just in case it's already there. */ + remove_pg_info_from_hash(info); - p = buf; - p += sprintf(p, "Params: count %llu min_pkt_size: %u max_pkt_size: %u\n", - (unsigned long long) pkt_dev->count, - pkt_dev->min_pkt_size, pkt_dev->max_pkt_size); - - p += sprintf(p, " frags: %d delay: %u clone_skb: %d ifname: %s\n", - pkt_dev->nfrags, 1000*pkt_dev->delay_us+pkt_dev->delay_ns, pkt_dev->clone_skb, pkt_dev->ifname); - - p += sprintf(p, " flows: %u flowlen: %u\n", pkt_dev->cflows, pkt_dev->lflow); - - - if(pkt_dev->flags & F_IPV6) { - char b1[128], b2[128], b3[128]; - fmt_ip6(b1, pkt_dev->in6_saddr.s6_addr); - fmt_ip6(b2, pkt_dev->min_in6_saddr.s6_addr); - fmt_ip6(b3, pkt_dev->max_in6_saddr.s6_addr); - p += sprintf(p, " saddr: %s min_saddr: %s max_saddr: %s\n", b1, b2, b3); - - fmt_ip6(b1, pkt_dev->in6_daddr.s6_addr); - fmt_ip6(b2, pkt_dev->min_in6_daddr.s6_addr); - fmt_ip6(b3, pkt_dev->max_in6_daddr.s6_addr); - p += sprintf(p, " daddr: %s min_daddr: %s max_daddr: %s\n", b1, b2, b3); - - } - else - p += sprintf(p, " dst_min: %s dst_max: %s\n src_min: %s src_max: %s\n", - pkt_dev->dst_min, pkt_dev->dst_max, pkt_dev->src_min, pkt_dev->src_max); + pg_lock_hash(__FUNCTION__); + { + int device_idx = info->odev ? info->odev->ifindex : 0; + int b = device_idx % PG_INFO_HASH_MAX; - p += sprintf(p, " src_mac: "); - - if ((pkt_dev->src_mac[0] == 0) && - (pkt_dev->src_mac[1] == 0) && - (pkt_dev->src_mac[2] == 0) && - (pkt_dev->src_mac[3] == 0) && - (pkt_dev->src_mac[4] == 0) && - (pkt_dev->src_mac[5] == 0)) - - for (i = 0; i < 6; i++) - p += sprintf(p, "%02X%s", pkt_dev->odev->dev_addr[i], i == 5 ? " " : ":"); - - else - for (i = 0; i < 6; i++) - p += sprintf(p, "%02X%s", pkt_dev->src_mac[i], i == 5 ? " " : ":"); - - p += sprintf(p, "dst_mac: "); - for (i = 0; i < 6; i++) - p += sprintf(p, "%02X%s", pkt_dev->dst_mac[i], i == 5 ? "\n" : ":"); + PG_DEBUG(printk("add_pg_info_from_hash, b: %i info: %p device_idx: %i\n", + b, info, device_idx)); - p += sprintf(p, " udp_src_min: %d udp_src_max: %d udp_dst_min: %d udp_dst_max: %d\n", - pkt_dev->udp_src_min, pkt_dev->udp_src_max, pkt_dev->udp_dst_min, - pkt_dev->udp_dst_max); - - p += sprintf(p, " src_mac_count: %d dst_mac_count: %d \n Flags: ", - pkt_dev->src_mac_count, pkt_dev->dst_mac_count); + info->next_hash = pg_info_hash[b]; + pg_info_hash[b] = info; - if (pkt_dev->flags & F_IPV6) - p += sprintf(p, "IPV6 "); - - if (pkt_dev->flags & F_IPSRC_RND) - p += sprintf(p, "IPSRC_RND "); + if (info->odev) { + set_nqw_hook(pg_thread, info, info->odev); + info->odev->priv_flags |= (IFF_PKTGEN_RCV); + } + } + pg_unlock_hash(__FUNCTION__); +}/* add_pg_info_to_hash */ - if (pkt_dev->flags & F_IPDST_RND) - p += sprintf(p, "IPDST_RND "); - - if (pkt_dev->flags & F_TXSIZE_RND) - p += sprintf(p, "TXSIZE_RND "); - - if (pkt_dev->flags & F_UDPSRC_RND) - p += sprintf(p, "UDPSRC_RND "); - - if (pkt_dev->flags & F_UDPDST_RND) - p += sprintf(p, "UDPDST_RND "); - - if (pkt_dev->flags & F_MACSRC_RND) - p += sprintf(p, "MACSRC_RND "); - - if (pkt_dev->flags & F_MACDST_RND) - p += sprintf(p, "MACDST_RND "); - - p += sprintf(p, "\n"); - - sa = pkt_dev->started_at; - stopped = pkt_dev->stopped_at; - if (pkt_dev->running) - stopped = now; /* not really stopped, more like last-running-at */ - - p += sprintf(p, "Current:\n pkts-sofar: %llu errors: %llu\n started: %lluus stopped: %lluus idle: %lluus\n", - (unsigned long long) pkt_dev->sofar, - (unsigned long long) pkt_dev->errors, - (unsigned long long) sa, - (unsigned long long) stopped, - (unsigned long long) pkt_dev->idle_acc); +/* Find the pktgen_interface_info for a device idx */ +struct pktgen_interface_info* find_pg_info(int device_idx) { + struct pktgen_interface_info* p = NULL; + if (debug > 1) { + printk("in find_pg_info...\n"); + } + pg_lock_hash(__FUNCTION__); + { + int b = device_idx % PG_INFO_HASH_MAX; + p = pg_info_hash[b]; + while (p) { + if (p->odev && (p->odev->ifindex == device_idx)) { + break; + } + p = p->next_hash; + } + } + pg_unlock_hash(__FUNCTION__); + return p; +} - p += sprintf(p, " seq_num: %d cur_dst_mac_offset: %d cur_src_mac_offset: %d\n", - pkt_dev->seq_num, pkt_dev->cur_dst_mac_offset, pkt_dev->cur_src_mac_offset); - if(pkt_dev->flags & F_IPV6) { - char b1[128], b2[128]; - fmt_ip6(b1, pkt_dev->cur_in6_daddr.s6_addr); - fmt_ip6(b2, pkt_dev->cur_in6_saddr.s6_addr); - p += sprintf(p, " cur_saddr: %s cur_daddr: %s\n", b2, b1); - } - else - p += sprintf(p, " cur_saddr: 0x%x cur_daddr: 0x%x\n", - pkt_dev->cur_saddr, pkt_dev->cur_daddr); +/* Remove an interface from our hash, dissassociate pktgen_interface_info + * from interface + */ +static void check_remove_device(struct pktgen_interface_info* info) { + struct pktgen_interface_info* pi = NULL; + if (info->odev) { + pi = find_pg_info(info->odev->ifindex); + if (pi != info) { + printk("ERROR: pi != info, pi: %p info: %p\n", pi, info); + } + else { + /* Remove info from our hash */ + remove_pg_info_from_hash(info); + } + /* TODO: Wonder if we need locking here? Had rtnl_lock, but + * that can sleep and this is called with irqs disabled... + */ + info->odev->priv_flags &= ~(IFF_PKTGEN_RCV); + atomic_dec(&(info->odev->refcnt)); + info->odev = NULL; + } +}/* check_remove_device */ + + +static int pg_remove_interface_from_all_threads(const char* dev_name) { + int cnt = 0; + pg_lock_thread_list(__FUNCTION__); + { + struct pktgen_thread_info* tmp = pktgen_threads; + struct pktgen_interface_info* info = NULL; + + while (tmp) { + info = pg_find_interface(tmp, dev_name); + if (info) { + printk("pktgen: Removing interface: %s from pktgen control.\n", + dev_name); + pg_rem_interface_info(tmp, info); + cnt++; + } + else { + /* printk("pktgen: Could not find interface: %s in rem_from_all.\n", + dev_name); */ + } + tmp = tmp->next; + } + } + pg_unlock_thread_list(__FUNCTION__); + return cnt; +}/* pg_rem_interface_from_all_threads */ - p += sprintf(p, " cur_udp_dst: %d cur_udp_src: %d\n", - pkt_dev->cur_udp_dst, pkt_dev->cur_udp_src); - p += sprintf(p, " flows: %u\n", pkt_dev->nflows); +static int pktgen_device_event(struct notifier_block *unused, unsigned long event, void *ptr) { + struct net_device *dev = (struct net_device *)(ptr); - if (pkt_dev->result[0]) - p += sprintf(p, "Result: %s\n", pkt_dev->result); - else - p += sprintf(p, "Result: Idle\n"); - *eof = 1; - - return p - buf; -} - - -static int count_trail_chars(const char __user *user_buffer, unsigned int maxlen) -{ - int i; - - for (i = 0; i < maxlen; i++) { - char c; - if (get_user(c, &user_buffer[i])) - return -EFAULT; - switch (c) { - case '\"': - case '\n': - case '\r': - case '\t': - case ' ': - case '=': - break; - default: - goto done; - }; - } -done: - return i; -} - -static unsigned long num_arg(const char __user *user_buffer, unsigned long maxlen, - unsigned long *num) -{ - int i = 0; - *num = 0; - - for(; i < maxlen; i++) { - char c; - if (get_user(c, &user_buffer[i])) - return -EFAULT; - if ((c >= '0') && (c <= '9')) { - *num *= 10; - *num += c -'0'; - } else - break; - } - return i; -} - -static int strn_len(const char __user *user_buffer, unsigned int maxlen) -{ - int i = 0; - - for(; i < maxlen; i++) { - char c; - if (get_user(c, &user_buffer[i])) - return -EFAULT; - switch (c) { - case '\"': - case '\n': - case '\r': - case '\t': - case ' ': - goto done_str; - break; - default: - break; - }; - } -done_str: - - return i; -} - -static int proc_if_write(struct file *file, const char __user *user_buffer, - unsigned long count, void *data) -{ - int i = 0, max, len; - char name[16], valstr[32]; - unsigned long value = 0; - struct pktgen_dev *pkt_dev = (struct pktgen_dev*)(data); - char* pg_result = NULL; - int tmp = 0; - char buf[128]; - - pg_result = &(pkt_dev->result[0]); - - if (count < 1) { - printk("pktgen: wrong command format\n"); - return -EINVAL; - } - - max = count - i; - tmp = count_trail_chars(&user_buffer[i], max); - if (tmp < 0) { - printk("pktgen: illegal format\n"); - return tmp; - } - i += tmp; - - /* Read variable name */ - - len = strn_len(&user_buffer[i], sizeof(name) - 1); - if (len < 0) { return len; } - memset(name, 0, sizeof(name)); - if (copy_from_user(name, &user_buffer[i], len) ) - return -EFAULT; - i += len; - - max = count -i; - len = count_trail_chars(&user_buffer[i], max); - if (len < 0) - return len; - - i += len; - - if (debug) { - char tb[count + 1]; - if (copy_from_user(tb, user_buffer, count)) - return -EFAULT; - tb[count] = 0; - printk("pktgen: %s,%lu buffer -:%s:-\n", name, count, tb); - } - - if (!strcmp(name, "min_pkt_size")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - if (value < 14+20+8) - value = 14+20+8; - if (value != pkt_dev->min_pkt_size) { - pkt_dev->min_pkt_size = value; - pkt_dev->cur_pkt_size = value; - } - sprintf(pg_result, "OK: min_pkt_size=%u", pkt_dev->min_pkt_size); - return count; - } - - if (!strcmp(name, "max_pkt_size")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - if (value < 14+20+8) - value = 14+20+8; - if (value != pkt_dev->max_pkt_size) { - pkt_dev->max_pkt_size = value; - pkt_dev->cur_pkt_size = value; - } - sprintf(pg_result, "OK: max_pkt_size=%u", pkt_dev->max_pkt_size); - return count; - } - - /* Shortcut for min = max */ - - if (!strcmp(name, "pkt_size")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - if (value < 14+20+8) - value = 14+20+8; - if (value != pkt_dev->min_pkt_size) { - pkt_dev->min_pkt_size = value; - pkt_dev->max_pkt_size = value; - pkt_dev->cur_pkt_size = value; - } - sprintf(pg_result, "OK: pkt_size=%u", pkt_dev->min_pkt_size); - return count; - } - - if (!strcmp(name, "debug")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - debug = value; - sprintf(pg_result, "OK: debug=%u", debug); - return count; - } - - if (!strcmp(name, "frags")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - pkt_dev->nfrags = value; - sprintf(pg_result, "OK: frags=%u", pkt_dev->nfrags); - return count; - } - if (!strcmp(name, "delay")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - if (value == 0x7FFFFFFF) { - pkt_dev->delay_us = 0x7FFFFFFF; - pkt_dev->delay_ns = 0; - } else { - pkt_dev->delay_us = value / 1000; - pkt_dev->delay_ns = value % 1000; - } - sprintf(pg_result, "OK: delay=%u", 1000*pkt_dev->delay_us+pkt_dev->delay_ns); - return count; - } - if (!strcmp(name, "udp_src_min")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - if (value != pkt_dev->udp_src_min) { - pkt_dev->udp_src_min = value; - pkt_dev->cur_udp_src = value; - } - sprintf(pg_result, "OK: udp_src_min=%u", pkt_dev->udp_src_min); - return count; - } - if (!strcmp(name, "udp_dst_min")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - if (value != pkt_dev->udp_dst_min) { - pkt_dev->udp_dst_min = value; - pkt_dev->cur_udp_dst = value; - } - sprintf(pg_result, "OK: udp_dst_min=%u", pkt_dev->udp_dst_min); - return count; - } - if (!strcmp(name, "udp_src_max")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - if (value != pkt_dev->udp_src_max) { - pkt_dev->udp_src_max = value; - pkt_dev->cur_udp_src = value; - } - sprintf(pg_result, "OK: udp_src_max=%u", pkt_dev->udp_src_max); - return count; - } - if (!strcmp(name, "udp_dst_max")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - if (value != pkt_dev->udp_dst_max) { - pkt_dev->udp_dst_max = value; - pkt_dev->cur_udp_dst = value; - } - sprintf(pg_result, "OK: udp_dst_max=%u", pkt_dev->udp_dst_max); - return count; - } - if (!strcmp(name, "clone_skb")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - pkt_dev->clone_skb = value; - - sprintf(pg_result, "OK: clone_skb=%d", pkt_dev->clone_skb); - return count; - } - if (!strcmp(name, "count")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - pkt_dev->count = value; - sprintf(pg_result, "OK: count=%llu", - (unsigned long long) pkt_dev->count); - return count; - } - if (!strcmp(name, "src_mac_count")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - if (pkt_dev->src_mac_count != value) { - pkt_dev->src_mac_count = value; - pkt_dev->cur_src_mac_offset = 0; - } - sprintf(pg_result, "OK: src_mac_count=%d", pkt_dev->src_mac_count); - return count; - } - if (!strcmp(name, "dst_mac_count")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - if (pkt_dev->dst_mac_count != value) { - pkt_dev->dst_mac_count = value; - pkt_dev->cur_dst_mac_offset = 0; - } - sprintf(pg_result, "OK: dst_mac_count=%d", pkt_dev->dst_mac_count); - return count; - } - if (!strcmp(name, "flag")) { - char f[32]; - memset(f, 0, 32); - len = strn_len(&user_buffer[i], sizeof(f) - 1); - if (len < 0) { return len; } - if (copy_from_user(f, &user_buffer[i], len)) - return -EFAULT; - i += len; - if (strcmp(f, "IPSRC_RND") == 0) - pkt_dev->flags |= F_IPSRC_RND; - - else if (strcmp(f, "!IPSRC_RND") == 0) - pkt_dev->flags &= ~F_IPSRC_RND; - - else if (strcmp(f, "TXSIZE_RND") == 0) - pkt_dev->flags |= F_TXSIZE_RND; - - else if (strcmp(f, "!TXSIZE_RND") == 0) - pkt_dev->flags &= ~F_TXSIZE_RND; - - else if (strcmp(f, "IPDST_RND") == 0) - pkt_dev->flags |= F_IPDST_RND; - - else if (strcmp(f, "!IPDST_RND") == 0) - pkt_dev->flags &= ~F_IPDST_RND; - - else if (strcmp(f, "UDPSRC_RND") == 0) - pkt_dev->flags |= F_UDPSRC_RND; - - else if (strcmp(f, "!UDPSRC_RND") == 0) - pkt_dev->flags &= ~F_UDPSRC_RND; - - else if (strcmp(f, "UDPDST_RND") == 0) - pkt_dev->flags |= F_UDPDST_RND; - - else if (strcmp(f, "!UDPDST_RND") == 0) - pkt_dev->flags &= ~F_UDPDST_RND; - - else if (strcmp(f, "MACSRC_RND") == 0) - pkt_dev->flags |= F_MACSRC_RND; - - else if (strcmp(f, "!MACSRC_RND") == 0) - pkt_dev->flags &= ~F_MACSRC_RND; - - else if (strcmp(f, "MACDST_RND") == 0) - pkt_dev->flags |= F_MACDST_RND; - - else if (strcmp(f, "!MACDST_RND") == 0) - pkt_dev->flags &= ~F_MACDST_RND; - - else { - sprintf(pg_result, "Flag -:%s:- unknown\nAvailable flags, (prepend ! to un-set flag):\n%s", - f, - "IPSRC_RND, IPDST_RND, TXSIZE_RND, UDPSRC_RND, UDPDST_RND, MACSRC_RND, MACDST_RND\n"); - return count; - } - sprintf(pg_result, "OK: flags=0x%x", pkt_dev->flags); - return count; - } - if (!strcmp(name, "dst_min") || !strcmp(name, "dst")) { - len = strn_len(&user_buffer[i], sizeof(pkt_dev->dst_min) - 1); - if (len < 0) { return len; } - - if (copy_from_user(buf, &user_buffer[i], len)) - return -EFAULT; - buf[len] = 0; - if (strcmp(buf, pkt_dev->dst_min) != 0) { - memset(pkt_dev->dst_min, 0, sizeof(pkt_dev->dst_min)); - strncpy(pkt_dev->dst_min, buf, len); - pkt_dev->daddr_min = in_aton(pkt_dev->dst_min); - pkt_dev->cur_daddr = pkt_dev->daddr_min; - } - if(debug) - printk("pktgen: dst_min set to: %s\n", pkt_dev->dst_min); - i += len; - sprintf(pg_result, "OK: dst_min=%s", pkt_dev->dst_min); - return count; - } - if (!strcmp(name, "dst_max")) { - len = strn_len(&user_buffer[i], sizeof(pkt_dev->dst_max) - 1); - if (len < 0) { return len; } - - if (copy_from_user(buf, &user_buffer[i], len)) - return -EFAULT; - - buf[len] = 0; - if (strcmp(buf, pkt_dev->dst_max) != 0) { - memset(pkt_dev->dst_max, 0, sizeof(pkt_dev->dst_max)); - strncpy(pkt_dev->dst_max, buf, len); - pkt_dev->daddr_max = in_aton(pkt_dev->dst_max); - pkt_dev->cur_daddr = pkt_dev->daddr_max; - } - if(debug) - printk("pktgen: dst_max set to: %s\n", pkt_dev->dst_max); - i += len; - sprintf(pg_result, "OK: dst_max=%s", pkt_dev->dst_max); - return count; - } - if (!strcmp(name, "dst6")) { - len = strn_len(&user_buffer[i], sizeof(buf) - 1); - if (len < 0) return len; - - pkt_dev->flags |= F_IPV6; - - if (copy_from_user(buf, &user_buffer[i], len)) - return -EFAULT; - buf[len] = 0; - - scan_ip6(buf, pkt_dev->in6_daddr.s6_addr); - fmt_ip6(buf, pkt_dev->in6_daddr.s6_addr); - - ipv6_addr_copy(&pkt_dev->cur_in6_daddr, &pkt_dev->in6_daddr); - - if(debug) - printk("pktgen: dst6 set to: %s\n", buf); - - i += len; - sprintf(pg_result, "OK: dst6=%s", buf); - return count; - } - if (!strcmp(name, "dst6_min")) { - len = strn_len(&user_buffer[i], sizeof(buf) - 1); - if (len < 0) return len; - - pkt_dev->flags |= F_IPV6; - - if (copy_from_user(buf, &user_buffer[i], len)) - return -EFAULT; - buf[len] = 0; - - scan_ip6(buf, pkt_dev->min_in6_daddr.s6_addr); - fmt_ip6(buf, pkt_dev->min_in6_daddr.s6_addr); - - ipv6_addr_copy(&pkt_dev->cur_in6_daddr, &pkt_dev->min_in6_daddr); - if(debug) - printk("pktgen: dst6_min set to: %s\n", buf); - - i += len; - sprintf(pg_result, "OK: dst6_min=%s", buf); - return count; - } - if (!strcmp(name, "dst6_max")) { - len = strn_len(&user_buffer[i], sizeof(buf) - 1); - if (len < 0) return len; - - pkt_dev->flags |= F_IPV6; - - if (copy_from_user(buf, &user_buffer[i], len)) - return -EFAULT; - buf[len] = 0; - - scan_ip6(buf, pkt_dev->max_in6_daddr.s6_addr); - fmt_ip6(buf, pkt_dev->max_in6_daddr.s6_addr); - - if(debug) - printk("pktgen: dst6_max set to: %s\n", buf); - - i += len; - sprintf(pg_result, "OK: dst6_max=%s", buf); - return count; - } - if (!strcmp(name, "src6")) { - len = strn_len(&user_buffer[i], sizeof(buf) - 1); - if (len < 0) return len; - - pkt_dev->flags |= F_IPV6; - - if (copy_from_user(buf, &user_buffer[i], len)) - return -EFAULT; - buf[len] = 0; - - scan_ip6(buf, pkt_dev->in6_saddr.s6_addr); - fmt_ip6(buf, pkt_dev->in6_saddr.s6_addr); - - ipv6_addr_copy(&pkt_dev->cur_in6_saddr, &pkt_dev->in6_saddr); - - if(debug) - printk("pktgen: src6 set to: %s\n", buf); - - i += len; - sprintf(pg_result, "OK: src6=%s", buf); - return count; - } - if (!strcmp(name, "src_min")) { - len = strn_len(&user_buffer[i], sizeof(pkt_dev->src_min) - 1); - if (len < 0) { return len; } - if (copy_from_user(buf, &user_buffer[i], len)) - return -EFAULT; - buf[len] = 0; - if (strcmp(buf, pkt_dev->src_min) != 0) { - memset(pkt_dev->src_min, 0, sizeof(pkt_dev->src_min)); - strncpy(pkt_dev->src_min, buf, len); - pkt_dev->saddr_min = in_aton(pkt_dev->src_min); - pkt_dev->cur_saddr = pkt_dev->saddr_min; - } - if(debug) - printk("pktgen: src_min set to: %s\n", pkt_dev->src_min); - i += len; - sprintf(pg_result, "OK: src_min=%s", pkt_dev->src_min); - return count; - } - if (!strcmp(name, "src_max")) { - len = strn_len(&user_buffer[i], sizeof(pkt_dev->src_max) - 1); - if (len < 0) { return len; } - if (copy_from_user(buf, &user_buffer[i], len)) - return -EFAULT; - buf[len] = 0; - if (strcmp(buf, pkt_dev->src_max) != 0) { - memset(pkt_dev->src_max, 0, sizeof(pkt_dev->src_max)); - strncpy(pkt_dev->src_max, buf, len); - pkt_dev->saddr_max = in_aton(pkt_dev->src_max); - pkt_dev->cur_saddr = pkt_dev->saddr_max; - } - if(debug) - printk("pktgen: src_max set to: %s\n", pkt_dev->src_max); - i += len; - sprintf(pg_result, "OK: src_max=%s", pkt_dev->src_max); - return count; - } - if (!strcmp(name, "dst_mac")) { - char *v = valstr; - unsigned char old_dmac[6]; - unsigned char *m = pkt_dev->dst_mac; - memcpy(old_dmac, pkt_dev->dst_mac, 6); - - len = strn_len(&user_buffer[i], sizeof(valstr) - 1); - if (len < 0) { return len; } - memset(valstr, 0, sizeof(valstr)); - if( copy_from_user(valstr, &user_buffer[i], len)) - return -EFAULT; - i += len; - - for(*m = 0;*v && m < pkt_dev->dst_mac + 6; v++) { - if (*v >= '0' && *v <= '9') { - *m *= 16; - *m += *v - '0'; - } - if (*v >= 'A' && *v <= 'F') { - *m *= 16; - *m += *v - 'A' + 10; - } - if (*v >= 'a' && *v <= 'f') { - *m *= 16; - *m += *v - 'a' + 10; - } - if (*v == ':') { - m++; - *m = 0; - } - } - - /* Set up Dest MAC */ - if (memcmp(old_dmac, pkt_dev->dst_mac, 6) != 0) - memcpy(&(pkt_dev->hh[0]), pkt_dev->dst_mac, 6); - - sprintf(pg_result, "OK: dstmac"); - return count; - } - if (!strcmp(name, "src_mac")) { - char *v = valstr; - unsigned char *m = pkt_dev->src_mac; - - len = strn_len(&user_buffer[i], sizeof(valstr) - 1); - if (len < 0) { return len; } - memset(valstr, 0, sizeof(valstr)); - if( copy_from_user(valstr, &user_buffer[i], len)) - return -EFAULT; - i += len; - - for(*m = 0;*v && m < pkt_dev->src_mac + 6; v++) { - if (*v >= '0' && *v <= '9') { - *m *= 16; - *m += *v - '0'; - } - if (*v >= 'A' && *v <= 'F') { - *m *= 16; - *m += *v - 'A' + 10; - } - if (*v >= 'a' && *v <= 'f') { - *m *= 16; - *m += *v - 'a' + 10; - } - if (*v == ':') { - m++; - *m = 0; - } - } - - sprintf(pg_result, "OK: srcmac"); - return count; - } - - if (!strcmp(name, "clear_counters")) { - pktgen_clear_counters(pkt_dev); - sprintf(pg_result, "OK: Clearing counters.\n"); - return count; - } - - if (!strcmp(name, "flows")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - if (value > MAX_CFLOWS) - value = MAX_CFLOWS; - - pkt_dev->cflows = value; - sprintf(pg_result, "OK: flows=%u", pkt_dev->cflows); - return count; - } - - if (!strcmp(name, "flowlen")) { - len = num_arg(&user_buffer[i], 10, &value); - if (len < 0) { return len; } - i += len; - pkt_dev->lflow = value; - sprintf(pg_result, "OK: flowlen=%u", pkt_dev->lflow); - return count; - } - - sprintf(pkt_dev->result, "No such parameter \"%s\"", name); - return -EINVAL; -} - -static int proc_thread_read(char *buf , char **start, off_t offset, - int len, int *eof, void *data) -{ - char *p; - struct pktgen_thread *t = (struct pktgen_thread*)(data); - struct pktgen_dev *pkt_dev = NULL; - - - if (!t) { - printk("pktgen: ERROR: could not find thread in proc_thread_read\n"); - return -EINVAL; - } - - p = buf; - p += sprintf(p, "Name: %s max_before_softirq: %d\n", - t->name, t->max_before_softirq); - - p += sprintf(p, "Running: "); - - if_lock(t); - for(pkt_dev = t->if_list;pkt_dev; pkt_dev = pkt_dev->next) - if(pkt_dev->running) - p += sprintf(p, "%s ", pkt_dev->ifname); - - p += sprintf(p, "\nStopped: "); - - for(pkt_dev = t->if_list;pkt_dev; pkt_dev = pkt_dev->next) - if(!pkt_dev->running) - p += sprintf(p, "%s ", pkt_dev->ifname); - - if (t->result[0]) - p += sprintf(p, "\nResult: %s\n", t->result); - else - p += sprintf(p, "\nResult: NA\n"); - - *eof = 1; - - if_unlock(t); - - return p - buf; -} - -static int proc_thread_write(struct file *file, const char __user *user_buffer, - unsigned long count, void *data) -{ - int i = 0, max, len, ret; - char name[40]; - struct pktgen_thread *t; - char *pg_result; - unsigned long value = 0; - - if (count < 1) { - // sprintf(pg_result, "Wrong command format"); - return -EINVAL; - } - - max = count - i; - len = count_trail_chars(&user_buffer[i], max); - if (len < 0) - return len; - - i += len; - - /* Read variable name */ - - len = strn_len(&user_buffer[i], sizeof(name) - 1); - if (len < 0) - return len; - - memset(name, 0, sizeof(name)); - if (copy_from_user(name, &user_buffer[i], len)) - return -EFAULT; - i += len; - - max = count -i; - len = count_trail_chars(&user_buffer[i], max); - if (len < 0) - return len; - - i += len; - - if (debug) - printk("pktgen: t=%s, count=%lu\n", name, count); - - thread_lock(); - - t = (struct pktgen_thread*)(data); - if(!t) { - printk("pktgen: ERROR: No thread\n"); - ret = -EINVAL; - goto out; - } - - pg_result = &(t->result[0]); - - if (!strcmp(name, "add_device")) { - char f[32]; - memset(f, 0, 32); - len = strn_len(&user_buffer[i], sizeof(f) - 1); - if (len < 0) { - ret = len; - goto out; - } - if( copy_from_user(f, &user_buffer[i], len) ) - return -EFAULT; - i += len; - pktgen_add_device(t, f); - ret = count; - sprintf(pg_result, "OK: add_device=%s", f); - goto out; - } - - if (!strcmp(name, "rem_device_all")) { - t->control |= T_REMDEV; - current->state = TASK_INTERRUPTIBLE; - schedule_timeout(HZ/8); /* Propagate thread->control */ - ret = count; - sprintf(pg_result, "OK: rem_device_all"); - goto out; - } - - - if (!strcmp(name, "max_before_softirq")) { - len = num_arg(&user_buffer[i], 10, &value); - t->max_before_softirq = value; - ret = count; - sprintf(pg_result, "OK: max_before_softirq=%lu", value); - goto out; - } - - ret = -EINVAL; - out: - thread_unlock(); - - return ret; -} - -static int create_proc_dir(void) -{ - int len; - /* does proc_dir already exists */ - len = strlen(PG_PROC_DIR); - - for (pg_proc_dir = proc_net->subdir; pg_proc_dir; pg_proc_dir=pg_proc_dir->next) { - if ((pg_proc_dir->namelen == len) && - (! memcmp(pg_proc_dir->name, PG_PROC_DIR, len))) - break; - } - - if (!pg_proc_dir) - pg_proc_dir = create_proc_entry(PG_PROC_DIR, S_IFDIR, proc_net); - - if (!pg_proc_dir) - return -ENODEV; - - return 0; -} - -static int remove_proc_dir(void) -{ - remove_proc_entry(PG_PROC_DIR, proc_net); - return 0; -} - -/* Think find or remove for NN */ -static struct pktgen_dev *__pktgen_NN_threads(const char* ifname, int remove) -{ - struct pktgen_thread *t; - struct pktgen_dev *pkt_dev = NULL; - - t = pktgen_threads; - - while (t) { - pkt_dev = pktgen_find_dev(t, ifname); - if (pkt_dev) { - if(remove) { - if_lock(t); - pktgen_remove_device(t, pkt_dev); - if_unlock(t); - } - break; - } - t = t->next; - } - return pkt_dev; -} - -static struct pktgen_dev *pktgen_NN_threads(const char* ifname, int remove) -{ - struct pktgen_dev *pkt_dev = NULL; - thread_lock(); - pkt_dev = __pktgen_NN_threads(ifname, remove); - thread_unlock(); - return pkt_dev; -} - -static int pktgen_device_event(struct notifier_block *unused, unsigned long event, void *ptr) -{ - struct net_device *dev = (struct net_device *)(ptr); - - /* It is OK that we do not hold the group lock right now, - * as we run under the RTNL lock. - */ + /* It is OK that we do not hold the group lock right now, + * as we run under the RTNL lock. + */ switch (event) { case NETDEV_CHANGEADDR: @@ -1548,368 +725,364 @@ break; case NETDEV_UNREGISTER: - pktgen_NN_threads(dev->name, REMOVE); + pg_remove_interface_from_all_threads(dev->name); break; }; return NOTIFY_DONE; } -/* Associate pktgen_dev with a device. */ -static struct net_device* pktgen_setup_dev(struct pktgen_dev *pkt_dev) { +/* Associate pktgen_interface_info with a device. + */ +static struct net_device* pg_setup_interface(struct pktgen_thread_info* pg_thread, + struct pktgen_interface_info* info) { struct net_device *odev; - - /* Clean old setups */ - - if (pkt_dev->odev) { - dev_put(pkt_dev->odev); - pkt_dev->odev = NULL; - } - - odev = dev_get_by_name(pkt_dev->ifname); - + int keep_it = 0; + + check_remove_device(info); + + odev = dev_get_by_name(info->ifname); if (!odev) { - printk("pktgen: no such netdevice: \"%s\"\n", pkt_dev->ifname); - goto out; + printk("No such netdevice: \"%s\"\n", info->ifname); } - if (odev->type != ARPHRD_ETHER) { - printk("pktgen: not an ethernet device: \"%s\"\n", pkt_dev->ifname); - goto out_put; - } - if (!netif_running(odev)) { - printk("pktgen: device is down: \"%s\"\n", pkt_dev->ifname); - goto out_put; + else if (odev->type != ARPHRD_ETHER) { + printk("Not an ethernet device: \"%s\"\n", info->ifname); } - pkt_dev->odev = odev; - - return pkt_dev->odev; + else if (!netif_running(odev)) { + printk("Device is down: \"%s\"\n", info->ifname); + } + else if (odev->priv_flags & IFF_PKTGEN_RCV) { + printk("ERROR: Device: \"%s\" is already assigned to a pktgen interface.\n", + info->ifname); + } + else { + info->odev = odev; + info->odev->priv_flags |= (IFF_PKTGEN_RCV); -out_put: - dev_put(odev); -out: - return NULL; + /* Can't use multi-skb > 0 with virtual interfaces, because they change + * the skb->dev pointer (at least) and so it's really impossible to send + * the exact same pkt over and over again + */ + if ((odev->priv_flags & IFF_MAC_VLAN) || + (odev->priv_flags & IFF_802_1Q_VLAN)) { + if (info->multiskb > 0) { + printk("pktgen: WARNING: Cannot use multi-skb > 0 on virtual interfaces, setting to zero.\n"); + info->multiskb = 0; + } + } + + keep_it = 1; + } + + if (info->odev) { + add_pg_info_to_hash(pg_thread, info); + } -} + if ((!keep_it) && odev) { + dev_put(odev); + } + + return info->odev; +}/* pg_setup_interface */ -/* Read pkt_dev from the interface and set up internal pktgen_dev +/* Read info from the interface and set up internal pktgen_interface_info * structure to have the right information to create/send packets */ -static void pktgen_setup_inject(struct pktgen_dev *pkt_dev) +static void pg_setup_inject(struct pktgen_interface_info* info) { - /* Try once more, just in case it works now. */ - if (!pkt_dev->odev) - pktgen_setup_dev(pkt_dev); - - if (!pkt_dev->odev) { - printk("pktgen: ERROR: pkt_dev->odev == NULL in setup_inject.\n"); - sprintf(pkt_dev->result, "ERROR: pkt_dev->odev == NULL in setup_inject.\n"); + if (!info->odev) { + /* Try once more, just in case it works now. */ + pg_setup_interface(info->pg_thread, info); + } + + if (!info->odev) { + printk("ERROR: info->odev == NULL in setup_inject.\n"); + sprintf(info->result, "ERROR: info->odev == NULL in setup_inject.\n"); return; } /* Default to the interface's mac if not explicitly set. */ + if (!(info->flags & F_SET_SRCMAC)) { + memcpy(&(info->hh[6]), info->odev->dev_addr, 6); + } + else { + memcpy(&(info->hh[6]), info->src_mac, 6); + } - if ((pkt_dev->src_mac[0] == 0) && - (pkt_dev->src_mac[1] == 0) && - (pkt_dev->src_mac[2] == 0) && - (pkt_dev->src_mac[3] == 0) && - (pkt_dev->src_mac[4] == 0) && - (pkt_dev->src_mac[5] == 0)) { - - memcpy(&(pkt_dev->hh[6]), pkt_dev->odev->dev_addr, 6); - } /* Set up Dest MAC */ - memcpy(&(pkt_dev->hh[0]), pkt_dev->dst_mac, 6); + memcpy(&(info->hh[0]), info->dst_mac, 6); /* Set up pkt size */ - pkt_dev->cur_pkt_size = pkt_dev->min_pkt_size; - - if(pkt_dev->flags & F_IPV6) { - /* - * Skip this automatic address setting until locks or functions - * gets exported - */ - -#ifdef NOTNOW - int i, set = 0, err=1; - struct inet6_dev *idev; - - for(i=0; i< IN6_ADDR_HSIZE; i++) - if(pkt_dev->cur_in6_saddr.s6_addr[i]) { - set = 1; - break; - } + info->cur_pkt_size = info->min_pkt_size; + + info->saddr_min = 0; + info->saddr_max = 0; + if (strlen(info->src_min) == 0) { + if (info->odev->ip_ptr) { + struct in_device *in_dev = info->odev->ip_ptr; + + if (in_dev->ifa_list) { + info->saddr_min = in_dev->ifa_list->ifa_address; + info->saddr_max = info->saddr_min; + } + } + } + else { + info->saddr_min = in_aton(info->src_min); + info->saddr_max = in_aton(info->src_max); + } - if(!set) { - - /* - * Use linklevel address if unconfigured. - * - * use ipv6_get_lladdr if/when it's get exported - */ + info->daddr_min = in_aton(info->dst_min); + info->daddr_max = in_aton(info->dst_max); + /* Initialize current values. */ + info->cur_dst_mac_offset = 0; + info->cur_src_mac_offset = 0; + info->cur_saddr = info->saddr_min; + info->cur_daddr = info->daddr_min; + info->cur_udp_dst = info->udp_dst_min; + info->cur_udp_src = info->udp_src_min; +} + +/* delay_ns is in nano-seconds */ +static void pg_nanodelay(int delay_ns, struct pktgen_interface_info* info, + struct pktgen_thread_info* pg_thread) +{ + u64 idle_start = getRelativeCurNs(); + u64 last_time; + u64 itmp = idle_start; + info->nanodelays++; + info->accum_delay_ns += delay_ns; + while (info->accum_delay_ns > PG_MAX_ACCUM_DELAY_NS) { + info->sleeps++; + pg_thread->sleeping = 1; + interruptible_sleep_on_timeout(&(pg_thread->queue), 1); + pg_thread->sleeping = 0; + /* will wake after one tick */ + last_time = itmp; + itmp = getRelativeCurNs(); + info->accum_delay_ns -= (itmp - last_time); + info->idle_acc += (itmp - last_time); - read_lock(&addrconf_lock); - if ((idev = __in6_dev_get(pkt_dev->odev)) != NULL) { - struct inet6_ifaddr *ifp; - - read_lock_bh(&idev->lock); - for (ifp=idev->addr_list; ifp; ifp=ifp->if_next) { - if (ifp->scope == IFA_LINK && !(ifp->flags&IFA_F_TENTATIVE)) { - ipv6_addr_copy(&pkt_dev->cur_in6_saddr, &ifp->addr); - err = 0; - break; - } - } - read_unlock_bh(&idev->lock); - } - read_unlock(&addrconf_lock); - if(err) printk("pktgen: ERROR: IPv6 link address not availble.\n"); - } -#endif - } - else { - pkt_dev->saddr_min = 0; - pkt_dev->saddr_max = 0; - if (strlen(pkt_dev->src_min) == 0) { - - struct in_device *in_dev; - - rcu_read_lock(); - in_dev = __in_dev_get(pkt_dev->odev); - if (in_dev) { - if (in_dev->ifa_list) { - pkt_dev->saddr_min = in_dev->ifa_list->ifa_address; - pkt_dev->saddr_max = pkt_dev->saddr_min; - } - in_dev_put(in_dev); - } - rcu_read_unlock(); - } - else { - pkt_dev->saddr_min = in_aton(pkt_dev->src_min); - pkt_dev->saddr_max = in_aton(pkt_dev->src_max); + if (!info->do_run_run) { + break; } + }/* while */ +}//pg_nanodelay - pkt_dev->daddr_min = in_aton(pkt_dev->dst_min); - pkt_dev->daddr_max = in_aton(pkt_dev->dst_max); - } - /* Initialize current values. */ - pkt_dev->cur_dst_mac_offset = 0; - pkt_dev->cur_src_mac_offset = 0; - pkt_dev->cur_saddr = pkt_dev->saddr_min; - pkt_dev->cur_daddr = pkt_dev->daddr_min; - pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min; - pkt_dev->cur_udp_src = pkt_dev->udp_src_min; - pkt_dev->nflows = 0; -} -static void spin(struct pktgen_dev *pkt_dev, __u64 spin_until_us) +/* Returns: cycles per micro-second */ +static int calc_mhz(void) { - __u64 start; - __u64 now; + struct timeval start, stop; + u64 start_s; + u64 t1, t2; + u32 elapsed; + u32 clock_time = 0; + + do_gettimeofday(&start); + start_s = get_cycles(); + /* Spin for 50,000,000 cycles */ + do { + barrier(); + elapsed = (u32)(get_cycles() - start_s); + if (elapsed == 0) + return 0; + } while (elapsed < 50000000); + do_gettimeofday(&stop); + + t1 = tv_to_us(&start); + t2 = tv_to_us(&stop); + + clock_time = (u32)(t2 - t1); + if (clock_time == 0) { + printk("pktgen: ERROR: clock_time was zero..things may not work right, t1: %u t2: %u ...\n", + (u32)(t1), (u32)(t2)); + return 0x7FFFFFFF; + } + return elapsed / clock_time; +} - start = now = getCurUs(); - printk(KERN_INFO "sleeping for %d\n", (int)(spin_until_us - now)); - while (now < spin_until_us) { - /* TODO: optimise sleeping behavior */ - if (spin_until_us - now > (1000000/HZ)+1) { - current->state = TASK_INTERRUPTIBLE; - schedule_timeout(1); - } else if (spin_until_us - now > 100) { - do_softirq(); - if (!pkt_dev->running) - return; - if (need_resched()) - schedule(); - } +/* Calibrate cycles per micro-second */ +static void cycles_calibrate(void) +{ + int i; - now = getCurUs(); + for (i = 0; i < 3; i++) { + u32 res = calc_mhz(); + if (res > pg_cycles_per_us) + pg_cycles_per_us = res; } - pkt_dev->idle_acc += now - start; + /* Set these up too, only need to calculate these once. */ + pg_cycles_per_ns = pg_cycles_per_us / 1000; + if (pg_cycles_per_ns == 0) { + pg_cycles_per_ns = 1; + } + pg_cycles_per_ms = pg_cycles_per_us * 1000; + + printk("pktgen: cycles_calibrate, cycles_per_ns: %d per_us: %d per_ms: %d\n", + pg_cycles_per_ns, pg_cycles_per_us, pg_cycles_per_ms); } /* Increment/randomize headers according to flags and current values * for IP src/dest, UDP src/dst port, MAC-Addr src/dst */ -static void mod_cur_headers(struct pktgen_dev *pkt_dev) { +static void mod_cur_headers(struct pktgen_interface_info* info) { __u32 imn; __u32 imx; - int flow = 0; - - if(pkt_dev->cflows) { - flow = pktgen_random() % pkt_dev->cflows; - - if (pkt_dev->flows[flow].count > pkt_dev->lflow) - pkt_dev->flows[flow].count = 0; - } - - + /* Deal with source MAC */ - if (pkt_dev->src_mac_count > 1) { + if (info->src_mac_count > 1) { __u32 mc; __u32 tmp; - - if (pkt_dev->flags & F_MACSRC_RND) - mc = pktgen_random() % (pkt_dev->src_mac_count); + if (info->flags & F_MACSRC_RND) { + mc = net_random() % (info->src_mac_count); + } else { - mc = pkt_dev->cur_src_mac_offset++; - if (pkt_dev->cur_src_mac_offset > pkt_dev->src_mac_count) - pkt_dev->cur_src_mac_offset = 0; + mc = info->cur_src_mac_offset++; + if (info->cur_src_mac_offset > info->src_mac_count) { + info->cur_src_mac_offset = 0; + } } - tmp = pkt_dev->src_mac[5] + (mc & 0xFF); - pkt_dev->hh[11] = tmp; - tmp = (pkt_dev->src_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); - pkt_dev->hh[10] = tmp; - tmp = (pkt_dev->src_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); - pkt_dev->hh[9] = tmp; - tmp = (pkt_dev->src_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); - pkt_dev->hh[8] = tmp; - tmp = (pkt_dev->src_mac[1] + (tmp >> 8)); - pkt_dev->hh[7] = tmp; + tmp = info->src_mac[5] + (mc & 0xFF); + info->hh[11] = tmp; + tmp = (info->src_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); + info->hh[10] = tmp; + tmp = (info->src_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); + info->hh[9] = tmp; + tmp = (info->src_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); + info->hh[8] = tmp; + tmp = (info->src_mac[1] + (tmp >> 8)); + info->hh[7] = tmp; } /* Deal with Destination MAC */ - if (pkt_dev->dst_mac_count > 1) { + if (info->dst_mac_count > 1) { __u32 mc; __u32 tmp; - - if (pkt_dev->flags & F_MACDST_RND) - mc = pktgen_random() % (pkt_dev->dst_mac_count); - + if (info->flags & F_MACDST_RND) { + mc = net_random() % (info->dst_mac_count); + } else { - mc = pkt_dev->cur_dst_mac_offset++; - if (pkt_dev->cur_dst_mac_offset > pkt_dev->dst_mac_count) { - pkt_dev->cur_dst_mac_offset = 0; + mc = info->cur_dst_mac_offset++; + if (info->cur_dst_mac_offset > info->dst_mac_count) { + info->cur_dst_mac_offset = 0; } } - tmp = pkt_dev->dst_mac[5] + (mc & 0xFF); - pkt_dev->hh[5] = tmp; - tmp = (pkt_dev->dst_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); - pkt_dev->hh[4] = tmp; - tmp = (pkt_dev->dst_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); - pkt_dev->hh[3] = tmp; - tmp = (pkt_dev->dst_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); - pkt_dev->hh[2] = tmp; - tmp = (pkt_dev->dst_mac[1] + (tmp >> 8)); - pkt_dev->hh[1] = tmp; - } - - if (pkt_dev->udp_src_min < pkt_dev->udp_src_max) { - if (pkt_dev->flags & F_UDPSRC_RND) - pkt_dev->cur_udp_src = ((pktgen_random() % (pkt_dev->udp_src_max - pkt_dev->udp_src_min)) + pkt_dev->udp_src_min); - + tmp = info->dst_mac[5] + (mc & 0xFF); + info->hh[5] = tmp; + tmp = (info->dst_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); + info->hh[4] = tmp; + tmp = (info->dst_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); + info->hh[3] = tmp; + tmp = (info->dst_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); + info->hh[2] = tmp; + tmp = (info->dst_mac[1] + (tmp >> 8)); + info->hh[1] = tmp; + } + + if (info->udp_src_min < info->udp_src_max) { + if (info->flags & F_UDPSRC_RND) { + info->cur_udp_src = ((net_random() % (info->udp_src_max - info->udp_src_min)) + + info->udp_src_min); + } else { - pkt_dev->cur_udp_src++; - if (pkt_dev->cur_udp_src >= pkt_dev->udp_src_max) - pkt_dev->cur_udp_src = pkt_dev->udp_src_min; + info->cur_udp_src++; + if (info->cur_udp_src >= info->udp_src_max) { + info->cur_udp_src = info->udp_src_min; + } } } - if (pkt_dev->udp_dst_min < pkt_dev->udp_dst_max) { - if (pkt_dev->flags & F_UDPDST_RND) { - pkt_dev->cur_udp_dst = ((pktgen_random() % (pkt_dev->udp_dst_max - pkt_dev->udp_dst_min)) + pkt_dev->udp_dst_min); + if (info->udp_dst_min < info->udp_dst_max) { + if (info->flags & F_UDPDST_RND) { + info->cur_udp_dst = ((net_random() % (info->udp_dst_max - info->udp_dst_min)) + + info->udp_dst_min); } else { - pkt_dev->cur_udp_dst++; - if (pkt_dev->cur_udp_dst >= pkt_dev->udp_dst_max) - pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min; + info->cur_udp_dst++; + if (info->cur_udp_dst >= info->udp_dst_max) { + info->cur_udp_dst = info->udp_dst_min; + } } } - if (!(pkt_dev->flags & F_IPV6)) { - - if ((imn = ntohl(pkt_dev->saddr_min)) < (imx = ntohl(pkt_dev->saddr_max))) { - __u32 t; - if (pkt_dev->flags & F_IPSRC_RND) - t = ((pktgen_random() % (imx - imn)) + imn); - else { - t = ntohl(pkt_dev->cur_saddr); - t++; - if (t > imx) { - t = imn; - } - } - pkt_dev->cur_saddr = htonl(t); - } - - if (pkt_dev->cflows && pkt_dev->flows[flow].count != 0) { - pkt_dev->cur_daddr = pkt_dev->flows[flow].cur_daddr; - } else { - - if ((imn = ntohl(pkt_dev->daddr_min)) < (imx = ntohl(pkt_dev->daddr_max))) { - __u32 t; - if (pkt_dev->flags & F_IPDST_RND) { - - t = ((pktgen_random() % (imx - imn)) + imn); - t = htonl(t); - - while( LOOPBACK(t) || MULTICAST(t) || BADCLASS(t) || ZERONET(t) || LOCAL_MCAST(t) ) { - t = ((pktgen_random() % (imx - imn)) + imn); - t = htonl(t); - } - pkt_dev->cur_daddr = t; - } - - else { - t = ntohl(pkt_dev->cur_daddr); - t++; - if (t > imx) { - t = imn; - } - pkt_dev->cur_daddr = htonl(t); - } - } - if(pkt_dev->cflows) { - pkt_dev->flows[flow].cur_daddr = pkt_dev->cur_daddr; - pkt_dev->nflows++; - } - } - } - else /* IPV6 * */ - { - if(pkt_dev->min_in6_daddr.s6_addr32[0] == 0 && - pkt_dev->min_in6_daddr.s6_addr32[1] == 0 && - pkt_dev->min_in6_daddr.s6_addr32[2] == 0 && - pkt_dev->min_in6_daddr.s6_addr32[3] == 0); - else { - int i; - - /* Only random destinations yet */ - - for(i=0; i < 4; i++) { - pkt_dev->cur_in6_daddr.s6_addr32[i] = - ((pktgen_random() | - pkt_dev->min_in6_daddr.s6_addr32[i]) & - pkt_dev->max_in6_daddr.s6_addr32[i]); - } - } - } + if ((imn = ntohl(info->saddr_min)) < (imx = ntohl(info->saddr_max))) { + __u32 t; + if (info->flags & F_IPSRC_RND) { + t = ((net_random() % (imx - imn)) + imn); + } + else { + t = ntohl(info->cur_saddr); + t++; + if (t > imx) { + t = imn; + } + } + info->cur_saddr = htonl(t); + } - if (pkt_dev->min_pkt_size < pkt_dev->max_pkt_size) { + if ((imn = ntohl(info->daddr_min)) < (imx = ntohl(info->daddr_max))) { __u32 t; - if (pkt_dev->flags & F_TXSIZE_RND) { - t = ((pktgen_random() % (pkt_dev->max_pkt_size - pkt_dev->min_pkt_size)) - + pkt_dev->min_pkt_size); + if (info->flags & F_IPDST_RND) { + t = ((net_random() % (imx - imn)) + imn); } else { - t = pkt_dev->cur_pkt_size + 1; - if (t > pkt_dev->max_pkt_size) - t = pkt_dev->min_pkt_size; + t = ntohl(info->cur_daddr); + t++; + if (t > imx) { + t = imn; + } } - pkt_dev->cur_pkt_size = t; + info->cur_daddr = htonl(t); } - pkt_dev->flows[flow].count++; -} + /* dhetheri - Make MAC address = 00:00:IP address */ + if (info->flags & F_IPMAC) { + __u32 tmp; + __u32 t; + + /* SRC MAC = 00:00:IP address */ + t = ntohl(info->cur_saddr); + tmp = info->src_mac[5] + (t & 0xFF); + info->hh[11] = tmp; + tmp = (info->src_mac[4] + ((t >> 8) & 0xFF) + (tmp >> 8)); + info->hh[10] = tmp; + tmp = (info->src_mac[3] + ((t >> 16) & 0xFF) + (tmp >> 8)); + info->hh[9] = tmp; + tmp = (info->src_mac[2] + ((t >> 24) & 0xFF) + (tmp >> 8)); + info->hh[8] = tmp; + tmp = (info->src_mac[1] + (tmp >> 8)); + info->hh[7] = tmp; + + info->cur_saddr = htonl(t); + + /* DST MAC = 00:00:IP address */ + t = ntohl(info->cur_daddr); + + tmp = info->dst_mac[5] + (t & 0xFF); + info->hh[5] = tmp; + tmp = (info->dst_mac[4] + ((t >> 8) & 0xFF) + (tmp >> 8)); + info->hh[4] = tmp; + tmp = (info->dst_mac[3] + ((t >> 16) & 0xFF) + (tmp >> 8)); + info->hh[3] = tmp; + tmp = (info->dst_mac[2] + ((t >> 24) & 0xFF) + (tmp >> 8)); + info->hh[2] = tmp; + tmp = (info->dst_mac[1] + (tmp >> 8)); + info->hh[1] = tmp; + + info->cur_daddr = htonl(t); + } /* MAC = 00:00:IP address (dhetheri) */ +}/* mod_cur_headers */ -static struct sk_buff *fill_packet_ipv4(struct net_device *odev, - struct pktgen_dev *pkt_dev) + +static struct sk_buff *fill_packet(struct net_device *odev, struct pktgen_interface_info* info) { struct sk_buff *skb = NULL; __u8 *eth; @@ -1917,10 +1090,26 @@ int datalen, iplen; struct iphdr *iph; struct pktgen_hdr *pgh = NULL; - - skb = alloc_skb(pkt_dev->cur_pkt_size + 64 + 16, GFP_ATOMIC); + + /* dhetheri - Moved out of mod_cur_headers. */ + if (info->min_pkt_size < info->max_pkt_size) { + __u32 t; + if (info->flags & F_TXSIZE_RND) { + t = ((net_random() % (info->max_pkt_size - info->min_pkt_size)) + + info->min_pkt_size); + } + else { + t = info->cur_pkt_size + 1; + if (t > info->max_pkt_size) { + t = info->min_pkt_size; + } + } + info->cur_pkt_size = t; + } + + skb = alloc_skb(info->cur_pkt_size + 64 + 16, GFP_ATOMIC); if (!skb) { - sprintf(pkt_dev->result, "No memory"); + sprintf(info->result, "No memory"); return NULL; } @@ -1934,17 +1123,17 @@ /* Update any of the values, used when we're incrementing various * fields. */ - mod_cur_headers(pkt_dev); + mod_cur_headers(info); - memcpy(eth, pkt_dev->hh, 12); - *(u16*)ð[12] = __constant_htons(ETH_P_IP); - - datalen = pkt_dev->cur_pkt_size - 14 - 20 - 8; /* Eth + IPh + UDPh */ - if (datalen < sizeof(struct pktgen_hdr)) + memcpy(eth, info->hh, 14); + + datalen = info->cur_pkt_size - 14 - 20 - 8; /* Eth + IPh + UDPh */ + if (datalen < sizeof(struct pktgen_hdr)) { datalen = sizeof(struct pktgen_hdr); + } - udph->source = htons(pkt_dev->cur_udp_src); - udph->dest = htons(pkt_dev->cur_udp_dst); + udph->source = htons(info->cur_udp_src); + udph->dest = htons(info->cur_udp_dst); udph->len = htons(datalen + 8); /* DATA + udphdr */ udph->check = 0; /* No checksum */ @@ -1952,9 +1141,14 @@ iph->version = 4; iph->ttl = 32; iph->tos = 0; - iph->protocol = IPPROTO_UDP; /* UDP */ - iph->saddr = pkt_dev->cur_saddr; - iph->daddr = pkt_dev->cur_daddr; + if (info->prot) { /* dhetheri */ + iph->protocol = info->prot; /* dhetheri */ + } + else { + iph->protocol = IPPROTO_UDP; /* UDP */ + } + iph->saddr = info->cur_saddr; + iph->daddr = info->cur_daddr; iph->frag_off = 0; iplen = 20 + 8 + datalen; iph->tot_len = htons(iplen); @@ -1965,10 +1159,10 @@ skb->dev = odev; skb->pkt_type = PACKET_HOST; - if (pkt_dev->nfrags <= 0) + if (info->nfrags <= 0) { pgh = (struct pktgen_hdr *)skb_put(skb, datalen); - else { - int frags = pkt_dev->nfrags; + } else { + int frags = info->nfrags; int i; pgh = (struct pktgen_hdr*)(((char*)(udph)) + 8); @@ -2016,1121 +1210,2321 @@ } } - /* Stamp the time, and sequence number, convert them to network byte order */ + /* Stamp the time, and sequence number, convert them to network byte order */ + if (pgh) { + pgh->pgh_magic = __constant_htonl(PKTGEN_MAGIC); + do_gettimeofday(&(pgh->timestamp)); + pgh->timestamp.tv_usec = htonl(pgh->timestamp.tv_usec); + pgh->timestamp.tv_sec = htonl(pgh->timestamp.tv_sec); + pgh->seq_num = htonl(info->seq_num); + } + info->seq_num++; + + return skb; +} + + +static void record_latency(struct pktgen_interface_info* info, int latency) { + /* NOTE: Latency can be negative */ + int div = 100; + int diff; + int vl; + int i; + + info->pkts_rcvd_since_clear++; + + if (info->pkts_rcvd_since_clear < 100) { + div = info->pkts_rcvd; + if (info->pkts_rcvd_since_clear == 1) { + info->avg_latency = latency; + } + } + + if ((div + 1) == 0) { + info->avg_latency = 0; + } + else { + info->avg_latency = ((info->avg_latency * div + latency) / (div + 1)); + } + + if (latency < info->min_latency) { + info->min_latency = latency; + } + if (latency > info->max_latency) { + info->max_latency = latency; + } + + /* Place the latency in the right 'bucket' */ + diff = (latency - info->min_latency); + for (i = 0; ilatency_bkts[i]++; + break; + } + } +}/* record latency */ + + +/* Returns < 0 if the skb is not a pktgen buffer. */ +int pktgen_receive(struct sk_buff* skb) { + /* int i; */ /* Debugging only */ + /* unsigned char* tmp; */ + + /* dhetheri */ + //printk("pktgen receive:\n"); + //tmp=(char *)(skb->data); + //for (i=0; i<90; i++) { + // printk("%02hx ", tmp[i]); + // if (((i+1) % 15) == 0) { + // printk("\n"); + // } + //} + //printk("\n"); + /* dhetheri */ + + /* See if we have a pktgen packet */ + if ((skb->len >= (20 + 8 + sizeof(struct pktgen_hdr))) && + (skb->protocol == __constant_htons(ETH_P_IP))) { + struct pktgen_hdr* pgh; + + /* It's IP, and long enough, lets check the magic number. + * TODO: This is a hack not always guaranteed to catch the right + * packets. + */ + + /* printk("Length & protocol passed, skb->data: %p, raw: %p\n", + skb->data, skb->h.raw); */ + + pgh = (struct pktgen_hdr*)(skb->data + 20 + 8); + + /* + tmp = (char*)(skb->data); + for (i = 0; i<90; i++) { + printk("%02hx ", tmp[i]); + if (((i + 1) % 15) == 0) { + printk("\n"); + } + } + printk("\n"); + */ + + if (pgh->pgh_magic == __constant_ntohl(PKTGEN_MAGIC)) { + struct net_device* dev = skb->dev; + struct pktgen_interface_info* info = find_pg_info(dev->ifindex); + + /* Got one! */ + /* TODO: Check UDP checksum ?? */ + __u32 seq = ntohl(pgh->seq_num); + + if (!info) { + return -1; + } + + info->pkts_rcvd++; + info->bytes_rcvd += ((skb->tail - skb->mac.raw) + 4); /* +4 for the checksum */ + + /* Check for out-of-sequence packets */ + if (info->last_seq_rcvd == seq) { + info->dup_rcvd++; + info->dup_since_incr++; + } + else { + __s64 rx; + __s64 tx; + struct timeval txtv; + if (!skb->stamp.tv_sec) { + do_gettimeofday(&skb->stamp); + } + rx = tv_to_us(&(skb->stamp)); + + txtv.tv_usec = ntohl(pgh->timestamp.tv_usec); + txtv.tv_sec = ntohl(pgh->timestamp.tv_sec); + tx = tv_to_us(&txtv); + record_latency(info, rx - tx); + + if ((info->last_seq_rcvd + 1) == seq) { + if ((info->peer_multiskb > 1) && + (info->peer_multiskb > (info->dup_since_incr + 1))) { + + info->seq_gap_rcvd += (info->peer_multiskb - + info->dup_since_incr - 1); + } + /* Great, in order...all is well */ + } + else if (info->last_seq_rcvd < seq) { + /* sequence gap, means we dropped a pkt most likely */ + if (info->peer_multiskb > 1) { + /* We dropped more than one sequence number's worth, + * and if we're using multiskb, then this is quite + * a few. This number still will not be exact, but + * it will be closer. + */ + info->seq_gap_rcvd += (((seq - info->last_seq_rcvd) * + info->peer_multiskb) - + info->dup_since_incr); + } + else { + info->seq_gap_rcvd += (seq - info->last_seq_rcvd - 1); + } + } + else { + info->ooo_rcvd++; /* out-of-order */ + } + + info->dup_since_incr = 0; + } + info->last_seq_rcvd = seq; + kfree_skb(skb); + if (debug > 1) { + printk("done with pktgen_receive, free'd pkt\n"); + } + return 0; + } + } + return -1; /* Let another protocol handle it, it's not for us! */ +}/* pktgen_receive */ + +static void pg_reset_latency_counters(struct pktgen_interface_info* info) { + int i; + info->avg_latency = 0; + info->min_latency = 0x7fffffff; /* largest integer */ + info->max_latency = 0x80000000; /* smallest integer */ + info->pkts_rcvd_since_clear = 0; + for (i = 0; ilatency_bkts[i] = 0; + } +} + +static void pg_clear_counters(struct pktgen_interface_info* info, int seq_too) { + info->idle_acc = 0; + info->sofar = 0; + info->tx_bytes = 0; + info->errors = 0; + info->ooo_rcvd = 0; + info->dup_rcvd = 0; + info->pkts_rcvd = 0; + info->bytes_rcvd = 0; + info->non_pg_pkts_rcvd = 0; + info->seq_gap_rcvd = 0; /* dropped */ + + /* Clear some transient state */ + info->accum_delay_ns = 0; + info->sleeps = 0; + info->nanodelays = 0; + + /* This is a bit of a hack, but it gets the dup counters + * in line so we don't have false alarms on dropped pkts. + */ + if (seq_too) { + info->dup_since_incr = info->peer_multiskb - 1; + info->seq_num = 1; + info->last_seq_rcvd = 0; + } + + pg_reset_latency_counters(info); +} + +/* Adds an interface to the thread. The interface will be in + * the stopped queue untill started. + */ +static int add_interface_to_thread(struct pktgen_thread_info* pg_thread, + struct pktgen_interface_info* info) { + int rv = 0; + /* grab lock & insert into the stopped list */ + pg_lock(pg_thread, __FUNCTION__); + + if (info->pg_thread) { + printk("pktgen: ERROR: Already assigned to a thread.\n"); + rv = -EBUSY; + goto out; + } + + info->next = pg_thread->stopped_if_infos; + pg_thread->stopped_if_infos = info; + info->pg_thread = pg_thread; + + out: + pg_unlock(pg_thread, __FUNCTION__); + return rv; +} + +/* Set up structure for sending pkts, clear counters, add to rcv hash, + * create initial packet, and move from the stopped to the running + * interface_info list + */ +static int pg_start_interface(struct pktgen_thread_info* pg_thread, + struct pktgen_interface_info* info) { + PG_DEBUG(printk("Entering pg_start_interface..\n")); + pg_setup_inject(info); + + if (!info->odev) { + return -1; + } + + PG_DEBUG(printk("About to clean counters..\n")); + pg_clear_counters(info, 1); + + info->do_run_run = 1; /* Cranke yeself! */ + + info->skb = NULL; + + info->started_at = getCurUs(); + + pg_lock(pg_thread, __FUNCTION__); + { + /* Remove from the stopped list */ + struct pktgen_interface_info* p = pg_thread->stopped_if_infos; + if (p == info) { + pg_thread->stopped_if_infos = p->next; + p->next = NULL; + } + else { + while (p) { + if (p->next == info) { + p->next = p->next->next; + info->next = NULL; + break; + } + p = p->next; + } + } + + info->next_tx_ns = 0; /* Transmit immediately */ + pg_thread->next_tx_ns = 0; + + /* Move to the front of the running list */ + info->next = pg_thread->running_if_infos; + pg_thread->running_if_infos = info; + pg_thread->running_if_sz++; + } + pg_unlock(pg_thread, __FUNCTION__); + PG_DEBUG(printk("Leaving pg_start_interface..\n")); + return 0; +}/* pg_start_interface */ + + +/* set stopped-at timer, remove from running list, do counters & statistics + * NOTE: We do not remove from the rcv hash. + */ +static int pg_stop_interface(struct pktgen_thread_info* pg_thread, + struct pktgen_interface_info* info) { + __u64 total_us; + if (!info->do_run_run) { + printk("pktgen interface: %s is already stopped\n", info->ifname); + return -EINVAL; + } + + info->stopped_at = getCurMs(); + info->do_run_run = 0; + + /* The main worker loop will place it onto the stopped list if needed, + * next time this interface is asked to be re-inserted into the + * list. + */ + + total_us = info->stopped_at - info->started_at; + + { + __u64 idle = pg_div(info->idle_acc, 1000); /* convert to us */ + char *p = info->result; + __u64 pps = divremdi3(info->sofar * 1000, pg_div(total_us, 1000), PG_DIV); + __u64 bps = pps * 8 * (info->cur_pkt_size + 4); /* take 32bit ethernet CRC into account */ + + p += sprintf(p, "OK: %llu(c%llu+d%llu) usec, %llu (%dbyte) %llupps %lluMb/sec (%llubps) errors: %llu", + total_us, total_us - idle, idle, + info->sofar, + info->cur_pkt_size + 4, /* Add 4 to account for the ethernet checksum */ + pps, + bps >> 20, bps, info->errors + ); + } + return 0; +}/* pg_stop_interface */ + + +/* Re-inserts 'last' into the pg_thread's list. Calling code should + * make sure that 'last' is not already in the list. + */ +static struct pktgen_interface_info* pg_resort_pginfos(struct pktgen_thread_info* pg_thread, + struct pktgen_interface_info* last, + int setup_cur_if) { + struct pktgen_interface_info* rv = NULL; + + pg_lock(pg_thread, __FUNCTION__); + { + struct pktgen_interface_info* p = pg_thread->running_if_infos; + + if (last) { + if (!last->do_run_run) { + /* If this guy was stopped while 'current', then + * we'll want to place him on the stopped list + * here. + */ + last->next = pg_thread->stopped_if_infos; + pg_thread->stopped_if_infos = last; + pg_thread->running_if_sz--; + } + else { + /* re-insert */ + if (!p) { + pg_thread->running_if_infos = last; + last->next = NULL; + } + else { + /* Another special case, check to see if we should go at the + * front of the queue. + */ + if (p->next_tx_ns > last->next_tx_ns) { + last->next = p; + pg_thread->running_if_infos = last; + } + else { + int inserted = 0; + while (p->next) { + if (p->next->next_tx_ns > last->next_tx_ns) { + /* Insert into the list */ + last->next = p->next; + p->next = last; + inserted = 1; + break; + } + p = p->next; + } + if (!inserted) { + /* place at the end */ + last->next = NULL; + p->next = last; + } + } + } + } + } + + /* List is re-sorted, so grab the first one to return */ + rv = pg_thread->running_if_infos; + if (rv) { + /* Pop him off of the list. We do this here because we already + * have the lock. Calling code just has to be aware of this + * feature. + */ + pg_thread->running_if_infos = rv->next; + } + } + + if (setup_cur_if) { + pg_thread->cur_if = rv; + } + + pg_unlock(pg_thread, __FUNCTION__); + return rv; +}/* pg_resort_pginfos */ + + +void pg_stop_all_ifs(struct pktgen_thread_info* pg_thread) { + struct pktgen_interface_info* next = NULL; + + pg_lock(pg_thread, __FUNCTION__); + if (pg_thread->cur_if) { + /* Move it onto the stopped list */ + pg_stop_interface(pg_thread, pg_thread->cur_if); + pg_thread->cur_if->next = pg_thread->stopped_if_infos; + pg_thread->stopped_if_infos = pg_thread->cur_if; + pg_thread->cur_if = NULL; + } + pg_unlock(pg_thread, __FUNCTION__); + + /* These have their own locking */ + next = pg_resort_pginfos(pg_thread, NULL, 0); + while (next) { + pg_stop_interface(pg_thread, next); + next = pg_resort_pginfos(pg_thread, NULL, 0); + } +}/* pg_stop_all_ifs */ + + +void pg_rem_all_ifs(struct pktgen_thread_info* pg_thread) { + struct pktgen_interface_info* next = NULL; + + /* Remove all interfaces, clean up memory */ + while ((next = pg_thread->stopped_if_infos)) { + int rv = pg_rem_interface_info(pg_thread, next); + if (rv >= 0) { + kfree(next); + } + else { + printk("ERROR: failed to rem_interface: %i\n", rv); + } + } +}/* pg_rem_all_ifs */ + + +void pg_rem_from_thread_list(struct pktgen_thread_info* pg_thread) { + /* Remove from the thread list */ + pg_lock_thread_list(__FUNCTION__); + { + struct pktgen_thread_info* tmp = pktgen_threads; + if (tmp == pg_thread) { + pktgen_threads = tmp->next; + } + else { + while (tmp) { + if (tmp->next == pg_thread) { + tmp->next = pg_thread->next; + pg_thread->next = NULL; + break; + } + tmp = tmp->next; + } + } + } + pg_unlock_thread_list(__FUNCTION__); +}/* pg_rem_from_thread_list */ + + +/* Main loop of the thread. Send pkts. + */ +void pg_thread_worker(struct pktgen_thread_info* pg_thread) { + struct net_device *odev = NULL; + __u64 idle_start = 0; + struct pktgen_interface_info* next = NULL; + u32 next_ipg = 0; + u64 now = 0; /* in nano-seconds */ + u32 tx_since_softirq = 0; + u32 queue_stopped = 0; + + /* setup the thread environment */ + init_pktgen_kthread(pg_thread, "kpktgend"); + + PG_DEBUG(printk("Starting up pktgen thread: %s\n", pg_thread->name)); + + /* an endless loop in which we are doing our work */ + while (! pg_thread->terminate) { + + if (signal_pending(current)) { + /* we received a request to terminate ourself */ + break; + } + + /* Re-sorts the list, inserting 'next' (which is really the last one + * we used). It pops the top one off of the queue and returns it. + * Calling code must make sure to re-insert the returned value + */ + next = pg_resort_pginfos(pg_thread, next, 1); + + /* Make sure the notify_queue_woken callback will act appropriately */ + if (next) { + pg_thread->next_tx_ns = next->next_tx_ns; + } + + if (queue_stopped > pg_thread->running_if_sz) { + /* All our devices are all fulled up, schedule and hope to run + * again soon. + */ + /* Take this opportunity to run the soft-irq */ + do_softirq(); + tx_since_softirq = 0; + + pg_thread->queues_stopped++; + pg_thread->sleeping = 1; + interruptible_sleep_on_timeout(&(pg_thread->queue), 1); + pg_thread->sleeping = 0; + queue_stopped = 0; + } + + if (next) { + + odev = next->odev; + + if (next->ipg || (next->accum_delay_ns > 0)) { + + now = getRelativeCurNs(); + if (now < next->next_tx_ns) { + next_ipg = (u32)(next->next_tx_ns - now); + + /* These will not actually busy-spin now. Will run as + * much as 1ms fast, and will sleep in 1ms units, assuming + * our tick is 1ms. + */ + pg_nanodelay(next_ipg, next, pg_thread); + if (!next->do_run_run) { + /* We were stopped while sleeping */ + continue; + } + } + + /* This is max IPG, this has special meaning of + * "never transmit" + */ + if (next->ipg == 0x7FFFFFFF) { + next->next_tx_ns = getRelativeCurNs() + next->ipg; + continue; + } + } + + if (need_resched()) { + idle_start = getRelativeCurNs(); + schedule(); + next->idle_acc += getRelativeCurNs() - idle_start; + } + + if (netif_queue_stopped(odev)) { + next->queue_stopped++; + queue_stopped++; + + if (!netif_running(odev)) { + pg_stop_interface(pg_thread, next); + } + + continue; /* Try the next interface */ + } + + if (next->last_ok || !next->skb) { + if ((++next->fp_tmp >= next->multiskb ) || (!next->skb)) { + /* build a new pkt */ + if (next->skb) { + kfree_skb(next->skb); + } + next->skb = fill_packet(odev, next); + if (next->skb == NULL) { + if (net_ratelimit()) { + printk(KERN_INFO "pktgen: Couldn't allocate skb in fill_packet.\n"); + } + schedule(); + next->fp_tmp--; /* back out increment, OOM */ + continue; + } + next->fp++; + next->fp_tmp = 0; /* reset counter */ + /* Not sure what good knowing nr_frags is... + next->nr_frags = skb_shinfo(skb)->nr_frags; + */ + } + atomic_inc(&(next->skb->users)); + } + + spin_lock_bh(&odev->xmit_lock); + if (!netif_queue_stopped(odev)) { + if (odev->hard_start_xmit(next->skb, odev)) { + if (net_ratelimit()) { + printk(KERN_INFO "pktgen: Hard xmit error\n"); + } + next->errors++; + next->last_ok = 0; + next->queue_stopped++; + queue_stopped++; + if (next->ntx_fudge < 10000) { + next->ntx_fudge = 10000; + } + else { + next->ntx_fudge = next->ntx_fudge << 1; + } + } + else { + queue_stopped = 0; /* reset this, we tx'd one successfully */ + next->last_ok = 1; + next->sofar++; + next->tx_bytes += (next->cur_pkt_size + 4); /* count csum */ + next->ntx_fudge = 0; + } + } + else { /* Re-try it next time */ + queue_stopped++; + next->queue_stopped++; + next->last_ok = 0; + if (next->ntx_fudge < 10000) { + next->ntx_fudge = 10000; + } + else { + next->ntx_fudge = next->ntx_fudge << 1; + } + } + spin_unlock_bh(&odev->xmit_lock); + + next->next_tx_ns = getRelativeCurNs() + next->ipg + next->ntx_fudge; + + if (++tx_since_softirq > pg_thread->max_before_softirq) { + do_softirq(); + tx_since_softirq = 0; + } + + /* If next->count is zero, then run forever */ + if ((next->count != 0) && (next->sofar >= next->count)) { + if (atomic_read(&(next->skb->users)) != 1) { + idle_start = getRelativeCurNs(); + while (atomic_read(&(next->skb->users)) != 1) { + if (signal_pending(current)) { + break; + } + schedule(); + } + next->idle_acc += getRelativeCurNs() - idle_start; + } + pg_stop_interface(pg_thread, next); + }/* if we're done with a particular interface. */ + + }/* if could find the next interface to send on. */ + else { + /* fall asleep for a bit */ + pg_thread->sleeping = 1; + interruptible_sleep_on_timeout(&(pg_thread->queue), HZ/10); + pg_thread->sleeping = 0; + } + }//while true + + /* here we go only in case of termination of the thread */ + + PG_DEBUG(printk("pgthread: %s stopping all Interfaces.\n", pg_thread->name)); + pg_stop_all_ifs(pg_thread); + + PG_DEBUG(printk("pgthread: %s removing all Interfaces.\n", pg_thread->name)); + pg_rem_all_ifs(pg_thread); + + pg_rem_from_thread_list(pg_thread); + + /* cleanup the thread, leave */ + PG_DEBUG(printk("pgthread: %s calling exit_pktgen_kthread.\n", pg_thread->name)); + exit_pktgen_kthread(pg_thread); +} + +/* private functions */ +static void kthread_launcher(void *data) { + struct pktgen_thread_info *kthread = data; + kernel_thread((int (*)(void *))kthread->function, (void *)kthread, 0); +} + +/* create a new kernel thread. Called by the creator. */ +void start_pktgen_kthread(struct pktgen_thread_info *kthread) { + + /* initialize the semaphore: + we start with the semaphore locked. The new kernel + thread will setup its stuff and unlock it. This + control flow (the one that creates the thread) blocks + in the down operation below until the thread has reached + the up() operation. + */ + init_MUTEX_LOCKED(&kthread->startstop_sem); + + /* store the function to be executed in the data passed to + the launcher */ + kthread->function = pg_thread_worker; + + /* create the new thread by running a task through keventd */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) + /* initialize the task queue structure */ + kthread->tq.sync = 0; + INIT_LIST_HEAD(&kthread->tq.list); + kthread->tq.routine = kthread_launcher; + kthread->tq.data = kthread; + + /* and schedule it for execution */ + schedule_task(&kthread->tq); +#else + INIT_WORK(&(kthread->wq), kthread_launcher, kthread); + /* and schedule it for execution */ + schedule_work(&kthread->wq); + +#endif + + /* wait till it has reached the setup_thread routine */ + down(&kthread->startstop_sem); +} + +/* stop a kernel thread. Called by the removing instance */ +static void stop_pktgen_kthread(struct pktgen_thread_info *kthread) { + printk("pgthread: %s stop_pktgen_kthread.\n", kthread->name); + + if (kthread->thread == NULL) { + printk("stop_kthread: killing non existing thread!\n"); + return; + } + + /* Stop each interface */ + pg_lock(kthread, __FUNCTION__); + { + struct pktgen_interface_info* tmp = kthread->running_if_infos; + while (tmp) { + tmp->do_run_run = 0; + tmp->next_tx_ns = 0; + tmp = tmp->next; + } + if (kthread->cur_if) { + kthread->cur_if->do_run_run = 0; + kthread->cur_if->ne