diff --git a/net/core/pktgen.c b/net/core/pktgen.c index 6a7320b..fcbe3c6 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -156,44 +156,28 @@ #include #include #include /* do_div */ #include +#include /* sched_clock() */ +#include "pktgen.h" + +#define USE_NQW_CALLBACK + +#ifdef USE_NQW_CALLBACK +#include + +#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE) +#include +#include "../macvlan/macvlan.h" +#endif +#endif + +#define VERSION "pktgen v2.68: Packet Generator for packet performance testing.\n" + +#define PG_DEBUG(a) a +//#define PG_DEBUG(a) -#define VERSION "pktgen v2.67: Packet Generator for packet performance testing.\n" - -/* #define PG_DEBUG(a) a */ -#define PG_DEBUG(a) - -/* The buckets are exponential in 'width' */ -#define LAT_BUCKETS_MAX 32 -#define IP_NAME_SZ 32 -#define MAX_MPLS_LABELS 16 /* This is the max label stack depth */ -#define MPLS_STACK_BOTTOM __constant_htonl(0x00000100) - -/* Device flag bits */ -#define F_IPSRC_RND (1<<0) /* IP-Src Random */ -#define F_IPDST_RND (1<<1) /* IP-Dst Random */ -#define F_UDPSRC_RND (1<<2) /* UDP-Src Random */ -#define F_UDPDST_RND (1<<3) /* UDP-Dst Random */ -#define F_MACSRC_RND (1<<4) /* MAC-Src Random */ -#define F_MACDST_RND (1<<5) /* MAC-Dst Random */ -#define F_TXSIZE_RND (1<<6) /* Transmit size is random */ -#define F_IPV6 (1<<7) /* Interface in IPV6 Mode */ -#define F_MPLS_RND (1<<8) /* Random MPLS labels */ - -/* Thread control flag bits */ -#define T_TERMINATE (1<<0) -#define T_STOP (1<<1) /* Stop run */ -#define T_RUN (1<<2) /* Start run */ -#define T_REMDEVALL (1<<3) /* Remove all devs */ -#define T_REMDEV (1<<4) /* Remove one dev */ /* If lock -- can be removed after some work */ -#define if_lock(t) spin_lock(&(t->if_lock)); -#define if_unlock(t) spin_unlock(&(t->if_lock)); -/* Used to help with determining the pkts on receive */ -#define PKTGEN_MAGIC 0xbe9be955 -#define PG_PROC_DIR "pktgen" -#define PGCTRL "pgctrl" static struct proc_dir_entry *pg_proc_dir = NULL; #define MAX_CFLOWS 65536 @@ -203,158 +187,43 @@ struct flow_state { int count; }; -struct pktgen_dev { - - /* - * Try to keep frequent/infrequent used vars. separated. - */ - - char ifname[IFNAMSIZ]; - char result[512]; - - struct pktgen_thread *pg_thread; /* the owner */ - struct list_head list; /* Used for chaining in the thread's run-queue */ - - int running; /* if this changes to false, the test will stop */ - - /* If min != max, then we will either do a linear iteration, or - * we will do a random selection from within the range. - */ - __u32 flags; - int removal_mark; /* non-zero => the device is marked for - * removal by worker thread */ - - int min_pkt_size; /* = ETH_ZLEN; */ - int max_pkt_size; /* = ETH_ZLEN; */ - int nfrags; - __u32 delay_us; /* Default delay */ - __u32 delay_ns; - __u64 count; /* Default No packets to send */ - __u64 sofar; /* How many pkts we've sent so far */ - __u64 tx_bytes; /* How many bytes we've transmitted */ - __u64 errors; /* Errors when trying to transmit, pkts will be re-sent */ - - /* runtime counters relating to clone_skb */ - __u64 next_tx_us; /* timestamp of when to tx next */ - __u32 next_tx_ns; - - __u64 allocated_skbs; - __u32 clone_count; - int last_ok; /* Was last skb sent? - * Or a failed transmit of some sort? This will keep - * sequence numbers in order, for example. - */ - __u64 started_at; /* micro-seconds */ - __u64 stopped_at; /* micro-seconds */ - __u64 idle_acc; /* micro-seconds */ - __u32 seq_num; - - int clone_skb; /* Use multiple SKBs during packet gen. If this number - * is greater than 1, then that many copies of the same - * packet will be sent before a new packet is allocated. - * For instance, if you want to send 1024 identical packets - * before creating a new packet, set clone_skb to 1024. - */ - - char dst_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ - char dst_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ - char src_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ - char src_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ - - struct in6_addr in6_saddr; - struct in6_addr in6_daddr; - struct in6_addr cur_in6_daddr; - struct in6_addr cur_in6_saddr; - /* For ranges */ - struct in6_addr min_in6_daddr; - struct in6_addr max_in6_daddr; - struct in6_addr min_in6_saddr; - struct in6_addr max_in6_saddr; - - /* If we're doing ranges, random or incremental, then this - * defines the min/max for those ranges. - */ - __u32 saddr_min; /* inclusive, source IP address */ - __u32 saddr_max; /* exclusive, source IP address */ - __u32 daddr_min; /* inclusive, dest IP address */ - __u32 daddr_max; /* exclusive, dest IP address */ - - __u16 udp_src_min; /* inclusive, source UDP port */ - __u16 udp_src_max; /* exclusive, source UDP port */ - __u16 udp_dst_min; /* inclusive, dest UDP port */ - __u16 udp_dst_max; /* exclusive, dest UDP port */ - - /* MPLS */ - unsigned nr_labels; /* Depth of stack, 0 = no MPLS */ - __be32 labels[MAX_MPLS_LABELS]; - - __u32 src_mac_count; /* How many MACs to iterate through */ - __u32 dst_mac_count; /* How many MACs to iterate through */ - - unsigned char dst_mac[ETH_ALEN]; - unsigned char src_mac[ETH_ALEN]; +static char version[] __initdata = VERSION; - __u32 cur_dst_mac_offset; - __u32 cur_src_mac_offset; - __u32 cur_saddr; - __u32 cur_daddr; - __u16 cur_udp_dst; - __u16 cur_udp_src; - __u32 cur_pkt_size; - - __u8 hh[14]; - /* = { - 0x00, 0x80, 0xC8, 0x79, 0xB3, 0xCB, - - We fill in SRC address later - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x00 - }; - */ - __u16 pad; /* pad out the hh struct to an even 16 bytes */ - struct sk_buff *skb; /* skb we are to transmit next, mainly used for when we - * are transmitting the same one multiple times - */ - struct net_device *odev; /* The out-going device. Note that the device should - * have it's pg_info pointer pointing back to this - * device. This will be set when the user specifies - * the out-going device name (not when the inject is - * started as it used to do.) - */ - struct flow_state *flows; - unsigned cflows; /* Concurrent flows (config) */ - unsigned lflow; /* Flow length (config) */ - unsigned nflows; /* accumulated flows (stats) */ -}; +#define REMOVE 1 +#define FIND 0 -struct pktgen_hdr { - __u32 pgh_magic; - __u32 seq_num; - __u32 tv_sec; - __u32 tv_usec; -}; +static struct pktgen_dev *__pktgen_NN_threads(const char *ifname, int remove); +static int pktgen_remove_device(struct pktgen_thread *t, struct pktgen_dev *i); +static int pktgen_add_device(struct pktgen_thread *t, const char *ifname); +static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t, + const char *ifname); +static int pktgen_device_event(struct notifier_block *, unsigned long, void *); +static void pktgen_run_all_threads(int background); +static void pktgen_stop_all_threads_ifs(void); +static int pktgen_stop_device(struct pktgen_dev *pkt_dev); +static void pktgen_stop(struct pktgen_thread *t); +static void pktgen_clear_counters(struct pktgen_dev *pkt_dev, int seq_too); +static int pktgen_mark_device(const char *ifname); +static unsigned int scan_ip6(const char *s, char ip[16]); +static unsigned int fmt_ip6(char *s, const char ip[16]); +static void clear_nqw_hook(struct pktgen_thread* t, struct net_device* dev); +static int set_nqw_hook(struct pktgen_thread* t, struct net_device* dev, int gfp); -struct pktgen_thread { - spinlock_t if_lock; - struct list_head if_list; /* All device here */ - struct list_head th_list; - int removed; - char name[32]; - char result[512]; - u32 max_before_softirq; /* We'll call do_softirq to prevent starvation. */ +/* Module parameters, defaults. */ +static int pg_count_d = 1000; /* 1000 pkts by default */ +static int pg_delay_d = 0x7FFFFFFF; /* Don't run until someone sets a different delay. */ - /* Field for thread to receive "posted" events terminate, stop ifs etc. */ +static int pg_clone_skb_d; +static int debug; - u32 control; - int pid; - int cpu; +static DEFINE_MUTEX(pktgen_thread_lock); +static LIST_HEAD(pktgen_threads); - wait_queue_head_t queue; +static struct notifier_block pktgen_notifier_block = { + .notifier_call = pktgen_device_event, }; -#define REMOVE 1 -#define FIND 0 /* This code works around the fact that do_div cannot handle two 64-bit numbers, and regular 64-bit division doesn't work on x86 kernels. @@ -476,36 +345,17 @@ static inline __u64 tv_diff(const struct return tv_to_us(a) - tv_to_us(b); } -/* old include end */ - -static char version[] __initdata = VERSION; - -static int pktgen_remove_device(struct pktgen_thread *t, struct pktgen_dev *i); -static int pktgen_add_device(struct pktgen_thread *t, const char *ifname); -static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t, - const char *ifname); -static int pktgen_device_event(struct notifier_block *, unsigned long, void *); -static void pktgen_run_all_threads(void); -static void pktgen_stop_all_threads_ifs(void); -static int pktgen_stop_device(struct pktgen_dev *pkt_dev); -static void pktgen_stop(struct pktgen_thread *t); -static void pktgen_clear_counters(struct pktgen_dev *pkt_dev); -static int pktgen_mark_device(const char *ifname); -static unsigned int scan_ip6(const char *s, char ip[16]); -static unsigned int fmt_ip6(char *s, const char ip[16]); - -/* Module parameters, defaults. */ -static int pg_count_d = 1000; /* 1000 pkts by default */ -static int pg_delay_d; -static int pg_clone_skb_d; -static int debug; +/* Since the machine booted. */ +static inline __u64 getRelativeCurUs(void) { + return pg_div(sched_clock(), 1000); +} -static DEFINE_MUTEX(pktgen_thread_lock); -static LIST_HEAD(pktgen_threads); +/* Since the machine booted. */ +static inline __u64 getRelativeCurNs(void) { + return sched_clock(); +} -static struct notifier_block pktgen_notifier_block = { - .notifier_call = pktgen_device_event, -}; +/* old include end */ /* * /proc handling functions @@ -542,10 +392,14 @@ static ssize_t pgctrl_write(struct file pktgen_stop_all_threads_ifs(); else if (!strcmp(data, "start")) - pktgen_run_all_threads(); + pktgen_run_all_threads(0); + + /* Run in the background. */ + else if (!strcmp(data, "bg_start")) + pktgen_run_all_threads(1); else - printk("pktgen: Unknown command: %s\n", data); + printk("pktgen: Unknown command: \"%s\"\n", data); err = count; @@ -558,6 +412,137 @@ static int pgctrl_open(struct inode *ino return single_open(file, pgctrl_show, PDE(inode)->data); } +static int pg_populate_report(struct pktgen_dev_report* rpt, struct pktgen_dev* pkt_dev) { + int i; + + memset(rpt, 0, sizeof(*rpt)); + rpt->api_version = 1; + rpt->flags = pkt_dev->flags; + strncpy(rpt->thread_name, pkt_dev->pg_thread->name, 32); + strncpy(rpt->interface_name, pkt_dev->ifname, 32); + rpt->min_pkt_size = pkt_dev->min_pkt_size; + rpt->max_pkt_size = pkt_dev->max_pkt_size; + rpt->clone_skb = pkt_dev->clone_skb; + rpt->peer_clone_skb = pkt_dev->peer_clone_skb; + rpt->nfrags = pkt_dev->nfrags; + + strncpy(rpt->dst_min, pkt_dev->dst_min, IP_NAME_SZ); + strncpy(rpt->dst_max, pkt_dev->dst_max, IP_NAME_SZ); + strncpy(rpt->src_min, pkt_dev->src_min, IP_NAME_SZ); + strncpy(rpt->src_max, pkt_dev->src_max, IP_NAME_SZ); + + memcpy(&rpt->in6_saddr, &pkt_dev->in6_saddr, sizeof(struct in6_addr)); + memcpy(&rpt->in6_daddr, &pkt_dev->in6_daddr, sizeof(struct in6_addr)); + + /* For ranges */ + memcpy(&rpt->min_in6_daddr, &pkt_dev->min_in6_daddr, sizeof(struct in6_addr)); + memcpy(&rpt->max_in6_daddr, &pkt_dev->max_in6_daddr, sizeof(struct in6_addr)); + memcpy(&rpt->min_in6_saddr, &pkt_dev->min_in6_saddr, sizeof(struct in6_addr)); + memcpy(&rpt->max_in6_saddr, &pkt_dev->max_in6_saddr, sizeof(struct in6_addr)); + + /* If we're doing ranges, random or incremental, then this + * defines the min/max for those ranges. + */ + rpt->saddr_min = pkt_dev->saddr_min; + rpt->saddr_max = pkt_dev->saddr_max; + rpt->daddr_min = pkt_dev->daddr_min; + rpt->daddr_max = pkt_dev->daddr_max; + + rpt->udp_src_min = pkt_dev->udp_src_min; + rpt->udp_src_max = pkt_dev->udp_src_max; + rpt->udp_dst_min = pkt_dev->udp_dst_min; + rpt->udp_dst_max = pkt_dev->udp_dst_max; + + /* MPLS */ + rpt->nr_labels = pkt_dev->nr_labels; /* Depth of stack, 0 = no MPLS */ + for (i = 0; ilabels[i] = pkt_dev->labels[i]; + } + + rpt->src_mac_count = pkt_dev->src_mac_count; + rpt->dst_mac_count = pkt_dev->dst_mac_count; + + memcpy(&rpt->dst_mac, &pkt_dev->dst_mac, ETH_ALEN); + memcpy(&rpt->src_mac, &pkt_dev->src_mac, ETH_ALEN); + + rpt->nflows = pkt_dev->nflows; + rpt->cflows = pkt_dev->cflows; + rpt->lflow = pkt_dev->lflow; + + rpt->delay_ns = pkt_dev->delay_ns; + rpt->count = pkt_dev->count; /* Default No packets to send */ + rpt->sofar = pkt_dev->sofar; /* How many pkts we've sent so far */ + rpt->tx_bytes = pkt_dev->tx_bytes; /* How many bytes we've transmitted */ + rpt->errors = pkt_dev->errors; /* Errors when trying to transmit, pkts will be re-sent */ + + /* Fields relating to receiving pkts */ + rpt->avg_latency = pkt_dev->avg_latency; /* in micro-seconds */ + rpt->min_latency = pkt_dev->min_latency; + rpt->max_latency = pkt_dev->max_latency; + for (i = 0; ilatency_bkts[i] = pkt_dev->latency_bkts[i]; + } + rpt->pkts_rcvd_since_clear = pkt_dev->pkts_rcvd_since_clear; + + rpt->ooo_rcvd = pkt_dev->ooo_rcvd; + rpt->pkts_rcvd = pkt_dev->pkts_rcvd; + rpt->dup_rcvd = pkt_dev->dup_rcvd; + rpt->bytes_rcvd = pkt_dev->bytes_rcvd; + rpt->seq_gap_rcvd = pkt_dev->seq_gap_rcvd; + rpt->non_pg_pkts_rcvd = pkt_dev->non_pg_pkts_rcvd; + return 0; +}; /* populate report */ + + +int pktgen_proc_ioctl(struct inode* inode, struct file* file, unsigned int cmd, + unsigned long arg) { + int err = 0; + struct pktgen_ioctl_info args; + struct pktgen_dev* pkt_dev = NULL; + + if (copy_from_user(&args, (void*)arg, sizeof(args))) { + return -EFAULT; + } + + /* Null terminate the names */ + args.thread_name[31] = 0; + args.interface_name[31] = 0; + + /* printk("pktgen: thread_name: %s interface_name: %s\n", + * args.thread_name, args.interface_name); + */ + + switch (cmd) { + case GET_PKTGEN_INTERFACE_INFO: { + mutex_lock(&pktgen_thread_lock); + pkt_dev = __pktgen_NN_threads(args.interface_name, FIND); + if (pkt_dev) { + pg_populate_report(&(args.report), pkt_dev); + if (copy_to_user((void*)(arg), &args, sizeof(args))) { + printk("ERROR: pktgen: copy_to_user failed.\n"); + err = -EFAULT; + } + else { + err = 0; + } + } + else { + printk("ERROR: pktgen: Could not find interface -:%s:-\n", + args.interface_name); + err = -ENODEV; + } + mutex_unlock(&pktgen_thread_lock); + break; + } + default: + printk("%s: Unknown pktgen IOCTL: %x \n", __FUNCTION__, + cmd); + return -EINVAL; + } + + return err; +}/* pktgen_proc_ioctl */ + static struct file_operations pktgen_fops = { .owner = THIS_MODULE, .open = pgctrl_open, @@ -565,6 +550,7 @@ static struct file_operations pktgen_fop .llseek = seq_lseek, .write = pgctrl_write, .release = single_release, + .ioctl = pktgen_proc_ioctl, }; static int pktgen_if_show(struct seq_file *seq, void *v) @@ -581,10 +567,11 @@ static int pktgen_if_show(struct seq_fil pkt_dev->max_pkt_size); seq_printf(seq, - " frags: %d delay: %u clone_skb: %d ifname: %s\n", + " frags: %d delay: %lluns clone_skb: %d peer_clone_skb: %d ifname: %s\n", pkt_dev->nfrags, - 1000 * pkt_dev->delay_us + pkt_dev->delay_ns, - pkt_dev->clone_skb, pkt_dev->ifname); + (unsigned long long)pkt_dev->delay_ns, + pkt_dev->clone_skb, pkt_dev->peer_clone_skb, + pkt_dev->ifname); seq_printf(seq, " flows: %u flowlen: %u\n", pkt_dev->cflows, pkt_dev->lflow); @@ -681,11 +668,29 @@ static int pktgen_if_show(struct seq_fil stopped = now; /* not really stopped, more like last-running-at */ seq_printf(seq, - "Current:\n pkts-sofar: %llu errors: %llu\n started: %lluus stopped: %lluus idle: %lluus\n", + "Current:\n tx-pkts: %llu tx-errors: %llu tx-bytes: %llu\n", (unsigned long long)pkt_dev->sofar, - (unsigned long long)pkt_dev->errors, (unsigned long long)sa, + (unsigned long long)pkt_dev->errors, + (unsigned long long)pkt_dev->tx_bytes); + seq_printf(seq, + " rx-pkts: %llu rx-bytes: %llu\n", + (unsigned long long)pkt_dev->pkts_rcvd, + (unsigned long long)pkt_dev->bytes_rcvd); + + seq_printf(seq, + " blocked: %s next-tx-ns: %llu (%lli) started: %lluus stopped: %lluus idle: %lluns\n", + pkt_dev->tx_blocked ? "TRUE" : "false", + (unsigned long long)pkt_dev->next_tx_ns, + (long long)(pkt_dev->next_tx_ns - getRelativeCurNs()), + (unsigned long long)sa, (unsigned long long)stopped, - (unsigned long long)pkt_dev->idle_acc); + (unsigned long long)pkt_dev->idle_acc_ns); + seq_printf(seq, + " nanodelays: %llu sleeps: %llu queue_stopped: %llu tx-early: %llu\n", + (unsigned long long)pkt_dev->nanodelays, + (unsigned long long)pkt_dev->sleeps, + (unsigned long long)pkt_dev->queue_stopped, + (unsigned long long)pkt_dev->req_tx_early); seq_printf(seq, " seq_num: %d cur_dst_mac_offset: %d cur_src_mac_offset: %d\n", @@ -969,15 +974,11 @@ static ssize_t pktgen_if_write(struct fi return len; } i += len; - if (value == 0x7FFFFFFF) { - pkt_dev->delay_us = 0x7FFFFFFF; - pkt_dev->delay_ns = 0; - } else { - pkt_dev->delay_us = value / 1000; - pkt_dev->delay_ns = value % 1000; + pkt_dev->delay_ns = value; + if ((getRelativeCurNs() + pkt_dev->delay_ns) > pkt_dev->next_tx_ns) { + pkt_dev->next_tx_ns = getRelativeCurNs() + pkt_dev->delay_ns; } - sprintf(pg_result, "OK: delay=%u", - 1000 * pkt_dev->delay_us + pkt_dev->delay_ns); + sprintf(pg_result, "OK: delay=%lluns", (unsigned long long)pkt_dev->delay_ns); return count; } if (!strcmp(name, "udp_src_min")) { @@ -1043,6 +1044,17 @@ static ssize_t pktgen_if_write(struct fi sprintf(pg_result, "OK: clone_skb=%d", pkt_dev->clone_skb); return count; } + if (!strcmp(name, "peer_clone_skb")) { + len = num_arg(&user_buffer[i], 10, &value); + if (len < 0) { + return len; + } + i += len; + pkt_dev->peer_clone_skb = value; + + sprintf(pg_result, "OK: peer_clone_skb=%d", pkt_dev->peer_clone_skb); + return count; + } if (!strcmp(name, "count")) { len = num_arg(&user_buffer[i], 10, &value); if (len < 0) { @@ -1141,6 +1153,7 @@ static ssize_t pktgen_if_write(struct fi pkt_dev->flags &= ~F_MPLS_RND; else { + printk("pktgen: Flag -:%s:- unknown\n", f); sprintf(pg_result, "Flag -:%s:- unknown\nAvailable flags, (prepend ! to un-set flag):\n%s", f, @@ -1405,7 +1418,7 @@ static ssize_t pktgen_if_write(struct fi } if (!strcmp(name, "clear_counters")) { - pktgen_clear_counters(pkt_dev); + pktgen_clear_counters(pkt_dev, 0); sprintf(pg_result, "OK: Clearing counters.\n"); return count; } @@ -1448,6 +1461,7 @@ static ssize_t pktgen_if_write(struct fi return count; } + printk("pktgen: No such parameter \"%s\"\n", name); sprintf(pkt_dev->result, "No such parameter \"%s\"", name); return -EINVAL; } @@ -1464,6 +1478,7 @@ static struct file_operations pktgen_if_ .llseek = seq_lseek, .write = pktgen_if_write, .release = single_release, + .ioctl = pktgen_proc_ioctl, }; static int pktgen_thread_show(struct seq_file *seq, void *v) @@ -1473,12 +1488,17 @@ static int pktgen_thread_show(struct seq BUG_ON(!t); - seq_printf(seq, "Name: %s max_before_softirq: %d\n", - t->name, t->max_before_softirq); + mutex_lock(&pktgen_thread_lock); + + /* versioning info. CFG_RT means we do not busy-spin, so can be configured for + * real-time scheduling if user-space so desires. */ + seq_printf(seq, "VERSION-2 CFG_RT\n"); + + seq_printf(seq, "PID: %d Name: %s max_before_softirq: %d\n", + t->pid, t->name, t->max_before_softirq); seq_printf(seq, "Running: "); - if_lock(t); list_for_each_entry(pkt_dev, &t->if_list, list) if (pkt_dev->running) seq_printf(seq, "%s ", pkt_dev->ifname); @@ -1494,8 +1514,7 @@ static int pktgen_thread_show(struct seq else seq_printf(seq, "\nResult: NA\n"); - if_unlock(t); - + mutex_unlock(&pktgen_thread_lock); return 0; } @@ -1563,13 +1582,35 @@ static ssize_t pktgen_thread_write(struc return -EFAULT; i += len; mutex_lock(&pktgen_thread_lock); - pktgen_add_device(t, f); + t->control_arg = f; + t->control |= T_ADD_DEV; + while (t->control & T_ADD_DEV) { + schedule_timeout_interruptible(msecs_to_jiffies(125)); + } + t->control_arg = 0; mutex_unlock(&pktgen_thread_lock); ret = count; sprintf(pg_result, "OK: add_device=%s", f); goto out; } + if (!strcmp(name, "rem_device")) { + char f[32]; + memset(f, 0, 32); + len = strn_len(&user_buffer[i], sizeof(f) - 1); + if (len < 0) { + ret = len; + goto out; + } + if (copy_from_user(f, &user_buffer[i], len)) + return -EFAULT; + i += len; + pktgen_mark_device(f); + ret = count; + sprintf(pg_result, "OK: rem_device=%s", f); + goto out; + } + if (!strcmp(name, "rem_device_all")) { mutex_lock(&pktgen_thread_lock); t->control |= T_REMDEVALL; @@ -1590,6 +1631,8 @@ static ssize_t pktgen_thread_write(struc goto out; } + printk("pktgen: un-known command to pktgen_thread: -:%s:-\n", name); + ret = -EINVAL; out: return ret; @@ -1607,8 +1650,10 @@ static struct file_operations pktgen_thr .llseek = seq_lseek, .write = pktgen_thread_write, .release = single_release, + .ioctl = pktgen_proc_ioctl, }; + /* Think find or remove for NN */ static struct pktgen_dev *__pktgen_NN_threads(const char *ifname, int remove) { @@ -1619,14 +1664,14 @@ static struct pktgen_dev *__pktgen_NN_th pkt_dev = pktgen_find_dev(t, ifname); if (pkt_dev) { if (remove) { - if_lock(t); pkt_dev->removal_mark = 1; t->control |= T_REMDEV; - if_unlock(t); } break; } } + /*PG_DEBUG(printk("pktgen_NN_threads, ifname: %s remove: %d pkt_dev: 0x%p\n", + ifname, remove, pkt_dev)); */ return pkt_dev; } @@ -1698,13 +1743,21 @@ static int pktgen_device_event(struct no /* Associate pktgen_dev with a device. */ -static struct net_device *pktgen_setup_dev(struct pktgen_dev *pkt_dev) +static struct net_device *pktgen_setup_dev(struct pktgen_dev *pkt_dev, struct pktgen_thread* t) { struct net_device *odev; /* Clean old setups */ if (pkt_dev->odev) { + +#ifdef USE_NQW_CALLBACK + /* Set the nqw callback hooks */ + rtnl_lock(); + clear_nqw_hook(t, pkt_dev->odev); + rtnl_unlock(); +#endif + pkt_dev->odev->pkt_dev = NULL; dev_put(pkt_dev->odev); pkt_dev->odev = NULL; } @@ -1726,6 +1779,14 @@ static struct net_device *pktgen_setup_d } pkt_dev->odev = odev; +#ifdef USE_NQW_CALLBACK + /* Set the nqw callback hooks */ + rtnl_lock(); + set_nqw_hook(t, pkt_dev->odev, GFP_ATOMIC); + rtnl_unlock(); +#endif + + pkt_dev->odev->pkt_dev = pkt_dev; return pkt_dev->odev; out_put: @@ -1742,7 +1803,7 @@ static void pktgen_setup_inject(struct p { /* Try once more, just in case it works now. */ if (!pkt_dev->odev) - pktgen_setup_dev(pkt_dev); + pktgen_setup_dev(pkt_dev, pkt_dev->pg_thread); if (!pkt_dev->odev) { printk("pktgen: ERROR: pkt_dev->odev == NULL in setup_inject.\n"); @@ -1845,6 +1906,189 @@ #endif pkt_dev->nflows = 0; } + +#ifdef USE_NQW_CALLBACK +/* Runs from interrupt */ +int pg_notify_queue_woken(struct net_device* dev) { + /* Find the thread that needs waking. */ + struct pktgen_thread* t = ((struct pg_nqw_data*)(dev->nqw_data))->pg_thread; + t->control |= T_WAKE_BLOCKED; + wake_up_interruptible(&(t->queue)); + return 0; +} + +/* Must hold RTNL lock while calling this. */ +static int set_nqw_hook(struct pktgen_thread* t, struct net_device* dev, int gfp) { + /* The notify-queue-woken magic only works for physical + * devices at this time. So, apply hook to underlying + * device. + */ + struct pg_nqw_data* nqwd; + ASSERT_RTNL(); + BUG_ON(!t); + + if (!dev) { + WARN_ON(!dev); + return -ENODEV; + } +#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE) + if (dev->priv_flags & IFF_MAC_VLAN) { + struct macvlan_vlan *vlan = dev->priv; + printk("pktgen: setting nqw_hook on lower mac-vlan dev: %p\n", vlan->lowerdev); + return set_nqw_hook(t, vlan->lowerdev, gfp); + } +#endif + + if (dev->priv_flags & IFF_802_1Q_VLAN) { + printk("pktgen: setting nqw_hook on real-dev of .1q vlan: %s\n", dev->name); + return set_nqw_hook(t, VLAN_DEV_INFO(dev)->real_dev, gfp); + } + + nqwd = (struct pg_nqw_data*)(dev->nqw_data); + + if (nqwd) { + if (nqwd->magic == PG_NQW_MAGIC) { + if (nqwd->pg_thread == t) { + atomic_inc(&(nqwd->nqw_ref_count)); + + printk("pktgen: Incremented nqw_ref_count: %d device: %s\n", + (int)(atomic_read(&(nqwd->nqw_ref_count))), dev->name); + return 0; + } + else { + printk("pktgen: ERROR: set_nqw_hook: nqwd thread does not match, dev: %s", + dev->name); + return -EINVAL; + } + } + else { + printk("wanlink: WARNING: set_nqw_hook: nqwd magic is NOT WanLink, dev: %s magic: 0x%x", + dev->name, nqwd->magic); + return 0; + } + } + else { + nqwd = kmalloc(sizeof(*nqwd), gfp); + if (nqwd) { + memset(nqwd, 0, sizeof(*nqwd)); + nqwd->magic = PG_NQW_MAGIC; + atomic_inc(&(nqwd->nqw_ref_count)); + nqwd->pg_thread = t; + dev->nqw_data = nqwd; + dev->notify_queue_woken = pg_notify_queue_woken; + printk("pktgen: Added nqw callback to device: %s\n", + dev->name); + return 0; + } + else { + printk("pktgen: ERROR: could not allocate nqwd for dev: %s\n", dev->name); + return -ENOBUFS; + } + } +}//set_nqw_hook + + +/* Must hold RTNL lock while calling this. */ +static void clear_nqw_hook(struct pktgen_thread* t, struct net_device* dev) { + /* The notify-queue-woken magic only works for physical + * devices at this time. So, apply hook to underlying + * device. + */ + ASSERT_RTNL(); + BUG_ON(!t); + +#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE) + if (dev->priv_flags & IFF_MAC_VLAN) { + struct macvlan_vlan *vlan = dev->priv; + clear_nqw_hook(t, vlan->lowerdev); + return; + } +#endif + + if (dev->priv_flags & IFF_802_1Q_VLAN) { + clear_nqw_hook(t, VLAN_DEV_INFO(dev)->real_dev); + return; + } + + if (dev->nqw_data) { + struct pg_nqw_data* nqwd = (struct pg_nqw_data*)(dev->nqw_data); + if (nqwd->magic == PG_NQW_MAGIC) { + if (t != nqwd->pg_thread) { + printk("pktgen ERROR: t != nqwd->pg_thread\n"); + } + atomic_dec(&(nqwd->nqw_ref_count)); + + printk("pktgen: Decremented nqw_ref_count: %d device: %s\n", + (int)(atomic_read(&(nqwd->nqw_ref_count))), + dev->name); + + BUG_ON(atomic_read(&(nqwd->nqw_ref_count)) < 0); + + if (atomic_read(&(nqwd->nqw_ref_count)) == 0) { + printk("pktgen: Removing nqw reference from device: %s\n", + dev->name); + dev->notify_queue_woken = NULL; + dev->nqw_data = NULL; + kfree(nqwd); + } + } + else { + printk("pktgen: WARNING: clear_nqw_hook: nqwd magic is NOT PKT-GEN, dev: %s magic: 0x%x", + dev->name, nqwd->magic); + } + } + else { + printk("pktgen: Warning: nqw_data is null in clear_nqw_hook, dev: %s\n", + dev->name); + } +}//clear_nqw_hook + +#endif + + +/* delay_ns is in nano-seconds */ +static void pg_nanodelay(u64 delay_ns, struct pktgen_dev* info) { + u64 idle_start = getRelativeCurNs(); + u64 last_time; + u64 _diff; + u64 itmp = idle_start; + struct pktgen_dev *p = NULL; + struct pktgen_thread* t = info->pg_thread; + + info->nanodelays++; + info->accum_delay_ns += delay_ns; + while (info->accum_delay_ns > PG_MAX_ACCUM_DELAY_NS) { + info->sleeps++; + interruptible_sleep_on_timeout(&(t->queue), 1); + /* will wake after one tick */ + last_time = itmp; + + /* Subtract delay from all interfaces for this thread, since all are blocked when + * any are blocked. + */ + itmp = getRelativeCurNs(); + _diff = (itmp - last_time); + list_for_each_entry(p, &t->if_list, list) { + p->accum_delay_ns -= _diff; + /* Limit saving up too much time... */ + if (p->accum_delay_ns < -10000000) { + p->accum_delay_ns = -10000000; + } + } + + /* For accounting, only charge this guy for the idle though...*/ + info->idle_acc_ns += _diff; + + /* break out if we are stopped or if we should transmit (maybe our ipg changed?) */ + if (info->removal_mark || (itmp >= info->next_tx_ns) || + (t->control && T_WAKE_BLOCKED) || + (t->control && T_STOP)) { + break; + } + }/* while */ +}//pg_nanodelay + +#if 0 static void spin(struct pktgen_dev *pkt_dev, __u64 spin_until_us) { __u64 start; @@ -1867,8 +2111,9 @@ static void spin(struct pktgen_dev *pkt_ now = getCurUs(); } - pkt_dev->idle_acc += now - start; + pkt_dev->idle_acc_ns += (now - start) * 1000; } +#endif /* Increment/randomize headers according to flags and current values * for IP src/dest, UDP src/dst port, MAC-Addr src/dst @@ -2545,13 +2790,202 @@ static inline struct sk_buff *fill_packe return fill_packet_ipv4(odev, pkt_dev); } -static void pktgen_clear_counters(struct pktgen_dev *pkt_dev) -{ - pkt_dev->seq_num = 1; - pkt_dev->idle_acc = 0; + +static void record_latency(struct pktgen_dev* pkt_dev, int latency) { + /* NOTE: Latency can be negative */ + int div = 100; + int diff; + int vl; + int i; + + pkt_dev->pkts_rcvd_since_clear++; + + if (pkt_dev->pkts_rcvd_since_clear < 100) { + div = pkt_dev->pkts_rcvd; + if (pkt_dev->pkts_rcvd_since_clear == 1) { + pkt_dev->avg_latency = latency; + } + } + + if ((div + 1) == 0) { + pkt_dev->avg_latency = 0; + } + else { + pkt_dev->avg_latency = ((pkt_dev->avg_latency * div + latency) / (div + 1)); + } + + if (latency < pkt_dev->min_latency) { + pkt_dev->min_latency = latency; + } + if (latency > pkt_dev->max_latency) { + pkt_dev->max_latency = latency; + } + + /* Place the latency in the right 'bucket' */ + diff = (latency - pkt_dev->min_latency); + for (i = 0; ilatency_bkts[i]++; + break; + } + } +}/* record latency */ + + +/* Returns < 0 if the skb is not a pktgen buffer. */ +int pktgen_receive(struct sk_buff* skb) { + /* See if we have a pktgen packet */ + /* TODO: Add support for detecting IPv6, TCP packets too. This will only + * catch UDP at the moment. --Ben + */ + /*printk("pktgen-rcv, skb->len: %d\n", skb->len);*/ + + if ((skb->len >= (20 + 8 + sizeof(struct pktgen_hdr))) && + (skb->protocol == __constant_htons(ETH_P_IP))) { + struct pktgen_hdr* pgh; + + /* It's IP, and long enough, lets check the magic number. + * TODO: This is a hack not always guaranteed to catch the right + * packets. + */ + + /*printk("Length & protocol passed, skb->data: %p, raw: %p\n", + skb->data, skb->h.raw);*/ + + pgh = (struct pktgen_hdr*)(skb->data + 20 + 8); + + /* + tmp = (char*)(skb->data); + for (i = 0; i<90; i++) { + printk("%02hx ", tmp[i]); + if (((i + 1) % 15) == 0) { + printk("\n"); + } + } + printk("\n"); + */ + + if (pgh->pgh_magic == __constant_ntohl(PKTGEN_MAGIC)) { + struct net_device* dev = skb->dev; + struct pktgen_dev* pkt_dev; + __u32 seq = ntohl(pgh->seq_num); + + // TODO: Need lock..maybe + pkt_dev = dev->pkt_dev; + + if (!pkt_dev) { + return -1; + } + + pkt_dev->pkts_rcvd++; + pkt_dev->bytes_rcvd += ((skb->tail - skb->mac.raw) + 4); /* +4 for the checksum */ + + /* Check for out-of-sequence packets */ + if (pkt_dev->last_seq_rcvd == seq) { + pkt_dev->dup_rcvd++; + pkt_dev->dup_since_incr++; + } + else { + __s64 rx; + __s64 tx; + struct timeval txtv; + if (skb->tstamp.off_sec || skb->tstamp.off_usec) { + skb_get_timestamp(skb, &txtv); + } + else { + do_gettimeofday(&txtv); + skb_set_timestamp(skb, &txtv); + } + rx = tv_to_us(&txtv); + + txtv.tv_usec = ntohl(pgh->tv_usec); + txtv.tv_sec = ntohl(pgh->tv_sec); + tx = tv_to_us(&txtv); + record_latency(pkt_dev, rx - tx); + + if ((pkt_dev->last_seq_rcvd + 1) == seq) { + if ((pkt_dev->peer_clone_skb > 1) && + (pkt_dev->peer_clone_skb > (pkt_dev->dup_since_incr + 1))) { + + pkt_dev->seq_gap_rcvd += (pkt_dev->peer_clone_skb - + pkt_dev->dup_since_incr - 1); + } + /* Great, in order...all is well */ + } + else if (pkt_dev->last_seq_rcvd < seq) { + /* sequence gap, means we dropped a pkt most likely */ + if (pkt_dev->peer_clone_skb > 1) { + /* We dropped more than one sequence number's worth, + * and if we're using clone_skb, then this is quite + * a few. This number still will not be exact, but + * it will be closer. + */ + pkt_dev->seq_gap_rcvd += (((seq - pkt_dev->last_seq_rcvd) * + pkt_dev->peer_clone_skb) - + pkt_dev->dup_since_incr); + } + else { + pkt_dev->seq_gap_rcvd += (seq - pkt_dev->last_seq_rcvd - 1); + } + } + else { + pkt_dev->ooo_rcvd++; /* out-of-order */ + } + + pkt_dev->dup_since_incr = 0; + } + pkt_dev->last_seq_rcvd = seq; + kfree_skb(skb); + if (debug > 1) { + printk("done with pktgen_receive, free'd pkt\n"); + } + return 0; + } + } + return -1; /* Let another protocol handle it, it's not for us! */ +}/* pktgen_receive */ + +static void pg_reset_latency_counters(struct pktgen_dev* pkt_dev) { + int i; + pkt_dev->avg_latency = 0; + pkt_dev->min_latency = 0x7fffffff; /* largest integer */ + pkt_dev->max_latency = 0x80000000; /* smallest integer */ + pkt_dev->pkts_rcvd_since_clear = 0; + for (i = 0; ilatency_bkts[i] = 0; + } +} + + +static void pktgen_clear_counters(struct pktgen_dev *pkt_dev, int seq_too) { + pkt_dev->idle_acc_ns = 0; pkt_dev->sofar = 0; pkt_dev->tx_bytes = 0; pkt_dev->errors = 0; + + pkt_dev->ooo_rcvd = 0; + pkt_dev->dup_rcvd = 0; + pkt_dev->pkts_rcvd = 0; + pkt_dev->bytes_rcvd = 0; + pkt_dev->non_pg_pkts_rcvd = 0; + pkt_dev->seq_gap_rcvd = 0; /* dropped */ + + /* Clear some transient state */ + pkt_dev->accum_delay_ns = 0; + pkt_dev->sleeps = 0; + pkt_dev->nanodelays = 0; + + /* This is a bit of a hack, but it gets the dup counters + * in line so we don't have false alarms on dropped pkts. + */ + if (seq_too) { + pkt_dev->dup_since_incr = pkt_dev->peer_clone_skb - 1; + pkt_dev->seq_num = 1; + pkt_dev->last_seq_rcvd = 0; + } + + pg_reset_latency_counters(pkt_dev); } /* Set up structure for sending pkts, clear counters */ @@ -2563,7 +2997,6 @@ static void pktgen_run(struct pktgen_thr PG_DEBUG(printk("pktgen: entering pktgen_run. %p\n", t)); - if_lock(t); list_for_each_entry(pkt_dev, &t->if_list, list) { /* @@ -2572,19 +3005,17 @@ static void pktgen_run(struct pktgen_thr pktgen_setup_inject(pkt_dev); if (pkt_dev->odev) { - pktgen_clear_counters(pkt_dev); + pktgen_clear_counters(pkt_dev, 1); pkt_dev->running = 1; /* Cranke yeself! */ pkt_dev->skb = NULL; pkt_dev->started_at = getCurUs(); - pkt_dev->next_tx_us = getCurUs(); /* Transmit immediately */ - pkt_dev->next_tx_ns = 0; + pkt_dev->next_tx_ns = getRelativeCurNs(); /* Transmit immediately */ strcpy(pkt_dev->result, "Starting"); started++; } else strcpy(pkt_dev->result, "Error starting"); } - if_unlock(t); if (started) t->control &= ~(T_STOP); } @@ -2603,65 +3034,12 @@ static void pktgen_stop_all_threads_ifs( mutex_unlock(&pktgen_thread_lock); } -static int thread_is_running(struct pktgen_thread *t) -{ - struct pktgen_dev *pkt_dev; - int res = 0; - - list_for_each_entry(pkt_dev, &t->if_list, list) - if (pkt_dev->running) { - res = 1; - break; - } - return res; -} - -static int pktgen_wait_thread_run(struct pktgen_thread *t) -{ - if_lock(t); - - while (thread_is_running(t)) { - - if_unlock(t); - msleep_interruptible(100); - - if (signal_pending(current)) - goto signal; - if_lock(t); - } - if_unlock(t); - return 1; -signal: - return 0; -} - -static int pktgen_wait_all_threads_run(void) -{ - struct pktgen_thread *t; - int sig = 1; - - mutex_lock(&pktgen_thread_lock); - - list_for_each_entry(t, &pktgen_threads, th_list) { - sig = pktgen_wait_thread_run(t); - if (sig == 0) - break; - } - - if (sig == 0) - list_for_each_entry(t, &pktgen_threads, th_list) - t->control |= (T_STOP); - - mutex_unlock(&pktgen_thread_lock); - return sig; -} - -static void pktgen_run_all_threads(void) -{ +static void pktgen_run_all_threads(int background) { struct pktgen_thread *t; - PG_DEBUG(printk("pktgen: entering pktgen_run_all_threads.\n")); + PG_DEBUG(printk("pktgen: entering pktgen_run_all_threads, background: %d\n", + background)); mutex_lock(&pktgen_thread_lock); @@ -2672,7 +3050,11 @@ static void pktgen_run_all_threads(void) schedule_timeout_interruptible(msecs_to_jiffies(125)); /* Propagate thread->control */ - pktgen_wait_all_threads_run(); + // Much harder to get rid of the if_lock if we allow this to block... + if (!background) { + printk("ERROR: non-background mode no longer supported.\n"); + //pktgen_wait_all_threads_run(); + } } static void show_results(struct pktgen_dev *pkt_dev, int nr_frags) @@ -2682,7 +3064,7 @@ static void show_results(struct pktgen_d total_us = pkt_dev->stopped_at - pkt_dev->started_at; - idle = pkt_dev->idle_acc; + idle = do_div(pkt_dev->idle_acc_ns, 1000); p += sprintf(p, "OK: %llu(c%llu+d%llu) usec, %llu (%dbyte,%dfrags)\n", (unsigned long long)total_us, @@ -2731,22 +3113,62 @@ static int pktgen_stop_device(struct pkt return 0; } -static struct pktgen_dev *next_to_run(struct pktgen_thread *t) -{ - struct pktgen_dev *pkt_dev, *best = NULL; - - if_lock(t); +/** Find the adapter that needs to tx next. + * We need to take the blocked adapters into account, but can't ignore + * them forever just in case we missed the tx-queue-wake event for some + * reason. + */ +static struct pktgen_dev *next_to_run(struct pktgen_thread *t, u64 now, u64* next_running_delay) { + struct pktgen_dev *pkt_dev = NULL; + struct pktgen_dev *best = NULL; + struct pktgen_dev *best_blocked = NULL; list_for_each_entry(pkt_dev, &t->if_list, list) { if (!pkt_dev->running) continue; - if (best == NULL) - best = pkt_dev; - else if (pkt_dev->next_tx_us < best->next_tx_us) - best = pkt_dev; + if (pkt_dev->tx_blocked) { + if (best_blocked == NULL) + best_blocked = pkt_dev; + else { + if (pkt_dev->next_tx_ns < best_blocked->next_tx_ns) { + best_blocked = pkt_dev; + } + } + } + else { + if (best == NULL) + best = pkt_dev; + else { + if (pkt_dev->next_tx_ns < best->next_tx_ns) { + best = pkt_dev; + } + } + } + } + + /** If we have a blocked device that is more than 1ms late, then try it again first. + * Otherwise, take best non-blocked device. + */ + if (best) { + if (best->next_tx_ns <= now) { + *next_running_delay = 0; + } + else { + *next_running_delay = best->next_tx_ns - now; + } } - if_unlock(t); - return best; + else { + *next_running_delay = 10000000; /* 10ms */ + } + + if (best_blocked && (best_blocked->next_tx_ns < (now - PG_TRY_TX_ANYWAY_NS))) { + return best_blocked; + } + + if (best) { + return best; + } + return best_blocked; } static void pktgen_stop(struct pktgen_thread *t) @@ -2755,8 +3177,6 @@ static void pktgen_stop(struct pktgen_th PG_DEBUG(printk("pktgen: entering pktgen_stop\n")); - if_lock(t); - list_for_each_entry(pkt_dev, &t->if_list, list) { pktgen_stop_device(pkt_dev); if (pkt_dev->skb) @@ -2764,8 +3184,6 @@ static void pktgen_stop(struct pktgen_th pkt_dev->skb = NULL; } - - if_unlock(t); } /* @@ -2779,8 +3197,6 @@ static void pktgen_rem_one_if(struct pkt PG_DEBUG(printk("pktgen: entering pktgen_rem_one_if\n")); - if_lock(t); - list_for_each_safe(q, n, &t->if_list) { cur = list_entry(q, struct pktgen_dev, list); @@ -2795,10 +3211,15 @@ static void pktgen_rem_one_if(struct pkt break; } - - if_unlock(t); } +static void pktgen_unblock_all_ifs(struct pktgen_thread *t) { + struct pktgen_dev *p = NULL;; + list_for_each_entry(p, &t->if_list, list) + p->tx_blocked = 0; +}/* wake all writers */ + + static void pktgen_rem_all_ifs(struct pktgen_thread *t) { struct list_head *q, *n; @@ -2807,8 +3228,6 @@ static void pktgen_rem_all_ifs(struct pk /* Remove all devices, free mem */ PG_DEBUG(printk("pktgen: entering pktgen_rem_all_ifs\n")); - if_lock(t); - list_for_each_safe(q, n, &t->if_list) { cur = list_entry(q, struct pktgen_dev, list); @@ -2818,8 +3237,6 @@ static void pktgen_rem_all_ifs(struct pk pktgen_remove_device(t, cur); } - - if_unlock(t); } static void pktgen_rem_thread(struct pktgen_thread *t) @@ -2835,34 +3252,41 @@ static void pktgen_rem_thread(struct pkt mutex_unlock(&pktgen_thread_lock); } -static __inline__ void pktgen_xmit(struct pktgen_dev *pkt_dev) -{ - struct net_device *odev = NULL; +static __inline__ void pktgen_xmit(struct pktgen_dev *pkt_dev, u64 now) { + struct net_device *odev; __u64 idle_start = 0; int ret; - + odev = pkt_dev->odev; - if (pkt_dev->delay_us || pkt_dev->delay_ns) { - u64 now; - - now = getCurUs(); - if (now < pkt_dev->next_tx_us) - spin(pkt_dev, pkt_dev->next_tx_us); - + if (pkt_dev->delay_ns || (pkt_dev->accum_delay_ns > 0)) { + if (now < pkt_dev->next_tx_ns) { + /* Don't tx early..*/ + pkt_dev->req_tx_early++; + goto out; + } + /* This is max DELAY, this has special meaning of * "never transmit" */ - if (pkt_dev->delay_us == 0x7FFFFFFF) { - pkt_dev->next_tx_us = getCurUs() + pkt_dev->delay_us; - pkt_dev->next_tx_ns = pkt_dev->delay_ns; + if (pkt_dev->delay_ns == 0x7FFFFFFF) { + pkt_dev->next_tx_ns = getRelativeCurNs() + pkt_dev->delay_ns; goto out; } } - if (netif_queue_stopped(odev) || need_resched()) { - idle_start = getCurUs(); - + if (need_resched()) { + idle_start = getRelativeCurNs(); + schedule(); + pkt_dev->idle_acc_ns += getRelativeCurNs() - idle_start; + } + + if (netif_queue_stopped(odev)) { + pkt_dev->queue_stopped++; + pkt_dev->tx_blocked = 1; + /* change tx time to now to show work was at least attempted. */ + pkt_dev->next_tx_ns = now; + if (!netif_running(odev)) { pktgen_stop_device(pkt_dev); if (pkt_dev->skb) @@ -2870,16 +3294,8 @@ static __inline__ void pktgen_xmit(struc pkt_dev->skb = NULL; goto out; } - if (need_resched()) - schedule(); - - pkt_dev->idle_acc += getCurUs() - idle_start; - if (netif_queue_stopped(odev)) { - pkt_dev->next_tx_us = getCurUs(); /* TODO */ - pkt_dev->next_tx_ns = 0; - goto out; /* Try the next interface */ - } + goto out; /* Try the next interface */ } if (pkt_dev->last_ok || !pkt_dev->skb) { @@ -2910,40 +3326,44 @@ static __inline__ void pktgen_xmit(struc if (likely(ret == NETDEV_TX_OK)) { pkt_dev->last_ok = 1; pkt_dev->sofar++; - pkt_dev->seq_num++; pkt_dev->tx_bytes += pkt_dev->cur_pkt_size; + pkt_dev->next_tx_ns = getRelativeCurNs() + pkt_dev->delay_ns; + pkt_dev->tx_blocked = 0; } else if (ret == NETDEV_TX_LOCKED && (odev->features & NETIF_F_LLTX)) { cpu_relax(); goto retry_now; } else { /* Retry it next time */ - + static int do_once_hsx_wrn = 1; + if (do_once_hsx_wrn) { + printk(KERN_INFO "pktgen: Hard xmit error, driver for %s doesn't do queue-stopped quite right.\n", odev->name); + printk(KERN_INFO "pktgen: Transmit request will be retried, and this error msg will not be printed again..\n"); + do_once_hsx_wrn = 0; + } + atomic_dec(&(pkt_dev->skb->users)); - if (debug && net_ratelimit()) - printk(KERN_INFO "pktgen: Hard xmit error\n"); - + pkt_dev->queue_stopped++; pkt_dev->errors++; pkt_dev->last_ok = 0; - } - - pkt_dev->next_tx_us = getCurUs(); - pkt_dev->next_tx_ns = 0; - - pkt_dev->next_tx_us += pkt_dev->delay_us; - pkt_dev->next_tx_ns += pkt_dev->delay_ns; - if (pkt_dev->next_tx_ns > 1000) { - pkt_dev->next_tx_us++; - pkt_dev->next_tx_ns -= 1000; + /* Try a little later..flag us as wanting to tx, but unable. Will try again shortly. + */ + pkt_dev->tx_blocked = 1; + /* change tx time to now to show work was at least attempted. */ + pkt_dev->next_tx_ns = now; } } else { /* Retry it next time */ + pkt_dev->queue_stopped++; pkt_dev->last_ok = 0; - pkt_dev->next_tx_us = getCurUs(); /* TODO */ - pkt_dev->next_tx_ns = 0; + /* Try a little later..flag us as wanting to tx, but unable. Will try again shortly. + */ + pkt_dev->tx_blocked = 1; + /* change tx time to now to show work was at least attempted. */ + pkt_dev->next_tx_ns = now; } netif_tx_unlock_bh(odev); @@ -2951,14 +3371,14 @@ static __inline__ void pktgen_xmit(struc /* If pkt_dev->count is zero, then run forever */ if ((pkt_dev->count != 0) && (pkt_dev->sofar >= pkt_dev->count)) { if (atomic_read(&(pkt_dev->skb->users)) != 1) { - idle_start = getCurUs(); + idle_start = getRelativeCurNs(); while (atomic_read(&(pkt_dev->skb->users)) != 1) { if (signal_pending(current)) { break; } schedule(); } - pkt_dev->idle_acc += getCurUs() - idle_start; + pkt_dev->idle_acc_ns += getRelativeCurNs() - idle_start; } /* Done with this */ @@ -2982,7 +3402,9 @@ static void pktgen_thread_worker(struct sigset_t tmpsig; u32 max_before_softirq; u32 tx_since_softirq = 0; - + u64 now; + u64 next_running_delay; + daemonize("pktgen/%d", cpu); /* Block all signals except SIGKILL, SIGSTOP and SIGTERM */ @@ -3007,6 +3429,8 @@ static void pktgen_thread_worker(struct t->control &= ~(T_STOP); t->control &= ~(T_REMDEVALL); t->control &= ~(T_REMDEV); + t->control &= ~(T_WAKE_BLOCKED); + t->control &= ~(T_ADD_DEV); t->pid = current->pid; @@ -3024,13 +3448,52 @@ static void pktgen_thread_worker(struct /* * Get next dev to xmit -- if any. */ + find_best: - pkt_dev = next_to_run(t); + if (t->control & T_WAKE_BLOCKED) { + pktgen_unblock_all_ifs(t); + t->control &= ~(T_WAKE_BLOCKED); + } + + now = getRelativeCurNs(); + pkt_dev = next_to_run(t, now, &next_running_delay); if (pkt_dev) { - pktgen_xmit(pkt_dev); + if (pkt_dev->tx_blocked) { + /* If blocked for less than 1ms, then sleep for up to 1ms. If the + * device un-blocks, then we will be woken by the wait-queue callback. + */ + u64 tx_anyway_ns = (now - PG_TRY_TX_ANYWAY_NS); + if (pkt_dev->next_tx_ns > tx_anyway_ns) { + pg_nanodelay(min(next_running_delay, (u64)(PG_TRY_TX_ANYWAY_NS)), + pkt_dev); + /* Maybe things have changed since we went to sleep. */ + goto find_best; + } + } + + /* If the best to run should not run yet, then sleep (or accumulate sleep) */ + if (now < pkt_dev->next_tx_ns) { + /* spin(pkt_dev, pkt_dev->next_tx_us); */ + u64 next_ipg = pkt_dev->next_tx_ns - now; + + /* These will not actually busy-spin now. Will run as + * much as 1ms fast, and will sleep in 1ms units, assuming + * our tick is 1ms. + */ + pg_nanodelay(next_ipg, pkt_dev); + now = getRelativeCurNs(); + if (pkt_dev->removal_mark || + (pkt_dev->pg_thread->control && T_STOP)) { + goto skip_tx; + } + } + + pktgen_xmit(pkt_dev, now); + + skip_tx: /* * We like to stay RUNNING but must also give * others fair share. @@ -3068,6 +3531,11 @@ static void pktgen_thread_worker(struct t->control &= ~(T_RUN); } + if (t->control & T_ADD_DEV) { + pktgen_add_device(t, (char*)(t->control_arg)); + t->control &= ~(T_ADD_DEV); + } + if (t->control & T_REMDEVALL) { pktgen_rem_all_ifs(t); t->control &= ~(T_REMDEVALL); @@ -3098,16 +3566,12 @@ static struct pktgen_dev *pktgen_find_de const char *ifname) { struct pktgen_dev *p, *pkt_dev = NULL; - if_lock(t); - list_for_each_entry(p, &t->if_list, list) if (strncmp(p->ifname, ifname, IFNAMSIZ) == 0) { pkt_dev = p; break; } - - if_unlock(t); - PG_DEBUG(printk("pktgen: find_dev(%s) returning %p\n", ifname, pkt_dev)); + /* PG_DEBUG(printk("pktgen: find_dev(%s) returning %p\n", ifname, pkt_dev)); */ return pkt_dev; } @@ -3120,8 +3584,6 @@ static int add_dev_to_thread(struct pktg { int rv = 0; - if_lock(t); - if (pkt_dev->pg_thread) { printk("pktgen: ERROR: already assigned to a thread.\n"); rv = -EBUSY; @@ -3133,12 +3595,9 @@ static int add_dev_to_thread(struct pktg pkt_dev->running = 0; out: - if_unlock(t); return rv; } -/* Called under thread lock */ - static int pktgen_add_device(struct pktgen_thread *t, const char *ifname) { struct pktgen_dev *pkt_dev; @@ -3151,7 +3610,10 @@ static int pktgen_add_device(struct pktg printk("pktgen: ERROR: interface already used.\n"); return -EBUSY; } - + else { + printk("pktgen: Attempting to add device: %s\n", ifname); + } + pkt_dev = kzalloc(sizeof(struct pktgen_dev), GFP_KERNEL); if (!pkt_dev) return -ENOMEM; @@ -3168,8 +3630,7 @@ static int pktgen_add_device(struct pktg pkt_dev->max_pkt_size = ETH_ZLEN; pkt_dev->nfrags = 0; pkt_dev->clone_skb = pg_clone_skb_d; - pkt_dev->delay_us = pg_delay_d / 1000; - pkt_dev->delay_ns = pg_delay_d % 1000; + pkt_dev->delay_ns = pg_delay_d; pkt_dev->count = pg_count_d; pkt_dev->sofar = 0; pkt_dev->udp_src_min = 9; /* sink port */ @@ -3179,7 +3640,7 @@ static int pktgen_add_device(struct pktg strncpy(pkt_dev->ifname, ifname, IFNAMSIZ); - if (!pktgen_setup_dev(pkt_dev)) { + if (!pktgen_setup_dev(pkt_dev, t)) { printk("pktgen: ERROR: pktgen_setup_dev failed.\n"); if (pkt_dev->flows) vfree(pkt_dev->flows); @@ -3202,7 +3663,7 @@ static int pktgen_add_device(struct pktg return add_dev_to_thread(t, pkt_dev); } -static struct pktgen_thread *__init pktgen_find_thread(const char *name) +static struct pktgen_thread* pktgen_find_thread(const char *name) { struct pktgen_thread *t; @@ -3241,7 +3702,6 @@ static int __init pktgen_create_thread(c } strcpy(t->name, name); - spin_lock_init(&t->if_lock); t->cpu = cpu; pe = create_proc_entry(t->name, 0600, pg_proc_dir); @@ -3304,6 +3764,14 @@ static int pktgen_remove_device(struct p /* Dis-associate from the interface */ if (pkt_dev->odev) { + +#ifdef USE_NQW_CALLBACK + /* Set the nqw callback hooks */ + rtnl_lock(); + clear_nqw_hook(t, pkt_dev->odev); + rtnl_unlock(); +#endif + pkt_dev->odev->pkt_dev = NULL; dev_put(pkt_dev->odev); pkt_dev->odev = NULL; } @@ -3314,6 +3782,7 @@ static int pktgen_remove_device(struct p /* Clean up proc file system */ + PG_DEBUG(printk("pktgen: removing proc entry: %s (0x%p)\n", pkt_dev->ifname, pg_proc_dir)); remove_proc_entry(pkt_dev->ifname, pg_proc_dir); if (pkt_dev->flows) @@ -3328,7 +3797,17 @@ static int __init pg_init(void) struct proc_dir_entry *pe; printk(version); - + printk("sizeof report: %d, sizeof in6_addr: %d\n", + sizeof(struct pktgen_dev_report), + sizeof(struct in6_addr)); + + if (handle_pktgen_hook) { + printk("pktgen: ERROR: pktgen is already loaded it seems..\n"); + /* Already loaded */ + return -EEXIST; + } + + pg_proc_dir = proc_mkdir(PG_PROC_DIR, proc_net); if (!pg_proc_dir) return -ENODEV; @@ -3367,6 +3846,9 @@ static int __init pg_init(void) return -ENODEV; } + handle_pktgen_hook = pktgen_receive; + PG_DEBUG(printk("pktgen initialization complete.\n")); + return 0; } @@ -3377,10 +3859,15 @@ static void __exit pg_cleanup(void) wait_queue_head_t queue; init_waitqueue_head(&queue); + /* Un-Register receive handler */ + handle_pktgen_hook = NULL; + /* Stop all interfaces & threads */ - + pktgen_stop_all_threads_ifs(); + list_for_each_safe(q, n, &pktgen_threads) { t = list_entry(q, struct pktgen_thread, th_list); + t->control |= (T_STOP); t->control |= (T_TERMINATE); wait_event_interruptible_timeout(queue, (t->removed == 1), HZ); @@ -3390,6 +3877,7 @@ static void __exit pg_cleanup(void) unregister_netdevice_notifier(&pktgen_notifier_block); /* Clean up proc file system */ + PG_DEBUG(printk("pktgen: removing proc entry: %s (0x%p)\n", PGCTRL, pg_proc_dir)); remove_proc_entry(PGCTRL, pg_proc_dir); proc_net_remove(PG_PROC_DIR); } diff --git a/net/core/pktgen.h b/net/core/pktgen.h new file mode 100644 index 0000000..9f133cd --- /dev/null +++ b/net/core/pktgen.h @@ -0,0 +1,334 @@ +/* -*-linux-c-*- + * $Id: candela_2.6.13.patch,v 1.3 2005/09/30 04:45:31 greear Exp $ + * pktgen.c: Packet Generator for performance evaluation. + * + * See pktgen.c for details of changes, etc. +*/ + + +#ifndef PKTGEN_H_INCLUDE_KERNEL__ +#define PKTGEN_H_INCLUDE_KERNEL__ + +#include + +/* The buckets are exponential in 'width' */ +#define LAT_BUCKETS_MAX 32 +#define PG_MAX_ACCUM_DELAY_NS 1000000 /* one ms */ +#define PG_TRY_TX_ANYWAY_NS 1000000 /* try a blocked tx queue after 1 ms. */ + +#define IP_NAME_SZ 32 +#define MAX_MPLS_LABELS 16 /* This is the max label stack depth */ +#define MPLS_STACK_BOTTOM __constant_htonl(0x00000100) + +/* Device flag bits */ +#define F_IPSRC_RND (1<<0) /* IP-Src Random */ +#define F_IPDST_RND (1<<1) /* IP-Dst Random */ +#define F_UDPSRC_RND (1<<2) /* UDP-Src Random */ +#define F_UDPDST_RND (1<<3) /* UDP-Dst Random */ +#define F_MACSRC_RND (1<<4) /* MAC-Src Random */ +#define F_MACDST_RND (1<<5) /* MAC-Dst Random */ +#define F_TXSIZE_RND (1<<6) /* Transmit packet size is random */ +#define F_IPV6 (1<<7) /* Interface in IPV6 Mode */ +#define F_MPLS_RND (1<<8) /* Random MPLS labels */ + +/* Thread control flag bits */ +#define T_TERMINATE (1<<0) +#define T_STOP (1<<1) /* Stop run */ +#define T_RUN (1<<2) /* Start run */ +#define T_REMDEVALL (1<<3) /* Remove all devs */ +#define T_REMDEV (1<<4) /* Remove one dev */ +#define T_WAKE_BLOCKED (1<<5) /* Wake up all blocked net-devices. */ +#define T_ADD_DEV (1<<6) /* Add a device. */ + +/* Used to help with determining the pkts on receive */ +#define PKTGEN_MAGIC 0xbe9be955 +#define PG_PROC_DIR "pktgen" +#define PGCTRL "pgctrl" + +struct pktgen_dev { + + /* + * Try to keep frequent/infrequent used vars. separated. + */ + + char ifname[IFNAMSIZ]; + char result[512]; + + struct pktgen_thread *pg_thread; /* the owner */ + struct list_head list; /* Used for chaining in the thread's run-queue */ + + int running; /* if this changes to false, the test will stop */ + + /* If min != max, then we will either do a linear iteration, or + * we will do a random selection from within the range. + */ + __u32 flags; + int removal_mark; /* non-zero => the device is marked for + * removal by worker thread */ + + __u32 min_pkt_size; /* = ETH_ZLEN; */ + __u32 max_pkt_size; /* = ETH_ZLEN; */ + __u32 nfrags; + __u64 delay_ns; /* Delay this much between sending packets. */ + __u64 count; /* Default No packets to send */ + __u64 sofar; /* How many pkts we've sent so far */ + __u64 tx_bytes; /* How many bytes we've transmitted */ + __u64 errors; /* Errors when trying to transmit, pkts will be re-sent */ + __u64 nanodelays; /* how many times have we called nano-delay on this device? */ + __s64 accum_delay_ns; /* Accumulated delay..when >= 1ms, we'll sleep on a wait queue. */ + __u64 sleeps; /* How many times have we gone to sleep on the wait queue. */ + __u64 queue_stopped; /* How many times was queue stopped when we tried to xmit? */ + /* runtime counters relating to clone_skb */ + __u64 next_tx_ns; /* timestamp of when to tx next */ + __u64 req_tx_early; /* requested to tx, but is too early for us to tx. */ + + __u64 allocated_skbs; + __u32 clone_count; + + int tx_blocked; /* Need to tx as soon as able... */ + int last_ok; /* Was last skb sent? + * Or a failed transmit of some sort? This will keep + * sequence numbers in order, for example. + */ + __u64 started_at; /* micro-seconds */ + __u64 stopped_at; /* micro-seconds */ + __u64 idle_acc_ns; /* nano-seconds */ + __u32 seq_num; + + __u32 clone_skb; /* Use multiple SKBs during packet gen. If this number + * is greater than 1, then that many copies of the same + * packet will be sent before a new packet is allocated. + * For instance, if you want to send 1024 identical packets + * before creating a new packet, set clone_skb to 1024. + */ + __u32 peer_clone_skb; /* Peer (transmitter's) clone setting. */ + + char dst_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ + char dst_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ + char src_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ + char src_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ + + struct in6_addr in6_saddr; + struct in6_addr in6_daddr; + struct in6_addr cur_in6_daddr; + struct in6_addr cur_in6_saddr; + /* For ranges */ + struct in6_addr min_in6_daddr; + struct in6_addr max_in6_daddr; + struct in6_addr min_in6_saddr; + struct in6_addr max_in6_saddr; + + /* If we're doing ranges, random or incremental, then this + * defines the min/max for those ranges. + */ + __u32 saddr_min; /* inclusive, source IP address */ + __u32 saddr_max; /* exclusive, source IP address */ + __u32 daddr_min; /* inclusive, dest IP address */ + __u32 daddr_max; /* exclusive, dest IP address */ + + __u16 udp_src_min; /* inclusive, source UDP port */ + __u16 udp_src_max; /* exclusive, source UDP port */ + __u16 udp_dst_min; /* inclusive, dest UDP port */ + __u16 udp_dst_max; /* exclusive, dest UDP port */ + + /* MPLS */ + unsigned nr_labels; /* Depth of stack, 0 = no MPLS */ + __be32 labels[MAX_MPLS_LABELS]; + + __u32 src_mac_count; /* How many MACs to iterate through */ + __u32 dst_mac_count; /* How many MACs to iterate through */ + + unsigned char dst_mac[ETH_ALEN]; + unsigned char src_mac[ETH_ALEN]; + + __u32 cur_dst_mac_offset; + __u32 cur_src_mac_offset; + __u32 cur_saddr; + __u32 cur_daddr; + __u16 cur_udp_dst; + __u16 cur_udp_src; + __u32 cur_pkt_size; + + __u8 hh[14]; + /* = { + 0x00, 0x80, 0xC8, 0x79, 0xB3, 0xCB, + + We fill in SRC address later + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00 + }; + */ + __u16 pad; /* pad out the hh struct to an even 16 bytes */ + + struct sk_buff *skb; /* skb we are to transmit next, mainly used for when we + * are transmitting the same one multiple times + */ + struct net_device *odev; /* The out-going device. Note that the device should + * have it's pg_info pointer pointing back to this + * device. This will be set when the user specifies + * the out-going device name (not when the inject is + * started as it used to do.) + */ + struct flow_state *flows; + unsigned cflows; /* Concurrent flows (config) */ + unsigned lflow; /* Flow length (config) */ + unsigned nflows; /* accumulated flows (stats) */ + + int avg_latency; /* in micro-seconds */ + int min_latency; + int max_latency; + __u64 latency_bkts[LAT_BUCKETS_MAX]; + __u64 pkts_rcvd_since_clear; /* with regard to clearing/resetting the latency logic */ + + + /* Fields relating to receiving pkts */ + __u32 last_seq_rcvd; + __u64 ooo_rcvd; /* out-of-order packets received */ + __u64 pkts_rcvd; /* packets received */ + __u64 dup_rcvd; /* duplicate packets received */ + __u64 bytes_rcvd; /* total bytes received, as obtained from the skb */ + __u64 seq_gap_rcvd; /* how many gaps we received. This coorelates to + * dropped pkts, except perhaps in cases where we also + * have re-ordered pkts. In that case, you have to tie-break + * by looking at send v/s received pkt totals for the interfaces + * involved. + */ + __u64 non_pg_pkts_rcvd; /* Count how many non-pktgen skb's we are sent to check. */ + __u64 dup_since_incr; /* How many dumplicates since the last seq number increment, + * used to detect gaps when multiskb > 1 + */ +}; + +struct pktgen_hdr { + __u32 pgh_magic; + __u32 seq_num; + __u32 tv_sec; + __u32 tv_usec; +}; + +struct pktgen_thread { + struct list_head if_list; /* All device here */ + struct list_head th_list; + int removed; + char name[32]; + char result[512]; + u32 max_before_softirq; /* We'll call do_softirq to prevent starvation. */ + + /* Field for thread to receive "posted" events terminate, stop ifs etc. */ + + u32 control; + char* control_arg; + int pid; + int cpu; + + wait_queue_head_t queue; +}; + +struct pg_nqw_data { + #define PG_NQW_MAGIC 0x82743ab6 + u32 magic; + atomic_t nqw_ref_count; + struct pktgen_thread* pg_thread; +}; + +struct pktgen_dev_report { + __u32 api_version; + __u32 flags; + __u32 min_pkt_size; + __u32 max_pkt_size; + __u32 nfrags; + + __u32 clone_skb; /* Use multiple SKBs during packet gen. If this number + * is greater than 1, then that many copies of the same + * packet will be sent before a new packet is allocated. + * For instance, if you want to send 1024 identical packets + * before creating a new packet, set clone_skb to 1024. + */ + __u32 peer_clone_skb; /* Peer (transmitter's) clone setting. */ + __s32 avg_latency; /* in micro-seconds */ + __s32 min_latency; + __s32 max_latency; + + char thread_name[32]; + char interface_name[32]; + char dst_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ + char dst_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ + char src_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ + char src_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ + unsigned char dst_mac[ETH_ALEN]; + unsigned char src_mac[ETH_ALEN]; + __u32 pad_32; /* pad to 8-byte boundary */ + + /* If we're doing ranges, random or incremental, then this + * defines the min/max for those ranges. + */ + __u32 saddr_min; /* inclusive, source IP address */ + __u32 saddr_max; /* exclusive, source IP address */ + __u32 daddr_min; /* inclusive, dest IP address */ + __u32 daddr_max; /* exclusive, dest IP address */ + + __u16 udp_src_min; /* inclusive, source UDP port */ + __u16 udp_src_max; /* exclusive, source UDP port */ + __u16 udp_dst_min; /* inclusive, dest UDP port */ + __u16 udp_dst_max; /* exclusive, dest UDP port */ + + /* MPLS */ + __u32 nr_labels; /* Depth of stack, 0 = no MPLS */ + __be32 labels[MAX_MPLS_LABELS]; + + __u32 src_mac_count; /* How many MACs to iterate through */ + __u32 dst_mac_count; /* How many MACs to iterate through */ + + __u64 nflows; /* accumulated flows (stats) */ + __u32 cflows; /* Concurrent flows (config) */ + __u32 lflow; /* Flow length (config) */ + + __u64 delay_ns; /* Delay this much between sending packets. */ + __u64 count; /* Default No packets to send */ + __u64 sofar; /* How many pkts we've sent so far */ + __u64 tx_bytes; /* How many bytes we've transmitted */ + __u64 errors; /* Errors when trying to transmit, pkts will be re-sent */ + __u64 latency_bkts[LAT_BUCKETS_MAX]; + __u64 pkts_rcvd_since_clear; /* with regard to clearing/resetting the latency logic */ + + /* Fields relating to receiving pkts */ + __u64 ooo_rcvd; /* out-of-order packets received */ + __u64 pkts_rcvd; /* packets received */ + __u64 dup_rcvd; /* duplicate packets received */ + __u64 bytes_rcvd; /* total bytes received, as obtained from the skb */ + __u64 seq_gap_rcvd; /* how many gaps we received. This coorelates to + * dropped pkts, except perhaps in cases where we also + * have re-ordered pkts. In that case, you have to tie-break + * by looking at send v/s received pkt totals for the interfaces + * involved. + */ + __u64 non_pg_pkts_rcvd; /* Count how many non-pktgen skb's we are sent to check. */ + + struct in6_addr in6_saddr; + struct in6_addr in6_daddr; + /* For ranges */ + struct in6_addr min_in6_daddr; + struct in6_addr max_in6_daddr; + struct in6_addr min_in6_saddr; + struct in6_addr max_in6_saddr; + + char future_use[256]; /* Give us some room for growth w/out changing structure size */ +} __attribute__((__packed__)); + +/* Define some IOCTLs. Just picking random numbers, basically. */ +#define GET_PKTGEN_INTERFACE_INFO 0x7450 +struct pktgen_ioctl_info { + char thread_name[32]; + char interface_name[32]; + struct pktgen_dev_report report; +}; + + +/* Defined in dev.c */ +extern int (*handle_pktgen_hook)(struct sk_buff *skb); + +/* Returns < 0 if the skb is not a pktgen buffer. */ +int pktgen_receive(struct sk_buff* skb); + + +#endif