mod_tcplib.c

/*
 * Copyright (c) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001,
 *               2002, 2003, 2004
 *	Ohio University.
 *
 * ---
 * 
 * Starting with the release of tcptrace version 6 in 2001, tcptrace
 * is licensed under the GNU General Public License (GPL).  We believe
 * that, among the available licenses, the GPL will do the best job of
 * allowing tcptrace to continue to be a valuable, freely-available
 * and well-maintained tool for the networking community.
 *
 * Previous versions of tcptrace were released under a license that
 * was much less restrictive with respect to how tcptrace could be
 * used in commercial products.  Because of this, I am willing to
 * consider alternate license arrangements as allowed in Section 10 of
 * the GNU GPL.  Before I would consider licensing tcptrace under an
 * alternate agreement with a particular individual or company,
 * however, I would have to be convinced that such an alternative
 * would be to the greater benefit of the networking community.
 * 
 * ---
 *
 * This file is part of Tcptrace.
 *
 * Tcptrace was originally written and continues to be maintained by
 * Shawn Ostermann with the help of a group of devoted students and
 * users (see the file 'THANKS').  The work on tcptrace has been made
 * possible over the years through the generous support of NASA GRC,
 * the National Science Foundation, and Sun Microsystems.
 *
 * Tcptrace is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * Tcptrace is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Tcptrace (in the file 'COPYING'); if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 * MA 02111-1307 USA
 * 
 * Original Author: Eric Helvey
 * 		School of Electrical Engineering and Computer Science
 * 		Ohio University
 * 		Athens, OH
 *		ehelvey@cs.ohiou.edu
 *		http://www.tcptrace.org/
 * Extensively Modified:    Shawn Ostermann
 */
#include "tcptrace.h"
static char const GCC_UNUSED rcsid[] =
   "$Header$";

#ifdef LOAD_MODULE_TCPLIB

/****************************************************************************
 * 
 * Module Title: Mod_TCPLib 
 * 
 * Author: Eric Helvey
 * 
 * Purpose: To generate data files needed by TCPLib and TrafGen.
 * 
 ****************************************************************************/
#include "mod_tcplib.h"
#include "dyncounter.h"


/* reading old files is problematic and I never use it anyway!!!
   it probably doesn't work anymore.
   sdo - Thu Aug  5, 1999 */
#undef READ_OLD_FILES

/* we're no longer interested in the old phone/conv columns */
#undef INCLUDE_PHONE_CONV

/* Local global variables */

/* different types of "directions" */
#define NUM_DIRECTION_TYPES 4
enum t_dtype {LOCAL = 0, INCOMING = 1, OUTGOING = 2, REMOTE = 3};
static char *dtype_names[NUM_DIRECTION_TYPES] = {"local","incoming","outgoing", "remote"};

/* structure to keep track of "inside" */
struct insidenode {
    ipaddr min;
    ipaddr max;
    struct insidenode *next;
} *inside_head = NULL;
#define LOCAL_ONLY (inside_head == NULL)



/* for the parallelism hack */
#define BURST_KEY_MAGIC 0x49524720 /* 'I' 'R' 'G' '<space>' */
struct burstkey {
    unsigned long magic;	/* MUST be BURST_KEY_MAGIC */
    unsigned long nbytes;	/* bytes in burst (INCLUDING this struct) */
    unsigned char key;		/* one character key to return */
    unsigned char unused[3];	/* (explicit padding) */
    unsigned long groupnum;	/* for keeping track of parallel HTTP */
};



#ifdef BROKEN
char *BREAKDOWN_APPS_NAMES[] = {
    "app 1",
    "app 2",
    "app 3",
    "app 4",
    "app 5",
    "app 6",
    "app 7",
    "app 8"
};
#endif /* BROKEN */


/* for VM efficiency, we pull the info that we want out of the tcptrace
   structures into THIS structure (or large files thrash) */
typedef struct module_conninfo_tcb {
    /* cached connection type (incoming, remote, etc) */
    enum t_dtype dtype;

    /* cached data bytes */
    u_llong	data_bytes;

    /*
     * FTP: number of data connections against this control conn
     * HTTP:
     * NNTP: number of bursts
     * HTTP: number of bursts
     */
    u_long numitems;

    /* burst info */
    u_long	burst_bytes;	/* size of the current burst */
    struct burstdata *pburst;

    /* was the last segment PUSHed? */
    Bool last_seg_pushed;	/* Thu Aug 26, 1999 - not used  */

    /* last time new data was sent */
    timeval	last_data_time;

    /* link back to REAL information */
    tcb 	*ptcb;

    /* previous connection of same type */
    struct module_conninfo *prev_dtype_all;/* for ALL app types */
    struct module_conninfo *prev_dtype_byapp; /* just for THIS app type */
} module_conninfo_tcb;


/* structure that this module keeps for each connection */
#define TCB_CACHE_A2B 0
#define TCB_CACHE_B2A 1
#define LOOP_OVER_BOTH_TCBS(var) var=TCB_CACHE_A2B; var<=TCB_CACHE_B2A; ++var
typedef struct module_conninfo {
    /* cached info */
    struct module_conninfo_tcb tcb_cache[2];

    /* this connection should be ignored for breakdown/convarrival */
    Bool ignore_conn;

    /* breakdown type */
    short btype;

    /* cached copy of address pair */
    tcp_pair_addrblock	addr_pair;

    /* link back to the tcb's */
    tcp_pair *ptp;

    /* time of connection start */
    timeval	first_time;
    timeval	last_time;

    /* previous connection in linked list of all connections */
    struct module_conninfo *prev;

    /* for parallel http sessions */
    struct parallelism *pparallelism;

    /* unidirectional conns ignored totally */
    Bool unidirectional_http;

    /* for determining bursts */
    tcb *tcb_lastdata;

    /* to determine parallelism in conns, trafgen encodes a group
       number in the data */
    u_long http_groupnum;
    
    /* next connection in linked list by endpoint pairs */
    struct module_conninfo *next_pair;
} module_conninfo;
module_conninfo *module_conninfo_tail = NULL;


/* data structure to store endpoint pairs */
typedef struct endpoint_pair {
    /* endpoint identification */
    tcp_pair_addrblock	addr_pair;

    /* linked list of connections using that pair */
    module_conninfo *pmchead;

    /* next address pair */
    struct endpoint_pair *pepnext;
} endpoint_pair;
#define ENDPOINT_PAIR_HASHSIZE 1023


/* for tracking burst data */
struct burstdata {
    dyn_counter nitems;		/* total items (bursts) in connection */
    dyn_counter size;		/* size of the items */
    dyn_counter idletime;	/* idle time between bursts */
};

/* for tracking number of connections */
struct parallelism {
    Bool	counted[NUM_DIRECTION_TYPES];
    				/* have we already accumulated this? */
				/* (in each of the 4 directions) */
    Bool	persistant[2];	/* is this persistant (for each TCB) */
    u_short	maxparallel;	/* maximum degree of parallelism */
    u_long	ttlitems[2];	/* across entire group (each dir) */
};



static struct tcplibstats {
    /* telnet packet sizes */
    dyn_counter telnet_pktsize;

    /* telnet interarrival times */
    dyn_counter telnet_interarrival;

    /* conversation interarrival times */
    dyn_counter conv_interarrival_all;

    /* protocol-specific interarrival times */
    dyn_counter conv_interarrival_byapp[NUM_APPS];

    /* conversation duration */
    dyn_counter conv_duration;

    /* for the interval breakdowns */
    int interval_count;
    timeval last_interval;
    int tcplib_breakdown_interval[NUM_APPS];

    /* histogram files */
    MFILE *hist_file;

    /* for NNTP, we track: */
    /* # items per connection */
    /* idletime between items */
    /* burst size */
    struct burstdata nntp_bursts;


    /* for HTTP1.0, we track: */
    /* # items per connection */
    /* # connections */
    /* idletime between items */
    /* burst size */
    struct burstdata http_P_bursts;
    dyn_counter http_P_maxconns; /* max degree of concurrency */
    dyn_counter http_P_ttlitems; /* ttl items across whole parallel group */
    dyn_counter http_P_persistant; /* which parallel groups are persistant */

    /* for HTTP1.1, we track: */
    /* # items per connection */
    /* idletime between items */
    /* burst size */
    struct burstdata http_S_bursts;

    /* telnet packet sizes */
    dyn_counter throughput;
    int throughput_bytes;
} *global_pstats[NUM_DIRECTION_TYPES] = {NULL};


/* local debugging flag */
static int ldebug = 0;

/* parallelism for our TRAFGEN files */
static Bool trafgen_generated = FALSE;

/* offset for all ports */
static int ipport_offset = 0;

/* the name of the directory (prefix) for the output */
static char *output_dir = DEFAULT_TCPLIB_DATADIR;

/* the name of the current tcptrace input file */
static char *current_file = NULL;

/* characters to print in interval breakdown file */
static const char breakdown_hash_char[] = { 'S', 'N', 'T', 'F', 'H', 'f'};


/* FTP endpoints hash table */
endpoint_pair *ftp_endpoints[ENDPOINT_PAIR_HASHSIZE];

/* HTTP endpoints hash table */
endpoint_pair *http_endpoints[ENDPOINT_PAIR_HASHSIZE];


/* internal types */
typedef Bool (*f_testinside) (module_conninfo *pmc,
			      module_conninfo_tcb *ptcbc);

/* various statistics and counters */
static u_long debug_newconn_counter;	/* total conns */
static u_long debug_newconn_badport;	/* a port we don't want */
static u_long debug_newconn_goodport;	/* we want the port */
static u_long debug_newconn_ftp_data_heuristic; /* merely ASSUMED to be ftp data */
static u_llong debug_total_bytes; /* total "bytes" accepted */

/* parallel http counters */
static u_long debug_http_total; /* all HTTP conns */
static u_long debug_http_parallel; /* parallel HTTP, not counted in breakdown/conv */
static u_long debug_http_single;
static u_long debug_http_groups;
static u_long debug_http_slaves;
static u_long debug_http_uni_conns; /* data in at most one direction, ignored */
static u_llong debug_http_uni_bytes; /* data in at most one direction, ignored */
static u_long debug_http_persistant;
static u_long debug_http_nonpersistant;
/* conns by type */
static u_long conntype_counter[NUM_DIRECTION_TYPES];
/* both flows have data */
static u_long conntype_duplex_counter[NUM_DIRECTION_TYPES];
/* this flow has data, twin is empty */
static u_long conntype_uni_counter[NUM_DIRECTION_TYPES];
/* this flow has NO data, twin is NOT empty */
static u_long conntype_nodata_counter[NUM_DIRECTION_TYPES];
/* neither this flow OR its twin has data */
static u_long conntype_noplex_counter[NUM_DIRECTION_TYPES];



/* Function Prototypes */
static void ParseArgs(char *argstring);
static int breakdown_type(tcp_pair *ptp);
static void do_final_breakdown(char* filename, f_testinside p_tester,
			       struct tcplibstats *pstats);
static void do_all_final_breakdowns(void);
static void do_all_conv_arrivals(void);
static void do_tcplib_final_converse(char *filename,
				     char *protocol, dyn_counter psizes);
static void do_tcplib_next_converse(module_conninfo_tcb *ptcbc,
				    module_conninfo *pmc);
static void do_tcplib_conv_duration(char *filename,
				    dyn_counter psizes);
static void do_tcplib_next_duration(module_conninfo_tcb *ptcbc,
				    module_conninfo *pmc);
static void tcplib_cleanup_bursts(void);
static void tcplib_save_bursts(void);
static Bool is_parallel_http(module_conninfo *pmc_new);
static void tcplib_filter_http_uni(void);

/* prototypes for connection-type determination */
static Bool is_ftp_ctrl_port(portnum port);
static Bool is_ftp_data_port(portnum port);
static Bool is_http_port(portnum port);
static Bool is_nntp_port(portnum port);
static Bool is_smtp_port(portnum port);
static Bool is_telnet_port(portnum port);

/* shorthand */
#define is_ftp_ctrl_conn(pmc)	(pmc->btype == TCPLIBPORT_FTPCTRL)
#define is_ftp_data_conn(pmc)	(pmc->btype == TCPLIBPORT_FTPDATA)
#define is_http_conn(pmc)	(pmc->btype == TCPLIBPORT_HTTP)
#define is_nntp_conn(pmc)	(pmc->btype == TCPLIBPORT_NNTP)
#define is_smtp_conn(pmc)	(pmc->btype == TCPLIBPORT_SMTP)
#define is_telnet_conn(pmc)	(pmc->btype == TCPLIBPORT_TELNET)


static char* namedfile(char *localsuffix, char * file);
static void setup_breakdown(void);
static void tcplib_add_telnet_interarrival(tcp_pair *ptp,
					   module_conninfo *pmc,
					   dyn_counter *psizes);
static void tcplib_add_telnet_packetsize(struct tcplibstats *pstats,
					 int length);
static void tcplib_do_ftp_control_size(char *filename, f_testinside p_tester);
static void tcplib_do_ftp_itemsize(char *filename, f_testinside p_tester);
static void tcplib_do_ftp_numitems(char *filename, f_testinside p_tester);
static void tcplib_do_smtp_itemsize(char *filename, f_testinside p_tester);
static void tcplib_do_telnet_duration(char *filename, f_testinside p_tester);
static void tcplib_do_telnet_interarrival(char *filename,
					  f_testinside p_tester);
static void tcplib_do_telnet_packetsize(char *filename,
					f_testinside p_tester);
static void tcplib_init_setup(void);
static void update_breakdown(tcp_pair *ptp, struct tcplibstats *pstats);
module_conninfo *FindPrevConnection(module_conninfo *pmc,
				    enum t_dtype dtype, int app_type);
static char *FormatBrief(tcp_pair *ptp,tcb *ptcb);
static char *FormatAddrBrief(tcp_pair_addrblock *addr_pair);
static void ModuleConnFillcache(void);


/* prototypes for determining "insideness" */
static void DefineInside(char *iplist);
static Bool IsInside(ipaddr *pipaddr);

static Bool TestOutgoing(module_conninfo*, module_conninfo_tcb *ptcbc);
static Bool TestIncoming(module_conninfo*, module_conninfo_tcb *ptcbc);
static Bool TestLocal(module_conninfo*, module_conninfo_tcb *ptcbc);
static Bool TestRemote(module_conninfo*, module_conninfo_tcb *ptcbc);

static int InsideBytes(module_conninfo*, f_testinside);
static enum t_dtype traffic_type(module_conninfo *pmc,
				   module_conninfo_tcb *ptcbc);

/* prototypes for endpoint pairs */
static void TrackEndpoints(module_conninfo *pmc);
static hash EndpointHash(tcp_pair_addrblock *addr_pair);
static hash IPHash(ipaddr *paddr);
static Bool SameEndpoints(tcp_pair_addrblock *paddr_pair1,
			  tcp_pair_addrblock *paddr_pair2);
static struct module_conninfo_tcb *MostRecentFtpControl(endpoint_pair *pep);
static Bool CouldBeFtpData(tcp_pair *ptp);
static void AddEndpointPair(endpoint_pair *hashtable[],
			    module_conninfo *pmc);
static endpoint_pair *FindEndpointPair(endpoint_pair *hashtable[],
				       tcp_pair_addrblock *paddr_pair);
static Bool IsNewBurst(module_conninfo *pmc, tcb *ptcb,
		       module_conninfo_tcb *ptcbc,
		       struct tcphdr *tcp);


/* various helper routines used by many others -- sdo */
static Bool ActiveConn(module_conninfo *pmc);
static Bool RecentlyActiveConn(module_conninfo *pmc);
static void tcplib_do_GENERIC_itemsize(
    char *filename, int btype,
    f_testinside p_tester, int bucketsize);
static void tcplib_do_GENERIC_burstsize(
    char *filename, dyn_counter counter);
static void tcplib_do_GENERIC_P_maxconns(
    char *filename, dyn_counter counter);
static void tcplib_do_GENERIC_nitems(
    char *filename, dyn_counter counter);
static void tcplib_do_GENERIC_idletime(
    char *filename, dyn_counter counter);
static void StoreCounters(char *filename, char *header1, char *header2,
			  int bucketsize, dyn_counter psizes);
#ifdef READ_OLD_FILES
static dyn_counter ReadOldFile(char *filename, int bucketsize,
				 int maxlegal, dyn_counter psizes);
#endif /* READ_OLD_FILES */



/* First section is comprised of functions that TCPTrace will call
 * for all modules.
 */

/***************************************************************************
 * 
 * Function Name: tcplib_init
 * 
 * Returns:  TRUE/FALSE whether or not the tcplib module for tcptrace
 *           has been requested on the command line.
 *
 * Purpose: To parse the command line arguments for the tcplib module's
 *          command line flags, return whether or not to run the module,
 *          and to set up the local global variables needed to generate
 *          the tcplib data files.
 *
 * Called by: LoadModules() in tcptrace.c
 * 
 * 
 ****************************************************************************/
int tcplib_init(
    int argc,      /* Number of command line arguments */
    char *argv[]   /* Command line arguments */
    )
{
    int i;             /* Runner for command line arguments */
    int enable = 0;    /* Do we turn on this module, or not? */
    char *args = NULL;

    for(i = 0; i < argc; i++) {
	if (!argv[i])
	    continue;

	/* See if they want to use us */
	if ((argv[i] != NULL) && (strncmp(argv[i], "-xtcplib", 8) == 0)) {
	    /* Calling the Tcplib part */
	    enable = 1;
	    args = argv[i]+(sizeof("-xtcplib")-1);

	    printf("Capturing TCPLib traffic\n");

	    /* We free this argument so that no other modules
	     * or the main program mis-interprets this flag.
	     */
	    argv[i] = NULL;

	    continue;
	}
    }

    /* If enable is not true, then all tcplib functions will
     * be ignored during this run of the program.
     */
    if (!enable)
	return(0);	/* don't call me again */

    /* parse the encoded args */
    ParseArgs(args);

    /* init internal data */
    tcplib_init_setup();

    /* don't care for detailed output! */
    printsuppress = TRUE; 

    return TRUE;
}


/* wants strings of the form IP1-IP2 */
static struct insidenode *
DefineInsideRange(
    char *ip_pair)
{
    char *pdash;
    struct insidenode *pnode;
    ipaddr *paddr;
    char *paddr1;
    char *paddr2;

    if (ldebug>2)
	printf("DefineInsideRange('%s') called\n", ip_pair);

    pdash = strchr(ip_pair,'-');
    if (pdash == NULL) {
	/* just one address, treat it as a range */
	paddr1 = ip_pair;
	paddr2 = ip_pair;
    } else {
	/* a pair */
	*pdash = '\00';
	paddr1 = ip_pair;
	paddr2 = pdash+1;
    }


    pnode = MallocZ(sizeof(struct insidenode));

    paddr = str2ipaddr(paddr1);
    if (paddr == NULL) {
	fprintf(stderr,"invalid IP address: '%s'\n", paddr1);
	exit(-1);
    }
    pnode->min = *paddr;


    paddr = str2ipaddr(paddr2);
    if (paddr == NULL) {
	fprintf(stderr,"invalid IP address: '%s'\n", paddr2);
	exit(-1);
    }
    pnode->max = *paddr;

    return(pnode);
}


static struct insidenode *
DefineInsideRecurse(
    char *iplist)
{
    char *pcomma;
    struct insidenode *left;

    /* find commas and recurse */
    pcomma = strchr(iplist,',');

    if (pcomma) {
	*pcomma = '\00';
	left = DefineInsideRecurse(iplist);
	left->next = DefineInsideRecurse(pcomma+1);

	return(left);
    } else {
	/* just one term left */
	return(DefineInsideRange(iplist));
    }
}



static void
DefineInside(
    char *iplist)
{
    
    if (ldebug>2)
	printf("DefineInside(%s) called\n", iplist);

    inside_head = DefineInsideRecurse(iplist);

    if (ldebug) {
	struct insidenode *phead;
	printf("DefineInside: result:\n  ");
	for (phead=inside_head; phead; phead=phead->next) {
	    printf("(%s <= addr", HostAddr(phead->min));
	    printf(" <= %s)", HostAddr(phead->max));
	    if (phead->next)
		printf(" OR ");
	}
	printf("\n");
    }
}


static Bool
IsInside(
    ipaddr *paddr)
{
    struct insidenode *phead;

    /* if use didn't specify "inside", then EVERYTHING is "inside" */
    if (LOCAL_ONLY)
	return(TRUE);

    for (phead = inside_head; phead; phead=phead->next) {
	int cmp1 = IPcmp(&phead->min, paddr);
	int cmp2 = IPcmp(&phead->max, paddr);

	if ((cmp1 == -2) || (cmp2 == -2)) {
	    /* not all the same address type, fail */
	    return(FALSE);
	}

	if ((cmp1 <= 0) &&	/* min <= addr */
	    (cmp2 >= 0))	/* max >= addr */
	    return(TRUE);
    }
    return(FALSE);
}

static Bool
TestOutgoing(
    module_conninfo *pmc,
    module_conninfo_tcb *ptcbc)
{
    if (ptcbc == &pmc->tcb_cache[TCB_CACHE_A2B])
	return( IsInside(&pmc->addr_pair.a_address) &&
	       !IsInside(&pmc->addr_pair.b_address));
    else
	return( IsInside(&pmc->addr_pair.b_address) &&
	       !IsInside(&pmc->addr_pair.a_address));
}

static Bool
TestIncoming(
    module_conninfo *pmc,
    module_conninfo_tcb *ptcbc)
{
    if (ptcbc == &pmc->tcb_cache[TCB_CACHE_A2B])
	return(!IsInside(&pmc->addr_pair.a_address) &&
	        IsInside(&pmc->addr_pair.b_address));
    else
	return(!IsInside(&pmc->addr_pair.b_address) &&
	        IsInside(&pmc->addr_pair.a_address));
}


static Bool
TestLocal(
    module_conninfo *pmc,
    module_conninfo_tcb *ptcbc)
{
    return(IsInside(&pmc->addr_pair.a_address) &&
	   IsInside(&pmc->addr_pair.b_address));
}


static Bool
TestRemote(
    module_conninfo *pmc,
    module_conninfo_tcb *ptcbc)
{
    return(!IsInside(&pmc->addr_pair.a_address) &&
	   !IsInside(&pmc->addr_pair.b_address));
}


static int InsideBytes(
    module_conninfo *pmc,
    f_testinside p_tester)	/* function to test "insideness" */
{
    int temp = 0;
    int dir;

    for (LOOP_OVER_BOTH_TCBS(dir)) {
	/* if "p_tester" likes this side of the connection, count the bytes */
	if ((*p_tester)(pmc, &pmc->tcb_cache[dir]))
	    temp += pmc->tcb_cache[dir].data_bytes;
    }

    return(temp);
}


static void
ParseArgs(char *argstring)
{
    int argc;
    char **argv;
    int i;
    
    /* make sure there ARE arguments */
    if (!(argstring && *argstring))
	return;

    /* break the string into normal arguments */
    StringToArgv(argstring,&argc,&argv);

    /* check the module args */
    for (i=1; i < argc; ++i) {
	/* The "-o####" flag sets the offset that we're going
	 * to consider for tcplib data files.  The reason is that
	 * for verification purposes, when trafgen creates traffic 
	 * it sends it to non-standard ports.  So, in order to get
	 * a data set from generated traffic, we'd have to remove
	 * the offset.  The -o allows us to do that.
	 */
	if (argv[i] && !strncmp(argv[i], "-o", 2)) {
	    ipport_offset = atoi(argv[i]+2);

	    if (!ipport_offset) {
		fprintf(stderr, "\
Invalid argument to flag \"-o\".\n\
Must be integer value greater than 0.\n");
		exit(1);
	    }

	    printf("TCPLib port offset - %d\n", ipport_offset);
	}


	/* The "-iIPs" gives the definition of "inside".  When it's used,
	 * we divide the the data into four sets:
	 * data.incoming:
	 *    for all data flowing from "inside" to "outside"
	 * data.outgoing:
	 *    for all data flowing from "outside" to "inside"
	 * data.local:
	 *    for all data flowing from "inside" to "inside"
	 * data.remote:
	 *    for all data flowing from "outside" to "outside"
	 *        (probably an error)
	 */
	else
	if (argv[i] && !strncmp(argv[i], "-i", 2)) {
	    if (!isdigit((int)*(argv[i]+2))) {
		fprintf(stderr,"-i requires IP address list\n");
		tcplib_usage();
		exit(-1);
	    }

	    DefineInside(argv[i]+2);
	}


	/* parallelism hack */
	else
	if (argv[i] && !strncmp(argv[i], "-H", 2)) {
	    trafgen_generated = TRUE;
	}

	/* local debugging flag */
	else
	if (argv[i] && !strncmp(argv[i], "-d", 2)) {
	    ++ldebug;
	}



	/* We will probably need to add another flag here to
	 * specify the directory in which to place the data
	 * files.  And here it is.
	 */
	else if (argv[i] && !strncmp(argv[i], "-D", 2)) {
	    char *pdir = argv[i]+2;

	    if (!pdir) {
		fprintf(stderr,"argument -DDIR requires directory name\n");
		exit(-1);
	    }

	    output_dir = strdup(pdir);

	    printf("TCPLib output directory - %sdata\n", output_dir);
	}

	/*  ... else invalid */
	else {
	    fprintf(stderr,"tcplib module: bad argument '%s'\n",
		    argv[i]);
	    exit(-1);
	}
    }

}

static void
tcplib_save_bursts()
{
    int dtype;
    module_conninfo *pmc;
    int non_parallel = 0;
    char *filename;

    tcplib_cleanup_bursts();
    

    /* accumulate parallelism stats */
    for (dtype=0; dtype < NUM_DIRECTION_TYPES; ++dtype) {
	non_parallel = 0;
	for (pmc = module_conninfo_tail; pmc; pmc = pmc->prev) {
	    struct parallelism *pp = pmc->pparallelism;
	    int dir;

	    /* make sure it's http */
	    if (!is_http_conn(pmc))
		continue;

	    /* ignore unidirectional */
	    if (pmc->unidirectional_http)
		continue;

	    /* check each TCB */
	    for (LOOP_OVER_BOTH_TCBS(dir)) {
		module_conninfo_tcb *ptcbc = &pmc->tcb_cache[dir];

		/* make sure it's
		    -- the right direction
		    -- and parallel
		    -- not already counted */
		if (ptcbc->dtype != dtype)
		    continue;

		if (pp == NULL) {
		    ++non_parallel;
		    continue;
		    }

		if (pp->counted[dtype])
		    continue;

		/* count the max connections */
		AddToCounter(&global_pstats[dtype]->http_P_maxconns,
			     pp->maxparallel, 1, 1);

		/* count the ttl items in the parallel group */
		AddToCounter(&global_pstats[dtype]->http_P_ttlitems,
			     pp->ttlitems[dir],
			     1, GRAN_NUMITEMS);

		/* binary counter, one sample of either: */
		/*  1: NOT persistant */
		/*  2: persistant */
		AddToCounter(&global_pstats[dtype]->http_P_persistant,
			     pp->persistant[dir]?2:1,
			     1, 1);

		/* debugging */
		if (pp->persistant[dir])
		    ++debug_http_persistant;
		else
		    ++debug_http_nonpersistant;

		/* don't count it again! */
		pmc->pparallelism->counted[dtype] = TRUE;
	    }
	}

	/* add the NON-parallel HTTP to the counter */
	AddToCounter(&global_pstats[dtype]->http_P_maxconns, 1,
		     non_parallel, 1);
    }


    /* write all the counters */
    for (dtype=0; dtype < NUM_DIRECTION_TYPES; ++dtype) {
	if (ldebug>1)
	    printf("tcplib: running burstsizes (%s)\n", dtype_names[dtype]);

	/* ---------------------*/
	/*   Burstsize		*/
	/* ---------------------*/
	/* HTTP 1.0 */
	filename = namedfile(dtype_names[dtype],TCPLIB_HTTP_P_BURSTSIZE_FILE);
	tcplib_do_GENERIC_burstsize(filename,
				    global_pstats[dtype]->http_P_bursts.size);

	/* HTTP 1.1 */
	filename = namedfile(dtype_names[dtype],TCPLIB_HTTP_S_BURSTSIZE_FILE);
	tcplib_do_GENERIC_burstsize(filename,
				    global_pstats[dtype]->http_S_bursts.size);

	/* NNTP */
	filename = namedfile(dtype_names[dtype],TCPLIB_NNTP_BURSTSIZE_FILE);
	tcplib_do_GENERIC_burstsize(filename,
				    global_pstats[dtype]->nntp_bursts.size);

	/* ---------------------*/
	/*   Total parallel items */
	/* ---------------------*/
	/* HTTP 1.0 */
	filename = namedfile(dtype_names[dtype],TCPLIB_HTTP_P_TTLITEMS_FILE);
	tcplib_do_GENERIC_nitems(filename,
				 global_pstats[dtype]->http_P_ttlitems);

	/* ---------------------*/
	/*   Num Items in Burst */
	/* ---------------------*/
	/* HTTP 1.1 */
	filename = namedfile(dtype_names[dtype],TCPLIB_HTTP_S_NITEMS_FILE);
	tcplib_do_GENERIC_nitems(filename,
				 global_pstats[dtype]->http_S_bursts.nitems);

	/* NNTP */
	filename = namedfile(dtype_names[dtype],TCPLIB_NNTP_NITEMS_FILE);
	tcplib_do_GENERIC_nitems(filename,
				 global_pstats[dtype]->nntp_bursts.nitems);

	/* ---------------------*/
	/*   Idletime		*/
	/* ---------------------*/

	/* HTTP 1.0 */
	filename = namedfile(dtype_names[dtype],TCPLIB_HTTP_P_IDLETIME_FILE);
	tcplib_do_GENERIC_idletime(filename,
				   global_pstats[dtype]->http_P_bursts.idletime);

	/* HTTP 1.1 */
	filename = namedfile(dtype_names[dtype],TCPLIB_HTTP_S_IDLETIME_FILE);
	tcplib_do_GENERIC_idletime(filename,
				   global_pstats[dtype]->http_S_bursts.idletime);

	/* NNTP */
	filename = namedfile(dtype_names[dtype],TCPLIB_NNTP_IDLETIME_FILE);
	tcplib_do_GENERIC_idletime(filename,
				   global_pstats[dtype]->nntp_bursts.idletime);

	/* store the counters */
	filename = namedfile(dtype_names[dtype],TCPLIB_HTTP_P_MAXCONNS_FILE);
	tcplib_do_GENERIC_P_maxconns(filename,
				     global_pstats[dtype]->http_P_maxconns);
	
	/* store the persistance */
	filename = namedfile(dtype_names[dtype],TCPLIB_HTTP_P_PERSIST_FILE);
	tcplib_do_GENERIC_nitems(filename,
				 global_pstats[dtype]->http_P_persistant);
	

	if (LOCAL_ONLY)
	    break;
    }
}




/***************************************************************************
 * 
 * Function Name: tcplib_done
 * 
 * Returns: Nothing
 *
 * Purpose: This function runs after all the packets have been read in
 *          and filed.  The functions that tcplib_done calls are the ones
 *          that generate the data files.
 *
 * Called by: FinishModules() in tcptrace.c
 * 
 * 
 ****************************************************************************/
static void
RunAllFour(
    void (*f_runme) (char *,f_testinside),
    char *thefile)
{
    char *filename;

    filename = namedfile("local",thefile);
    (*f_runme)(filename,TestLocal);

    if (LOCAL_ONLY)
	return;  /* none of the rest will match anyway */

    filename = namedfile("incoming",thefile);
    (*f_runme)(filename,TestIncoming);

    filename = namedfile("outgoing",thefile);
    (*f_runme)(filename,TestOutgoing);

    filename = namedfile("remote",thefile);
    (*f_runme)(filename,TestRemote);

}
void tcplib_done()
{
    char *filename;
    int i;

    /* fill the info cache */
    if (ldebug)
	printf("tcplib: completing data structure\n");
    ModuleConnFillcache();
    
    /* do TELNET */
    if (ldebug)
	printf("tcplib: running telnet\n");
    RunAllFour(tcplib_do_telnet_packetsize,TCPLIB_TELNET_PACKETSIZE_FILE);
    RunAllFour(tcplib_do_telnet_interarrival,TCPLIB_TELNET_INTERARRIVAL_FILE);
    RunAllFour(tcplib_do_telnet_duration,TCPLIB_TELNET_DURATION_FILE);



    /* do FTP */
    if (ldebug)
	printf("tcplib: running ftp\n");
    RunAllFour(tcplib_do_ftp_control_size,TCPLIB_FTP_CTRLSIZE_FILE);
    RunAllFour(tcplib_do_ftp_itemsize,TCPLIB_FTP_ITEMSIZE_FILE);
    RunAllFour(tcplib_do_ftp_numitems,TCPLIB_FTP_NITEMS_FILE);



    /* do SMTP */
    if (ldebug)
	printf("tcplib: running smtp\n");
    RunAllFour(tcplib_do_smtp_itemsize,TCPLIB_SMTP_ITEMSIZE_FILE);



    /* do NNTP */
    if (ldebug)
	printf("tcplib: running nntp\n");


    /* do HTTP */
    if (ldebug)
	printf("tcplib: running http\n");

    /* filter out the unidirectional HTTP (server pushes) */
    tcplib_filter_http_uni();


    /* for efficiency, do all burst size stuff together */
    if (ldebug)
	printf("tcplib: running burstsizes\n");
    tcplib_save_bursts();


    /* do the breakdown stuff */
    if (ldebug)
	printf("tcplib: running breakdowns\n");
    do_all_final_breakdowns();


    /* do the conversation interrival time */
    if (ldebug)
	printf("tcplib: running conversation interarrival times\n");
    do_all_conv_arrivals();
    for (i=0; i < NUM_DIRECTION_TYPES; ++i) {
	if (ldebug>1)
	    printf("tcplib: running conversation arrivals (%s)\n",
		   dtype_names[i]);
	filename = namedfile(dtype_names[i],TCPLIB_NEXT_CONVERSE_FILE);
	do_tcplib_final_converse(filename, "total",
				 global_pstats[i]->conv_interarrival_all);

#ifdef BROKEN
	/* do the application-specific tables for Mark */
	for (j=0; j < NUM_APPS; ++j) {
	    char new_filename[128];
	    char *app_name = BREAKDOWN_APPS_NAMES[j];
	    snprintf(new_filename,sizeof(new_filename),"%s_%s", filename, app_name);
	    
	    do_tcplib_final_converse(new_filename, app_name,
				     global_pstats[i]->conv_interarrival_byapp[j]);
	}
#endif /* BROKEN */

	/* do conversation durations */
	filename = namedfile(dtype_names[i],TCPLIB_CONV_DURATION_FILE);
	do_tcplib_conv_duration(filename,
				global_pstats[i]->conv_duration);

	if (LOCAL_ONLY)
	    break;
    }

    /* print stats */
    debug_http_single = debug_http_total - debug_http_parallel;
    printf("tcplib: total connections seen: %lu (%lu accepted, %lu bad port)\n",
	   debug_newconn_counter, debug_newconn_goodport, debug_newconn_badport);
    printf("tcplib: total bytes seen: %" FS_ULL "\n",
	   debug_total_bytes);
    printf("tcplib: %lu random connections accepted under FTP data heuristic\n",
	   debug_newconn_ftp_data_heuristic);
    printf("tcplib: %lu HTTP conns (%lu parallel, %lu single)\n",
	   debug_http_total,
	   debug_http_parallel,
	   debug_http_single);
    printf("tcplib: %lu HTTP conns (%lu single, %lu leaders, %lu slaves)\n",
	   debug_http_total,
	   debug_http_single,
	   debug_http_groups,
	   debug_http_slaves);
    printf("tcplib: %lu groups (%lu persistant ||, %lu nonpersistant ||)\n",
	   debug_http_groups,
	   debug_http_persistant,
	   debug_http_nonpersistant);
    printf("tcplib: %lu (%.2f%%) unidir. HTTP conns (%" FS_ULL " bytes, %.2f%%) ignored\n",
	   debug_http_uni_conns,
	   100.0 * ((float)debug_http_uni_conns /
		    (float)(debug_newconn_counter + debug_http_uni_conns)),
	   debug_http_uni_bytes,
	   100.0 * ((float)debug_http_uni_bytes /
		    (float)(debug_total_bytes + debug_http_uni_bytes)));
	   
    for (i=0; i < NUM_DIRECTION_TYPES; ++i) {
	printf("  Flows of type %-8s %5lu (%lu duplex, %lu noplex, %lu unidir, %lu nodata)\n",
	       dtype_names[i],
	       conntype_counter[i],
	       conntype_duplex_counter[i],
	       conntype_noplex_counter[i],
	       conntype_uni_counter[i],
	       conntype_nodata_counter[i]);
    }

    /* dump HTTP groups for debugging */
    {
	module_conninfo *pmc;
	printf("Group Numbers for HTTP conns\n");
	for (pmc = module_conninfo_tail; pmc; pmc = pmc->prev) {
	    tcb *ptcb = pmc->tcb_cache[TCB_CACHE_A2B].ptcb;

	    if (pmc->btype != TCPLIBPORT_HTTP)
		continue;
	    
	    if (pmc->unidirectional_http)
		continue;

	    printf("%s: %30s\tGROUPNUM %5lu\tdata %" FS_ULL ":%" FS_ULL "\n",
		   ts2ascii(&ptcb->ptp->first_time),
		   FormatBrief(ptcb->ptp, ptcb),
		   pmc->http_groupnum,
		   pmc->tcb_cache[0].data_bytes,
		   pmc->tcb_cache[1].data_bytes);
	}

	printf("Unidirectional HTTP conns (ignored)\n");
	for (pmc = module_conninfo_tail; pmc; pmc = pmc->prev) {
	    tcb *ptcb = pmc->tcb_cache[TCB_CACHE_A2B].ptcb;

	    if (pmc->btype != TCPLIBPORT_HTTP)
		continue;
	    
	    if (!pmc->unidirectional_http)
		continue;
		
		printf("%s: %30s\tGROUPNUM %5lu\tdata %" FS_ULL ":%" FS_ULL "\n",
		   ts2ascii(&ptcb->ptp->first_time),
		   FormatBrief(ptcb->ptp, ptcb),
		   pmc->http_groupnum,
		   pmc->tcb_cache[0].data_bytes,
		   pmc->tcb_cache[1].data_bytes);
	}
    }
    

    return;
}








/***************************************************************************
 * 
 * Function Name: tcplib_read
 * 
 * Returns: Nothing
 *
 * Purpose: This function is called each time a packet is read in by
 *          tcptrace.  tcplib_read examines the packet, and keeps track
 *          of certain information about the packet based on the packet's
 *          source and/or destination ports.
 *
 * Called by: ModulesPerPacket() in tcptrace.c
 * 
 * 
 ****************************************************************************/
void tcplib_read(
    struct ip *pip,    /* The packet */
    tcp_pair *ptp,     /* The pair of hosts - basically the conversation */
    void *plast,       /* Unused here */
    void *pmodstruct   /* Nebulous structure used to hold data that the module
			* feels is important. */
    )
{
    struct tcphdr *tcp;  /* TCP header information */
    int data_len = 0;    /* Length of the data cargo in the packet, and
			  * the period of time between the last two packets
			  * in a conversation */
    tcb *ptcb;
    module_conninfo_tcb *ptcbc;
    struct tcplibstats *pstats;
    module_conninfo *pmc = pmodstruct;
    enum t_dtype dtype;
    int dir;

    /* first, discard any connections that we aren't interested in. */
    /* That means that pmodstruct is NULL */
    if (pmc == NULL) {
	return;
    }


    /* Setting a pointer to the beginning of the TCP header */
    tcp = (struct tcphdr *) ((char *)pip + (4 * IP_HL(pip)));

    /* calculate the amount of user data */
    data_len = pip->ip_len -	/* size of entire IP packet (and IP header) */
	(4 * IP_HL(pip)) -	/* less the IP header */
	(4 * TH_OFF(tcp));	/* less the TCP header */

    /* stats */
    debug_total_bytes += data_len;

    /* see which of the 2 TCB's this goes with */
    if (ptp->addr_pair.a_port == ntohs(tcp->th_sport)) {
	ptcb = &ptp->a2b;
	dir = TCB_CACHE_A2B;
    } else {
	ptcb = &ptp->b2a;
	dir = TCB_CACHE_B2A;
    }
    ptcbc = &pmc->tcb_cache[dir];


    /* see where to keep the stats */
    dtype = traffic_type(pmc,ptcbc);
    pstats = global_pstats[dtype];

    /* Let's do the telnet packet sizes.  Telnet packets are the only
     * ones where we actually care about the sizes of individual packets.
     * All the other connection types are a "send as fast as possible" 
     * kind of setup where the packet sizes are always optimal.  Because
     * of this, we need the size of each and every telnet packet that 
     * comes our way. */
    if (is_telnet_conn(pmc)) {
	if (data_len > 0) {
	    if (ldebug>2)
		printf("read: adding %d byte telnet packet to %s\n",
		       data_len, dtype_names[dtype]);
	    tcplib_add_telnet_packetsize(pstats,data_len);
	}
    }


    /* Here's where we'd need to do telnet interarrival times.  The
     * same basic scenario applies with telnet packet interarrival
     * times.  Because telnet type traffic is "stop and go", we need
     * to be able to model how long the "stops" are.  So we measure
     * the time in between successive packets in a single telnet
     * conversation. */
    if (is_telnet_conn(pmc)) {
	tcplib_add_telnet_interarrival(
	    ptp, pmc, &pstats->telnet_interarrival);
    }


    /* keep track of bytes/second too */
    if (data_len > 0) {
	static timeval last_time = {0,0};
	unsigned etime;

	/* accumulate total bytes */
	pstats->throughput_bytes += data_len;

	/* elapsed time in milliseconds */
	etime = (int)(elapsed(last_time, current_time)/1000.0);

	/* every 15 seconds, gather throughput stats */
	if (etime > 15000) {
	    AddToCounter(&pstats->throughput,
			 pstats->throughput_bytes, etime, 1024);
	    pstats->throughput_bytes = 0;
	    last_time = current_time;
	}

    }


    /* create data for traffic breakdown over time file */
    /* (sdo - only count packets with DATA) */
    if (data_len > 0) {
	int a2b_btype = pmc->btype;

	if (a2b_btype != TCPLIBPORT_NONE) {
	    pstats->tcplib_breakdown_interval[a2b_btype] +=
		ptp->a2b.data_bytes;
	}
    }

    /* if it's http and we don't already know that it's
       parallel, and this is the first data, and we're
       looking at trafgen data, check the group number encoded
       in the data stream */
    if (is_http_conn(pmc) &&
	trafgen_generated &&
	(ptcb->data_bytes == data_len) &&
	data_len >= sizeof(struct burstkey)) {
	u_char *pdata = (u_char *)tcp + TH_OFF(tcp)*4;
	int available =  (char *)plast - (char *)pdata + 1;
	struct burstkey *pburst;

	if (0)
	    printf("Looking for burst key (in %d bytes of data, %d bytes available)\n",
		   data_len, available);

	/* see where the burst key should be */
	pburst = (void *) pdata;

	if (pburst->magic == BURST_KEY_MAGIC) {
	    pmc->http_groupnum = ntohl(pburst->groupnum);
	    if (ldebug>1)
		printf("FOUND BURST KEY in %s, group num is %lu!\n",
		       FormatBrief(ptcb->ptp, ptcb),
		       pmc->http_groupnum);

	    /* check for parallelism */
	    if (is_parallel_http(pmc)) {
		pmc->ignore_conn = TRUE;
	    }
	}
    }


    /* DATA Burst checking (NNTP and HTTP only) */
    if ((data_len > 0) &&
	(is_nntp_conn(pmc) || is_http_conn(pmc))) {


	/* see if it's a new burst */
	if (IsNewBurst(pmc, ptcb, ptcbc, tcp)) {
	    int etime;

	    if (ldebug > 1)
		printf("New burst starts at time %s for %s\n",
		       ts2ascii(&current_time),
		       FormatBrief(pmc->ptp,ptcb));


	    /* count the PREVIOUS burst item */
	    /* NB: the last is counted in tcplib_cleanup_bursts() */
	    ++ptcbc->numitems;

	    /* special burst handling for HTTP */
	    if (is_http_conn(pmc) && pmc->pparallelism) {
		struct parallelism *pp = pmc->pparallelism;

		if (ptcbc->numitems > 1) {
		    pp->persistant[dir] = TRUE;
		}

		/* add to total bursts in parallel group */
		++pp->ttlitems[dir];
	    }

	    /* accumulate burst size stats */
	    if (ldebug>1)
		printf("Adding burst size %ld to %s\n",
		       ptcbc->burst_bytes,
		       FormatBrief(pmc->ptp,ptcb));
	    AddToCounter(&ptcbc->pburst->size,
			 ptcbc->burst_bytes,
			 1, GRAN_BURSTSIZE);

	    /* reset counter for next burst */
	    ptcbc->burst_bytes = 0;

	    /* determine idle time (elapsed time in milliseconds) */
	    etime = (int)(elapsed(ptcbc->last_data_time,
				  current_time)/1000.0);

	    /* accumulate idletime stats */

	    /* version 2.0 - Thu Aug 26, 1999, subtract the RTT */
	    /* use rtt_last, RTT of last "good ack" */
	    if (ptcb->rtt_last > 0.0) {
		int last_good_rtt = ptcb->rtt_last / 1000.0;
#ifdef OLD
		printf("last_good: %d (%f), etime_0: %d  etime_1: %d\n",
		       last_good_rtt, ptcb->rtt_last, etime, etime-last_good_rtt);
#endif /* OLD */
		etime -= last_good_rtt;

		if (etime >= 0)
		    AddToCounter(&ptcbc->pburst->idletime,
				 etime, 1, GRAN_BURSTIDLETIME);
	    }
	}

	/* accumulate size of current burst */
	ptcbc->burst_bytes += data_len;

	/* remember when the last data was sent (for idletime) */
	ptcbc->last_data_time = current_time;
    }

    /* This is just a sanity check to make sure that we've got at least
     * one time, and that our breakdown section is working on the same
     * file that we are. */
    data_len = (current_time.tv_sec - pstats->last_interval.tv_sec);
    
    if (data_len >= TIMER_VAL) {
	update_breakdown(ptp, pstats);
    }

    /* analysis done, remember the last packet and PUSH status */
    pmc->last_time = current_time;
    ptcbc->last_seg_pushed = PUSH_SET(tcp);

    return;
}





/******************************************************************
 *
 * fill the tcb cache, make the 'previous' linked lists
 *
 ******************************************************************/
static void
ModuleConnFillcache(
    void)
{
    module_conninfo *pmc;
    enum t_dtype dtype;
    int dir;

    /* fill the cache */
    for (pmc = module_conninfo_tail; pmc ; pmc=pmc->prev) {
	tcp_pair *ptp = pmc->ptp;	/* shorthand */
	int a2b_bytes = ptp->a2b.data_bytes;
	int b2a_bytes = ptp->b2a.data_bytes;

	/* both sides byte counters */
	pmc->tcb_cache[TCB_CACHE_A2B].data_bytes = a2b_bytes;
	pmc->tcb_cache[TCB_CACHE_B2A].data_bytes = b2a_bytes;

	/* debugging stats */
	if ((a2b_bytes == 0) && (b2a_bytes == 0)) {
	    /* no bytes at all */
	    ++conntype_noplex_counter[pmc->tcb_cache[TCB_CACHE_A2B].dtype];
	    ++conntype_noplex_counter[pmc->tcb_cache[TCB_CACHE_B2A].dtype];
	} else if ((a2b_bytes != 0) && (b2a_bytes == 0)) {
	    /* only A2B has bytes */
	    ++conntype_uni_counter[pmc->tcb_cache[TCB_CACHE_A2B].dtype];
	    ++conntype_nodata_counter[pmc->tcb_cache[TCB_CACHE_B2A].dtype];
	} else if ((a2b_bytes == 0) && (b2a_bytes != 0)) {
	    /* only B2A has bytes */
	    ++conntype_nodata_counter[pmc->tcb_cache[TCB_CACHE_A2B].dtype];
	    ++conntype_uni_counter[pmc->tcb_cache[TCB_CACHE_B2A].dtype];
	} else {
	    /* both sides have bytes */
	    ++conntype_duplex_counter[pmc->tcb_cache[TCB_CACHE_A2B].dtype];
	    ++conntype_duplex_counter[pmc->tcb_cache[TCB_CACHE_B2A].dtype];
	}

	    
	/* globals */
	pmc->last_time = ptp->last_time;
    }


    for (dtype = LOCAL; dtype <= REMOTE; ++dtype) {
	/* do the A sides, then the B sides */
	for (LOOP_OVER_BOTH_TCBS(dir)) {
	    int app;
	    if (ldebug>1)
		printf("  Making previous for %s, side %s\n",
		       dtype_names[dir], (dir==TCB_CACHE_A2B)?"A":"B");

	    /* do conversation interravial calculations by app */
	    for (app=-1; app <= NUM_APPS; ++app) {
		/* note: app==-1 means ALL apps */
		for (pmc = module_conninfo_tail; pmc ; ) {
		    module_conninfo_tcb *ptcbc = &pmc->tcb_cache[dir];

		    /* if app != -1, we just want SOME of them */
		    if ((app != -1) && (pmc->btype != app)) {
			/* don't want this one, try the next one */
			pmc = pmc->prev;
			continue;
		    }

		    if (ptcbc->dtype == dtype) {
			module_conninfo *prev
			    = FindPrevConnection(pmc,dtype,app);
			if (app == -1)
			    ptcbc->prev_dtype_all = prev;
			else
			    ptcbc->prev_dtype_byapp = prev;
			pmc = prev;
		    } else {
			pmc = pmc->prev;
		    }
		}
	    }
	}
    }
}



/******************************************************************
 *
 * to improve efficiency, we try to keep all of these on the
 * same virual pages
 *
 ******************************************************************/
static module_conninfo *
NewModuleConn()
{
#define CACHE_SIZE 128
    static module_conninfo *pcache[CACHE_SIZE];
    static int num_cached = 0;
    module_conninfo *p;

    if (num_cached == 0) {
	int i;
	char *ptmp;

	ptmp = MallocZ(CACHE_SIZE*sizeof(module_conninfo));

	for (i=0; i < CACHE_SIZE; ++i) {
	    pcache[i] = (module_conninfo *)ptmp;
	    ptmp += sizeof(module_conninfo);
	}

	num_cached = CACHE_SIZE;
    }

    /* grab one from the cache and return it */
    p = pcache[--num_cached];
    return(p);
}



/***************************************************************************
 * 
 * Function Name: tcplib_newconn
 * 
 * Returns: The time of this connection.  This becomes the pmodstruct that
 *          is returned with each call to tcplib_read.
 *
 * Purpose: To setup and handle new connections.
 *
 * Called by: ModulesPerConn() in tcptrace.c
 * 
 * 
 ****************************************************************************/
void *
tcplib_newconn(
    tcp_pair *ptp)   /* This conversation */
{
    int btype;			/* breakdown type */
    module_conninfo *pmc;
                                  /* Pointer to a timeval structure.  The
				   * timeval structure becomes the time of
				   * the last connection.  The pmc
				   * is tcptrace's way of allowing modules
				   * to keep track of information about
				   * connections */

    /* trafgen only uses a few ports... */
    if (trafgen_generated) {
	u_short server_port = ptp->addr_pair.b_port;

	if ((server_port < ipport_offset+IPPORT_FTP_DATA) ||
	    (server_port > ipport_offset+IPPORT_NNTP)) {
	    ++debug_newconn_badport;
	    return(NULL);
	}
    }

    /* verify that it's a connection we're interested in! */
    ++debug_newconn_counter;
    btype = breakdown_type(ptp);
    if (btype == TCPLIBPORT_NONE) {
	++debug_newconn_badport;
	return(NULL); /* so we won't get it back in tcplib_read() */
    } else {
	/* else, it's acceptable, count it */
	++debug_newconn_goodport;
    }

    /* create the connection-specific data structure */
    pmc = NewModuleConn();
    pmc->first_time = current_time;
    pmc->ptp = ptp;
    pmc->tcb_cache[TCB_CACHE_A2B].ptcb = &ptp->a2b;
    pmc->tcb_cache[TCB_CACHE_B2A].ptcb = &ptp->b2a;

    /* cache the address info */
    pmc->addr_pair = ptp->addr_pair;

    /* determine its "insideness" */
    pmc->tcb_cache[TCB_CACHE_A2B].dtype = traffic_type(pmc, &pmc->tcb_cache[TCB_CACHE_A2B]);
    pmc->tcb_cache[TCB_CACHE_B2A].dtype = traffic_type(pmc, &pmc->tcb_cache[TCB_CACHE_B2A]);
    ++conntype_counter[pmc->tcb_cache[TCB_CACHE_A2B].dtype];
    ++conntype_counter[pmc->tcb_cache[TCB_CACHE_B2A].dtype];

    /* determine the breakdown type */
    pmc->btype = btype;

    /* chain it in */
    pmc->prev = module_conninfo_tail;
    module_conninfo_tail = pmc;

    /* setup the burst counter shorthand */
    if ((btype == TCPLIBPORT_NNTP) ||
	(btype == TCPLIBPORT_HTTP)) {
	module_conninfo_tcb *ptcbc;
	struct tcplibstats *pstats;
	int dir;

/* 	printf("NewConn, saw btype %d for %s\n", btype, */
/* 	       FormatBrief(ptp)); */
	

	for (LOOP_OVER_BOTH_TCBS(dir)) {
	    ptcbc = &pmc->tcb_cache[dir];
	    pstats = global_pstats[ptcbc->dtype];

	    if (btype == TCPLIBPORT_NNTP) {
		ptcbc->pburst = &pstats->nntp_bursts;
	    } else if (btype == TCPLIBPORT_HTTP) {
		/* assume 1.1 unless we see parallelism later */
		ptcbc->pburst = &pstats->http_S_bursts;
	    }

	    ptcbc->last_data_time = current_time;
	}
    }


    /* debugging counter */
    if (btype == TCPLIBPORT_HTTP)
	++debug_http_total;


    /* add to list of endpoints we track */
    TrackEndpoints(pmc);

    /* if it's NOT trafgen generated and it's HTTP, check if it's
       parallel */
    if (is_http_conn(pmc) && !trafgen_generated) {
	if (is_parallel_http(pmc)) {
	    pmc->ignore_conn = TRUE;
	}
    }


    return (pmc);
}






/***************************************************************************
 * 
 * Function Name: tcplib_newfile
 * 
 * Returns: Nothing
 *
 * Purpose: This function is called by tcptrace every time that a new
 *          trace file is opened.  tcplib_newfile basically sets up a new
 *          line in the breakdown file, so that we can get a picture of
 *          the traffic distribution for a single trace.
 *
 * Called by: ModulesPerFile() in tcptrace.c
 * 
 * 
 ****************************************************************************/
void tcplib_newfile(
    char *filename,     /* Name of the file just opened. */
    u_long filesize,
    Bool fcompressed
    )
{
    static int first_file = TRUE;

    /* If this isn't the first file that we've seen this run, then
     * we want to run do_final_breakdown on the file we ran BEFORE
     * this one. */
    if (!first_file) {
	do_all_final_breakdowns();
	free(current_file);
    } else {
	/* If this is the first file we've seen, then we just want to 
	 * record the name of this file, and do nothing until the file
	 * is done. */
	printf("%s", filename);
	first_file = FALSE;
    }	

    /* remember the current file name */
    current_file = (char *) strdup(filename);

    setup_breakdown();

    return;
}







/***************************************************************************
 * 
 * Function Name: tcplib_usage
 * 
 * Returns: Nothing
 *
 * Purpose: To print out usage instructions for this module.
 *
 * Called by: ListModules() in tcptrace.c
 * 
 * 
 ****************************************************************************/
void tcplib_usage()
{
    printf("\
\t-xtcplib\"[ARGS]\"\tgenerate tcplib-format data files from trace\n");
    printf("\
\t  -oN      set port offset to N, default is 0\n\
\t           for example, we normally find telnet at 23, but\n\
\t           if it's at 9023, then use \"-o9000\"\n\
\t  -iIPLIST\n\
\t           define the IP addresses which are \"inside\".  Format allows\n\
\t           ranges and commas, as in:\n\
\t               -i128.1.0.0-128.2.255.255\n\
\t               -i128.1.0.0-128.2.255.255,192.10.1.0-192.10.2.240\n\
\t  -H       use hacks to find data from trafgen-generated files\n\
\t  -DDIR    store the results in directory DIR, default is \"data\"\n\
");
}

/* End of the tcptrace standard function section */








/***************************************************************************
 * 
 * Function Name: tcplib_init_setup
 * 
 * Returns: Nothing
 *
 * Purpose:  To setup and initialize the tcplib module's set of 
 *           global variables.
 *
 * Called by: tcplib_init() in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
static void tcplib_init_setup(void)
{
    int i;   /* Loop Counter */
    enum t_dtype ix;
    struct tcplibstats *pstats;

    /* We need to save the contents in order to piece together the answers
     * later on
     *
     * sdo - so why does Eric turn it OFF?
     */
    save_tcp_data = FALSE;


    for (ix = LOCAL; ix <= REMOTE; ++ix) {
	/* create the big data structure */
	global_pstats[ix] = pstats = MallocZ(sizeof(struct tcplibstats));

	for(i = 0; i < NUM_APPS; i++) {
	    pstats->tcplib_breakdown_interval[i] = 0;
	}
    }

    setup_breakdown();

    return;
}



/***************************************************************************
 * 
 * Function Name: setup_breakdown
 * 
 * Returns: Nothing
 *
 * Purpose: To open the traffic breakdown graph file, and to set the
 *          interval count.
 *
 * Called by: tcplib_init_setup() in mod_tcplib.c
 *            tcplib_newfile()    in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
static void setup_breakdown(void)
{
    int ix;
    
    for (ix = LOCAL; ix <= REMOTE; ++ix) {
	struct tcplibstats *pstats = global_pstats[ix];
	char *prefix = dtype_names[ix];
	char *filename = namedfile(prefix,TCPLIB_BREAKDOWN_GRAPH_FILE);

	if (!(pstats->hist_file = Mfopen(filename, "w"))) {
	    perror(filename);
	    exit(1);
	}

	pstats->interval_count = 0;
    }
}




/***************************************************************************
 * 
 * Function Name: update_breakdown
 * 
 * Returns: Nothing
 *
 * Purpose: To create a file containing a kind of histogram of traffic
 *          seen in this file.  The histogram would contain one row per
 *          a set # of seconds, and would display one characteristic
 *          character per a specified number of bytes.
 *
 * Called by: tcplib_read() in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
static void update_breakdown(
    tcp_pair *ptp,      /* This conversation */
    struct tcplibstats *pstats)
{
    int i;        /* Looping variable */
    int count;
    
    /* Displays the interval number.  A new histogram line is displayed
     * at TIMER_VALUE seconds. */
    Mfprintf(pstats->hist_file, "%d\t", pstats->interval_count);

    /* Display some characters for each type of traffic */
    for(i = 0; i < NUM_APPS; i++) {
	struct tcplibstats *pstats = global_pstats[LOCAL];

	/* We'll be displaying one character per BREAKDOWN_HASH number
	   of bytes */
	count = (pstats->tcplib_breakdown_interval[i] / BREAKDOWN_HASH) + 1;

	/* If there was actually NO traffic of that type, then we don't
	 * want to display any characters.  But if there was a little bit
	 * of traffic, even much less than BREAKDOWN_HASH, we want to 
	 * acknowledge it. */
	if (!pstats->tcplib_breakdown_interval[i])
	    count--;

	/* Print one hash char per count. */
	while(count > 0) {
	    Mfprintf(pstats->hist_file, "%c", breakdown_hash_char[i]);
	    count--;
	}
    }

    /* After we've done all the applications, end the line */
    Mfprintf(pstats->hist_file, "\n");

    /* Zero out the counters */
    for(i = 0; i < NUM_APPS; i++) {
	pstats->tcplib_breakdown_interval[i] = 0;
    }

    /* Update the breakdown interval */
    pstats->interval_count++;

    /* Update the time that the last breakdown interval occurred. */
    pstats->last_interval = current_time;
}



/***************************************************************************
 * 
 * Function Name: namedfile
 * 
 * Returns: Relative path name attached to output file name.
 *
 * Purpose: The namedfile uses the -D command line argument to take a data
 *          directory and puts it together with its default file name to
 *          come up with the file name needed for output.
 *
 * Called by: do_final_breakdown() in mod_tcplib.c
 *            do_tcplib_final_converse() in mod_tcplib.c
 *            tcplib_do_telnet_duration() in mod_tcplib.c
 *            tcplib_do_telnet_interarrival() in mod_tcplib.c
 *            tcplib_do_telnet_pktsize() in mod_tcplib.c
 *            tcplib_do_ftp_itemsize() in mod_tcplib.c
 *            tcplib_do_ftp_control_size() in mod_tcplib.c
 *            tcplib_do_smtp_itemsize() in mod_tcplib.c
 *            tcplib_do_nntp_itemsize() in mod_tcplib.c
 *            tcplib_do_http_itemsize() in mod_tcplib.c
 * 
 ****************************************************************************/
static char *
namedfile(
    char * localsuffix,
    char * real)  /* Default file name for the output file */
{
    char directory[256];
    static char buffer[256];    /* Buffer to store the full file name */

    if (!LOCAL_ONLY)
	snprintf(directory,sizeof(directory),"%s_%s", output_dir, localsuffix);
    else
	snprintf(directory,sizeof(directory),"%s", output_dir);

    /* try to CREATE the directory if it doesn't exist */
    if (access(directory,F_OK) != 0) {
	if (mkdir(directory,0755) != 0) {
	    perror(directory);
	    exit(-1);
	}
	if (ldebug>1)
	    printf("Created directory '%s'\n", directory);
    }

    snprintf(buffer,sizeof(buffer),"%s/%s", directory, real);

    return buffer;
}




/***************************************************************************
 * 
 * Function Name: do_final_breakdown
 * 
 * Returns: Nothing
 *
 * Purpose: To generate the final breakdown file.  More specifically, to
 *          generate the one line in the breakdown file associated with
 *          the input file that is currently being traced.
 *
 * Called by: tcplib_done()    in mod_tcplib.c
 *            tcplib_newfile() in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
static void do_final_breakdown(
    char *filename,
    f_testinside p_tester,
    struct tcplibstats *pstats)
{
    module_conninfo *pmc;
    MFILE* fil;        /* File descriptor for the traffic breakdown file */
    long file_pos;    /* Offset within the traffic breakdown file */
    u_long num_parallel_http = 0;


    /* This is the header for the traffic breakdown file.  It follows the
     * basic format of the original TCPLib breakdown file, but has been
     * modified to accomodate the additions that were made to TCPLib */
#ifdef INCLUDE_PHONE_CONV
    char *header = "stub\tsmtp\tnntp\ttelnet\tftp\thttp\tphone\tconv\n";
#else /* INCLUDE_PHONE_CONV */
    char *header = "stub             smtp\tnntp\ttelnet\tftp\thttp\n";
#endif /* INCLUDE_PHONE_CONV */

    if (!(fil = Mfopen(filename, "a"))) {
	perror("Opening Breakdown File");
	exit(1);
    }

    Mfseek(fil, 0, SEEK_END);
    file_pos = Mftell(fil);

    /* Basically, we're checking to see if this file has already been
     * used.  We have the capability to both start a new set of data
     * based on a trace file, or we have the ability to incorporate one
     * trace file's data into the data from another trace.  This would
     * have the effect of creating a hybrid traffic pattern, that matches
     * neither of the sources, but shares characteristics of both. */
    if (file_pos < strlen(header)) {
	Mfprintf(fil, "%s", header);
    }

    /* We only do this next part if we actually have a file name.  In
     * earlier revisions, sending a NULL filename signified the end of
     * all trace files.  At this point, a NULL file name has no useful
     * purpose, so we ignore it completely. */
    if (current_file) {
	int bad_port = 0;
	int no_data = 0;
	int bad_dir = 0;

	/* for protocol breakdowns */
	int breakdown_protocol[NUM_APPS] = {0};

	/* The breakdown file line associated with each trace file is
	 * prefaced with the trace file's name.  This was part of the
	 * original TCPLib format. */
	Mfprintf(fil, "%-16s ", current_file);

	/* count the connections of each protocol type */
	for (pmc = module_conninfo_tail; pmc ; pmc = pmc->prev) {
	    int protocol_type;
	    module_conninfo_tcb *ptcbc;

	    /* check the protocol type */
	    protocol_type = pmc->btype;
	    if (protocol_type == TCPLIBPORT_NONE) {
		++bad_port;
		continue;	/* not interested, loop to next conn */
	    }

	    /* count the parallel HTTP separately */
	    if (pmc->ignore_conn) {
		if (protocol_type == TCPLIBPORT_HTTP) {
		    ++num_parallel_http;
		    continue;
		}
	    }

	    /* see if we want A->B */
	    ptcbc = &pmc->tcb_cache[TCB_CACHE_A2B];
	    if ((*p_tester)(pmc, ptcbc)) {
		/* count it if there's data */
		if (ptcbc->data_bytes > 0) {
		    if (pmc->ignore_conn && is_http_conn(pmc))
			++num_parallel_http;
		    else
			++breakdown_protocol[protocol_type];
		} else {
		    ++no_data;
		}
	    } else {
		/* see if we want B->A */
		ptcbc = &pmc->tcb_cache[TCB_CACHE_B2A];
		if ((*p_tester)(pmc, ptcbc)) {
		    /* count it if there's data */
		    if (ptcbc->data_bytes > 0) {
			if (pmc->ignore_conn && is_http_conn(pmc))
			    ++num_parallel_http;
			else
			    ++breakdown_protocol[protocol_type];
		    } else {
			++no_data;
		    }
		} else {
		    ++bad_dir;
		}
	    }
	}

	/* Print out each of the columns we like */
	/* SMTP */
	Mfprintf(fil, "%.4f\t",
		((float)breakdown_protocol[TCPLIBPORT_SMTP])/ num_tcp_pairs);

	/* NNTP */
	Mfprintf(fil, "%.4f\t",
		((float)breakdown_protocol[TCPLIBPORT_NNTP])/ num_tcp_pairs);

	/* TELNET */
	Mfprintf(fil, "%.4f\t",
		((float)breakdown_protocol[TCPLIBPORT_TELNET])/ num_tcp_pairs);

	/* FTP */
	Mfprintf(fil, "%.4f\t",
		((float)breakdown_protocol[TCPLIBPORT_FTPCTRL])/ num_tcp_pairs);

	/* HTTP */
	Mfprintf(fil, "%.4f\t",
		((float)breakdown_protocol[TCPLIBPORT_HTTP])/ num_tcp_pairs);

#ifdef UNDEF
	/* FTP Data */
	Mfprintf(fil, "%.4f\t",
		((float)breakdown_protocol[TCPLIBPORT_FTPDATA])/ num_tcp_pairs);

	/* Parallel HTTP */
	Mfprintf(fil, "%.4f\t",
		((float)num_parallel_http)/ num_tcp_pairs);
#endif /* UNDEF */

#ifdef INCLUDE_PHONE_CONV
	/* Place holders for phone and converstation intervals.  The
	 * phone type was never fully developed in the original TCPLib
	 * implementation.  At the current time, we don't consider
	 * phone type conversations.  The placeholder for conversation
	 * intervals allows us to use TCPLib's existing setup for
	 * aquiring statistics.  Without a placeholder in the
	 * breakdown file, TCPLib won't recognize this particular
	 * item, and in the generation of statistically equivalent
	 * traffic patterns, the interval between converstaions is of
	 * utmost importance, especially as far as the scalability of
	 * traffic is concerned. */
	Mfprintf(fil, "%.4f\t%.4f", (float)0, (float)0);
#endif /* INCLUDE_PHONE_CONV */
	Mfprintf(fil, "\n");

    }

    Mfclose(fil);
    Mfclose(pstats->hist_file);
}

static void do_all_final_breakdowns(void)
{
    char *filename;
    
    filename = namedfile("local",TCPLIB_BREAKDOWN_FILE);
    do_final_breakdown(filename, TestLocal,
		       global_pstats[LOCAL]);

    if (LOCAL_ONLY)
	return;  /* none of the rest will match anyway */

    filename = namedfile("incoming",TCPLIB_BREAKDOWN_FILE);
    do_final_breakdown(filename, TestIncoming,
		       global_pstats[INCOMING]);

    filename = namedfile("outgoing",TCPLIB_BREAKDOWN_FILE);
    do_final_breakdown(filename, TestOutgoing,
		       global_pstats[OUTGOING]);

    filename = namedfile("remote",TCPLIB_BREAKDOWN_FILE);
    do_final_breakdown(filename, TestRemote,
		       global_pstats[REMOTE]);
}



/***************************************************************************
 * 
 * Function Name: breakdown_type
 * 
 * Returns: The generic type of connection associated with "port"
 *
 * Purpose: To convert the port given to the function to the appropriate
 *          TCPLib type port.  As we come across other ports that have the
 *          same basic characteristics as TCPLib type, we can just add
 *          them here.
 *
 * Called by: tcplib_read()        in mod_tcplib.c
 *            do_final_breakdown() in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
static int breakdown_type(
    tcp_pair *ptp)
{
    /* shorthand */
    portnum porta = ptp->addr_pair.a_port;
    portnum portb = ptp->addr_pair.b_port;

    if (is_telnet_port(porta) || 
	is_telnet_port(portb))
	return(TCPLIBPORT_TELNET);

    if (is_ftp_ctrl_port(porta) ||
	is_ftp_ctrl_port(portb))
	return(TCPLIBPORT_FTPCTRL);

    if (is_smtp_port(porta) ||
	is_smtp_port(portb))
	return(TCPLIBPORT_SMTP);

    if (is_nntp_port(porta) ||
	is_nntp_port(portb))
	return(TCPLIBPORT_NNTP);

    if (is_http_port(porta) ||
	is_http_port(portb))
	return(TCPLIBPORT_HTTP);
    
    if (is_ftp_data_port(porta) ||
	is_ftp_data_port(portb) ||
	CouldBeFtpData(ptp))
	return(TCPLIBPORT_FTPDATA);

    return TCPLIBPORT_NONE;
}

/* End Breakdown Stuff */







/* Begin Next Conversation Stuff */

/***************************************************************************
 * 
 * Function Name: do_tcplib_next_converse
 * 
 * Returns: Nothing
 *
 * Purpose: This function takes a new conversation and deals with the time
 *          between successive conversations.  If an entry in the breakdown
 *          table already exists with that particular time, then the counter
 *          is simply incremented.  If not, then a new table is made with a
 *          space for the new table item.  We're using arrays, but a change
 *          might be made to use a linked list before too long.
 *
 * Called by: tcplib_newconn() in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
static void do_tcplib_next_converse(
    module_conninfo_tcb *ptcbc,
    module_conninfo *pmc)
{
    struct tcplibstats *pstats;
    module_conninfo *pmc_previous;
    enum t_dtype dtype;
    int etime;   /* Time difference between the first packet in this
		  * conversation and the first packet in the previous
		  * conversation.  Basically, this is the time between
		  * new conversations. */

    /* see where to keep the stats */
    dtype = traffic_type(pmc, ptcbc);
    pstats = global_pstats[dtype];

    if (ldebug>2) {
	printf("do_tcplib_next_converse: %s, %s\n",
	       FormatBrief(pmc->ptp, ptcbc->ptcb), dtype_names[dtype]);
    }


    /* sdo - Wed Jun 16, 1999 */
    /* new method, search backward to find the previous connection that had */
    /* data flowing in the same "direction" and then use the difference */
    /* between the starting times of those two connections as the conn */
    /* interrival time */
    /* sdo - Fri Jul  9, 1999 (information already computed in Fillcache) */

    /* FIRST, do conversation interarrivals for ALL conns */
    if (ptcbc->prev_dtype_all != NULL) {
	pmc_previous = ptcbc->prev_dtype_all;
	/* elapsed time since that previous connection started */
	etime = (int)(elapsed(pmc_previous->first_time,
			      pmc->first_time)/1000.0); /* convert us to ms */

	/* keep stats */
	AddToCounter(&pstats->conv_interarrival_all, etime, 1, 1);
    }


    /* THEN, do conversation interarrivals by APP type */
    if (ptcbc->prev_dtype_byapp != NULL) {
	pmc_previous = ptcbc->prev_dtype_byapp;
	/* elapsed time since that previous connection started */
	etime = (int)(elapsed(pmc_previous->first_time,
			      pmc->first_time)/1000.0); /* convert us to ms */

	/* keep stats */
	AddToCounter(&pstats->conv_interarrival_byapp[pmc->btype],
		     etime, 1, 1);
    }

    return;
}

/* End of the breakdown section */


/* return the previous connection that passes data in the direction */
/* given in "dtype" and has app type apptype (or -1 for any) */
module_conninfo *
FindPrevConnection(
    module_conninfo *pmc,
    enum t_dtype dtype,
    int app_type)
{
    module_conninfo_tcb *ptcbc;
    int count = 0;

    /* loop back further in time */
    for (pmc = pmc->prev; pmc; pmc = pmc->prev) {
	int dir;

	/* ignore FTP Data and parallel HTTP */
	if (pmc->ignore_conn)
	    continue;
	
	for (LOOP_OVER_BOTH_TCBS(dir)) {
	    ptcbc = &pmc->tcb_cache[dir];
	    if ((app_type != -1) && (app_type != pmc->btype)) {
		/* skip it, wrong app */
	    }
	    if (ptcbc->dtype == dtype) {
		if (ptcbc->data_bytes != 0)
		    return(pmc);
	    }

	    if (ldebug)
		++count;
	}
    }

    if (ldebug > 1)
	printf("FindPrevConnection %s returned NULL, took %d searches\n",
	       dtype_names[dtype], count);

    return(NULL);
}



/***************************************************************************
 * 
 * Function Name: do_tcplib_final_converse
 * 
 * Returns: Nothing
 *
 * Purpose: To generate a new line in the breakdown file which shows the
 *          conversation percentages viewed in the file that is currently
 *          open, but has just been ended.
 *
 * Called by: tcplib_done() in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
static void
do_tcplib_final_converse(
    char *filename,
    char *protocol,
    dyn_counter psizes)
{
    const int bucketsize = GRAN_CONVARRIVAL;
    char title[80];


#ifdef READ_OLD_FILES
    /* sdo - OK, pstats->conv_interarrival already has the counts we */
    /* made.  First, include anything from an existing file. */
    psizes = ReadOldFile(filename, bucketsize, 0, psizes);
#endif /* READ_OLD_FILES */

    /* generate the graph title of the table */
    snprintf(title,sizeof(title),"Conversation Interval Time (ms) - %s", protocol);

    /* Now, dump out the combined data */
    StoreCounters(filename,title, "% Interarrivals", bucketsize, psizes);

    return;
}

static void
do_tcplib_conv_duration(
    char *filename,
    dyn_counter psizes)
{
    const int bucketsize = GRAN_CONVDURATION;

#ifdef READ_OLD_FILES
    psizes = ReadOldFile(filename, bucketsize, 0, psizes);
#endif /* READ_OLD_FILES */

    /* Now, dump out the combined data */
    StoreCounters(filename,"Conversation Duration (ms)",
		  "% Conversations", bucketsize, psizes);

    return;
}

static void do_tcplib_next_duration(
    module_conninfo_tcb *ptcbc,
    module_conninfo *pmc)
{
    struct tcplibstats *pstats;
    enum t_dtype dtype;
    int etime;   /* Time difference between the first packet in this
		  * conversation and the last packet */

    /* see where to keep the stats */
    dtype = traffic_type(pmc, ptcbc);
    pstats = global_pstats[dtype];

    if (ldebug>2) {
	printf("do_tcplib_next_duration: %s, %s\n",
	       FormatBrief(pmc->ptp, ptcbc->ptcb), dtype_names[dtype]);
    }


    /* elapsed time since that previous connection started */
    etime = (int)(elapsed(pmc->first_time,
			  pmc->last_time)/1000.0); /* convert us to ms */

    /* keep stats */
    AddToCounter(&pstats->conv_duration, etime, 1, GRAN_CONVDURATION);
    
    return;
}

/* End Next Conversation Stuff */










/* Begin Telnet stuff */

/***************************************************************************
 * 
 * Function Name: is_telnet_port
 * 
 * Returns: TRUE/FALSE whether a given port is a telnet/login type port.
 *
 * Purpose: To accept a port number and determine whenter or not the port
 *          would exhibit the characteristics of a telnet/login port.
 *
 * Called by: tcplib_read()                in mod_tcplib.c
 *            tcplib_do_telnet_duration()  in mod_tcplib.c
 *            tcplib_add_telnet_interval() in mod_tcplib.c
 * 
 ****************************************************************************/
Bool is_telnet_port(
    portnum port)       /* The port we're looking at */
{
    port -= ipport_offset;

    switch(port) {
      case IPPORT_LOGIN:
      case IPPORT_KLOGIN:
      case IPPORT_OLDLOGIN:
      case IPPORT_FLN_SPX:
      case IPPORT_UUCP_LOGIN:
      case IPPORT_KLOGIN2:
      case IPPORT_NLOGIN:
      case IPPORT_TELNET:
/*       case IPPORT_SSH: */ /* not considered safe to assume -- sdo */
	return TRUE;
	break;

      default:
	return FALSE;
    }
}








/***************************************************************************
 * 
 * Function Name: tcplib_do_telnet_duration
 * 
 * Returns: Nothing
 *
 * Purpose: To collect information about the duration of a telnet
 *          conversation, and merge this information with data from
 *          previous runs of this module, if such data exists.
 *
 * Called by: tcplib_do_telnet() in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
void tcplib_do_telnet_duration(
    char *filename,		/* where to store the output */
    f_testinside p_tester)	/* functions to test "insideness" */
{
    dyn_counter psizes = NULL;
    const int bucketsize = GRAN_TELNET_DURATION;
    module_conninfo *pmc;
    

#ifdef READ_OLD_FILES
    /* This section reads in the data from the existing telnet duration
     * file in preparation for merging with the current data. */
    psizes = ReadOldFile(filename, bucketsize, 0, psizes);
#endif /* READ_OLD_FILES */


    /* Fill the array with the current data */
    for (pmc = module_conninfo_tail; pmc; pmc=pmc->prev) {
	/* if there wasn't data flowing in this direction, skip it */
	if (InsideBytes(pmc,p_tester) == 0)
	    continue;

	
	/* Only work this for telnet connections */
	if (is_telnet_conn(pmc)) {
	    /* convert the time difference to ms */
	    int temp = (int)(
		elapsed(pmc->first_time,
			pmc->last_time)/1000.0); /* convert us to ms */

	    /* increment the number of instances at this time. */
	    AddToCounter(&psizes, temp, 1, bucketsize);
	}
    }


    /* Output data to the file */
    StoreCounters(filename,"Duration (ms)", "% Conversations",
		  bucketsize,psizes);

    /* free the dynamic memory */
    DestroyCounters(&psizes);
}



/***************************************************************************
 * 
 * Function Name: do_all_conv_arrivals
 * 
 * Returns: Nothing
 *
 * Purpose: collect all the conversation interarrival times
 *
 * Called by: tcplib_done mod_tcplib.c
 * 
 * 
 ****************************************************************************/
void do_all_conv_arrivals()
{
    module_conninfo *pmc;

    if (ldebug>1)
	printf("do_all_conv_arrivals: there are %d tcp pairs\n",
	       num_tcp_pairs);

    /* process each connection */
    for (pmc = module_conninfo_tail; pmc; pmc=pmc->prev) {
	int dir;

	if (ldebug>2) {
	    static int count = 0;
	    printf("do_all_conv_arrivals: processing pmc %d: %p\n",
		   ++count, pmc);
	}

	/* ignore unidirectional HTTP */
	if (is_http_conn(pmc) && pmc->unidirectional_http)
	    continue;


	for (LOOP_OVER_BOTH_TCBS(dir)) {
	    if (pmc->tcb_cache[dir].data_bytes != 0) {
		do_tcplib_next_converse(&pmc->tcb_cache[dir], pmc);
		do_tcplib_next_duration(&pmc->tcb_cache[dir], pmc);
	    }
	}
    }
}









/***************************************************************************
 * 
 * Function Name: tcplib_add_telnet_interarrival
 * 
 * Returns: Nothing
 *
 * Purpose: This function takes the current packet and computes the time
 *          between the current packet and the previous packet.  This value
 *          is then added to the list of telnet interarrivals.  These values
 *          will be used at a later time.
 *
 * Called by: tcplib_read() in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
void tcplib_add_telnet_interarrival(
    tcp_pair *ptp,              /* This conversation */
    module_conninfo *pmc,
    dyn_counter *psizes)
{
    int temp = 0;    /* time differential between packets */

    /* Basically, I need the current time AND the time of the previous
     * packet BOTH right now.  As far as I can see, this function
     * won't get called until the previous packet time has already
     * been overwritten by the current time.  This makes obtaining
     * interarrival times more difficult.  */

    /* Answer - changed the original program.  We added the pmstruct thing
     * to the original TCPTrace which allows a module to store information
     * about a connection.  Quite handy.  Thanks, Dr. Ostermann */

    /* First packet has no interarrival time */
    if (tv_same(ptp->last_time,ptp->first_time)) {
	/* If this is the first packet we've seen, nothing to do.
	 * We'll be able to get some data the next
	 * time. */
	return;
    }
	
    /* Determining the time difference in ms */
    temp = (int)(elapsed(pmc->last_time, current_time)/1000.0); /* us to ms */

    /* We're going to set an artificial maximum for telnet interarrivals
     * for the case when someone (like me) would open a telnet session
     * and just leave it open and not do anything on it for minutes or
     * hours, or in some cases days.  Keeping track of the exact time
     * for a connection like that is not worth the effort, so we just
     * set a ceiling and if it's over the ceiling, we make it the
     * ceiling. */
    if (temp > MAX_TEL_INTER_COUNT - 1)
	temp = MAX_TEL_INTER_COUNT - 1;

    /* In this case, we know for a fact that we don't have a value of
     * temp that larger than the array, so we just increment the count
     */
    (void) AddToCounter(psizes, temp, 1, 1);

    return;
}









    
/***************************************************************************
 * 
 * Function Name: tcplib_do_telnet_interarrival
 * 
 * Returns: Nothing
 *
 * Purpose: To model integrate the old data for telnet interarrival times
 *          with the data gathered during this execution of the program.
 *
 * Called by: tcplib_do_telnet() in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
void tcplib_do_telnet_interarrival(
    char *filename,		/* where to store the output */
    f_testinside p_tester)	/* stuck with the interface :-(  */
{
    const int bucketsize = GRAN_TELNET_ARRIVAL;
    dyn_counter psizes = NULL;

    /* ugly interface conversion :-( */
    if (p_tester == TestIncoming)
	psizes = global_pstats[INCOMING]->telnet_interarrival;
    else if (p_tester == TestOutgoing)
	psizes = global_pstats[OUTGOING]->telnet_interarrival;
    else if (p_tester == TestLocal)
	psizes = global_pstats[LOCAL]->telnet_interarrival;
    else if (p_tester == TestRemote)
	psizes = global_pstats[REMOTE]->telnet_interarrival;
    else {
	fprintf(stderr,
		"tcplib_do_telnet_interarrival: internal inconsistancy!\n");
	exit(-1);
    }



#ifdef READ_OLD_FILES
    /* add in the data from the old run (if it exists) */
    psizes = ReadOldFile(filename, bucketsize, MAX_TEL_INTER_COUNT, psizes);
#endif /* READ_OLD_FILES */


    /* Dumping the data out to the data file */
    StoreCounters(filename, "Interarrival Time (ms)", "% Interarrivals",
		  bucketsize,psizes);
}





/***************************************************************************
 * 
 * Function Name: tcplib_do_telnet_packetsize
 * 
 * Returns: Nothing
 *
 * Purpose: To take the data on telnet packet sizes measured during this
 *          run of the program, merge them with any existing data, and 
 *          drop a data file.
 *
 * Called by: tcplib_do_telnet() in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
void tcplib_do_telnet_packetsize(
    char *filename,		/* where to store the output */
    f_testinside p_tester)	/* stuck with the interface :-(  */
{
    const int bucketsize = GRAN_TELNET_PACKETSIZE;
    dyn_counter psizes = NULL;

    /* ugly interface conversion :-( */
    if (p_tester == TestIncoming)
	psizes = global_pstats[INCOMING]->telnet_pktsize;
    else if (p_tester == TestOutgoing)
	psizes = global_pstats[OUTGOING]->telnet_pktsize;
    else if (p_tester == TestLocal)
	psizes = global_pstats[LOCAL]->telnet_pktsize;
    else if (p_tester == TestRemote)
	psizes = global_pstats[REMOTE]->telnet_pktsize;
    else {
	fprintf(stderr,
		"tcplib_do_telnet_packetsize: internal inconsistancy!\n");
	exit(-1);
    }


#ifdef READ_OLD_FILES
    /* In this section, we're reading in from the previous data file,
     * applying the data contained there to the data set that we've 
     * acquired during this run, and then dumping the merged data set
     * back out to the data file */
    psizes = ReadOldFile(filename, bucketsize, 0, psizes);
#endif /* READ_OLD_FILES */


    /* Dumping the data out to the data file */
    StoreCounters(filename, "Packet Size (bytes)", "% Packets",
		  bucketsize,psizes);
}








/***************************************************************************
 * 
 * Function Name: tcplib_add_telnet_packetsize
 * 
 * Returns: Nothing
 *
 * Purpose: Takes a length as acquired from a telnet packet data size and
 *          increments the count of the telnet packet size table by one for
 *          entry which corresponds to that length.  If the packet size is
 *          larger than the allocated table allows, we truncate the packet
 *          size.
 *
 * Called by: tcplib_read() in mod_tcplib.c
 * 
 * 
 ****************************************************************************/
void tcplib_add_telnet_packetsize(
    struct tcplibstats *pstats,
    int length)  /* The length of the packet to be added to the table */
{
    /* Incrementing the table */
    AddToCounter(&pstats->telnet_pktsize, length, 1, 1);
}


/* End Telnet Stuff */








/* Begin FTP Stuff */

/***************************************************************************
 * 
 * Function Name: is_ftp_ctrl_conn
 * 
 * Returns: Boolean value
 *
 * Purpose: To determine if the connection is an FTP control port.
 *
 * Called by: tcplib_do_ftp_control_size() in mod_tcplib.c
 * 
 ****************************************************************************/
Bool is_ftp_ctrl_port(
    portnum port)
{
    port -= ipport_offset;
    return (port == IPPORT_FTP_CONTROL);
}
Bool is_ftp_data_port(
    portnum port)
{
    port -= ipport_offset;
    return (port == IPPORT_FTP_DATA);
}

	    



/***************************************************************************
 * 
 * Function Name: tcplib_do_ftp_itemsize
 * 
 * Returns: Nothing
 *
 * Purpose: To generate the ftp.itemsize data file from the information
 *          collected on ftp transfer sizes.  This function also integrates
 *          new data with old data, if any old data exists.
 *
 * Called by: tcplib_do_ftp() in mod_tcplib.c
 * 
 ****************************************************************************/
void tcplib_do_ftp_itemsize(
    char *filename,		/* where to store the output */
    f_testinside p_tester)	/* functions to test "insideness" */
{
    const int bucketsize = GRAN_FTP_ITEMSIZE;

    tcplib_do_GENERIC_itemsize(filename, TCPLIBPORT_FTPDATA,
			       p_tester, bucketsize);
}


void tcplib_do_ftp_numitems(
    char *filename,		/* where to store the output */
    f_testinside p_tester)	/* functions to test "insideness" */
{
    int bucketsize = GRAN_NUMITEMS;
    module_conninfo *pmc;
    dyn_counter psizes = NULL;
    

#ifdef READ_OLD_FILES
    /* If an old data file exists, open it, read in its contents
     * and store them until they are integrated with the current
     * data */
    psizes = ReadOldFile(filename, bucketsize, 0, psizes);
#endif /* READ_OLD_FILES */


    /* fill out the array with data from the current connections */
    for (pmc = module_conninfo_tail; pmc; pmc = pmc->prev) {
	/* We only need the stats if it's the right port */
	if (is_ftp_ctrl_conn(pmc)) {
	    if ((*p_tester)(pmc, &pmc->tcb_cache[TCB_CACHE_A2B])) {
		if (ldebug && (pmc->tcb_cache[TCB_CACHE_A2B].numitems == 0))
			printf("numitems: control %s has NONE\n",
			       FormatBrief(pmc->ptp, NULL));
		AddToCounter(&psizes, pmc->tcb_cache[TCB_CACHE_A2B].numitems,
			     1, 1);
	    }
	}
    }


    /* store all the data (old and new) into the file */
    StoreCounters(filename,"Total Articles", "% Conversation",
		  bucketsize,psizes);

    /* free the dynamic memory */
    DestroyCounters(&psizes);
}


static void tcplib_do_ftp_control_size(
    char *filename,		/* where to store the output */
    f_testinside p_tester)	/* functions to test "insideness" */
{
    const int bucketsize = GRAN_FTP_CTRLSIZE;

    tcplib_do_GENERIC_itemsize(filename, TCPLIBPORT_FTPCTRL,
			       p_tester, bucketsize);
}
/* End of FTP Stuff */




/* Begin SMTP Stuff */
static Bool
is_smtp_port(
    portnum port)
{
    port -= ipport_offset;
    return (port == IPPORT_SMTP);
}


static void tcplib_do_smtp_itemsize(
    char *filename,		/* where to store the output */
    f_testinside p_tester)	/* functions to test "insideness" */
{
    const int bucketsize = GRAN_SMTP_ITEMSIZE;

    tcplib_do_GENERIC_itemsize(filename, TCPLIBPORT_SMTP,
			       p_tester, bucketsize);
}
/* Done SMTP Stuff */




/* Begin NNTP Stuff */
static Bool
is_nntp_port(
    portnum port)
{
    port -= ipport_offset;
    return (port == IPPORT_NNTP);
}


/* Done NNTP Stuff */



/* Begin HTTP Stuff */
static Bool
is_http_port(
    portnum port)
{
    port -= ipport_offset;
    return ((port == IPPORT_HTTP) ||
	    (port == IPPORT_HTTPS));
}



/***************************************************************************
 **
 ** Support Routines
 **
 ***************************************************************************/


/***************************************************************************
 *
 * StoreCounters -- store a dyncounter structure into a file
 *   this get's a little interesting because of the way tcplib works
 *   for example, given the simple table
 *	1 0.3333
 *	2 0.6667
 *	3 1.0000
 *   tcplib will generate integer samples (from a random test):
 *	1	66756	0.6676	0.6676
 *	2	33244	0.3324	1.0000
 *
 *   as another example, given the simple table (same as before except
 *   for the first line)
 *	0 0.0000
 *	1 0.3333
 *	2 0.6667
 *	3 1.0000
 *   tcplib will generate integer samples (from a random test):
 *	0	33609	0.3361	0.3361
 *	1	33044	0.3304	0.6665
 *	2	33347	0.3335	1.0000
 *   SO... if we really want 1/3 1's, 2's, and 3's, we need the table:
 *	1 0.0000
 *	2 0.3333
 *	3 0.6667
 *	4 1.0000
 *    and a random test gives us
 *	1	33609	0.3361	0.3361
 *	2	33044	0.3304	0.6665
 *	3	33347	0.3335	1.0000
 *    which is exactly what we wanted.
 *
 *    ... therefore, for each counter, we store counter+GRANULARITY
 *    in the table, and also store a 0.0000 value for the FIRST entry
 * 
 **************************************************************************/
static void
StoreCounters(
    char *filename,
    char *header1,
    char *header2,
    int bucketsize,
    dyn_counter psizes)
{
    MFILE *fil;
    int running_total = 0;
    int lines = 0;

    if (ldebug>1)
	printf("Saving data for file '%s'\n", filename);

    /* verify bucketsize, but not needed anymore */
    if (bucketsize != GetGran(psizes)) {
	/* probably because the counter was never used */
	if (GetTotalCounter(psizes) != 0) {
	    fprintf(stderr,"StoreCounters: bad bucketsize (%s)\n",
		    filename);
	    exit(-1);
	}
    }

    if (!(fil = Mfopen(filename, "w"))) {
	perror(filename);
	exit(1);
    }

    Mfprintf(fil, "%s\t%s\tRunning Sum\tCounts\n", header1, header2);

    if (psizes == NULL) {
	if (ldebug>1)
	    printf("  (No data for file '%s')\n", filename);
    } else {
	int cookie = 0;
	int first = TRUE;
	while (1) {
	    u_long ix;
	    int value;
	    u_long count;
	    u_long total_counter = GetTotalCounter(psizes);
	    u_long gran = GetGran(psizes);

	    if (NextCounter(&psizes, &cookie, &ix, &count) == 0)
		break;

	    value = ix;
	    running_total += count;

	    if (count) {
		if (first) {
		    /* see comments above! */
		    Mfprintf(fil, "%.3f\t%.4f\t%d\t%d\n",
			     (float)(value), 0.0, 0, 0);
		    first = FALSE;
		}
		Mfprintf(fil, "%.3f\t%.4f\t%d\t%lu\n",
			 (float)(value+gran),
			 (float)running_total/(float)total_counter,
			 running_total,
			 count);
		++lines;
	    }
	}
    }

    Mfclose(fil);

    if (ldebug>1)
	printf("  Stored %d values into %d lines of '%s'\n",
	       running_total, lines, filename);
}


#ifdef READ_OLD_FILES
static dyn_counter 
ReadOldFile(
    char *filename,
    int bucketsize,
    int maxlegal,		/* upper limit on array IX (or 0) */
    dyn_counter psizes)
{
    FILE* old;                /* File pointer for old data file */
    float bytes;
    int count;
    int linesread = 0;

    /* If the an old data file exists, open it, read in its contents
     * and store them until they are integrated with the current
     * data */
    if ((old = fopen(filename, "r"))) {
	char buffer[256];

	/* read and discard the first line */
	fgets(buffer, sizeof(buffer)-1, old);


	/* Read in each line in the file and pick out the pieces of */
	/* the file.  Store each important piece is psizes */
	/* format is: 
		Total Bytes	% Conversations	Running Sum	Counts
		5.000		0.0278		1		1
		170.000		0.1111		4		3
	*/
	/* (we only need the 1st and 4th fields) */
	while (fscanf(old, "%f\t%*f\t%*d\t%d\n", &bytes, &count) == 4) {
	    ++linesread;
	    if ((maxlegal != 0) && (bytes > maxlegal))
		bytes = maxlegal;
	    AddToCounter(&psizes, (((int)bytes)/bucketsize), count);
	}

	if (ldebug>2) {
	    if (psizes && (linesread > 0))
		printf("Read data from old file '%s' (%lu values)\n",
		       filename, TotalCounter(psizes));
	    else
		printf("Old data file '%s' had no data\n", filename);
	}

	fclose(old);
    }

    return(psizes);
}
#endif /* READ_OLD_FILES */


/* all of the itemsize routines look like this */
static void
tcplib_do_GENERIC_itemsize(
    char *filename,		/* where to store the output */
    int btype,
    f_testinside p_tester,	/* functions to test "insideness" */
    int bucketsize)		/* how much data to group together */
{
    module_conninfo *pmc;
    dyn_counter psizes = NULL;
    

#ifdef READ_OLD_FILES
    /* If an old data file exists, open it, read in its contents
     * and store them until they are integrated with the current
     * data */
    psizes = ReadOldFile(filename, bucketsize, 0, psizes);
#endif /* READ_OLD_FILES */


    /* fill out the array with data from the current connections */
    for (pmc = module_conninfo_tail; pmc; pmc = pmc->prev) {
	/* We only need the stats if it's the right breakdown type */
	if (pmc->btype == btype) {
	    int nbytes = InsideBytes(pmc,p_tester);

	    /* if there's no DATA, don't count it!  (sdo change!) */
	    if (nbytes != 0)
		AddToCounter(&psizes, nbytes, 1, bucketsize);
	}
    }


    /* store all the data (old and new) into the file */
    StoreCounters(filename,"Article Size (bytes)", "% Articles",
		  bucketsize,psizes);

    /* free the dynamic memory */
    DestroyCounters(&psizes);
}



/* cleanup all the burstsize counters */
/* called for HTTP_P, HTTP_S, and NNTP */
static void
tcplib_cleanup_bursts()
{
    module_conninfo *pmc;

    /* all but the last burst was ALREADY recorded, so we just clean
       up any burst that might be left */
    for (pmc = module_conninfo_tail; pmc; pmc = pmc->prev) {
	int dir;

	for (LOOP_OVER_BOTH_TCBS(dir)) {
	    module_conninfo_tcb *ptcbc = &pmc->tcb_cache[dir];

	    /* check for burst data */
	    if (ptcbc->pburst == NULL)
		continue;

	    /* count the LAST burst */
	    if (ptcbc->burst_bytes != 0) {
		++ptcbc->numitems;
	    }

	    /* add the last burst into the ttl for the parallel stream */
	    if (ptcbc->burst_bytes != 0) {
		struct parallelism *pp = pmc->pparallelism;
		if (pp) {
		    ++pp->ttlitems[dir];
		}
	    }


	    if (ptcbc->burst_bytes != 0) {
		AddToCounter(&ptcbc->pburst->size,
			     ptcbc->burst_bytes,
			     1, GRAN_BURSTSIZE);
	    }

	    if (ptcbc->numitems != 0) {
		AddToCounter(&ptcbc->pburst->nitems,
			     ptcbc->numitems,
			     1,GRAN_NUMITEMS);
	    }
	}
    }
}
/* both ftp and nntp look the same */
static void
tcplib_do_GENERIC_burstsize(
    char *filename,		/* where to store the output */
    dyn_counter counter)
{
    int bucketsize = GRAN_BURSTSIZE;


    /* store all the data (old and new) into the file */
    StoreCounters(filename,"Burst Size", "% Bursts",
		  bucketsize, counter);
}
static void
tcplib_do_GENERIC_P_maxconns(
    char *filename,		/* where to store the output */
    dyn_counter counter)
{
    int bucketsize = GRAN_MAXCONNS;

    /* store all the data (old and new) into the file */
    StoreCounters(filename,"Max Conns", "% Streams",
		  bucketsize, counter);
}
static void
tcplib_do_GENERIC_nitems(
    char *filename,		/* where to store the output */
    dyn_counter counter)
{
    int bucketsize = GRAN_NUMITEMS;

    /* store all the data (old and new) into the file */
    StoreCounters(filename,"Num Items", "% Conns",
		  bucketsize, counter);
}
/* both ftp and nntp look the same */
static void
tcplib_do_GENERIC_idletime(
    char *filename,		/* where to store the output */
    dyn_counter counter)
{
    int bucketsize = GRAN_BURSTIDLETIME;

    /* store all the data (old and new) into the file */
    StoreCounters(filename,"Idle Time", "% Bursts",
		  bucketsize, counter);
}

static enum t_dtype
traffic_type(
    module_conninfo *pmc,
    module_conninfo_tcb *ptcbc)
{
    if (TestLocal(pmc,ptcbc))
	return(LOCAL);

    if (TestIncoming(pmc,ptcbc))
	return(INCOMING);

    if (TestOutgoing(pmc,ptcbc))
	return(OUTGOING);

    if (TestRemote(pmc,ptcbc))
	return(REMOTE);

    fprintf(stderr,"Internal error in traffic_type\n");
    exit(-1);
}

static char *
FormatBrief(
    tcp_pair *ptp,
    tcb *ptcb)
{
    tcb *pab = &ptp->a2b;
    tcb *pba = &ptp->b2a;
    static char infobuf[100];

    if (ptcb == pba)
	snprintf(infobuf,sizeof(infobuf),"%s - %s (%s2%s)",
		ptp->b_endpoint, ptp->a_endpoint,
		pba->host_letter, pab->host_letter);
    else
	snprintf(infobuf,sizeof(infobuf),"%s - %s (%s2%s)",
		ptp->a_endpoint, ptp->b_endpoint,
		pab->host_letter, pba->host_letter);

    return(infobuf);
}

static char *
FormatAddrBrief(
    tcp_pair_addrblock	*paddr_pair)
{
    static char infobuf[100];
    char infobuf1[100];
    char infobuf2[100];

    snprintf(infobuf1,sizeof(infobuf1),"%s", HostName(paddr_pair->a_address));
    snprintf(infobuf2,sizeof(infobuf2),"%s", HostName(paddr_pair->b_address));
    snprintf(infobuf,sizeof(infobuf),"%s - %s", infobuf1, infobuf2);

    return(infobuf);
}


/* find a connection pair */
static endpoint_pair *
FindEndpointPair(
    endpoint_pair *hashtable[],
    tcp_pair_addrblock	*paddr_pair)
{
    endpoint_pair **ppep_head;
    endpoint_pair *pep_search;

    /* find the correct hash bucket */
    ppep_head = &hashtable[EndpointHash(paddr_pair)];

    /* search the bucket for the correct pair */
    for (pep_search = *ppep_head; pep_search;
	 pep_search = pep_search->pepnext) {

	/* see if it's the same connection */
	if (SameEndpoints(&pep_search->addr_pair,
			  paddr_pair))
	    return(pep_search);
    }

    /* after loop, pep_search is NON-NULL if we found it */

    return(NULL);
}




/* add a connection to the list of all connections on that address pair */
static void
AddEndpointPair(
    endpoint_pair *hashtable[],
    module_conninfo *pmc)
{
    endpoint_pair **ppep_head;
    endpoint_pair *pep_search;

    /* search the bucket for the correct pair */
    ppep_head = &hashtable[EndpointHash(&pmc->addr_pair)];
    pep_search = FindEndpointPair(hashtable,&pmc->addr_pair);

    if (pep_search == NULL) {
	/* not found, create it */
	pep_search = MallocZ(sizeof(endpoint_pair));

	/* fill in the address info */
	pep_search->addr_pair = pmc->ptp->addr_pair;

	/* put at the front of the bucket */
	pep_search->pepnext = *ppep_head;
	*ppep_head = pep_search;
    }

    /* put the new connection at the front of the list for this
       endpoint pair */
    pmc->next_pair = pep_search->pmchead;
    pep_search->pmchead = pmc;

    if (ldebug>1) {
	printf("\nEndpoint pair bucket\n");

	/* for each thing on the bucket list */
	for (pep_search = *ppep_head; pep_search;
	     pep_search = pep_search->pepnext) {
	    module_conninfo *pmc;
	    printf("  %s:\n", FormatAddrBrief(&pep_search->addr_pair));
	    /* for each connection on that pair */
	    for (pmc = pep_search->pmchead; pmc; pmc = pmc->next_pair) {
		printf("    %u <-> %u\n",
		       pmc->addr_pair.a_port,
		       pmc->addr_pair.b_port);
	    }
	}
    }
}



/* remove unidirectional conns from consideration */
static void
tcplib_filter_http_uni()
{
    module_conninfo *pmc;

    for (pmc = module_conninfo_tail; pmc; pmc = pmc->prev) {
	if ((pmc->tcb_cache[0].data_bytes == 0) ||
	    (pmc->tcb_cache[1].data_bytes == 0)) {
	    /* unidirectional (or no data at all) */
	    pmc->unidirectional_http = TRUE;

	    /* update counters */
	    ++debug_http_uni_conns;
	    debug_http_uni_bytes +=
		pmc->tcb_cache[0].data_bytes +
		pmc->tcb_cache[1].data_bytes;

	    /* UNDO other counters */
	    --debug_http_total;
	    if (pmc->pparallelism &&
		(pmc->pparallelism->maxparallel > 1)) {
		--pmc->pparallelism->maxparallel;
		--debug_http_parallel;

		/* if this is NOT the last one, decrement slave count */
		if (pmc->pparallelism->maxparallel > 0) {
		    --debug_http_slaves;
		} else {
		    /* nobody left, empty group */
		    --debug_http_groups;
		}
	    }

	    /* persistance is OK, as those are already ignored
	       in tcplib_save_bursts */

	}
    }
}


static Bool
is_parallel_http(
    module_conninfo *pmc_new)
{
    endpoint_pair *pep;
    module_conninfo *pmc;
    struct parallelism *pp = NULL;
    int dir;
    int parallel_conns = 0;
    u_long parent_groupnum = 0;

    /* see if there are any other connections on these endpoints */
    pep = FindEndpointPair(http_endpoints, &pmc_new->addr_pair);

    /* if none, we're done */
    if (pep == NULL)
	return(FALSE);

    /* search that pep chain for PARALLEL conns */
    for (pmc = pep->pmchead; pmc; pmc = pmc->next_pair) {

	/* for efficiency, as we search we remove old, inactive entries */
	/* NOTE that this is the NEXT entry, not current */
	if (pmc->next_pair) {
	    if (!RecentlyActiveConn(pmc->next_pair)) {
		/* remove it (by linking around it) */
		pmc->next_pair = pmc->next_pair->next_pair;
	    }
	}

	/* if it's ME or it's not ACTIVE, skip it */
	if ((pmc_new == pmc) || !RecentlyActiveConn(pmc))
	    continue;

	if (trafgen_generated) {
	    /* burst key group nums must also be the same */
	    if (pmc_new->http_groupnum != pmc->http_groupnum) {
		continue;	/* skip it */
	    }
	}


	/* OK, it's ACTIVE */

	/* mark it as parallel if not already done */
	if (pmc->pparallelism == NULL) {
	    pmc->pparallelism = MallocZ(sizeof(struct parallelism));

	    /* if HE isn't marked, then he must be the "parent",
	       and this must be a new group */
	    ++debug_http_groups;
	    ++debug_http_parallel;

	    /* if this isn't trafgen-generated, give it a group number */
	    /* (mostly for debugging) */
	    if (!trafgen_generated) {
		static u_long groupnum = 0;
		parent_groupnum = ++groupnum;
		pmc->http_groupnum = parent_groupnum;
	    }
	      

	    /* switch its stats to parallel */
	    for (LOOP_OVER_BOTH_TCBS(dir)) {
		module_conninfo_tcb *ptcbc = &pmc->tcb_cache[dir];
		struct tcplibstats *pstats = global_pstats[ptcbc->dtype];
		ptcbc->pburst = &pstats->http_P_bursts;
	    }
	} else {
	    /* it's already known to be parallel, so he's my brother */
	    if (!trafgen_generated) {
		parent_groupnum = pmc->http_groupnum;
	    }
	}

	/* remember the parallel struct for these connections */
	pp = pmc->pparallelism;

	/* sanity check, if we've already found one, it better be */
	/* the same as this one!! */
	if ((pp != NULL) && (pp != pmc->pparallelism)) {
	    fprintf(stderr,"FindParallelHttp: bad data structure!!\n");
	    exit(-1);
	}

	/* found one more */
	++parallel_conns;
    }

    /* if we didn't find any, we're done */
    if (parallel_conns == 0)
	return(FALSE);

    /* mark ME as parallel too */
    ++debug_http_slaves;
    ++debug_http_parallel;
    for (LOOP_OVER_BOTH_TCBS(dir)) {
	module_conninfo_tcb *ptcbc = &pmc_new->tcb_cache[dir];
	struct tcplibstats *pstats = global_pstats[ptcbc->dtype];
	ptcbc->pburst = &pstats->http_P_bursts;
    }

/*     printf("FindParallel, marking as parallel %s\n",  */
/* 	   FormatBrief(pmc_new->ptp)); */

    /* if this isn't trafgen generated, take the group number */
    pmc_new->http_groupnum = parent_groupnum;
	

    /* update stats on this parallel system */
    pmc_new->pparallelism = pp;
    ++parallel_conns;	/* include ME */

    /* update maximum parallelism, if required */
    if (parallel_conns > pmc_new->pparallelism->maxparallel)
	pmc_new->pparallelism->maxparallel = parallel_conns;

    /* this _IS_ parallel */
    return(TRUE);
}


static void
TrackEndpoints(
    module_conninfo *pmc)
{
    /* remember the endpoints (ftp and HTTP) */
    if (is_ftp_ctrl_conn(pmc)) {
	AddEndpointPair(ftp_endpoints,pmc);
    } else if (is_http_conn(pmc)) {
	AddEndpointPair(http_endpoints,pmc);
    }

    /* if it's an FTP data connection, find the control conn */
    if (is_ftp_data_conn(pmc)) {
	endpoint_pair *pep;

	/* for FTP Data, we ignore this one */
	pmc->ignore_conn = TRUE;

	pep = FindEndpointPair(ftp_endpoints, &pmc->addr_pair);

	if (pep) {
	    /* "charge" this new DATA connection to the most
	       recently-active ftp control connection */
	    struct module_conninfo_tcb *tcbc_control;
	    tcbc_control = MostRecentFtpControl(pep);
	    ++tcbc_control->numitems;
	    if (ldebug>1) {
		printf("Charging ftp data to %s, count %lu\n",
		       FormatBrief(tcbc_control->ptcb->ptp,
				   tcbc_control->ptcb),
		       tcbc_control->numitems);
	    }
	} else {
	    if (ldebug>1)
		fprintf(stderr,"WARNING: no FTP control conn for %s???\n",
			FormatBrief(pmc->ptp, NULL));
	}
    }
}




/* could this connection be an FTP data connection that's NOT
   on port 21? */
static Bool
CouldBeFtpData(
    tcp_pair *ptp)
{
    endpoint_pair *pep;
    struct module_conninfo_tcb *ptcbc;

    /* make sure NEITHER port is reserved */
    if (ptp->addr_pair.a_port < 1024 ||
	ptp->addr_pair.b_port < 1024)
	return(FALSE);
    
    /* see if there's any active FTP control connection on
       these endpoints... */
    pep = FindEndpointPair(ftp_endpoints,&ptp->addr_pair);
    if (pep == NULL)
	return(FALSE);

    /* find the most recent FTP control connection */
    ptcbc = MostRecentFtpControl(pep);
    if (pep == NULL)
	return(FALSE);

    /* OK, I guess it COULD be... */
    ++debug_newconn_ftp_data_heuristic;
    return(TRUE);
}


/* find the TCB (client side) for the most recently-active control
   connection on this pair of endpoints */
static module_conninfo_tcb *
MostRecentFtpControl(
    endpoint_pair *pep)
{
    struct module_conninfo *pmc;
    static module_conninfo_tcb *tcbc_newest = NULL;
    tcb *tcb_newest;
    timeval time_newest;

    if (pep->pmchead == NULL) {
	/* None at all, that's odd... */
	fprintf(stderr,"MostRecentFtpControl: unexpected empty list \n");
	exit(-1);
    }


    /* search the rest looking for something newer */
    for (pmc = pep->pmchead; pmc; pmc = pmc->next_pair) {
	tcb *ptcb_client = &pmc->ptp->a2b;

	/* if it's not "active", we're not interested */
	if (!ActiveConn(pmc))
	    continue;

	/* have we found anyone yet? */
	if (tcbc_newest == NULL) {
	    tcb_newest = &pmc->ptp->a2b;
	    tcbc_newest = &pmc->tcb_cache[TCB_CACHE_A2B];
	    time_newest = tcb_newest->last_data_time;
	} else if (tv_gt(ptcb_client->last_data_time, time_newest)) {
	    /* this is "most recent" */
	    tcb_newest = ptcb_client;
	    tcbc_newest = &pmc->tcb_cache[TCB_CACHE_A2B];
	    time_newest = ptcb_client->last_data_time;
	}
    }

    return(tcbc_newest);
}


static hash
IPHash(
    ipaddr *paddr)
{
    hash hval = 0;
    int i;

    if (ADDR_ISV4(paddr)) { /* V4 */
	hval = paddr->un.ip4.s_addr;
    } else if (ADDR_ISV6(paddr)) { /* V6 */
	for (i=0; i < 16; ++i)
	    hval += paddr->un.ip6.s6_addr[i];
    } else {
	/* address type unknown */
	fprintf(stderr,"Unknown IP address type %d encountered\n",
		ADDR_VERSION(paddr));
	exit(-1);
    }

    return(hval);
}


static hash
EndpointHash(
    tcp_pair_addrblock *addr_pair)
{
    hash hval;

    hval =
	IPHash(&addr_pair->a_address) +
	IPHash(&addr_pair->b_address);

    return(hval % ENDPOINT_PAIR_HASHSIZE);
}



static Bool
SameEndpoints(
    tcp_pair_addrblock	*pap1,
    tcp_pair_addrblock	*pap2)
{
    if (IPcmp(&pap1->a_address,&pap2->a_address) == 0) {
	if (IPcmp(&pap1->b_address,&pap2->b_address) == 0) {
	    return(TRUE);
	}
    } else if (IPcmp(&pap1->a_address,&pap2->b_address) == 0) {
	if (IPcmp(&pap1->b_address,&pap2->a_address) == 0) {
	    return(TRUE);
	}
    }

    return(FALSE);
}


/* Data is considered a NEW burst if:
 *  1) All previous data was ACKed
 *  2) There was intervening data in the other direction
 *  3) idletime > RTT
 */
static Bool
IsNewBurst(
    module_conninfo *pmc,
    tcb *ptcb,
    module_conninfo_tcb *ptcbc,
    struct tcphdr *tcp)
{
    seqnum seq = ntohl(tcp->th_seq);
    tcb *orig_lastdata;

    tcb *ptcb_otherdir = ptcb->ptwin;


    /* remember the last direction the data flowed */
    orig_lastdata = pmc->tcb_lastdata;
    pmc->tcb_lastdata = ptcb;


    if (graph_tsg) {
	plotter_perm_color(ptcb->tsg_plotter, "green");
	plotter_text(ptcb->tsg_plotter, current_time, seq, "a", "?");
    }

    /* it's only a NEW burst if there was a PREVIOUS burst */
    if (ptcbc->burst_bytes == 0) {
	if (graph_tsg)
	    plotter_text(ptcb->tsg_plotter, current_time, seq, "b", "==0");
	return(FALSE);
    }

    /* check for old data ACKed */
    if (SEQ_LESSTHAN(ptcb_otherdir->ack,seq)) {
	/* not ACKed */
	if (graph_tsg)
	    plotter_text(ptcb->tsg_plotter, current_time, seq, "b", "noack");
	return(FALSE);
    }

    /* check for idletime > RTT */
    {
	u_long etime_usecs = elapsed(ptcbc->last_data_time, current_time);
	u_long last_rtt_usecs = ptcb->rtt_last;
	if ((last_rtt_usecs != 0) && (etime_usecs < last_rtt_usecs)) {
	    if (graph_tsg) {
		char buf[100];
		snprintf(buf,sizeof(buf),"short (%ld < %ld)", etime_usecs, last_rtt_usecs);
		plotter_text(ptcb->tsg_plotter, current_time, seq, "b", buf);
	    }
	    return(FALSE);
	}
    }

    /* check for intervening data */
    if (ptcb == orig_lastdata) {
	/* no intervening data */
	if (graph_tsg)
	    plotter_text(ptcb->tsg_plotter, current_time, seq, "b", "!data");
	return(FALSE);
    }

    /* ... else, it's a new burst */

    if (graph_tsg) {
	plotter_perm_color(ptcb->tsg_plotter, "magenta");
	plotter_text(ptcb->tsg_plotter, current_time, seq, "r", "YES!!");
    }
    return(TRUE);
}

/* is this connection "active" */
/* 1: not reset */
/* 2: either 0 or 1 fins */
static Bool
ActiveConn(
    module_conninfo *pmc)
{
    if (FinCount(pmc->ptp) > 1)
	return(FALSE);

    if (ConnReset(pmc->ptp))
	return(FALSE);
    
    return(TRUE);
}


/* is this connection "parallel" */
/* 1: ActiveConn() */
/* 2: last packets sent "recently" (defined as within 10 seconds) */
static Bool
RecentlyActiveConn(
    module_conninfo *pmc)
{
    int dir;
    
    if (ActiveConn(pmc))
	return(TRUE);

    for (LOOP_OVER_BOTH_TCBS(dir)) {
	timeval last_packet = pmc->tcb_cache[dir].ptcb->last_time;

	/* elapsed time from last packet (in MICROseconds) */
	if (elapsed(last_packet,current_time) < 10*US_PER_SEC) {
	    /* 10 seconds for now, hope it works! */
	    return(TRUE);
	}
    }
    
    return(FALSE);
}


#endif /* LOAD_MODULE_TCPLIB */