PUBLIC HTStream * HTRules (HTRequest *	request,
			   void *	param,
			   HTFormat	input_format,
			   HTFormat	output_format,
			   HTStream *	output_stream)
{
    HTAlertCallback *cbf = HTAlert_find(HT_A_CONFIRM);
    
    /*
    **  If the library has been compiled so that we automatically accept
    **  rule files then it's OK not to ask the user.
    */
#ifdef HT_AUTOMATIC_RULES
    if (!cbf || (cbf && (*cbf)(request,HT_A_CONFIRM, HT_MSG_RULES, NULL,NULL,NULL))) {
#else
    if ((cbf && (*cbf)(request,HT_A_CONFIRM, HT_MSG_RULES, NULL,NULL,NULL))) {
#endif
	HTStream * me;
	HTTRACE(APP_TRACE, "Rule file... Parser object created\n");
	if ((me = (HTStream *) HT_CALLOC(1, sizeof(HTStream))) == NULL)
	    HT_OUTOFMEM("HTRules");
	me->isa = &HTRuleClass;
	me->request = request;
	me->buffer = HTChunk_new(512);
	me->EOLstate = EOL_BEGIN;
	if (!rules) rules = HTList_new();
	return me;
    } else {
	HTRequest_addError(request, ERR_FATAL, NO, HTERR_NO_AUTO_RULES,
			   NULL, 0, "HTRules");
	return HTErrorStream();
    }
}

/*
**  Parse a rule file - don't ask don't tell - be carefull with this one!
*/
PUBLIC HTStream * HTRules_parseAutomatically (HTRequest *	request,
					      void *		param,
					      HTFormat		input_format,
					      HTFormat		output_format,
					      HTStream *	output_stream)
{
    if (request) {
	HTStream * me;
	HTTRACE(APP_TRACE, "Rule file... Automatic parser object created\n");
	if ((me = (HTStream *) HT_CALLOC(1, sizeof(HTStream))) == NULL)
	    HT_OUTOFMEM("HTRules");
	me->isa = &HTRuleClass;
	me->request = request;
	me->buffer = HTChunk_new(512);
	me->EOLstate = EOL_BEGIN;
	if (!rules) rules = HTList_new();
	return me;
    } else {
	HTRequest_addError(request, ERR_FATAL, NO, HTERR_NO_AUTO_RULES,
			   NULL, 0, "HTRules");
	return HTErrorStream();
    }
}

/*	HTDNS_add
**	---------
**	Add an element to the cache of visited hosts. Note that this function
**	requires the system implemented structure hostent and not our own
**	host_info. The homes variable indicates the number of IP addresses 
**	found. A host name must NOT contain a port number.
**	Returns address of new HTdns object
*/
PUBLIC HTdns * HTDNS_add (HTList * list, struct hostent * element,
			  char *host, int *homes)
{
    HTdns *me;
    char *addr = NULL;
    char **index = element->h_addr_list;
    int cnt = 1;

    while(*index++) cnt++;
    if ((me = (HTdns *) HT_CALLOC(1, sizeof(HTdns))) == NULL ||
	(me->addrlist = (char **) HT_CALLOC(1, cnt*sizeof(char*))) == NULL ||
	(addr = (char *) HT_CALLOC(1, cnt*element->h_length)) == NULL)
	HT_OUTOFMEM("HTDNS_add");
    StrAllocCopy(me->hostname, host);
    me->ntime = time(NULL);
    index = element->h_addr_list;
    cnt = 0;
    while (*index) {
	*(me->addrlist+cnt) = addr+cnt*element->h_length;
	memcpy((void *) *(me->addrlist+cnt++), *index++, element->h_length);
    }
    me->homes = cnt;
    *homes = cnt;
    if ((me->weight = (double *) HT_CALLOC(me->homes, sizeof(double))) == NULL)
        HT_OUTOFMEM("HTDNS_add");
    me->addrlength = element->h_length;
    HTTRACE(PROT_TRACE, "DNS Add..... `%s\' with %d home(s) to %p\n" _ 
		host _ *homes _ list);
    HTList_addObject(list, (void *) me);
    return me;
}

/*	Ensure a certain size
**	---------------------
*/
PUBLIC void HTChunk_ensure (HTChunk * ch, int len)
{
    if (ch && len > 0) {
	int needed = ch->size+len;
	if (needed >= ch->allocated) {
	    ch->allocated = needed - needed%ch->growby + ch->growby;
	    if (ch->data) {
		if ((ch->data = (char  *) HT_REALLOC(ch->data, ch->allocated)) == NULL)
		    HT_OUTOFMEM("HTChunk_ensure");
	        memset((void *) (ch->data + ch->size), '\0', ch->allocated-ch->size);
	    } else {
		if ((ch->data = (char  *) HT_CALLOC(1, ch->allocated)) == NULL)
		    HT_OUTOFMEM("HTChunk_ensure");
	    }
	}
    }
#if 0
    if (needed <= ch->allocated) return;
    ch->allocated = needed-1 - ((needed-1) % ch->growby)
    			     + ch->growby; /* Round up */
    ch->data = ch->data ? (char *)realloc(ch->data, ch->allocated)
			: (char *)HT_MALLOC(ch->allocated);
    if (ch->data == NULL) HT_OUTOFMEM(__FILE__, "HTChunk_ensure");
#endif
}

PUBLIC HTMuxChannel * HTMuxChannel_new (HTHost * host)
{
    if (host) {
	HTMuxChannel * me = NULL;

	/* Create new object */
	if ((me = (HTMuxChannel *) HT_CALLOC(1, sizeof(HTMuxChannel))) == NULL)
	    HT_OUTOFMEM("HTMuxChannel_new");
	me->hash = HTHost_hash(host);
	me->host = host;

	/*
	**  Make sure that we are in interleave mode
	*/
	HTHost_setMode(host, HT_TP_INTERLEAVE);

	/*
	**  Get a special MUX Net object that we keep to our selves. We don't
	**  associate a request object as the Net object lives longer.
	*/
	me->net = HTNet_new(NULL);
	HTNet_setReadStream(me->net, HTDemux_new(host, me));

	/* Insert into hash table */
	if (!muxchs) {
	    if ((muxchs=(HTList **) HT_CALLOC(HOST_HASH_SIZE, sizeof(HTList *))) == NULL)
		HT_OUTOFMEM("HTMuxChannel_new");
	}
	if (!muxchs[me->hash]) muxchs[me->hash] = HTList_new();
	HTList_addObject(muxchs[me->hash], (void *) me);
	HTTRACE(MUX_TRACE, "Mux Channel. %p created with hash %d\n" _ me _ me->hash);
	return me;
    }
    return NULL;
}

/* C O N S T R U C T O R S */
PUBLIC CSParse_t * CSParse_new(void)
{
    CSParse_t * me;
	if ((me = (CSParse_t *) HT_CALLOC(1, sizeof(CSParse_t))) == NULL)
	    HT_OUTOFMEM("CSParse");
    me->nowIn = NowIn_NEEDOPEN;
    me->token = HTChunk_new(0x10);
	if ((me->pParseContext = (ParseContext_t *) HT_CALLOC(1, sizeof(ParseContext_t))) == NULL)
	    HT_OUTOFMEM("ParseContext_t");
    return me;
}

PUBLIC HText *	LMHText_new (
	HTRequest * request,
	HTParentAnchor * anchor,
	HTStream *outstrm)
{
    HTLine * line;
    HText * self;
    if ((self = (HText  *) HT_CALLOC(1, sizeof(*self))) == NULL)
/*        HT_OUTOFMEM("HText"); */
	return self;
    
    self->pLm = Context_getLineMode(request);
    if (!loaded_texts) loaded_texts = HTList_new();
    HTList_addObject(loaded_texts, self);
    if (HTList_count(loaded_texts) >= LOADED_LIMIT) {
        HTTRACE(CACHE_TRACE, "MemoryCache. Freeing off cached doc.\n"); 
        HText_free((HText *)HTList_removeFirstObject(loaded_texts));
    }
    
    if ((line = self->last_line = (HTLine *) HT_MALLOC(LINE_SIZE(MAX_LINE))) == NULL)
	HT_OUTOFMEM("HText_New");
    line->next = line->prev = line;
    line->offset = line->size = 0;
    self->lines = self->chars = 0;
    self->title = 0;
    self->first_anchor = self->last_anchor = self->current_anchor = 0;
    self->style = &default_style;
    self->top_of_screen = 0;
    self->node_anchor = anchor;
    self->last_anchor_number = 0;	/* Numbering of them for references */
    self->stale = YES;
    
    self->target = NULL;
    
    HTAnchor_setDocument(anchor, (void *) self);

    clear_screen();
    HTMainText = self;
    HTMainAnchor = anchor;
    self->display_on_the_fly = DISPLAY_LINES;
    self->all_pages = NO;	/* One page at a time on the fly */
    
    if (!space_string) {	/* Make a blank line */
        char *p;
	if ((space_string = (char  *) HT_MALLOC(HTScreenWidth+1)) == NULL)
	    HT_OUTOFMEM("HText_New");
        for (p=space_string; p<space_string+HTScreenWidth; p++) 
            *p = ' '; 		/* Used for printfs later */
        space_string[HTScreenWidth] = '\0'; 
    }
    
    return self;
}

PUBLIC HTHashtable * HTHashtable_new (int size)
{
    HTHashtable *newHashtable;
    int c = size > 0 ? size : HT_L_HASH_SIZE;
    if ((newHashtable = (HTHashtable  *) HT_CALLOC(1, sizeof (HTHashtable))) == NULL)
        HT_OUTOFMEM("HTHashtable_new");

    if((newHashtable->table = (void **) HT_CALLOC(c, sizeof (void *))) == NULL)
	HT_OUTOFMEM("HTHashtable_new");

    newHashtable->count = 0;
    newHashtable->size = c;
    return newHashtable;
}

PUBLIC HTLink * HTLink_new (void)
{
    HTLink * link;
    if ((link = (HTLink *) HT_CALLOC(1, sizeof (HTLink))) == NULL)
        HT_OUTOFMEM("HTLink_new");
    return link;
}

PUBLIC HTStream * HTBoundary   (HTRequest *	request,
				void *		param,
				HTFormat	input_format,
				HTFormat	output_format,
				HTStream *	output_stream)
{
    HTResponse * response = HTRequest_response(request);
    HTParentAnchor * anchor = HTRequest_anchor(request);
    HTAssocList * type_param = response ?
	HTResponse_formatParam(response) :
	HTAnchor_formatParam(anchor);
    char * boundary = HTAssocList_findObject(type_param, "boundary");
    if (boundary) {
	HTStream * me;
	if ((me = (HTStream  *) HT_CALLOC(1, sizeof(HTStream))) == NULL)
	    HT_OUTOFMEM("HTBoundary");
	me->isa = &HTBoundaryClass;
	me->request = request;
	me->format = output_format;
	me->orig_target = output_stream;
	me->debug = HTRequest_debugStream(request);
	me->state = EOL_FLF;
	StrAllocCopy(me->boundary, boundary);		       /* Local copy */
	me->bpos = me->boundary;
	HTTRACE(STREAM_TRACE, "Boundary.... Stream created with boundary '%s\'\n" _ me->boundary);
	return me;
    } else {
	HTTRACE(STREAM_TRACE, "Boundary.... UNKNOWN boundary!\n");
	return HTErrorStream();
    }
}

PRIVATE HTNewsNode * HTNewsNode_new (int index, char * subject, char * from,
				     time_t date, char * name,
				     int refs, HTList * refNames)
{
    HTNewsNode * node;
    if ((node = (HTNewsNode *) HT_CALLOC(1, sizeof(HTNewsNode))) == NULL)
        HT_OUTOFMEM("HTNewsNode_new");
    if (name) StrAllocCopy(node->name, name);
    if (subject) {
	StrAllocCopy(node->subject, subject);
	node->subject = HTStrip(node->subject);
    }
    if (from) StrAllocCopy(node->from, from);
    node->index = index;
    node->date = date;
    node->refs = refs;
    node->refNames = refNames;
    node->show = (name != NULL);
    node->fake = (name == NULL);
    node->minRefIndex = index;
    node->maxRefIndex = index;
    node->minRefDate = date;
    node->maxRefDate = date;
    return node;
}

/*	Add an entry to a list of host names
**	------------------------------------
**	Existing entries are replaced with new ones
*/
PRIVATE BOOL add_hostname (HTList * list, const char * host,
			   const char * access, unsigned port,
			   BOOL regex, int regex_flags)
{
    HTHostList *me;
    if (!list || !host || !*host)
	return NO;
    if ((me = (HTHostList *) HT_CALLOC(1, sizeof(HTHostList))) == NULL)
        HT_OUTOFMEM("add_hostname");
#ifdef HT_POSIX_REGEX
    if (regex)
	me->regex = get_regex_t(host,
				regex_flags < 0 ?
				W3C_DEFAULT_REGEX_FLAGS : regex_flags);
#endif

    if (access) {
	char *ptr;
	StrAllocCopy(me->access, access);      	     	    /* Access method */
	ptr = me->access;
	while ((*ptr = TOLOWER(*ptr))) ptr++;
    }
    StrAllocCopy(me->host, host);		     	    	/* Host name */
    {
	char *ptr = me->host;
	while ((*ptr = TOLOWER(*ptr))) ptr++;
    }
    me->port = port;					      /* Port number */
    HTTRACE(PROT_TRACE, "HTHostList.. adding `%s\' to list\n" _ me->host);
    HTList_addObject(list, (void *) me);
    return YES;
}

PRIVATE Robot * Robot_new (void)
{
    Robot * me;
    if ((me = (Robot *) HT_CALLOC(1, sizeof(Robot))) == NULL ||
	(me->tv = (struct timeval*) HT_CALLOC(1, sizeof(struct timeval))) == NULL)
	HT_OUTOFMEM("Robot_new");
    me->htext = 0;
    me->tv->tv_sec = DEFAULT_TIMEOUT;
    me->cwd = HTGetCurrentDirectoryURL();
    me->output = OUTPUT;
    me->urilist = HTList_new();
    me->count = 0;

    /* We keep an extra timeout request object for the timeout_handler */
    me->timeout = HTRequest_new();

    /* Bind the Robot object together with the Request Object */
    me->request = HTRequest_new();
    HTRequest_setContext (me->request, me);
    HTRequest_setPreemptive(me->request, YES);

    /* Make a new profile */
    HTProfile_newPreemptiveRobot ("w3clibtcl", "1.0");

    return me;
}

PRIVATE HTChildAnchor * HTChildAnchor_new (void)
{
    HTChildAnchor *child;
    if ((child = (HTChildAnchor  *) HT_CALLOC(1, sizeof(HTChildAnchor))) == NULL)
        HT_OUTOFMEM("HTChildAnchor_new");
    return child;
}

PUBLIC int HTServHTTP (SOCKET soc, HTRequest * request)
{
    HTNet * net = HTRequest_net(request);
    https_info * http;			    /* Specific protocol information */

    /*
    ** Initiate a new https object and bind to request object
    ** This is actually state HTTPS_BEGIN, but it can't be in the state
    ** machine as we need the object first (chicken and egg problem).
    */
    HTTRACE(PROT_TRACE, "Serv HTTP... on socket %d\n" _ soc);
    if ((http = (https_info *) HT_CALLOC(1, sizeof(https_info))) == NULL)
        HT_OUTOFMEM("HTServHTTP");
    http->server = request;
    http->state = HTTPS_BEGIN;
    http->clients = HTList_new();
    HTNet_setContext(net, http);

    /*
    ** Create the stream pipe FROM the channel to the server request.
    */
    net->readStream = HTTPReceive_new(request, http);
    HTRequest_setOutputConnected(request, YES);
    http->state = HTTPS_BEGIN;

    HTNet_setEventCallback(net, ServEvent);
    HTNet_setEventParam(net, http);  /* callbacks get http* */

    return ServEvent(soc, http, HTEvent_BEGIN);		/* get it started - ops is ignored */
}

/*	Define a presentation system command for a content-type
**	-------------------------------------------------------
** INPUT:
**	conversions:	The list of conveters and presenters
**	representation:	the MIME-style format name
**	command:	the MAILCAP-style command template
**	quality:	A degradation faction [0..1]
**	maxbytes:	A limit on the length acceptable as input (0 infinite)
**	maxsecs:	A limit on the time user will wait (0 for infinity)
*/
PUBLIC void HTPresentation_add (HTList *	conversions,
				const char *	representation,
				const char *	command,
				const char *	test_command,
				double		quality,
				double		secs, 
				double		secs_per_byte)
{
    HTPresentation * pres;
    if (presentation_converter) {
	if ((pres = (HTPresentation  *) HT_CALLOC(1,sizeof(HTPresentation))) == NULL)
	    HT_OUTOFMEM("HTSetPresentation");
        pres->rep = HTAtom_for(representation);
	pres->rep_out = WWW_PRESENT;		/* Fixed for now ... :-) */
	pres->converter = presentation_converter;
	pres->quality = quality;
	pres->secs = secs;
	pres->secs_per_byte = secs_per_byte;
	pres->rep = HTAtom_for(representation);
	pres->command = NULL;
	StrAllocCopy(pres->command, command);
	pres->test_command = NULL;
	StrAllocCopy(pres->test_command, test_command);
	HTTRACE(CORE_TRACE, "Presentation Adding `%s\' with quality %.2f\n" _ 
		    command _ quality);
	HTList_addObject(conversions, pres);
    }
}

PUBLIC char * StrAllocMCopy (char ** dest, ...)
{
    va_list pArgs;
    char * p, * argp;

    /* How much space do we need? */
    int needed = 0;
    va_start(pArgs, dest);
    while ((p = va_arg(pArgs, char *)) != NULL) 
	needed += strlen(p);
    va_end(pArgs);

    if (*dest) HT_FREE(*dest);
    if (needed) {

	/* Allocate that amount of memory */
	if ((*dest = (char *) HT_MALLOC(needed + 1)) == NULL)
	    HT_OUTOFMEM("HTStrCpy");
	p = *dest;

	/* Fill the string */
	va_start(pArgs, dest);
	while ((argp = va_arg (pArgs, char *)) != NULL) {
	    strcpy(p, argp);
	    p += strlen(argp);
	}
	va_end (pArgs);
    }
    return *dest;
}

/*	HTDir_addElement
**	---------------
**    	This function accepts a directory line. "data" and "size", and
**	"description" can all be NULL
**	Returns YES if OK, else NO
*/
PUBLIC BOOL HTDir_addElement (HTDir *dir, char *name, char *date, char *size,
			      HTFileMode mode)
{
    HTDirNode *node = HTDirNode_new();
    if (!dir || !name) return NO;
    if ((node->fname = (char *) HT_MALLOC(strlen(name) + 2)) == NULL)
	HT_OUTOFMEM("HTDir_addElement");
    strcpy(node->fname, name);					/* Mandatory */
    if (dir->show & HT_DS_DATE && date) StrAllocCopy(node->date, date);
    if (dir->show & HT_DS_SIZE && size) StrAllocCopy(node->size, size);
    if (dir->show & HT_DS_DES) {
#if 0

	/* FIND DESCRIPTION */

#endif
    }


    /* Set the mode of the file */
    node->mode = mode;

    /* Should we display now or later? */
    if (dir->key == HT_DK_NONE) {
	if (!dir->size++) HTDir_headLine(dir);
	HTDirNode_print(dir, node);
	HTDirNode_free(node);
    } else {
	int slen = strlen(name);
	if (slen > dir->curfw)
	    dir->curfw = slen < MaxFileW ? slen : MaxFileW;
	HTArray_addObject(dir->array, (void *) node);
    }
    return YES;
}

/*	String Allocate and Concatenate
*/
PUBLIC char * HTSACat (char ** dest, const char * src)
{
  if (src && *src) {
    if (*dest) {
      int length = strlen (*dest);
      if ((*dest  = (char  *) HT_REALLOC(*dest, length + strlen(src) + 1)) == NULL)
          HT_OUTOFMEM("HTSACat");
      strcpy (*dest + length, src);
    } else {
      if ((*dest  = (char  *) HT_MALLOC(strlen(src) + 1)) == NULL)
          HT_OUTOFMEM("HTSACat");
      strcpy (*dest, src);
    }
  }
  return *dest;
}

PUBLIC HTAlertPar * HTAlert_newReply (void)
{
    HTAlertPar * me;
    if ((me = (HTAlertPar  *) HT_CALLOC(1, sizeof(HTAlertPar))) == NULL)
        HT_OUTOFMEM("HTAlert_newReply");
    return me;
}

void HText_beginAnchor (HText * text, HTChildAnchor * anchor)
{
    TextAnchor * a;
    if (text && anchor) {
	Robot * mr = (Robot *) HTRequest_context(text->request);
	HTAnchor * dest = HTAnchor_followMainLink((HTAnchor *) anchor);
	HTParentAnchor * dest_parent = HTAnchor_parent(dest);
	char * uri = HTAnchor_address((HTAnchor *) dest_parent);

#if 0
	if (SHOW_MSG) HTTrace("Robot....... Found `%s\' \n", uri ? uri : "NULL");
#endif	
	if (uri) {
	    HTList_addObject(mr->urilist, (void *) uri);
	    mr->count++;
	}

	if ((a = (TextAnchor  *) HT_MALLOC(sizeof(*a))) == NULL)
	    HT_OUTOFMEM("HText_beginAnchor");
	if (text->last_anchor) {
	    text->last_anchor->next = a;
	} else {
	    text->first_anchor = a;
	}
	a->next = 0;
	a->anchor = anchor;
	text->last_anchor = a;
    
	if (HTAnchor_followMainLink((HTAnchor*)anchor)) {
	    a->number = ++(text->anchors);
	} else {
	    a->number = 0;
	}
    }
}