// $Id: HTTP.cc,v 1.45 2003/08/24 18:43:09 vern Exp $
//
// Copyright (c) 1998, 1999, 2001, 2002, 2003
//      The Regents of the University of California.  All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that: (1) source code distributions
// retain the above copyright notice and this paragraph in its entirety, (2)
// distributions including binary code include the above copyright notice and
// this paragraph in its entirety in the documentation or other materials
// provided with the distribution, and (3) all advertising materials mentioning
// features or use of this software display the following acknowledgement:
// ``This product includes software developed by the University of California,
// Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
// the University nor the names of its contributors may be used to endorse
// or promote products derived from this software without specific prior
// written permission.
// THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
// WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

#include "config.h"

#include <ctype.h>
#include <math.h>
#include <stdlib.h>

#include "NetVar.h"
#include "HTTP.h"
#include "Event.h"
#include "MIME.h"
#include "TCP_Rewriter.h"

enum {
	EXPECT_REQUEST_LINE,
	EXPECT_REQUEST_MESSAGE,
	EXPECT_REQUEST_TRAILER,
};

enum {
	EXPECT_REPLY_LINE,
	EXPECT_REPLY_MESSAGE,
	EXPECT_REPLY_TRAILER,
};

HTTP_Entity::HTTP_Entity(HTTP_Message *arg_message, MIME_Entity* parent_entity, int arg_expect_body)
:MIME_Entity(arg_message, parent_entity)
	{
	http_message = arg_message;
	expect_body = arg_expect_body;
	chunked_transfer_state = NON_CHUNKED_TRANSFER;
	content_length = -1;	// unspecified
	expect_data_length = 0;
	body_length = 0;
	deliver_body = (http_entity_data != 0);
	}

void HTTP_Entity::EndOfData()
	{
	MIME_Entity::EndOfData();
	}

void HTTP_Entity::Deliver(int len, const char* data, int trailing_CRLF)
	{
	if ( end_of_data )
		{
		// Multipart entities may have trailers
		if ( content_type != CONTENT_TYPE_MULTIPART )
			IllegalFormat("data trailing the end of entity");
		return;
		}

	if ( in_header )
		{
		ASSERT(trailing_CRLF);
		MIME_Entity::Deliver(len, data, trailing_CRLF);
		return;
		}

	// Entity body.
	if ( chunked_transfer_state != NON_CHUNKED_TRANSFER )
		{
		switch ( chunked_transfer_state ) {
		case EXPECT_CHUNK_SIZE:
			ASSERT(trailing_CRLF);
			if ( ! atoi_n(len, data, 0, 16, expect_data_length) )
				{
				http_message->Weird("HTTP_bad_chunk_size");
				expect_data_length = 0;
				}

			if ( expect_data_length > 0 )
				{
				chunked_transfer_state = EXPECT_CHUNK_DATA;
				SetPlainDelivery(expect_data_length);
				}
			else
				{
				// This is the last chunk
				in_header = 1;
				chunked_transfer_state = EXPECT_CHUNK_TRAILER;
				}
			break;

		case EXPECT_CHUNK_DATA:
			ASSERT(! trailing_CRLF);
			ASSERT(len <= expect_data_length);
			expect_data_length -= len;
			if ( expect_data_length <= 0 )
				{
				SetPlainDelivery(0);
				chunked_transfer_state = EXPECT_CHUNK_DATA_CRLF;
				}
			DeliverBody(len, data, 0);
			break;

		case EXPECT_CHUNK_DATA_CRLF:
			ASSERT(trailing_CRLF);
			if ( len > 0 )
				IllegalFormat("inaccurate chunk size: data before <CR><LF>");
			chunked_transfer_state = EXPECT_CHUNK_SIZE;
			break;
		}
		}

	else if ( content_length >= 0 )
		{
		ASSERT(! trailing_CRLF);
		ASSERT(len <= expect_data_length);

		DeliverBody(len, data, 0);

		expect_data_length -= len;
		if ( expect_data_length <= 0 )
			{
			SetPlainDelivery(0);
			EndOfData();
			}
		}

	else if ( content_type == CONTENT_TYPE_MULTIPART ||
		  content_type == CONTENT_TYPE_MESSAGE )
		DeliverBody(len, data, trailing_CRLF);

	else
		DeliverBody(len, data, trailing_CRLF);
	}

void HTTP_Entity::DeliverBody(int len, const char* data, int trailing_CRLF)
	{
	body_length += len;
	if ( trailing_CRLF )
		body_length += 2;

	if ( deliver_body )
		MIME_Entity::Deliver(len, data, trailing_CRLF);
	}

// Returns 1 if the undelivered bytes are completely within the body,
// otherwise returns 0.
int HTTP_Entity::Undelivered(int len)
	{
	if ( end_of_data && in_header )
		return 0;

	if ( chunked_transfer_state != NON_CHUNKED_TRANSFER )
		{
		if ( chunked_transfer_state == EXPECT_CHUNK_DATA &&
		     expect_data_length >= len )
			{
			body_length += len;
			expect_data_length -= len;

			SetPlainDelivery(expect_data_length);

			if ( expect_data_length <= 0 )
				chunked_transfer_state = EXPECT_CHUNK_DATA_CRLF;

			return 1;
			}
		else
			return 0;
		}

	else if ( content_length >= 0 )
		{
		if ( expect_data_length >= len )
			{
			body_length += len;
			expect_data_length -= len;

			SetPlainDelivery(expect_data_length);

			if ( expect_data_length <= 0 )
				EndOfData();

			return 1;
			}

		else
			return 0;
		}

	return 0;
	}

void HTTP_Entity::SubmitData(int len, const char* buf)
	{
	if ( deliver_body )
		MIME_Entity::SubmitData(len, buf);
	}

void HTTP_Entity::SetPlainDelivery(int length)
	{
	http_message->SetPlainDelivery(length);
	}

void HTTP_Entity::SubmitHeader(MIME_Header* h)
	{
	if ( strcasecmp_n(h->get_name(), "content-length") == 0 )
		{
		data_chunk_t vt = h->get_value_token();
		if ( ! is_null_data_chunk(vt) )
			{
			int n;
			if ( atoi_n(vt.length, vt.data, 0, 10, n) )
				content_length = n;
			else
				content_length = 0;
			}
		}

	else if ( strcasecmp_n(h->get_name(), "transfer-encoding") == 0 )
		{
		data_chunk_t vt = h->get_value_token();
		if ( strcasecmp_n(vt, "chunked") == 0 )
			chunked_transfer_state = BEFORE_CHUNK;
		}

	MIME_Entity::SubmitHeader(h);
	}

void HTTP_Entity::SubmitAllHeaders()
	{
	// The presence of a message-body in a request is signaled by
	// the inclusion of a Content-Length or Transfer-Encoding
	// header field in the request's message-headers.
	if ( chunked_transfer_state == EXPECT_CHUNK_TRAILER )
		{
		http_message->SubmitTrailingHeaders(headers);
		chunked_transfer_state = EXPECT_NOTHING;
		EndOfData();
		return;
		}

	MIME_Entity::SubmitAllHeaders();

	if ( expect_body == HTTP_BODY_NOT_EXPECTED )
		{
		EndOfData();
		return;
		}

	if ( chunked_transfer_state != NON_CHUNKED_TRANSFER )
		chunked_transfer_state = EXPECT_CHUNK_SIZE;
	else if ( content_length >= 0 )
		{
		if ( content_length > 0 )
			{
			SetPlainDelivery(content_length);
			expect_data_length = content_length;
			}
		else
			EndOfData(); // handle the case that content-length = 0
		}
	else if ( content_type == CONTENT_TYPE_MULTIPART ||
		  content_type == CONTENT_TYPE_MESSAGE )
		{
		}
	else
		{
		if ( expect_body != HTTP_BODY_EXPECTED )
			// there is no body
			EndOfData();
		}
	}

HTTP_Message::HTTP_Message(HTTP_Conn* arg_conn, HTTP_Endpoint* arg_endp, int expect_body)
: MIME_Message ()
	{
	conn = arg_conn;
	endp = arg_endp;
	is_orig = conn->IsOrig(endp);

	current_entity = 0;
	top_level = new HTTP_Entity(this, 0, expect_body);
	BeginEntity(top_level);

	data_buffer = 0;
	total_buffer_size = 0;

	body_length = 0;
	content_gap_length = 0;
	}

HTTP_Message::~HTTP_Message()
	{
	delete top_level;
	}

Val* HTTP_Message::BuildMessageStat(const int interrupted, const char* msg)
	{
	RecordVal* stat = new RecordVal(http_message_stat);
	stat->Assign(0, new Val(interrupted, TYPE_BOOL));
	stat->Assign(1, new StringVal(msg));
	stat->Assign(2, new Val(body_length, TYPE_COUNT));
	stat->Assign(3, new Val(content_gap_length, TYPE_COUNT));
	return stat;
	}

void HTTP_Message::Done(const int interrupted, const char* detail)
	{
	if ( finished )
		return;

	MIME_Message::Done();

	// DEBUG_MSG("%.6f HTTP message done.\n", network_time);
	top_level->EndOfData();

	if ( http_message_done )
		{
		val_list* vl = new val_list;
		vl->append(conn->BuildConnVal());
		vl->append(new Val(is_orig, TYPE_BOOL));
		vl->append(BuildMessageStat(interrupted, detail));
		conn->ConnectionEvent(http_message_done, vl);
		}

	conn->HTTP_MessageDone(endp, this);

	delete_strings(buffers);

	if ( data_buffer )
		{
		delete data_buffer;
		data_buffer = 0;
		}
	}

int HTTP_Message::Undelivered(int len)
	{
	if ( ! top_level )
		return 0;

	if ( ((HTTP_Entity*) top_level)->Undelivered(len) )
		{
		content_gap_length += len;
		return 1;
		}

	return 0;
	}

void HTTP_Message::BeginEntity(MIME_Entity* entity)
	{
	// DEBUG_MSG("%.6f: begin entity (%d)\n", network_time, is_orig);
	current_entity = (HTTP_Entity*) entity;

	if ( http_begin_entity )
		{
		val_list* vl = new val_list();
		vl->append(conn->BuildConnVal());
		vl->append(new Val(is_orig, TYPE_BOOL));
		conn->ConnectionEvent(http_begin_entity, vl);
		}
	}

void HTTP_Message::EndEntity(MIME_Entity* entity)
	{
	// DEBUG_MSG("%.6f: end entity (%d)\n", network_time, is_orig);
	body_length += ((HTTP_Entity*) entity)->BodyLength();

	DeliverEntityData();

	if ( http_end_entity )
		{
		val_list* vl = new val_list();
		vl->append(conn->BuildConnVal());
		vl->append(new Val(is_orig, TYPE_BOOL));
		conn->ConnectionEvent(http_end_entity, vl);
		}

	current_entity = (HTTP_Entity*) entity->Parent();

	// It is necessary to call Done when EndEntity is triggered by
	// SubmitAllHeaders (through EndOfData).
	if ( entity == top_level )
		Done();
	}

void HTTP_Message::SubmitHeader(MIME_Header* h)
	{
	conn->HTTP_Header(endp, h);
	}

void HTTP_Message::SubmitAllHeaders(MIME_HeaderList& hlist)
	{
	if ( http_all_headers )
		{
		val_list* vl = new val_list();
		vl->append(conn->BuildConnVal());
		vl->append(new Val(is_orig, TYPE_BOOL));
		vl->append(BuildHeaderTable(hlist));
		conn->ConnectionEvent(http_all_headers, vl);
		}

	if ( http_content_type )
		{
		StringVal *ty = current_entity->ContentType();
		StringVal *subty = current_entity->ContentSubType();
		ty->Ref();
		subty->Ref();

		val_list* vl = new val_list();
		vl->append(conn->BuildConnVal());
		vl->append(new Val(is_orig, TYPE_BOOL));
		vl->append(ty);
		vl->append(subty);
		conn->ConnectionEvent(http_content_type, vl);
		}
	}

void HTTP_Message::SubmitTrailingHeaders(MIME_HeaderList& /* hlist */)
	{
	// Do nothing for now.
	}

void HTTP_Message::SubmitData(int len, const char* buf)
	{
	if ( buf != (const char*) data_buffer->Bytes() + buffer_offset ||
	     buffer_offset + len > buffer_size )
		internal_error("buffer misalignment");

	buffer_offset += len;
	if ( buffer_offset >= buffer_size )
		{
		buffers.push_back(data_buffer);
		data_buffer = 0;
		}
	}

int HTTP_Message::RequestBuffer(int* plen, char** pbuf)
	{
	if ( ! http_entity_data )
		return 0;

	if ( ! data_buffer )
		if ( ! InitBuffer(mime_segment_length) )
			return 0;

	*plen = data_buffer->Len() - buffer_offset;
	*pbuf = (char*) data_buffer->Bytes() + buffer_offset;

	return 1;
	}

void HTTP_Message::SubmitAllData()
	{
	// This marks the end of message
	}

void HTTP_Message::SubmitEvent(int event_type, const char* detail)
	{
	const char* category = "";

	switch ( event_type ) {
	case MIME_EVENT_ILLEGAL_FORMAT:
		category = "illegal format";
		break;

	case MIME_EVENT_ILLEGAL_ENCODING:
		category = "illegal encoding";
		break;

	case MIME_EVENT_CONTENT_GAP:
		category = "content gap";
		break;

	default:
		internal_error("unrecoginzied HTTP message event");
	}

	conn->HTTP_Event(category, detail);
	}

void HTTP_Message::SetPlainDelivery(int length)
	{
	endp->SetPlainDelivery(length);
	if ( ! data_buffer )
		InitBuffer(length);
	}

void HTTP_Message::SkipEntityData()
	{
	if ( current_entity )
		current_entity->SkipBody();
	}

void HTTP_Message::DeliverEntityData()
	{
	if ( http_entity_data )
		{
		BroString* entity_data = 0;

		if ( data_buffer && buffer_offset > 0 )
			{
			if ( buffer_offset < buffer_size )
				{
				entity_data = new BroString(data_buffer->Bytes(), buffer_offset, 0);
				delete data_buffer;
				}
			else
				entity_data = data_buffer;

			data_buffer = 0;

			if ( buffers.empty() )
				conn->HTTP_EntityData(endp, entity_data);
			else
				buffers.push_back(entity_data);

			entity_data = 0;
			}

		if ( ! buffers.empty() )
			{
			if ( buffers.size() == 1 )
				{
				entity_data = buffers[0];
				buffers.clear();
				}
			else
				{
				entity_data = concatenate(buffers);
				delete_strings(buffers);
				}

			conn->HTTP_EntityData(endp, entity_data);
			}
		}
	else
		{
		delete_strings(buffers);

		if ( data_buffer )
			delete data_buffer;

		data_buffer = 0;
		}

	total_buffer_size = 0;
	}

int HTTP_Message::InitBuffer(int length)
	{
	if ( length <= 0 )
		return 0;

	if ( total_buffer_size >= http_entity_data_delivery_size )
		DeliverEntityData();

	if ( total_buffer_size + length > http_entity_data_delivery_size )
		{
		length = http_entity_data_delivery_size - total_buffer_size;
		if ( length <= 0 )
			return 0;
		}

	u_char* b = new u_char[length];
	data_buffer = new BroString(0, b, length);

	buffer_size = length;
	total_buffer_size += length;
	buffer_offset = 0;

	return 1;
	}

void HTTP_Message::Weird(const char* msg)
	{
	conn->Weird(msg);
	}

HTTP_Endpoint::HTTP_Endpoint(HTTP_Conn* conn, int is_orig)
:TCP_EndpointLine(conn, is_orig,
		  0 /* !NUL_sensitive */,
		  1 /* skip partial */)
	{
	plain_delivery_length = 0;
	}

void HTTP_Endpoint::SetPlainDelivery(int length)
	{
	if ( length < 0 )
		internal_error("negative length for plain delivery");
	plain_delivery_length = length;
	}

int HTTP_Endpoint::DoDeliverOnce(int len, u_char* data)
	{
	if ( plain_delivery_length > 0 )
		{
		int deliver_plain =
			plain_delivery_length > len ? len : plain_delivery_length;

		last_char = 0; // clear last_char
		plain_delivery_length -= deliver_plain;
		((HTTP_Conn*) conn)->DeliverPlainSegment(this, deliver_plain, (const char*) data);
		return deliver_plain;
		}
	else
		return TCP_EndpointLine::DoDeliverOnce(len, data);
	}

HTTP_Conn::HTTP_Conn(NetSessions* s, HashKey* k, double t, const ConnID* id,
		const struct tcphdr* tp)
: TCP_Connection(s, k, t, id, tp)
	{
	num_requests = num_replies = 0;
	num_request_lines = num_reply_lines = 0;
	request_version = reply_version = 0.0;	// unknown version
	keep_alive = 0;

	request_message = reply_message = 0;
	request_state = EXPECT_REQUEST_LINE;
	reply_state = EXPECT_REPLY_LINE;

	request_ongoing = 0;
	request_method = request_URI = 0;
	unescaped_URI = 0;

	reply_ongoing = 0;
	reply_code = 0;
	reply_reason_phrase = 0;
	}

void HTTP_Conn::BuildEndpoints()
	{
	orig = new HTTP_Endpoint(this, 1);

	if ( http_reply )
		resp = new HTTP_Endpoint(this, 0);
	else
		resp = new TCP_Endpoint(this, 0);
	}

HTTP_Conn::~HTTP_Conn()
	{
	Unref(request_method);
	Unref(request_URI);
	Unref(unescaped_URI);
	Unref(reply_reason_phrase);
	}

void HTTP_Conn::Done()
	{
	if ( finished )
		return;

	finished = 1;

	RequestMade(1, "message interrupted when connection done");
	ReplyMade(1, "message interrupted when connection done");

	delete request_message;
	request_message = 0;

	delete reply_message;
	reply_message = 0;

	GenStats();

	while ( ! unanswered_requests.empty() )
		{
		Unref(unanswered_requests.front());
		unanswered_requests.pop();
		}

	TCP_Connection::Done();
	}

void HTTP_Conn::EndpointEOF(TCP_EndpointContents* endp)
	{
	// DEBUG_MSG("%.6f eof\n", network_time);

	if ( IsOrig(endp) )
		RequestMade(0, "message ends as connection contents are completely delivered");
	else
		ReplyMade(0, "message ends as connection contents are completely delivered");

	TCP_Connection::EndpointEOF(endp);
	}

void HTTP_Conn::ConnectionFinished(int half_finished)
	{
	// DEBUG_MSG("%.6f connection finished\n", network_time);

	RequestMade(1, "message ends as connection is finished");
	ReplyMade(1, "message ends as connection is finished");

	TCP_Connection::ConnectionFinished(half_finished);
	}

void HTTP_Conn::PacketWithRST()
	{
	RequestMade(1, "message interrupted by RST");
	ReplyMade(1, "message interrupted by RST");

	TCP_Connection::PacketWithRST();
	}

void HTTP_Conn::ConnectionReset()
	{
	RequestMade(1, "message interrupted by RST");
	ReplyMade(1, "message interrupted by RST");

	TCP_Connection::ConnectionReset();
	}

void HTTP_Conn::GenStats()
	{
	if ( http_stats )
		{
		RecordVal* r = new RecordVal(http_stats_rec);
		r->Assign(0, new Val(num_requests, TYPE_COUNT));
		r->Assign(1, new Val(num_replies, TYPE_COUNT));
		r->Assign(2, new Val(request_version, TYPE_DOUBLE));
		r->Assign(3, new Val(reply_version, TYPE_DOUBLE));

		val_list* vl = new val_list;
		vl->append(BuildConnVal());
		vl->append(r);

		// DEBUG_MSG("%.6f http_stats\n", network_time);
		ConnectionEvent(http_stats, vl);
		}
	}

void HTTP_Conn::NewLine(TCP_Endpoint* s, int length, const char* line, int
/* is_const */)
	{
	// HTTP_Event("HTTP line", new_string_val(length, line));

	const char* end_of_line = line + length;
	if ( s == orig )
		{
		++num_request_lines;

		switch ( request_state ) {
		case EXPECT_REQUEST_LINE:
			if ( HTTP_RequestLine(line, end_of_line) )
				{
				++num_requests;

				if ( ! keep_alive && num_requests > 1 )
					Weird("unexpected_multiple_HTTP_requests");

				request_state = EXPECT_REQUEST_MESSAGE;
				request_ongoing = 1;
				Ref(request_method);
				Ref(request_URI);
				Ref(unescaped_URI);
				unanswered_requests.push(request_method->Ref());
				HTTP_Request();
				InitHTTPMessage((HTTP_Endpoint *) s,
						request_message, HTTP_BODY_MAYBE);
				}

			else
				{
				if ( ! RequestExpected() )
					HTTP_Event("crud_trailing_HTTP_request",
						   new_string_val(line, end_of_line));
				}
			break;

		case EXPECT_REQUEST_MESSAGE:
			request_message->Deliver(length, line, 1);
			break;

		case EXPECT_REQUEST_TRAILER:
			break;
		}
		}
	else
		{ // HTTP reply
		switch ( reply_state ) {
		case EXPECT_REPLY_LINE:
			if ( HTTP_ReplyLine(line, end_of_line) )
				{
				++num_replies;

				if ( unanswered_requests.empty() )
					Weird("unmatched_HTTP_reply");

				reply_state = EXPECT_REPLY_MESSAGE;
				reply_ongoing = 1;

				HTTP_Reply();

				InitHTTPMessage((HTTP_Endpoint *) s,
						reply_message,
						ExpectReplyMessageBody());
				}
			break;

		case EXPECT_REPLY_MESSAGE:
			reply_message->Deliver(length, line, 1);
			break;

		case EXPECT_REPLY_TRAILER:
			break;
		}
		}
	}

void HTTP_Conn::DeliverPlainSegment(TCP_Endpoint* sender,
				    int length, const char* data)
	{
	if ( IsOrig(sender) )
		request_message->Deliver(length, data, 0);
	else
		reply_message->Deliver(length, data, 0);
	}

void HTTP_Conn::Undelivered(TCP_Endpoint* sender, int seq, int length)
	{
	TCP_Connection::Undelivered(sender, seq, length);

	HTTP_Message* msg = IsOrig(sender) ? request_message : reply_message;
	if ( msg )
		msg->SubmitEvent(MIME_EVENT_CONTENT_GAP, fmt("seq=%d, len=%d", seq, length));

	// Check if the content gap falls completely within a message body
	if ( msg && msg->Undelivered(length) )
		// If so, we are safe to skip the content and go on parsing
		return;

	// Otherwise stop parsing the connection
	if ( IsOrig(sender) )
		{
		// Stop parsing reply messages too, because whether a
		// reply contains a body may depend on knowing the
		// request method

		RequestMade(1, "message interrupted by a content gap");
		ReplyMade(1, "message interrupted by a content gap");

		orig->AsEndpointContents()->SetSkipDeliveries(1);
		TCP_EndpointContents* rc = resp->AsEndpointContents();
		if ( rc )
			rc->SetSkipDeliveries(1);
		}
	else
		{
		ReplyMade(1, "message interrupted by a content gap");
		sender->AsEndpointContents()->SetSkipDeliveries(1);
		}
	}

const char* HTTP_Conn::PrefixMatch(const char* line, const char* end_of_line,
				   const char* prefix)
	{
	while ( *prefix && line < end_of_line && *prefix == *line )
		{
		++prefix;
		++line;
		}

	if ( *prefix )
		// It didn't match.
		return 0;

	return line;
	}

const char* HTTP_Conn::PrefixWordMatch(const char* line, const char*
end_of_line, const char* prefix)
	{
	if ( (line = PrefixMatch(line, end_of_line, prefix)) == 0 )
		return 0;

	const char* orig_line = line;
	line = skip_whitespace(line, end_of_line);

	if ( line == orig_line )
		// Word didn't end at prefix.
		return 0;

	return line;
	}

int HTTP_Conn::HTTP_RequestLine(const char* line, const char* end_of_line)
	{
	const char* rest;
	static const char* http_methods[] = {
		"GET", "POST", "HEAD",

		"OPTIONS", "PUT", "DELETE", "TRACE", "CONNECT",

		// HTTP methods for distributed authoring.
		"PROPFIND", "PROPPATCH", "MKCOL", "DELETE", "PUT",
		"COPY", "MOVE", "LOCK", "UNLOCK",

		0,
	};

	int i;
	for ( i = 0; http_methods[i]; ++i )
		if ( (rest = PrefixWordMatch(line, end_of_line, http_methods[i])) != 0 )
			break;

	if ( ! http_methods[i] )
		{
		// Weird("HTTP_unknown_method");
		if ( RequestExpected() )
			HTTP_Event("unknown_HTTP_method", new_string_val(line, end_of_line));
		return 0;
		}

	request_method = new StringVal(http_methods[i]);

	if ( ! ParseRequest(rest, end_of_line) )
		internal_error("HTTP ParseRequest failed");

	if ( rule_matcher )
		Match(Rule::HTTP, (const u_char *) request_URI->AsString()->Bytes(),
			request_URI->AsString()->Len(), true, true, 1);

	return 1;
	}

int HTTP_Conn::ParseRequest(const char* line, const char* end_of_line)
	{
	const char* end_of_uri;
	const char* version_start;
	const char* version_end;

	for ( end_of_uri = line; end_of_uri < end_of_line; ++end_of_uri )
		{
		if ( ! is_reserved_URI_char(*end_of_uri) &&
		     ! is_unreserved_URI_char(*end_of_uri) &&
		     *end_of_uri != '%' )
			break;
		}

	for ( version_start = end_of_uri; version_start < end_of_line; ++version_start )
		{
		end_of_uri = version_start;
		version_start = skip_whitespace(version_start, end_of_line);
		if ( PrefixMatch(version_start, end_of_line, "HTTP/") )
			break;
		}

	if ( version_start >= end_of_line )
		{
		// If no version is found
		SetVersion(request_version, 0.9);
		}
	else
		{
		if ( version_start + 8 <= end_of_line )
			{
			version_start += 5; // "HTTP/"
			SetVersion(request_version,
					HTTP_Version(end_of_line - version_start,
							version_start));

			version_end = version_start + 3;
			if ( skip_whitespace(version_end, end_of_line) != end_of_line )
				HTTP_Event("crud after HTTP version is ignored",
					   new_string_val(line, end_of_line));
			}
		else
			HTTP_Event("bad_HTTP_version", new_string_val(line, end_of_line));
		}

	// NormalizeURI(line, end_of_uri);

	request_URI = new StringVal(end_of_uri - line, line);
	unescaped_URI = new StringVal(unescape_URI((const u_char*) line,
					(const u_char*) end_of_uri, this));

	return 1;
	}

// Only recognize [0-9][.][0-9].
double HTTP_Conn::HTTP_Version(int len, const char* data)
	{
	if ( len >= 3 &&
	     data[0] >= '0' && data[0] <= '9' &&
	     data[1] == '.' &&
	     data[2] >= '0' && data[2] <= '9' )
		{
		return double(data[0] - '0') + 0.1 * double(data[2] - '0');
		}
	else
		{
	        HTTP_Event("bad_HTTP_version", new_string_val(len, data));
		return 0;
		}
	}

void HTTP_Conn::SetVersion(double& version, double new_version)
	{
	if ( version == 0.0 )
		version = new_version;

	else if ( version != new_version )
		Weird("HTTP_version_mismatch");

	if ( version > 1.05 )
		keep_alive = 1;
	}

void HTTP_Conn::HTTP_Event(const char* category, const char* detail)
	{
	HTTP_Event(category, new StringVal(detail));
	}

void HTTP_Conn::HTTP_Event(const char* category, StringVal* detail)
	{
	if ( http_event )
		{
		val_list* vl = new val_list();
		vl->append(BuildConnVal());
		vl->append(new StringVal(category));
		vl->append(detail);

		// DEBUG_MSG("%.6f http_event\n", network_time);
		ConnectionEvent(http_event, vl);
		}
	}

void HTTP_Conn::HTTP_Request()
	{
	if ( http_request )
		{
		val_list* vl = new val_list;
		vl->append(BuildConnVal());
		vl->append(request_method);
		vl->append(request_URI);
		vl->append(unescaped_URI);
		vl->append(new StringVal(fmt("%.1f", request_version)));
		// DEBUG_MSG("%.6f http_request\n", network_time);
		ConnectionEvent(http_request, vl);
		}
	}

void HTTP_Conn::HTTP_Reply()
	{
	if ( http_reply )
		{
		val_list* vl = new val_list;
		vl->append(BuildConnVal());
		vl->append(new StringVal(fmt("%.1f", reply_version)));
		vl->append(new Val(reply_code, TYPE_COUNT));
		if ( reply_reason_phrase )
			vl->append(reply_reason_phrase);
		else
			vl->append(new StringVal("<empty>"));
		ConnectionEvent(http_reply, vl);
		}
	else
		{
		Unref(reply_reason_phrase);
		reply_reason_phrase = 0;
		}
	}

void HTTP_Conn::RequestMade(const int interrupted, const char* msg)
	{
	if ( ! request_ongoing )
		return;

	request_ongoing = 0;

	if ( request_message )
		request_message->Done(interrupted, msg);

	// DEBUG_MSG("%.6f request made\n", network_time);

	Unref(request_method);
	Unref(unescaped_URI);
	Unref(request_URI);

	request_method = request_URI = unescaped_URI = 0;

	num_request_lines = 0;

	request_state = EXPECT_REQUEST_LINE;
	}

void HTTP_Conn::ReplyMade(const int interrupted, const char* msg)
	{
	if ( ! reply_ongoing )
		return;

	reply_ongoing = 0;

	// DEBUG_MSG("%.6f reply made\n", network_time);

	if ( reply_message )
		reply_message->Done(interrupted, msg);

	if ( ! unanswered_requests.empty() )
		{
		Unref(unanswered_requests.front());
		unanswered_requests.pop();
		}

	reply_code = 0;
	reply_reason_phrase = 0;

	reply_state = EXPECT_REPLY_LINE;
	}

void HTTP_Conn::RequestClash(Val* /* clash_val */)
	{
	Weird("multiple_HTTP_request_elements");

	// Flush out old values.
	RequestMade(1, "request clash");
	}

BroString* HTTP_Conn::UnansweredRequestMethod()
	{
	return unanswered_requests.empty() ? 0 : unanswered_requests.front()->AsString();
	}

int HTTP_Conn::HTTP_ReplyLine(const char* line, const char* end_of_line)
	{
	const char* rest;

	if ( ! (rest = PrefixMatch(line, end_of_line, "HTTP/")) )
		{
		// ##TODO: some server replies with an HTML document
		// without a status line and a MIME header, when the
		// request is malformed.
		HTTP_Event("bad_HTTP_reply", new_string_val(line, end_of_line));
		return 0;
		}

	SetVersion(reply_version, HTTP_Version(end_of_line - rest, rest));

	for ( ; rest < end_of_line; ++rest )
		if ( is_lws(*rest) )
			break;

	if ( rest >= end_of_line )
		{
		HTTP_Event("HTTP_reply_code_missing",
				new_string_val(line, end_of_line));
		return 0;
		}

	rest = skip_whitespace(rest, end_of_line);

	if ( rest + 3 > end_of_line )
		{
		HTTP_Event("HTTP_reply_code_missing",
			new_string_val(line, end_of_line));
		return 0;
		}

	reply_code = HTTP_ReplyCode(rest);

	for ( rest += 3; rest < end_of_line; ++rest )
		if ( is_lws(*rest) )
			break;

	if ( rest >= end_of_line )
		{
		HTTP_Event("HTTP_reply_reason_phrase_missing",
			new_string_val(line, end_of_line));
		// Tolerate missing reason phrase?
		return 1;
		}

	rest = skip_whitespace(rest, end_of_line);
	reply_reason_phrase =
		new StringVal(end_of_line - rest, (const char *) rest);

	return 1;
	}

int HTTP_Conn::HTTP_ReplyCode(const char* code_str)
	{
	if ( isdigit(code_str[0]) && isdigit(code_str[1]) && isdigit(code_str[2]) )
		return	(code_str[0] - '0') * 100 +
			(code_str[1] - '0') * 10 +
			(code_str[2] - '0');
	else
		return 0;
	}

int HTTP_Conn::ExpectReplyMessageBody()
	{
	// RFC 2616:
	//
	//     For response messages, whether or not a message-body is included with
	//     a message is dependent on both the request method and the response
	//     status code (section 6.1.1). All responses to the HEAD request method
	//     MUST NOT include a message-body, even though the presence of entity-
	//     header fields might lead one to believe they do. All 1xx
	//     (informational), 204 (no content), and 304 (not modified) responses
	//     MUST NOT include a message-body. All other responses do include a
	//     message-body, although it MAY be of zero length.

	BroString* method = UnansweredRequestMethod();

	if ( method && strcasecmp_n(method->Len(), (const char*) (method->Bytes()), "HEAD") == 0 )
		return HTTP_BODY_NOT_EXPECTED;

	if ( (reply_code >= 100 && reply_code < 200) ||
	     reply_code == 204 || reply_code == 304 )
		return HTTP_BODY_NOT_EXPECTED;

	return HTTP_BODY_EXPECTED;
	}

void HTTP_Conn::HTTP_Header(TCP_Endpoint* sender, MIME_Header* h)
	{
	if ( strcasecmp_n(h->get_name(), "server") == 0 )
		{
		ParseServerVersion(h->get_value());
		}

	// To be "liberal", we only look at "keep-alive" on the client
	// side, and if seen assume the connection to be persistent.
	// This seems fairly safe - at worst, the client does indeed
	// send additional requests, and the server ignores them.
	else if ( IsOrig(sender) &&
		  strcasecmp_n(h->get_name(), "connection") == 0 )
		{
		if ( strcasecmp_n(h->get_value_token(), "keep-alive") == 0 )
			keep_alive = 1;
		}

	if ( http_header )
		{
		val_list* vl = new val_list();
		vl->append(BuildConnVal());
		vl->append(new Val(IsOrig(sender), TYPE_BOOL));
		vl->append(new_string_val(h->get_name())->ToUpper());
		vl->append(new_string_val(h->get_value()));
		// DEBUG_MSG("%.6f http_header\n", network_time);
		ConnectionEvent(http_header, vl);
		}
	}

void HTTP_Conn::ParseServerVersion(data_chunk_t ver)
	{
	int len = ver.length;
	const char* data = ver.data;

	// The RFC defines:
	//
	//	product		= token ["/" product-version]
	//	product-version = token
	//	Server		= "Server" ":" 1*( product | comment )

	int offset;
	data_chunk_t product, product_version;
	int num_version = 0;

	while ( len > 0 )
		{
		offset = MIME_get_slash_token_pair(len, data, &product, &product_version);
		if ( offset < 0 )
			{
			// I guess version detection is best-effort,
			// so we do not complain in the final version
			if ( num_version == 0 )
				{
				HTTP_Event("bad_HTTP_version", new_string_val(len, data));
				}
			break;
			}
		else
			{
			len -= offset;
			data += offset;

			int version_len = product.length + 1 + product_version.length;
			char* version_str = new char[version_len+1];
			char* s = version_str;
			memcpy(s, product.data, product.length);
			s += product.length;
			*(s++) = '/';
			memcpy(s, product_version.data, product_version.length);
			s += product_version.length;
			*s = 0;
			VersionFoundEvent(Resp()->dst_addr, version_str, version_len);
			delete [] version_str;
			++num_version;
			}
		}
	}

void HTTP_Conn::HTTP_EntityData(TCP_Endpoint* sender, BroString* entity_data)
	{
	if ( http_entity_data )
		{
		val_list* vl = new val_list();
		vl->append(BuildConnVal());
		vl->append(new Val(IsOrig(sender), TYPE_BOOL));
		vl->append(new Val(entity_data->Len(), TYPE_COUNT));
		vl->append(new StringVal(entity_data));
		ConnectionEvent(http_entity_data, vl);
		}
	}

// Calls request/reply done
void HTTP_Conn::HTTP_MessageDone(TCP_Endpoint* sender, HTTP_Message* /* message */)
	{
	if ( IsOrig(sender) )
		RequestMade(0, "message ends normally");
	else
		ReplyMade(0, "message ends normally");
	}

void HTTP_Conn::InitHTTPMessage(HTTP_Endpoint* endp, HTTP_Message*& message, int expect_body)
	{
	if ( message )
		{
		if ( ! message->Finished() )
			Weird("HTTP_overlapping_messages");

		delete message;
		}

	// DEBUG_MSG("%.6f init http message\n", network_time);
	message = new HTTP_Message(this, endp, expect_body);
	}

void HTTP_Conn::SkipEntityData(int is_orig)
	{
	HTTP_Message* msg = is_orig ? request_message : reply_message;

	if ( msg )
		msg->SkipEntityData();
	}

int is_reserved_URI_char(unsigned char ch) // see RFC 2396 (definition of URI)
	{
	return strchr(";/?:@&=+$,", ch) != 0;
	}

int is_unreserved_URI_char(unsigned char ch) // see RFC 2396 (definition of URI)
	{
	return isalnum(ch) || strchr("-_.!~*\'()", ch) != 0;
	}

void escape_URI_char(unsigned char ch, unsigned char*& p)
	{
	*p++ = '%';
	*p++ = encode_hex((ch >> 4) & 0xf);
	*p++ = encode_hex(ch & 0xf);
	}

BroString* unescape_URI(const u_char* line, const u_char* line_end,
			Connection* conn)
	{
	byte_vec decoded_URI = new u_char[line_end - line + 1];
	byte_vec URI_p = decoded_URI;

	// An 'unescaped_special_char' here means a character that *should*
	// be escaped, but isn't in the URI.  A control characters that
	// appears directly in the URI would be an example.  The RFC implies
	// that if we do not unescape the URI that we see in the trace, every
	// character should be a printable one -- either reserved or unreserved
	// (or '%').
	//
	// Counting the number of unescaped characters and generating a weird
	// event on URI's with unescaped characters (which are rare) will
	// let us locate strange-looking URI's in the trace -- those URI's
	// are often interesting.
	int unescaped_special_char = 0;

	while ( line < line_end )
		{
		if ( *line == '%' )
			{
			++line;

			if ( line == line_end )
				{
				// How to deal with % at end of line?
				// *URI_p++ = '%';
				if ( conn )
					conn->Weird("illegal_%_at_end_of_URI");
				break;
				}

			else if ( *line == '%' )
				{
				// Double '%' might be either due to
				// software bug, or more likely, an
				// evasion (e.g. used by Nimda).
				// *URI_p++ = '%';
				if ( conn )
					conn->Weird("double_%_in_URI");
				--line;	// ignore the first '%'
				}

			else if ( isxdigit(line[0]) && isxdigit(line[1]) )
				{
				*URI_p++ = (decode_hex(line[0]) << 4) +
					   decode_hex(line[1]);
				++line; // place line at the last hex digit
				}

			else
				{
				if ( conn )
					conn->Weird("unescaped_%_in_URI");
				*URI_p++ = '%';	// put back initial '%'
				*URI_p++ = *line;	// take char w/o interp.
				}
			}

		else
			{
			if ( ! is_reserved_URI_char(*line) &&
			     ! is_unreserved_URI_char(*line) )
				// Count these up as a way to compress
				// the corresponding Weird event to a
				// single instance.
				++unescaped_special_char;
			*URI_p++ = *line;
			}

		++line;
		}

	URI_p[0] = 0;

	if ( unescaped_special_char && conn )
		conn->Weird("unescaped_special_URI_char");

	return new BroString(1, decoded_URI, URI_p - decoded_URI);
	}

#include "http-rw.bif.func_def"
