diff -ru htdig-3.1.5/CONFIG htdig-3.1.5_patched/CONFIG --- htdig-3.1.5/CONFIG Fri Feb 25 03:29:48 2000 +++ htdig-3.1.5_patched/CONFIG Wed Apr 18 16:57:02 2001 @@ -8,7 +8,7 @@ # These variables are set by configure # # This specifies the root of the directory tree to be used by ht://Dig -prefix= /opt/www/htdig +prefix= /home/www/htdig # This specifies the root of the directory tree to be used for programs # installed by ht://Dig @@ -62,14 +62,14 @@ # The directory where your HTTP server looks for CGI programs. This is where # htsearch will get installed. # -CGIBIN_DIR= /opt/www/cgi-bin +CGIBIN_DIR= /home/www/cgi-bin # # IMAGE_DIR # Define this to be a place that can be accessed by your web server. This is # where a couple of images will be placed. # -IMAGE_DIR= /opt/www/htdocs/htdig +IMAGE_DIR= /home/www/htdocs/htdig # # IMAGE_URL_PREFIX @@ -82,7 +82,7 @@ # Set this to the absolute path where you want the sample search form to # be installed. # -SEARCH_DIR= /opt/www/htdocs/htdig +SEARCH_DIR= /home/www/htdocs/htdig # # SEARCH_FORM diff -ru htdig-3.1.5/Makefile.config.in htdig-3.1.5_patched/Makefile.config.in --- htdig-3.1.5/Makefile.config.in Fri Feb 25 03:28:58 2000 +++ htdig-3.1.5_patched/Makefile.config.in Wed Apr 18 13:37:41 2001 @@ -24,13 +24,13 @@ SENDMAIL= @SENDMAIL@ DEFINES= -DDEFAULT_CONFIG_FILE=\"$(DEFAULT_CONFIG_FILE)\" -LIBDIRS= -L../htlib -L../htcommon -L../db/dist -L/usr/lib +LIBDIRS= -L../htlib -L../htcommon -L../db/dist -L/usr/lib -L$(OPENSSL)/lib INCS= -I$(top_srcdir)/htlib -I$(top_srcdir)/htcommon \ - -I../db/dist -I../include + -I../db/dist -I../include -I$(OPENSSL)/include HTLIBS= ../htcommon/libcommon.a \ ../htlib/libht.a \ ../db/dist/libdb.a -LIBS= $(HTLIBS) @LIBS@ +LIBS= $(HTLIBS) @LIBS@ -lssl -lcrypto DIST= @PACKAGE@-@VERSION@ DISTDIR= $(top_srcdir)/../$(DIST) diff -ru htdig-3.1.5/configure htdig-3.1.5_patched/configure --- htdig-3.1.5/configure Fri Feb 25 03:28:58 2000 +++ htdig-3.1.5_patched/configure Wed Apr 18 17:04:12 2001 @@ -11,7 +11,7 @@ ac_help= ac_default_prefix=/usr/local # Any additions from configure.in: -ac_default_prefix=/opt/www/htdig +ac_default_prefix=/home/www/htdig ac_help="$ac_help --disable-zlib do not compile with zlib support" ac_help="$ac_help @@ -806,7 +806,7 @@ withval="$with_cgi_bin_dir" CGIBIN_DIR="$withval" else - CGIBIN_DIR="/opt/www/cgi-bin" + CGIBIN_DIR="/home/www/cgi-bin" fi @@ -815,7 +815,7 @@ withval="$with_image_dir" IMAGE_DIR="$withval" else - IMAGE_DIR="/opt/www/htdocs/htdig" + IMAGE_DIR="/home/www/htdocs/htdig" fi @@ -824,7 +824,7 @@ withval="$with_search_dir" SEARCH_DIR="$withval" else - SEARCH_DIR="/opt/www/htdocs/htdig" + SEARCH_DIR="/home/www/htdocs/htdig" fi @@ -2107,7 +2107,7 @@ int main() { /* Ultrix mips cc rejects this. */ -typedef int charset[2]; const charset x; +typedef int charset[2]; const charset x = {0,0}; /* SunOS 4.1.1 cc rejects this. */ char const *const *ccp; char **p; diff -ru htdig-3.1.5/configure.in htdig-3.1.5_patched/configure.in --- htdig-3.1.5/configure.in Fri Feb 25 03:28:58 2000 +++ htdig-3.1.5_patched/configure.in Wed Apr 18 17:02:46 2001 @@ -22,7 +22,7 @@ AC_PREREQ(2.13) AC_CONFIG_HEADER(include/htconfig.h) -AC_PREFIX_DEFAULT(/opt/www/htdig) +AC_PREFIX_DEFAULT(/home/www/htdig) HTDIG_TOP=`pwd` AC_SUBST(HTDIG_TOP) @@ -31,13 +31,13 @@ AC_ARG_ENABLE(zlib,[ --disable-zlib do not compile with zlib support], WITH_ZLIB=$enableval, WITH_ZLIB="yes") AC_ARG_WITH(cgi-bin-dir, [ --with-cgi-bin-dir=DIR where your cgi-bin directory is], - CGIBIN_DIR="$withval", CGIBIN_DIR="/opt/www/cgi-bin") + CGIBIN_DIR="$withval", CGIBIN_DIR="/home/www/cgi-bin") AC_SUBST(CGIBIN_DIR) AC_ARG_WITH(image-dir, [ --with-image-dir=DIR where the ht://Dig images are installed], - IMAGE_DIR="$withval", IMAGE_DIR="/opt/www/htdocs/htdig") + IMAGE_DIR="$withval", IMAGE_DIR="/home/www/htdocs/htdig") AC_SUBST(IMAGE_DIR) AC_ARG_WITH(search-dir, [ --with-search-dir=DIR where the sample search form should be installed], - SEARCH_DIR="$withval", SEARCH_DIR="/opt/www/htdocs/htdig") + SEARCH_DIR="$withval", SEARCH_DIR="/home/www/htdocs/htdig") AC_SUBST(SEARCH_DIR) dnl While we're at it, let's work out the program transformations diff -ru htdig-3.1.5/htcommon/DocumentDB.cc htdig-3.1.5_patched/htcommon/DocumentDB.cc --- htdig-3.1.5/htcommon/DocumentDB.cc Fri Feb 25 03:29:10 2000 +++ htdig-3.1.5_patched/htcommon/DocumentDB.cc Wed Apr 18 13:37:41 2001 @@ -217,7 +217,7 @@ while ((key = dbf->Get_Next())) { dbf->Get(key, data); - if (strncmp(HtURLCodec::instance()->decode(key), "http:", 5) == 0) + if (strncmp(HtURLCodec::instance()->decode(key), "http:", 5) == 0 || strncmp(HtURLCodec::instance()->decode(key), "https:", 6) == 0) { ref = new DocumentRef; ref->Deserialize(data); @@ -284,7 +284,7 @@ while ((coded_key = dbf->Get_Next())) { String key = HtURLCodec::instance()->decode(coded_key); - if (mystrncasecmp(key, "http:", 5) == 0) + if (mystrncasecmp(key, "http:", 5) == 0 || mystrncasecmp(key, "https:", 6) == 0) { DocumentRef *ref = (*this)[key]; if (ref) diff -ru htdig-3.1.5/htcommon/defaults.cc htdig-3.1.5_patched/htcommon/defaults.cc --- htdig-3.1.5/htcommon/defaults.cc Fri Feb 25 03:29:10 2000 +++ htdig-3.1.5_patched/htcommon/defaults.cc Wed Apr 18 13:37:41 2001 @@ -38,7 +38,7 @@ {"bad_word_list", "${common_dir}/bad_words"}, {"build_select_lists", ""}, {"case_sensitive", "true"}, - {"common_url_parts", "http:// http://www. ftp:// ftp://ftp. /pub/ .html .htm .gif .jpg .jpeg /index.html /index.htm .com/ .com mailto:"}, + {"common_url_parts", "https:// https://www. http:// http://www. ftp:// ftp://ftp. /pub/ .html .htm .gif .jpg .jpeg /index.html /index.htm .com/ .com mailto:"}, {"create_image_list", "false"}, {"create_url_list", "false"}, {"compression_level", "0"}, diff -ru htdig-3.1.5/htdig/Document.cc htdig-3.1.5_patched/htdig/Document.cc --- htdig-3.1.5/htdig/Document.cc Fri Feb 25 03:29:10 2000 +++ htdig-3.1.5_patched/htdig/Document.cc Wed Apr 18 13:37:41 2001 @@ -220,6 +220,7 @@ tm.tm_year += 1900; tm.tm_yday = 0; // clear these to prevent problems in strftime() tm.tm_wday = 0; + tm.tm_isdst = -1; if (debug > 2) { @@ -331,7 +332,7 @@ return Document_no_host; } } - + c.assign_ssl(strcmp(url->service(), "https") == 0); if (c.connect(1) == NOTOK) { if (debug) diff -ru htdig-3.1.5/htdig/Images.cc htdig-3.1.5_patched/htdig/Images.cc --- htdig-3.1.5/htdig/Images.cc Fri Feb 25 03:29:10 2000 +++ htdig-3.1.5_patched/htdig/Images.cc Wed Apr 18 13:37:41 2001 @@ -61,7 +61,7 @@ { String u = url; URL Url(url); - if (strcmp(Url.service(), "http") != 0) + if (strcmp(Url.service(), "http") != 0 && strcmp(Url.service(), "https") != 0) return 0; u.lowercase(); @@ -81,6 +81,7 @@ return 0; if (c.assign_server(Url.host()) == NOTOK) return 0; + c.assign_ssl(strcmp(Url.service(), "https") == 0); if (c.connect(1) == NOTOK) { diff -ru htdig-3.1.5/htdig/Retriever.cc htdig-3.1.5_patched/htdig/Retriever.cc --- htdig-3.1.5/htdig/Retriever.cc Fri Feb 25 03:29:10 2000 +++ htdig-3.1.5_patched/htdig/Retriever.cc Wed Apr 18 13:37:41 2001 @@ -117,8 +117,7 @@ // from == 2 add url from db.log // from == 3 urls in db.docs and there was a db.log // -void -Retriever::Initial(char *list, int from) +void Retriever::Initial(char *list, int from) { // // Split the list of urls up into individual urls. @@ -133,14 +132,15 @@ URL u(tokens[i]); server = (Server *) servers[u.signature()]; url = u.get(); +cout << "URL: " << url; if (debug > 2) cout << "\t" << from << ":" << (int) log << ":" << url; if (!server) { - String robotsURL = "http://"; - robotsURL << u.host() << "/robots.txt"; + String robotsURL = u.service(); + robotsURL << "://" << u.host() << "/robots.txt"; StringList *localRobotsFiles = GetLocal(robotsURL.get()); - server = new Server(u.host(), u.port(), localRobotsFiles); + server = new Server(u.host(), u.port(), strcmp(u.service(), "https") == 0, localRobotsFiles); servers.Add(u.signature(), server); delete localRobotsFiles; } @@ -668,10 +668,10 @@ // Currently, we only deal with HTTP URLs. Gopher and ftp will // come later... ***FIX*** // - if (strstr(u, "/../") || strncmp(u, "http://", 7) != 0) + if (strstr(u, "/../") || (strncmp(u, "http://", 7) != 0 && strncmp(u, "https://", 8) != 0)) { if (debug > 2) - cout << endl <<" Rejected: Not an http or relative link!"; + cout << endl <<" Rejected: Not an http, https or relative link!"; return FALSE; } @@ -1232,10 +1232,10 @@ // // Hadn't seen this server, yet. Register it // - String robotsURL = "http://"; - robotsURL << url.host() << "/robots.txt"; + String robotsURL = url.service(); + robotsURL << "://" << url.host() << "/robots.txt"; StringList *localRobotsFile = GetLocal(robotsURL.get()); - server = new Server(url.host(), url.port(), localRobotsFile); + server = new Server(url.host(), url.port(), (int) (strcmp(url.service(), "https") == 0), localRobotsFile); servers.Add(url.signature(), server); delete localRobotsFile; } @@ -1365,10 +1365,10 @@ // // Hadn't seen this server, yet. Register it // - String robotsURL = "http://"; - robotsURL << url.host() << "/robots.txt"; + String robotsURL = url.service(); + robotsURL << "://" << url.host() << "/robots.txt"; StringList *localRobotsFile = GetLocal(robotsURL.get()); - server = new Server(url.host(), url.port(), localRobotsFile); + server = new Server(url.host(), url.port(), strcmp(url.service(), "https") == 0, localRobotsFile); servers.Add(url.signature(), server); delete localRobotsFile; } diff -ru htdig-3.1.5/htdig/Server.cc htdig-3.1.5_patched/htdig/Server.cc --- htdig-3.1.5/htdig/Server.cc Fri Feb 25 03:29:10 2000 +++ htdig-3.1.5_patched/htdig/Server.cc Wed Apr 18 14:05:15 2001 @@ -20,9 +20,9 @@ //***************************************************************************** -// Server::Server(char *host, int port, StringList *local_robots_files) +// Server::Server(char *host, int port, int ssl, StringList *local_robots_file) // -Server::Server(char *host, int port, StringList *local_robots_files) +Server::Server(char *host, int port, int ssl, StringList *local_robots_files) { if (debug > 0) cout << endl << "New server: " << host << ", " << port << endl; @@ -40,7 +40,8 @@ // // Attempt to get a robots.txt file from the specified server // - String url = "http://"; + String url = "http://"; + if (ssl) url = "https://"; url << host << ':' << port << "/robots.txt"; Document doc(url, 0); Only in htdig-3.1.5_patched/htdig: Server.cc.orig Only in htdig-3.1.5_patched/htdig: Server.cc.rej diff -ru htdig-3.1.5/htdig/Server.h htdig-3.1.5_patched/htdig/Server.h --- htdig-3.1.5/htdig/Server.h Fri Feb 25 03:29:10 2000 +++ htdig-3.1.5_patched/htdig/Server.h Wed Apr 18 13:37:41 2001 @@ -26,7 +26,7 @@ // // Construction/Destruction // - Server(char *host, int port, StringList *local_robots_files = NULL); + Server(char *host, int port, int ssl, StringList *local_robots_file = NULL); ~Server(); // diff -ru htdig-3.1.5/htlib/Connection.cc htdig-3.1.5_patched/htlib/Connection.cc --- htdig-3.1.5/htlib/Connection.cc Fri Feb 25 03:29:11 2000 +++ htdig-3.1.5_patched/htlib/Connection.cc Wed Apr 18 13:37:41 2001 @@ -39,6 +39,10 @@ int rresvport(int *); } +SSL_CTX *Connection::ctx = NULL; +SSL_METHOD *Connection::meth = NULL; + + List all_connections; Connection::Connection() @@ -49,9 +53,27 @@ server_name = 0; all_connections.Add(this); timeout_value = 0; + ssl = NULL; + m_ssl_on = 0; + initSSL(); +} + +void Connection::initSSL() +{ + if (ctx == NULL) + { + SSLeay_add_ssl_algorithms(); + meth = SSLv2_client_method(); + SSL_load_error_strings(); + ctx = SSL_CTX_new(meth); + if (ctx == NULL) + { + printf("ctx NULL\n"); + exit(1); + } + } } - //************************************************************************* // Connection::Connection(int socket) // PURPOSE: @@ -72,6 +94,9 @@ server_name = 0; all_connections.Add(this); timeout_value = 0; + ssl = NULL; + m_ssl_on = 0; + initSSL(); } @@ -94,15 +119,15 @@ { if (priv) { - int aport = IPPORT_RESERVED - 1; + int aport = IPPORT_RESERVED - 1; - sock = rresvport(&aport); + sock = rresvport(&aport); } else - sock = socket(AF_INET, SOCK_STREAM, 0); + sock = socket(AF_INET, SOCK_STREAM, 0); if (sock == NOTOK) - return NOTOK; + return NOTOK; int on = 1; setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on)); @@ -149,6 +174,8 @@ if (sock >= 0) { int ret = ::close(sock); + SSL_free(ssl); + ssl = NULL; sock = -1; return ret; } @@ -224,6 +251,16 @@ //***************************************************************************** +// int Connection::assign_ssl(int ssl_on) +// +int Connection::assign_ssl(int ssl_on) +{ + m_ssl_on = ssl_on; + return OK; +} + + +//***************************************************************************** // int Connection::connect(int allow_EINTR) // int Connection::connect(int allow_EINTR) @@ -244,8 +281,24 @@ if (status == 0 || errno == EALREADY || errno == EISCONN) { - connected = 1; - return OK; + if (m_ssl_on) + { + ssl = SSL_new(ctx); + if (ssl != NULL) + { + SSL_set_fd(ssl, sock); + if (SSL_connect(ssl) != -1) + { + connected = 1; + return OK; + } + } + } + else + { + connected = 1; + return OK; + } } #if 0 if (status == ECONNREFUSED) @@ -373,26 +426,32 @@ need_io_stop = 0; do { - errno = 0; + errno = 0; - if (timeout_value > 0) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(sock, &fds); - - timeval tv; - tv.tv_sec = timeout_value; - tv.tv_usec = 0; - - int selected = ::select(sock+1, &fds, 0, 0, &tv); - if (selected <= 0) - need_io_stop++; - } - - if (!need_io_stop) - count = ::read(sock, buffer, maxlength); - else - count = -1; // Input timed out + if (timeout_value > 0) + { + fd_set fds; + FD_ZERO(&fds); + FD_SET(sock, &fds); + + timeval tv; + tv.tv_sec = timeout_value; + tv.tv_usec = 0; + + int selected = ::select(sock+1, &fds, 0, 0, &tv); + if (selected <= 0) + need_io_stop++; + } + + if (!need_io_stop) + { + if (ssl != NULL) + count = SSL_read(ssl, buffer, maxlength); + else + count = ::read(sock, buffer, maxlength); + } + else + count = -1; // Input timed out } while (count < 0 && errno == EINTR && !need_io_stop); need_io_stop = 0; @@ -410,7 +469,10 @@ do { - count = ::write(sock, buffer, maxlength); + if (ssl != NULL) + count = SSL_write(ssl, buffer, maxlength); + else + count = ::write(sock, buffer, maxlength); } while (count < 0 && errno == EINTR && !need_io_stop); need_io_stop = 0; diff -ru htdig-3.1.5/htlib/Connection.h htdig-3.1.5_patched/htlib/Connection.h --- htdig-3.1.5/htlib/Connection.h Fri Feb 25 03:29:11 2000 +++ htdig-3.1.5_patched/htlib/Connection.h Wed Apr 18 13:37:41 2001 @@ -36,6 +36,14 @@ #include #include +#include +#include +#include +#include +#include +#include + + class String; class Connection : public io @@ -45,6 +53,7 @@ Connection(); Connection(int socket); ~Connection(); + void initSSL(); // (De)initialization int open(int priv = 0); @@ -64,6 +73,9 @@ int assign_server(unsigned int addr = INADDR_ANY); char *get_server() {return server_name;} + // SLL stuff + int assign_ssl(int ssl_on); + // Connection establishment int connect(int allow_EINTR = 0); Connection *accept(int priv = 0); @@ -90,6 +102,10 @@ private: int sock; + int m_ssl_on; + SSL *ssl; + static SSL_CTX *ctx; + static SSL_METHOD *meth; struct sockaddr_in server; int connected; char *peer; diff -ru htdig-3.1.5/htlib/URL.cc htdig-3.1.5_patched/htlib/URL.cc --- htdig-3.1.5/htlib/URL.cc Fri Feb 25 03:29:11 2000 +++ htdig-3.1.5_patched/htlib/URL.cc Wed Apr 18 13:37:41 2001 @@ -130,9 +130,15 @@ while (isalpha(*p)) p++; int hasService = (*p == ':'); - if ((hasService && ((strncmp(ref, "http://", 7) == 0) || - (strncmp(ref, "http:", 5) != 0))) - || strncmp(ref, "//", 2) == 0) + + if ((hasService && + ( (strncmp(ref, "http://", 7) == 0) || + (strncmp(ref, "http:", 5) != 0) || + (strncmp(ref, "https://", 8) == 0) || + (strncmp(ref, "https:", 6) != 0) + ) + ) + || (strncmp(ref, "//", 2) == 0)) { // // No need to look at the parent url since this is a complete url... @@ -217,7 +223,12 @@ _url << ":"; if (_host.length()) _url << "//" << _host; - if (_port != 80 && strcmp(_service, "http") == 0) + + // Add on the port if its not one of the standard ones + // + if ((strcmp(_service, "https") == 0 && _port != 443)|| + (strcmp(_service, "http" ) == 0 && _port != 80) + ) _url << ':' << _port; _url << _path; } @@ -284,13 +295,21 @@ if (p) _port = atoi(p); if (!p || _port <= 0) - _port = 80; + { + if( strcmp( _service, "https" ) == 0 ) + _port = 443; + else + _port = 80; + } } else { _host = strtok(p, "/"); _host.chop(" \t"); - _port = 80; + if( strcmp( _service, "https" ) == 0 ) + _port = 443; + else + _port = 80; } // @@ -309,7 +328,8 @@ // _url = _service; _url << "://" << _host; - if (_port != 80) + if ((strcmp(_service, "https") == 0 && _port != 443)|| + (strcmp(_service, "http" ) == 0 && _port != 80)) _url << ':' << _port; _url << _path; } @@ -438,7 +458,8 @@ _path.lowercase(); _url = _service; _url << "://" << _host; - if (_port != 80) + if ((strcmp(_service, "https") == 0 && _port != 443)|| + (strcmp(_service, "http" ) == 0 && _port != 80)) _url << ':' << _port; _url << _path; } @@ -486,7 +507,8 @@ if (_service.length() == 0 || _normal) return; - if (strcmp(_service, "http") != 0) + if (strcmp(_service, "http") != 0 && + strcmp(_service, "https") != 0) return; removeIndex(_path); @@ -543,7 +565,8 @@ _url << ":"; if (_host.length()) _url << "//" << _host; - if (_port != 80 && strcmp(_service, "http") == 0) + if ((strcmp(_service, "https") == 0 && _port != 443)|| + (strcmp(_service, "http" ) == 0 && _port != 80)) _url << ':' << _port; _url << _path; _normal = 1; diff -ru htdig-3.1.5/htlib/URL.h htdig-3.1.5_patched/htlib/URL.h --- htdig-3.1.5/htlib/URL.h Fri Feb 25 03:29:11 2000 +++ htdig-3.1.5_patched/htlib/URL.h Wed Apr 18 13:37:41 2001 @@ -48,7 +48,7 @@ void host(char *h) {_host = h;} int port() {return _port;} void port(int p) {_port = p;} - char *service() {return _service;} + char *service() {return _service.get();} void service(char *s) {_service = s;} char *path() {return _path;} void path(char *p);