diff -urN curl-7.19.4.orig/configure.ac curl-7.19.4/configure.ac --- curl-7.19.4.orig/configure.ac 2009-02-13 23:25:15.000000000 +0900 +++ curl-7.19.4/configure.ac 2009-03-03 23:49:58.000000000 +0900 @@ -1439,6 +1439,98 @@ AM_CONDITIONAL(HAVE_LIBZ, test x"$AMFIXLIB" = x1) dnl ********************************************************************** +dnl Check for the presence of LIBMETALINK libraries and headers +dnl ********************************************************************** + +dnl Check for & handle argument to --with-metalink. + +_cppflags=$CPPFLAGS +_ldflags=$LDFLAGS +AC_ARG_WITH(metalink, +AC_HELP_STRING([--with-metalink=PATH],[search for libmetalink in PATH]) +AC_HELP_STRING([--without-metalink],[disable use of libmetalink.]), + [OPT_METALINK="$withval"]) + +if test "$OPT_METALINK" = "no" ; then + AC_MSG_WARN([metalink disabled]) +else + if test "$OPT_METALINK" = "yes" ; then + OPT_METALINK="" + fi + + if test -z "$OPT_METALINK"; then + dnl check for the lib first without setting any new path, since many + dnl people have it in the default path + + AC_CHECK_LIB(metalink, metalink_parse_file, + dnl libmetalink found, set the variable + [HAVE_LIBMETALINK="1"], + dnl if no lib found, try /usr/local + [OPT_METALINK="/usr/local"]) + fi + + dnl Add a nonempty path to the compiler flags + if test -n "$OPT_METALINK"; then + CPPFLAGS="$CPPFLAGS -I$OPT_METALINK/include" + LDFLAGS="$LDFLAGS -L$OPT_METALINK/lib$libsuff" + fi + + AC_CHECK_HEADER([metalink/metalink_parser.h], + [ + dnl metalink/metalink_parser.h was found + HAVE_METALINK_H="1" + dnl if the lib wan't found already, try again with the new paths + if test "$HAVE_LIBMETALINK" != "1"; then + AC_CHECK_LIB(metalink, metalink_parse_file, + [ + dnl the lib was found! + HAVE_LIBMETALINK="1" + ], + [ CPPFLAGS=$_cppflags + LDFLAGS=$_ldflags]) + fi + ], + [ + dnl metalink/metalink_parser.h was not found, restore the flags + CPPFLAGS=$_cppflags + LDFLAGS=$_ldflags] + ) + + if test "$HAVE_LIBMETALINK" = "1" && test "$HAVE_METALINK_H" != "1" + then + AC_MSG_WARN([configure found only the libmetalink lib, not the header file!]) + HAVE_LIBMETALINK="" + elif test "$HAVE_LIBMETALINK" != "1" && test "$HAVE_METALINK_H" = "1" + then + AC_MSG_WARN([configure found only the libmetalink header file, not the lib!]) + elif test "$HAVE_LIBMETALINK" = "1" && test "$HAVE_METALINK_H" = "1" + then + dnl both header and lib were found! + AC_SUBST(HAVE_LIBMETALINK) + AC_DEFINE(HAVE_METALINK_H, 1, [if you have the metalink/metalink_parser.h header file]) + AC_DEFINE(HAVE_LIBMETALINK, 1, [if libmetalink is available]) + + CURL_LIBS="$CURL_LIBS -lmetalink" + LIBS="$LIBS -lmetalink" + + dnl replace 'HAVE_LIBMETALINK' in the automake makefile.ams + AMFIXLIB="1" + AC_MSG_NOTICE([found both libmetalink and metalink/metalink_parser.h header]) + curl_metalink_msg="enabled" + + dnl when shared libs were found in a path that the run-time + dnl linker doesn't search through, we need to add it to + dnl LD_LIBRARY_PATH to prevent further configure tests to fail + dnl due to this + + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$OPT_METALINK/lib$libsuff" + export LD_LIBRARY_PATH + AC_MSG_NOTICE([Added $OPT_METALINK/lib$libsuff to LD_LIBRARY_PATH]) + + fi +fi + +dnl ********************************************************************** dnl Check for the presence of LIBSSH2 libraries and headers dnl ********************************************************************** @@ -2580,6 +2672,7 @@ ca cert path: ${capath} LDAP support: ${curl_ldap_msg} LDAPS support: ${curl_ldaps_msg} + metalink support:${curl_metalink_msg} ]) if test "x$soname_bump" = "xyes"; then diff -urN curl-7.19.4.orig/src/main.c curl-7.19.4/src/main.c --- curl-7.19.4.orig/src/main.c 2009-02-17 18:10:21.000000000 +0900 +++ curl-7.19.4/src/main.c 2009-03-03 23:49:58.000000000 +0900 @@ -114,6 +114,25 @@ #include /* for TCP_KEEPIDLE, TCP_KEEPINTVL */ #endif +#ifdef HAVE_LIBMETALINK +#include + +struct metalinkfile { + struct metalinkfile *next; + metalink_file_t *file; +}; + +struct metalink { + struct metalink *next; + metalink_t* metalink; +}; + +struct metalinkfile *new_metalinkfile(metalink_file_t *metalinkfile); + +struct metalink *new_metalink(metalink_t *metalink); + +#endif // HAVE_LIBMETALINK + /* The last #include file should be: */ #ifdef CURLDEBUG #ifndef CURLTOOLDEBUG @@ -564,6 +583,14 @@ int default_node_flags; /* default flags to seach for each 'node', which is basically each given URL to transfer */ struct OutStruct *outs; + +#ifdef HAVE_LIBMETALINK + struct metalinkfile *metalinkfile_list; /* point to the first node */ + struct metalinkfile *metalinkfile_last; /* point to the last/current node */ + + struct metalink *metalink_list; /* point to the first node */ + struct metalink *metalink_last; /* point to the last/current node */ +#endif // HAVE_LIBMETALINK }; #define WARN_PREFIX "Warning: " @@ -684,6 +711,9 @@ #define GETOUT_USEREMOTE (1<<2) /* use remote file name locally */ #define GETOUT_UPLOAD (1<<3) /* if set, -T has been used */ #define GETOUT_NOUPLOAD (1<<4) /* if set, -T "" has been used */ +#ifdef HAVE_LIBMETALINK +#define GETOUT_METALINK (1<<5) /* set when Metalink download */ +#endif /* HAVE_LIBMETALINK */ static void help(void) { @@ -1684,6 +1714,9 @@ {"$7", "socks5-gssapi-nec", FALSE}, #endif {"$8", "proxy1.0", TRUE}, +#ifdef HAVE_LIBMETALINK + {"$9", "metalink", TRUE}, +#endif /* HAVE_LIBMETALINK */ {"0", "http1.0", FALSE}, {"1", "tlsv1", FALSE}, @@ -2210,8 +2243,74 @@ GetStr(&config->proxy, nextarg); config->proxyver = CURLPROXY_HTTP_1_0; break; +#ifdef HAVE_LIBMETALINK + case '9': /* --metalink */ + { + metalink_error_t r; + metalink_t* metalink; + metalink_file_t **files; + metalink_checksum_t** checksums; + struct metalink *ml; + + r = metalink_parse_file(nextarg, &metalink); + + if(r != 0) { + fprintf(stderr, "ERROR: code=%d\n", r); + exit(EXIT_FAILURE); + } + ml = new_metalink(metalink); + + if(config->metalink_list) { + config->metalink_last->next = ml; + config->metalink_last = ml; + } else { + config->metalink_list = config->metalink_last = ml; + } + + for(files = metalink->files; *files; files++) { + /* Skip an entry which has no resource. */ + if(!(*files)->resources[0]) continue; + struct getout *url; + if(config->url_get || (config->url_get=config->url_list)) { + /* there's a node here, if it already is filled-in continue to + find an "empty" node */ + while(config->url_get && (config->url_get->flags&GETOUT_URL)) + config->url_get = config->url_get->next; + } + + /* now there might or might not be an available node to fill in! */ + + if(config->url_get) + /* existing node */ + url = config->url_get; + else + /* there was no free node, create one! */ + url=new_getout(config); + + if(url) { + struct metalinkfile *mlfile; + /* Set name as url */ + GetStr(&url->url, (*files)->name); + + /* set flag metalink here */ + url->flags |= GETOUT_METALINK; + url->flags |= GETOUT_URL; + + mlfile = new_metalinkfile(*files); + + if(config->metalinkfile_list) { + config->metalinkfile_last->next = mlfile; + config->metalinkfile_last = mlfile; + } else { + config->metalinkfile_list = config->metalinkfile_last = mlfile; + } + } + } + break; + } } break; +#endif /* HAVE_LIBMETALINK */ case '#': /* --progress-bar */ config->progressmode = toggle?CURL_PROGRESS_BAR:0; break; @@ -3932,6 +4031,683 @@ curl_slist_free_all(easycode); } +#ifdef HAVE_LIBMETALINK + +struct metalinkfile *new_metalinkfile(metalink_file_t *metalinkfile) { + struct metalinkfile *f; + f = (struct metalinkfile*)malloc(sizeof(struct metalinkfile)); + f->file = metalinkfile; + f->next = NULL; + return f; +} + +struct metalink *new_metalink(metalink_t *metalink) { + struct metalink *ml; + ml = (struct metalink*)malloc(sizeof(struct metalink)); + ml->metalink = metalink; + ml->next = NULL; + return ml; +} + +static int operatemetalink(CURL *curl, + struct getout *urlnode, + long retry_sleep_default, + struct OutStruct outs, + struct OutStruct heads, + char *outfiles, + struct Configurable *config) +{ + long retry_numretries; + int infd = STDIN_FILENO; + bool infdopen; + char *outfile; + struct timeval retrystart; + struct metalinkfile *mlfile; + int res = 0; + char *filename; + metalink_resource_t **mlres; + char errorbuffer[CURL_ERROR_SIZE]; + struct ProgressData progressbar; + long retry_sleep; + struct InStruct input; + curl_off_t uploadfilesize; /* -1 means unknown */ + char *uploadfile=NULL; /* a single file, never a glob */ + + uploadfilesize=-1; + + mlfile = config->metalinkfile_last; + config->metalinkfile_last = config->metalinkfile_last->next; + + filename = strdup(mlfile->file->name); + + outfile = outfiles?strdup(outfiles):NULL; + + if((urlnode->flags&GETOUT_USEREMOTE) || + (outfile && !curlx_strequal("-", outfile)) ) { + + /* + * We have specified a file name to store the result in, or we have + * decided we want to use the name attribute of file element in Metalink + * XML. + */ + + if(!outfile) { + /* Find and get file name */ + char * pc; + pc = strrchr(filename, '/'); + + if(pc) { + /* duplicate the string beyond the slash */ + pc++; + outfile = *pc ? strdup(pc): NULL; + } else { + outfile = strdup(filename); + } + if(!outfile || !*outfile) { + helpf(config->errors, "Metalink file[@name] has no length!\n"); + res = CURLE_WRITE_ERROR; + free(filename); + /* break; */ + return 1; + } +#if defined(MSDOS) + { + /* This is for DOS, and then we do some major replacing of + bad characters in the file name before using it */ + char file1[PATH_MAX]; + if(strlen(outfile) >= PATH_MAX) + outfile[PATH_MAX-1]=0; /* cut it */ + strcpy(file1, msdosify(outfile)); + free(outfile); + + outfile = strdup(rename_if_dos_device_name(file1)); + if(!outfile) { + res = CURLE_OUT_OF_MEMORY; + break; + } + } +#endif /* MSDOS */ + } + /* Create the directory hierarchy, if not pre-existant to a multiple + file output call */ + + if(config->create_dirs && + (-1 == create_dir_hierarchy(outfile, config->errors))) + return CURLE_WRITE_ERROR; + + if(config->resume_from_current) { + /* We're told to continue from where we are now. Get the + size of the file as it is now and open it for append instead */ + + struct_stat fileinfo; + + /* VMS -- Danger, the filesize is only valid for stream files */ + if(0 == stat(outfile, &fileinfo)) + /* set offset to current file size: */ + config->resume_from = fileinfo.st_size; + else + /* let offset be 0 */ + config->resume_from = 0; + } + + outs.filename = outfile; + + if(config->resume_from) { + outs.init = config->resume_from; + /* open file for output: */ + outs.stream=(FILE *) fopen(outfile, config->resume_from?"ab":"wb"); + if (!outs.stream) { + helpf(config->errors, "Can't open '%s'!\n", outfile); + return CURLE_WRITE_ERROR; + } + } + else { + outs.stream = NULL; /* open when needed */ + } + } + infdopen=FALSE; + + if(!config->errors) + config->errors = stderr; + + if(!outfile && !config->use_ascii) { + /* We get the output to stdout and we have not got the ASCII/text + flag, then set stdout to be binary */ + SET_BINMODE(stdout); + } + + /* Loop though all resources in Metalink */ + for(mlres = mlfile->file->resources; *mlres; ++mlres) { + if(1 == config->tcp_nodelay) + my_setopt(curl, CURLOPT_TCP_NODELAY, 1); + + /* where to store */ + my_setopt(curl, CURLOPT_WRITEDATA, &outs); + /* what call to write */ + my_setopt(curl, CURLOPT_WRITEFUNCTION, my_fwrite); + + /* for uploads */ + input.fd = infd; + input.config = config; + my_setopt(curl, CURLOPT_READDATA, &input); + /* what call to read */ + my_setopt(curl, CURLOPT_READFUNCTION, my_fread); + + /* in 7.18.0, the CURLOPT_SEEKFUNCTION/DATA pair is taking over what + CURLOPT_IOCTLFUNCTION/DATA pair previously provided for seeking */ + my_setopt(curl, CURLOPT_SEEKDATA, &input); + my_setopt(curl, CURLOPT_SEEKFUNCTION, my_seek); + + if(config->recvpersecond) + /* tell libcurl to use a smaller sized buffer as it allows us to + make better sleeps! 7.9.9 stuff! */ + my_setopt(curl, CURLOPT_BUFFERSIZE, config->recvpersecond); + + /* size of uploaded file: */ + my_setopt(curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize); + my_setopt(curl, CURLOPT_URL, (*mlres)->url); /* what to fetch */ + my_setopt(curl, CURLOPT_PROXY, config->proxy); /* proxy to use */ + my_setopt(curl, CURLOPT_NOPROGRESS, config->noprogress); + if(config->no_body) { + my_setopt(curl, CURLOPT_NOBODY, 1); + my_setopt(curl, CURLOPT_HEADER, 1); + } + else + my_setopt(curl, CURLOPT_HEADER, config->include_headers); + + my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror); + my_setopt(curl, CURLOPT_UPLOAD, uploadfile?TRUE:FALSE); + my_setopt(curl, CURLOPT_DIRLISTONLY, config->dirlistonly); + my_setopt(curl, CURLOPT_APPEND, config->ftp_append); + + if (config->netrc_opt) + my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + else if (config->netrc) + my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_REQUIRED); + else + my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_IGNORED); + + my_setopt(curl, CURLOPT_FOLLOWLOCATION, config->followlocation); + my_setopt(curl, CURLOPT_UNRESTRICTED_AUTH, config->unrestricted_auth); + my_setopt(curl, CURLOPT_TRANSFERTEXT, config->use_ascii); + my_setopt(curl, CURLOPT_USERPWD, config->userpwd); + my_setopt(curl, CURLOPT_PROXYUSERPWD, config->proxyuserpwd); + my_setopt(curl, CURLOPT_RANGE, config->range); + my_setopt(curl, CURLOPT_ERRORBUFFER, errorbuffer); + my_setopt(curl, CURLOPT_TIMEOUT, config->timeout); + + switch(config->httpreq) { + case HTTPREQ_SIMPLEPOST: + my_setopt(curl, CURLOPT_POSTFIELDS, config->postfields); + my_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, config->postfieldsize); + break; + case HTTPREQ_POST: + my_setopt(curl, CURLOPT_HTTPPOST, config->httppost); + break; + default: + break; + } + my_setopt(curl, CURLOPT_REFERER, config->referer); + my_setopt(curl, CURLOPT_AUTOREFERER, config->autoreferer); + my_setopt(curl, CURLOPT_USERAGENT, config->useragent); + my_setopt(curl, CURLOPT_FTPPORT, config->ftpport); + my_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, + config->low_speed_limit); + my_setopt(curl, CURLOPT_LOW_SPEED_TIME, config->low_speed_time); + my_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE, + config->sendpersecond); + my_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, + config->recvpersecond); + my_setopt(curl, CURLOPT_RESUME_FROM_LARGE, + config->use_resume?config->resume_from:0); + my_setopt(curl, CURLOPT_COOKIE, config->cookie); + my_setopt(curl, CURLOPT_HTTPHEADER, config->headers); + my_setopt(curl, CURLOPT_SSLCERT, config->cert); + my_setopt(curl, CURLOPT_SSLCERTTYPE, config->cert_type); + my_setopt(curl, CURLOPT_SSLKEY, config->key); + my_setopt(curl, CURLOPT_SSLKEYTYPE, config->key_type); + my_setopt(curl, CURLOPT_KEYPASSWD, config->key_passwd); + + /* SSH private key uses the same command-line option as SSL private + key */ + my_setopt(curl, CURLOPT_SSH_PRIVATE_KEYFILE, config->key); + my_setopt(curl, CURLOPT_SSH_PUBLIC_KEYFILE, config->pubkey); + + /* SSH host key md5 checking allows us to fail if we are + * not talking to who we think we should + */ + my_setopt(curl, CURLOPT_SSH_HOST_PUBLIC_KEY_MD5, config->hostpubmd5); + + + /* default to strict verifyhost */ + my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2); + if(config->cacert || config->capath) { + if (config->cacert) + my_setopt(curl, CURLOPT_CAINFO, config->cacert); + + if (config->capath) + my_setopt(curl, CURLOPT_CAPATH, config->capath); + my_setopt(curl, CURLOPT_SSL_VERIFYPEER, TRUE); + } + if(config->insecure_ok) { + /* new stuff needed for libcurl 7.10 */ + my_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 1); + } + + if(config->no_body || config->remote_time) { + /* no body or use remote time */ + my_setopt(curl, CURLOPT_FILETIME, TRUE); + } + + my_setopt(curl, CURLOPT_MAXREDIRS, config->maxredirs); + my_setopt(curl, CURLOPT_CRLF, config->crlf); + my_setopt(curl, CURLOPT_QUOTE, config->quote); + my_setopt(curl, CURLOPT_POSTQUOTE, config->postquote); + my_setopt(curl, CURLOPT_PREQUOTE, config->prequote); + my_setopt(curl, CURLOPT_WRITEHEADER, + config->headerfile?&heads:NULL); + my_setopt(curl, CURLOPT_COOKIEFILE, config->cookiefile); + /* cookie jar was added in 7.9 */ + if(config->cookiejar) + my_setopt(curl, CURLOPT_COOKIEJAR, config->cookiejar); + /* cookie session added in 7.9.7 */ + my_setopt(curl, CURLOPT_COOKIESESSION, config->cookiesession); + + my_setopt(curl, CURLOPT_SSLVERSION, config->ssl_version); + my_setopt(curl, CURLOPT_TIMECONDITION, config->timecond); + my_setopt(curl, CURLOPT_TIMEVALUE, config->condtime); + my_setopt(curl, CURLOPT_CUSTOMREQUEST, config->customrequest); + my_setopt(curl, CURLOPT_STDERR, config->errors); + + /* three new ones in libcurl 7.3: */ + my_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, config->proxytunnel); + my_setopt(curl, CURLOPT_INTERFACE, config->iface); + my_setopt(curl, CURLOPT_KRBLEVEL, config->krblevel); + + progressbarinit(&progressbar, config); + if((config->progressmode == CURL_PROGRESS_BAR) && + !config->noprogress && !config->mute) { + /* we want the alternative style, then we have to implement it + ourselves! */ + my_setopt(curl, CURLOPT_PROGRESSFUNCTION, myprogress); + my_setopt(curl, CURLOPT_PROGRESSDATA, &progressbar); + } + + /* new in libcurl 7.6.2: */ + my_setopt(curl, CURLOPT_TELNETOPTIONS, config->telnet_options); + + /* new in libcurl 7.7: */ + my_setopt(curl, CURLOPT_RANDOM_FILE, config->random_file); + my_setopt(curl, CURLOPT_EGDSOCKET, config->egd_file); + my_setopt(curl, CURLOPT_CONNECTTIMEOUT, config->connecttimeout); + + if(config->cipher_list) + my_setopt(curl, CURLOPT_SSL_CIPHER_LIST, config->cipher_list); + + if(config->httpversion) + my_setopt(curl, CURLOPT_HTTP_VERSION, config->httpversion); + + /* new in libcurl 7.9.2: */ + if(config->disable_epsv) + /* disable it */ + my_setopt(curl, CURLOPT_FTP_USE_EPSV, FALSE); + + /* new in libcurl 7.10.5 */ + if(config->disable_eprt) + /* disable it */ + my_setopt(curl, CURLOPT_FTP_USE_EPRT, FALSE); + + /* new in libcurl 7.10.6 (default is Basic) */ + if(config->authtype) + my_setopt(curl, CURLOPT_HTTPAUTH, config->authtype); + + if(config->tracetype != TRACE_NONE) { + my_setopt(curl, CURLOPT_DEBUGFUNCTION, my_trace); + my_setopt(curl, CURLOPT_DEBUGDATA, config); + my_setopt(curl, CURLOPT_VERBOSE, TRUE); + } + + res = CURLE_OK; + + /* new in curl ?? */ + if (config->engine) { + res = my_setopt(curl, CURLOPT_SSLENGINE, config->engine); + my_setopt(curl, CURLOPT_SSLENGINE_DEFAULT, 1); + } + + if (res != CURLE_OK) + goto show_error; + + /* new in curl 7.10 */ + my_setopt(curl, CURLOPT_ENCODING, + (config->encoding) ? "" : NULL); + + /* new in curl 7.10.7 */ + my_setopt(curl, CURLOPT_FTP_CREATE_MISSING_DIRS, + config->ftp_create_dirs); + if(config->proxyanyauth) + my_setopt(curl, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + else if(config->proxynegotiate) + my_setopt(curl, CURLOPT_PROXYAUTH, CURLAUTH_GSSNEGOTIATE); + else if(config->proxyntlm) + my_setopt(curl, CURLOPT_PROXYAUTH, CURLAUTH_NTLM); + else if(config->proxydigest) + my_setopt(curl, CURLOPT_PROXYAUTH, CURLAUTH_DIGEST); + else if(config->proxybasic) + my_setopt(curl, CURLOPT_PROXYAUTH, CURLAUTH_BASIC); + + /* new in curl 7.10.8 */ + if(config->max_filesize) + my_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, + config->max_filesize); + + if(4 == config->ip_version) + my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + else if(6 == config->ip_version) + my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); + else + my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_WHATEVER); + + /* new in curl 7.15.5 */ + if(config->ftp_ssl_reqd) + my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_ALL); + + /* new in curl 7.11.0 */ + else if(config->ftp_ssl) + my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY); + + /* new in curl 7.16.0 */ + else if(config->ftp_ssl_control) + my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_CONTROL); + + /* new in curl 7.16.1 */ + if(config->ftp_ssl_ccc) + my_setopt(curl, CURLOPT_FTP_SSL_CCC, config->ftp_ssl_ccc_mode); + + /* new in curl 7.11.1, modified in 7.15.2 */ + if(config->socksproxy) { + my_setopt(curl, CURLOPT_PROXY, config->socksproxy); + my_setopt(curl, CURLOPT_PROXYTYPE, config->socksver); + } + + /* curl 7.13.0 */ + my_setopt(curl, CURLOPT_FTP_ACCOUNT, config->ftp_account); + + my_setopt(curl, CURLOPT_IGNORE_CONTENT_LENGTH, config->ignorecl); + + /* curl 7.14.2 */ + my_setopt(curl, CURLOPT_FTP_SKIP_PASV_IP, config->ftp_skip_ip); + + /* curl 7.15.1 */ + my_setopt(curl, CURLOPT_FTP_FILEMETHOD, config->ftp_filemethod); + + /* curl 7.15.2 */ + if(config->localport) { + my_setopt(curl, CURLOPT_LOCALPORT, config->localport); + my_setopt(curl, CURLOPT_LOCALPORTRANGE, + config->localportrange); + } + + /* curl 7.15.5 */ + my_setopt(curl, CURLOPT_FTP_ALTERNATIVE_TO_USER, + config->ftp_alternative_to_user); + + /* curl 7.16.0 */ + my_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE, + !config->disable_sessionid); + + /* curl 7.16.2 */ + if(config->raw) { + my_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, FALSE); + my_setopt(curl, CURLOPT_HTTP_TRANSFER_DECODING, FALSE); + } + + /* curl 7.17.1 */ + if (!config->nokeepalive) { + my_setopt(curl, CURLOPT_SOCKOPTFUNCTION, sockoptcallback); + my_setopt(curl, CURLOPT_SOCKOPTDATA, config); + } + + /* curl 7.19.1 (the 301 version existed in 7.18.2) */ + my_setopt(curl, CURLOPT_POSTREDIR, config->post301 | + (config->post302 ? CURL_REDIR_POST_302 : FALSE)); + + retry_numretries = config->req_retry; + + retrystart = cutil_tvnow(); + + do { + res = curl_easy_perform(curl); + if (!curl_slist_append(easycode, "ret = curl_easy_perform(hnd);")) { + res = CURLE_OUT_OF_MEMORY; + break; + } + /* if retry-max-time is non-zero, make sure we haven't exceeded the + time */ + if(retry_numretries && + (!config->retry_maxtime || + (cutil_tvdiff(cutil_tvnow(), retrystart)< + config->retry_maxtime*1000)) ) { + enum { + RETRY_NO, + RETRY_TIMEOUT, + RETRY_HTTP, + RETRY_FTP, + RETRY_LAST /* not used */ + } retry = RETRY_NO; + long response; + if(CURLE_OPERATION_TIMEDOUT == res) + /* retry timeout always */ + retry = RETRY_TIMEOUT; + else if(CURLE_OK == res) { + /* Check for HTTP transient errors */ + char *this_url=NULL; + curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &this_url); + if(this_url && + curlx_strnequal(this_url, "http", 4)) { + /* This was HTTP(S) */ + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); + + switch(response) { + case 500: /* Internal Server Error */ + case 502: /* Bad Gateway */ + case 503: /* Service Unavailable */ + case 504: /* Gateway Timeout */ + retry = RETRY_HTTP; + /* + * At this point, we have already written data to the output + * file (or terminal). If we write to a file, we must rewind + * or close/re-open the file so that the next attempt starts + * over from the beginning. + * + * TODO: similar action for the upload case. We might need + * to start over reading from a previous point if we have + * uploaded something when this was returned. + */ + break; + } + } + } /* if CURLE_OK */ + else if(CURLE_LOGIN_DENIED == res) { + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); + + if(response/100 == 5) + /* + * This is typically when the FTP server only allows a certain + * amount of users and we are not one of them. It mostly + * returns 530 in this case, but all 5xx codes are transient. + */ + retry = RETRY_FTP; + } + + if(retry) { + static const char * const m[]={NULL, + "timeout", + "HTTP error", + "FTP error" + }; + warnf(config, "Transient problem: %s " + "Will retry in %ld seconds. " + "%ld retries left.\n", + m[retry], + retry_sleep/1000, + retry_numretries); + + go_sleep(retry_sleep); + retry_numretries--; + if(!config->retry_delay) { + retry_sleep *= 2; + if(retry_sleep > RETRY_SLEEP_MAX) + retry_sleep = RETRY_SLEEP_MAX; + } + if(outs.bytes && outs.filename) { + /* We have written data to a output file, we truncate file + */ + if(!config->mute) + fprintf(config->errors, "Throwing away %Od bytes\n", + outs.bytes); + fflush(outs.stream); + /* truncate file at the position where we started appending */ +#ifdef HAVE_FTRUNCATE + ftruncate( fileno(outs.stream), outs.init); + /* now seek to the end of the file, the position where we + just truncated the file in a large file-safe way */ + fseek(outs.stream, 0, SEEK_END); +#else + /* ftruncate is not available, so just reposition the file + to the location we would have truncated it. This won't + work properly with large files on 32-bit systems, but + most of those will have ftruncate. */ + fseek(outs.stream, (long)outs.init, SEEK_SET); +#endif + outs.bytes = 0; /* clear for next round */ + } + continue; + } + } /* if retry_numretries */ + + /* In all ordinary cases, just break out of loop here */ + retry_sleep = retry_sleep_default; + break; + + } while(1); + if(res == CURLE_OK) + break; + } + if((config->progressmode == CURL_PROGRESS_BAR) && + progressbar.calls) { + /* if the custom progress bar has been displayed, we output a + newline here */ + fputs("\n", progressbar.out); + } + + if(config->writeout) { + ourWriteOut(curl, config->writeout); + } +#ifdef USE_ENVIRONMENT + if (config->writeenv) + ourWriteEnv(curl); +#endif + +show_error: + +#ifdef VMS + if (!config->showerror) { + vms_show = VMSSTS_HIDE; + } +#else + if((res!=CURLE_OK) && config->showerror) { + fprintf(config->errors, "curl: (%d) %s\n", res, + errorbuffer[0]? errorbuffer: + curl_easy_strerror((CURLcode)res)); + if(CURLE_SSL_CACERT == res) { +#define CURL_CA_CERT_ERRORMSG1 \ +"More details here: http://curl.haxx.se/docs/sslcerts.html\n\n" \ +"curl performs SSL certificate verification by default, using a \"bundle\"\n" \ +" of Certificate Authority (CA) public keys (CA certs). If the default\n" \ +" bundle file isn't adequate, you can specify an alternate file\n" \ +" using the --cacert option.\n" + +#define CURL_CA_CERT_ERRORMSG2 \ +"If this HTTPS server uses a certificate signed by a CA represented in\n" \ +" the bundle, the certificate verification probably failed due to a\n" \ +" problem with the certificate (it might be expired, or the name might\n" \ +" not match the domain name in the URL).\n" \ +"If you'd like to turn off curl's verification of the certificate, use\n" \ +" the -k (or --insecure) option.\n" + + fprintf(config->errors, "%s%s", + CURL_CA_CERT_ERRORMSG1, + CURL_CA_CERT_ERRORMSG2 ); + } + } +#endif + + if (outfile && !curlx_strequal(outfile, "-") && outs.stream) + fclose(outs.stream); + +#ifdef HAVE_UTIME + /* Important that we set the time _after_ the file has been + closed, as is done above here */ + if(config->remote_time && outs.filename) { + /* ask libcurl if we got a time. Pretty please */ + long filetime; + curl_easy_getinfo(curl, CURLINFO_FILETIME, &filetime); + if(filetime >= 0) { + struct utimbuf times; + times.actime = (time_t)filetime; + times.modtime = (time_t)filetime; + utime(outs.filename, ×); /* set the time we got */ + } + } +#endif +#ifdef __AMIGA__ + /* Set the url as comment for the file. (up to 80 chars are allowed) + */ + if( strlen(url) > 78 ) + url[79] = '\0'; + + SetComment( outs.filename, url); +#endif + + if(filename) + free(filename); + if(outfile) + free(outfile); + + if(infdopen) + close(infd); + if(outfiles) + free(outfiles); + + /* empty this urlnode struct */ + if(urlnode->url) + free(urlnode->url); + if(urlnode->outfile) + free(urlnode->outfile); + + return 0; +} + +static void clean_metalink(struct Configurable *config) +{ + while(config->metalinkfile_list) { + struct metalinkfile *mlfile = config->metalinkfile_list; + config->metalinkfile_list = config->metalinkfile_list->next; + free(mlfile); + } + config->metalinkfile_last = 0; + while(config->metalink_list) { + struct metalink *ml = config->metalink_list; + config->metalink_list = config->metalink_list->next; + metalink_delete(ml->metalink); + free(ml); + } + config->metalink_last = 0; +} + +#endif // HAVE_LIBMETALINK static int operate(struct Configurable *config, int argc, argv_item_t argv[]) @@ -4191,6 +4967,11 @@ heads.config = config; } +#ifdef HAVE_LIBMETALINK + config->metalinkfile_last = config->metalinkfile_list; + config->metalink_last = config->metalink_list; +#endif // HAVE_LIBMETALINK + /* loop through the list of given URLs */ while(urlnode) { int up; /* upload file counter within a single upload glob */ @@ -4244,6 +5025,17 @@ } } + /* process metalink download in the separate function */ + if(urlnode->flags&GETOUT_METALINK) { + operatemetalink(curl, urlnode, retry_sleep_default, outs, heads, + outfiles, config); + /* move on to the next URL */ + nextnode=urlnode->next; + free(urlnode); /* free the node */ + urlnode = nextnode; + continue; + } + /* Here's the loop for uploading multiple files within the same single globbed string. If no upload, we enter the loop once anyway. */ for(up = 0; @@ -5122,6 +5914,11 @@ if(config->errors_fopened) fclose(config->errors); +#ifdef HAVE_LIBMETALINK + /* Release metalink related resources here */ + clean_metalink(config); +#endif // HAVE_LIBMETALINK + main_free(); /* cleanup */ return res;