使用 libcurl 加载 http URL

Loading http URL using libcurl

我从 libcurl 教程中获取示例代码并添加代码以使用进度回调函数验证每个 URL 下载了多少数据。

但是我得到的输出没有给我确切的下载日期值。对于第一个 url (http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv) 下载率,它总是以垃圾值开头。我需要以不同的方式初始化吗?或者有没有其他方法可以有效地获取下载数据值。如果我做错了什么,请纠正我。

Code: 

/***************************************************************************
 *                                  _   _ ____  _
 *  Project                     ___| | | |  _ \| |
 *                             / __| | | | |_) | |
 *                            | (__| |_| |  _ <| |___
 *                             \___|\___/|_| \_\_____|
 *
 * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel@haxx.se>, et al.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution. The terms
 * are also available at http://curl.haxx.se/docs/copyright.html.
 *
 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
 * copies of the Software, and permit persons to whom the Software is
 * furnished to do so, under the terms of the COPYING file.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 ***************************************************************************/
/* A multi-threaded example that uses pthreads extensively to fetch
 * X remote files at once */

//#include <Python.h>
#include <stdio.h>
#include <pthread.h>
#include <curl/curl.h>

#define NUMT 4

/*
  List of URLs to fetch.

  If you intend to use a SSL-based protocol here you MUST setup the OpenSSL
  callback functions as described here:

  http://www.openssl.org/docs/crypto/threads.html#DESCRIPTION

*/
const char * const urls[NUMT]= {
  "http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv",
  "http://www.yahoo.com/",
  "http://www.contactor.se/",
  "http://www.haxx.se/"
};

#define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL     3

struct myprogress {
  double lastruntime;
  curl_off_t totdnld;
  void *url;
  CURL *curl;
};

int i = 0;

/* this is how the CURLOPT_XFERINFOFUNCTION callback works */
static int xferinfo(void *p,
                    curl_off_t dltotal, curl_off_t dlnow,
                    curl_off_t ultotal, curl_off_t ulnow)
{
  struct myprogress *myp = (struct myprogress *)p;
  CURL *curl = myp->curl;
  double curtime = 0;

  curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &curtime);

  /* under certain circumstances it may be desirable for certain functionality
     to only run every N seconds, in order to do this the transaction time can
     be used */
  if((curtime - myp->lastruntime) >= MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL) {
    myp->lastruntime = curtime;
    fprintf(stderr, "TOTAL TIME: %f \r\n", curtime);
  }

  fprintf(stderr, "UP: %" CURL_FORMAT_CURL_OFF_T " of %" CURL_FORMAT_CURL_OFF_T
          "  DOWN: %" CURL_FORMAT_CURL_OFF_T " of %" CURL_FORMAT_CURL_OFF_T
          "\r\n",
          ulnow, ultotal, dlnow, dltotal);

  myp->totdnld = myp->totdnld + dltotal;
  fprintf(stderr, "TOTAL Download: %" CURL_FORMAT_CURL_OFF_T " url is: %s \r\n", myp->totdnld, myp->url);

 // if(dlnow > STOP_DOWNLOAD_AFTER_THIS_MANY_BYTES)
  //  return 1;
  return 0;
}

/* for libcurl older than 7.32.0 (CURLOPT_PROGRESSFUNCTION) */
static int older_progress(void *p,
                          double dltotal, double dlnow,
                          double ultotal, double ulnow)
{
  return xferinfo(p,
                  (curl_off_t)dltotal,
                  (curl_off_t)dlnow,
                  (curl_off_t)ultotal,
                  (curl_off_t)ulnow);
}

static void *pull_one_url(void *url)
{
  CURL *curl;
  CURLcode res = CURLE_OK;
  struct myprogress prog;

  curl = curl_easy_init();
  if(curl) {
    prog.lastruntime = 0;
    prog.curl = curl;
    prog.url = url;
    prog.totdnld = (curl_off_t) 0;
    curl_easy_setopt(curl, CURLOPT_URL, url);
    curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, older_progress);
    /* pass the struct pointer into the progress function */
    curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &prog);

#if LIBCURL_VERSION_NUM >= 0x072000
    /* xferinfo was introduced in 7.32.0, no earlier libcurl versions will
       compile as they won't have the symbols around.

       If built with a newer libcurl, but running with an older libcurl:
       curl_easy_setopt() will fail in run-time trying to set the new
       callback, making the older callback get used.

       New libcurls will prefer the new callback and instead use that one even
       if both callbacks are set. */

    curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, xferinfo);
    /* pass the struct pointer into the xferinfo function, note that this is
       an alias to CURLOPT_PROGRESSDATA */
    curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &prog);
#endif

    curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
    res = curl_easy_perform(curl);
    if(res != CURLE_OK)
      fprintf(stderr, "%s\n", curl_easy_strerror(res));
    curl_easy_cleanup(curl);

  }
  return NULL;
}


/*
   int pthread_create(pthread_t *new_thread_ID,
   const pthread_attr_t *attr,
   void * (*start_func)(void *), void *arg);
*/

int main(int argc, char **argv)
{
  pthread_t tid[NUMT];
  int i;
  int error;

  /* Must initialize libcurl before any threads are started */
  curl_global_init(CURL_GLOBAL_ALL);

  for(i=0; i< NUMT; i++) {
    error = pthread_create(&tid[i],
                           NULL, /* default attributes please */
                           pull_one_url,
                           (void *)urls[i]);
    if(0 != error)
      fprintf(stderr, "Couldn't run thread number %d, errno %d\n", i, error);
    else
      fprintf(stderr, "Thread %d, gets %s\n", i, urls[i]);
  }

  /* now wait for all threads to terminate */
  for(i=0; i< NUMT; i++) {
    error = pthread_join(tid[i], NULL);
    fprintf(stderr, "Thread %d terminated\n", i);
  }

  return 0;
}

输出:

Thread 0, gets http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
Thread 1, gets http://www.yahoo.com/
Thread 2, gets http://www.contactor.se/
Thread 3, gets http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 140094997247744 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 140094997247744 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 140094997247744 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 140094997247744 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 140094997247744 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 140094997247744 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 140094997247744 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 140094997247744 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 140094997247744 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 1450 of 1450
TOTAL Download: 1450 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 1450 of 1450
TOTAL Download: 2900 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 1450 of 1450
TOTAL Download: 4350 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 1450 of 1450
TOTAL Download: 5800 url is: http://www.yahoo.com/
UP: 0 of 0  DOWN: 392 of 392
TOTAL Download: 392 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 392 of 392
TOTAL Download: 784 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 392 of 392
TOTAL Download: 1176 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 392 of 392
TOTAL Download: 1568 url is: http://www.contactor.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 0 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 140094997247744 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 0 of 0
TOTAL Download: 140094997247744 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 2685 of 2685
TOTAL Download: 2685 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 2685 of 2685
TOTAL Download: 5370 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 2685 of 2685
TOTAL Download: 8055 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 2685 of 2685
TOTAL Download: 10740 url is: http://www.haxx.se/
UP: 0 of 0  DOWN: 3763 of 83353
TOTAL Download: 140094997331097 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 3763 of 83353
TOTAL Download: 140094997414450 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 10603 of 83353
TOTAL Download: 140094997497803 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 10603 of 83353
TOTAL Download: 140094997581156 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 20155 of 83353
TOTAL Download: 140094997664509 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 20155 of 83353
TOTAL Download: 140094997747862 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 25627 of 83353
TOTAL Download: 140094997831215 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 25627 of 83353
TOTAL Download: 140094997914568 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 39299 of 83353
TOTAL Download: 140094997997921 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 39299 of 83353
TOTAL Download: 140094998081274 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 42035 of 83353
TOTAL Download: 140094998164627 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 42035 of 83353
TOTAL Download: 140094998247980 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 52979 of 83353
TOTAL Download: 140094998331333 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 52979 of 83353
TOTAL Download: 140094998414686 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 62555 of 83353
TOTAL Download: 140094998498039 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 62555 of 83353
TOTAL Download: 140094998581392 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 65291 of 83353
TOTAL Download: 140094998664745 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 65291 of 83353
TOTAL Download: 140094998748098 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 78971 of 83353
TOTAL Download: 140094998831451 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 78971 of 83353
TOTAL Download: 140094998914804 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 83075 of 83353
TOTAL Download: 140094998998157 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 83075 of 83353
TOTAL Download: 140094999081510 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 83353 of 83353
TOTAL Download: 140094999164863 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 83353 of 83353
TOTAL Download: 140094999248216 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 83353 of 83353
TOTAL Download: 140094999331569 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
UP: 0 of 0  DOWN: 83353 of 83353
TOTAL Download: 140094999414922 url is: http://curl.haxx.se/docs/faq.html#How_do_I_make_libcurl_not_receiv
Thread 0 terminated
Thread 1 terminated
Thread 2 terminated
Thread 3 terminated

您从未初始化 progtotdnld 成员,因此它的值是不确定的。