I read through some of the links posted in response to my issue and the
below code seems to work quite well.
I ran several iterations in threaded mode and did not have a single
failure. EXCEPT that if the number of threads gets much above 40
then failures do happen, not always but to often. I ran several
iterations at 100 threads that worked then that started failing
consistently.
I'm wondering if its related to CRYPTO_num_locks() and if that can be
increased dynamically at runtime?
The  thread_setup() and thread_cleanup() routines handle everything that
i need.
Thanks All!

#include <stdio.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <pthread.h>
#include <errno.h>

#include <curl/curl.h>

pthread_t threadIDs[200];
char ebuf[512]={0}; // for errno msgs, could be a lot smaller but for
now 1/2k is fine

pthread_t start_thread();
void *test_thread(void *info);

void handle_error(const char *file, int lineno, const char *msg);
static void locking_function(int mode, int n, const char *file, int line);
static unsigned long id_function(void);
int thread_setup(void);
int thread_cleanup(void);

//---------------------------------------------------------------------------------------------
//---------------------------------------------------------------------------------------------

int main(int argc, char **argv)
{
int i, r;

         if(0!=curl_global_init(CURL_GLOBAL_ALL)) {
             printf("Curl globasl Inoit failed\n");
             exit(0);
         }

         if(argc>1) { // call thread routine as an ordinary function call

             for(r=0; r<700; r++) {
                 printf("Begin test run #%d\n", r);
                 test_thread(&i);
                 printf("end test run #%d\n", r);
             }

         } else { // run it as a pthread

             for(r=0; r<20; r++) {
                 printf("Begin test run #%d\n", r);

                 for(i=0; i<40; i++)
                     threadIDs[i] = start_thread();
                 printf("All test threads started\n");

                 for(i=0; i<40; i++)
                     pthread_join(threadIDs[i], NULL);
                 printf("All test threads ended\n");
                 printf("end test run #%d\n", r);
             } // end for r

         } // end if/else

        curl_global_cleanup();

return 0;
}
//---------------------------------------------------------------------------------------------

pthread_t start_thread(void)
{
pthread_attr_t ThreadAttr;
pthread_t      Thread_id;
int            err;

   pthread_attr_init(&ThreadAttr);
   pthread_attr_setdetachstate( &ThreadAttr, PTHREAD_CREATE_JOINABLE );
   errno = 0;
   err = pthread_create(&Thread_id, &ThreadAttr, &test_thread, NULL);

   pthread_attr_destroy(&ThreadAttr);

   if(err != 0)    {

         switch(err)
         {
             case EAGAIN:
                 printf("The system lacked the necessary resources to
create\n");
                 printf("another thread, or the system-imposed limit on
the total\n");
                 printf("number of threads in a process
PTHREAD_THREADS_MAX would be exceeded.\n");
                 break;
             case EINVAL:
                 printf("The value specified by attr is invalid.\n");
                 break;
             case EPERM:
                 printf("The caller does not have appropriate permission
to set the required\n");
                 printf("scheduling parameters or scheduling policy.\n");
                 break;
             default:
                 printf("Some weird error code returned from
pthread_create\n");
                 break;
         }
         printf("Error Code 0x%08X %s: pthread failed to start\n", err,
strerror_r(errno, ebuf, sizeof(ebuf)));
         return -1; // return negative thread id

     } // end if err

return Thread_id;
}
//---------------------------------------------------------------------------------------------

void *test_thread(void *info)
{
CURL *curl_handle;
CURLcode res;
char URL[64] = {0};
FILE *out;

     thread_setup();
     out = fopen("/dev/null", "w");
     strcpy(URL, "https://www.google.com");

     curl_handle = curl_easy_init();
     if(curl_handle) {
        //curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 1);
        curl_easy_setopt(curl_handle, CURLOPT_URL, URL);
        curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 0L);
        curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L);
        //curl_easy_setopt(curl_handle, CURLOPT_HTTPGET, 1L);
        curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, out);
        res = curl_easy_perform(curl_handle);
        curl_easy_cleanup(curl_handle);
     }
     if(out) fclose(out);
     thread_cleanup();

return (void *)0;
}
//---------------------------------------------------------------------------------------------
//---------------------------------------------------------------------------------------------
//
// stuff below was copied directly from
// https://curl.haxx.se/libcurl/c/opensslthreadlock.html
//
#include <stdio.h>
#include <pthread.h>
#include <openssl/err.h>

#define MUTEX_TYPE       pthread_mutex_t
#define MUTEX_SETUP(x)   pthread_mutex_init(&(x), NULL)
#define MUTEX_CLEANUP(x) pthread_mutex_destroy(&(x))
#define MUTEX_LOCK(x)    pthread_mutex_lock(&(x))
#define MUTEX_UNLOCK(x)  pthread_mutex_unlock(&(x))
#define THREAD_ID        pthread_self()
//---------------------------------------------------------------------------------------------

void handle_error(const char *file, int lineno, const char *msg)
{
   fprintf(stderr, "** %s:%d %s\n", file, lineno, msg);
   ERR_print_errors_fp(stderr);
   /* exit(-1); */
}
//---------------------------------------------------------------------------------------------

/* This array will store all of the mutexes available to OpenSSL. */
static MUTEX_TYPE *mutex_buf = NULL;

static void locking_function(int mode, int n, const char *file, int line)
{
   if(mode & CRYPTO_LOCK)
     MUTEX_LOCK(mutex_buf[n]);
   else
     MUTEX_UNLOCK(mutex_buf[n]);
}
//---------------------------------------------------------------------------------------------

static unsigned long id_function(void)
{
   return ((unsigned long)THREAD_ID);
}
//---------------------------------------------------------------------------------------------

int thread_setup(void)
{
   int i;

   mutex_buf = malloc(CRYPTO_num_locks() * sizeof(MUTEX_TYPE));
   if(!mutex_buf)
     return 0;
   for(i = 0;  i < CRYPTO_num_locks();  i++)
     MUTEX_SETUP(mutex_buf[i]);
   CRYPTO_set_id_callback(id_function);
   CRYPTO_set_locking_callback(locking_function);
   return 1;
}
//---------------------------------------------------------------------------------------------

int thread_cleanup(void)
{
   int i;

   if(!mutex_buf)
     return 0;
   CRYPTO_set_id_callback(NULL);
   CRYPTO_set_locking_callback(NULL);
   for(i = 0;  i < CRYPTO_num_locks();  i++)
     MUTEX_CLEANUP(mutex_buf[i]);
   free(mutex_buf);
   mutex_buf = NULL;
   return 1;
}
//---------------------------------------------------------------------------------------------
//---------------------------------------------------------------------------------------------

-------------------------------------------------------------------
Unsubscribe: https://cool.haxx.se/list/listinfo/curl-library
Etiquette: https://curl.haxx.se/mail/etiquette.html
Received on 2018-08-01