mastering libcurl part 2

@bagder
/* set the options (I left out a few, you will get the point anyway) */
curl_easy_setopt(handles[HTTP_HANDLE], CURLOPT_URL, "https://example.com");
curl_easy_setopt(handles[FTP_HANDLE], CURLOPT_URL, "ftp://example.com");
curl_easy_setopt(handles[FTP_HANDLE], CURLOPT_UPLOAD, 1L);
/* init a multi stack */
multi_handle = curl_multi_init();
/* add the individual transfers */
for(i = 0; i<HANDLECOUNT; i++)
curl_multi_add_handle(multi_handle, handles[i]);
while(still_running) {
CURLMcode mc = curl_multi_perform(multi_handle, &still_running);
if(still_running)
/* wait for activity, timeout or "nothing" */
mc = curl_multi_poll(multi_handle, NULL, 0, 1000, NULL);
if(mc)
break;
}
/* See how the transfers went */
while((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
if(msg->msg == CURLMSG_DONE) {
int idx;
/* Find out which handle this message is about */
for(idx = 0; idx<HANDLECOUNT; idx++) {
int found = (msg->easy_handle == handles[idx]);
if(found)
break;
}
switch(idx) {
case HTTP_HANDLE:
printf("HTTP transfer completed with status %dn", msg->data.result);
break;
case FTP_HANDLE:
printf("FTP transfer completed with status %dn", msg->data.result);
break;
}
}
}
/* remove the transfers and cleanup the handles */
for(i = 0; i<HANDLECOUNT; i++) {
curl_multi_remove_handle(multi_handle, handles[i]);
curl_easy_cleanup(handles[i]);
}
#include <stdio.h>
#include <curl/curl.h>
int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "https://example.com");
/* example.com is redirected, so we tell libcurl to follow redirection */
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %sn",
curl_easy_strerror(res));
/* always cleanup */
curl_easy_cleanup(curl);
}
return 0;
}
mastering libcurl
November 20, 2023 Daniel Stenberg
more libcurl source code and details in a single video than you ever saw before
part two

Daniel Stenberg @bagder
@mastodon.social
https://daniel.haxx.se

Setup - November 20, 2023
Live-streamed
Expected to last multiple hours
Recorded
Lots of material never previously presented
There will be LOTS of source code on display
https://github.com/bagder/mastering-libcurl
@bagder

@bagder
Just ask!
Live questions from non-visible chat
@bagder

mastering libcurl
The project
Getting it
API and ABI
Architecture
API fundamentals
Setting up
@bagder
Transfers
Share API
TLS
Proxies
HTTP
Header API
URL API
WebSocket
Future
Part 1 Part 2

Transfers
@bagder
slide 7 of 99
your app
your
transfers

Downloads: storing
libcurl delivers data to CURLOPT_WRITEFUNCTION
Pass in a custom pointer to the callback with CURLOPT_WRITEDATA
Defaults to fwrite() to stdout: rarely what you want
The function is called none, one or many times.
Gets 1 to 16kB data per call
The exact amounts depends on factors beyond your control
and vary - do not presume!
@bagder

Downloads: storing
Return a different value than number of
bytes to deal with == error
@bagder
struct memory {
char *ptr;
size_t size;
};
static size_t
write_cb(void *contents, size_t size, size_t nmemb, void *userp)
{
struct memory *mem = userp;
size_t realsize = size * nmemb;
char *ptr = realloc(mem->ptr, mem->size + realsize + 1);
if(!ptr) {
fprintf(stderr, "not enough memoryn");
return 0;
}
mem->ptr = ptr;
memcpy(&mem->ptr[mem->size], contents, realsize);
mem->size += realsize;
mem->ptr[mem->size] = 0;
return realsize;
}
write-callback.c

Downloads: compression
no data compression is done by default
HTTP compression is download-only
(HTTP/2 and HTTP/3 header compression is not optional)
For HTTP, set CURLOPT_ACCEPT_ENCODING to “”
For SSH, set CURLOPT_SSH_COMPRESSION to 1L
Beware of decompression bombing
The write callback is called the same way
@bagder

Downloads: compression
@bagder
compress.c
int main(void)
{
CURL *curl;
CURLcode res = CURLE_OK;
if(curl) {
curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "");
curl_easy_setopt(curl, CURLOPT_URL, "https://curl.se/");
}
return (int)res;
}

Downloads: multiple
Reuse easy handles and call curl_easy_perform() again for serial
Add multiple easy handles to a multi handle for parallel
Multi handle transfers can be made to multiplex (more details later)
Run multiple instances in separate threads
@bagder

Downloads: maximum ﬁle size
A default ﬁle transfer has no size nor time limit
CURLOPT_MAXFILESIZE_LARGE
CURLOPT_TIMEOUT_MS
Or limit yourself in the write callback
or in the progress callback
@bagder

Downloads: resume and ranges
libcurl can continue a previous transfer or get a partial resource
CURLOPT_RESUME_FROM_LARGE
CURLOPT_RANGE
@bagder
range.c
int main(void)
{
CURL *curl;
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "https://curl.se/typo.html");
curl_easy_setopt(curl, CURLOPT_RANGE, “200-999”);
}
return (int)res;
}

Downloads: buffer size
CURLOPT_BUFFERSIZE is 16kB by default
Allocated and associated with the easy handle
10MB maximum
May affect maximum possible transfer speed
@bagder

Uploads: providing data
CURLOPT_READFUNCTION
Returning error stops transfer
@bagder
int main(void)
{
CURL *curl;
CURLcode res;
struct WriteThis wt;
wt.readptr = data;
wt.sizeleft = strlen(data);
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "https://curl.se");
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_cb);
curl_easy_setopt(curl, CURLOPT_READDATA, &wt);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
if(res)
}
return 0;
}
read-callback.c
struct WriteThis {
const char *readptr;
size_t sizeleft;
};
static size_t read_cb(char *dest, size_t size, size_t nmemb,
void *userp)
{
struct WriteThis *wt = (struct WriteThis *)userp;
size_t buffer_size = size*nmemb;
if(wt->sizeleft) {
/* [some code left out] */
memcpy(dest, wt->readptr, copy_this_much);
return copy_this_much; /* we copied this many bytes */
}
return 0; /* no more data left to deliver */
}
read-callback.c

Uploads: providing data
Two more common ways, details follow later in the HTTP section
CURLOPT_POSTFIELDS
CUROPT_MIMEPOST
@bagder

Uploads: multiple uploads
Of course you can mix uploads and downloads
The same rules apply for multiple uploads as for multiple downloads
@bagder

Uploads: buffer size
CURLOPT_UPLOAD_BUFFERSIZE is 64kB by default
Allocated when needed and associated with the easy handle
2MB maximum
May affect maximum possible transfer speed
@bagder

multiplexing
multiple simultaneous transfers over the same connection
Default is separate connections
@bagder
client
curl.se

multiplexing
Better bandwidth use
fewer resources
@bagder
client
curl.se

multiplexing
HTTP/2 or HTTP/3
transfers added to a multi handle
CURLMOPT_PIPELINING
There is a max number of streams
Connections can GOAWAY
Multiplexing (or not) is done transparently
@bagder
same
[ multi handle +
HTTP(S) +
port number +
host name +
HTTP version ]
==
multiplexing possible

Transfer controls: stop
Transfers continue until done (success or fail)
There are several timeouts
Return error from a callback
Careful with threads
With multi interface, remove the easy handle from the multi
@bagder

Transfer controls: stop slow transfers
By default, a transfer can stall for any
period without that being an error.
Stop transfer if below N bytes/sec
during M seconds:
N - CURLOPT_LOW_SPEED_LIMIT
M - CURLOPT_LOW_SPEED_TIME
@bagder
lowspeed.c
int main(void)
{
CURL *curl;
if(curl) {
/* abort if slower than 30 bytes/sec during 60 seconds */
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 60L);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 30L);
}
return (int)res;
}

Transfer controls: rate limit
Do not transfer data faster than N
bytes/sec
Separate options for receiving and
sending
attempts to keep the average speed
below the given threshold over a
period time
@bagder
maxspeed.c
int main(void)
{
CURL *curl;
if(curl) {
curl_off_t maxrecv = 31415;
curl_off_t maxsend = 67954;
curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, maxrecv);
curl_easy_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE, maxsend);
}
return (int)res;
}

Transfer controls: progress meter
libcurl can output a progress meter
on stderr
Disabled by default
Awkward reverse option:
CURLOPT_NOPROGRESS - set to 1L to
disable progress meter
Return error to stop transfer
@bagder
meter.c
int main(void)
{
CURL *curl;
if(curl) {
/* enable progress meter */
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
}
return (int)res;
}

Transfer controls: progress callback
keep track of transfer progress yourself
also called on idle with easy interface
@bagder
progress-cb.c
int main(void)
{
CURL *curl;
if(curl) {
struct progress data = {curl, 1000000 };
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &data);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION,
progress_cb);
}
return (int)res;
}
progress-cb.c
struct progress {
CURL *handle;
size_t size;
};
static size_t progress_cb(void *clientp, curl_off_t dltotal,
curl_off_t dlnow, curl_off_t ultotal,
curl_off_t ulnow)
{
struct progress *memory = clientp;
/* use the values */
return 0; /* all is good */
}

Timeouts
By default, libcurl typically has no or very liberal timeouts
You might want to narrow things down
CURLOPT_TIMEOUT[_MS]
CURLOPT_CONNECTTIMEOUT[_MS]
Make your own with the progress callback
@bagder

post transfer meta-data
curl_easy_getinfo() returns info about the previous transfer
There are 71 different CURLINFO_* options
See their man pages for details
@bagder
getinfo.c
int main(void)
{
CURL *curl;
if(curl) {
char *ct;
char *ip;
curl_off_t dlsize;
curl_easy_perform(curl);
curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ct);
curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD_T,
&dlsize);
curl_easy_getinfo(curl, CURLINFO_PRIMARY_IP, &ip);
}
return (int)res;
}
ACTIVESOCKET, APPCONNECT_TIME, APPCONNECT_TIME_T, CAINFO,
CAPATH, CERTINFO, CONDITION_UNMET, CONNECT_TIME,
CONNECT_TIME_T, CONN_ID, CONTENT_LENGTH_DOWNLOAD,
CONTENT_LENGTH_DOWNLOAD_T, CONTENT_LENGTH_UPLOAD,
CONTENT_LENGTH_UPLOAD_T, CONTENT_TYPE, COOKIELIST,
EFFECTIVE_METHOD, EFFECTIVE_URL, FILETIME, FILETIME_T,
FTP_ENTRY_PATH, HEADER_SIZE, HTTPAUTH_AVAIL, HTTP_CONNECTCODE,
HTTP_VERSION, LASTSOCKET, LOCAL_IP, LOCAL_PORT, NAMELOOKUP_TIME,
NAMELOOKUP_TIME_T, NUM_CONNECTS, OS_ERRNO, PRETRANSFER_TIME,
PRETRANSFER_TIME_T, PRIMARY_IP, PRIMARY_PORT, PRIVATE, PROTOCOL,
PROXYAUTH_AVAIL, PROXY_ERROR, PROXY_SSL_VERIFYRESULT,
REDIRECT_COUNT, REDIRECT_TIME, REDIRECT_TIME_T, REDIRECT_URL,
REFERER, REQUEST_SIZE, RESPONSE_CODE, RETRY_AFTER,
RTSP_CLIENT_CSEQ, RTSP_CSEQ_RECV, RTSP_SERVER_CSEQ,
RTSP_SESSION_ID, SCHEME, SIZE_DOWNLOAD, SIZE_DOWNLOAD_T,
SIZE_UPLOAD, SIZE_UPLOAD_T, SPEED_DOWNLOAD, SPEED_DOWNLOAD_T,
SPEED_UPLOAD, SPEED_UPLOAD_T, SSL_ENGINES, SSL_VERIFYRESULT,
STARTTRANSFER_TIME, STARTTRANSFER_TIME_T, TLS_SESSION,
TLS_SSL_PTR, TOTAL_TIME, TOTAL_TIME_T, XFER_ID

threading
Never share curl handles simultaneously across multiple threads
Use separate ones in separate threads ﬁne
Share partial data between handles in separate threads with the share API
Multi-threaded is a sensible option if CPU-bound
All libcurl calls work in the same thread (but... )
@bagder

error handling
Always check return codes from libcurl function calls
CURLOPT_ERRORBUFFER is your friend
Your application decides and acts on retry strategies
@bagder

convert curl command lines to libcurl source code embryos
excellent initial get-started step
--libcurl
@bagder
$ curl -H "foo: bar" https://curl.se/ libcurl dashdash.c
[lots of HTML output]
dashdash.c
int main(int argc, char *argv[])
{
CURLcode ret;
CURL *hnd;
struct curl_slist *slist1;
slist1 = NULL;
slist1 = curl_slist_append(slist1, "foo: bar");
hnd = curl_easy_init();
curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
curl_easy_setopt(hnd, CURLOPT_URL, "https://curl.se/");
curl_easy_setopt(hnd, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, slist1);
curl_easy_setopt(hnd, CURLOPT_USERAGENT, "curl/8.4.0");
curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L);
curl_easy_setopt(hnd, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_2TLS);
curl_easy_setopt(hnd, CURLOPT_FTP_SKIP_PASV_IP, 1L);
curl_easy_setopt(hnd, CURLOPT_TCP_KEEPALIVE, 1L);
/* Here is a list of options the curl code used that cannot get generated
as source easily. You may choose to either not use them or implement
them yourself.

share API
@bagder
slide 33 of 99

share data between handles
(some) caches and state are kept in the easy handle
transfers using different easy handles might not be entirely independent
the share API:
1. create a share object
2. decide what data the object should hold
3. specify which easy handles should use the object
4. one share object per transfer
@bagder

shareable data
cookie jar
DNS cache
connection pool
TLS session-id cache
PSL
HSTS cache
[future]
@bagder

share data between handles
Share object A: share cookies and DNS cache
Share object B: share connection cache
Let transfers share A and B as you wish
@bagder
easy transfer 1
easy transfer 2
easy transfer 3
easy transfer 4
easy transfer 5
share object B
share object A

share-cookies.c
@bagder
share-cookies.c
int main(void)
{
CURL *e;
CURL *e2;
CURLSH *sh = curl_share_init();
curl_share_setopt(sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
e = curl_easy_init();
e2 = curl_easy_init();
curl_easy_setopt(e, CURLOPT_URL, "https://curl.se/");
curl_easy_setopt(e2, CURLOPT_URL, "https://curl.se/");
curl_easy_setopt(e, CURLOPT_SHARE, sh);
curl_easy_setopt(e2, CURLOPT_SHARE, sh);
curl_easy_perform(e);
curl_easy_perform(e2);
curl_easy_cleanup(e);
curl_easy_cleanup(e2);
curl_share_cleanup(sh);
return 0;
}

shares and threads
if you share data cross-threads: setup mutex callbacks
the connection pool cannot be shared cross-threads (known bug)
@bagder

enable TLS
For communication with the peer
TLS is implied for “S-protocols”: HTTPS, FTPS, IMAPS, POP3S, SMTPS etc
Use the correct scheme in the URL
For FTP, IMAP, POP3, SMTP, LDAP etc: CURLOPT_USE_SSL is needed
@bagder
/* require TLS or fail */
curl_easy_setopt(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_ALL);

enable TLS for the proxy
the proxy connection is controlled separately
use HTTPS:// proxy for best privacy
@bagder
curl_easy_setopt(curl, CURLOPT_PROXY, “https://proxy.example:8081”);

ciphers
libcurl defaults to sensible, modern and safe ciphers
assuming you use modern TLS library
CURLOPT_SSL_CIPHER_LIST
CURLOPT_TLS13_CIPHERS
again, the proxy conﬁg is set and managed separately
@bagder

verifying server certificates
libcurl verifies TLS server certificates by default (CURLOPT_SSL_VERIFYPEER)
using the default CA store
...or the one you point to (CURLOPT_CAINFO)
craft your own verification with CURLOPT_SSL_CTX_FUNCTION
https proxy config is set and managed separately
never disable server certificate verification in prodution
@bagder
https://curl.se/docs/caextract.html

“blob” alternatives
For your systems where you don’t have a ﬁle system or want to avoid using ﬁles
CURLOPT_CAINFO_BLOB
CURLOPT_ISSUERCERT_BLOB
CURLOPT_PROXY_CAINFO_BLOB
CURLOPT_PROXY_ISSUERCERT_BLOB
CURLOPT_PROXY_SSLCERT_BLOB
CURLOPT_PROXY_SSLKEY_BLOB
CURLOPT_SSLCERT_BLOB
CURLOPT_SSLKEY_BLOB
@bagder
#def ne CURL_BLOB_COPY 1
#def ne CURL_BLOB_NOCOPY 0
struct curl_blob {
void *data;
size_t len;
unsigned int flags;
};
struct curl_blob blob = { bufptr, buflen, CURL_BLOB_COPY };
curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &blob);

TLS backend(s)
libcurl supports more than one TLS backend built-in
one of backends is set as default
set desired TLS backend ﬁrst thing, and only then
curl_global_sslset()
@bagder
BearSSL
AWS-LC
GnuTLS
mbedSSL
OpenSSL
Schannel
wolfSSL
Secure Transport
rustls
BoringSSL
libressl
AmiSSL

SSLKEYLOGFILE
@bagder
TLS transfers are encrypted
encrypted transfers can’t be snooped upon
unless we can extract the secrets in run-time
set the environment variable named SSLKEYLOGFILE to a ﬁle name
tell wireshark to read secrets from that ﬁle name
then run your libcurl using application as normal
(also works with browsers)

Proxies
@bagder
slide 49 of 99

a proxy is an intermediary
a server application that acts as an intermediary between a client requesting a
resource and the server providing that resource
proxy website
Network A Network B
client
@bagder

remote
name
resolve
local
name
resolve
proxy types
@bagder
SOCKS
HTTP
SOCKS4
SOCKS4a
SOCKS5
SOCKS5h
HTTP
HTTPS
HTTPS
HTTP/1.1
HTTPS
HTTP/2
SOCKS5
SOCKS4

set proxy type
CURLOPT_PROXY with “scheme” preﬁx
CURLOPT_PROXTYPE
@bagder
http:// CURLPROXY_HTTP
https:// CURLPROXY_HTTPS
https:// CURLPROXY_HTTPS2 (*)
socks4:// CURLPROXY_SOCKS4
socks4a:// CURLPROXY_SOCKS4A
socks5:// CURLPROXY_SOCKS5
socks5h:// CURLPROXY_SOCKS5_HOSTNAME

proxy.c
@bagder
proxy.c
int main(void)
{
CURL *curl = curl_easy_init();
CURLcode ret = CURLE_OK;
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "https://example.com/");
curl_easy_setopt(curl, CURLOPT_PROXY, "local.example.com:1080");
/* set the proxy type */
curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
ret = curl_easy_perform(curl);
}
return (int)ret;
}
proxy-scheme.c
int main(void)
{
CURL *curl = curl_easy_init();
CURLcode ret = CURLE_OK;
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "https://example.com/");
curl_easy_setopt(curl, CURLOPT_PROXY, "socks5://local.example.com:1080");
ret = curl_easy_perform(curl);
}
return (int)ret;
}

HTTP versions: CURLOPT_HTTP_VERSION
libcurl supports HTTP/0.9, HTTP/1.0, HTTP/1.1, HTTP/2 and HTTP/3
Generally: you don’t need to care
Different over the wire, made to look similar for the application
HTTP/0.9 must be enabled with CURLOPT_HTTP09_ALLOWED
HTTP/1.0 with CURL_HTTP_VERSION_1_0
HTTP/1.1 is a general default or CURL_HTTP_VERSION_1_1
HTTP/2 is default over HTTPS (CURL_HTTP_VERSION_2TLS), or used for clear text
HTTP as well with CURL_HTTP_VERSION_2_0
HTTP/3 is asked for with CURL_HTTP_VERSION_3 or CURL_HTTP_VERSION_3ONLY
@bagder

Kernel space
HTTP/1, HTTP/2, HTTP/3
IPv4 / IPv6
TLS 1.2+
UDP
TCP
connections
HTTP/1
@bagder
QUIC
streams
connections
TLS 1.3
HTTP Semantics
HTTP/3
header compression
server push
User
space
HTTP/2
streams
header compression
server push

Response code
Every HTTP response has a three-digit response code
1xx informational response
2xx success
3xx redirection
4xx client errors
5xx server errors
libcurl returns CURLE_OK for a successful transfer
Independently of response code!
@bagder

Response code: 404.c
@bagder
404.c
int main(void)
{
CURL *curl;
if(curl) {
long code;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
printf("Result: %d Response code: %ldn",
(int)res, code);
}
return (int)res;
}
$ gcc 404.c -lcurl
$ ./a.out
[HTML]
Result: 0 Response code: 404

Redirects
HTTP often redirects the client
HTTP response code 30x + a
Location: header
By default libcurl does not follow
redirects
CURLOPT_FOLLOWLOCATION
@bagder
follow.c
int main(void)
{
CURL *curl;
if(curl) {
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
}
return (int)res;
}

HTTP METHOD
GET is used by default
POST is used for CURLOPT_POST, CURLPOST_POSTFIELDS or CURLOPT_MIMEPOST
PUT is used for CURLOPT_UPLOAD
Change the method with CURLOPT_CUSTOMREQUEST
@bagder
WARNING

HTTP POST
CURLOPT_POSTFIELDS - provide data in a buffer
CURLOPT_POSTFIELDSIZE_LARGE - if not zero terminated
CURLOPT_COPYPOSTFIELDS - if you want libcurl to copy
CURLOPT_READFUNCTION - as seen on slide 16
CURLOPT_POST - tell it is a POST
CURLOPT_MIMEPOST - structured data in one or many “parts”, see next slide
@bagder

CURLOPT_POSTFIELDS
@bagder
postﬁelds.c
int main(void)
{
CURL *curl;
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "https://example.com/postit.cgi");
curl_easy_setopt(curl, CURLOPT_POSTFIELDS,
"name=Daniel&slides=yes&title=mastering-libcurl");
}
return (int)res;
}

HTTP multipart formpost
This is a POST sending data in a special multipart format
Content-Type multipart/form-data
The data is sent as a series of “parts”, one or more
Each part has a name, separate headers, ﬁle name and more
Each part is separated by a “mime boundary”
@bagder

The curl MIME API
CURLOPT_MIMEPOST wants a curl_mime * argument.
curl_mime * is a handle to a complete “multipart”
curl_mime *multipart = curl_mime_init(curl_handle);
Then add parts with curl_mime_addpart(multipart);
curl_mimepart *part = curl_mime_addpart(multipart);
Each part has a set of properties, name and data being the key ones.
curl_mime_name(part, "name");
curl_mime_data(part, "daniel", CURL_ZERO_TERMINATED);
Then you can add another part
@bagder

More HTTP mimepost
We can insert part data from a ﬁle with curl_mime_ﬁledata() or from a callback with
curl_mime_data_cb()
Provide a set of headers for a part with curl_mime_headers()
@bagder

HTTP request headers
A HTTP request is a method, path and a
sequence of headers
libcurl inserts the set of headers it thinks
are necessary, a bare minimum
CURLOPT_HTTPHEADER lets the
application add, change, remove headers
@bagder
mod-headers.c
int main(void)
{
CURL *curl;
struct curl_slist *list = NULL;
if(curl) {
/* add a custom one */
list = curl_slist_append(list, "Shoesize: 10");
/* remove an internally generated one */
list = curl_slist_append(list, "Accept:");
/* change an internally generated one */
list = curl_slist_append(list, "Host: curl.example");
/* provide one without content */
list = curl_slist_append(list, "Empty;");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
}
return (int)res;
}

Conditionals
only get the resource if newer (or older)
than this
CURLOPT_TIMECONDITION +
CURLOPT_TIMEVALUE_LARGE
Also works with FTP, FILE and RTSP
@bagder
ifmodiﬁed.c
int main(void)
{
CURL *curl;
if(curl) {
curl_off_t when = 1698793200; /* November 1, 2023 */
curl_easy_setopt(curl, CURLOPT_URL, "https://curl.se/index.html");
curl_easy_setopt(curl, CURLOPT_TIMEVALUE_LARGE, when);
curl_easy_setopt(curl, CURLOPT_TIMECONDITION,
(long)CURL_TIMECOND_IFMODSINCE);
}
return (int)res;
}

ranges
Ask for a piece of a remote resource
The server may ignore the ask 🙁
@bagder
range.c
int main(void)
{
CURL *curl;
if(curl) {
curl_easy_setopt(curl, CURLOPT_RANGE, “200-999”);
}
return (int)res;
}

PUT
PUT is upload for HTTP to “replace
the remote resource”
CURLOPT_UPLOAD +
CURLOPT_READFUNCTION
@bagder
put.c
struct reader {
FILE *fp;
};
static size_t read_cb(char *dest, size_t size, size_t nmemb, void *userp)
{
struct reader *r = userp;
return fread(dest, size, nmemb, r->fp);
}
int main(void)
{
CURL *curl;
CURLcode res;
FILE *this;
struct reader readctx;
this = fopen("hugef le.tar.gz", "r");
if(!this)
return 1;
readctx.fp = this;
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "https://example.com/upload.cgi");
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_cb);
curl_easy_setopt(curl, CURLOPT_READDATA, &readctx);
if(res)
}
fclose(this);
return 0;
}

Cookies
Cookies are name=value pairs for specific destinations
For a specific domain + path
When enabling the “cookie engine”, cookies are held in memory
When cookies are enabled, they are received and sent following cookies rules
INPUT:
CURLOPT_COOKIEFILE - read cookies from this file
CURLOPT_COOKIESESSION - consider this a new cookie session
CURLOPT_COOKIE - send only this specific cookie
OUTPUT:
CURLOPT_COOKIEJAR - write cookies to this file
@bagder

Cookies
*COOKIEFILE and *COOKIEJAR start the engine
cookies are per-easy handle - even when multi is used
@bagder
cookies

alt-svc
Alt-Svc: is a response header
“this service is available on that host for the next N seconds”
also says which HTTP versions it supports there
www.example.com:443 can then be provided by another.example.org:8765
The original way to bootstrap into sing HTTP/3
in its nature for “the next connect attempt”
this cache is kept in memory
can be saved to and load from ﬁle
specify which HTTP versions you want to “follow”
@bagder
Alt-svc
data

alt-svc controls
CURLOPT_ALTSVC specifies the alt-svc cache file name
CURLOPT_ALTSVC_CTRL specifies which protocols to allow
@bagder
Alt-svc
data
altsvc.c
int main(void)
{
CURL *curl;
if(curl) {
curl_easy_setopt(curl, CURLOPT_ALTSVC_CTRL,
CURLALTSVC_H1|CURLALTSVC_H2|CURLALTSVC_H3);
curl_easy_setopt(curl, CURLOPT_ALTSVC, "altsvc-cache.txt");
}
return (int)res;
}

HSTS
Strict-Transport-Security: is a response header
“access this host name only over HTTPS for the next N seconds”
this cache is kept in memory
can be saved to and load from ﬁle
makes subsequent requests avoid clear text transfers
@bagder
HSTS data

HSTS control
CURLOPT_HSTS speciﬁes the cache
ﬁle name
CURLOPT_HSTS_CTRL controls
behavior: enable and readonly
With CURLOPT_HSTSREADFUNCTION
and CURLOPT_HSTSWRITEFUNCTION
you can change storage.
@bagder
HSTS data
hsts.c
int main(void)
{
CURL *curl;
if(curl) {
curl_easy_setopt(curl, CURLOPT_ALTSVC_CTRL,
CURLALTSVC_H1|CURLALTSVC_H2|CURLALTSVC_H3);
curl_easy_setopt(curl, CURLOPT_ALTSVC, "altsvc-cache.txt");
}
return (int)res;
}

HTTP/2
HTTP version is transparent to the application
libcurl defaults to negotiating HTTP/2 for HTTPS
When using the multi interface, libcurl can multiplex HTTP/2
CURLOPT_STREAM_DEPENDS, CURLOPT_STREAM_DEPENDS_E and
CURLOPT_STREAM_WEIGHT
@bagder

HTTP/3
HTTP/3 needs to be asked for
HTTP/3 is only used over HTTPS
CURL_HTTP_VERSION_3 means try HTTP/3 but fall back gracefully if needed
CURL_HTTP_VERSION_3ONLY does not allow falling back
When using the multi interface, libcurl can multiplex HTTP/3
@bagder

Header API
@bagder
slide 78 of 99

Get HTTP response headers
in callbacks (an exception!)
after a transfer
numerate all headers
get a speciﬁc header ﬁeld
@bagder

curl_easy_header()
@bagder
curl_easy_header(CURL *easy,
const char *name,
size_t index,
unsigned int origin,
int request,
struct curl_header * hout);
struct curl_header {
char *name;
char *value;
size_t amount;
size_t index;
unsigned int origin;
void *anchor;
};
content-type.c
int main(void)
{
CURL *curl;
if(curl) {
struct curl_header *type;
CURLHcode h;
h = curl_easy_header(easy, "Content-Type", 0, CURLH_HEADER, -1, &type);
if(!h type)
printf("Content-Type: %sn", type->value);
}
return (int)res;
}
Origins
CURLH_HEADER
CURLH_TRAILER
CURLH_CONNECT
CURLH_1XX
CURLH_PSEUDO

curl_easy_nextheader()
@bagder
struct curl_header *
curl_easy_nextheader(CURL *easy,
unsigned int origin,
int request,
struct curl_header *prev);
struct curl_header {
char *name;
char *value;
size_t amount;
size_t index;
unsigned int origin;
void *anchor;
};
list-headers.c
int main(void)
{
CURL *curl;
if(curl) {
struct curl_header *h;
struct curl_header *prev = NULL;
while((h = curl_easy_nextheader(curl, CURLH_HEADER, 0, prev))) {
printf("%s: %sn", h->name, h->value);
prev = h;
}
}
return (int)res;
}
Origins
CURLH_HEADER
CURLH_TRAILER
CURLH_CONNECT
CURLH_1XX
CURLH_PSEUDO

URL API
@bagder
slide 82 of 99

URL API : basics
RFC 3986+
This is not the WHATWG URL Speciﬁcation
Parse, change, generate URLs
The CURLU * handle represents a URL
Create a URL handle with curl_url()
Or duplicate an existing with curl_url_dup()
@bagder

URL API: set and get parts
You set URL parts to add them to the handle
You get URL parts to extract them from the handle
@bagder

URL API: parts
CURLUPART_[name]
URL is the entire thing
@bagder
scheme://user:password@host:1234/path?query#fragment
URL
SCHEME
USER
PASSWORD
OPTIONS
HOST
ZONEID
PORT
PATH
QUERY
FRAGMENT

Parse a complete URL
By setting the URL in the handle
Get it from the handle to ﬁgure out what it holds
@bagder
url-set.c
int main(void)
{
CURLUcode rc;
CURLU *url = curl_url();
rc = curl_url_set(url, CURLUPART_URL,
"https://example.com", 0);
if(!rc) {
char *norm;
rc = curl_url_get(url, CURLUPART_URL, &norm, 0);
if(!rc norm) {
printf("URL: %sn", norm);
curl_free(norm);
}
}
curl_url_cleanup(url);
return 0;
}

Set URL components
By setting the components in the handle
Get the full URL from the handle to ﬁgure out what it holds
@bagder
url-set-parts.c
int main(void)
{
char *output;
CURLUcode rc;
curl_url_set(url, CURLUPART_SCHEME, "https", 0);
curl_url_set(url, CURLUPART_HOST, "curl.se", 0);
curl_url_set(url, CURLUPART_PORT, "443", 0);
curl_url_set(url, CURLUPART_PATH, "/index.html", 0);
rc = curl_url_get(url, CURLUPART_URL, &output, 0);
if(!rc output) {
printf("URL: %sn", output);
curl_free(output);
}
return 0;
}
$ gcc url-set-parts.c -lcurl
$ ./a.out
URL: https://curl.se:443/index.html

Extract URL components
First we set a URL
Get components from the URL
@bagder
url-get-parts.c
int main(void)
{
char *port;
char *query;
curl_url_set(url, CURLUPART_URL,
"https://example.com:8080/donkey.php?age=7", 0);
curl_url_get(url, CURLUPART_PORT, &port, 0);
curl_url_get(url, CURLUPART_QUERY, &query, 0);
if(port) {
printf("Port: %sn", port);
curl_free(port);
}
if(query) {
printf("Query: %sn", query);
curl_free(query);
}
return 0;
}
$ gcc url-get-parts.c -lcurl
$ ./a.out
Port: 8080
Query: age=7

Redirect
Set an absolute URL
Then set a relative URL
@bagder
redirect.c
int main(void)
{
char *dest;
CURLUcode rc;
"https://curl.se/this/cool/path/here.html", 0);
curl_url_set(url, CURLUPART_URL, " / /second/take/moo.jpg", 0);
rc = curl_url_get(url, CURLUPART_URL, &dest, 0);
if(!rc dest) {
printf("URL: %sn", dest);
curl_free(dest);
}
return 0;
}
$ gcc redirect.c -lcurl
$ ./a.out
URL: https://curl.se/this/second/take/moo.jpg

URL encoding
URL encode or URL decode the
strings at set or get
@bagder
$ gcc url-encode.c -lcurl
$ ./a.out
Raw Decoded Description
/%20works%3f/here.html / works?/here.html f rst
/no%20no%20no/ /no no no/ update
/%3f%3f%3f /??? qs
url-encode.c
static void showpath(CURL *url, const char *desc)
{
char *path;
curl_url_get(url, CURLUPART_PATH, &path, 0);
printf("%-22s", path);
curl_free(path);
curl_url_get(url, CURLUPART_PATH, &path, CURLU_URLDECODE);
printf(" %-22s %sn", path, desc);
curl_free(path);
}
int main(void)
{
"https:// .example/%20works%3f/here.html", 0);
printf("%-22s %-22s Descriptionn", "Raw", "Decoded");
showpath(url, "f rst");
curl_url_set(url, CURLUPART_PATH, "/no%20no%20no/", 0);
showpath(url, "update");
curl_url_set(url, CURLUPART_PATH, "???", CURLU_URLENCODE);
showpath(url, "qs");
return 0;
}

International Domain Names
@bagder
punycode.c
int main(void)
{
char *host;
curl_url_set(url, CURLUPART_URL, "https://räksmörgås.se/", 0);
curl_url_get(url, CURLUPART_HOST, &host, CURLU_PUNYCODE);
printf("host: %sn", host);
curl_free(host);
return 0;
}
$ gcc punycode.c -lcurl
$ ./a.out
host: xn rksmrgs-5wao1o.se
idn.c
int main(void)
{
char *host;
curl_url_set(url, CURLUPART_URL, "https://xn rksmrgs-5wao1o.se/", 0);
curl_url_get(url, CURLUPART_HOST, &host, CURLU_PUNY2IDN);
printf("host: %sn", host);
curl_free(host);
return 0;
}
$ gcc idn.c -lcurl
$ ./a.out
host: räksmörgås.se

WebSocket
@bagder
slide 92 of 99

WebSocket API
EXPERIMENTAL
curl_ws_recv
curl_ws_send
@bagder
https://youtu.be/NLIhd0wYO24

How to dig deeper
curl is the accumulated results and experiences from 25 years of improvements
almost 500 man pages
Everything curl
source code
ask the community!
@bagder

https://everything.curl.dev/
@bagder

Going next?
curl is 25 years old
curl has been growing and developed its entire lifetime
curl development speed is increasing
the Internet does not stop or slow down
protocols and new ways of doing Internet transfers keep popping up
new versions, new systems, new concepts and new ideas keep coming
there is no slowdown in sight
reasonably, curl will keep develop
curl will keep expanding, get new features, get taught new things
we, the community, make it do what we think it should do
you can affect what’s next for curl
@bagder

Daniel Stenberg
@bagder@mastodon.social
https://daniel.haxx.se/
Thank you!
Questions?
@bagder

mastering libcurl part 2

Recommended

Recommended

More Related Content

Similar to mastering libcurl part 2

Similar to mastering libcurl part 2 (20)

More from Daniel Stenberg

More from Daniel Stenberg (20)

Recently uploaded

Recently uploaded (20)

mastering libcurl part 2