[med-svn] [r-cran-curl] 02/05: New upstream version 3.0
Andreas Tille
tille at debian.org
Fri Oct 13 09:35:00 UTC 2017
This is an automated email from the git hooks/post-receive script.
tille pushed a commit to branch master
in repository r-cran-curl.
commit 7ad1804fd8a8fa87a0b1769ecc0fab6e2f9c0414
Author: Andreas Tille <tille at debian.org>
Date: Fri Oct 13 11:30:51 2017 +0200
New upstream version 3.0
---
DESCRIPTION | 6 ++--
MD5 | 39 +++++++++++----------
NAMESPACE | 2 ++
NEWS | 11 ++++++
R/fetch.R | 5 +--
R/handle.R | 2 +-
R/multi.R | 72 ++++++++++++++++++++++++++++++++++----
R/nslookup.R | 1 +
inst/doc/intro.html | 20 +++++------
man/curl_fetch.Rd | 6 +++-
man/multi.Rd | 43 ++++++++++++++++++++---
src/Makevars.win | 25 ++++---------
src/curl-common.h | 2 ++
src/handle.c | 6 ++++
src/init.c | 2 ++
src/multi.c | 66 +++++++++++++++++++++++++++++++---
src/ssl.c | 35 ++++++++++++++++++
src/utils.c | 14 ++++++++
tests/testthat/test-certificates.R | 6 ++--
tests/testthat/test-multi.R | 57 ++++++++++++++++++++++++++++++
tools/winlibs.R | 5 +--
21 files changed, 352 insertions(+), 73 deletions(-)
diff --git a/DESCRIPTION b/DESCRIPTION
index 2ed9db1..fc1f237 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,7 +1,7 @@
Package: curl
Type: Package
Title: A Modern and Flexible Web Client for R
-Version: 2.8.1
+Version: 3.0
Authors at R: c(
person("Jeroen", "Ooms", , "jeroen at berkeley.edu", role = c("cre", "aut")),
person("Hadley", "Wickham", , "hadley at rstudio.com", role = "ctb"),
@@ -29,10 +29,10 @@ Depends: R (>= 3.0.0)
LazyData: true
RoxygenNote: 6.0.1
NeedsCompilation: yes
-Packaged: 2017-07-20 10:47:34 UTC; jeroen
+Packaged: 2017-10-05 11:21:57 UTC; jeroen
Author: Jeroen Ooms [cre, aut],
Hadley Wickham [ctb],
RStudio [cph]
Maintainer: Jeroen Ooms <jeroen at berkeley.edu>
Repository: CRAN
-Date/Publication: 2017-07-21 23:02:20 UTC
+Date/Publication: 2017-10-06 10:18:36 UTC
diff --git a/MD5 b/MD5
index a773d62..9ea329f 100644
--- a/MD5
+++ b/MD5
@@ -1,16 +1,16 @@
-777eb3d19c39cfec86dea77f3effd1f8 *DESCRIPTION
+0f8221142d12e6825c8dec4159258d0a *DESCRIPTION
3d90003e73983dbcc48aa8a9aba50c5a *LICENSE
-493180de7b982326d0812811234fa9d0 *NAMESPACE
-5b255990bb811f5206c7fae773055ee1 *NEWS
+2873a38d29bee5851f627c0bcea7ddd1 *NAMESPACE
+0098722858cec95dcd61b627278778f1 *NEWS
a8d57b37295e51b5336a7d649f2655c9 *R/curl.R
3c3e1de8ce698ab9d5a8d2ad5dcf6243 *R/download.R
d5a884a9572387ca7ba6bc7d8af20b1a *R/echo.R
c6c5182b1629ba6080707e80026cf3ca *R/escape.R
-987eb506e56631bf185d17d7965343e0 *R/fetch.R
+b96d051532b04d88b269a9c92849ace6 *R/fetch.R
826b6c6f8a393b4377c7c6e61451c2a8 *R/form.R
-335e3fd0dbfa6feac1a15036bae9b5f7 *R/handle.R
-9bbad1a5258633609fd6cbfd817c9831 *R/multi.R
-0eb54202268bceedd53ad2129d447d7b *R/nslookup.R
+44aa3e4f30b73ee94c8ba73e21cb01e7 *R/handle.R
+cbe2413cf26767df3004d39789cfc399 *R/multi.R
+10eee1e326522a74f7006d11cf288be5 *R/nslookup.R
44c8b59af7ae3fdb01f1e4e60cc2350c *R/onload.R
736948f114335f0d630583653865f99e *R/options.R
341deaec71f85196697042ede275d232 *R/parse_headers.R
@@ -23,26 +23,26 @@ d41d8cd98f00b204e9800998ecf8427e *configure.win
c39c444068e373bfb2c8ed9ed4e3ac5f *data/curl_symbols.rda
611b46a4e232014c7ae278b68d8feb03 *inst/doc/intro.R
405dc07df5149c76d91ee28fdc948c5f *inst/doc/intro.Rmd
-94d3edf4227ac497cfa8352812c78f64 *inst/doc/intro.html
+e6d038923ed16f38d8e419960fdebb09 *inst/doc/intro.html
d57f6d9e5c62e5a803cb6a0986ead273 *man/curl.Rd
5d3ad4ce0121c05dd1c57a480cfa0a8c *man/curl_download.Rd
cb407e5e439cc862c074502158d4303d *man/curl_echo.Rd
a485ea24e0b0f48300fd5326dadfe926 *man/curl_escape.Rd
-78f33f00b206fac85719739d68ae1f52 *man/curl_fetch.Rd
+2a91c3f6be44b1fc3ae8b262aa71e986 *man/curl_fetch.Rd
c052e459cb2cf67a0091670567dc8926 *man/curl_options.Rd
a37a546454cd67c61686643e82ea61ed *man/handle.Rd
4f516d10514eb7cb39ac8954a0794905 *man/handle_cookies.Rd
9c70d4d244f200a1a86337131cc6a1a8 *man/ie_proxy.Rd
-c05fb6a2f7868427b2650cd3ea81c286 *man/multi.Rd
+5acef6d806308bd293165d7470aea2cc *man/multi.Rd
cefd4119ea6f827b69a4d8e6c49358ea *man/multipart.Rd
837636046e875dde74e265de8d9c1f6f *man/nslookup.Rd
c3175e1371fa28d57c203d2af8fcd60b *man/parse_date.Rd
d0a186c57e0a7306a4c892f5498d8f85 *man/parse_headers.Rd
907343d8176df8e5611a85761a3e91f1 *src/Makevars.in
-9c4f36d80d26084373473ef924704b8d *src/Makevars.win
+9e40e661e50f0e7abe0d542e8f1926c5 *src/Makevars.win
6919a902921153ad6bcc2fc133653860 *src/callbacks.c
6d391e7654d5f35cc336ab174454e0ae *src/callbacks.h
-8ac023718b3b49b910d4e4961dc1144c *src/curl-common.h
+cf4af6362a219a5ebb687dda3229a6aa *src/curl-common.h
ca60d50a5e73b486510be5d802fdea28 *src/curl-symbols.h
351234356a2bff7a629a0b9628c35197 *src/curl.c
53ef5e1c1fae06308fef2c35ab2c2357 *src/download.c
@@ -50,15 +50,16 @@ ca60d50a5e73b486510be5d802fdea28 *src/curl-symbols.h
8efd3934c757642b7eb3cfcd6f3b874a *src/fetch.c
7f6f09915f019020eec775a4dabdb7ac *src/form.c
0c66ee6a65fa291ad390119953beb871 *src/getdate.c
-f4011970be658707d4c52924eebac93f *src/handle.c
+e62c657fee741f8de2973ee529b703f0 *src/handle.c
752bda61360b3cf594c9943d43cef8b5 *src/ieproxy.c
-2f2b792c8784fde145db53f87132d7e9 *src/init.c
+b9464c6eb4b92188434848795182e179 *src/init.c
a8e71725b7f154879b6873c87782dad3 *src/interrupt.c
-9c6c1084dc63fe03aed20839032350f4 *src/multi.c
+c8df9c1a16d12e347295dd7b0c8b2333 *src/multi.c
ec2d29b9fbbaece65e54651626cd198b *src/nslookup.c
91703e066a9e848a1d63eede9f9dd255 *src/reflist.c
f5cd7f3315e332bd18a38bb494ad640e *src/split.c
-2cfc7d3f20b0ca8f856c34fd3add73d0 *src/utils.c
+7f3f51c21577e71603903ef20ad5ffcf *src/ssl.c
+7099eb18cd00b2f2ec19324663a0050c *src/utils.c
365c211a7f307461ad46a74952b97d57 *src/version.c
5c32d93f54a58c1382926a61792bfaed *src/winhttp32.def.in
245dea4d9024e0381bc9f9bd3a8b6627 *src/winhttp64.def.in
@@ -67,17 +68,17 @@ f5cd7f3315e332bd18a38bb494ad640e *src/split.c
0d6b531f198306999a2219c399277b02 *tests/testthat/helper-version.R
bbbd697e81545d0a97fa262c18262155 *tests/testthat/test-auth.R
92f1d068681f06347ab39b81050df8de *tests/testthat/test-blockopen.R
-c0a9a4cb45d94973ccf13f572d0ba80d *tests/testthat/test-certificates.R
+8875a1092b706fc014a8dfffc3d0eb5c *tests/testthat/test-certificates.R
17857acb1bd8f16af0c43be20a5c9d51 *tests/testthat/test-connection.R
f2bc95bcceef2dfeebb48bb59f502712 *tests/testthat/test-cookies.R
c48bedfac6e4bf4220023269451ef0e6 *tests/testthat/test-escape.R
db8323549c034fef1a207400b0fdfc72 *tests/testthat/test-gc.R
dd85f72fc445554e74af3a8c78174447 *tests/testthat/test-handle.R
df03468bf874c557a0d11bd4b779c518 *tests/testthat/test-idn.R
-3ce80dfaff40cee787657b51e700e77f *tests/testthat/test-multi.R
+ec5aa1b4fc58b7fae0c991812743be77 *tests/testthat/test-multi.R
c5282786b5363c42d50e8941145bd661 *tests/testthat/test-nonblocking.R
24d68aa9da0801de878d899deb7355b3 *tests/testthat/test-post.R
2019a062987f868e186ae4b144455727 *tools/symbols-in-versions
1f5c1ab1572d962e4de4f67a2323a04a *tools/symbols.R
-23996666180a531897b4bcef45178229 *tools/winlibs.R
+5b4aef5a162bd6e6dfcd8e82b05ac2e2 *tools/winlibs.R
405dc07df5149c76d91ee28fdc948c5f *vignettes/intro.Rmd
diff --git a/NAMESPACE b/NAMESPACE
index 176999a..4ef52dc 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -28,6 +28,7 @@ export(ie_get_proxy_for_url)
export(ie_proxy_info)
export(multi_add)
export(multi_cancel)
+export(multi_fdset)
export(multi_list)
export(multi_run)
export(multi_set)
@@ -54,6 +55,7 @@ useDynLib(curl,R_handle_setheaders)
useDynLib(curl,R_handle_setopt)
useDynLib(curl,R_multi_add)
useDynLib(curl,R_multi_cancel)
+useDynLib(curl,R_multi_fdset)
useDynLib(curl,R_multi_list)
useDynLib(curl,R_multi_new)
useDynLib(curl,R_multi_run)
diff --git a/NEWS b/NEWS
index 624b5a8..1277eba 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,14 @@
+3.0
+ - MAJOR CHANGE ON WINDOWS: On Windows 7 / 2008-R2 and up we switch from OpenSSL to native Windows SSL
+ (secure channel). Therefore curl now uses certificates from the windows certificate manager.
+ This enables users on corporate/government networks to connect through enterprise proxies and such.
+ On Windows Vista/2008 and older (including CRAN) we still use OpenSSL to guarantee TLS 1.1/1.2 support.
+ Other changes:
+ - Windows: updated libcurl to v7.56.0 with dual SSL and native IDN/SSPI
+ - multi_add() and curl_fetch_multi() gain an optional 'data' callback parameter
+ - New function multi_fdset() returns the file descriptors of a multi pool
+ - Add CURLOPT_PROXYAUTH = CURLAUTH_ANY to default handle settings
+
2.8.1
- Windows: switch back to OpenSSL instead of SecureChannel because Windows 2008 (CRAN) does not
support TLS 1.1 and TLS 1.2 which is required for many servers now.
diff --git a/R/fetch.R b/R/fetch.R
index ea4c78b..0732550 100644
--- a/R/fetch.R
+++ b/R/fetch.R
@@ -102,8 +102,9 @@ curl_fetch_stream <- function(url, fun, handle = new_handle()){
#' @rdname curl_fetch
#' @inheritParams multi
#' @useDynLib curl R_curl_connection
-curl_fetch_multi <- function(url, done = NULL, fail = NULL, pool = NULL, handle = new_handle()){
+curl_fetch_multi <- function(url, done = NULL, fail = NULL, pool = NULL,
+ data = NULL, handle = new_handle()){
handle_setopt(handle, url = enc2utf8(url))
- multi_add(handle = handle, done = done, fail = fail, pool = pool)
+ multi_add(handle = handle, done = done, fail = fail, data = data, pool = pool)
invisible(handle)
}
diff --git a/R/handle.R b/R/handle.R
index 57c6c86..1b070d9 100644
--- a/R/handle.R
+++ b/R/handle.R
@@ -96,7 +96,7 @@ handle_setform <- function(handle, ..., .list = list()){
if(is.character(val)){
form[[i]] <- charToRaw(enc2utf8(val))
} else if(!is.raw(val) && !inherits(val, "form_file") && !inherits(val, "form_data")){
- stop("Insupported value type for form field '", names(form[i]), "'.")
+ stop("Unsupported value type for form field '", names(form[i]), "'.")
}
}
.Call(R_handle_setform, handle, form)
diff --git a/R/multi.R b/R/multi.R
index f0b06f9..1860c47 100644
--- a/R/multi.R
+++ b/R/multi.R
@@ -30,6 +30,14 @@
#' The \link{multi_cancel} function can be used to cancel a pending request.
#' It has no effect if the request was already completed or canceled.
#'
+#' The \link{multi_fdset} function returns the file descriptors curl is
+#' polling currently, and also a timeout parameter, the number of
+#' milliseconds an application should wait (at most) before proceeding. It
+#' is equivalent to the \code{curl_multi_fdset} and
+#' \code{curl_multi_timeout} calls. It is handy for applications that is
+#' expecting input (or writing output) through both curl, and other file
+#' descriptors.
+#'
#' @name multi
#' @rdname multi
#' @useDynLib curl R_multi_add
@@ -38,24 +46,65 @@
#' response data in same structure as \link{curl_fetch_memory}.
#' @param fail callback function called on failed request. Argument contains
#' error message.
+#' @param data callback function or open connection object for receiving data.
+#' If \code{NULL} the entire response content gets buffered and is returned
+#' in the \code{done} callback.
#' @param pool a multi handle created by \link{new_pool}. Default uses a global pool.
#' @export
-#' @examples h1 <- new_handle(url = "https://eu.httpbin.org/delay/3")
+#' @examples
+#' results <- list()
+#' success <- function(x){
+#' results <<- append(results, list(x))
+#' }
+#' failure <- function(str){
+#' cat(paste("Failed request:", str), file = stderr())
+#' }
+#' # This handle will take longest (3sec)
+#' h1 <- new_handle(url = "https://eu.httpbin.org/delay/3")
+#' multi_add(h1, done = success, fail = failure)
+#'
+#' # This handle writes data to a file
+#' con <- file("output.txt", open = "wb")
#' h2 <- new_handle(url = "https://eu.httpbin.org/post", postfields = "bla bla")
+#' multi_add(h2, done = success, fail = failure, data = con)
+#'
+#' # This handle raises an error
#' h3 <- new_handle(url = "https://urldoesnotexist.xyz")
-#' multi_add(h1, done = print, fail = print)
-#' multi_add(h2, done = print, fail = print)
-#' multi_add(h3, done = print, fail = print)
+#' multi_add(h3, done = success, fail = failure)
+#'
+#' # Actually perform the requests
#' multi_run(timeout = 2)
#' multi_run()
-multi_add <- function(handle, done = NULL, fail = NULL, pool = NULL){
+#'
+#' # Check the file
+#' close(con)
+#' readLines("output.txt")
+#' unlink("output.txt")
+multi_add <- function(handle, done = NULL, fail = NULL, data = NULL, pool = NULL){
if(is.null(pool))
pool <- multi_default()
+ if(inherits(data, "connection")){
+ con <- data
+ if(!isOpen(con))
+ stop("Connection for 'data' argument is not open")
+ data <- if(identical(summary(data)$text, "text")){
+ function(x){
+ cat(rawToChar(x), file = con)
+ flush(con)
+ }
+ } else {
+ function(x){
+ writeBin(x, con = con)
+ flush(con)
+ }
+ }
+ }
stopifnot(inherits(handle, "curl_handle"))
stopifnot(inherits(pool, "curl_multi"))
stopifnot(is.null(done) || is.function(done))
stopifnot(is.null(fail) || is.function(fail))
- .Call(R_multi_add, handle, done, fail, pool)
+ stopifnot(is.null(data) || is.function(data))
+ .Call(R_multi_add, handle, done, fail, data, pool)
}
#' @param timeout max time in seconds to wait for results. Use \code{0} to poll for results without
@@ -131,3 +180,14 @@ print.curl_multi <- function(x, ...){
len <- length(multi_list(x))
cat(sprintf("<curl multi-pool> (%d pending requests)\n", len))
}
+
+#' @export
+#' @useDynLib curl R_multi_fdset
+#' @rdname multi
+
+multi_fdset <- function(pool = NULL){
+ if(is.null(pool))
+ pool <- multi_default()
+ stopifnot(inherits(pool, "curl_multi"))
+ .Call(R_multi_fdset, pool)
+}
diff --git a/R/nslookup.R b/R/nslookup.R
index 64df1aa..93688c1 100644
--- a/R/nslookup.R
+++ b/R/nslookup.R
@@ -19,6 +19,7 @@
#' nslookup("ipv6.test-ipv6.com", ipv4_only = FALSE, error = FALSE)
nslookup <- function(host, ipv4_only = FALSE, multiple = FALSE, error = TRUE){
stopifnot(is.character(host))
+ host <- enc2utf8(host)
if(grepl("://", host, fixed = TRUE))
stop("This looks like a URL, not a hostname")
out <- .Call(R_nslookup, host[1], as.logical(ipv4_only))
diff --git a/inst/doc/intro.html b/inst/doc/intro.html
index 0ae8939..c92c0e1 100644
--- a/inst/doc/intro.html
+++ b/inst/doc/intro.html
@@ -10,7 +10,7 @@
-<meta name="date" content="2017-07-20" />
+<meta name="date" content="2017-10-05" />
<title>The curl package: a modern R interface to libcurl</title>
@@ -221,7 +221,7 @@ div.tocify {
<h1 class="title toc-ignore">The curl package: a modern R interface to libcurl</h1>
-<h4 class="date"><em>2017-07-20</em></h4>
+<h4 class="date"><em>2017-10-05</em></h4>
</div>
@@ -245,17 +245,17 @@ str(req)</code></pre>
<pre><code>List of 6
$ url : chr "https://httpbin.org/get"
$ status_code: int 200
- $ headers : raw [1:302] 48 54 54 50 ...
+ $ headers : raw [1:303] 48 54 54 50 ...
$ modified : POSIXct[1:1], format: NA
- $ times : Named num [1:6] 0 0.0228 0.1229 0.3557 2.822 ...
+ $ times : Named num [1:6] 0 0.0317 0.1293 0.4425 0.5442 ...
..- attr(*, "names")= chr [1:6] "redirect" "namelookup" "connect" "pretransfer" ...
$ content : raw [1:300] 7b 0a 20 20 ...</code></pre>
<pre class="r"><code>parse_headers(req$headers)</code></pre>
<pre><code> [1] "HTTP/1.1 200 OK" "Connection: keep-alive"
- [3] "Server: meinheld/0.6.1" "Date: Thu, 20 Jul 2017 10:47:02 GMT"
+ [3] "Server: meinheld/0.6.1" "Date: Thu, 05 Oct 2017 11:21:50 GMT"
[5] "Content-Type: application/json" "Access-Control-Allow-Origin: *"
[7] "Access-Control-Allow-Credentials: true" "X-Powered-By: Flask"
- [9] "X-Processed-Time: 0.00144004821777" "Content-Length: 300"
+ [9] "X-Processed-Time: 0.000731945037842" "Content-Length: 300"
[11] "Via: 1.1 vegur" </code></pre>
<pre class="r"><code>cat(rawToChar(req$content))</code></pre>
<pre><code>{
@@ -346,7 +346,7 @@ curl_fetch_multi('https://httpbin.org/blabla', done = cb, pool = pool)</code></p
<p>When we call <code>multi_run()</code>, all scheduled requests are performed concurrently. The callback functions get triggered when each request completes.</p>
<pre class="r"><code># This actually performs requests:
out <- multi_run(pool = pool)</code></pre>
-<pre><code>done: https://www.google.nl/?gfe_rd=cr&ei=O4pwWdjyC4vc8AfipLq4Cg : HTTP: 200
+<pre><code>done: https://www.google.nl/?gfe_rd=cr&dcr=0&ei=0BXWWa-SGqGk8wfAlo7YAw : HTTP: 200
done: https://httpbin.org/blabla : HTTP: 404
done: https://cloud.r-project.org/ : HTTP: 200 </code></pre>
<pre class="r"><code>print(out)</code></pre>
@@ -539,7 +539,7 @@ req <- curl_fetch_memory("http://httpbin.org/cookies/delete?foo", h
handle_cookies(h)</code></pre>
<pre><code> domain flag path secure expiration name value
1 httpbin.org FALSE / FALSE <NA> bar ftw
-2 httpbin.org FALSE / FALSE 2017-07-20 12:47:30 foo <NA>
+2 httpbin.org FALSE / FALSE 2017-10-05 11:21:55 foo <NA>
3 httpbin.org FALSE / FALSE <NA> baz moooo</code></pre>
<p>The <code>handle_cookies</code> function returns a data frame with 7 columns as specified in the <a href="http://www.cookiecentral.com/faq/#3.5">netscape cookie file format</a>.</p>
</div>
@@ -553,11 +553,11 @@ req2 <- curl_fetch_memory("http://www.r-project.org")</code></pre>
<pre class="r"><code>req <- curl_fetch_memory("https://api.github.com/users/ropensci")
req$times</code></pre>
<pre><code> redirect namelookup connect pretransfer starttransfer total
- 0.000000 0.011790 0.102725 0.302215 0.419966 0.420094 </code></pre>
+ 0.000000 0.011717 0.102780 0.303632 0.426000 0.426091 </code></pre>
<pre class="r"><code>req2 <- curl_fetch_memory("https://api.github.com/users/rstudio")
req2$times</code></pre>
<pre><code> redirect namelookup connect pretransfer starttransfer total
- 0.000000 0.000033 0.000035 0.000094 0.115455 0.115580 </code></pre>
+ 0.000000 0.000042 0.000044 0.000138 0.117330 0.117430 </code></pre>
<p>If you really need to re-use a handle, do note that that curl does not cleanup the handle after each request. All of the options and internal fields will linger around for all future request until explicitly reset or overwritten. This can sometimes leads to unexpected behavior.</p>
<pre class="r"><code>handle_reset(h)</code></pre>
<p>The <code>handle_reset</code> function will reset all curl options and request headers to the default values. It will <strong>not</strong> erase cookies and it will still keep alive the connections. Therefore it is good practice to call <code>handle_reset</code> after performing a request if you want to reuse the handle for a subsequent request. Still it is always safer to create a new fresh handle when possible, rather than recycling old ones.</p>
diff --git a/man/curl_fetch.Rd b/man/curl_fetch.Rd
index 159c109..d9a4273 100644
--- a/man/curl_fetch.Rd
+++ b/man/curl_fetch.Rd
@@ -14,7 +14,7 @@ curl_fetch_disk(url, path, handle = new_handle())
curl_fetch_stream(url, fun, handle = new_handle())
curl_fetch_multi(url, done = NULL, fail = NULL, pool = NULL,
- handle = new_handle())
+ data = NULL, handle = new_handle())
}
\arguments{
\item{url}{A character string naming the URL of a resource to be downloaded.}
@@ -33,6 +33,10 @@ response data in same structure as \link{curl_fetch_memory}.}
error message.}
\item{pool}{a multi handle created by \link{new_pool}. Default uses a global pool.}
+
+\item{data}{callback function or open connection object for receiving data.
+If \code{NULL} the entire response content gets buffered and is returned
+in the \code{done} callback.}
}
\description{
Low-level bindings to write data from a URL into memory, disk or a callback
diff --git a/man/multi.Rd b/man/multi.Rd
index 5566058..b9d3e8f 100644
--- a/man/multi.Rd
+++ b/man/multi.Rd
@@ -8,9 +8,10 @@
\alias{multi_list}
\alias{multi_cancel}
\alias{new_pool}
+\alias{multi_fdset}
\title{Async Multi Download}
\usage{
-multi_add(handle, done = NULL, fail = NULL, pool = NULL)
+multi_add(handle, done = NULL, fail = NULL, data = NULL, pool = NULL)
multi_run(timeout = Inf, poll = FALSE, pool = NULL)
@@ -21,6 +22,8 @@ multi_list(pool = NULL)
multi_cancel(handle)
new_pool(total_con = 100, host_con = 6, multiplex = TRUE)
+
+multi_fdset(pool = NULL)
}
\arguments{
\item{handle}{a curl \link{handle} with preconfigured \code{url} option.}
@@ -31,6 +34,10 @@ response data in same structure as \link{curl_fetch_memory}.}
\item{fail}{callback function called on failed request. Argument contains
error message.}
+\item{data}{callback function or open connection object for receiving data.
+If \code{NULL} the entire response content gets buffered and is returned
+in the \code{done} callback.}
+
\item{pool}{a multi handle created by \link{new_pool}. Default uses a global pool.}
\item{timeout}{max time in seconds to wait for results. Use \code{0} to poll for results without
@@ -76,14 +83,42 @@ requests.
The \link{multi_cancel} function can be used to cancel a pending request.
It has no effect if the request was already completed or canceled.
+
+The \link{multi_fdset} function returns the file descriptors curl is
+polling currently, and also a timeout parameter, the number of
+milliseconds an application should wait (at most) before proceeding. It
+is equivalent to the \code{curl_multi_fdset} and
+\code{curl_multi_timeout} calls. It is handy for applications that is
+expecting input (or writing output) through both curl, and other file
+descriptors.
}
\examples{
+results <- list()
+success <- function(x){
+ results <<- append(results, list(x))
+}
+failure <- function(str){
+ cat(paste("Failed request:", str), file = stderr())
+}
+# This handle will take longest (3sec)
h1 <- new_handle(url = "https://eu.httpbin.org/delay/3")
+multi_add(h1, done = success, fail = failure)
+
+# This handle writes data to a file
+con <- file("output.txt", open = "wb")
h2 <- new_handle(url = "https://eu.httpbin.org/post", postfields = "bla bla")
+multi_add(h2, done = success, fail = failure, data = con)
+
+# This handle raises an error
h3 <- new_handle(url = "https://urldoesnotexist.xyz")
-multi_add(h1, done = print, fail = print)
-multi_add(h2, done = print, fail = print)
-multi_add(h3, done = print, fail = print)
+multi_add(h3, done = success, fail = failure)
+
+# Actually perform the requests
multi_run(timeout = 2)
multi_run()
+
+# Check the file
+close(con)
+readLines("output.txt")
+unlink("output.txt")
}
diff --git a/src/Makevars.win b/src/Makevars.win
index cedf965..17d4219 100644
--- a/src/Makevars.win
+++ b/src/Makevars.win
@@ -1,30 +1,17 @@
-# Set a default
-LIBCURL_BUILD ?= openssl
-#LIBCURL_BUILD ?= winssl
-
-# Switches between OpenSSL and SecureChannel builds of libcurl stack
-ifeq "${LIBCURL_BUILD}" "openssl"
-CURL_LIBS = -lcurl -lssh2 -lz -lssl -lcrypto -lgdi32 -lws2_32 -lcrypt32 -lwldap32
-else
-CURL_LIBS = -lcurl -lz -lws2_32 -lcrypt32 -lwldap32
-endif
-
-PKG_LIBS= -L../windows/libcurl-7.54.1/lib-${LIBCURL_BUILD}${R_ARCH} \
- -L. -lwinhttp $(CURL_LIBS)
+VERSION=7.56.0
+PKG_LIBS= -L../windows/libcurl-$(VERSION)/lib${R_ARCH} \
+ -L. -lwinhttp -lcurl-dualssl -lz -lssl -lcrypto -lgdi32 -lws2_32 -lcrypt32 -lwldap32
PKG_CPPFLAGS= \
- -I../windows/libcurl-7.54.1/include -DCURL_STATICLIB
+ -I../windows/libcurl-$(VERSION)/include -DCURL_STATICLIB
-all: info clean winlibs libwinhttp.dll.a
+all: clean winlibs libwinhttp.dll.a
clean:
rm -f $(SHLIB) $(OBJECTS) libwinhttp.dll.a winhttp.def
-info:
- @echo "Building curl with '$(LIBCURL_BUILD)' crypto backend."
-
winlibs: clean
- "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" --vanilla "../tools/winlibs.R"
+ "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" --vanilla "../tools/winlibs.R" $(VERSION)
echo '#include <curl/curl.h>' | $(CPP) $(PKG_CPPFLAGS) -std=gnu99 -xc - | grep "^[ \t]*CURLOPT_.*," | sed s/,// > ../tools/option_table.txt
winhttp.def:
diff --git a/src/curl-common.h b/src/curl-common.h
index 7487611..5a8bc36 100644
--- a/src/curl-common.h
+++ b/src/curl-common.h
@@ -25,6 +25,7 @@ typedef struct {
memory content;
SEXP complete;
SEXP error;
+ SEXP data;
} async;
typedef struct {
@@ -55,6 +56,7 @@ void reset_errbuf(reference *ref);
void clean_handle(reference *ref);
size_t push_disk(void* contents, size_t sz, size_t nmemb, FILE *ctx);
size_t append_buffer(void *contents, size_t sz, size_t nmemb, void *ctx);
+size_t data_callback(void * data, size_t sz, size_t nmemb, SEXP fun);
CURLcode curl_perform_with_interrupt(CURL *handle);
int pending_interrupt();
SEXP make_handle_response(reference *ref);
diff --git a/src/handle.c b/src/handle.c
index b223e40..9d67689 100644
--- a/src/handle.c
+++ b/src/handle.c
@@ -113,6 +113,7 @@ void set_handle_defaults(reference *ref){
/* allow all authentication methods */
assert(curl_easy_setopt(handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY));
assert(curl_easy_setopt(handle, CURLOPT_UNRESTRICTED_AUTH, 1L));
+ assert(curl_easy_setopt(handle, CURLOPT_PROXYAUTH, CURLAUTH_ANY));
/* enables HTTP2 on HTTPS (match behavior of curl cmd util) */
#if defined(CURL_VERSION_HTTP2) && defined(HAS_HTTP_VERSION_2TLS)
@@ -126,6 +127,11 @@ void set_handle_defaults(reference *ref){
/* dummy readfunction because default can freeze R */
assert(curl_easy_setopt(handle, CURLOPT_READFUNCTION, dummy_read));
+ /* seems to be needed for native WinSSL */
+#ifdef _WIN32
+ curl_easy_setopt(handle, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NO_REVOKE);
+#endif
+
/* set default headers (disables the Expect: http 100)*/
#ifdef HAS_CURLOPT_EXPECT_100_TIMEOUT_MS
assert(curl_easy_setopt(handle, CURLOPT_EXPECT_100_TIMEOUT_MS, 0L));
diff --git a/src/init.c b/src/init.c
index 93b5a16..fbc272d 100644
--- a/src/init.c
+++ b/src/init.c
@@ -1,10 +1,12 @@
#include <R_ext/Rdynload.h>
#include <curl/curl.h>
+void select_ssl_backend();
CURLM *multi_handle = NULL;
static struct curl_slist * default_headers = NULL;
void R_init_curl(DllInfo *info) {
+ select_ssl_backend();
curl_global_init(CURL_GLOBAL_DEFAULT);
multi_handle = curl_multi_init();
default_headers = curl_slist_append(default_headers, "Expect:");
diff --git a/src/multi.c b/src/multi.c
index 65ee725..439ce35 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -43,6 +43,7 @@ void multi_release(reference *ref){
ref->async.content.size = 0;
ref->async.complete = NULL;
ref->async.error = NULL;
+ ref->async.data = NULL;
ref->async.node = NULL;
/* Unlock handle (and cleanup if needed) */
@@ -58,7 +59,7 @@ SEXP R_multi_cancel(SEXP handle_ptr){
return handle_ptr;
}
-SEXP R_multi_add(SEXP handle_ptr, SEXP cb_complete, SEXP cb_error, SEXP pool_ptr){
+SEXP R_multi_add(SEXP handle_ptr, SEXP cb_complete, SEXP cb_error, SEXP cb_data, SEXP pool_ptr){
multiref *mref = get_multiref(pool_ptr);
CURLM *multi = mref->m;
@@ -67,8 +68,13 @@ SEXP R_multi_add(SEXP handle_ptr, SEXP cb_complete, SEXP cb_error, SEXP pool_ptr
Rf_error("Handle is locked. Probably in use in a connection or async request.");
/* placeholder body */
- curl_easy_setopt(ref->handle, CURLOPT_WRITEFUNCTION, append_buffer);
- curl_easy_setopt(ref->handle, CURLOPT_WRITEDATA, &(ref->async.content));
+ if(Rf_isFunction(cb_data)){
+ curl_easy_setopt(ref->handle, CURLOPT_WRITEFUNCTION, (curl_write_callback) data_callback);
+ curl_easy_setopt(ref->handle, CURLOPT_WRITEDATA, cb_data);
+ } else {
+ curl_easy_setopt(ref->handle, CURLOPT_WRITEFUNCTION, append_buffer);
+ curl_easy_setopt(ref->handle, CURLOPT_WRITEDATA, &(ref->async.content));
+ }
/* add to scheduler */
massert(curl_multi_add_handle(multi, ref->handle));
@@ -81,7 +87,8 @@ SEXP R_multi_add(SEXP handle_ptr, SEXP cb_complete, SEXP cb_error, SEXP pool_ptr
/* set multi callbacks */
ref->async.complete = cb_complete;
ref->async.error = cb_error;
- R_SetExternalPtrProtected(handle_ptr, Rf_list2(cb_error, cb_complete));
+ ref->async.data = cb_data;
+ R_SetExternalPtrProtected(handle_ptr, Rf_list3(cb_error, cb_complete, cb_data));
/* lock and protect handle */
ref->refCount++;
@@ -245,3 +252,54 @@ SEXP R_multi_setopt(SEXP pool_ptr, SEXP total_con, SEXP host_con, SEXP multiplex
SEXP R_multi_list(SEXP pool_ptr){
return get_multiref(pool_ptr)->handles;
}
+
+SEXP R_multi_fdset(SEXP pool_ptr){
+ multiref *mref = get_multiref(pool_ptr);
+ CURLM *multi = mref->m;
+ fd_set read_fd_set, write_fd_set, exc_fd_set;
+ int max_fd, i, num_read = 0, num_write = 0, num_exc = 0;
+ int *pread, *pwrite, *pexc;
+ long timeout;
+ SEXP result, names;
+
+ FD_ZERO(&read_fd_set);
+ FD_ZERO(&write_fd_set);
+ FD_ZERO(&exc_fd_set);
+
+ massert(curl_multi_fdset(multi, &read_fd_set, &write_fd_set,
+ &exc_fd_set, &max_fd));
+
+ massert(curl_multi_timeout(multi, &timeout));
+
+ for (i = 0; i <= max_fd; i++){
+ if (FD_ISSET(i, &read_fd_set)) num_read++;
+ if (FD_ISSET(i, &write_fd_set)) num_write++;
+ if (FD_ISSET(i, &exc_fd_set)) num_exc++;
+ }
+
+ result = PROTECT(allocVector(VECSXP, 4));
+ SET_VECTOR_ELT(result, 0, allocVector(INTSXP, num_read));
+ SET_VECTOR_ELT(result, 1, allocVector(INTSXP, num_write));
+ SET_VECTOR_ELT(result, 2, allocVector(INTSXP, num_exc));
+ SET_VECTOR_ELT(result, 3, ScalarReal((double) timeout));
+
+ names = PROTECT(allocVector(STRSXP, 4));
+ SET_STRING_ELT(names, 0, mkChar("reads"));
+ SET_STRING_ELT(names, 1, mkChar("writes"));
+ SET_STRING_ELT(names, 2, mkChar("exceptions"));
+ SET_STRING_ELT(names, 3, mkChar("timeout"));
+ setAttrib(result, R_NamesSymbol, names);
+
+ pread = INTEGER(VECTOR_ELT(result, 0));
+ pwrite = INTEGER(VECTOR_ELT(result, 1));
+ pexc = INTEGER(VECTOR_ELT(result, 2));
+
+ for (i = 0; i <= max_fd; i++){
+ if (FD_ISSET(i, &read_fd_set)) *(pread++) = i;
+ if (FD_ISSET(i, &write_fd_set)) *(pwrite++) = i;
+ if (FD_ISSET(i, &exc_fd_set)) *(pexc++) = i;
+ }
+
+ UNPROTECT(2);
+ return result;
+}
diff --git a/src/ssl.c b/src/ssl.c
new file mode 100644
index 0000000..92ac801
--- /dev/null
+++ b/src/ssl.c
@@ -0,0 +1,35 @@
+#include <curl/curl.h>
+#include <Rinternals.h>
+
+#if LIBCURL_VERSION_MAJOR > 7 || (LIBCURL_VERSION_MAJOR == 7 && LIBCURL_VERSION_MINOR >= 56)
+#define HAS_MULTI_SSL 1
+#endif
+
+/* Fall back on OpenSSL on Legacy Windows (Vista/2008) which do not support TLS 1.2 natively */
+void select_ssl_backend(){
+#if defined(_WIN32) && defined(HAS_MULTI_SSL)
+ DWORD dwBuild = 0;
+ DWORD dwVersion = GetVersion();
+ if (dwVersion < 0x80000000)
+ dwBuild = (DWORD)(HIWORD(dwVersion));
+
+ /* TLS 1.2 requires at least Windows 7 or 2008-R2 */
+ curl_sslbackend backend = dwBuild < 7600 ? CURLSSLBACKEND_OPENSSL : CURLSSLBACKEND_SCHANNEL;
+
+ /* Try to set the backend */
+ switch(curl_global_sslset(backend, NULL, NULL)){
+ case CURLSSLSET_OK :
+ break;
+ case CURLSSLSET_TOO_LATE:
+ Rf_warning("Failed to set libcurl SSL: already initiated");
+ break;
+ case CURLSSLSET_UNKNOWN_BACKEND:
+ Rf_warning("Failed to set libcurl SSL: unsupported backend");
+ break;
+ default:
+ Rf_warning("Failed to set libcurl SSL: unknown error");
+ break;
+ }
+
+#endif
+}
diff --git a/src/utils.c b/src/utils.c
index 07f8a5e..5a5193f 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -130,3 +130,17 @@ size_t append_buffer(void *contents, size_t sz, size_t nmemb, void *ctx) {
mem->size += realsize;
return realsize;
}
+
+size_t data_callback(void * data, size_t sz, size_t nmemb, SEXP fun) {
+ size_t size = sz * nmemb;
+ SEXP buf = PROTECT(allocVector(RAWSXP, size));
+ memcpy(RAW(buf), data, size);
+
+ /* call the R function */
+ int err;
+ SEXP call = PROTECT(LCONS(fun, LCONS(buf, R_NilValue)));
+ R_tryEval(call, R_GlobalEnv, &err);
+ UNPROTECT(2);
+ return err ? 0 : size;
+}
+
diff --git a/tests/testthat/test-certificates.R b/tests/testthat/test-certificates.R
index 7b2a92c..4f852c0 100644
--- a/tests/testthat/test-certificates.R
+++ b/tests/testthat/test-certificates.R
@@ -1,10 +1,12 @@
context("Certificate validation")
+# This tests TLS 1.2
test_that("CloudFlare / LetsEncrypt certs", {
- if(is.numeric(get_windows_build()))
- skip_if_not(get_windows_build() >= 7600, "TLS 1.2 requires at least Windows 7 / Windows Server 2008 R2")
expect_equal(curl_fetch_memory('https://www.opencpu.org')$status_code, 200)
expect_equal(curl_fetch_memory('https://rud.is')$status_code, 200)
+
+ # Test HTTP -> HTTPS (TLS 1.2) redirection
+ expect_equal(curl_fetch_memory('http://curl.haxx.se')$status_code, 200)
})
test_that("Invalid domain raises an error", {
diff --git a/tests/testthat/test-multi.R b/tests/testthat/test-multi.R
index 37e7a4e..84d98fe 100644
--- a/tests/testthat/test-multi.R
+++ b/tests/testthat/test-multi.R
@@ -1,6 +1,7 @@
context("Multi handle")
test_that("Max connections works", {
+ skip_on_os("solaris")
skip_if_not(curl_version()$version >= as.numeric_version("7.30"),
"libcurl does not support host_connections")
multi_set(host_con = 2, multiplex = FALSE)
@@ -25,6 +26,7 @@ test_that("Max connections reset", {
})
test_that("Timeout works", {
+ skip_on_os("solaris")
h1 <- new_handle(url = httpbin("delay/3"))
h2 <- new_handle(url = httpbin("post"), postfields = "bla bla")
h3 <- new_handle(url = "https://urldoesnotexist.xyz", connecttimeout = 1)
@@ -97,8 +99,63 @@ test_that("Errors in Callbacks", {
expect_equal(multi_run(pool = pool), list(success = 0, error = 0, pending = 0))
})
+test_that("Data callback", {
+ con <- rawConnection(raw(0), "r+")
+ on.exit(close(con))
+ hx <- new_handle()
+ handle_setopt(hx, COPYPOSTFIELDS = jsonlite::toJSON(mtcars));
+ handle_setheaders(hx, "Content-Type" = "application/json")
+ status <- NULL
+ curl_fetch_multi(httpbin("post"), done = function(res){
+ status <<- res$status_code
+ }, fail = stop, data = function(x){
+ writeBin(x, con)
+ }, handle = hx)
+
+ curl_fetch_multi(httpbin("get"), done = function(res){
+ #this somehow breaks the gc
+ #expect_equal(res$status_code, 200)
+ }, fail = stop, data = function(x){
+ expect_is(x, "raw")
+ })
+
+ # test protect
+ gc()
+
+ # perform requests
+ out <- multi_run()
+ expect_equal(out$success, 2)
+ expect_equal(status, 200)
+
+ # get data from buffer
+ content <- rawConnectionValue(con)
+ output <- jsonlite::fromJSON(rawToChar(content))
+ expect_is(output$json, "data.frame")
+ expect_equal(sort(names(output$json)), sort(names(mtcars)))
+})
+
+test_that("callback protection", {
+ done <- function(res){
+ expect_is(res$status_code, "integer")
+ }
+ fail <- function(...){
+ print("error")
+ }
+ data <- function(x){
+ expect_is(x, "raw")
+ }
+ pool <- new_pool()
+ handle <- new_handle(url = httpbin("get"))
+ multi_add(handle, done = done, fail = fail, data = data, pool = pool)
+ rm(handle, done, fail, data)
+ gc(); gc();
+ out <- multi_run(pool = pool)
+ expect_equal(out$success, 1)
+})
+
test_that("GC works", {
gc()
expect_equal(total_handles(), 0L)
})
+
diff --git a/tools/winlibs.R b/tools/winlibs.R
index 78d064d..7eb7e6b 100644
--- a/tools/winlibs.R
+++ b/tools/winlibs.R
@@ -1,7 +1,8 @@
# Build against static libraries from rwinlib
-if(!file.exists("../windows/libcurl-7.54.1/include/curl/curl.h")){
+VERSION <- commandArgs(TRUE)
+if(!file.exists(sprintf("../windows/libcurl-%s/include/curl/curl.h", VERSION))){
if(getRversion() < "3.3.0") setInternet2()
- download.file("https://github.com/rwinlib/libcurl/archive/v7.54.1.zip", "lib.zip", quiet = TRUE)
+ download.file(sprintf("https://github.com/rwinlib/libcurl/archive/v%s.zip", VERSION), "lib.zip", quiet = TRUE)
dir.create("../windows", showWarnings = FALSE)
unzip("lib.zip", exdir = "../windows")
unlink("lib.zip")
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/r-cran-curl.git
More information about the debian-med-commit
mailing list