From bd8184c724d55d6145cae09d2779d2b7482a50ae Mon Sep 17 00:00:00 2001 From: jbnadal Date: Mon, 20 Mar 2017 18:30:48 +0100 Subject: [PATCH] Bump uclient version 2016-12-09 --- .../package/uclient/Config.in | 14 + .../package/uclient/uclient.mk | 24 + src/3P/uclient/.gitignore | 9 + src/3P/uclient/CMakeLists.txt | 25 + src/3P/uclient/progress.c | 229 ++++ src/3P/uclient/progress.h | 25 + src/3P/uclient/uclient-backend.h | 45 + src/3P/uclient/uclient-fetch.c | 717 ++++++++++ src/3P/uclient/uclient-http.c | 1189 +++++++++++++++++ src/3P/uclient/uclient-utils.c | 184 +++ src/3P/uclient/uclient-utils.h | 49 + src/3P/uclient/uclient.c | 421 ++++++ src/3P/uclient/uclient.h | 130 ++ 13 files changed, 3061 insertions(+) create mode 100644 bsp/buildroot_external/package/uclient/Config.in create mode 100644 bsp/buildroot_external/package/uclient/uclient.mk create mode 100644 src/3P/uclient/.gitignore create mode 100644 src/3P/uclient/CMakeLists.txt create mode 100644 src/3P/uclient/progress.c create mode 100644 src/3P/uclient/progress.h create mode 100644 src/3P/uclient/uclient-backend.h create mode 100644 src/3P/uclient/uclient-fetch.c create mode 100644 src/3P/uclient/uclient-http.c create mode 100644 src/3P/uclient/uclient-utils.c create mode 100644 src/3P/uclient/uclient-utils.h create mode 100644 src/3P/uclient/uclient.c create mode 100644 src/3P/uclient/uclient.h diff --git a/bsp/buildroot_external/package/uclient/Config.in b/bsp/buildroot_external/package/uclient/Config.in new file mode 100644 index 00000000..349ddd3a --- /dev/null +++ b/bsp/buildroot_external/package/uclient/Config.in @@ -0,0 +1,14 @@ +config BR2_PACKAGE_LIBUBOX + bool "libubox" + depends on !BR2_STATIC_LIBS + select BR2_PACKAGE_JSON_C + help + This library originates from the OpenWrt project to + handle the configuration file infrastructure, but can + also be used for the same purposes in projects other + than OpenWrt. + + http://nbd.name/gitweb.cgi?p=luci2/libubox.git;a=summary + +comment "libubox needs a toolchain w/ dynamic library" + depends on BR2_STATIC_LIBS diff --git a/bsp/buildroot_external/package/uclient/uclient.mk b/bsp/buildroot_external/package/uclient/uclient.mk new file mode 100644 index 00000000..c6bf40d8 --- /dev/null +++ b/bsp/buildroot_external/package/uclient/uclient.mk @@ -0,0 +1,24 @@ +################################################################################ +# +# LIB UBOX +# +################################################################################ + +UCLIENT_VERSION:= 2016-12-09 + +UCLIENT_SITE = $(TOPDIR)/../../src/3P/libubox +UCLIENT_SITE_METHOD = local +UCLIENT_LICENSE = LGPLv2.1, GPLv2, BSD-3c, MIT +UCLIENT_INSTALL_STAGING = YES + +UCLIENT_DEPENDENCIES = json-c + +UCLIENT_CONF = SRC_DIR=$(TOPDIR)/../.. + +UCLIENT_CONF_OPTS +=-DBUILD_LUA=OFF + +UCLIENT_CONF_ENV = $(UCLIENT_CONF) +UCLIENT_MAKE_ENV = $(UCLIENT_CONF) +UCLIENT_CONF_OPTS += -DMODULE_PATH=$(TOPDIR)/../../bsp/cmake-modules -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) + +$(eval $(cmake-package)) diff --git a/src/3P/uclient/.gitignore b/src/3P/uclient/.gitignore new file mode 100644 index 00000000..8902d55c --- /dev/null +++ b/src/3P/uclient/.gitignore @@ -0,0 +1,9 @@ +Makefile +CMakeCache.txt +CMakeFiles +*.cmake +*.a +*.so +*.dylib +install_manifest.txt +uclient-fetch diff --git a/src/3P/uclient/CMakeLists.txt b/src/3P/uclient/CMakeLists.txt new file mode 100644 index 00000000..072afef9 --- /dev/null +++ b/src/3P/uclient/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required(VERSION 2.6) + +INCLUDE(CheckIncludeFiles) + +PROJECT(uclient C) +ADD_DEFINITIONS(-Os -Wall -Werror --std=gnu99 -g3 -Wmissing-declarations) + +SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") + +FIND_PATH(ubox_include_dir libubox/ustream-ssl.h) +INCLUDE_DIRECTORIES(${ubox_include_dir}) + +ADD_LIBRARY(uclient SHARED uclient.c uclient-http.c uclient-utils.c) +TARGET_LINK_LIBRARIES(uclient ubox dl) + +ADD_EXECUTABLE(uclient-fetch uclient-fetch.c progress.c) +TARGET_LINK_LIBRARIES(uclient-fetch uclient) + +INSTALL(FILES uclient.h uclient-utils.h + DESTINATION include/libubox +) +INSTALL(TARGETS uclient uclient-fetch + LIBRARY DESTINATION lib + RUNTIME DESTINATION bin +) diff --git a/src/3P/uclient/progress.c b/src/3P/uclient/progress.c new file mode 100644 index 00000000..f62bc9c1 --- /dev/null +++ b/src/3P/uclient/progress.c @@ -0,0 +1,229 @@ +/* vi: set sw=4 ts=4: */ +/* + * Progress bar code. + */ +/* Original copyright notice which applies to the CONFIG_FEATURE_WGET_STATUSBAR stuff, + * much of which was blatantly stolen from openssh. + */ +/*- + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. BSD Advertising Clause omitted per the July 22, 1999 licensing change + * ftp://ftp.cs.berkeley.edu/pub/4bsd/README.Impt.License.Change + * + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include "progress.h" + +enum { + /* Seconds when xfer considered "stalled" */ + STALLTIME = 5 +}; + +static int +wh_helper(int value, int def_val, const char *env_name) +{ + if (value == 0) { + char *s = getenv(env_name); + if (s) + value = atoi(s); + } + if (value <= 1 || value >= 30000) + value = def_val; + return value; +} + +static unsigned int +get_tty2_width(void) +{ +#ifndef TIOCGWINSZ + return wh_helper(0, 80, "COLUMNS"); +#else + struct winsize win = {0}; + ioctl(2, TIOCGWINSZ, &win); + return wh_helper(win.ws_col, 80, "COLUMNS"); +#endif +} + +static unsigned int +monotonic_sec(void) +{ + struct timespec tv; + clock_gettime(CLOCK_MONOTONIC, &tv); + return tv.tv_sec; +} + +void +progress_init(struct progress *p, const char *curfile) +{ + p->curfile = strdup(curfile); + p->start_sec = monotonic_sec(); + p->last_update_sec = p->start_sec; + p->last_change_sec = p->start_sec; + p->last_size = 0; +} + +/* File already had beg_size bytes. + * Then we started downloading. + * We downloaded "transferred" bytes so far. + * Download is expected to stop when total size (beg_size + transferred) + * will be "totalsize" bytes. + * If totalsize == 0, then it is unknown. + */ +void +progress_update(struct progress *p, off_t beg_size, + off_t transferred, off_t totalsize) +{ + off_t beg_and_transferred; + unsigned since_last_update, elapsed; + int barlength; + int kiloscale; + + //transferred = 1234; /* use for stall detection testing */ + //totalsize = 0; /* use for unknown size download testing */ + + elapsed = monotonic_sec(); + since_last_update = elapsed - p->last_update_sec; + p->last_update_sec = elapsed; + + if (totalsize != 0 && transferred >= totalsize - beg_size) { + /* Last call. Do not skip this update */ + transferred = totalsize - beg_size; /* sanitize just in case */ + } + else if (since_last_update == 0) { + /* + * Do not update on every call + * (we can be called on every network read!) + */ + return; + } + + kiloscale = 0; + /* + * Scale sizes down if they are close to overflowing. + * This allows calculations like (100 * transferred / totalsize) + * without risking overflow: we guarantee 10 highest bits to be 0. + * Introduced error is less than 1 / 2^12 ~= 0.025% + */ + if (ULONG_MAX > 0xffffffff || sizeof(off_t) == 4 || sizeof(off_t) != 8) { + /* + * 64-bit CPU || small off_t: in either case, + * >> is cheap, single-word operation. + * ... || strange off_t: also use this code + * (it is safe, just suboptimal wrt code size), + * because 32/64 optimized one works only for 64-bit off_t. + */ + if (totalsize >= (1 << 22)) { + totalsize >>= 10; + beg_size >>= 10; + transferred >>= 10; + kiloscale = 1; + } + } else { + /* 32-bit CPU and 64-bit off_t. + * Use a 40-bit shift, it is easier to do on 32-bit CPU. + */ +/* ONE suppresses "warning: shift count >= width of type" */ +#define ONE (sizeof(off_t) > 4) + if (totalsize >= (off_t)(1ULL << 54*ONE)) { + totalsize = (uint32_t)(totalsize >> 32*ONE) >> 8; + beg_size = (uint32_t)(beg_size >> 32*ONE) >> 8; + transferred = (uint32_t)(transferred >> 32*ONE) >> 8; + kiloscale = 4; + } + } + + fprintf(stderr, "\r%-20.20s", p->curfile); + + beg_and_transferred = beg_size + transferred; + + if (totalsize != 0) { + unsigned ratio = 100 * beg_and_transferred / totalsize; + fprintf(stderr, "%4u%%", ratio); + + barlength = get_tty2_width() - 49; + if (barlength > 0) { + /* god bless gcc for variable arrays :) */ + char buf[barlength + 1]; + unsigned stars = (unsigned)barlength * beg_and_transferred / totalsize; + memset(buf, ' ', barlength); + buf[barlength] = '\0'; + memset(buf, '*', stars); + fprintf(stderr, " |%s|", buf); + } + } + + while (beg_and_transferred >= 100000) { + beg_and_transferred >>= 10; + kiloscale++; + } + /* see http://en.wikipedia.org/wiki/Tera */ + fprintf(stderr, "%6u%c", (unsigned)beg_and_transferred, " kMGTPEZY"[kiloscale]); + + since_last_update = elapsed - p->last_change_sec; + if ((unsigned)transferred != p->last_size) { + p->last_change_sec = elapsed; + p->last_size = (unsigned)transferred; + if (since_last_update >= STALLTIME) { + /* We "cut out" these seconds from elapsed time + * by adjusting start time */ + p->start_sec += since_last_update; + } + since_last_update = 0; /* we are un-stalled now */ + } + + elapsed -= p->start_sec; /* now it's "elapsed since start" */ + + if (since_last_update >= STALLTIME) { + fprintf(stderr, " - stalled -"); + } else if (!totalsize || !transferred || (int)elapsed < 0) { + fprintf(stderr, " --:--:-- ETA"); + } else { + unsigned eta, secs, hours; + + totalsize -= beg_size; /* now it's "total to upload" */ + + /* Estimated remaining time = + * estimated_sec_to_dl_totalsize_bytes - elapsed_sec = + * totalsize / average_bytes_sec_so_far - elapsed = + * totalsize / (transferred/elapsed) - elapsed = + * totalsize * elapsed / transferred - elapsed + */ + eta = totalsize * elapsed / transferred - elapsed; + if (eta >= 1000*60*60) + eta = 1000*60*60 - 1; + secs = eta % 3600; + hours = eta / 3600; + fprintf(stderr, "%3u:%02u:%02u ETA", hours, secs / 60, secs % 60); + } +} diff --git a/src/3P/uclient/progress.h b/src/3P/uclient/progress.h new file mode 100644 index 00000000..720060d8 --- /dev/null +++ b/src/3P/uclient/progress.h @@ -0,0 +1,25 @@ +#ifndef __PROGRESS_H +#define __PROGRESS_H + +#include + +struct progress { + unsigned int last_size; + unsigned int last_update_sec; + unsigned int last_change_sec; + unsigned int start_sec; + char *curfile; +}; + + +void progress_init(struct progress *p, const char *curfile); +void progress_update(struct progress *p, off_t beg_size, + off_t transferred, off_t totalsize); + +static inline void +progress_free(struct progress *p) +{ + free(p->curfile); +} + +#endif diff --git a/src/3P/uclient/uclient-backend.h b/src/3P/uclient/uclient-backend.h new file mode 100644 index 00000000..c2b9fd53 --- /dev/null +++ b/src/3P/uclient/uclient-backend.h @@ -0,0 +1,45 @@ +/* + * uclient - ustream based protocol client library + * + * Copyright (C) 2014 Felix Fietkau + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef __UCLIENT_INTERNAL_H +#define __UCLIENT_INTERNAL_H + +struct uclient_url; + +struct uclient_backend { + const char * const * prefix; + + struct uclient *(*alloc)(void); + void (*free)(struct uclient *cl); + void (*update_proxy_url)(struct uclient *cl); + void (*update_url)(struct uclient *cl); + + int (*connect)(struct uclient *cl); + int (*request)(struct uclient *cl); + void (*disconnect)(struct uclient *cl); + + int (*read)(struct uclient *cl, char *buf, unsigned int len); + int (*write)(struct uclient *cl, const char *buf, unsigned int len); +}; + +void uclient_backend_set_error(struct uclient *cl, int code); +void uclient_backend_set_eof(struct uclient *cl); +void uclient_backend_reset_state(struct uclient *cl); +struct uclient_url *uclient_get_url(const char *url_str, const char *auth_str); +struct uclient_url *uclient_get_url_location(struct uclient_url *url, const char *location); + +#endif diff --git a/src/3P/uclient/uclient-fetch.c b/src/3P/uclient/uclient-fetch.c new file mode 100644 index 00000000..d9582f55 --- /dev/null +++ b/src/3P/uclient/uclient-fetch.c @@ -0,0 +1,717 @@ +/* + * uclient - ustream based protocol client library + * + * Copyright (C) 2014 Felix Fietkau + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "progress.h" +#include "uclient.h" +#include "uclient-utils.h" + +#ifdef __APPLE__ +#define LIB_EXT "dylib" +#else +#define LIB_EXT "so" +#endif + +static const char *user_agent = "uclient-fetch"; +static const char *post_data; +static struct ustream_ssl_ctx *ssl_ctx; +static const struct ustream_ssl_ops *ssl_ops; +static int quiet = false; +static bool verify = true; +static bool proxy = true; +static bool default_certs = false; +static bool no_output; +static const char *output_file; +static int output_fd = -1; +static int error_ret; +static off_t out_offset; +static off_t out_bytes; +static off_t out_len; +static char *auth_str; +static char **urls; +static int n_urls; +static int timeout; +static bool resume, cur_resume; + +static struct progress pmt; +static struct uloop_timeout pmt_timer; + +static int init_request(struct uclient *cl); +static void request_done(struct uclient *cl); + +static void pmt_update(struct uloop_timeout *t) +{ + progress_update(&pmt, out_offset, out_bytes, out_len); + uloop_timeout_set(t, 1000); +} + +static const char * +get_proxy_url(char *url) +{ + char prefix[16]; + char *sep; + + if (!proxy) + return NULL; + + sep = strchr(url, ':'); + if (!sep) + return NULL; + + if (sep - url > 5) + return NULL; + + memcpy(prefix, url, sep - url); + strcpy(prefix + (sep - url), "_proxy"); + return getenv(prefix); +} + +static int open_output_file(const char *path, uint64_t resume_offset) +{ + char *filename = NULL; + int flags; + int ret; + + if (cur_resume) + flags = O_RDWR; + else + flags = O_WRONLY | O_TRUNC; + + if (!cur_resume && !output_file) + flags |= O_EXCL; + + flags |= O_CREAT; + + if (output_file) { + if (!strcmp(output_file, "-")) { + if (!quiet) + fprintf(stderr, "Writing to stdout\n"); + + ret = STDOUT_FILENO; + goto done; + } + } else { + filename = uclient_get_url_filename(path, "index.html"); + output_file = filename; + } + + if (!quiet) + fprintf(stderr, "Writing to '%s'\n", output_file); + ret = open(output_file, flags, 0644); + if (ret < 0) + goto free; + + if (resume_offset && + lseek(ret, resume_offset, SEEK_SET) < 0) { + if (!quiet) + fprintf(stderr, "Failed to seek %"PRIu64" bytes in output file\n", resume_offset); + close(ret); + ret = -1; + goto free; + } + + out_offset = resume_offset; + out_bytes += resume_offset; +done: + if (!quiet) { + progress_init(&pmt, output_file); + pmt_timer.cb = pmt_update; + pmt_timer.cb(&pmt_timer); + } + +free: + free(filename); + return ret; +} + +static void header_done_cb(struct uclient *cl) +{ + enum { + H_RANGE, + H_LEN, + __H_MAX + }; + static const struct blobmsg_policy policy[__H_MAX] = { + [H_RANGE] = { .name = "content-range", .type = BLOBMSG_TYPE_STRING }, + [H_LEN] = { .name = "content-length", .type = BLOBMSG_TYPE_STRING }, + }; + struct blob_attr *tb[__H_MAX]; + uint64_t resume_offset = 0, resume_end, resume_size; + static int retries; + + if (retries < 10) { + int ret = uclient_http_redirect(cl); + if (ret < 0) { + if (!quiet) + fprintf(stderr, "Failed to redirect to %s on %s\n", cl->url->location, cl->url->host); + error_ret = 8; + request_done(cl); + return; + } + if (ret > 0) { + if (!quiet) + fprintf(stderr, "Redirected to %s on %s\n", cl->url->location, cl->url->host); + + retries++; + return; + } + } + + if (cl->status_code == 204 && cur_resume) { + /* Resume attempt failed, try normal download */ + cur_resume = false; + init_request(cl); + return; + } + + blobmsg_parse(policy, __H_MAX, tb, blob_data(cl->meta), blob_len(cl->meta)); + + switch (cl->status_code) { + case 416: + if (!quiet) + fprintf(stderr, "File download already fully retrieved; nothing to do.\n"); + request_done(cl); + break; + case 206: + if (!cur_resume) { + if (!quiet) + fprintf(stderr, "Error: Partial content received, full content requested\n"); + error_ret = 8; + request_done(cl); + break; + } + + if (!tb[H_RANGE]) { + if (!quiet) + fprintf(stderr, "Content-Range header is missing\n"); + error_ret = 8; + break; + } + + if (sscanf(blobmsg_get_string(tb[H_RANGE]), + "bytes %"PRIu64"-%"PRIu64"/%"PRIu64, + &resume_offset, &resume_end, &resume_size) != 3) { + if (!quiet) + fprintf(stderr, "Content-Range header is invalid\n"); + error_ret = 8; + break; + } + case 204: + case 200: + if (no_output) + break; + + if (tb[H_LEN]) + out_len = strtoul(blobmsg_get_string(tb[H_LEN]), NULL, 10); + + output_fd = open_output_file(cl->url->location, resume_offset); + if (output_fd < 0) { + if (!quiet) + perror("Cannot open output file"); + error_ret = 3; + request_done(cl); + } + break; + + default: + if (!quiet) + fprintf(stderr, "HTTP error %d\n", cl->status_code); + request_done(cl); + error_ret = 8; + break; + } +} + +static void read_data_cb(struct uclient *cl) +{ + char buf[256]; + ssize_t n; + int len; + + if (!no_output && output_fd < 0) + return; + + while (1) { + len = uclient_read(cl, buf, sizeof(buf)); + if (!len) + return; + + out_bytes += len; + if (!no_output) { + n = write(output_fd, buf, len); + if (n < 0) + return; + } + } +} + +static void msg_connecting(struct uclient *cl) +{ + char addr[INET6_ADDRSTRLEN]; + int port; + + if (quiet) + return; + + uclient_get_addr(addr, &port, &cl->remote_addr); + fprintf(stderr, "Connecting to %s:%d\n", addr, port); +} + +static void check_resume_offset(struct uclient *cl) +{ + char range_str[64]; + struct stat st; + char *file; + int ret; + + file = uclient_get_url_filename(cl->url->location, "index.html"); + if (!file) + return; + + ret = stat(file, &st); + free(file); + if (ret) + return; + + if (!st.st_size) + return; + + snprintf(range_str, sizeof(range_str), "bytes=%"PRIu64"-", (uint64_t) st.st_size); + uclient_http_set_header(cl, "Range", range_str); +} + +static int init_request(struct uclient *cl) +{ + int rc; + + out_offset = 0; + out_bytes = 0; + out_len = 0; + uclient_http_set_ssl_ctx(cl, ssl_ops, ssl_ctx, verify); + + if (timeout) + cl->timeout_msecs = timeout * 1000; + + rc = uclient_connect(cl); + if (rc) + return rc; + + msg_connecting(cl); + + rc = uclient_http_set_request_type(cl, post_data ? "POST" : "GET"); + if (rc) + return rc; + + uclient_http_reset_headers(cl); + uclient_http_set_header(cl, "User-Agent", user_agent); + if (cur_resume) + check_resume_offset(cl); + + if (post_data) { + uclient_http_set_header(cl, "Content-Type", "application/x-www-form-urlencoded"); + uclient_write(cl, post_data, strlen(post_data)); + } + + rc = uclient_request(cl); + if (rc) + return rc; + + return 0; +} + +static void request_done(struct uclient *cl) +{ + const char *proxy_url; + + if (n_urls) { + proxy_url = get_proxy_url(*urls); + if (proxy_url) { + uclient_set_url(cl, proxy_url, NULL); + uclient_set_proxy_url(cl, *urls, auth_str); + } else { + uclient_set_url(cl, *urls, auth_str); + } + n_urls--; + cur_resume = resume; + error_ret = init_request(cl); + if (error_ret == 0) + return; + } + + if (output_fd >= 0 && !output_file) { + close(output_fd); + output_fd = -1; + } + uclient_disconnect(cl); + uloop_end(); +} + + +static void eof_cb(struct uclient *cl) +{ + if (!quiet) { + pmt_update(&pmt_timer); + uloop_timeout_cancel(&pmt_timer); + fprintf(stderr, "\n"); + } + + if (!cl->data_eof) { + if (!quiet) + fprintf(stderr, "Connection reset prematurely\n"); + error_ret = 4; + } else if (!quiet) { + fprintf(stderr, "Download completed (%"PRIu64" bytes)\n", (uint64_t) out_bytes); + } + request_done(cl); +} + +static void handle_uclient_error(struct uclient *cl, int code) +{ + const char *type = "Unknown error"; + bool ignore = false; + + switch(code) { + case UCLIENT_ERROR_CONNECT: + type = "Connection failed"; + error_ret = 4; + break; + case UCLIENT_ERROR_TIMEDOUT: + type = "Connection timed out"; + error_ret = 4; + break; + case UCLIENT_ERROR_SSL_INVALID_CERT: + type = "Invalid SSL certificate"; + ignore = !verify; + error_ret = 5; + break; + case UCLIENT_ERROR_SSL_CN_MISMATCH: + type = "Server hostname does not match SSL certificate"; + ignore = !verify; + error_ret = 5; + break; + default: + error_ret = 1; + break; + } + + if (!quiet) + fprintf(stderr, "Connection error: %s%s\n", type, ignore ? " (ignored)" : ""); + + if (ignore) + error_ret = 0; + else + request_done(cl); +} + +static const struct uclient_cb cb = { + .header_done = header_done_cb, + .data_read = read_data_cb, + .data_eof = eof_cb, + .error = handle_uclient_error, +}; + +static int usage(const char *progname) +{ + fprintf(stderr, + "Usage: %s [options] \n" + "Options:\n" + " -4 Use IPv4 only\n" + " -6 Use IPv6 only\n" + " -q Turn off status messages\n" + " -O Redirect output to file (use \"-\" for stdout)\n" + " -P Set directory for output files\n" + " --user= HTTP authentication username\n" + " --password= HTTP authentication password\n" + " --user-agent|-U Set HTTP user agent\n" + " --post-data=STRING use the POST method; send STRING as the data\n" + " --spider|-s Spider mode - only check file existence\n" + " --timeout=N|-T N Set connect/request timeout to N seconds\n" + " --proxy=on|off|-Y on|off Enable/disable env var configured proxy\n" + "\n" + "HTTPS options:\n" + " --ca-certificate= Load CA certificates from file \n" + " --no-check-certificate don't validate the server's certificate\n" + "\n", progname); + return 1; +} + +static void init_ca_cert(void) +{ + glob_t gl; + int i; + + glob("/etc/ssl/certs/*.crt", 0, NULL, &gl); + for (i = 0; i < gl.gl_pathc; i++) + ssl_ops->context_add_ca_crt_file(ssl_ctx, gl.gl_pathv[i]); +} + +static void init_ustream_ssl(void) +{ + void *dlh; + + dlh = dlopen("libustream-ssl." LIB_EXT, RTLD_LAZY | RTLD_LOCAL); + if (!dlh) + return; + + ssl_ops = dlsym(dlh, "ustream_ssl_ops"); + if (!ssl_ops) + return; + + ssl_ctx = ssl_ops->context_new(false); +} + +static int no_ssl(const char *progname) +{ + fprintf(stderr, + "%s: SSL support not available, please install one of the " + "libustream-ssl-* libraries as well as the ca-bundle and " + "ca-certificates packages.\n", + progname); + + return 1; +} + +enum { + L_NO_CHECK_CERTIFICATE, + L_CA_CERTIFICATE, + L_USER, + L_PASSWORD, + L_USER_AGENT, + L_POST_DATA, + L_SPIDER, + L_TIMEOUT, + L_CONTINUE, + L_PROXY, + L_NO_PROXY, + L_QUIET, +}; + +static const struct option longopts[] = { + [L_NO_CHECK_CERTIFICATE] = { "no-check-certificate", no_argument }, + [L_CA_CERTIFICATE] = { "ca-certificate", required_argument }, + [L_USER] = { "user", required_argument }, + [L_PASSWORD] = { "password", required_argument }, + [L_USER_AGENT] = { "user-agent", required_argument }, + [L_POST_DATA] = { "post-data", required_argument }, + [L_SPIDER] = { "spider", no_argument }, + [L_TIMEOUT] = { "timeout", required_argument }, + [L_CONTINUE] = { "continue", no_argument }, + [L_PROXY] = { "proxy", required_argument }, + [L_NO_PROXY] = { "no-proxy", no_argument }, + [L_QUIET] = { "quiet", no_argument }, + {} +}; + + + +int main(int argc, char **argv) +{ + const char *progname = argv[0]; + const char *proxy_url; + char *username = NULL; + char *password = NULL; + struct uclient *cl; + int longopt_idx = 0; + bool has_cert = false; + int i, ch; + int rc; + int af = -1; + + signal(SIGPIPE, SIG_IGN); + init_ustream_ssl(); + + while ((ch = getopt_long(argc, argv, "46cO:P:qsT:U:Y:", longopts, &longopt_idx)) != -1) { + switch(ch) { + case 0: + switch (longopt_idx) { + case L_NO_CHECK_CERTIFICATE: + verify = false; + break; + case L_CA_CERTIFICATE: + has_cert = true; + if (ssl_ctx) + ssl_ops->context_add_ca_crt_file(ssl_ctx, optarg); + break; + case L_USER: + if (!strlen(optarg)) + break; + username = strdup(optarg); + memset(optarg, '*', strlen(optarg)); + break; + case L_PASSWORD: + if (!strlen(optarg)) + break; + password = strdup(optarg); + memset(optarg, '*', strlen(optarg)); + break; + case L_USER_AGENT: + user_agent = optarg; + break; + case L_POST_DATA: + post_data = optarg; + break; + case L_SPIDER: + no_output = true; + break; + case L_TIMEOUT: + timeout = atoi(optarg); + break; + case L_CONTINUE: + resume = true; + break; + case L_PROXY: + if (strcmp(optarg, "on") != 0) + proxy = false; + break; + case L_NO_PROXY: + proxy = false; + break; + case L_QUIET: + quiet = true; + break; + default: + return usage(progname); + } + break; + case '4': + af = AF_INET; + break; + case '6': + af = AF_INET6; + break; + case 'c': + resume = true; + break; + case 'U': + user_agent = optarg; + break; + case 'O': + output_file = optarg; + break; + case 'P': + if (chdir(optarg)) { + if (!quiet) + perror("Change output directory"); + exit(1); + } + break; + case 'q': + quiet = true; + break; + case 's': + no_output = true; + break; + case 'T': + timeout = atoi(optarg); + break; + case 'Y': + if (strcmp(optarg, "on") != 0) + proxy = false; + break; + default: + return usage(progname); + } + } + + argv += optind; + argc -= optind; + + if (verify && !has_cert) + default_certs = true; + + if (argc < 1) + return usage(progname); + + if (!ssl_ctx) { + for (i = 0; i < argc; i++) { + if (!strncmp(argv[i], "https", 5)) + return no_ssl(progname); + } + } + + urls = argv + 1; + n_urls = argc - 1; + + uloop_init(); + + if (username) { + if (password) { + rc = asprintf(&auth_str, "%s:%s", username, password); + if (rc < 0) + return rc; + } else + auth_str = username; + } + + if (!quiet) + fprintf(stderr, "Downloading '%s'\n", argv[0]); + + proxy_url = get_proxy_url(argv[0]); + if (proxy_url) { + cl = uclient_new(proxy_url, auth_str, &cb); + if (cl) + uclient_set_proxy_url(cl, argv[0], NULL); + } else { + cl = uclient_new(argv[0], auth_str, &cb); + } + if (!cl) { + fprintf(stderr, "Failed to allocate uclient context\n"); + return 1; + } + if (af >= 0) + uclient_http_set_address_family(cl, af); + + if (ssl_ctx && default_certs) + init_ca_cert(); + + cur_resume = resume; + rc = init_request(cl); + if (!rc) { + /* no error received, we can enter main loop */ + uloop_run(); + } else { + fprintf(stderr, "Failed to establish connection\n"); + error_ret = 4; + } + + uloop_done(); + + uclient_free(cl); + + if (output_fd >= 0 && output_fd != STDOUT_FILENO) + close(output_fd); + + if (ssl_ctx) + ssl_ops->context_free(ssl_ctx); + + return error_ret; +} diff --git a/src/3P/uclient/uclient-http.c b/src/3P/uclient/uclient-http.c new file mode 100644 index 00000000..ac9d50f4 --- /dev/null +++ b/src/3P/uclient/uclient-http.c @@ -0,0 +1,1189 @@ +/* + * uclient - ustream based protocol client library + * + * Copyright (C) 2014 Felix Fietkau + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "uclient.h" +#include "uclient-utils.h" +#include "uclient-backend.h" + +enum auth_type { + AUTH_TYPE_UNKNOWN, + AUTH_TYPE_NONE, + AUTH_TYPE_BASIC, + AUTH_TYPE_DIGEST, +}; + +enum request_type { + REQ_GET, + REQ_HEAD, + REQ_POST, + REQ_PUT, + REQ_DELETE, + __REQ_MAX +}; + +enum http_state { + HTTP_STATE_INIT, + HTTP_STATE_HEADERS_SENT, + HTTP_STATE_REQUEST_DONE, + HTTP_STATE_RECV_HEADERS, + HTTP_STATE_RECV_DATA, + HTTP_STATE_ERROR, +}; + +static const char * const request_types[__REQ_MAX] = { + [REQ_GET] = "GET", + [REQ_HEAD] = "HEAD", + [REQ_POST] = "POST", + [REQ_PUT] = "PUT", + [REQ_DELETE] = "DELETE", +}; + +struct uclient_http { + struct uclient uc; + + const struct ustream_ssl_ops *ssl_ops; + struct ustream_ssl_ctx *ssl_ctx; + struct ustream *us; + + struct ustream_fd ufd; + struct ustream_ssl ussl; + + struct uloop_timeout disconnect_t; + unsigned int seq; + + bool ssl_require_validation; + bool ssl; + bool eof; + bool connection_close; + bool disconnect; + enum request_type req_type; + enum http_state state; + + enum auth_type auth_type; + char *auth_str; + + long read_chunked; + long content_length; + + int usock_flags; + + uint32_t nc; + + struct blob_buf headers; + struct blob_buf meta; +}; + +enum { + PREFIX_HTTP, + PREFIX_HTTPS, + __PREFIX_MAX, +}; + +static const char * const uclient_http_prefix[] = { + [PREFIX_HTTP] = "http://", + [PREFIX_HTTPS] = "https://", + [__PREFIX_MAX] = NULL +}; + +static int uclient_http_connect(struct uclient *cl); + +static int uclient_do_connect(struct uclient_http *uh, const char *port) +{ + socklen_t sl; + int fd; + + if (uh->uc.url->port) + port = uh->uc.url->port; + + memset(&uh->uc.remote_addr, 0, sizeof(uh->uc.remote_addr)); + + fd = usock_inet_timeout(USOCK_TCP | USOCK_NONBLOCK | uh->usock_flags, + uh->uc.url->host, port, &uh->uc.remote_addr, + uh->uc.timeout_msecs); + if (fd < 0) + return -1; + + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK); + ustream_fd_init(&uh->ufd, fd); + + sl = sizeof(uh->uc.local_addr); + memset(&uh->uc.local_addr, 0, sl); + getsockname(fd, &uh->uc.local_addr.sa, &sl); + + return 0; +} + +static void uclient_http_disconnect(struct uclient_http *uh) +{ + uloop_timeout_cancel(&uh->disconnect_t); + if (!uh->us) + return; + + if (uh->ssl) + ustream_free(&uh->ussl.stream); + ustream_free(&uh->ufd.stream); + close(uh->ufd.fd.fd); + uh->us = NULL; +} + +static void uclient_http_free_url_state(struct uclient *cl) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + + uh->auth_type = AUTH_TYPE_UNKNOWN; + free(uh->auth_str); + uh->auth_str = NULL; + uclient_http_disconnect(uh); +} + +static void uclient_http_error(struct uclient_http *uh, int code) +{ + uh->state = HTTP_STATE_ERROR; + uh->us->eof = true; + ustream_state_change(uh->us); + uclient_backend_set_error(&uh->uc, code); +} + +static void uclient_http_request_disconnect(struct uclient *cl) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + + if (!uh->us) + return; + + uh->eof = true; + uh->disconnect = true; + uloop_timeout_set(&uh->disconnect_t, 1); +} + +static void uclient_notify_eof(struct uclient_http *uh) +{ + struct ustream *us = uh->us; + + if (uh->disconnect) + return; + + if (!uh->eof) { + if (!us->eof && !us->write_error) + return; + + if (ustream_pending_data(us, false)) + return; + } + + if (uh->content_length < 0 && uh->read_chunked >= 0) + uh->uc.data_eof = true; + + uclient_backend_set_eof(&uh->uc); + + if (uh->connection_close) + uclient_http_request_disconnect(&uh->uc); +} + +static void uclient_http_reset_state(struct uclient_http *uh) +{ + uh->seq++; + uclient_backend_reset_state(&uh->uc); + uh->read_chunked = -1; + uh->content_length = -1; + uh->eof = false; + uh->disconnect = false; + uh->connection_close = false; + uh->state = HTTP_STATE_INIT; + + if (uh->auth_type == AUTH_TYPE_UNKNOWN && !uh->uc.url->auth) + uh->auth_type = AUTH_TYPE_NONE; +} + +static void uclient_http_init_request(struct uclient_http *uh) +{ + uh->seq++; + uclient_http_reset_state(uh); + blob_buf_init(&uh->meta, 0); +} + +static enum auth_type +uclient_http_update_auth_type(struct uclient_http *uh) +{ + if (!uh->auth_str) + return AUTH_TYPE_NONE; + + if (!strncasecmp(uh->auth_str, "basic", 5)) + return AUTH_TYPE_BASIC; + + if (!strncasecmp(uh->auth_str, "digest", 6)) + return AUTH_TYPE_DIGEST; + + return AUTH_TYPE_NONE; +} + +static void uclient_http_process_headers(struct uclient_http *uh) +{ + enum { + HTTP_HDR_TRANSFER_ENCODING, + HTTP_HDR_CONNECTION, + HTTP_HDR_CONTENT_LENGTH, + HTTP_HDR_AUTH, + __HTTP_HDR_MAX, + }; + static const struct blobmsg_policy hdr_policy[__HTTP_HDR_MAX] = { +#define hdr(_name) { .name = _name, .type = BLOBMSG_TYPE_STRING } + [HTTP_HDR_TRANSFER_ENCODING] = hdr("transfer-encoding"), + [HTTP_HDR_CONNECTION] = hdr("connection"), + [HTTP_HDR_CONTENT_LENGTH] = hdr("content-length"), + [HTTP_HDR_AUTH] = hdr("www-authenticate"), +#undef hdr + }; + struct blob_attr *tb[__HTTP_HDR_MAX]; + struct blob_attr *cur; + + blobmsg_parse(hdr_policy, __HTTP_HDR_MAX, tb, blob_data(uh->meta.head), blob_len(uh->meta.head)); + + cur = tb[HTTP_HDR_TRANSFER_ENCODING]; + if (cur && strstr(blobmsg_data(cur), "chunked")) + uh->read_chunked = 0; + + cur = tb[HTTP_HDR_CONNECTION]; + if (cur && strstr(blobmsg_data(cur), "close")) + uh->connection_close = true; + + cur = tb[HTTP_HDR_CONTENT_LENGTH]; + if (cur) + uh->content_length = strtoul(blobmsg_data(cur), NULL, 10); + + cur = tb[HTTP_HDR_AUTH]; + if (cur) { + free(uh->auth_str); + uh->auth_str = strdup(blobmsg_data(cur)); + } + + uh->auth_type = uclient_http_update_auth_type(uh); +} + +static bool uclient_request_supports_body(enum request_type req_type) +{ + switch (req_type) { + case REQ_POST: + case REQ_PUT: + case REQ_DELETE: + return true; + default: + return false; + } +} + +static void +uclient_http_add_auth_basic(struct uclient_http *uh) +{ + struct uclient_url *url = uh->uc.url; + int auth_len = strlen(url->auth); + char *auth_buf; + + if (auth_len > 512) + return; + + auth_buf = alloca(base64_len(auth_len) + 1); + base64_encode(url->auth, auth_len, auth_buf); + ustream_printf(uh->us, "Authorization: Basic %s\r\n", auth_buf); +} + +static char *digest_unquote_sep(char **str) +{ + char *cur = *str + 1; + char *start = cur; + char *out; + + if (**str != '"') + return NULL; + + out = cur; + while (1) { + if (!*cur) + return NULL; + + if (*cur == '"') { + cur++; + break; + } + + if (*cur == '\\') + cur++; + + *(out++) = *(cur++); + } + + if (*cur == ',') + cur++; + + *out = 0; + *str = cur; + + return start; +} + +static char *digest_sep(char **str) +{ + char *cur, *next; + + cur = *str; + next = strchr(*str, ','); + if (next) { + *str = next + 1; + *next = 0; + } else { + *str += strlen(*str); + } + + return cur; +} + +static bool strmatch(char **str, const char *prefix) +{ + int len = strlen(prefix); + + if (strncmp(*str, prefix, len) != 0 || (*str)[len] != '=') + return false; + + *str += len + 1; + return true; +} + +static void +get_cnonce(char *dest) +{ + uint32_t val = 0; + FILE *f; + size_t n; + + f = fopen("/dev/urandom", "r"); + if (f) { + n = fread(&val, sizeof(val), 1, f); + fclose(f); + if (n != 1) + return; + } + + bin_to_hex(dest, &val, sizeof(val)); +} + +static void add_field(char **buf, int *ofs, int *len, const char *name, const char *val) +{ + int available = *len - *ofs; + int required; + const char *next; + char *cur; + + if (*len && !*buf) + return; + + required = strlen(name) + 4 + strlen(val) * 2; + if (required > available) + *len += required - available + 64; + + *buf = realloc(*buf, *len); + if (!*buf) + return; + + cur = *buf + *ofs; + cur += sprintf(cur, ", %s=\"", name); + + while ((next = strchr(val, '"'))) { + if (next > val) { + memcpy(cur, val, next - val); + cur += next - val; + } + + cur += sprintf(cur, "\\\""); + val = next + 1; + } + + cur += sprintf(cur, "%s\"", val); + *ofs = cur - *buf; +} + +static void +uclient_http_add_auth_digest(struct uclient_http *uh) +{ + struct uclient_url *url = uh->uc.url; + const char *realm = NULL, *opaque = NULL; + const char *user, *password; + char *buf, *next; + int len, ofs; + + char cnonce_str[9]; + char nc_str[9]; + char ahash[33]; + char hash[33]; + + struct http_digest_data data = { + .nc = nc_str, + .cnonce = cnonce_str, + .auth_hash = ahash, + }; + + len = strlen(uh->auth_str) + 1; + if (len > 512) + return; + + buf = alloca(len); + strcpy(buf, uh->auth_str); + + /* skip auth type */ + strsep(&buf, " "); + + next = buf; + while (*next) { + const char **dest = NULL; + const char *tmp; + + while (*next && isspace(*next)) + next++; + + if (strmatch(&next, "realm")) + dest = &realm; + else if (strmatch(&next, "qop")) + dest = &data.qop; + else if (strmatch(&next, "nonce")) + dest = &data.nonce; + else if (strmatch(&next, "opaque")) + dest = &opaque; + else if (strmatch(&next, "stale") || + strmatch(&next, "algorithm") || + strmatch(&next, "auth-param")) { + digest_sep(&next); + continue; + } else if (strmatch(&next, "domain") || + strmatch(&next, "qop-options")) + dest = &tmp; + else { + digest_sep(&next); + continue; + } + + *dest = digest_unquote_sep(&next); + } + + if (!realm || !data.qop || !data.nonce) + return; + + sprintf(nc_str, "%08x", uh->nc++); + get_cnonce(cnonce_str); + + data.qop = "auth"; + data.uri = url->location; + data.method = request_types[uh->req_type]; + + password = strchr(url->auth, ':'); + if (password) { + char *user_buf; + + len = password - url->auth; + if (len > 256) + return; + + user_buf = alloca(len + 1); + strncpy(user_buf, url->auth, len); + user_buf[len] = 0; + user = user_buf; + password++; + } else { + user = url->auth; + password = ""; + } + + http_digest_calculate_auth_hash(ahash, user, realm, password); + http_digest_calculate_response(hash, &data); + + buf = NULL; + len = 0; + ofs = 0; + + add_field(&buf, &ofs, &len, "username", user); + add_field(&buf, &ofs, &len, "realm", realm); + add_field(&buf, &ofs, &len, "nonce", data.nonce); + add_field(&buf, &ofs, &len, "uri", data.uri); + add_field(&buf, &ofs, &len, "cnonce", data.cnonce); + add_field(&buf, &ofs, &len, "response", hash); + if (opaque) + add_field(&buf, &ofs, &len, "opaque", opaque); + + ustream_printf(uh->us, "Authorization: Digest nc=%s, qop=%s%s\r\n", data.nc, data.qop, buf); + free(buf); +} + +static void +uclient_http_add_auth_header(struct uclient_http *uh) +{ + if (!uh->uc.url->auth) + return; + + switch (uh->auth_type) { + case AUTH_TYPE_UNKNOWN: + case AUTH_TYPE_NONE: + break; + case AUTH_TYPE_BASIC: + uclient_http_add_auth_basic(uh); + break; + case AUTH_TYPE_DIGEST: + uclient_http_add_auth_digest(uh); + break; + } +} + +static void +uclient_http_send_headers(struct uclient_http *uh) +{ + struct uclient_url *url = uh->uc.url; + struct blob_attr *cur; + enum request_type req_type = uh->req_type; + int rem; + + if (uh->state >= HTTP_STATE_HEADERS_SENT) + return; + + if (uh->uc.proxy_url) + url = uh->uc.proxy_url; + + ustream_printf(uh->us, + "%s %s HTTP/1.1\r\n" + "Host: %s%s%s\r\n", + request_types[req_type], + url->location, url->host, + url->port ? ":" : "", + url->port ? url->port : ""); + + blobmsg_for_each_attr(cur, uh->headers.head, rem) + ustream_printf(uh->us, "%s: %s\r\n", blobmsg_name(cur), (char *) blobmsg_data(cur)); + + if (uclient_request_supports_body(uh->req_type)) + ustream_printf(uh->us, "Transfer-Encoding: chunked\r\n"); + + uclient_http_add_auth_header(uh); + + ustream_printf(uh->us, "\r\n"); + + uh->state = HTTP_STATE_HEADERS_SENT; +} + +static void uclient_http_headers_complete(struct uclient_http *uh) +{ + enum auth_type auth_type = uh->auth_type; + int seq = uh->uc.seq; + + uh->state = HTTP_STATE_RECV_DATA; + uh->uc.meta = uh->meta.head; + uclient_http_process_headers(uh); + + if (auth_type == AUTH_TYPE_UNKNOWN && uh->uc.status_code == 401 && + (uh->req_type == REQ_HEAD || uh->req_type == REQ_GET)) { + uclient_http_connect(&uh->uc); + uclient_http_send_headers(uh); + uh->state = HTTP_STATE_REQUEST_DONE; + return; + } + + if (uh->uc.cb->header_done) + uh->uc.cb->header_done(&uh->uc); + + if (uh->eof || seq != uh->uc.seq) + return; + + if (uh->req_type == REQ_HEAD || uh->uc.status_code == 204) { + uh->eof = true; + uclient_notify_eof(uh); + } +} + +static void uclient_parse_http_line(struct uclient_http *uh, char *data) +{ + char *name; + char *sep; + + if (uh->state == HTTP_STATE_REQUEST_DONE) { + char *code; + + if (!strlen(data)) + return; + + /* HTTP/1.1 */ + strsep(&data, " "); + + code = strsep(&data, " "); + if (!code) + goto error; + + uh->uc.status_code = strtoul(code, &sep, 10); + if (sep && *sep) + goto error; + + uh->state = HTTP_STATE_RECV_HEADERS; + return; + } + + if (!*data) { + uclient_http_headers_complete(uh); + return; + } + + sep = strchr(data, ':'); + if (!sep) + return; + + *(sep++) = 0; + + for (name = data; *name; name++) + *name = tolower(*name); + + name = data; + while (isspace(*sep)) + sep++; + + blobmsg_add_string(&uh->meta, name, sep); + return; + +error: + uh->uc.status_code = 400; + uh->eof = true; + uclient_notify_eof(uh); +} + +static void __uclient_notify_read(struct uclient_http *uh) +{ + struct uclient *uc = &uh->uc; + unsigned int seq = uh->seq; + char *data; + int len; + + if (uh->state < HTTP_STATE_REQUEST_DONE || uh->state == HTTP_STATE_ERROR) + return; + + data = ustream_get_read_buf(uh->us, &len); + if (!data || !len) + return; + + if (uh->state < HTTP_STATE_RECV_DATA) { + char *sep, *next; + int cur_len; + + do { + sep = strchr(data, '\n'); + if (!sep) + break; + + next = sep + 1; + if (sep > data && sep[-1] == '\r') + sep--; + + /* Check for multi-line HTTP headers */ + if (sep > data) { + if (!*next) + return; + + if (isspace(*next) && *next != '\r' && *next != '\n') { + sep[0] = ' '; + if (sep + 1 < next) + sep[1] = ' '; + continue; + } + } + + *sep = 0; + cur_len = next - data; + uclient_parse_http_line(uh, data); + if (seq != uh->seq) + return; + + ustream_consume(uh->us, cur_len); + len -= cur_len; + + if (uh->eof) + return; + + data = ustream_get_read_buf(uh->us, &len); + } while (data && uh->state < HTTP_STATE_RECV_DATA); + + if (!len) + return; + } + + if (uh->eof) + return; + + if (uh->state == HTTP_STATE_RECV_DATA) { + /* Now it's uclient user turn to read some data */ + uloop_timeout_cancel(&uc->connection_timeout); + + if (uc->cb->data_read) + uc->cb->data_read(uc); + } +} + +static void __uclient_notify_write(struct uclient_http *uh) +{ + struct uclient *uc = &uh->uc; + + if (uc->cb->data_sent) + uc->cb->data_sent(uc); +} + +static void uclient_notify_read(struct ustream *us, int bytes) +{ + struct uclient_http *uh = container_of(us, struct uclient_http, ufd.stream); + + __uclient_notify_read(uh); +} + +static void uclient_notify_write(struct ustream *us, int bytes) +{ + struct uclient_http *uh = container_of(us, struct uclient_http, ufd.stream); + + __uclient_notify_write(uh); +} + +static void uclient_notify_state(struct ustream *us) +{ + struct uclient_http *uh = container_of(us, struct uclient_http, ufd.stream); + + if (uh->ufd.stream.write_error) { + uclient_http_error(uh, UCLIENT_ERROR_CONNECT); + return; + } + uclient_notify_eof(uh); +} + +static int uclient_setup_http(struct uclient_http *uh) +{ + struct ustream *us = &uh->ufd.stream; + int ret; + + uh->us = us; + uh->ssl = false; + + us->string_data = true; + us->notify_state = uclient_notify_state; + us->notify_read = uclient_notify_read; + us->notify_write = uclient_notify_write; + + ret = uclient_do_connect(uh, "80"); + if (ret) + return UCLIENT_ERROR_CONNECT; + + return 0; +} + +static void uclient_ssl_notify_read(struct ustream *us, int bytes) +{ + struct uclient_http *uh = container_of(us, struct uclient_http, ussl.stream); + + __uclient_notify_read(uh); +} + +static void uclient_ssl_notify_write(struct ustream *us, int bytes) +{ + struct uclient_http *uh = container_of(us, struct uclient_http, ussl.stream); + + __uclient_notify_write(uh); +} + +static void uclient_ssl_notify_state(struct ustream *us) +{ + struct uclient_http *uh = container_of(us, struct uclient_http, ussl.stream); + + uclient_notify_eof(uh); +} + +static void uclient_ssl_notify_error(struct ustream_ssl *ssl, int error, const char *str) +{ + struct uclient_http *uh = container_of(ssl, struct uclient_http, ussl); + + uclient_http_error(uh, UCLIENT_ERROR_CONNECT); +} + +static void uclient_ssl_notify_verify_error(struct ustream_ssl *ssl, int error, const char *str) +{ + struct uclient_http *uh = container_of(ssl, struct uclient_http, ussl); + + if (!uh->ssl_require_validation) + return; + + uclient_http_error(uh, UCLIENT_ERROR_SSL_INVALID_CERT); +} + +static void uclient_ssl_notify_connected(struct ustream_ssl *ssl) +{ + struct uclient_http *uh = container_of(ssl, struct uclient_http, ussl); + + if (!uh->ssl_require_validation) + return; + + if (!uh->ussl.valid_cn) + uclient_http_error(uh, UCLIENT_ERROR_SSL_CN_MISMATCH); +} + +static int uclient_setup_https(struct uclient_http *uh) +{ + struct ustream *us = &uh->ussl.stream; + int ret; + + uh->ssl = true; + uh->us = us; + + if (!uh->ssl_ctx) + return UCLIENT_ERROR_MISSING_SSL_CONTEXT; + + ret = uclient_do_connect(uh, "443"); + if (ret) + return UCLIENT_ERROR_CONNECT; + + us->string_data = true; + us->notify_state = uclient_ssl_notify_state; + us->notify_read = uclient_ssl_notify_read; + us->notify_write = uclient_ssl_notify_write; + uh->ussl.notify_error = uclient_ssl_notify_error; + uh->ussl.notify_verify_error = uclient_ssl_notify_verify_error; + uh->ussl.notify_connected = uclient_ssl_notify_connected; + uh->ussl.server_name = uh->uc.url->host; + uh->ssl_ops->init(&uh->ussl, &uh->ufd.stream, uh->ssl_ctx, false); + uh->ssl_ops->set_peer_cn(&uh->ussl, uh->uc.url->host); + + return 0; +} + +static int uclient_http_connect(struct uclient *cl) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + int ret; + + if (!cl->eof || uh->disconnect || uh->connection_close) + uclient_http_disconnect(uh); + + uclient_http_init_request(uh); + + if (uh->us) + return 0; + + uh->ssl = cl->url->prefix == PREFIX_HTTPS; + + if (uh->ssl) + ret = uclient_setup_https(uh); + else + ret = uclient_setup_http(uh); + + return ret; +} + +static void uclient_http_disconnect_cb(struct uloop_timeout *timeout) +{ + struct uclient_http *uh = container_of(timeout, struct uclient_http, disconnect_t); + + uclient_http_disconnect(uh); +} + +static struct uclient *uclient_http_alloc(void) +{ + struct uclient_http *uh; + + uh = calloc_a(sizeof(*uh)); + uh->disconnect_t.cb = uclient_http_disconnect_cb; + blob_buf_init(&uh->headers, 0); + + return &uh->uc; +} + +static void uclient_http_free_ssl_ctx(struct uclient_http *uh) +{ + uh->ssl_ops = NULL; + uh->ssl_ctx = NULL; +} + +static void uclient_http_free(struct uclient *cl) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + + uclient_http_free_url_state(cl); + uclient_http_free_ssl_ctx(uh); + blob_buf_free(&uh->headers); + blob_buf_free(&uh->meta); + free(uh); +} + +int +uclient_http_set_request_type(struct uclient *cl, const char *type) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + int i; + + if (cl->backend != &uclient_backend_http) + return -1; + + if (uh->state > HTTP_STATE_INIT) + return -1; + + for (i = 0; i < ARRAY_SIZE(request_types); i++) { + if (strcmp(request_types[i], type) != 0) + continue; + + uh->req_type = i; + return 0; + } + + return -1; +} + +int +uclient_http_reset_headers(struct uclient *cl) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + + blob_buf_init(&uh->headers, 0); + + return 0; +} + +int +uclient_http_set_header(struct uclient *cl, const char *name, const char *value) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + + if (cl->backend != &uclient_backend_http) + return -1; + + if (uh->state > HTTP_STATE_INIT) + return -1; + + blobmsg_add_string(&uh->headers, name, value); + return 0; +} + +static int +uclient_http_send_data(struct uclient *cl, const char *buf, unsigned int len) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + + if (uh->state >= HTTP_STATE_REQUEST_DONE) + return -1; + + uclient_http_send_headers(uh); + + if (len > 0) { + ustream_printf(uh->us, "%X\r\n", len); + ustream_write(uh->us, buf, len, false); + ustream_printf(uh->us, "\r\n"); + } + + return len; +} + +static int +uclient_http_request_done(struct uclient *cl) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + + if (uh->state >= HTTP_STATE_REQUEST_DONE) + return -1; + + uclient_http_send_headers(uh); + if (uclient_request_supports_body(uh->req_type)) + ustream_printf(uh->us, "0\r\n\r\n"); + uh->state = HTTP_STATE_REQUEST_DONE; + + return 0; +} + +static int +uclient_http_read(struct uclient *cl, char *buf, unsigned int len) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + int read_len = 0; + char *data, *data_end; + + if (uh->state < HTTP_STATE_RECV_DATA || !uh->us) + return 0; + + data = ustream_get_read_buf(uh->us, &read_len); + if (!data || !read_len) + return 0; + + data_end = data + read_len; + read_len = 0; + + if (uh->read_chunked == 0) { + char *sep; + + if (data[0] == '\r' && data[1] == '\n') { + data += 2; + read_len += 2; + } + + sep = strstr(data, "\r\n"); + if (!sep) + return 0; + + *sep = 0; + uh->read_chunked = strtoul(data, NULL, 16); + + read_len += sep + 2 - data; + data = sep + 2; + + if (!uh->read_chunked) { + uh->eof = true; + uh->uc.data_eof = true; + } + } + + if (len > data_end - data) + len = data_end - data; + + if (uh->read_chunked >= 0) { + if (len > uh->read_chunked) + len = uh->read_chunked; + + uh->read_chunked -= len; + } else if (uh->content_length >= 0) { + if (len > uh->content_length) + len = uh->content_length; + + uh->content_length -= len; + if (!uh->content_length) { + uh->eof = true; + uh->uc.data_eof = true; + } + } + + if (len > 0) { + read_len += len; + memcpy(buf, data, len); + } + + if (read_len > 0) + ustream_consume(uh->us, read_len); + + uclient_notify_eof(uh); + + /* Now that we consumed something and if this isn't EOF, start timer again */ + if (!uh->uc.eof && !cl->connection_timeout.pending) + uloop_timeout_set(&cl->connection_timeout, cl->timeout_msecs); + + return len; +} + +int uclient_http_redirect(struct uclient *cl) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + struct blobmsg_policy location = { + .name = "location", + .type = BLOBMSG_TYPE_STRING, + }; + struct uclient_url *url = cl->url; + struct blob_attr *tb; + + if (cl->backend != &uclient_backend_http) + return false; + + switch (cl->status_code) { + case 301: + case 302: + case 307: + break; + default: + return false; + } + + blobmsg_parse(&location, 1, &tb, blob_data(uh->meta.head), blob_len(uh->meta.head)); + if (!tb) + return false; + + url = uclient_get_url_location(url, blobmsg_data(tb)); + if (!url) + return false; + + free(cl->url); + cl->url = url; + if (uclient_http_connect(cl)) + return -1; + + uclient_http_request_done(cl); + + return true; +} + +int uclient_http_set_ssl_ctx(struct uclient *cl, const struct ustream_ssl_ops *ops, + struct ustream_ssl_ctx *ctx, bool require_validation) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + + if (cl->backend != &uclient_backend_http) + return -1; + + uclient_http_free_url_state(cl); + + uclient_http_free_ssl_ctx(uh); + uh->ssl_ops = ops; + uh->ssl_ctx = ctx; + uh->ssl_require_validation = !!ctx && require_validation; + + return 0; +} + +int uclient_http_set_address_family(struct uclient *cl, int af) +{ + struct uclient_http *uh = container_of(cl, struct uclient_http, uc); + + if (cl->backend != &uclient_backend_http) + return -1; + + switch (af) { + case AF_INET: + uh->usock_flags = USOCK_IPV4ONLY; + break; + case AF_INET6: + uh->usock_flags = USOCK_IPV6ONLY; + break; + default: + uh->usock_flags = 0; + break; + } + + return 0; +} + +const struct uclient_backend uclient_backend_http = { + .prefix = uclient_http_prefix, + + .alloc = uclient_http_alloc, + .free = uclient_http_free, + .connect = uclient_http_connect, + .disconnect = uclient_http_request_disconnect, + .update_url = uclient_http_free_url_state, + .update_proxy_url = uclient_http_free_url_state, + + .read = uclient_http_read, + .write = uclient_http_send_data, + .request = uclient_http_request_done, +}; diff --git a/src/3P/uclient/uclient-utils.c b/src/3P/uclient/uclient-utils.c new file mode 100644 index 00000000..a375eeac --- /dev/null +++ b/src/3P/uclient/uclient-utils.c @@ -0,0 +1,184 @@ +/* + * uclient - ustream based protocol client library + * + * Copyright (C) 2014 Felix Fietkau + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include +#include +#include +#include + +#include +#include + +#include "uclient-utils.h" + +static const char *b64 = +"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +void base64_encode(const void *inbuf, unsigned int len, void *outbuf) +{ + unsigned char *out = outbuf; + const uint8_t *in = inbuf; + unsigned int i; + int pad = len % 3; + + for (i = 0; i < len - pad; i += 3) { + uint32_t in3 = (in[0] << 16) | (in[1] << 8) | in[2]; + int k; + + for (k = 3; k >= 0; k--) { + out[k] = b64[in3 & 0x3f]; + in3 >>= 6; + } + in += 3; + out += 4; + } + + if (pad) { + uint32_t in2 = in[0] << (16 - 6); + + out[3] = '='; + + if (pad > 1) { + in2 |= in[1] << (8 - 6); + out[2] = b64[in2 & 0x3f]; + } else { + out[2] = '='; + } + + in2 >>= 6; + out[1] = b64[in2 & 0x3f]; + in2 >>= 6; + out[0] = b64[in2 & 0x3f]; + + out += 4; + } + + *out = '\0'; +} + +int uclient_urldecode(const char *in, char *out, bool decode_plus) +{ + static char dec[3]; + int ret = 0; + char c; + + while ((c = *(in++))) { + if (c == '%') { + if (!isxdigit(in[0]) || !isxdigit(in[1])) + return -1; + + dec[0] = in[0]; + dec[1] = in[1]; + c = strtol(dec, NULL, 16); + in += 2; + } else if (decode_plus && c == '+') { + c = ' '; + } + + *(out++) = c; + ret++; + } + + *out = 0; + return ret; +} + +static char hex_digit(char val) +{ + val += val > 9 ? 'a' - 10 : '0'; + return val; +} + +void bin_to_hex(char *dest, const void *buf, int len) +{ + const uint8_t *data = buf; + int i; + + for (i = 0; i < len; i++) { + *(dest++) = hex_digit(data[i] >> 4); + *(dest++) = hex_digit(data[i] & 0xf); + } + *dest = 0; +} + +static void http_create_hash(char *dest, const char * const * str, int n_str) +{ + uint32_t hash[4]; + md5_ctx_t md5; + int i; + + md5_begin(&md5); + for (i = 0; i < n_str; i++) { + if (i) + md5_hash(":", 1, &md5); + md5_hash(str[i], strlen(str[i]), &md5); + } + md5_end(hash, &md5); + bin_to_hex(dest, &hash, sizeof(hash)); +} + +void http_digest_calculate_auth_hash(char *dest, const char *user, const char *realm, const char *password) +{ + const char *hash_str[] = { + user, + realm, + password + }; + + http_create_hash(dest, hash_str, ARRAY_SIZE(hash_str)); +} + +void http_digest_calculate_response(char *dest, const struct http_digest_data *data) +{ + const char *h_a2_strings[] = { + data->method, + data->uri, + }; + const char *resp_strings[] = { + data->auth_hash, + data->nonce, + data->nc, + data->cnonce, + data->qop, + dest, /* initialized to H(A2) first */ + }; + + http_create_hash(dest, h_a2_strings, ARRAY_SIZE(h_a2_strings)); + http_create_hash(dest, resp_strings, ARRAY_SIZE(resp_strings)); +} + +char *uclient_get_url_filename(const char *url, const char *default_name) +{ + const char *str; + int len = strcspn(url, ";&"); + + while (len > 0 && url[len - 1] == '/') + len--; + + for (str = url + len - 1; str >= url; str--) { + if (*str == '/') + break; + } + + str++; + len -= str - url; + + if (len > 0) + return strncpy(calloc(1, len + 1), str, len); + + return strdup(default_name); +} diff --git a/src/3P/uclient/uclient-utils.h b/src/3P/uclient/uclient-utils.h new file mode 100644 index 00000000..a4f975d6 --- /dev/null +++ b/src/3P/uclient/uclient-utils.h @@ -0,0 +1,49 @@ +/* + * uclient - ustream based protocol client library + * + * Copyright (C) 2014 Felix Fietkau + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef __UCLIENT_UTILS_H +#define __UCLIENT_UTILS_H + +#include + +struct http_digest_data { + const char *uri; + const char *method; + + const char *auth_hash; /* H(A1) */ + const char *qop; + const char *nc; + const char *nonce; + const char *cnonce; +}; + +static inline int base64_len(int len) +{ + return ((len + 2) / 3) * 4; +} + +void base64_encode(const void *inbuf, unsigned int len, void *out); +void bin_to_hex(char *dest, const void *buf, int len); + +int uclient_urldecode(const char *in, char *out, bool decode_plus); + +void http_digest_calculate_auth_hash(char *dest, const char *user, const char *realm, const char *password); +void http_digest_calculate_response(char *dest, const struct http_digest_data *data); + +char *uclient_get_url_filename(const char *url, const char *default_name); + +#endif diff --git a/src/3P/uclient/uclient.c b/src/3P/uclient/uclient.c new file mode 100644 index 00000000..8862b557 --- /dev/null +++ b/src/3P/uclient/uclient.c @@ -0,0 +1,421 @@ +/* + * uclient - ustream based protocol client library + * + * Copyright (C) 2014 Felix Fietkau + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include +#include +#include "uclient.h" +#include "uclient-utils.h" +#include "uclient-backend.h" + +char *uclient_get_addr(char *dest, int *port, union uclient_addr *a) +{ + int portval; + void *ptr; + + switch(a->sa.sa_family) { + case AF_INET: + ptr = &a->sin.sin_addr; + portval = a->sin.sin_port; + break; + case AF_INET6: + ptr = &a->sin6.sin6_addr; + portval = a->sin6.sin6_port; + break; + default: + return strcpy(dest, "Unknown"); + } + + inet_ntop(a->sa.sa_family, ptr, dest, INET6_ADDRSTRLEN); + if (port) + *port = ntohs(portval); + + return dest; +} + +static struct uclient_url * +__uclient_get_url(const struct uclient_backend *backend, + const char *host, int host_len, + const char *location, const char *auth_str) +{ + struct uclient_url *url; + char *host_buf, *uri_buf, *auth_buf, *next; + + url = calloc_a(sizeof(*url), + &host_buf, host_len + 1, + &uri_buf, strlen(location) + 1, + &auth_buf, auth_str ? strlen(auth_str) + 1 : 0); + + url->backend = backend; + url->location = strcpy(uri_buf, location); + if (host) + url->host = strncpy(host_buf, host, host_len); + + next = strchr(host_buf, '@'); + if (next) { + *next = 0; + url->host = next + 1; + + if (uclient_urldecode(host_buf, host_buf, false) < 0) + goto free; + + url->auth = host_buf; + } + + if (!url->auth && auth_str) + url->auth = strcpy(auth_buf, auth_str); + + /* Literal IPv6 address */ + if (*url->host == '[') { + url->host++; + next = strrchr(url->host, ']'); + if (!next) + goto free; + + *(next++) = 0; + if (*next == ':') + url->port = next + 1; + } else { + next = strrchr(url->host, ':'); + if (next) { + *next = 0; + url->port = next + 1; + } + } + + return url; + +free: + free(url); + return NULL; +} + +static const char * +uclient_split_host(const char *base, int *host_len) +{ + char *next, *location; + + next = strchr(base, '/'); + if (next) { + location = next; + *host_len = next - base; + } else { + location = "/"; + *host_len = strlen(base); + } + + return location; +} + +struct uclient_url __hidden * +uclient_get_url_location(struct uclient_url *url, const char *location) +{ + struct uclient_url *new_url; + char *host_buf, *uri_buf, *auth_buf, *port_buf; + int host_len = strlen(url->host) + 1; + int auth_len = url->auth ? strlen(url->auth) + 1 : 0; + int port_len = url->port ? strlen(url->port) + 1 : 0; + int uri_len; + + if (strstr(location, "://")) + return uclient_get_url(location, url->auth); + + if (location[0] == '/') + uri_len = strlen(location) + 1; + else + uri_len = strlen(url->location) + strlen(location) + 2; + + new_url = calloc_a(sizeof(*url), + &host_buf, host_len, + &port_buf, port_len, + &uri_buf, uri_len, + &auth_buf, auth_len); + + if (!new_url) + return NULL; + + new_url->backend = url->backend; + new_url->prefix = url->prefix; + new_url->host = strcpy(host_buf, url->host); + if (url->port) + new_url->port = strcpy(port_buf, url->port); + if (url->auth) + new_url->auth = strcpy(auth_buf, url->auth); + + new_url->location = uri_buf; + if (location[0] == '/') + strcpy(uri_buf, location); + else { + int len = strcspn(url->location, "?#"); + char *buf = uri_buf; + + memcpy(buf, url->location, len); + if (buf[len - 1] != '/') { + buf[len] = '/'; + len++; + } + + buf += len; + strcpy(buf, location); + } + + return new_url; +} + +struct uclient_url __hidden * +uclient_get_url(const char *url_str, const char *auth_str) +{ + static const struct uclient_backend *backends[] = { + &uclient_backend_http, + }; + + const struct uclient_backend *backend; + const char * const *prefix = NULL; + struct uclient_url *url; + const char *location; + int host_len; + int i; + + for (i = 0; i < ARRAY_SIZE(backends); i++) { + int prefix_len = 0; + + for (prefix = backends[i]->prefix; *prefix; prefix++) { + prefix_len = strlen(*prefix); + + if (!strncmp(url_str, *prefix, prefix_len)) + break; + } + + if (!*prefix) + continue; + + url_str += prefix_len; + backend = backends[i]; + break; + } + + if (!*prefix) + return NULL; + + location = uclient_split_host(url_str, &host_len); + url = __uclient_get_url(backend, url_str, host_len, location, auth_str); + if (!url) + return NULL; + + url->prefix = prefix - backend->prefix; + return url; +} + +static void uclient_connection_timeout(struct uloop_timeout *timeout) +{ + struct uclient *cl = container_of(timeout, struct uclient, connection_timeout); + + if (cl->backend->disconnect) + cl->backend->disconnect(cl); + + uclient_backend_set_error(cl, UCLIENT_ERROR_TIMEDOUT); +} + +struct uclient *uclient_new(const char *url_str, const char *auth_str, const struct uclient_cb *cb) +{ + struct uclient *cl; + struct uclient_url *url; + + url = uclient_get_url(url_str, auth_str); + if (!url) + return NULL; + + cl = url->backend->alloc(); + if (!cl) + return NULL; + + cl->backend = url->backend; + cl->cb = cb; + cl->url = url; + cl->timeout_msecs = UCLIENT_DEFAULT_TIMEOUT_MS; + cl->connection_timeout.cb = uclient_connection_timeout; + + return cl; +} + +int uclient_set_proxy_url(struct uclient *cl, const char *url_str, const char *auth_str) +{ + const struct uclient_backend *backend = cl->backend; + struct uclient_url *url; + int host_len; + char *next, *host; + + if (!backend->update_proxy_url) + return -1; + + next = strstr(url_str, "://"); + if (!next) + return -1; + + host = next + 3; + uclient_split_host(host, &host_len); + + url = __uclient_get_url(NULL, host, host_len, url_str, auth_str); + if (!url) + return -1; + + free(cl->proxy_url); + cl->proxy_url = url; + + if (backend->update_proxy_url) + backend->update_proxy_url(cl); + + return 0; +} + +int uclient_set_url(struct uclient *cl, const char *url_str, const char *auth_str) +{ + const struct uclient_backend *backend = cl->backend; + struct uclient_url *url = cl->url; + + url = uclient_get_url(url_str, auth_str); + if (!url) + return -1; + + if (url->backend != cl->backend) { + free(url); + return -1; + } + + free(cl->proxy_url); + cl->proxy_url = NULL; + + free(cl->url); + cl->url = url; + + if (backend->update_url) + backend->update_url(cl); + + return 0; +} + +int uclient_set_timeout(struct uclient *cl, int msecs) +{ + if (msecs <= 0) + return -EINVAL; + + cl->timeout_msecs = msecs; + + return 0; +} + +int uclient_connect(struct uclient *cl) +{ + return cl->backend->connect(cl); +} + +void uclient_free(struct uclient *cl) +{ + struct uclient_url *url = cl->url; + + if (cl->backend->free) + cl->backend->free(cl); + else + free(cl); + + free(url); +} + +int uclient_write(struct uclient *cl, const char *buf, int len) +{ + if (!cl->backend->write) + return -1; + + return cl->backend->write(cl, buf, len); +} + +int uclient_request(struct uclient *cl) +{ + int err; + + if (!cl->backend->request) + return -1; + + err = cl->backend->request(cl); + if (err) + return err; + + uloop_timeout_set(&cl->connection_timeout, cl->timeout_msecs); + + return 0; +} + +int uclient_read(struct uclient *cl, char *buf, int len) +{ + if (!cl->backend->read) + return -1; + + return cl->backend->read(cl, buf, len); +} + +void uclient_disconnect(struct uclient *cl) +{ + uloop_timeout_cancel(&cl->connection_timeout); + + if (!cl->backend->disconnect) + return; + + cl->backend->disconnect(cl); +} + +static void __uclient_backend_change_state(struct uloop_timeout *timeout) +{ + struct uclient *cl = container_of(timeout, struct uclient, timeout); + + if (cl->error_code && cl->cb->error) + cl->cb->error(cl, cl->error_code); + else if (cl->eof && cl->cb->data_eof) + cl->cb->data_eof(cl); +} + +static void uclient_backend_change_state(struct uclient *cl) +{ + cl->timeout.cb = __uclient_backend_change_state; + uloop_timeout_set(&cl->timeout, 1); +} + +void __hidden uclient_backend_set_error(struct uclient *cl, int code) +{ + if (cl->error_code) + return; + + uloop_timeout_cancel(&cl->connection_timeout); + cl->error_code = code; + uclient_backend_change_state(cl); +} + +void __hidden uclient_backend_set_eof(struct uclient *cl) +{ + if (cl->eof || cl->error_code) + return; + + uloop_timeout_cancel(&cl->connection_timeout); + cl->eof = true; + uclient_backend_change_state(cl); +} + +void __hidden uclient_backend_reset_state(struct uclient *cl) +{ + cl->data_eof = false; + cl->eof = false; + cl->error_code = 0; + uloop_timeout_cancel(&cl->timeout); +} diff --git a/src/3P/uclient/uclient.h b/src/3P/uclient/uclient.h new file mode 100644 index 00000000..e3695db6 --- /dev/null +++ b/src/3P/uclient/uclient.h @@ -0,0 +1,130 @@ +/* + * uclient - ustream based protocol client library + * + * Copyright (C) 2014 Felix Fietkau + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef __LIBUBOX_UCLIENT_H +#define __LIBUBOX_UCLIENT_H + +#include + +#include +#include +#include + +#define UCLIENT_DEFAULT_TIMEOUT_MS 30000 + +struct uclient_cb; +struct uclient_backend; + +enum uclient_error_code { + UCLIENT_ERROR_UNKNOWN, + UCLIENT_ERROR_CONNECT, + UCLIENT_ERROR_TIMEDOUT, + UCLIENT_ERROR_SSL_INVALID_CERT, + UCLIENT_ERROR_SSL_CN_MISMATCH, + UCLIENT_ERROR_MISSING_SSL_CONTEXT, +}; + +union uclient_addr { + struct sockaddr sa; + struct sockaddr_in sin; + struct sockaddr_in6 sin6; +}; + +struct uclient_url { + const struct uclient_backend *backend; + int prefix; + + const char *host; + const char *port; + const char *location; + + const char *auth; +}; + +struct uclient { + const struct uclient_backend *backend; + const struct uclient_cb *cb; + + union uclient_addr local_addr, remote_addr; + + struct uclient_url *proxy_url; + struct uclient_url *url; + int timeout_msecs; + void *priv; + + bool eof; + bool data_eof; + int error_code; + int status_code; + int seq; + struct blob_attr *meta; + + struct uloop_timeout connection_timeout; + struct uloop_timeout timeout; +}; + +struct uclient_cb { + void (*data_read)(struct uclient *cl); + void (*data_sent)(struct uclient *cl); + void (*data_eof)(struct uclient *cl); + void (*header_done)(struct uclient *cl); + void (*error)(struct uclient *cl, int code); +}; + +struct uclient *uclient_new(const char *url, const char *auth_str, const struct uclient_cb *cb); +void uclient_free(struct uclient *cl); + +int uclient_set_url(struct uclient *cl, const char *url, const char *auth); +int uclient_set_proxy_url(struct uclient *cl, const char *url_str, const char *auth_str); + + +/** + * Sets connection timeout. + * + * Provided timeout value will be used for: + * 1) Receiving HTTP response + * 2) Receiving data + * + * In case of timeout uclient will use error callback with + * UCLIENT_ERROR_TIMEDOUT code. + * + * @param msecs timeout in milliseconds + */ +int uclient_set_timeout(struct uclient *cl, int msecs); + +int uclient_connect(struct uclient *cl); +void uclient_disconnect(struct uclient *cl); + +int uclient_read(struct uclient *cl, char *buf, int len); +int uclient_write(struct uclient *cl, const char *buf, int len); +int uclient_request(struct uclient *cl); + +char *uclient_get_addr(char *dest, int *port, union uclient_addr *a); + +/* HTTP */ +extern const struct uclient_backend uclient_backend_http; + +int uclient_http_reset_headers(struct uclient *cl); +int uclient_http_set_header(struct uclient *cl, const char *name, const char *value); +int uclient_http_set_request_type(struct uclient *cl, const char *type); +int uclient_http_redirect(struct uclient *cl); + +int uclient_http_set_ssl_ctx(struct uclient *cl, const struct ustream_ssl_ops *ops, + struct ustream_ssl_ctx *ctx, bool require_validation); +int uclient_http_set_address_family(struct uclient *cl, int af); + +#endif