1/* see license file for copyright and license details */
2
3#include "arg.h"
4#include "tls.h"
5#include "util.h"
6
7#include <arpa/inet.h>
8#include <ctype.h>
9#include <errno.h>
10#include <fcntl.h>
11#include <netdb.h>
12#include <netinet/in.h>
13#include <stdio.h>
14#include <stdlib.h>
15#include <string.h>
16#include <sys/socket.h>
17#include <sys/stat.h>
18#include <sys/types.h>
19#include <unistd.h>
20
21struct Stream {
22 struct TlsSocket *ts;
23 char buf[8192];
24 size_t len;
25 size_t idx;
26};
27
28static int qflag = 0;
29static int Sflag = 0;
30static int cflag = 0;
31static int spider = 0;
32static int no_check_certificate = 0;
33static int timeout_sec = 900;
34static char *Pflag = NULL;
35static char *Oflag = NULL;
36static char *user_agent = "wget/aruu";
37static char *post_data = NULL;
38static char *post_file = NULL;
39static char **custom_headers = NULL;
40static size_t custom_headers_num = 0;
41
42static void
43usage(void)
44{
45 eprintf(
46 "usage: %s [-cqS] [-O file] [-P dir] [-T timeout] [-U "
47 "user_agent] "
48 "[-post-data data] [-post-file file] [-header header] "
49 "[-no-check-certificate] [-spider] url\n",
50 argv0
51 );
52}
53
54static void
55add_header(const char *hdr)
56{
57 custom_headers = ereallocarray(custom_headers, custom_headers_num + 1, sizeof(*custom_headers));
58 custom_headers[custom_headers_num++] = estrdup(hdr);
59}
60
61static int
62dial(const char *host, const char *port)
63{
64 struct addrinfo hints, *res, *rp;
65 int fd = -1, r;
66
67 memset(&hints, 0, sizeof(hints));
68 hints.ai_family = AF_UNSPEC;
69 hints.ai_socktype = SOCK_STREAM;
70
71 r = getaddrinfo(host, port, &hints, &res);
72 if (r != 0) {
73 if (!qflag)
74 weprintf("getaddrinfo %s:%s: %s\n", host, port, gai_strerror(r));
75 return -1;
76 }
77
78 for (rp = res; rp; rp = rp->ai_next) {
79 fd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
80 if (fd < 0)
81 continue;
82 if (timeout_sec > 0) {
83 struct timeval tv;
84 tv.tv_sec = timeout_sec;
85 tv.tv_usec = 0;
86 setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
87 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
88 }
89 if (connect(fd, rp->ai_addr, rp->ai_addrlen) == 0)
90 break;
91 close(fd);
92 fd = -1;
93 }
94
95 freeaddrinfo(res);
96 return fd;
97}
98
99static void
100parse_url(char *url, char **host, char **port, char **path, int *is_tls)
101{
102 char *p, *ss;
103
104 *is_tls = 0;
105 if (strncasecmp(url, "http://", 7) == 0) {
106 url += 7;
107 } else if (strncasecmp(url, "https://", 8) == 0) {
108 url += 8;
109 *is_tls = 1;
110 } else {
111 eprintf("unsupported protocol or invalid url: %s\n", url);
112 }
113
114 *host = url;
115 p = strchr(url, '/');
116 if (p) {
117 *p = '\0';
118 *path = p + 1;
119 } else {
120 *path = "";
121 }
122
123 /* handle ipv6 brackets or host:port */
124 if (**host == '[') {
125 (*host)++;
126 ss = strchr(*host, ']');
127 if (ss) {
128 *ss = '\0';
129 ss++;
130 if (*ss == ':')
131 *port = ss + 1;
132 else
133 *port = *is_tls ? "443" : "80";
134 } else {
135 eprintf("invalid ipv6 literal: %s\n", *host);
136 }
137 } else {
138 p = strrchr(*host, ':');
139 if (p) {
140 *p = '\0';
141 *port = p + 1;
142 } else {
143 *port = *is_tls ? "443" : "80";
144 }
145 }
146}
147
148static char *
149find_header(const char *headers, const char *name)
150{
151 const char *p;
152 size_t len = strlen(name);
153
154 p = headers;
155 while (p && *p) {
156 if (strncasecmp(p, name, len) == 0) {
157 p += len;
158 while (*p == ' ' || *p == '\t')
159 p++;
160 len = strcspn(p, "\r\n");
161 return estrndup(p, len);
162 }
163 p = strchr(p, '\n');
164 if (p)
165 p++;
166 }
167 return NULL;
168}
169
170static int
171stream_getc(struct Stream *s)
172{
173 ssize_t r;
174
175 if (s->idx < s->len) {
176 return (unsigned char)s->buf[s->idx++];
177 }
178 s->idx = 0;
179 r = tlss_read(s->ts, s->buf, sizeof(s->buf));
180 if (r <= 0) {
181 s->len = 0;
182 return EOF;
183 }
184 s->len = (size_t)r;
185 return (unsigned char)s->buf[s->idx++];
186}
187
188static size_t
189stream_read(struct Stream *s, void *ptr, size_t size)
190{
191 size_t total = 0;
192 size_t n;
193 char *p = ptr;
194 ssize_t r;
195
196 while (total < size) {
197 if (s->idx < s->len) {
198 n = MIN(size - total, s->len - s->idx);
199 memcpy(p + total, s->buf + s->idx, n);
200 s->idx += n;
201 total += n;
202 } else {
203 s->idx = 0;
204 r = tlss_read(s->ts, s->buf, sizeof(s->buf));
205 if (r <= 0) {
206 s->len = 0;
207 break;
208 }
209 s->len = (size_t)r;
210 }
211 }
212 return total;
213}
214
215static void
216read_chunked(struct Stream *s, int out_fd)
217{
218 char line[128];
219 char chunk_buf[8192];
220 size_t line_len, n;
221 long long chunk_size, remaining;
222 int c;
223
224 for (;;) {
225 line_len = 0;
226 for (;;) {
227 c = stream_getc(s);
228 if (c == EOF)
229 eprintf(
230 "unexpected end of file reading chunk "
231 "size\n"
232 );
233 if (c == '\n') {
234 line[line_len] = '\0';
235 break;
236 }
237 if (c != '\r' && line_len < sizeof(line) - 1) {
238 line[line_len++] = c;
239 }
240 }
241
242 chunk_size = strtoll(line, NULL, 16);
243 if (chunk_size == 0) {
244 stream_getc(s);
245 stream_getc(s);
246 break;
247 }
248
249 remaining = chunk_size;
250 while (remaining > 0) {
251 n = stream_read(s, chunk_buf, MIN(remaining, (long long)sizeof(chunk_buf)));
252 if (n == 0)
253 eprintf(
254 "unexpected end of file in chunk "
255 "data\n"
256 );
257 if (writeall(out_fd, chunk_buf, n) < 0)
258 eprintf("write output:\n");
259 remaining -= n;
260 }
261
262 stream_getc(s);
263 stream_getc(s);
264 }
265}
266
267static void
268read_non_chunked(struct Stream *s, int out_fd, long long content_len)
269{
270 char chunk_buf[8192];
271 long long remaining = content_len;
272 size_t n, to_read;
273
274 while (content_len < 0 || remaining > 0) {
275 to_read = sizeof(chunk_buf);
276 if (content_len >= 0)
277 to_read = (size_t)MIN(remaining, (long long)sizeof(chunk_buf));
278 n = stream_read(s, chunk_buf, to_read);
279 if (n == 0) {
280 if (content_len >= 0)
281 eprintf("unexpected end of file\n");
282 break;
283 }
284 if (writeall(out_fd, chunk_buf, n) < 0)
285 eprintf("write output:\n");
286 if (content_len >= 0)
287 remaining -= n;
288 }
289}
290
291static void
292req_printf(struct TlsSocket *ts, const char *fmt, ...)
293{
294 va_list ap;
295 char buf[1024];
296 int len;
297
298 va_start(ap, fmt);
299 len = vsnprintf(buf, sizeof(buf), fmt, ap);
300 va_end(ap);
301 if (len > 0)
302 tlss_write(ts, buf, len);
303}
304
305// ?man wget: retrieve files from the web
306// ?man arguments: url
307// ?man download files over http or https
308int
309main(int argc, char *argv[])
310{
311 struct Stream s;
312 char *url, *host, *port, *path, *loc;
313 char *curr_host, *curr_port, *curr_path;
314 char *new_url;
315 char *cl_str;
316 char *te_str;
317 char *header_end;
318 char *out_name;
319 int redirects = 0;
320 int max_redirects = 20;
321 int sock_fd = -1;
322 int out_fd = 1;
323 int chunked;
324 int status;
325 long long content_len;
326 size_t total_read;
327 ssize_t n;
328 size_t dir_len;
329 char *last_slash;
330 int is_tls = 0;
331 struct TlsSocket *tls_sock = NULL;
332 off_t resume_offset = 0;
333 int out_mode = O_WRONLY | O_CREAT | O_TRUNC;
334 long long post_len = 0;
335 int post_fd = -1;
336 size_t i;
337
338 ARGBEGIN
339 {
340 // ?man -O:str: specify output file path
341 case 'O':
342 Oflag = EARGF(usage());
343 break;
344 // ?man -P:str: specify output directory prefix
345 case 'P':
346 Pflag = EARGF(usage());
347 break;
348 // ?man -T:num: set network read and connect timeout
349 case 'T':
350 timeout_sec = estrtonum(EARGF(usage()), 0, 100000);
351 break;
352 // ?man -U:str: set User-Agent header
353 case 'U':
354 user_agent = EARGF(usage());
355 break;
356 // ?man -c: continue retrieval of aborted transfer
357 case 'c':
358 cflag = 1;
359 break;
360 // ?man -q: quiet mode to suppress stderr output
361 case 'q':
362 qflag = 1;
363 break;
364 // ?man -S: print server response headers to stderr
365 case 'S':
366 Sflag = 1;
367 break;
368 // ?man --: specify - option
369 case '-':
370 if (strcmp(argv[0], "-no-check-certificate") == 0) {
371 no_check_certificate = 1;
372 brk_ = 1;
373 } else if (strncmp(argv[0], "-header=", 8) == 0) {
374 add_header(argv[0] + 8);
375 brk_ = 1;
376 } else if (strcmp(argv[0], "-header") == 0) {
377 brk_ = 1;
378 if (!argv[1])
379 usage();
380 add_header(argv[1]);
381 argv++;
382 argc--;
383 } else if (strncmp(argv[0], "-post-data=", 11) == 0) {
384 post_data = argv[0] + 11;
385 brk_ = 1;
386 } else if (strcmp(argv[0], "-post-data") == 0) {
387 brk_ = 1;
388 if (!argv[1])
389 usage();
390 post_data = argv[1];
391 argv++;
392 argc--;
393 } else if (strncmp(argv[0], "-post-file=", 11) == 0) {
394 post_file = argv[0] + 11;
395 brk_ = 1;
396 } else if (strcmp(argv[0], "-post-file") == 0) {
397 brk_ = 1;
398 if (!argv[1])
399 usage();
400 post_file = argv[1];
401 argv++;
402 argc--;
403 } else if (strcmp(argv[0], "-spider") == 0) {
404 spider = 1;
405 brk_ = 1;
406 } else {
407 usage();
408 }
409 break;
410 default:
411 usage();
412 }
413 ARGEND
414
415 if (argc < 1)
416 usage();
417
418 url = estrdup(argv[0]);
419
420 /* determine output filename early to check for resume */
421 out_name = NULL;
422 if (Oflag) {
423 out_name = Oflag;
424 } else {
425 last_slash = strrchr(url, '/');
426 if (last_slash && *(last_slash + 1))
427 out_name = last_slash + 1;
428 else
429 out_name = "index.html";
430
431 if (Pflag) {
432 char *tmp = emalloc(strlen(Pflag) + 1 + strlen(out_name) + 1);
433 sprintf(tmp, "%s/%s", Pflag, out_name);
434 out_name = tmp;
435 }
436 }
437
438 if (cflag && out_name && strcmp(out_name, "-") != 0) {
439 struct stat st;
440 if (stat(out_name, &st) == 0 && S_ISREG(st.st_mode)) {
441 resume_offset = st.st_size;
442 }
443 }
444
445 if (post_data) {
446 post_len = strlen(post_data);
447 } else if (post_file) {
448 struct stat st;
449 post_fd = open(post_file, O_RDONLY);
450 if (post_fd < 0)
451 eprintf("open %s:\n", post_file);
452 if (fstat(post_fd, &st) < 0)
453 eprintf("stat %s:\n", post_file);
454 post_len = st.st_size;
455 }
456
457 while (!tls_sock) {
458 if (redirects > max_redirects)
459 eprintf("too many redirects\n");
460
461 curr_host = curr_port = curr_path = NULL;
462 parse_url(url, &curr_host, &curr_port, &curr_path, &is_tls);
463
464 host = estrdup(curr_host);
465 port = estrdup(curr_port);
466 path = estrdup(curr_path);
467
468 sock_fd = dial(host, port);
469 if (sock_fd < 0)
470 eprintf("failed to connect to %s:%s\n", host, port);
471
472 tls_sock = tlss_connect(sock_fd, host, !no_check_certificate, is_tls);
473 if (!tls_sock) {
474 close(sock_fd);
475 eprintf("failed to establish TLS connection with %s\n", host);
476 }
477
478 /* send http request */
479 const char *method = spider ? "HEAD" : ((post_data || post_file) ? "POST" : "GET");
480 req_printf(tls_sock, "%s /%s HTTP/1.1\r\n", method, path);
481 req_printf(tls_sock, "Host: %s\r\n", host);
482 req_printf(tls_sock, "User-Agent: %s\r\n", user_agent);
483 req_printf(tls_sock, "Connection: close\r\n");
484
485 if (resume_offset > 0) {
486 req_printf(tls_sock, "Range: bytes=%lld-\r\n", (long long)resume_offset);
487 }
488
489 if (post_data || post_file) {
490 int has_ct = 0;
491 for (i = 0; i < custom_headers_num; i++) {
492 if (strncasecmp(custom_headers[i], "Content-Type:", 13) == 0) {
493 has_ct = 1;
494 break;
495 }
496 }
497 if (!has_ct) {
498 req_printf(
499 tls_sock,
500 "Content-Type: "
501 "application/"
502 "x-www-form-urlencoded\r\n"
503 );
504 }
505 req_printf(tls_sock, "Content-Length: %lld\r\n", post_len);
506 }
507
508 for (i = 0; i < custom_headers_num; i++) {
509 req_printf(tls_sock, "%s\r\n", custom_headers[i]);
510 }
511
512 req_printf(tls_sock, "\r\n");
513
514 if (post_data) {
515 tlss_write(tls_sock, post_data, strlen(post_data));
516 } else if (post_file) {
517 char io_buf[8192];
518 ssize_t r;
519 while ((r = read(post_fd, io_buf, sizeof(io_buf))) > 0) {
520 if (tlss_write(tls_sock, io_buf, r) < 0) {
521 eprintf("failed to write post data:\n");
522 }
523 }
524 close(post_fd);
525 post_fd = -1;
526 }
527
528 /* read headers */
529 total_read = 0;
530 header_end = NULL;
531 memset(s.buf, 0, sizeof(s.buf));
532 while (total_read < sizeof(s.buf) - 1) {
533 n = tlss_read(tls_sock, s.buf + total_read, sizeof(s.buf) - 1 - total_read);
534 if (n <= 0) {
535 if (n < 0)
536 eprintf("read socket:\n");
537 else
538 eprintf(
539 "connection closed by "
540 "server\n"
541 );
542 }
543 total_read += n;
544 s.buf[total_read] = '\0';
545 header_end = strstr(s.buf, "\r\n\r\n");
546 if (header_end)
547 break;
548 }
549
550 if (!header_end)
551 eprintf("http header too large or not found\n");
552
553 *header_end = '\0';
554 s.ts = tls_sock;
555 s.len = total_read;
556 s.idx = (header_end + 4) - s.buf;
557
558 if (Sflag) {
559 fprintf(stderr, "%s\n\n", s.buf);
560 }
561
562 if (strncasecmp(s.buf, "HTTP/1.1 ", 9) != 0 && strncasecmp(s.buf, "HTTP/1.0 ", 9) != 0) {
563 eprintf("invalid http response: %s\n", s.buf);
564 }
565 status = atoi(s.buf + 9);
566
567 if (status >= 300 && status < 400) {
568 loc = find_header(s.buf, "Location:");
569 if (!loc)
570 eprintf(
571 "redirect response without location "
572 "header\n"
573 );
574
575 if (strncasecmp(loc, "http://", 7) == 0 || strncasecmp(loc, "https://", 8) == 0) {
576 new_url = estrdup(loc);
577 } else if (loc[0] == '/') {
578 new_url = emalloc(8 + strlen(host) + strlen(port) + strlen(loc) + 2);
579 sprintf(new_url, "%s://%s:%s%s", is_tls ? "https" : "http", host, port, loc);
580 } else {
581 last_slash = strrchr(path, '/');
582 dir_len = 0;
583 if (last_slash)
584 dir_len = last_slash - path + 1;
585 new_url = emalloc(8 + strlen(host) + strlen(port) + 1 + dir_len + strlen(loc) + 2);
586 sprintf(new_url, "%s://%s:%s/", is_tls ? "https" : "http", host, port);
587 if (dir_len > 0)
588 strncat(new_url, path, dir_len);
589 strcat(new_url, loc);
590 }
591
592 free(loc);
593 free(url);
594 url = new_url;
595 tlss_close(tls_sock, 1);
596 tls_sock = NULL;
597 redirects++;
598 } else if (status == 206) {
599 out_mode = O_WRONLY | O_CREAT | O_APPEND;
600 } else if (status == 200) {
601 out_mode = O_WRONLY | O_CREAT | O_TRUNC;
602 } else if (status == 416) {
603 if (!qflag)
604 weprintf(
605 "file already fully retrieved or "
606 "range invalid\n"
607 );
608 tlss_close(tls_sock, 1);
609 free(url);
610 free(host);
611 free(port);
612 free(path);
613 return 0;
614 } else {
615 eprintf("server returned status: %d\n", status);
616 }
617
618 free(host);
619 free(port);
620 free(path);
621 }
622
623 if (spider) {
624 tlss_close(tls_sock, 1);
625 free(url);
626 return 0;
627 }
628
629 cl_str = find_header(s.buf, "Content-Length:");
630 content_len = -1;
631 if (cl_str) {
632 content_len = strtoll(cl_str, NULL, 10);
633 free(cl_str);
634 }
635
636 te_str = find_header(s.buf, "Transfer-Encoding:");
637 chunked = 0;
638 if (te_str) {
639 if (strcasecmp(te_str, "chunked") == 0)
640 chunked = 1;
641 free(te_str);
642 }
643
644 if (strcmp(out_name, "-") != 0) {
645 out_fd = open(out_name, out_mode, 0644);
646 if (out_fd < 0)
647 eprintf("open %s:\n", out_name);
648 }
649
650 if (chunked)
651 read_chunked(&s, out_fd);
652 else
653 read_non_chunked(&s, out_fd, content_len);
654
655 tlss_close(tls_sock, 1);
656 if (out_fd != 1)
657 close(out_fd);
658 if (Oflag != out_name && Pflag)
659 free(out_name);
660 free(url);
661
662 for (i = 0; i < custom_headers_num; i++) {
663 free(custom_headers[i]);
664 }
665 free(custom_headers);
666
667 return 0;
668}
