1/* see license file for copyright and license details */
2
3#include "arg.h"
4#include "tls.h"
5#include "util.h"
6
7#include <arpa/inet.h>
8#include <ctype.h>
9#include <errno.h>
10#include <fcntl.h>
11#include <limits.h>
12#include <netdb.h>
13#include <netinet/in.h>
14#include <stdio.h>
15#include <stdlib.h>
16#include <string.h>
17#include <sys/socket.h>
18#include <sys/stat.h>
19#include <sys/types.h>
20#include <unistd.h>
21
22struct Stream {
23 struct TlsSocket *ts;
24 char buf[8192];
25 size_t len;
26 size_t idx;
27};
28
29static int qflag = 0;
30static int Sflag = 0;
31static int cflag = 0;
32static int spider = 0;
33static int no_check_certificate = 0;
34static int timeout_sec = 900;
35static int tries = 20;
36static char *Pflag = NULL;
37static char *Oflag = NULL;
38static char *user_agent = "wget/aruu";
39static char *post_data = NULL;
40static char *post_file = NULL;
41static char **custom_headers = NULL;
42static size_t custom_headers_num = 0;
43
44static void
45usage(void)
46{
47 eprintf(
48 "usage: %s [-cqS] [-O file] [-P dir] [-T timeout] [-t tries] [-U "
49 "user_agent] "
50 "[-post-data data] [-post-file file] [-header header] "
51 "[-no-check-certificate] [-spider] url\n",
52 argv0
53 );
54}
55
56static void
57add_header(const char *hdr)
58{
59 custom_headers = ereallocarray(custom_headers, custom_headers_num + 1, sizeof(*custom_headers));
60 custom_headers[custom_headers_num++] = estrdup(hdr);
61}
62
63static int
64dial(const char *host, const char *port)
65{
66 struct addrinfo hints, *res, *rp;
67 int fd = -1, r;
68
69 memset(&hints, 0, sizeof(hints));
70 hints.ai_family = AF_UNSPEC;
71 hints.ai_socktype = SOCK_STREAM;
72
73 r = getaddrinfo(host, port, &hints, &res);
74 if (r != 0) {
75 if (!qflag)
76 weprintf("getaddrinfo %s:%s: %s\n", host, port, gai_strerror(r));
77 return -1;
78 }
79
80 for (rp = res; rp; rp = rp->ai_next) {
81 fd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
82 if (fd < 0)
83 continue;
84 if (timeout_sec > 0) {
85 struct timeval tv;
86 tv.tv_sec = timeout_sec;
87 tv.tv_usec = 0;
88 setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
89 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
90 }
91 if (connect(fd, rp->ai_addr, rp->ai_addrlen) == 0)
92 break;
93 close(fd);
94 fd = -1;
95 }
96
97 freeaddrinfo(res);
98 return fd;
99}
100
101static void
102parse_url(char *url, char **host, char **port, char **path, int *is_tls)
103{
104 char *p, *ss;
105
106 *is_tls = 0;
107 if (strncasecmp(url, "http:// ", 7) == 0) {
108 url += 7;
109 } else if (strncasecmp(url, "https:// ", 8) == 0) {
110 url += 8;
111 *is_tls = 1;
112 } else {
113 eprintf("unsupported protocol or invalid url: %s\n", url);
114 }
115
116 *host = url;
117 p = strchr(url, '/');
118 if (p) {
119 *p = '\0';
120 *path = p + 1;
121 } else {
122 *path = "";
123 }
124
125 /* handle ipv6 brackets or host:port */
126 if (**host == '[') {
127 (*host)++;
128 ss = strchr(*host, ']');
129 if (ss) {
130 *ss = '\0';
131 ss++;
132 if (*ss == ':')
133 *port = ss + 1;
134 else
135 *port = *is_tls ? "443" : "80";
136 } else {
137 eprintf("invalid ipv6 literal: %s\n", *host);
138 }
139 } else {
140 p = strrchr(*host, ':');
141 if (p) {
142 *p = '\0';
143 *port = p + 1;
144 } else {
145 *port = *is_tls ? "443" : "80";
146 }
147 }
148}
149
150static char *
151find_header(const char *headers, const char *name)
152{
153 const char *p;
154 size_t len = strlen(name);
155
156 p = headers;
157 while (p && *p) {
158 if (strncasecmp(p, name, len) == 0) {
159 p += len;
160 while (*p == ' ' || *p == '\t')
161 p++;
162 len = strcspn(p, "\r\n");
163 return estrndup(p, len);
164 }
165 p = strchr(p, '\n');
166 if (p)
167 p++;
168 }
169 return NULL;
170}
171
172static int
173stream_getc(struct Stream *s)
174{
175 ssize_t r;
176
177 if (s->idx < s->len) {
178 return (unsigned char)s->buf[s->idx++];
179 }
180 s->idx = 0;
181 r = tlss_read(s->ts, s->buf, sizeof(s->buf));
182 if (r <= 0) {
183 s->len = 0;
184 return EOF;
185 }
186 s->len = (size_t)r;
187 return (unsigned char)s->buf[s->idx++];
188}
189
190static size_t
191stream_read(struct Stream *s, void *ptr, size_t size)
192{
193 size_t total = 0;
194 size_t n;
195 char *p = ptr;
196 ssize_t r;
197
198 while (total < size) {
199 if (s->idx < s->len) {
200 n = MIN(size - total, s->len - s->idx);
201 memcpy(p + total, s->buf + s->idx, n);
202 s->idx += n;
203 total += n;
204 } else {
205 s->idx = 0;
206 r = tlss_read(s->ts, s->buf, sizeof(s->buf));
207 if (r <= 0) {
208 s->len = 0;
209 break;
210 }
211 s->len = (size_t)r;
212 }
213 }
214 return total;
215}
216
217static void
218read_chunked(struct Stream *s, int out_fd)
219{
220 char line[128];
221 char chunk_buf[8192];
222 size_t line_len, n;
223 long long chunk_size, remaining;
224 int c;
225
226 for (;;) {
227 line_len = 0;
228 for (;;) {
229 c = stream_getc(s);
230 if (c == EOF)
231 eprintf(
232 "unexpected end of file reading chunk "
233 "size\n"
234 );
235 if (c == '\n') {
236 line[line_len] = '\0';
237 break;
238 }
239 if (c != '\r' && line_len < sizeof(line) - 1) {
240 line[line_len++] = c;
241 }
242 }
243
244 chunk_size = strtoll(line, NULL, 16);
245 if (chunk_size == 0) {
246 stream_getc(s);
247 stream_getc(s);
248 break;
249 }
250
251 remaining = chunk_size;
252 while (remaining > 0) {
253 n = stream_read(s, chunk_buf, MIN(remaining, (long long)sizeof(chunk_buf)));
254 if (n == 0)
255 eprintf(
256 "unexpected end of file in chunk "
257 "data\n"
258 );
259 if (writeall(out_fd, chunk_buf, n) < 0)
260 eprintf("write output:\n");
261 remaining -= n;
262 }
263
264 stream_getc(s);
265 stream_getc(s);
266 }
267}
268
269static void
270read_non_chunked(struct Stream *s, int out_fd, long long content_len)
271{
272 char chunk_buf[8192];
273 long long remaining = content_len;
274 size_t n, to_read;
275
276 while (content_len < 0 || remaining > 0) {
277 to_read = sizeof(chunk_buf);
278 if (content_len >= 0)
279 to_read = (size_t)MIN(remaining, (long long)sizeof(chunk_buf));
280 n = stream_read(s, chunk_buf, to_read);
281 if (n == 0) {
282 if (content_len >= 0)
283 eprintf("unexpected end of file\n");
284 break;
285 }
286 if (writeall(out_fd, chunk_buf, n) < 0)
287 eprintf("write output:\n");
288 if (content_len >= 0)
289 remaining -= n;
290 }
291}
292
293static void
294req_printf(struct TlsSocket *ts, const char *fmt, ...)
295{
296 va_list ap;
297 char buf[1024];
298 int len;
299
300 va_start(ap, fmt);
301 len = vsnprintf(buf, sizeof(buf), fmt, ap);
302 va_end(ap);
303 if (len > 0)
304 tlss_write(ts, buf, len);
305}
306
307// ?man wget: retrieve files from the web
308// ?man arguments: url
309// ?man download files over http or https
310int
311main(int argc, char *argv[])
312{
313 struct Stream s;
314 char *url, *host, *port, *path, *loc;
315 char *curr_host, *curr_port, *curr_path;
316 char *new_url;
317 char *cl_str;
318 char *te_str;
319 char *header_end;
320 char *out_name;
321 int redirects = 0;
322 int max_redirects = 20;
323 int sock_fd = -1;
324 int out_fd = 1;
325 int chunked;
326 int status;
327 long long content_len;
328 size_t total_read;
329 ssize_t n;
330 size_t dir_len;
331 char *last_slash;
332 int is_tls = 0;
333 struct TlsSocket *tls_sock = NULL;
334 off_t resume_offset = 0;
335 int out_mode = O_WRONLY | O_CREAT | O_TRUNC;
336 long long post_len = 0;
337 int post_fd = -1;
338 size_t i;
339 int attempt;
340 int max_tries;
341
342 ARGBEGIN
343 {
344 // ?man -O:str: specify output file path
345 case 'O':
346 Oflag = EARGF(usage());
347 break;
348 // ?man -P:str: specify output directory prefix
349 case 'P':
350 Pflag = EARGF(usage());
351 break;
352 // ?man -T:num: set network read and connect timeout
353 case 'T':
354 timeout_sec = estrtonum(EARGF(usage()), 0, 100000);
355 break;
356 // ?man -t:num: set number of connection retries, 0 retries forever
357 case 't':
358 tries = estrtonum(EARGF(usage()), 0, 1000000);
359 break;
360 // ?man -U:str: set User-Agent header
361 case 'U':
362 user_agent = EARGF(usage());
363 break;
364 // ?man -c: continue retrieval of aborted transfer
365 case 'c':
366 cflag = 1;
367 break;
368 // ?man -q: quiet mode to suppress stderr output
369 case 'q':
370 qflag = 1;
371 break;
372 // ?man -S: print server response headers to stderr
373 case 'S':
374 Sflag = 1;
375 break;
376 // ?man --: specify - option
377 case '-':
378 if (strcmp(argv[0], "-no-check-certificate") == 0) {
379 no_check_certificate = 1;
380 brk_ = 1;
381 } else if (strncmp(argv[0], "-header=", 8) == 0) {
382 add_header(argv[0] + 8);
383 brk_ = 1;
384 } else if (strcmp(argv[0], "-header") == 0) {
385 brk_ = 1;
386 if (!argv[1])
387 usage();
388 add_header(argv[1]);
389 argv++;
390 argc--;
391 } else if (strncmp(argv[0], "-post-data=", 11) == 0) {
392 post_data = argv[0] + 11;
393 brk_ = 1;
394 } else if (strcmp(argv[0], "-post-data") == 0) {
395 brk_ = 1;
396 if (!argv[1])
397 usage();
398 post_data = argv[1];
399 argv++;
400 argc--;
401 } else if (strncmp(argv[0], "-post-file=", 11) == 0) {
402 post_file = argv[0] + 11;
403 brk_ = 1;
404 } else if (strcmp(argv[0], "-post-file") == 0) {
405 brk_ = 1;
406 if (!argv[1])
407 usage();
408 post_file = argv[1];
409 argv++;
410 argc--;
411 } else if (strcmp(argv[0], "-spider") == 0) {
412 spider = 1;
413 brk_ = 1;
414 } else {
415 usage();
416 }
417 break;
418 default:
419 usage();
420 }
421 ARGEND
422
423 if (argc < 1)
424 usage();
425
426 url = estrdup(argv[0]);
427
428 /* determine output filename early to check for resume */
429 out_name = NULL;
430 if (Oflag) {
431 out_name = Oflag;
432 } else {
433 last_slash = strrchr(url, '/');
434 if (last_slash && *(last_slash + 1))
435 out_name = last_slash + 1;
436 else
437 out_name = "index.html";
438
439 if (Pflag) {
440 char *tmp = emalloc(strlen(Pflag) + 1 + strlen(out_name) + 1);
441 sprintf(tmp, "%s/%s", Pflag, out_name);
442 out_name = tmp;
443 }
444 }
445
446 if (cflag && out_name && strcmp(out_name, "-") != 0) {
447 struct stat st;
448 if (stat(out_name, &st) == 0 && S_ISREG(st.st_mode)) {
449 resume_offset = st.st_size;
450 }
451 }
452
453 if (post_data) {
454 post_len = strlen(post_data);
455 } else if (post_file) {
456 struct stat st;
457 post_fd = open(post_file, O_RDONLY);
458 if (post_fd < 0)
459 eprintf("open %s:\n", post_file);
460 if (fstat(post_fd, &st) < 0)
461 eprintf("stat %s:\n", post_file);
462 post_len = st.st_size;
463 }
464
465 /* wgets own -t: 0 means retry forever */
466 max_tries = tries == 0 ? INT_MAX : tries;
467
468 while (!tls_sock) {
469 if (redirects > max_redirects)
470 eprintf("too many redirects\n");
471
472 curr_host = curr_port = curr_path = NULL;
473 parse_url(url, &curr_host, &curr_port, &curr_path, &is_tls);
474
475 host = estrdup(curr_host);
476 port = estrdup(curr_port);
477 path = estrdup(curr_path);
478
479 for (attempt = 1; attempt <= max_tries; attempt++) {
480 sock_fd = dial(host, port);
481 if (sock_fd < 0) {
482 if (attempt == max_tries)
483 eprintf("failed to connect to %s:%s\n", host, port);
484 continue;
485 }
486
487 tls_sock = tlss_connect(sock_fd, host, !no_check_certificate, is_tls);
488 if (!tls_sock) {
489 close(sock_fd);
490 if (attempt == max_tries)
491 eprintf("failed to establish TLS connection with %s\n", host);
492 continue;
493 }
494
495 break;
496 }
497
498 /* send http request */
499 const char *method = spider ? "HEAD" : ((post_data || post_file) ? "POST" : "GET");
500 req_printf(tls_sock, "%s /%s HTTP/1.1\r\n", method, path);
501 req_printf(tls_sock, "Host: %s\r\n", host);
502 req_printf(tls_sock, "User-Agent: %s\r\n", user_agent);
503 req_printf(tls_sock, "Connection: close\r\n");
504
505 if (resume_offset > 0) {
506 req_printf(tls_sock, "Range: bytes=%lld-\r\n", (long long)resume_offset);
507 }
508
509 if (post_data || post_file) {
510 int has_ct = 0;
511 for (i = 0; i < custom_headers_num; i++) {
512 if (strncasecmp(custom_headers[i], "Content-Type:", 13) == 0) {
513 has_ct = 1;
514 break;
515 }
516 }
517 if (!has_ct) {
518 req_printf(
519 tls_sock,
520 "Content-Type: "
521 "application/"
522 "x-www-form-urlencoded\r\n"
523 );
524 }
525 req_printf(tls_sock, "Content-Length: %lld\r\n", post_len);
526 }
527
528 for (i = 0; i < custom_headers_num; i++) {
529 req_printf(tls_sock, "%s\r\n", custom_headers[i]);
530 }
531
532 req_printf(tls_sock, "\r\n");
533
534 if (post_data) {
535 tlss_write(tls_sock, post_data, strlen(post_data));
536 } else if (post_file) {
537 char io_buf[8192];
538 ssize_t r;
539 while ((r = read(post_fd, io_buf, sizeof(io_buf))) > 0) {
540 if (tlss_write(tls_sock, io_buf, r) < 0) {
541 eprintf("failed to write post data:\n");
542 }
543 }
544 close(post_fd);
545 post_fd = -1;
546 }
547
548 /* read headers */
549 total_read = 0;
550 header_end = NULL;
551 memset(s.buf, 0, sizeof(s.buf));
552 while (total_read < sizeof(s.buf) - 1) {
553 n = tlss_read(tls_sock, s.buf + total_read, sizeof(s.buf) - 1 - total_read);
554 if (n <= 0) {
555 if (n < 0)
556 eprintf("read socket:\n");
557 else
558 eprintf(
559 "connection closed by "
560 "server\n"
561 );
562 }
563 total_read += n;
564 s.buf[total_read] = '\0';
565 header_end = strstr(s.buf, "\r\n\r\n");
566 if (header_end)
567 break;
568 }
569
570 if (!header_end)
571 eprintf("http header too large or not found\n");
572
573 *header_end = '\0';
574 s.ts = tls_sock;
575 s.len = total_read;
576 s.idx = (header_end + 4) - s.buf;
577
578 if (Sflag) {
579 fprintf(stderr, "%s\n\n", s.buf);
580 }
581
582 if (strncasecmp(s.buf, "HTTP/1.1 ", 9) != 0 && strncasecmp(s.buf, "HTTP/1.0 ", 9) != 0) {
583 eprintf("invalid http response: %s\n", s.buf);
584 }
585 status = atoi(s.buf + 9);
586
587 if (status >= 300 && status < 400) {
588 loc = find_header(s.buf, "Location:");
589 if (!loc)
590 eprintf(
591 "redirect response without location "
592 "header\n"
593 );
594
595 if (strncasecmp(loc, "http:// ", 7) == 0 || strncasecmp(loc, "https://", 8) == 0) {
596 new_url = estrdup(loc);
597 } else if (loc[0] == '/') {
598 new_url = emalloc(8 + strlen(host) + strlen(port) + strlen(loc) + 2);
599 sprintf(new_url, "%s:// %s:%s%s", is_tls ? "https" : "http", host, port, loc);
600 } else {
601 last_slash = strrchr(path, '/');
602 dir_len = 0;
603 if (last_slash)
604 dir_len = last_slash - path + 1;
605 new_url = emalloc(8 + strlen(host) + strlen(port) + 1 + dir_len + strlen(loc) + 2);
606 sprintf(new_url, "%s:// %s:%s/", is_tls ? "https" : "http", host, port);
607 if (dir_len > 0)
608 strncat(new_url, path, dir_len);
609 strcat(new_url, loc);
610 }
611
612 free(loc);
613 free(url);
614 url = new_url;
615 tlss_close(tls_sock, 1);
616 tls_sock = NULL;
617 redirects++;
618 } else if (status == 206) {
619 out_mode = O_WRONLY | O_CREAT | O_APPEND;
620 } else if (status == 200) {
621 out_mode = O_WRONLY | O_CREAT | O_TRUNC;
622 } else if (status == 416) {
623 if (!qflag)
624 weprintf(
625 "file already fully retrieved or "
626 "range invalid\n"
627 );
628 tlss_close(tls_sock, 1);
629 free(url);
630 free(host);
631 free(port);
632 free(path);
633 return 0;
634 } else {
635 eprintf("server returned status: %d\n", status);
636 }
637
638 free(host);
639 free(port);
640 free(path);
641 }
642
643 if (spider) {
644 tlss_close(tls_sock, 1);
645 free(url);
646 return 0;
647 }
648
649 cl_str = find_header(s.buf, "Content-Length:");
650 content_len = -1;
651 if (cl_str) {
652 content_len = strtoll(cl_str, NULL, 10);
653 free(cl_str);
654 }
655
656 te_str = find_header(s.buf, "Transfer-Encoding:");
657 chunked = 0;
658 if (te_str) {
659 if (strcasecmp(te_str, "chunked") == 0)
660 chunked = 1;
661 free(te_str);
662 }
663
664 if (strcmp(out_name, "-") != 0) {
665 out_fd = open(out_name, out_mode, 0644);
666 if (out_fd < 0)
667 eprintf("open %s:\n", out_name);
668 }
669
670 if (chunked)
671 read_chunked(&s, out_fd);
672 else
673 read_non_chunked(&s, out_fd, content_len);
674
675 tlss_close(tls_sock, 1);
676 if (out_fd != 1)
677 close(out_fd);
678 if (Oflag != out_name && Pflag)
679 free(out_name);
680 free(url);
681
682 for (i = 0; i < custom_headers_num; i++) {
683 free(custom_headers[i]);
684 }
685 free(custom_headers);
686
687 return 0;
688}
