3 eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}'
4 if 0; # not running under some shell
6 # Simple user agent using LWP library.
10 lwp-request, GET, POST, HEAD - Simple command line user agent
14 B<lwp-request> [B<-afPuUsSedvhx>] [B<-m> I<method>] [B<-b> I<base URL>] [B<-t> I<timeout>]
15 [B<-i> I<if-modified-since>] [B<-c> I<content-type>]
16 [B<-C> I<credentials>] [B<-p> I<proxy-url>] [B<-o> I<format>] I<url>...
20 This program can be used to send requests to WWW servers and your
21 local file system. The request content for POST and PUT
22 methods is read from stdin. The content of the response is printed on
23 stdout. Error messages are printed on stderr. The program returns a
24 status value indicating the number of URLs that failed.
32 Set which method to use for the request. If this option is not used,
33 then the method is derived from the name of the program.
37 Force request through, even if the program believes that the method is
38 illegal. The server might reject the request eventually.
42 This URI will be used as the base URI for resolving all relative URIs
47 Set the timeout value for the requests. The timeout is the amount of
48 time that the program will wait for a response from the remote server
49 before it fails. The default unit for the timeout value is seconds.
50 You might append "m" or "h" to the timeout value to make it minutes or
51 hours, respectively. The default timeout is '3m', i.e. 3 minutes.
55 Set the If-Modified-Since header in the request. If I<time> is the
56 name of a file, use the modification timestamp for this file. If
57 I<time> is not a file, it is parsed as a literal date. Take a look at
58 L<HTTP::Date> for recognized formats.
60 =item -c <content-type>
62 Set the Content-Type for the request. This option is only allowed for
63 requests that take a content, i.e. POST and PUT. You can
64 force methods to take content by using the C<-f> option together with
65 C<-c>. The default Content-Type for POST is
66 C<application/x-www-form-urlencoded>. The default Content-type for
67 the others is C<text/plain>.
71 Set the proxy to be used for the requests. The program also loads
72 proxy settings from the environment. You can disable this with the
77 Don't load proxy settings from environment.
81 Send this HTTP header with each request. You can specify several, e.g.:
84 -H 'Referer: http://other.url/' \
88 =item -C <username>:<password>
90 Provide credentials for documents that are protected by Basic
91 Authentication. If the document is protected and you did not specify
92 the username and password with this option, then you will be prompted
93 to provide these values.
97 The following options controls what is displayed by the program:
103 Print request method and absolute URL as requests are made.
107 Print request headers in addition to request method and absolute URL.
111 Print response status code. This option is always on for HEAD requests.
115 Print response status chain. This shows redirect and authorization
116 requests that are handled by the library.
120 Print response headers. This option is always on for HEAD requests.
124 Do B<not> print the content of the response.
128 Process HTML content in various ways before printing it. If the
129 content type of the response is not HTML, then this option has no
130 effect. The legal format values are; I<text>, I<ps>, I<links>,
133 If you specify the I<text> format then the HTML will be formatted as
134 plain latin1 text. If you specify the I<ps> format then it will be
135 formatted as Postscript.
137 The I<links> format will output all links found in the HTML document.
138 Relative links will be expanded to absolute ones.
140 The I<html> format will reformat the HTML code and the I<dump> format
141 will just dump the HTML syntax tree.
143 Note that the C<HTML-Tree> distribution needs to be installed for this
144 option to work. In addition the C<HTML-Format> distribution needs to
145 be installed for I<-o text> or I<-o ps> to work.
149 Print the version number of the program and quit.
153 Print usage message and quit.
157 Extra debugging output.
161 Set text(ascii) mode for content input and output. If this option is not
162 used, content input and output is done in binary mode.
166 Because this program is implemented using the LWP library, it will
167 only support the protocols that LWP supports.
171 L<lwp-mirror>, L<LWP>
175 Copyright 1995-1999 Gisle Aas.
177 This library is free software; you can redistribute it and/or
178 modify it under the same terms as Perl itself.
182 Gisle Aas <gisle@aas.no>
187 $progname =~ s,.*[\\/],,; # use basename only
188 $progname =~ s/\.\w*$//; # strip extension, if any
197 use URI::Heuristic qw(uf_uri);
199 use HTTP::Status qw(status_message);
200 use HTTP::Date qw(time2str str2time);
203 # This table lists the methods that are allowed. It should really be
204 # a superset for all methods supported for every scheme that may be
205 # supported by the library. Currently it might be a bit too HTTP
206 # specific. You might use the -f option to force a method through.
208 # "" = No content in request, "C" = Needs content in request
221 # We make our own specialization of LWP::UserAgent that asks for
222 # user/password if document is protected.
224 package RequestAgent;
225 @ISA = qw(LWP::UserAgent);
229 my $self = LWP::UserAgent::new(@_);
230 $self->agent("lwp-request/$main::VERSION");
234 sub get_basic_credentials
236 my($self, $realm, $uri) = @_;
237 if ($main::options{'C'}) {
238 return split(':', $main::options{'C'}, 2);
241 my $netloc = $uri->host_port;
242 print "Enter username for $realm at $netloc: ";
245 return (undef, undef) unless length $user;
247 system("stty -echo");
248 my $password = <STDIN>;
250 print "\n"; # because we disabled echo
252 return ($user, $password);
255 return (undef, undef)
260 $method = uc(lc($progname) eq "lwp-request" ? "GET" : $progname);
266 'a', # content i/o in text(ascii) mode
268 'f', # make request even if method is not in %allowed_methods
271 'i=s', # if-modified-since
272 'c=s', # content type for POST
273 'C=s', # credentials for basic authorization
274 'H=s@', # extra headers, form "Header: value string"
276 'u', # display method, URL and headers of request
277 'U', # display request headers also
278 's', # display status code
279 'S', # display whole chain of status codes
280 'e', # display response headers (default for HEAD)
281 'd', # don't display content
286 'x', # extra debugging info
288 'P', # don't load proxy setting from environment
290 'o=s', # output format
293 Getopt::Long::config("noignorecase", "bundling");
294 unless (GetOptions(\%options, @getopt_args)) {
299 my $DISTNAME = 'libwww-perl-' . LWP::Version();
301 This is lwp-request version $VERSION ($DISTNAME)
303 Copyright 1995-1999, Gisle Aas.
305 This program is free software; you can redistribute it and/or
306 modify it under the same terms as Perl itself.
310 usage() if $options{'h'} || !@ARGV;
312 LWP::Debug::level('+') if $options{'x'};
314 # Create the user agent object
315 $ua = RequestAgent->new;
317 # Load proxy settings from *_proxy environment variables.
318 $ua->env_proxy unless $options{'P'};
320 $method = uc($options{'m'}) if defined $options{'m'};
324 $allowed_methods{$method} = "C"; # force content
327 $allowed_methods{$method} = "";
330 elsif (!defined $allowed_methods{$method}) {
331 die "$progname: $method is not an allowed method\n";
334 if ($method eq "HEAD") {
336 $options{'e'} = 1 unless $options{'d'};
340 if (defined $options{'t'}) {
341 $options{'t'} =~ /^(\d+)([smh])?/;
342 die "$progname: Illegal timeout value!\n" unless defined $1;
345 $timeout *= 60 if $2 eq "m";
346 $timeout *= 3600 if $2 eq "h";
348 $ua->timeout($timeout);
351 if (defined $options{'i'}) {
352 if (-e $options{'i'}) {
356 $time = str2time($options{'i'});
357 die "$progname: Illegal time syntax for -i option\n"
358 unless defined $time;
360 $options{'i'} = time2str($time);
364 if ($allowed_methods{$method} eq "C") {
365 # This request needs some content
366 unless (defined $options{'c'}) {
367 # set default content type
368 $options{'c'} = ($method eq "POST") ?
369 "application/x-www-form-urlencoded"
373 die "$progname: Illegal Content-type format\n"
374 unless $options{'c'} =~ m,^[\w\-]+/[\w\-.+]+(?:\s*;.*)?$,
376 print "Please enter content ($options{'c'}) to be ${method}ed:\n"
378 binmode STDIN unless -t or $options{'a'};
379 $content = join("", <STDIN>);
382 die "$progname: Can't set Content-type for $method requests\n"
383 if defined $options{'c'};
386 # Set up a request. We will use the same request object for all URLs.
387 $request = HTTP::Request->new($method);
388 $request->header('If-Modified-Since', $options{'i'}) if defined $options{'i'};
389 for my $user_header (@{ $options{'H'} || [] }) {
390 my ($header_name, $header_value) = split /:\s*/, $user_header, 2;
391 $request->header($header_name, $header_value);
392 $ua->agent($header_value) if lc($header_name) eq "user-agent"; # Ugh!
394 #$request->header('Accept', '*/*');
395 if ($options{'c'}) { # will always be set for request that wants content
396 $request->header('Content-Type', $options{'c'});
397 $request->header('Content-Length', length $content); # Not really needed
398 $request->content($content);
403 # Ok, now we perform the requests, one URL at a time
404 while ($url = shift) {
405 # Create the URL object, but protect us against bad URLs
407 if ($url =~ /^\w+:/ || $options{'b'}) { # is there any scheme specification
408 $url = URI->new($url, $options{'b'});
409 $url = $url->abs($options{'b'}) if $options{'b'};
416 $@ =~ s/ at .* line \d+.*//;
422 $ua->proxy($url->scheme, $options{'p'}) if $options{'p'};
424 # Send the request and get a response back from the server
426 $response = $ua->request($request);
428 if ($options{'u'} || $options{'U'}) {
429 my $url = $response->request->url->as_string;
430 print "$method $url\n";
431 print $response->request->headers_as_string, "\n" if $options{'U'};
435 printResponseChain($response);
437 elsif ($options{'s'}) {
438 print $response->status_line, "\n";
443 print $response->headers_as_string;
444 print "\n"; # separate headers and content
447 unless ($options{'d'}) {
449 $response->content_type eq 'text/html') {
454 if ($@ =~ m,^Can't locate HTML/Parse.pm in \@INC,) {
455 die "The HTML-Tree distribution need to be installed for the -o option to be used.\n";
461 my $html = HTML::Parse::parse_html($response->content);
463 $options{'o'} eq 'ps' && do {
464 require HTML::FormatPS;
465 my $f = HTML::FormatPS->new;
466 print $f->format($html);
469 $options{'o'} eq 'text' && do {
470 require HTML::FormatText;
471 my $f = HTML::FormatText->new;
472 print $f->format($html);
475 $options{'o'} eq 'html' && do {
476 print $html->as_HTML;
479 $options{'o'} eq 'links' && do {
480 my $base = $response->base;
481 $base = $options{'b'} if $options{'b'};
482 for ( @{ $html->extract_links } ) {
483 my($link, $elem) = @$_;
484 my $tag = uc $elem->tag;
485 $link = URI->new($link)->abs($base)->as_string;
486 print "$tag\t$link\n";
490 $options{'o'} eq 'dump' && do {
494 # It is bad to not notice this before now :-(
495 die "Illegal -o option value ($options{'o'})\n";
499 binmode STDOUT unless $options{'a'};
500 print $response->content;
504 $errors++ unless $response->is_success;
510 sub printResponseChain
513 return unless defined $response;
514 printResponseChain($response->previous);
515 my $method = $response->request->method;
516 my $url = $response->request->url->as_string;
517 my $code = $response->code;
518 print "$method $url --> ", $response->status_line, "\n";
525 Usage: $progname [-options] <url>...
526 -m <method> use method for the request (default is '$method')
527 -f make request even if $progname believes method is illegal
528 -b <base> Use the specified URL as base
529 -t <timeout> Set timeout value
530 -i <time> Set the If-Modified-Since header on the request
531 -c <conttype> use this content-type for POST, PUT, CHECKIN
532 -a Use text mode for content I/O
533 -p <proxyurl> use this as a proxy
534 -P don't load proxy settings from environment
535 -H <header> send this HTTP header (you can specify several)
536 -C <username>:<password>
537 provide credentials for basic authentication
539 -u Display method and URL before any response
540 -U Display request headers (implies -u)
541 -s Display response status code
542 -S Display response status chain
543 -e Display response headers
544 -d Do not display content
545 -o <format> Process HTML content in various ways
547 -v Show program version
548 -h Print this message
550 -x Extra debugging output