#!/usr/bin/perl
# vim: set ts=4 :
use strict; use warnings FATAL => 'all'; use feature qw(switch say state);

use version 0.77; our $VERSION = version->declare(sprintf "v%s", q$Revision: 1.4 $ =~ /(\d[.\d]+)/);

use File::Basename qw(basename dirname);

#
# Download files in parallel, both internally (multiple connections per file)
# and multiple files at once.  See comments in the FastDownloader.pm library.
#
# Author: Jim Avera  (jim.avera AT gmail dot com)
#         http://abhweb.org/jima

my $USAGE = "USAGE:
${\basename($0)} [GENERAL OPTIONS] file...

Each <file> may be any of these forms:

  A single url string (same as -u thestring)

  -letters arg arg ...  where <letters> indicates what kind of args follow,
                         and may be any of the following in any order:
                           u - url to download from 
                           s - expected file size
                           c - hex checksum (md5 by default; may be ALG:sum)
                           o - outpath filename.  May be absolute or relative
                               to destdir (default: current directory)

For example, to download to /tmp/myfile.pdf:

  ${\basename($0)} -d /tmp -uo http://srvr/x/foo.pdf myfile.pdf
  
To download /cache/dir/foo.deb and /cache/dir/bar.deb, checking
the size of the first and the md5 sum of the second,
and downloading a third file to an absolute path checking both size an md5:
 
  ${\basename($0)} -d /cache/dir \\
    -us http://host1/x/y/foo.deb 1234 \\
    -uc http://host2/p/q/bar.deb 401b30e3b8b5d629635a5c613cdb7919 \\
    -uosc http://host3/a/b/mum.ble /path/to/mymum.ble 1234 401b3...

GENERAL OPTIONS
  -d,--destdir default-destination-directory
  -n,--dry-run
  -j,--num-threads <num
  -q,--quiet
  -s,--silent
  --debug
";    

use threads;
use threads::shared;
use Getopt::Long qw(GetOptions);
use Digest ();

# The parallel downloader has now been made into a library
use lib "/home/jima/lib/perl";
use FastDownloader;

my $MiB = 1024*1024;

#------------------------------------------------------
# Get the general options, leaving the rest in @ARGV
#------------------------------------------------------
my ($destdir, $num_threads, $dryrun, $quiet, $silent, $debug, $help);
Getopt::Long::Configure ("default", "gnu_getopt", "pass_through", "auto_version");
GetOptions(
  "d|destdir=s"          => \$destdir,
  "n|dry-run"            => \$dryrun,
  "q|quiet"              => \$quiet,
  "s|silent"             => \$silent,
  "debug"                => \$debug,
  "j|jobs|num-threads:i" => \$num_threads,
  "h|help"               => \$help,
) or die $USAGE."(bad args)\n";
if ($help) { print $USAGE; exit 0; }
$num_threads //= 2;
die "Invalid number of threads: $num_threads\n" 
    unless $num_threads =~ /^[1-9]\d*$/;
$destdir //= "."; 
die "westdir '$destdir' is not a directory\n" unless -d $destdir;
$quiet = $silent = 0 if $debug;
$quiet //= $silent;

#------------------------------------------------------
# Parse the special-form args giving the files to download etc.
#------------------------------------------------------
my @items;
while (@ARGV) {
    if ($ARGV[0] =~ /^-([a-z]+)$/) {
        my $letters = $1;
        shift @ARGV;
        my %args;
        foreach (split //, $letters) {
            die "Unrecognized letter '$_' in -$letters\n"
              unless /^[usco]$/;
            die "Not enough args following '-$letters'\n" if @ARGV==0;
            $args{$_} = shift @ARGV;
        }
        die "URL must be specified ('-$letters' is not valid)\n" 
          unless $args{u};
        push @items, [ @args{qw/u s c o/} ];
    }
    elsif ($ARGV[0] =~ /^\w+:\/\//) {
        push @items, [$ARGV[0],undef,undef,undef]; 
        shift @ARGV;
    }
    else {
        die "Invalid special argument '$ARGV[0]'\n";
    }
}
foreach (@items) {
    my ($u, $s, $c, $o) = @$_;
    die "Invalid URL '$u'\n" if $u !~ m#^\w+://[^/]+/#;
    die "Invalid size '$s'\n" if defined($s) && $s !~ /^\d+$/;
    die "Invalid digest checksum'$c'\n" if defined($c) && $c !~ /^(\w+:)?[0-9a-fA-F]+$/;
}

#------------------------------------------------------
# Do the downloads
#------------------------------------------------------

my @fspecs;
foreach (@items) {
    my ($u, $s, $c, $o) = @$_;
    push @fspecs, { url           =>  $u,
                    local_file    =>  $o,
                    destdir       =>  $destdir,
                    callback      =>  "filecheck_callback",
                    # These are for us, used by the callback:
                    ExpectedSize  =>  $s,
                    Csum          =>  $c,
                  };
}
  
my $obj = FastDownloader->new(
                               filespecs   => \@fspecs,
                               num_threads => $num_threads,
                               dryrun      => $dryrun,
                               quiet       => $quiet,
                               debug       => $debug,
                             );
$obj->wait();
my $stats = $obj->get_stats();

#------------------------------------------------------
# Print statistics
#------------------------------------------------------

unless ($silent) {

    if (! $stats->{failed_file_count} && $stats->{file_count} == 0) {
        warn "> No files were downloaded.\n";
    } else {
        printf STDERR "> Downloaded $stats->{file_count} files (%.1f MiB",
                      ($stats->{total_bytes} / $MiB);
        if ($stats->{elapsed_secs} != 0) {
            printf STDERR ' @ %.1f MiB/sec', 
                          ($stats->{total_bytes} / $MiB)/$stats->{elapsed_secs};
        } else {
            print STDERR " in less than one second";
        }
        print STDERR ") using $num_threads threads\n";

        if ($stats->{failed_file_count}) {
            warn "> $stats->{failed_file_count} FAILED to download successfully\n";
        }
    }
} 

exit 0;

sub filecheck_callback($$$) {
    my ($fspec, $temp_path, $final_path) = @_;

    # Check the file size
    if (defined $fspec->{ExpectedSize}) {
        my $actual_size = (-s $temp_path);
        if ($actual_size != $fspec->{ExpectedSize}) {
            warn "File-size mis-match on $temp_path\n",
                "   Expected $fspec->{ExpectedSize} bytes, got $actual_size bytes\n";
            return 0;
        }
    }

    # Check the hash.  The Csum prefix is normally "MD5Sum:" 
    # but could specify another algorithm.
    if (defined $fspec->{Csum}) {
        my ($hashname, $expected_digest) = ($fspec->{Csum} =~ /(\w+)Sum:(.*)/);
        my $hasher = Digest->new($hashname); # e.g "MD5"
        open(my $fhandle, "<$temp_path") or die "$temp_path : $!";
        binmode $fhandle;
        $hasher->addfile($fhandle);
        my $actual_digest = $hasher->hexdigest();
        if ($actual_digest ne $expected_digest) {
            warn "$hashname mis-match on $temp_path\n",
                "   expected: $expected_digest\n",
                "     actual: $actual_digest\n";
            return 0;
        }
    }

    return 1;
}