#!/usr/bin/perl
package Scripts::cpanelsync;

# cpanel - cpanelsync                             Copyright(c) 2010 cPanel, Inc.
#                                                           All rights Reserved.
# copyright@cpanel.net                                         http://cpanel.net
# This code is subject to the cPanel license. Unauthorized copying is prohibited

## "grep '###' cpanelsync" gives an overview of the logic and files downloaded

BEGIN {
    my $running_in_debugger = exists $INC{'perl5db.pl'};
    if ($running_in_debugger) {
        $ENV{'LANG'} = 'C';
    }

    if ( $ENV{'LANG'} ne 'C' ) {
        $ENV{'LANG'} = 'C';
        exec $0, @ARGV;
        die 'Failed to recreate self in a sane env';
    }
    unshift @INC, '/usr/local/cpanel';
}

use strict;
use warnings;
use Socket;
use Cpanel::Tar         ();
use Cpanel::HttpRequest ();
use Cpanel::SafeDir::MK ();
use bytes;    # must import no () allowed

# Package global; redefined in unit tests
our $cpanelsync_excludes       = '/etc/cpanelsync.exclude';
our $cpanelsync_chmod_excludes = '/etc/cpanelsync.no_chmod';

## if invoked as a script, there is nothing in the call stack
my $invoked_as_script = !caller();
__PACKAGE__->script(@ARGV) if ($invoked_as_script);

sub script {
    my ( $package, @argv ) = @_;
    $| = 1;

    my $gotSigALRM = 0;

    $SIG{'INT'} = 'IGNORE';

    my $hasmd5 = 0;
    eval { require Digest::MD5; $hasmd5 = 1; };
    my $hasbzip2 = 0;
    eval { require Compress::Bzip2; $hasbzip2 = 1; };
    my $has_rollback;
    eval { require Cpanel::RollBack; $has_rollback = 1; };

    my $nfok = 0;
    if ( $argv[0] && $argv[0] =~ m/404/ ) {
        $nfok = 1;
        shift @argv;
    }
    my $learn_repo = 0;
    my $repomode   = 0;
    my $repo;
    if ( $argv[0] && $argv[0] =~ m/\-\-repo/ ) {
        shift @argv;
        $repomode = 1;
        $repo     = shift(@argv);
    }
    if ( $argv[0] && $argv[0] =~ m/\-\-learnrepo/ ) {
        shift(@argv);
        $learn_repo = 1;
    }

    my $exit_code = 0;

    my $connecttimeout = 25;
    my $host           = $argv[0] || '';
    my $url            = $argv[1] || '/';
    my $root           = $argv[2] || '/';

    if ( !$host || $host eq '' ) {
        die "$0: Usage $0 <host> <uri> <syncroot>";
    }

    eval {
        no warnings;
        local $SIG{'__DIE__'};
        local $SIG{'__WARN__'};
        require Cpanel::Carp;
        local $SIG{'__DIE__'};
        local $SIG{'__WARN__'};
        Cpanel::Carp::enable();
        $Cpanel::Carp::OUTPUT_FORMAT = 'supress';
    };

    if ($has_rollback) {
        my $append_to_rollback_conf_ref = UNIVERSAL::can( 'Cpanel::RollBack', 'append_to_rollback_conf' );
        if ( defined $append_to_rollback_conf_ref ) {
            $append_to_rollback_conf_ref->($root);
        }
    }

    my $httpClient = Cpanel::HttpRequest->new();
    if ( !-d $root ) {
        Cpanel::SafeDir::MK::safemkdir( $root, '0755', 2 );
        if ( !-d $root ) {
            die "Unable to create directory $root";
        }
    }

    my $lock = 'locked';

    my $sleep      = 30;
    my $lock_count = 0;
    my $trycount   = 0;

    # Check for Locked file
    ### "$url/.cpanelsync.lock",
    while ( $lock =~ m/locked/i ) {
        my ( $lock, $status ) = $httpClient->request(
            'exitOn404' => $nfok,
            'host'      => $host,
            'url'       => "$url/.cpanelsync.lock",

            #http 1.1 on the first request, but drop the connection on the
            #second one to not tie up the server
            'protocol' => ( $lock_count == 0 ? 1 : 0 )
        );
        exit if ( $status == 0 );
        last if ( !$lock || $lock !~ m/locked/i );
        $lock_count++;
        if ( $lock_count > 20 ) {
            $sleep += 30;
        }
        elsif ( $lock_count == 20 ) {
            $sleep = 120;
        }

        # if these next lines are changed, update /scripts/cpanel_easy_sanity_check's check for them as well
        print "The update server is currently updating its files.\n";
        print "It may take up to 30 minutes before access can be obtained.\n";
        print "Waiting $sleep seconds for access to the update server......\n";

        $httpClient->disconnect();    #do not leave the connection open

        if ( ++$trycount % 30 == 0 ) { $httpClient->skiphost(); }
        sleep $sleep;
        print "Checking again....\n";

    }

    ### "$url/.cpanelsync.version",
    if ($repomode) {
        $exit_code = check_repo_version( $repo, $learn_repo, $httpClient, $host, $url );
    }

    my $dotcpanelsync = "$root/.cpanelsync";

    my %OLDFILES;
    my %NEWFILES;
    if ( !-e $dotcpanelsync ) {    ###
        ## if .cpanelsync does not exist, download/extract the .tar.bz2
        my $basedir = $root;
        my @DIRS = split m{ [/] }xms, $basedir;
        pop @DIRS;
        $basedir = join '/', @DIRS;

        my $basename = $url;
        my @BDIR = split m{ [/] }xms, $basename;
        $basename = pop @BDIR;

        if ( $basedir eq '' ) { $basedir = '/'; }

        my $bz2 = "$basedir/$basename.tar.bz2";
        unlink $bz2;
        ### "http://${host}${url}.tar.bz2"
        downloadfile( $httpClient, "http://${host}${url}.tar.bz2", $bz2 );

        my $tarcfg = Cpanel::Tar::load_tarcfg();
        system( $tarcfg->{'bin'}, '-x', '-p', $tarcfg->{'no_same_owner'}, '-j', '-v', '-C', $basedir, '-f', $bz2 );
        unlink $bz2;

        ## TODO?: take the md5sums of the new files, and manipulate %OLDFILES (or %MD5LIST?)
    }
    else {
        if ( open my $cpsync_fh, '<', $dotcpanelsync ) {
            while (<$cpsync_fh>) {
                chomp;
                my ( $type, $file ) = ( split( /===/, $_ ) )[ 0, 1 ];
                if ($file) {
                    $OLDFILES{$file} = $type;
                }
            }
            close $cpsync_fh;
        }
    }

    my @FILELIST;
    $trycount = 0;
    my $usebz2  = 1;
    my $skipbz2 = 0;

    # Download file list to sync
    while (1) {    ###
        $trycount++;

        if   ( $trycount % 2 == 0 ) { $usebz2 = 0; }
        else                        { $usebz2 = 1; }

        ### "$url/.cpanelsync.bz2",  -or-
        ### "$url/.cpanelsync"
        ## note: this overwrites the local $dotcpanelsync[.bz2] file
        if ( !$skipbz2 && $usebz2 ) {
            $httpClient->request(
                'host'     => $host,
                'url'      => "$url/.cpanelsync.bz2",
                'protocol' => 1,
                'destfile' => "$dotcpanelsync.bz2"
            );

            my $size = ( stat("$dotcpanelsync.bz2") )[7];
            if ( $size && $size > 0 ) {
                unbzip2( $hasbzip2, "$dotcpanelsync.bz2" );
            }
        }
        else {
            $httpClient->request(
                'host'     => $host,
                'url'      => "$url/.cpanelsync",
                'protocol' => 1,
                'destfile' => $dotcpanelsync
            );
        }

        if ( -e $dotcpanelsync ) {
            open( FL, $dotcpanelsync );
            while (<FL>) {
                chomp;
                push( @FILELIST, $_ );
            }
            close(FL);
        }

        if ( -e "$dotcpanelsync.bz2" ) {
            unlink "$dotcpanelsync.bz2";
            $skipbz2 = 1;
        }

        last if ( @FILELIST && $FILELIST[-1] eq '.' );

        if ( $trycount > 1 ) {
            if ( $trycount == 10 ) {
                print "Tried to download the sync file 10 times and failed!\n";
                ## TODO?: document exit codes
                exit 1;
            }
            downloadfailed($httpClient);
        }
    }

    # Global excludes for handling excluded files from update or permission checks
    my @excludes       = get_excludes($cpanelsync_excludes);
    my @chmod_excludes = get_excludes($cpanelsync_chmod_excludes);

    my %MD5LIST;
    if ( -e "$dotcpanelsync.md5s" ) {
        loadmd5s( \%MD5LIST, $root );
    }

    foreach my $fileinfo (@FILELIST) {    ###
        chomp $fileinfo;
        next if ( $fileinfo eq '.' );

        ## note: appending 'r' to these vars to denote that they represent info on the
        ##   remote (incoming) resource. Ideally, they should be packaged in a hash,
        ##   similar to how $target is handled.
        ## $rextra is either an md5 for 'file' $ftype, or symlink destination
        my ( $rtype, $rfile, $rperm, $rextra ) = split( /===/, $fileinfo );

        my $target_info = lstat_target( $root, $rfile );

        ## using %04d as $rperm is a string (comes from the .cpanelsync file)
        $rperm = sprintf( "%04d", $rperm );

        prune_OLDFILES( \%OLDFILES, $rfile, $rtype );

        next if is_excluded( \@excludes, $root, $rfile );

        if ( $rtype eq 'f' ) {
            handle_file( $target_info, \%MD5LIST, $root, $rfile, $hasmd5, $rextra, $skipbz2, $httpClient, $host, $url, $hasbzip2, $repomode, $exit_code, $repo, \@chmod_excludes, $rperm, \%NEWFILES );    ###
        }
        elsif ( $rtype eq 'd' ) {
            handle_dir( $target_info, \@chmod_excludes, $root, $rfile, $rperm );
        }
        elsif ( $rtype eq 'l' ) {
            handle_symlink( $target_info, $rextra );
        }
    }

    my $saferoot = $root;
    $saferoot =~ s/\.\///g;

    handle_deletes( \@FILELIST, \%OLDFILES, $saferoot );

    write_newlist( \%NEWFILES, $saferoot );

    writemd5s( \%MD5LIST, $saferoot );

    ### "$url/.cpanelsync.version",
    if ( $exit_code == 0 && $repomode ) {
        $exit_code = check_repo_version( $repo, 0, $httpClient, $host, $url );
    }

    if ( -x '/scripts/cpanelsync_postprocessor' ) {
        system '/scripts/cpanelsync_postprocessor', $saferoot;
    }
    if ( -x '/scripts/cpanelsync_postprocessor.custom' ) {
        system '/scripts/cpanelsync_postprocessor.custom', $saferoot;
    }

    if ($invoked_as_script) {
        exit $exit_code;
    }
    return $exit_code;
}

sub downloadfile {
    my ( $httpClient, $file, $where ) = @_;

    $file =~ m!http://([^/]+)(.*)!;

    my $host = $1;
    my $url  = $2;

    $httpClient->request(
        'host'     => $1,
        'url'      => $2,
        'protocol' => 1,
        'destfile' => $where,
    );

}

sub getmd5sum {
    my ( $hr_MD5LIST, $root, $file, $hasmd5, %OPTS ) = @_;

    return if ( !defined $file || $file eq '' );

    my $md5;
    my $skipcache = $OPTS{'skipcache'};
    my $mtime     = $OPTS{'mtime'};
    my $size      = $OPTS{'size'};

    my $fullfile = $root . '/' . $file;
    $fullfile =~ s/\/\.\//\//g;
    $fullfile =~ s/\/+/\//g;

    if ( defined $OPTS{'is_normal_file'} ) {
        if ( !$OPTS{'is_normal_file'} ) {
            return '';
        }
    }
    else {
        return '' if ( !-f $fullfile );
    }

    if ( !$size || !$mtime ) {
        ( $size, $mtime ) = ( stat($fullfile) )[ 7, 9 ];
    }

    if (   !$skipcache
        && defined $hr_MD5LIST->{$file}{'size'}
        && $hr_MD5LIST->{$file}{'size'} == $size
        && defined $hr_MD5LIST->{$file}{'mtime'}
        && $hr_MD5LIST->{$file}{'mtime'} == $mtime
        && $hr_MD5LIST->{$file}{'md5'} ne '' ) {
        $hr_MD5LIST->{$file}{'used'} = 1;
        return ( $hr_MD5LIST->{$file}{'md5'} );
    }

    if ($hasmd5) {
        my $fsize = ( stat($fullfile) )[7];
        if ( open my $md5_fh, '<', $fullfile ) {
            my $ctx = Digest::MD5->new;

            my $buf = '';
            my $len = 0;

            while ( my $n = read( $md5_fh, $buf, 65535 ) ) {
                $len += bytes::length($buf);
                $ctx->add($buf);
            }

            $md5 = $ctx->hexdigest() if $len == $fsize;
        }
        else {
            warn "Unable to open $fullfile: $!";
            system 'stat', $fullfile;    #give more debug info
            return '';
        }
    }
    else {
        if ( $ENV{'PATH'} !~ m/\/sbin/ ) {
            $ENV{'PATH'} .= ':/sbin';
        }

        if (   -e '/bin/md5sum'
            || -e '/usr/bin/md5sum'
            || -e '/usr/local/bin/md5sum' ) {
            open( MD5, '-|' ) || exec 'md5sum', $fullfile;
        }
        else {
            open( MD5, '-|' ) || exec 'md5', '-r', $fullfile;
        }
        while (<MD5>) {
            chomp();
            ( $md5, undef ) = split( /\s+/, $_ );
            last();
        }
        close(MD5);
    }

    $hr_MD5LIST->{$file}{'size'}  = $size;
    $hr_MD5LIST->{$file}{'mtime'} = $mtime;
    $hr_MD5LIST->{$file}{'md5'}   = $md5;
    $hr_MD5LIST->{$file}{'used'}  = 1;

    return $md5;
}

sub get_excludes {
    my ($file) = @_;

    return if ( !-e $file || -z _ );

    my @excludes;
    open( EX, '<', $file ) or return undef;
    while (<EX>) {
        next if m/^\s*$/;
        chomp;
        s!/$!!;
        push @excludes, $_;
    }
    close(EX);

    return @excludes;
}

sub loadmd5s {
    my ( $hr_MD5LIST, $dir ) = @_;

    open( MD5, '<', $dir . '/.cpanelsync.md5s' );
    while (<MD5>) {
        chomp();
        my ( $filename, $size, $mtime, $md5 ) = split( /:::/, $_, 4 );
        $hr_MD5LIST->{$filename}{'size'}  = $size;
        $hr_MD5LIST->{$filename}{'mtime'} = $mtime;
        $hr_MD5LIST->{$filename}{'md5'}   = $md5;
    }
    return;
}

sub write_newlist {
    my ( $hr_NEWFILES, $dir ) = @_;

    $dir =~ s/\/$//g;
    open( my $new_fh, '>', $dir . '/.cpanelsync.new' ) || do {
        warn "Could not write new list: " . $dir . '/.cpanelsync.new';
        return;
    };
    foreach my $filename ( keys %$hr_NEWFILES ) {
        print {$new_fh} $filename . "\n";
    }
    close($new_fh);
}

sub writemd5s {
    my ( $hr_MD5LIST, $dir ) = @_;

    $dir =~ s/\/$//g;
    open( MD5, '>', $dir . '/.cpanelsync.md5s' ) || do {
        warn "Could not write md5 cache: " . $dir . '/.cpanelsync.md5s';
        return;
    };
    foreach my $filename ( keys %$hr_MD5LIST ) {
        next if ( !$hr_MD5LIST->{$filename}{'used'} || substr( $filename, 0, 1 ) eq '/' );
        print MD5 join( ':::', $filename, $hr_MD5LIST->{$filename}{'size'}, $hr_MD5LIST->{$filename}{'mtime'}, $hr_MD5LIST->{$filename}{'md5'} ) . "\n";
    }
    close(MD5);
}

sub downloadfailed {
    my ($httpClient) = @_;
    print 'Download Failed... trying again...in..';
    if ($httpClient) {
        $httpClient->disconnect();
    }
    my $sleepsecs = 60;
    for ( my $i = $sleepsecs; $i > 0; $i-- ) {
        print '..' . $i . '..';
        sleep 1;
    }
}

sub unbzip2 {
    my ( $hasbzip2, $file ) = @_;

    if ($hasbzip2) {
        my $outfile = $file;
        $outfile =~ s/\.bz2$//g;
        if ( $outfile eq $file ) { return; }
        my $out_fh;
        open( $out_fh, '>', $outfile ) || do {
            print STDERR "cpanelsync: unbzip2: error opening $outfile\n";
            return;
        };
        my $bzip2 = Compress::Bzip2->new( '-verbosity' => 0 );
        $bzip2->bzopen( $file, 'r' );
        my $buf;
        my $read;
        while ( $read = $bzip2->bzread( $buf, 65535 ) ) {    #65535 is about 35% faster then 512
            if ( $read < 0 ) {
                print "error: $Compress::Bzip2::bzerrno\n";
                print STDERR "error: $Compress::Bzip2::bzerrno\n";
                last;
            }
            syswrite( $out_fh, $buf, $read );
        }
        close($out_fh);
        unlink($file);
    }
    else {
        system( 'bzip2', '-df', $file );
    }
}

sub check_repo_version {
    my ( $repo, $learn, $httpClient, $host, $url ) = @_;
    my $local_repo_v;
    if ( !-e '/var/cpanel' )            { mkdir( '/var/cpanel',            0755 ); }
    if ( !-e '/var/cpanel/cpanelsync' ) { mkdir( '/var/cpanel/cpanelsync', 0755 ); }
    if ( !-e '/var/cpanel/cpanelsync/repoversions' ) {
        mkdir( '/var/cpanel/cpanelsync/repoversions', 0755 );
    }
    my $repo_fh;
    open( $repo_fh, '<', '/var/cpanel/cpanelsync/repoversions/' . $repo ) && do {
        $local_repo_v = readline($repo_fh);
        chomp($local_repo_v);
        close($repo_fh);
    };
    my ( $remote_repo_v, $status ) = $httpClient->request(
        'exitOn404' => 0,
        'host'      => $host,
        'url'       => "$url/.cpanelsync.version",
        'protocol'  => 1,
    );
    if ($remote_repo_v) {
        if ( !$local_repo_v || $remote_repo_v ne $local_repo_v ) {
            open( my $repo_fh, '>', '/var/cpanel/cpanelsync/repoversions/' . $repo );
            print {$repo_fh} $remote_repo_v;
            close($repo_fh);
            if ( !$learn && $local_repo_v ) {
                print "Repo: $repo : version changed from $local_repo_v to $remote_repo_v in mid sync.  Sync needs to be restarted. (learn=$learn)\n";
                return 16;
            }
            else {
                print "Repo: $repo : learned new version: $remote_repo_v (learn=$learn)\n";
            }
        }
        else {
            print "Repo: $repo : check passed : local=$local_repo_v & remote=$remote_repo_v (learn=$learn)\n";
        }
    }
    return 0;
}

sub is_excluded {
    my ( $ar_excludes, $root, $rfile ) = @_;
    my @excludes = @$ar_excludes;

    if (@excludes) {
        my $clean_rfile = $rfile;
        $clean_rfile =~ s!^\./!!;
        my $absfile = $root . '/' . $clean_rfile;
        $absfile =~ tr{/}{}s;
        $absfile =~ s{ [/] \z }{}xmsg;

        ## Note: to take advantage of the "implicit" exclusion of a directory's contents, as written
        ##   the explicitly listed exclude directory must exist at the installation site.
        if ( grep { $_ eq $absfile || ( -d $_ && $absfile =~ m/^\Q$_\E\// ) } @excludes ) {
            print "Skipping sync of $absfile (check /etc/cpanelsync.exclude)\n";
            return 1;
        }

        ## Maintain support for old broken behavior --------------
        $absfile = $root . '/' . $rfile;
        $absfile =~ tr{/}{}s;
        $absfile =~ s{ [/] \z }{}xmsg;
        if ( grep { $_ eq $absfile } @excludes ) {
            print "Skipping sync of $absfile (check /etc/cpanelsync.exclude)\n";
            return 1;
        }
        ## -------------------------------------------------------
    }
    return;
}

sub in_chmod_excludes {
    my ( $ar_chmod_excludes, $root, $rfile ) = @_;
    if ( scalar @$ar_chmod_excludes ) {
        my $clean_rfile = $rfile;
        $clean_rfile =~ s/^\.\///;
        my $absfile = $root . '/' . $clean_rfile;
        $absfile =~ tr{/}{}s;
        $absfile =~ s{ [/] \z }{}xmsg;
        return 1 if ( grep { $_ eq $absfile } @$ar_chmod_excludes );
    }
    return;
}

sub handle_symlink {
    my ( $target, $rextra ) = @_;

    my $dolink = 0;
    if ( !$target->{'exists'} ) {
        $dolink = 1;
    }
    elsif ( $target->{'islnk'} ) {
        if ( readlink $target->{'path'} ne $rextra ) {
            unlink $target->{'path'};
            $dolink = 1;
        }
    }
    elsif ( $target->{'isnormfile'} ) {
        unlink $target->{'path'};
        $dolink = 1;
    }
    elsif ( $target->{'isdir'} ) {
        system 'rm', '-rf', '--', $target->{'path'};
        $dolink = 1;
    }

    if ($dolink) {
        if ( symlink( $rextra, $target->{'path'} ) ) {
            print "Created symlink $target->{'path'} -> $rextra successfully\n";
        }
        else {
            print "Failed to create symlink $target->{'path'} -> $rextra: $!\n";
        }
    }

    return;
}

sub handle_dir {
    my ( $target, $ar_chmod_excludes, $root, $rfile, $rperm ) = @_;
    ## note: $rperm is an octal string (e.g. '0751')

    if ( $target->{'islnk'} || $target->{'isnormfile'} ) {
        unlink $target->{'path'};
        $target->{'exists'} = 0;
    }

    if ( !$target->{'exists'} ) {
        ## FIX: used to be created with a hardcoded mode of '0755'. The only case this
        ##   will not account for is a new directory that is also in chmod_excludes. I believe
        ##   this to be a very edge case.
        if ( Cpanel::SafeDir::MK::safemkdir( $target->{'path'}, $rperm, 2 ) ) {
            print "Created directory $target->{'path'} successfully\n";
        }
    }
    elsif ( !in_chmod_excludes( $ar_chmod_excludes, $root, $rfile )
        && sprintf( "%04o", ( $target->{'perm'} & 07777 ) ) ne $rperm ) {
        if ( chmod( oct($rperm), $target->{'path'} ) ) {
            print "Directory $target->{'path'} verified\n";
        }
        else {
            print "Failed to update permissions on directory $target->{'path'}: $!";
        }
    }
}

sub handle_file {    ###
    my ( $target, $hr_MD5LIST, $root, $rfile, $hasmd5, $rextra, $skipbz2, $httpClient, $host, $url, $hasbzip2, $repomode, $exit_code, $repo, $ar_chmod_excludes, $rperm, $newfiles_ref ) = @_;

    if ( $target->{'isdir'} ) {
        system 'rm', '-rf', '--', $target->{'path'};
    }
    elsif ( $target->{'islnk'} ) {
        unlink $target->{'path'};
    }

    if (
        ( $target->{'isdir'} || $target->{'islnk'} || !$target->{'exists'} )
        || getmd5sum(
            $hr_MD5LIST, $root, $rfile, $hasmd5,
            'mtime'          => $target->{'mtime'},
            'size'           => $target->{'size'},
            'is_normal_file' => $target->{'isnormfile'}
        ) ne $rextra
      ) {
        my $dfile = $rfile;
        $dfile =~ s/^\.//g;

        my $trycount = 0;
        my $goodfile = 1;
        my $usebz2   = 1;

        my $pathtemp = $target->{'path'} . '-cpanelsync';
        while (1) {    ###
            $trycount++;
            if   ( $trycount % 2 == 0 ) { $usebz2 = 0; }
            else                        { $usebz2 = 1; }

            unlink($pathtemp);
            ### "http://${host}${url}${dfile}.bz2"  -or-
            ### "http://${host}${url}${dfile}"
            if ( !$skipbz2 && $usebz2 && $dfile !~ m/\.bz2$/ ) {
                downloadfile( $httpClient, "http://${host}${url}${dfile}.bz2", "$pathtemp.bz2" );
                my $size = ( stat("$pathtemp.bz2") )[7];
                if ( $size && $size > 0 ) {
                    unbzip2( $hasbzip2, "$pathtemp.bz2" );
                }
                if ( -e "$pathtemp.bz2" ) {
                    ## TODO: meaning what exactly? test with dashk
                    print "$pathtemp.bz2 still exists\n";
                    unlink "$pathtemp.bz2";
                    $skipbz2 = 1;
                    next;
                }
            }
            else {
                downloadfile( $httpClient, "http://${host}${url}${dfile}", $pathtemp );
            }

            my $check_md5sum = getmd5sum(
                $hr_MD5LIST, $root, $rfile . '-cpanelsync',
                $hasmd5, 'skipcache' => 1
            );
            last if ( $check_md5sum eq $rextra );
            my $size = ( stat( $rfile . '-cpanelsync' ) )[7] || 0;
            print "md5sum mismatch (actual: $check_md5sum) (expected: $rextra) (size: $size)\n";

            if ( $trycount > 1 ) {
                if ( $trycount % 3 == 0 ) { $httpClient->skiphost(); }
                if ( $trycount == 10 ) {
                    print "Tried to download the file $rfile 10 times and failed!\n";
                    if ($repomode) {
                        ## TODO: docuement exit codes
                        exit 16;
                    }
                    $goodfile = 0;
                    last;
                }

                ### "$url/.cpanelsync.version",
                if ($repomode) {
                    ## TODO: this does not read well.
                    $exit_code = check_repo_version( $repo, 0, $httpClient, $host, $url );
                    if ( $exit_code != 0 ) {
                        exit $exit_code;
                    }
                }

                downloadfailed($httpClient);
            }
        }
        if ($goodfile) {
            print "Got file $rfile ok (md5 matches)\n";
            _goodfile_handle_chmod(
                $ar_chmod_excludes, $root, $rfile, $target->{'perm'},
                $pathtemp,          $rperm
            );
            _goodfile_handle_rename( $target->{'path'} );
            $newfiles_ref->{ $target->{'path'} } = 1;
        }
        else {
            unlink $pathtemp;
        }
    }
    else {
        if (   !in_chmod_excludes( $ar_chmod_excludes, $root, $rfile )
            && $target->{'exists'}
            && ( sprintf( "%04o", ( $target->{'perm'} & 07777 ) ) ne $rperm ) ) {
            chmod( oct($rperm), $target->{'path'} );
        }
    }
}

sub _goodfile_handle_chmod {
    my ( $ar_chmod_excludes, $root, $rfile, $origperm, $pathtemp, $rperm ) = @_;
    ## if file matches the chmod exclude list, chmod the new temp file with
    ##   the mode from the old file
    if ( in_chmod_excludes( $ar_chmod_excludes, $root, $rfile ) ) {
        my $real_origperm = sprintf( "%04o", ( $origperm & 07777 ) );
        chmod( oct($real_origperm), $pathtemp );
    }
    else {
        chmod( oct($rperm), $pathtemp );
    }
}

sub _goodfile_handle_rename {
    my ($path) = @_;
    unlink $path;
    if ( -e $path ) {
        if ( rename( $path, $path . '.unlink' ) ) {
            unlink $path . '.unlink';
        }
        else {
            unlink $path;
        }
    }
    ## the "rename || unlink" clause ideally should warn the user that the file did not make it to
    ##   its production location. but this, as this runs as root, is hard-to-replicate.
    rename( $path . '-cpanelsync', $path ) || unlink( $path . '-cpanelsync' );
    return;
}

sub prune_OLDFILES {
    my ( $hr_OLDFILES, $rfile, $rtype ) = @_;
    if ( exists $hr_OLDFILES->{$rfile} ) {

        # Handle transition from directory to a symlink
        if ( $rtype eq 'l' && $hr_OLDFILES->{$rfile} ne 'l' ) {
            foreach my $old_file ( keys %$hr_OLDFILES ) {
                ## delete from hash all subdirs of $rfile, which is becoming a link
                if ( $old_file =~ m/^\Q$rfile\E\// ) {
                    delete $hr_OLDFILES->{$old_file};
                }
            }
        }
        delete $hr_OLDFILES->{$rfile};
    }
    return;
}

sub handle_deletes {
    my ( $ar_FILELIST, $hr_OLDFILES, $saferoot ) = @_;
    my @olddirectories;
    my %EXCLUDE_DELETE;

    if ( -e $saferoot . '/.cpanelsync.delete.exclude' && open( my $exc_fh, '<', $saferoot . '/.cpanelsync.delete.exclude' ) ) {
        %EXCLUDE_DELETE = map { chomp($_); $_ => undef } (<$exc_fh>);
        close($exc_fh);
    }

    ## note: loop on @FILELIST to prevent mass deletion on an inadvertantly empty .cpanelsync
    if ( scalar @$ar_FILELIST ) {
        ## adding 'sort' to guarantee an order for keys()
        foreach my $oldfile ( sort keys %$hr_OLDFILES ) {
            $oldfile =~ s/^\.\///g;

            my @BASEDIR = split( /\//, $saferoot . '/' . $oldfile );
            pop(@BASEDIR);
            my $basedir = join( '/', @BASEDIR );

            if ( -l $basedir ) {
                print "Skipping cleanse of $saferoot/$oldfile (within symlinked directory)\n";
                next;
            }

            if ( exists $EXCLUDE_DELETE{ $saferoot . '/' . $oldfile } ) {
                print "Excluding file removal from previous tree: $saferoot/$oldfile\n";
                next;
            }

            if ( -l $saferoot . '/' . $oldfile ) {
                ## ???: what sets the .keep files?
                next if -e $saferoot . '/' . $oldfile . '.keep';
                print "Removing symlink from previous tree: $saferoot/$oldfile\n";
                unlink $saferoot . '/' . $oldfile or print "Unable to remove deprecated symlink $saferoot/$oldfile: $!\n";
            }
            elsif ( -d $saferoot . '/' . $oldfile ) {
                push @olddirectories, $saferoot . '/' . $oldfile;
            }
            elsif ( -e _ ) {
                ## ???: what sets the .keep files?
                next if -e $saferoot . '/' . $oldfile . '.keep';
                print "Removing file from previous tree: $saferoot/$oldfile\n";
                unlink $saferoot . '/' . $oldfile or print "Unable to remove deprecated file $saferoot/$oldfile: $!\n";
            }
        }
    }
    foreach my $dir ( reverse sort @olddirectories ) {
        print "Removing directory from previous tree: $dir\n";
        rmdir $dir or print "Unable to remove deprecated directory $dir: $!\n";
    }

    return;
}

sub lstat_target {
    my ( $root, $rfile ) = @_;
    my %target;

    $target{'path'} = $root . '/' . $rfile;
    $target{'path'} =~ s/^\.\///;
    $target{'path'} =~ s/\/(?:\.\/)+/\//g;
    $target{'path'} =~ s/\/{2,}/\//g;

    my @_lstat = lstat( $target{'path'} );
    ## two slices to assign @_lstat indexes into %target
    @target{ 'perm', 'size', 'mtime' } = @_lstat[ 2, 7, 9 ];

    # if it was a symlink lstat didn't fall back to stat and we got the info on the
    # symlink instead of the info we wanted.  We call lstat as we know it will give the
    # information for the file if its not a symlink and -l will still work.  This saves us
    # from having to stat the file twice if its not a symlink which will be most of the time.

    $target{'isdir'}      = -d _;
    $target{'exists'}     = -e _;
    $target{'isnormfile'} = -f _;

    $target{'islnk'} = 0;
    ## note: I can not find a situation where "-l _" is true, where isdir and isnormfile are not
    ##   already false (namely '', as returned by stat)
    if ( -l _ ) {
        $target{'islnk'}      = 1;
        $target{'isdir'}      = 0;
        $target{'isnormfile'} = 0;
    }
    return \%target;
}

1;