#!/usr/bin/perl -w

# This program can take a base Fedora or related OS install and
# automatically install and configure LANforge, update the OS,
# kernel, etc.

# TODO: Script needs to check if an user specified option is already configured in files
#       instead of blindly adding options.

# NOTE:  To disable funky interface naming, use do_biosdevname=0
#  Rename and/or otherwise fix up the ifcfg-eth0 (or whatever is mgt dev)
#  Fix up LANforge port layout, mgt-dev.
#
#  From Isaac's notes on Fedora 17:
#- Cinnamon installed, but only runs on local display. VNC session is GNOME.
#
#  Start gnome-tweak-tool as user lanforge, go to Desktop menu,
#  turn ON 'Have file manager handle the desktop'
#
#-  Use gnome-applets to add system-monitor to top panel.
#
#   To add applets in VNC, hit F8 key to bring up VNC Menu, select Alt
#   then right-click top panel. F8 again to toggle Alt, then add applets.
#
#  Reboot problems:  edit /etc/modprobe.d/blacklist.conf and add:  blacklist mei
#
#
#  This can be run directly from a web site, on the to-be LANforge machine:
#  perl <(curl -s http://www.candelatech.com/lf_kinstall.txt) --do_all_ct --kver 3.5.7+ --lfver 5.2.7
#
# 2023-01-03 JBR: Updated file to 4-space intents!
#
package main;
use POSIX qw(ceil floor abs);
#use Time::HiRes qw( usleep );
use strict;
use warnings;
use Carp;
$SIG{ __DIE__ } = sub {
    $Carp::CarpLevel = 1;
    unshift(@_, "[STOPPING] ");
    Carp::confess(@_)
};
$| = 1;
our $debug_on = 0;
our @CARP_NOT;
my @debugsign = grep {/^-(d|debug|verbose)$/} @ARGV;
if (@debugsign > 0) {
    $debug_on++;
    # use diagnostics;
    use Data::Dumper;
    $SIG{ __WARN__ } = sub {
        $Carp::CarpLevel = 1;
        unshift(@_, "[warn] ");
        Carp::cluck(@_);
    };
}
use URI::Escape qw(uri_escape);
use Getopt::Long;
# use File::Copy qw(cp);
use File::Temp qw(tempfile tempdir);
use POSIX qw(strftime);
use MIME::Base64 qw();
use Cwd;
sub print_slow {
    for (@_) {
        if ($_ =~ "/\n$/") {
            print "$_";
        }
        else {
            print "$_\n";
        }
    }
    print "[press ctrl-c to exit] ...";
    my $wait_sec = 4;
    my $counter = 1;
    while(($wait_sec - $counter) >= 0) {
        sleep(1);
        print "\b" x $counter;
        print " ";
        $counter++;
    }
    print "\r                             \n";
}
my $md5sum = "md5sum";
my $user_group = "lanforge";

# capture calling commandline to report at end of run:
my $commandline0 = join(" ", @ARGV);
#TODO determine if "lf_kinstall-test.pl" and enable debug.
# our $debug = 1 if ("$0" eq "lf_kinstall-test.pl");
#TODO Since we can detect LF0202 hardware make mgt_dev eth5 and issue correctly laid out lanforge_eth_1.cfg .
#TODO Will probably need to help installer reconfigure pXpY names for ethX or at least help determine mgt_dev
#TODO and assign it correctly.  When listing out interfaces indicate if ip address is present and whether there is link.
#TODO Will probably need lfconfig to accept --mgt_dev.
#TODO if ip addr show $do_mgt_dev true get ipaddr
#     else exit indicating the $do_mgt_dev doesn't exist.
#     if ipaddr is missing then exit indicating that $do_mgt_dev is missing IP address.
#     pass $do_mgt_dev to lfconfig or edit config.values.

#TODO implement cURL time-out and max-time and other timeouts and on a timeout
#TODO implement cURL cache busting:
#if you have an http_proxy env var that might encourage it, you can set --noproxy
#-H "Pragma: no-cache" -H "Cache-Control: no-cache" will put cache busting headers in request

#cURL debug: curl -qvs -o/dev/null http://localhost/

our $q = "'";
our $Q = '"';
our $t = '`';
our $d = '$';
our $dd = '$$';
our $B = "\\"; # $b is builtin sort {$a <=> $b}
our $L = '{';
our $R = '}';
our @ROT = ('-', $B, '|', '/');
our $at = '@';
our $dat = '${@}';
our $proxy_re = '^[^#]* *proxy\s*=\s*http';
our $broken_logfile = 0;
my $curr_dir = `pwd`;
chomp $curr_dir;
my $epath = `dirname $0`;
chomp $epath;
our $tmp_dir = "/var/tmp";
my @tempargv = @ARGV;
my $lfkinargs = "";
foreach (@tempargv) {
    $lfkinargs = $lfkinargs . " $_";
}
my $cp = "rsync -ahq --progress"; # "cp"
my $ttys = "ttyS0";
my $old_hostname = "";
if (-f "/etc/board.json") {
    $old_hostname = "OpenWrt";
}
else {
    $old_hostname = `hostname`;
    chomp($old_hostname);
}

our $is_vm = 0;
if ( -f "/proc/cpuinfo") {
    `grep -c hypervisor /proc/cpuinfo`;
    chomp($is_vm);
    $is_vm = ($is_vm =~ /\d/) ? int($is_vm) : 0;
}
our $osveri = 13;
my $xorp_ver = "1.8.6-WIP";
my $nocache = 0;
my $cachebuster = "";
my $curl = "curl -4 -k"; # IPv4, insecure HTTPs
if ( -x "/bin/curl") {
    $curl =~ s!curl!/bin/curl!;
}
elsif ( -x "/usr/bin/curl") {
    $curl =~ s!curl!/usr/bin/curl!
}
my $curl_resume = "-C -";
my $user = "guest:guest";
our $home = "/home/lanforge";
our $lf_tmp = "/var/tmp";
our $tmp_template = "kinst_XXXXXXXX";
if (! (-d $lf_tmp && -w $lf_tmp)) {
    my $next_tmp = "/tmp";
    print "lf_tmp[$lf_tmp] does not exist and/or is not writable, using $next_tmp...\n";
    sleep(1) if ($::debug_on);
    if (!mkdir($lf_tmp)) {
        print "Unable to create temp directory $lf_tmp: $! \n";
        print "Logging to $next_tmp...\n";
        $lf_tmp = $next_tmp;
    }
}
our $install_log = "$lf_tmp/lf_kinstall_" . ts() . ".log";
if (-w "$home/tmp") {
    $lf_tmp = "$home/tmp";
}
our $mgt_dev = ""; # install LANforge Server and make this value mgt_dev in lfconfig
our $mgt_dns = ""; # use this if we need to specify static dns entry during --do_interop or --regen_nm_conf
our $mgt_ip = ""; # use for dhcp or static ip entry during --do_interop or --regen_nm_conf
my $ctgnome = "/usr/local/bin/ctgnome.bash";
my $nextboot = "";
my $grub_cfg = "/boot/grub2/grub.cfg";
my $grub_mkconfig = "grub2-mkconfig"; # This is for Fedora, maybe others.
my $grub1_cfg = "/boot/grub/grub.conf";
my $etc_default_grub = "/etc/default/grub";
my $grub_40 = "40_custom";
my $etc_grubd_40_custom = "/etc/grub.d/$grub_40";
my $force_osver = "";
our $osver = "F13"; # works with most modern systems
my $force_osveri = -1;
my $uveri = 14;
my $arch = "x86";
my $archbit = "32";
my $lfserver = "";
my $interop = "";
my $lfserver_nodebver = "";
my $lfgui = "";
my $kern = "";
my $fw_board = "board.bin";
my $fw_ct_wmi = "firmware-2.bin";                             # since 5.2.12
my $fw_ct_htt = "firmware-2-htt-mgt.bin";                     # since 5.3.2
my $fw_ct_htt5_4_old = "firmware-5-htt-mgt.bin-10.4";         # since 5.3.4
my $fw_ct_htt5_4_new = "firmware-5-htt-mgt.bin-10-4";         # since 5.3.5
my $fw_ct_htt5_4_9984 = "firmware-5-htt-mgt.bin-9984-10-4";   # since 5.3.5
my $fw_ct_htt5_4_9984b = "firmware-5-htt-mgt.bin-9984-10-4b"; # since 5.3.8
my $fw_ct_htt5_4_9888 = "firmware-5-htt-mgt.bin-9888-10-4";   # since 5.3.5, 2x2 mu-mimo
my $fw_ct_htt2_9887 = "firmware-2-htt-mgt.bin-9887-10-1";     # since 5.3.5, 2x2 peregrine
my $fw_ath10k_stock = "ath10k_stock_fw.tgz";
my $reg_ct = "regulatory.bin";
my $fw_upstream = "ath10k-firmware-2.bin-upstream";
my $xorp = "";
my $xorp_nodebver = "";
my $is_64 = 0;
our $http_root = "/var/www/html";
our $downloaded_md5 = 0;  # Do this once per script run.
our $print_only = 0;      # set this when printing what would download
our $print_w32 = 0;       # set this when printing a windows batch file
our $w32get = 'curl.exe'; # wget.exe has dll problems
our $is_deb_based = 0;
my $is_chr_ubuntu = 0;
my $is_arm = 0;
my $is_rpi3 = 0;
my $is_rpi4 = 0;
my $is_rpi5 = 0;
our $is_openwrt = 0;
my $openwrt_board = "";
my $is_x86 = 1;
my $hwver = "UNKNOWN";
my $lf2100_8x = 0;
my $ubuntu_deb_ver = ""; # deprecated
my $curl_ssl = "";
my $do_fedrepos_default = 0;
my $do_restore_metalinks = 0;
my $do_enable_archive_baseurl = 0;
my $add_random_www_data = 0;
my $bind_apache_mgt_port = 0;
my $do_enable_max_zram = 0;
my $do_interop = 0;
my $do_webui = 0;
our $regen_nm_conf = -1;
my $do_limited_nm_config = 0;
our $skip_ifrename = 0; # on virtual machines this tends to go wild
our $use_systemctl = 0;
our $remove_pipewire = -1; # 1 == yes
our $pip = "/usr/bin/pip3";
if ((!-f $pip) && (-f "/usr/bin/python3-pip")) {
    $::pip = "/usr/bin/python3-pip";
}
elsif ((!-f "/usr/bin/pip3") && (-f "/usr/bin/pip")) {
    $::pip = "/usr/bin/pip3";
}

if ($curl =~ m|(?:/usr)*(?:/bin)*curl| ) {
    $curl_ssl = "--ssl";
}
my $IW = "/sbin/iw";

# check for lanforge user
# better not have actual root uid for lanforge :-(
our $lanforge_uid = 0;
our $should_create_user = 0;
$lanforge_uid = `id -u lanforge 2>/dev/null`;
if ($? != 0) {
    $::should_create_user = 1;
    # print "Should: $? [$::should_create_user]\n";
}
#else {
#   print "Should? $? [$::should_create_user]\n";
#}
#print "Should create user? [$::should_create_user]\n";
if (!-d $home) {
    print("Unable to find '$home', changing lf_tmp to /var/tmp/lf_tmp\n");
    $lf_tmp = "/var/tmp/lf_tmp";
}
mkdir($lf_tmp) unless (-d $lf_tmp && -w $lf_tmp);
chmod(0777, $lf_tmp);
my $staged_in_candela = 0;
my $other_deb_pkgs = "";
# Install logs are here, so we cannot use do_cmd since that itself
# wants to log.
if ((!-d $lf_tmp) && (system(qw(mkdir -p $lf_tmp)) != 0)) {
    print("* Unable to create '$lf_tmp', switching lf_tmp to /tmp\n");
    $lf_tmp = "/tmp";
}
print("\nINSTALL_LOG located at $install_log\n") if ($debug_on);

my @df_tmp = grep {/tmp/} split(/\n/, `df -P /tmp`);
if (@df_tmp < 1) {
    @df_tmp = grep {/dev/} split(/\n/, `df -P /tmp`);
}
if (@df_tmp > 0) {
    chomp @df_tmp;
    my @df_hunks = split(/\s+/, $df_tmp[0]);

    if ((0 + $df_hunks[3]) < 768000) {
        if (!-f "/etc/board.json") {
            # board.json means we are openwrt
            note("# Limited space in /tmp; Switching download from /tmp to /var/tmp\n");
            $::tmp_dir = "/var/tmp";
        }
    }
}
our %md5_file_map = ();
our %file_md5_map = ();
my $do_print_label = 0;
my $print_host = 0;
my $print_queue = 0;
my $serialno = 0;
our $is_macos = 0;
our $is_new_523c = 0;
# another way to check for MacOS is to inspect the
# version of bash running: /bin/bash -version
# GNU bash, version 3.2.57(1)-release (x86_64-apple-darwin17)
my $uname = `uname -a`;
if ($uname =~ /Darwin/) {
    $is_macos = 1;
    $::home = "/Users/lanforge";
    $md5sum = "md5";
    $user_group = "staff";
}
else {
    my $uname_i = `uname -i`;
    chomp($uname_i);
    $::yum_xclude = "-x '*.i686'" if ("x86_64" eq $uname_i);
}

our %metalinks = (
    # better make sure these values are single-quoted!
    "fedora"                                   => 'fedora-$releasever&arch=$basearch',
    "fedora-debug"                             => 'fedora-debug-$releasever&arch=$basearch',
    "fedora-debuginfo"                         => 'fedora-debug-$releasever&arch=$basearch',
    "fedora-source"                            => 'fedora-source-$releasever&arch=$basearch',
    "fedora-modular"                           => 'fedora-modular-$releasever&arch=$basearch',
    "fedora-modular-debug"                     => 'fedora-modular-debug-$releasever&arch=$basearch',
    "fedora-modular-debuginfo"                 => 'fedora-modular-debug-$releasever&arch=$basearch',
    "fedora-modular-source"                    => 'fedora-modular-source-$releasever&arch=$basearch',
    "fedora-updates"                           => 'updates-released-f$releasever&arch=$basearch',
    "fedora-updates-debug"                     => 'updates-released-debug-f$releasever&arch=$basearch',
    "fedora-updates-debuginfo"                 => 'updates-released-debug-f$releasever&arch=$basearch',
    "fedora-updates-source"                    => 'updates-released-source-f$releasever&arch=$basearch',
    "fedora-updates-modular"                   => 'updates-released-modular-f$releasever&arch=$basearch',
    "fedora-updates-modular-debug"             => 'updates-released-modular-debug-f$releasever&arch=$basearch',
    "fedora-updates-modular-debuginfo"         => 'updates-released-modular-debug-f$releasever&arch=$basearch',
    "fedora-updates-modular-source"            => 'updates-released-modular-source-f$releasever&arch=$basearch',
    "fedora-updates-testing"                   => 'updates-testing-f$releasever&arch=$basearch',
    "fedora-updates-testing-debug"             => 'updates-testing-debug-f$releasever&arch=$basearch',
    "fedora-updates-testing-debuginfo"         => 'updates-testing-debug-f$releasever&arch=$basearch',
    "fedora-updates-testing-source"            => 'updates-testing-source-f$releasever&arch=$basearch',
    "fedora-updates-testing-modular"           => 'updates-testing-modular-f$releasever&arch=$basearch',
    "fedora-updates-testing-modular-debug"     => 'updates-testing-modular-debug-f$releasever&arch=$basearch',
    "fedora-updates-testing-modular-debuginfo" => 'updates-testing-modular-debug-f$releasever&arch=$basearch',
    "fedora-updates-testing-modular-source"    => 'updates-testing-modular-source-f$releasever&arch=$basearch',
    "updates"                                  => 'updates-released-f$releasever&arch=$basearch',
    "updates-debug"                            => 'updates-released-debug-f$releasever&arch=$basearch',
    "updates-debuginfo"                        => 'updates-released-debug-f$releasever&arch=$basearch',
    "updates-source"                           => 'updates-released-source-f$releasever&arch=$basearch',
    "updates-modular"                          => 'updates-released-modular-f$releasever&arch=$basearch',
    "updates-modular-debug"                    => 'updates-released-modular-debug-f$releasever&arch=$basearch',
    "updates-modular-debuginfo"                => 'updates-released-modular-debug-f$releasever&arch=$basearch',
    "updates-modular-source"                   => 'updates-released-modular-source-f$releasever&arch=$basearch',
    "updates-testing"                          => 'updates-testing-f$releasever&arch=$basearch',
    "updates-testing-debug"                    => 'updates-testing-debug-f$releasever&arch=$basearch',
    "updates-testing-debuginfo"                => 'updates-testing-debug-f$releasever&arch=$basearch',
    "updates-testing-source"                   => 'updates-testing-source-f$releasever&arch=$basearch',
    "updates-testing-modular"                  => 'updates-testing-modular-f$releasever&arch=$basearch',
    "updates-testing-modular-debug"            => 'updates-testing-modular-debug-f$releasever&arch=$basearch',
    "updates-testing-modular-debuginfo"        => 'updates-testing-modular-debug-f$releasever&arch=$basearch',
    "updates-testing-modular-source"           => 'updates-testing-modular-source-f$releasever&arch=$basearch',
);

our %vm_oui_patterns = (
    'vmware.5056'       => '^00:50:56',
    'virtualbox.0800'   => '^08:00:27',
    'virtualbox.5254'   => '^52:54:00',
    'virtualbox.21f6'   => '^00:21:f6',
    'virtualbox.144f'   => '^00:14:4f',
    'virtualbox.0f4b'   => '^00:0f:4b',
    'oracle.10e0'       => '^00:10:e0',
    'oracle.007d'       => '^00:00:7d',
    'oracle.2128'       => '^00:21:28',
    'oracle.015d'       => '^00:01:5d',
    'oracle.21f6'       => '^00:21:f6',
    'parallels.1c42'    => '^00:1c:42',
    'docker.0242'       => '^02:42:',
    'hyperv.155d'       => '^00:15:5d',
    'qemu.5254'         => '^52:54:00',
    'xen.153e'          => '^00:16:3e',
    'xen.cafe'          => '^00:ca:fe',
);

=pod
greearb> I think if <= 8, cma 128, if > 8 ax radios, then cma 256, and if > 16 radios, then cmd 512
greearb> maybe pull radio count logic up near the top, out of the do-sys-reconfig code.
greearb> and if >0 intel radios, --do_noaer 1 needs set
=cut

our %radio_map = ();
sub survey_radios() {
    return if (!-d "/sys/class/ieee80211");
    my @radios = `cd /sys/class/ieee80211 && ls`;
    chomp(@radios);
    return if (@radios < 1);
    my @ieee = `cd /sys/class/ieee80211; ls | xargs readlink`;
    if (@ieee < 1) {
        note("- survey_radios: saw no radios");
        return;
    }
    chomp(@ieee);
    for my $line (@ieee) {
        my @hunks = split(/[\/]/, $line);
        # print Dumper(\@hunks);
        printf("radio %s pci %s\n", $hunks[-1], $hunks[-3]);
        $::radio_map{$hunks[-1]} =  $hunks[-3];
    }
}

survey_radios();

sub cma_recommendation {
    my $num_radios = 0 + (scalar keys %::radio_map);
    my $recommendation = 0;

    $recommendation = 128  if ($num_radios <= 8);
    $recommendation = 256  if ($num_radios > 8);
    $recommendation = 512  if ($num_radios > 16);
    # note("CMA Recommendation: $recommendation MB\n");
    return $recommendation;
}

sub write_defaults_db() {
    # Create default db.  This holds profiles and such.
    if (open(my $fh, ">", "$home/defaults.db")) {
        note("# creating default database\n");
        print $fh "add_profile upstream 4 0 0 1 65535 '[BLANK]' '[BLANK]' 0 0xFFFFFFFF' '[BLANK]'\n";
        print $fh "add_profile routed-AP 3 0 0 1 5180 'routed-AP' '[BLANK]' 1 0xFFFFFFFF' '[BLANK]'\n";
        print $fh "add_profile bridged-AP 2 0 0 1 5180 'bridged-AP' '[BLANK]' 0 0xFFFFFFFF' '[BLANK]'\n";
        print $fh "add_profile monitor 5 0 0 1 65535 '[BLANK]' '[BLANK]' 0 0xFFFFFFFF '[BLANK]'\n";
        print $fh "add_profile upstream-dhcp 4 0 0 1 65535 '[BLANK]' '[BLANK]' 1 0xFFFFFFFF '[BLANK]'\n";
        print $fh "add_profile STA-AUTO 1 0 0 1 65535 '[BLANK]' '[BLANK]' 0 0xFFFFFFFF '[BLANK]'\n";
        print $fh "add_profile STA-BE 1 16 0 1 65535 '[BLANK]' '[BLANK]' 0 0xFFFFFFFF '[BLANK]'\n";
        print $fh "add_profile STA-AX 1 12 0 1 65535 '[BLANK]' '[BLANK]' 0 0xFFFFFFFF '[BLANK]'\n";
        print $fh "add_profile STA-AC 1 8 0 1 65535 '[BLANK]' '[BLANK]' 0 0xFFFFFFFF '[BLANK]'\n";
        print $fh "add_profile STA-n 1 5 0 1 65535 '[BLANK]' '[BLANK]' 0 0xFFFFFFFF '[BLANK]'\n";
        print $fh "add_profile STA-abg 1 4 0 1 65535 '[BLANK]' '[BLANK]' 0 0xFFFFFFFF '[BLANK]'\n";
        print $fh "add_profile as-is 0 0 0 1 65535 '[BLANK]' '[BLANK]' 0 0xFFFFFFFF '[BLANK]'\n";
        print $fh "add_profile rdd 7 0 0 1 65535 '[BLANK]' '[BLANK]' 0 0xFFFFFFFF '[BLANK]'\n";
        print $fh "add_profile peer 10 0 0 1 65535 '[BLANK]' '[BLANK]' 0 0xFFFFFFFF '[BLANK]' 0 '[BLANK]'\n";
        print $fh "add_profile uplink-nat 11 0 0 1 65535 '[BLANK]' '[BLANK]' 256 0xFFFFFFFF '[BLANK]' 0\n";

        print $fh "\n";
        print $fh "add_traffic_profile slow-dl-udp 1 56000 0 -1 0 0 1 0 0xFFFFFFFF\n";
        print $fh "add_traffic_profile tcp-dl-6m-vi 2 6000000 0 -1 0 128 1 0 0xFFFFFFFF\n";
        print $fh "add_traffic_profile tcp-dl-3m-vi 2 3000000 0 -1 0 128 1 0 0xFFFFFFFF\n";
        print $fh "add_traffic_profile tcp-dl-1m-vi 2 1000000 0 -1 0 128 1 0 0xFFFFFFFF\n";
        print $fh "add_traffic_profile voip 9 56000 0 -1 0 0 1 0 0xFFFFFFFF  0 0\n";
        print $fh "add_traffic_profile http 3 1544000 0 -1 0 0 1 0 0xFFFFFFFF  0 0\n";
        close($fh);
    }
    else {
        note("# not creating default database\n");
    }
}


sub createAbandonedCleanup {
    my $fcacname = "/etc/cron.hourly/50-clean-abandoned";
    my $contents = <<__END__
#!/bin/bash
/bin/systemctl list-units | awk '/abandoned/{print \$1}' | xargs -i systemctl stop {}

#
__END__
    ;
    die("could not open $fcacname")
        unless open(my $fch, ">", $fcacname);
    print $fch $contents, "\n";
    close $fch;
    chmod 0755, $fcacname;
}

sub newWebRequest {
    my ($url, $destfile, $method) = @_;
    my $tmptmpdir = $::tmp_dir;
    # some behaviors change tmp_dir to $basedir, detect and override
    $tmptmpdir = "/tmp" if ($::tmp_dir !~ /^(\/var)?\/tmp/);
    if ($destfile eq "") {
        my (undef, $destfile) = tempfile($::tmp_template, DIR => $tmptmpdir);
    }
    my $rh = {
        "method"          => "GET",
        "follow"          => 1,
        "connect-timeout" => 20,
        "resume"          => 0,
        "retry"           => 5,
        "retry-delay"     => 20,
        "retry-max-time"  => 300,
        "options"         => undef,
        "url"             => $url,
        "destfile"        => $destfile,
        "errors"          => undef,
        "status"          => undef,
        "retval"          => -1,
        "headers"         => undef,
        "output"          => "ignore",
        "user"            => "guest",
        "passwd"          => "guest",
        "notify_on_start" => 0,
        "notify_on_ok"    => 0,
        "notify_on_fail"  => 1,
        "die_on_fail"     => 1,
        "req_zero_exit"   => 0,
        "status_200_ok"   => 1,
    };
    if (defined $method && $method =~ /head/i) {
        $rh->{"method"} = "HEAD";
    }
    return $rh;
}

sub newHeadRequest {
    my $url = shift;
    my $rh = newWebRequest($url, "/dev/null", "HEAD");
    $rh->{"output"} = "/tmp/kinstall_head";
    $rh->{"resume"} = undef;
    $rh->{"options"} = "-s";
    return $rh;
}

sub webget {
    my $rh_req = shift;
    my $tmptmpdir = $::tmp_dir;
    # some behaviors change tmp_dir to $basedir, detect and override
    $tmptmpdir = "/tmp" if ($::tmp_dir !~ /^(\/var)?\/tmp/);
    if ( (! -x "/bin/curl") && (! -x "/usr/bin/curl")) {
        print("\n** curl not installed, install curl? [y|n|ctrl-c] ");
        my $answer = <STDIN>;
        if ($answer !~ /^[Yy]/) {
            die("cannot continue without curl");
        }
        if ($::is_deb_based) {
            do_cmd("apt-get install -y curl");
        }
        elsif ($::is_fedora || $::is_redhat || $::is_centos) {
            do_cmd("yum install -y curl");
        }
        else {
            die("No install command for your platform right now: OSVER[$::osver], bye");
        }
    }
    my ($fh1, $stdout) = tempfile($::tmp_template, DIR => $tmptmpdir);
    my ($fh2, $stderr) = tempfile($::tmp_template, DIR => $tmptmpdir);
    my ($fh3, $headers) = tempfile($::tmp_template, DIR => $tmptmpdir);
    my $cmd = "$curl -D $headers ";

    if ((defined $rh_req->{"connect-timeout"}) && ($rh_req->{"connect-timeout"} ne "")) {
        $cmd .= " --connect-timeout " . $rh_req->{"connect-timeout"};
    }
    if ((defined $rh_req->{"retry"}) && ($rh_req->{"retry"} ne "")) {
        $cmd .= " --retry " . $rh_req->{"retry"};
    }
    if ((defined $rh_req->{"retry-delay"}) && ($rh_req->{"retry-delay"} ne "")) {
        $cmd .= " --retry-delay " . $rh_req->{"retry-delay"};
    }
    if ((defined $rh_req->{"retry-max-time"}) && ($rh_req->{"retry-max-time"} ne "")) {
        $cmd .= " --retry-max-time " . $rh_req->{"retry-max-time"};
    }
    if ((defined $rh_req->{"follow"}) && ($rh_req->{"follow"} ne "")) {
        $cmd .= " -L "
    }
    if ((defined $rh_req->{"user"}) && ($rh_req->{"user"} ne "")) {
        $cmd .= " -u $rh_req->{'user'}:$rh_req->{'passwd'}"
    }
    if ($rh_req->{"url"} =~ /^https:/) {
        $cmd .= " -k --ssl";
    }
    if ((defined $::proxy) && ($::proxy ne "")) {
        $cmd .= " --proxy $::proxy";
    }

    #$cmd .=" --max-time 360 -o $rh_req->{'destfile'}";
    $cmd .= " -o $rh_req->{'destfile'}";
    if (defined $rh_req->{'resume'}) {
        if ($rh_req->{'resume'} =~ /^(1|on|true|yes)$/i) {
            $cmd .= " -C -";
        }
        elsif ($rh_req->{'resume'} =~ /^(0|off|false|no)$/i) {
            $cmd .= " ";
        }
        elsif ($rh_req->{'resume'} ne "") {
            $cmd .= " " . $rh_req->{'resume'}
        }
    }
    if ((defined $rh_req->{'options'}) && ($rh_req->{'options'} ne "")) {
        $cmd .= " " . $rh_req->{'options'};
    }
    if ($rh_req->{"method"} eq "HEAD") {
        $cmd .= " --head -X HEAD";
    }
    $cmd .= " ${Q}$rh_req->{'url'}${Q}";
    # print "---------------------------------------------------------------------\n" if ($debug_on);
    if ($rh_req->{'notify_on_start'} ne "0") {
        note("Begin $cmd") if ($debug_on);
    }
    #/bin/bash -c ${Q}${cmd}${Q}
    if ($::is_openwrt || ($cmd =~ / -s /)) {
        if ($::is_openwrt) {
            print("##CMD: $cmd > $stdout") if ($debug_on);
            system("$cmd > $stdout");
        }
        else {
            print("##CMD: $cmd > $stdout 2> $stderr") if ($debug_on);
            system("$cmd > $stdout 2> $stderr");
        }
        # print("Done with webget bash cmd") if ($debug_on);
    }
    else {
        $cmd =~ s/["]/\\"/g;
        # note("##CMD: /bin/bash -c ${Q}${cmd} > >(tee $stdout)${Q}");
        # openwrt bash does not understand this double > > logic
        system("/bin/bash -c ${Q}${cmd} > >(tee $stdout)${Q}");
    }
    $rh_req->{"retval"} = 0 + $?;
    # print "---------------------------------------------------------------------\n" if ($debug_on);
    close($fh1);
    close($fh2);
    close($fh3);

    my @headers = split(/\r*\n/, `cat $headers`);
    chomp(@headers);
    $rh_req->{"headers"} = \@headers;
    $rh_req->{"status"} = $headers[0];

    if ($rh_req->{"output"} ne "ignore") {
        my @lines = split(/\r*\n/, `cat $rh_req->{"destfile"}`);
        $rh_req->{"output"} = \@lines;
    }

    if (($rh_req->{"req_zero_exit"} != 0) && ($rh_req->{"retval"} == 0)) {
        if ($rh_req->{"notify_on_ok"} != 0) {
            note("$rh_req->{'method'} $rh_req->{'url'} -> $rh_req->{'destfile'}");
        }
        unlink $stdout;
        unlink $stderr;
        return 0;
    }
    if (($rh_req->{"status_200_ok"} > 0) && (defined $rh_req->{'status'}) && ($rh_req->{'status'} =~ / 200 /)) {
        if ($rh_req->{"notify_on_ok"} != 0) {
            note("$rh_req->{'method'} $rh_req->{'url'} -> $rh_req->{'destfile'}  retval $rh_req->{'retval'}");
        }
        unlink $stdout;
        unlink $stderr;
        return 0;
    }
    elsif ((defined $rh_req->{'status'}) && ($rh_req->{'status'} =~ / 416 /)) {
        if ($rh_req->{"notify_on_ok"} != 0) {
            note("Already downloaded");
        }
        unlink $stdout;
        unlink $stderr;
        return 0;
    }

    if (($rh_req->{'notify_on_fail'} > 0) || ($rh_req->{'die_on_fail'} > 0)) {
        my @err = `head $stderr`;
        my @out = `head $stdout`;
        chomp(@headers);
        chomp(@err);
        chomp(@out);
        unshift(@headers, "");
        unshift(@err, "") if (@err > 0);
        unshift(@out, "") if (@out > 0);
        if ($debug_on) {
            print "Headers:", join("\n||| ", @headers), "\n";
            print("STDERR:", join("\n||| ", @err), "\n") if (@err > 0);
            print("STDOUT:", join("\n||| ", @out), "\n") if (@out > 0);
        }
        if ($::debug_on) {
            print "---------------------------------------------------------------------\n";
            print Dumper($rh_req);
            print "---------------------------------------------------------------------\n";
        }
        if ($rh_req->{'die_on_fail'} > 0) {
            err("Failed to download $rh_req->{'url'}\n");
        }
    }
    return $rh_req->{"retval"};
} #~webget

# Use remove_passwords() to make the lanforge system accessible without the need of
# either passwords or ssh keys. This downgrades ssh built in security enormously.
# This system really should not be placed on a routable connection, at this point
# we've created the computer world's tastiest bait.
sub remove_passwords {
    # back up ssh/sshd_config
    print "#############################################\n";
    print "######    Removing System Passwords    ######\n";
    print "#############################################\n";
    print "Continue? [y/N] ";
    my $confirm = <STDIN>;
    if ($confirm =~ /^y/i) {
        print "#############################################\n";
        print "#### removing system passwords in 4 sec  ####\n";
        print "#############################################\n";
        sleep 4;
    }
    else {
        return;
    }
    my $etc_ssh = "/etc/ssh";
    my $sshd_conf = "$etc_ssh/sshd_config";
    if (!-f $sshd_conf) {
        die("No $sshd_conf found, cannot continue");
    }
    if (!-f "$etc_ssh/.sshd_config.bak") {
        `cp $sshd_conf $etc_ssh/.sshd_config.bak`;
    }
    my @new_sshd_lines = ();
    my @sshd_lines = `cat $sshd_conf`;
    chomp(@sshd_lines);
    my $dt = `date`;
    chomp($dt);
    push(@new_sshd_lines, (
        "#############################################################################",
        "#############################################################################",
        "#",
        "#                   LANforge NO PASSWORD Configuration",
        "#",
        "#    You should never place a LANforge within access of the Internet!",
        "#                And now doubly true for this system!",
        "#    Backup file in /etc/ssh/.sshd_config.bak",
        "#    Modified on $dt",
        "#",
        "#############################################################################",
        "#############################################################################",
        "PermitRootLogin yes",
        "StrictModes no",
        "PasswordAuthentication yes",
        "PermitEmptyPasswords yes",
        "#")
    );
    for my $line (@sshd_lines) {
        next if ($line =~ /^\s*#/); # do not copy comments
        next if ($line =~ /^\s*$/); # do not copy blank lines
        next if ($line =~ /(PermitRootLogin|StrictModes|PasswordAuthentication|PermitEmptyPasswords)/);

        push(@new_sshd_lines, $line);
    }
    push(@new_sshd_lines, "#");
    print "Writing new sshd_config...";
    die("Unable to open $sshd_conf")
        unless open(my $fh, ">", $sshd_conf);
    print $fh join("\n", @new_sshd_lines), "\n";
    close $fh;
    print "done\n";
    print "Deleting passwords...";
    do_cmd("passwd -d root");
    do_cmd("passwd -d lanforge");
    do_cmd("systemctl restart sshd.service");
    print "done\n";
}

sub create_lf_user {
    my $M = '';
    if (!-d $::home) { # create home dir if not found
        print "::create_lf_user: home[$::home] is_macos[$::is_macos]\n";
        if ($::is_macos) {
            do_cmd("sysadminctl -addUser lanforge -password lanforge -admin -shell /bin/bash");
            print " User lanforge created.\n";
            return;
        }
        $M = "-m -k/etc/skel";
    }
    if ($::is_macos) {
        # TODO:  Fix up other logic below for macos sometime.
        print "::create_lf_user: home[$::home] exists, not creating user\n";
        return;
    }

    # what groups are likely admin?
    my @group_lines = `cat /etc/group`;
    chomp(@group_lines);
    my @admin_groups = grep {
        /^(adm|admin|dialout|lpadmin|tty|plugdev|root|sudo|video|wheel):/
    } @group_lines;
    my $adm_groups = '';
    if ($#admin_groups) {
        my @col_one = ();
        for my $line (@admin_groups) {
            push(@col_one, (split(/:/, $line))[0]);
        }
        $adm_groups = join(',', @col_one);
    }
    print("* * Adding lanforge user...\n");
    do_cmd("getent group lanforge || groupadd lanforge");
    if (($::home ne "/home/lanforge") && ($::home ne "/root")) {
        print_slow("Unusual home directory for lanforge: [$::home]");
    }
    if ($::home =~ /root$/) {
        print_slow(" ** ** Changing \$HOME to /home/lanforge ",
            "    If you need a home directory ending with 'root' you better know how to edit this file.");
        $::home = "/home/lanforge";
    }

    my $useradd_cmd = "useradd -d $::home $M -g lanforge -G $adm_groups -s /bin/bash lanforge";
    print_slow("    USERADD command: $useradd_cmd\n\n");
    do_cmd("/bin/bash -c 'id lanforge &>/dev/null || $useradd_cmd'");
    do_cmd(qq(bash -c 'echo "lanforge:lanforge" | chpasswd'));
    $::should_create_user = ($? == 0) ? 0 : 1;
    if ($::should_create_user) {
        print_slow( "===== ===== ===== ===== ===== ===== \n",
            " Creating lanforge user failed.\n",
            "===== ===== ===== ===== ===== ===== \n"
        );
    }
    else {
        print " User 'lanforge' created!\n";
    }
    # create some expected home directories
    mkdir("$::home/tmp", 0775)                if ( !-d "$::home/tmp" );
    do_cmd("chown lanforge:$user_group $::home/tmp");
    mkdir("$::home/Pictures", 0750)           if ( !-d "$::home/Pictures" );
    do_cmd("chown lanforge:$user_group $::home/Pictures");
    mkdir("$::home/.config", 0700)            if ( !-d "$::home/.config" );
    mkdir("$::home/.config/autostart", 0700)  if ( !-d "$::home/.config/autostart" );
    mkdir("$::home/Desktop", 0700)            if ( !-d "$::home/Desktop" );
    do_cmd("chown lanforge:$user_group $::home/Desktop", 1);
    mkdir ("$::home/.vnc", 0700)              if ( !-d "$::home/.vnc" );
} #~create_lf_user

sub update_motd {
    die("LANforge version not found [$::lfver]") if ($::lfver eq "-1");
    rename("/etc/motd", "/etc/motd.orig");
    open(my $fh, ">", "/etc/motd");
    print $fh "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --\n";
    print $fh "LANforgeServer $::lfver installed. Many scripts need the environment\n";
    print $fh "variables defined in /home/lanforge/lanforge.profile. To use those, type:\n";
    print $fh "    . /home/lanforge/lanforge.profile\n";
    print $fh "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --\n";
    close($fh);

    open($fh, ">", "/etc/issue");
    print $fh "[ LANforge $::lfver ] [ \\m kernel \\r ] (\\l)\n\n";
    close($fh);
}

sub enable_video_group {
    if ($::is_macos) {
        return;
    }
    do_cmd("gpasswd -a lanforge video", 1);
}

# Please avoid /etc/issue, use /etc/os-release
my $issue = "";
if (-f "/etc/issue") {
    $issue = `cat /etc/issue`;
}
if (($issue eq "") || ($issue =~ /^\\S.*/)) { # this matches \S, meaning nuttin' useful in issue file because templating
    if (-f "/etc/system-release") {
        $issue = `cat /etc/system-release`;
    }
    else {
        print "file /etc/system-release does not exist\n" if ($debug_on);
    }
}

my $os_version_id = -1;
my $os_name = "";

my @os_rel_lines = ();
if (-f "/etc/os-release") {
    @os_rel_lines = `cat /etc/os-release`;
    chomp(@os_rel_lines);
    my @osr_matches = grep {/VERSION_ID=/} @os_rel_lines;
    if (@osr_matches > 0) {
        ($os_version_id) = $osr_matches[0] =~ /VERSION_ID=['"]?([^"']+)['"]?/;
        if ($os_version_id =~ /(\d+)\.(\d+)-snapshot/) { # openwrt version looks like: 19.07-snapshot
            $os_version_id = "$1$2";
        }
    }
    @osr_matches = grep {/NAME=/} @os_rel_lines;
    if (@osr_matches > 0) {
        ($os_name) = $osr_matches[0] =~ /NAME=["']?([^"']+)["']?/;
    }
}
elsif ($is_macos) {
    $os_version_id = `uname -r`;
    ($os_version_id) = $os_version_id =~ /^(\d+)\./;
    $os_name = `uname -s`;
    chomp($os_name);
}
if ((defined $os_version_id) && ($os_version_id ne "")) {
    $::osveri = int($os_version_id);
}
else {
    $os_version_id = -1; # make sure this is not undef
}
if ((defined $os_name) && ($os_name ne "")) {
    $issue = "$os_name release $os_version_id (kinstall)";
}
if ($::debug_on) {
    print("os_version_id[$os_version_id] os_name[$os_name] issue[$issue]\n");
}
# print "552: osveri $::osveri\n";
# print "553: issue $issue\n";

# use this to find default kernel for each LF release
our %kver_map = (
    '5.3.8' => '4.16.18+',
    '5.3.9' => '4.20.17+',
    '5.4.1' => '5.2.21+',
    '5.4.2' => '5.4.52+',
    '5.4.3' => '5.12.14+',
    '5.4.4' => '5.15.7+',
    '5.4.5' => '5.19.11+',
    '5.4.6' => '6.2.16+',
    '5.4.7' => '6.7.5+',
    '5.4.8' => '6.7.9+',
);

# check for leftover dumptrucks
my @old_dumptrucks = (
    "/boot/DUMPTRUCK",
    "/DUMPTRUCK",
    "/home/DUMPTRUCK"
);
for my $truck (@old_dumptrucks) {
    unlink($truck) if (-f $truck);
}
# Radio layout for wiphy systems.
my @macs = ("UNKNOWN", "UNKNOWN", "UNKNOWN", "UNKNOWN", "UNKNOWN", "UNKNOWN");
my @wlans = ();
my @busses = ();
my @desc = ();
my @driver = ();

my $fs3 = "192.168.100.3";
#my $ctproxy = "juno.candelatech.com";
my $ctproxy = ""; # disabling because this was not getting unset

our $notes = "";
our $warnings = "";
our $errors = "";

our $kver = "-1";
our $skip_kern = -1;
our $lfver = "-1";
our $build_path = "";
my $lfrls = "";
my $ser_speed = 115200;
my $source_dir = "";
our $proxy = "";

if ((defined $ENV{"ALL_PROXY"}) && ($ENV{ALL_PROXY} ne "")) {
    $::proxy = $ENV{"ALL_PROXY"};
    note("# Using all_proxy [$ENV{ALL_PROXY}]\n");
}
elsif ((defined $ENV{"HTTP_PROXY"}) && ($ENV{HTTP_PROXY} ne "")) {
    note("** If you have \$HTTP_PROXY set, please consider setting \$ALL_PROXY instead **\n");
    sleep 2;
    $::proxy = $ENV{"HTTP_PROXY"};
    note("# Using http_proxy [$ENV{HTTP_PROXY}]\n");
}
elsif ((defined $ENV{"all_proxy"}) && ($ENV{all_proxy} ne "")) {
    $::proxy = $ENV{"all_proxy"};
    note("# Using all_proxy [$ENV{all_proxy}]\n");
}
elsif ((defined $ENV{"http_proxy"}) && ($ENV{http_proxy} ne "")) {
    note("** If you have \$http_proxy set, please consider setting \$all_proxy instead **\n");
    sleep 2;
    $::proxy = $ENV{"http_proxy"};
    note("# Using http_proxy [$ENV{http_proxy}]\n");
}
my $do_grub = 0;
my $skip_grub = 0;
my $skip_gui = 0;
my $skip_xorp = 0;
my $skip_fmirror = 0;
my $skip_resume = 0;
my $skip_installer_check = 0;
my $do_radius = 0;
my $skip_radius = 0;
my $force_new_certs = 0;
my $do_services = 0;
my $do_pkgs = 0;
my $do_pkgs_f = 0;
my $do_only_pkgs = 0;
my $show_large_pkgs = 0;
my $do_yum_update = 0;
my $do_selinux = -1;
my $do_iommu = -1;
my $do_nomitigations = 1;
my $do_noaer = 1;
my $do_cma = cma_recommendation();
my $do_slub_debug = -1;
my $do_all_ct = 0;
our $ignore_err = 0;
my $do_all = 0;
my $do_upgrade = 0;
my $do_ct_st = 0;
my $do_ct_swak = 0;
my $ntwk_mgr_ok = 0;
my $do_xrandr = 0;
my $do_gnome = 0;
my $do_install_mate = 0;
my $mate_installed = 0;
my $do_vnc = 0;
my $do_http = 0;
my $fix_web_root = "";
my $do_biosdevname = -1;
my $do_serial = -1;
my $do_elevator = -1;
my $do_kmemleak = -1;
my $do_ssd_fstab = 0;
my $using_serial = 0;
my $do_udev = 0;
my $do_runlevel = 0;
my $do_lanforge = 0;
my $build_scrcpy = 0;
my $do_kern = 0;
my $disable_audit_logs = -1;
my $remove_kern = 0;
my $do_firmware = 0;
my $skip_firmware = 0;
my $do_save_yumc = 0;
my $do_local_install = 0;
my $force_web = 0;
our $create_install_bundle = 0;
our $use_install_bundle = 0;
our $show_urls = 0; # see $print_only
my $force_notes = 0;
# my $do_make_clean = 0; # not implemented
my $skip_yum_update = 0;
my $skip_yum_all = 0;
my $is_offline = 0;
my $do_cpu_burn = 0;
my $burner = "stress-ng"; # was burnP6
my $do_cups_pdf = 0;
my $skip_cpu_burn = 0;
my $skip_disk_test = 0;
my $do_disk_test = 0;
my $burnt = 600; # Default cpuburn time.
my $download_only = 0;
our %download_list = ();
my $hs20_server_ip = "127.0.0.1";
my $do_hs20 = 0;
my $do_xrdp = 0;
our $do_sys_reconfig = 0;
our $do_vm_reconfig = 0;  # use this in a vm or to force do_sys_reconfig
our $do_ff_homepage = 0; # use this in a vm or to force do_sys_reconfig
my $do_mgt_dev = "eth0";
my $do_release_mirror = 0; # for mirroring r5.4.x into /var/www/html/private/downloads/r5.4.x
my $help = 0;
my $helpall = 0;
my $no_fmirror = 0;
my $set_swiotlb = 0; # 65536 is presumably safe default for most linux systems
                    # but the unit is probably 8K slabs of memory, so this number
                    # computes to 512MB of ram
my $make_ifcfg_eth0 = 0;
my $create_webpage = 0;

# sg.candelatech.com : singapore
# sf.candelatech.com : san francisco
my $www = "https://www.candelatech.com";
my $download_host = $www;
our $apt = "apt";
our $apt_update = 0; # tracks if apt update has been run

# Figure out what OS version we are using (or whatever we are closest to)
our $os_string = "UNKNOWN OS";
our $is_fedora = 0;
our $is_centos = 0;
our $is_redhat = 0;
our $is_ubuntu = 0;

our $do_abandoned_cleanup = 0;
our $yum_xclude = "";
our $yum_has_pending_kernel = 0;
my $use_yum_cache = 0;
our $disable_resolved = 0; # Change NM.conf dns=systemd-resolved to dns=default

our $usage = qq[
--------------------------------------------------------------------------------
Usage:
   lf_kinstall.pl --lfver {lanforge version} --kver {kernel version} {command}
Note: the parameters --do_kern and --do_grub requires the --kver option to specify the kernel version.
Candelatech Linux kernels end with "+" to denote extra patches.
Use --debug|--verbose|-d to enable debugging.
   Example: ./lf_kinstall.pl --kver 3.5.7+ --lfver 5.2.7 --do_lanforge
--
Please refer to the Install Guide and the options reference:
    • Install guide: http://www.candelatech.com/lfserver_install.php#webauto
    • Reference: http://www.candelatech.com/cookbook.php?vol=misc&book=lf_kinstall
--
 ## Common Options ##

help:         This message.
lfver:        Specify LANforge version (i.e., 5.2.6)
kver:         Specify kernel version, use a + at the end of kernel versions for Candelatech kernels
proxy:        Use a proxy for curl, e.g. http://%user:%password\@%proxy-ip:%port
_
do_all_ct:    Enables all options except: --xrandr, --runlevel
_
do_upgrade:   Also --do_update. Upgrades LANforge software, kernel, firmware, packages and OS packages.
              No system settings altered.  Skips cpu-burn.
_
do_lanforge:  Installs LANforge software ONLY, i.e. no kernel installed.
              No system settings altered.
_
do_firmware:  Download and install ath10k 802.11AC NIC firmware.
_
do_kern:      Install the kernel via download or --source_dir.
              Enables 'do_grub' option.
_
help_all:     Show advanced options
skip_pip:     Avoid doing pip upgrades; might be necessary if you have proxies
skip_yum_all: Don't yum update packages, or install new ones.
];

our $advanced_help = qq[
--
Please refer to the Install Guide and the options reference:
    • Install guide: http://www.candelatech.com/lfserver_install.php#webauto
    • Reference: http://www.candelatech.com/cookbook.php?vol=misc&book=lf_kinstall
--
## Advanced Options ##

acknowledge: Also -g. Acknowledge that your licenses are out of date to continue updates
add_random_www_data: creates a series of files in $http_root accessable using
  the url /slug_list.html
_
bind_apache_mgt_port: configures an /etc/hosts entry lanforge-srv that matches
                      the ip address of the mgt_dev in /home/lanforge/config.values. Updates the
                      /etc/httpd or /etc/apache2 files to bind to that address. Use this option
                      each time you change the mgt_dev.
_
build_scrcpy:  Clone and build scrcpy tool (should not be needed for latest 5.4.6 builds)
build_url:    Also: --build_path. This path is appended directly after the --download_from hostname. The default
              download path will not be used. The string  \$lfver will not be inserted.
              Build paths will not adhere to a directory pattern, use the full path.
_
create_install_bundle: Also: --install_bundle, --create_bundle, --create_tarball
                        Create a tarball in tmp_dir for copying to another LANforge system,
                        will infer --osver, --osveri for current system unless you specify
                        --force_osver, --force_osveri
_
create_lanforge_user: adds user lanforge and directory /home/lanforge
create:pypirc: create pypirc file
create_raid: Also --make_raid, --create_lvm, --make_lvm [drive1,drive2...]
            Turn two or more blank drives into a mirrored LVM filesystem
            If there have been filesystems installed on these drives before,
            use --wipe_raid to remove the filesystems.
            Example: $0 --make_raid nvme0n1p1,nvme1n1p1
_
com-speed:    Also --com_speed. Specify serial com speed (defaults to 115200)
com_port:     Specify the serial com port (defaults to ttyS0)
create_webpage: turn this help into HTML text for publication
debug:      Also -d, --verbose. Enable extra output.
disable_audit_logs: Affects kernel audit messages. Use with --do_grub and reboot to have this setting take effect.
                     0: enable kernel audit logs (auditd.service not enabled)
                     1: disable kernel audit logs
_
disable_resolved: change NetworkManager dns=systemd-resolved to dns=default
                  and disable systemd-resolved.service
_
do_all:       Enables all options except: --xrandr, --biosdevname, --runlevel, and --serial
              Skips the CPU burn-in test.
_
do_abandoned_cleanup: create cronjob that erases systemd slices from
                     logged out sessions every hour (might have occurred between F24-F27)
_
do_biosdevname: Enable biosdevname for interface names (uses terms like enp0s1 instead of eth0).
                Only takes affect when --do_grub is enabled.
                 0:  Disabled
                 1:  Enabled
                -1:  Use current settings (default)
_
do_cma:       Configure cma buffer for extra VRF buffer space.
               Use with --do_grub, applies on reboot. Use these choices:
               -1: do not change
               0: disable
               1: apply 64 megabyte value
               >1: apply this value in megabytes
               Example: --do_grub --do_cma 48
_
do_cpu_burn:  Attemps the CPU burn task.
              NOTE:  The --skip_cpu_burn flag below has precedence.
                0:  Do not attempt it.
                1:  Do it if we have not already run it on this system previously.
                2:  Always run it.
              >10:  Run CPU burn test for specified seconds.
_
do_ct_st:     Download and install LANforge software and kernel only.
              Enables 'do_lanforge do_kern do_grub do_http do_selinux=0 do_iommu=0 do_firmware'
              No yum update.
              No system settings altered, except for selinux and iommu.
_
do_ct_swak:   Download and install LANforge software but only update grub to
              point to an already installed kernel.
              Enables 'do_lanforge do_grub do_http do_selinux=0 do_iommu=0 do_firmware'
              No system settings altered, except for selinux and iommu.
_
do_cups_pdf: install cups-pdf package
do_disk_test: Also: --disk_test. Fill and re-read disk to prove read errors?
               0: Do not attempt it
               1: Do it if $::home/did_disktest not found unless --skip_disk_test specified
               >1: Always run it, ignoring --skip_disk_test
_
do_elevator:  Add kernel option elevator=noop (i.e. not deadline) for single SSD CFQ optimisation.
                 0:  Disabled
                 1:  Enabled
                -1:  Use current settings (default)
              Only takes affect when do_grub is enabled.
_
do_enable_archive_baseurl: Select this option if your mirrors.fedoraproject.org URLs are
   unable to provide your repository because the content has been moved to archives.fedoraproject.org
   This option is pretty much opposite of --do_restore_metalinks.
_
do_enable_max_zram: write /etc/sysctl.d/70-lanforge.conf and apply agressive zram configuration (Fedora >= 30)
                     To undo, remove file and reboot.
_
do_fedrepos_default: Select this option to use the fedrepos default command
   if your yum.repos.d repository URLs have become misconfigured. If fedrepos
   is not available, consider --do_restore_metalinks option.
_
do_ff_homepage: updates Firefox Homepage
do_gnome:     Tweak gnome settings (enble desktop icons, etc)
do_grub:      Modify the grub config files to boot the specified kernel.
do_gui_autostart: Start GUI in VNC server session which starts on boot.
                  Requires LANforgeGUI of specified version to already be installed.
                  0: disable GUI autostart
                  1: enable GUI autostart
_
do_hs20:      Build keys, configure apache ssl and other actions to enable
              this system to act as an HS20-R2 server.  Requires specific
              LANforge configuration as well before this will actually work.
              This option must be explicitly enabled:  It is not enable as part
              of any of the other option groupings.
_
do_http:      Serve LANforge related files at http://localhost
do_image_prep: Erase the network settings and dnf cache in preparation for
               making a disk image for a clone. Expects /root/resize-home.sh
_
do_interop: installs LANforge server, GUI, firmware and required packages on system to sufficient to allow it
            to be a resource in a LANforge cluster. Right now, this focuses on Ubuntu based systems.
            This also will configure Networkmanager to ignore ports except the management interface. We cannot
            necessarily disable NM on interop equipment.
_
do_iommu:     Configure system to enable/disable intel_iommu.  This kernel feature decreases performance,
              so LANforge typically wants this disabled for optimal Ethernet performance.  This can also be disabled in
              the BIOS by disabling the VT-d option and IOMMU options.
                 0:  Disabled (default for do_all_ct, do_all, do_ct_swak, do_ct_st)
                 1:  Enabled
                -1:  Use current settings (default)
              Only takes affect when do_grub is enabled.
              IOMMU is useful in these conditions:
              * virtual machine hosting
              * MediaTek radios
              * Ath10k radios
_
do_kmemleak:  Configure kernel option for kmemleak.  Requires kernel to be compiled with
              appropriate options to actually enable this.
                 0:  Disabled
                 1:  Enabled
                -1:  Use current settings (default)
              Only takes affect when do_grub is enabled.
_
do_mgt_dev: Also --md. Specify the device to dedicate with management network.
            Use this with --do_sys_reconfig.
_
do_noaer:      Configure system to enable/disable pci error reporting. Requires --do_grub.
                  0: Disabled (removes grub bootline parameter)
                  1: Enabled (adds pci=noaer bootline parameter, DEFAULT)
                 -1: Use current settings
_
do_nomitigations: Configure system to enable/disable spectre and related mitigations.
              We try to compile out most of these security features since LANforge
              is not designed to be secure and performance is more important to us.
              To help make sure all of these mitigations are disabled, we will also
              pass command-line args to the kernel on bootup to request disabling
              mitigations.  Default is '1'.
                 0:  Do not add the mitigations=off option
                 1:  Do add the mitigations=off (default)
                -1:  Use current settings
              Only takes affect when do_grub is enabled.
_
do_only_pkgs: Only install packages (and groups) and exit. Use when creating VMs or if you
              want to install all distro packages and reboot before proceeding.
              If you want to download everything needed, install yum packages for development imaging:
               1) touch /home/lanforge/did_cpuburn /home/lanforge/did_disktest
               2) ./lf_kinstall.pl --lfver ___ --kver ___ --do_selinux=0 --skip_yum_all
               3) ./lf_kinstall.pl --lfver ___ --kver ___ --skip_yum_all --do_all_ct --force_web \\
                  --tmp_dir /home/lanforge/Downloads --download_from http://ctdownloads/ --download_only
               4) reboot
               5) ./lf_kinstall.pl --lfver ___ --kver ___ --do_only_pkgs
               6) poweroff and make your snapshot
_
do_pkgs:      Install packages from Internet needed by LANforge.
do_print_label:   Use the following two options to print a label with model and mac address information
   1) print_host:    hostname owning printer
   2) print_queue:   name of print queue, often 'QL-800' or 'LaserWriter-450'
   3) serialno:      provide the serial number for chassis, or use 'HOSTNAME'
   Example: --do_print_label --print_host 192.168.100.14:8082 --print_queue QL-800 --serialno HOSTNAME
_
do_radius:    Install and configure radius server (with default values).
do_release_mirror: for downloading install files necessary to host new releases on the system.
                    These files are placed in /var/www/html/downloads and /var/www/html/private/downloads.
                    In the GUI Release Mgr tab, Download From can list http://192.168.1.101 or whatever the
                    mananagement port IP is.
_
do_restore_metalinks: Select this option if you see errors like below:
   1) Repository updates-debuginfo has no mirror or baseurl set.
   2) Repository updates-source has no mirror or baseurl set.
   3) Repository fedora-debuginfo has no mirror or baseurl set.
   4) Repository fedora-source has no mirror or baseurl set.
   5) Error: Failed to synchronize cache for repo 'updates'
   6) Your /etc/yum.repos.d repo files have had changes and are unable to reach mirrors.fedoraproject.org
_
do_rfgen: shortcut to install packages necessary for using RF generator hardware
do_runlevel:  Configure system to run-level 3 or 5
                 0:  Use current settings (do nothing)
                 3:  Set to use runlevel 3 (non graphical login)
                 5:  Set to use runlevel 5 (graphical login, needed for cinnamon)
_
do_save_yumc: Specify:
                 1: Yum update then save cache.
                20: Delete local cache first then update and save.
_
do_selinux:   Configure selinux (it conflicts with LANforge.)
              do_grub should also be enabled to modify the kernel boot commands.
                 0:  Disabled  (default, if do_all and related options are selected)
                 1:  Enabled
                -1:  Use current settings
_
do_serial:    Enable serial console configuration in grub.
                 0:  Disabled
                 1:  Enabled
                -1:  Use current settings (default)
              Only takes affect when do_grub is enabled.
_
do_services:  Enable/Disable services to work well with LANforge.
do_slub_debug: Enable/Disable SLUB memory debugging (at least on debugging kernels)
               Use with --do_grub, applies on reboot. Use these choices:
               -1: do not change
               0: disable
               1: enable slub debugging
               Example: --do_grub --do_slub_debug 1
_
do_ssd_fstab: Modify rw behaviour for SSDs in /etc/fstab.
do_sys_reconfig:  Attempt to re-configure system config files.
                 Only works on certain platforms (APU2, Jetway, Noah2, Axiomtek)
_
do_udev:      Create /etc/udev/rules.d/70-persistent-net.rules file if
              it does not already exist.  It may still need hand-editing.
_
do_vm_prep: Erase the network settings and dnf cache in preparation for
            making a OVA image. Expects /root/resize-home.sh
_
do_vm_reconfig: forces --do_sys_reconfig to run, looking for enpX interfaces and no radios
do_vnc:       Configure VNC for user lanforge.
do_webui:     installs and configures LANforge WebUI components.  Fedora-34 and higher.
do_xrandr:    Add work-around to disable LVDS1 using xrandr.
              This works around Gnome issues on the Lanner WiFi appliances,
              and perhaps other systems.
_
do_xrdp:      should we adjust the xrdp.ini and sesman.ini files:
               0: ignore files
               1: adjust files
_
do_yum_update: Update OS packages from Internet.
                Does yum --upgrade or dnf --upgrade or apt-get upgrade.
_
download_from: Also: --from, --download_host. Specify web url to download LANforge packages from.
                Implies --force_web.
_
download_only: Download files to tmp-dir, but do not install them or make other changes.
download_videos: shortcut for downloading Candelatech videos into $http_root/videos
   so that they might be served for video testing. See /usr/local/lanforge/nginx/video.inc
   for stanza when enabling directory listings when using nginx.
_
fix_web_root: Enable do_http and reset the /etc/httpd/httpd.conf DocumentRoot and Directory
              from $::home/candela_cdrom to $http_root
_
force_new_certs:   Re-generate the HotSpot 2.0 R2 (HS20-R2) and/or RADIUS
              certs, even if we have previously created them.
_
force_notes:  Force creating lfnotes.html
force_osver:  Override the detected osver string with this value.
force_osveri: Override the detected osversion integer with this value.
force_web:    Force script to use webserver. Do not use with --source_dir.
hs20_server_ip: IP Address to advertise as HS20 servers.  This could be IP
                address of management port in simple configurations, and for
                more realistic configurations, perhaps the IP address of something
                in same network as the HS20 OSEN and .1x APs.
_
hwver:        Specify the hardware string:
              CT521, CT52x-PR, CT521, CT520, LF0202, CT523, CT523c, CT314, LF0312, LF0313, CT522
              Can help with html info page configuration.  Leave blank if unsure.
_
ignore_err:   Ignore any (otherwise fatal) errors.  May be useful for offline
              installs where some functionality is better than none.
_
install_large_file_cron: Creates a crontab entry that runs $home/scripts/check_large_files.bash
                        on a daily basis. If you want to run that check more frequently, you
                        will have to modify the crontab rules yourself.
_
install_pip_lanforge_scripts: this does a pip3 install of lanforge-scripts. This is not necessary
                              if you just need to use /home/lanforge/scripts/py-scripts casually.
_
install_selenium: install the selenium package using pip3 (implemented in 5.4.8)
_
install_vlc: VLC video player requires RPMFusion repository. This adds the repository
                and then installs VLC.
_
lf2100_8x:     Specify adapter board used is the 8x internal splitter/combiner board, for do_sys_reconfig
_
make_ifcfg_eth0: Creates /etc/sysconfig/network-scripts/ifcfg-eth0 from the
                 eth0 entry found in /etc/udev/rules.d/70-persistent-net.rules.
                 Fedora only. Edit 70-persistent first.
_
mgt_dev: indicate what interface should be configured as the management interface.
        LANforge works best in cases where there is a dedicated management interface and management network
        (out of band management). In the case there is only one interface and it has to serve as a traffic port
        and a management port, that is possible (in band management). Currently this option is only used by
        the --regen_nm_conf option. If this option is not specified Specify one of these options:
        1) {interface_name}: name of the interface to set
        2) "existing": force the use of the mgt_dev in /home/lanforge/config.values
_
mgt_dns: specify IP address to add to [ipv4]mgt_dev.dns value in mgt_dev.nmconnection
mgt_ip: specify either "dhcp" or a "ip/cird/gateway" combination for mgt_dev.nmconnection:
        1) mgt_ip=dhcp
        2) mgt_ip=192.168.208.24/20/192.168.208.1
_
no_fmirror:   Uninstall yum-fastest mirror.
nocache:      Add fake URL argument to disable any HTTP caching.
ntwk_mgr_ok:  Enable NetworkManager.  It is disabled by default.
offline:      Specify that the system is on an isolated network. Turns off:
              --do_only_pkgs --do_pkgs
              Enables:  --skip_fmirror --skip_installer_check --skip_pip --skip_yum_all --skip_yum_update
_
print_host:   Use with --print_label
print_only:   Implies --show_urls without website check
print_windows: print out .bat file contents to help with downloading LANforge updates
rebuild_pip: this will uninstall the python pip user environments and build a
             new one from scripts/requirements.txt; if you see errors, you probably
             are missing newly required library packages provided by the distro.
             To make sure your packages are up to date, use:
             Example:  $0 --do_pkgs
_
regen_https_key:    regenerate the /etc/pki/tls/\$hostname.crt and .key files so that they have
                    F33+ 4096 bit keys, not smaller keys.
_
regen_nm_conf: Also --regen_nm_config. Backs up /etc/NetworkManager/NetworkManager.conf and recreates one that
                specifies that no interfaces except the one with the default route be managed.
                Enabled by default during do_interop. NM settings you do not want modified should
                live in separate conf files in /etc/NetworkManager/conf.d. When this option is used
                without the --mgt_dev option, a new management device will be determined. Default value is 1.
                0: do not modify NetworkManager.conf
                1: update NetworkManager.conf and mgt_dev.nmconnection
                2: update NetworkManager.conf, mgt_dev.nmconnection, but do not restart NetworkManager
_
remove_kern:    Uninstall a LANforge kernel from /boot and /etc/grub.d.
                It will update the /boot/grub2/grub.cfg.
_
remove_large_file_cron: removes the cronjob created by --install_large_file_cron
remove_passwords: removes the passwords from accounts root and lanforge and reconfigures
                  sshd to accept empty passwords. Yes...really, it IS crazy, right?
_
remove_pipewire: Removes pulseaudio-pipewire packages.
                 0: do not remove pipewire (default)
                 1: remove pipewire
_
reset_pip: Use when pip3 updates were installed as root and you have system pip3 package conflicts.
           This will reinstall the python-pip3 package and 'pip3 remove -r requirements.txt' to remove
           pip3 libraries from system folder. Follow this with --update_pip to get pip3 libraries into
           user lanforge folder.
_
save_gui_cfg: Use this to restore GUI settings on reboot. Running this a second
              time copies a changed LANforgeGUI/lfcfg.txt file to \$home to save new changes.
              1) backup: copies the LANforgeGUI/lfcfg.txt file to \$home. Disables lfcustom_gui.bash if it exists.
              2) static: implies backup, creates lfcustom_gui.bash that replaces lfcfg.txt into LANforgeGUI directory
              every time LANforgeServer is started.
_
serialno:   use with --do_print_label, specifies serial number on label

set_swiotlb:  Also: --swiotlb, --do_swiotlb. Configure IOMMU buffer size.
              Requires you use IOMMU setting, so use with --do_grub --do_iommu=1
              Examples: --do_grub --do_iommu=1 --set_swiotlb=65536
_
show_large_pkgs: Also: --show_larges_pkgs, --largest_pkgs, --large_pkgs.
                Show top 20 largest packages. Useful during do_image_prep if you
                want to reduce install footprint.
_
show_urls:    Show URLS for all files that should be downloaded, and exit.
skip_cpu_burn:   Don't burn-in CPU.
skip_disk_test: avoid do_disk_test if $::home/did_disktest is missing
skip_fmirror: Don't alter systems existing use of yum-plugin-fastestmirror.
skip_grub:    Don't do kernel install, even if other options would have selected it.
skip_gui:     Don't install LANforge-GUI, even if other options would have selected it.
skip_instller_check: avoids checking MD5 sum of lf_kinstall.pl
skip_ifrename: avoids renaming interfaces and rewriting 70-persistent-net.
              If OUIs of VMs are detected, the file $home/LF_NO_IFRENAME is
              created. If $home/LF_NO_IFRENAME is found, this option is
              automatically enabled.
skip_installer_check: turn off checking md5sum of $0
skip_kern:    Do not install a kernel package, suppresses do_grub, do_kern
skip_radius:  Do not attempt to reconfigure radius. Useful when attempting
              to speed up do_sys_reconfig on slow systems.
_
skip_resume:  Don't try to use HTTP resume when downloading packages.
skip_xorp:    Don't install Xorp virtual-router package, even if other options would have selected it.
skip_yum_update: Don't execute dnf/yum --update
source_dir:   Specify location of installation files (must be absolute path).
              Implied by --use_bundle. Useful when downloading install packages on an
              isolated system (typically to $home/Downloads). Example:
              $0 --lfver 5.4.7 --kver 6.7.5+ --do_upgrade --source_dir $home/Downloads --offline
_
tmp_dir:      Specify the script temporary working directory and backups of system files.
              Typically defaults to /var/tmp.
_
update_pip: Upgrades pip userland
use_install_bundle: Also --use_bundle, --bundle [bundle file]
                    Upgrade LANforge using an install bundle file. Bundle files
                    Sets \$source_dir, \$tmp_dir to directory containing bundle.tar file.
                    See http://www.candelatech.com/lfserver_install.php#offline_bundle
                    Bundle installations require the system OS matches compiled OS
                    version in the bundle. Confirm your OS version using:
                    "grep Y /etc/os-release". Not intended for --do_interop.
_
use_yum_cache: do not erase and rebuild yum cache
wipe_raid: Also --remove_raid --remove_lvm [drive1,drive2...]
            Remove LVM volumes and signatures from specified disks.
            Use this before doing --create_raid the first time on any
            disk that already has a filesystem.
            Example: $0 --wipe_raid nvme0n1p1,nvme1n1p1
_


## Note: ##
If you use commands "yum update" or "dnf update", and you need to use a kernel
    provided by the repository, use grub2-mkconfig to create the boot entry, or
    consider using --do_interop
      Example: grub2-mkconfig -o /boot/grub2/grub.cfg

--------------------------------------------------------------------------------
];

sub usage {
    for my $line (split("\n", $::usage)) {
        if ($line =~ /[*_]+dd/) {
            $line =~ s|\*_?dd| |;
            $line =~ s|\*dd| |;
        }
        elsif ($line =~ /^_$/) {
            $line = "\n";
        }
        print $line, "\n";
    }
}

my $yum_gnome_pkgs = "gnome-tweak-tool gnome-session-classic gnome-applets gnome-applet-sensors";

my $yum_mate_pkgs = "mate-sensors-applet";

my $candela_yum_pkgs = "";
my $do_image_prep = 0;
my $do_vm_prep = 0;
my $yum_groups = q( development-tools system-tools "DNS Name Server" "Directory Server" "FTP Server" "Windows File Server");
my $install_vlc = 0;
my $do_rfgen = 0;
my $download_videos = 0;
my $create_raid = "";
my $wipe_raid = 0;
my $create_pypirc = 0;
our $acknowledged = 0;
my $install_large_file_cron = 0;
my $remove_large_file_cron = 0;
my $regen_https_key = 0;
my $update_pip = 0;
my $skip_pip = 0;
my $install_pip_lanforge_scripts = 0;
my $reset_pip = 0;
my $rebuild_pip = 0;
my $do_remove_passwds = 0;
our $do_gui_autostart = -1;
our $save_gui_cfg = "";
my $install_selenium = 0;

if (! GetOptions(
        # TODO: upgrade feature with the option of having only one drive
        # create_lvm=drive_a,...drive_n
        'acknowledge|g'                                                     => \$::acknowledged,
        'add_random_www_data'                                               => \$add_random_www_data,
        'bind_apache_mgt_port'                                              => \$bind_apache_mgt_port,
        'build_scrcpy'                                                      => \$build_scrcpy,
        'build_url|build_path=s'                                            => \$::build_path,
        'com-speed|com_speed|s=i'                                           => \$ser_speed,
        'com_port=s'                                                        => \$ttys,
        'create_install_bundle|install_bundle|create_bundle|create_tarball' => \$::create_install_bundle,
        'create_lanforge_user|u'                                            => \$::should_create_user,
        'create_pypirc'                                                     => \$create_pypirc,
        'create_raid|make_raid|create_lvm|make_lvm=s'                       => \$create_raid,
        'create_webpage'                                                    => \$create_webpage,
        'debug|verbose|d'                                                   => \$debug_on,
        'disable_audit_log|disable_audit_logs=i'                            => \$disable_audit_logs,
        'disable_resolved'                                                  => \$disable_resolved,
        'do_abandoned_cleanup'                                              => \$do_abandoned_cleanup,
        'do_all_ct|c'                                                       => \$do_all_ct,
        'do_all|a'                                                          => \$do_all,
        'do_biosdevname|b=i'                                                => \$do_biosdevname,
        'do_cma=i'                                                          => \$do_cma,
        'do_cpu_burn|dcp=i'                                                 => \$do_cpu_burn,
        'do_ct_st'                                                          => \$do_ct_st,
        'do_ct_swak'                                                        => \$do_ct_swak,
        'do_cups_pdf'                                                       => \$do_cups_pdf,
        'do_disk_test|disk_test=i'                                          => \$do_disk_test,
        'do_elevator|el=i'                                                  => \$do_elevator,
        'do_enable_archive_baseurl'                                         => \$do_enable_archive_baseurl,
        'do_enable_max_zram'                                                => \$do_enable_max_zram,
        'do_fedrepos_default'                                               => \$do_fedrepos_default,
        'do_ff_homepage'                                                    => \$::do_ff_homepage,
        'do_firmware'                                                       => \$do_firmware,
        'do_gnome'                                                          => \$do_gnome,
        'do_grub'                                                           => \$do_grub,
        'do_gui_autostart=i'                                                => \$do_gui_autostart,
        'do_hs20'                                                           => \$do_hs20,
        'do_http'                                                           => \$do_http,
        'do_image_prep'                                                     => \$do_image_prep,
        'do_interop'                                                        => \$do_interop,
        'do_iommu=i'                                                        => \$do_iommu,
        'do_kern'                                                           => \$do_kern,
        'do_kmemleak=i'                                                     => \$do_kmemleak,
        'do_lanforge'                                                       => \$do_lanforge,
        #'do_make_clean'                                                     => \$do_make_clean,
        'do_mgt_dev|md=s'                                                   => \$do_mgt_dev,
        'do_noaer=i'                                                        => \$do_noaer,
        'do_nomitigations=i'                                                => \$do_nomitigations,
        'do_only_pkgs'                                                      => \$do_only_pkgs,
        'do_pkgs'                                                           => \$do_pkgs_f,
        'do_print_label'                                                    => \$do_print_label,
        'do_radius'                                                         => \$do_radius,
        'do_release_mirror'                                                 => \$do_release_mirror,
        'do_restore_metalinks'                                              => \$do_restore_metalinks,
        'do_rfgen'                                                          => \$do_rfgen,
        'do_runlevel|r=i'                                                   => \$do_runlevel,
        'do_save_yumc|ys=s'                                                 => \$do_save_yumc,
        'do_selinux=i'                                                      => \$do_selinux,
        'do_serial|e=i'                                                     => \$do_serial,
        'do_services'                                                       => \$do_services,
        'do_slub_debug=i'                                                   => \$do_slub_debug,
        'do_ssd_fstab'                                                      => \$do_ssd_fstab,
        'do_sys_reconfig'                                                   => \$::do_sys_reconfig,
        'do_udev'                                                           => \$do_udev,
        'do_upgrade|do_update'                                              => \$do_upgrade,
        'do_vm_prep'                                                        => \$do_vm_prep,
        'do_vm_reconfig'                                                    => \$do_vm_reconfig,
        'do_vnc'                                                            => \$do_vnc,
        'do_webui'                                                          => \$do_webui,
        'do_xrandr'                                                         => \$do_xrandr,
        'do_xrdp|i'                                                         => \$do_xrdp,
        'do_yum_update'                                                     => \$do_yum_update,
        'download_only'                                                     => \$download_only,
        'download_videos'                                                   => \$download_videos,
        'fix_web_root'                                                      => \$fix_web_root,
        'force_new_certs'                                                   => \$force_new_certs,
        'force_notes'                                                       => \$force_notes,
        'force_osver=s'                                                     => \$force_osver,
        'force_osveri=i'                                                    => \$force_osveri,
        'force_web'                                                         => \$force_web,
        'from|download_from|download_host=s'                                => \$download_host,
        'help'                                                              => \$help,
        'help_all'                                                          => \$helpall,
        'hs20_server_ip=s'                                                  => \$hs20_server_ip,
        'hwver|hw=s'                                                        => \$hwver,
        'ignore_err'                                                        => \$::ignore_err,
        'install_large_file_cron'                                           => \$install_large_file_cron,
        'install_pip_lanforge_scripts'                                      => \$install_pip_lanforge_scripts,
        'install_selenium'                                                  => \$install_selenium,
        'install_vlc'                                                       => \$install_vlc,
        'kver|k=s'                                                          => \$::kver,
        'lf2100_8x'                                                         => \$lf2100_8x,
        'lfver|v=s'                                                         => \$::lfver,
        'make_ifcfg_eth0'                                                   => \$make_ifcfg_eth0,
        'mgt_dev=s'                                                         => \$::mgt_dev,
        'mgt_dns=s'                                                         => \$::mgt_dns,
        'mgt_ip=s'                                                          => \$::mgt_ip,
        'no_fmirror'                                                        => \$no_fmirror,
        'nocache'                                                           => \$nocache,
        'ntwk_mgr_ok|m'                                                     => \$ntwk_mgr_ok,
        'offline'                                                           => \$is_offline,
        'print_host=s'                                                      => \$print_host,
        'print_only'                                                        => \$print_only,
        'print_queue=s'                                                     => \$print_queue,
        'print_windows'                                                     => \$print_w32,
        'proxy|prxy=s'                                                      => \$::proxy,
        'rebuild_pip'                                                       => \$rebuild_pip,
        'regen_https_key'                                                   => \$regen_https_key,
        'regen_nm_conf|regen_nm_config=i'                                   => \$::regen_nm_conf,
        'remove_kern'                                                       => \$remove_kern,
        'remove_large_file_cron'                                            => \$remove_large_file_cron,
        'remove_passwords'                                                  => \$do_remove_passwds,
        'remove_pipewire=i'                                                 => \$remove_pipewire,
        'reset_pip'                                                         => \$reset_pip,
        'save_gui_cfg=s'                                                    => \$save_gui_cfg,
        'serialno=s'                                                        => \$serialno,
        'set_swiotlb|swiotlb|do_swiotlb=s'                                  => \$set_swiotlb,
        'show_large_pkgs|show_largest_pkgs|largest_pkgs|large_pkgs'         => \$show_large_pkgs,
        'show_urls'                                                         => \$show_urls,
        'skip_cpu_burn'                                                     => \$skip_cpu_burn,
        'skip_disk_test'                                                    => \$skip_disk_test,
        'skip_firmware'                                                     => \$skip_firmware,
        'skip_fmirror'                                                      => \$skip_fmirror,
        'skip_grub'                                                         => \$skip_grub,
        'skip_gui'                                                          => \$skip_gui,
        'skip_ifrename'                                                     => \$skip_ifrename,
        'skip_installer_check'                                              => \$skip_installer_check,
        'skip_kern'                                                         => \$::skip_kern,
        'skip_pip'                                                          => \$skip_pip,
        'skip_radius'                                                       => \$skip_radius,
        'skip_resume'                                                       => \$skip_resume,
        'skip_xorp'                                                         => \$skip_xorp,
        'skip_yum_all'                                                      => \$skip_yum_all,
        'skip_yum_update'                                                   => \$skip_yum_update,
        'source_dir|sdir=s'                                                 => \$source_dir,
        'tmp_dir|tdir=s'                                                    => \$::tmp_dir,
        'update_pip'                                                        => \$update_pip,
        'use_install_bundle|use_bundle|bundle=s'                            => \$::use_install_bundle,
        'use_yum_cache'                                                     => \$use_yum_cache,
        'wipe_raid'                                                         => \$wipe_raid,
    )
) {
    print("ERROR:\n     $0 $lfkinargs\nUnknown or incomplete option(s) use --help to show options\n");
    exit 1;
}

our $update_dep = "$::home/scripts/py-scripts/update_dependencies.py";
our $webui_update_dep = "$::home/scripts/py-scripts/webGUI_update_dependencies.py";

### Do not put any complicated logic here, we first need to probe
# some system details so that we can make smarter decisions when doing
# more complicated actions like trying to auto-calculate the
# lfver/kver and so forth.
####

if ($::proxy eq "none") {
    $::proxy = "";
    $ctproxy = "";
}

if ($help) {
    print(usage());
    exit(0);
}
if ($helpall) {
    print(usage());
    print($advanced_help, "\n");
    exit(0);
}

if (-f "$::home/LF_INTEROP") {
    print("found $::home/LF_INTEROP...");
    if ($regen_nm_conf < 1) {
        print(" * Changing to interop install mode\n");
        $do_interop = 1;
    }
}
if (-f "$::home/LF_NO_IFRENAME") {
    print "found $::home/LF_NO_IFRENAME, will not rename any devices\n";
    $skip_ifrename = 1;
}

if ($regen_nm_conf > 0) {
    $do_grub = 0;
    $do_kern = 0;
    $do_lanforge = 0;
    $do_pkgs = 0;
    $do_upgrade = 0;
    $do_yum_update = 0;
    $ntwk_mgr_ok = 1;
    $skip_installer_check = 1;
    $skip_yum_all = 1;
}

if ($uname =~ "x86_64") {
    $arch = "x64";
    $archbit = "64";
    $is_64 = 1;
}
elsif (($uname =~ "armv7l") || ($uname =~ "aarch64") || ($uname =~ "arm64")) {
    $arch = "arm";
    $is_x86 = 0;
    $is_arm = 1;

    if (($uname =~ "aarch64") || ($uname =~ "arm64")) {
        $archbit = "64";
        $is_64 = 1;
    }

    my $ln = `grep "Raspberry Pi" /proc/cpuinfo`;
    if ($ln =~ /Raspberry Pi 5/) {
        $is_rpi5 = 1;
    }
    elsif ($ln =~ /Raspberry Pi 4/) {
        $is_rpi4 = 1;
    }
    elsif (-f "/boot/firmware/bcm2710-rpi-3-b.dtb") {
        $is_rpi3 = 1;
    }
}

if (-f "/etc/board.json") {
    $is_openwrt = 1;
    $os_string = "OpenWRT";
    $IW = "/usr/sbin/iw";
    my $content = `cat /etc/board.json`;
    if ($content =~ /"name": "Netgear Nighthawk X4S R7800"/g) {
        $openwrt_board = "R7800";
    }
    elsif ($content =~ /"name": "FriendlyARM NanoPi NEO 2"/g) {
        $openwrt_board = "NEO2";
    }
    elsif ($content =~ /"name": "Edgecore ECW5410"/g) {
        $openwrt_board = "ECW5410";
    }
    elsif ($content =~ /"name": "Edgecore ECW5211"/g) {
        $openwrt_board = "ECW5211";
    }
    elsif ($content =~ /"name": "tp-link EC420 G1"/g) {
        $openwrt_board = "EC420";
    }
    elsif ($content =~ /"name": "Linksys E8450 \(UBI\)"/g) {
        if ($do_mgt_dev eq "eth0") {
            $do_mgt_dev = "wan";
        }
        $openwrt_board = "E8450";
    }
    elsif ($content =~ /"name": "CIG WF194C"/g) {
        $openwrt_board = "CIG194C";
    }
}

if ($wipe_raid) {
    wipe_raid();
    exit 0;
}
if ($create_raid ne "") {
    create_raid($create_raid);
    exit 0;
}

if ($do_kern && !($do_upgrade ||$do_lanforge||$do_all||$do_yum_update)) {
    $do_grub = 1;
    $skip_yum_all = 1;
    $skip_yum_update = 1;
    $skip_pip = 1;
    $skip_cpu_burn = 1;
    $skip_disk_test = 1;
    $skip_installer_check = 1;
    $skip_radius = 1;
    $skip_xorp = 1;
}

### Now that we know more about our board, we can attempt to fetch install data
## and such.
###
sub fetchInstallData() {
    if (system("ping -q -c 1 -n -w 2 www.candelatech.com > /dev/null 2>&1") != 0) {
        err("* * Unable to ping www.candelatech.com, is your network up? * * *\n");
        return;
    }
    if ($www =~ /www\.candelatech\.com/) {
        my $lfs_url = "$www/lfserver_install.php";
        my $lfs_file = "/tmp/lfs_in.txt";
        unlink($lfs_file) if (-f $lfs_file);
        my $rh_request = newWebRequest($lfs_url, $lfs_file, "GET");
        $rh_request->{'quiet'} = 1;
        $rh_request->{"options"} = "-#";
        webget($rh_request);
        if ($rh_request->{"status"} !~ / 200 /) {
            warning("Unable to retrieve $lfs_url to $lfs_file\n");
            return;
        }
        my @lfs_lines = `cat $lfs_file`;
        chomp(@lfs_lines);
        my @meta_lines = grep {/<meta\s+name=['"]x-lanforge-version['"]\s+content=['"][^'"]+['"]/} @lfs_lines;
        if (@meta_lines < 1) {
            `head -20 $lfs_file`;
            warning("* Unable to find metadata about lanforge release version.\n");
        }
        my ($hunk) = ($meta_lines[0] =~ /content=['"]([^'"]+)['"]/);
        $hunk =~ s/--lfver\s+//g;
        note("# Determined release: $hunk\n");
        $::lfver = $hunk;

        @meta_lines = grep {/<meta\s+name=['"]x-kernel-version['"]\s+content=['"][^'"]+['"]/} @lfs_lines;
        if (@meta_lines < 1) {
            `head -20 $lfs_file`;
            die("* Unable to find metadata about kernel version. Cannot continue.\n");
        }
        ($hunk) = ($meta_lines[0] =~ /content=['"]([^'"]+)['"]/);
        $hunk =~ s/--kver\s+//g;
        note("# Determined kernel: $hunk\n");
        $::kver = $hunk;
    }
    else {
        note("# since you are not downloading from www.candelatech.com we assume you know how to set --lfver and --kver\n");
    }
} # ~fetchInstallData

# TODO: some of these methods need to be moved up or others of these statements need to be moved down
if (!(defined $::lfver) || ($::lfver =~ /^\s*$/) || ($::lfver le "5.3.9") || ($::lfver eq "-1")) {
    if (($do_all_ct || $do_all || $do_firmware || $download_only || $do_lanforge) && !($::use_install_bundle || $do_grub)) {
        note("# No --lfver specified. Checking www.candelatech.com/lfserver_install.php for current release...");
        fetchInstallData();
    }
}

sub regen_https_selfcert() {
    my $hostname = `hostname`;
    chomp $hostname;

    if ($hostname =~ /\./) {
        my @hunks = split('.', $hostname);
        $hostname = $hunks[0];
    }
    print("HOSTNAME: $hostname\n") if ($::debug_on);
    my $crtfile = "/etc/pki/tls/certs/$hostname.crt";
    my $csrfile = "/etc/pki/tls/certs/$hostname.csr";
    my $reqconf = "/etc/pki/tls/certs/$hostname.conf";
    print("$crtfile\n") if ($::debug_on);
    my $keyfile = "/etc/pki/tls/private/$hostname.key";
    print("$keyfile\n") if ($::debug_on);
    die("Cannot open $reqconf $?")
        unless open(my $conf_fh, ">", $reqconf);
    my $conf_txt = qq([ req ]
default_bits            = 4096
default_keyfile         = $keyfile
distinguished_name      = req_distinguished_name
prompt                  = no

[ req_distinguished_name ]
C                       = US
ST                      = Washington
L                       = Ferndale
O                       = Candela Technologies, Inc.
OU                      = LANforge
CN                      = $hostname
emailAddress            = support\@candelatech.com

);
    print $conf_fh $conf_txt;
    close $conf_fh;

    unlink($crtfile) if (-f $crtfile);
    unlink($keyfile) if (-f $keyfile);
    #openssl here
    print("openssl genrsa -out $keyfile 4096\n");
    do_cmd("openssl genrsa -out $keyfile 4096");

    print("openssl req -new -key $keyfile -out $csrfile -config $reqconf\n");
    do_cmd("openssl req -new -key $keyfile -out $csrfile -config $reqconf\n");

    print("openssl x509 -req -days 7200 -in $csrfile -signkey $keyfile -out $crtfile\n");
    do_cmd("openssl x509 -req -days 7200 -in $csrfile -signkey $keyfile -out $crtfile");
    print "Keys should be generated. Please try restarting apache.\n";

    if (-f "$::home/do_https_key") {
        unlink("$::home/do_https_key");
    }
}

sub fix_chrony_conf() {
    if ($::is_macos) {
        return;
    }

    if (!-f "/etc/chrony.conf") {
        warn("No /etc/chrony.conf file") if ($::is_fedora && $::debug_on);
        return;
    }
    if ($::osveri > 35) {
        # we don't run secconf module in our kernels, turn chronyd off
        my @lines = `systemctl is-enabled chronyd.service`;
        chomp(@lines);
        if ((@lines > 0) && ($lines[0] eq "enabled")) {
            note("F36+ requires secconf in kernel to run chronyd, switching to journalctl-timesyncd");
            do_cmd("systemctl disable chronyd.service", 1);
            do_cmd("systemctl stop chronyd.service", 1);
            do_cmd("systemctl enable systemd-timesyncd.service", 1);
            do_cmd("systemctl restart systemd-timesyncd.service", 1);
            return;
        }
    }
    # rewrite the chrony.conf file in case we've done something stupid like encountered a local NTP source
    # we should just stick to fedora or ubuntu time servers
    print("# found chrony.conf...") if ($debug_on);
    my @chronylines = `cat /etc/chrony.conf`;
    chomp(@chronylines);
    my $chrony_modifications = 0;
    my @chrony_pool_matches = grep {/^pool pool\.ntp\.org/} @chronylines;
    my @chrony_server_matches = grep {/^server \d\.pool\.ntp\.org/} @chronylines;
    my @chrony_bad_matches = grep {/^server \S*(proxy|bitratchet|firewall|candelatech).*$/} @chronylines;
    if ((@chrony_pool_matches < 1) && ($::osveri > 24)) {
        note("# pool.ntp.org not listed in chrony.conf...");
        push(@chronylines, "pool pool.ntp.org iburst maxsources 4");
        $chrony_modifications++;
    }
    if (@chrony_server_matches < 1) {
        note("# server 0.pool.ntp.org not listed in chrony.conf...");
        $chrony_modifications++;
        push(@chronylines, "server 0.pool.ntp.org iburst auto_offline");
        push(@chronylines, "server 1.pool.ntp.org iburst auto_offline");
        push(@chronylines, "server 2.pool.ntp.org iburst auto_offline");
        push(@chronylines, "server 3.pool.ntp.org iburst auto_offline");
    }
    if (@chrony_bad_matches > 0) {
        for (my $i = 0; $i < @chronylines; $i++) {
            my $chline = $chronylines[$i];
            if ($chline =~ /^server \S*(proxy|bitratchet|firewall|candelatech).*$/) {
                note("# Disabling $chline\n");
                $chronylines[$i] = "# $chline";
                $chrony_modifications++;
            }
        }
    }
    if ($chrony_modifications) {
        die("Unable to open /etc/chrony.conf")
            unless open(my $chronyfile, ">", "/etc/chrony.conf");
        print $chronyfile join("\n", @chronylines), "\n";
        close $chronyfile;
        note("# updated chrony.conf");
        do_cmd("systemctl restart chronyd.service", 1);
        sleep 10 if ($::debug_on);
    }
} #~fix_chrony_conf

sub requirements_txt_name {
    if ($::osveri <= 26) {
        warning("requirements_txt_name: $::osveri version less than F27, no requirements.txt\n");
        return "";
    }
    if (($::osveri < 30) && ($::osveri >= 27)) {
        return "python.3.6.requirements.txt";
    }
    return "requirements.txt" if ($::osveri >= 30);
}

sub fetch_requirements_txt {
    if (-f "$::home/.freeze-python") {
        warning("# not fetching new " . requirements_txt_name() . ", python frozen.");
        return;
    }
    if ("" eq requirements_txt_name()) {
        warning("* this OS version lacks " . requirements_txt_name());
        return;
    }
    my $reqtxt_url = "https://raw.githubusercontent.com/greearb/lanforge-scripts/master/" . requirements_txt_name();
    note("Downloading requirements txt from $reqtxt_url\n");
    my $rh_request = newWebRequest($reqtxt_url,
        "$::home/scripts/" . requirements_txt_name(),
        "GET");
    # $rh_request->{'quiet'} = 0;
    $rh_request->{"options"} = "-#";
    webget($rh_request);
    if (!-f "$::home/scripts/" . requirements_txt_name()) {
        err("unable to download " . requirements_txt_name() . "\n");
    }
    else {
        print("...downloaded $::home/scripts/" . requirements_txt_name() . "\n");
    }
}

# Various types of proxy settings.
my $l_proxy = (defined $::proxy) ? $::proxy : "";
if (("" eq $l_proxy) && (defined $ENV{'ALL_PROXY'}) && ("" ne $ENV{'ALL_PROXY'})) {
    $l_proxy = $ENV{'ALL_PROXY'};
}
if (("" eq $l_proxy) && (defined $ENV{'HTTP_PROXY'}) && ("" ne $ENV{'HTTP_PROXY'})) {
    $l_proxy = $ENV{'HTTP_PROXY'};
}
my $m_proxy = "";
my $proxy_env = "";
if ("" ne $l_proxy) {
    $l_proxy =~ s/http:/https:/g;
    $m_proxy = "--proxy=${q}$l_proxy${q}";
    $::proxy = $l_proxy if ($::proxy eq "");
    $proxy_env = "env https_proxy=$l_proxy";
}

sub update_pip {
    if (-f "$::home/.freeze-python") {
        warning("# not updating pip, python frozen.\n");
        return;
    }
    if ($skip_pip || $is_offline) {
        warning("* skipping pip: --skip_pip[$skip_pip] or --is_offline[$is_offline] set\n");
        return;
    }
    if ((defined $::proxy) && ($::proxy ne "")) {
        print_slow("update_pip: proxy environment variable detected.\nThis can make pip fail to update.\n");
    }
    if (($is_fedora || $is_centos) && ($::osveri < 27)) {
        warning("* Fedora $::osveri python upgrades not supported. Fedora >= 27 supported.");
        return;
    }
    # do_all_ct should not respect this file
    unlink("$::home/did_pylib")
        if ($do_all_ct && -f "$::home/did_pylib");

    if (-f "$::home/did_pylib") {
        if (!$::do_sys_reconfig) {
            note("* skipping python pip upgrade this time but not next time.\n");
            unlink("$::home/did_pylib");
        }
        return;
    }
    # we do a pip system upgrade upgrade first in order to allow the userspace to upgrade
    #do_cmd("su -l lanforge -c ${Q}$proxy_env python3 -m pip install $m_proxy --upgrade pip ${Q}", 1);
    # this is for the pip virtual environment, this call can fail if the userspace pip is newer than the system pip
    #do_cmd("su -l lanforge -c ${Q}$proxy_env python3 -m pip install $m_proxy --upgrade pip --user${Q}", 1);
    #do_cmd("su -l lanforge -c ${Q}$proxy_env pip3 install $m_proxy wheel --user${Q}", 1);
    #fetch_requirements_txt();  # we might have changed version requirements

    print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
    print("su -l lanforge -c ${Q}$proxy_env $update_dep ${Q}");
    print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
    do_cmd("su -l lanforge -c ${Q}$proxy_env $update_dep ${Q}");
    print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");

    if ($do_webui) {
        print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
        print("su -l lanforge -c ${Q}$proxy_env $webui_update_dep ${Q}");
        print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
        do_cmd("su -l lanforge -c ${Q}$proxy_env $webui_update_dep ${Q}");
        print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
    }

    # if (! -f "$home/scripts/". requirements_txt_name()) {
    #   err("* ". requirements_txt_name()." not found, this must be a LANforge 5.4.3 or earlier system.\n".
    #    " Use install_pip_lanforge_scripts() to upgrade script pip requirements.\n");
    # }

    # if ( -f "$home/scripts/". requirements_txt_name()) {
    #    warning("# found ".requirements_txt_name().", updating pip3...\n");
    #    do_cmd("su -l lanforge -c ${Q}pip3 install $m_proxy -r $home/scripts/". requirements_txt_name()." --user${Q}", 1);
    #    do_cmd("touch $home/did_pylib");
    #    if ($::osveri <= 27) {
    #       do_cmd("touch $home/.freeze-python");
    #    }
    #    return;
    # }
}

=pod
   Use this method to uninstall python-pip3 packages that were installed as root
   which come from requirements.txt. Then reset the Fedora python3-pip library by
   doing a dnf reinstall of the package.
=cut
sub reset_pip {
    if (-f "$::home/.freeze-python") {
        warning("# not resetting pip libraries, python frozen. Remove the freeze-python file if really need to.");
        return;
    }
    fetch_requirements_txt(); # we might have changed version requirements
    if (!-f "$::home/scripts/" . requirements_txt_name()) {
        err("* reset_pip: unable to fetch " . requirements_txt_name());
        return;
    }
    my $fname = "$::home/scripts/" . requirements_txt_name();
    my @requirements_lines = `csat $fname`;
    my %requirements_map = ();
    for my $req (@requirements_lines) {
        my ($name) = $req =~ /^([^=<>]+)[=<>]+/;
        $requirements_map{$name} = 1;
    }
    chomp(@requirements_lines);
    # it is possible for pip to be so broken it needs reinstallation immediately
    my $immediate_reinstall = 0;
    die("pip3 variable \$pip not declared yet") if (!(defined $::pip));
    system(qq($::pip --help &>/dev/null));
    my $rv = $?;
    if ($rv == -1) {
        $immediate_reinstall++;
    }
    if ($rv & 127) {
        $immediate_reinstall++;
    }
    if (($rv >> 8) != 0) {
        $immediate_reinstall++;
    }
    if ($immediate_reinstall) {
        note("# pip3 appears broken already, reinstalling it first");
        do_cmd("dnf reinstall -y python3-pip");
    }
    # my @installed_list = `$pip list`;
    # chomp(@installed_list);
    note("# pip3 removing requirements.txt items...");
    # that uninstall command can fail on pexpect or any other distutils related package
    do_cmd("$pip uninstall -r $::home/scripts/" . requirements_txt_name(), 1);
    if (!$immediate_reinstall) {
        note("# pip3 getting reinstalled after removing " . requirements_txt_name());
        do_cmd("dnf reinstall -y python3-pip");
    }
} # ~reset pip

sub rebuild_pip_userenv {
    if ($skip_pip) {
        warning("* skipping pip\n");
        return;
    }
    if (-f "$::home/.freeze-python") {
        warning("# not updating pip, python frozen.");
        return;
    }
    if (($is_fedora || $is_centos) && ($::osveri < 27)) {
        err("* Fedora $::osveri python upgrades not supported. Fedora >= 27 supported.");
        return;
    }
    chdir("$::home/scripts/");
    my $reqtxt = requirements_txt_name();
    unlink "uninstall.txt"
        if (-f "uninstall.txt");
    `grep -v pexpect < $reqtxt > uninstall.txt`;
    die("empty or missing uninstall.txt, cannot continue")
        if (!-f "uninstall.txt" || -z "uninstall.txt");
    print("# ----- Uninstalling pip user space ----- ----- ----- ----- ----- ----- ----- ----- -----\n");
    do_cmd("su -l lanforge -c ${Q}$pip uninstall -r $::home/scripts/uninstall.txt -y${Q}", 1);
    #print("# ----- Updating requirements.tzt ----- ----- ----- ----- ----- ----- ----- ----- -----\n");
    #fetch_requirements_txt();
    # Upgrading pip can force resetting pip packages already installed. This should not be necessary
    # and will frequently fail in an isolated environment.
    #print("# ----- Upgrading pip  ----- ----- ----- ----- ----- ----- ----- ----- ----- -----\n");
    #do_cmd("su -l lanforge -c ${Q}$proxy_env python3 -m pip install --upgrade pip${Q}", 1);
    #do_cmd("su -l lanforge -c ${Q}$proxy_env python3 -m pip install --upgrade pip --user${Q}", 1);
    #do_cmd("su -l lanforge -c ${Q}$proxy_env $pip install --upgrade pip${Q}", 1);

    # this https://stackoverflow.com/questions/28002897/wheel-file-installation indicates to JBR wheels are
    # not necessary for deployments. Not to say they are unecessary for developers.
    #do_cmd("su -l lanforge -c ${Q}$proxy_env $pip install $m_proxy wheel --user${Q}", 1);
    print("# ----- Installing pip user space ----- ----- ----- ----- ----- ----- ----- ----- ----- -----\n");
    do_cmd("su -l lanforge -c ${Q}$proxy_env $pip install --upgrade pip_search", 1);
    do_cmd("su -l lanforge -c ${Q}$proxy_env $pip install --upgrade pip_tools", 1);
    # This command does a strict source compile and requires a lot of *-devel rpm packages to complete
    # and still is not a reliable way to compile the numpy package. On an APU2 this process literally takes
    # over 4 hours because of SATA2 disk speeds. We are electing to using update_dependencies.py
    # instead as a manner of avoiding problematic package compilation
    # do_cmd("su -l lanforge -c ${Q}$proxy_env $pip install -r $::home/scripts/$reqtxt${Q}", 1);

    die("$update_dep not found or not executable")
        unless (-x $update_dep);

    print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
    print("su -l lanforge -c ${Q}$proxy_env $update_dep ${Q}");
    print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
    do_cmd("su -l lanforge -c ${Q}$proxy_env $update_dep ${Q}");
    print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");

    if ($do_webui) {
        print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
        print("su -l lanforge -c ${Q}$proxy_env $webui_update_dep ${Q}");
        print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
        do_cmd("su -l lanforge -c ${Q}$proxy_env $webui_update_dep ${Q}");
        print("\n--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---\n");
    }
}

if ($rebuild_pip) {
    rebuild_pip_userenv();
    exit(0);
}

if ($install_pip_lanforge_scripts) {
    install_pip_lanforge_scripts();
    exit(0);
}

if (!$skip_pip && $update_pip) {
    update_pip();
    exit(0);
}
if ($reset_pip) {
    reset_pip();
    exit(0);
}

if ($::do_gui_autostart > -1) {
    cfg_gui_autostart($::do_gui_autostart);
}

if ($::save_gui_cfg =~ /backup|static/) {
    cfg_save_gui_cfg($::save_gui_cfg);
}
elsif ($::save_gui_cfg ne "") {
    err("  --save_gui_cfg: options should only be backup or static.");
    exit(1);
}

if ($do_remove_passwds) {
    remove_passwords();
    exit(0);
}

sub install_pip_lanforge_scripts {
    if (-f "$::home/did_pylib") {
        note("# not installing pip lanforge_scripts, marker found.");
        unlink("$::home/did_pylib") if (!$::do_sys_reconfig);
    }
    else {
        note("# doing pip install upgrade");
        do_cmd("su -l lanforge -c ${Q}python3 -m pip install --upgrade pip --user${Q}", 1);
        do_cmd("su -l lanforge -c ${Q}pip3 install wheel --user${Q}", 1);
        note("# installing pip lanforge_scripts");
        do_cmd("su -l lanforge -c ${Q}pip3 install lanforge-scripts --user${Q}", 1);
        note("# setting did_pylib marker");
        do_cmd("touch $::home/did_pylib");
    }
}

sub do_release_mirror {
    if (! defined($::lfver) || $::lfver eq "" ) {
        print "*** do_release_mirror: please specify --lfver ***";
        exit 1;
    }
    chmod(0755, "$home");
    my @dir_list = qw(
        /home/lanforge/public_html
        /home/lanforge/public_html/downloads
        /home/lanforge/public_html/private
        /home/lanforge/public_html/private/downloads
    );
    @dir_list = (@dir_list, "/home/lanforge/public_html/private/downloads/r$::lfver");
    for my $dname (@dir_list) {
        next if (-d $dname);
        if (!mkdir($dname, 0775)) {
            err("* Unable to create directory $dname: $!");
            exit(1);
        }
    }
    die($!) unless
        open(my $fh, ">", "/etc/httpd/conf.d/lf-release-mirror.conf");
    print $fh "Alias /home/lanforge/public_html/downloads /downloads\n";
    print $fh "Alias /home/lanforge/public_html/private /private\n";
    print $fh "<Directory /home/lanforge/public_html>\n";
    print $fh "    AllowOverride all\n";
    print $fh "    Options +FollowSymlinks +Indexes +MultiViews\n";
    print $fh "    IndexOptions FancyIndexing FoldersFirst HTMLTable NameWidth=*\n";
    print $fh "    Require all granted\n";
    print $fh "</Directory>\n";
    close($fh);
    do_cmd("apachectl graceful", 1);

    die("** /var/www/html/downloads is not a symlink, cannot continue")
        if (-d "/var/www/html/downloads" && ! -l "/var/www/html/downloads");
    die("** /var/www/html/private is not a symlink, cannot contnue")
        if (-d "/var/www/html/private" && ! -l "/var/www/html/downloads");
    symlink("/home/lanforge/public_html/downloads", "/var/www/html/downloads")
        if (! -l "/var/www/html/downloads");
    symlink("/home/lanforge/public_html/private", "/var/www/html/private")
        if (! -l "/var/www/html/private");
    die("Unable to cd to $home/public_html: $!")
        unless chdir("$home/public_html");
    my $local_urls = "urls_manifest.txt";
    my $cropped_urls = "/tmp/urls_manifest.txt";
    my $manifest_url = "$download_host/private/downloads/r$::lfver/urls_manifest.txt";
    my $rh_request = newWebRequest($manifest_url, "urls_manifest.txt", "GET");
    $rh_request->{'quiet'} = 1;
    $rh_request->{"options"} = "-#";
    webget($rh_request);
    if ($rh_request->{"status"} !~ / 200 /) {
        err("Unable to retrieve $manifest_url \n");
        exit 1;
    }
    # crop out the comments at the start of the file
    do_cmd("grep -v ^# $local_urls > $cropped_urls");
    do_cmd("wget -r -q -P $home/public_html --show-progress --progress=bar:force:noscroll -nH -i $cropped_urls");
    do_cmd("chown -R lanforge:lanforge /home/lanforge/public_html", 1);
}

sub install_selenium {
    # it is possible to install selenium using dnf or pip3
    # if ($selenium_use_dnf) {
    #   do_cmd("$yum install
    print(" ... installing selenium not configured at this time...\n");
}

sub create_webpage {
    die("no ../html directory: $!") unless( -d "../html");
    die("cannot write to ../html directory: $!") unless( -w "../html");
    my $webpage = "../html/lf_kinstall.html";
    print("* Creating webpage $webpage");
    die($!) unless open(my $web_fh, ">", $webpage);
    my $P = '%';
    print $web_fh qq%<!DOCTYPE html>
  <html lang="en">
  <head>
    <!-- GENERATED BY lf_kinstall.pl --create_webpage DO NOT EDIT -->
    <title>lf_kinstall.pl | Candelatech</title>
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <meta charset="UTF-8" />
    <link rel="stylesheet" type="text/css" media="all"   href="default.css" />
    <link rel="stylesheet" type="text/css" media="all"   href="menutext.css" />
    <link rel="stylesheet" type="text/css"               href="media-query-mobile.css" />
    <link rel="stylesheet" type="text/css" media="print" href="print.css" />
    <!--
      Any styles in this section get removed when being browsed as a cookbook
      Apply persistent styles in the <section id=chapter> node.
    -->
    <style>
      body {
         margin:1em;
         padding: 1em;
         background: #eaeaea;
      }
      div#overview {
        width: 75$P;
        margin: 1em auto;
      }
      div#chapter, section#chapter {
         padding: 2em 2em 2em 8em;
      }
      dt {
         margin: 1em 0 0 0;
         font-weight: bold;
      }
      dt:target {
         border: 2px solid #ddffaa;
         border-right-style: none;
         border-bottom-style: none;
         border-left-style: none;
         background-color: #ffff99;
      }
      dt:target + dd {
         border: 2px solid #ddffaa;
         border-right-style: none;
         border-top-style: none;
         border-left-style: none;
         background-color: white;
      }
      .unch {
         display:inline-block;
         min-width:3em;
         width:4em;
         max-width:5em;
      }
      p#subject {
         font-size: 46px;
         font-weight: bold;
         text-align: center;
      }
      p#goal {
         font-size: 14px;
         font-weight: normal;
      }
      section#chapter {
         width: 85$P;
         padding: 0;
         margin: 0 0 0 -4em;
      }
      section#chapter dl#optionsdl {
        margin-left: 0;
        width: 85$P;
      }
      section#chapter nav#leftnav {
         background: #e5e5e5;
         border: 2px solid gray;
         height: 450px;
         margin: 0 -50$P;
         min-width: 50px;
         padding: 0;
         position: sticky;
         overflow: scroll;
      }
      nav#leftnav ol {
         margin: 0;
      }
      nav#leftnav ol li {
         margin 0;
         padding: 0;
      }
      nav#leftnav ol li a {
         padding: 4px 0 1px 0.9em;
      }

    </style>
  </head>
  <body>
    <!-- GENERATED BY lf_kinstall.pl --create_webpage DO NOT EDIT -->
    <p id="subject">Install Script lf_kinstall.pl</p>
    <p id="goal">
      Install and reconfigure your LANforge server with the lf_kinstall.pl script.
    </p>
    <div id="overview">
      The <kbd>lf_kinstall.pl</kbd> script can configure a majority of Linux OS features
      that LANforge requires changes to. This includes:
      <ul>
        <li>Disabling SELinux</li>
        <li>Disableing firewall</li>
        <li>Downloading dnf and LANforce updates</li>
        <li>Adding VNC and RDP access</li>
        <li>Adding firmware</li>
        <li>Disabling or reconfiguring NetworkManager</li>
        <li>Enabling serial console</li>
        <li>Modifying kernel options for iommu, pci-aer and kernel memory</li>
        <li>...and more...</li>
      </ul>
    </div>
    <p><small>If you are looking at the HTML version of this page,
    <a href="/cookbook.php?vol=misc&book=lf_kinstall">the Cookbook version looks better!</a>
    </small></p>
    <!-- GENERATED BY lf_kinstall.pl --create_webpage DO NOT EDIT -->
    <h1 class="big_heading">LANforge Install Script lf_kinstall.pl</h1>
    <section id="chapter">
      <style>
      dt:before {
         content: "--";
      }
      dt:target {
         border: 2px solid #ddffaa;
         border-right-style: none;
         border-bottom-style: none;
         border-left-style: none;
         background: #ffff99;
      }
      dt:target + dd {
         border: 2px solid #ddffaa;
         border-right-style: none;
         border-top-style: none;
         border-left-style: none;
         background-color: white;
      }
      section#chapter {
         width: 90\%;
         padding: 0;
         margin: 0 0 1em 150px;
      }
      nav#leftnav {
         position: sticky;
         top: 100px;
         left: 0px;
         text-align: left;
         margin: 0 -50$P;
         padding: 5px 5px;
         font-size: 10px;
         float: left;
         min-width: 100px;
         width: 150px;
         max-width: 250px;
         background: #e5e5e5;
         height: 450px;
         overflow: scroll;
         border: 2px solid gray;
      }
      nav#leftnav ol {
         margin: 0 0.25em 0 -2em;
         font-size:10px;
         line-height: 0.9;
      }
      nav#leftnav ol li {
         margin: 0;
         padding: 0;
         font-size: 10px;
         line-height: 0.9;
         padding: 4px 0 1px 0.3em;
      }
      nav#leftnav ol li a {
         font-size: 12px;
         line-height: 1.1;
         margin: 0;
         padding: 1px;
      }
      </style>
      <nav id="leftnav">
        <small>Contents</small>
         <ol id="leftnav_list">
         </ol>
     </nav>
     <dl id="optionsdl">
    %;
    my $inch = '<span class="unch">&nbsp;</span>';
    my $tt = '<code>';
    my $_tt = '</code>';
    my $kbd = '<kbd>';
    my $_kbd = '</kbd>';
    my $h2 = '<h2>';
    my $_h2 = '</h2>';
    my $th2 = '##';
    my $dd_tags = 0;
    for my $line (split("\n", $::usage),
                    ("<br/>"),
                    split("\n", $advanced_help))
    {
        if ($line =~ /^[-]+$/) {
            print $web_fh "<hr />\n";
            next;
        }
        if (index($line, "$th2") != -1) {
            $line =~ s[^\s*$th2(.*?)$th2][$h2$1$_h2];
        }
        if ($line =~ /kinstall\.pl .*}/) {
           $line =~ s|((\./)?lf_kinstall\.pl .*[}])|$kbd$1$_kbd|;
        }
        elsif ($line =~ /kinstall\.pl .*\\/) {
           $line =~ s|((\./)?lf_kinstall\.pl .*)$|$kbd$1$_kbd<br/>|;
        }
        elsif ($line =~ /kinstall\.pl .*/) {
           $line =~ s|((\./)?lf_kinstall\.pl .*)$|$kbd$1$_kbd|;
        }
        else {
           if (($dd_tags > 0) && ($line =~ /[*_]+dd/)) {
              $line =~ s|\*_dd|</dd> |;
              $dd_tags--;
           }
           elsif (($dd_tags > 0) && ($line =~ /[*]+dd/)) {
              $line =~ s|\*dd|<dd> |;
              $dd_tags++;
           }

           if ($line =~ /\s+\$\S+/ ) {
              $line =~ s|(\$\S+)|$kbd$1$_kbd|g;
           }
           if ($line =~ m|https?://\S+| ) {
              $line =~ s|(https?://\S+)|$kbd$1$_kbd|g;
           }
           elsif ($line =~ m|\s+/\S+| ) {
              $line =~ s|(/\S+)|$kbd$1$_kbd|g;
           }
           if ($line =~ m|--[a-z]+\S+| ) {
              $line =~ s|(--[a-z]+\S+)|$kbd$1$_kbd|g;
           }
           if ($line =~ m|\\$|) {
               $line =~ s|\\|\\<br/>\n$inch|;
           }
           if ($line =~ m!\S+\.(bash|pl|pm|php|py|sh|tar|txt)\b! ) {
              $line =~ s!([\/A-Za-z0-9_-]+\.(bash|pl|pm|php|py|sh|tar|txt))!$kbd$1$_kbd!g;
           }
           if ($line =~ /^\s+[•]/) {
               $line = "<br/>$line</br>";
           }
           elsif ($line =~ /['"].*?['"]/ ) {
              $line =~ s!(['"])(.*?)(['"])!<span class="DarkGreen">$1$2$3</span>!g;
           }
        }
        if ($line =~ /^Usage:/i) {
            print $web_fh "<h2>Usage</h2>\n";
            next;
        }
        if ($line =~ /^Parameters:/i) {
            print $web_fh "<h2>Parameters</h2>\n<dl id='optionsdl'>\n";
            next;
        }
        if ($line =~ /^(Notes?):(.*)/i) {
            print $web_fh "<br /><strong>$1:</strong> $2\n";
            next;
        }
        elsif ($line =~ /\b-?\d+[):] /) {
            $line =~ s/^\s+//g;
            $line =~ s/>(\d+):/&gt;$1:/;
            $line =~ s/<(\d+):/&lt;$1:/;
            print $web_fh "<br />$inch $line\n";
        }
        elsif ($line =~ /^_$/) {
            print $web_fh "</dd>\n";
            $dd_tags--;
        }
        # this is the parameter keyword:
        elsif ($line =~ /^[=><0-9a-z_-]+: /) {
            #$print "dd_tags[$dd_tags] $line\n";
            if ($dd_tags > 0) {
                print $web_fh "</dd>\n";
                $dd_tags--;
            }
            $line =~ s|(^[=><0-9a-z_-]+):\s*(.*)$|<dt id='$1'>$1</dt>\n<dd>$2|;
            $dd_tags++;
            print $web_fh $line, "\n";
        }
        elsif ($line =~ /\s+Examples?: /) {
            $line =~ s|Examples?: (.*)|<br />$inch<span class="notetext">Example: <code>$1</code></span>|;
            print $web_fh $line, "\n";
        }
        else {
            print $web_fh $line, "\n";
        }
    }

  print $web_fh qq%
        </dl>
<script>
function populate_nav() {
   var leftnav = document.getElementById("leftnav_list");
   var options_dl = document.getElementById("optionsdl");
   var options_dt_list = options_dl.getElementsByTagName("dt");
   console.log("options_dt_list: "+options_dt_list);
   if (!options_dt_list || options_dt_list.length < 1) {
       console.log("populate_nav: no dt elements, bye");
       return false;
   }
   if (!leftnav) {
      console.log("populate_nav: NO LEFTNAV");
      return false;
   }
   console.log("populate_nav starting");
   for (let term of Array.from(options_dl.getElementsByTagName("dt"))) {
      console.log("populate_nav item: "+term.outerHTML);
      if (term.nodeName != "DT") {
         continue;
      }
      var anchor = document.createElement("a");
      anchor.innerHTML = term.id;
      anchor.href="#"+term.id;
      var item = document.createElement("li");
      item.appendChild(anchor)
      leftnav.appendChild(item);
   }
   console.log("populate_nav finished");
}
populate_nav();
</script>
    </section>
  </body>
</html>%;
  close($web_fh);
  print("* ...created");
  exit(0);
} # ~sub create_webpage/

if ($create_webpage == 1) {
  create_webpage();
  exit 0;
}

# if kver is functionally unset, we might have a default
print "KVER [$::kver] skip_kver[$::skip_kern]\n" if ($debug_on);

if (!($is_macos || defined $kver || $::skip_kern) || ($kver =~ /^\s*$/) || ($kver =~ /^-1$/)) {
    if ($remove_kern) {
        err("** Option --remove_kern requires --kver to be specified. Will not remove booted kernel or assume default kernel.");
        exit 1;
    }
    if (exists $kver_map{$::lfver}) {
        $kver = $kver_map{$::lfver};
        note("# Defaulting kernel version to $kver\n");
    }
    else {
        if ($do_kern || $do_grub) {
            err("** Option --kver is unset and there is no default for kernel for LANforge ver $::lfver\n");
        }
    }
}

if ($::lfver ne "") {
    $lfrls = $::lfver;
    $lfrls =~ tr/.//d;
    # Pad to ensure that 5.2.10 is less than 5.3.1.
    # Will need more complexity when middle release number hits double digits.
    # Something like split into array and pad each single digit point number.
    if (length($lfrls) < 4) {
        substr($lfrls, 2, 0) .= '0';
    }
}

if ($force_osveri != -1) {
    note("# Forcing osveri to: $force_osveri\n");
    $::osveri = int($force_osveri);
}

if ($is_macos) {
    my $swvers = `sw_vers | grep ProductVersion`;
    $swvers =~ /ProductVersion:\s+(\d+)\.(\d+)\.(\d+)/;
    $::osveri = $1 * 1000 + $2 * 100 + $3;
    if ($::osveri >= 12000 + 600) {
        $::osver = "12.6";
    }
    elsif ($::osveri >= 10000 + 1500) {
        $::osver = "10.15";
    }
    else {
        $::osver = "10.8";
    }
}
elsif ($issue =~ /Fedora( Linux)? release (\d+)\s+.*/) {
    if (($os_version_id < 0) && ($force_osveri == -1)) {
        $::osveri = int($1);
    }
    $is_fedora = 1;
    $os_string = "Fedora $::osveri";
    if ($::osveri == 12) {
        $::osver = "F11";
    }
    elsif ($::osveri == 15 || $::osveri == 16) {
        $::osver = "F14";
    }
    #  elsif (($::osveri == 17 || $::osveri == 18) ||
    #         ($::osveri >= 17 && (($lfver eq "5.2.8") || ($lfver eq "5.2.9")))) {
    elsif (($::osveri == 17 || $::osveri == 18) ||
        ($::osveri >= 17 && ($lfrls gt "5205") && ($lfrls lt "5210"))) {
        # We only started F19 builds in LF 5.2.10 and higher.
        $::osver = "F17";
    }
    elsif ($::osveri >= 19) {
        $::osver = "F" . $::osveri;
        # Older releases do not have builds for OS greater than Fedora 19.
        if ($lfrls lt "5301") {
            $::osver = "F19";
        }
        # 5.3.1 has Fedora 20 support
        if (($::osveri > 20) && ($lfrls lt "5302")) {
            $::osver = "F20";
        }
        # 5.3.2 gains Fedora 21 support
        if (($::osveri > 21) && ($lfrls lt "5305")) {
            $::osver = "F21";
        }
        # 5.3.4 gains Fedora 23 support
        # F23 support since removed. --Ben
        #if (($::osveri >= 23) && ($lfrls eq "5304")) {
        #  $::osver = "F23";
        #}
        # 5.3.5 gains Fedora 24 support
        # 5.3.7 gains Fedora 27 support
        if ($::osveri >= 36) {
            $::osver = "F36";
        }
        elsif ($::osveri >= 34) {
            $::osver = "F34";
        }
        elsif ($::osveri > 29) {
            $::osver = "F30";
        }
        elsif (($::osveri > 27) && ($::osveri <= 29) && ($lfrls ge "5309")) {
            $::osver = "F29";
        }
        elsif (($::osveri > 27) && ($lfrls ge "5401")) {
            $::osver = "F27";
        }
        elsif ((($::osveri > 24) && ($::osveri < 27)) && ($lfrls ge "5307")) {
            $::osver = "F24";
        }
        elsif ($::osveri > 26) {
            if ($lfrls ge "5307") {
                $::osver = "F27";
            }
            elsif ($lfrls ge "5305") {
                $::osver = "F24";
            }
            else {
                # If older than this, well, things are likely well broken anyway.
                $::osver = "F21";
            }
        }
        elsif ((($::osveri > 21) && ($::osveri < 24)) && ($lfrls ge "5305")) {
            $::osver = "F21";
        }
        #note("osver: $::osver  osveri: $::osveri  lfrls: $lfrls\n");
    }
    else { # implies that osveri < 19
        $::osver = "F" . $::osveri;
    }
}
elsif ($issue =~ /Red Hat/) {
    my ($v) = $issue =~ /Red\s+Hat\s+.*?release\s+([.0-9]+)\s*/i;
    $is_redhat = 1;
    $no_fmirror = 1;
    $os_string = "Red Hat $v";
    if ($v =~ /^6\./) {
        $::osver = "C6X";
        $::osveri = 13;
    }
    elsif ($v =~ /^7\./) {
        $::osver = "C7X";
        $::osveri = 19;
    }
    elsif ($v =~ /^8\./) {
        $::osver = "C8X";
        $::osveri = 28;
        $yum_groups = q( "Development Tools" "System Tools" "Network Servers");
    }
}
elsif ($issue =~ /^CentOS /) {
    my ($v) = $issue =~ /^CentOS .*?release ([0-9.]+)\b/i;
    $is_centos = 1;
    $os_string = "CentOS $v";
    note("# Using CentOS, ver: $v\n");

    if (($v eq "6.4") || ($v eq "6.3") || ($v eq "6.2")) {
        $::osver = "F13";
        $curl_ssl = "";
    }
    elsif (($v eq "6.5") ||
        ($v eq "6.6") ||
        ($v eq "6.7") ||
        ($v eq "6.8") ||
        ($v eq "6.9")) {
        $curl_ssl = "";
        if ($lfrls lt "5301") {
            $::osver = "F13";
        }
        else {
            # We only build 'Centos-6.6' images, but they will work on 6.7
            # A 'yum update' of 6.5, etc, will take it to 6.7, by the way.
            $::osver = "C66";
        }
    }
    elsif ($v =~ /^7\.[0-9].*/) {
        if ($force_osveri == -1) {
            $::osveri = 19;
        }
        $::osver = "C7X";
    }
    elsif ($v =~ /^8\.[0-9].*/) {
        if ($force_osveri == -1) {
            $::osveri = 28;
        }
        $::osver = "C8X";
    }
}
elsif ($issue =~ /Ubuntu (12\.\S+).*/) {
    my $v = $1;
    $is_ubuntu = 1;
    $os_string = "Ubuntu";
    if ($force_osveri == -1) {
        $::osveri = 12;
    }
    $::osver = "Ubuntu 12";
    $is_deb_based = 1;
    $apt = "apt-get";
    $http_root = "/var/www";
    $grub_mkconfig = "grub-mkconfig";
    $grub_cfg = "/boot/grub/grub.cfg"; # This is grub2
}
elsif ($issue =~ /Ubuntu (?:release )?([0-9]+)[.]\S+/) {
    my $v1 = $1;
    my $v2 = $1;
    $apt = "apt";
    $is_ubuntu = 1;
    $os_string = "Ubuntu";
    $uveri = $v1;
    if ($v1 <= 13) {
        $apt = "apt-get";
        if ($force_osveri == -1) {
            $::osveri = 17; # Fedora 17 builds should work for Ubuntu 13
        }
        $http_root = "/var/www";
    }
    else {
        if ($force_osveri == -1) {
            if ($v1 == 14) {
                $::osveri = 19; # Fedora 19 builds should work for Ubuntu 14
            }
            elsif ($v1 > 14 && $v1 <= 16) {
                $::osveri = 21;
            }
            # shifting U20.04 to F30 because the logic to
            # replace all the other downloaded libraries is
            # a lot more work than shifting the version of the
            # server package.
            elsif ($v1 > 16 && $v1 < 20) {
                $::osveri = 27;
            }
            elsif ($v1 >= 20) {
                $::osveri = 30;
            }
            else {
               die("un-handled version of ubuntu: $v1.$v2");
            }
        }
        chomp($uname);
        if ($is_x86) {
            if ($is_64) {
                # Install 32-bit compat libraries for gua, pesqserver
                $other_deb_pkgs .= " libc6:i386 zlib1g:i386 libldap-2.4-2:i386";
            }
            $ubuntu_deb_ver = ""; #"-U14";
        }
    }
    $::osver = "Ubuntu $v1";
    $is_deb_based = 1;
    $grub_cfg = "/boot/grub/grub.cfg"; # This is grub2
    $grub1_cfg = "NOT_HERE";
    $grub_mkconfig = "grub-mkconfig";
}
elsif ($issue =~ /Raspbian GNU\/Linux (\d+)/) {
    $apt = "apt-get";
    if ($force_osveri == -1) {
        $::osveri = int($1);
    }
    $os_string = "Raspbian";
    $::osver = "Raspbian $::osveri";
    $is_deb_based = 1;
    $http_root = "/var/www";
}
elsif ($issue =~ /Debian GNU\/Linux (\d+)/) {
    $apt = "apt-get";
    if ($force_osveri == -1) {
        $::osveri = int($1);
    }
    $os_string = "Debian";
    $::osver = "Debian $::osveri";
    $is_deb_based = 1;
    $http_root = "/var/www";
}
else {
    print_slow("# Unknown OS version, will use defaults.");
}

if (defined $download_host && "$download_host" ne "") {
    #$force_web = 1; # setting it this high turns off detection of /mnt/d2
    if ($download_host eq "1") {
        die("download_host must be form of https?://hostname");
    }
    $www = $download_host;
}

# ====== ====== ====== ======       You can use $osveri below here      ====== ====== ====== ====== ======

if ($do_release_mirror == 1) {
    do_release_mirror();
    exit 0;
}

if ($is_fedora && ($osveri < 27)) {
    $skip_pip = 1;
}
if ($is_redhat || $is_centos || $is_ubuntu || $is_deb_based) {
    $skip_pip = 1;
}

if (!$::create_install_bundle && (! -w $home)) {
    print("WARNING: \$home [$home] is not writable, does it exist?\n");
    sleep(3);
    if (! (-e $home && -d $home)) {
        print("\$HOME [$home] does not exist!\nYou will be prompted to create the lanforge user.\n");
        sleep(1);
    }
    # $home = $ENV{'HOME'};
    if ( $ENV{'HOME'} ne "/home/lanforge") {
        my $logged_in = (defined $ENV{'SUDO_USER'}) ? $ENV{'SUDO_USER'} : $ENV{'LOGNAME'};
        print(qq(    \$HOME [$ENV{'HOME'}] is NOT $home, and $home does not exist.\n)
            .qq(        Logged in user:   "$logged_in"\n)
            .qq(        Effective User:   "$ENV{'LOGNAME'}"\n));
        sleep(2);
        print "===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== \n";
        print "We suggest creating a 'lanforge' user with home directory $home.\n";
        print "===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== \n";
        print "Proceed? [Y|n] ";
        my $answer = <STDIN>;
        chomp $answer;
        if ($answer =~ /^[Yy][EeSs]*/) {
            print "Creating lanforge user...\n";
            $::home = ($is_macos) ? "/Users/lanforge" : "/home/lanforge";
            create_lf_user();
        }
        else {
            print "===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== \n";
            print "Not creating lanforge user. Files and directories for this LANforge\n";
            print "installation will appear in '$home', and many programs might\n";
            print "fail from this unanticipated path.\n";
            print_slow("===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== \n");
        }
    }
} # ~if !-w $home

if ($is_ubuntu || $is_deb_based || $is_fedora || $is_centos || $is_redhat ) {
    if (system("which systemctl &>/dev/null") == 0) {
        $use_systemctl = 1;
    }
}
if ($do_enable_max_zram) {
    enable_max_zram();
    exit 0;
}

if ($::do_abandoned_cleanup) {
    createAbandonedCleanup();
    exit 0;
}
if ($regen_https_key) {
    regen_https_selfcert();
    exit;
}
if ($do_print_label && !$is_macos) {
    if (!defined($print_host) || ("" eq $print_host) || ($print_host eq "0")) {
        print "What host? Example: --print_label --print_host 10.3.0.1:8082 --print_queue QL-800 --serialno HOSTNAME\n";
        exit 1;
    }
    if (!defined($print_queue) || ("" eq $print_queue) || ($print_queue eq "0")) {
        print "What print queue? Example: --print_label --print_host 10.3.0.1:8082: --print_queue QL-800 --serialno HOSTNAME\n";
        exit 1;
    }
    if (!-f "/etc/udev/rules.d/70-persistent-net.rules") {
        print "File 70-persistent-net.rules not found. Print labels after --do_sys_reconfig. Will not print.\n";
        exit 1;
    }
    ## Iain, look here:
    my %cpu_map = (
        "AMD GX-412TC SOC"               => "ct521a",
        "Intel... Core.... i5-6300U CPU" => "ct522",
        "Intel... Core.... i7-7700T CPU" => "ct523c"
    );
    my %mac_map = (
        "00:0d:b9" => "ct521a",
        "00:30:18" => "ct522",
        "00:60:e0" => "ct523c",
        "00:0e:8e" => "n",
        "04:f0:21" => "ac"
    );
    my %lspci_map = (
        "Qualcomm Atheros AR93xx Wireless Network Adapter"                   => "n",
        "Qualcomm Atheros QCA986x\/988x 802\.11ac Wireless Network Adapter"  => "ac",
        "Qualcomm Atheros QCA9984 802\.11ac Wave 2 Wireless Network Adapter" => "ac2"
    );
    my $adapter_counter = ();
    my @proccpu_lines = `cat /proc/cpuinfo`;
    chomp(@proccpu_lines);
    my @lspci_lines = `lspci | grep Network`;
    chomp(@lspci_lines);
    my %adapter_counter = ();
    my $model_info = "";
    for my $cpu_key (keys %cpu_map) {
        my @matches = grep {/$cpu_key/} @proccpu_lines;
        if (@matches > 0) {
            $model_info .= $cpu_map{$cpu_key};
            last;
        }
    }
    # print "LSPCI_LINES:\n".join("\n||| ", @lspci_lines), "\n";
    for my $network_key (keys %lspci_map) {
        # print "        Checking $network_key\n";
        my @matches = grep {/$network_key/} @lspci_lines;
        if (@matches > 0) {
            $adapter_counter{ $lspci_map{$network_key} } = 0
                if (!exists $adapter_counter{ $lspci_map{$network_key} });
            $adapter_counter{ $lspci_map{$network_key} }++;
        }
    }

    for my $adapter_key (sort keys %adapter_counter) {
        $model_info .= "-" . $adapter_counter{$adapter_key} . $adapter_key;
    }
    print "model info " . $model_info, "\n";

    my $url = $print_host;
    $url = "http://" . $url if ($url !~ m!^http://!);
    $url = $url . ":8082/" if ($url !~ m!:8082/?$!);
    my $hostname = `hostname`;
    chomp($hostname);
    my $model = "Custom";
    if ($hostname =~ /-/) {
        my @hunks = split("-", $hostname);
        $model = $hunks[0];
    }
    $model = $model_info if ($model_info ne "");

    if (!(defined $serialno) || ("" eq $serialno) || ($serialno eq "0")) {
        print "If there is no serial number, use HOSTNAME to copy value `hostname`\n";
        exit 1;
    }
    if ($serialno =~ /^HOST*/i) {
        $serialno = $hostname;
    }
    my @persistent_rules = `cat /etc/udev/rules.d/70-persistent-net.rules`;
    chomp(@persistent_rules);
    my @matches = grep {/ NAME="eth0"/} @persistent_rules;
    if (@matches < 1) {
        print "Unable to find eth0 udev rule. Will not print.\n";
        print join("\n  > ", @persistent_rules), "\n";
        exit 1;
    }
    my ($mac) = $matches[0] =~ / ATTR\{address\}==\"([^\"]+)\", /;
    if (!(defined $mac)) {
        print "Unable to determine eth0 mac, cannot print.\n";
        exit 1;
    }
    my $qs = "printer=" . uri_escape($print_queue)
            . "&mac=" . uri_escape($mac)
            . "&model=" . uri_escape($model)
            . "&hostname=" . uri_escape($hostname)
            . "&serial=" . uri_escape($serialno);
    #print "Submit print job [$qs]\n";
    if ($print_queue eq "debug") {
        print "Would post: $qs to $url\n";
    }
    else {
        do_cmd("$curl -v -d '$qs' $url", 1);
        print "...submitted\n";
    }

    exit 0;
}

sub collect_ouis() {
    my $ra_found_oui_list = [];
    if ($is_fedora || $is_redhat || $is_centos || $is_ubuntu ) {
        my @mac_addr_list = (`cat /sys/class/net/*/address`);
        if (@mac_addr_list > 0) {
            chomp(@mac_addr_list);
            for my $mac (@mac_addr_list) {
                push(@$ra_found_oui_list, substr($mac, 0, 8));
            }
        }
    }
    return $ra_found_oui_list;
}

sub using_vm_oui() {
    my $ra_ouis = collect_ouis();
    my $is_vm = 0;
    #print Dumper(["discovered ouis", $ra_ouis]) if ($::debug_on);
    while(my ($name, $pattern) = each(%::vm_oui_patterns)) {
        foreach my $oui (@$ra_ouis) {
            if ($oui =~ qr/$pattern/i) {
                $is_vm ++;
                print ("Found $name mac address\n") if ($::debug_on);
                last;
            }
        }
    }
    return $is_vm;
}

our $has_vm_oui = using_vm_oui();

my $check_large_cron_fn = "/etc/cron.hourly/lf_check_large_files.sh";
if ($install_large_file_cron) {
    if (-f $check_large_cron_fn) {
        note("replacing $check_large_cron_fn");
        unlink $check_large_cron_fn;
    }
    my $lfc_text = qq(# LANforge cron job to check for large files
if [ -x $::home/scripts/check_large_files.bash ]; then
    $::home/scripts/check_large_files.bash -a 2>&1 | logger -t lf_check_large
fi
);
    die("Unable to write $check_large_cron_fn")
        unless open(my $lfc_fh, ">", $check_large_cron_fn);
    print $lfc_fh $lfc_text, "\n";
    close($lfc_fh);
    chmod(0755, ($check_large_cron_fn));
    do_cmd("systemctl restart crond.service", 1);
    exit;
}

if ($remove_large_file_cron) {
    if (-f $check_large_cron_fn) {
        note("Removing $check_large_cron_fn");
        unlink $check_large_cron_fn;
        do_cmd("systemctl restart crond.service", 1);
    }
    exit;
}

if ($::use_install_bundle) {
    if ("" eq $::use_install_bundle) {
        err("Use_install_bundle wants name of tar file, bye.");
        exit 1;
    }
    my @previous_installs = `ls -1d /home/lanforge/LANforgeServer-*`;
    note("Use_install_bundle opening $::use_install_bundle\n");
    my $basedir = undef;
    my $bundle = undef;
    my @tar_items = ();

    # should force relative paths into canonical paths
    $::use_install_bundle = `readlink -f $::use_install_bundle`;
    chomp $::use_install_bundle;
    my @hunks = split(/\//, $::use_install_bundle);
    if ("" eq $hunks[$#hunks]) {
        err("install bundle looks like a blank name, bye");
        exit 1;
    }
    $bundle = $hunks[$#hunks];
    my $pos = rindex($::use_install_bundle, '/');
    $basedir = substr($::use_install_bundle, 0, $pos);
    my $old_dir = $curr_dir;
    if ($::tmp_dir ne $basedir) {
        $::tmp_dir = $basedir;
        $source_dir = $basedir;
    }
    chdir($basedir);
    my $lastver = "unknown";
    my $lastkern = `uname -r`;
    chomp $lastkern;
    chomp(@previous_installs);
    if (@previous_installs) {
        ($lastver) = $previous_installs[$#previous_installs] =~ /LANforgeServer-([.0-9]+)/;
    }
    note("In $basedir, inspecting $bundle...");
    @tar_items = `tar -tf $bundle`;
    chomp(@tar_items);
    my @lfs = grep {/LANforgeServer-/} @tar_items;
    if (!@lfs) {
        warn("ERROR: unable to find LANforgeServer in $bundle; Items seen:\n");
        warn(join("\n", @tar_items) . "\n");
        exit(1);
    }
    ($::lfver) = $lfs[0] =~ /LANforgeServer-(.*)_/;
    $lfrls = $::lfver;
    $lfrls =~ tr/.//d;
    if (length($lfrls) < 4) {
        substr($lfrls, 2, 0) .= '0';
    }

    if (!(defined $::lfver) || ($::lfver eq "") || ($::lfver lt "5.3.8")) {
        warn("ERROR: unable to determine LANforgeServer version in $bundle; Items seen:\n");
        warn(join("\n", splice(@lfs, 0, 10)) . "\n....\n");
        warn("You might want Bundle_lfver-${lastver}_kern-${lastkern}_osver-${main::osver}-i-${main::osveri}_x64.tar.xz\n");
        exit(1);
    }
    if ($::osver ne "F$::osveri") {
        #note("** Adjusting osver[$osver] to [F$::osveri]\n");
        $::osver = "F$::osveri";
    }
    fix_chrony_conf();
    my @kerns = grep {/ct\d+\.\d+\.\d+[+]/} @tar_items;
    if (!@kerns) {
        warn("ERROR: unable to find kernel in $bundle; Items seen:\n");
        warn(join("\n", splice(@tar_items, 0, 10)) . "\n....\n");
        my $v = $::kver_map{$::lfver};
        if ($::lfver ne "") {
            warn("You might want Bundle_lfver-${main::lfver}_kern-${lastkern}_osver-${main::osver}-i-${main::osveri}_x64.tar.xz\n");
        }
        else {
            warn("You might want Bundle_lfver-${lastver}_kern-${lastkern}_osver-${main::osver}-i-${main::osveri}_x64.tar.xz\n");
        }
        exit(1);
    }
    $::kver = pop @kerns;
    $::kver = substr($::kver, 2, index($::kver, '+') - 1);
    if (!(defined $::osveri) || "" eq $::osveri || $::osveri < 21) {
        die("Unable to determine OSVERI; osver[$::osver] osveri[$::osveri] from ${lfs [ 0 ]}\n");
    }
    note("lfver [$::lfver] lfrls[$lfrls] osver[$::osver] osveri[$::osveri]\n");
    note("In $basedir, extracting $bundle...");
    do_cmd("tar -xvf $bundle");
    chdir($old_dir);
    print "Equivalent command becomes:\n";
    print "lf_kinstall.pl --lfver $::lfver --kver $::kver --force_osver $::osver --force_osveri $::osveri --do_lanforge --do_kern --do_firmware --source_dir $source_dir\n";
    sleep(4);

    if (!(defined $::tmp_dir) || ("" eq $tmp_dir)) {
        err("Cannot use install bundle if tmp_dir unset.");
        exit 1;
    }
    $disable_audit_logs = 1;
    $do_cma = cma_recommendation() if ($do_cma == 1);
    $do_firmware = 1;
    $do_grub = 1;
    $do_http = 1;
    $do_kern = 1;
    $do_lanforge = 1;
    $do_pkgs = 0;
    $do_upgrade = 1;
    $do_xrandr = 0;
    $do_yum_update = 0;
    $is_offline = 1;
    $skip_cpu_burn = 1;
    $skip_fmirror = 1;
    $skip_pip = 1;
    $skip_xorp = 0;
    $skip_yum_all = 1;
    $skip_yum_all = 1;
    $skip_yum_update = 1;
    $skip_yum_update = 1;
    $source_dir = $::tmp_dir;
} # ~if use_install_bundle

if ($do_kern && ($skip_kern==1)) {
    warn("* * Both --do_kern and --skip_kern specified. Disabling --do_kern and --do_grub");
    $do_kern = 0;
    $do_grub = 0;
}
if ($do_kern && $remove_kern) {
    err("** --do_kern enabled and --remove_kern specified. Choose one.");
    exit(1);
}

if ($do_kern || $remove_kern) {
    $do_grub = 1;
    if (! $do_upgrade || $do_all || $do_all_ct) {
        $skip_yum_all = 1;
        $skip_yum_update = 1;
        $do_pkgs = 0;
        $use_yum_cache = 1;
        $skip_pip = 1;
    }
}

if ($show_large_pkgs) {
    if ($is_centos || $is_fedora || $is_redhat) {
        do_cmd("rpm -qa --queryformat '%${L}SIZE${R} %${L}NAME${R} \n' | sort -n -r | head -20", 1);
    }
    elsif ($is_ubuntu) {
        #dpkg-query -Wf '${Installed-Size} ${Package}\n' | sort -nr | head -10
        do_cmd("dpkg-query -Wf ${q}${d}${L}Installed-Size${R}   ${d}${L}Package${R}\n${q} | sort -rn | head -20", 1);
    }
    exit(0);
}
# TODO: split up the nginx config versus the vide downloads
if ($download_videos) {
    note("Downloading items from www.candelatech.com/videos....");
    my $num_v = 0;
    my $video_dir = "$http_root/videos";
    my $video_link = "/usr/local/lanforge/nginx/html/videos";
    mkdir($video_dir) unless (-d $video_dir);
    symlink($video_dir, $video_link) unless ((-l $video_link) || (-d $video_link));
    my $destination = "";
    my @url_list = ();
    my $urldir = "$www/videos/";
    unlink("/tmp/vidlist") if (-f "/tmp/vidlist");
    my $rh_request = newWebRequest($urldir, "/tmp/vidlist", "GET");
    $rh_request->{'quiet'} = 1;
    $rh_request->{"options"} = "-#";
    webget($rh_request);
    if ($rh_request->{"status"} !~ / 200 /) {
        err("Unable to retrieve $urldir to /tmp/vidlist\n");
        exit 1;
    }
    my @lines = `cat /tmp/vidlist`;
    chomp(@lines);
    my @matches = grep {/href=".*\.mp4"/} @lines;
    if (@matches < 1) {
        err("Unable to see mp4 entries in /tmp/vidlist");
        exit 1;
    }
    for my $line (@matches) {
        #print "L> $line\n";
        my ($hunk) = $line =~ m|href="([^"]+\.mp4)"|;
        push(@url_list, "http://www.candelatech.com/videos/" . $hunk) if ($hunk);
    }
    $num_v = @url_list;
    note("Found $num_v items to download");
    for my $url (@url_list) {
        my ($name) = $url =~ m|/([^/]+)$|;
        $name =~ s/%20/_/g;
        $destination = "$http_root/videos/$name";
        note("Pulling $name from $url");
        $rh_request = newWebRequest($url, $destination, "GET");
        $rh_request->{"options"} = "-#";
        $rh_request->{"resume"} = 1;
        webget($rh_request);
        # stuff
    }
    my $nginx_inc = "/usr/local/lanforge/nginx/video.inc";
    if (!-f $nginx_inc) {
        die("unable to open $nginx_inc") unless open(my $n_fh, '>', $nginx_inc);
        note("creating nginx include in /usr/local/lanforge/nginx");
        print $n_fh qq[# include this to gain access to video dir listing
location /videos {
   root   html/videos;
   index  index.html index.htm;
   autoindex on;
   autoindex_exact_size off;
}
];
        close $n_fh;
    }
    exit 0;
}

sub create_ifcfg_eth0 {
    if (!($is_fedora || $is_centos || $is_redhat)) {
        err("make_ifcfg_eth0 is only a fedora/centos/redhat option");
        exit(1);
    }
    my $seventy = "/etc/udev/rules.d/70-persistent-net.rules";
    if (!-f $seventy) {
        err("make_ifcfg_eth0: /etc/udev/rules.d/70-persistent-net.rules not found. Use --do_udev to create it.");
        exit(1);
    }
    my @seventy_lines = `cat $seventy`;
    my @matches = grep {/^[^\x23].* NAME="eth0"/} @seventy_lines;
    if (!@matches || (@matches > 1)) {
        err("make_ifcfg_eth0: No *single* rule for eth0 exists in 70-persistent-net.rules. Please inspect.");
        exit(1);
    }
    my ($addr) = $matches[0] =~ /{address}=="([^"]+)"/;
    if (!(defined $addr) || ("" eq $addr)) {
        err("make_ifcfg_eth0: No MAC address for eth0 rule defined in 70-persistent-net.rules. Please inspect.");
        err("LINE: $addr");
        exit(1);
    }
    my $ifcfg_fname = "/etc/sysconfig/network-scripts/ifcfg-eth0";
    note("writing new $ifcfg_fname for eth0/$addr");
    open(FILE, ">", $ifcfg_fname) or die("Couldnt open file: $ifcfg_fname for writing: $!\n\n");
    print FILE qq[TYPE=Ethernet
BOOTPROTO=dhcp
ONBOOT=yes
DEFROUTE=yes
PEERDNS=yes
PEERROUTES=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_PEERDNS=yes
IPV6_PEERROUTES=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=eth0
DEV=eth0
DEVICE=eth0
HWADDR=$addr
AUTOCONNECT_PRIORITY=-999
];
    close FILE;
}

if ($make_ifcfg_eth0) {
    create_ifcfg_eth0();
    exit(0);
}

if ($do_fedrepos_default) {
    fedrepos_default();
    exit 0;
}
if ($do_restore_metalinks) {
    restore_all_metalinks();
    exit 0;
}
if ($do_enable_archive_baseurl) {
    enable_archive_baseurl();
    exit 0;
}
if ($add_random_www_data) {
    if ($lfrls > 5308) {
        create_random_www_data();
    }
    else {
        note("# add_random_www_data option available in 5.3.9 or later");
    }
    exit 0;
}
if ($bind_apache_mgt_port) {
    if ($lfrls > 5308) {
        config_apache_mgt_port();
    }
    else {
        note("# bind_apache_mgt_port option available in 5.3.9 or later");
    }
    exit 0;
}
if ($do_vm_prep) {
    if ($::osveri < 21) {
        err("do_vm_prep useful only on F21 and later");
        exit(1);
    }
    if (!-f "/root/resize-home.sh") {
        err("Missing: /root/resize-home.sh");
        err("Try: wget http://ctweb/bxb/cf-image/resize-home.sh");
        exit(1);
    }
    if ($::osveri < 30) {
        my $deft = "/lib/systemd/system/default.target";
        my $graft = "/lib/systemd/system/graphical.target";
        unlink($deft) if (-f $deft);
        symlink($graft, $deft);
    }
    else {
        do_cmd("systemctl unmask graphical.target", 1);
        do_cmd("systemctl set-default graphical.target");
    }
    my @stat = `systemctl is-active --quiet vncserver\@:1.service`;
    if ($? != 0) {
        print "vncserver\@:1.service not enabled or not installed\n";
        print join("\n", @stat) . "\n";
    }
    else {
        do_cmd("systemctl stop vncserver\@:1.service", 1);
        do_cmd("systemctl disable vncserver\@:1.service", 1);
    }

    @stat = `systemctl is-active --quiet xrdp.service`;
    if ($? != 0) {
        print "xrdp.service not enabled or not installed\n";
        print join("\n", @stat) . "\n";
    }
    else {
        do_cmd("systemctl stop xrdp.service", 1);
        do_cmd("systemctl disable xrdp.service", 1);
    }
    fix_chrony_conf();
    @stat = `systemctl is-active --quiet pcscd.service`;
    if ($? != 0) {
        print "pcscd.service not enabled or not installed\n";
        print join("\n", @stat) . "\n";
    }
    else {
        do_cmd("systemctl stop pcscd.service", 1);
        do_cmd("systemctl stop pcscd.socket", 1);
        do_cmd("systemctl disable pcscd.service", 1);
        do_cmd("systemctl disable pcscd.socket", 1);
    }
    do_cmd("systemctl disable abrtd.service", 1);
    do_cmd("systemctl disable abrt-xorg.service", 1);
    do_cmd("systemctl disable abrt-vmcore.service", 1);
    do_cmd("systemctl disable abrt-pstoreoops.service", 1);
    do_cmd("systemctl disable abrt-oops.service", 1);
    do_cmd("systemctl disable abrt-journal-core.service", 1);
    do_cmd("systemctl disable bluetooth.service", 1);
    do_cmd("systemctl disable dbus-org.bluez.service", 1);
    do_cmd("systemctl mask dbus-org.bluez.service", 1);
    do_cmd("systemctl disable hddtemp.service", 1);
    do_cmd("systemctl disable lm_sensors.service", 1);
    do_cmd("systemctl disable mcelog.service", 1);
    do_cmd("systemctl disable radiusd.service", 1);
    do_cmd("systemctl disable radiusd.service", 1);
    do_cmd("systemctl daemon-reload", 1);

    rename("/etc/issue", "/etc/issue.orig");
    open(my $fh, ">", "/etc/issue");
    print $fh "${B}S\nCandelatech LANforge $::lfver Kernel ${B}r on ${B}m\n";
    print $fh "LANforge $::lfver\n\n";
    print $fh "If you need to resize /home, use resize-home.sh\n";
    print $fh "\nUse the Configure LANforge icon to set up your networking.\n\n";
    close($fh);
    open($fh, ">", "/etc/issue.resized");
    print $fh "${B}S\nCandelatech LANforge $::lfver Kernel ${B}r on ${B}m\n";
    print $fh "LANforge $::lfver\n\n";
    print $fh "\nUse the Configure LANforge icon to set up your networking.\n\n";
    close($fh);
    rename("/etc/motd", "/etc/motd.orig");
    open($fh, ">", "/etc/motd");
    print $fh "\nWelcome to Candelatech LANforge $::lfver\n\n";
    print $fh "\nUse the Configure LANforge icon to set up your networking.\n\n";
    close($fh);

    do_cmd("echo 'LANforge-vm' > /etc/hostname", 1);
    if ($::is_fedora && ($osveri < 34)) {
        do_cmd("hostnamectl --static set-hostname LANforge-vm", 1);
    }
    @stat = glob q(/etc/sysconfig/network-scripts/ifcfg-e*);
    my $f;
    foreach $f (@stat) {
        print "Removing network config for: $f\n";
        unlink $f;
    }
    @stat = (
        "/etc/udev/rules.d/70-persistent-net.rules",
        "/etc/udev/rules.d/70-persistent.rules",
        "/etc/udev/rules.d/70-phyname.rules",
        "/etc/resolv.conf"
    );
    for $f (@stat) {
        unlink($f) if (-f $f);
    }
    do_cmd("dnf clean all -y", 1);

    @stat = ("/home/lanforge/Downloads",
        "/home/lanforge/tmp",
        "/home/lanforge/.cache",
        "/home/lanforge/wifi",
        "/home/lanforge/vr_conf",
        "/home/lanforge/old_logs_1",
        "/home/lanforge/l4logs",
        "/home/lanforge/gui-eula-ok.txt",
        "/home/lanforge/server-eula-ok.txt",
        "/root/.cache",
        "/var/tmp",
        "/var/cache/yum",
        "/var/lib/dhcpd",
        "/var/lib/dhclient",
        "/var/lib/NetworkManager",
    );
    for $f (@stat) {
        print "find $f -type f -exec rm -f {} ${B};\n";
        do_cmd("find $f -type f -exec rm -f {} \\;", 1)
            if (-d $f);
    }
    @stat = glob("/home/lanforge/*_log*.txt");
    push(@stat,
        "/home/lanforge/.bash_history",
        "/root/.bash_history"
    );
    for $f (@stat) {
        print "rm -f $f\n";
        do_cmd("rm -f $f", 1);
    }
    `bash -c "history -c"`;
    @stat = (
        "uname -a",
        "uname -r",
        "df -h",
        "cat /home/lanforge/license.txt",
        "telnet localhost 4001",
        "ip li show",
        "ip a show",
        "ping www.candelatech.com",
        "host www.candelatech.com",
        "service lanforge restart",
        "ip vrf exec _vrf1 ping 8.8.8.8",
        "cd /home/lanforge; ./lfdebug.bash",
        "cd /root; ./lf_kinstall.pl --lfver 5.4.2 --kver 5.4.52+ --do_upgrade --print_only",
        "killall dhclient; ip a flush eth0; dhclient eth0",
        "dnf clean all; dnf makecache",
    );
    open($fh, ">", "/home/lanforge/.bash_history");
    `chown lanforge:$user_group /home/lanforge/.bash_history`;
    `chmod 640 /home/lanforge/.bash_history`;
    print $fh join("\n", @stat);
    print $fh "\n";
    close $fh;
    open($fh, ">", "/etc/hosts");
    print $fh "::1           localhost.localdomain localhost localhost6.loaldomain6 localhost6\n";
    print $fh "127.0.0.1     localhost.localdomain localhost localhost4.localdomain4 localhost4\n";
    print $fh "192.168.1.101 lanforge.localnet lanforge.localdomain\n\n";
    close $fh;

    `cp /home/lanforge/.bash_history /root`;
    `sync`;
    print "Shutdown system now.\n";
    exit 0;
} # ~do_vm_prep

# this is what you use when you want to prep a lanforge install for imaging
# copy the resize-home.sh script in first
# it wipes a lot of information out
#print "do_image_prep:$do_image_prep osveri:$::osveri\n" if $debug_on;
if ($do_image_prep) {
    die("LANforge version not found [$::lfver]") if ($::lfver eq "-1");
    if ($::osveri < 21) {
        err("do_image_prep useful only on F21 and later");
        exit(1);
    }
    if (!-f "/root/resize-home.sh") {
        err("Missing: /root/resize-home.sh");
        err("Try: wget http://ctweb/bxb/cf-image/resize-home.sh");
        exit(1);
    }
    do_cmd("service lanforge stop", 1);
    my @stat = `systemctl is-active --quiet vncserver\@:1.service`;
    if ($? != 0) {
        print "vncserver\@:1.service not enabled or not installed\n";
        print join("\n", @stat) . "\n";
    }
    else {
        do_cmd("systemctl stop vncserver\@:1.service", 1);
        do_cmd("systemctl disable vncserver\@:1.service", 1);
    }
    @stat = `systemctl is-active --quiet xrdp.service`;
    if ($? != 0) {
        print "xrdp.service not enabled or not installed\n";
        print join("\n", @stat) . "\n";
    }
    else {
        do_cmd("systemctl stop xrdp.service", 1);
        do_cmd("systemctl disable xrdp.service", 1);
    }
    @stat = `systemctl is-active --quiet pcscd.service`;
    if ($? != 0) {
        print "pcscd.service not enabled or not installed\n";
        print join("\n", @stat) . "\n";
    }
    else {
        do_cmd("systemctl stop pcscd.service", 1);
        do_cmd("systemctl stop pcscd.socket", 1);
        do_cmd("systemctl disable pcscd.service", 1);
        do_cmd("systemctl disable pcscd.socket", 1);
    }
    do_cmd("chkconfig lanforge off", 1);

    fix_chrony_conf();
    do_cmd("echo 'rename-this-clone' > /etc/hostname", 1);
    if ($::is_fedora && ($osveri < 34)) {
        do_cmd("hostnamectl --static set-hostname rename-this-clone", 1);
    }
    rename("/etc/issue", "/etc/issue.orig");
    open(my $fh, ">", "/etc/issue");
    print $fh "${B}S\nCandelatech LANforge $::lfver Kernel ${B}r on ${B}m\n";
    print $fh "LANforge $::lfver\n\n";
    print $fh "Log in as root and use resize-home.sh *first*\n\n";
    close($fh);
    open($fh, ">", "/etc/issue.resized");
    print $fh "${B}S\nCandelatech LANforge $::lfver Kernel ${B}r on ${B}m\n";
    print $fh "LANforge $::lfver\n\n";
    close($fh);
    update_motd();

    @stat = glob q(/etc/sysconfig/network-scripts/ifcfg-e*);
    my $f;
    foreach $f (@stat) {
        print "Removing network config for: $f\n";
        unlink $f;
    }
    @stat = (
        "/etc/udev/rules.d/70-persistent-net.rules",
        "/etc/udev/rules.d/70-persistent.rules",
        "/etc/udev/rules.d/70-phyname.rules",
        "/etc/resolv.conf"
    );
    for $f (@stat) {
        unlink($f) if (-f $f);
    }
    do_cmd("dnf clean all -y", 1);

    @stat = ("/home/lanforge/Downloads",
        "/home/lanforge/tmp",
        "/home/lanforge/.cache",
        "/home/lanforge/wifi",
        "/home/lanforge/vr_conf",
        "/home/lanforge/old_logs_1",
        "/home/lanforge/l4logs",
        "/home/lanforge/gui-eula-ok.txt",
        "/home/lanforge/server-eula-ok.txt",
        "/root/.cache",
        "/var/tmp",
        "/var/cache/yum",
        "/var/lib/dhcpd",
        "/var/lib/dhclient",
        "/var/lib/NetworkManager",
    );
    for $f (@stat) {
        print "find $f -type f -exec rm -f {} ${B};\n";
        do_cmd("find $f -type f -exec rm -f {} \\;", 1)
            if (-d $f);
    }
    @stat = glob("/home/lanforge/*_log*.txt");
    push(@stat,
        "/home/lanforge/did_cpuburn",
        "/home/lanforge/did_disktest",
        "/home/lanforge/.bash_history",
        "/root/.bash_history"
    );
    for $f (@stat) {
        print "rm -f $f\n";
        do_cmd("rm -f $f", 1);
    }
    `bash -c "history -c"`;
    @stat = (
        "uname -a",
        "uname -r",
        "df -h",
        "cat /home/lanforge/license.txt",
        "telnet localhost 4001",
        "ip li show",
        "ip a show",
        "ping www.candelatech.com",
        "host www.candelatech.com",
        "service lanforge restart",
        "ip vrf exec _vrf1 ping 8.8.8.8",
        "cd /home/lanforge; ./lfdebug.bash",
        "cd /root; ./lf_kinstall.pl --lfver 5.4.1 --kver 5.2.21+ --do_upgrade --print_only",
        "killall dhclient; ip a flush eth0; dhclient eth0",
        "dnf clean all; dnf makecache",
    );
    open($fh, ">", "/home/lanforge/.bash_history");
    `chown lanforge:$user_group /home/lanforge/.bash_history`;
    `chmod 640 /home/lanforge/.bash_history`;
    print $fh join("\n", @stat);
    print $fh "\n";
    close $fh;

    open($fh, ">", "/etc/hosts");
    print $fh "::1           localhost.localdomain localhost localhost6.loaldomain6 localhost6\n";
    print $fh "127.0.0.1     localhost.localdomain localhost localhost4.localdomain4 localhost4\n";
    print $fh "192.168.1.101 lanforge.localnet lanforge.localdomain\n\n";
    close $fh;

    `cp /home/lanforge/.bash_history /root`;
    `sync`;
    # this assumes we are actually on an ssd
    print "fstrim -v /boot...\n";
    `fstrim -v /boot`;
    print "fstrim /home...\n";
    `fstrim -v /home`;
    print "fstrim /var...\n";
    `fstrim -v /var`;
    print "fstrim /usr...\n";
    `fstrim -v /usr`;
    print "Shutdown system now.\n";
    exit 0;
} # ~do_image_prep

sub enable_max_zram {
    if (!$is_fedora || ($::osveri < 30)) {
        note("Zram: Version $::osver not supported. Fedora 30+ supported\n");
        return;
    }
    # check if settings already enabled
    my $sysctl_conf = "/etc/sysctl.d/70-lanforge.conf";
    die("Unable to open $sysctl_conf: $!")
        unless open(my $sysctl_fh, ">", $sysctl_conf);
    print $sysctl_fh "vm.vfs_cache_pressure=100\n";
    print $sysctl_fh "vm.swappiness=100\n";
    print $sysctl_fh "vm.dirty_background_ratio=10\n";
    print $sysctl_fh "vm.dirty_ratio=50\n";
    close $sysctl_fh;
    do_cmd("sysctl -p /etc/sysctl.d/70-lanforge.conf");
}

sub cfg_save_gui_cfg {
    my $action = shift(@_);
    die("cfg_save_gui_cfg: no action provided") unless ($action ne "");
    if (($::lfver eq "") || ($::lfver lt "0") || ($::lfver lt "5.4.5")) {
        die("cfg_save_gui_cfg: requires version 5.4.5 or better, specify --lfver please");
    }
    if (($action eq "static") || ($action eq "backup")) {
        do_cmd("cp $::home/LANforgeGUI_$::lfver/lfcnf.txt $::home/lfcnf.txt", 1);
    }
    if ($action eq "backup") {
        if (-x "$::home/lfcustom_gui.bash") {
            note(" * disabling lfcustom_gui.bash");
            chmod(0664, "$::home/lfcustom_gui.bash");
        }
        return;
    }
    if ($action eq "static") {
        if (!-x "$::home/lfcustom_gui.bash") {
            open(my $lfc, ">", "$::home/lfcustom_gui.bash")
                or die("Unable to open $::home/lfcustom_gui.bash: $!");
            print $lfc "#!/bin/bash\n",
                "if [[ -d $::home/LANforgeGUI_$::lfver ]] && [[ -f $::home/lfcnf.txt ]]; then\n",
                "    cp $::home/lfcnf.txt $::home/LANforgeGUI_$::lfver/\n",
                "fi\n";
            close($lfc);
            chmod(0775, "$::home/lfcustom_gui.bash");
        }
        if (!-x "$::home/lfcustom_ctrl.bash") {
            my @custom_ctrl_sh = (
                "",
                "# Created by lf_kinstall as part of cfg_save_gui_cfg()",
                "if [ -x $::home/lfcustom_gui.bash ]; then",
                "    $::home/lfcustom_gui.bash",
                "fi");
            my $open_append = ">";
            if (!-e "$::home/lfcustom_ctrl.bash") {
                unshift(@custom_ctrl_sh, "#!/bin/bash");
                $open_append = ">>";
            }

            `fgrep -q 'lfcustom_gui.bash' "$::home/lfcustom_ctrl.bash" &>/dev/null`;
            if ($? != 0) {
                open(my $lfc, $open_append, "$::home/lfcustom_ctrl.bash")
                    or die("Unable to open $::home/lfcustom_ctrl.bash: $!");

                print $lfc join("\n", @custom_ctrl_sh);
                close($lfc);
            }
            chmod(0775, "$::home/lfcustom_ctrl.bash");
        } # ~if lfcustom_ctrl does not exit
    }     # if action is static
}         # ~sub cfg_save_gui_cfg

sub cfg_gui_autostart {
    die("cfg_gui_autostart requires 0=disable 1=enable")
        if (@_ < 1);
    print("LFVER: [$::lfver]\n");
    if (($::lfver =~ /^\s*$/) || ($::lfver lt 0)) {
        die("cannot configure LANforge GUI autostart without knowing LF version");
    }

    my $on_off = int(shift(@_));
    if (($on_off < 0) || ($on_off > 1)) {
        die("cfg_gui_autostart requires 0=disable 1=enable");
    }

    if (!$on_off) {
        if (!-d "$::home/.config/autostart") {
            note(" * GUI autostart: $::home/.config/autostart directory not found");
            return;
        }
        if (!-e "$::home/.config/autostart/LANforge-auto.desktop") {
            note(" * GUI autostart: $::home/.config/autostart/LANforge-auto.desktop not found");
            return;
        }
        warning(" ** Unable to remove LANforge-auto.desktop: $!")
            unless unlink("$::home/.config/autostart/LANforge-auto.desktop");
        note("# disabled LANforge-auto.desktop session autostart");
        return;
    } # ~ remove autostart

    if (!-e "$::home/LANforgeGUI_$::lfver/LANforge-auto.desktop") {
        warning("# LANforgeGUI $::lfver not installed, $::home/LANforgeGUI_$::lfver/LANforge-auto.desktop not found");
        return;
    }
    if (!-d "$::home/.config/autostart") {
        warning(" * GUI autostart: $::home/.config/autostart directory not found, creating!");
        mkdir("$::home/.config/autostart", 0755);
    }
    # remove old symlink because if might not point to correct version
    if (-f "$::home/.config/autostart/LANforge-auto.desktop") {
        unlink("$::home/.config/autostart/LANforge-auto.desktop")
            or warning("* could not remove old symlink [$::home/.config/autostart/LANforge-auto.desktop]");
    }
    symlink("$::home/LANforgeGUI_$::lfver/LANforge-auto.desktop",
        "$::home/.config/autostart/LANforge-auto.desktop")
        or warning(" * GUI autostart: unable to create symlink: $! ");
    if (-f "$::home/.config/autostart/LANforge-auto.desktop") {
        note("# enabled LANforgeGUI autostart. Restart VNC server to take effect.\n");
    }
} # ~if do_gui_autostart

sub route_check {
    if ($::is_macos) {
        return;
    }

    my @route_lines = `ip route show`;
    chomp(@route_lines);
    my @route_matches = grep {/^(default|0\.0\.0\.0\/0) via /} @route_lines;
    my $proxyinfo = "";
    if (defined $ENV{HTTP_PROXY}) {
        $proxyinfo .= " proxy: $ENV{HTTP_PROXY}";
    }
    if (defined $ENV{ALL_PROXY}) {
        $proxyinfo .= " proxy: $ENV{ALL_PROXY}";
    }
    if (@route_matches < 1) {
        print "# ----- ----- ----- ----- ----- ----- ----- ----- ----- \n";
        print "#    Default route not found!\n";
        print "#    Unless you have a proxy, your upgrade might not complete.\n";
        print "#    Proxies: $proxyinfo\n";
        print "#    Routes found: ";
        print join("\n#    ", @route_lines);
        print "\n# ----- ----- ----- ----- ----- ----- ----- ----- ----- \n";
        print "     Press ctl-c to stop, or wait 5 sec to continue\n";
        sleep 5;
    }
}

# check to see if xrdp is already an exception in dnf/dnf.conf
# not a bad idea to freeze xrdp because if it breaks on upgrade
# all kindsa frustration ensues
my $xrdp_pkg = "xrdp";
if (-f "/etc/dnf/dnf.conf") {
    my @exclude_lines = `grep xrdp /etc/dnf/dnf.conf`;
    chomp(@exclude_lines);
    my @exclude_matches = grep {/^exclude=xrdp/} @exclude_lines;
    if (@exclude_matches > 0) {
        $xrdp_pkg = "";
    }
}

my @f19_pkgs = ($::osveri < 19)
    ? qw(system-config-display distcache mod_python mkinitrd)
    : ();

my @f23_pkgs = ($::osveri <= 23)
    ? qw(vim-enhanced wireshark)
    : ();

my @f24_pkgs = (($::osveri <= 24) && $is_fedora)
    ? qw(vim-enhanced php-mysqlnd system-config-httpd system-config-nfs system-config-date
    system-config-nfs-docs system-config-samba-docs yum-utils)
    : ();

my @f26_pkgs = ($::osveri == 25)
    ? qw(python-argparse yum-utils)
    : ();
my @rf_gen_pkgs = qw(gr-osmosdr gnuradio);
if ($::osveri < 34) {
    @rf_gen_pkgs = (@rf_gen_pkgs, "hackrf")
}
my @f27_pkgs = ($::osveri == 27)
    ? (qw(gtk-recordmydesktop fros fros-gnome wireshark-gtk python37 crda
    wkhtmltopdf sqlite php-pdo xawtv camorama gnome-terminal-nautilus lzop
    perl-Text-CSV_XS perl-Text-CSV-Separator iperf3 perl-JSON-XS yum-utils),
    @rf_gen_pkgs)
    : ();
@f27_pkgs = ($::osveri >= 27)
    ? (($::osver ne "C8X" ? (qw(xorg-x11-xauth atril compat-readline6 dhcp-server
    libjpeg-turbo-devel libjpeg-turbo-utils roxterm xterm remmina ),
    ($xrdp_pkg, "python3-virtualenv")) # python37 not found in F30+
    : ()),
    qw(pulseaudio-libs-glib2), @f27_pkgs)
    : (qw(dstat wireshark), @f27_pkgs);

if ($osveri <= 35) {
    if ($osveri > 27) {
        @f27_pkgs = (@f27_pkgs, "ffmpeg-devel");
    }
    @f27_pkgs = (@f27_pkgs, qw(meson SDL2-devel libusb-devel));
}
my @f29_pkgs = ($::osveri >= 29)
    ? (qw(network-scripts), @rf_gen_pkgs)
    : ();

# crda is an alias for wireless-regdb
my @f30_pkgs = ($::osveri == 30)
    ? qw(xorg-x11-apps yum-utils wireless-regdb jack-audio-connection-kit ipmiutil)
    : ();
#  python3-matplotlib - should be managed by update_dependencies.py
@f30_pkgs = ($::osveri >= 30)
    ? (qw(vokoscreenNG fros fros-gnome wkhtmltopdf sqlite php-pdo xawtv camorama
    gnome-terminal-nautilus lzop libmateweather libmateweather-data remmina
    perl-Text-CSV_XS perl-Text-CSV-Separator iperf3 perl-JSON-XS perl-JSON-Parse
    android-tools adb-enhanced),
    @f30_pkgs, @rf_gen_pkgs)
    : (@f30_pkgs);

my @f32_pkgs = ($::osveri == 32)
    ? qw(jack-audio-connection-kit ipmiutil)
    : ();
@f32_pkgs = ($::osveri >= 32)
    ? (qq( dhcp-server roxterm xterm $xrdp_pkg), @f32_pkgs)
    : (@f32_pkgs);

my @f34_pkgs = (($::osveri == 34) || ($::osveri == 35))
    ? qw(wireless-tools ipmiutil pipewire-jack-audio-connection-kit)
    : ();
@f34_pkgs = ($::osveri >= 34)
    ? (qw(dnf-utils libidn beesu iftop htop sipcalc mosh dhcping
    tigervnc-server remmina spamassassin squid avrdude httpd-manual freerdp stress-ng
    tcptrack ntpsec  aria2 bridge-utils
    perl-Devel-NYTProf perl-RRD-Simple perl-Net-Telnet perl-JSON-Parse cups-pdf gnuplot colordiff trash-cli),
    @f34_pkgs)
    : (@f34_pkgs);
# xorg-x11-apps xeyes
my @f36_pkgs = (($::osveri >= 36) && ($::osveri < 40))
    ? (qw( meson libavformat-free-devel libavdevice-free-devel SDL2-devel ffmpeg-free-devel pipewire-jack-audio-connection-kit))
    : ();
@f36_pkgs = ($::osveri == 36)
    ? (@f36_pkgs, qw(ipmiutil))
    : ();

my @rh7_pkgs = ($is_redhat && $::osveri <= 19)
    ? qw(php-mysql perl-IO-Pty-Easy perl-IO-Tty perl-Expect )
    : ();

my @c8x_pkgs = ($::osver eq "C8X")
    ? qw(readline perl-Number-Misc perl-JSON perl-JSON-PP python3-lxml screen tigervnc-server)
    : ();
my @rh8_pkgs = (($::osver eq "C8X") && $is_redhat)
    ? qq( xorg-x11-utils iftop htop mosh dhcping tigervnc-server xorg-x11-apps xterm spamassassin  php-pdo
         httpd-manual $xrdp_pkg freerdp p7zip aria2 perl-Text-CSV_XS dhcp-server perl-JSON-XS iperf3 screen perl-JSON-PP )
    # squid
    : ();

#print Dumper({
#    "F30" => \@f30_pkgs,
#    "F34" => \@f34_pkgs,
#    "CX8" => \@c8x_pkgs,
#    "CX8R" => \@rh8_pkgs
#});

# C8X lacks these:
# atril # no mate-desktop rh8
# compat-readline6 ; readline available
# bridge-utils # rh8 removed
# dhcp  # skip
# perl-Number-Format > perl-Number-Misc
# perl-JSON-XS > perl-json-pp
# perl-Devel-NYTProf # gone
# perl-RRD-Simple # gone
# perl-Net-Telnet # gone
# cups-pdf # gone
# crypto-utils # gone
# libxslt-python => prolly python3-lxml
# gnumeric
# iperf
# wireless-tools
# ntpdate
# tcptrack

# Recent Java defaults of note:
# F30: openjdk-1.8.0
# F34: openjdk-11
# F36: openjdk-17

my @yum_pkgs_on_fedora_only = ($is_fedora)
    ? qw(python3-websocket-client python3-devel  python3-pexpect python3-ptyprocess
    php-opcache freeradius-utils lapack-devel atlas-devel)
    : ();
my @yum_pkgs = (@f19_pkgs, @f23_pkgs, @f24_pkgs, @f26_pkgs, @f27_pkgs, @f29_pkgs,
    @f30_pkgs, @f34_pkgs, @f36_pkgs, @yum_pkgs_on_fedora_only, @rh7_pkgs, @c8x_pkgs, @rh8_pkgs,
    qw(bind-utils pciutils tree lsof wireshark expect haveged),
    qw(telnet telnet-server perl-CPAN perl-JSON lbzip2 pxz bzip2 patch jq),
    qw(perl-Text-CSV perl-Expect perl-libwww-perl),
    qw(rrdtool rrdtool-perl perl-Net-SSLeay perl-IO-Socket-SSL perl-Time-HiRes),
    qw(tidy libxml2 yajl python3-pip ),
    qw(lm_sensors dconf-editor cups hdparm smartmontools wget ftp),
    qw(kernel-devel mod_perl mod_ssl php php-ldap),
    qw(freeradius freeradius-utils nginx libnet tcpdump radvd sendmail-cf),
    qw(perf lksctp-tools openssh-server openssh-clients openssh man-pages),
    qw(iscsi-initiator-utils conntrack-tools pcsc-lite-libs),
    qw(sysstat ethtool cifs-utils minicom gdb strace dmidecode),
    qw(dos2unix openvpn easy-rsa ),
    qw(lapack lapack-devel openblas openblas-devel atlas atlas-devel) # required for numpy, scipy
);

# these have to be installed separately after all the x86_64 packages otherwise they
# skew the x86_64 package solver
# pesq server needs i686, as does g.729 gua codec
my @yum_i686_pkgs = (
    qw(pciutils-libs.i686 libpcap.i686),
    qw(libstdc++.i686 glibc.i686 zlib.i686 libgcc.i686),
    qw(openldap.i686)
);

# These will warn but not fail the install.
#  crypto-utils - abandoned
#  F36 abandons wireless-tools

my @rh7_pkgs_warn = ($is_redhat && ($::osveri <= 19))
    ? qw()
    : ();
my @fedora_pkgs_warn = ($is_fedora)
    ? qw(roxterm tcptrack squid python3-lxml camorama
    httpd-manual freerdp perl-RRD-Simple trash-cli ffms2-devel xawtv)
    : ();
@fedora_pkgs_warn = (($::osveri >= 30) && ($::osveri <= 36))
    ?(@fedora_pkgs_warn, qw(libusb-devel))
    :@fedora_pkgs_warn;
@fedora_pkgs_warn = (($::osveri >= 37) && ($::osveri <= 40))
    ?(@fedora_pkgs_warn, qw(libusb1-devel))
    :@fedora_pkgs_warn;

my @yum_pkgs_warn = (@rh7_pkgs_warn, @fedora_pkgs_warn,  $xrdp_pkg,
    qw(beesu iftop htop sipcalc mosh dhcping tigervnc-server fping),
    qw(xterm wkhtmltopdf sqlite php-pdo php-xdebug),
    qw(avrdude firefox stress-ng valgrind ),
    qw(ntpdate gnome-terminal-nautilus p7zip lzop aria2),
    qw(wireless-tools perl-Text-CSV_XS perl-Text-CSV-Separator iperf3 xauth),
    qw(dhcp-server bridge-utils perl-JSON-XS perl-Devel-NYTProf perl-Net-Telnet cups-pdf),
    qw(gnuplot colordiff trash-cli android-tools SDL2-devel ffms2-devel meson gcc make),
    qw(d-feet bluez bluez-tools mlocate libreoffice)
); # pulseaudio   pulseaudio-utils  python3-pulsectl xorg-x11-apps  jnettop spamassassin
if ($::osveri < 30) {
    @yum_pkgs_warn = (@yum_pkgs_warn, qw(xorg-x11-utils));
}
if (($::osveri >= 27) && $::osveri <= 30) {
    @yum_pkgs_warn = (@yum_pkgs_warn, qw(pulseaudio-module-bluetooth libxslt-python));
}
if ($::osveri > 30) {
    @yum_pkgs_warn = (@yum_pkgs_warn, qw(tuna tuned numactl kernel-tools),
         qw(java-1.8.0-openjdk-devel java-17-openjdk-devel)
         );
}

# perl-core -> perl-4 # ...was probably necessary for a CentOS 7 server minimal install maybe?
# However requiring perl-CPAN should pull it in

# xorgxrdp skipped
# this array collects list of items to add if we're F30 and --best might install something different
my @force_yum_pkgs = qw(wkhtmltopdf xorgxrdp); # crda can break F34+ yum installs

# dnf --best implies 'only consider latest packages' which is of debatable value
# when earlier dnf versions do no also offer a --nobest option. The --best option
# can actually get in the way of updating packages because there might be dependency
# conflicts that will not resolve.
our $skip_broken = ($::osveri <= 21) ? "--skip-broken" : " --allowerasing";
our $yum = ($::osveri <= 21) ? "yum" : "dnf";
if ($::osveri >= 34) {
    $skip_broken .= " --skip-broken";
}
#
#   YUM / DNF activity area
#
#   Begin by determining up-to-date repositories and updates
#
my @pending_yum_updates = ();
if (!($::create_install_bundle || $skip_yum_all) && ($is_centos || $is_fedora || $is_redhat )) {
    if (($yum eq "yum") && !($is_deb_based || $download_only || $is_macos)) {
        if ($do_save_yumc == 20) {
            note("# Deleting /var/cache/yum.\n");
            do_cmd("rm -fr /var/cache/yum/");
        }
        if (!$skip_fmirror && ($do_yum_update || $do_pkgs)) {
            if ($no_fmirror) {
                do_cmd("$yum -y remove yum-fastestmirror");
            }
            else {
                do_cmd("$yum -y install yum-fastestmirror");
            }
        }
    }
    if (!($is_offline||$use_yum_cache||$skip_yum_all||$skip_yum_update)) {
        # only clean cache if it is older than 1 week
        if ($main::osveri >= 24) {
            my $cache_date_epoch = `stat --format %Y /var/cache/dnf/packages.db`;
            chomp $cache_date_epoch;
            my $now_date_epoch = `date +%s`;
            chomp $now_date_epoch;
            if ($cache_date_epoch) {
                if ((int($cache_date_epoch) - int($cache_date_epoch)) > 25200){
                    do_cmd("$yum clean all");
                    do_cmd("$yum makecache");
                }
            }
        }
    }

    # return code for updates is 100
    do_cmd("$yum check-update > /tmp/dnf-updates.txt ||:");
    @pending_yum_updates = `awk '{print \$1}' /tmp/dnf-updates.txt`;
    chomp(@pending_yum_updates);
    #if (@pending_yum_updates>0) {
    #    print_slow(" %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%% ",
    #                Dumper(["pending_yum_updates", \@pending_yum_updates]),
    #                " %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%% " )
    #        if ($debug_on);
    #}
    my @matches = grep {/^kernel/} @pending_yum_updates;
    $yum_has_pending_kernel = scalar(@matches);
    print "Computing non-kernel updates...\n";
    for(my $i=scalar(@pending_yum_updates)-1; $i>=0; $i--) {
        # delete blank trash
        if (!(exists $pending_yum_updates[$i]) || ($pending_yum_updates[$i] =~ /^\s*$/)) {
            splice(@pending_yum_updates, $i, 1);
            next;
        }
        if ($pending_yum_updates[$i] =~ /^(kernel|Last)/){
            splice(@pending_yum_updates, $i, 1);
        }
    }
    #if (@pending_yum_updates>0) {
    #    print_slow( " %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%% ",
    #                Dumper(["pending_yum_updates", \@pending_yum_updates]),
    #                " %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%% " )
    #    if ($debug_on);
    #}
    #print_slow(" %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%%  %%%%% " );
}

# Install RFGEN/HACKRF packages if requested.
if (!$::create_install_bundle && ($do_rfgen && !$skip_yum_all)) {
    if ($is_fedora) {
        my $packages = "hackrf gr-osmosdr gnuradio";
        note("# updating dnf cache...\n");
        note("# Installing $packages...\n");
        do_cmd("dnf install $::yum_xclude $packages");

        # Generate HackRF udev rule to allow non-root users to control HackRF
        my $fname = "/etc/udev/rules.d/99-hackrf.rules";
        if (-f "$fname") {
            note("HackRF udev rules file already detected. Will not regenerate.\n\n");
        }
        else {
            note("# Generating HackRF udev rules file...\n");
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            my @new_99hackrf_lines = (
                "#",
                "#  Auto-created by LANforge lf_kinstall.pl.",
                "#  This file is recreated when run with the '--do_rfgen' option.",
                "#",
                "",
                "# HackRF One",
                "SUBSYSTEM==\"usb\", ATTRS{idVendor}==\"1d50\", ATTRS{idProduct}==\"6089\", MODE=\"0666\"",
                "",
            );
            print FILE join("\n", @new_99hackrf_lines);
            close FILE;

            # Reload udev rules. Otherwise changes won't have affect until next boot
            do_cmd("udevadm control --reload-rules");
            do_cmd("udevadm trigger");
        }
    }
    else {
        err("* do_rfgen not supported except for fedora");
    }
} # ~hackrf packages

# Install VLC and exit
if ($install_vlc  && ! ($create_install_bundle || $skip_yum_all || $$skip_yum_update)) {
    if ($is_fedora && ($::osveri > 21)) {
        my @repolines = `dnf -C repolist`;
        if (@repolines > 1) {
            my @matches = grep {/rpmfusion-free/} @repolines;
            if (@matches < 1) {
                note("Adding rpmfusion url");
                my $rpmfusionurl = "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-${main::osveri}.noarch.rpm";
                do_cmd("dnf install $rpmfusionurl -y");
                do_cmd("dnf check-update");
                do_cmd("dnf install vlc -y");
            }
        }
        my $v_rv = system('grep geteuid /usr/bin/vlc');
        if ($v_rv == 0) {
            note('enabling vlc to run as root');
            do_cmd("sed -i 's/geteuid/getppid/' /usr/bin/vlc", 1);
        }
        exit 0;
    }
    else {
        err("* install_vlc not supported except for fedora 21+");
        exit 1;
    }
} # ~install vnc


if ($::tmp_dir eq "") {
    die(usage() . "\n\nERROR:  When using --tmp_dir option you must specify a location for the LANforge installation software.\n\n");
}

if ($do_only_pkgs) {
    $do_pkgs_f = 1;
    $do_pkgs = 1;
}
if ($do_pkgs_f) {
    $do_pkgs = 1;
}

my $rem_yum_proxy = 0;

my $cmd;
my $i;

my $d2dir = "";
my $url_base = "";

if ($::lfver ne "") {
    #
    # Documentation about qr// patterns is they are used to compile a re-usable regex efficiency within loops.
    # The qr// operator actually returns a regex object. Please see https://perldoc.perl.org/perlop#Regexp-Quote-Like-Operators
    #
    # The regex m|.*/$| applies well only when the the URL provided looks like http://www.candelatech.com/
    # It should be valid to use all remaining pathing to indicate --build_path (a more intuitive use of the parameter)
    #
    if ($www =~ m|https?://[^/]+/+\S+|) {
        ($www, $build_path) = $www =~ m|(https?://[^/]+)(/\S+)|;
        print "Found combined download/build URL:\n         host:[$www]\n   build_path:[$build_path]\n";
    }
    elsif ($www =~ m|http.*/$|) {
        ($www) = $www =~ m|^(.*)/$|;
    }

    if ($do_kern) {
        if (!(defined $::lfver) || $::lfver =~ /^\s*$/ || $::lfver =~ /^-1$/) {
            die("unable to parse lfver[$::lfver], missing --lfver argument?");
        }
    }

    $d2dir = "/mnt/d2/pub/candela_cdrom.$::lfver";
    if ($::build_path ne "") {
        my $p = '';
        if (index($::build_path, '/') != 0) {
            $p = '/';
        }
        $url_base = "${www}${p}$::build_path";
        $d2dir = "/mnt/d2${p}$::build_path";
    }
    else {
        $url_base = "$www/private/downloads/r$::lfver";
    }
    $force_web = 1;
}

if ($force_osver ne "") {
    note("# Forcing osver to: $force_osver\n");
    $::osver = $force_osver;
}

my @which_mate = `which -a mate-session 2>/dev/null`;
chomp @which_mate;
if (@which_mate > 0) {
    $mate_installed = 1;
}

if (($is_centos || $is_redhat || $is_fedora) && ($::osveri >= 19) && ($mate_installed == 0) && !$is_arm) {
    $do_install_mate = 1 if ($::osver ne "C8X");
}

if ($is_ubuntu && ($mate_installed == 0) && !$is_arm) {
    $do_install_mate = 1;
}

if ($skip_resume) {
    $curl_resume = "";
}

if ($nocache) {
    $cachebuster = "?footime=" . time();
}

if ($do_lanforge || $is_arm || $is_macos) {
    $skip_cpu_burn = 1;
}

if ($do_kern || $remove_kern) {
    $do_grub = 1;
    $do_cma = cma_recommendation() if ($do_cma == 1);
    $skip_cpu_burn = 1;
}

if ($::create_install_bundle) {
    $skip_installer_check = 1;
    $skip_cpu_burn = 2;
    $do_cpu_burn = 0;
    $skip_disk_test = 1;
    $do_disk_test = 0;
    $skip_fmirror = 1;
    $do_http = 0;
    $show_urls = 0; # setting to 1 will prematurely quit
    $do_firmware = 1;
    $do_interop = 0;
    $do_webui = 0;
    $do_kern = 1;
    $do_lanforge = 1;
    if ($tmp_dir eq "/tmp") {
        my @mount_lines = `mount | grep /tmp`;
        if ((@mount_lines > 0) && ($mount_lines[0] =~ /tmpfs/)) {
            print "## - - - - - - - - - - - - - - - - - -  ##\n";
            print "##                                      ##\n";
            print "## /tmp appears to be a tmpfs directory ##\n";
            print "## We typically need more space so use  ##\n";
            print "## the --tmp_dir option in the command, ##\n";
            print "   Try --tmp_dir $::home/Downloads\n";
            print "##                                      ##\n";
            print "## - - - - - - - - - - - - - - - - - -  ##\n";
            exit 1;
        }
    }
}

if ($print_only) {
    $do_cpu_burn = 0;
    $do_disk_test = 0;
    $do_interop = 0;
    $do_webui = 0;
    $do_only_pkgs = 0;
    $do_pkgs = 0;
    $force_web = 1;
    $show_urls = 1;
    $skip_cpu_burn = 2;
    $skip_disk_test = 1;
    $skip_installer_check = 1;
    $skip_yum_all = 1;
    $skip_yum_update = 1;
}
if ($show_urls) {
    $do_disk_test = 0;
    $force_web = 1;
    $skip_cpu_burn = 2;
    $skip_disk_test = 1;
    $skip_installer_check = 1;
    $skip_yum_all = 1;
    $skip_yum_update = 1;
}
if ((defined $is_vm) && ($is_vm > 0)) {
    $skip_cpu_burn = 2;
    $do_cpu_burn = 0;
    $do_disk_test = 0;
}

if ($do_webui) {
    if ($skip_yum_all || $skip_yum_update) {
        note("#  Disabling do_webui because yum is requested to be skipped.\n");
        $do_webui = 0;
    }
    elsif ($::osveri < 34) {
        note("#  Disabling do_webui because OS version is less than Fedora-34.\n");
        $do_webui = 0;
    }
}

note("#     is-vm $is_vm         is-ubuntu: $is_ubuntu      mate-inst'd: $mate_installed    install-mate: $do_install_mate ");
note("#     do-burn: $do_cpu_burn      disk-test: $do_disk_test      is-arm: $is_arm         is-fedora: $is_fedora\n");
note("#     skip-burn: $skip_cpu_burn    do-gnome: $do_gnome       uveri: $uveri         rpi4: $is_rpi4  rpi5: $is_rpi5\n");
note("#     do_sys_rcfg: $do_sys_reconfig  is-deb-based: $is_deb_based   osveri: $::osveri        osver: $::osver\n");
note("#     create-bndl: $::create_install_bundle  use-bundle: $::use_install_bundle     is-64: $is_64\n");
note("#     skip-kern: $skip_kern   kver: $kver   do_upgrade:   $do_upgrade     do_grub: $do_grub\n");

if ($remove_kern) {
    my $kfile = "/boot/ct$kver.img";
    my $krd = "/boot/initrd-ct$kver.img";
    if ((! -e $kfile) || (! -e $krd)) {
        err("* Option --do_grub or --remove_kern requested but kernel $kver is not installed.
* Please check /boot/ directory to see what kernels are installed.");
    }
}

if ($debug_on) {
    print_slow("-- please read --")
}

my $base_deb_pkgs = "apache2 curl nfs-kernel-server openssh-server patch screen ssh tightvncserver vim host";
my $gnome_debs = "gnome-session-fallback gnome-terminal";
if ($do_install_mate || $mate_installed || $is_arm) {
    $gnome_debs = "";
    $do_gnome = 0;
}
my $libnl_pkg = "libnl1";
my $deb_php = "php5 libapache2-mod-php5";
my $iscsi_pkg = "open-iscsi-utils";
my $lubuntu_debs = "";
if ($::osver eq "Ubuntu 12") {
    $lubuntu_debs = "lubuntu-desktop gnumeric";
}
if ($is_arm) {
    # Assume ARM distro is already running something slim.
    $lubuntu_debs = "";
}

if ($::osveri >= 21 || $is_rpi5) {
    $deb_php = "php libapache2-mod-php";
    $libnl_pkg = "libnl-3-dev";
    $iscsi_pkg = "open-iscsi";
}

my $non_rpi4 = "linux-tools-common"; #dconf-tools not in 18.04 ?
if ($is_rpi4 || $is_rpi5) {
    $non_rpi4 = "";
    $iscsi_pkg = "open-iscsi";
    $deb_php = "php";
}
# do we want libnet1-dev or libnet-dev?
$other_deb_pkgs = " vim tshark radvd $non_rpi4 gdb strace ethtool  telnet " .
    "avrdude libnet1 iftop tcpdump iw apt-file bridge-utils bzip2 lbzip2 lzop " .
    "iperf traceroute libnet-telnet-perl ntpdate haveged locate $libnl_pkg " .
    "apache2 freeradius libssh2-1 isc-dhcp-server x11-utils $xrdp_pkg xterm " .
    "libnumber-format-perl dos2unix htop sipcalc mosh libnet1-dev " .
    "libjson-pp-perl libjson-xs-perl jq perltidy libxml2-utils yajl-tools " .
    "libpcsclite1 wireless-tools $lubuntu_debs $iscsi_pkg conntrack dhcping $gnome_debs $deb_php " .
    "nfs-kernel-server ethtool tcpdump adb";

if ($is_deb_based && !$is_arm && ($do_install_mate || $mate_installed)) {
    $other_deb_pkgs .= " tightvncserver";
    if ($::osveri >= 20) {
        $other_deb_pkgs .= " libgcrypt20 libgcrypt20-dev wireshark-gtk ubuntu-mate-desktop xz-utils";
    }
    elsif (($::osveri >= 19) && ($::osveri < 20)) {
        $other_deb_pkgs .= "python3-websocket  libgcrypt20 libgcrypt20-dev wireshark-gtk ubuntu-mate-desktop xz-utils pxz  dconf-tools";
    }
    elsif (($::osveri >= 16) && ($::osveri < 19)) {
        $other_deb_pkgs .= "python-websocket  libgcrypt11 libgcrypt11-dev wireshark-gtk ubuntu-mate-desktop xz-utils pxz  dconf-tools";
    }
    else {
        $other_deb_pkgs .= " libgcrypt11-dev wireshark";
    }
    if ($::osveri == 12) {
        $other_deb_pkgs .= "libgcrypt11-dev xz-lzma xz-utils dconf-tools";
    }
}

if ($is_deb_based && ($apt_update == 0) && ($skip_yum_all < 1)) {
    if ($do_interop) {
        $ENV{LD_LIBRARY_PATH} = "";
    }
    do_cmd("$apt update", 1);
    $apt_update = 1;
}
# this section needs to be moved into a function that is just about
# setting do_iommu and do_swiotlb, because those items are getting set
# even when determining unrelated behaviours, such as --regen_https_key or --create_install_bundle
if (!($is_macos || $create_install_bundle)) {
    if (system("which lspci &>/dev/null") != 0) {
        if ($skip_yum_all) {
            print (" --skip_yum_all set, not installing pciutils");
        }
        else {
            if ($is_deb_based) {
                do_cmd("$apt install -y pciutils", 1);
            }
            else {
                do_cmd("$yum install $skip_broken $::yum_xclude -y pciutils", 1);
            }
        }
    }
    my $lspci = `lspci`;
    if (($lspci =~ /Qualcomm Atheros Device 0040/g) ||
           ($lspci =~ /Qualcomm Atheros Device 0056/g) ||
              ($lspci =~ /Qualcomm Atheros Device 0046/g)) {
        note("# Found ath10k wave2 Adapter.\n");
        if ($do_iommu == -1) {
            note("# Found ath10k wave2 Adapter, will default do_iommu to enabled.\n");
            $do_iommu = 1;
        }
        if ($do_noaer < 1) {
            note("# Ath10k wave2 detected and not setting pci=noaer.\n");
        }
        else {
            note("# Ath10k wave2 detected and enabling pci=noaer.\n");
        }
    }

    # MTK radios need iommu too
    if (($lspci =~ /MEDIATEK Corp. Device 7906/g) ||
           ($lspci =~ /MEDIATEK Corp. Device 7921/g) ||
              ($lspci =~ /MEDIATEK Corp. Device 7922/g) ||
              ($lspci =~ /MEDIATEK Corp. Device 7925/g) ||
                  ($lspci =~ /MEDIATEK Corp. Device 7915/g) ||
                      ($lspci =~ /MT7915E/g) ||
                         ($lspci =~ /MEDIATEK Corp. Device 0608/g)) {
        note("# Found MTK pcie adapter.\n");
        if ($do_iommu == -1) {
            note("# Found MTK pcie Adapter, will default do_iommu to enabled.\n");
            $do_iommu = 1;
        }
        if ($do_iommu == 1) {
            if ($set_swiotlb == 0) {
                note("# Found MTK pcie Adapter, will default swiotlb to 65536.\n");
                # cat /debug/swiotlb/io_tlb_nslabs
                # 65536 <- this is units of 8K SLABS, not memory in KB
                $set_swiotlb = 65536;
            }
        }
        # IOMMU is on by default, setting cmdline intel_iommu=off kernel_iommu=off is the
        # actual cmdline you will see.
        # Possible that the next higher number of slabs is 512000 for MT7922 radios or
        # multiple (> 6 MTK) radios
    }

    my $ath_count = () = $lspci =~ /Network controller: Qualcomm/g;
    my $mtk_count = () = $lspci =~ /(Network controller: MEDIATEK|MT7915E)/g;

    if ($ath_count + $mtk_count > 10) {
        note("# Greater than 10 appropriate radios, disabling IOMMU, increasing swiotlb size.\n");
        $do_iommu = 0;
        $set_swiotlb = 65536;
    }
    if ($ath_count + $mtk_count > 20) {
        $set_swiotlb = 65536 * 2;
    }
} # if not mac-os

if ($do_sys_reconfig || $do_vm_reconfig) {
    $::do_ff_homepage = 1;
    $force_notes = 1;
    $do_udev = 1;
    $do_radius = 1;
    $force_new_certs = 1;
}

if ($do_all_ct) {
    $do_cpu_burn = 1;
    $do_cups_pdf = 1;
    $do_biosdevname = 0;
    $do_serial = 1;
    $do_firmware = 1;
    $do_kern = 1;
    $do_grub = 1;
    $disable_audit_logs = 1;
    $do_cma = cma_recommendation() if ($do_cma == 1);
    $do_services = 1;
    $do_radius = 1;
    $do_pkgs = 1;
    $do_yum_update = 1;
    if ($do_selinux == -1) {
        $do_selinux = 0; # disable it by default
    }
    if ($do_iommu == -1) {
        $do_iommu = 0; # Disable it by default
    }
    $do_gnome = 1 unless ($do_install_mate || $mate_installed);
    $do_vnc = 1;
    $do_udev = 1;
    $do_lanforge = 1;
    $do_xrdp = 1;
    $do_http = 1;
    $fix_web_root = 1;
    $force_notes = 1;
    $do_disk_test = ($is_vm) ? 0 : 1;
    if (($is_fedora && $::osveri >= 19)
        || ($is_centos && $::osveri >= 19)
        || ($is_redhat && $::osveri >= 19)
        || ($is_ubuntu && $::osveri >= 21)) {
        $do_runlevel = 3;
    }
}

if ($do_all) {
    $do_cpu_burn = 1;
    $do_disk_test = ($is_vm) ? 0 : 1;
    $do_cups_pdf = 1;
    $do_firmware = 1;
    $do_kern = ($skip_kern!=1)?1:0;
    $do_grub = ($skip_kern!=1)?1:0;
    $disable_audit_logs = 1;
    $do_cma = cma_recommendation() if ($do_cma == 1);
    $do_radius = 1;
    $do_services = 1;
    $do_pkgs = 1;
    $do_yum_update = 1;
    if ($do_selinux == -1) {
        $do_selinux = 0; # disable it by default
    }
    if ($do_iommu == -1) {
        $do_iommu = 0; # Disable it by default
    }
    $do_gnome = 1 unless ($do_install_mate || $mate_installed);
    $do_vnc = 1;
    $do_udev = 1;
    $do_lanforge = 1;
    $do_http = 1;
    $do_xrdp = 1;
    $skip_cpu_burn = 1;
}

if ($skip_radius) {
    $do_radius = 0;
}

if ($do_upgrade) {
    $do_cups_pdf = 1;
    $do_gnome = 1 unless ($do_install_mate || $mate_installed);
    $do_firmware = 1;
    $do_kern = ($skip_kern!=1)?1:0;
    $do_grub = ($skip_kern!=1)?1:0;
    $disable_audit_logs = 1;
    $do_cma = cma_recommendation() if ($do_cma == 1);
    $do_pkgs = 1;
    $do_yum_update = 1;
    $do_lanforge = 1;
    $do_http = 1;
    $do_xrdp = 1;
    $skip_fmirror = 1;
}

if ($do_ct_st) {
    $do_cups_pdf = 1;
    $do_firmware = 1;
    $do_kern = ($skip_kern!=1)?1:0;
    $do_grub = ($skip_kern!=1)?1:0;
    $disable_audit_logs = 1;
    $do_cma = cma_recommendation() if ($do_cma == 1);
    $do_lanforge = 1;
    $do_xrdp = 1;
    $do_http = 1;
    $skip_fmirror = 1;
    $skip_cpu_burn = 1;
    $skip_disk_test = 1;
    if ($do_selinux == -1) {
        $do_selinux = 0; # disable it by default
    }
    if ($do_iommu == -1) {
        $do_iommu = 0; # Disable it by default
    }
    if ($is_fedora && $::osveri >= 19) {
        $do_runlevel = 3;
    }
}

if ($do_ct_swak) {
    $do_firmware = 1;
    $do_grub = 1;
    $disable_audit_logs = 1;
    $do_cma = cma_recommendation() if ($do_cma == 1);
    $do_lanforge = 1;
    $do_http = 1;
    $do_xrdp = 1;
    $skip_fmirror = 1;
    $skip_cpu_burn = 1;
    $skip_disk_test = 1;
    if ($do_selinux == -1) {
        $do_selinux = 0; # disable it by default
    }
    if ($do_iommu == -1) {
        $do_iommu = 0; # Disable it by default
    }
    if ($is_fedora && $::osveri >= 19) {
        $do_runlevel = 3;
    }
}

if ($fix_web_root) {
    $do_http = 1;
}

if ($is_offline) {
    $do_only_pkgs = 0;
    $do_pkgs = 0;
    $skip_fmirror = 1;
    $skip_installer_check = 1;
    $skip_pip = 1;
    $skip_yum_all = 1;
    $skip_yum_update = 1;
}

if ($download_only) {
    $skip_yum_all = 1;
    $disable_audit_logs = -1;
    $do_xrandr = 0;
    $do_selinux = -1;
    $do_http = 1;
}

if ($::create_install_bundle) {
    $skip_yum_all = 1;
    $do_xrandr = 0;
    $do_selinux = -1;
    $skip_xorp = 0;
    $do_grub = 1;
    $skip_fmirror = 1;
    $skip_cpu_burn = 2;
    $skip_disk_test = 1;
    $do_lanforge = 1;
    $do_firmware = 1;
    $do_upgrade = 1;
    $do_kern = 1;
    $do_yum_update = 0;
    $do_xrandr = 0;
    $do_pkgs = 0;
}

if ($skip_yum_update) {
    $do_yum_update = 0;
}

if ($skip_firmware) {
    $do_firmware = 0;
}

if ($skip_yum_all) {
    $do_yum_update = 0;
    $do_pkgs = 0;
    $skip_fmirror = 1;
    $do_save_yumc = 0;
}

if ($do_ssd_fstab) {
    $do_elevator = 1;
}

if ($skip_grub) {
    $do_grub = 0;
    if ($remove_kern) {
        err("** --remove_kern and --skip_grub conflict");
        exit(1);
    }
}
if (!($do_upgrade || $do_all || $do_lanforge) && ($do_grub || $do_kern)) {
    $skip_yum_all = 1;
    $skip_yum_update = 1;
    $skip_pip = 1;
}

# Never install gnome on ARM platforms, they likely have either
# no desktop or something less bloated than gnome.
if ($is_arm) {
    $do_gnome = 0;
    $do_install_mate = 0;
    if (!$is_openwrt) {
        $do_firmware = 0; # don't need this on rpi
    }
}

if ($is_macos) {
    $do_kern = 0;
    $do_grub = 0;
    $remove_kern = 0;
    $skip_cpu_burn = 1;
    $do_yum_update = 0;
    $skip_yum_all = 1;
    $do_http = 0;
    $do_firmware = 0;
    $do_xrandr = 0;
    $do_selinux = -1;
    $skip_xorp = 1;
    $do_services = 0;
    $do_radius = 0;
    $do_hs20 = 0;
    $do_cups_pdf = 0;
    $do_xrdp = 0;
    #$do_vnc = 0;
    $do_udev = 0;
    $do_runlevel = 0;
    $skip_pip = 1;
}

if ($do_interop && $do_kern) {
    $do_kern=0;
    note("* do_interop disables do_kern\n");
}
if ($do_interop) {
    # todo: desktop icons should get copied to $HOME/.local/share/applications/
    if ((!-d "$::home") || (!-d "$::home/DB")) {
        $skip_yum_all = 0;
        $do_yum_update = 1;
    }
    else {
        $do_yum_update = $do_upgrade;
        $skip_yum_all = ($do_upgrade==0) ? 1: 0;
    }
    $do_cpu_burn = 0;
    $do_cups_pdf = 0;
    $do_firmware = 0;
    $do_grub = 0;
    $do_hs20 = 0;
    $do_http = 0;
    $do_kern = 0;
    $do_lanforge = 1;
    $do_limited_nm_config = 1;
    $do_pkgs = 1;
    $do_radius = 0;
    # avoid changing default runlevel if possible
    # because ubuntu systemctl might not handle disabling GDM well
    $do_runlevel = -1;
    $do_selinux = -1;
    $do_services = 1;
    $do_udev = 0;
    $do_vnc = 1;
    $do_xrandr = 0;
    $do_xrdp = 0;
    $ntwk_mgr_ok = 1;
    $remove_kern = 0;
    $skip_cpu_burn = 1;
    $skip_disk_test = 1;
    $skip_kern = 1;
    $skip_pip = 1;
    $skip_xorp = 1;
    if ($regen_nm_conf != 0) {
        $regen_nm_conf = 1;
    }
}
elsif ($regen_nm_conf >= 1) {
    $do_grub = 0;
    $do_kern = 0;
    $do_lanforge = 0;
    $do_pkgs = 0;
    $do_upgrade = 0;
    $do_yum_update = 0;
    $ntwk_mgr_ok = 1;
    $skip_installer_check = 1;
    $skip_yum_all = 1;
}

if ($source_dir ne "") {
    if (file_readable("$source_dir", "--source_dir")) {
        $do_local_install = 1;
        $skip_cpu_burn = 2;
        $skip_disk_test = 1;
        if (!$do_pkgs_f) {
            $do_yum_update = 0;
            $do_pkgs = 0;
        }
        $skip_fmirror = 1;
        $do_save_yumc = 0;
    }
    else {
        die("Unable to use --source_dir option");
    }
}

if (!$show_urls) {
    file_readable("$::tmp_dir", "--tmp_dir");
    file_writable("$::tmp_dir", "--tmp_dir");
}

if ($proxy ne "") {
    $proxy =~ s/@/\\@/g;
    # $curl = $curl . " -x $proxy";
    note("# Using proxy, curl now: $curl\n");
}

my %df_notes = ("Root" => 2001, # in megabytes
    "Tmp"              => 1501,
    "Home"             => 2501,
    "Boot"             =>   41);

if (!($::create_install_bundle || $is_macos || $print_only)
    && ($do_kern || $do_firmware || $do_lanforge || $do_upgrade || $do_sys_reconfig || $do_vm_reconfig)) {
    my @found_certificates = `find /etc/pki -name '*.cert' -o -name '*.crt'`;
    chomp(@found_certificates);
    #print "x" x 72, "\n";
    my $one_year = `date --date "+1 year" +'\%Y-\%m-\%d'`;
    chomp $one_year;
    my $today = `date +'\%Y-\%m-\%d'`;
    chomp $today;
    for my $cert (@found_certificates) {
        #print "Checking $cert...\n";
        my $notafter = `openssl x509 -enddate -noout -in $cert 2>/dev/null`;
        next if (!(defined $notafter) || ("" eq $notafter));
        $notafter =~ s/notAfter=(.*)$/$1/;
        $notafter = `date --date "$notafter" +"\%Y-\%m-\%d"`;
        chomp $notafter;
        if ($notafter le $today) {
            print "$cert expired on $notafter\n";
            $force_new_certs = 1;
        }
        elsif ($notafter le $one_year) {
            print "$cert exires soon, on $notafter\n";
            $force_new_certs = 1;
        }
        #else {
        #   print "$notafter is older than one year\n";
        #}
    }
    #print "x" x 72, "\n";
    #exit 0;
}

if (!($::create_install_bundle || $print_only || $do_local_install)
    && ($do_kern || $do_firmware || $do_lanforge || $do_upgrade)) {
    lcheck();
    route_check();
}

if ($do_kern || $do_firmware || $do_lanforge || $show_urls || $force_notes) {
    if (!$do_local_install) {
        if ($::lfver eq "") {
            die(usage() . "\n\nERROR:  Must specify --lfver if configuring LANforge, showing URLs, or installing kernel from remote source.\n\n");
        }
    }
    if (!$is_macos) {
        # do disk space warning, col 4 is available MB
        my @df_lines = `df -m / /boot $home /tmp /var/tmp`;
        chomp @df_lines;
        my @df_warns = ();
        # root
        my @vol_lines = grep {/^\/dev\/.*? \/$/} @df_lines;
        if (@vol_lines) {
            my $fwee = (split(/[ ]+/, $vol_lines[0]))[3];
            if ($fwee < $df_notes{"Root"}) {
                push(@df_warns, "Careful! We suggest " . $df_notes{"Root"} . " MB or more free space on /, we see $fwee\n");
            }
        }
        # home
        @vol_lines = grep {/^\/dev\/.*? \/home$/} @df_lines;
        if (@vol_lines) {
            my $fwee = (split(/[ ]+/, $vol_lines[0]))[3];
            if ($fwee < $df_notes{"Home"}) {
                push(@df_warns, "Careful! We suggest " . $df_notes{"Home"} . " MB or more free space on /home, we see $fwee\n");
            }
        }

        # tmp
        @vol_lines = grep {/^\/dev\/.*? \/tmp$/} @df_lines;
        if (@vol_lines) {
            my $fwee = (split(/[ ]+/, $vol_lines[0]))[3];
            if ($fwee < $df_notes{"Tmp"}) {
                push(@df_warns, "Careful! We suggest " . $df_notes{"Tmp"} . " MB or more free space on /tmp, we see $fwee\n");
            }
        }

        # boot
        @vol_lines = grep {/^\/dev\/.*? \/boot$/} @df_lines;
        if (@vol_lines) {
            my $fwee = (split(/[ ]+/, $vol_lines[0]))[3];
            if ($fwee < $df_notes{"Boot"}) {
                push(@df_warns, "Careful! We suggest " . $df_notes{"Boot"} . " MB or more free space on /boot, we see $fwee\n");
            }
        }

        if (@df_warns) {
            print_slow( "##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### #####",
                        @df_warns,
                        "##### ##### ##### ##### ##### ##### ##### ##### ##### ##### ##### #####");
        }
    } # if not mac-os
}
# check for chr_ubuntu
if ($old_hostname eq "ChrUbuntu") {
    $is_chr_ubuntu = 1;
}
elsif (!$::create_install_bundle) {
    # Check dmidecode too
    my $has_dmi_decode = system('which dmidecode &>/dev/null');
    if ($has_dmi_decode == 0) {
        my $dmi = `dmidecode -q`;
        if ($dmi =~ /Product Name: Lumpy/) {
            $is_chr_ubuntu = 1;
        }
    }
}
if (($do_kern || $do_grub || $remove_kern || $show_urls) && (!$is_macos)) {
    if ($kver eq "") {
        die(usage(). "\n\nERROR:  You must specify --kver when configuring showing URLs or installing or removing kernel.\n\n");
    }
    if ($is_64) {
        if ($is_chr_ubuntu) {
            $kern = "ct$kver.$arch-chr.tar.gz";
        }
        elsif ($is_arm) {
            # no 64-bit kernel currently
            $do_kern = 0;
        }
        else {
            $kern = "ct$kver.$arch.tar.gz";
        }
    }
    else {
        if ($is_chr_ubuntu) {
            $kern = "ct$kver.p4s-chr.tar.gz";
        }
        elsif ($is_arm) {
            # No generic arm kernel at this time
            if ($is_rpi3) {
                $kern = "ct$kver.rpi3.tar.gz";
            }
            else {
                $do_kern = 0;
            }
        }
        else {
            $kern = "ct$kver.p4s.tar.gz";
        }
    }
}

if ($do_http && ($lfrls lt "5207")) {
    warning("# --lfver $::lfver (lfrls[$lfrls]) is older than 5.2.7 which has no package for LANforge web interface.  Skipping --do_http.\n");
    $do_http = 0;
}

if ($do_lanforge || $show_urls || $force_notes) {
    if ($lfrls gt "5404" && !$is_macos) {
        $interop = "interop-${lfver}.tar.gz";
    }
    if ($is_deb_based && !$is_64) {
        die("Ubuntu 32-bit LANforge packages are no longer available");
    }
    if ($is_deb_based && !$is_arm) {
        $lfserver = "LANforgeServer-${lfver}_Linux-F${osveri}-x64.tar.gz";
        $lfgui = "LANforgeGUI_".$lfver."_Linux64.tar.bz2";
        if (!$skip_xorp) {
            $xorp = "xorp_64-F${osveri}.tgz";
            $xorp_nodebver = $xorp;
        }
        print_slow("    GUI package: $lfgui",
                   "    Server package: $lfserver",
                   "    Xorp package: $xorp") if ($::debug_on);
    }
    else { # ~resolve ubuntu-x86 package names
        if ($is_macos) {
            if ($is_arm) {
                # Not building for different OS versions at this time.
                $lfserver = "LANforgeServer-${main::lfver}_MacOS-ARM.tar.gz";
            }
            else {
                $lfserver = "LANforgeServer-${main::lfver}_MacOS-$::osver.tar.gz";
            }
            $xorp = "";
            #$lfgui = "LANforgeGUI_${lfver}_Linux.tar.bz2";
            $skip_gui = 1;
        }
        elsif ($is_arm) {
            if ($uveri >= 16) {
                $lfserver = "LANforgeServer-${main::lfver}_Linux-U16-a$archbit.tar.gz";
                $xorp = "xorp_$archbit-U16-a$archbit.tgz";
                #$lfgui = "LANforgeGUI_${lfver}_Linux.tar.bz2";
                $skip_gui = 1;
            }
            elsif ($is_rpi4) {
                $lfserver = "LANforgeServer-${main::lfver}_Linux-rpi4.tar.gz";
                $xorp = "xorp_32-rpi4.tgz";
                #$lfgui = "LANforgeGUI_${lfver}_Linux.tar.bz2";
                $skip_gui = 1;
            }
            elsif ($is_rpi5) {
                $lfserver = "LANforgeServer-${main::lfver}_Linux-rpi5.tar.gz";
                #$xorp = "xorp_32-rpi5.tgz";
                #$lfgui = "LANforgeGUI_${lfver}_Linux.tar.bz2";
                $skip_xorp = 1; # Not compiled for RPI5 currently.
                $skip_gui = 1;
            }
            else {
                # TODO:  Maybe rename imx6?
                $lfserver = "LANforgeServer-${main::lfver}_Linux-imx6.tar.gz";
                $xorp = "xorp_32-imx6.tgz";
                #$lfgui = "LANforgeGUI_${lfver}_Linux.tar.bz2";
                $skip_gui = 1;
            }
        } # ~resolve dev-board ubuntu-x86 package names
        else {
            $lfserver = "LANforgeServer-${main::lfver}_Linux-$osver-$arch.tar.gz";
            $xorp = "xorp_$archbit-$osver.tgz";

            if ($is_64) {
                $lfgui = "LANforgeGUI_${main::lfver}_Linux$archbit.tar.bz2";
            }
            else {
                $lfgui = "LANforgeGUI_${main::lfver}_Linux.tar.bz2";
            }
        } # ~resolve RH/Fedora package names
    } # ~resolve non-ubuntu-x86 package names
} #~do lanforge, force notes

sub generate_nmconnection_file {
    my $ifname = shift;
    if (!(defined $ifname) or ($ifname eq "")) {
        die("generate_nmconnection_file: no interface specified, bye");
    }
    my $connfile = shift;
    if (!(defined $connfile) or ($connfile eq "")) {
        #print("No connection file name provided, setting name to mgt_dev.nmconnection");
        $connfile = "mgt_dev.nmconnection";
    }
    # check for /etc/network/interfaces because this file will
    # contradict the entries this method will configure
    if ( -f "/etc/network/interfaces") {
        note("* Networking configuration in etc/network/interfaces found. Removing!\n");
        do_cmd("mv /etc/network/interfaces /etc/network/un_terfaces");
    }
    my $fq_connfile = "/etc/NetworkManager/system-connections/$connfile";
    if ( -f $fq_connfile ) {
        note("* replacing NM connection: $connfile\n");
        unlink($fq_connfile);
    }
    # find the interface the mac address
    if ($ifname eq "existing") {
        if ( -f "$::home/config.values") {
            my $posb_name = `grep ^mgt_dev $::home/config.values`;
            if (defined $posb_name) {
                chomp($posb_name);
                my @hunks = split(/\s+/, $posb_name);
                if (@hunks > 1) {
                    $ifname = $hunks[1];
                }
                else {
                    die("No existing interface name found");
                }
            }
        }
    }
    my @ipbr_lines = `ip -br addr show $ifname`;
    chomp(@ipbr_lines);
    #print_slow("===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ",
    #    Dumper(["ifname", $ifname, "ipbr_lines", \@ipbr_lines]),
    #    "===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ===== ",
    #    );
    my $macaddr = "00:00:00:00:00:00";
    my $detected_ip = "";
    my $detected_gateway = "";
    my $detected_mask = "";
    my $detected_dns = "";
    my @iplishow_lines = `ip -o link show $ifname`;
    if (@iplishow_lines < 1) {
        die("ip li sho $ifname FAILED: $!");
    }
    ($macaddr) = $iplishow_lines[0] =~ m| link/ether (\S+) |;
    if (!(defined $macaddr) || ($macaddr eq "")) {
        die("failed to match macaddr from [$iplishow_lines[0]]");
    }
    $macaddr = uc($macaddr);
    my @matches = ();
    my @ipbradshow_lines = `ip -br addr show $ifname`;
    if (@ipbradshow_lines < 1) {
        warning("* DID NOT SEE IP ADDR for $ifname\n");
    }
    else {
        #print_slow(
        #    " -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --",
        #    Dumper(["ipbradshow_lines", \@ipbradshow_lines]),
        #    " -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --",
        #);
        ($detected_ip, $detected_mask) = $ipbradshow_lines[0] =~ m|(\d+\.\d+\.\d+\.\d+)/(\d+)|;
    }
    my @nmdevshow_lines = `nmcli device show "$ifname" | grep -v NAME`;
    if (@nmdevshow_lines < 1) {
        warning("* DID NOT SEE NMCLI DEV data for $ifname\n");
    }
    else {
        chomp(@nmdevshow_lines);
        @matches = grep {/IP4\.GATEWAY:/} @nmdevshow_lines;
        if (@matches > 0) {
            ($detected_gateway) = $matches[0] =~ m|IP4\.GATEWAY:\s+(\d+\.\d+\.\d+\.\d+)|i;
            #print_slow(Dumper([
            #    "CMD", "nmcli device show $ifname | grep -v NAME",
            #    "detected_gateway", $detected_gateway,
            #    "MaTCHes", \@matches,
            #    "nmDEVshow_lines", \@nmdevshow_lines ])
            #);
        }
    }
    my @nmconnshow_lines = `nmcli conn show | grep -v NAME`;
    chomp(@nmconnshow_lines);
    #print_slow(
    #   " -- -- x -- -- x -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --",
    #   Dumper(["nmcli conn show | grep -v NAME", \@nmconnshow_lines]),
    #   " -- -- x -- -- x -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --",
    #);

    my $nmconnshow_name = "";
    @matches = grep {/\b$ifname\b/} @nmconnshow_lines;
    if (@matches < 1) {
        #print Dumper(["NO MATCHES for [$ifname] nmconnshow_lines", \@nmconnshow_lines]);
        warn("nothing matches /\\b$ifname\$/");
    }
    else {
        ($nmconnshow_name) = $matches[0] =~ /^(\S+)/;
        if (!(defined $nmconnshow_name) || ($nmconnshow_name eq "")) {
            warn("missing connshow_name[$nmconnshow_name]");
        }
        #warn("nmconnshow_name[$nmconnshow_name]");
    }

    # if we need to reset from command line:
    # nmcli conn modify mgt_dev ipv4.dns 192.168.45.3
    # nmcli conn up mgt_dev
    # sleep 3
    # ping ctweb
    chomp($nmconnshow_name);
    warn("NO NMCONNSHOW_NAME $!") if (! $nmconnshow_name);
    @nmconnshow_lines = `nmcli conn show "$nmconnshow_name" | grep -v NAME`;

    my $ipv4_method = "method=auto";
    if (@nmconnshow_lines < 1) {
        warning("no output from; nmcli conn show '$nmconnshow_name': $!");
    }
    else {
        chomp(@nmconnshow_lines);
        my @matches = grep { /ipv4.dns:/} @nmconnshow_lines;
        if (@matches>0) {
            ($detected_dns) = $matches[0] =~ m|ipv4\.dns:\s+([0-9.]+)|i;
        }
        print Dumper([
            # "nmconnshow_lines", \@nmconnshow_lines,
            "detected_ip",      $detected_ip,
            "detected_mask",    $detected_mask,
            "detected_dns",     $detected_dns,
            "detected_gateway", $detected_gateway,
        ]);        # want to detect manual/static IP assignment
        @matches = grep { /^ipv4\.method:\s+\w+$/i } @nmconnshow_lines;
        #print_slow(
        #    " -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --",
        #    "     nmconnshow_lines command:  nmcli conn show $nmconnshow_name | grep -v NAME ",
        #    Dumper(\@nmconnshow_lines),
        #    " -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --",
        #);
        if (@matches < 1) {
            warning("** missing ipv4.method from nmcli comm show $ifname output ");
        }
        ($ipv4_method) = $matches[0] =~ /ipv4.method:\s+(\w+)/;
    }
    my $dns = $detected_dns;
    if ((defined $::mgt_dns) && ($::mgt_dns ne "")) {
        print "* Using --mgt_dns option [$::mgt_dns] instead of $detected_dns\n";
        $dns = $::mgt_dns;
    }
    if ($dns) {
        $dns ="dns=$dns";
    }
    if ((defined $::mgt_ip) && ($::mgt_ip ne "")) {
        if ($::mgt_ip =~ /dhcp/i) {
            $ipv4_method = "method=auto"
        }
        else {
            my @new_ip = $::mgt_ip =~ m|^([0-9.]+)/(\d+)/([0-9.]+)|;
            if (exists $new_ip[0] && $new_ip[0] ne "") {
                # method=static not valid, use 'manual'
                $ipv4_method = "method=manual\naddress1=$new_ip[0]";
                if (exists $new_ip[1] && $new_ip[1] ne "") {
                    $ipv4_method .= "/$new_ip[1]";
                }
                else {
                    $ipv4_method .= "/24";
                }
                if (exists $new_ip[2] && $new_ip[2] ne "") {
                    $ipv4_method .= "\ngateway=$new_ip[2]";
                }
            }
            else {
                err("* mgt_ip set auto! does not match 'ip/cidr/gateway': $::mgt_ip\n");
                $ipv4_method = "method=auto";
            }
        }
    }
    else { # --mgt_ip was not specified on the command-line
        if ($ipv4_method eq "manual") {
            $ipv4_method = "method=manual\naddress1=$detected_ip/$detected_mask";
            if ($detected_gateway) {
                $ipv4_method .= "\ngateway=$detected_gateway";
            }
            $dns = "dns=$detected_dns";
        }
        elsif ($ipv4_method eq "auto") {
            $ipv4_method = "method=auto";
            $dns = "";
        }
        else {
            warn("Un-handled ipv4.method [$ipv4_method]");
        }
    }
    my $unixtime=`date +"\%s"`;
    chomp $unixtime;
    my @connfile_lines = (
        "[connection]",
        "id=mgt_dev",
        "type=ethernet",
        "timestamp=$unixtime",
        "interface-name=$ifname",
        "autoconnect=true",
        "metered=2",
        "autoconnect-priority=10",
        "",
        "[ethernet]",
        "mac-address=$macaddr",
        "auto-negotiate=true",
        "",
        "[ipv4]",
        "dns-search=",
        $ipv4_method,
        "may-fail=false",
        $dns,
        "",
        "[ipv6]",
        "method=disabled",
        "dns-search=",
        "method=auto",
        "ip6-privacy=0",
        "",
        "[proxy]",
        "",
    );
    open(my $fh, ">", $fq_connfile) or die($!);
    print $fh join("\n", @connfile_lines);
    close($fh);
    chmod(0600, "/etc/NetworkManager/system-connections/mgt_dev.nmconnection");
    chown(0, 0, "/etc/NetworkManager/system-connections/mgt_dev.nmconnection");
    print("* mgt_dev.nmconnection regenerated\n");
} # ~generate_nmconnection_file()

sub restart_nm {
    if ($::regen_nm_conf == 2) {
        print "not restarting NetworkManager\n";
        return;
    }
    my @future_ip = `grep -E '(method|address1)=' /etc/NetworkManager/system-connections/mgt_dev.nmconnection`;
    chomp @future_ip;
    my @present_ips = `ip -br addr show | grep UP`;
    chomp @present_ips;
    # my @matches = grep { /=auto/ } @future_ip;
    print(" =====          NetworkManager.conf           =====\n");
    system("cat /etc/NetworkManager/NetworkManager.conf");
    print(" =====        mgt_dev.nmconnection            =====\n");
    system("cat /etc/NetworkManager/system-connections/mgt_dev.nmconnection");
    print(" =====        Keyfiles                        =====\n");
    system("ls /etc/NetworkManager/system-connections/*.nmconnection");
    print(" =====        nmcli device status             =====\n");
    do_cmd("nmcli device status");
    print_slow(
        "===== ----- ===== ----- ===== ----- ===== ----- ===== ----- ===== ----- ===== ----- =====",
        "               About to restart NetworkManager. You could lose your connection.",
        "       Present IPs:",
        @present_ips,
        "",
        "       Planned mgt_dev IP:",
        @future_ip,
        "",
        "===== ----- ===== ----- ===== ----- ===== ----- ===== ----- ===== ----- ===== ----- =====",
        "....about to restart NetworkMananger...",
        );
    my $nmrestartsh = "/root/restart-nm.sh";
    unlink $nmrestartsh if (-f $nmrestartsh);
    open(my $nmf_fh, ">", $nmrestartsh) or warn("Unable to open $nmrestartsh: $!");
    if ($nmf_fh) {
        print $nmf_fh "#!/bin/bash\n";
        print $nmf_fh "systemctl stop NetworkManager\n";
        print $nmf_fh "rm -f /var/lib/NetworkManager/*\n";
        print $nmf_fh "systemctl start NetworkManager\n";
        print $nmf_fh "nmcli connection up mgt_dev ||: \n";
        close($nmf_fh);
        chmod(0775, $nmrestartsh);
        system("nohup $nmrestartsh | logger -t nmrestart 2>&1 </dev/null");
    }
} # ~restart_nm

sub rewrite_resolvconf {
    my $dns = "8.8.8.8";
    if ((defined $::mgt_dns) && ($::mgt_dns ne "")) {
        $dns = $::mgt_dns;
    }
    unlink("/etc/resolv.conf") if (-l "/etc/resolv.conf");
    open(my $resolv_fh, ">", "/etc/resolv.conf") or warn("Unable to open /etc/resolv.conf: $!");
    if ($resolv_fh) {
        print $resolv_fh "; created by lf_kinstall.pl\n";
        print $resolv_fh "; if this file is not replaced by dhclient then networking has not been restarted\n";
        print $resolv_fh "; or the mgt_dev interface is configured to use a static IP. If you have DNS trouble\n";
        print $resolv_fh "; try:            host -v www.candelatech.com $dns\n";
        print $resolv_fh "; and replace $dns with the IP of your DNS server\n";
        print $resolv_fh "options timeout:2\n";
        print $resolv_fh "nameserver $dns\n";
        close $resolv_fh;
        print("* Updated /etc/resolv.conf");
    }
}
# use this method to reconfigure your system to not use systemd-resolved
sub stop_using_systemdresolved {
    note("* Disabling systemd-resolved\n");
    do_cmd("systemctl stop systemd-resolved.service", 1);
    do_cmd("systemctl disable systemd-resolved.service", 1);
    do_cmd("systemctl daemon-reload", 1);
} # ~stop_using...

# update the /etc/NetworkManager.conf file to use main/no-auto-default
sub get_interfaces_list {
    my $ra_interfaces = [];
    my @ipli_lines = `ip -o link show`;
    chomp @ipli_lines;
    for my $line (@ipli_lines) {
        next if ($line =~ /: lo:/);
        my ($ifname) = $line =~ /^\d+: ([^:]+):/;
        next if (!(defined $ifname) || ($ifname eq ""));
        push(@$ra_interfaces, $ifname);
    }
    #print Dumper(["ra_interfaces", $ra_interfaces]) if ($::debug_on);
    return $ra_interfaces;
} # ~get_interfaces_list()

sub non_default_interfaces {
    # determine which of these interfaces are not providing default routes
    my $ra_interfaces = shift;
    my %ip_to_ifname = ();
    my %ifname_to_ip = ();
    my $dns = "8.8.8.8";
    if ((defined $::mgt_dns) && ($::mgt_dns ne "")) {
        $dns = $::mgt_dns;
    }
    my @ip_list = `ip -br addr show`;
    chomp(@ip_list);
    my $line = undef;
    for $line (@ip_list) {
      next if ($line =~ /^\s+$/); # skip blank things
      my @hunks = split(' ', $line);
      if (@hunks < 3) {
        # interface probably unplugged or down
        #print_slow("    non_default_interfaces does not break into three: [$line]");
        next;
      }
      my $ip = substr($hunks[2], 0, index($hunks[2], '/'));
      $ip_to_ifname{$ip} = $hunks[0];
      $ifname_to_ip{$hunks[0]} = $ip;
    }
    # print Dumper(["ip_to_ifname", \%ip_to_ifname, "ifname_to_ip", \%ifname_to_ip]) if ($::debug_on);
    if (!(defined $ra_interfaces) or (@$ra_interfaces < 1)) {
        err("Non default interfaces was given nothing. Please pass in a list of interfaces.\n");
        return [];
    }
    my %have_def_route = ();
    my %non_defaults = map { $_ => 1 } @$ra_interfaces;
    # add devices that will never be managable
    $non_defaults{"_vrf*"} = 1;
    $non_defaults{"vrf*"} = 1;
    $non_defaults{"wlan*"} = 1;
    $non_defaults{"sta*"} = 1;
    $non_defaults{"moni*"} = 1;

    #print_slow(" ===== ===== ===== ::non_default_interfaces: Evaluating These  Interfaces  ===== ===== ===== =====",
    #           keys(%non_defaults),
    #           "===== ===== =====  ===== ===== =====  ===== ===== =====  ===== ===== ===== ") if ($::debug_on);
    # if we have multiple instances of '1' as values something is wrong
    my $winner = undef;

    if ((defined $::mgt_dev) && ($::mgt_dev ne "")) {
        #print_slow(
        #    "===== ===== =====  ===== ===== =====  ===== ===== =====  ===== ===== ===== ",
        #    "       mgt_dev parameter has been specified on command line: [$::mgt_dev]",
        #    "===== ===== =====  ===== ===== =====  ===== ===== =====  ===== ===== ===== ")
        #    if ($::debug_on);
        if ($::mgt_dev =~ /[:]/) {
            print ("mgt_dev appears to be a mac address, ignoring for now.\n");
        }
        elsif ($::mgt_dev eq "existing") {
            # use config.values mgt_dev value
            if (-f "$::home/config.values") {
                my @cv_lines = `cat $::home/config.values`;
                chomp(@cv_lines);
                my @results = grep {/^mgt_dev/} @cv_lines;
                if (@results > 0) {
                    for my $line (@results) {
                        ($winner) = $line =~ /^mgt_dev\s+(.*)$/;
                        if (!(defined $winner)||($winner eq "")) {
                            print_slow("BAD mgt_dev line [$line]\n");
                        }
                        else {
                            last;
                        }
                    }
                    print("* Winner is existing mgt dev [$::mgt_dev]\n");
                    $::mgt_dev = $winner;
                }
                else {
                    print_slow("* no mgt_dev option in config.values!");
                }
            } # if config.values
            else {
                note("* $::home/config.values not found, new mgt_dev will be identified.");
            }
        }
        else {
            print("* Winning interface is: $::mgt_dev\n");
            $winner = $::mgt_dev;
        }
    } # ~if mgt_dev
    else {
        print( "* Non_default_interfaces: evaluating the device with the default gateway...\n");
        my @ipro_lines = `ip -o route show`;
        chomp @ipro_lines;
        my $ifname = undef;
        for $line (@ipro_lines) {
            ($ifname) = $line =~ m|^.*?\s+dev\s+(\S+)\s+|;
            if (!(defined $ifname) || ($ifname eq "")) {
                print_slow("no ifname here: [$line]");
                next;
            }
            # print "IFNAME [$ifname]\n";
            if (exists $have_def_route{$ifname}) {
                next;
            }
            # print Dumper(["dev", $ifname]) if ($::debug_on);
            # this might need to be modified to account for routing metrics, but ping is probably better
            my $metric = 1;
            if ($line =~ /^default\b/) {
                $metric = 1; # this can only apply if there exactly 1 default route
                if ($line =~ / metric \d+/) {
                    ($metric) = $line =~ / metric (\d+)/;
                    die("non_default_interfaces: bad regex!") if (!(defined $metric));
                }
                $have_def_route{$ifname} = $metric;
                next;
            }
            $non_defaults{$ifname} = 1;
        }
        if (!(defined $winner)) {
            if (keys(%have_def_route) > 1) {
                # we can land in a situation where the lowest route metric is
                # actually a dead route, it just responded first. So ping our way out of this
                # some hosts have a type of internet access daemon that does this, so
                # the metric numbers actually change as either the dhcp lease gets refreshed
                # or the access daemon updates the metric
                my $metric_1_count = 0;
                while( my($ifname, $metric) = each( %have_def_route)) {
                    $metric_1_count++ if ($metric == 1);
                }
                die("multiple metric-1 routes found, trouble afoot!")
                    if ($metric_1_count > 1);
                print Dumper(["have_def_route", \%have_def_route, "non_defaults", \%non_defaults ])
                    if ($::debug_on);
                my %ping_success = ();
                print("-- multiple gateways detected, discovering useful gateway --\n");
                while( my($ifname, $metric) = each( %have_def_route)) {
                    my $pingcmd = "ping -q  -W2 -w4 -I $ifname_to_ip{$ifname} 8.8.8.8";
                    print_slow("    PING CMD:   $pingcmd") if ($::debug_on);
                    my $pingrez = `$pingcmd`;
                    if ($? == 0) {
                        $ping_success{$ifname} = $metric;
                    }
                    else {
                        $ping_success{$ifname} = 0;
                    }
                }
                for my $ifname (keys(%ping_success)) {
                    my $itype = "Unknown";
                    if ($ifname =~ /^e/) {
                        $itype = "Ethernet";
                    }
                    elsif ($ifname =~ /^w/) {
                        $itype = "Wireless";
                    }
                    my $success = $ping_success{$ifname};
                    print "* $itype -- $ifname -- $success\n";
                    if (($success > 0) && ($itype eq "Ethernet")) {
                        print "     ** $ifname is winning **\n";
                        $winner = $ifname;
                    }
                }
                if (!(defined $winner)) {
                    print_slow("\n* No ethernet interface was able to ping google *\n",
                                "Please update your NetworkManager.conf by hand.",
                                "Ideally you want an ethernet management interface");
                    for my $ifname (keys(%ping_success)) {
                        my $itype = "Unknown";
                        if ($ifname =~ /^e/) {
                            $itype = "Ethernet";
                        }
                        elsif ($ifname =~ /^w/) {
                            $itype = "Wireless";
                        }
                        my $success = $ping_success{$ifname};
                        print "* $itype -- $ifname -- $success\n";
                        if ($success > 0) {
                            print "     ** $ifname is winning **\n";
                            $winner = $ifname;
                        }
                    }
                    if (!(defined $winner)) {
                        print_slow("\n* No interface appears to have been able to ping google. *");
                    }
                    else {
                        print_slow("\n* Interface $winner is now the mangement interface. *") if ($::debug_on);
                    }
               } # ~no winner
            } # ~multiple interfaces have default route
            elsif (keys(%have_def_route) == 1) {
                # winner
                $winner = (keys(%have_def_route))[0];
            }
            else {
                $winner = undef;
                print_slow( " ===== ===== =====  ===== ===== =====  ===== ===== =====  ===== ===== ===== ",
                            "           No interfaces have a default route.",
                            "           Please configuring mgt_dev by hand",
                            " ===== ===== =====  ===== ===== =====  ===== ===== =====  ===== ===== ===== ");
            }
        } # if not winner
    } # ~else --mgt_dev= not provided on command line

    if (defined $winner) {
        if (exists $non_defaults{$winner}) {
            delete $non_defaults{$winner};
        }
        if ($winner ne $::mgt_dev) {
            print("setting mgt_dev to $winner\n");
            $::mgt_dev = $winner;
        }
    }
    else {
        print_slow("non_default_interfaces: no MGT DEV determined as winner");
    }
    #print_slow( " ===== ===== =====  ===== ===== =====  ===== ===== =====  ===== ===== ===== ",
    #            Dumper(["non_defaults minus the winner", \%non_defaults]),
    #            " ===== ===== =====  ===== ===== =====  ===== ===== =====  ===== ===== ===== ")
    #    if ($::debug_on);
    return keys(%non_defaults);
} # ~non_default_interfaces()

sub networkmanager_interop_config {
    my $netmgr_d = "/etc/NetworkManager";
    my $netmgr_conf = "$netmgr_d/NetworkManager.conf";
    die("No $netmgr_d directory, bye")
        if (! -d $netmgr_d);

    my $ra_iinterfaces = get_interfaces_list();
    # print Dumper(["ra_iinterfaces", $ra_iinterfaces]);
    my @nnondefaults = non_default_interfaces($ra_iinterfaces);
    # print Dumper(["ra_nnondefaults", \@nnondefaults]);
    my $no_auto = "";
    if (@nnondefaults < 1) {
        $no_auto = '*'; # unable to deterine
    }
    else {
        $no_auto = join(',', @nnondefaults);
    }
    my $write_netmgr_conf = 0;
    my @nmconf_lines = ();
    if ((!-f $netmgr_conf) || (-z $netmgr_conf)) {
        print("* creating $netmgr_d ...\n");
        # important to note that the no-auto-default=* does not actually seem to
        # disable. So this really needs a list of every interface except the one holding the default route.
        # do not enable ifupdown, let systemd manage resolvconf
        @nmconf_lines = ( "[main]",
                            "plugins=keyfile",
                            "no-auto-default=${no_auto}",
                            "dns=default",
                            "",
                            "[ifupdown]",
                            "managed=false",
                            "",
                            "[device]",
                            "wifi.scan-rand-mac-address=no",
                            "",
                        );
        $write_netmgr_conf = 1;
    }
    else {
        @nmconf_lines = `cat $netmgr_conf`;
        do_cmd("cp $netmgr_conf $netmgr_conf.old", 1);
        chomp(@nmconf_lines);
        die("Error reading $netmgr_conf") if (@nmconf_lines < 1);

        my $lno = 0;
        my $noauto_ct = 0;
        my @matches = grep {/^no-auto-default=.*/} @nmconf_lines;
        $noauto_ct = @matches;

        if ($noauto_ct > 0) {
            print("* NetworkManager.conf lists no-auto-default\n");
        }
        else {
            print("* adding no-auto-default to NetworkManager.conf\n");
            for my $line (@nmconf_lines) {
                if ($line =~ /^\[main\]$/) {
                    my @replacement = (
                        'no-auto-default=*'
                    );
                    print("* Splicing in stuff to $netmgr_conf: ");
                    splice(@nmconf_lines, $lno+1, 0, @replacement);
                    last;
                }
                $lno++;
            }
            $write_netmgr_conf = 1;
        }
    } # else we had a conf file

    if (!(defined $::mgt_dev) || ($::mgt_dev eq "") || ($::mgt_dev eq "existing")) {
        die("network_mananger_interop_config: did not assign winning interface to MGT_DEV");
    }
    my $write_resolvconf = 0;
    my $stop_resolved = 0;
    if ($write_netmgr_conf > 0) {
        open(my $fh, ">", $netmgr_conf) or die("Unable to open $netmgr_conf: $!");
        print $fh join("\n", @nmconf_lines), "\n";
        close($fh);
        print("* updated $netmgr_conf\n");
        # print Dumper(@nmconf_lines) if ($::debug_on);
        print("* update lfconfig with mgt_dev[$::mgt_dev]...\n");
        do_cmd("cd $::home; echo 'mgt_dev $::mgt_dev' | ./lfconfig");
        print("* updated lanforge mgt_dev\n");

        # rewrite the 99-unmananged.conf file
        my $unmanaged_conf = "${netmgr_d}/conf.d/99-unmanaged.conf";
        open($fh, ">", $unmanaged_conf) or die("Unable to open $unmanaged_conf: $!");
        print $fh "[keyfile]\n";
        print $fh "unmanaged-devices=";
        for my $ndname (@nnondefaults) {
            print $fh "interface-name:$ndname;"
        }
        print $fh "\n";
        close($fh);

        print("* NetworkManager.conf has been updated, checking dns= setting\n");
        my @dnslines = `grep '^dns=' $netmgr_conf`;
        chomp(@dnslines);
        my @resolvlines = `cat /etc/resolv.conf`;
        chomp(@resolvlines);
        if (@dnslines > 0) {
            if (@dnslines > 1) {
                print_slow( " ===== ===== =====  ===== ===== =====  ===== ===== =====  ===== ===== ===== ",
                            "       !! multiple dns= options in $netmgr_conf, I smell trouble !!",
                            Dumper(\@dnslines),
                            " ===== ===== =====  ===== ===== =====  ===== ===== =====  ===== ===== ===== ");
            }
            my @matches = grep { /^dns=systemd-resolved/ } @dnslines;
            if (@matches > 0) {
                print("Network manager dns=systemd-resolved set. Not editing resolv.conf\n");
            }
            else {
                @matches = grep { /^dns=default/ } @dnslines;
                if (@matches > 0) {
                    print("* $netmgr_conf has dns=default set, checking resolv.conf\n");
                    # we expect links like /etc/resolv.conf -> ../run/systemd/resolve/stub-resolv.conf
                    if (-l "/etc/resolv.conf") {
                        print("resolv.conf is a link, we should replace it");
                        $write_resolvconf = 1;
                        $stop_resolved = 1;
                    }
                    else {
                        @matches = grep { /; generated by .*dhclient.*/ } @resolvlines;
                        if (@matches > 0) {
                            print("* dhclient generated resolv.conf, should not edit\n");
                        }
                        else {
                            @matches = grep { /nameserver 127.0.0.53/ } @resolvlines;
                            $write_resolvconf = 1;
                            $stop_resolved = 1;
                            print("NM is set to dns=default but resolvconf using 127.0.0.53, will rewrite\n");
                        }
                    }
                } # ~if dns=default
           } # ~else dns=systemd
        } # ~if nmconf dns= set
        else {
            warn("XX no dns= setting in $netmgr_conf\n");
        }
    }
    elsif($::debug_on)  {
        print_slow("not writing $netmgr_conf ");
    }
    if ($write_resolvconf) {
        rewrite_resolvconf();
    }
    if ($stop_resolved || $::disable_resolved) {
        stop_using_systemdresolved();
    }
    generate_nmconnection_file($::mgt_dev);
    restart_nm() if ($::regen_nm_conf == 1);
} # ~networkmanager_interop_config()

if (($do_interop != 1) && ($regen_nm_conf > 0)) {
    if (! -d "/etc/NetworkManager") {
        die("Cannot run --regen_nm_conf on system lacking /etc/NetworkManager, bye.");
    }
    if (-f "/etc/NetworkManager/NetworkManager.conf") {
        do_cmd("mv /etc/NetworkManager/NetworkManager.conf /etc/NetworkManager/_NetworkManager.old");
    }
    networkmanager_interop_config();
    exit(0);
}

sub bu_fedora_repo {
    if (!-f "/etc/yum.repos.d/fedora.repo") {
        err("** no fedora.repo file");
        exit 1;
    }
    do_cmd("cp /etc/yum.repos.d/fedora.repo /etc/yum.repos.d/fedora.repo.orig")
        unless (-f "/etc/yum.repos.d/fedora.repo.orig");

    if (!-f "/etc/yum.repos.d/fedora-updates.repo") {
        err("** no fedora-updates.repo file");
        exit 1;
    }

    do_cmd("cp /etc/yum.repos.d/fedora-updates.repo /etc/yum.repos.d/fedora-updates.repo.orig")
        unless (-f "/etc/yum.repos.d/fedora-updates.repo.orig");
}

sub fedrepos_default {
    if (-x "/usr/bin/fedrepos" || -x "/bin/fedrepos") {
        note("# checking fedora repositorys for default state");
        my $rslt2 = `grep LEAVE_ME_BE /etc/yum.repos.d/fedora.repo`;
        if ($rslt2 =~ /LEAVE_ME_BE/g) {
            note("# NOT setting fedrepos to default, LEAVE_ME_BE is found in the fedora.repo file\n")
        }
        else {
            bu_fedora_repo();
            # In the past, we may have disabled the metalink..that breaks things
            # when a distribution moves to legacy state.  So, try to repair this.
            my $rsltM = `grep metalink /etc/yum.repos.d/fedora.repo`;
            if (!($rsltM =~ /metalink/g)) {
                note("# Requesting fedrepos to set metalink config back to defaults.\n");
                do_cmd("fedrepos default", 1);
            }
        }
        exit 0;
    }
    err("* fedrepos not installed.");
    exit 1;
}

sub restore_all_metalinks {
    if (-f "/etc/yum.repos.d/fedora.repo") {
        bu_fedora_repo();
        restore_metalinks("/etc/yum.repos.d/fedora.repo");
        restore_metalinks("/etc/yum.repos.d/fedora-updates.repo");
        exit 0;
    }
    err("* /etc/yum.repos.d/fedora.repo not found.");
    exit 1;
}

sub enable_archive_baseurl {
    if (-f "/etc/yum.repos.d/fedora.repo") {
        bu_fedora_repo();
        enable_archive_baseurl("/etc/yum.repos.d/fedora.repo");
        enable_archive_baseurl("/etc/yum.repos.d/fedora-updates.repo");
        exit 0;
    }
    err("* /etc/yum.repos.d/fedora.repo not found.");
    exit 1;
}

sub create_random_www_data {
    if ($lfrls <= 5308) {
        note("# add_random_www_data option available in 5.3.9 or later\n");
        return;
    }
    die("Missing $::home/scripts/create_file_assortment.bash\n"
        . "Install LANforge 5.3.9+ first, then make sure you have: "
        . " $::home/scripts/create_file_assortment.bash")
        unless (-f "$::home/scripts/create_file_assortment.bash");
    if (! -f "$::http_root/data_slug_2048K.bin") {
        note("# creating files of random data\n");
        do_cmd("$::home/scripts/create_file_assortment.bash $::http_root");
        note("#...done\n");
    }
}

sub config_apache_mgt_port {
    if ($lfrls <= 5308) {
        note("# config_apache_mgt_port option available in 5.3.9 or later\n");
        return;
    }

    die("Missing $::home/scripts/adjust_apache.pl\n"
        . "Install lanforge first, then make sure you have: "
        . " $::home/scripts/adjust_apache.pl")
        unless (-f "$::home/scripts/adjust_apache.pl");
    note("# binding apache to mgt port\n");
    #do_cmd("$::home/scripts/adjust_apache.pl");
    adjust_apache();
    note("# config_apache_mgt_port: restarting apache\n");
    sleep 2;
    do_cmd("hostname -F /etc/hostname", 1);
    do_cmd("systemctl restart --quiet httpd") if (-d "/etc/httpd");
    do_cmd("systemctl restart apache2") if (-d "/etc/apache2");
}

if (!(-x "/bin/curl" || -x "/usr/bin/curl")) {
    err("* No curl found, skipping installer check");
    $skip_installer_check = 1;
}

if (($do_lanforge || $do_upgrade)
    && !($::create_install_bundle || $do_all || $do_all_ct || $do_interop)) {
    my @prev_lfserv = glob("$::home/LANforgeServer-*");
    if (@prev_lfserv < 1) {
        die("* --do_upgrade called on system missing any previous LANforgeServer-5.x.y folders in $::home\n");
    }
}

# check status of this file
if ($www !~ /www\.candelatech\.com/) {
    $skip_installer_check = 1;
}
if (($skip_installer_check == 0)
    && ($debug_on != 1)
    && !($do_local_install || $print_only || $show_urls || $is_offline)
    && ($do_all_ct || $do_all || $do_lanforge || $do_sys_reconfig)
    # checking our $0 for kinstall keeps us from inspecting $ perl <( curl )
    && ($0 =~ /kinstall\.(pl|txt)/)) {
    note("Checking md5sum of $0...");
    my $mymd5sig = `$md5sum $0`;
    chomp($mymd5sig);
    $mymd5sig = (split(/\s+/, $mymd5sig))[0];
    print "$0 md5sum is $mymd5sig\n" if ($::debug_on);
    my $lfs_url = "$www/kinstallmd5.txt";
    print "lfs_url $lfs_url\n" if ($::debug_on);
    my $lfs_file = "/tmp/lf_kinstall.txt";
    unlink($lfs_file) if (-f $lfs_file);
    my $rh_request = newWebRequest($lfs_url, $lfs_file, "GET");
    $rh_request->{'connect-timeout'} = 5;
    $rh_request->{'retry'} = 2;
    $rh_request->{'retry-delay'} = 2;
    $rh_request->{'max-retry-time'} = 15;
    $rh_request->{'die_on_fail'} = 0;
    if ($::debug_on) {
        $rh_request->{'quiet'} = 0;
        $rh_request->{"options"} = "-v";
        $rh_request->{"notify_on_start"} = 1;
    }
    else {
        $rh_request->{'quiet'} = 1;
        $rh_request->{"options"} = "-#";
    }
    webget($rh_request);
    if (!(defined $rh_request->{"status"}) || ($rh_request->{"status"} !~ / 200 /)) {
        warning("Unable to retrieve $lfs_url to $lfs_file\n");
    }
    else {
        my @kchecklines = `cat $lfs_file`;
        chomp(@kchecklines);
        my @kcheckmatches = grep {/lf_kinstall\.pl$/} @kchecklines;
        if (@kcheckmatches > 0) {
            my $kmd5sig = "-";
            ($kmd5sig) = $kcheckmatches[0] =~ /(\w+)\s+.*lf_kinstall/;
            $kmd5sig = "--" if (!defined($kmd5sig));
            if ($mymd5sig ne $kmd5sig) {
                note("**  MD5 signature mismatch: me($mymd5sig)\n                           www($kmd5sig)\n");
                sleep 3;
                note("**   This means that the lf_kinstall.pl file you are using does not match\n"
                    . "**   the version on www.candelatech.com. You can avoid this warning with --skip_installer_check.\n");
                sleep 3;
            }
            else {
                print("  lf_kinstall MD5 signature matches\n") if ($::do_debug);
            }
        }
    }
}
else {
    print("did not attempt to check md5sum of lf_kinstall.pl") if ($::do_debug);
}

if (!($::create_install_bundle || $::use_install_bundle || $skip_yum_all || $is_deb_based || $download_only
     || $print_only || $show_urls || $do_local_install)) {
    clean_yum_proxy();
}

# save yum cache
if (($do_save_yumc == 1 || $do_save_yumc == 20)
    && !($::create_install_bundle || $is_deb_based || $download_only || $print_only || $show_urls || $do_local_install || $skip_yum_all)) {
    $do_yum_update = 1;
    #Change in yum,conf, keepcache=1
    do_cmd("cp /etc/yum.conf /etc/yum.conf.bak." . ts());
    my $found_kc = 0;
    my $yum_cfg = "/etc/yum.conf";

    note("# Updating $yum_cfg to enable KEEPCACHE.\n");

    my @cmds = `cat $yum_cfg`;
    open(FILE, ">$yum_cfg") or die "Couldn't open file: $yum_cfg for writing: $!\n\n";
    for ($i = 0; $i < @cmds; $i++) {
        my $ln = $cmds[$i];
        chomp($ln);
        if ($ln =~ /^\s*keepcache=.*/) {
            print FILE "keepcache=1\n";
            $found_kc = 1;
        }
        else {
            print FILE "$ln\n";
        }
    }
    if (!$found_kc) {
        print FILE "keepcache=1\n";
    }
    close FILE;
}
elsif ($do_save_yumc && !$is_deb_based) {
    die(usage() . "\n\nERROR:  Unspecified option for --do_save_yumc.\n\n");
}

if ($do_http) {
    if (($::lfver eq "") || ($::lfver eq "-1")) {
        die(usage() . "\n\nERROR:  Must specify lfver if configuring HTTP.\n\n");
    }
}

if ($do_upgrade || $do_lanforge || $do_http) {
    if ((-f '/lib/systemd/system/vncserver@.service') && !($do_only_pkgs)) {
        # we don't want mention of vncserver_wrapper or <USER>
        #  = ("grep", "-q", 'vncserver_wrapper|\<USER\>',
        my @vncservicelines = `cat /lib/systemd/system/vncserver\@.service`;
        my @matches = grep {/vncserver_wrapper|<USER>/i} @vncservicelines;
        if (@matches > 0) {
            # we found those strings
            note("# Broken vncserver\@.service detected, enabling --do_vnc\n");
            $do_vnc = 1;
        }
        else {
            note("# Checked vncserver\@.service and found no vncserver_wrapper\n");
        }
    }
    if ($create_pypirc) {
        my $pprc = q([distutils]
   index-servers =
       pypi

   [pypi]
   repository: https://upload.pypi.org/legacy/
   );
        print("# creating /home/lanforge/.pypirc\n");
        die($!) unless open(my $fh, ">", "/home/lanforge/.pypirc");
        print $fh $pprc;
        close $fh;
        `chown lanforge:$user_group /home/lanforge/.pypirc`;

        print("# creating /root/.pypirc\n");
        die($!) unless open($fh, ">", "/root/.pypirc");
        print $fh $pprc;
        close $fh;

        exit(0);
    }
}

if (($do_yum_update || $do_pkgs) && !($skip_yum_all || $download_only || $::create_install_bundle)) {
    if ($is_fedora) {
        if ($::osveri == 26) {
            my @rpmi_lines = `rpm -qi dnf`;
            # rpm -qi dnf | perl -ne '/Version\s*:\s*([0-9.]+)$/ && print "$1\n";'
            my @rver = grep {/Version\s*:\s*[0-9.]+$/} @rpmi_lines;
            if (@rver > 1) {
                my ($rpm_ver) = $rver[0] =~ /Version\s*:\s*([0-9.]+)$/;
                if (defined $rpm_ver && $rpm_ver <= 2.5.1) {
                    warn("Please update F26 kernel and dnf with the command before continuing:");
                    warn(" \$ sudo dnf update -y dnf kernel-core ");
                    warn("And then reboot.");
                }
            }
        }
        if ($::osveri >= 26) {
            print("# dnfdragora-updater is not helpful\n");
            note("# disabling dnfdragora...");
            my @dnfdragora_lines = `rpm -qa 'PackageKit*'`;
            if (@dnfdragora_lines > 0) {
                do_cmd("dnf remove -y PackageKit", 1);
            }
            @dnfdragora_lines = `rpm -qa dnfdragora*`;
            if (@dnfdragora_lines > 0) {
                do_cmd("dnf remove -y dnfdragora*", 1);
            }
            @dnfdragora_lines = `rpm -qa openvswitch`;
            if (@dnfdragora_lines > 0) {
                note("# disabling openvswitch");
                do_cmd("dnf remove -y openvswitch", 1);
            }
        }
        if ($main::osveri == 33) {
            # spice-vdagent is broken and sucks up a processor *schooop!*
            note("# disabling spice-vdagent");
            system(qw(/usr/bin/killall spice-vdagent));
            my @grepcmd = ('/usr/bin/grep', '^exclude=.*spice-vdagent', '/etc/dnf/dnf.conf');
            my $greprv = system(@grepcmd);
            if ($greprv == 0) {
                print("# spice-vdagent excluded\n");
            }
            else {
                if (open(my $dnfsettings, ">>", "/etc/dnf/dnf.conf")) {
                    print $dnfsettings "exclude=spice-vdagent\n";
                    close $dnfsettings;
                }
                else {
                    warn("* Unable to add exclude=spice-vdagent to dnf.conf");
                }
            }
            do_cmd("dnf remove -y spice-vdagent", 1);
            system(qw(chmod a-x /usr/bin/spice-vdagent)) if (-x "/usr/bin/spice-vdagent");
        }
        if ($::osveri >= 33) {
            my @dnfdragora_lines = `rpm -qa 'open-vm-tools*'`;
            if (@dnfdragora_lines > 0) {
                print("# disabling open-vm-tools");
                do_cmd("dnf remove -y 'open-vm-tools*'", 1);
                do_cmd("dnf remove -y setroubleshoot", 1);
                do_cmd("dnf remove -y virtualbox-guest-additions", 1);
            }
        }
    } # ~if remove fedora packages before doing updates
} # ~do yum update/rpm replacements

# We need nfs to be installed before we can attempt to mount
# our local storage.
if ($is_deb_based && $do_pkgs && !($download_only || $::create_install_bundle)) {
    if ($do_interop) {
        $ENV{LD_LIBRARY_PATH} = "";
    }
    if ($apt_update == 0) {
        do_cmd("dpkg --configure -a", 1);
        do_cmd("$apt update");
        $apt_update = 1;
    }
    # At least in some cases, even 'curl' is not installed..so
    # Need to run this very early.
    note("# Installing required deb packages.\n");
    do_cmd("dpkg --configure -a", 1);
    do_cmd("$apt install -y $base_deb_pkgs");
    do_cmd("$apt -f install"); # to make sure things are tamped down
    # check if root accoun is locked. Unlock it.
    my $root_p_status = `passwd -S root`;
    if ($? == 0) {
        my @root_status = split(/\s+/, $root_p_status);
        if ($root_status[1] ne "P") {
            note("* unlocking and setting root password\n");
            do_cmd(qq(bash -c 'echo "root:lanforge" | chpasswd'), 1);
            do_cmd("passwd -u root", 1);
        }
    }
}

# check if the root password is locked and has never been set
# this is the default for vanilla Fedora 34+ installations
if (!($::create_install_bundle || $download_only)
    && ($do_all_ct || $do_all || ($do_interop && $is_deb_based)
    && -f "/etc/shadow")) {
    print "Checking for locked root password...\n";
    my @shadlines = `grep root /etc/shadow`;
    chomp(@shadlines);
    # print Dumper(\@shadlines);
    my @lockmatches = grep {/^root:[!]{1,2}:/} @shadlines;
    # print Dumper(\@lockmatches);
    if (@lockmatches > 0) {
        note("Locked and unset root password detected. Setting root password to 'lanforge'\n");
        do_cmd(qq(bash -c 'echo "root:lanforge" | chpasswd'), 1);
        do_cmd("passwd -u root", 1);
    }
}

if ($is_fedora &&
    !($::create_install_bundle || $skip_yum_all || $download_only || $::use_install_bundle)) {
    # For scrcpy (adb tool) and possibly others in the future
    if ($::osveri >= 30) {
        note("Checking for rpmfusion repo...");
        `$yum repolist | grep rpmfusion &>/dev/null`;
        if ($? != 0) {
            note("Adding rpmfusion url for scrcpy");
            my $rpmfusionurl = "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-${main::osveri}.noarch.rpm";
            do_cmd("dnf install $rpmfusionurl -y", 1);
        }
        else {
            note("...found\n");
        }
    }
} # ~if do scrcpy

if ($is_redhat && $::osveri <= 19) {
    note("Configuring subscription manager...\n");
    do_cmd("subscription-manager repos --enable rhel-7-server-rpms", 1);
    do_cmd("subscription-manager repos --enable rhel-7-server-optional-rpms", 1);
}

if (($is_centos || $is_redhat)
    && !($::create_install_bundle || $skip_yum_all || $download_only || $::use_install_bundle)) {
    my @repos_already = `$yum repolist | awk '{print \$1}'`;
    chomp(@repos_already);
    my %repo_names = ();
    for my $r (@repos_already) {
        $repo_names{$r} = 1;
    }
    do_cmd("rpm --import https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official", 1);
    if (!($do_local_install || $::create_install_bundle)) {
        # Enable the EPEL repo on Centos
        if ($::osveri == 13) {
            #if ( system("rpm -qi epel-release >/dev/null") != 0 ) {
            if (!exists($repo_names{"epel"})) {
                do_cmd("rpm -Uvh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm", 1);
            }
        }
        elsif (($::osveri > 13) && ($::osveri < 28)) {
            if (!exists($repo_names{"epel"})) {
                do_cmd("rpm -U https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm", 1);
                do_cmd("$yum search epel-release && $yum install -y epel-release", 1);
            }
            if (!exists($repo_names{"elrepo"})) {
                do_cmd("rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org", 1);
                do_cmd("rpm -U https://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm", 1);
            }
        }
        elsif (($::osveri > 27) && ($::osveri < 32)) {
            if (!exists($repo_names{"elrepo"})) {
                do_cmd("rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org", 1);
                do_cmd("rpm -U https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm", 1);
            }
            if (!exists($repo_names{"epel"})) {
                do_cmd("rpm -U https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm", 1);
                do_cmd("$yum search epel-release && $yum install -y epel-release", 1);
            }
        }
        if ($::osver eq "C8X") {
            $yum_gnome_pkgs = "";
            if (!exists($repo_names{"elrepo"})) {
                do_cmd("rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org", 1);
                do_cmd("$yum install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm", 1);
            }
            if (!exists($repo_names{"epel"})) {
                do_cmd("rpm --import https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official");
                do_cmd("rpm -U https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm");
            }
            if (($::osver eq "C8X") && (!-f "/etc/yum.repos.d/CentOS-PowerTools.repo")) {
                # create it
                die("Unable to create CentOS-PowerTools.repo")
                    unless (open(my $fhpt, ">", "/etc/yum.repos.d/CentOS-PowerTools.repo"));
                print $fhpt q([PowerTools]
name=CentOS-$releasever - PowerTools
mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=PowerTools&infra=$infra
#baseurl=http://mirror.centos.org/$contentdir/$releasever/PowerTools/$basearch/os/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
);
                close $fhpt;
            }
            my @haslist = (`fgrep -h '[' /etc/yum.repos.d/* | grep -v -E debug -e source | tr -d \\[\\] |sort`);
            chomp(@haslist);
            my @repolist = ();
            my %desired = (
                'AppStream'                                => 1,
                'rhel-8-for-x86_64-appstream-rpms'         => 1,
                'cr'                                       => 1,
                'centosplus'                               => 1,
                'extras'                                   => 1,
                'PowerTools'                               => 1,
                'epel'                                     => 1,
                'epel-playground'                          => 1,
                'epel-testing'                             => 1,
                'epel-testing-modular'                     => 1,
                'fasttrack'                                => 1,
                'codeready-builder-for-rhel-8-x86_64-rpms' => 1
            );
            for my $repo (@haslist) {
                push(@repolist, $repo) if (exists $desired{$repo});
            }
            for my $repo (@repolist) {
                do_cmd("dnf config-manager --enable $repo", 1);
            }
        } # ~if CS8
        elsif ($::osveri > 13) { #centos
            # we want to set a few repos to enabled=1
            my @enable_list = (
                "CentOS-Base.repo:[centosplus]",
                "CentOS-fasttrack.repo:[fasttrack]",
                "epel.repo:[epel]",
                "elrepo.repo:[elrepo-extras]"
            );
            for my $repo (@enable_list) {
                my ($repo_file, $repo_name) = split(/:/, $repo);
                $repo_file = "/etc/yum.repos.d/$repo_file";
                if (!-w $repo_file) {
                    warn "Repo file $repo_file not writable\n";
                    next;
                }
                note("enabling $repo_name in $repo_file\n");
                my @old_lines = ();
                my @new_lines = ();
                open(my $fh, "<", $repo_file) or warn "couldn't open $repo_file";
                next unless ($fh);
                @old_lines = <$fh>;
                close $fh;
                my $start = 0;
                for my $line (@old_lines) {
                    chomp $line;
                    if ($line eq $repo_name) {
                        $start = 1;
                    }
                    if ($start) {
                        if ($line =~ /^enabled\s*=\s*0/) {
                            $line = "enabled=1";
                            $start = 0;
                        }
                    }
                    push @new_lines, $line;
                }
                open($fh, ">", $repo_file) or warn "couldn't open $repo_file";
                if ($fh) {
                    print $fh join("\n", @new_lines) . "\n";
                    close $fh;
                }
            }
            #for my $pkg (qw(ipmitool freerdp mod_python)) {
            #   do_cmd("yum search all $pkg &>/dev/null", 1);
            #}
        }
        if (!(defined $::use_install_bundle && "" ne $::use_install_bundle) && $do_yum_update) {
            if (!($is_offline||$use_yum_cache||$skip_yum_update||$skip_yum_all)) {
                note("# cleaning $yum cache...\n");
                do_cmd("$yum clean all&>/dev/null", 1);
                note("# updating Red Hat subscription-manager...\n");
                do_cmd("subscription-manager refresh", 1) if ($is_redhat || ($::osver eq "C8X"));
                note("# updating dnf cache...\n");
                do_cmd("$yum makecache -v &>/dev/null", 1);
            }
            note("# checking for updates...\n");
            do_cmd("$yum check-update &>/dev/null", 1);
        }
    } #~  If (!($do_local_install||$::create_install_bundle))
} # ~if redhat/centos

if (!($do_local_install || $::create_install_bundle || $force_web || $show_urls || $download_only)) {
    # Create mount points if we can.
    my $wants_nfs = (!$force_web) && ($do_upgrade || $do_lanforge || $do_pkgs || $do_firmware || $do_kern);

    # require that 192.168.100.1 or 192.168.92.1 be the default gateways
    my @rolines = `ip route show| grep default`;
    if (($? >> 8) > 0) {
        err("* No default route?");
        print(join("\n", @rolines));
        err("Unable to find default route. Enabling --force_web");
        $wants_nfs = 0;
        $force_web = 1;
    }
    elsif (@rolines > 0) {
        if ($rolines[0] =~ /^default via 192\.168\.(92|100)\.1 /) {
            $wants_nfs = 1;
            $force_web = 0;
        }
        else {
            $wants_nfs = 0;
            $force_web = 1;
        }
    }
    else {
        $wants_nfs = 0;
        $force_web = 1;
    }

    if ($wants_nfs && (system("ping -q -c 1 -n -w 2 $fs3 > /dev/null 2>&1") == 0)) {
        note("# Able to ping 192.168.100.3 \n");
        # if we have correct web page ciontent for fs3, then we're almost certainly on correct network
        unlink("/tmp/fs3.html") if (-f "/tmp/fs3.html");
        `curl --connect-timeout 15 -D /tmp/fs3.txt -o /tmp/fs3.html http://192.168.100.3/index.html`;
        if ($? != 0) {
            note("# Do not see http://192.168.100.3/index.html, not attempting NFS\n");
            $wants_nfs = 0;
            $force_web = 1;
        }
        elsif (-s "/tmp/fs3.html") {
            note("# inspecting http://192.168.100.3/index.html headers....");
            my @headerlines = `cat /tmp/fs3.txt`;
            my @matches = grep {m[HTTP.*404]} @headerlines;
            if (@matches >= 1) {
                note("# inauthentic fs3.candelatech.com: got a 404\n");
                $staged_in_candela = 0;
                $wants_nfs = 0;
                $force_web = 1;
            }
            else {
                note("# inspecting http://192.168.100.3/index.html: got a 200\n");
                my @indexlines = `cat /tmp/fs3.html`;
                my @matches = grep {m[<head><title>Candela</title></head>]} @indexlines;
                if (@matches >= 1) {
                    note("# found authentic fs3.candelatech.com\n");
                    $staged_in_candela++;
                }
                else {
                    note("# found inauthentic fs3.candelatech.com:\n");
                    note("# -----------------------------------------\n");
                    note(join("\n", @indexlines));
                    note("# -----------------------------------------\n");
                    $staged_in_candela = 0;
                    $wants_nfs = 0;
                    $force_web = 1;
                }
            }
        }
        elsif (-d "/mnt/d2/pub") {
            note("# Appears fs3.candelatech.com already mounted \n");
            $staged_in_candela++
        }
        else {
            note("# Do not see fs3.candelatech.com, not attempting NFS\n");
            $staged_in_candela = 0;
            $wants_nfs = 0;
            $force_web = 1;
        }

        note("# -----------------------------------------\n");
        sleep(5);

        if ($::create_install_bundle) {
            $staged_in_candela = 0;
        }
        note("# Staged on candelatech.com network: $staged_in_candela\n");

        #note("# Found $fs3, will try to get builds from it first.\n");
        #On Fedora 20 mount.nfs can be missing.  Check and install if missing.
        if (!$is_openwrt && !$is_macos) {
            if (system("which mount.nfs &>/dev/null") != 0) {
                note("# mount.nfs missing.  Attempting to install from web.\n");
                if ($::osveri >= 24) {
                    do_cmd("$yum --allowerasing --best -y install nfs-utils", 1);
                    do_cmd("$yum --allowerasing --best -y install mount.nfs", 1);
                }
                else {
                    do_cmd("$yum -y install nfs-utils", 1);
                    do_cmd("$yum -y install mount.nfs", 1);
                }
            }
            else {
                note("# mount.nfs is present\n");
            }

            if ($::osveri > 13) {
                note("# enabling nfs and rpcbind services...\n");
                if ($is_centos || $is_redhat) {
                    mkdir("/run/rpcbind") unless (-d "/run/rpcbind/");
                    do_cmd("systemctl enable rpcbind.service", 1);
                    do_cmd("systemctl restart rpcbind.service", 1);
                    do_cmd("systemctl restart nfs-utils.service", 1);
                }
                elsif (-f "/lib/systemd/system/nfs-client.target") {
                    note("enabling nfs-client.target...");
                    do_cmd("systemctl enable    nfs-client.target", 1);
                    do_cmd("systemctl restart   nfs-client.target", 1);
                }
                elsif (-f "/lib/systemd/system/nfs-mountd.service") {
                    note("enabling nfs-mountd.service...");
                    do_cmd("systemctl add-wants multi-user.target nfs-mountd.service", 1);
                    do_cmd("systemctl enable    nfs-mountd.service", 1);
                    do_cmd("systemctl restart   nfs-mountd.service", 1);
                }
                sleep 3;
                do_cmd("systemctl --system daemon-reload", 1);
                sleep 3;
            }
        } # if not openwrt or macos image

        if ($wants_nfs && $staged_in_candela) {
            if (system("grep /mnt/d2 /etc/fstab > /dev/null 2>&1") != 0) {
                note("# Updating fstab so it can attempt to mount /mnt/d2 on fs3.\n");
                my $nfsopts = "noauto";
                if ($is_redhat || $is_centos || $::osveri >= 19 || $is_rpi4 || $is_rpi5) {
                    $nfsopts = "noauto,nfsvers=3";
                }

                system("echo $fs3:/mnt/d2 /mnt/d2         nfs  $nfsopts 0 0 >> /etc/fstab");
                if (!-d "/mnt/d2") {
                    if (!mkdir("/mnt/d2")) {
                        note("# WARNING:  Unable to mkdir and read /mnt/d2, will try to download from $www\n");
                        $force_web = 1;
                        $wants_nfs = 0;
                        sleep 3;
                    }
                }
            }
        }
        else {
            note("# not adding mnt/d2 entry");
        }

        if (!$force_web && $wants_nfs && $staged_in_candela) {
            #note("# Found /mnt/d2 in /etc/fstab...\n");
            if (!-d "/mnt/d2/pub") {
                note("# Adding mount d2; staged:[$staged_in_candela] wantsnfs[$wants_nfs] forceweb[$force_web]\n");
                note("# Attempt to mount /mnt/d2...\n");
                if (system("mount /mnt/d2") != 0) {
                    note("# NOTE:  Could not mount /mnt/d2, will try to download from $www\n");
                    $force_web = 1;
                    $wants_nfs = 0;
                    sleep 3;
                }
                else {
                    $staged_in_candela++;
                }
            }
        }
        else {
            note("# not attempting to mount d2; staged:[$staged_in_candela] wantsnfs[$wants_nfs] forceweb[$force_web]\n");
        }


        if ($staged_in_candela) {
            # silence speaker
            my $fn = "/etc/modprobe.d/pcspkr-blacklist.conf";
            if (!-f $fn) {
                do_cmd("echo 'blacklist pcspkr' > $fn", 1);
            }
            # blacklist mei
            $fn = "/etc/modprobe.d/mei-blacklist.conf";
            if (!-f $fn) {
                do_cmd("echo 'blacklist mei mei_me mei_txe' > $fn", 1);
            }
        }

        # this prolly should be move out of the fs3 check block

        # update root system mount point to be data=journal because
        # doing fsck in field is not easy for customers to do
        #
        # possibly the data=ordered mount options that I see might be
        # because some of these options might be getting added properly
        if ($staged_in_candela && ($do_all_ct || $do_all || $do_grub)) {
            my @fstab_lines = `cat /etc/fstab`;
            my @new_fstab_lines = ();
            my @devlist = ();
            my $fstab_modified = 0;
            chomp @fstab_lines;
            for my $line (@fstab_lines) {
                my $new_line = $line;
                # update this regex so we can make sure that the last column is 1 for /, 2 for anything ext4
                if ($line =~ m{^\s*(.*?)\s+(/|/boot|/home)\s+ext4\s+([^ ]+)\s+1 }) {
                    my ($f_dev, $f_point, $f_sys, $f_opts, $remainder) = $line =~ m{^\s*(.*?)\s+(/\S*)\s+(ext4)\s+([^ ]+)\s+(1.*)$};
                    do_cmd("tune2fs -o journal_data $f_dev", 1);
                    do_cmd("tune2fs -c 1 $f_dev", 1);
                    push(@devlist, $f_dev);
                    if ($line =~ /,?data=/) {
                        $fstab_modified++;
                        $line =~ s/(,?data=[^, ]+)//;
                    }
                }
                push(@new_fstab_lines, $line);
            }
            if ($::osveri < 17) {
                do_cmd("touch /forcefsck", 1);
            }

            # if there are any data=(ordered|writeback) flags on the mount, just erase them
            # they will force a newly data=journal filesystem to reboot into ro mode
            if ($fstab_modified > 0) {
                open(my $fh_fstab, ">", "/etc/fstab") or die("unable to write fstab");
                print $fh_fstab join("\n", @new_fstab_lines) . "\n";
                close $fh_fstab;
            }
            note("# set ext3 option data=journal to mountpoints:" . join(", ", @devlist) . "\n") if (@devlist);
            `sync`; # wouldn't that be ironic?
        }           # ~if at candela

        # here woule be a opportune place to create a swap file
        # maybe we should check on largest partition, however a good standard spot might be
        # /home/lanforge/.swap
        # dd if=/dev/zero bs=1G count=6 of=/home/lanforge/.swap
        # mkswap /home/lanforge/.swap; swapon /home/lanforge/.swap
        # then append "swapon /home/lanforge.swap" in /etc/rc.local
        # make sure rc-local.service is enabled!

        # If we successfully mounted fs3, then try using it for squid proxy
        # as well.

        if (!-d $d2dir) {
            note("# Could not access /mnt/d2/pub on fs3($fs3), will attempt to download from web site.\n");
            $force_web = 1;
        }
    }
    else {
        # Perhaps we are on a customers network so attempt to install from web.
        $force_web = 1;
    }
} #~ create mount points
if ($ctproxy ne "") {
    note("# Found /mnt/d2/pub for local file system downloads.\n");
    if ($do_upgrade || $do_yum_update || $do_pkgs) {
        if (-f "/etc/dnf/dnf.conf") {
            if (system("grep proxy /etc/dnf/dnf.conf > /dev/null 2>&1") != 0) {
                do_cmd("cp /etc/dnf/dnf.conf /etc/dnf/dnf.conf.orig");
                do_cmd("echo proxy=http://$ctproxy:3128 \>\> /etc/dnf/dnf.conf");
                $rem_yum_proxy = 1;
            }
        }
        if (-f "/etc/yum.conf") {
            if (system("grep proxy /etc/yum.conf > /dev/null 2>&1") != 0) {
                do_cmd("cp /etc/yum.conf /etc/yum.conf.orig");
                do_cmd("echo proxy=http://$ctproxy:3128 \>\> /etc/yum.conf");
                $rem_yum_proxy = 1;
            }
        }
    }
}
if ($force_web && ($download_host ne $www)) {
    #note("[Force web] Checking source dir...\n");
    if ($source_dir eq "") {
        #note("[Force web] clearing d2dir...\n");
        $d2dir = "";
    }
    else {
        die(usage(). "\n\nERROR:  Cannot use both --force_web and --source_dir together!\n\n");
    }
}

if ($is_offline) {
    if ($show_urls) {
        warning ("--offline and --show_urls should not be combined. This turns off --force_web\n");
    }
    $force_web = 0;
}

if (!($do_grub || $do_pkgs || $do_services || $do_radius || $do_xrdp
    || $do_yum_update || ($do_selinux != -1) || $do_xrandr
    || $do_gnome || $do_udev || $do_runlevel || $do_vnc || $do_http || $do_lanforge
    || $do_upgrade || $do_ct_st || $no_fmirror || $show_urls || $force_notes
    || $do_cpu_burn || $do_firmware || $do_gui_autostart || $save_gui_cfg
    || $do_interop )) {
    die(usage(). "\n\nERROR:\n $0 $lfkinargs\n\nNo options selected..nothing to do!\n\n");
}

# reconfigure grub for fedora here
if ($do_grub && !$is_deb_based && !($download_only || $::create_install_bundle)) {
    note(printf("CMA recommendataion: %d\n", cma_recommendation()));

    if (!($do_biosdevname == -1 || $do_biosdevname == 0 || $do_biosdevname == 1)) {
        die(usage() . "\n\nERROR:  do_biosdevname must be -1, 0, or 1.\n\n");
    }
    if (!($do_elevator == -1 || $do_elevator == 0 || $do_elevator == 1)) {
        die(usage(). "\n\nERROR:  do_elevator must be -1, 0, or 1.\n\n");
    }

    if ($do_cma < -1) {
        die(usage() . "\n\nERROR:  do_cma must be -1, 0, or >=1, preferrably 64.\n\n");
    }
    if (!($do_noaer == -1 || $do_noaer == 0 || $do_noaer == 1)) {
        die(usage() . "\n\nERROR:  do_noaer must be -1, 0, or 1.\n\n");
    }

    if (!($do_iommu == -1 || $do_iommu == 0 || $do_iommu == 1)) {
        die(usage() . "\n\nERROR:  do_iommu must be -1, 0, or 1.\n\n");
    }
    if (!($do_nomitigations == -1 || $do_nomitigations == 0 || $do_nomitigations == 1)) {
        die(usage() . "\n\nERROR:  do_nomitigations must be -1, 0, or 1.\n\n");
    }
    if (!($do_kmemleak == -1 || $do_kmemleak == 0 || $do_kmemleak == 1)) {
        die(usage() . "\n\nERROR:  do_kmemleak must be -1, 0, or 1.\n\n");
    }
    if ($kver eq "") {
        die(usage() . "\n\nERROR:  You must specify --kver when updating grub.\n\n");
    }
    if ($skip_kern==1) {
        die("Cannot use --do_grub and --skip_kern at the same time. (do_upgrade:$do_upgrade)\n\n")
    }
    if (-f "$grub_cfg") {
        if (!-f "$etc_default_grub") {
            die("ERROR: $etc_default_grub does not exist, cannot update boot options?\n\n");
        }
    }
    else {
        if (!-f "$grub1_cfg") {
            # Maybe it's EFI?
            my $is_ok = 0;
            if (-l "/etc/grub.conf") {
                my $lfn = readlink("/etc/grub.conf");
                if ($lfn =~ "boot/efi/EFI/redhat/grub.conf") {
                    if (-f "/boot/efi/EFI/redhat/grub.conf") {
                        note("# This appears to be an EFI based grub.conf system.\n");
                        $is_ok = 1;
                        $grub1_cfg = "/boot/efi/EFI/redhat/grub.conf";
                    }
                }
            }

            if ((!$is_ok) && -l "/etc/grub2-efi.cfg") {
                my $lfn = readlink("/etc/grub2-efi.cfg");
                if ($lfn =~ qr{boot/.*/grub\.cfg}) {
                    my @efi_files = glob("/boot/efi/EFI/*/grub.cfg");
                    if (@efi_files > 0 && -f $efi_files[0]) {
                        note("# This appears to be an EFI based grub2-efi.cfg system.\n");
                        note("# using: $efi_files[0]\n");
                        $is_ok = 1;
                        $grub_cfg = $efi_files[0];
                    }
                    else {
                        note("# No EFI based grub2-efi.cfg files found.\n");
                    }
                }
            }

            if (!$is_ok) {
                die("ERROR: Neither $grub1_cfg nor $grub_cfg exists, maybe this machine uses a different bootloader?\n\n");
            }
        }
    }
} # ~grub files

my $docroot = $http_root;
my $doc_pkg = "LANforgeDocs-$::lfver.tar.gz";
my $userdocs = "$docroot/LANforgeDocs-$::lfver";

#Check if files are accessible before commencing so that user can leave install until completed.
if (!$force_web) {
    my $rd_errs = 0;
    if ($do_local_install) {
        if ($do_firmware) {
            if ($lfrls >= 5212) {
                file_readable("$source_dir/$fw_board", "--source_dir --do_firmware") || $rd_errs++;
                file_readable("$source_dir/$fw_ct_wmi", "--source_dir --do_firmware") || $rd_errs++;
                if ($lfrls >= 5303) {
                    file_readable("$source_dir/$fw_ct_htt", "--source_dir --do_firmware") || $rd_errs++;
                }
                if ($lfrls >= 5304) {
                    file_readable("$source_dir/$fw_ath10k_stock", "--source_dir --do_firmware") || $rd_errs++;
                }
                if ($lfrls == 5304) {
                    file_readable("$source_dir/$fw_ct_htt5_4_old", "--source_dir --do_firmware") || $rd_errs++;
                }
                if ($lfrls >= 5305) {
                    file_readable("$source_dir/$fw_ct_htt2_9887", "--source_dir --do_firmware") || $rd_errs++;
                    file_readable("$source_dir/$fw_ct_htt5_4_9888", "--source_dir --do_firmware") || $rd_errs++;
                    file_readable("$source_dir/$fw_ct_htt5_4_9984", "--source_dir --do_firmware") || $rd_errs++;
                    file_readable("$source_dir/$fw_ct_htt5_4_new", "--source_dir --do_firmware") || $rd_errs++;
                }
                if ($lfrls >= 5308) {
                    file_readable("$source_dir/$fw_ct_htt5_4_9984b", "--source_dir --do_firmware") || $rd_errs++;
                }
                if ($lfrls < 5304) {
                    file_readable("$source_dir/$reg_ct", "--source_dir --do_firmware") || $rd_errs++;
                }
                file_readable("$source_dir/$fw_upstream", "--source_dir --do_firmware") || $rd_errs++;
            }
        }
        if ($do_kern) {
            file_readable("$source_dir/$kern", "--source_dir --do_kern") || $rd_errs++;
        }
        if ($do_lanforge) {
            file_readable("$source_dir/$lfserver", "--source_dir --do_lanforge") || $rd_errs++;
            if (!$skip_gui) {
                file_readable("$source_dir/$lfgui", "--source_dir --do_lanforge") || $rd_errs++;
            }
            if (!$skip_xorp) {
                file_readable("$source_dir/$xorp", "--source_dir --do_lanforge") || $rd_errs++;
            }
            if ($interop ne "") {
                file_readable("$source_dir/$interop", "--source_dir --do_lanforge") || $rd_errs++;
            }
        }
        if ($::create_install_bundle || $do_http) {
            file_readable("$source_dir/$doc_pkg", "--source_dir --do_http") || $rd_errs++;
            file_readable("$source_dir/LANforge-GUI-${lfver}-Installer.exe", "--source_dir --do_http") || $rd_errs++;
            file_readable("$source_dir/LANforgeGUI_${lfver}-x64-Installer.exe", "--source_dir --do_http") || $rd_errs++;
            file_readable("$source_dir/LANforgeGUI_${lfver}_Linux64.tar.bz2", "--source_dir --do_http") || $rd_errs++;
            file_readable("$source_dir/LANforgeGUI_${lfver}_Linux.tar.bz2", "--source_dir --do_http") || $rd_errs++;
            file_readable("$source_dir/LANforgeGUI_${lfver}_MacOS.dmg.bz2", "--source_dir --do_http") || $rd_errs++;
        }
        if ($rd_errs > 0) {
            die("\nUnable to read $rd_errs files from $source_dir\n\n");
        }
    } # ~ do_local_install
    elsif (-d "$d2dir") {
        if ($do_firmware) {
            if ($lfrls >= 5212) {
                file_readable("$d2dir/$fw_board", "$d2dir --do_firmware") || $rd_errs++;
                file_readable("$d2dir/$fw_ct_wmi", "$d2dir --do_firmware") || $rd_errs++;
                if ($lfrls >= 5303) {
                    file_readable("$d2dir/$fw_ct_htt", "$d2dir --do_firmware") || $rd_errs++;
                }
                if ($lfrls >= 5304) {
                    file_readable("$d2dir/$fw_ath10k_stock", "--source_dir --do_firmware") || $rd_errs++;
                }
                if ($lfrls == 5304) {
                    file_readable("$d2dir/$fw_ct_htt5_4_old", "--source_dir --do_firmware") || $rd_errs++;
                }
                if ($lfrls >= 5305) {
                    file_readable("$d2dir/$fw_ct_htt2_9887", "--source_dir --do_firmware") || $rd_errs++;
                    file_readable("$d2dir/$fw_ct_htt5_4_9888", "--source_dir --do_firmware") || $rd_errs++;
                    file_readable("$d2dir/$fw_ct_htt5_4_9984", "--source_dir --do_firmware") || $rd_errs++;
                    file_readable("$d2dir/$fw_ct_htt5_4_new", "--source_dir --do_firmware") || $rd_errs++;
                }

                if ($lfrls >= 5308) {
                    file_readable("$d2dir/$fw_ct_htt5_4_9984b", "--source_dir --do_firmware") || $rd_errs++;
                }

                if ($lfrls < 5304) {
                    file_readable("$d2dir/$reg_ct", "$d2dir --do_firmware") || $rd_errs++;
                }
                file_readable("$d2dir/$fw_upstream", "$d2dir --do_firmware") || $rd_errs++;
            }
        }
        if ($do_kern) {
            file_readable("$d2dir/$kern", "$d2dir --do_kern") || $rd_errs++;
        }
        if ($do_lanforge) {
            file_readable("$d2dir/$lfserver", "$d2dir --do_lanforge") || $rd_errs++;
            if (!$skip_gui) {
                file_readable("$d2dir/$lfgui", "$d2dir --do_lanforge") || $rd_errs++;
            }
            if (!$skip_xorp) {
                file_readable("$d2dir/$xorp", "$d2dir --do_lanforge") || $rd_errs++;
            }
            if ($interop ne "") {
                file_readable("$d2dir/$interop", "$d2dir --do_lanforge") || $rd_errs++;
            }
        }
        if ($do_http) {
            file_readable("$d2dir/$doc_pkg", "--do_http") || $rd_errs++;
            file_readable("$d2dir/LANforge-GUI-${lfver}-Installer.exe", "--source_dir --do_http") || $rd_errs++;
            file_readable("$d2dir/LANforgeGUI_${lfver}-x64-Installer.exe", "--source_dir --do_http") || $rd_errs++;
            file_readable("$d2dir/LANforgeGUI_${lfver}_Linux64.tar.bz2", "--source_dir --do_http") || $rd_errs++;
            file_readable("$d2dir/LANforgeGUI_${lfver}_Linux.tar.bz2", "--source_dir --do_http") || $rd_errs++;
            file_readable("$d2dir/LANforgeGUI_${lfver}_MacOS.dmg.bz2", "--source_dir --do_http") || $rd_errs++;
        }
        if ($rd_errs > 0) {
            die("\nUnable to read $rd_errs files from $d2dir\n\n");
        }
    }
    elsif (!$do_kern && !$do_elevator && !$do_iommu) {
        if (!file_readable("$d2dir", "Enabling --force_web")) {
            warning("#  Enabling --force_web\n");
            $force_web = 1;
        }
    }
}

# $print_only should imply show_urls=1 by here

my $internet_available = 0;
my ($download_hostname) = $download_host =~ qr{^https?://([^/]+)};
if ((!$::create_install_bundle)
    && $force_web
    && ($show_urls || $do_kern || $do_firmware || $do_lanforge || $do_http))
{
    my $web_check = 1;
    if ($print_only) {
        note("\n# lf_kinstall offline install will need file listed below.\n\n");
    }
    else {
        print("# Checking for access to Candela website: $curl -sv -u $user $curl_ssl --head $url_base/\n");
        my $host = "host";
        if ($is_openwrt) {
            $host = "nslookup";
        }
        if (system("$host $download_hostname") == 0) {
            $internet_available++;
        }
        else {
            note(" * Unable to resolve [$download_hostname]\n");
        }
        if (system("$host www.candelatech.com") == 0) {
            $internet_available++;
        }
        else {
            note(" * Unable to resolve www.candelatech.com\n");
        }
        if ($internet_available == 0) {
            die(" * Unable to resolve $www. Please check DNS settings.\n");
        }
        my $rh_req = newHeadRequest("$url_base/");
        webget($rh_req);
        if ($rh_req->{"status"} =~ / 200 /) {
            $web_check = 0;
        }
    }
    if (($web_check == 0) || $show_urls) {
        my @failed_urls = ();
        if ($do_kern || $show_urls) {
            if ($print_w32) {
                note("Create a Windows .bat file to help download all items for update.\n");
                note("--------------------------8<------------------------\n");
                note("${at}echo off\n");
                note("TITLE Download LANforge $::lfver\n");
                note("SET ${Q}GET=%USERPROFILE%${B}Downloads${B}$w32get${Q}\n");
                note("SET ${Q}OPTS=-L -O -C - -u guest:guest${Q}\n");
                note("IF NOT EXIST $w32get (\n");
                note("  ${at}echo Download $w32get from $www/downloads/$w32get\n");
                note("  ${at}echo If using PowerShell, try this:\n");
                note("  ${at}echo     Invoke-WebRequest -Uri '$www/downloads/$w32get' -OutFile ${Q}${d}env:USERPROFILE${B}Downloads${B}$w32get${Q}\n");
                note("  EXIT /B 1");
                note(")\n");
                note("cd %USERPROFILE%${B}Downloads\n");
                note("IF NOT EXIST $::lfver MKDIR $::lfver\n");
                note("cd $::lfver\n");
                check_show_url("$www/downloads/putty-0.70-installer.msi", \@failed_urls);
            }
            if ($skip_kern!=1) {
                check_show_url("$url_base/$kern", \@failed_urls);
            }
        }
        if ($do_firmware || $show_urls) {
            if ($lfrls >= 5212) {
                check_show_url("$url_base/$fw_board", \@failed_urls);
                check_show_url("$url_base/$fw_ct_wmi", \@failed_urls);
                if ($lfrls >= 5303) {
                    check_show_url("$url_base/$fw_ct_htt", \@failed_urls);
                }
                if ($lfrls == 5304) {
                    check_show_url("$url_base/$fw_ct_htt5_4_old", \@failed_urls);
                }
                if ($lfrls >= 5304) {
                    check_show_url("$url_base/$fw_ath10k_stock", \@failed_urls);
                }
                if ($lfrls >= 5305) {
                    check_show_url("$url_base/$fw_ct_htt2_9887", \@failed_urls);
                    check_show_url("$url_base/$fw_ct_htt5_4_9888", \@failed_urls);
                    check_show_url("$url_base/$fw_ct_htt5_4_9984", \@failed_urls);
                    check_show_url("$url_base/$fw_ct_htt5_4_new", \@failed_urls);
                }
                if ($lfrls >= 5308) {
                    check_show_url("$url_base/$fw_ct_htt5_4_9984b", \@failed_urls);
                }
                if ($lfrls < 5304) {
                    check_show_url("$url_base/$reg_ct", \@failed_urls);
                }
                check_show_url("$url_base/$fw_upstream", \@failed_urls);
            }
        }
        if ($do_lanforge || $show_urls) {
            check_show_url("$url_base/$lfserver", \@failed_urls);
            if (!$skip_gui) {
                check_show_url("$url_base/$lfgui", \@failed_urls);
            }
            if (($show_urls || !$skip_xorp) && $xorp ne "") {
                check_show_url("$url_base/$xorp", \@failed_urls);
            }
            if ($interop ne "") {
                check_show_url("$url_base/$interop", \@failed_urls);
            }
        }
        if ($do_http || $show_urls) {
            check_show_url("$url_base/$doc_pkg", \@failed_urls);
            check_show_url("$url_base/LANforge-GUI-${lfver}-Installer.exe", \@failed_urls);
            check_show_url("$url_base/LANforgeGUI_${lfver}-x64-Installer.exe", \@failed_urls);
            check_show_url("$url_base/LANforgeGUI_${lfver}_Linux64.tar.bz2", \@failed_urls);
            check_show_url("$url_base/LANforgeGUI_${lfver}_Linux.tar.bz2", \@failed_urls);
            check_show_url("$url_base/LANforgeGUI_${lfver}_MacOS.dmg.bz2", \@failed_urls);
            if ($lfrls > 5405) {
                check_show_url("$url_base/LANforge-Server-${lfver}-Installer.exe", \@failed_urls);
                check_show_url("$url_base/LANforge-Server-${lfver}-upgrade.zip", \@failed_urls);
                check_show_url("$url_base/interop-${lfver}.tar.gz", \@failed_urls);
            }
        }
        if ($show_urls) {
            exit(0);
        }
        if (@failed_urls > 0) {
            foreach (@failed_urls) {
                note("# Could not find: $_\n");
            }
            die("\n\nUnable to read " . @failed_urls . " files from $url_base\n\n");
        }
        else {
            note("# Found all files on URL: $url_base\n");
            sleep(3) if ($::debug_on);
        }
    }
    else {
        die("\n\nERROR:  Unable to access URL (force_web), url-base: $url_base\n\n");
    }
} # ~web check and or show urls to download

if (!($is_arm || $is_macos)) {
    my $c = 0;
    while (system("ps -aef | grep $burner | grep -v grep > /dev/null 2>&1") == 0) {
        system("pkill -9 burnP");
        if ($c > 50) {
            die("\n\nFailed to kill burnPX $c times, exiting script...\n\n");
        }
        $c++;
    }
}
if ($debug_on) {
    print "do_c_b:$do_cpu_burn, skip_c_b:$skip_cpu_burn, do_d_t:$do_disk_test, skip_d_t:$skip_disk_test\n";
    print "force_web:$force_web, dl_only:$download_only, is_arm:$is_arm, c_i_b:$::create_install_bundle\n";
    sleep(3);
}

if (($skip_cpu_burn >= $do_cpu_burn) || $download_only || $is_arm || $::create_install_bundle) {
    note("# skipping cpu_burn\n");
}
elsif ($do_cpu_burn) {
    my $did_cpuburn = "$::home/did_cpuburn";
    if (-f "$did_cpuburn" && ($do_cpu_burn == 1)) {
        note("# Skipping cpu-burn, already ran it once on this system.\n");
    }
    else {
        note("# Doing cpu_burn\n");
        if (system("which $burner") != 0) {
            do_cmd("$yum -y install $burner");
        }
        if ($do_cpu_burn >= 10) {
            $burnt = $do_cpu_burn;
        }
        if ($burner =~ /^stress-ng/) {
            do_cmd("stress-ng -c 0 --ignite-cpu -t $burnt", 1); # just does cpu
        }
        system("touch $did_cpuburn");
    }
}
else {
    note("# skipped cpu burn\n");
}

if ($skip_disk_test || $download_only || $is_arm || $is_macos || $::create_install_bundle) {
    note("# skipping disk test\n");
}
elsif (((0 + $do_disk_test) - (0 + $skip_disk_test)) > 0) {
    my $did_disktest = "$::home/did_disktest";
    print "Checking for marker [$did_disktest]...\n";
    if (-f "$did_disktest" && ($do_disk_test == 1)) {
        note("# Skipping disk test: [$did_disktest]: found.\n");
        sleep(3);
    }
    elsif ((($staged_in_candela == 0) && ($do_disk_test == 1))
        || (!(-f "$did_disktest") && ($do_disk_test > 0))
        || ($do_disk_test > 1)) {
        note("# Doing disk test...\n");
        my %parts = ();
        my @pvs_matches = ();
        my @pvs_lines = `/usr/sbin/pvs`;
        if (@pvs_lines > 0) {
            note("# using lvs...\n");
            @pvs_matches = grep {m|/dev/sd.. |} @pvs_lines;
            if (@pvs_matches > 0) {
                for my $line (@pvs_matches) {
                    my ($name) = $line =~ m|/dev/(sd[^ ]+) |;
                    $parts{$name} = "/dev/$name";
                }
            }
        }
        # actually using LVM makes this harder to parse
        my $lsblk_cmd = "/usr/bin/lsblk -l -o NAME,MOUNTPOINT,TYPE,PARTTYPE,LABEL";
        my $lsblk_lvm_cmd = "/usr/bin/lsblk -l -o NAME,MOUNTPOINT,TYPE,LABEL | sort | uniq";
        my $lsblk_cmd_ok = (@pvs_matches < 1) ? $lsblk_cmd : $lsblk_lvm_cmd;
        print "Using cmd: $lsblk_cmd_ok\n" if ($::debug_on);
        my @lsblk_lines = `$lsblk_cmd_ok`;
        chomp @lsblk_lines;
        if ($debug_on) {
            print Dumper(\@lsblk_lines);
        }
        for my $line (@lsblk_lines) {
            print "LSBLKLINE: $line\n" if ($::debug_on);
            next if ($line =~ /^\s*$/);
            next if ($line =~ /^NAME/);
            next if ($line =~ /disk\s*$/);
            next if ($line =~ /lvm\s*$/);
            next if ($line =~ /SWAP\s*$/);
            next if ($line !~ /\s+(BOOT|ROOT|HOME)\s*$/);
            next if ($line =~ /^sd\w+\s+part\s+[-a-fx0-9]{3,}+\s*$/);

            my ($l_name, $l_mnt, $l_type, $l_label, $l_parttype);
            if (@pvs_matches < 1) {
                print "No pvs_matches\n" if ($::debug_on);
                if ($line =~ /(BOOT|ROOT|HOME)/) {
                    ($l_name, $l_mnt, $l_type, $l_parttype, $l_label) =
                        $line =~ /([vs]d\w+)\s+(\S+)\s+(part|disk|rom)\s+(\S+)\s+(BOOT|ROOT|HOME)$/;
                }
                if ($::debug_on) {
                    print "LL: $line\n";
                    print "($l_name) ($l_mnt) ($l_type) ($l_parttype) ($l_label)\n";
                }
                $l_mnt = "swap" if (($l_mnt =~ /^\s+$/) || ($l_parttype eq "0x82"));
            }
            else {
                ($l_name, $l_mnt, $l_type, $l_label) = split(/\s+/, $line);
                if ($::debug_on) {
                    print "LLL: $line\n";
                    print "($l_name) ($l_mnt) ($l_type)  ($l_label)\n" if ($::debug_on);
                }
            }
            next if (!(defined $l_name));
            #$l_mnt = "swap" if (($l_mnt =~ /^\s+$/) || ($l_parttype eq "0x82"));
            $parts{$l_mnt} = $l_name;
        }
        if ($::debug_on) {
            print Dumper(\%parts);
            sleep 3;
        }
        my %dumptrucks = ();
        my $part;
        my %devs_to_read = ();
        while (my ($l_mnt, $l_part) = each %parts) {
            print "DEVS from ($l_mnt)($l_part)\n" if ($::debug_on);
            if ($l_mnt =~ /^sd[abcdefghijkl]/) {
                my ($dev) = $l_mnt =~ /^(sd[abcdefghijkl])/;
                $devs_to_read{$dev} = 0 unless (exists $devs_to_read{$dev});
                $devs_to_read{$dev}++;
            }
            if ($l_part =~ m|/?sd[abcdefghijkl]\d?$|) {
                my ($dev) = $l_part =~ /(sd[abcdefghijkl])/;
                $devs_to_read{$dev} = 0 unless (exists $devs_to_read{$dev});
                $devs_to_read{$dev}++;
            }
        }
        if ($::debug_on) {
            print Dumper(\%devs_to_read);
            sleep 3;
        }
        do_cmd("/usr/bin/dd bs=1M count=1 status=none iflag=fullblock if=/dev/urandom "
            . "| base64 "
            . "| dd bs=1M count=1 iflag=fullblock > /tmp/skiploader", 1);
        while (my ($l_mnt, $l_part) = each %parts) {
            print "l_mnt $l_mnt, l_part $l_part\n" if ($::debug_on);
            next if (($l_mnt eq "swap") or ($l_mnt eq '[SWAP]'));
            my $par_path = '';
            if ($l_mnt =~ m|/boot\s*$|) {
                $par_path = '/boot';
            }
            elsif ($l_mnt =~ m|/\s*$|) {
                $par_path = '/';
                $part = $l_part;
            }
            elsif ($l_mnt =~ m|/home\s*$|) {
                $par_path = '/home';
                $part = $l_part;
            }
            else {
                print "Skipping $l_mnt...";
                next;
            }
            if ($par_path eq "/") {
                $dumptrucks{$par_path} = "/DUMPTRUCK";
            }
            else {
                $dumptrucks{$par_path} = "$par_path/DUMPTRUCK";
            }
            `touch $dumptrucks{$par_path}`;
            #print "df -BM --output=avail $par_path | tail -1\n";
            my $mfree = `df -BM --output=avail $par_path | tail -1`;
            ($mfree) = $mfree =~ /(\d+)M$/;
            print "$par_path free: $mfree\n";
            note("# filling $par_path\n");
            my $i = 0;
            if (!-f "/tmp/skip-skiploader") {
                while ($i < $mfree) {
                    `nice cat /tmp/skiploader >> $dumptrucks{$par_path} 2>/dev/null`;
                    `sleep 0.01 &>/dev/null`;
                    $i++;
                    print "\r" . $ROT[ $i % $#ROT ];
                    if (0 == ($i % 64)) {
                        my $prog = `df -BM --output=avail $par_path | tail -1`;
                        chomp $prog;
                        print " $prog";
                        `sleep 0.1`;
                    }
                }
            }
            else {
                print "I see /tmp/skip-skiploader, will not fill dumptrucks\n";
                sleep 4 if ($::debug_on);
            }
            `sync`;
            note("rm $dumptrucks{$par_path} \n");
            unlink($dumptrucks{$par_path});
        } #~while fill each partition

        for my $dev (keys %devs_to_read) {
            note("reading $dev..."); # todo: this should be done per sdX, not per root file system
            if ($dev =~ m|/dev/|) {
                ($dev) = $dev =~ m|/dev/(\S+)$|;
            }
            do_cmd("dd status=progress bs=16M iflag=fullblock if=/dev/$dev | $md5sum", 1);
        }
        `touch $did_disktest`;
    }
    else {
        note("# skipping disk test, saw $did_disktest\n");
    }
} #~if do_disk_test

if ($do_yum_update && !($is_offline|| $skip_yum_all || $download_only || $::create_install_bundle)) {
    note("# checking for updates");
    if ($is_deb_based) {
        if ($do_interop) {
            $ENV{LD_LIBRARY_PATH} = "";
        }
        print(" ===== ===== ===== ===== ===== ===== ===== ===== =====\n");
        note("# Updating system with:  $apt\n");
         # to repair possibly interrupted dpkg process
        do_cmd("dpkg --configure -a");
        do_cmd("$apt update");
        do_cmd("$apt install -y $other_deb_pkgs");
        do_cmd("$apt dist-upgrade -y");
        do_cmd("$apt -f install");
        do_cmd("$apt autoremove -y");
        print_slow(" ===== ===== ===== ===== ===== ===== ===== ===== =====\n") if ($debug_on);
    }
    else {
        if (!$use_yum_cache) {
            note("# cleaning package cache...\n");
            do_cmd("$yum clean packages");
            note("# rebuilding package cache...\n");
            do_cmd("$yum makecache -v");
        }
        # disable deltarpms is useful when proxies misbehave and also
        # when units with slow storage (ct521a etc) are involved. Disk activity
        # involved in delta-rpm has not kept pace with average network speed
        # of most customers.
        my @excludes = (
            "kernel",
            "kernel-core",
            # "kernel-devel",
            "kernel-headers",
            "kernel-modules",
            "kernel-modules-extra",
            # "kernel-tools-libs"
        );
        if (@excludes > 0) {
            $yum_xclude .= "-x " . join(" -x ", @excludes);
        }
        if ($::osveri >= 24) {
            note("# disabling deltarpms for dnf updates\n");
            my @output =`dnf updateinfo -v | grep 'plugins:'`;
            chomp(@output);
            my @matches = grep { /deltarpm/ } @output;
            print Dumper(["dnf updateinfo:", \@output]) if ($debug_on);
            if (@matches > 0) {
                $yum_xclude .= " --disableplugin deltarpms";
            }
        }
        # dnf does not have --skip-broken ? maybe it does
        if (@pending_yum_updates < 1) {
            print_slow( "No non-kernel package need updating....") if ($debug_on);
        }
        else {
            print "Computing updates that are not kernel related...\n";
            my $pkg_list = join(" ", @pending_yum_updates);
            my $updatecmd = "$yum update -y $skip_broken $yum_xclude --skip-broken $pkg_list";
            #print_slow( "==============================================================================",
            #            "Updating system with: $updatecmd\n",
            #            "==============================================================================")
            #    if ($debug_on);
            do_cmd($updatecmd, 1);
        }
    }
} # ~if we should attempt yum or apt-get update
elsif (!$::create_install_bundle) {
    print_slow(" ===== ===== ===== Skipped updates:",
                "       do_yum_update:$do_yum_update",
                "       skip_yum_all:$skip_yum_all",
                "       skip_yum_update:$skip_yum_update",
                "       print_only:$print_only",
                "       show_urls:$show_urls",
                "       force_web:$force_web",
                "       download_only:$download_only",
                "       create_install_bundle:$::create_install_bundle",
                "===== ===== ===== ===== =====") if ($debug_on);
}
my $gver = "";
my @gnome_shell = `which -a gnome-shell 2>/dev/null`;
chomp @gnome_shell;
my $gv1 = 0;
my $gv2 = 0;
my $gv3 = 0;

if (@gnome_shell > 0) {
    $gver = `gnome-shell --version`;
    chomp($gver);
    if ($gver =~ /.*\s+(\d+)\.(\d+)\.(\d+).*/) {
        $gv1 = $1;
        $gv2 = $2;
        $gv3 = $3;
    }
}

my $cinnamon_repo = "/etc/yum.repos.d/fedora-cinnamon.repo";
my $do_cinnamon = 0;

# Make sure we clean out old cinnamon repo file..it could be corrupt.
if ($is_fedora && -f $cinnamon_repo && !($download_only || $::create_install_bundle)) {
    unlink($cinnamon_repo);
}
if (!$create_install_bundle && $do_interop) {
    do_cmd("echo 1 > $::home/LF_INTEROP");
}
if ($do_pkgs && !($is_offline || $skip_yum_all || $download_only || $::create_install_bundle)) {
    my @uninstalled_pkgs = ();
    if ($is_macos) {
        do_cmd("port install md5sha1sum", 1);
    }
    elsif ($is_deb_based) {
        print("----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- \n");
        note("# Installing deb packages.\n");
        sleep(3);
        if ($staged_in_candela) {
            if ($ctproxy ne "") {
                do_cmd(qq(echo 'Apt::http::Proxy "http://$ctproxy:3128";' >> /etc/apt/apt.conf));
            }
            # TODO:  else clean up the old proxy config???
        }
        if ($do_interop) {
            $ENV{LD_LIBRARY_PATH} = "";
        }
        do_cmd("$apt update");
        # Apt-File develops an index of package names optimized for searching
        # https://wiki.debian.org/apt-file
        do_cmd("$apt install apt-file");
        do_cmd("apt-file update");
        do_cmd("$apt -y install $other_deb_pkgs");
        do_cmd("$apt -f install");
        do_cmd("$apt autoremove -y");
        do_cmd("$apt dist-upgrade -y");
        do_cmd("$apt -f install");
        do_cmd("$apt autoremove -y");

        if (!$is_arm) {
            # fixup wireshark-gtk using etc/alternatives
            my @list = `update-alternatives --list wireshark 2>/dev/null`;
            if (-f "/usr/bin/wireshark-gtk" && -f "/usr/bin/wireshark" && !-l "/usr/bin/wireshark") {
                note("# installing wireshark");
                do_cmd("dpkg -r --force-depends wireshark-qt", 1);
            }
            if ($? != 0 || @list < 1) {
                if (!-f "/usr/bin/wireshark" && -x "/usr/bin/wireshark-gtk") {
                    note(" * updating wirehark-gtk to wireshark using update-alternatives\n");
                    do_cmd("update-alternatives --install /usr/bin/wireshark wireshark /usr/bin/wireshark-gtk 1", 1);
                }
            }
        }
    } # ~deb-based
    else {
        note("# Installing yum groups: $yum_groups\n");
        if ($is_redhat && !$use_yum_cache) {
            note("# refreshing Red Hat subscription manager....\n");
            do_cmd("subscription-manager refresh");
            note("# updating $yum cache....\n");
            do_cmd("$yum makecache -v");
        }
        note("# Installing yum packages: $yum_groups...\n");
        do_cmd("$yum groupinstall $skip_broken -y $yum_groups");

        for my $pkg (@yum_pkgs) {
            system("rpm -q $pkg >/dev/null");
            if ($? != 0) {
                push(@uninstalled_pkgs, $pkg);
                print "+";
            }
            else {
                print "-";
            }
        }
        print "\n";
        if ($::debug_on && (@uninstalled_pkgs > 0) && (join(' ', @uninstalled_pkgs) !~ /^\s*$/)) {
            print "== == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == \n";
            print Dumper([ "Missing required $arch pkgs", @uninstalled_pkgs ]);
            print "== == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == \n";
            sleep(5);
        }
        # the pipewire exclude might be a requirement for F36, if this logic starts
        # to look old, it can prolly be removed.
        my $pw_exclude = ""; # ($::osveri > 30) ? "-x 'pipewire*'" : "";
        my $yum_pkgs_str = join(" ", @uninstalled_pkgs);
        if ($yum_pkgs_str !~ /^\s*$/) {
            note("Installing $yum_pkgs_str\n");
            do_cmd("$yum install $skip_broken $::yum_xclude $pw_exclude -y $yum_pkgs_str");
            sleep(2);
        }

        note("Checking for skipped packages: ");
        my @skipped_pkgs = ();
        my $num_tries = 3;
        for ($num_tries = 3; $num_tries >= 0; $num_tries--) {
            @skipped_pkgs = ();
            for my $pkg (@uninstalled_pkgs) {
                next if ($pkg =~ /^\s*$/);
                if (system("rpm -qi $pkg >/dev/null") != 0) {
                    push(@skipped_pkgs, $pkg);
                    print "!";
                }
                else {
                    print ".";
                }
            }
            # check that we haven't somehow compiled a bunch of blank strings into the list
            if ((@skipped_pkgs > 0) && (join(' ', @skipped_pkgs) !~ /^\s*$/)) {
                print(Dumper([ "Try $num_tries: Skipped required pkgs:", @skipped_pkgs ]));
                do_cmd("$yum install $skip_broken $::yum_xclude $pw_exclude -y " . join(' ', @skipped_pkgs));
                sleep 3;
            }
        } # tries
        if ((@skipped_pkgs > 0)  && (join(' ', @skipped_pkgs) !~ /^\s*$/)){
            print(Dumper([ "Skipped required pkgs:", @skipped_pkgs ]));
            die("Required packages were skipped.")
        }

        # WARN only packages; first check that we haven't already installed them
        @uninstalled_pkgs = ();
        # uninstall any pipewire for now
        if (($::osveri > 30) && ($::remove_pipewire == 1)) {
            my $pw_pkg_cnt = 0 + `rpm -qa 'pipewire*' |wc -l`;
            if ($pw_pkg_cnt > 0) {
                warn("# removing pipewire packages ");
                do_cmd("rpm -qa 'pipewire*' | xargs rpm --nodeps -ehv", 1);
            }
        }
        # check firefox
        my $ff_lib = "/usr/lib64/firefox";
        if (-d $ff_lib) {
            my $err_count = 0;
            my @ff_files = glob(qq[$ff_lib/{firefox,lib}*]);
            for my $fname (@ff_files) {
                system("/usr/bin/ldd $fname | grep 'libpipewire.* not found'");
                if ($? == 0) {
                    $err_count++;
                    err("a* Firefox library $fname missing dependencies\n");
                }
            }
            if ($err_count > 0) {
                warning("Re-installing firefox...\n");
                do_cmd("$yum reinstall -y firefox");
            }
        }

        for my $pkg (@yum_pkgs_warn) {
            system("rpm -q $pkg >/dev/null");
            if ($? != 0) {
                push(@uninstalled_pkgs, $pkg);
                print "+";
            }
            else {
                print "-";
            }
        }
        print "\n";
        note("Searching for optional packages...\n");
        my @found_optional = ();
        my $cpu_arch = `uname -p`;
        chomp($cpu_arch);
        for my $pkg (@uninstalled_pkgs) {
            #my @possible_matches = `$yum -q search all $pkg 2>/dev/null | grep -v ==`;
            #print qq(XXXXXXXXX $yum -q whatprovides $pkg 2>/dev/null | grep -E "\\.(noarch|$cpu_arch)" XXXXXXXXXXXXXXXXXX\n);
            my @possible_matches = `$yum $yum_xclude -q whatprovides $pkg 2>/dev/null`;
            #| grep -E "\\.(noarch|$cpu_arch)"`;
            chomp(@possible_matches);
            my @likely_matches = grep {/^\S+\.(noarch|$cpu_arch)\s*:/} @possible_matches;
            #print "Whatprovides:  ", Dumper(\@possible_matches);
            #print "Likely matches:", Dumper(\@likely_matches);
            #sleep 5;
            if (@likely_matches > 0) {
                my ($pblpkg) = $possible_matches[0] =~ /^(\S+)\s*:/;
                if ((defined $pblpkg) && ("" ne $pblpkg)) {
                    print(" +$pkg provided by $pblpkg\n");
                    push(@found_optional, $pblpkg);
                }
            }
            else {
                #@possible_matches  = `$yum -q whatprovides $pkg 2>/dev/null | head -1`;
                @possible_matches = `$yum $yum_xclude -q search all $pkg 2>/dev/null | grep -v ==`;
                chomp(@possible_matches);
                if (@possible_matches > 0) {
                    #print "Search all: ",Dumper(\@possible_matches);
                    print(" +$pkg");
                    push(@found_optional, $pkg);
                }
                else {
                    print(" -$pkg");
                }
            }
        } # ~for uninstalled pkgs
        print "\n";
        if ($::debug_on) {
            print "== == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == \n";
            print Dumper([ "Selected missing optional $arch pkgs", @found_optional ]);
            print_slow("== == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == \n");
        }

        $yum_pkgs_str = join(" ", @found_optional);
        if ($yum_pkgs_str !~ /^\s*$/) {
            print("\nInstalling optional [$yum_pkgs_str]...\n");
            do_cmd("$yum -v install $skip_broken $::yum_xclude $pw_exclude -y $yum_pkgs_str", 1);
            sleep 2;
        }
        else {
            print "\n...no optional packages to install.\n";
        }
        note("Checking for skipped optional packages: ");
        @skipped_pkgs = ();
        for my $pkg (@found_optional) {
            system("rpm -q $pkg >/dev/null");
            if ($? != 0) {
                push(@skipped_pkgs, $pkg);
                print "+";
            }
            else {
                print ".";
            }
        }
        if ((@skipped_pkgs > 0) && (join(' ', @skipped_pkgs) !~ /^\s*$/)) {
            if ($::debug_on) {
                print(Dumper([ "\nSkipped optional pkgs:", @skipped_pkgs ]));
                warning("Optional packages were skipped.");
            }
        }

        if ($::osveri >= 29) {
            # think we have a dnf problem installing this many packages at once
            print "\n# Installing specific yum packages\n";
            for my $pkg (sort @force_yum_pkgs) {
                next if ($pkg =~ /^\s*$/);
                print("$yum install -y $skip_broken $pkg\n");
                do_cmd("$yum install -y $skip_broken $pkg");
                #sleep 1;
            }
        }

        if (($::osveri == 30) && ($xrdp_pkg ne "")) {
            system("rpm -qi xorgxrdp");
            my $rv = $?;
            # print("RETVAL $rv\n");

            # we want xorgxrdp-0.2.12-4.fc30 or later
            if (($rv >> 8) > 0) {
                err("* xorgxrdp appears to have not installed. Installing F30 advisory patches:\n");
                system("$yum upgrade --advisory=FEDORA-2019-0e9a61f3a6");
                $rv = $?;
                if (($rv >> 8) > 0) {
                    err("** Xorgxrdp advisory appears to have failed.");
                }
                else {
                    system("$yum install $skip_broken -y xorgxrdp");
                    $rv = $?;
                    if (($rv >> 8) > 0) {
                        err("** installation of Xorgxrdp appears to have failed.");
                    }
                }
            } # if xorgxrdp install failed
        }     # if checking xrdp

        if ($::debug_on) {
            print "== == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == \n";
            print Dumper([ "Selected missing optional i686 pkgs", @found_optional ]);
            print_slow("== == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == \n");
        }
        note("# installing i686 packages");
        if (@yum_i686_pkgs > 0) {
            @yum_i686_pkgs = ();
            my $ra_i6_pkgs = [ @yum_i686_pkgs ];
            for my $pkg (@$ra_i6_pkgs) {
                system("rpm -q $pkg >/dev/null");
                if ($? != 0) {
                    push(@yum_i686_pkgs, $pkg);
                    print "-";
                }
                else {
                    print ".";
                }
            }
            if (@yum_i686_pkgs > 0) {
                do_cmd("$yum install --skip-broken -x '*.x86_64' -y " . join(' ', @yum_i686_pkgs));
            }
        }
        note("Checking for i686 skipped packages: ");
        $num_tries = 3;
        @skipped_pkgs = ();
        for my $pkg (@yum_i686_pkgs) {
            system("rpm -q $pkg >/dev/null");
            if ($? != 0) {
                push(@skipped_pkgs, $pkg);
                print "+";
            }
            else {
                print ".";
            }
        }
        if (@skipped_pkgs) {
            for ($num_tries = 3; $num_tries >= 0; $num_tries--) {
                if (@yum_i686_pkgs) {
                    do_cmd("$yum install --skip-broken -x '*.x86_64' -y " . join(' ', @skipped_pkgs));
                }
                @skipped_pkgs = ();
                for my $pkg (@uninstalled_pkgs) {
                    if (system("rpm -qi $pkg >/dev/null") != 0) {
                        push(@skipped_pkgs, $pkg);
                        print "!";
                    }
                    else {
                        print ".";
                    }
                }
                if (@skipped_pkgs) {
                    print(Dumper([ "Try $num_tries: Skipped required pkgs:", @skipped_pkgs ]));
                    sleep 3;
                }
            } # tries
        }

        if (@skipped_pkgs) {
            print(Dumper([ "Skipped i686 required pkgs:", @skipped_pkgs ]));
            die("Required i686 packages were skipped.")
        }

        note("# installing wireshark....");

        my $wsbin = "/usr/bin/wireshark";
        my $wsgtkbin = "";
        if (-x "/sbin/wireshark-gtk") {
            $wsgtkbin = "/sbin/wireshark-gtk";
        }
        elsif (-x "/usr/sbin/wireshark-gtk") {
            $wsgtkbin = "/usr/sbin/wireshark-gtk";
        }
        if (!-x "/usr/bin/wireshark-gtk") {
            do_cmd("update-alternatives --remove wireshark /usr/bin/wireshark-gtk ||:", 1);
            do_cmd("update-alternatives --remove wireshark /usr/sbin/wireshark-gtk ||:", 1);
        }
        if (!-x "/usr/sbin/wireshark-gtk" && -l "/usr/sbin/wireshark-gtk") {
            unlink "/usr/sbin/wireshark-gtk";
        }
        if (!-x "/etc/alternatives/wireshark" && -l "/etc/alternatives/wireshark") {
            my $lnk = `readlink /etc/alternatives/wireshark`;
            unlink "/etc/alternatives/wireshark-gtk" if (-f "/etc/alternatives/wireshark-gtk");
            unlink "/etc/alternatives/wireshark" if (-f "/etc/alternatives/wireshark");
        }
        if (($wsgtkbin ne "") && (-f $wsgtkbin) && (-x $wsgtkbin)) {
            do_cmd("update-alternatives --install $wsbin wireshark $wsgtkbin 1", 1);
            do_cmd("update-alternatives --set wireshark $wsgtkbin", 1);
        }

        if ($::osveri > 19) {
            do_cmd("$yum install $skip_broken -y ipmitool", 1);
        }
        else {
            do_cmd("$yum install -y ipmitool", 1);
        }
        if ($::osveri > 19) {
            my @nfs_lines = `systemctl list-unit-files | awk '/^nfs-/{print \$1}'`;
            chomp @nfs_lines;
            my @units = split("\n", `systemctl list-unit-files`);
            my @s_units = grep {/^rpcbind.*/} @units;
            if (@s_units > 0) {
                mkdir("/run/rpcbind") unless (-d "/run/rpcbind/");
                do_cmd("systemctl restart rpcbind");
            }
            my @matches = grep {/nfs-utils/} @nfs_lines;
            if (@matches > 0) {
                # print Dumper(\@matches);
                do_cmd("systemctl restart nfs-utils", 1);
            }
            if (-f "/lib/systemd/system/nfs.service") {
                do_cmd("systemctl restart nfs.service", 1);
            }
            if (-f "/lib/systemd/system/nfs-server.service") {
                do_cmd("systemctl restart nfs-server.service", 1);
            }
            if (-f "/lib/systemd/system/nfs-client.service") {
                do_cmd("systemctl restart nfs-client.service", 1);
            }
            @s_units = grep {/^NetworkManager-wait-online/} @units;
            if (@s_units > 0) {
                do_cmd("systemctl disable NetworkManager-wait-online");
                do_cmd("systemctl mask NetworkManager-wait-online");
                unlink("/usr/lib/systemd/system/network-online.target.wants/NetworkManager-wait-online.service")
                    if (-f "/usr/lib/systemd/system/network-online.target.wants/NetworkManager-wait-online.service");
            }
            @s_units = grep {/^systemd-hostnamed/} @units;
            if (@s_units > 0) {
                do_cmd("systemctl disable systemd-hostnamed");
                do_cmd("systemctl mask systemd-hostnamed");
            }
            # Not sure if we need this, but maybe... --Ben
            #@s_units  = grep {/^systemd-resolved/} @units;
            #if (@s_units >0) {
            #  do_cmd("systemctl disable systemd-resolved");
            #  do_cmd("systemctl stop systemd-resolved");
            #  do_cmd("systemctl mask systemd-resolved");
            #}
            @s_units = grep {/pcscd.service/} @units;
            if (@s_units > 0) {
                do_cmd("systemctl stop pcscd.service");
                do_cmd("systemctl stop pcscd.socket");
                do_cmd("systemctl disable pcscd.service");
                do_cmd("systemctl disable pcscd.socket");
                do_cmd("systemctl mask pcscd.service");
            }
            do_cmd("systemctl daemon-reload");
        }

        if ($staged_in_candela) {
            if ($candela_yum_pkgs ne "") {
                note("# Installing yum packages for Candela staged systems.\n");
                do_cmd("$yum -y install $candela_yum_pkgs $skip_broken");
            }
            # we don't want evolution components
            my @hunks = split("\n", `rpm -qa evolution*`);
            if (@hunks > 0) {
                my $hunkstr = join(" ", @hunks);
                if ($hunkstr !~ /^\s*$/) {
                    note("# removing " . join(" ", @hunks) . "\n");
                    do_cmd("rpm -e --nodeps " . join(" ", @hunks), 1);
                }
            }
            # we definitely don't want PackageKit and gnome-software
            # definitely reports of cache file growing out of control
            if (-f "/var/cache/PackageKit") {
                note("# remove PackageKit and gnome-software\n");
                do_cmd("rpm -e --nodeps PackageKit gnome-software", 1);
                do_cmd("rm -rf /var/cache/PackageKit", 1);
            }
        }

        if ($do_install_mate) {
            if ($::osveri >= 27) {
                do_cmd("$yum -y install --best --allowerasing \@mate-desktop");
            }
            else {
                do_cmd("$yum -y install \@mate-desktop");
            }
            note("# And erase some gnome stuff.\n");
            $nextboot .= "$yum -y erase gnome-shell\n";
        }
        elsif (($do_gnome || !$mate_installed) && ($yum_gnome_pkgs !~ /^\s*$/)) {
            note("# Install some gnome specific packages...\n");
            do_cmd("$yum -y install $yum_gnome_pkgs");
        }

        if (($do_install_mate || $mate_installed) && ($yum_mate_pkgs !~ /^\s*$/)) {
            note("# Install some mate specific packages...\n");
            do_cmd("$yum -y install $yum_mate_pkgs");
        }

        # 2013-08-06: Appears leigh123linux repository no longer exists and with Cinnamon installed,
        #             moving desktop icons causes user to be logged out.
        #  Will use standard upstream cinnamon packages or nothing at all.
        # Install cinnamon (gnome-shell replacement) on F15 +
        if ($do_cinnamon) {
            note("# Installing Cinnamon Gnome-3 shell replacement.\n");
            # NOTE:  This will NOT abort if the commands fail, happens when offline.
            do_cmd("$yum -y install cinnamon", 1);
        }
        else {
            note("# Gnome shell version: $gv1.$gv2.$gv3.  Skipping \"cinnamon\" package.\n");
        }
        `rpm -qi network-manager-applet 2>/dev/null`;
        if ($? == 0) {
            do_cmd("rpm -e --nodeps network-manager-applet", 1);
        }
        do_cmd("which ksensors &>/dev/null && ($yum remove -y ksensors) ||:", 1);
        if ($is_fedora && $osveri >= 34) {
            do_cmd("rpm -e mate-power-manager ||:", 1);
        }
    }
    if ($do_only_pkgs) {
        note("All packages are installed. If this is a VM, you might want to poweroff and snapshot at this point.");
        exit(0);
    }
}
else {
    note("# Skipping extra yum packages.\n");
}

# TODO: if we have a blank getcap result on ping, we need to
# reinstall iputils; F24 problem
if (!($is_offline || $download_only || $::create_install_bundle)
    && (($os_string =~ /Fedora/) && ($::osveri > 21))
    && ($do_upgrade || $do_all || $do_lanforge)) {
    # my @lines = `getcap /usr/bin/ping`;
    do_cmd("chmod u+s /usr/bin/ping", 1);
}

if (!($download_only || $::create_install_bundle) && $do_xrandr) {
    # Fix up gdm start script too
    #  /etc/gdm/Init/Default
    my $gidf = "/etc/gdm/Init/Default";
    my $gid = `cat $gidf`;
    my $xrcmd = "xrandr --output LVDS1 --off";
    if (!($gid =~ /$xrcmd/)) {
        if ($gid =~ s/exit 0/$xrcmd\nexit 0/g) {
            note("# Adding $xrcmd to $gidf\n");
            open(GID, ">$gidf") or die "Couldn't open file: $gidf\n\n";
            print GID $gid;
            close GID
        }
    }

    my $fn = "$::home/.config/autostart/xrandr.desktop";
    if (-d "$::home") {
        create_lf_user() if ($::should_create_user);
        if (!-d "$::home/.config/autostart") {
            do_cmd("mkdir -p $::home/.config/autostart");
            do_cmd("chown lanforge:$user_group $::home/.config");
            do_cmd("chown lanforge:$user_group $::home/.config/autostart");
        }
        open(FILE, ">$fn") or die "Couldn't open file: $fn  $!\n\n";
        print FILE "[Desktop Entry]\n";
        print FILE "Type=Application\n";
        print FILE "Exec=xrandr --output LVDS1 --off\n";
        print FILE "Hidden=false\n";
        print FILE "X-GNOME-Autostart-enabled=true\n";
        print FILE "Name[en_US]=xrandr\n";
        print FILE "Name=xrandr\n";
        print FILE "Comment[en_US]=\n";
        print FILE "Comment=\n";

        close FILE;
        do_cmd("chown lanforge:$user_group $fn");
    }
    else {
        note("# ERROR: $::home does not exist.\nCannot create XRANDR work-around file: $fn\n");
        note("# Maybe you need to add the lanforge user?\n");
        exit(1);
    }
} # ~ if do xrandr

if (!($download_only || $::create_install_bundle) && ($do_selinux != -1)) {
    note("# disabling selinux");
    my $found_se = 0;
    my $selinux_cfg = "/etc/selinux/config";

    do_cmd("which setenforce && setenforce 0 ||true", 1);

    if (-f $selinux_cfg) {
        if ($do_selinux == 0) {
            note("# Updating $selinux_cfg to disable SELINUX.\n");
        }
        else {
            note("# Updating $selinux_cfg to enable SELINUX.\n");
        }

        my @cmds = `cat $selinux_cfg`;
        open(FILE, ">$selinux_cfg") or die "Couldn't open file: $selinux_cfg for writing: $!\n\n";
        for ($i = 0; $i < @cmds; $i++) {
            my $ln = $cmds[$i];
            chomp($ln);
            if ($ln =~ /^\s*SELINUX=.*/) {
                if ($do_selinux == 0) {
                    print FILE "SELINUX=disabled\n";
                }
                else {
                    print FILE "SELINUX=enabled\n";
                }
                $found_se = 1;
            }
            else {
                print FILE "$ln\n";
            }
        }
        if (!$found_se) {
            if ($do_selinux == 0) {
                print FILE "SELINUX=disabled\n";
            }
            else {
                print FILE "SELINUX=enabled\n";
            }
        }
        close FILE;
    }
    elsif (!$is_deb_based) {
        warning("# Could not find selinux config file: $selinux_cfg, hopefully it is just not installed.\n");
    }
} # ~change selinux

note("Checking do_lanforge: $do_lanforge do_kern: $do_kern skip_kern: $skip_kern do_firmware: $do_firmware create_install_bundle $::create_install_bundle\n");

if ($::create_install_bundle) {
    if (!(defined $tmp_dir) || ("" eq $tmp_dir)) {
        err("*** create_install_bundle will not work without setting tmp_dir");
        exit(1);
    }
}

sub configure_rabbitmq {
    my $fh;
    my $fname = "/etc/systemd/system/multi-user.target.wants/rabbitmq-server.service";
    if (-f $fname) {
        open($fh, "<", "$fname");
        my @lns = <$fh>;
        close $fh;
        if (open FILE, ">", $fname) {
            for my $ln (@lns) {
                chomp($ln);
                if ($ln =~ /^# Restart=/) {
                    print FILE "Restart=always\n";
                } elsif ($ln =~ /^# RestartSec=/) {
                    print FILE "RestartSec=3\n";
                }
                else {
                    print FILE "$ln\n";
                }
            }
            close(FILE);
        }
    }
}

sub grub_menuentry {
    die("grub_menuentry() unknown kver") if (!(defined $kver));

    my $local_ver = $kver;
    $local_ver = $_[0] if (@_ > 0);
    # print Dumper(["grub_menuentry", $local_ver, \@_]) if ($debug_on);
    return "Candela Technologies $local_ver COM1 ${ser_speed} 8n1" if ($do_serial);
    return "Candela Technologies $local_ver"
}
if (!(defined $cachebuster) || ($cachebuster eq "")) {
    $cachebuster = "?footime=" . time();
    print "Cachebuster now [$cachebuster]\n" if ($debug_on);
}
#my $v = ($debug_on) ? "-v" : "";
if ($do_lanforge || $do_kern || $do_firmware) {
    if (!$::create_install_bundle && $do_local_install) {
        note("Noticing local install\n");
        if ($do_lanforge) {
            if ($source_dir ne $::tmp_dir) {
                note("Copying LANforge server package\n");
                do_cmd("$cp $source_dir/$lfserver $::tmp_dir/");
                if ($interop ne "") {
                    do_cmd("$cp $source_dir/$interop $::tmp_dir/");
                }
                if (!$skip_gui) {
                    note("Copying LANforge GUI package\n");
                    do_cmd("$cp $source_dir/$lfgui $::tmp_dir/");
                }
                if (!$skip_xorp) {
                    note("Copying Xorp package\n");
                    do_cmd("$cp $source_dir/$xorp $::tmp_dir/");
                }

                # Fixup for F19 and LF 5.2.9
                if ($is_fedora && ($::osveri >= 19) &&
                    (($::lfver eq "5.2.8") || ($::lfver eq "5.2.9"))) {
                    if ($is_64) {
                        do_cmd("$cp $source_dir/libgnutls.so.26-x64-f17 $::tmp_dir/");
                        do_cmd("$cp $source_dir/libtasn1.so.3-x64-f17 $::tmp_dir/");
                    }
                    else {
                        do_cmd("$cp $source_dir/libgnutls.so.26-i32-f17 $::tmp_dir/");
                        do_cmd("$cp $source_dir/libtasn1.so.3-i32-f17 $::tmp_dir/");
                    }
                }
            }
            if ($do_kern) {
                note("Copying Kernel package\n");
                do_cmd("$cp $source_dir/$kern $::tmp_dir/");
            }
            if ($do_firmware) {
                note("Copying firmware packages\n");
                if ($lfrls >= 5212) {
                    do_cmd("$cp $source_dir/$fw_board $::tmp_dir/");
                    do_cmd("$cp $source_dir/$fw_ct_wmi $::tmp_dir/");
                    if ($lfrls >= 5303) {
                        do_cmd("$cp $source_dir/$fw_ct_htt $::tmp_dir/");
                    }
                    if ($lfrls == 5304) {
                        do_cmd("$cp $source_dir/$fw_ct_htt5_4_old $::tmp_dir/");
                    }
                    if ($lfrls >= 5304) {
                        do_cmd("$cp $source_dir/$fw_ath10k_stock $::tmp_dir/");
                    }
                    if ($lfrls >= 5305) {
                        do_cmd("$cp $source_dir/$fw_ct_htt2_9887 $::tmp_dir/");
                        do_cmd("$cp $source_dir/$fw_ct_htt5_4_9888 $::tmp_dir/");
                        do_cmd("$cp $source_dir/$fw_ct_htt5_4_9984 $::tmp_dir/");
                        do_cmd("$cp $source_dir/$fw_ct_htt5_4_new $::tmp_dir/");
                    }
                    if ($lfrls >= 5308) {
                        do_cmd("$cp $source_dir/$fw_ct_htt5_4_9984b $::tmp_dir/");
                    }
                    if ($lfrls < 5304) {
                        do_cmd("$cp $source_dir/$reg_ct $::tmp_dir/");
                    }
                    do_cmd("$cp $source_dir/$fw_upstream $::tmp_dir/");
                }
            }
        } # source-dir ~ tmp_dir
    }     # ~do_local_install
    elsif (!$::create_install_bundle && -d "$d2dir") {
        # Get from local NFS storage
        note("# Noticing mnt-d2 resources :3605\n");
        if ($do_lanforge) {
            note("# copying mnt-d2 lfserver\n");

            do_cmd("$cp $d2dir/$doc_pkg $::tmp_dir/");
            add_to_download_list($doc_pkg);

            do_cmd("$cp $d2dir/LANforge-GUI-${lfver}-Installer.exe $::tmp_dir/");
            add_to_download_list("LANforge-GUI-${lfver}-Installer.exe");

            do_cmd("$cp $d2dir/LANforgeGUI_${lfver}-x64-Installer.exe $::tmp_dir/");
            add_to_download_list("LANforgeGUI_${lfver}-x64-Installer.exe");

            do_cmd("$cp $d2dir/LANforgeGUI_${lfver}_Linux64.tar.bz2 $::tmp_dir/");
            add_to_download_list("LANforgeGUI_${lfver}_Linux64.tar.bz2");

            do_cmd("$cp $d2dir/LANforgeGUI_${lfver}_Linux.tar.bz2 $::tmp_dir/");
            add_to_download_list("LANforgeGUI_${lfver}_Linux.tar.bz2");

            do_cmd("$cp $d2dir/LANforgeGUI_${lfver}_MacOS.dmg.bz2 $::tmp_dir/");
            add_to_download_list("LANforgeGUI_${lfver}_MacOS.dmg.bz2");

            if ($lfrls > 5405) {
                do_cmd("$cp $d2dir/LANforge-Server-${lfver}-Installer.exe $::tmp_dir/");
                add_to_download_list("LANforge-Server-${lfver}-Installer.exe");

                do_cmd("$cp $d2dir/LANforge-Server-${lfver}-upgrade.zip $::tmp_dir/");
                add_to_download_list("LANforge-Server-${lfver}-upgrade.zip");

                do_cmd("$cp $d2dir/interop-${lfver}.tar.gz $::tmp_dir/");
                add_to_download_list("interop-${lfver}.tar.gz");
            }

            do_cmd("$cp $d2dir/$lfserver $::tmp_dir/");
            add_to_download_list($lfserver);

            if ($interop ne "") {
                do_cmd("$cp $d2dir/$interop $::tmp_dir/");
                add_to_download_list($interop);
            }

            if (!$skip_gui) {
                note("# copying mnt-d2 GUI\n");
                do_cmd("$cp $d2dir/$lfgui $::tmp_dir/");
                add_to_download_list($lfgui);
            }
            if (!$skip_xorp) {
                note("# copying mnt-d2 Xorp\n");
                do_cmd("$cp $d2dir/$xorp $::tmp_dir/");
                add_to_download_list($xorp);
            }

            # Fixup for F19 and LF 5.2.9
            if ($is_fedora && ($::osveri >= 19) &&
                (($::lfver eq "5.2.8") || ($::lfver eq "5.2.9"))) {
                if ($is_64) {
                    do_cmd("$cp $d2dir/libgnutls.so.26-x64-f17 $::tmp_dir/");
                    add_to_download_list("libgnutls.so.26-x64-f17");
                    do_cmd("$cp $d2dir/libtasn1.so.3-x64-f17 $::tmp_dir/");
                    add_to_download_list("libtasn1.so.3-x64-f17");
                }
                else {
                    do_cmd("$cp $d2dir/libgnutls.so.26-i32-f17 $::tmp_dir/");
                    add_to_download_list("libgnutls.so.26-i32-f17");
                    do_cmd("$cp $d2dir/libtasn1.so.3-i32-f17 $::tmp_dir/");
                    add_to_download_list("libtasn1.so.3-i32-f17");
                }
            }
        }
        if ($do_firmware) {
            note("# copying mnt-d2 firmware\n");
            if ($lfrls >= 5212) {
                do_cmd("$cp $d2dir/$fw_board $::tmp_dir/");
                add_to_download_list($fw_board);
                do_cmd("$cp $d2dir/$fw_ct_wmi $::tmp_dir/");
                add_to_download_list($fw_ct_wmi);
                if ($lfrls >= 5303) {
                    do_cmd("$cp $d2dir/$fw_ct_htt $::tmp_dir/");
                    add_to_download_list($fw_ct_htt);
                }
                if ($lfrls == 5304) {
                    do_cmd("$cp $d2dir/$fw_ct_htt5_4_old $::tmp_dir/");
                    add_to_download_list($fw_ct_htt5_4_old);
                }
                if ($lfrls >= 5304) {
                    do_cmd("$cp $d2dir/$fw_ath10k_stock $::tmp_dir/");
                    add_to_download_list($fw_ath10k_stock);
                }
                if ($lfrls >= 5305) {
                    do_cmd("$cp $d2dir/$fw_ct_htt2_9887 $::tmp_dir/");
                    add_to_download_list($fw_ct_htt2_9887);
                    do_cmd("$cp $d2dir/$fw_ct_htt5_4_9888 $::tmp_dir/");
                    add_to_download_list($fw_ct_htt5_4_9888);
                    do_cmd("$cp $d2dir/$fw_ct_htt5_4_9984 $::tmp_dir/");
                    add_to_download_list($fw_ct_htt5_4_9984);
                    do_cmd("$cp $d2dir/$fw_ct_htt5_4_new $::tmp_dir/");
                    add_to_download_list($fw_ct_htt5_4_new);
                }
                if ($lfrls >= 5308) {
                    do_cmd("$cp $d2dir/$fw_ct_htt5_4_9984b $::tmp_dir/");
                    add_to_download_list($fw_ct_htt5_4_9984b);
                }
                if ($lfrls < 5304) {
                    do_cmd("$cp $d2dir/$reg_ct $::tmp_dir/");
                    add_to_download_list($reg_ct);
                }
                do_cmd("$cp $d2dir/$fw_upstream $::tmp_dir/");
                add_to_download_list($fw_upstream);
            }
        } # ~do_firmware
        if ($do_kern) {
            do_cmd("$cp $d2dir/$kern $::tmp_dir/");
            add_to_download_list($kern);
        }
    } # endif do local d2 install
    else {
        # else do download
        print("# Noticing download option\n") if ($debug_on);
        if ($do_lanforge) {
            note("# downloading LANforge server\n");
            download("$url_base/$lfserver", "$cachebuster", "$::tmp_dir/$lfserver");
            add_to_download_list($lfserver);

            if ($lfrls > 5405) {
                note("# downloading Windows LFServer installer\n");
                download("$url_base/LANforge-Server-${lfver}-Installer.exe", "$cachebuster", "$::tmp_dir/LANforge-Server-${lfver}-Installer.exe");
                add_to_download_list("LANforge-Server-${lfver}-Installer.exe");

                note("# downloading Windows LFServer upgrade\n");
                download("$url_base/LANforge-Server-${lfver}-upgrade.zip", "$cachebuster", "$::tmp_dir/LANforge-Server-${lfver}-upgrade.zip");
                add_to_download_list("LANforge-Server-${lfver}-upgrade.zip");
            }
            # interop introduced in 5.4.5
            if ($interop ne "") {
                note("# downloading interop apk package\n");
                download("$url_base/$interop", "$cachebuster", "$::tmp_dir/$interop");
                add_to_download_list($interop);
            }
            #else {
            #    note("# Skipping interop.tar.gz download, interop name is blank.\n");
            #}

            if (!$skip_gui) {
                note("# downloading LANforge GUI\n");
                download("$url_base/$lfgui", "$cachebuster", "$::tmp_dir/$lfgui");
                add_to_download_list($lfgui);
            }
            if (!$skip_xorp) {
                note("# downloading Xorp\n");
                download("$url_base/$xorp", "$cachebuster", "$::tmp_dir/$xorp");
                add_to_download_list($xorp);
            }
            else {
                note("# skip-xorp: no $lfserver\n");
            }

            # Fixup for F19 and LF 5.2.9
            if ($is_fedora && ($::osveri >= 19) &&
                (($lfrls == 5208) || ($lfrls == 5209))) {
                if ($is_64) {
                    download("$url_base/libgnutls.so.26-x64-f17", "$cachebuster", "$::tmp_dir/libgnutls.so.26-x64-f17");
                    add_to_download_list("libgnutls.so.26-x64-f17");
                    download("$source_dir/libtasn1.so.3-x64-f17", "$cachebuster", "$::tmp_dir/libtasn1.so.3-x64-f17");
                    add_to_download_list("libtasn1.so.3-x64-f17");
                }
                else {
                    download("$source_dir/libgnutls.so.26-i32-f17", "$cachebuster", "$::tmp_dir/libgnutls.so.26-i32-f17");
                    add_to_download_list("libgnutls.so.26-i32-f17");
                    download("$source_dir/libtasn1.so.3-i32-f17", "$cachebuster", "$::tmp_dir/libtasn1.so.3-i32-f17");
                    add_to_download_list("libtasn1.so.3-i32-f17");
                }
            }
        }
        if ($::create_install_bundle || $do_http || $download_only) {
            note("# downloading documentation\n");
            download("$url_base/$doc_pkg", "$cachebuster", "$::tmp_dir/$doc_pkg");
            add_to_download_list($doc_pkg);

            note("# downloading 32-bit Windows GUI\n");
            download("$url_base/LANforge-GUI-${lfver}-Installer.exe", "$cachebuster", "$::tmp_dir/LANforge-GUI-${lfver}-Installer.exe");
            add_to_download_list("LANforge-GUI-${lfver}-Installer.exe");

            note("# downloading 64-bit Windows GUI\n");
            download("$url_base/LANforgeGUI_${lfver}-x64-Installer.exe", "$cachebuster", "$::tmp_dir/LANforgeGUI_${lfver}-x64-Installer.exe");
            add_to_download_list("LANforgeGUI_${lfver}-x64-Installer.exe");


            if ($lfrls > 5002) {
                note("# downloading Windows LFServer installer\n");
                download("$url_base/LANforge-Server-${lfver}-Installer.exe", "$cachebuster", "$::tmp_dir/LANforge-Server-${lfver}-Installer.exe");
                add_to_download_list("LANforge-Server-${lfver}-Installer.exe");
            }
            if ($lfrls > 5405) {
                note("# downloading Windows LFServer upgrade\n");
                download("$url_base/LANforge-Server-${lfver}-upgrade.zip", "$cachebuster", "$::tmp_dir/LANforge-Server-${lfver}-upgrade.zip");
                add_to_download_list("LANforge-Server-${lfver}-upgrade.zip");
            }

            note("# downloading LANforge GUI Linux x64\n");
            download("$url_base/LANforgeGUI_${lfver}_Linux64.tar.bz2", "$cachebuster", "$::tmp_dir/LANforgeGUI_${lfver}_Linux64.tar.bz2");
            add_to_download_list("LANforgeGUI_${lfver}_Linux64.tar.bz2");

            note("# downloading LANforge GUI x32\n");
            download("$url_base/LANforgeGUI_${lfver}_Linux.tar.bz2", "$cachebuster", "$::tmp_dir/LANforgeGUI_${lfver}_Linux.tar.bz2");
            add_to_download_list("LANforgeGUI_${lfver}_Linux.tar.bz2");

            note("# downloading LANforge GUI MacOS\n");
            download("$url_base/LANforgeGUI_${lfver}_MacOS.dmg.bz2", "$cachebuster", "$::tmp_dir/LANforgeGUI_${lfver}_MacOS.dmg.bz2");
            add_to_download_list("LANforgeGUI_${lfver}_MacOS.dmg.bz2");
        }
        if ($do_firmware) {
            note("# downloading firmware\n");
            if ($lfrls >= 5212) {
                download("$url_base/$fw_board", "$cachebuster", "$::tmp_dir/$fw_board");
                add_to_download_list($fw_board);
                download("$url_base/$fw_ct_wmi", "$cachebuster", "$::tmp_dir/$fw_ct_wmi");
                add_to_download_list($fw_ct_wmi);
                if ($lfrls >= 5303) {
                    download("$url_base/$fw_ct_htt", "$cachebuster", "$::tmp_dir/$fw_ct_htt");
                    add_to_download_list($fw_ct_htt);
                }
                if ($lfrls == 5304) {
                    download("$url_base/$fw_ct_htt5_4_old", "$cachebuster", "$::tmp_dir/$fw_ct_htt5_4_old");
                    add_to_download_list($fw_ct_htt5_4_old);
                }
                if ($lfrls >= 5304) {
                    download("$url_base/$fw_ath10k_stock", "$cachebuster", "$::tmp_dir/$fw_ath10k_stock");
                    add_to_download_list($fw_ath10k_stock);
                }
                if ($lfrls >= 5305) {
                    download("$url_base/$fw_ct_htt2_9887", "$cachebuster", "$::tmp_dir/$fw_ct_htt2_9887");
                    add_to_download_list($fw_ct_htt2_9887);
                    download("$url_base/$fw_ct_htt5_4_9888", "$cachebuster", "$::tmp_dir/$fw_ct_htt5_4_9888");
                    add_to_download_list($fw_ct_htt5_4_9888);
                    download("$url_base/$fw_ct_htt5_4_9984", "$cachebuster", "$::tmp_dir/$fw_ct_htt5_4_9984");
                    add_to_download_list($fw_ct_htt5_4_9984);
                    download("$url_base/$fw_ct_htt5_4_new", "$cachebuster", "$::tmp_dir/$fw_ct_htt5_4_new");
                    add_to_download_list($fw_ct_htt5_4_new);
                }
                if ($lfrls >= 5308) {
                    download("$url_base/$fw_ct_htt5_4_9984b", "$cachebuster", "$::tmp_dir/$fw_ct_htt5_4_9984b");
                    add_to_download_list($fw_ct_htt5_4_9984b);
                }
                if ($lfrls < 5304) {
                    download("$url_base/$reg_ct", "$cachebuster", "$::tmp_dir/$reg_ct");
                    add_to_download_list($reg_ct);
                }
                download("$url_base/$fw_upstream", "$cachebuster", "$::tmp_dir/$fw_upstream");
                add_to_download_list($fw_upstream);
            }
        }
        if ($do_kern && $do_interop) {
            err("--do_interop set, disabling --do_kern\n");
            $do_kern = 0;
        }
        if ($do_kern) {
            note("# downloading kernel $kern\n");
            download("$url_base/$kern", "$cachebuster", "$::tmp_dir/$kern");
            add_to_download_list($kern);
        }
    } # ~else do web download

    # Things are downloaded to $::tmp_dir now...
    if ($download_only) {
        note("# Download only set, bye.\n");
        show_summary();
        exit(0);
    }
}
if ($::create_install_bundle) {
    note("* creating manifest of files to bundle...\n");
    open(my $mf, ">", "$::tmp_dir/manifest.txt") || die("unable to open $::tmp_dir/manifest.txt: " . $?);
    foreach my $key (sort keys(%::download_list)) {
        print "$key\n";
        print $mf "$key\n";
    }
    close($mf);
    my $x64 = $is_64 ? "x64" : "x32";
    my $bundle_name = "Bundle_lfver-${lfver}_kern-${kver}_osver-${main::osver}-i-${main::osveri}_${x64}.tar.xz";
    note("* creating bundle $::tmp_dir/$bundle_name\n");
    chdir($::tmp_dir);

    print "========================================\n";
    print `cat manifest.txt`;
    print "========================================\n";
    print "All packages are being added to a highly compressed tar archive.\n";
    print_slow("This takes a while!");
    # pxz is outdated, use xz -T0 now for parallel compression
    my $use_cores = `grep 'processor' /proc/cpuinfo  | tail -1 | awk -F: '{print \$2}'`;
    chomp($use_cores);
    if (!(defined $use_cores) || ($use_cores =~ /^\s*$/)) {
        $use_cores = 6;
    }
    print_slow("Compressing with $use_cores cores...\n");
    do_cmd("tar cf $::tmp_dir/$bundle_name -I '/usr/bin/xz -9e -T$use_cores' -T $::tmp_dir/manifest.txt");
    note("* created bundle $::tmp_dir/$bundle_name \n");
    note("* Copy this and the lf_kinstall.pl script to remote host and use the command: \n");
    note("  $0 --use_install_bundle ${main::tmp_dir}/$bundle_name \n");
    exit(0);
}
else { # not creating install bundle
    if ($is_deb_based && $do_interop) {
        # we need ssh operational before installing lanforge
        do_cmd("systemctl is-enabled ssh.service || touch /tmp/install-ssh", 1);
        if ( -f "/tmp/install-ssh") {
            note("* installing openssh-server\n");
            do_cmd("$apt install -y openssh-server", 1);
            do_cmd("systemctl enable ssh.service", 1);
            do_cmd("systemctl start ssh.service", 1);
            note("* installed ssh server");
            unlink("/tmp/install-ssh");
        }
    }
    if ($do_kern) {
        # Turn down logging on console.
        if (-d "/etc/sysctl.d") {
            note("# Making console logging less verbose...\n");
            do_cmd("echo \"#Auto generated by lf_kinstall, this will be over-written.\" > /etc/sysctl.d/lanforge.conf", 1);
            do_cmd("echo \"# verbose console messages: kernel.printk = 8 4 1 7\" >> /etc/sysctl.d/lanforge.conf", 1);
            do_cmd("echo \"kernel.printk = 4 4 1 7\" >> /etc/sysctl.d/lanforge.conf", 1);
        }

        # if we had dnf kernel updates pending, we should install those now before we do our final do_grub config
        if ($yum_has_pending_kernel && !($skip_yum_all || $skip_yum_update)) {
            print_slow("* doing deferred kernel package updates...\n");
            do_cmd("$yum update -y --best --skip-broken", );
        }
        # Install kernel
        chdir("/") || die "Couldn't chdir to /:  $!\n\n";
        note("# Installing $kern...\n");
        do_cmd("tar --no-same-owner -xzf $::tmp_dir/$kern");
        if (!$is_chr_ubuntu) {
            if ($is_rpi3) {
                note("# kinitramfs $kern...\n");
                # Raspberry PI-3 (ubuntu) special kernel install section
                do_cmd("mkinitramfs $kver -o /boot/firmware/initrd.img");
                do_cmd("depmod $kver");
            }
            elsif ($is_arm) {
                do_cmd("depmod $kver");
            }
            else {
                note("# kinstall_ct$kver...\n");
                do_cmd("/usr/local/bin/kinstall_ct$kver.bash");
            }
        }
        else {
            # Install an updated kernel on chromebooks.
            my $kfs = "/dev/sda6";
            my $rfs = "/dev/sda7";
            my $kern_img = "ct$kver\.img";
            do_cmd("echo \"console=tty1 debug verbose root=$rfs rootwait rw lsm.module_locking=0\" > kernel-config");
            do_cmd("vbutil_kernel --pack newkern --keyblock /usr/share/vboot/devkeys/kernel.keyblock --version 1 --signprivate /usr/share/vboot/devkeys/kernel_data_key.vbprivk --config kernel-config --vmlinuz /boot/$kern_img");
            do_cmd("dd if=newkern of=$kfs");
            #Set Ubuntu partition as top priority for next boot
            do_cmd("cgpt add -i 6 -P 5 -T 1 /dev/sda");
            # Tell user how to make it permanent.
            open(FILE, ">$::home/perm_ubuntu.bash") or die "Couldn't open file: $::home/perm_ubuntu.bash for writing $!\n\n";
            print FILE "#!/bin/bash\n";
            print FILE "sudo cgpt add -i 6 -P 5 -S 1 /dev/sda\n";
            close FILE;
            do_cmd("chmod a+x $::home/perm_ubuntu.bash");
            warning("The system will boot into Ubuntu on next boot, but if you wish to have it continue to boot into Ubuntu instead of Google Chrome, then you must run the command: $::home/perm_ubuntu.bash.  Please run this command after a successful reboot to protect against bricking the system with a bad Ubuntu/LANforge kernel.\n");
        }
    } # ~if do kern

    if ($do_firmware && !$is_openwrt) {
        note("# Installing firmware ...\n");
        if ($lfrls >= 5212) {
            my $fwbase = "/lib/firmware/ath10k";
            my $fwdir = "$fwbase/QCA988X/hw2.0/";
            my $fwdir2 = "$fwbase/QCA99X0/hw2.0/";
            my $fwdir9984 = "$fwbase/QCA9984/hw1.0/";
            my $fwdir9888 = "$fwbase/QCA9888/hw2.0/";
            my $fwdir9887 = "$fwbase/QCA9887/hw1.0/";
            if (!-d "$fwdir") {
                do_cmd("mkdir -p $fwdir");
            }
            if (!$is_openwrt) {
                do_cmd("cp -f $::tmp_dir/$fw_board $fwdir/$fw_board");
            }
            if ($lfrls >= 5303) {
                do_cmd("cp -f $::tmp_dir/$fw_ct_htt $fwdir/firmware-2.bin");
                do_cmd("cp -f $::tmp_dir/$fw_ct_wmi $fwdir/firmware-2-wmi.bin");
                if ($lfrls >= 5304) {
                    if (!-d "$fwdir2") {
                        do_cmd("mkdir -p $fwdir2");
                    }
                    do_cmd("cp -f $::tmp_dir/$fw_ath10k_stock $fwbase");
                    do_cmd("cd $fwbase && tar -xvzf $fw_ath10k_stock");

                    if ($lfrls == 5304) {
                        do_cmd("cp -f $::tmp_dir/$fw_ct_htt5_4_old $fwdir2/firmware-5-htt-mgt.bin");
                    }
                    if ($lfrls >= 5305) {
                        do_cmd("cp -f $::tmp_dir/$fw_ct_htt5_4_new $fwdir2/firmware-5-htt-mgt.bin");
                        if (!-d "$fwdir9984") {
                            do_cmd("mkdir -p $fwdir9984");
                        }
                        if ($lfrls >= 5308) {
                            do_cmd("cp -f $::tmp_dir/$fw_ct_htt5_4_9984b $fwdir9984/firmware-5-htt-mgt-b.bin");
                        }
                        do_cmd("cp -f $::tmp_dir/$fw_ct_htt5_4_9984 $fwdir9984/firmware-5-htt-mgt.bin");
                        if (!-f "$fwdir9984/firmware-5.bin.orig" && -f "$fwdir9984/firmware-5.bin") {
                            do_cmd("cp $fwdir9984/firmware-5.bin $fwdir9984/firmware-5.bin.orig");
                        }
                        # We don't want to auto-load default firmware by accident...it will not boot with our
                        # normal firmware config.
                        do_cmd("cp -f $::tmp_dir/$fw_ct_htt5_4_9984 $fwdir9984/firmware-5.bin");
                        if (!-d "$fwdir9888") {
                            do_cmd("mkdir -p $fwdir9888");
                        }
                        do_cmd("cp -f $::tmp_dir/$fw_ct_htt5_4_9888 $fwdir9888/firmware-5-htt-mgt.bin");
                        if (!-f "$fwdir9888/firmware-5.bin") {
                            do_cmd("cp -f $::tmp_dir/$fw_ct_htt5_4_9888 $fwdir9888/firmware-5.bin");
                        }

                        if (!-d "$fwdir9887") {
                            do_cmd("mkdir -p $fwdir9887");
                        }
                        do_cmd("cp -f $::tmp_dir/$fw_ct_htt2_9887 $fwdir9887/firmware-2-htt-mgt.bin");
                        if (!-f "$fwdir9887/firmware-2.bin") {
                            do_cmd("cp -f $::tmp_dir/$fw_ct_htt2_9887 $fwdir9887/firmware-2.bin");
                        }
                        if (-f "$fwdir9887/firmware-5.bin") { # Move stock firmware out of the way
                            do_cmd("mv $fwdir9887/firmware-5.bin $fwdir9887/firmware-5.bin.orig");
                        }
                    }
                    if (!-f "$fwdir2/firmware-5.bin") {
                        do_cmd("cp -f $fwdir2/firmware-5-htt-mgt.bin $fwdir2/firmware-5.bin");
                    }
                }
            }
            else {
                do_cmd("cp -f $::tmp_dir/$fw_ct_wmi $fwdir/firmware-2.bin");
            }

            if ($lfrls < 5304) {
                note("# Installing regulatory.bin\n");
                if (-f "/usr/lib/crda/regulatory.bin") {
                    do_cmd("cp -f $::tmp_dir/$reg_ct /usr/lib/crda/regulatory.bin", 1);
                }
                else {
                    do_cmd("cp -f $::tmp_dir/$reg_ct /lib/crda/regulatory.bin", 1);
                }
            }

            do_cmd("cp -f $::tmp_dir/$fw_upstream $fwdir/$fw_upstream");

            # Move un-wanted ath10k firmware out of the way.
            if (-e "$fwdir/firmware-5.bin") {
                do_cmd("mv $fwdir/firmware-5.bin $fwdir/firmware-5.bin.orig");
            }
            if (-e "$fwdir/firmware-4.bin") {
                do_cmd("mv $fwdir/firmware-4.bin $fwdir/firmware-4.bin.orig");
            }
            if (-e "$fwdir/firmware-3.bin") {
                do_cmd("mv $fwdir/firmware-3.bin $fwdir/firmware-3.bin.orig");
            }
        }
    } # ~if do firmware

    if ($do_lanforge) {
        print_slow(" ==== Configuring system for LANforge ==== ==== ==== ==== ==== ==== ==== ==== ==== ====\n")
            if ($::debug_on);
        my $fname = "/etc/nsswitch.conf";
        my $nsswitch_grep = `grep '^hosts:' $fname`;
        # print("NSS: $nsswitch_grep\n");
        if ((!$is_macos) && ($nsswitch_grep =~ /files myhostname mdns4_minimal \[NOTFOUND=return/)) {
            note("Updating /etc/nsswitch.conf\n");
            sleep(3);
            my @nsswitch_lines = `cat /etc/nsswitch.conf`;
            my @new_nsswitch_lines = ();
            chomp(@nsswitch_lines);
            for my $nss_line (@nsswitch_lines) {
                if ($nss_line !~ /^hosts: /) {
                    push(@new_nsswitch_lines, $nss_line);
                    next;
                }
                push(@new_nsswitch_lines, "hosts: files myhostname mdns4_minimal resolve dns [NOTFOUND=return] [!UNAVAIL=return]");
            }
            open(FILE, ">", $fname) or die "Could not write $fname $!\n\n";
            print FILE join("\n", @new_nsswitch_lines), "\n";
            close(FILE);
        }

        # Fix pulseaudio to support ofono
        my $pa_cfg = "/etc/pulse/default.pa";
        if (-f $pa_cfg && !$is_macos) {
            my $found_pa_include = 0;
            my @cmds = `cat $pa_cfg`;
            open(FILE, ">$pa_cfg") or die "Couldn't open file: $pa_cfg for writing: $!\n\n";
            for ($i = 0; $i < @cmds; $i++) {
                my $ln = $cmds[$i];
                chomp($ln);
                if ($ln =~ /^\s*load-module module-bluetooth-discover.*/) {
                    print FILE "load-module module-bluetooth-discover headset=ofono\n";
                }
                elsif ($ln =~ /^\s*\.include \/etc\/pulse\/default\.pa\.d.*/) {
                    $found_pa_include = 1;
                    print FILE "$ln\n";
                }
                else {
                    print FILE "$ln\n";
                }
            }

            if (!$found_pa_include) {
                print FILE "\n### Allow including a default.pa.d directory, which if present, can be used
### for additional configuration snippets.
### Note that those snippet files must have a .pa file extension, not .conf
.nofail
.include /etc/pulse/default.pa.d
";
            }
            close(FILE);
        } # if pulse audio
        if (-f "$::home/local/etc/ofono.conf") {
            do_cmd("mkdir -p /etc/ofono");
            open(FILE, ">/etc/ofono/phonesim.conf") or warn "Couldn't open file: /etc/ofono/phonesim.conf for writing: $!\n\n";
            print FILE "[phonesim]
Driver=phonesim
Address=127.0.0.1
Port=12345
";
            close(FILE);

            do_cmd("cp $::home/local/etc/ofono.conf /etc/dbus-1/system.d/", 1);

            my $ofono_svc = "/usr/lib/systemd/system/ofono.service";
            open(FILE, ">$ofono_svc") or warn "Couldn't open file: $ofono_svc for writing: $!\n\n";
            print FILE "[Unit]
Description=Telephony service
After=syslog.target

[Service]
Type=dbus
BusName=org.ofono
ExecStart=$::home/local/sbin/ofonod -n
StandardError=null

[Install]
WantedBy=multi-user.target
";
            close(FILE);
        } # ~if ofono

        fix_chrony_conf();
        if (!$skip_xorp) {
            # Install xorp
            if ($is_deb_based && !$is_arm) {
                # we should not be using debs
                # if ($ubuntu_deb_ver ne "") {
                #    do_cmd("cp $::tmp_dir/$xorp $::tmp_dir/$xorp_nodebver");
                #}
                # do_cmd("dpkg -i $::tmp_dir/$xorp_nodebver");
                print_slow(" = = not installing XORP = = \n");
            }
            else {
                chdir("/usr/local") || die "Couldn't chdir to /usr/local:  $!";
                do_cmd("tar -xzf $::tmp_dir/$xorp");
            }
            chdir("/usr/local/xorp") || die "Couldn't chdir to /usr/local/xorp:  $!";
            do_cmd("./xorp_install.bash", 1);
            chdir("$::home") || die "Couldn't chdir to $::home:  $!\n\n";
        }

        if ($interop ne "") {
            chdir("$::home") || die "Couldn't chdir to $::home:  $!\n\n";
            do_cmd("tar -xzf $::tmp_dir/$interop");
        }

        # check that DefaultTasksMax is set in /etc/systemd/system.conf
        if ((!$is_macos) && (-f "/etc/systemd/system.conf")) {
            `grep 'DefaultTasksMax=' /etc/systemd/system.conf`;
            if ($? == 0) {
                my $count = 0;
                my @lines = `cat /etc/systemd/system.conf`;
                chomp(@lines);
                my @new_lines = ();
                my @matching = grep {/^DefaultTasksMax=65535/} @lines;
                if (@matching > 0) {
                    @lines = ();
                    note("DefaultTasksMax is set properly\n");
                }
                else {
                    if (!-f "/etc/systemd/system.conf.orig") {
                        `cp /etc/systemd/system.conf /etc/systemd/system.conf.orig`;
                    }
                    for my $line (@lines) {
                        if ($line =~ /DefaultTasksMax=/) {
                            $count++;
                            if ($count == 1) {
                                push(@new_lines, "# " . $line);
                                $line = "DefaultTasksMax=65535";
                                push(@new_lines, $line);
                                next;
                            }
                            elsif ($line !~ /^\x23/) {
                                push(@new_lines, "# $line");
                                next;
                            }
                        }
                        push(@new_lines, $line);
                    } # ~foreach
                    open(my $fh_system_conf, ">", "/etc/systemd/system.conf") or die("Unable to write /etc/systemd/system.conf");
                    print $fh_system_conf  join("\n", @new_lines) . "\n";
                    close($fh_system_conf);
                    do_cmd("systemctl daemon-reload");
                } # ~else need to modify system.conf
            } # grep default tasks max set
        } #  system.conf exists
        if ($::should_create_user) {
            print_slow("* About to create user lanforge....");
            create_lf_user();
        }
        if (!$is_macos) {
            do_cmd("gpasswd -a lanforge dialout", 1);
            do_cmd("gpasswd -a lanforge tty", 1);
        }

        note("Configuring journalctl.conf file to keep only 128M of logs\n");
        my $jc_conf = "/etc/systemd/journald.conf";
        if (!$is_macos && -f $jc_conf) {
            my @jc_lines = `cat $jc_conf`;
            chomp @jc_lines;
            my @new_jc_lines = ();
            my @jc_matches = grep {/^SystemMaxUse=/} @jc_lines;
            if (@jc_matches > 0) {
                if ($jc_matches[0] !~ /^SystemMaxUse=128M/) {
                    for my $jc_line (@jc_lines) {
                        next if ($jc_line =~ /SystemMaxUse=/);
                        push(@new_jc_lines, $jc_line);
                    }
                }
                @jc_lines = @new_jc_lines;
            }
            push(@jc_lines, (
               "SystemMaxUse=128M",
               "SystemMaxFiles=17",
               "SystemMaxFileSize=8M",
               ));
            die("Unable to write $jc_conf: $!") unless open(my $jc_fh, ">", $jc_conf);
            for my $jc_line (@jc_lines) {
                print $jc_fh "$jc_line\n";
            }
            close $jc_fh;
            do_cmd("systemctl restart systemd-journald.service", 1);
            if ($::osveri >= 24) {
                note("# vacuuming logs\n");
                do_cmd("journalctl --vacuum-size=10M", 1);
            }
        } # ~if journalctl.conf

        print("* Creating default LANforge scenarios...\n");
        write_defaults_db();

        if (!$is_macos) {
            my $profile_lanforge = "/etc/profile.d/lanforge.sh";
            print("# updating $profile_lanforge\n");
            open(my $fh_profile, ">", $profile_lanforge)
                or die("unable to open $profile_lanforge");
            print $fh_profile qq(# Created by lf_kinstall.pl
export PERL5LIB=".:/home/lanforge/scripts"
);
            close $fh_profile;
            print("# updating .lfaliases.sh\n");
            open(my $fh_lfprofile, ">", "$::home/.lfaliases.sh")
              or die("unable to create $::home/.lfaliases.sh");
            my $lfprofile = qq[# regular set of defaults that help with customer support
#
alias r='ls -ltr'
alias h='df -h'
alias u='du -h'
alias ..='cd ..'
alias ,='ls -CFs'
alias Ip='ip -br addr sh'
alias Li='ip -br link sh'
alias Ro='ip -br ro sh'
alias gui='cd `ls -1d ~/LANforgeGUI*|tail -1` && ./lfclient.bash -s localhost'
function bu() {
   bn=${t}basename ${d}1${t}
   dn=${t}dirname ${d}1${t}
   olddt=${t}ls -l --time-style=+%s ${d}1 | awk '{print ${d}6}'${t}
   cp -v "${d}1" "${d}dn/.${d}bn.${d}olddt"
}
alias StopCX='echo "set_cx_state all all STOPPED"|nc localhost 4001'
export VISUAL=vim
export EDITOR=vim
alias vi='/usr/bin/vim'
alias va='vi ~/.lfaliases.sh && . ~/.lfaliases.sh'
alias watchlog='watch -n60 ls -l /core* $::home/ath10* $::home/core* 2>/dev/null'
alias tail_manager='cd /home/lanforge; tail -F lanforge_log_?.txt run_[cm]*out'
alias logreboot='journalctl -b0 > dmesg.txt; echo s > /proc/sysrq-trigger; echo u > /proc/sysrq-trigger; echo b > /proc/sysrq-trigger'
alias now=`date +%Y%m%d-%H%M%S`
function Now() {
  date +%Y%m%d-%H%M%S
}
function NewHprofFile() {
    N=`Now`
    DFILE="\$HOME/Documents/gui-\$N.hprof"
}
function JmapDump() {
    # test for jps and jmap
    # wget https://www.candelatech.com/downloads/jdk-8u144-linux-x64.tar.gz
    # wget https://www.candelatech.com/downloads/jdk-8u144-windows-x64.exe
    # or /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.382.b05-2.fc37.x86_64/bin/
    # or /usr/lib/jvm/java-17-openjdk-17.0.8.0.7-1.fc37.x86_64/bin/
   local pid=`jps | awk '/lfclient/{print \$1}'`

   (( \$? != 0 )) && echo "No java running" && return 1
   [ -z "\$pid" ] && echo "No java pid" && return 1
   local n=`Now`
   local dfile="\$HOME/Documents/gui-\${n}.hprof"
   # Some versions of jdk do not allow -F -J-d64
   J=jmap
   # set -x
   if [[ -d /usr/lib/jvm/java-17-openjdk*.x86_64/bin ]]; then
        JAVA_HOME=/usr/lib/jvm/java-17-openjdk*.x86_64
        J=jhsdb
        env JAVA_HOME="\$JAVA_HOME" \\
            \$J jmap --binaryheap --dumpfile \$dfile --pid \$pid

   elif [[ -d /usr/lib/jvm/java-1.8.0-openjdk*.x86_64/bin ]]; then
        JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk*.x86_64
        J=\$JAVA_HOME/bin/jmap
        env JAVA_HOME="\$JAVA_HOME" \\
            \$J -F -J-d64 -dump:live,format=b,file=\$dfile \$pid

   elif [[ -d /home/lanforge/jdk1.8.0_144 ]]; then
        JAVA_HOME=/home/lanforge/jdk1.8.0_144
        J=\$JAVA_HOME/bin/jmap
        env JAVA_HOME="\$JAVA_HOME" \\
            \$J -F -J-d64 -dump:live,format=b,file=\$dfile \$pid
   elif [[ ! -z "\$distro_jmap" ]]; then
        JAVA_HOME=\$( /usr/bin/dirname "\$distro_jmap" )
        if [[ \$JAVA_HOME = */bin* ]]; then
           JAVA_HOME="\${JAVA_HOME%/bin*}"
        fi
        J="\$distro_jmap"
        env JAVA_HOME="\$JAVA_HOME" \
            \$J -dump:live,format=b,file="\$dfile" \$pid
   fi
   set +x
}
function Krl() {
   local u=${dat}
   if [[ ${d}u != http* ]]; then
      u="http://localhost:8080${dat}"
   fi
   curl -o /tmp/result.${dd} ${B}
         -H 'Accept: application/json' ${B}
         -sq ${d}u

   if [ ${d}? -eq 0 ]; then
      json_pp < /tmp/result.${dd}
   else
      echo "? [${d}?]:"
      less /tmp/result.${dd}
   fi
}
function Frl() {
   if [ ! -e "/tmp/curl_data" ]; then
      echo "no /tmp/curl_data"
      return
   fi
   curl -H 'Accept: application/json' ${B}
      -X POST -d${at}/tmp/curl_data ${B}
    -sqv http://localhost:8080${dat}
}
function Jrl() {
   if [ ! -e "/tmp/curl_json" ]; then
      echo "no curl json"
      return
   fi
   curl -sv -H 'Accept: application/json' ${B}
      -H 'Content-Type: application/json' ${B}
      -X POST -d${at}/tmp/curl_json ${B}
    -sq http://localhost:8080${dat}
}
];
            print $fh_lfprofile $lfprofile;
            close $fh_lfprofile;
            do_cmd("chown lanforge:$user_group .lfaliases.sh", 1);
        } # ~create .bash_aliases

        if (!-f "$::home/.vimrc") {
            note("# creating .vimrc\n");
            open(my $fh_vimrc, ">", "$::home/.vimrc") or warn("unable to create .vimrc");
            if ($fh_vimrc) {
                (my $vimrc = qq{set si
                   set si
                   set ai
                   set hlsearch
                   set expandtab
                   set smarttab
                   set shiftwidth=4
                   set softtabstop=4
                   set background=light
                   set number
                   set wrap linebreak nolist
                   hi CursorLine   cterm=NONE ctermbg=darkred ctermfg=white guibg=darkred guifg=white
                   hi CursorColumn cterm=NONE ctermbg=darkred ctermfg=white guibg=darkred guifg=white
                   hi Comment     cterm=NONE  ctermbg=darkblue ctermfg=LightBlue guibg=darkblue guifg=LightBlue
                   }) =~ s/^ +//gm;
                print $fh_vimrc $vimrc;
                close $fh_vimrc;
                if (!-f "/root/.vimrc") {
                    do_cmd("cp $::home/.vimrc /root", 1);
                }
                do_cmd("chown lanforge:$user_group $::home/.vimrc", 1);
            }
        } # ~if create .vimrc
        if (!-f "$::home/.screenrc") {
            print("# creating .screenrc\n");
            open(my $fh_screenrc, ">", "$::home/.screenrc") or die("unable to create .screenrc");
            (my $screenrc = qq{escape ^yY
               defescape ^yY
               startup_message     off
               autodetach          on
               multiuser           on
               deflogin            off
               hardcopy_append     on
               vbell               off
               defscrollback       10000
               silencewait         10
               shell               bash
               hardstatus          on
               hardstatus          alwayslastline
               hardstatus string "Use C-y %{.bW}| %-w%{.rW}%n %t%{-}%+w %=%{..G} %H %{..Y} %m/%d %C%a"
               screen -t Shell 0 bash
               screen -t Shell 1 bash
               select 0
               }) =~ s/^ +//gm;
            print $fh_screenrc $screenrc;
            close $fh_screenrc;
            do_cmd("chown lanforge:$user_group $::home/.screenrc", 1);
        }
        mkdir("$::home/wifi", 0775)     if ( !-d "$::home/wifi" );
        do_cmd("chown -R lanforge:$user_group $::home/wifi", 1);
        mkdir("$::home/tmp",  0775)     if ( !-d "$::home/tmp" );
        do_cmd("chown lanforge:$user_group $::home/tmp", 1);
        mkdir("$::home/Pictures", 0775) if ( !-d "$::home/Pictures" );
        do_cmd("chown -R lanforge:$user_group $::home/Pictures", 1);

        if (!$skip_gui) {
            print_slow("# Installing GUI...lfgui[$lfgui] to home[$::home]\n") if ($::debug_on);
            # Install LANforge-GUI
            chdir("$::home") || die "Couldn't chdir to $::home:  $!\n\n";
            do_cmd("tar -xjf $::tmp_dir/$lfgui");
            chdir("$::home/LANforgeGUI_$::lfver") || die "Couldn't chdir to $::home/LANforgeGUI_$::lfver:  $!\n\n";
            do_cmd("./lfgui_install.bash");
            note("#...installed GUI.\n");
            # set background image
            if (! -d "$::home/Pictures") {
                mkdir("$::home/Pictures", 0775);
                `chown lanforge: $::home/Pictures`;
            }
            if (-f "$::home/LANforgeGUI_$::lfver/images/anvil-right.svg") {
                do_cmd("cp $::home/LANforgeGUI_$::lfver/images/anvil-right.svg $::home/Pictures/", 1);
            }
        } # ~if install gui

        if (-f "$::home/serverctl.bash") {
            chdir("$::home") || die "Couldn't chdir to $::home:  $!\n\n";
            note("# stopping lanforge...\n");
            do_cmd("./serverctl.bash stop", 1);
        }

        # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- #
        # Install LANforge-Server
        # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- #
        # backup previous databases
        if (-d "$::home/DB") {
            note("# backing up previous LF scenarios...");
            chdir("$::home") || die "Couldn't chdir to $::home:  $!\n\n";
            my $preDBname = "DB-pre-$::lfver-" . ts() . ".tar.gz";
            #unlink($preDBname);
            do_cmd("tar -czf $preDBname DB");
        }

        # Create btserver key/cert files if not already existing.
        if (! -f "$::home/btserver_priv_key.pem") {
            #https://www.scottbrady91.com/openssl/creating-rsa-keys-using-openssl

            my $fname = "$::home/btserver_ca.cnf";
            if (!-f $fname) {
                open(FILE, ">$fname");
                print FILE "[ ca ]
default_ca              = CA_default

[ CA_default ]
dir                     = ./
certs                   = \$dir
crl_dir                 = \$dir/crl
database                = \$dir/index.txt
new_certs_dir           = \$dir
certificate             = \$dir/ca.pem
serial                  = \$dir/serial
crl                     = \$dir/crl.pem
private_key             = \$dir/ca.key
RANDFILE                = \$dir/.rand
name_opt                = ca_default
cert_opt                = ca_default
default_days            = 5000
default_crl_days        = 30
default_md              = sha256
preserve                = no
policy                  = policy_match
crlDistributionPoints   = URI:http://www.example.org/example_ca.crl

[ policy_match ]
countryName             = match
stateOrProvinceName     = match
organizationName        = match
organizationalUnitName  = optional
commonName              = supplied
emailAddress            = optional

[ policy_anything ]
countryName             = optional
stateOrProvinceName     = optional
localityName            = optional
organizationName        = optional
organizationalUnitName  = optional
commonName              = supplied
emailAddress            = optional

[ req ]
prompt                  = no
distinguished_name      = certificate_authority
default_bits            = 3072
#input_password         = lanforge
#output_password        = lanforge
#x509_extensions        = v3_ca

[certificate_authority]
countryName             = US
stateOrProvinceName     = WA
localityName            = Ferndale
organizationName        = Candela Technologies Inc.
emailAddress            = support\@candelatech.com
commonName              = \"Candela - CA\"

[v3_ca]
subjectKeyIdentifier    = hash
authorityKeyIdentifier  = keyid:always,issuer:always
basicConstraints        = critical,CA:true
crlDistributionPoints   = URI:http://www.example.org/example_ca.crl
";
                close FILE;
            } # create btserver-ca.cnf

            note("# Generating btserver private ssl key...");
            do_cmd("cd $::home; openssl genrsa -out btserver_priv_key.pem 3072; cd -", 1);

            note("# Generating btserver key cert...");
            do_cmd("cd $::home; openssl req -new -x509 -key btserver_priv_key.pem -out btserver_cert.pem -days 7200 -config $fname; cd -", 1);

            #note("# Generating pkcs12 for java applications...");
            #do_cmd("cd $::home; openssl pkcs12 -export -out cert-and-key.p12 -in btserver_cert.pem -inkey btserver_priv_key.pem -CAfile btserver_ca.cnf -passout pass:; cd -");
        } # ~if create bt_server private key

        # Check for lanforge/.ssh/localhost_rsa and create
        # an authorized keys entry so lanforge and ssh to localhost as root
        if (!$is_macos) {
            my ($x, $y, $uid, $gid) = getpwnam("lanforge") or die "lanforge not in passwd file";
            if (!-d "$::home/.ssh") {
                mkdir("$::home/.ssh", 0700);
                chown $uid, $gid, "$::home/.ssh";
                chmod 0700, "$::home/.ssh";
            }

            my $identity_file = "$::home/.ssh/id_rsa";
            if (!-f $identity_file) {
                note("# generating lanforge\@localhost ssh identity [$identity_file]\n");
                do_cmd(qq(/usr/bin/ssh-keygen -t rsa -f "$identity_file" -N '' -C 'lanforge\@localhost' ));
                chown $uid, $gid, "$identity_file", "$identity_file.pub";
                chmod 0700, ("$identity_file", "$identity_file.pub");
            }
            open(my $fh_lf_ssh_pub, "<", "$identity_file.pub") or die("unable to open $identity_file.pub");
            my $lf_ssh_pub = <$fh_lf_ssh_pub>;
            close($fh_lf_ssh_pub);
            chomp($lf_ssh_pub);

            if (!-d "/root/.ssh") {
                mkdir("/root/.ssh", 0700);
            }
            my $do_id_install = 0;
            my $root_auth = "/root/.ssh/authorized_keys";
            if (!-f $root_auth) {
                $do_id_install = 1;
            }
            else {
                open(my $fh_ssh_pub, "<", $root_auth) or die("unable to open $root_auth");
                my @root_auth_lines = <$fh_ssh_pub>;
                # TODO: check that we're using rsa keys, dsa now deprecated
                # TODO: check if existing key is installed

                my $q;
                $do_id_install = 1;
                for ($q = 0; $q < @root_auth_lines; $q++) {
                    my $ln = $root_auth_lines[$q];
                    if ($ln =~ /\S+\s+(.*)/) {
                        my $key_str = $1;
                        #note("ssh: Checking key-str -:$key_str:-  lf-ssh-pub -:$lf_ssh_pub:-\n");
                        if ($lf_ssh_pub eq $key_str) {
                            note("# Found lanforge ssh key in root authorized keys...continuing.\n");
                            $do_id_install = 0;
                            last;
                        }
                        else {
                            #note("ssh:  Key did not match\n");
                        }
                    }
                } # ~for
            } # else check root auth keys

            if ($do_id_install) {
                note("# adding ssh/authorized_keys entry for lanforge\@localhost\n");
                do_cmd(qq(echo -n 'from="localhost,127.0.0.1,::1" ' >> $root_auth));
                do_cmd(qq(cat $identity_file.pub >> $root_auth));
                chmod 0600, "$root_auth";
            }
            else {
                note("# $root_auth already has entry for lanforge\@localhost\n");
            }

            # replace host key if necessary
            if ($::osveri >= 29) {
                # sshd not running on f29 or f30 on first install
                do_cmd(qq(systemctl restart sshd.service));
            }
            do_cmd(qq(ssh-keygen -R localhost || true));
            do_cmd(qq(ssh-keyscan -t rsa localhost 2>/dev/null >> /root/.ssh/known_hosts), 1);
            do_cmd(qq(ssh-keyscan -t rsa localhost 2>/dev/null >> $::home/.ssh/known_hosts), 1);
            #do_cmd(qq(ssh-keyscan -t dsa localhost 2>/dev/null >> /root/.ssh/known_hosts));
            #do_cmd(qq(ssh-keyscan -t dsa localhost 2>/dev/null >> $::home/.ssh/known_hosts));
            chown $uid, $gid, "$::home/.ssh/known_hosts";

            # provide policy for local X11 forwarding over ssh
            open(my $fh_ssh_config, "<", "/etc/ssh/ssh_config") or die("could not read ssh_config");
            my @ssh_config = <$fh_ssh_config>;
            close($fh_ssh_config);

            # if ssh allows config files, add forwarding entry:
            my $ssh_confd = "/etc/ssh/ssh_config.d/";
            if (-d $ssh_confd) {
                my $ssh_lfconf = "$ssh_confd/lanforge.conf";
                if (!-r $ssh_lfconf) {
                    err("* Unable to open $ssh_lfconf")
                       unless (open(my $fh_ssh_config, ">", $ssh_lfconf));
                    #print $fh_ssh_config "X11UseLocalhost no\n\n";
                    print $fh_ssh_config "Host *\n";
                    print $fh_ssh_config "  ForwardAgent yes\n";
                    print $fh_ssh_config "  ForwardX11 yes\n";
                    print $fh_ssh_config "  PasswordAuthentication yes\n";
                    close $fh_ssh_config;
                    do_cmd("service sshd restart", 1);
                }
            }
            else {
                print_slow("            = = = I don't see ssh_config.d = = =");
            }
            # make sure to remove X11UseLocalHost in ssh_config because it can break outgoing ssh
            # X11UseLocalhost is a config directive intended for sshd_config NOT ssh_config, this
            # is documented in man pages.
            my $save_ssh_config = 0;
            if ($is_deb_based) {
                my $lcount=0;
                # perl allows for negative subscripts to avoid $#size - $pos style subscripts
                # we are reversing the original array to iterate in reverse order to take advantage
                # of this negative subscript feature
                for my $line (reverse @ssh_config) {
                    $lcount--;
                    if ($line =~ /^\s*X11UseLocalhost/i) {
                        print_slow("FOUND THE x11uselocalhost at line [$lcount]");
                        splice(@ssh_config, $lcount, 1);
                        $save_ssh_config++;
                    }
                }
                #print Dumper(["ssh_config", \@ssh_config]);
                #print_slow(" ---- ---- Check to see if I found the X11UseLocalhost ---- ---- ");
                #die("testing");
            }
            my @x11forward = grep {/^\s*ForwardX11 *yes *$/} @ssh_config;
            if (@x11forward < 1) {
                #print_slow(" ---- ---- Abt to configure ForwardX11 ---- ---- ");
                my $i = 0;
                my $start_lf_lines = -1;
                my $finish_lf_lines = -1;

                # do not add x11uselocalhost to this file
                my @insert = ("# BEGIN LANforge\n",
                              "    ForwardX11 yes\n",
                              "# END LANforge\n");

                note(" X11: checking for previous ForwardX11 modification\n");
                my @x11lanforge = grep {/^\x23 BEGIN LANforge$/} @ssh_config;
                if (@x11lanforge > 0) {
                    note(" X11: replacing LF begin/end in ssh_config\n");
                    foreach my $line (@ssh_config) {
                        $start_lf_lines = $i if ($line =~ /^\x23 BEGIN LANforge$/);
                        $finish_lf_lines = $i if ($line =~ /^\x23 END LANforge$/);
                        last if ($finish_lf_lines > 0);
                        $i++;
                    }
                    # this is the case where we have BEGIN/END but "forwardx11 no"
                    if ($finish_lf_lines > 0) {
                        note(" X11: replacing LF stanza in ssh_config\n");
                        splice(@ssh_config, $start_lf_lines, 1 + $finish_lf_lines - $start_lf_lines, @insert);
                        $save_ssh_config++;
                    }
                }
                else {
                    note(" X11: checking for disabled ForwardX11\n");
                    @x11forward = grep {/^\s*ForwardX11 no *$/} @ssh_config;
                    if (@x11forward > 0) {
                        note(" X11: found ForwardX11=no declarations...\n");
                        $i = 0;
                        foreach my $line (@ssh_config) {
                            if ($line =~ /^\s*ForwardX11 no *$/) {
                                splice(@ssh_config, $i, 1, @insert);
                                $save_ssh_config++;
                                last;
                            }
                            $i++;
                        }
                    }
                    else {
                        note(" X11: Found no ForwardX11 declarations...\n");
                        $i = 0;
                        foreach my $line (@ssh_config) {
                            $i++;
                            if ($line =~ /^Host \*$/) {
                                splice(@ssh_config, $i, 0, @insert);
                                note(" X11: added ForwardX11 declaration\n");
                                $save_ssh_config++;
                                last;
                            }
                        }
                    } # if/else forwardx11 no
                } # if/else lanforge section
                die("misconfigured ssh_config, bye.") if ($#ssh_config < 1);
            } # ~if no x11forward
            else {
                note("# X11: ssh_config already provides X11 forwarding\n");
            }
            if ($save_ssh_config > 0) {
                note("# Updating ssh_config...\n");
                open($fh_ssh_config, ">", "/etc/ssh/ssh_config") or die("could not write ssh_config");
                print $fh_ssh_config  join("", @ssh_config) . "\n";
                close($fh_ssh_config);
            }
            undef @ssh_config;
            undef @x11forward;
        } # if not macos

        my @motd = `([ -f /etc/motd ] && cat /etc/motd)||:`;
        my @motd_lines = grep {/Destroy this message:/} @motd;
        if (@motd_lines) {
            note("# turning off motd...\n");
            do_cmd(qq(echo "" > /etc/motd), 1);
        }
        note("# SSH: checking sshd_config for PermitRootLogin...\n");

        if (!$is_macos) {
            open(my $fh_sshd_config, "<", "/etc/ssh/sshd_config") or die("could not read sshd_config");
            my @sshd_config = <$fh_sshd_config>;
            close($fh_sshd_config);

            my @sshd_login = grep {/^\s*PermitRootLogin *yes *$/} @sshd_config;
            my @sshd_permit = grep {/^\s*PasswordAuthentication *yes *$/} @sshd_config;
            my @sshd_nopermit = grep {/^\s*PasswordAuthentication\s+no/} @sshd_config;
            if ((@sshd_login < 1) || (@sshd_permit < 1) || (@sshd_nopermit > 0)) {
                my $i = 0;
                my $start_lf_lines = -1;
                my $finish_lf_lines = -1;
                my @insert = ("# BEGIN LANforge\n",
                              "PermitRootLogin yes\n",
                              "PasswordAuthentication yes\n",
                              "# END LANforge\n");
                # remove all PasswordAuthentication lines, we will replace them
                foreach my $line (@sshd_config) {
                    if ($line =~ /^#*\s*PasswordAuthentication (\S+) *$/) {
                        splice(@sshd_config, $i, 1);
                    }
                    $i++;
                }

                note("# SSH: checking for previous ForwardX11 modification\n");
                my @sshd_lanforge = grep {/^# BEGIN LANforge$/} @sshd_config;
                if (@sshd_lanforge > 0) {
                    note(" SSH: replacing LF stanza in sshd_config\n");
                    foreach my $line (@sshd_config) {
                        $start_lf_lines = $i if ($line =~ /^# BEGIN LANforge$/);
                        $finish_lf_lines = $i if ($line =~ /^# END LANforge$/);
                        last if ($finish_lf_lines > 0);
                        $i++;
                    }
                    # this is the case where we have BEGIN/END but "PermitRootLogin no"
                    if ($finish_lf_lines > 0) {
                        note(" SSH: replacing LF stanza in sshd_config\n");
                        splice(@sshd_config, $start_lf_lines, 1 + $finish_lf_lines - $start_lf_lines, @insert);
                    }
                }
                else {
                    note(" SSH: checking for disabled RootLogin\n");
                    @sshd_lanforge = grep {/^\s*PermitRootLogin (\S+)\s*$/} @sshd_config;

                    if (@sshd_lanforge > 0) {
                        note(" SSH: found 'PermitRootLogin no' declarations...\n");
                        $i = 0;
                        foreach my $line (@sshd_config) {
                            if ($line =~ /^\s*PermitRootLogin\s+(\S+)\s*$/) {
                                splice(@sshd_config, $i, 1, @insert);
                                note(" PermitRootLogin==no line 6545 ");
                            }
                            $i++;
                            last if (@sshd_lanforge < $i);
                        }
                    }
                    @sshd_lanforge = grep {/^#\s*PermitRootLogin (\S+) *$/} @sshd_config;
                    if (@sshd_lanforge > 0) {
                        note(" SSH: No PermitRootLogin declarations...\n");
                        $i = 0;
                        foreach my $line (@sshd_config) {
                            if ($line =~ /^#\s*PermitRootLogin (\S+) *$/) {
                                splice(@sshd_config, $i, 1, @insert);
                                note(" SSH: set PermitRootLogin yes declarations\n");
                                last;
                            }
                            $i++;
                        }
                    }
                    else {
                        note(" SSH: No PermitRootLogin declarations...\n");
                        $i = 0;
                        foreach my $line (@sshd_config) {
                            if ($line =~ /^\s*AcceptEnv .*$/) {
                                splice(@sshd_config, $i, 0, @insert);
                                note(" SSH: set PermitRootLogin yes declarations\n");
                                last;
                            }
                            $i++;
                        }
                    }
                }
                die("misconfigured sshd_config, bye.") if (@sshd_config < 1);
                note("# Updating sshd_config...\n");
                open($fh_sshd_config, ">", "/etc/ssh/sshd_config") or die("could not write sshd_config");
                print $fh_sshd_config  join("", @sshd_config) . "\n";
                close($fh_sshd_config);
                warn("# trouble restarting sshd: $!\n")
                    unless !system("systemctl restart sshd.service");
            }
            else {
                note(" ssh: sshd_config already provides root login\n");
            }
        }# if not macos

        note("# Creating blank scenario...\n");
        do_cmd("mkdir -p $::home/DB/BLANK");

        note("# Create reporting directories...\n");
        do_cmd("mkdir -p $::home/report-data", 1);
        if (!$is_macos) {
            if (-d $http_root) {
                if ((!-l "$http_root/report-data") && (!-d "$http_root/report-data")) {
                    if ($do_interop) {
                        # this needs debugging, do not create /root/report-data to begin with
                        do_cmd("ln -s $::home/report-data $http_root/report-data", 1);
                        print_slow("==  ==  ==  check $http_root/report-data symlink: ==  ==  ==",
                                    `ls -l $http_root/report-data`
                                    );
                    }
                    else {
                        do_cmd("ln -s $::home/report-data $http_root/report-data", 1);
                    }

                }
                do_cmd("mkdir -p $::home/html-reports", 1);
                if ((!-l "$http_root/html-reports") && (!-d "$http_root/html-reports")) {
                    do_cmd("ln -s $::home/html-reports $http_root/html-reports", 1);
                }
            }
            else {
                warn("* $http_root not found. Did httpd or apache2 not get installed?");
            }
        }

        if ((-d "/mnt/d2/pub") && $is_fedora && ($::osveri >= 19) && ($::osveri < 34)) {
            note("# copying saved fedora-mate-dconf-panel...");
            my $dcnf_cfg = "/mnt/d2/pub/script-test/f19-mate-panel.lanforge.config.dconf.user";
            if ($::osveri >= 30) {
                $dcnf_cfg = "/mnt/d2/pub/script-test/f30-mate-panel.lanforge.config.dconf.user";
            }
            elsif ($::osveri >= 24) {
                $dcnf_cfg = "/mnt/d2/pub/script-test/f24-mate-panel.lanforge.config.dconf.user";
            }
            if (-f $dcnf_cfg) {
                if (!-d "$::home/.config/dconf") {
                    do_cmd("mkdir -p $::home/.config/dconf", 1);
                }
                if (!-f "$::home/.config/dconf/user.orig") {
                    do_cmd("cp $::home/.config/dconf/user $::home/.config/dconf/user.orig", 1);
                }
                do_cmd("cp $dcnf_cfg $::home/.config/dconf/user", 1);
                do_cmd("dconf update", 1);
            }
        }
        else {
            note("# saved fedora-mate-panel.dconf unavailable, continuing\n");
        }
        if ((-d "/mnt/d2/pub") && $is_fedora && ($::osveri >= 34)) {
            note("# Applying desktop dconf settings (f34-dconf.bash)\n");
            if (-f "/mnt/d2/pub/script-test/f34-dconf.bash") {
                do_cmd("cp /mnt/d2/pub/script-test/f34-dconf.bash $::home", 1);
                if (-x "$::home/f34-dconf.bash") {
                    # do_cmd("su -l lanforge $::home/f34-dconf.bash", 1); # does not run as root well
                    warn("** Run f34-dconf.bash as lanforge in a startx session to configure desktop **");
                }
                else {
                    warn("Unable to find $::home/f34-dconf.bash");
                }
            }
            else {
                note("# Unable to find /mnt/d2/pub/script-test/f34-dconf.bash");
            }
        }

        # debian install
        my $do_bt_tar = 1;
        if ($is_deb_based) {
            print_slow("# updating desktop at $::home/Desktop\n");
            if ($::home !~ /lanforge/) {
                die("we should have home == /home/lanforge by now, right?");
            }
            do_cmd("mkdir -p $::home/Desktop", 1);
            do_cmd("chown lanforge:$user_group $::home/Desktop", 1);
            my $is_lxe = 0;

            if (-f "/usr/share/applications/gnome-terminal.desktop") {
                do_cmd("cp /usr/share/applications/gnome-terminal.desktop $::home/Desktop/Terminal.desktop", 1);
            }
            elsif (-f "/usr/share/app-install/desktop/gnome-terminal\:gnome-terminal.desktop") {
                do_cmd("cp /usr/share/app-install/desktop/gnome-terminal\:gnome-terminal.desktop $::home/Desktop/Terminal.desktop", 1);
            }
            elsif (-f "/usr/share/applications/lxterminal.desktop") {
                $is_lxe = 1;
                if (open(FILE, ">$::home/Desktop/lxterminal.desktop")) {
                    print FILE "[Desktop Entry]\nType=Link\nName=LXTerminal\nIcon=lxterminal\nURL=/usr/share/applications/lxterminal.desktop\n";
                    close(FILE);
                    do_cmd("chown lanforge:$user_group $::home/Desktop/lxterminal.desktop");
                }
            }
            elsif (-f "/usr/share/applications/mate-terminal.desktop") {
                do_cmd("cp /usr/share/applications/mate-terminal.desktop $::home/Desktop/Terminal.desktop", 1);
            }
            elsif (-f "/usr/share/app-install/desktop/mate-terminal\:mate-terminal.desktop") {
                do_cmd("cp /usr/share/app-install/desktop/mate-terminal\:mate-terminal.desktop $::home/Desktop/Terminal.desktop", 1);
            }
            if (-f "$::home/Desktop/Terminal.desktop") {
                do_cmd("chown lanforge:$user_group $::home/Desktop/Terminal.desktop", 1);
                if (!$is_lxe) {
                    do_cmd("chmod a+x $::home/Desktop/Terminal.desktop", 1);
                }
            }
            do_cmd("rm -f $::home/do_interactive_start", 1);
        }

        if ($is_deb_based && $do_bt_tar) {
            # print_slow("===== copy updated libraries =====");
            my @download_these = qw(
                libcrypt.so
                libcrypto.so
                libcrypto.so.1.1
                libldap-2.4.so.2
                libnet.so
                libnet.so.1
                libpcap.so
                libpcap.so.1
                libssh2.so
                libssh2.so.1
                libssl.so.1.1
                libstdc++.so.6
                libtinfo.so
                libtinfo.so.6
            );
            my $equiv_rel = "f21";
            if ($osver =~ /Ubuntu 1[67]/) {
                $equiv_rel = "f21";
                @download_these = (
                    @download_these,
                    qw(
                        libcrypt.so.2
                        libcrypto.so.10
                        libcrypto.so.1.0.2o
                        libcrypto.so.1.1
                        libcrypto.so.1.1.0i
                        libgnutls.so
                        libgnutls.so.28
                        libgnutls.so.28.41.10
                        libhogweed.so
                        libhogweed.so.2
                        libhogweed.so.2.5
                        libldap-2.4.so.2.10.8
                        libldap.so
                        libnettle.so.4
                        libpcre.so
                        libpcre.so.1
                        libpcre.so.1.2.3
                        libsasl2.so
                        libsasl2.so.3
                        libsasl2.so.3.0.0
                        libsctp.so
                        libsctp.so.1
                        libsctp.so.1.0.16
                        libssl.so.10
                        libssl.so.1.0.1k
                        libssl.so.1.0.2o
                        libssl.so.1.1.0i
                        libtinfo.so.6.0
                        libtspi.so.1
                        libtspi.so.1.2.0
                    )
                );
                # libncurses.so.5
                # libncurses.so.5.0
            }
            elsif ($osver =~ /Ubuntu 1[89]/) {
                $equiv_rel = "f27";
                @download_these = (
                    @download_these,
                    qw(
                        libldap-2.4.so.2.10.8
                        libncurses.so.6
                        libncurses.so.6.0
                        libnettle.so.6
                        libpcre.so.1
                        libsasl2.so.3
                        libsctp.so.1
                        libssl.so.1.0.2o
                        libssl.so.1.1.0h
                        libssl.so.1.1.0i
                        libssl.so.10
                        libstdc++.so.6.0.24
                        libtinfo.so.6
                        libtinfo.so.6.0
                        libtirpc.so.3
                    )
                );
            }
            elsif ($osver =~ /Ubuntu 2[01234]/) {
                $equiv_rel = "f30";
                @download_these = (
                    @download_these,
                    qw(
                        libcrypt.so.1
                        libcrypt.so.1.1.0
                        libcrypt.so.2
                        libcrypt.so.2.0.0
                        libcrypto.so.1.1.1g
                        libidn.so
                        libidn.so.12
                        libidn.so.12.6.0
                        liblber-2.4.so.2
                        liblber-2.4.so.2.10.10
                        liblber.so
                        libldap-2.4.so.2.10.10
                        libldap.so
                        libncurses.so.6
                        libncurses.so.6.1
                        libnet.so.1
                        libnettle.so.6
                        libpcap.so.1
                        libpcre.so.1
                        libsasl2.so
                        libsasl2.so.3
                        libsasl2.so.3.0.0
                        libsctp.so.1
                        libssh2.so.1-x64
                        libssl.so.1.1.1g
                        libtinfo.so.6
                    )
                );
            }
            print("Checking for Fedora libraries that LANforge is compiled against. Erase these files if you want to replace them.\n");
            for my $lib_file (@download_these) {
                my $fq_lib_file = "/home/lanforge/$lib_file";
                # print("= = = = = Checking for $fq_lib_file = = = = =\n") if ($debug_on);
                my $dl = "/downloads/$equiv_rel";
                if ($download_host =~ m|/$|) {
                    $dl = "downloads/$equiv_rel";
                }
                if (-f $fq_lib_file && ! -z $fq_lib_file) {
                    print " $fq_lib_file " if ($::debug_on);
                }
                else {
                    print "Fetching ${download_host}${dl}/$lib_file -> $fq_lib_file\n";
                    my $rh_req = newWebRequest("${download_host}${dl}/$lib_file", "$fq_lib_file", "GET");
                    $rh_req->{"resume"} = undef;
                    $rh_req->{"notify_on_fail"} = 1;
                    $rh_req->{"die_on_fail"} = 1;
                    $rh_req->{"notify_on_start"} = 'yes';
                    if (webget($rh_req) != 0) {
                        die("problems at the Circle-K")
                    }
                }
            }
            print "\n";
            print_slow("===== ===== ===== ===== ===== ===== ===== ===== ") if ($::debug_on);
            # make sure that pkexec is suid, this is used for launching wireshark, etc
            chmod (04755, "/usr/bin/pkexec");
        }
        if ($do_bt_tar) {
            note("# applying mitigation for CVE-2021-4034");
            my @pke_locations = ("/usr/sbin/pkexec", "/sbin/pkexec", "/usr/bin/pkexec", "/bin/pkexec");
            for my $pke_filename (@pke_locations) {
                next unless (-f $pke_filename);
                chmod(0755, $pke_filename);
            }
            if ($is_fedora && !$ntwk_mgr_ok) {
                `systemctl is-enabled NetworkManager.service &>/dev/null`;
                if ($? == 0) {
                    warning("\n\nWARNING: Fedora NetworManager.service is enabled!\n\n");
                    print_slow("",
                        "*** *** *** *** *** *** *** *** *** *** *** *** ***",
                        "    WARNING: NetworkManager is running!",
                        "",
                        " NetworkManager is usually disabled on Fedora LANforge installations.",
                        " Press Ctrl-C to stop and disable NetworkManager.",
                        "*** *** *** *** *** *** *** *** *** *** *** *** ***",
                        "");
                }
            }
            print_slow("# installing lfserver from $::tmp_dir/$lfserver\n") if ($::debug_on);
            chdir("$::home") || die "Couldn't chdir to $::home:  $!\n\n";
            do_cmd("tar -xzf $::tmp_dir/$lfserver");
            if ($has_vm_oui) {
                `echo 1 > LF_NO_IFRENAME`;
            }
            # check if we have installed a version of LANforge that brings its own hackrf commands
            # if so, warn if the OS version of HackRF is installed
            if ($is_fedora && -f "LANforgeServer-$::lfver/local/bin/hackrf_info") {
                my @hackrf_pkgs = `rpm -qa hackrf`;
                if ((@hackrf_pkgs > 0) && ($hackrf_pkgs[0] =~ /^h/)) {
                    chomp(@hackrf_pkgs);
                    warn("\n    Fedora version of hackrf package is installed, and that \n"
                        ."conflicts with the version placed in /home/lanforge/local/bin.\n");
                    #print("     Remove this package now? [y|n] ");
                    #my $answer = <STDIN>;
                    #chomp $answer;
                    #if ($answer =~ /^[Yy][EeSs]*/) {
                    do_cmd("rpm -ehv --nodeps $hackrf_pkgs[0]", 1);
                    #}
                    #else {
                    #    print "\n....not removing $hackrf_pkgs[0]\n";
                    #}
                }
            }
            if (!$is_macos && -e "/home/lanforge/local/lib/libhackrf.so.0.8.0") {
                # Clean up any old links.
                do_cmd("rm -f /lib64/libhackrf.so.0 /lib64/libhackrf.so.0.8.0");

                # Soft-link to our custom build libraries.
                do_cmd("ln -s /home/lanforge/local/lib/libhackrf.so.0.8.0 /lib64/libhackrf.so.0");
                do_cmd("ln -s /home/lanforge/local/lib/libhackrf.so.0.8.0 /lib64/libhackrf.so.0.8.0");
            }

            # check for residual libpci libraries that could be left over from a F34 -> F36 transition of build servers
            # this needs to be done after the server install.bash is run because it is what copies them into place.
            # The /home/lanforge/LANforgeServer-5.4.6 directory is never destroyed between upgrades so any libraries
            # imported from LF-Server_F34.x86 archive are never removed
            if ($osver eq "F36") {
                my $libnam = 'libpci.so.3';
                my @local_pci_libs = ( glob("$::home/${libnam}*"),
                                        glob("$::home/LANforgeServer-5.4.6/${libnam}*"));
                if (@local_pci_libs > 0) {
                    #print_slow("* Found libcpi.so.3 libraries that should be removed:",
                    #    Dumper(["local_pci_libs", \@local_pci_libs]),
                    #    "= = = = = == = = = = == = = = = == = = = = == = = = = == = = = = =");
                    for my $fname (@local_pci_libs) {
                        next if (!(defined $fname) || ("" eq $fname));
                        my @stringgrep = `strings $fname | fgrep '.debug'`;
                        if (@stringgrep > 0) {
                            my @matches = grep { /libpci\.so\.3.*?\.fc3[012345]\.(i386|x86_64)\.debug/ } @stringgrep;
                            # print Dumper(["matches", \@matches, "stringgrep", \@stringgrep]);
                            do_cmd("mv -v $fname /var/tmp", 1) if (@matches > 0);
                        }
                    }
                } # ~if local pci libs
            } # if F36
            chdir("$::home/LANforgeServer-$::lfver") || die "Couldn't chdir to $::home/LANforgeServer-$::lfver:  $!\n\n";
            do_cmd("LF_AUTO_INSTALL=1 ./install.bash 2>&1 </dev/null | tee /tmp/lfinstall.txt ");

            if (($::osveri >= 34) && ($lfrls gt "5404")) {
                # move the python 3 version of lf_hackrf into place
                if (-f "lf_hackrf3.py") {
                    if (-f "../lf_hackrf.py") {
                        my $firstline = `head -1 ../lf_hackrf.py`;
                        if ((defined $firstline) && ($firstline ne "")) {
                            chomp($firstline);
                            if (index($firstline, "python2") > 1) {
                                do_cmd("mv ../lf_hackrf.py ../lf_hackrf2.py");
                                do_cmd("cp lf_hackrf3.py ../lf_hackrf.py");
                                chmod(0755, "../lf_hackrf.py");
                            }
                        }
                        else {
                            warn("inspection of $::home/lf_hackrf.py failed")
                        }
                    }
                    else {
                        warn("lf_hackrf.py missing");
                    }
                }
                else {
                    warn("lf_hackrf3.py missing");
                }
            } # ~if modern enough to replace lf_hackrf with py3 version

            if (($lfrls ge "5407") && !$is_macos) {
                note("Checking for permissions to ping...");
                if ( -f "$home/local/bin/ping") {
                    do_cmd("chown root:root $home/local/bin/ping", 1);
                    do_cmd("chmod u+s $home/local/bin/ping", 1);
                }
                else {
                    note("CT ping not found");
                }
            }

            note("# Updating message of the day...\n");
            update_motd();
            note("Checking for LANforge installation warnings...");
            my @lns = `cat /tmp/lfinstall.txt`;
            for ($i = 0; $i < @lns; $i++) {
                my $ln = $lns[$i];
                chomp($ln);
                if (($ln =~ /.*FAILED.*/) ||
                    ($ln =~ /.*ERROR.*/)) {
                    err("$ln\n");
                }
                elsif ($ln =~ /.*WARNING.*/) {
                    warning("$ln\n");
                }
            } # For all btserver install log text.
            if ($skip_pip) {
                note("* skipping pip upgrade after doing server/install.bash\n");
            }
            elsif ((defined $::proxy) && ($::proxy ne "")) {
                note("* skipping update_pip because proxy detected. Use --update_pip option to update pip\n");
            }
            else {
                # appropriate here because we're doing upgrades, not just grub or kernel
                # and we should not have any proxy problems. If we do, it will become obvious
                if (!($is_offline||$use_install_bundle||$skip_pip)) {
                    update_pip();
                }
            }
        } # if do_bt_tar

        if (!$is_macos) {
            # update the apache index styling
            my $apachedir = "/etc/httpd/conf.d";
            if ($is_deb_based) {
                $apachedir = "/etc/apache2/conf-available"
            }
            if (! -d $apachedir) {
                warning("* Apache is not installed? could not find [$apachedir]\n");
                warning("* Not creating autoindex or random www data\n");
                sleep(3) if ($debug_on);
            }
            else {
                if (!-f "$apachedir/autoindex-lf.conf") {
                    note("# updating webserver autoindex configuration [$apachedir/autoindex-lf.conf]...\n");
                    die($!) unless open(FILE, ">", "$apachedir/autoindex-lf.conf");
                    print FILE "# * Created by lf_kinstall.pl *\n";
                    print FILE "#\n";
                    print FILE "#    This file is created only once. The file autodir.css is updated regularly.\n";
                    print FILE "#    If you wish to disable the branding in this file, comment it out\n";
                    print FILE "#    or replace this with an empty file.\n\n";
                    print FILE "IndexStyleSheet ${Q}/autodir.css${Q}\n";
                    print FILE "IndexOptions NameWidth=* SuppressDescription FoldersFirst\n\n";
                    close FILE;
                    if ($is_deb_based) {
                        symlink("/etc/apache2/conf-available/autoindex-lf.conf",
                                "/etc/apache2/conf-available/autoindex-lf.conf");
                    }
                }
                create_random_www_data();
            }
            #config_apache_mgt_port(); # on first installation, this fails because enp0sFoodle isn't spotted
        }

        # Hack to work around library mis-match problem with Fedora 19,
        # and LANforge 5.2.9 and earlier.
        if ($is_fedora && ($::osveri >= 19) &&
            (($::lfver eq "5.2.8") || ($::lfver eq "5.2.9"))) {
            # Native libraries should work fine, but in 5.2.9 LANforge was
            # installing a libcrypto from build machine, but not installing
            # the matching libssl.
            do_cmd("mkdir -p $::home/old_libs", 1);
            do_cmd("mv $::home/libcrypto* $::home/old_libs/", 1);

            # And fixup # 2:
            # Need to get some libraries to make F17 build of suplicant work
            # on Fedora 19.
            if ($is_64) {
                do_cmd("cp $::tmp_dir/libgnutls.so.26-x64-f17 $::home/libgnutls.so.26");
                do_cmd("cp $::tmp_dir/libtasn1.so.3-x64-f17 $::home/libtasn1.so.3");
            }
            else {
                do_cmd("cp $::tmp_dir/libgnutls.so.26-i32-f17 $::home/libgnutls.so.26");
                do_cmd("cp $::tmp_dir/libtasn1.so.3-i32-f17 $::home/libtasn1.so.3");
            }
            create_lf_user() if ($::should_create_user);
            do_cmd("chown lanforge:$user_group $::home/lib*");
        }

        if (!$is_macos) {
            # Add lanforge user to wireshark group...but not a huge deal if
            # Also make sure user lanforge is in root's group.
            # this fails, so continue on if it does fail.
            note("# updating user lanforge group membership for wireshark...\n");
            do_cmd("(grep -q wireshark /etc/group 2>/dev/null && usermod -a -G wireshark lanforge)||:", 1);
            my $lf_groups = `id lanforge`;
            foreach my $group (qw(cdrom dialout root tty video wheel wireshark xorp)) {
                if (index($lf_groups, $group) < 0) {
                    do_cmd("gpasswd -a lanforge $group", 1);
                    note("# Please reboot or logout and login again")
                }
            }
        }
    } # ~if do lanforge

    if ($do_webui) {
        do_cmd("dnf -y module install nodejs:14", 1);
        do_cmd("su -l lanforge -c ${Q}$proxy_env (cd $::home/local/ws-scrcpy && npm install && cd -)${Q}", 1);
        do_cmd("su -l lanforge -c ${Q}$proxy_env (cd $::home/local/ws-scrcpy && npm audit fix && cd -)${Q}", 1);
        do_cmd("dnf -y install rabbitmq-server", 1);
        configure_rabbitmq();
        do_cmd("systemctl enable rabbitmq-server", 1);
        do_cmd("systemctl start rabbitmq-server", 1);
        do_cmd("su -l lanforge -c ${Q}$proxy_env (cd $::home/local/interop-webGUI && python3 manage.py makemigrations && cd -)${Q}", 1);
        do_cmd("su -l lanforge -c ${Q}$proxy_env (cd $::home/local/interop-webGUI && python3 manage.py migrate && cd -)${Q}", 1);
        do_cmd("su -l lanforge -c ${Q}$proxy_env (cd $::home/local/interop-webGUI && python3 createsuperuser.py && cd -)${Q}", 1);
    }
} # ~else we're not creating an install bundle

=pod
 Create a formatted block of text can be printed to a grub.d/50_candela file
 We have to format all the text blocks first because we're likely going to be
 holding grub.d/40_custom open while we read it.
=cut
sub format_grub_file_txt {
    my ($grub_fname,
        $using_serial,
        $lcal_kver,
        $ser_speed,
        $grub_section_hdr,
        $efi_suffix,
        $boot_prefix,
        $root,
        $kargs) = @_;
    # print Dumper({ "grub_fname" => $grub_fname,
    #     "using_serial"          => $using_serial,
    #     "kver"                  => $lcal_kver,
    #     "ser_speed"             => $ser_speed,
    #     "grub_section_hdr"      => $grub_section_hdr,
    #     "efi_suffix"            => $efi_suffix,
    #     "boot_prefix"           => $boot_prefix,
    #     "root"                  => $root,
    #     "kargs"                 => $kargs }) if ($debug_on);
    if (@_ != 9) {
        die("format_grub_file_txt wants 9 arguments.");
    }

    my $ra_lines = [
        "#!/usr/bin/sh",
        "exec tail -n +3 \$0",
        "# Generated by LANforge lf_kinstall.pl",
        "# Be careful not to change the 'exec tail' line above."
    ];

    push(@$ra_lines, "menuentry ${Q}" . grub_menuentry($lcal_kver) . "${Q} {");
    push(@$ra_lines, $grub_section_hdr);
    push(@$ra_lines, "\techo\t${q}Loading " . grub_menuentry($lcal_kver) . " ...${q}");

    push(@$ra_lines, "\tlinux$efi_suffix\t$boot_prefix/ct$lcal_kver.img root=$root ro $kargs",
        "\techo\t'Loading initial ramdisk ...'",
        "\tinitrd$efi_suffix\t$boot_prefix/initrd-ct$lcal_kver.img",
        "}");

    # print Dumper({ "format_grub_file_txt" => $ra_lines }) if ($debug_on);
    return $ra_lines;
} # ~format_grub_file_txt
=pod
   Use save_grub_file to actually write the file to the correct directory
=cut
sub save_grub_file {
    my ($grub_fname, $ra_lines) = @_;
    if ($grub_fname !~ m|^/etc/grub\.d|) {
        $grub_fname = "/etc/grub.d/$grub_fname";
        $grub_fname =~ s|//|/|g;
    }
    note("# saving $grub_fname");
    open(my $grub_fh, ">", $grub_fname) or die("Unable to open $grub_fname: $!");
    print $grub_fh join("\n", @$ra_lines), "\n";
    close $grub_fh;
    chmod(0755, $grub_fname);
} # ~save_grub_file

# reconfigure grub.d files
if ((!$::create_install_bundle) && $do_grub && !($is_chr_ubuntu || $is_arm)) {
    if ($kver =~ /^\s*$/ || $kver =~ /^-1$/) {
        die("* cannot continue with kver[${kver}], no --kver specified?");
    }
    note("# Using kernel: $kver\n");
    my $now = ts();
    my $kfile = "/boot/ct$kver.img";
    my $krd = "/boot/initrd-ct$kver.img";
    if (!$remove_kern) {
        if (!-f "$kfile") {
            die("ERROR:  Could not find kernel file: $kfile\n\n");
        }
        if (!-f "$krd") {
            die("ERROR:  Could not find kernel initrd file: $krd\n\n");
        }
    }
    if ($remove_kern) {
        my $booted_ver = `uname -r`;
        chomp($booted_ver);
        if ($booted_ver eq $kver) {
            err("** do_grub: will not remove presently booted kernel version.");
            exit(1);
        }
    }
    if (-f "$etc_default_grub") {
        my $boot_prefix = ""; # This works for Fedora
        # Grub-2 based system.
        note("# Updating grub-2 defaults in file: $etc_default_grub\n");
        # Update the default config file.
        my $kargs = "";
        my $found_gts = 0;
        my $found_gtc = 0;
        my $blanks = 0;
        my @cmds = `cat $etc_default_grub`;
        if (!-f "$etc_default_grub.lf-backup") {
            do_cmd("cp $etc_default_grub $etc_default_grub.lf-backup.$now");
            do_cmd("chmod 644 $etc_default_grub.lf-backup.$now");
        }
        open(FILE, ">$etc_default_grub") or die "Couldn't open file: $etc_default_grub for writing: $!\n\n";

        if ($do_serial == -1) {
            for ($i = 0; $i < @cmds; $i++) {
                my $ln = $cmds[$i];
                chomp($ln);
                if (($ln =~ /^\s*GRUB_CMDLINE_LINUX=(.*)/) ||
                    ($ln =~ /^\s*GRUB_CMDLINE_LINUX_DEFAULT=(.*)/)) {
                    my $args = $1;
                    # Figure out if we are using serial or not.
                    if ($args =~ /console=ttyS.*\,(\d+)/) {
                        $using_serial = 1;
                        $ser_speed = $1;
                    }
                    last;
                }
            }
        }

        for ($i = 0; $i < @cmds; $i++) {
            my $ln = $cmds[$i];
            chomp($ln);

            # Don't keep printing more and more blank lines....
            if ($ln eq "") {
                $blanks++;
                if ($blanks > 2) {
                    next;
                }
            }
            else {
                $blanks = 0;
            }

            if ($ln =~ /^\s*GRUB_TIMEOUT=(\d+)/) {
                if ($1 < 5) {
                    note("# Increasing timeout to 5 seconds.\n");
                    print FILE "GRUB_TIMEOUT=5\n";
                    next;
                }
            }
            elsif ($ln =~ /^\s*GRUB_DEFAULT=(.*)/) {
                # Will do this line below.
                next;
            }
            elsif (($ln =~ /^\s*GRUB_CMDLINE_LINUX=(.*)/) ||
                ($ln =~ /^\s*GRUB_CMDLINE_LINUX_DEFAULT=(.*)/)) {
                # Ubuntu uses this latter format.
                my $old_args = $1;
                my $cmd = "echo $old_args";
                print "Old-args cmd -:$cmd:-\n";
                # Total hack.  Use the vt_handoff thing on Ubuntu.
                my $put_back = "";
                if ($::osver =~ "Ubuntu 13") {
                    $put_back = " \$vt_handoff";
                }
                $old_args = `$cmd`; # This should do the shell processing needed.
                chomp($old_args);
                # Put back shell-ish commands that were removed improperly.
                $old_args = "$old_args" . $put_back;
                # Put the quotes back.
                $old_args = "\" $old_args \"";
                my $args = $old_args;
                # Remove quiet in line   ; Or end of line
                $args =~ s/ quiet / /g;

                # Remove rhgb
                $args =~ s/ rhgb / /g;

                if ($do_biosdevname != -1) {
                    # Remove previous biosdevname stuff if it exists.
                    $args =~ s/\s+biosdevname=\d+//;

                    if ($do_biosdevname == 1) {
                        note("# Enabling biosdevname..\n");
                        $cmd = " biosdevname=1\"";
                    }
                    else {
                        note("# Disabling biosdevname..\n");
                        $cmd = " biosdevname=0\"";
                    }
                    $args =~ s/\"$/$cmd/;
                }

                if ($do_elevator != -1) {
                    # Remove previous elevator stuff if it exists.
                    $args =~ s/\s+elevator=\S+//;

                    if ($do_elevator == 1) {
                        note("# Setting elevator=noop..\n");
                        $cmd = " elevator=noop\"";
                    }
                    else {
                        note("# Removing elevator..\n");
                        $cmd = "";
                    }
                    $args =~ s/\"$/$cmd/;
                }

                if ($do_iommu != -1) {
                    # Remove previous elevator stuff if it exists.
                    $args =~ s/\s+intel_iommu=\S+//;

                    if ($do_iommu == 1) {
                        # This is default kernel behaviour, no options required.
                        note("# Enabling intel_iommu (if hardware/BIOS otherwise supports it)..\n");
                        $cmd = "";

                        if ((defined $set_swiotlb) && (($set_swiotlb ne "") || ($set_swiotlb > 0))) {
                            $args =~ s/\s+swiotlb=\S+//;
                            $cmd .= " swiotlb=$set_swiotlb\"";
                        }
                    }
                    else {
                        note("# Disabling iommu with: intel_iommu=off\n");
                        $cmd = " intel_iommu=off\"";
                    }
                    $args =~ s/\"$/$cmd/;
                }

                if ($do_nomitigations != -1) {
                    # Remove previous elevator stuff if it exists.
                    $args =~ s/\s+mitigations=\S+//;

                    if ($do_nomitigations == 1) {
                        note("# Disabling all spectre and related mitigations..\n");
                        $cmd = " mitigations=off\"";
                        $args =~ s/\"$/$cmd/;
                    }
                    else {
                        # This is default kernel behaviour (if compiled to support spectre mitigations at all).
                    }
                }

                if ($do_cma != -1) {
                    # apparently 'M' and 'MB' are acceptable suffixes, let's be case insensitive on matching
                    $args =~ s/\s+cma=\d+[MmBb]+//;
                    if ($do_cma == 1) {
                        $do_cma = cma_recommendation();
                    }
                    if ($do_cma == 1) {
                        warn("**BROKEN** Enabling cma=128M bootline param\n");
                        $cmd = qq( cma=).cma_recommendation().qq(M");
                    }
                    if ($do_cma > 1) {
                        if ($do_cma < 64) {
                            warn("* do_cma is < 64M, chosen value will be ${do_cma}M");
                        }
                        $cmd = qq( cma=${do_cma}M");
                    }
                    elsif ($do_cma == 0) {
                        note("# Removing cma= bootline param\n");
                        $cmd = "";
                    }
                    $args =~ s/\"?$/$cmd/;
                }

                if ($do_slub_debug != -1) {
                    $args =~ s/\s+slub_debug//;
                    if ($do_slub_debug == 1) {
                        note("# Enabling slub_debug bootline param\n");
                        $cmd = ' slub_debug"';
                    }
                    else {
                        note("# Removing slub_debug bootline param\n");
                        $cmd = "";
                    }
                    $args =~ s/\"?$/$cmd/;
                }

                if ($do_noaer > -1) {
                    $args =~ s/\s+pci=noaer//;
                    if ($do_noaer == 1) {
                        note("# Enabling pci=noaer bootline param\n");
                        $cmd = ' pci=noaer"';
                    }
                    else {
                        note("# Removing pci=noaer bootline param\n");
                        $cmd = "";
                    }
                    $args =~ s/\"?$/$cmd/;
                }

                if ($do_kmemleak != -1) {
                    # Remove previous kmemleak stuff if it exists.
                    $args =~ s/\s+kmemleak=on//;
                    $args =~ s/\s+kmemleak=off//;

                    if ($do_kmemleak == 1) {
                        note("# Setting kmemleak on..\n");
                        $cmd = " kmemleak=on\"";
                    }
                    else {
                        note("# Setting kmemleak off..\n");
                        $cmd = " kmemleak=off\"";
                    }
                    $args =~ s/\"$/$cmd/;
                }

                if ($disable_audit_logs != -1) {
                    $args =~ s/\s+audit=\d+//;
                    if ($disable_audit_logs == 1) {
                        note("# Diabling audit system...\n");
                        $cmd = " audit=0\"";
                    }
                    else {
                        note("# Enabling audit system...\n");
                        $cmd = " audit=1\"";
                    }
                    $args =~ s/\"$/$cmd/;
                }
                if ($do_selinux != -1) {
                    # Remove previous selinux stuff if it exists.
                    $args =~ s/\s+selinux=\d+//;

                    if ($do_selinux == 1) {
                        note("# Setting selinux on..\n");
                        $cmd = " selinux=1\"";
                    }
                    else {
                        note("# Setting selinux off..\n");
                        $cmd = " selinux=0\"";
                    }
                    $args =~ s/\"$/$cmd/;
                }

                if ($do_serial != -1) {
                    # Remove previous LF console stuff if it exists.
                    $args =~ s/\s+console=tty\S+\s+console=tty\S+\,\S+//;
                    if (!($args =~ /\"$/)) {
                        $args = "$args\"";
                    }

                    if ($do_serial == 1) {
                        note("# Adding serial-console logic, tty: $ttys\n");
                        # Add serial console logic.
                        $cmd = " console=tty0 console=${ttys},${ser_speed}n8\"";
                        $args =~ s/\"$/$cmd/;
                        $using_serial = 1;
                    }
                    else {
                        note("# NOT using serial console...\n");
                        if (!$args =~ /console=tty0/) {
                            $cmd = " console=tty0\"";
                            $args =~ s/\"$/$cmd/;
                        }
                    }
                } # ~do_serial

                # forcefsck
                # https://www.linuxuprising.com/2019/05/how-to-force-fsck-filesystem.html
                # we want to make sure that last column of fstab is 1 for l and 2 for every other
                # ext4 partition
                if ($args !~ /fsck\.mode=/) {
                    $args =~ s/\"$/ fsck\.mode=force\"/;
                }
                if ($args !~ /fsck\.repair=/) {
                    $args =~ s/\"$/ fsck\.repair=yes\"/;
                }

                if (!($args =~ /\"$/)) {
                    $args = "$args\"";
                }
                if ($args =~ /^\"(.*)\"$/) {
                    $kargs = $1;
                }
                $args =~ s/\s*\"\s*/\"/g;
                if ($old_args ne $args) {
                    note("# Updating kernel command line, old:\n$old_args\nnew:\n$args\n");
                    if ($ln =~ /^\s*GRUB_CMDLINE_LINUX=(.*)/) {
                        print FILE "GRUB_CMDLINE_LINUX=$args\n";
                    }
                    else {
                        print FILE "GRUB_CMDLINE_LINUX_DEFAULT=$args\n";
                    }
                    next;
                }
            } # ~if has GRUB_CMDLINE_LINUX
            elsif (($ln =~ /^\s*GRUB_TERMINAL=serial/) ||
                ($ln =~ /^\s*GRUB_TERMINAL=\"serial.*/)) {
                if (!$found_gts) {
                    $found_gts = 1;
                    if ($using_serial) {
                        if ($do_serial == -1) {
                            print FILE $ln . "\n";
                        }
                        elsif ($do_serial == 1) {
                            print FILE "GRUB_TERMINAL=\"serial console\"\n";
                        }
                    }
                }
                next;
            }
            elsif ($ln =~ /^\s*GRUB_SERIAL_COMMAND=(.*)/) {
                if (!$found_gtc) {
                    my $old_args = $1;
                    my $args = $old_args;
                    $found_gtc = 1;
                    if ($do_serial == -1) {
                        print FILE $ln . "\n";
                    }
                    elsif ($do_serial == 1) {
                        if (!($args =~ /^\"serial --unit=0 --speed=$ser_speed --word=8 --parity=no --stop=1\"$/)) {
                            # Update it with new speed
                            $args = "\"serial --unit=0 --speed=$ser_speed --word=8 --parity=no --stop=1\"";
                            note("# Updating serial command arguments, old:\n$old_args\nnew:\n$args\n");
                        }
                        print FILE "GRUB_SERIAL_COMMAND=$args\n";
                    }
                }
                next;
            }
            elsif ($ln =~ /^\s*GRUB_HIDDEN_TIMEOUT=(.*)/) {
                note("# Disabling GRUB_HIDDEN_TIMEOUT.\n");
                print FILE "#$ln\n";
                next;
            }
            elsif ($ln =~ /^\s*GRUB_HIDDEN_TIMEOUT_QUIET=(.*)/) {
                note("# Disabling GRUB_HIDDEN_TIMEOUT_QUIET.\n");
                print FILE "#$ln\n";
                next;
            }

            # Line is un-changed.
            print FILE "$ln\n";
        } # ~for each line in etc/default/grub
        if ($kver =~ /^\s*$/ || $kver =~ /^-1$/) {
            die("* cannot continue with kver[${kver}], no --kver specified?");
        }
        note("# Using serial: $using_serial  found-gtc: $found_gtc  found_gts: $found_gts\n");
        $cmd = grub_menuentry($kver);
        note("# Setting default kernel to: $cmd\n");

        # Grub2 Wiki:
        #  Note: There are other, simpler, ways of setting the default entry, but
        #  they are prone to error if/when grub2-mkconfig is re-run. These include
        #  directly setting the default in /boot/grub2/grub.cfg or setting
        #  GRUB_DEFAULT to either a number or an entry title in /etc/default/grub.
        #  Neither of these methods is recommended.
        #  for grub2-set-default this value needs to be 'saved'
        #  https://www.gnu.org/software/grub/manual/grub/html_node/Simple-configuration.html
        # Fedora 24 does not appear to have a working grubenv feature: https://www.gnu.org/software/grub/manual/grub/html_node/Environment-block.html
        # this appears to work on F27. Thus, only on F27 and later can we use
        # GRUB_DEFAULT=saved

        if ($::osveri <= 24) {
            print FILE "GRUB_DEFAULT='$cmd'\n";
        }
        else {
            print FILE "GRUB_DEFAULT=saved\n";
        }

        if ($using_serial == 1) {
            if (!$found_gts) {
                $cmd = "GRUB_TERMINAL=\"serial console\"";
                note("# Adding line: $cmd\n");
                print FILE "$cmd\n";
            }
            if (!$found_gtc) {
                $cmd = "GRUB_SERIAL_COMMAND=\"serial --unit=0 --speed=$ser_speed --word=8 --parity=no --stop=1\"";
                note("# Adding line: $cmd\n");
                print FILE "$cmd\n";
            }
        }
        close FILE;

        # Organize LANforge kernels into individual 50_candela_x files.
        # this has the advantage of being able to be cleaned up by
        # other scripts, like check_large_files, and erased kernels
        # can be taken out of the grub menu without doing a kinstall,
        # rather just using a normal grub2-mkconfig command.
        # First, figure out the default stuff by looking at existing
        # menu-entries in the output /boot/grub2/grub.cfg This is not
        # a file we want to edit, but collect existing boot entries from.
        my $in_section = 0;
        my $grub_section_hdr = "";
        my $root = "";
        my $efi_suffix = "";

        # this section looks for grub_section_hdr, root and efi_suffix
        # TODO: turn this into a subroutine
        @cmds = `cat $grub_cfg`; # looking thru /boot/grub2/grub.cfg can be wrong
        for ($i = 0; $i < @cmds; $i++) {
            my $ln = $cmds[$i];
            chomp($ln);
            #print "$grub_cfg: in: $in_section, root: $root  line: $ln\n";
            if ($ln =~ /^\s*menuentry\s+.*\{\s*$/) {
                $in_section = 1;
            }
            else {
                if ($in_section) {
                    my $done_with_hdr = 0;
                    if ($ln =~ /\s*echo\s+.*Loading/ || $ln =~ /fwconfig|fwsetup/) {
                        $done_with_hdr = 1;
                        $i++;
                    }
                    if (!$done_with_hdr) {
                        # Fedora 19 doesn't have the 'Loading' line in it, so detect
                        # header ending in this way as well.
                        if ($ln =~ /^\s*linux\s+.*/) {
                            $done_with_hdr = 1;
                        }
                        elsif ($ln =~ /^\s*linux16\s+.*/) {
                            $done_with_hdr = 1;
                            $efi_suffix = "16"; # This is needed on a new Nexcom system
                        }
                        elsif ($ln =~ /^\s*linuxefi\s+.*/) {
                            # Make fedora 20 efi configurations work.
                            $done_with_hdr = 1;
                            $efi_suffix = "efi";
                        }
                    }
                    if ($done_with_hdr) {
                        # Find the 'linux' line to extract root=foo
                        # hard to notice that we're using nested loop to speed up the outer loop index variable

                        for (; $i < @cmds; $i++) {
                            $ln = $cmds[$i];
                            chomp($ln);
                            # if we come across a bootmenu item that is 'fwsetup' that is for an UEFI
                            # boot entry and we've dropped into the wrong area
                            if ($ln =~ /fwsetup/) {
                                die("No configured to handle fwsetup");
                            }
                            if ($ln =~ /.*\s+root=(\S+)\s+ro\s+.*/) {
                                $root = $1;

                                if ($ln =~ /^\s*linux\s+(\S+)\/vmlin.*/) {
                                    $boot_prefix = $1;
                                }
                                last;
                            }
                        } # ~for each cmd to find boot prefix
                        last;
                    } # ~if done with header
                    else {
                        $grub_section_hdr = "$grub_section_hdr$ln\n";
                    }
                } # ~if in_section
                else {
                    # Find the 'linux' line to extract root=foo
                    if ($ln =~ /.*=\"root=(\S+)\s+ro\s+.*/) {
                        $root = $1;
                        if ($ln =~ /^\s*linux\s+(\S+)\/vmlin.*/) {
                            $boot_prefix = $1;
                        }
                    } # ~if root=
                }     # ~else not in_section
            }         # else not in section, looking for menuenuentry
        }             # ~foreach line looking for $boot_prefix, $efi_suffix

        note("# Updating custom grub file: $etc_grubd_40_custom  prefix: $boot_prefix\n");
        my $found_ct = 0;
        $blanks = 0;

        my @grub_file_list1 = glob("/etc/grub.d/{4,5}0*");
        # print Dumper({"grub_file_list" => \@grub_file_list1});
        # ignore backup files
        my $found_kver = 0;
        my @grub_file_list2 = ();
        my @matching_list = ();
        if ($kver =~ /^\s*$/ || $kver =~ /^-1$/) {
            die("* cannot continue with kver[${kver}], no --kver specified?");
        }
        for my $fname (sort @grub_file_list1) {
            next if ($fname =~ /lf-backup$/);
            # since kver contains metacharacters ([.+]) we should not
            # use a regex to match but an exact substring
            if (index($fname, $kver) > -1) {
                note("do_grub: $fname matches kver[$kver]\n");
                push(@matching_list, $fname);
                $found_kver++;
            }
            push(@grub_file_list2, $fname);
        }
        undef @grub_file_list1;
        if ($remove_kern) {
            if ($found_kver > 1) {
                err("** remove_kver: $kver found in multiple file names. Refusing to remove multiple files.");
                print Dumper([ "matching_list", \@matching_list ]) if ($debug_on);
                exit(1);
            }
            if ($found_kver == 1) {
                note("# remove_kver: deleting $matching_list[0]\n");
                if (-f $matching_list[0]) {
                    unlink($matching_list[0]);
                }
                else {
                    err("** Unable to find $matching_list[0]");
                    exit(1);
                }
            }
        }

        ## REFORMAT 40_custom
        my %section_map = (
            "40_custom" => []
        );

        # unlink("/root/.${grub_40}.lf-backup") if ( -f "/root/.${grub_40}.lf-backup");
        do_cmd("cp $etc_grubd_40_custom /root/.${grub_40}.lf-backup.$now");
        do_cmd("chmod 644 /root/.${grub_40}.lf-backup.$now");
        if ($kver =~ /^\s*$/ || $kver =~ /^-1$/) {
            die("* cannot continue with kver[${kver}], no --kver specified?");
        }

        # remove old kernel files from /boot
        if ($remove_kern) {
            my @remove_these = ();
            push(@remove_these, glob("/boot/ct${kver}.img"));
            push(@remove_these, glob("/boot/initrd-ct${kver}.img"));
            push(@remove_these, glob("/boot/System.map-${kver}*"));
            # print Dumper([ "remove_these", \@remove_these ]) if ($debug_on);
            for my $remove_fname (@remove_these) {
                note(" removing [$remove_fname] ");
                if (-f $remove_fname) {
                    unlink($remove_fname);
                }
                else {
                    err("** unable to remove [$remove_fname] $!\n");
                }
            }
            # remove the associated /lib/modules directory as well
            if (-d "/lib/modules/$kver") {
                note("# removing kernel modules /lib/modules/$kver\n");
                do_cmd("rm -rf /lib/modules/$kver");
            }
            else {
                err("** unable to find [/lib/modules/$kver]\n");
            }
            err("* Please select new default kernel. The first Distro kernel on the list will load on next reboot.\n");
            my @remaining = ();
            push(@remaining, glob("/lib/modules/*\+"));
            for my $suggestion (sort @remaining) {
                $suggestion =~ s|/lib/modules/||g;
                # ignore possibly mis-removed kernels that have modules left over
                next if (!-f "/boot/ct${suggestion}.img");
                err("* suggestion: $0 --kver $suggestion --do_kern\n");
                last;
            }
        } # ~remove kernel

        # make a list of ct kernels and matching /lib/modules directories; turn these into the
        # the section map, and warn if any do not match up
        my @ct_kernels = glob("/boot/ct*img");
        my @ct_libdirs = glob("/lib/modules/*+");
        die("Unable to find any Candela kernels in /boot. Cannot continue.")
            if (@ct_kernels < 1);

        $grub_section_hdr = "# blank" if ($grub_section_hdr =~ /^\s*$/s);
        for my $ctkern (@ct_kernels) {
            my ($verno) = $ctkern =~ /ct(.*?)\.img/;
            die("bad pattern match for verno: [$ctkern]") if (!(defined $verno));
            my @vernorez = map {index($_, $verno) > 1} @ct_libdirs;
            if (@vernorez < 1) {
                print Dumper([ "ct_libdirs", \@ct_libdirs ]);
                die("Unable to find kernel [$verno] in ct_libdirs, missing module directory?");
            }
            my $grub_fn = "50_candela_$verno";
            # print("GRUB SEC HRD $grub_section_hdr\n");

            $section_map{$grub_fn} = format_grub_file_txt($grub_fn, $using_serial, $verno, $ser_speed, $grub_section_hdr,
                $efi_suffix, $boot_prefix, $root, $kargs);;
        }
        # so if we survived that, we need to create the 50_candela grub files

        if ((keys %section_map) < 2) {
            print(Dumper([ "section_map", \%section_map ]));
            die("Too few grub entries, debugging needed");
        }
        # TODO: this flag should probe and set before any other parsing
        if (!$found_ct && !$remove_kern) {
            note("# Didn't find existing ct section..adding new one.\n");
            if ($kver =~ /^\s*$/ || $kver =~ /^-1$/) {
                die("* cannot continue with kver[${kver}], no --kver specified?");
            }
            my $grub_fn = "50_candela_$kver"; # kver should have been set
            # print("GRUB SEC HRD $grub_section_hdr\n");
            my $ra_grub_entry = format_grub_file_txt($grub_fn, $using_serial, $kver, $ser_speed, $grub_section_hdr, $efi_suffix,
                $boot_prefix, $root, $kargs);
            $section_map{$grub_fn} = $ra_grub_entry;
        }
        # print(Dumper([ "section_map", \%section_map ])) if ($debug_on);
        my $grub_cfg_bak = "/root/.${grub_40}.lf-backup.$now";
        if (!-f $grub_cfg_bak) {
            do_cmd("cp $grub_cfg $grub_cfg_bak");
            do_cmd("chmod 644 $grub_cfg_bak");
        }
        # while it was inefficient to just re-create the 40_custom file there, it will be cleaner on the
        # logic to not change it for now, and re-create
        while (my ($grub_fn, $ra_grub_entry) = each(%section_map)) {
            die("about to save a blank file to [$grub_fn]") if (!defined($ra_grub_entry));
            if ($remove_kern && (index($grub_fn, $kver) > -1)) {
                die("do_grub: --remove_kern specified yet about to save_grub_file($kver)");
            }
            save_grub_file($grub_fn, $ra_grub_entry);
        }

        #$cmd = "grub2-mkconfig > $grub_cfg";
        #Note on Fedora 18 the > method did not generate any errors whereas -o is
        #more helpful.
        #Grub2 Wiki:
        #  /etc/default/grub and ensure this line exists: GRUB_DEFAULT=saved
        #  grub2-mkconfig -o /boot/grub2/grub.cfg
        #  List all possible menu entries: grep ^menuentry /boot/grub2/grub.cfg | cut -d "'" -f2
        #  NOTE: ' and not " which lf_kinstall uses.
        #  Set default menu entry: grub2-set-default <menu entry title>
        #  Verify default menu entry in /boot/grub2/grubenv: grub2-editenv list

        # Fedora Wiki now has cautions about how long to use grub2-install with UEFI systems.
        # https://fedoraproject.org/wiki/GRUB_2
        my $grub_set_default = ($grub_mkconfig =~ /grub2/) ? "grub2-set-default" : "grub-set-default";
        note("- - - - $grub_set_default ${q}" . grub_menuentry() . "${q} - - - - - - \n") if ($debug_on);
        do_cmd("$grub_set_default ${q}" . grub_menuentry() . "${q}");
        $cmd = "$grub_mkconfig -o $grub_cfg";
        note("\n\n# Re-generating grub2 config file with command: $cmd\n");
        do_cmd($cmd);
    } # ~if -e grub_dflt for grub2
    else {
        # Grub1 configuration
        # We are increasingly unlikely to see grub1
        note("# Updating grub-1 file: $grub1_cfg\n");
        if ($remove_kern) {
            err("** do_grub: --remove_kern not supported for grub-1 formats");
        }
        my $kargs = "";
        my $found_gts = 0;
        my $found_gtc = 0;
        my $blanks = 0;
        my $cur_default = 0;
        my $kernel_sections = 0; # counter
        my $replacing_existing = 0;

        my @cmds = `cat $grub1_cfg`;
        open(FILE, ">$grub1_cfg") or die "Couldn't open file: $grub1_cfg for writing: $!\n\n";

        if ($do_serial == -1) {
            for ($i = 0; $i < @cmds; $i++) {
                my $ln = $cmds[$i];
                chomp($ln);
                if ($ln =~ /^\s*serial\s+.*\s+--speed=(\d+).*/) {
                    $using_serial = 1;
                    $ser_speed = $1;
                    last;
                }
            }
        }
        else {
            $using_serial = $do_serial;
        }

        for ($i = 0; $i < @cmds; $i++) {
            my $ln = $cmds[$i];
            chomp($ln);

            # Don't keep printing more and more blank lines....
            if ($ln eq "") {
                $blanks++;
                if ($blanks > 2) {
                    next;
                }
            }
            else {
                $blanks = 0;
            }

            if ($ln =~ /^\s*timeout=(\d+)/) {
                if ($1 < 5) {
                    note("# Increasing timeout to 5 seconds.\n");
                    print FILE "timeout=5\n";
                    next;
                }
            }
            elsif ($ln =~ /^\s*default=(.*)/) {
                $cur_default = $1;
                print FILE "default=$cur_default\n";
                next;
            }
            elsif ($ln =~ /^\s*serial\s+(.*)/) {
                if (!$found_gts) {
                    $found_gts = 1;
                    if ($using_serial) {
                        if ($do_serial == -1) {
                            print FILE $ln . "\n";
                        }
                        else {
                            print FILE "serial --unit=0 --speed=$ser_speed --word=8 --parity=no --stop=1\n";
                        }
                    }
                }
                next;
            }
            elsif ($ln =~ /^\s*terminal\s+(.*)/) {
                if ($found_gts) {
                    if (!$found_gtc) {
                        $found_gtc = 1;
                        if ($do_serial == -1) {
                            print FILE $ln . "\n";
                        }
                        elsif ($do_serial == 1) {
                            print FILE "terminal --timeout=10 serial console\n";
                        }
                    }
                    next;
                }
            }
            elsif ($ln =~ /^title\s+(.*)/) {
                my $old_title = $1;
                my $new_title;

                if ($do_serial == 1) {
                    if (!$found_gts) {
                        $found_gts = 1;
                        print FILE "serial --unit=0 --speed=$ser_speed --word=8 --parity=no --stop=1\n";
                        print FILE "terminal --timeout=10 serial console\n\n";
                    }
                }

                $new_title = grub_menuentry();

                if ($cur_default == $kernel_sections) {
                    # Our settings need to go here..make a copy if it's different than what
                    # we are about to load.
                    my $old_section = "";

                    note("#old-title -:$old_title:-\n#new-title -:$new_title:-\n");

                    if ($old_title eq $new_title) {
                        note("# Replacing existing kernel section.\n");
                        $replacing_existing = 1;
                    }
                    else {
                        $old_section = "$ln\n";
                    }

                    print FILE "title $new_title\n";
                    # Read the rest of the section
                    while (($i + 1) < @cmds) {
                        my $ln = $cmds[$i + 1];
                        chomp($ln);
                        if (($ln eq "") || (!($ln =~ /^\s+\S+/))) {
                            last;
                        }
                        $i++;

                        if (!$replacing_existing) {
                            $old_section .= "$ln\n";
                        }

                        if ($ln =~ /^\s+kernel\s+(.*)/) {
                            # Deal with 'kernel' line
                            my $old_args = $1;
                            my $args = " $old_args ";
                            # Remove quiet in line
                            $args =~ s/ quiet / /g;
                            # Remove rhgb
                            $args =~ s/ rhgb / /g;
                            if ($do_biosdevname != -1) {
                                # Remove previous biosdevname stuff if it exists.
                                $args =~ s/\s+biosdevname=\d+//;
                                if ($do_biosdevname == 1) {
                                    note("# Enabling biosdevname..\n");
                                    $cmd = " biosdevname=1";
                                }
                                else {
                                    note("# Disabling biosdevname..\n");
                                    $cmd = " biosdevname=0";
                                }
                                $args .= $cmd;
                            }
                            if ($do_elevator != -1) {
                                # Remove previous elevator stuff if it exists.
                                $args =~ s/\s+elevator=\S+//;
                                if ($do_elevator == 1) {
                                    note("# Enabling elevator=noop..\n");
                                    $cmd = " elevator=noop";
                                }
                                else {
                                    note("# Removing elevator..\n");
                                    $cmd = "";
                                }
                                $args .= $cmd;
                            }

                            if ($do_iommu != -1) {
                                # Remove previous elevator stuff if it exists.
                                $args =~ s/\s+intel_iommu=\S+//;

                                if ($do_iommu == 1) {
                                    # This is default kernel behaviour, no options required.
                                    note("# Enabling intel_iommu (if hardware/BIOS otherwise supports it)..\n");
                                    $cmd = "";
                                }
                                else {
                                    note("# Disabling iommu with: intel_iommu=off\n");
                                    $cmd = " intel_iommu=off";
                                }
                                $args .= $cmd;
                            }

                            if ($do_kmemleak != -1) {
                                # Remove previous kmemleak stuff if it exists.
                                $args =~ s/\s+kmemleak=on//;
                                $args =~ s/\s+kmemleak=off//;

                                if ($do_kmemleak == 1) {
                                    note("# Setting kmemleak on..\n");
                                    $cmd = " kmemleak=on";
                                }
                                else {
                                    note("# Setting kmemleak off..\n");
                                    $cmd = " kmemleak=off";
                                }
                                $args .= $cmd;
                            }

                            if ($do_selinux != -1) {
                                # Remove previous selinux stuff if it exists.
                                $args =~ s/\s+selinux=\d+//;

                                if ($do_selinux == 1) {
                                    note("# Setting selinux on..\n");
                                    $cmd = " selinux=1";
                                }
                                else {
                                    note("# Setting selinux off..\n");
                                    $cmd = " selinux=0";
                                }
                                $args .= $cmd;
                            }

                            # Deprecated sysfs for older releases.
                            if ($::osveri < 11) {
                                $args =~ s/\s+sysfs.deprecated=\d+//;
                                $args .= " sysfs.deprecated=1";
                            }
                            if ($do_serial != -1) {
                                # Remove previous LF console stuff if it exists.
                                $args =~ s/\s+console=tty0\s+console=tty\S+\,\S+//;
                                if ($do_serial == 1) {
                                    note("# Adding serial-console logic:  tty: $ttys\n");
                                    # Add serial console logic.
                                    $args .= " console=tty0 console=${ttys},${ser_speed}n8";
                                }
                                else {
                                    note("# NOT using serial console...\n");
                                    if (!$args =~ /console=tty0/) {
                                        $args .= " console=tty0";
                                    }
                                }
                            }
                            # First token is our kernel version..fix that up.
                            $args =~ /(\S+)\s+(.*)/;
                            my $old_ki = $1;
                            $args = "/ct$kver.img $2";
                            if ($old_ki =~ /^\/boot\/.*/) {
                                print FILE "\tkernel /boot$args\n";
                            }
                            else {
                                print FILE "\tkernel $args\n";
                            }
                        }
                        elsif ($ln =~ /^\s+initrd\s+(.*)/) {
                            if ($ln =~ /^\s+initrd \/boot\/.*/) {
                                print FILE "\tinitrd /boot/initrd-ct$kver.img\n";
                            }
                            else {
                                print FILE "\tinitrd /initrd-ct$kver.img\n";
                            }
                        }
                        else {
                            print FILE "$ln\n";
                        }
                    } # while in section
                    if ($old_section ne "") {
                        note("# Printing old section: $old_section\n");
                        print FILE "\n$old_section\n";
                        $old_section = "";
                    }
                    $kernel_sections++;
                    next;
                }
                else {
                    # If title matches our title..get rid of the entry
                    if ($old_title eq $new_title) {
                        while (($i + 1) < @cmds) {
                            my $ln = $cmds[$i + 1];
                            chomp($ln);
                            if (($ln eq "") || (!($ln =~ /^\s+\S+/))) {
                                last;
                            }
                            $i++;
                        }
                        $kernel_sections++;
                        next;
                    }
                }
                $kernel_sections++;
            } # found a kernel section starting with 'title'
            # Line is un-changed.
            print FILE "$ln\n";
        } # for all lines in the grub1 file
    }     # grub-1 configuration:  /etc/grub.conf typically
    close FILE;

    # check if we are on a lvm mirrored system
    if (-e "/dev/disk/by-partlabel/boot_b") {
        note("# discovered lvm boot_b partition\n");
        if (-d "/boot2") {
            my @lines = `mount | grep boot2`;
            if (@lines < 1) {
                `mount /boot2`;
            }
            @lines = `mount | grep boot2`;
            if (@lines < 1) {
                err("** Unable to see /boot2, cannot backup boot configuration **\n");
            }
            else {
                # best to remove all the contents of boot2 because whatever is on there
                # could be filling the partition to 100% and we'll fail here when rsync
                # cannot continue.
                do_cmd('find /boot2 -type f -exec rm -f {} \;', 1);
                do_cmd("rsync -a /boot/ /boot2/", 1);
                my $target_name = `basename /lib/grub/*`;
                chomp $target_name;
                my $sdb = `readlink /dev/disk/by-partlabel/boot_b`;
                chomp $sdb;
                #print "SDB[$sdb]\n";
                $sdb =~ s|[./]||g;
                #print "SDB[$sdb]\n";
                if ($sdb =~ m|vdb|) {
                    $sdb = "/dev/vdb";
                }
                elsif ($sdb =~ m|sdb|) {
                    $sdb = "/dev/sdb";
                }
                elsif ($sdb =~ m|nvme|) {
                    $sdb = "/dev/" . substr($sdb, 0, 7);
                }
                #print "SDB[$sdb]\n";

                do_cmd("grub2-install --target $target_name $sdb")
            }
        }
        else {
            err("** See a boot_b partition but no /boot2 mountpoint. Cannot mirror /boot.\n");
        }
    }
    else {
        note("No /boot2 or boot_b partition seen.\n");
    }
} # do-grub
else {
    note("# do-grub: $do_grub  chr-ubuntu: $is_chr_ubuntu  is-arm: $is_arm\n");
    note("# Skipping grub modification, LANforge specific kernel will not be installed.\n");
    note("#  Some features (Armageddon, virtual wifi stations, etc may not work, but most normal\n");
    note("#  traffic generation and network emulation features will work fine.\n");
}

if (!$::create_install_bundle && ($do_serial == 1)) {
    checkTtyConf();
}
if (!$::create_install_bundle && $do_services) {
    my $fok = 0;
    if ($is_deb_based) {
        if (-f "/etc/init/ufw.conf") {
            note("# Disabling firewall (it inteferes with LANforge in some cases.)\n");
            do_cmd("mv /etc/init/ufw.conf /etc/init/ufw.conf.orig");
        }
        if ($do_interop) {
            print "Entirely disable firewall\n";
            do_cmd("systemctl disable ufw.service", 1);
            do_cmd("systemctl stop ufw.service", 1);
            if ($regen_nm_conf > 0) {
                networkmanager_interop_config();
            }
            else {
                print_slow("* do_interop told to ignore regen_nm_conf option");
            }
        }
        elsif ($is_rpi3 || $is_rpi4 || $is_rpi5) {
            if (-f "/etc/init/network-manager.conf") {
                warning("# Disabling NetworkManager (it interferes with LANforge)\n");
                do_cmd("mv /etc/init/network-manager.conf /etc/init/network-manager.conf.orig");
            }
            if ($use_systemctl) {
                do_cmd("systemctl daemon-reload");

                do_cmd("systemctl disable abrtd.service", 1) if (-f "/lib/systemd/system/abrtd.service");
                do_cmd("systemctl disable avahi-daemon.service", 1);
                #do_cmd("systemctl disable firewalld.service");
                #do_cmd("systemctl mask firewalld.service", 1);

                # disable these
                do_cmd("(systemctl is-enabled -q ip6tables.service &>/dev/null && systemctl disable ip6tables.service)||:", 1);
                do_cmd("(systemctl is-enabled -q iptables.service  &>/dev/null && systemctl disable iptables.service )||:", 1);
                do_cmd("(systemctl is-enabled -q sendmail.service  &>/dev/null && systemctl disable sendmail.service )||:", 1);
                do_cmd("(systemctl is-enabled -q sm-client.service &>/dev/null && systemctl disable sm-client.service)||:", 1);
                do_cmd("(systemctl is-enabled -q libvirtd.service  &>/dev/null && systemctl disable libvirtd         )||:", 1);
                do_cmd("(systemctl is-enabled -q snapd.service     &>/dev/null && systemctl disable snapd.service    )||:", 1);
                do_cmd("(systemctl is-enabled -q lxd.service       &>/dev/null && systemctl disable lxd.service      )||:", 1);
                do_cmd("(systemctl is-enabled -q auditd.service    &>/dev/null && systemctl disable auditd.service   )||:", 1);
                do_cmd("(systemctl is-enabled -q systemd-journald-audit &>/dev/null && systemctl disable systemd-journald-audit.socket )||:", 1);
                do_cmd("systemctl mask systemd-journald-audit.socket ||:", 1);

                # enable these
                # network.service is a nutty one, it's typically not a native systemd unit on fedora
                #do_cmd("systemctl enable network.service ||true");
                #do_cmd("[ -x /sbin/chkconfig ] && /sbin/chkconfig network on");
                do_cmd("systemctl enable ssh.service", 1);
                do_cmd("systemctl start ssh.service", 1);
                do_cmd("systemctl enable cups.service", 1);
                do_cmd("systemctl start cups.service", 1);
                do_cmd("systemctl enable xrdp.service", 1);
            }

            # Do not uninstall network manager. It breaks the desktop and updates. Putting desktop components
            # on hold (apt-mark hold) will block updates. Just make sure NetworkManager is disabled and
            # is non-executable

            note("* Enabling eth0 dhcp...\n");
            my $fn = "/etc/network/interfaces";
            my @lines = `cat $fn`;
            my @matches = grep {/^iface eth0 inet /} @lines;
            if (@matches < 1) {
                push(@lines, "auto eth0");
                push(@lines, "iface eth0 inet dhcp");
                open(my $fh, ">", $fn) || die("Unable to write $fn");
                print $fh join("\n", @lines) . "\n";
                close $fh;
            }

            if ($::osver =~ /Ubuntu 16/) {
                # systemctl list-unit-files Network* | awk '/NetworkManager.service/ {print $2}'
                my @unitlist = `systemctl list-unit-files Network*`;
                my @nm_units = grep {/NetworkManager\.service/} @unitlist;
                if (@nm_units > 0 && (grep(/disabled/, @nm_units) < 1)) {
                    note("* attempting to disable Network manager\n");
                    do_cmd("(systemctl -q is-enabled NetworkManager &>/dev/null && systemctl disable NetworkManager)||:", 1);
                    do_cmd("(systemctl -q is-enabled network-manager &>/dev/null && systemctl disable network-manager)||:", 1);
                    if (-x "/usr/sbin/NetworkManager") {
                        chmod(0644, "/usr/sbin/NetworkManager");
                    }
                    #do_cmd("(systemctl -q is-enabled dnsmasq &>/dev/null && systemctl disable dnsmasq)||:", 1);
                    do_cmd("systemctl daemon-reload", 1);
                }
            }
            @lines = `which -a NetworkManager 2>/dev/null||:`;
            if (@lines > 0) {
                warning("# WARNING:  Could not figure out how to disable NetworkManager.\n");
            }

            # ifplugd should only manage eth0.
            #my $fname = "/etc/default/ifplugd";
            #note("# Set ifplugd to only manage eth0 (assume this is mgt device)\n");
            #my $sed = `cat $fname`;
            #$sed =~ s/INTERFACES=\"\S+\"/INTERFACES=\"eth0\"/g;
            #open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            #print FILE $sed;
            #close FILE;

            if ($is_arm) {
                note("# Disabling accountservice binaries..they eat CPU for no obvious purpose.\n");
                do_cmd("chmod a-x /usr/lib/accountsservice/*", 1);
                note("# Disabling avahi-daemon, it uses too much CPU when there are lots of interfaces.\n");
                do_cmd("echo manual > /etc/init/avahi-daemon.override", 1);
            }

            if ($is_rpi4 || $is_rpi5) {
                # Probably should enable for other recent OSs too?  Or check if systemctl exists?

                # Special hack, not sure why this is needed.
                if (!-e "/usr/lib/arm-linux-gnueabihf/libpcap.so.1") {
                    if (-e "/usr/lib/arm-linux-gnueabihf/libpcap.so") {
                        do_cmd("ln -s usr/lib/arm-linux-gnueabihf/libpcap.so /usr/lib/arm-linux-gnueabihf/libpcap.so.1");
                    }
                }
                if (!-e "/usr/lib/aarch64-linux-gnu/libpcap.so.1") {
                    if (-e "/usr/lib/aarch64-linux-gnu/libpcap.so") {
                        do_cmd("ln -s usr/lib/aarch64-linux-gnu/libpcap.so /usr/lib/aarch64-linux-gnu/libpcap.so.1");
                    }
                }

                if ($? != 0) {
                    do_cmd("systemctl start xrdp.service", 1);
                }

                if (-f "/lib/systemd/system/nfs-client.target") {
                    # Fedora 21 needs this variant.  See
                    # https://bugzilla.redhat.com/show_bug.cgi?id=1173564
                    note(" enable nfs-client.target...\n");
                    do_cmd("systemctl enable      nfs-client.target", 1);
                    do_cmd("systemctl start       nfs-client.target", 1);
                }
                if (-f "/lib/systemd/system/nfs-mountd.service") {
                    note(" enable nfs-mountd.service...\n");
                    do_cmd("systemctl add-wants multi-user.target nfs-mountd.service");
                    do_cmd("systemctl enable        nfs-mountd.service &>/dev/null ||:", 1);
                    do_cmd("systemctl restart       nfs-mountd.service", 1);
                }
                if (-f "/lib/systemd/system/nfs-utils.service") {
                    note(" enable nfs-utils.service...\n");
                    do_cmd("systemctl enable      nfs-utils.service", 1);
                    do_cmd("systemctl start       nfs-utils.service", 1);
                }
                if (-f "/lib/systemd/system/haveged.service") {
                    note(" enable haveged.service...\n");
                    do_cmd("systemctl enable      haveged.service", 1);
                    do_cmd("systemctl start       haveged.service", 1);
                }
                do_cmd("systemctl daemon-reload");
                note("# To see active system components, run:\nsystemctl list-units\n");
            } # end of rpi4/rpi5 section
        } # rpi
    } # ~deb_based
    elsif ($::osveri > 14) {
        $fok = 0;
        if (system("which NetworkManager &>/dev/null") != 0) {
            $fok = 1;
        }
        if ($ntwk_mgr_ok) {
            do_cmd("systemctl enable NetworkManager.service", $fok);
            do_cmd("systemctl unmask NetworkManager.service", $fok);
        }
        else {
            do_cmd("systemctl disable NetworkManager.service", $fok);
            do_cmd("systemctl mask NetworkManager.service", $fok);
            my $rclocal = "/etc/rc.local";
            my $rcrdlocal = "/etc/rc.d/rc.local";

            if (!-f $rclocal || -l $rclocal) {
                my $new_rc_local = q(#!/bin/bash
# Automatically generated by LANforge lf_kinstall.pl because
# it did not detect /etc/rc.local; this rc.local will create
# an /etc/resolv.conf in the case where resolv.conf is a
# symlink (as used by systemd-resolved) typically pointing
# to /run/systemd/resolve/stub-resolv.conf
# This symlink hinders the operation of dhclient when not
# controlled by systemd-resolved, as is the case when the system
# relies on simple dhclient operation *and* LANforge settles on
# a static address because it could not gain a DHCP lease
resolvc="/etc/resolv.conf"
`systemctl is-enabled NetworkManager.service &>/dev/null` || {
  [ ! -f $resolvc -o -L $resolvc ] && {
    [ -L $resolvc ] && rm -f $resolvc
    cat > $resolvc <<EOF
nameserver 192.168.100.1
nameserver 8.8.8.8
options timeout:1
EOF
   chmod 664 $resolvc
  } ||:
} ||:
);
                do_cmd("touch $rclocal") if (!-f $rclocal);
                if (system("grep -q NetworkManager.service $rclocal") != 0) {
                    do_cmd("cp $rclocal /tmp/rc.local");
                    open(my $FH, ">", $rclocal) or die("Unable to open $rclocal");
                    print $FH $new_rc_local;
                    close $FH;
                    do_cmd("cat /tmp/rc.local >> $rclocal");
                }
                do_cmd("chmod +x $rclocal") if (!-x $rclocal);
                if (!-f $rcrdlocal) {
                    #symlink("/etc/rc.local", "/etc/rc.d/rc.local" ) if ( ! -l "/etc/rc.d/rc.local" );
                    do_cmd("ln $rclocal $rcrdlocal");
                }
                do_cmd("chmod +x $rcrdlocal") if (!-x $rcrdlocal);
                note("enable rc-local.service...\n");
                # this is not obeyed if there is a systemd-generator step and this is auto-added to multi-user.target
                # https://unix.stackexchange.com/questions/569740/enable-rc-local-service-in-centos-8-x
                do_cmd("systemctl enable rc-local.service &>/dev/null ||:", 1);
            }
        }
        do_cmd("systemctl daemon-reload");

        do_cmd("systemctl disable abrtd.service", 1);
        do_cmd("systemctl disable avahi-daemon.service", 1);
        if ($::osveri > 17) {
            do_cmd("systemctl disable firewalld.service");
            # Fedora-27 still starts firewall unless we mask it.
            do_cmd("systemctl mask firewalld.service", 1);
        }
        # disable these
        note("disable other services...\n");
        do_cmd("(systemctl is-enabled -q ip6tables.service &>/dev/null && systemctl disable ip6tables.service)||:", 1);
        do_cmd("(systemctl is-enabled -q iptables.service  &>/dev/null && systemctl disable iptables.service )||:", 1);
        do_cmd("(systemctl is-enabled -q sendmail.service  &>/dev/null && systemctl disable sendmail.service )||:", 1);
        do_cmd("(systemctl is-enabled -q sm-client.service &>/dev/null && systemctl disable sm-client.service)||:", 1);
        do_cmd("(systemctl is-enabled -q libvirtd.service  &>/dev/null && systemctl disable libvirtd         )||:", 1);
        do_cmd("(systemctl is-enabled -q snapd.service     &>/dev/null && systemctl disable snapd.service    )||:", 1);
        do_cmd("(systemctl is-enabled -q lxd.service       &>/dev/null && systemctl disable lxd.service      )||:", 1);
        do_cmd("(systemctl is-enabled -q auditd.service    &>/dev/null && systemctl disable auditd.service   )||:", 1);
        do_cmd("(systemctl is-enabled -q systemd-journald-auditd.socket &>/dev/null && systemctl disable systemd-journald-audit.socket )||:", 1);
        do_cmd("systemctl mask systemd-journald-audit.socket ||:", 1);

        # enable these
        # network.service is a nutty one, it's typically not a native systemd unit on fedora
        note("enable network.service...\n");
        do_cmd("systemctl enable network.service ||true");
        do_cmd("[ -x /sbin/chkconfig ] && /sbin/chkconfig network on");
        note("enable sshd.service...\n");
        do_cmd("systemctl enable sshd.service", 1);
        do_cmd("systemctl start sshd.service", 1);
        note("enable cups.service...\n");
        do_cmd("systemctl enable cups.service", 1);
        do_cmd("systemctl start cups.service", 1);
        note("enable xrdp.service...\n");
        do_cmd("systemctl enable xrdp.service", 1);
        `systemctl -q is-active --quiet xrdp.service`;
        if ($? != 0) {
            do_cmd("systemctl start xrdp.service", 1);
        }

        if ($::osveri < 21) {
            # This appears to be required for NFS on 3.12+ kernels??
            do_cmd("systemctl enable nfs-secure.service", 1);
            do_cmd("systemctl start nfs-secure.service", 1);
        }
        if (-f "/lib/systemd/system/nfs-client.target") {
            # Fedora 21 needs this variant.  See
            # https://bugzilla.redhat.com/show_bug.cgi?id=1173564
            note("enable nfs-client.target...\n");
            do_cmd("systemctl enable      nfs-client.target", 1);
            do_cmd("systemctl start       nfs-client.target", 1);
        }
        if (-f "/lib/systemd/system/nfs-mountd.service") {
            note("enable nfs-mountd.service...\n");
            do_cmd("systemctl add-wants multi-user.target nfs-mountd.service");
            do_cmd("systemctl enable      nfs-mountd.service &> /dev/null ||:", 1);
            do_cmd("systemctl start       nfs-mountd.service", 1);
        }
        if (-f "/lib/systemd/system/nfs-utils.service") {
            note("enable nfs-utils.service...\n");
            # this might not have a WantedBy= property, it's not a real service in F33+
            do_cmd("systemctl enable      nfs-utils.service &>/dev/null ||:", 1);
            do_cmd("systemctl start       nfs-utils.service", 1);
        }

        do_cmd("systemctl daemon-reload");
        note("# To see active system components, run:\nsystemctl list-units\n");
    }
    else {
        do_cmd("chkconfig --level 2345 network on");
        $fok = 0;
        if (system("which NetworkManager &>/dev/null") != 0) {
            $fok = 1;
        }
        if ($ntwk_mgr_ok) {
            do_cmd("chkconfig --level 2345 NetworkManager on", $fok);
        }
        else {
            do_cmd("chkconfig --level 2345 NetworkManager off", $fok);
        }
        do_cmd("chkconfig --level 2345 abrtd off", 1);
        do_cmd("chkconfig --level 2345 avahi-daemon off", 1);
        do_cmd("chkconfig --level 2345 ip6tables off", 1);
        do_cmd("chkconfig --level 2345 iptables off");
        do_cmd("chkconfig --level 2345 sendmail off", 1);

        do_cmd("chkconfig --level 2345 sshd on", 1);
        do_cmd("/etc/init.d/sshd start", 1);
        do_cmd("chkconfig --level 2345 cups on", 1);
        do_cmd("/etc/init.d/cups start", 1);
        do_cmd("chkconfig --level 2345 xrdp on", 1);
        do_cmd("/etc/init.d/xrdp start", 1);
        note("# To see active system components, run:\nchkconfig\n");
    }
}

if ($do_lanforge && !($::create_install_bundle || $download_only || $skip_yum_all)) {
    # need to avoid this in a proxy environment!
    # Install scrcpy, used for Interop (Android) display.

    # F30 needs some extra things to support scrcpy.


    if ((defined $::proxy) && ($::proxy ne "")) {
        warning("* proxy detected, will not build scrcpy git repo [$proxy]\n");
    }
    elsif ($is_fedora && $build_scrcpy) {
        # NOTE:  I have seen this fail because there were some i686 versions of SDL2-devel on my system.
        # Work-around:  yum erase SDL2-2.0.9-3.fc30.i686; yum install SDL2-devel
        # If this becomes common thing, then we can automate the fix.
        # To compile this, we need at least these packages (on F37): meson libavformat-free-devel libavdevice-free-devel
        if (!-d "/home/lanforge/git/scrcpy-ct") {
            do_cmd("mkdir -p /home/lanforge/git");
            # want to inject PROXY environment here because it fails for cisco
            do_cmd("cd /home/lanforge/git && git clone https://github.com/greearb/scrcpy-ct.git; cd -", 1);
        }
        else {
            do_cmd("cd /home/lanforge/git/scrcpy-ct && git pull; cd -", 1);
        }
        do_cmd("cd /home/lanforge/git/scrcpy-ct && ./install_release.sh; cd -", 1);
    }
}

if ((!$::create_install_bundle) && $do_radius) {
    note("# DO RADIUS\n");
    # Assume it's already installed..just need to verify config.
    my $fname = "/etc/freeradius/users";
    my $rbase = "/etc/freeradius";
    my $restart_rad = 0;
    if (!-f $fname) {
        note("# do_radius: creating /etc/raddb/users\n");
        $fname = "/etc/raddb/users";
        $rbase = "/etc/raddb";
        if (!-f $fname) {
            warning("#WARNING:  Cannot find freeradius users file, cannot configure radius.\n");
            $fname = "";
        }
    }
    if ($fname ne "") {
        do_cmd("chown root:radiusd $rbase/certs/*");
        note("# do_radius: fname $fname\n");
        my $rusers = `cat $fname`;
        my $rusers_has_lanforge = ($rusers =~ /lanforge Auth-Type/);
        note("# do_radius: force_new_certs:$force_new_certs rusers_has_lanforge $rusers_has_lanforge\n");
        sleep(10);
        if ($force_new_certs || !$rusers_has_lanforge) {
            my @fnamelines = `cat $fname`;
            chomp(@fnamelines);
            my @matches = grep {/^testuser .*"testpasswd"$/} @fnamelines;
            if (@matches < 1) {
                note("# Adding lanforge user to radius users file: $fname\n");
                open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
                print FILE "# EAP-TLS user (private key login)\n";
                print FILE "lanforge Auth-Type := EAP\n\n";
                print FILE "\n# EAP-TTLS user (name + password login)\n";
                print FILE "testuser Cleartext-Password := \"testpasswd\"\n\n";
                print FILE $rusers;
                close FILE;
                $restart_rad = 1;
            }

            # Ubuntu puts the example .cnf files elsewhere.
            if (!-x "$rbase/certs/bootstrap") {
                if (-x "/usr/share/doc/freeradius/examples/certs/bootstrap") {
                    do_cmd("cp -a /usr/share/doc/freeradius/examples/certs/* $rbase/certs/");
                }
            }

            # Assume we only need to muck with certs if we had to modify the users file.
            make_cacnf("$rbase/certs/ca.cnf");
            make_cacnf("$rbase/certs/client.cnf");
            make_cacnf("$rbase/certs/server.cnf");

            incrementRadiusSerialNumber($rbase);

            # Comment out preprocess in the sites-available/default file.
            $fname = "$rbase/sites-available/default";
            note("# Comment out 'preprocess' in file: $fname\n");
            my $sed = `cat $fname`;
            $sed =~ s/\n\s*preprocess/\n\t\#preprocess/g;
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            print FILE $sed;
            close FILE;

            # Set the default eap type to tls
            $fname = "$rbase/mods-available/eap";
            if (!-f $fname) {
                $fname = "$rbase/eap.conf";
            }
            if (-f $fname) {
                note("# Setting eap type to tls in file: $fname\n");
                $sed = `cat $fname`;
                $sed =~ s/\n\s*default_eap_type\s+\=\s+\S+/\n\t\tdefault_eap_type = tls/g;
                $sed =~ s/\n\s*private_key_password\s+\=\s*whatever/\n\tprivate_key_password = lanforge/g;
                open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
                print FILE $sed;
                close FILE;
            }
            else {
                warning("# WARNING:  Could not find eap config file!  RADIUS server may not work properly.\n");
            }

            # Fix up clients.conf localhost entry, change password to lanforge.
            $fname = "$rbase/clients.conf";
            if (-f $fname) {
                note("# Updating localhost entry in file: $fname\n");
                $sed = `cat $fname`;
                $sed =~ s/\n\s*secret\s+\=\s*testing123/\n\tsecret = lanforge/g;
                open(FILE, ">", $fname) or die "Couldn't open file: $fname for writing: $!\n\n";
                print FILE $sed;
                close FILE;
            }
            else {
                warning("# [$fname] not found, locahost clients.conf not updated\n");
            }

            # Allow duplicate subjects.
            $fname = "$rbase/certs/index.txt.attr";
            if (-f $fname) {
                note("# Updating attr file: $fname\n");
                $sed = `cat $fname`;
                $sed =~ s/unique_subject\s+=\s+yes/unique_subject = no/g;
                open(FILE, ">", $fname) or die "Couldn't open file: $fname for writing: $!\n\n";
                print FILE $sed;
                close FILE;
            }
            else {
                warning("# [$fname] not found, unique_subject not altered\n");
            }

            # Allow duplicate subjects (openssl.cnf)
            if (-f "/etc/pki/tls/openssl.cnf") {
                $fname = "/etc/pki/tls/openssl.cnf";
            }
            else {
                # Ubuntu puts it here
                $fname = "/etc/ssl/openssl.cnf";
            }
            note("# Setting unique_subect = no in file: $fname\n");
            $sed = `cat $fname`;
            $sed =~ s/\n\s*\#unique_subject\s+=\s+no/\nunique_subject = no/g;
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            print FILE $sed;
            close FILE;


            # Build the new .pem files.
            note("# build new pem files.\n");
            do_cmd("cd $rbase/certs; make; cd -", 1);

            # Build client files.
            note("# build client_req.pem file.\n");
            do_cmd("cd $rbase/certs; openssl req -days 7200 -new -config client.cnf -keyout client_key.pem -out client_req.pem; cd -");

            note("# build client_cert.pem file. (rbase[$rbase])\n");
            incrementRadiusSerialNumber($rbase);
            do_cmd("cd $rbase/certs; openssl ca -days 7200 -config ca.cnf -in client_req.pem -key lanforge -batch -out client_cert.pem; cd -");
            note("# build client.p12.\n");

            incrementRadiusSerialNumber($rbase);
            do_cmd("cd $rbase/certs; openssl pkcs12 -export -clcerts -in client_cert.pem -inkey client_key.pem -out client.p12 -passin pass:lanforge -passout pass:lanforge; cd -");

            # Copy client files to $home
            do_cmd("cd $rbase/certs; cp ca.pem client.p12 $home; cd -");
            do_cmd("chown root:radiusd $rbase/certs/*");

            if (-x "/etc/init.d/freeradius") {
                # Ubuntu uses this.
                do_cmd("/etc/init.d/freeradius restart");
            }
            else {
                # Fedora uses this.
                if ($::osveri > 14) {
                    do_cmd("systemctl stop radiusd.service", 1);
                    do_cmd("systemctl start radiusd.service", 1);
                    do_cmd("systemctl enable radiusd.service", 1);
                }
                else {
                    do_cmd("service radiusd stop", 1);
                    do_cmd("service radiusd start", 1);
                    do_cmd("service enable radiusd", 1);
                }
            }
        } # If lanforge is not in users file.
        else {
            note("# do_radius: lanforge is already in users file\n");
        }
        do_cmd("chown root:radiusd $rbase/certs/*");
    }
    else {
        note("# do_radius: fname was blank. No freeradius or radiusd configured\n");
    }

    # HotSpot 2.0 stuff
    note("# do_hs20 $do_hs20");
    if ($do_hs20 && -d "$home/local/hs20") {
        note("# configuring HS20\n");
        my $logo = "logo-64x64.png";
        my $company = "lanforge.com";
        my $password = "lanforge";
        my $hs_nai = "osen\@$company";
        my $hs_domain = ".lanforge.local";
        my $hs_realm = "$old_hostname$hs_domain";
        my $osu_client = "osu-client";
        my $osu_server = "osu-server";
        my $osu_server_port = 443;
        my $subscription_server = "osu-signup";
        my $subscription_port = 435;
        my $osu = "osu";
        my $ocsp = "ocsp";
        my $osu_revoked = "osu-revoked";
        my $do_sql_inserts = 0;

        my $hs20_base = "$home/hs20";
        do_cmd("mkdir -p $hs20_base/www", 1);
        do_cmd("mkdir -p $hs20_base/spp/policy", 1);
        do_cmd("mkdir -p $hs20_base/AS/DB", 1);
        do_cmd("chgrp apache $hs20_base/AS/DB", 1);
        do_cmd("chmod g+xw $hs20_base/AS/DB", 1);
        if ($force_new_certs || !-f "$hs20_base/AS/DB/eap_user.db") {
            note("# force_new_certs: $force_new_certs; replacing eap_user.db\n");
            if (-f "$hs20_base/AS/DB/eap_user.db") {
                unlink("$hs20_base/AS/DB/eap_user.db");
            }
            do_cmd("sqlite3 $hs20_base/AS/DB/eap_user.db < $home/local/hs20/sql.txt", 1);
            do_cmd("chgrp apache $hs20_base/AS/DB/eap_user.db", 1);
            do_cmd("chmod g+xw $hs20_base/AS/DB/eap_user.db", 1);
            $do_sql_inserts = 1;
            do_cmd("chown root:radiusd $rbase/certs/*");
        }
        else {
            note("# force_new_certs: not replacing eap_user.db\n");
        }

        # Configure subscription policies
        $fname = "$hs20_base/spp/policy/default.xml";
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        print FILE "<Policy>
   <PolicyUpdate>
      <UpdateInterval>30</UpdateInterval>
      <UpdateMethod>ClientInitiated</UpdateMethod>
      <Restriction>Unrestricted</Restriction>
      <URI>https://$osu_server.$hs_realm:$osu_server_port/hs20/spp.php</URI>
   </PolicyUpdate>
</Policy>
";
        close FILE;

        # Create file to manage ocsp service.
        $fname = "$home/ocsp.bash";
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        print FILE "#!/bin/bash

# This (re)starts the ocsp responder and updates the
# cache every minute.
# This should be run to support HS20-R2 server features.
# It is not needed for HS20-R2 station mode.

cd $home/hs20/ca
killall openssl
./ocsp-responder.sh&

while true
do
   ./ocsp-update-cache-ct.sh
   sleep 60
done
";
        close FILE;
        do_cmd("chmod a+x $fname");

        # Build some new ssl self-signed cert files.
        # Build client files.
        # Create CA certificate
        #my $key_dir = "$hs20_base/AS/Key-OSEN";  # For OSEN radius authentication keys
        #my $key_dir_as = "$hs20_base/AS/Key";  # For 'real' AP radius authentication keys
        # Turns out HS20 is way more complicated that this, so use hostapd scripts to create keys instead. --Ben
        #do_make_radius_keys($rbase, $key_dir);
        #do_make_radius_keys($rbase, $key_dir_as);

        my $key_dir = "$hs20_base/ca";
        my $key_dir_as = "$hs20_base/ca";

        do_cmd("chown root:radiusd $rbase/certs/*");
        if ($force_new_certs) {
            note("# removing hs20/ca certificates\n");
            do_cmd("rm -fr $hs20_base/ca");
        }
        else {
            note("# force_new_certs: not replacing $hs20_base/ca certificates\n");
        }

        if (!-d "$hs20_base/ca") {
            note("# replacing hs20/ca certificates\n");
            do_cmd("cp -ar $home/local/hs20/ca $hs20_base/");
            my $wd = getcwd();

            # Clean up old keys & certs.
            chdir("$hs20_base/ca");
            do_cmd("./clean.sh");
            do_cmd("rm -fr rootCA");

            do_cmd("cp $home/local/images/canvil2-32x32-gray-yel-ico.png $hs20_base/www/logo-32x32.png");
            do_cmd("cp $home/local/images/canvil2-64x64-gray-yel-ico.png $hs20_base/www/logo-64x64.png");

            my $cmd = "sha1sum $hs20_base/www/$logo";
            my $logo_sha1 = `$cmd`;
            if ($logo_sha1 =~ /(\S+).*/) {
                $logo_sha1 = $1;
            }
            else {
                warning("Could not generate icon hash-sha1: $cmd\n");
            }
            $cmd = "sha256sum $hs20_base/www/$logo";
            my $logo_sha256 = `$cmd`;
            if ($logo_sha256 =~ /(\S+).*/) {
                $logo_sha256 = $1;
            }
            else {
                warning("Could not generate icon hash-sha256: $cmd\n");
            }

            # Use HS20 script to generate new local certs and keys and other related things.
            # NOTE:  -C option has format constraints...do not change w/out reason.
            # NOTE:  -r and -R options must coorelate to the 'real' hostapd osu_friendly_name configuration
            #        for instance:  osu_friendly_name=eng:LANforge HS20 Operator
            # NOTE:  I could not get OCSP stapling to work on secondary virtual hosts in apache.  Not sure if it
            #        was my problem or apache problem.  To work-around, use standard-port for osu_server.
            # NOTE:  'real' hostapd cofig osu_server_uri relates to this as well, for instance:
            #        osu_server_uri=https://osu-server.ben-ota-2.lanforge.local:443/hs20/spp.php/signup?realm=ben-ota-2.lanforge.local
            note("# executing hs20 setup\n");
            do_cmd("./setup.sh -c $company -C \"Hotspot 2.0 Trust Root CA - 99\" -o $osu_client.$hs_realm -O $ocsp.$hs_realm -p $password -S $osu_server.$hs_realm -P $subscription_server.$hs_realm -V $osu_revoked.$hs_realm -m $hs_realm -r \"engLANforge HS20 Operator\" -R \"finLANforge HS20 Operator\" -l http://$osu_server.$hs_realm:$osu_server_port/$logo -g $logo_sha1 -G $logo_sha256 -u http://$ocsp.$hs_realm:8888/");
            chdir("$wd");

            # Copy some files created by setup.sh into desired location.
            do_cmd("mkdir -p $hs20_base/www/files");
            do_cmd("cp $hs20_base/ca/rootCA/cacert.der $hs20_base/www/files/spp-root-ca.der");
            do_cmd("cp $hs20_base/ca/rootCA/cacert.fingerprint $hs20_base/www/files/spp-root-ca.fingerprint");
            do_cmd("cp $hs20_base/ca/rootCA/cacert.der $hs20_base/www/files/aaa-root-ca.der");
            do_cmd("cp $hs20_base/ca/rootCA/cacert.fingerprint $hs20_base/www/files/aaa-root-ca.fingerprint");
        }
        else {
            note("# hs20: $hs20_base/ca already exists\n");
        }

        do_cmd("mv $hs20_base/ca/ocsp-update-cache.sh $hs20_base/ca/ocsp-update-cache.sh.orig");
        $fname = "$hs20_base/ca/ocsp-update-cache-ct.sh";
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        print FILE "#!/bin/sh\n\n# NOTE:  You may need to replace 'localhost' with your ocsp hostname instead.\n";
        print FILE "openssl ocsp -no_nonce -CAfile ca.pem -verify_other demoCA/cacert.pem -issuer demoCA/cacert.pem -cert server.pem -url http://$ocsp.$hs_realm:8888/ -respout ocsp-server-cache.der\n";
        close(FILE);
        do_cmd("chmod a+x $fname");

        if ($do_sql_inserts) {
            note("# hs20 creating osu_config database\n");
            my $spp_finger = "BUG";
            my $aaa_finger = "BUG";
            my $tmp = `cat $hs20_base/www/files/spp-root-ca.fingerprint`;
            chomp($tmp);
            if ($tmp =~ /(\S+).*/) {
                $spp_finger = $1;
            }
            $tmp = `cat $hs20_base/www/files/aaa-root-ca.fingerprint`;
            chomp($tmp);
            if ($tmp =~ /(\S+).*/) {
                $aaa_finger = $1;
            }
            $fname = "$hs20_base/spp/sql-example.txt";
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            print FILE "
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','fqdn','$hs_realm');
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','friendly_name','LANforge Example Operator');
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','spp_http_auth_url','https://$osu_server.$hs_realm:$osu_server_port/hs20/spp.php?realm=$hs_realm');
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','trust_root_cert_url','https://$osu_server.$hs_realm:$osu_server_port/hs20/files/spp-root-ca.der');
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','trust_root_cert_fingerprint','$spp_finger');
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','aaa_trust_root_cert_url','https://$osu_server.$hs_realm:$osu_server_port/hs20/files/aaa-root-ca.der');
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','aaa_trust_root_cert_fingerprint','$aaa_finger');
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','free_account','free');
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','policy_url','https://$subscription_server.$hs_realm:$subscription_port/hs20/spp.php?realm=$hs_realm');
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','remediation_url','https://$subscription_server.$hs_realm:$subscription_port/hs20/remediation.php?session_id=');
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','free_remediation_url','https://$subscription_server.$hs_realm:$subscription_port/hs20/free-remediation.php?session_id=');
INSERT INTO osu_config(realm,field,value) VALUES('$hs_realm','signup_url','https://$subscription_server.$hs_realm:$subscription_port/hs20/signup.php?session_id=');

INSERT INTO users(identity,realm,methods,password,phase2,shared) VALUES('free','$hs_realm','TTLS-MSCHAPV2','free',1,1);

INSERT INTO wildcards(identity,methods) VALUES('','TTLS,TLS');
";
            close FILE;

            do_cmd("sqlite3 $hs20_base/AS/DB/eap_user.db <  $hs20_base/spp/sql-example.txt", 1);
            # Open permissions so that the php scripts and things they call can
            # write to the database.  Not secure.
            do_cmd("chmod a+rwx $hs20_base/AS/DB  $hs20_base/AS/DB/eap_user.db");
        }
        else {
            note("# hs20: not doing sql_inserts\n");
        }

        $fname = "$hs20_base/AS/hostapd-radius-sql-example.conf";
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        print FILE "
# hostapd-radius config for the radius used by the 'real' AP
interface=eth0#1
driver=none
logger_syslog=-1
logger_syslog_level=2
logger_stdout=-1
logger_stdout_level=2
ctrl_interface=/var/run/hostapd
ctrl_interface_group=0
#ieee8021x=1
eapol_key_index_workaround=0
eap_server=1
eap_user_file=sqlite:$hs20_base/AS/DB/eap_user.db
server_id=$old_hostname-sql
radius_server_auth_port=1810
radius_server_clients=$hs20_base/AS/hostap.radius_clients
eap_sim_db=unix:/tmp/hlr_auc_gw.sock db=$hs20_base/AS/DB/eap_sim.db

# This shares certs with 'real' hostapd-radius and http server.
# Should probably be different.
ca_cert=$hs20_base/ca/ca.pem
server_cert=$hs20_base/ca/server.pem
private_key=$hs20_base/ca/server.key
private_key_passwd=$password

ocsp_stapling_response=$hs20_base/ca/ocsp-server-cache.der

# Not sure about this. --Ben
subscr_remediation_url=https://$osu_server.$old_hostname.$hs_realm:$osu_server_port/hs20/spp.php
";
        close FILE;

        $fname = "$hs20_base/AS/hostapd-radius-osen-example.conf";
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        print FILE "
# hostapd-radius config for the radius used by the OSEN AP
interface=eth0#0
driver=none
logger_syslog=-1
logger_syslog_level=2
logger_stdout=-1
logger_stdout_level=2
ctrl_interface=/var/run/hostapd
ctrl_interface_group=0
eap_server=1
eap_user_file=$hs20_base/AS/hostapd-osen.eap_user
server_id=$old_hostname-osen
radius_server_auth_port=1811
radius_server_clients=$hs20_base/AS/hostap.radius_clients

# This shares certs with 'real' hostapd-radius and http server.
# Should probably be different.
ca_cert=$hs20_base/ca/ca.pem
server_cert=$hs20_base/ca/server.pem
private_key=$hs20_base/ca/server.key
private_key_passwd=$password

ocsp_stapling_response=$hs20_base/ca/ocsp-server-cache.der

";
        close FILE;

        $fname = "$hs20_base/AS/hostapd-osen-example.conf";
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        print FILE "
# hostapd config for the OSEN AP.
interface=vap1
driver=nl80211
logger_syslog=-1
logger_syslog_level=2
logger_stdout=-1
logger_stdout_level=2
ctrl_interface=/var/run/hostapd
ctrl_interface_group=0
ssid=ota-9k-osen
bssid=00:0e:8e:2b:e5:96
country_code=US
ieee80211d=0
ieee80211h=0
ieee80211w=0
hw_mode=g
ieee80211n=1
beacon_int=240
dtim_period=2
max_num_sta=2007
rts_threshold=2347
fragm_threshold=2346
preamble=0
macaddr_acl=0
auth_algs=1
ignore_broadcast_ssid=0
# Enable HT modes if you want 300Mbps+ throughput.
#ht_capab=[HT20][HT40-][HT40+][GF][SHORT-GI-20][SHORT-GI-40]
#         [TX-STBC][RX-STBC123][MAX-AMSDU-7935][DSSS_CCK-40][PSMP][LSIG-TXOP-PROT]
ht_capab=[HT20][HT40+][SHORT-GI-40][SHORT-GI-20]
#vht_capab=[MAX-MPDU-11454][RXLDPC][SHORT-GI-80][TX-STBC-2BY1][RX-STBC-1][MAX-A-MPDU-LEN-EXP0][RX-ANTENNA-PATTERN][TX-ANTENNA-PATTERN]

wmm_enabled=1
wmm_ac_bk_cwmin=4
wmm_ac_bk_cwmax=10
wmm_ac_bk_aifs=7
wmm_ac_bk_txop_limit=0
wmm_ac_bk_acm=0
wmm_ac_be_aifs=3
wmm_ac_be_cwmin=4
wmm_ac_be_cwmax=10
wmm_ac_be_txop_limit=0
wmm_ac_be_acm=0
wmm_ac_vi_aifs=2
wmm_ac_vi_cwmin=3
wmm_ac_vi_cwmax=4
wmm_ac_vi_txop_limit=94
wmm_ac_vi_acm=0
wmm_ac_vo_aifs=2
wmm_ac_vo_cwmin=2
wmm_ac_vo_cwmax=3
wmm_ac_vo_txop_limit=47
wmm_ac_vo_acm=0
channel=1
supported_rates=10 20 55 110 60 90 120 180 240 360 480 540
ieee8021x=1
own_ip_addr=127.0.0.1
auth_server_addr=127.0.0.1
auth_server_port=1811
auth_server_shared_secret=$password

# Error emulation settings.
ignore_probe_probability=0.000000
ignore_auth_probability=0.000000
ignore_assoc_probability=0.000000
ignore_reassoc_probability=0.000000
corrupt_gtk_rekey_mic_probability=0.000000
osen=1

#Wifi Custom Config specified by user directly.
ocsp_stapling_response=$hs20_base/ca/ocsp-server-cache.der

";
        close FILE;

        $fname = "$hs20_base/AS/hostapd-1x-example.conf";
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        print FILE qq(
# hostapd config for the real AP
interface=vap2
driver=nl80211
logger_syslog=-1
logger_syslog_level=2
logger_stdout=-1
logger_stdout_level=2
ctrl_interface=/var/run/hostapd
ctrl_interface_group=0
ssid=ota-9k-2
bssid=00:0e:8e:f8:73:96
country_code=US
ieee80211d=0
ieee80211h=0
ieee80211w=0
hw_mode=g
ieee80211n=1
beacon_int=240
dtim_period=2
max_num_sta=2007
rts_threshold=2347
fragm_threshold=2346
preamble=0
macaddr_acl=0
auth_algs=1
ignore_broadcast_ssid=0
# Enable HT modes if you want 300Mbps+ throughput.
#ht_capab=[HT20][HT40-][HT40+][GF][SHORT-GI-20][SHORT-GI-40]
#         [TX-STBC][RX-STBC123][MAX-AMSDU-7935][DSSS_CCK-40][PSMP][LSIG-TXOP-PROT]
ht_capab=[HT20][HT40+][SHORT-GI-40][SHORT-GI-20]
#vht_capab=[MAX-MPDU-11454][RXLDPC][SHORT-GI-80][TX-STBC-2BY1][RX-STBC-1][MAX-A-MPDU-LEN-EXP0][RX-ANTENNA-PATTERN][TX-ANTENNA-PATTERN]

wmm_enabled=1
wmm_ac_bk_cwmin=4
wmm_ac_bk_cwmax=10
wmm_ac_bk_aifs=7
wmm_ac_bk_txop_limit=0
wmm_ac_bk_acm=0
wmm_ac_be_aifs=3
wmm_ac_be_cwmin=4
wmm_ac_be_cwmax=10
wmm_ac_be_txop_limit=0
wmm_ac_be_acm=0
wmm_ac_vi_aifs=2
wmm_ac_vi_cwmin=3
wmm_ac_vi_cwmax=4
wmm_ac_vi_txop_limit=94
wmm_ac_vi_acm=0
wmm_ac_vo_aifs=2
wmm_ac_vo_cwmin=2
wmm_ac_vo_cwmax=3
wmm_ac_vo_txop_limit=47
wmm_ac_vo_acm=0
channel=1
supported_rates=10 20 55 110 60 90 120 180 240 360 480 540
ieee8021x=1
own_ip_addr=127.0.0.1
auth_server_addr=127.0.0.1
auth_server_port=1810
auth_server_shared_secret=$password
wpa=2
wpa_pairwise=CCMP
wpa_key_mgmt=WPA-EAP WPA-EAP-SHA256

# 802.11u configuration
interworking=1
access_network_type=0
internet=0
asra=1
esr=0
uesa=0
venue_group=0
venue_type=0
venue_name=eng:LANforge Test Venue
ipaddr_type_availability=00
domain_name=$hs_realm
nai_realm=0,$hs_realm,12[5:6],21[2:4][5:7]

# HotSpot 2.0 configuration
hs20=1
hs20_oper_friendly_name=eng:LANforge HotSpot 2.0

# Error emulation settings.
ignore_probe_probability=0.000000
ignore_auth_probability=0.000000
ignore_assoc_probability=0.000000
ignore_reassoc_probability=0.000000
corrupt_gtk_rekey_mic_probability=0.000000

#Wifi Custom Config specified by user directly.
hs20_icon=64:64:eng:image/png:$logo:$hs20_base/www/logo-64x64.png
osu_ssid="ota-9k-osen"
# OSU Providers
# One or more sets of following parameter. Each OSU provider is started by the
# mandatory osu_server_uri item. The other parameters add information for the
# last added OSU provider.
#
osu_server_uri=https://$osu_server.$hs_realm:$osu_server_port/hs20/spp.php/signup?realm=$hs_realm
osu_friendly_name=eng:LANforge HS20 Operator
osu_nai=$hs_nai
osu_method_list=1 0
osu_icon=$logo
osu_service_desc=eng:LANforge Example services

);
        close FILE;


        # Set RADIUS passphrase for the APs
        # Note: Modify to match the setup

        # Open permissions for apache to find these files.
        do_cmd("chmod a+rx $home", 1);
        do_cmd("chmod a+rx $hs20_base", 1);
        do_cmd("chmod a+rx $hs20_base/www", 1);

        note("# configuring hs20/AS hostap.radius_clients\n");
        $fname = "$hs20_base/AS/hostap.radius_clients";
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        print FILE "0.0.0.0/0    $password\n";
        close FILE;
        note("# updating $home/local/hs20/spp files\n");
        do_cmd("cp -ar $home/local/hs20/www/* $hs20_base/www/");
        do_cmd("cp $home/local/hs20/spp/dm_ddf-v1_2.dtd $hs20_base/spp/");
        do_cmd("cp $home/local/hs20/spp/spp.xsd $hs20_base/spp/");
        do_cmd("cp $home/local/hs20/spp/hs20_spp_server $hs20_base/spp/");

        $fname = "$hs20_base/www/config.php";
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        print FILE "<?php
\$osu_root = \"$hs20_base\";
\$osu_db = \"sqlite:\$osu_root/AS/DB/eap_user.db\";
?>
";
        close FILE;

        # Create some default files that make it easier to start using hostapd as radius server.

        $fname = "$hs20_base/AS/hostapd.eap_user";
        if (!-f $fname) {
            note("# configuring hs20/AS/hostapd.eap_user\n");
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            print FILE "\"*\@$company\" TLS
\"0\"* AKA
\"2\"* AKA
\"4\"* AKA

# For OSEN authentication (HS20 r2)
\"$hs_nai\"      WFA-UNAUTH-TLS

";
            close FILE;
        }
        else {
            note("# hs20/AS hostapd.eap_user already exists\n");
        }

        # Create some default files that make it easier to start using hostapd as OSEN radius server.
        $fname = "$hs20_base/AS/hostapd-osen.eap_user";
        if (!-f $fname) {
            note("# configuring hs20/AS/hostapd-osen.eap_user\n");
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            print FILE "
# For OSEN authentication (HS20 r2)
\"$hs_nai\"      WFA-UNAUTH-TLS

";
            close FILE;
        }

        $fname = "/etc/hlr_auc_gw.milenage_db";
        if (!-f $fname) {
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            print FILE "
# Parameters for Milenage (Example algorithms for AKA).
# The example Ki, OPc, and AMF values here are from 3GPP TS 35.208 v6.0.0
# 4.3.20 Test Set 20. SQN is the last used SQN value.
# These values can be used for both UMTS (EAP-AKA) and GSM (EAP-SIM)
# authentication. In case of GSM/EAP-SIM, AMF and SQN values are not used, but
# dummy values will need to be included in this file.
# IMSI Ki OPc AMF SQN
232010000000000 90dca4eda45b53cf0f12d7c9c3bc6a89 cb9cccc4b9258e6dca4760379fb82581 61df 000000000000

# These values are from Test Set 19 which has the AMF separation bit set to 1
# and as such, is suitable for EAP-AKA' test.
555444333222111 5122250214c33e723a5dd523fc145fc0 981d464c7c52eb6e5036234984ad0bcf c3ab 16f3b3f70fc1

";
            close FILE;
        }

        my $ssl_txt = "
# Enable OCSP stapling
SSLUseStapling on

ServerName $osu_server.$hs_realm:443/

Alias /hs20/ \"$hs20_base/www/\"
        <Directory \"$hs20_base/www/\">
                Options Indexes MultiViews FollowSymLinks
                AllowOverride None
                Require all granted
        </Directory>

SSLCertificateFile $hs20_base/ca/server.pem
SSLCertificateKeyFile $hs20_base/ca/server.key
SSLCertificateChainFile $hs20_base/ca/ca.pem
";

        my $ssl_txt2 = "

# Enable OCSP stapling cache
SSLStaplingCache shmcb:/tmp/stapling_cache(128000)

# HS20 signup server.
Listen osu-signup.$hs_realm:$subscription_port https
<VirtualHost *:$subscription_port>
 ServerName osu-signup.$hs_realm
 DocumentRoot $hs20_base/www
 ErrorLog logs/ssl_error_log_hs20_signup
 TransferLog logs/ssl_access_log_hs20_signup
 CustomLog logs/ssl_request_log_hs20_signup \"\%t \%h \%\{SSL_PROTOCOL\}x \%\{SSL_CIPHER\}x \\\"\%r\\\" \%b\"
 SSLEngine on
 SSLUseStapling on
 SSLCertificateFile $hs20_base/ca/signup-server.pem
 SSLCertificateKeyFile $hs20_base/ca/signup-server.key
 SSLCertificateChainFile $hs20_base/ca/ca.pem
 Alias /hs20/ \"$hs20_base/www/\"
 <Directory \"$hs20_base/www\">
    Options Indexes MultiViews FollowSymLinks
    AllowOverride None
    Require all granted
 </Directory>
</VirtualHost>
";

        do_cmd("cp -ar /etc/httpd/conf.d /etc/httpd/conf.d-hs20");
        do_cmd("cp /etc/httpd/conf/httpd.conf /etc/httpd/conf/httpd-hs20.conf");

        $fname = "/etc/httpd/conf/httpd-hs20.conf";
        if (-f "$fname") {
            if (!-f "$fname.orig") {
                do_cmd("cp $fname $fname.orig");
            }
            my @lines = `cat $fname`;
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            my $found_it = 0;
            my $found_it2 = 0;
            my $in_skip = 0;
            my $listen = 0;
            my $found_sn = 0;
            for ($i = 0; $i < @lines; $i++) {
                my $ln = $lines[$i];
                chomp($ln);

                if ($ln =~ /^PidFile/) {
                    print FILE "PidFile \"/run/httpd/httpd-hs20.pid\"\n";
                    next;
                }
                elsif ($ln =~ /^Listen/) {
                    print FILE "Listen $osu-server.$hs_realm:80\n";
                    next;
                }
                elsif ($ln =~ /^#ServerName/) {
                    print FILE "$ln\n";
                    print FILE "ServerName $osu-server.$hs_realm\n";
                    $found_sn = 1;
                    next;
                }
                elsif ($ln =~ /^ServerName/) {
                    if (!$found_sn) {
                        print FILE "ServerName $osu-server.$hs_realm\n";
                        $found_sn = 1;
                    }
                    next;
                }
                elsif ($ln =~ /^IncludeOptional/) {
                    print FILE "IncludeOptional conf.d-hs20/*.conf\n";
                    next;
                }
                elsif ($ln =~ /^ErrorLog /) {
                    print FILE "ErrorLog logs/error_log_hs20\n";
                    next;
                }
                elsif ($ln =~ /^(\s*)CustomLog /) {
                    print FILE "$1" . "CustomLog logs/access_log_hs20 combined\n";
                    next;
                }
                print FILE "$ln\n";
            }
            close FILE;
        }

        $fname = "/etc/httpd/conf.d-hs20/ssl.conf";
        if (-f "$fname") {
            if (!-f "$fname.orig") {
                do_cmd("cp $fname $fname.orig");
            }
            my @lines = `cat $fname`;
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            my $found_it = 0;
            my $found_it2 = 0;
            my $in_skip = 0;
            my $listen = 0;
            for ($i = 0; $i < @lines; $i++) {
                my $ln = $lines[$i];
                chomp($ln);

                if ($ln =~ /^Listen/) {
                    if ($listen == 0) {
                        print FILE "Listen $osu-server.$hs_realm:443 https\n";
                        $listen++;
                        next;
                    }
                }
                elsif ($ln =~ /^ErrorLog /) {
                    print FILE "ErrorLog logs/ssl_error_log_hs20\n";
                    next;
                }
                elsif ($ln =~ /^TransferLog /) {
                    print FILE "TransferLog logs/ssl_access_log_hs20\n";
                    next;
                }
                elsif ($ln =~ /^CustomLog /) {
                    print FILE "CustomLog logs/ssl_request_log_hs20 \\\n";
                    next;
                }

                if ($in_skip) {
                    if (($ln =~ /###LANFORGE-SSL-STOP###/) ||
                        ($ln =~ /###LANFORGE-SSL2-STOP###/)) {
                        $in_skip = 0;
                        print FILE "$ln\n";
                    }
                    next;
                }
                if ($ln =~ /###LANFORGE-SSL-START###/) {
                    print FILE "$ln\n$ssl_txt";
                    $found_it = 1;
                    $in_skip = 1;
                    next;
                }
                elsif ($ln =~ /###LANFORGE-SSL2-START###/) {
                    print FILE "$ln\n$ssl_txt2";
                    $found_it2 = 1;
                    $in_skip = 1;
                    next;
                }
                elsif ($ln =~ /SSL Engine Switch/) {
                    if (!$found_it) {
                        print FILE "###LANFORGE-SSL-START###\n$ssl_txt\n###LANFORGE-SSL-STOP###\n\n";
                        $found_it = 1;
                    }
                }
                # Skip some lines we will be over-riding in our own section.
                elsif ($ln =~ /^SSLCertificateFile.*/) {
                    print FILE "# Commented out by LANforge lf_kinstall.pl script.\n";
                    print FILE "#$ln\n";
                    next;
                }
                elsif ($ln =~ /^SSLCertificateKeyFile.*/) {
                    print FILE "# Commented out by LANforge lf_kinstall.pl script.\n";
                    print FILE "#$ln\n";
                    next;
                }
                elsif ($ln =~ /^SSLCertificateChainFile.*/) {
                    print FILE "# Commented out by LANforge lf_kinstall.pl script.\n";
                    print FILE "#$ln\n";
                    next;
                }
                print FILE "$ln\n";
            } # for all lines in http ssl.conf file

            if (!$found_it2) {
                print FILE "###LANFORGE-SSL2-START###\n$ssl_txt2\n###LANFORGE-SSL-STOP###\n\n";
                $found_it2 = 1;
            }
            close FILE;

            # http will need to be per vrf
            #do_restart_httpd();
        } # If we can find ssl conf file
        else {
            warning("# Cannot find ssl conf file, web-server will not handle hs20 requests.\n");
        }

        $fname = "/etc/httpd/conf.modules.d/00-mpm.conf";
        if (-f "$fname") {
            if (!-f "$fname.orig") {
                do_cmd("cp $fname $fname.orig");
            }
            my @lines = `cat $fname`;
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            my $found_it = 0;
            for ($i = 0; $i < @lines; $i++) {
                my $ln = $lines[$i];
                chomp($ln);

                if ($ln =~ /^LoadModule mpm_prefork_module modules\/mod_mpm_prefork.so/) {
                    $found_it = 1;
                }
                elsif ($ln =~ /^LoadModule mpm_event_module modules\/mod_mpm_event.so/) {
                    print FILE "#" . $ln . "\n";
                    next;
                }
                print FILE "$ln\n";
            } # for all lines in http ssl.conf file

            if (!$found_it) {
                print FILE "LoadModule mpm_prefork_module modules/mod_mpm_prefork.so\n";
            }
            close FILE;

            #do_restart_httpd();
        } # If we can find ssl conf file
        else {
            warning("# Cannot find ssl conf file, web-server will not handle hs20 requests.\n");
        }

        $fname = "/etc/hosts";
        if (-f "$fname") {
            my @lines = `cat $fname`;
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
            my $found_it = 0;
            my $in_skip = 0;
            for ($i = 0; $i < @lines; $i++) {
                my $ln = $lines[$i];
                chomp($ln);

                if ($in_skip) {
                    if ($ln =~ /###LANFORGE-HOSTS-STOP###/) {
                        $in_skip = 0;
                        print FILE "$ln\n";
                    }
                    next;
                }
                if ($ln =~ /###LANFORGE-HOSTS-START###/) {
                    print FILE "# This section may be over-written by lf_kinstall.pl\n";
                    print FILE "$hs20_server_ip $ocsp.$hs_realm\n";
                    print FILE "$hs20_server_ip $osu_server.$hs_realm\n";
                    print FILE "$hs20_server_ip $osu_client.$hs_realm\n";
                    print FILE "$hs20_server_ip $subscription_server.$hs_realm\n";
                    print FILE "$hs20_server_ip $osu_revoked.$hs_realm\n";
                    $found_it = 1;
                    $in_skip = 1;
                    next;
                }
                print FILE "$ln\n";
            } # for all lines in http ssl.conf file

            if (!$found_it) {
                print FILE "###LANFORGE-HOSTS-START###\n";
                print FILE "# This section may be over-written by lf_kinstall.pl\n";
                print FILE "$hs20_server_ip $ocsp.$hs_realm\n";
                print FILE "$hs20_server_ip $osu_server.$hs_realm\n";
                print FILE "$hs20_server_ip $osu_client.$hs_realm\n";
                print FILE "$hs20_server_ip $subscription_server.$hs_realm\n";
                print FILE "$hs20_server_ip $osu_revoked.$hs_realm\n";
                print FILE "###LANFORGE-HOSTS-STOP###\n\n";
            }
            close FILE;

            $fname = "/etc/dnsmasq.d/lanforge.conf";
            if (-d "/etc/dnsmasq.d") {
                open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
                print FILE "local=/local/\n";
                close FILE;

                # Start per vrf instead.
                #do_cmd("systemctl enable dnsmasq");
                #do_cmd("systemctl start dnsmasq");
            }
            else {
                warning("WARNING:  dnsmasq does not appear to be installed.  HS20 station machines may need /etc/hosts updated manually!\n");
            }

            # http will need to be per vrf
            #do_restart_httpd();
        } # If we can find /etc/hosts file
        else {
            warning("# Cannot find ssl conf file, web-server will not handle hs20 requests.\n");
        }

    } # End of hotspot 2.0 stuff
    if (!($is_openwrt || $is_macos)) {
        do_cmd("chown root:radiusd $rbase/certs/*", 1);
    }
} # ~hotspot20

# check for cups-pdf
if (!$::create_install_bundle) {
    my $lpadmin = `which lpadmin`;
    my $lpstat = `which lpstat`;
    chomp($lpadmin);
    chomp($lpstat);
    if ($do_cups_pdf && $lpadmin ne "") {
        note("# configuring CUPS/pdf");
        my @printer_lines = `$lpstat -p`;
        my @matches = grep {/cups/i} @printer_lines;
        if (@matches < 1) {
            if ($::osveri >= 30) {
                do_cmd("$lpadmin -p Cups-PDF -E -v cups-pdf:/ -m CUPS-PDF_opt.ppd", 1);
            }
            elsif ($::osveri >= 27) {
                do_cmd("$lpadmin -p Cups-PDF -E -v cups-pdf:/ -m /usr/share/cups/model/CUPS-PDF_opt.ppd", 1);
            }
            elsif ($::osveri > 14) {
                do_cmd("$lpadmin -p Cups-PDF -E -v cups-pdf:/ -m /usr/share/cups/model/CUPS-PDF.ppd", 1);
            }
            else {
                do_cmd("$lpadmin -p Cups-PDF -E -v 'cups-pdf:/' -m /usr/share/cups/model/CUPS-PDF.ppd", 1);
            }
        }
        # sets default printer, continue on if we fail this step.
        do_cmd("$lpadmin -d Cups-PDF", 1);
    }
}
# X display and tcp-listen
if (!($::create_install_bundle || $is_macos) && ($do_gnome || $do_install_mate ||
    (($do_all || $do_all_ct || $do_upgrade || $do_lanforge || $do_sys_reconfig || $do_vm_reconfig)
        && $mate_installed && $::osveri > 14))) {

    create_lf_user() if ($::should_create_user);

    my $fn;
    # centos7 might not quickly load avahi, prime it
    # ubuntu does not seem to enjoy this
    if (!$do_interop) {
        note("# gsettings: do-gnome: $do_gnome  install-mate: $do_install_mate  mate-installed: $mate_installed  osveri: $::osveri\n");
        my @glines = `which -a gsettings`;
        if (@glines > 0) {
            do_cmd("gsettings list-schemas > /dev/null", 1);
            sleep 2;
            my @mate_schemas = `(gsettings list-schemas | grep org.mate)|| :`;

            $fn = "$ctgnome";
            open(TFILE, ">$fn") or die "Couldn't open file: $fn  $!\n\n";
            print TFILE "#!/bin/bash\n\n";

            my @desktop_tasks = (
                # "gsettings set org.gnome.desktop.background draw-background true",
                "dconf dump / &>/dev/null",
                #"dconf compile /run/user/\$UID/dconf $home/.config/dconf ",
                #"sleep 1",
                "gsettings set org.gnome.desktop.background show-desktop-icons true",
                "gsettings set org.gnome.desktop.background primary-color '#209040'",
                "gsettings set org.gnome.desktop.background secondary-color '#002000'",
                "gsettings set org.gnome.desktop.background color-shading-type 'vertical'",
                "gsettings set org.gnome.desktop.background picture-options centered",
                "[ -d $home/Pictures ] || mkdir -p $home/Pictures",
                "chown lanforge:$user_group $home/Pictures",
                "chmod 0750 $home/Pictures",
            );
            push(@desktop_tasks, "gsettings set org.gnome.desktop.background picture-uri 'file://$home/Pictures/anvil-right.svg'")
                if (-f "$home/Pictures/anvil-right.svg");

            if (@mate_schemas > 0) {
                # below does not fix F34 terminal opening in '/' not '/home/lanforge'
                # push(@desktop_tasks, "gsettings set org.mate.caja-open-terminal desktop-opens-home-dir true");
                push(@desktop_tasks, "gsettings set org.mate.background draw-background true");
                push(@desktop_tasks, "gsettings set org.mate.background show-desktop-icons true");
                push(@desktop_tasks, "gsettings set org.mate.background primary-color '#209020'");
                push(@desktop_tasks, "gsettings set org.mate.background secondary-color '#002000'");
                push(@desktop_tasks, "gsettings set org.mate.background color-shading-type 'vertical-gradient'");
                push(@desktop_tasks, "gsettings set org.mate.background picture-filename '$home/Pictures/anvil-right.svg'")
                    if (-f "$home/Pictures/anvil-right.svg");
                push(@desktop_tasks, "gsettings set org.mate.background picture-options centered");
            }

            if ($os_string =~ /Fedora/) {
                if ($::osveri >= 19) {
                    push(@desktop_tasks, "gsettings set org.gnome.desktop.session session-name gnome-classic");
                }
                else {
                    push(@desktop_tasks, "gsettings set org.gnome.desktop.session session-name gnome-fallback");
                }
            }
            my $has_mate = 0;
            if (system("which mate-session &>/dev/null") == 0) {
                $has_mate = 1;
            }
            if ($do_gnome || $do_install_mate || $mate_installed) {
                # Determined through dconf-editor:
                # - Disable any lock so that users don't get locked out on remote sessions.
                push(@desktop_tasks, "gsettings set org.gnome.desktop.lockdown disable-lock-screen true");
                push(@desktop_tasks, "gsettings set org.gnome.desktop.lockdown disable-log-out true");
                push(@desktop_tasks, "gsettings set org.gnome.desktop.screensaver lock-enabled false");

                # - Disable idle lock on screensaver.
                push(@desktop_tasks, "gsettings set org.gnome.desktop.screensaver idle-activation-enabled false");

                if ((@mate_schemas > 0) && ($mate_installed || $do_install_mate)) {
                    push(@desktop_tasks, "gsettings set org.mate.screensaver lock-enabled false");
                    push(@desktop_tasks, "gsettings set org.mate.screensaver idle-activation-enabled false");
                }
                push(@desktop_tasks, "dconf update /");
                #push(@desktop_tasks, "dconf compile /run/user/\$UID/dconf $home/.config/dconf ");
                #push(@desktop_tasks, "vncserver -kill :1");
                push(@desktop_tasks, "sleep 1");
            }

            my $prefix = "sudo -u lanforge dbus-launch";
            for my $task (@desktop_tasks) {
                do_cmd("$prefix $task", 1);
            }
            my $config_dt = "$home/config-desktop.sh";
            unlink($config_dt) if (-f $config_dt);

            if (open(my $fh_config_dt, ">", $config_dt)) {
                print $fh_config_dt "#!/bin/bash\n";
                for my $task (@desktop_tasks) {
                    print $fh_config_dt "$task ||:\n";
                }
                print $fh_config_dt "chmod -x $home/config-desktop.sh\n";
                #print $fh_config_dt "vncserver -kill :1";
                close $fh_config_dt;
                chmod(0755, $config_dt);
                `chown lanforge:$user_group $config_dt`;
            }
            # Use fallback/classic mode
            $cmd = "";
            close TFILE;
            do_cmd("chmod a+x $fn");
        }
        else {
            note("# gsettings not found, skipping gsettings commands.\n");
        }
    } # ~do interop
    do_cmd("mkdir -p $home/.config/autostart", 1);
    do_cmd("mkdir -p /root/.config/autostart", 1);
    my $fileman = "";
    if (!$do_interop) {
        if (-f "/bin/caja") {
            $fileman = "caja";
            if (-f "/bin/nautilus") {
                note("Removing nautilus");
                if ($is_deb_based) {
                    #do_cmd("dpkg -e nautilus", 1);
                    note("cannot uninstall nautilus, it will break ubuntu-desktop")
                }
                else {
                    do_cmd("rpm -e  --nodeps nautilus", 1);
                }
            }
            if (-f "$home/.config/autostart/nautilus.desktop") {
                unlink("$home/.config/autostart/nautilus.desktop");
            }
        }
        elsif (-f "/bin/nautilus") {
            $fileman = "nautilus";
            if (-f "$home/.config/autostart/caja.desktop") {
                unlink("$home/.config/autostart/caja.desktop");
            }
        }
    }

    # on older desktops, starting the filemanager was a benefit
    my $autostart_fileman = 0;
    if ($autostart_fileman && !$do_interop) {
        $fn = "$home/.config/autostart/$fileman.desktop";
        if (-d "$home") {
            open(FILE, ">$fn") or die "Couldn't open file: $fn  $!\n\n";
            print FILE "[Desktop Entry]
Type=Application
Exec=$fileman -n
Hidden=false
X-GNOME-Autostart-enabled=true
Name[en_US]=$fileman
Name=$fileman
Comment[en_US]=File Manager
Comment=File Manager
";
            close FILE;
            do_cmd("chown lanforge:$user_group $fn");

            $fn = "$home/.config/autostart/vncconfig.desktop";
            open(FILE, ">$fn") or die "Couldn't open file: $fn  $!\n\n";
            print FILE "[Desktop Entry]
Type=Application
Exec=vncconfig -iconic
Hidden=false
X-GNOME-Autostart-enabled=true
Name[en_US]=vncconfig
Name=vncconfig
Comment[en_US]=VNC Config
Comment=VNC Config
";
            close FILE;
            do_cmd("chown lanforge:$user_group $fn");
        }
    } # ~autostart file manager

    # Stop auto-start of some things on recent mate systems.
    note("# Stopping auto-start for some mate utilities.\n");
    my $asf = "mate-screensaver.desktop";
    if (-f "/etc/xdg/autostart/$asf") {
        do_cmd("cp /etc/xdg/autostart/$asf $home/.config/autostart");
        do_cmd("echo X-MATE-Autostart-enabled=false >> $home/.config/autostart/$asf");
        do_cmd("echo Hidden=true >> $home/.config/autostart/$asf");
        do_cmd("cp $home/.config/autostart/$asf /root/.config/autostart/");
    }

    $asf = "mate-power-manager.desktop";
    if (-f "/etc/xdg/autostart/$asf") {
        do_cmd("cp /etc/xdg/autostart/$asf $home/.config/autostart");
        do_cmd("echo X-MATE-Autostart-enabled=false >> $home/.config/autostart/$asf");
        do_cmd("echo Hidden=true >> $home/.config/autostart/$asf");
        do_cmd("cp $home/.config/autostart/$asf /root/.config/autostart/");
    }

    $asf = "mate-volume-control-applet.desktop";
    if (-f "/etc/xdg/autostart/$asf") {
        do_cmd("cp /etc/xdg/autostart/$asf $home/.config/autostart");
        do_cmd("echo X-MATE-Autostart-enabled=false >> $home/.config/autostart/$asf");
        do_cmd("echo Hidden=true >> $home/.config/autostart/$asf");
        do_cmd("cp $home/.config/autostart/$asf /root/.config/autostart/");
    }

    # Startup Terminal
    $fn = "$home/.config/autostart/terminal.desktop";
    my $tname = "xterm";
    if (-d "$home") {
        $tname = "xterm" if (-x "/bin/xterm");
        $tname = "gnome-terminal" if (-x "/bin/gnome-terminal");
        $tname = "mate-terminal" if (-x "/bin/mate-terminal");
        $tname = "roxterm" if ($is_arm);
        open(FILE, ">$fn") or die "Couldn't open file: $fn  $!\n\n";
        print FILE "[Desktop Entry]
Type=Application
Path=$home
Exec=$tname
Hidden=false
X-GNOME-Autostart-enabled=true
Name[en_US]=terminal
Name=terminal
Comment[en_US]=
Comment=
";
        close FILE;
        do_cmd("chown lanforge:$user_group $fn");
    } # ~home terminal.desktop

    # - Workaround MATE black desktop background affecting font readability by using a 1x1 black PNG for background.
    if ($do_install_mate || $mate_installed) {
        my $dflt_bkgnd = "/usr/share/backgrounds/default.png";
        my $pngOut = "$lf_tmp/default.png";
        my $image64 = <<IMAGE64;
iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAAXNSR0IArs4c6QAAAAxJREFUCNdj
YGBgAAAABAABJzQnCgAAAABJRU5ErkJggg==
IMAGE64
        if (open FILE, ">", $pngOut) {
            binmode FILE;
            print FILE MIME::Base64::decode($image64);
            close FILE;
        }
        do_cmd("cp $pngOut $dflt_bkgnd", 1);
    }

    do_cmd("chown -R lanforge:$user_group $home/.config");

    # we might have our default.target updated
    if (($::osveri >= 27) && ($do_runlevel == 3 || $do_runlevel == 0)) {
        my $dft_tg = readlink("/usr/lib/systemd/system/default.target");
        print("Default target: $dft_tg\n");
        if ($dft_tg =~ /graphical\.target$/) {
            do_cmd("systemctl disable graphical.target");
            do_cmd("systemctl mask graphical.target");
            if ($::osveri >= 30) {
                do_cmd("systemctl set-default multi-user.target");
            }
            else {
                unlink("/usr/lib/systemd/system/default.target");
                symlink("/usr/lib/systemd/system/multi-user.target", "/usr/lib/systemd/system/default.target");
            }
        }
    }
} # ~gnome/mate desktop and screen lock

# Fix up TCP X-11 sessions..this is similar to the
# old DisallowTCP=false hacks to gdm.
if ($do_lanforge) {

    # Ensure macos doesn't go to sleep.
    if ($is_macos) {
        # Consider: dscl . -append /groups/admin GroupMembership lanforge
        # But need to check first or it will be appeded over and over.
        # dscl . -read /groups/admin GroupMembership
        do_cmd("systemsetup -setcomputersleep Never", 1);
    }

    fixup_lightdm_conf();

    my $fn;
    if (-d "/etc/lightdm/lightdm.conf.d/") {
        $fn = "/etc/lightdm/lightdm.conf.d/100-lanforge.conf";
        if (!-f $fn) {
            if (open(FILE, ">", $fn)) {
                print FILE "[SeatDefaults]\nxserver-allow-tcp=true\n";
                close FILE;
            }
            else {
                warning("# unable to open $fn\n");
            }
        }
    }

    $fn = "/etc/X11/xinit/xserverrc";
    if (-f $fn) {
        my @new_lines = ();
        my @lines = `cat $fn`;
        chomp(@lines);
        note("# Disabling -nolisten tcp in $fn\n");
        for my $ln (@lines) {
            if ($ln !~ /^\w*\#/ && $ln =~ /-nolisten tcp/) {
                $ln =~ s/-nolisten tcp//g;
            }
            push(@new_lines, $ln);
        }
        if (open(FILE, ">", $fn)) {
            print FILE join("\n", @new_lines) . "\n";
            close FILE;
        }
        else {
            warning("# unable to open $fn\n");
        }
    }

    # In case user is starting with 'startx', then that needs hacking too.
    $fn = "/usr/bin/startx";
    if (-f $fn && !$is_macos) {
        my @new_text = ();
        my $line_limit = 0;
        my $found_listen = 0;
        my @lns = `cat $fn`;
        chomp(@lns);
        note("# Disabling -nolisten tcp in $fn\n");
        for my $ln (@lns) {
            if ($ln =~ m{Automatically determine an unused .DISPLAY}) {
                $line_limit = 1;
            }
            if ($line_limit) {
                if ($::osveri > 21 && $found_listen == 0) {
                    push(@new_text, qq[listenarg="-listen tcp"]);
                    $found_listen = 1;
                }
                push(@new_text, $ln);
                next;
            }
            elsif (($ln =~ m{^defaultserverargs=}) && ($::osveri > 21)) {
                push(@new_text, qq[defaultserverargs="-listen tcp"]);
            }
            elsif ($ln =~ m{^listenarg=".+"} || $ln =~ m{^listenarg=""}) {
                $found_listen = 1;
                if ($::osveri le 21) {
                    push(@new_text, qq[listenarg=""]);
                }
                else {
                    push(@new_text, qq[listenarg="-listen tcp"]);
                }
            }
            else {
                push(@new_text, $ln);
            }
        } #~for
        if (open FILE, ">$fn") {
            print FILE join("\n", @new_text) . "\n";
            close FILE;
        }
    } # ~ -f bin/startx

    $fn = "/usr/bin/X11/X";
    if (-f "/usr/bin/X11/X && !$is_macos") {
        my @new_text = ();
        my $line_limit = 0;
        my $found_listen = 0;
        my @lns = `cat $fn`;
        chomp(@lns);
        for my $ln (@lns) {
            if ($ln !~ /^\w*\#.*$/ && $ln =~ /-nolisten tcp/) {
                $ln =~ s/-nolisten tcp//;
            }
            push(@new_text, $ln);
        }
        note("# Disabling -nolisten tcp in $fn\n");
        if (open FILE, ">", $fn) {
            print FILE join("\n", @new_text) . "\n";
            close FILE;
        }
    }


    # check that wireshark isn't going to complain about XKEYBOARD extensions because of Qt
    if ($is_deb_based) {
        my $dpkgq = q(dpkg-query -f='${db:Status-Abbrev}\n' -W wireshark-gtk 2>/dev/null| grep 'ii ');
        my @lines = `$dpkgq||:`;
        if (@lines > 0) {
            my @list = `update-alternatives --display wireshark 2>/dev/null`;
            if ($? != 0 || @list < 1) {
                if (-x "/usr/bin/wireshark-gtk" && -x "/usr/bin/wireshark") {
                    do_cmd("dpkg -r --force-depends wireshark-qt", 1);
                }

                if (!-x "/usr/bin/wireshark") {
                    note(" * updating wirehark-gtk to wireshark using update-alternatives\n");
                    do_cmd("update-alternatives --install /usr/bin/wireshark wireshark /usr/bin/wireshark-gtk 1", 1);
                }
                note(" Cannot update wirehark-gtk to wireshark using update-alternatives, file exists\n");
            }
        }
    }
    if (!$is_macos) {
        # "$home/.bash_aliases", prolly unavailable
        my @check_these = ("$home/.bashrc", "$home/.bash_profile", "$home/.bash_history");
        for my $file (@check_these) {
            if (-f $file) {
                `chown lanforge:$user_group $file`;
            }
            else {
                my $alternate = $file;
                $alternate =~ s|$home|/etc/skel|;
                if (-f $alternate) {
                    warning("* Replacing missing $file from $alternate\n");
                    do_cmd("cp -v $alternate $file");
                    do_cmd("chown lanforge:$user_group $file");
                    chmod(0640, $file);
                }
                else {
                    note("* No alternate ($alternate) source for $file\n");
                }
            }
        }
    }
} # ~ do_lanforge

if ($do_vnc || $do_xrdp) {
    adjust_xrdp();
}
if ($do_udev) {
    _do_udev();
}

sub add_to_download_list {
    my $item = shift;
    $::download_list{$item}++;
    if ($item =~ m|/|) {
        err("add_to_download_list found poor path: $item");
        print Dumper(%::download_list);
        die("stuff! $!");
    }
}

sub adjust_xrdp {
    if ($is_macos) {
        return;
    }
    # This makes XRDP work but it has to run when xrdp is configured
    if (-d "/etc/sysconfig") {
        open(my $xorgdesktop, ">", "/etc/sysconfig/desktop")
            or die("Unable to open /etc/sysconfig/desktop: $!");
        print $xorgdesktop "PREFERRED=mate-session\n";
        print $xorgdesktop "DESKTOP=MATE\n";
        print $xorgdesktop "unset SESSION_MANAGER\n";
        print $xorgdesktop "unset DBUS_SESSION_BUS_ADDRESS\n";
        close($xorgdesktop);
    }
    else {
        print("* XRDP configuration abbreviated, /etc/sysconfig not present; are you on UBUNTU?\n");
    }

    # Fix up xrdp.ini

    my $add_xorg = ($is_fedora && ($::osveri >= 24)) ? 1 : 0;
    my $saw_autorun = 0;

    if (-d "/etc/xrdp") {
        note("# Fixing xrdp.ini...\n");
        my $fn = "/etc/xrdp/xrdp.ini";
        if (-f "$fn.rpmnew") {
            do_cmd("mv $fn.rpmnew $fn");
        }
        if (-f "$fn") {
            my @lns = `cat $fn`;
            my $buf = "";
            my $add_xrdp1 = 1;
            my @matches1 = grep {/\[xrdp1\]/} @lns;
            $add_xrdp1 = 0 if (@matches1 > 0);
            #my @matches2 = grep { /\[Xorg\]/ } @lns;
            #$add_xorg = 0 if (@matches2 > 0);

            for my $ln (@lns) {
                chomp($ln);

                if ($ln =~ /^autorun=/) {
                    if ($is_fedora && $add_xorg) {
                        note("autorun ADD_XORG\n");
                        $ln = "autorun=Xorg";
                        $saw_autorun = 1;
                    }
                    else {
                        note("autorun ADD_XRDP1 1\n");
                        $ln = "autorun=xrdp1";
                        $saw_autorun = 1;
                    }
                }
                elsif ($ln eq "; Session types") {
                    # Remove all text below that in the file.
                    $buf .= "$ln\n;\n\n";
                    last;
                }
                $buf .= "$ln\n";
            } # ~for

            # Xorg setting and autoren not available in F21?
            if ($add_xorg) {
                $buf .= "
[Xorg]
name=Xorg
lib=libxup.so
username=ask
password=ask
ip=127.0.0.1
port=-1
code=20
channelcliprdr=0
";
            }
            if ($add_xrdp1) {
                $buf .= "
[xrdp1]
name=sesman-Xvnc
lib=libvnc.so
username=ask
password=ask
ip=127.0.0.1
port=-1
delay=2000

";
            }
            if (open FILE, ">", $fn) {
                print FILE $buf;
                close FILE;
            }
        } # ~ done with xrdp.ini

        # adjust xrdp/sesman.ini next
        my @sesman_stanza = split("\n", "[Xorg]
param=Xorg
param=-config
param=xrdp/xorg.conf
param=-noreset
param=-listen
param=tcp
param=-logfile
param=.xorgxrdp.%s.log
");


        # Fix up Xrdp to allow xhost+ sessions to run.
        note("# Fixing sesman.ini...\n");
        $fn = "/etc/xrdp/sesman.ini";
        if (-f "$fn.rpmnew") {
            # In case newer xrdp was installed, modify from the new config file.
            do_cmd("mv $fn.rpmnew $fn");
        }
        if (-f "$fn") {
            my @x11rdp_opts_21 = qw(-bs);
            my @x11rdp_opts_23 = qw(-bs -listen tcp);
            my @xvnc_opts_21 = qw(-bs -localhost -dpi 96);
            my @xvnc_opts_23 = qw(-bs -localhost -dpi 96 -listen tcp);
            my @xorg_opts_24 = qw(Xorg -config xrdp/xorg.conf -noreset -listen tcp -logfile .xorgxrdp.%s.log);
            my @sesman_lines = `cat $fn`;
            push(@sesman_lines, ""); # force an extra blank line
            chomp(@sesman_lines);
            my @new_sesman_lines = ();
            my @extra_options = ();
            my $write_me = 0;
            my $interesting = 0;
            my $add_listen = ($::osveri > 21) ? 1 : 0;
            my $ra_new_options = undef;
            my $extra_conf = "";
            my $use_param_numbers = 1;
            my $saw_xorg = 0;
            my $saw_xvnc = 0;
            my $saw_x11rdp = 0;
            for my $ln (@sesman_lines) {
                if ($ln =~ /^\[X11rdp\]/) {
                    $interesting = 1;
                    $saw_x11rdp = 1;
                    push(@new_sesman_lines, $ln);
                    $ra_new_options = ($::osveri > 21) ? \@x11rdp_opts_23 : \@x11rdp_opts_21;
                    next;
                }
                if ($ln =~ /^\[Xvnc\]/) {
                    $interesting = 1;
                    $saw_xvnc = 1;
                    push(@new_sesman_lines, $ln);
                    $ra_new_options = ($::osveri > 21) ? \@xvnc_opts_23 : \@xvnc_opts_21;
                    next;
                }
                if ($ln =~ /^\[Xorg\]/) {
                    $interesting = 1;
                    $saw_xorg = 1;
                    push(@new_sesman_lines, $ln);
                    $ra_new_options = ($::osveri > 23) ? \@xorg_opts_24 : [];
                    next;
                }

                if ($interesting) {
                    # We are in one of the special sections above.  Newer F24, at least, needs
                    # an extra argument...
                    if (($ln eq "param=X11rdp") || ($ln eq "param=Xvnc")) {
                        $use_param_numbers = 0;
                        $extra_conf = $ln;
                    }
                    elsif ($ln eq "param=-uds") {
                        push(@extra_options, "-uds");
                    }
                }
                else {
                    push(@new_sesman_lines, $ln);
                    next;
                }

                # want to collect this stanza and re-write it with correct numerals
                if ($ln =~ /^\s*$/) {
                    my $param_no = 1;
                    if ($extra_conf ne "") {
                        push(@new_sesman_lines, $extra_conf);
                    }
                    for my $item (@$ra_new_options) {
                        if ($use_param_numbers) {
                            push(@new_sesman_lines, "param$param_no=$item");
                            $param_no++;
                        }
                        else {
                            push(@new_sesman_lines, "param=$item");
                        }
                    }
                    for my $item (@extra_options) {
                        if ($use_param_numbers) {
                            push(@new_sesman_lines, "param$param_no=$item");
                            $param_no++;
                        }
                        else {
                            push(@new_sesman_lines, "param=$item");
                        }
                    }

                    push(@new_sesman_lines, $ln);
                    $interesting = 0;
                    $extra_conf = "";
                    $write_me = 1;
                    @extra_options = ();
                } # ~if an interesting blank line
            }     #~for $ln
            if ($add_xorg && !$saw_xorg) {
                $write_me = 1;
                push(@new_sesman_lines, @sesman_stanza);
            }
            if ($write_me) {
                note("# Changed -nolisten arguments from xrdp sesman.ini\n");
                if (open FILE, ">$fn") {
                    print FILE join("\n", @new_sesman_lines) . "\n";
                    close FILE;
                }
                if (system("which systemctl &>/dev/null") == 0) {
                    #do_cmd("systemctl restart xrdp.service xrdp-sesman.service", 1);
                    note("# To restart xrdp use this command:     systemctl restart xrdp.service xrdp-sesman.service");
                }
                elsif (system("which service &>/dev/null") == 0) {
                    note("# To restart xrdp use this command:     service xrdp restart");
                    #do_cmd("service xrdp restart", 1);
                    #do_cmd("service xrdp-sesman restart", 1);
                }
            }
        } # ~if sesman.ini
    }
    else {
        warning("XRDP not installed? /etc/xrdp missing. Are you on UBUNTU?");
    }

    if ($add_xorg && $::osveri >= 27) {
        open(my $fh_xwrapper, ">", "/etc/X11/Xwrapper.config")
            or die("Unable to write /etc/X11/Xwrapper.config");
        print $fh_xwrapper "allowed_users = anybody\n";
        print $fh_xwrapper "needs_root_rights = yes\n";
        close($fh_xwrapper);
    }
    if ($::osveri >= 36 && $::osveri < 38) {
        if (! -f "/usr/lib64/xrdp/libxup.so") {
            my $rh_req = newWebRequest("$download_host/downloads/f36/libxup.so", "$::tmp_dir/libxup.so");
            $rh_req->{"notify_on_fail"} = 1;
            $rh_req->{"die_on_fail"} = 1;
            $rh_req->{"notify_on_start"} = 'yes';
            if (webget($rh_req) != 0) {
                warning("webget libxup.so DOWNLOAD ERROR\n");
                print Dumper($rh_req) if ($::debug_on);
                return;
            }
            do_cmd("cp $::tmp_dir/libxup.so /usr/lib64/xrdp/libxup.so");
            chmod(0755, "/usr/lib64/xrdp/libxup.so");
        }
    }
} # ~adjust_xrdp

sub set_xresources {
    if ($is_macos) {
        return;
    }
    my $file = shift;
    if (!defined($file) || $file eq "") {
        return;
    }
    my $do_remove_shadow = 0;
    $do_remove_shadow = 1 if (!-f "$file");
    if (-f "$file") {
        `fgrep 'Xcursor.core' $file 2>/dev/null`;
        $do_remove_shadow = 1 if ($? != 0);
    }
    if ($do_remove_shadow) {
        do_cmd(qq{echo 'Xcursor.core: 1' >> $file});
    }
    1;
}

if (!$::create_install_bundle && $do_runlevel) {
    # /lib/systemd/system is the correct spot for this
    my $wrong_default = "/etc/systemd/system/default.target";
    if (-f $wrong_default) {
        unlink($wrong_default);
    }

    my $targ = "/lib/systemd/system/default.target";
    if (!-f $targ) {
        if (-f "/usr/lib/systemd/system/default.target") {
            $targ = "/usr/lib/systemd/system/default.target";
        }
        else {
            warn(" * * Unable to find default.target in /lib or /usr/lib\n");
        }
    }
    if (-f $targ) {
        if ($do_runlevel <= 0) {
            print "No change to runlevels requested.\n";
        }
        elsif ($do_runlevel == 3) {
            note("# Changing runlevel to level 3 (multi-user, non graphical).\n");
            do_cmd("rm -f $targ");
            do_cmd("ln -s /lib/systemd/system/multi-user.target $targ");
            do_cmd("systemctl disable graphical.target");
            do_cmd("systemctl mask graphical.target");
            do_cmd("systemctl daemon-reload");

            if ($do_install_mate || $mate_installed) {
                create_lf_user() if ($::should_create_user);

                my $xinitrc_txt = q{#!/bin/sh
unset SESSION_MANAGER
unset DBUS_SESSION_BUS_ADDRESS
if [ -d /etc/X11/xinit/xinitrc.d ]; then
  for f in /etc/X11/xinit/xinitrc.d/*; do
    [ -x "$f" ] && . "$f"
  done
  unset f
fi
xsetroot -solid gray
exec /usr/bin/mate-session
};
                my $fn = "/root/.xinitrc"; # implies /root
                open my $fh, ">$fn" || die "Couldn't open file: $fn  $!\n\n";
                print $fh $xinitrc_txt;
                close $fh;
                $fn = "$home/.xinitrc"; # implies $home
                open $fh, ">$fn" || die "Couldn't open file: $fn  $!\n\n";
                print $fh $xinitrc_txt;
                close $fh;
                do_cmd("chown lanforge:$user_group $home/.xinitrc");
                do_cmd("chmod u+x $home/.xinitrc", 1);
                set_xresources("$home/.Xresources");
                set_xresources("~/.Xresources");
            }
            # Work around Fedora 19 bug when using serial console:
            # https://bugzilla.redhat.com/show_bug.cgi?id=979174
            if ($is_fedora && ($::osveri >= 19) && ($::osveri < 23)) {
                do_cmd("$yum -y erase initial-setup", 1);
            }
            if ($is_fedora && ($::osveri >= 24) && ($::osveri <= 26)) {
                do_cmd(q{echo "allowed_users=anybody" > /etc/X11/Xwrapper.config});
            }
        }
        elsif ($do_runlevel == 5) {
            note("# Changing runlevel to level 5 (multi-user, graphical).\n");
            do_cmd("rm -f $targ");
            do_cmd("ln -s /lib/systemd/system/graphical.target $targ");
            do_cmd("systemctl daemon-reload");
        }
        else {
            err("# ERROR:  Only runlevel 3 and 5 currently supported by this script, requested: $do_runlevel\n");
        }
    }
    else {
        warning("# WARNING:  Could not find file $targ, this script does not support setting runlevel on this system.\n");
    }
}
## ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ======
#           VNC Server
## ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ======
if (!$::create_install_bundle && $do_vnc && !($download_only || $::create_install_bundle)) {
    my $fn;
    if ($staged_in_candela && !$is_macos) {
        my $jd = "/mnt/d2/pub/lf-staging/vnc-client-java/";
        if (-d "$jd") {
            do_cmd("cp -ar /mnt/d2/pub/lf-staging/vnc-client-java/vnc/ /usr/share/");
        }
        else {
            warning("\n# WARNING:  Could not find $jd, no java VNC support installed.\n\n");
        }
    }

    if ($is_macos) {
        # Seems macos broke this with recent updates..have to do it by hand.
        #https://gist.github.com/nateware/3915757
        # Step 1: Set priveleges
        #do_cmd("/System/Library/CoreServices/RemoteManagement/ARDAgent.app/Contents/Resources/kickstart -configure -allowAccessFor -allUsers -privs -all", 1);
        # Step 2: Allow VNC clients
        #do_cmd("/System/Library/CoreServices/RemoteManagement/ARDAgent.app/Contents/Resources/kickstart -configure -clientopts -setvnclegacy -vnclegacy yes", 1);
        # Step 3: Set VNC password (change it at the end of the line (i.e. don't use supersecret))
        #do_cmd("/System/Library/CoreServices/RemoteManagement/ARDAgent.app/Contents/Resources/kickstart -configure -clientopts -setvncpw -vncpw lanforge", 1);
        # Step 4: Restart service
        #do_cmd("/System/Library/CoreServices/RemoteManagement/ARDAgent.app/Contents/Resources/kickstart -restart -agent -console", 1);
        # Step 5: If no ARD services have been activated on the machine before, it is also necessary to run the following command
        #do_cmd("/System/Library/CoreServices/RemoteManagement/ARDAgent.app/Contents/Resources/kickstart -activate", 1);

        # And ssh while we are at it.
        # https://apple.stackexchange.com/questions/278744/command-line-enable-remote-login-and-remote-management
        #do_cmd("systemsetup -setremotelogin on", 1);

        #do_cmd("/System/Library/CoreServices/RemoteManagement/ARDAgent.app/Contents/Resources/kickstart --configure --allowAccessFor --allUsers", 1);

        # Not strictly vnc, but this works around bug allowing ssh access on macos Catelina
        # https://apple.stackexchange.com/questions/442379/cant-enable-ssh-on-catalina-system-preferences-keeps-saying-remote-login-sta
        #do_cmd("launchctl load -w /System/Library/LaunchDaemons/ssh.plist", 1);
    }
    elsif ($is_deb_based) {
        # TODO: likely out of date now
        if ($uveri >= 16 && !$is_arm) {
            mkdir("$home/.vnc");
            my $fn = "$home/.vnc/xstartup";
            if (-f $fn) {
                # Make a backup if we have not done so already.
                if (!-f "$fn.orig") {
                    do_cmd("cp $fn $fn.orig");
                }
            }
            my $listen_tcp = "";

            # For Fedora, jump to line 5830
            #if ($::osveri >= 25) {
            #   $listen_tcp = "-listen tcp";
            #}
            my $autokill = "-autokill";
            if ($is_deb_based) {
                $autokill = "";
            }
            if ( -x "/usr/bin/vncserver" && -x "/usr/bin/vncpasswd") {
                open FILE, ">$fn" || die "Couldn't open file: $fn  $!\n\n";
                print FILE "#!/bin/sh
    # WARNING:  This will be over-written by LANforge lf_kinstall.pl script.
    unset SESSION_MANAGER
    unset DBUS_SESSION_BUS_ADDRESS
    [ -r $home/.Xresources ] && xrdp $home/.Xresources
    [ -x $home/config-desktop.sh ] && $home/config-desktop.sh ||:
    /usr/bin/mate-session
    ";
                close FILE;
                do_cmd("chmod a+x $fn");
                #`systemctl -q is-enabled vncserver`
                my $vnc_unit_fn = '/lib/systemd/system/vncserver@.service';
                if (!-f $vnc_unit_fn) {
                    my $vnc_unit = qq([Unit]
    Description=Remote desktop service (VNC)
    After=syslog.target network.target

    [Service]
    Type=forking
    User=lanforge

    # Clean any existing files in /tmp/.X11-unix environment
    ExecStartPre=-/usr/bin/vncserver -kill %i
    ExecStart=/usr/bin/vncserver -depth 16 $autokill -geometry 1600x900 $listen_tcp %i
    WorkingDirectory=$home
    PIDFile=$home/.vnc/%H%i.pid
    ExecStop=-/usr/bin/vncserver -kill %i

    [Install]
    WantedBy=multi-user.target

             );
                    #         my $vnc_unit = qq([Unit]
                    #Description=Start TightVNC server at startup
                    #After=syslog.target network.target
                    #
                    #[Service]
                    #Type=forking
                    #User=lanforge
                    #PAMName=login
                    #PIDFile=$home/.vnc/%H%i.pid
                    #ExecStartPre=/bin/sh -c '/usr/bin/vncserver -kill %i > /dev/null 2>&1 ||:'
                    #ExecStart=/usr/bin/vncserver -depth 24 -geometry 1280x800 %i
                    #ExecStop=/usr/bin/vncserver -kill %i
                    #WorkingDirectory=$home
                    #
                    #[Install]
                    #WantedBy=multi-user.target
                    #);
                    if (open(my $unit_fh, ">", $vnc_unit_fn)) {
                        print $unit_fh $vnc_unit;
                        close $unit_fh;
                        unlink('/etc/systemd/system/vncserver@:1.service')
                            if (-f '/etc/systemd/system/vncserver@:1.service');
                        symlink($vnc_unit_fn, '/etc/systemd/system/vncserver@:1.service');
                        unlink('/etc/systemd/system/multi-user.target.wants/vncserver@:1.service')
                            if (-f '/etc/systemd/system/multi-user.target.wants/vncserver@:1.service');
                        symlink($vnc_unit_fn, '/etc/systemd/system/multi-user.target.wants/vncserver@:1.service');
                    }
                    else {
                        warn("Problem changing systemd vncserver service files.");
                    }
                    do_cmd("loginctl enable-linger lanforge");
                    do_cmd("echo lanforge | vncpasswd -f > $home/.vnc/passwd");
                    do_cmd("chown -R lanforge:$user_group $home/.vnc");
                    do_cmd("chmod 600 $home/.vnc/passwd");
                    do_cmd('systemctl enable vncserver@:1.service', 1);
                    do_cmd("systemctl daemon-reload", 1);
                }
            }
            else {
                warning("* VNCServer not installed (missing /usr/bin/vncserver, /usr/bin/vncpasswd) Are you on UBUNTU?\n");
            }
        } # ~if ubuntu 16
        # Seems a total pain to configure VNC on Ubuntu, so will set up
        # xrdp instead.
        my $sessionmgr = undef;
        if ($do_install_mate || $mate_installed || -x "/usr/bin/mate-session") {
            $sessionmgr = "mate-session";
        }
        elsif (-x "/usr/bin/xfce4-session") {
            $sessionmgr = "xfce4-session";
        }
        elsif (-x "/usr/bin/lxsession") {
            $sessionmgr = "lxsession";
        }
        elsif (-x "/usr/bin/gnome-session") {
            $sessionmgr = "gnome-session --session=gnome-fallback";
        }
        else {
            warning("# NO X session manager discovered.");
        }
        if (defined $sessionmgr) {
            my @xsess = (
                "xsetroot -solid gray",
                "[ -x $home/config-desktop.sh ] && $home/config-desktop.sh",
                "$sessionmgr",
                q(#)
            );
            if (open FILE, ">$home/.xsession") {
                print FILE join("\n", @xsess);
                close FILE;
            }
            if (open FILE, ">/root/.xsession") {
                print FILE join("\n", @xsess);
                close FILE;
            }
        }

        # Looks like I might need to hack .dmrc as well.
        my $fname = "$home/.dmrc";
        my $lns = "";
        if (-f $fname) {
            $lns = `cat $fname`;
        }
        if ($lns =~ /Session=gnome-fallback/) {
            # Nothing to do.
        }
        else {
            if ($lns =~ /Session\=(\S+)/) {
                my $old = $1;
                note("# Changing .dmrc gnome-session from: $old to gnome-fallback.\n");
                $lns =~ s/Session=$old/Session=gnome-fallback/g;
            }
            else {
                # Add new section.
                note("# Adding .dmrc gnome-fallback session configuration.\n");
                $lns .= "[Desktop]\nSession=gnome-fallback\n";
            }
            if (open FILE, ">$fname") {
                print FILE $lns;
                close FILE;
            }
        }
    }
    elsif ($::osveri >= 34) {
        # newer version of VNCServer 1.11
        note("* setting up vncserver...\n");
        unlink('/etc/systemd/system/vncserver@:1.service')
            if (-f '/etc/systemd/system/vncserver@:1.service');
        unlink('/etc/systemd/system/vncserver@.service')
            if (-f '/etc/systemd/system/vncserver@.service');

        # update /etc/tigervnc/vncserver.users, make sure :1=lanforge
        my $create_vnc_lf_user = 0;
        if (!-f "/etc/tigervnc/vncserver.users") {
            $create_vnc_lf_user = 1;
        }
        else {
            `grep -q '^:1=lanforge' /etc/tigervnc/vncserver.users`;
            if ($? != 0) {
                $create_vnc_lf_user = 1;
            }
        }
        if ($create_vnc_lf_user) {
            do_cmd("echo ':1=lanforge' >> /etc/tigervnc/vncserver.users");
        }

        # write /home/lanforge/.vnc/config
        mkdir("$home/.vnc") unless (-d "$home/.vnc");
        do_cmd("chown lanforge:$user_group $home/.vnc");
        do_cmd("chmod 700 $home/.vnc");
        if (-f "$home/.vnc/config") {
            `grep -q '^geometry='  $home/.vnc/config`;
            do_cmd("echo 'geometry=1600x900' >> $home/.vnc/config")
                unless ($? == 0);
            `grep -q '^alwaysshared'  $home/.vnc/config`;
            do_cmd("echo 'alwaysshared' >> $home/.vnc/config")
                unless ($? == 0);
            `grep -q '^listen=TCP'  $home/.vnc/config`;
            do_cmd("echo 'listen=TCP' >> $home/.vnc/config")
                unless ($? == 0);
        }
        else {
            do_cmd("echo 'geometry=1600x900' >> $home/.vnc/config");
            do_cmd("echo 'alwaysshared' >> $home/.vnc/config");
            do_cmd("echo 'listen=TCP' >> $home/.vnc/config");
        }
        # create vncpassword
        do_cmd("echo lanforge | vncpasswd -f > $home/.vnc/passwd");
        do_cmd("chown -R lanforge:$user_group $home/.vnc");
        do_cmd("chown -R lanforge:$user_group $home/.config");
        do_cmd("chmod 600 $home/.vnc/passwd");

        # vnc session background
        if ($do_install_mate || $mate_installed) {
            note("* Creating config-desktop.desktop for Fedora 34+\n");
            $fn = "$home/config-desktop.desktop";
            if (-f $fn) {
                # Make a backup if we have not done so already.
                if (!-f "$fn.orig") {
                    do_cmd("cp $fn $fn.orig");
                }
            }
            open FILE, ">$fn" || die "Couldn't open file: $fn  $!\n\n";
            print FILE "[Desktop Entry]
Name=Config Desktop
Comment=Applies LANforge desktop background
Exec=$home/config-desktop.sh
Icon=
Terminal=false
Type=Application
";
            close FILE;
            do_cmd("chmod 640 $fn");
            do_cmd("chown lanforge: $fn");
            symlink("$home/$fn", "$home/.config/autostart/$fn");
        }

        # enable service
        do_cmd("systemctl enable vncserver\@:1.service");
        my $vnc_service_override_d = "/etc/systemd/system/vncserver\@:1.service.d";
        my $vnc_service_override_f = "${vnc_service_override_d}/override.conf";
        if (!-f $vnc_service_override_f) {
            mkdir($vnc_service_override_d)
                unless (-d $vnc_service_override_d);
            do_cmd("echo '[Service]' > ${q}${vnc_service_override_f}${q}");
            do_cmd("echo 'WorkingDirectory=$home' >> ${q}${vnc_service_override_f}${q}");
            do_cmd("systemctl daemon-reload");
        }
        do_cmd("systemctl restart vncserver\@:1.service");
    }
    elsif ($::osveri > 14) {
        # This does NOT apply if we are using MATE, which we now use by default in Fedora 19+.
        # --Ben
        # More needs to be done for Fedora 17-19 and gnome-classic mode, see comment 24:
        # https://bugzilla.redhat.com/show_bug.cgi?id=896648
        # Summary:
        # - $home/.vnc/xstartup
        #     #twm &
        #     exec gnome-session --session=gnome-classic &
        #     AND/OR edit /usr/bin/vncserver to generate gnome-session line in xstartup.
        #
        # - To remove vncconfig window from desktop panel change option to -nowin in: /usr/bin/vncserver, e.g.:
        #     system($exedir."vncconfig -nowin >> " . &quotedString($desktopLog) . " 2>&1 &");
        #
        # - Add "-session optional pam_systemd.so" line to /etc/pam.d/runuser-l so runuser-l will look like this:
        #     auth            include         runuser
        #     session         optional        pam_keyinit.so force revoke
        #     -session        optional        pam_systemd.so
        #     session         include         runuser
        #
        # - /etc/systemd/system/vncserver@:X.service
        #     "Type=simple" and add "-listen tcp -fg" to vncserver@.service...
        #     Should look like this:
        #
        #     [Service]
        #     Type=simple
        #     # Clean any existing files in /tmp/.X11-unix environment
        #     ExecStartPre=/bin/sh -c '/usr/bin/vncserver -kill %i > /dev/null 2>&1 || :'
        #     ExecStart=/sbin/runuser -l fedora -c "/usr/bin/vncserver -listen tcp -fg %i"
        #     ExecStop=/sbin/runuser -l fedora -c "/usr/bin/vncserver -kill %i"
        #
        # - Edit /usr/bin/vncserver:
        #     $geometry = "1280x1024";
        #

        # Notes for MATE.  Work around lots of 'caja' windows opening sometimes.
        # As a workaround i suggest to edit two desktop files.
        # /etc/xdg/autostart/caja-autostart.desktop
        # change here the Exec line to 'Exec=caja -n --sync'

        # /usr/share/applications/caja.desktop
        # change here the Exec line to 'Exec=caja -n --sync'
        # See:  https://bugzilla.redhat.com/show_bug.cgi?id=886029

        unlink('/etc/systemd/system/vncserver@:1.service')
            if (-f '/etc/systemd/system/vncserver@:1.service');
        unlink('/etc/systemd/system/multi-user.target.wants/vncserver@:1.service')
            if (-f '/etc/systemd/system/multi-user.target.wants/vncserver@:1.service');
        unlink('/etc/systemd/system/vncserver@.service')
            if (-f '/etc/systemd/system/vncserver@.service');
        unlink('/etc/systemd/system/multi-user.target.wants/vncserver@.service')
            if (-f '/etc/systemd/system/multi-user.target.wants/vncserver@.service');
        unlink('/lib/systemd/system/vncserver@:1.service')
            if (-f '/lib/systemd/system/vncserver@:1.service');

        #my $fn = '/lib/systemd/system/vncserver@:1.service';
        #    my @vncserver_lines = `cat /lib/systemd/system/vncserver@.service`;
        #    my @new_vnc_lines = ();
        #    chomp @vncserver_lines;
        #    my @workingdir_lines = grep { /WorkingDirectory=/ } @vncserver_lines;
        #    for my $vnc_line (@vncserver_lines) {
        #       $vnc_line =~ s{[<]USER[>]}{lanforge}   if ($vnc_line=~/<USER>/);
        #       $vnc_line =~ s{User=}{User=lanforge}   if ($vnc_line=~/^User=$/);
        #       if ($::osveri > 21 && $vnc_line =~ /^ExecStart=.usr.bin.vncserver %i/ ) {
        #         $vnc_line =~ s/vncserver %i/vncserver -listen tcp %i/;
        #       }
        #       push(@new_vnc_lines, $vnc_line);
        #
        #       if ($vnc_line =~ /^ExecStart=/ && @workingdir_lines < 1) {
        #         push(@new_vnc_lines, "WorkingDirectory=$home");
        #       }
        #    }

        my $vnc_unit_fn = '/lib/systemd/system/vncserver@.service';
        unlink($vnc_unit_fn)
            if (-f $vnc_unit_fn);
        my $listen_tcp = "";
        if ($::osveri >= 22) {
            $listen_tcp = "-listen tcp";
        }
        my $vnc_unit = qq(
[Unit]
Description=Remote desktop service (VNC)
After=syslog.target network.target

[Service]
Type=forking
User=lanforge

# Clean any existing files in /tmp/.X11-unix environment
ExecStartPre=-/usr/bin/vncserver -kill %i
ExecStart=/usr/bin/vncserver -depth 16 -autokill -geometry 1600x900 $listen_tcp %i
WorkingDirectory=$home
PIDFile=$home/.vnc/%H%i.pid
ExecStop=-/usr/bin/vncserver -kill %i

[Install]
WantedBy=multi-user.target
);
        open(FILE, ">", $vnc_unit_fn) or die "Couldn't open file: $vnc_unit_fn  $!\n\n";
        print FILE $vnc_unit;
        close FILE;

        unlink('/etc/systemd/system/vncserver@:1.service')
            if (-e '/etc/systemd/system/vncserver@:1.service');
        symlink($vnc_unit_fn, '/etc/systemd/system/vncserver@:1.service');

        unlink('/etc/systemd/system/multi-user.target.wants/vncserver@:1.service')
            if (-e '/etc/systemd/system/multi-user.target.wants/vncserver@:1.service');
        symlink($vnc_unit_fn, '/etc/systemd/system/multi-user.target.wants/vncserver@:1.service');
        do_cmd("systemctl daemon-reload");

        if (!-d "$home/.vnc") {
            do_cmd("mkdir -p $home/.vnc");
        }

        note("# Setting VNC default resolution to 1600x900\n");
        # the config/monitors.xml file configures Gnome and the file
        # is undocumented. https://unix.stackexchange.com/questions/152586/what-do-the-tags-in-monitors-xml-do
        # Some of the recommended methods of changing the desktop resolution also involve `gtf` as shown
        # https://superuser.com/questions/1086133/vnc-server-configuration-for-multi-monitor-support
        # https://help.realvnc.com/hc/en-us/articles/360016058212-How-do-I-adjust-the-screen-resolution-of-a-virtual-desktop-under-Linux-
        # And updating library files in /usr/share/X11/xorg.conf.d
        do_cmd("mkdir -p $home/.config/");
        $fn = "$home/.config/monitors.xml";
        open(FILE, ">", $fn) or die "Couldn't open file: $fn  $!\n\n";
        print FILE "<monitors version=\"1\">
  <configuration>
      <clone>no</clone>
      <output name=\"VNC-0\">
          <vendor>???</vendor>
          <product>0x0000</product>
          <serial>0x00000000</serial>
          <width>1600</width>
          <height>900</height>
          <rate>30</rate>
          <x>0</x>
          <y>0</y>
          <rotation>normal</rotation>
          <reflect_x>no</reflect_x>
          <reflect_y>no</reflect_y>
          <primary>yes</primary>
      </output>
  </configuration>
</monitors>
";
        close FILE;

        if ($do_install_mate || $mate_installed) {
            note("* Creating Mate desktop for Fedora 19+\n");
            $fn = "$home/.vnc/xstartup";
            if (-f $fn) {
                # Make a backup if we have not done so already.
                if (!-f "$fn.orig") {
                    do_cmd("cp $fn $fn.orig");
                }
            }
            open FILE, ">$fn" || die "Couldn't open file: $fn  $!\n\n";
            print FILE "#!/bin/sh

# WARNING:  This will be over-written by LANforge lf_kinstall.pl script.

unset SESSION_MANAGER
unset DBUS_SESSION_BUS_ADDRESS
DESKTOP=MATE
[ ! -d /run/user/\$UID/dconf ] && mkdir -p /run/user/\$UID/dconf
[ -r $home/.Xresources ] && xrdb $home/.Xresources
[ -x $home/config-desktop.sh ] && $home/config-desktop.sh ||:
/usr/bin/mate-session
";
            close FILE;
            do_cmd("chmod a+x $fn");
        }
        create_lf_user() if ($::should_create_user);
        if (system("which vncserver &>/dev/null") != 0) {
            note("Installing vncserver");
            if ($is_deb_based) {
                do_cmd("$apt install -y tightvncserver");
            }
            else {
                do_cmd("yum install $skip_broken -y tigervnc-server");
            }
        }
        do_cmd("echo lanforge | vncpasswd -f > $home/.vnc/passwd");
        do_cmd("chown -R lanforge:$user_group $home/.vnc");
        do_cmd("chown -R lanforge:$user_group $home/.config");
        do_cmd("chmod 600 $home/.vnc/passwd");
        if ($::osveri >= 17) {
            #do_cmd("cp /lib/systemd/system/vncserver@.service /etc/systemd/system/vncserver\@.service", 1);
            #do_cmd("systemctl enable vncserver\@:1.service", 1); # link bug, make link by hand
            symlink("/etc/systemd/system/vncserver\@:1.service", "/etc/systemd/system/multi-user.target.wants/vncserver\@:1.service");
            #do_cmd("systemctl restart vncserver\@:1.service", 1);
            warning("* Please restart vncserver to make changes take effect.\n");
            do_cmd("systemctl daemon-reload");
            #do_cmd("systemctl start vncserver\@:1.service", 1);
            note("# To restart vncserver, use the command:     systemctl restart vncserver\@:1.service");
        }
        else {
            if ($is_deb_based) {
                do_cmd("update-rc.d tightvncserver enable", 1);
            }
            else {
                do_cmd("chkconfig vncserver on", 1);
            }
            #do_cmd("service vncserver restart ", 1);
            warning("* Please restart vncserver to make changes take effect.\n");
        }
    }
    else {
        my $fname = "/etc/sysconfig/vncservers";
        my @cmds = `cat $fname`;
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        for ($i = 0; $i < @cmds; $i++) {
            my $ln = $cmds[$i];
            chomp($ln);
            if ($ln =~ /^VNCSERVERS=(.*)/) {
                my $srv = $1;
                if (!($srv =~ /\:lanforge/)) {
                    if ($srv =~ /^\"(.*)\"/) {
                        $srv = $1;
                    }
                    # Find a free display number
                    my $q;
                    for ($q = 1; $q < 10; $q++) {
                        if (!($srv =~ /$q\:\S+/)) {
                            $srv .= " $q:lanforge";
                            last;
                        }
                    }
                    print FILE "VNCSERVERS=\"$srv\"\n";
                    print FILE "VNCSERVERARGS[$q]=\"-geometry 1280x1024\"\n";
                }
                else {
                    print FILE "$ln\n";
                }
            }
            else {
                print FILE "$ln\n";
            }
        }
        do_cmd("/etc/init.d/vncserver stop", 1);
        do_cmd("pkill -9 vnc", 2);
        do_cmd("chkconfig --level 2345 vncserver on");
        do_cmd("/etc/init.d/vncserver start", 1);
    }
} # ~is deb-based
if (-d $docroot) {
    do_cmd("echo '$::lfver' > $docroot/installed-ver.txt");
}
else {
    print "docroot[$docroot] does not exist, printing installed-ver.txt to home[$home]\n";
    do_cmd("echo '$::lfver' > $home/installed-ver.txt", 1);
}

if ($do_http) {
    if ($fix_web_root && !$is_deb_based) {
        my $httpd_conf = "/etc/httpd/conf/httpd.conf";
        my $ts = ts();
        note("# Backing up $httpd_conf to: $httpd_conf.bak." . $ts . "\n");
        do_cmd("cp -a $httpd_conf $httpd_conf.bak." . $ts);
        my @cmds = `cat $httpd_conf`;
        open(FILE, ">$httpd_conf") or die "Couldn't open file: $httpd_conf for writing: $!\n\n";
        for ($i = 0; $i < @cmds; $i++) {
            my $ln = $cmds[$i];
            chomp($ln);
            if ($ln =~ /^\s*#/) {
                print FILE "$ln\n";
                next;
            }
            $ln =~ s/^\s*DocumentRoot \"\/home\/lanforge\/candela_cdrom\"/DocumentRoot \"\/var\/www\/html\"/;
            $ln =~ s/^\s*\<Directory \"\/home\/lanforge\/candela_cdrom\">/<Directory \"\/var\/www\/html\">/; # "
            print FILE "$ln\n";
        }
        close FILE;
    }
    if (!-l "$docroot/lf_reports") {
        mkdir("$home/lf_reports") if (!-d "$home/lf_reports");
        chmod(0775, "$home/lf_reports");
        do_cmd("ln -s $home/lf_reports $docroot/lf_reports")
    }
    if (-f "$docroot/index.html") {
        # We use index.php, so get rid of any conflicting index.html file.
        do_cmd("mv $docroot/index.html $docroot/index.html.orig");
    }

    if (!-d "$docroot/wslib") {
        do_cmd("mkdir -p $docroot/wslib");
    }

    if ($do_local_install) {
        if ($source_dir ne $::tmp_dir) {
            do_cmd("cp $source_dir/$doc_pkg $::tmp_dir/$doc_pkg");
        }
        add_to_download_list($doc_pkg);
    }
    elsif (-f "$d2dir/$doc_pkg") {
        do_cmd("cp $d2dir/$doc_pkg $::tmp_dir/$doc_pkg");
        add_to_download_list($doc_pkg);
    }
    else {
        download("$url_base/$doc_pkg", "$cachebuster", "$::tmp_dir/$doc_pkg");
        add_to_download_list($doc_pkg);
    }

    if ($do_local_install) {
        do_cmd("cp -ar $source_dir/*GUI* $docroot/");
        if ($lfrls >= 5405) {
            do_cmd("cp -ar $source_dir/interop-* $docroot/ ||:", 1);
            add_to_download_list("interop-${lfver}.tar.gz");
        }
        if ($lfrls >= 5406){
            do_cmd("cp -ar $source_dir/LANforge-Server-* $docroot/");
            add_to_download_list("LANforge-Server-${lfver}-Installer.exe");
            add_to_download_list("LANforge-Server-${lfver}-upgrade.zip");
        }
        do_cmd("cp -ar $source_dir/LANforgeServer-* $docroot/");
        add_to_download_list("LANforge-GUI-${lfver}-Installer.exe");
        add_to_download_list("LANforgeGUI_${lfver}-x64-Installer.exe");
        add_to_download_list("LANforgeGUI_${lfver}_Linux64.tar.bz2");
        add_to_download_list("LANforgeGUI_${lfver}_Linux.tar.bz2");
        add_to_download_list("LANforgeGUI_${lfver}_MacOS.dmg.bz2");
    }
    elsif (-d "$d2dir") {
        do_cmd("cp -ar $d2dir/*GUI* $docroot/");
        do_cmd("cp -ar $d2dir/LANforgeServer-* $docroot/");
        if ($lfrls >= 5405) {
            do_cmd("cp -ar $d2dir/interop-* $docroot/ ||:", 1);
            add_to_download_list("interop-${lfver}.tar.gz");
        }
        if ($lfrls >= 5406) {
            do_cmd("cp -ar $d2dir/LANforge-Server-* $docroot/");
            add_to_download_list("LANforge-GUI-${lfver}-Installer.exe");
            add_to_download_list("LANforge-GUI-${lfver}-upgrade.zip");
        }
        add_to_download_list("LANforgeGUI_${lfver}-x64-Installer.exe");
        add_to_download_list("LANforgeGUI_${lfver}_Linux64.tar.bz2");
        add_to_download_list("LANforgeGUI_${lfver}_Linux.tar.bz2");
        add_to_download_list("LANforgeGUI_${lfver}_MacOS.dmg.bz2");
    }
    else {
        # Cannot do wild-cards for URLs
        # Download to tmp dir so we can take advantage of any already-downloaded files,
        # then copy them to desired location.
        download("$url_base/LANforge-GUI-${lfver}-Installer.exe", "$cachebuster", "$::tmp_dir/LANforge-GUI-${lfver}-Installer.exe");
        add_to_download_list("LANforge-GUI-${lfver}-Installer.exe");
        do_cmd("cp \"$::tmp_dir/LANforge-GUI-${lfver}-Installer.exe\" \"$docroot/LANforge-GUI-${lfver}-Installer.exe\"");

        download("$url_base/LANforgeGUI_${lfver}-x64-Installer.exe", "$cachebuster", "$::tmp_dir/LANforgeGUI_${lfver}-x64-Installer.exe");
        add_to_download_list("LANforgeGUI_${lfver}-x64-Installer.exe");
        do_cmd("cp \"$::tmp_dir/LANforgeGUI_${lfver}-x64-Installer.exe\" \"$docroot/LANforgeGUI_${lfver}-x64-Installer.exe\"");

        download("$url_base/LANforgeGUI_${lfver}_Linux64.tar.bz2", "$cachebuster", "$::tmp_dir/LANforgeGUI_${lfver}_Linux64.tar.bz2");
        add_to_download_list("LANforgeGUI_${lfver}_Linux64.tar.bz2");
        do_cmd("cp \"$::tmp_dir/LANforgeGUI_${lfver}_Linux64.tar.bz2\" \"$docroot/LANforgeGUI_${lfver}_Linux64.tar.bz2\"");

        download("$url_base/LANforgeGUI_${lfver}_Linux.tar.bz2", "$cachebuster", "$::tmp_dir/LANforgeGUI_${lfver}_Linux.tar.bz2");
        add_to_download_list("LANforgeGUI_${lfver}_Linux.tar.bz2");
        do_cmd("cp \"$::tmp_dir/LANforgeGUI_${lfver}_Linux.tar.bz2\" \"$docroot/LANforgeGUI_${lfver}_Linux.tar.bz2\"");

        download("$url_base/LANforgeGUI_${lfver}_MacOS.dmg.bz2", "$cachebuster", "$::tmp_dir/LANforgeGUI_${lfver}_MacOS.dmg.bz2");
        add_to_download_list("LANforgeGUI_${lfver}_MacOS.dmg.bz2");
        do_cmd("cp \"$::tmp_dir/LANforgeGUI_${lfver}_MacOS.dmg.bz2\" \"$docroot/LANforgeGUI_${lfver}_MacOS.dmg.bz2\"");

        if ($lfrls >= 5405) {
            download("$url_base/interop-${lfver}.tar.gz", "$cachebuster", "$::tmp_dir/interop-${lfver}.tar.gz");
            add_to_download_list("interop-${lfver}.tar.gz");
            do_cmd("cp \"$::tmp_dir/interop-${lfver}.tar.gz\" \"$docroot/interop-${lfver}.tar.gz\"");
        }
        if ($lfrls >= 5406) {
            download("$url_base/LANforge-Server-${lfver}-Installer.exe", "$cachebuster", "$::tmp_dir/LANforge-Server-${lfver}-Installer.exe");
            add_to_download_list("LANforge-Server-${lfver}-Installer.exe");
            do_cmd("cp \"$::tmp_dir/LANforge-Server-${lfver}-Installer.exe\" \"$docroot/LANforge-Server-${lfver}-Installer.exe\"");

            download("$url_base/LANforge-Server-${lfver}-upgrade.zip", "$cachebuster", "$::tmp_dir/LANforge-Server-${lfver}-upgrade.zip");
            add_to_download_list("LANforge-Server-${lfver}-upgrade.zip");
            do_cmd("cp \"$::tmp_dir/LANforge-Server-${lfver}-upgrade.zip\" \"$docroot/LANforge-Server-${lfver}-upgrade.zip\"");
        }
    }

    do_cmd("cd $docroot && tar -xzf $::tmp_dir/$doc_pkg");
    do_cmd("cd $docroot && cp -a $userdocs/manager-style.css .", 1);
    my $autodircss = q(/* created by lf_kinstall
  If you want to disable this branding, please alter the file conf.d/autoindex-lf.conf
  If autoindex-lf.conf is present but empty, this css file can get upgraded but ignored.
*/
body {
   font-family: "Century Gothic",Arial,sans-serif,sans;
   background: #cdc;
}
h1 {
   text-align: center;
   color: green;
}
table#indexlist {
   background: #efe;
   width: 90%;
   margin: 0 auto;
}
);
    if (!-f "$docroot/autodir.css") {
        open(my $adfh, ">", "$docroot/autodir.css") || die "Unable to write to autodir.css: $!";
        print $adfh $autodircss;
        close $adfh;
    }
    if (!-f "$docroot/candela_swirl.png") {
        do_cmd("cp $userdocs/images/candela_swirl.png $docroot/");
        do_cmd("cp $userdocs/images/candela_swirl_micro.png $docroot/");
    }
    # test for index.php fix or give it a patch
    if ($::lfver =~ /^5.3.2/) {
        if (-f "$docroot/index.php") {
            my $ph;
            open($ph, "<", "$docroot/index.php");
            my @index_php = <$ph>;
            close $ph;
            my @found_lines = grep {qq|= file_get_contents(.installed_ver_note);|} @index_php;
            if (@found_lines > 0) {
                if (-w "$docroot/index.php") {
                    my $patch = "H4sIAKOR6FUAA41R3WqDMBi99ymC9KKFJrKujZOtwmAwyoq7KrsMqUYXyBQ0rbKxd98XbdqVKigE
SXJ+vnNCue+EaJIU8VGUCK2R6z46K5+vgjkNzI3MK82VEgkDAMsLLQzIO/LSq+va+9RfyjtjMGCI
bjRohEimaJpKJZhoZKWraY/UbIZ+AAnfvwlaTiY0i4tci3yAaRx+YQlViR6RCfwtxAxi9mgNk++i
t+j9I3Ivzi0DtaSOb3kOXcW+7zy154ksLWTPwZJACdvnKC3KTLwUcYVd0nlijE+NjqFYR+pD37Ef
zB/ujGF2kEzdL4bYr7sN6/yIy7YyPzRE85LsvxfQvKXT5Xg6Xd4K1MP+GBSwVcCb0+uURDTCPTcw
LkNXQE+McGyMa4WrJOHoJFbkJswfrwvriCEDAAA=";

                    open($ph, "|-", "gunzip -c >/tmp/index-php.patch");
                    if ($ph) {
                        my $zip = MIME::Base64::decode($patch);
                        print $ph $zip or warn("unable to decompress patch, $!\n");
                        close $ph;
                    }
                    do_cmd("cp $docroot/index.php $docroot/.index.php.txt", 1);
                    do_cmd("patch -i /tmp/index-php.patch $http_root/index.php", 1);
                    note("Patched index.php\n");
                }
                else {
                    warning("Unable to write to $docroot/index.php\n");
                }
            }
            undef @found_lines;
        }
    }

    do_cmd("chown -R root:root $docroot/*");
    do_cmd("chmod -R a+r $docroot/*");
    if (-d $docroot) {
        do_cmd("find $docroot -type d | xargs chmod a+x ");
    }

    if (-f "/etc/httpd/conf.d/welcome.conf") {
        unlink("/etc/httpd/conf.d/welcome.conf");
    }

    try_generate_lfnotes();

    do_restart_httpd();
}

if ($force_notes) {
    fix_chrony_conf();
    try_generate_lfnotes();
}

if ($do_gnome) {
    if (($::osveri > 14) && (!$is_deb_based)) {
        note("\n# NOTE: After logging in to desktop, run: $ctgnome to tweak desktop settings.\n\n");
    }
}

if (!($is_offline || $skip_yum_all || $::create_install_bundle)) {
    if (!(defined $::use_install_bundle && "" ne $::use_install_bundle) && $do_local_install && ($yum eq "dnf")) {
        if (!$use_yum_cache) {
            note("# updating dnf cache last time, why?....\n");
            if ($::osveri >= 26) {
                do_cmd("dnf makecache -v", 1);
            }
            else {
                do_cmd("dnf makecache fast -v", 1);
            }
        }
    }

    # Remove proxy from yum.conf if we added it earlier.
    if ($rem_yum_proxy) {
        clean_yum_proxy();
    }

    # Make 'yum-plugin-fastestmirror' update itself next time it's used.
    # This helps if machine ships elsewhere after this script has run.
    if (-d "/var/cache/yum") {
        print "Destroying yum timedhosts artifacts...\n";
        do_cmd("find /var/cache/yum \\( -name timedhosts -o -name timedhosts.txt \\) -exec rm {} \\;");
    }
    # If we have some commands to run on next start of LANforge, create that now.
    if ($nextboot ne "") {
        do_cmd("mkdir -p $home/nextboot");
        my $fn = "$home/nextboot/kinstall-nextboot.bash";
        open(TFILE, ">$fn") or die "Couldn't open file: $fn  $!\n\n";
        print TFILE "#!/bin/bash\n\n";
        print TFILE $nextboot;
        print TFILE "\n\n";
        close TFILE;
        do_cmd("chmod a+x $fn");
    }
}
show_summary();
exit(0);

## ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ======
##                              Library Methods Below
## ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ====== ======
sub checkTtyConf {
    # For Ubuntu 12/13, we need some additional things to enable serial console
    if ($::osver eq "Ubuntu 12" or $::osver eq "Ubuntu 14") {
        open(F2, ">/etc/init/$ttys.conf") or die "Couldn't open file: /etc/init/$ttys.conf:  $!\n\n";
        print F2 "
# $ttys - getty
#
# This service maintains a getty on $ttys from the point the system is
# started until it is shut down again.

# Auto-created by LANforge, and LANforge 'lf_kinstall.pl' script will
# over-write it.

   start on stopped rc or RUNLEVEL=[2345]
   stop on runlevel [!2345]

   respawn
   exec /sbin/getty -L $ser_speed $ttys vt102
   ";
        close F2;
        do_cmd("start $ttys", 1);
    }
    if ($::osver eq "Ubuntu 16") {
        # Doing this on RPI blows up its kernel.  Other ARM boards may be similarly twitchy,
        # so will add specific checks for them if we find systems that work.
        if (!$is_arm) {
            do_cmd('[ -L /etc/systemd/system/getty.target.wants/getty@ttyS0.service ] || ln -s /lib/systemd/system/getty@.service /etc/systemd/system/getty.target.wants/getty@ttyS0.service', 1);
            do_cmd('systemctl enable getty@ttyS0', 1);
            do_cmd('systemctl start getty@ttyS0', 1);
            do_cmd('systemctl daemon-reload', 1);
        }
    }

    if ($is_arm) {
        # TODO:  Maybe not for all platforms?
        # Move some un-needed files out of the way..keeps init from spamming logs.
        my $unneeded = "/root/unneeded";
        if (!-d $unneeded) {
            do_cmd("mkdir -p $unneeded", 1);
        }
        if (-f "/etc/init/ttyO0.conf") {
            do_cmd("mv /etc/init/ttyO0.conf $unneeded/", 1);
        }
        if (-f "/etc/init/ttyS0.conf") {
            do_cmd("mv /etc/init/ttyS0.conf $unneeded/", 1);
        }
    }
}

sub incrementRadiusSerialNumber {
    my $rbase = shift;
    if (!defined $rbase || "$rbase" eq "") {
        warning("# Cannot increment radius serial number, <rbase> not defined [$rbase]\n");
        return;
    }
    die("increamentRadiusSerialNumber: directory [$rbase] not found") if (!-d $rbase);
    die("increamentRadiusSerialNumber: directory [$rbase/certs] not found") if (!-d "$rbase/certs");
    do_cmd("chown root:radiusd $rbase/certs/*");
    # Increment serial number.

    my $sn = 0;
    if (-f "$rbase/certs/serial") {
        $sn = int(`cat $rbase/certs/serial`);
        chomp($sn);
    }
    else {
        warning("# file[$rbase/certs/serial] not found, serial number will be 1\n");
    }
    $sn++;
    # Seems a single digit won't work...maybe 1k plus is good idea.
    # openssl cert serial numbers are actually hexadecimal values
    if ($sn < 9000) {
        $sn = 9000 + $sn % 100;
    }
    note("# Increasing radius serial number to: $sn\n");
    `echo $sn > $rbase/certs/serial`;
    do_cmd("chown root:radiusd $rbase/certs/*");
}

sub do_restart_httpd {
    if ($is_deb_based) {
        # Nothing to do
        if (-d "/etc/apache2") {
            do_cmd("systemctl enable apache2.service", 1);
            do_cmd("hostname -F /etc/hostname", 1);
            do_cmd("systemctl restart --quiet apache2.service", 1);
        }
        else {
            print("Apache2 not installed\n");
        }
    }
    elsif (($is_fedora || $is_redhat || $is_centos) && ($::osveri > 14)) {
        if (system("which httpd &>/dev/null") != 0) {
            do_cmd("yum install $skip_broken -y httpd");
        }
        # I saw this fail on F24 because it was already enabled, it appears.  Stupid software.
        # So, make it OK to fail the cmds here...we can check after install anyway easily enough. --Ben
        do_cmd("systemctl enable  httpd.service", 1);
        do_cmd("hostname -F /etc/hostname", 1);
        do_cmd("systemctl restart --quiet httpd.service", 1);
    }
    else {
        do_cmd("chkconfig --level 2345 httpd on");
        do_cmd("/etc/init.d/httpd start");
    }
}

sub clean_yum_proxy {
    note("Inspecting for yum proxies, $proxy...\n");
    if (-f "/etc/dnf/dnf.conf") {
        my @cmds = `cat /etc/dnf/dnf.conf`;
        my @rslts = grep {/$::proxy_re/} @cmds;
        if (@rslts > 0) {
            note("Removing dnf proxies, $proxy...\n");
            open FILE, ">/etc/dnf/dnf.conf" || die "Couldn't open /etc/dnf/dnf.conf: $!\n";
            for ($i = 0; $i < @cmds; $i++) {
                my $ln = $cmds[$i];
                chomp($ln);
                if ($ln =~ /$::proxy_re/) {
                    print FILE "# ";
                }
                print FILE "$ln\n";
            }
        }
        close FILE;
    }
    if (-f "/etc/yum.conf") {
        my @cmds = `cat /etc/yum.conf`;
        my @rslts = grep {/$::proxy_re/} @cmds;
        if (@rslts > 0) {
            note("Removing yum proxies...\n");
            open FILE, ">/etc/yum.conf" || die "Couldn't open /etc/yum.conf: $!\n";
            for ($i = 0; $i < @cmds; $i++) {
                my $ln = $cmds[$i];
                chomp($ln);
                if ($ln =~ /$::proxy_re/) {
                    print FILE "# ";
                }
                print FILE "$ln\n";
            }
        }
        close FILE;
    }
}

sub show_summary {
    print("\n= = = = = = = = = = = = SUMMARY = = = = = = = = = = = = = =\n");
    print("- - - - - - - - - - - - All Notes - - - - - - - - - - - -\n"
        . $::notes . "\n\n");
    print("- - - - - - - - - - - - All Warnings - - - - - - - - - - - -\n"
        . $::warnings . "\n\n") if ("" ne $::warnings);
    print("- - - - - - - - - - - - All Errors - - - - - - - - - - - -\n"
        . $::errors . "\n\n") if ("" ne $::errors);
    print("# [$::osver] Command: $0 $commandline0\n");
    print("= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =\n");
    print("# Done.\n");
}

# Return 0 on success (basically, whatever system() returns
sub do_cmd {
    my $err = 1;
    my $cmd = shift;
    my $ok_to_fail = shift;
    my $rs_fname1 = (exists $_[0]) ? shift : undef;
    my $rs_fname2 = (exists $_[0]) ? shift : undef;
    my $using_redirection = 0;
    my $tmptmpdir = $::tmp_dir;
    $tmptmpdir = "/tmp" if ($::tmp_dir !~ /^(\/var)?\/tmp/);
    my ($fh1, $fname1) = tempfile($::tmp_template, DIR => $tmptmpdir);
    my ($fh2, $fname2) = tempfile($::tmp_template, DIR => $tmptmpdir);

    if ($cmd =~ m/["]/) {
        $cmd =~ s/["]/\\"/g;
    }
    if ($cmd =~ m/[>]{1,2}/) {
        $using_redirection = 1;
    }
    if (!$::is_openwrt && ($using_redirection == 0)) {
        $cmd = $cmd . " > >(tee $fname1) 2> >(tee $fname2)";
    }
    logit("do_cmd:[$cmd]\n", 1);
    $err = system("/bin/bash -c ${Q}${cmd}${Q}");
    close($fh1);
    close($fh2);

    if ($err == 0) {
        unlink($fname1);
        unlink($fname2);
        return $err;
    }

    my $stdout = `cat $fname1`;
    my $stderr = `cat $fname2`;

    $rs_fname1 = \$fname1;
    $rs_fname2 = \$fname2;

    if ($ok_to_fail || $::ignore_err) {
        warning("# WARNING:  Command failed, not fatal: $cmd\n");
        warning("# STDOUT: $stdout\n");
        warning("# STDERR: $stderr\n");
        return $err
    }

    err(qq(\n# FATAL:  Command failed, lf_kinstall.pl script is aborting!!
# K_INSTALL: $0 $lfkinargs
# FAILED CMD: $cmd [$!]
# STDOUT:[$stdout]
# STDERR:[$stderr]
"));
    die("$!");
} # ~do_cmd

sub check_show_url {
    my $url = shift;
    my $ra_failed_urls = shift;

    if ($::show_urls && !$::print_w32) {
        note("# $url\n");
    }
    if ($::print_only) {
        if ($::print_w32) {
            my ($lfilename) = $url =~ /([^\/]+)$/;
            note("echo $lfilename ....\n");
            #note("%USERPROFILE%${B}Downloads${B}$w32get -L -O -C - -u guest:guest $url\n");
            note("%GET% %OPTS% $url\n");
        }
        return;
    }
    my $rh_req = newHeadRequest($url . $cachebuster);
    webget($rh_req);
    if ($rh_req->{"status"} !~ / 200 /) {
        push(@$ra_failed_urls, $url);
    }
} # ~check_show_url

sub download {
    my $base_url = shift;
    my $cachebuster = shift;
    my $output = shift;
    my $resume_ok = 0;
    my $new_etag = "";
    my $skip_download = 0;
    my $etag_line = "";
    my $cl_line = "";

    if ($is_offline || $print_only || $show_urls) {
        print "     not downloading $base_url : is_offline[$is_offline] print_only[$print_only] show_urls[$show_urls]\n" if ($main::debug_on);
        return;
    }
    if (!(defined $base_url) || ($base_url eq "")) {
        die("::download called with blank \$1 value, please debug.");
    }
    if (!(defined $cachebuster) || ($cachebuster eq "")) {
        die("::download called with blank \$2 value, please debug.");
    }
    if (!(defined $output) || ($output eq "")) {
        die("::download called with blank \$3 value, please debug.");
    }
    if ($base_url =~ m|.*/$|) {
        die("::download: base_url[$base_url] appears to be a directory, you probably have an unresolved package") if ($::debug_on);
    }

    if ($::create_install_bundle) {
        note("Not downloading [$base_url], doing create_install_bundle\n");
        return;
    }
    #if ($::debug_on) {
    #    if (-e "$::tmp_dir/md5.txt") {
    #        print("$::tmp_dir/md5.txt already exists\n");
    #    }
    #    if (-z "$::tmp_dir/md5.txt") {
    #        print("$::tmp_dir/md5.txt and is empty\n");
    #    }
    #    else {
    #        print("$::tmp_dir/md5.txt and not empty\n");
    #    }
    #}
    if (!-e "$::tmp_dir/md5.txt") {
        $downloaded_md5 = 0;
        note("have to download md5.txt\n");
    }
    if (-z "$::tmp_dir/md5.txt") {
        $downloaded_md5 = 0;
        warning("$::tmp_dir/md5.txt appears empty, will download again");
    }
    if ($::downloaded_md5 < 1) {
        # First, get the md5
        my $rh_req = newWebRequest("$url_base/md5.txt$cachebuster", "$::tmp_dir/md5.txt", "GET");
        $rh_req->{"resume"} = undef;
        $rh_req->{"notify_on_fail"} = 1;
        $rh_req->{"die_on_fail"} = 1;
        $rh_req->{"notify_on_start"} = 'yes';
        if (webget($rh_req) == 0) {
            $::downloaded_md5++;
        }
        else {
            warning("webget md5.txt DOWNLOAD ERROR\n");
            print Dumper($rh_req) if ($::debug_on);
            if (-e "$::tmp_dir/md5.txt") {
                print("$::tmp_dir/md5.txt exists");
                system("ls -l $::tmp_dir/md5.txt");
            }
            else {
                warn("$::tmp_dir/md5.txt missing");
            }
        }
    }

    if (!-e "$::tmp_dir/md5.txt") {
        system("ls -l $::tmp_dir/");
        die("file not found:  $::tmp_dir/md5.txt");
    }
    if (-z "$::tmp_dir/md5.txt") {
        system("ls -l $::tmp_dir/md5.txt");
        die("$::tmp_dir/md5.txt appears empty");
    }
    if (($::downloaded_md5 > 0) && (keys(%::md5_file_map) < 1)) {
        print("# sorting MD5 entries...") if ($debug_on);
        my @md5s = `cat "$::tmp_dir/md5.txt"`;
        chomp(@md5s);
        for my $md5line (@md5s) {
            next if ($md5line=~/^\s*$/);
            my ($md5, $fname) = $md5line =~ /^(\S+)\s+(\S+)$/;
            next if ($fname=~/^\s*$/);
            next if ($md5=~/^\s*$/);
            $::md5_file_map{$md5} = $fname;
            $::file_md5_map{$fname} = $md5;
        }
    }
print "download($base_url)\n" if ($debug_on);
sleep(1);
    my $rh_request = newHeadRequest($base_url);
    webget($rh_request);
    my $rslts = join("\n", $rh_request->{"headers"});
    ($etag_line, $cl_line) = find_etag_info($rslts);
    if ($etag_line =~ /ETag:\s+\"(\S+)\"/) {
        $new_etag = $1;
    }
    #print("new-etag: $new_etag  results: $rslts\n skip-resume: $skip_resume  output-etag: $output.etag\n");

    if ($new_etag ne "" && !$skip_resume) {
        if (-f "$output.etag") {
            # Get info on this file on the server.
            my $old_rslts = `cat $output.etag`;
            my $old_etag_line = "";
            my $old_cl_line = "";
            ($old_etag_line, $old_cl_line) = find_etag_info($old_rslts);
            if ($old_etag_line =~ /^ETag:\s+\"(\S+)\"/) {
                my $old_etag = $1;
                note("# Found old-etag: $old_etag\n") if ($::debug_on);

                if ($old_etag ne $new_etag) {
                    warning("# ETags do not match for: $output, disabling http-resume, old: $old_etag  new: $new_etag\nFull Header:\n$rslts\n");
                }
                else {
                    # Check size, maybe it's all done already?
                    note("# Etags match, checking length.\n");
                    if ($cl_line =~ /^Content-Length:\s+(\d+)/) {
                        my $ln = $1;
                        my $local_ln = -s "$output";
                        if ($ln == $local_ln) {
                            note("# Skipping download for URL: $base_url, it appears to be already downloaded.\n") if ($::debug_on);
                            $skip_download = 1;
                        }
                        else {
                            if ($local_ln > $ln) {
                                err("# Local file: $output appears corrupted, will re-download it.\n");
                                $resume_ok = 0;
                            }
                            else {
                                note("# Lengths do not match, ln: $ln  local-length: $local_ln, enabling\n");
                                note("#   HTTP Resume for URL: $base_url\n");
                                $resume_ok = 1;
                            }
                        }
                    }
                    else {
                        err("Could not find content-length in rslts:\n$rslts\n");
                        # output debug info
                        #if (! $downloaded_md5) {
                        #  print("`$curl -sqv -u $user $url_base/md5.txt$cachebuster -o $tmp_dir/md5.txt`\n");
                        #}
                        #else {
                        #  print("`$curl -sqv --head -u $user $base_url`);
                        #}
                        exit(1);
                    }
                }
            }
        }
        else {
            #print("Did not find old etag file: $output.etag\n");
        }
    }

    #note("# save-new-etag: $new_etag\n");
    if ($new_etag ne "") {
        note("# Creating new etag file: $output.etag\n");
        open(FILE, ">$output.etag") or die("Couldn't open file: $output.etag for writing: $!\n\n");
        print FILE $rslts;
        close FILE;
    }
    my $new_md5 = "0";
    my $md5 = "0";
    my $fname = "-";
    my $ofname = "unset";
    my @hunks = split(/\//, $output);
    $ofname = pop(@hunks);
    if (exists $::file_md5_map{$ofname}) {
        note("# MD5 $ofname in file_md5_map\n") if ($::debug_on);
        $md5 = $::file_md5_map{$ofname};
    }
    else {
        err("# MD5 $ofname NOT in file_md5_map\n");
    }

    # If output file already exists, then check md5sum
    if (-f "$output") {
        my $new_md5 = `$md5sum $output`;
        $new_md5 =~ s/([A-Fa-f0-9])\s+.*$/$1/;
        chomp $new_md5;
        # print ("Previously downloaded $ofname, doing [md5sum $output]\n");
        if (exists $::md5_file_map{$new_md5}) {
            # note("# MD5 [$new_md5] is in md5_file_map\n") if ($::debug_on);
            $fname = $::md5_file_map{$new_md5};
            warning("bad md5_file_map entry") if (!(defined $fname) || ($fname =~ /^\s*$/) );
        }
        else {
            note("# Careful, $ofname MD5 $new_md5 not in md5_file_map\n") if ($debug_on);
        }

        # print ("Expected:  [$md5] [$ofname]\nDownloaded [$new_md5] [$ofname]\n");
        if ($new_md5 eq $md5) {
            note("# Success: MD5 matches for already-downloaded file: $output\n") if ($::debug_on);
            return;
        }
        if (!$resume_ok) {
            if ($debug_on) {
                note("# Exists:   [$new_md5] for $output exists.\n");
                note("# Expected: [$md5] does not match\n");
                note("# Will remove $output and re-try the download.");
            }
            unlink("$output");
            $skip_download = 0;
        }
        else {
            note("# MD5 does not match for already-downloaded file, will try to resume download: $output\n");
            $skip_download = 0;
        }
    }

    if (!$skip_download) {
        my $rh_req = newWebRequest($base_url . $cachebuster, $output);
        $rh_req->{"die_on_fail"} = 1;
        $rh_req->{"options"} = "-#"; # show progress bar == '-#'
        if ($resume_ok) {
            print("Will try to download with resume...\n") if ($::debug_on);
            $rh_req->{"resume"} = $curl_resume;
        }
        else {
            print("Will try to download without resume...\n") if ($::debug_on);
        }
        webget($rh_req);
    }

    # When here, successfully downloaded.  Check md5sum against server.
    #$ofname = pop((split(/\//, $output)));
    #print ("Downloaded: $ofname, doing [md5sum $output]\n");
    $new_md5 = `$md5sum $output`;
    if ($is_macos) {
        # funky md5 output
        $new_md5 =~ s/.*=\s+(\S+)/$1/;
    }
    else {
        $new_md5 =~ s/([A-Fa-f0-9])\s+.*$/$1/;
    }
    chomp $new_md5;
    $fname = "-";
    if (exists $::md5_file_map{$new_md5}) {
        note("# MD5 $new_md5 in md5_file_map\n") if ($::debug_on);
        $fname = $::md5_file_map{$new_md5};
    }
    else {
        note("# Careful, $ofname MD5 $new_md5 not in md5_file_map\n");
    }

    print("Expected:  $md5 $ofname\nDownloaded $new_md5 $ofname\n") if ($::debug_on);
    if ($new_md5 ne $md5) {
        err("# ERROR:  New md5sum: $md5 does not match downloaded: $new_md5.\n"
            . "#  Please delete: $output and try again, or contact support\@candelatech.com\n");
        exit(1);
    }
    note("# Success: MD5SUM matches for downloaded file: $output\n") if ($::debug_on);
} # ~download()

sub find_etag_info {
    my $rslts = shift;
    my $etag_line = "";
    my $cl_line = "";

    # Get the ETag
    my @rslts_a = split(/\n/, $rslts);
    my $i;
    for ($i = 0; $i < @rslts_a; $i++) {
        my $ln = $rslts_a[$i];
        chomp($ln);
        if ($ln =~ /^ETag:.*/) {
            $etag_line = $ln;
        }
        elsif ($ln =~ /^Content-Length:.*/) {
            $cl_line = $ln;
        }
    }

    return ($etag_line, $cl_line);
}

sub lcheck {
    if (!-f "$home/license.txt") {
        note("# no license.txt found\n");
        return;
    }
    my $now = `date +\%s`;
    chomp $now;
    $now = 0 + $now;
    my $expired = 0;
    my $expiring = 0;
    my @lic_lines = `cat $home/license.txt`;
    my $thisweek = 7 * 24 * 60 * 60;
    my $two_y = $thisweek * 200;
    my $indays = 0;
    my $licline = 0;
    chomp @lic_lines;
    for my $lic_line (@lic_lines) {
        $licline++;
        next if ($lic_line =~ /^\w*$/);
        next if ($lic_line =~ /^\w*#/);
        my @hunks = split(' ', $lic_line);
        if (@hunks < 5) {
            warning("# parse error $licline: $lic_line\n");
            $expired += 0x0FFE;
            next;
        }
        if ($hunks[4] =~ /forever/i) {
            note("# discovered perpetual license\n");
            next;
        }
        if ($hunks[4] !~ /^\d+$/i) {
            warning("# formatting error license line $licline:${hunks [ 4 ]}\n");
            $expired += 0x0FFE;
            next;
        }

        my $delta = ((0 + $hunks[4]) - $now);
        $indays = floor($delta / 86400);
        if ($delta > 0 && $delta <= $thisweek) {
            note("# The license for $hunks[0] expires in $indays days\n");
            sleep 1;
            $expiring++;
        }
        elsif ($delta < 1) {
            $indays = abs($indays);
            note("# The license for $hunks[0] has expired ($indays days ago)\n");
            sleep 1 unless ($::acknowledged == 1);
            $expired++;
        }
    }
    return if (($expired + $expiring) == 0);

    if ($expiring > 0) {
        print qq(
-  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -

Your LANforge system licenses are about to expire. If you are
upgrading, carefully consider if you need support for new features.

-  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
);
        sleep 6;
    }
    if ($expired >= 0x0FFE) {
        print qq(
-  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -

   Your LANforge system license.txt file appears to be damaged.
   Please contact support if you have problems with this upgrade.

-  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
);
        sleep 6;
    }
    if ($expired > 0 && $expired < 0x0FFE && $::acknowledged != 1) {
        my $option = "   Use the --acknowledge option to continue.";
        my $bad_idea = "   LANforge installs this old could break during an upgrade.";
        if ($indays > 800) {
            $option = "$bad_idea\n$option";
        }
        print qq(
-  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -

   Your LANforge system licenses are expired. We encourage you to
   renew support in case you have out of date software that will
   conflict with this upgrade.

$option
-  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -
);
        sleep 6;
        exit 1;
    }
}

sub file_readable {
    my $file = shift;
    my $option = shift;

    my $err = 1;
    if ($option eq "") {
        $option = "\nWARNING: Option not passed to file_readable()";
    }
    logit("# Determining if: $file is readable\n");
    if (!(-r $file) || !(-R $file)) {
        $err = 0;
        err("\n\nERROR:  Option: \"$option\", file unreadable: $file\n\n");
    }
    return $err
}

sub file_writable {
    my $err = 1;
    my $file = shift;
    my $option = shift;
    if ($option eq "") {
        $option = "\nWARNING: Option not passed to file_writable()";
    }
    logit("# Determining if $option $file is writable\n");
    if (!(-w $file) || !(-W $file)) {
        $err = 0;
        err("\n\nERROR:  Option: $option, file unwritable: " . "$file" . "\n\n");
    }
    return $err
}

sub err {
    my $msg = shift;
    warn("K-Err> $msg\n"); # print to std-err
    logit($msg, 1);
    $errors = $errors . $msg;
}

sub warning {
    my $msg = shift;
    # blank messages are not helpful
    return if ($msg !~ /\S+/);
    warn("K-warning> $msg\n"); # print to std-err
    logit($msg, 1);
    $warnings = $warnings . $msg;
}

sub note {
    my $msg = shift;
    if ($msg !~ /\n$/) {
        $msg = "$msg\n";
    }
    logit($msg);
    $::notes .= $msg;
}

sub logit {
    my $msg = shift;
    my $skip_print = shift;
    $skip_print = 0 if (!defined $skip_print);
    if ($skip_print == 0) {
        print $msg . (($msg !~ /\n$/) ? "\n" : '');
    }
    if ($::broken_logfile == 0) {
        open(my $LOGFILE, ">>", $::install_log) or
            warn("problem opening logfile[$::install_log]");
        if (!(defined $LOGFILE)) {
            $::broken_logfile++;
            warn("ERROR:  Unable to write to log file: $::install_log");
        }
        else {
            print $LOGFILE ts() . " $msg";
            close($LOGFILE);
        }
    }
    if ($::broken_logfile > 0) {
        warn $msg unless ($skip_print);
    }
}

sub ts {
    # grab the current time
    my @now = localtime();
    # rearrange the following to suit your stamping needs.
    # it currently generates YYYYMMDD_hhmmss
    my $timeStamp = sprintf("%04d%02d%02d_%02d%02d%02d",
        $now[5] + 1900, $now[4] + 1, $now[3],
        $now[2], $now[1], $now[0]);
    return $timeStamp
}

sub try_generate_lfnotes {
    my $fname;
    my $date = POSIX::strftime("%B %d, %Y", localtime);

    # Detect system
    my $cpu = "UNKNOWN";
    my @radios_pci = ();
    my @radios = ();
    my @usb_radios = ();
    my @netdevs = ();
    my @is_usb = ();
    my $found_usb = 0;

    if ($::is_macos) {
        $cpu = `sysctl machdep.cpu.brand_string`;
        chomp($cpu);
        $hwver = "MacOS";
        @netdevs = `networksetup -listallhardwareports | grep Port:`;
        chomp(@netdevs);
        @radios = grep {/Wi-Fi/} @netdevs ;
    }
    else {
        @radios = split(/\s+/, `([ -d /sys/class/ieee80211 ] && ls /sys/class/ieee80211/)||:`);
        my @usb_radios = ();
        my @netdevs = split(/\s+/, `ls /sys/class/net/`);
        $cpu = `cat /proc/cpuinfo`;
    }
    my $rad_count = @radios;

    # Don't count USB radios for HW matching.
    if (!$::is_macos) {
        my $q;
        for ($q = 0; $q < @radios; $q++) {
            my $rad = $radios[$q];
            my $subs = `ls -l /sys/class/ieee80211/$rad/device/|grep subsystem`;
            if ($subs =~ /bus\/usb/) {
                $rad_count--;
                $is_usb[$q] = 1;
                $found_usb = 1;
                note("Found USB radio: $rad\n");
                @usb_radios = (@usb_radios, $rad);
            }
            else {
                $is_usb[$q] = 0;
                if (-f "/sys/class/ieee80211/$rad/device/uevent") {
                    my $uevent = `cat /sys/class/ieee80211/$rad/device/uevent`;
                    if ($uevent =~ /PCI_SLOT_NAME=(\S+)/) {
                        note("# Found PCI radio: $1\n");
                        @radios_pci = (@radios_pci, $1);
                    }
                }
                if ($found_usb) {
                    err("\n\nERROR:  USB wiphy NICs do not come last in listing.\nPlease fix /etc/udev.d/rules.d/ persistent files for wlanX, etc.\n");
                }
            }
        }
    } # !mac
    note("# hwver:  Radios: $rad_count\n");

    if ($hwver ne "UNKNOWN") {
        note("# hwver set on command line: $hwver\n");
    }
    elsif ($cpu =~ /Raspberry Pi 4/) {
        $hwver = "LF0314";
    }
    elsif ($cpu =~ /Raspberry Pi 5/) {
        $hwver = "LF0314_5";
    }
    elsif ($cpu =~ /AMD GX-412TC SOC/) {
        $hwver = "LF0350";
    }
    elsif ($cpu =~ /M 620  \@ 2\.67GHz/) {
        #note("# Found CT521 cpu.\n");
        if ($rad_count == 2) {
            #note("# Found 2 non-USB radios.\n");
            $hwver = "CT521";
        }
        #note("# Found Platinum Ricer cpu.\n");
        if ($rad_count == 1) {
            #note("# Found 1 non-USB radios.\n");
            $hwver = "CT52x-PR";
        }
        else {
            note("# $hwver: Found: " . @radios . " radios: " . join(" ", @radios) . "\n");
        }
    }
    elsif (($cpu =~ /N270   \@ 1\.60GHz/) || ($cpu =~ /D525   \@ 1\.80GHz/)) {
        #note("# Found CT520 cpu.\n");
        if ($rad_count == 1) {
            #note("# Found 1 radios.\n");
            $hwver = "CT520";
        }
        else {
            note("# $hwver: Found: " . @radios . " radios: " . join(" ", @radios) . "\n");
        }
    }
    elsif (($cpu =~ /Atom\(TM\) CPU D510   \@ 1\.66GHz/) || (($cpu =~ /CPU\s+C2518/) && (@netdevs == 7))) {
        #note("# Found LF0202 cpu.\n");
        if (@netdevs == 7) {
            #note("# Found 7 network devices (6 + lo).\n");
            $hwver = "LF0202";
            # LF0202 mini-PCIe does not support PCIe but only USB NICs
        }
        else {
            note("# $hwver: Found: " . @netdevs . " netdevs: " . join(" ", @netdevs) . "\n");
        }
    }
    elsif ($cpu =~ /Intel\(R\)\ Core\(TM\)\ i7\-\d{4}[QL]E/) {
        #note("# Found CT523 cpu.\n");
        # Check for two built-in ethernets..that is tight enough.
        my $ecnt = `lspci|grep 82574|wc -l`;
        chomp($ecnt);
        if ($ecnt >= 2) {
            $hwver = "CT523";
        }
        else {
            note("# $hwver: Found: " . @radios . " radios: " . join(" ", @radios) . "\n");
        }
    }
    elsif ($cpu =~ /Intel\(R\)\ Core\(TM\)\ i7\-7700T/) {
        note("# Found CT523c (axiomtek cube) cpu.\n");
        # Check for two built-in ethernets..that is tight enough.
        # In future, could check dmidecode --handle 2 for 'SKYBAY' model name.
        my $ecnt = `lspci|grep I211|wc -l`;
        chomp($ecnt);
        if ($ecnt == 2) {
            $hwver = "CT523c";
        }
        else {
            note("# $hwver: Found: " . @radios . " radios: " . join(" ", @radios) . "\n");
        }
    }
    elsif ($cpu =~ /Intel\(R\)\ Core\(TM\)\ i7\-12700TE/) {
        note("# Found NEW CT523c (axiomtek cube) cpu.\n");
        # Check for the two built-in ethernets..that is tight enough.
        $hwver ="CT523c";
        my $ecnt = 'lspci|grep -e I226 -e I219|wc -l';
        chomp($ecnt);
        if ($ecnt >= 2) {
           $hwver = "CT523c";
           $is_new_523c = 1;
        }
        else {
            note("# $hwver: Found: " . @radios . " radios: " . join(" ", @radios) . "\n");
        }
    }
    elsif ($cpu =~ /Intel\(R\) Core\(TM\) i5-[76]300U CPU \@ 2\.[46]0GHz/) {
        # Well, could be a jetway CT522 (LF0313) system
        my $ecnt_350 = `lspci|grep I350|wc -l`;
        chomp($ecnt_350);
        if ($ecnt_350 == 4) {
            $hwver = "CT522";
        }
    }
    elsif ($cpu =~ /Intel\(R\) Core\(TM\) i5-6500TE CPU \@ 2\.30GHz/) {
        # Well, could be an iBase system (LF0312) system
        my $ecnt = `lspci|grep I219|wc -l`;
        chomp($ecnt);
        if ($ecnt == 1) {
            $hwver = "LF0312";
        }
    }
    elsif ($cpu =~ /BCM2708/) {
        # Raspbian
        $hwver = "CT314";
    }
    elsif ($::is_openwrt) {
        $hwver = $openwrt_board;
    }
    elsif ($cpu =~ /Intel\(R\) Core\(TM\) Xeon\(R\) CPU E3-1245 v5/) {
        $hwver = "CT524";
    }
    elsif ($cpu =~ /Intel\(R\) Atom\(TM\) CPU\s+E3845/) {
        $hwver = "LF0351";
        # if this has radios it is a CT521b
        if (@radios_pci > 0) {
            $hwver = "CT521b";
        }
    }
    else {
        my @model_matches = grep {/model name/} (split(/\r?\n/, $cpu));
        $cpu = $model_matches[0];
        note("# UNMATCHED Cpu-info: $cpu\n");
    }

    my $tmp;

    note("# hwver: $hwver\n");

    my $bus_map = {};
    if (!$::is_openwrt) {
        # we can detect QEMU status by looking at /dev/disk/by-id or systemd-detect-virt
        my $virtprobe = `systemd-detect-virt`;
        if (defined $virtprobe) {
            chomp $virtprobe;
            $::do_vm_reconfig = 1
                if (("" ne $virtprobe) && ("none" ne $virtprobe));
        }
    }

    if ($::do_vm_reconfig) {
        # get a list of virtual interfaces
        my @iflist = `ls -1d /sys/class/net/e*`;
        chomp(@iflist);
        die("Unable to find /sys/class/net/e* devices")
            if (@iflist < 1);

        for (my $i = 0; $i < @iflist; $i++) {
            my ($e) = $iflist[$i] =~ m|^/.*/([^/]+)$|;
            $iflist[$i] = $e;
        }
        my %ifmap = map {$_ => 1} @iflist;
        #print Dumper(\%ifmap);

        my @ports_to_remove = ();
        my $old_dev = $iflist[0];
        for my $pp (keys %ifmap) {
            next if ($pp eq "eth0");
            push(@ports_to_remove, $pp);
        }
        my @radios = ();
        my $port_layout = "PORT_CONF eth0 eth1 ~";

        note("do_vm_reconfig...");
        # TODO:  Should we pass $do_mgt_dev instead of 'eth0' below?
        _do_sys_reconfig($old_dev, \@ports_to_remove, \@radios, $port_layout, $rad_count, "vm", "eth0");
    }

    if ($::is_openwrt) {
        # Make sure this cruft is disabled.
        do_cmd("/etc/init.d/opensync stop", 1);
        do_cmd("/etc/init.d/opensync disable", 1);
        do_cmd("/etc/init.d/openvswitch stop", 1);
        do_cmd("/etc/init.d/openvswitch disable", 1);

        do_cmd("/etc/init.d/firewall disable", 1);
        do_cmd("/etc/init.d/dhcpsnoop disable", 1);
        do_cmd("/etc/init.d/igmpproxy disable", 1);
        do_cmd("/etc/init.d/radsecproxy disable", 1);
        do_cmd("/etc/init.d/usteer disable", 1);
        do_cmd("/etc/init.d/wpad disable", 1);

        do_cmd("/etc/init.d/opennds disable", 1);

        if (!-e "/home/lanforge/local/sbin/iw") {
            do_cmd("ln -s /usr/sbin/iw /home/lanforge/local/sbin/iw");
        }
        if (!-e "/home/lanforge/local/sbin/iptables") {
            do_cmd("ln -s /usr/sbin/iptables /home/lanforge/local/sbin/iptables");
        }
        if (!-e "/home/lanforge/local/sbin/ip") {
            do_cmd("ln -s /sbin/ip /home/lanforge/local/sbin/ip");
        }
        if (!-e "/home/lanforge/local/sbin/dhcpd") {
            do_cmd("ln -s /usr/sbin/dhcpd /home/lanforge/local/sbin/dhcpd");
        }
        if (!-e "/home/lanforge/local/sbin/dhclient") {
            do_cmd("ln -s /usr/sbin/dhclient /home/lanforge/local/sbin/dhclient");
        }
    } #!openwrt

    if (($hwver eq "LF0202") || ($hwver eq "LF0350") || ($hwver eq "LF0351") || ($hwver eq "CT520") || ($hwver eq "CT521")
        || ($hwver eq "CT523") || ($hwver eq "CT523c") || ($hwver eq "CT52x-PR") || ($hwver eq "LF0313") || ($hwver eq "CT522")
        || ($hwver eq "LF0312") || ($hwver eq "R7800") || ($hwver eq "ECW5410") || ($hwver eq "ECW5211") || ($hwver eq "EC420")
        || ($hwver eq "CIG194C") || ($hwver eq "E8450")
        || ($hwver eq "NEO2") || ($hwver eq "LF0314")|| ($hwver eq "LF0314_5") || ($hwver eq "CT521b")) {

        # CT523 with only 2 radios is all different..seems it changes its busses
        # around or something.
        if ($hwver eq "CT523") {
            # For 1-radio system:
            # wlan0 is on pci-e riser, and bus 5.

            # For 2-radio system:
            # wlan0 is on left pci-e built-in slot, and is bus 5
            # wlan1 is on pci-e riser, and bus 6.

            # For 3-radio system, it now seems to be this.  It may not have
            # always been so, so we need to double-check:
            # wlan0 is on left pci-e built-in slot, and is bus 5
            # wlan1 is on right pci-e built-in slot and is bus 6
            # wlan2 is on pci-e riser, and bus 7.

            my $old_dev = "enp1s0";
            my @ports = ("enp2s0", $old_dev);
            my @radios = ("0000:03:00.0", "0000:04:00.0", "0000:05:00.0");
            my $port_layout = "PORT_CONF eth1 eth0 ~";

            if ($rad_count == 1) {
                $old_dev = "enp3s0";
                @radios = ("0000:05:00.0");
                @ports = ("enp4s0", $old_dev)
            }

            if ($::do_sys_reconfig) {
                note(" do_sys_reconfig CT523");
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "ct523", "eth0");
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "CT52x-PR") {
            if ($rad_count == 2) {
                # Might require hacking on 70-phyname and other rules to
                # make this line up as is here.
                # If we put in a NIC in the spare slot, everything shifts.
                my $tmp = `ethtool -i wlan1`;
                if ($tmp =~ / 0000:05:00.0/) {
                    getPhyInfo(0, "wlan0", "0000:04:00.0", "<b>PCIe plug-in card</b>");
                    getPhyInfo(1, "wlan1", "0000:05:00.0", "<b>Built-in (Top)</b>");
                }
                else {
                    getPhyInfo(0, "wlan0", "0000:03:00.0", "<b>PCIe plug-in card</b>");
                    getPhyInfo(1, "wlan1", "0000:04:00.0", "<b>Built-in (Top)</b>");
                }
            }
            else {
                getPhyInfo(0, "wlan0", "0000:04:00.0", "<b>PCIe plug-in card</b>");
            }
        }
        elsif ($hwver eq "LF0350") {
            # APU2
            my $old_dev = "enp2s0";
            my @ports = ($old_dev, "enp3s0", "enp4s0");
            if ($rad_count == 0) {
                # go figure
                $old_dev = "enp1s0";
                @ports = ($old_dev, "enp2s0", "enp3s0");
            }
            my @radios = ("0000:01:00.0", "0000:05:00.0");
            my $port_layout = "PORT_CONF eth0 eth1 eth2 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "lf0350", "eth0");

                # These systems use igb, and it will work better with a single receive queue.
                if (!-f "/etc/modprobe.d/igb.conf") {
                    do_cmd("echo options igb max_rss_qs=1 > /etc/modprobe.d/igb.conf");
                }
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif (($hwver eq "LF0351") || ($hwver eq "CT521b")) {
            # left most port is enp8s0, then enp7s0, enp3s0
            my $old_dev = "enp8s0";
            my @ports = ($old_dev, "enp7s0", "enp3s0");
            if ($rad_count == 0) {
                # go figure
                $old_dev = "enp8s0";
                @ports = ($old_dev, "enp7s0", "enp3s0");
            }
            # slot mpcie2 on left is address 0000:04:00.0
            # slot mpcie1 on right is address 0000:06:00.0
            my @radios = ("0000:06:00.0", "0000:04:00.0");
            my $port_layout = "PORT_CONF eth0 eth1 eth2 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "ct521b", "eth0");

                # These systems use igb, and it will work better with a single receive queue.
                if (!-f "/etc/modprobe.d/igb.conf") {
                    do_cmd("echo options igb max_rss_qs=1 > /etc/modprobe.d/igb.conf");
                }
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "R7800") {
            # Netgear r7800 OpenWRT
            my $old_dev = "eth0.5";
            my @ports = ("eth1.4", "eth1.3", "eth1.2", "eth1.1", $old_dev);
            my @radios = ("0000:01:00.0", "0001:01:00.0", @usb_radios);
            $rad_count = @radios;
            my $port_layout = "PORT_CONF eth1.1 eth1.2 eth1.3 eth1.4 eth0.5 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "R7800", "eth0.5");
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "ECW5410") {
            # OpenWrt
            my $old_dev = "eth0";
            my @ports = ("eth1", $old_dev);
            my @radios = ("0001:01:00.0", "0002:01:00.0", @usb_radios);
            $rad_count = @radios;
            my $port_layout = "PORT_CONF eth1 eth0 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "ECW5410", "eth0");
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "ECW5211") {
            # OpenWrt
            my $old_dev = "eth0";
            my @ports = ($old_dev, "eth1");
            my @radios = ("a000000.wifi", "a800000.wifi", @usb_radios);
            $rad_count = @radios;
            my $port_layout = "PORT_CONF eth0 eth1 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "ECW5211", "eth0");
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "E8450") {
            # OpenWrt
            my $old_dev = "wan";
            my @ports = ($old_dev, "lan1", "lan2", "lan3", "lan4");
            my @radios = ("18000000.wmac", "0000:01:00.0", @usb_radios);
            $rad_count = @radios;
            my $port_layout = "PORT_CONF lan1 lan2 lan3 lan4 wan ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "E8450", $do_mgt_dev);
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "EC420") {
            # OpenWrt
            my $old_dev = "eth0";
            my @ports = ($old_dev);
            my @radios = ("0000:01:00.0", "a000000.wifi", @usb_radios);
            $rad_count = @radios;
            my $port_layout = "PORT_CONF eth0 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "EC420", "eth0");
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "CIG194C") {
            # OpenWrt
            my $old_dev = "eth1";
            my @ports = ($old_dev, "eth0");
            # ath11k puts multiple radios on one bus :(
            my @radios = ("c000000.wifi1", "c000000.wifi1", @usb_radios);
            $rad_count = @radios;
            my $port_layout = "PORT_CONF eth0 eth1 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "CIG194C", "eth1");
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "NEO2") {
            # NEO2 OpenWRT
            my $old_dev = "eth0";
            my @ports = ($old_dev);
            $rad_count = @radios;
            my $port_layout = "PORT_CONF eth0 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "NEO2", "eth0");
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "LF0314") {
            # RPI4
            my $old_dev = "eth0";
            my @ports = ($old_dev);
            $rad_count = @radios;
            my $port_layout = "PORT_CONF eth0 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "rpi4", "eth0");
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "LF0314_5") {
            # RPI5
            my $old_dev = "eth0";
            my @ports = ($old_dev);
            $rad_count = @radios;
            my $port_layout = "PORT_CONF eth0 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "rpi5", "eth0");
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "LF0312") {
                                       # ibase, ct523b
            my $old_dev = "enp0s31f6"; # Right port, mgt
            my @ports = ($old_dev, "enp3s0");
            # Bottom, slots 0-2, right to left as viewed with case cover off.
            # Wouldn't be suprised if different radio counts changed this.
            my @radios = ("0000:05:00.0", "0000:04:00.0", "0000:02:00.0", "0000:01:00.0");
            my $port_layout = "PORT_CONF eth1 eth0 ~";

            if ($::do_sys_reconfig) {
                note(" do_sys_reconfig LF0312");
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "lf0312", "eth0");
                if (-f "$home/scripts/fanctl_lf0312.pl" && !-f "/etc/cron.d/lf0312") {
                    note("...adding cron file to operate fan");
                    my $crontab = "* * * * * root $home/scripts/fanctl_lf0312.pl\n";
                    do_cmd("echo '$crontab' > /etc/cron.d/lf0312", 1);
                    chmod 0644, "/etc/cron.d/lf0312";
                }
                if (!$is_macos) {
                    if (!-d "$home/f81866_fan") {
                        warning("f81866_fan not found");
                    }

                    if (!-d "$home/scripts/fanctl_lf0312.pl") {
                        warning("fanctl_lf0312.pl not found");
                    }
                }
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "CT523c") {
            # New axiomtek ct523c
            # built-in device names change based on cards installed, so try various names
            my $old_dev = "enp1s0"; # mgt
            my @ports = ($old_dev, "enp2s0");

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp2s0";
                @ports = ($old_dev, "enp3s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp3s0";
                @ports = ($old_dev, "enp4s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp4s0";
                @ports = ($old_dev, "enp5s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp5s0";
                @ports = ($old_dev, "enp6s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp6s0";
                @ports = ($old_dev, "enp7s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp7s0";
                @ports = ($old_dev, "enp8s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp8s0";
                @ports = ($old_dev, "enp9s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp9s0";
                @ports = ($old_dev, "enp10s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp10s0";
                @ports = ($old_dev, "enp11s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp11s0";
                @ports = ($old_dev, "enp12s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp12s0";
                @ports = ($old_dev, "enp13s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp13s0";
                @ports = ($old_dev, "enp14s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp14s0";
                @ports = ($old_dev, "enp15s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp15s0";
                @ports = ($old_dev, "enp16s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp16s0";
                @ports = ($old_dev, "enp17s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp17s0";
                @ports = ($old_dev, "enp18s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp18s0";
                @ports = ($old_dev, "enp19s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp19s0";
                @ports = ($old_dev, "enp20s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp20s0";
                @ports = ($old_dev, "enp21s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp21s0";
                @ports = ($old_dev, "enp22s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp22s0";
                @ports = ($old_dev, "enp23s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp23s0";
                @ports = ($old_dev, "enp24s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp24s0";
                @ports = ($old_dev, "enp25s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp25s0";
                @ports = ($old_dev, "enp26s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp26s0";
                @ports = ($old_dev, "enp27s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp27s0";
                @ports = ($old_dev, "enp28s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp28s0";
                @ports = ($old_dev, "enp29s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp29s0";
                @ports = ($old_dev, "enp30s0");
            }

            if (!-e "/sys/class/net/$old_dev") {
                $old_dev = "enp44s0";
                @ports = ($old_dev, "eno1");
            }

            # Some common layouts are supported here, but not all possible ones.
            my @radios = @radios_pci;
            my $port_layout = "PORT_CONF eth1 eth0 ~";
            my $ax_count200 = `lspci|grep AX200|wc -l`;
            my $ax_count210 = `lspci|grep AX210|wc -l`;
            my $ax_count210a = `lspci|grep 'Intel Corporation Device 2725'|wc -l`;
            my $be_count200 = `lspci|grep 'Intel Corporation Device 272b'|wc -l`;
            my $amfeltec_gen3 = `lspci|grep 'PEX 8717'|wc -l`;  # 9 show up per 6-port adapter.
            my $mtk_7915_count = `lspci|grep MEDIATEK|grep 7915|wc -l`;
            my $mtk_7921_count = `lspci|grep MEDIATEK|grep 7921|wc -l`;
            my $mtk_7922_count = `lspci|grep MEDIATEK|grep 7922|wc -l`;
            my $mtk_7925_count = `lspci|grep MEDIATEK|grep 7925|wc -l`;
            my $w2_count = `lspci|grep QCA9984|wc -l`;
            my $e10g_count = `lspci|grep X550|wc -l`;

            chomp($ax_count200);
            chomp($ax_count210);
            chomp($ax_count210a);
            chomp($be_count200);
            chomp($mtk_7915_count);
            chomp($mtk_7921_count);
            chomp($mtk_7922_count);
            chomp($mtk_7925_count);
            chomp($w2_count);
            chomp($e10g_count);

            my $ax_count = 0;
            if ($ax_count200 ne "") {
                $ax_count += int($ax_count200);
            }
            if ($ax_count210 ne "") {
                $ax_count += int($ax_count210);
            }
            if ($ax_count210a ne "") {
                $ax_count += int($ax_count210a);
            }
            if ($be_count200 ne "") {
                $ax_count += int($be_count200);
            }
            if ($mtk_7921_count ne "") {
                $ax_count += int($mtk_7921_count)
            }
            if ($mtk_7922_count ne "") {
                $ax_count += int($mtk_7922_count)
            }
            if ($mtk_7925_count ne "") {
                $ax_count += int($mtk_7925_count)
            }

            if ($is_new_523c && $lf2100_8x && $ax_count == 24 && $e10g_count eq "2") {
                # internal splitter combiner x3 plus 10g
                note(" do_sys_reconfig, detected 8ax with internal 8x 4-1 splitter board, plus dual 10g ct523c");
                @radios = ("0000:08:00.0", "0000:04:00.0", "0000:06:00.0", "0000:03:00.0",
                           "0000:09:00.0", "0000:07:00.0", "0000:0b:00.0", "0000:0d:00.0",
                           "0000:11:00.0", "0000:12:00.0", "0000:14:00.0", "0000:15:00.0",
                           "0000:16:00.0", "0000:17:00.0", "0000:19:00.0", "0000:1b:00.0",
                           "0000:20:00.0", "0000:23:00.0", "0000:22:00.0", "0000:24:00.0",
                           "0000:25:00.0", "0000:1f:00.0", "0000:27:00.0", "0000:29:00.0");

                @ports = (@ports, "enp43s0f0", "enp43s0f1"); # Should be correct
                $port_layout = "PORT_CONF eth1 NA eth2 ~\nPORT_CONF eth0 NA eth3 ~\n";
            }
            elsif ($amfeltec_gen3 == 9 && $e10g_count eq "0") {
                # Amfeltec pcie gen-3 x6 adapter, no 10g
                note(" do_sys_reconfig, detected 6x gen-3 adapter");
                @radios = ("0000:09:00.0", "0000:05:00.0", "0000:06:00.0", # left to right, front side.
                           "0000:0a:00.0", "0000:03:00.0", "0000:04:00.0"); # right to left, back side (right is next to faceplate)

                $port_layout = "PORT_CONF eth1 NA ~\nPORT_CONF eth0 NA ~\n";
            }
            elsif ($amfeltec_gen3 == 18 && $e10g_count eq "0") {
                # Qty-2 Amfeltec pcie gen-3 x6 adapters, no 10g
                note(" do_sys_reconfig, detected two 6x gen-3 adapters");
                @radios = ("0000:13:00.0", "0000:0f:00.0", "0000:10:00.0",  # Slot3, left to right, front side.
                           "0000:14:00.0", "0000:0d:00.0", "0000:0e:00.0",  # Slot3, right to left, back side (right is next to faceplate)
                           "0000:09:00.0", "0000:05:00.0", "0000:06:00.0",  # Slot4, left to right, front side.
                           "0000:0a:00.0", "0000:03:00.0", "0000:04:00.0"); # Slot4, right to left, back side (right is next to faceplate)

                $port_layout = "PORT_CONF eth1 NA ~\nPORT_CONF eth0 NA ~\n";
            }
            elsif ($lf2100_8x && $ax_count == 8 && $e10g_count eq "2") {
                # internal splitter combiner plus 10g
                note(" do_sys_reconfig, detected 8ax with internal 8x 4-1 splitter board, plus dual 10g ct523c");
                @radios = ("0000:08:00.0", "0000:04:00.0", "0000:0c:00.0", "0000:0a:00.0",
                           "0000:09:00.0", "0000:07:00.0", "0000:0e:00.0", "0000:05:00.0");

                @ports = (@ports, "enp1s0f0", "enp1s0f1"); # Not sure this is correct.
                $port_layout = "PORT_CONF eth1 NA eth2 ~\nPORT_CONF eth0 NA eth3 ~\n";
            }
            elsif ($lf2100_8x && $ax_count == 16 && $e10g_count eq "2") {
                # internal splitter combiner x2 plus 10g
                note(" do_sys_reconfig, detected 8ax with internal 8x 4-1 splitter board, plus dual 10g ct523c");
                @radios = ("0000:08:00.0", "0000:04:00.0", "0000:0c:00.0", "0000:0a:00.0",
                           "0000:09:00.0", "0000:07:00.0", "0000:0e:00.0", "0000:05:00.0",
                           "0000:16:00.0", "0000:12:00.0", "0000:1a:00.0", "0000:18:00.0",
                           "0000:17:00.0", "0000:15:00.0", "0000:1c:00.0", "0000:13:00.0");

                @ports = (@ports, "enp1s0f0", "enp1s0f1"); # Not sure this is correct.
                $port_layout = "PORT_CONF eth1 NA eth2 ~\nPORT_CONF eth0 NA eth3 ~\n";
            }
            elsif ($lf2100_8x && $ax_count == 24 && $e10g_count eq "2") {
                # internal splitter combiner x3 plus 10g
                note(" do_sys_reconfig, detected 8ax with internal 8x 4-1 splitter board, plus dual 10g ct523c");
                @radios = ("0000:08:00.0", "0000:04:00.0", "0000:0c:00.0", "0000:0a:00.0",
                           "0000:09:00.0", "0000:07:00.0", "0000:0e:00.0", "0000:05:00.0",
                           "0000:24:00.0", "0000:20:00.0", "0000:28:00.0", "0000:26:00.0",
                           "0000:25:00.0", "0000:23:00.0", "0000:2a:00.0", "0000:21:00.0",
                           "0000:16:00.0", "0000:12:00.0", "0000:1a:00.0", "0000:18:00.0",
                           "0000:17:00.0", "0000:15:00.0", "0000:1c:00.0", "0000:13:00.0");

                @ports = (@ports, "enp1s0f0", "enp1s0f1"); # Not sure this is correct.
                $port_layout = "PORT_CONF eth1 NA eth2 ~\nPORT_CONF eth0 NA eth3 ~\n";
            }
            elsif ($lf2100_8x && $ax_count == 24 && $e10g_count eq "0") {
                # internal splitter combiner x3, no 10g
                note(" do_sys_reconfig, detected 8ax with internal 8x 4-1 splitter board, ct523c");
                @radios = ("0000:07:00.0", "0000:03:00.0", "0000:0b:00.0", "0000:09:00.0",
                           "0000:08:00.0", "0000:06:00.0", "0000:0d:00.0", "0000:04:00.0",
                           "0000:23:00.0", "0000:1f:00.0", "0000:27:00.0", "0000:25:00.0",
                           "0000:24:00.0", "0000:22:00.0", "0000:29:00.0", "0000:20:00.0",
                           "0000:15:00.0", "0000:11:00.0", "0000:19:00.0", "0000:17:00.0",
                           "0000:16:00.0", "0000:14:00.0", "0000:1b:00.0", "0000:12:00.0");

                @ports = (@ports); # Not sure this is correct.
                $port_layout = "PORT_CONF eth1 ~\nPORT_CONF eth0 ~\n";
            }
            # 12x x2 adapter plus 10g.
            elsif ($ax_count == 24 && $e10g_count eq "2") {
                note(" do_sys_reconfig, detected 24-ax, dual 10g ct523c");
                @radios = ("0000:05:00.0", "0000:0a:00.0", "0000:0c:00.0", "0000:0e:00.0",
                    "0000:04:00.0", "0000:07:00.0", "0000:08:00.0", "0000:09:00.0",
                    "0000:06:00.0", "0000:0b:00.0", "0000:0d:00.0", "0000:0f:00.0",

                    "0000:13:00.0", "0000:18:00.0", "0000:1a:00.0", "0000:1c:00.0",
                    "0000:12:00.0", "0000:15:00.0", "0000:16:00.0", "0000:17:00.0",
                    "0000:14:00.0", "0000:19:00.0", "0000:1b:00.0", "0000:1d:00.0");

                @ports = (@ports, "enp1s0f0", "enp1s0f1"); # Not sure this is correct.
                $port_layout = "PORT_CONF eth1 NA eth2 ~\nPORT_CONF eth0 NA eth3 ~\n";
            }
            # 12x adapter plus 10g.
            elsif ($ax_count == 12 && $e10g_count eq "2") {
                note(" do_sys_reconfig, detected 12-ax, dual 10g ct523c");
                @radios = ("0000:05:00.0", "0000:0a:00.0", "0000:0c:00.0", "0000:0e:00.0",
                    "0000:04:00.0", "0000:07:00.0", "0000:08:00.0", "0000:09:00.0",
                    "0000:06:00.0", "0000:0b:00.0", "0000:0d:00.0", "0000:0f:00.0");
                @ports = (@ports, "enp1s0f0", "enp1s0f1"); # Not sure this is correct.
                $port_layout = "PORT_CONF eth1 NA eth2 ~\nPORT_CONF eth0 NA eth3 ~\n";
            }
            elsif ($ax_count == 8 && $w2_count eq "2" && $e10g_count eq "2") {
                note(" do_sys_reconfig, detected 8-ax dual wave-2, dual 10g ct523c");
                # 8-ax, dual-band wave-2, 10g NIC.  wallytech uses 2 farthest slots, 5g in farthest.
                # each wallystech board is fully populated, closest radio to SMA plate is radio 1, and also lowest PCI
                @radios = ("0000:06:00.0", "0000:07:00.0", # wallytech 2.4 and 5g, respectively
                    "0000:12:00.0", "0000:13:00.0", "0000:14:00.0", "0000:15:00.0",
                    "0000:0b:00.0", "0000:0c:00.0", "0000:0d:00.0", "0000:0e:00.0");
                @ports = (@ports, "enp1s0f0", "enp1s0f1");
                $port_layout = "PORT_CONF eth1 NA eth2 ~\nPORT_CONF eth0 NA eth3 ~\n";
            }
            elsif ($ax_count == 4 && $w2_count eq "2" && $e10g_count eq "0") {
                note(" do_sys_reconfig, detected 4-ax, dual wave-2 ct523c");
                # 4-ax, dual-band wave-2. wallystech with wave-2s in slot 3 and amfeltec 4ax slot 4.
                # wallytech uses slot 2 and slot 3, 5g in furthest.
                # amfeltec board puts radio 1 in middle slot, 2 in far right slot.  On other side,
                #   middle left is radio 4, middle right is radio 3.
                @radios = ("0000:0e:00.0", "0000:0f:00.0",                           # wallytech 2.4 and 5g, respectively, wlan0 - 1
                    "0000:09:00.0", "0000:04:00.0", "0000:05:00.0", "0000:03:00.0"); # ax210, wlan2 - 5
                $old_dev = "enp18s0";
                @ports = ($old_dev, "enp19s0");
                $port_layout = "PORT_CONF eth1 ~\nPORT_CONF eth0 ~\n";
            }
            elsif ($ax_count == 4 && $w2_count eq "4" && $e10g_count eq "2") {
                note(" do_sys_reconfig, detected 4-ax, quad wave-2, dual 10g ct523c");
                # 4-ax, dual-band wave-2, 10g NIC.  wallytech next to 10g uses 2 farthest slots, 5g in farthest.
                # second wallytech uses slot 2 and slot 4, 5g in furthest.
                # amfeltec board puts radio 1 in middle slot, 2 in far right slot.  On other side,
                #   middle left is radio 4, middle right is radio 3.
                @radios = ("0000:06:00.0", "0000:07:00.0",                           # wallytech 2.4 and 5g, respectively, next to 10g
                    "0000:16:00.0", "0000:18:00.0",                                  # wallytech 2.4 and 5g, respectively, wlan2, wlan3, second
                    "0000:11:00.0", "0000:0c:00.0", "0000:0d:00.0", "0000:0b:00.0"); # ax200, wlan4 - 7
                $old_dev = "enp26s0";
                @ports = ($old_dev, "enp27s0", "enp1s0f0", "enp1s0f1");
                $port_layout = "PORT_CONF eth1 NA eth2 ~\nPORT_CONF eth0 NA eth3 ~\n";
            }
            elsif ($mtk_7915_count == 6 && $e10g_count eq "2") {
                note(" do_sys_reconfig, detected 6-mtk, dual 10g ct523c");
                # 6-mtk, 10g NIC.  10g in leftmost (blue) 523c slot. wallytech next to 10g mtk in 2 farthest slots
                # second wallystech uses slot 3 and slot 4
                # third wallystech uses slot 3 and slot 4
                @radios = ("0000:06:00.0", "0000:07:00.0", # wallystech x2 mtk next to 10G, wlan0, wlan1, radios in furthest two slots
                    "0000:14:00.0", "0000:15:00.0",        # wallystech x2 mtk wlan2, wlan3, second
                    "0000:0d:00.0", "0000:0e:00.0");       # wallystech x2 mtk wlan4, wlan5, third
                $old_dev = "enp23s0";
                @ports = ($old_dev, "enp24s0", "enp1s0f0", "enp1s0f1");
                $port_layout = "PORT_CONF eth1 NA eth2 ~\nPORT_CONF eth0 NA eth3 ~\n";
            }
            elsif ($ax_count eq "4" && $e10g_count eq "2") {
                note(" do_sys_reconfig, detected 4-ax, 10g ct523c");
                # one wallystech with 4-ax, in slot 3, 10G in blue slot 1
                @radios = ("0000:04:00.0", "0000:05:00.0", "0000:06:00.0", "0000:07:00.0"); # wallystech with 4-ax
                $old_dev = "enp9s0";
                @ports = ($old_dev, "enp10s0", "enp1s0f0", "enp1s0f1");
                $port_layout = "PORT_CONF eth1 NA eth2 ~\nPORT_CONF eth0 NA eth3 ~\n";
            }
            elsif ($ax_count eq "4" && $e10g_count eq "0") {
                note(" do_sys_reconfig, detected 4-ax, no 10g ct523c");
                # one wallystech with 4-ax, in slot 4, opposite side of blue 10g slot
                @radios = ("0000:03:00.0", "0000:04:00.0", "0000:05:00.0", "0000:06:00.0"); # wallystech with 4-ax
                $old_dev = "enp8s0";
                @ports = ($old_dev, "enp9s0");
                $port_layout = "PORT_CONF eth1 ~\nPORT_CONF eth0 ~\n";
            }
            elsif ($w2_count eq "6" && $e10g_count eq "0") {
                note(" do_sys_reconfig, detected six wave-2 ct523c");
                # six dual-band wave-2.  next to empty slot that would usually hold a 10G. wallystech uses 2 farthest slots, 5g in farthest.
                # second wallystech uses slot 2 and slot 4, 5g in furthest.
                # third wallystech uses 2 farthest slots, 5g in farthest
                @radios = ("0000:05:00.0", "0000:06:00.0", # wallystech 2.4 and 5g, respectively, wlan0, wlan1, next to what would normally be a 10g but empty slot
                    "0000:12:00.0", "0000:14:00.0",        # wallystech 2.4 and 5g, respectively, wlan2, wlan3, second
                    "0000:0c:00.0", "0000:0d:00.0");       # wallystech 2.4 and 5g, respectively, wlan4, wlan5, third
                $old_dev = "enp22s0";
                @ports = ($old_dev, "enp23s0");
                $port_layout = "PORT_CONF eth1 ~\nPORT_CONF eth0 ~\n";
            }

            $rad_count = @radios;

            if ($::do_sys_reconfig) {
                note(" do_sys_reconfig CT523c");
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "ct523c", "eth0");
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif (($hwver eq "LF0313") || ($hwver eq "CT522")) {
                                      # jetway, ct522
            my $old_dev = "enp1s0f0"; # Left port, mgt
            my @ports = ($old_dev, "enp1s0f1", "enp1s0f2", "enp1s0f3", "enp2s0", "enp0s31f6");
            my @radios = ("0000:03:00.0", "0000:04:00.0");
            my $port_layout = "PORT_CONF eth0 eth1 eth2 eth3 eth4 eth5 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "ct522", "eth0");
            }
            for ($i = 0; $i < $rad_count; $i++) {
                getPhyInfo($i, "wlan$i", $radios[$i], "<b>Radio $i</b>");
            }
        }
        elsif ($hwver eq "LF0202") {
            # Lanner 6-port
            my $old_dev = "enp0s20f3";
            my @ports = ("enp1s0", "enp2s0", "enp0s20f0", "enp0s20f1", "enp0s20f2", $old_dev);
            my @radios = ();
            my $port_layout = "PORT_CONF eth0 eth1 eth2 eth3 eth4 eth5 ~";

            if ($::do_sys_reconfig) {
                _do_sys_reconfig($old_dev, \@ports, \@radios, $port_layout, $rad_count, "ct920", "eth5");

                # These systems use igb, and it will work better with a single receive queue.
                if (!-f "/etc/modprobe.d/igb.conf") {
                    do_cmd("echo options igb max_rss_qs=1 > /etc/modprobe.d/igb.conf");
                }
            }
        }
        else {
            # Bus 0000:03:00.0 will be wiphy0 (pci-e NIC on 520, 521)
            getPhyInfo(0, "wlan0", "0000:03:00.0", "<b>Bottom</b> (PCIe plug-in card)");

            if ($hwver eq "CT521") {
                getPhyInfo(1, "wlan1", "0000:04:00.0", "<b>Top</b>");
            }
        }

        # This deals with USB adapters.
        my $mc_sofar = @desc;
        if (@radios > $mc_sofar) {
            for ($q = $mc_sofar; $q < @radios; $q++) {
                my $rad = $radios[$q];
                $tmp = `cat /sys/class/ieee80211/$rad/macaddress`;
                chomp($tmp);
                $macs[$q] = $tmp;
                $desc[$q] = "<b>USB</b>";
                $driver[$q] = "UNKNOWN";
            }
        }
    } # ~if we matched a LF chassis profile

    if ($hwver ne "UNKNOWN" || $force_notes) {
        $fname = "$::http_root/lfnotes.html";

        if (-f $fname) {
            do_cmd("cp $fname /root/lfnotes.html.bak-" . time, 1);
        }

        if (!open(FILE, ">$fname")) {
            warning("# Failed to create: $fname, error: $!  Not fatal.\n");
            return;
        }

        my $hn = "";
        if (-r "/etc/hostname") {
            $hn = `cat /etc/hostname`;
        }
        if ($hn eq "") {
            $hn = `hostname`;
        }
        chomp($hn);
        if ($hn ne "") {
            $hn = ": $hn";
        }

        print FILE "
<html>
<body>
<center><h2>Information about the LANforge System$hn</h2></center>
<P>
<table width=\"100%\" border='0' cellpadding='0' cellspacing='0'>
  <tr>
    <td>
      <table border=\"1\" cellpadding=\"2\" cellspacing=\"0\" width=240>
        <caption><b>Default&nbsp;Login&nbsp;Accounts</b></caption>
        <tr><td><b>User</b></td> <td><b>Password</b></td></tr>
        <tr><td>lanforge</td> <td>lanforge</td></tr>
        <tr><td>root</td> <td>lanforge</td></tr>
      </table>
    </td>\n";
        if (($hwver eq "CT520") || ($hwver eq "CT521") || ($hwver eq "CT521b") || ($hwver eq "CT523") || ($hwver eq "CT523c")
            || ($hwver eq "CT52x-PR") || ($hwver eq "LF0350") || ($hwver eq "LF0312") || ($hwver eq "LF0313") || ($hwver eq "CT522")
            || ($hwver eq "R7800") || ($hwver eq "ECW5410") || ($hwver eq "EC420") || ($hwver eq "CIG194C") || ($hwver eq "NEO2")) {
            print FILE "
    <td align=\"right\">
      <table border=\"1\" cellpadding=\"2\" cellspacing=\"0\">
        <caption><b>Wireless NIC(s) Installation</b></caption>\n";
            print FILE "        <tr><th>Location</th> <th>Radio</th> <th>MAC</th> <th>Driver</th></tr>\n";

            for ($q = 0; $q < @radios; $q++) {
                my $rad = $radios[$q];
                my $desc = $desc[$q];
                print FILE "
        <tr><td>$desc</td>  <td>$rad / W$q</td>  <td>$macs[$q]</td> <td>$driver[$q]</td> </tr>\n";
            }
            print FILE "
      </table>
    </td>\n";
        }
        print FILE "
  </tr>
</table>
<P>
Plug the management (<b>MGT</b>) port into your lab network. It is configured to use
DHCP and is reconfigurable. If the unit is powered up without the
<b>MGT</b> port plugged into a network serving DHCP, the IP address will default
to <b>192.168.1.101</b>.
<P>
A LANforge GUI manages the LANforge system and can be installed on a
separate PC or run on the
LANforge system desktop via VNC or Remote Desktop.
<P>
Connecting a browser to the IP address of the unit, e.g. http://192.168.1.101,
gives download access to the Windows and Linux versions of the LANforge GUI for
installation and links to documentation.
<P>
A PC running the GUI on the same subnet as the MGT port can discover the
LANforge system and other systems. Click '<b>Discover</b>' in the GUI connection
widget and it should populate the drop-down box for easy connecting,
otherwise, enter the IP address of the LANforge system.
<P>
Alternative means of accessing a LANforge system include:<br>
- Connect a browser to port 5801, e.g. http://192.168.1.101:5801, for unit
management with a RealVNC JAVA applet client.<br>
- Connect a VNC client to port 5901 or display :1, e.g. 192.168.1.101::5901
or 192.168.1.101:1 with the VNC password 'lanforge' in both cases.<br>
- Connect a Remote Desktop client to management interface, e.g. 192.168.1.101,
user and password lanforge.<br>
- Access to the system over serial console, <b>COM1</b> with serial port
settings of: <b>115200 8 N 1</b><br>
<P>
User 'lanforge' is the preferred login. The LANforge servers start automatically
when the machine starts.
<P>
Additional full documentation, including user and install guides, is available
at <a href=\"http://www.candelatech.com/doc.php\" target=_new>www.candelatech.com/doc.php</a>,
and in the LANforge GUI install directory under the html directory.
<P>
For support or questions, please send email to
<a href=\"mailto:support\@candelatech.com\">support\@candelatech.com</a> and/or
call us at: +1 360 380 1618.
<P>
Thank You, Candela Support Team!
<P>
<font size=-1>$date: $os_string, ${archbit} bit installed and updated and LANforge
$::lfver installed and configured.</font>
</body>
</html>
";
        close FILE;
    }
    else {
        warning("# Did not detect specific hardware chassis, not creating lfnotes.html\n");
    }

    my $hn = `cat /etc/hostname`;
    chomp($hn);
    my $e0mac = `ip li sho eth0 | awk '/link/ {print \$2}'| head -1 | tr -d :`;
    chomp($e0mac);
    if ($? != 0) {
        warning("# mac address not found for eth0?");
    }
    else {
        $e0mac = substr($e0mac, -4);
        print "HOSTNAME: $hn\nMAC:             $e0mac\n";
        if ($hn !~ /$e0mac$/) {
            warning("\n# = = = = = = = = Hostname does not end with MAC address = = = = = = = = #\n");
        }
    }
} # ~try_generate_lfnotes()

sub fixup_lightdm_conf {
    if ($::is_macos) {
        return;
    }

    my $fn = "/etc/lightdm/lightdm.conf";
    if (-f "$fn") {
        my $new_text = "";
        my @lns = `cat $fn`;
        for my $ln (@lns) {
            chomp($ln);
            if ($ln =~ /^#xserver-allow-tcp=/) {
                note("# Enabling lightdm xserver-allow-tcp=true.\n");
                $new_text .= "xserver-allow-tcp=true\n";
            }
            elsif ($ln =~ /^autologin-user=pi/) {
                $new_text .= "#### autologin-user=pi\n";
            }
            else {
                $new_text .= "$ln\n";
            }
        }
        if (open FILE, ">", $fn) {
            print FILE $new_text;
            close FILE;
        }
    }
}

sub create_https_script {
    my @lines = (
        "[Unit]",
        "Description=Regenerate the HTTPS certificate used by Apache on this system",
        "Before=systemd-user_sessions.service",
        "Wants=network-online.target",
        "Before=httpd.service",
        "After=network-online.target",
        "ConditionPathExists=$home/do_https_key",
        "",
        "[Service]",
        "Type=oneshot",
        "ExecStart=$home/lf_kinstall.pl --regen_https_key",
        "ExecStartPost=/usr/bin/rm -f $home/do_https_key",
        "RemainAfterExit=no",
        "",
        "[Install]",
        "WantedBy=httpd.target"
    );
    my $fname = "/etc/systemd/system/create_https_cert.service";
    open(my $create_https_f, ">", $fname) or die("Unable to open[$fname] $!");
    for my $line (@lines) {
        print $create_https_f "$line\n";
    }
    close($create_https_f);
    do_cmd("systemctl enable create_https_cert.service");
}

sub _do_sys_reconfig {
    note("In _do_sys_reconfig, openwrt: $::is_openwrt");
    # if we are on fedora >= 30 then create a systemd unit that will issue the regen_https_key action
    if ($is_fedora && ($::osveri >= 30)) {
        # recreate this file in case there are changes in future
        create_https_script();

        if (!-f "$home/do_https_key") {
            do_cmd("touch $home/do_https_key");
            do_cmd("chown lanforge: $home/do_https_key");
            do_cmd("chmod 666 $home/do_https_key");
            note("#  do_https_key task created... \n");
            note("* Rebooting system will generate a new HTTPS certificate. This is expected.\n");
        }

        # if .bash_profile/hosts does not exist
        # #.bash_profile
        # # created by lf_kinstall.pl
        # if [ -f ~/.bashrc ]; then
        #    . ~/.bashrc
        # fi
        # [ -d $HOME ] && cd $HOME ||:


        # if .bashrc does not exist
        #     cp /etc/skel/.bashrc .
        # else
        #     update bashrc
        #     add shopt PS1
        #     cd $HOME
        # shopt -q login_shell && cd $HOME || :
        # add PS1
        # [[ $- == *i* ]] && export PS1="[\u@\h ] \w\n $ "
    }

    my ($old_dev, $ports_ref, $radios_ref, $port_layout, $rad_count, $hn_prefix, $mgt_dev) = @_;
    my @ports = @{$ports_ref};
    my @radios = @{$radios_ref};
    my $fname = "";

    my $i;
    my $mac;

    fixup_lightdm_conf();

    if (!-e "$home/defaults.db") {
        write_defaults_db();
    }

    if ($is_rpi4 || $is_rpi5) {
        note("# Setting root password to lanforge");
        do_cmd(qq(bash -c 'echo "root:lanforge" | chpasswd'));

        # Remove auto-login
        $fname = "/etc/systemd/system/autologin@.service";
        if (-f "$fname") {
            my @lines = `cat $fname`;
            open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";

            for ($i = 0; $i < @lines; $i++) {
                my $ln = $lines[$i];
                chomp($ln);
                if ($ln =~ /(.*) --autologin pi (.*)/) {
                    print FILE "$1 $2\n";
                }
                else {
                    print FILE "$ln\n";
                }
            }
        }
    }

    if (-d "/sys/class/net/$old_dev") {
        $mac = `cat /sys/class/net/$old_dev/address`;
    }
    else {
        $mac = `cat /sys/class/net/$mgt_dev/address`;
    }
    chomp($mac);

    # Set the hostname.
    my $hn = "";
    if (-f "/etc/hostname") {
        $hn = `cat /etc/hostname`;
        chomp($hn);
    }
    if ($mac =~ /.*\:(\S\S):(\S\S)/) {
        my $mac_l2 = "$1$2";
        $hn = "$hn_prefix-$mac_l2";
        if ($::is_openwrt) {
            $hn = "LF-$hn";
            do_cmd("uci set system.\@system\[0\].hostname='$hn'");
            do_cmd("uci commit system");
        }
        else {
            do_cmd("echo $hn > /etc/hostname");
            do_cmd("hostname -F /etc/hostname");
        }
    }

    $fname = "/etc/hosts";
    if (-f "$fname") {
        my @lines = `cat $fname`;
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        my $found_l4 = 0;

        for ($i = 0; $i < @lines; $i++) {
            my $ln = $lines[$i];
            chomp($ln);
            if ($ln =~ /^127\.0\.0\.1\s+.*/) {
                print FILE "127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4 $hn-local\n";
                $found_l4 = 1;
            }
            else {
                print FILE "$ln\n";
            }
        }
        if (!$found_l4) {
            print FILE "127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4 $hn-local\n";
        }
        close FILE;
    }

    if (!$::is_openwrt && !$::is_macos) {
        # TODO: make this a subroutine
        note("Rebuilding HTTPS certs\n");
        sleep 2;
        $hn = `cat /etc/hostname`;
        chomp($hn);
        if (!(defined $hn) || ("" eq $hn)) {
            $hn = `hostname`;
            chomp($hn);
        }
        if (!(defined $hn) || ("" eq $hn)) {
            if ($::is_openwrt) {
                $hn = "OpenWrt";
            }
        }
        print "HOSTNAME: $hn\n";
        die("Hostname is blank") if (!(defined $hn) || ("" eq $hn));
        system("/usr/bin/openssl req -x509 -nodes -days 1460 -newkey rsa:1024 -keyout $hn.key -out $hn.crt -sha512"
            . " -subj '/C=US/ST=Washington/L=Ferndale/O=Candela Technologies, Inc./OU=LANforge/emailAddress=support\@candelatech.com/CN=$hn'");
        my $cert_pre = "/etc/pki/tls";
        if (!-d $cert_pre) {
            if (-d "/etc/ssl/certs") {
                $cert_pre = "/etc/ssl";
            }
        }
        note("Copying certificates...\n");
        do_cmd("cp $hn.crt $cert_pre/certs/");
        do_cmd("mv $hn.crt $cert_pre/certs/localhost.crt");
        do_cmd("cp $hn.key $cert_pre/private/");
        do_cmd("mv $hn.key $cert_pre/private/localhost.crt");
        if (-f "$hn.csr") {
            do_cmd("mv $hn.csr $cert_pre/private/");
        }
        # re-write the apache server hostname
        if (-d "/etc/httpd/conf.d") {
            die($!) unless open(FILE, ">", "/etc/httpd/conf.d/00-ServerName.conf");
            print FILE "ServerName $hn\n";
            close FILE;
        }
        else {
            # RPI-4, probably rest of newer debian based stuff
            if (-d "/etc/apache2/sites-enabled/") {
                die($!) unless open(FILE, ">", "/etc/apache2/sites-enabled/00-ServerName.conf");
                print FILE "ServerName $hn\n";
                close FILE;
            }
        }
        # set ff homepage
        if ($::do_ff_homepage) {
            update_ff_homepage();
        }
        # interface renaming might not succeed. Attempt some of this first.
        #do_cmd("$home/scripts/adjust_apache.pl"); # defer until interfaces are renamed
    }
    if ($::skip_ifrename) {
        note("--skip_ifrename is set, will not rename any interfaces or re-create udev/rules.d/70*.rules files\n");
        return;
    }
    if (!$::is_openwrt) {
        $fname = "/etc/sysconfig/network-scripts/ifcfg-$old_dev";
        my $mname = "/etc/sysconfig/network-scripts/ifcfg-$mgt_dev";
        my $uuid = `uuidgen $mgt_dev`;
        if (-f $fname && !-f $mname) {
            do_cmd("ifdown $old_dev > /dev/null 2>&1", 1);
            do_cmd("mv $fname /etc/sysconfig/network-scripts/ifcfg-$mgt_dev");
            $fname = "/etc/sysconfig/network-scripts/ifcfg-$mgt_dev";
            my $content = `cat $fname`;
            $content =~ s/$old_dev/$mgt_dev/g;
            $content =~ s/UUID=\S+/UUID=$uuid/g;
            open(FILE, ">$fname") or die("Couldn't open file: $fname for writing: $!\n\n");
            print FILE $content;
            close FILE;
        }
        elsif (!(-f $mname) && (-f "/sys/class/net/eth0")) {
            create_ifcfg_eth0()
        }
        else {
            if (-f "/etc/network/interfaces") {
                # Debian based
                $fname = "/etc/network/interfaces";
                open(FILE, ">$fname") or die("Couldn't open file: $fname for writing: $!\n\n");
                print FILE "# interfaces(5) file used by ifup(8) and ifdown(8)
# (re)Created by LANforge lf_kinstall.pl script.
# Please note that this file is written to be used with dhcpcd
# For static IP, consult /etc/dhcpcd.conf and 'man dhcpcd.conf'
# Include files from /etc/network/interfaces.d:

source-directory /etc/network/interfaces.d

auto eth0
iface eth0 inet dhcp
";
                close FILE;
            }
            else {
                $fname = "/etc/sysconfig/network-scripts/ifcfg-$mgt_dev";
                open(FILE, ">$fname") or die("Couldn't open file: $fname for writing: $!\n\n");
                print FILE "HWADDR=$mac
TYPE=Ethernet
BOOTPROTO=dhcp
DEFROUTE=yes
PEERDNS=yes
PEERROUTES=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_PEERDNS=yes
IPV6_PEERROUTES=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=$mgt_dev
DEVICE=$mgt_dev
UUID=$uuid
ONBOOT=yes
AUTOCONNECT_PRIORITY=-999
";
                close FILE;
            }
        }
        print "_do_sys_reconfig:          possibly rename_port old_dev[$old_dev] -> mgt_dev[$mgt_dev]\n";
        rename_port($old_dev, "$mgt_dev", 1);

        for ($i = 0; $i < @ports; $i++) {
            $fname = "/etc/sysconfig/network-scripts/ifcfg-$ports[$i]";
            if (-f $fname) {
                do_cmd("rm -f $fname");
            }
        }

        for ($i = 0; $i < @ports; $i++) {
            print "_do_sys_reconfig:          possibly rename_port ports[X][$ports[$i]] -> ethX[eth$i]\n";
            rename_port($ports[$i], "eth$i", 0);
        }
    } # if !openwrt

    $fname = "/etc/udev/rules.d/70-persistent-net.rules";
    unlink("$fname");
    print_slow(
        " ----- ----- _do_sys_reconfig \@ports: ----- -----",
        Dumper(["ports:", \@ports]),
        "  ----- ----- ----- ----- ----- ----- ----- -----"
    );
    _do_udev();

    my $rules70persist = `cat $fname`;

    note("do-sys-reconfig, after persistent-net.rules setup.\n");

    if (!$is_openwrt) {
        #  This is buggy on openwrt, possibly need to get rid of it entirely?
        # This really is probably not needed, but should also cause no harm.
        # is renaming of interfaces that are already 'ethX' actually necessary?
        # the condition  here seems to apply to rules that were not named ethX
        # on VMs we frequently see errors caused by 'renaming eth1 -> eth0' and
        # the result is two UDEV rules naming different devices to eth0. This
        # bit of logic should not be applied to virtual machines.
        if (!$is_vm) {
            for ($i = 0; $i < @ports; $i++) {
                print(" ----- ----- renaming [$ports[$i]] / [eth$i]  -----  -----\n");
                $rules70persist =~ s/$ports[$i]/eth$i/g;
            }
        }
    }
    print_slow( " -----  -----  -----  -----  ----- Renaming ethernets $fname  ----- ----- ----- ----- -----",
                $rules70persist,
                " ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----");
    open(FILE, ">$fname") or die("Couldn't open file: $fname for writing: $!\n\n");

    my @radnames = ();
    # Remove wlan interfaces, we'll add them back below with proper names.
    my @ufca = split(/\n/, $rules70persist);
    my $bus = "";
    my @macs = ();
    for ($i = 0; $i < $rad_count; $i++) {
        # Now, deal with wlanX interfaces.
        $bus = $radios[$i];
        if (-e "/sys/bus/pci/devices/$bus/ieee80211") {
            $macs[$i] = `cat /sys/bus/pci/devices/$bus/ieee80211/*/macaddress|head -1`;
            my $rn = `ls /sys/bus/pci/devices/$bus/ieee80211/`;
            chomp($rn);
            #print("1: radio-name from /sys/bus/pci/devices/$bus/ieee80211/:  $rn\n");
            @radnames = (@radnames, $rn);
        }
        elsif (-e "/sys/devices/platform/soc/$bus/ieee80211") {
            # ath11k puts two radios on one bus, or some such of a thing.
            my @mrads = split('\s', `ls /sys/devices/platform/soc/$bus/ieee80211`);
            if (@mrads > 1) {
                my $z;
                for ($z = 0; $z < @mrads; $z++) {
                    my $mrad = $mrads[$z];
                    chomp($mrad);
                    #print("2: radio-name from soc:  $mrad\n");
                    @radnames = (@radnames, $mrad);
                    $macs[$i] = `cat /sys/devices/platform/soc/$bus/ieee80211/$mrad/macaddress|head -1`;
                    chomp($macs[$i]);
                    @ufca = grep !/$macs[$i]/, @ufca;
                    if ($z != (@mrads - 1)) {
                        $i++;
                    }
                }
                next;
            }
            else {
                $macs[$i] = `cat /sys/devices/platform/soc/$bus/ieee80211/*/macaddress|head -1`;
                my $rn = `ls /sys/devices/platform/soc/$bus/ieee80211`;
                chomp($rn);
                #print("3: radio-name from soc:  $rn\n");
                @radnames = (@radnames, $rn);
            }
        }
        elsif (-e "/sys/devices/platform/$bus/ieee80211") {
            # MTK e8450 puts one radio here
            my @mrads = split('\s', `ls /sys/devices/platform/$bus/ieee80211`);
            if (@mrads > 1) {
                my $z;
                for ($z = 0; $z < @mrads; $z++) {
                    my $mrad = $mrads[$z];
                    chomp($mrad);
                    #print("4: radio-name from soc/platform:  $mrad\n");
                    @radnames = (@radnames, $mrad);
                    $macs[$i] = `cat /sys/devices/platform/$bus/ieee80211/$mrad/macaddress|head -1`;
                    chomp($macs[$i]);
                    @ufca = grep !/$macs[$i]/, @ufca;
                    if ($z != (@mrads - 1)) {
                        $i++;
                    }
                }
                next;
            }
            else {
                $macs[$i] = `cat /sys/devices/platform/$bus/ieee80211/*/macaddress|head -1`;
                my $rn = `ls /sys/devices/platform/$bus/ieee80211`;
                chomp($rn);
                #print("5: radio-name from /sys/devices/platform/$bus/ieee80211:  $rn\n");
                @radnames = (@radnames, $rn);
            }
        }
        # TODO:  Need ath11k help here?
        else {
            # Maybe usb and just based on name?
            $macs[$i] = `cat /sys/class/ieee80211/$bus/addresses|head -1`;
            @radnames = (@radnames, $bus);
            #print("6: radio-name from bus/usb:  $bus\n");
        }
        chomp($macs[$i]);
        @ufca = grep !/$macs[$i]/, @ufca;
    }
    my $template = qq(SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="__ADDR__", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="__DEVNAME__");

    for ($i = 0; $i < $rad_count; $i++) {
        # Now, deal with wlanX interfaces.
        my $wlan_line = $template;
        $wlan_line =~ s/__ADDR__/$macs[$i]/g;
        $wlan_line =~ s/__DEVNAME__/wlan$i/g;
        push(@ufca, "$wlan_line\n");
    }
    print_slow( " -----  -----  -----  -----  ----- ufca -> rules70persist $fname after radios ----- ----- ----- ----- -----",
                Dumper(['ufca', \@ufca] ),
                " ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----");


    $rules70persist = join("\n", @ufca);
    print FILE $rules70persist;
    close FILE;

    print("do-sys-reconfig, after phyname rules setup.\n");

    # Phyname rules
    $fname = "/etc/udev/rules.d/70-phyname.rules";
    if ($rad_count > 0) {
        open(FILE, ">$fname") or die("Couldn't open file: $fname for writing: $!\n\n");
        print FILE "# Created by LANforge (lf_kinstall).  Will not be over-written.

ACTION!=\"add\", GOTO=\"phyname_end\"
SUBSYSTEM!=\"ieee80211\", GOTO=\"phyname_end\"

# read MAC address
ENV\{MATCHADDR\}=\"\$attr\{macaddress\}\"\n";

        for ($i = 0; $i < $rad_count; $i++) {
            print FILE "ENV\{MATCHADDR\}==\"$macs[$i]\", RUN+=\"$IW \%k set name wiphy$i\"\n";
        }
        print FILE "\nLABEL=\"phyname_end\"\n";
        close FILE;
    }
    else {
        do_cmd("rm -f $fname");
    }

    note("Checking is_openwrt: $::is_openwrt in do_sys_reconfig.\n");
    if ($::is_openwrt) {
        my $usb_starting_phy = 0;
        note("Disabling firewall...\n");
        do_cmd("echo \\# Disabled by LANforge > /etc/config/firewall");
        do_cmd("/etc/init.d/firewall disable", 1);
        if ($openwrt_board eq "R7800") {
            do_cmd("uci set wireless.\@wifi-device[0].phyname=wiphy0");
            do_cmd("uci set wireless.\@wifi-device[0].disabled=1");
            do_cmd("uci set wireless.\@wifi-device[1].phyname=wiphy1");
            do_cmd("uci set wireless.\@wifi-device[1].disabled=1");
            $usb_starting_phy = 2;

            # Give each vlan its own MAC address (vlan 1 uses parent MAC addr)
            my $eth1_mac = `ip link show eth1`;
            if ($eth1_mac =~ /link\/ether (\S+)/g) {
                my $eth1_mac = $1;

                # eth0, eth1, wlan0, wlan1.  eth1 has lowest MAC, next are sequential
                # So we start with +4.
                my $vlan_mac = increment_mac($eth1_mac, 4);
                do_cmd("uci set network.lan2.macaddr=$vlan_mac");
                $vlan_mac = increment_mac($vlan_mac, 1);
                do_cmd("uci set network.lan3.macaddr=$vlan_mac");
                $vlan_mac = increment_mac($vlan_mac, 1);
                do_cmd("uci set network.lan4.macaddr=$vlan_mac");
                do_cmd("uci commit network");
            }
        }
        elsif ($openwrt_board eq "ECW5410") {
            do_cmd("uci set wireless.\@wifi-device[0].phyname=wiphy0");
            do_cmd("uci set wireless.\@wifi-device[0].disabled=1");
            do_cmd("uci set wireless.\@wifi-device[1].phyname=wiphy1");
            do_cmd("uci set wireless.\@wifi-device[1].disabled=1");
            $usb_starting_phy = 2;
        }
        elsif ($openwrt_board eq "ECW5211") {
            do_cmd("uci set wireless.\@wifi-device[0].phyname=wiphy0");
            do_cmd("uci set wireless.\@wifi-device[0].disabled=1");
            do_cmd("uci set wireless.\@wifi-device[1].phyname=wiphy1");
            do_cmd("uci set wireless.\@wifi-device[1].disabled=1");
            $usb_starting_phy = 2;
        }
        elsif ($openwrt_board eq "E8450") {
            do_cmd("uci set wireless.\@wifi-device[0].phyname=wiphy0");
            do_cmd("uci set wireless.\@wifi-device[0].disabled=1");
            do_cmd("uci set wireless.\@wifi-device[1].phyname=wiphy1");
            do_cmd("uci set wireless.\@wifi-device[1].disabled=1");
            $usb_starting_phy = 2;
        }
        elsif ($openwrt_board eq "EC420") {
            do_cmd("uci set wireless.\@wifi-device[0].phyname=wiphy0");
            do_cmd("uci set wireless.\@wifi-device[0].disabled=1");
            do_cmd("uci set wireless.\@wifi-device[1].phyname=wiphy1");
            do_cmd("uci set wireless.\@wifi-device[1].disabled=1");
            $usb_starting_phy = 2;
        }
        elsif ($openwrt_board eq "CIG194C") {
            do_cmd("uci set wireless.\@wifi-device[0].phyname=wiphy0");
            do_cmd("uci set wireless.\@wifi-device[0].disabled=1");
            do_cmd("uci set wireless.\@wifi-device[1].phyname=wiphy1");
            do_cmd("uci set wireless.\@wifi-device[1].disabled=1");
            $usb_starting_phy = 2;
        }

        # Check for USB ones as well.
        for ($i = 0; $i < $rad_count; $i++) {
            $bus = $radios[$i];
            if (!((-e "/sys/bus/pci/devices/$bus/ieee80211/") ||
                (-e "/sys/devices/platform/soc/$bus/ieee80211") ||
                (-e "/sys/devices/platform/$bus/ieee80211")
            )) {
                # bus is phy name instead
                my $path = `ls -l /sys/class/ieee80211/$bus/device/driver/|grep usb`;
                chomp($path);
                #note("usb $bus path: $path\n");
                if ($path =~ /.*devices\/(platform.*)/g) {
                    $path = $1;
                    #note("trimmed usb $bus path: $path\n");
                    my $q;
                    for ($q = $usb_starting_phy; $q < 10; $q++) {
                        my $uci_path = `uci get wireless.\@wifi-device[$q].path`;
                        chomp($uci_path);
                        #note("uci_path $q: $uci_path\n");
                        if ($uci_path eq $path) {
                            #note("Found match...\n");
                            do_cmd("uci set wireless.\@wifi-device[$q].phyname=wiphy$usb_starting_phy");
                            do_cmd("uci set wireless.\@wifi-device[$q].disabled=1");
                            $usb_starting_phy++;
                            last;
                        }
                    }
                }
            }
        }

        do_cmd("uci commit wireless");

        # Disable auto-starting these.
        if (-f "/etc/init.d/dhcpd") {
            do_cmd("mv /etc/init.d/dhcpd /etc/init.d/dncpd.nope");
        }
        if (-f "/etc/init.d/dnsmasq") {
            do_cmd("mv /etc/init.d/dnsmasq /etc/init.d/dnsmasq.nope");
        }
    } # ~if openwrt

    # Rename phy devices accordingly
    for ($i = 0; $i < $rad_count; $i++) {
        $bus = $radios[$i];
        my $old_name = $radnames[$i];
        chomp($old_name);
        if ($old_name ne "wiphy$i") {
            do_cmd("$IW $old_name set name wiphy$i", 1);
        }
    }

    print "* Updating mgt-dev to [$mgt_dev]\n";
    do_cmd("cd $home && echo mgt_dev $mgt_dev | ./lfconfig");

    $fname = "$home/lanforge_eth_1.cfg";
    open(FILE, ">$fname") or die("Couldn't open file: $fname for writing: $!\n\n");
    print FILE "# Contains config information.  Right now this
# is just for physical port layout on the GUI.  An example layout is:
#
#              PORT_CONF NA   eth0 NA   NA   NA  NA ~
#              PORT_CONF NA   eth1 NA   NA   NA  NA ~
#              PORT_CONF NA   eth2 eth4 NA   NA  NA ~
#              PORT_CONF eth7 eth3 eth5 eth6 NA  NA ~
#
#  Here is a guess generated by the lf_kinstall script.  This file will not be over-written,
#  unless forced with the ./lfconfig --new_layout command, so you can change it as you please.
#
$port_layout
";
    close FILE;

    # possible that you might lose all your network connections at this point
    # the apache server name will likely change again, so if apache does
    # not respond after this point, restarting lanforge (service lanforge restart)
    # will run the adjust_apache.pl script again.
    if (!$::is_openwrt) {
        # Rename devices, including wlans that may have not already been renamed.
        if ( ! ($skip_ifrename || $is_vm)) {
            do_cmd("$home/lf_ifrename.pl --all");
        }

        # re-write the apache server hostname
        my $apache_pre = "/etc/httpd/conf.d";
        if (!-d "/etc/httpd/conf.d") {
            $apache_pre = "/etc/apache2/sites-enabled/";
        }
        die($!) unless open(FILE, ">", "$apache_pre/00-ServerName.conf");
        print FILE "ServerName $hn\n";
        close FILE;
        note("# Updated $apache_pre/00-ServerName.conf\n");
        adjust_apache();
        note("# about to restart apache...\n");

        if (-d "/etc/httpd") {
            do_cmd("hostname -F /etc/hostname", 1);
            do_cmd("systemctl restart --quiet httpd", 1);
            sleep 1;
            if (system("systemctl is-active --quiet httpd") != 0) {
                regen_https_selfcert();
                do_cmd("hostname -F /etc/hostname", 1);
                do_cmd("systemctl restart --quiet httpd", 1);
                sleep 1;
                if (system("systemctl is-active --quiet httpd") != 0) {
                    warning("Apache did not restart with new certificate, regenerating hostname self signed cert...\n");
                    warning("    if installing LANforge the first time, check *systemctl status httpd* AFTER you reboot.\n");
                    warning("    Run *lf_kinstall.pl --regen_https_key; systemctl restart httpd*, to fix.\n");
                    err(" * * Unable to restart httpd, run *lf_kinstall.pl --regen_https_key; systemctl restart httpd* after a reboot.\n");
                }
            }
        }
        do_cmd("systemctl restart apache2", 1) if (-d "/etc/apache2");
    }
} # ~_do_sys_reconfig

sub increment_mac {
    my $mac_str = shift;
    my $amt = shift;

    (my $mac_hex = $mac_str) =~ s/://g;
    my ($mac_hi, $mac_lo) = unpack("nN", pack('H*', $mac_hex));

    if ($mac_lo > (0xFFFFFFFF - $amt)) {
        $mac_hi = ($mac_hi + 1) & 0xFFFF;
        # This is not 100% correct, would skip last few maybe, but good enough I think.
        $mac_lo = 0;
    }
    else {
        $mac_lo += $amt;
    }

    $mac_hex = sprintf("%04X%08X", $mac_hi, $mac_lo);
    $mac_str = join(':', $mac_hex =~ /../sg);
    return $mac_str;
} # ~increment mac

sub update_ff_homepage {
    note("# Updating Firefox homepage\n");
    my $ff_syspref = "firefox/browser/defaults/preferences/";
    my $fn = "/usr/lib/$ff_syspref/syspref.js";
    my @lines = ();
    if (-f $fn) {
        note("# Firefox homepage in Ubuntu location\n");
        $fn = "/usr/lib/$ff_syspref/syspref.js";
        @lines = `cat $fn`;
    }
    $fn = "/usr/lib64/$ff_syspref/firefox-redhat-default-prefs.js";
    if (-f $fn) {
        note("# Firefox homepage in Fedora location\n");
        @lines = `cat $fn`;
    }
    if (@lines > 1) {
        note("# Updating $fn\n");
        my $hn = `cat /etc/hostname`;
        chomp($hn);
        my @newlines = ();
        chomp(@lines);
        for my $line (@lines) {
            if ($line =~ m|browser\.startup\.homepage|) {
                print "Altering $line \n";
                #$line =~ s|homepage=[^"]+|homepage=https://$hn/|;
                #print "Now: $line \n";
                $line = qq[pref("browser.startup.homepage", "data:text/plain,browser.startup.homepage=http://$hn/");];
            }
            push(@newlines, $line);
        }
        if (open(my $FH, ">", $fn)) {
            note("Saving $fn\n");
            print $FH join("\n", @lines);
            print $FH "\n";
            close $FH;
        }
        else {
            note("Unable to open $fn, changes not saved.\n");
        }
    }
} # ~ set_ff_homepage3

sub rename_port {
    my $old_name = shift;
    my $new_name = shift;
    my $force_dhcp = shift;

    if (-f "/sys/class/net/$new_name/address") {
        print "/sys/class/net/$new_name/address - exists, not renaming from [$old_name]\n";
    }
    else {
        # do not use ifdown, it expects a corresponding e/s/network-scripts/ifcfg-$old_name file to exist
        do_cmd("ip link set down dev $old_name", 1); #  > /dev/null 2>&1
        do_cmd("ip link set dev $old_name name $new_name", 1); #  > /dev/null 2>&1
        my $rv = do_cmd("ifup $new_name", 1); # > /dev/null 2>&1
        if (($rv != 0) && $force_dhcp) {
            print("rename_port: using dhclient on [$new_name]...\n");
            do_cmd("dhclient $new_name", 1);
        }
    }
}

# this does not appear to be called
#sub do_make_radius_keys {
#  my $rbase = shift;
#  my $key_dir = shift;
#
#  do_cmd("mkdir -p $key_dir");
#  do_cmd("rm -fr $key_dir/*", 1);
#
#  do_cmd("openssl genrsa 2048 > $key_dir/ca-key.pem", 1);
#  do_cmd("openssl req -new -x509 -nodes -days 3600 -config $rbase/certs/client.cnf -key $key_dir/ca-key.pem -out $key_dir/ca.pem", 1);
#
#  # Create server certificate
#  do_cmd("openssl req -newkey rsa:2048 -days 3600 -nodes -config $rbase/certs/server.cnf -keyout $key_dir/server-key.pem -out $key_dir/server-req.pem", 1);
#  do_cmd("openssl x509 -req -in $key_dir/server-req.pem -days 3600 -CA $key_dir/ca.pem -CAkey $key_dir/ca-key.pem -set_serial 01 -out $key_dir/server-cert.pem", 1);
#
#  # Create client certificate
#  do_cmd("openssl req -newkey rsa:2048 -days 3600 -nodes -config $rbase/certs/client.cnf -keyout $key_dir/client-key.pem -out $key_dir/client-req.pem", 1);
#  do_cmd("openssl x509 -req -in $key_dir/client-req.pem -days 3600 -CA $key_dir/ca.pem -CAkey $key_dir/ca-key.pem -set_serial 01 -out $key_dir/client-cert.pem", 1);
#
#  # Verify certificates
#  do_cmd("openssl verify -CAfile $key_dir/ca.pem $key_dir/server-cert.pem $key_dir/client-cert.pem", 1);
#  do_cmd("chown root:radiusd $rbase/certs/*");
#}

sub make_cacnf {
    my $fname = shift;

    my $in_req = 0;
    my $in_ca = 0;
    my $in_cad = 0;
    my $in_srv = 0;
    my $in_client = 0;

    note("# Customizing radius file: $fname\n");

    my @cmds = `cat $fname`;
    open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
    for (my $i = 0; $i < @cmds; $i++) {
        my $ln = $cmds[$i];
        chomp($ln);
        if ($ln =~ /^\[/) {
            # Section entry
            $in_req = 0;
            $in_ca = 0;
            $in_cad = 0;
            $in_srv = 0;
            $in_client = 0;
            if ($ln =~ /^\[\s*req\s*\]/) {
                $in_req = 1;
            }
            elsif ($ln =~ /^\[\s*server\s*\]/) {
                $in_srv = 1;
            }
            elsif ($ln =~ /^\[\s*client\s*\]/) {
                $in_client = 1;
            }
            elsif ($ln =~ /^\[\s*certificate_authority\s*\]/) {
                $in_ca = 1;
            }
            elsif ($ln =~ /^\[\s*CA_default\s*\]/) {
                $in_cad = 1;
            }
        }

        if ($in_cad) {
            if ($ln =~ /^default_days\s+/) {
                $ln = "default_days            = 5000";
            }
        }
        elsif ($in_req) {
            if ($ln =~ /^input_password\s+/) {
                $ln = "input_password          = lanforge";
            }
            elsif ($ln =~ /^output_password\s+/) {
                $ln = "output_password         = lanforge";
            }
        }
        elsif ($in_ca || $in_srv || $in_client) {
            my $cne = "CA";
            if ($in_srv) {
                $cne = "SERVER";
            }
            elsif ($in_client) {
                $cne = "CLIENT";
            }

            if ($ln =~ /^countryName\s+/) {
                $ln = "countryName             = US";
            }
            elsif ($ln =~ /^stateOrProvinceName\s+/) {
                $ln = "stateOrProvinceName     = WA";
            }
            elsif ($ln =~ /^localityName\s+/) {
                $ln = "localityName            = Ferndale";
            }
            elsif ($ln =~ /^organizationName\s+/) {
                $ln = "organizationName        = Candela Technologies Inc.";
            }
            elsif ($ln =~ /^commonName\s+/) {
                $ln = "commonName              = \"Candela - $cne\"";
            }
            elsif ($ln =~ /^emailAddress\s+/) {
                $ln = "emailAddress            = support\@candelatech.com";
            }
        }
        print FILE "$ln\n";
    } # For all lines in ca.cnf file
    close FILE;
    do_cmd("chown root:radiusd $fname");
} # make_cacnf sub

sub getPhyInfo {
    my $idx = shift;
    my $wlan = shift;
    my $bus = shift;
    my $desc = shift;
    #print "wlan[$wlan] bus[$bus]\n";
    $macs[$idx] = "unknown";
    $desc[$idx] = "unknown";

    my $tmp = `ethtool -i $wlan`;
    if ($tmp =~ /driver:\s+(\S+)/) {
        $driver[$idx] = $1;
    }

    # consider this regex next time: /-([^ -]{4})-/)
    if ($tmp =~ /firmware-version:\s+10\.4-ct-(\S\S\S\S)-/) {
        $driver[$idx] .= "(wave-2 $1)";
    }
    my $macval = "no-mac-val";
    my $sysbusfile = "";
    if ($tmp =~ / $bus/) {
        my @globmatch = glob("/sys/bus/pci/devices/${bus}/ieee80211/*phy${idx}/macaddress");
        if (@globmatch > 0) {
            $sysbusfile = $globmatch[0];
            $macval = `cat $sysbusfile`;
        }
        if ($macval eq "no-mac-val") {
            @globmatch = glob("/sys/devices/platform/soc/${bus}/ieee80211/*phy${idx}/macaddress");
            if (@globmatch > 0) {
                $sysbusfile = $globmatch[0];
                $macval = `cat $sysbusfile`;
            }
            else {
                @globmatch = glob("/sys/devices/platform/${bus}/ieee80211/*phy${idx}/macaddress");
                if (@globmatch > 0) {
                    $sysbusfile = $globmatch[0];
                    $macval = `cat $sysbusfile`;
                }
            }
        }

        if ($macval eq "no-mac-val") {
            warning("## Files /sys/bus/pci/devices/$bus/ieee80211/*phy$idx/macaddress and\n/sys/devices/platform/soc/$bus/ieee80211/*phy$idx/macaddress and\n/sys/devices/platform/$bus/ieee80211/*phy$idx/macaddress appear missing!\n");
        }
        chomp($macval);
        $macs[$idx] = $macval;
        $desc[$idx] = $desc;
        return 0;
    }
    else {
        warning("# Un-expected bus info for $wlan: $tmp, bus: $bus  desc: $desc\n");
        return -1
    }
}

sub getBusInfo {
    my $bus = shift;
    my $addr = "";

    if (-e "/sys/devices/platform/soc/$bus/ieee80211/*/macaddress") {
        $addr = `cat /sys/devices/platform/soc/$bus/ieee80211/*/macaddress`;
    }
    elsif (-e "/sys/devices/platform/$bus/ieee80211/*/macaddress") {
        $addr = `cat /sys/devices/platform/$bus/ieee80211/*/macaddress`;
    }
    else {
        $addr = `cat /sys/bus/pci/devices/$bus/ieee80211/*/macaddress`;
    }
    return $addr;
}

sub _do_udev {
    my $template = qq(SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="__ADDR__", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="__DEVNAME__");
    if (!-f "/etc/sysctl.d/50-coredump.conf") {
        # Disable core file renaming...systemd steals them by default.
        do_cmd("echo kernel.core_pattern=core.\%p > /etc/sysctl.d/50-coredump.conf", 1);
        if (-f "/lib/systemd/systemd-sysctl") {
            do_cmd("/lib/systemd/systemd-sysctl", 1);
        }
        else {
            do_cmd("sysctl -p", 1);
        }
    }

    if ($::is_openwrt) {
        do_cmd("mkdir -p /etc/udev/rules.d/");
    }

    my $fname = "/etc/udev/rules.d/70-persistent-net.rules";
    if (-f "$fname") {
        note(qq(\n# WARNING:  Not updating udev rules since file: $fname already exists.
#  Remove it and re-run this script to have the file re-created.
"));
    }
    else {
        open(FILE, ">$fname") or die "Couldn't open file: $fname for writing: $!\n\n";
        my @new70lines = (
            "#",
            "#  Auto-created by LANforge lf_kinstall.pl::_do_udev.",
            "#  This file is destroyed by --do_sys_reconfig.",
            "#",
        );

        my %devnames = ();
        my @uevent_files = `ls /sys/class/net/*/device/uevent`;
        chomp(@uevent_files);
        foreach my $ln (@uevent_files) {
            print("sys/class/net device ln: $ln\n");
            my ($dn) = $ln =~ m|/sys/class/net/(.*)/device/uevent|;
            if (!(defined $dn) || ($dn eq "")) {
                err("XX BAD UDEV REGEX no device for [$ln]\n");
                next;
            }
            $devnames{$dn} = 1;
        }
        # we cannot have duplicate device names in this map
        foreach my $dn (keys %devnames) {
            if (! -e "/sys/class/net/$dn/address") {
                err("XX sys/class/net/$dn/address not found\n");
                next;
            }
            my $addr = `cat /sys/class/net/$dn/address`;
            chomp($addr);
            #print("dn: $dn  addr: $addr\n");
            # NOTE:  Jan 29, 2022:  Remove wlan|wlp from the pattern below, I think that any
            # system in last many years will be fine with that change, the original work-around
            # was probably for truly ancient systems.  Or, maybe I am missing something and we'll
            # re-add that code later...
            if ($dn !~ /^(sta|moni|vap|_vrf|vrf).*/) {
                my $uln = $template;
                $uln =~ s/__ADDR__/$addr/g;
                $uln =~ s/__DEVNAME__/$dn/g;
                #print("uln: $uln");
                push(@new70lines, "$uln", "");
            }
            else {
                # we should not be appending LF created wifi devices here
                # Skip wlan, causes renameXX interfaces to appear for some reason.
                push(@new70lines,  "# skipped device $dn");
            }
        }
        push(@new70lines, "");
        print_slow(
            "       --- --- --- new70lines --- --- ---",
            @new70lines,
            "       --- --- --- --- --- --- --- ---",
            );
        print FILE join("\n", @new70lines);
        close FILE;

        note("\n$fname has been created.  Please hand-edit to fix up ordering.\n");
        note("You may also need to create/edit /etc/sysconfig/network-scripts/ifcfg-eth* files as appropriate.\n\n");
        #print `cat $fname`;
    }

    if ($is_fedora && ($::osveri >= 19)) {
        # Get crappy new udev behaviour out of the way.
        if (-e "/usr/lib/udev/rules.d/80-net-name-slot.rules") {
            note("# Making sure UDEV net-name-slot rules is disabled.\n");
        }
        do_cmd("rm -f /etc/udev/rules.d/80-net-name-slot.rules", 1);
        do_cmd("ln -s /dev/null /etc/udev/rules.d/80-net-name-slot.rules", 1);
    }
}

#sub check_dnf_ok {
#   if ($skip_yum_all || $skip_yum_update || (defined $::use_install_bundle && "" ne $::use_install_bundle )) {
#     note("* Skipping check_dnf_ok: skip_yum_all[$skip_yum_all] skip_yum_update[$skip_yum_update] use_install_bundle[$::use_install_bundle]\n");
#     return 0;
#   }
#   my $stdout_fn ="";
#   my $stderr_fn ="";
#   do_cmd("dnf makecache", 1, \$stdout_fn, \$stderr_fn);
#   print "stdout_fn: $stdout_fn\n";
#   print "stderr_fn: $stderr_fn\n";
#}

sub metalink_for {
    my $repname = shift;
    die("metalinks not defined") unless %main::metalinks;
    unless ($repname) {
        note("** undefined argument for metalink_for(), using 'fedora'");
        sleep(5);
        $repname = 'fedora';
    }
    if (exists $::metalinks{$repname}) {
        note(" * metalink for $repname is $::metalinks{$repname}") if ($::debug_on);
        return "metalink=https://mirrors.fedoraproject.org/metalink?repo=" . $::metalinks{$repname};
    }
    note("** unable to find metalink for <<$repname>>");
    return $::metalinks{'fedora'};
}

sub restore_metalinks {
    my $repofilename = shift;
    if (!(defined $repofilename) || ($repofilename eq "")) {
        err("repofile name undefined, not resetting any URLs");
        exit 1;
    }
    note("# inspecting $repofilename for mirrorlist");
    my @lines = `cat $repofilename`;
    chomp(@lines);
    my @rslt2 = grep {/LEAVE_ME_BE/} @lines;
    if (@rslt2 > 0) {
        note("# ignoring $repofilename");
        return;
    }
    my @lines2 = ();
    my $ml_sightings = 0;
    my $ml_line;

    note("* setting default mirror list in $repofilename");
    for my $line (@lines) {
        if (($line =~ /^\s*$/) || ($line =~ /^\s*#/)) {
            note(" blankish: " . $line) if ($::debug_on);
        }
        elsif ($line =~ /^\s*\[/) {
            if ((defined $ml_line) && ($ml_line ne "") && ($ml_sightings < 1)) {
                push(@lines2, "$ml_line\n"); # push our current line, don't destroy [fedora] etc
                note(" saving reponame: $ml_line ") if ($::debug_on);
            }
            else {
                note(" I see a stanza but I lack a ml_line yet: $line") if ($::debug_on);
            }
            my ($repname) = $line =~ /^\[(.*)\]/;
            note("# seeing repository <$repname>");
            $ml_sightings = 0;
            $ml_line = metalink_for($repname);
            #$line = $ml_line;
            note(" new ml_line: $line ") if ($::debug_on);
        }
        elsif ($line =~ /^\s*baseurl/) {
            note(" commenting out: $line ") if ($::debug_on);
            $line = "# $line";
        }
        elsif ((defined $ml_line) && ($ml_line ne "") && ($line eq $ml_line)) {
            note("# found $ml_line") if ($::debug_on);
            $ml_sightings++;
        }
        elsif ($line =~ /^\s*metalink/) {
            note(" commenting out: $line ") if ($::debug_on);
            $line = "# $line";
        }
        else {
            note(" ok $line ") if ($::debug_on);
        }
        push(@lines2, $line);
    } #~for
    # we are at the last line
    if (defined $ml_line && ($ml_line ne "") && ($ml_sightings < 1)) {
        push(@lines2, $ml_line);
    }

    # now re-write file
    open(my $fh, ">", $repofilename)
        || die("Unable to open $repofilename: $!");
    print $fh join("\n", @lines2);
    print $fh "\n";
    close($fh);
}

sub wipe_raid {
    my @swap_lines = `cat /proc/swaps | grep -vE 'zram|Filename'`;
    if (@swap_lines > 0) {
        chomp(@swap_lines);
        for my $line (@swap_lines) {
            my ($swapdev) = m|(/dev/\S+)|;
            if ((defined $swapdev) && ($swapdev ne "")) {
                print("[wipe-raid] swapoff $swapdev\n");
                do_cmd("swapoff $swapdev", 1);
            }
        }
    }
    my @lvlines = ();
    my @vgslines = ();
    do_cmd("lvm vgs");
    @vgslines = `vgdisplay`;
    my @vgnames;
    if (@vgslines > 0) {
        @vgnames = grep {/VG Name/} @vgslines;
    }
    @lvlines = `lvdisplay`;
    chomp(@lvlines);
    if (@lvlines > 0) {
        my %pventries = ();
        my ($vgname) = $vgnames[0] =~ m|VG Name\s*(\S+)$|;
        print("[wipe_raid] vgreduce --removemissing --force $vgname\n");
        do_cmd("vgreduce --removemissing --force $vgname", 1);
    }

    for my $vgname (@vgnames) {
        chomp $vgname;
        my ($svgname) = $vgname =~ /VG Name\s+(\w+)$/;
        chomp $svgname;
        my @lvlinesb = `lvs $svgname | tail -n +2`;
        if (@lvlinesb > 0) {
            for my $lvlb (@lvlinesb) {
                print "lvlb $lvlb\n";
                sleep 2;
                my ($hunk) = $lvlb =~ m|^\s*(\S+)\s+$svgname|;
                print("[wipe-raid] lvremove -f -f $svgname/$hunk\n");
                do_cmd("lvremove -f -f $svgname/$hunk", 1);
            }
            print("[wipe-raid] vgreduce -f -f -a $svgname\n");
            do_cmd("vgreduce -f -f -a $svgname", 1);
        }
        print("[wipe-raid] vgremove $svgname\n");
        do_cmd("vgremove $svgname", 1);
        my @pvlines = `pvs | tail -n +2`;
        my @pvn = grep {/ $svgname /} @pvlines;
        for my $pvl (@pvn) {
            my @hunks = $pvl =~ m|^\s*(/\S+) |;
            print "[wipe-raid] pvremove $hunks[0] -f\n";
            do_cmd("pvremove $hunks[0] -f", 1);
            sleep 1;
        }
    }
    my @pvlines = `pvs | tail -n +2`;
    my @pvn = grep {/^\s*\S+\s+/} @pvlines;
    my @drives = ();
    for my $pvl (@pvn) {
        my @hunks = $pvl =~ m|^\s*(/\S+) |;
        push(@drives, $hunks[0]);
        print "[wipe-raid] pvremove --force --force $hunks[0]\n";
        do_cmd("pvremove  --force --force $hunks[0]", 1);
        sleep 1;
    }

    #elsif (-e "/dev/nvme0n1") {
    #    $sda = "/dev/nvme0n1";
    #    $sdb = "/dev/nvme1n1";
    #}
    if (@drives < 1) {
        print "[wipe-raid] NO DRIVES identified. Run wipefs -af by hand.\n";
        return;
    }
    for my $drv (@drives) {
        print("[wipe-raid] wipefs -af $drv\n");
        do_cmd("wipefs -af $drv", 1);
    }
    note("[wipe-raid] Best to reboot now! Partitions have been altered.");
} # ~wipe_raid()

sub create_raid {
    # TODO: shift out arguments that would be our target drives
    # if there are no drives listed, test for (nvme0, nvme1), (sda, sdb), (vda,vdb)
    # if there are more than two block storage devices detected, then complain that
    # the drives need to be specified.

    my $targets_str = shift;
    if (!(defined $targets_str) or ($targets_str eq "")) {
        die("create_raid: wants comma separated list of drive names");
    }
    my @targets = split(",", $targets_str);

    for( my $i=0; $i < @targets; $i++) {
        my $drv = $targets[$i];
        if ($drv eq "") {
            die("create_raid: cannot operate with blank entry, watch your commas");
        }
        my $idx = index($drv, "/dev");
        if ( $idx < 0 ) {
            $targets[$i] = "/dev/$drv";
        }
        elsif ($idx > 0) {
            die("create_raid: why is this legal? [$drv]");
        }
    }

    # TODO: use parameters from create_raid=a,b
    #elsif (-e "/dev/nvme0n1") {
    #    $sda = "/dev/nvme0n1";
    #    $sdb = "/dev/nvme1n1";
    #}
    my $using_efi = 0;
    if (-d "/sys/firmware/efi/efivars") {
        $using_efi = 1;
    }
    # TODO: split formatting commands up so that it processes a list of devices
    # so that we can create a single disk LVM or even a three-way mirror
    my @commands = (
        "sync",
        "swapoff -a");
    for my $drv (@targets) {
        push(@commands, "partprobe $drv");
    }
    push(@commands, "sync");
    for my $drv (@targets) {
        push(@commands,
            "sgdisk --zap-all $drv",
            "partprobe $drv");
    }
    my @sufs = qw(a b c d e f g);
    # create a EFI boot record if we are creating an MBR partition
    # this applies to all GPT formatted disks (BIOS or UEFI)
    for( my $i=0; $i < @targets; $i++) {
        my $drv = $targets[$i];
        my $label = "efi_mbr_".$sufs[$i];
        push(@commands,
            "sgdisk -n 0:0:+1MiB -t 0:ef02 -c 0:$label $drv");
    }
    if ($using_efi) {
        # create a EFI boot partition if we are booting in UEFI mode, not BIOS
        for( my $i=0; $i < @targets; $i++) {
            my $drv = $targets[$i];
            my $label = "boot_efi_".$sufs[$i];
            push(@commands,
                "sgdisk -n 0:0:+128MiB -t 0:ef00 -c 0:$label $drv");
        }
    }
    push(@commands, "sync");
    for( my $i=0; $i < @targets; $i++) {
        my $drv = $targets[$i];
        my $label = "boot_" . $sufs[$i];
        push(@commands, "sgdisk -n 0:0:+2GiB -t 0:ea00 -c 0:$label $drv");
        $label = "pv_" . $sufs[$i];
        push(@commands, 
            "sgdisk -n 0:0:0 -t 0:8e00 -c 0:$label $drv",
            "sync",
            "sleep 1",
            "partprobe $drv",
            "sleep 1",
            "pvcreate -ff --metadatacopies 2 /dev/disk/by-partlabel/$label");
    }
    push(@commands, "sync",
        "vgcreate ctvg /dev/disk/by-partlabel/pv_*",
        "sync",
        "lvcreate --type raid1 -m 1 -L 50G -n lv_root ctvg",
        "lvcreate --type raid1 -m 1 -L  4G -n lv_swap ctvg",
        "lvcreate --type raid1 -m 1 -l 100%FREE -n lv_home ctvg",
        "lvm vgscan",
        "lvs");
    for( my $i=0; $i < @targets; $i++) {
        my $drv = $targets[$i];
        my $label = "grub_" . $sufs[$i];
        push(@commands,
            "[ -e /dev/disk/by-partlabel/grub_a ] && mkfs -t vfat /dev/disk/by-partlabel/grub_${sufs [ $i ]} ||:",
            "mkfs -t ext4 -L BOOT /dev/disk/by-partlabel/boot_${sufs [ $i ]}");
    }
    push(@commands,
        "mkswap /dev/mapper/ctvg-lv_swap",
        "mkfs -t ext4 -L ROOT  /dev/mapper/ctvg-lv_root",
        "mkfs -t ext4 -L HOME /dev/mapper/ctvg-lv_home",
    );
    for my $cmd (@commands) {
        print "[create_raid] $cmd\n";
        do_cmd($cmd);
    }
    note("[create_raid] good time to reboot!\n");
} # ~create_raid()

sub adjust_apache {
    my $fetch = 0;
    my $aapache = "$home/scripts/adjust_apache.pl";
    # check if we have a IP
    my @iprosho = `ip route show`;
    if (@iprosho < 1) {
        err("Unable to determine any routes on your system. Skipping adjust_apache.");
        return;
    }
    if (!-f $aapache) {
        $fetch++;
    }
    else {
        # detects if the script is older than 2021-02-27
        my @zargs = ("/bin/grep", "-q", "use Getopt::Long;", $aapache);
        $fetch++ if (system(@zargs) != 0);
    }
    if ($fetch) {
        my $now = `date +"\%Y\%m\%d\%H\%M\%S"`;
        chomp($now);
        do_cmd("test -f '$aapache'&& mv '$aapache' '$aapache.$now' ||:", 1);
        do_cmd("$curl -o '$aapache' https://raw.githubusercontent.com/greearb/lanforge-scripts/master/adjust_apache.pl", 1);
    }
    if (-f $aapache) {
        do_cmd("chmod +x '$aapache'", 1);
        do_cmd("$aapache -a", 1);
    }
    else {
        err("Unable to fetch adjust_apache.pl, skipped.");
    }
} # ~adjust_apache


# Local Variables:
# mode: cperl
# indent-tabs-mode: nil
# cperl-indent-level: 4
# End:

# Doesn't work in emacs??
# set tabstop=4
