use HTTP::Request;
use LWP;
my $req = HTTP::Request->new(GET => 'http://challenge.ctf.games:31879/robots.txt');
my $ua = LWP::UserAgent->new();
$res = $ua->request($req);
foreach $line (split /\n/,$res->content) {
        if ($line =~ m/User-agent: (.*)/) { $agent = $1 }
        if ($line =~ m/Disallow: \/(.*)\.txt/) {
                my $req2 = HTTP::Request->new(GET => "http://challenge.ctf.games:31879/$1.txt");
                $req2->header('User-agent' => $agent);
                my $res2 = $ua->request($req2);
                print $res2->content . "Getting $1\n" unless ($res2->content =~ m/UNFORTUNATELY/);
        }

}