/** Downloader app following https://www.youtube.com/watch?v=_z-RS0rXg9s but * use libcurl (C API) with some adjusted API usage. * * Compile with * g++ -std=c++11 -DNO_PROXY SimpleDownloader.cpp -lpthread -lcurl * */ #include #include #include #include #include #include #include #include #include #include struct MemoryHolder { char *mem; size_t size; std::string filename; unsigned line; }; static const std::string ROOT_URL = "http://iki.fi/bisqwit/ctu85/"; static unsigned ln = 1; static std::string Color(int n, const std::string& s) { return "\33[" + std::to_string(n) + 'm' + s + "\33[m"; } static std::string Line(int l) { int m = l - ln; ln = l; return "\r" + (m < 0 ? "\33[" + std::to_string(-m) + 'A': std::string(m, '\n')); } static std::string ClearLine(int l) { return "\33[2K"; } static std::mutex PrintLock; static int ProgressFunction(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) { // scope mutex lock std::lock_guard lock(PrintLock); MemoryHolder mHolder = *(MemoryHolder*)clientp; std::cout << Line(mHolder.line) << ClearLine(mHolder.line) << Color(93, mHolder.filename + ": ") << dlnow << " of " << dltotal << " bytes received (" << int(dltotal ? dlnow * 100.0/dltotal : 0) << "%)" << std::flush; return 0; } static size_t WriteFunction(char *contents, std::size_t size, std::size_t nmemb, void *userp) { size_t realsize = size * nmemb; MemoryHolder *mholder = (MemoryHolder*)userp; // memory chunk needs to be allocated first // this will ensure it will before we call realloc() if (mholder->mem == nullptr) mholder->mem = (char*)std::malloc(1); char *ptr = (char*)std::realloc(mholder->mem, mholder->size + realsize + 1); if (!ptr) { // out of memory std::cout << Line(mholder->line) << ClearLine(mholder->line) << Color(31, "Not enough memory") << std::flush; // return number of bytes that handled which we didn't handle any return 0; } // set new memory location that has been re-allocated mholder->mem = ptr; std::memcpy(&(mholder->mem[mholder->size]), contents, realsize); mholder->size += realsize; // set null-terminated character mholder->mem[mholder->size] = 0; return realsize; } static std::size_t Download(const std::string& url, const std::string& filename, const unsigned line) { CURL *curl = curl_easy_init(); if (!curl) { std::cout << Line(line) << ClearLine(line) << Color(31, "Error initialize libcurl") << std::flush; return 0; } MemoryHolder mHolder {}; mHolder.filename = filename; mHolder.line = line; curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); #ifndef NO_PROXY curl_easy_setopt(curl, CURLOPT_PROXY, "http://127.0.0.1:1080"); curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); curl_easy_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, 1L); #endif curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, ProgressFunction); curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, (void*)&mHolder); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteFunction); // send in pre-initialized MemoryHolder for this particular download id // note: we need to know number of downloads before downloading curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&mHolder); // this is a blocking call // when the download is done, then this function call will be returned CURLcode res = curl_easy_perform(curl); if (res != CURLE_OK) { // print out error std::cout << Line(line) << ClearLine(line) << Color(91, filename + ": Download failed [" + curl_easy_strerror(res) + "]") << std::flush; // destroy MemoryHolder of this particular download if (mHolder.mem) { delete(mHolder.mem); mHolder.mem = nullptr; } } else { // write to file // good thing, we can define this just once here std::ofstream of(filename); of.write((const char*)(mHolder.mem), mHolder.size); // destroy MemoryHolder of this particular download // note: as we don't need this particular data anymore after successfully write delete(mHolder.mem); mHolder.mem = nullptr; } curl_easy_cleanup(curl); return mHolder.size; } int main(int argc, char *argv[]) { // initialize curl curl_global_init(CURL_GLOBAL_DEFAULT); std::vector> sizes; unsigned line = 1; for (const std::string& r: {"8859-1.TXT", "8859-2.TXT", "8859-3.TXT", "8859-4.TXT", "8859-5.TXT", "8859-6.TXT", "8859-7.TXT", "8859-8.TXT", "8859-9.TXT", "8859-10.TXT", "8859-11.TXT", "8859-13.TXT", "8859-14.TXT", "8859-15.TXT", "8859-16.TXT"}) { sizes.emplace_back(std::async(std::launch::async, [r, &line] { return Download(ROOT_URL + r, r, line++); })); std::this_thread::sleep_for(std::chrono::milliseconds(16)); } std::size_t total = 0; for (auto& s: sizes) total += s.get(); std::cout << Line(line) << Color(92, "Total bytes downloaded: " + std::to_string(total)) << std::endl; curl_global_cleanup(); return 0; }